Added support for MultiVector and cleaned up.

in some cases. This required changes in the internals of the class.

Added some new methods in class MultiVector.

.github/workflows/builds-and-tests.yml

@@ -142,10 +142,6 @@ jobs:
 
     continue-on-error: ${{ matrix.enzyme && true || false }}
 
-    # Enable ccache for all jobs except Windows (would need sccache).
-    env:
-      USE_CCACHE: ${{ matrix.os != 'windows-latest' }}
-
     steps:
       # Fix 'No space left on device' errors for Ubuntu builds.
       - name: Run Actions Cleaner
@@ -294,52 +290,6 @@ jobs:
           echo "OMPI_CC=$LLVM_PREFIX/bin/clang" >> $GITHUB_ENV
           echo "OMPI_CXX=$LLVM_PREFIX/bin/clang++" >> $GITHUB_ENV
 
-      # Restore the compiler cache (ccache). The key embeds the run id, so new
-      # runs save a fresh snapshot; the restore-keys prefix warm-starts from the
-      # most recent prior run (incl. the base branch for PRs).
-      - name: cache ccache
-        if: ${{ env.USE_CCACHE == 'true' }}
-        uses: actions/cache@v5
-        with:
-          path: .ccache
-          key: ccache-${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}${{ matrix.enzyme && '-enzyme' || '' }}-${{ github.run_id }}
-          restore-keys: |
-            ccache-${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}${{ matrix.enzyme && '-enzyme' || '' }}-
-
-      # Configure ccache and select how it is injected into the MFEM build:
-      # - make:  set CXX="ccache g++"; for MPI, OMPI_CXX="ccache g++" so mpicxx
-      #          runs ccache around g++ (not ccache around the mpicxx wrapper).
-      # - cmake: set CMAKE_<LANG>_COMPILER_LAUNCHER=ccache.
-      # - enzyme: wrap the brew clang++ via OMPI_CXX.
-      # The chosen options are passed through build-mfem's 'config-options'
-      # input (see the build step below).
-      - name: configure ccache
-        if: ${{ env.USE_CCACHE == 'true' }}
-        run: |
-          command -v ccache >/dev/null 2>&1 || {
-            if [[ "${{ runner.os }}" == "Linux" ]]; then
-              sudo apt-get update && sudo apt-get install -y ccache
-            else
-              brew install ccache
-            fi
-          }
-          echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV
-          echo "CCACHE_MAXSIZE=1G" >> $GITHUB_ENV
-          echo "CCACHE_COMPILERCHECK=content" >> $GITHUB_ENV
-          # Ignore header timestamps (restamped by each checkout) so direct mode hits.
-          echo "CCACHE_SLOPPINESS=include_file_mtime,include_file_ctime,time_macros" >> $GITHUB_ENV
-          # Hash absolute paths relative to the workspace.
-          echo "CCACHE_BASEDIR=${{ github.workspace }}" >> $GITHUB_ENV
-          if [[ "${{ matrix.enzyme }}" == "true" ]]; then
-            echo "OMPI_CXX=ccache $LLVM_PREFIX/bin/clang++" >> $GITHUB_ENV
-          elif [[ "${{ matrix.build-system }}" == "cmake" ]]; then
-            echo 'CCACHE_CONFIG_OPTS=-DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache' >> $GITHUB_ENV
-          else
-            echo "OMPI_CXX=ccache g++" >> $GITHUB_ENV
-            echo 'CCACHE_CONFIG_OPTS=CXX="ccache g++" MPICXX="mpicxx"' >> $GITHUB_ENV
-          fi
-        shell: bash
-
       # MFEM build and test
       - name: build
         uses: mfem/github-actions/build-mfem@v2.7
@@ -355,14 +305,9 @@ jobs:
           metis-dir: ${{ env.METIS_TOP_DIR }}
           mfem-dir: ${{ env.MFEM_TOP_DIR }}
           precision: ${{ matrix.precision }}
-          config-options: ${{ matrix.config-opts }} ${{ env.CCACHE_CONFIG_OPTS }}
+          config-options: ${{ matrix.config-opts }}
           library-only: ${{ matrix.target == 'dbg' && matrix.os != 'ubuntu-latest' }}
 
-      - name: ccache stats
-        if: ${{ env.USE_CCACHE == 'true' }}
-        run: ccache -s
-        shell: bash
-
       # Run checks (and only checks) on debug targets
       - name: checks
         if: matrix.build-system == 'make' && matrix.target == 'dbg'

.github/workflows/cleanup-pr-caches.yml

@@ -1,42 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
----
-# A closed PR's caches can never be restored again, so delete them to free
-# space against the 10 GB per-repo cache limit.
-name: Cleanup PR caches
-
-on:
-  pull_request:
-    types: [closed]
-
-permissions:
-  actions: write
-
-jobs:
-  cleanup:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Delete caches for the closed PR
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          GH_REPO: ${{ github.repository }}
-          PR_REF: refs/pull/${{ github.event.pull_request.number }}/merge
-        run: |
-          echo "Deleting caches for $PR_REF"
-          while :; do
-            ids=$(gh cache list --ref "$PR_REF" --limit 100 --json id --jq '.[].id')
-            [ -n "$ids" ] || break
-            echo "$ids" | while read -r id; do
-              [ -n "$id" ] || continue
-              echo "Deleting cache $id"
-              gh cache delete "$id" || echo "  (already gone)"
-            done
-          done

.github/workflows/repo-check.yml

@@ -13,7 +13,6 @@ name: "Checks"
 
 permissions:
   actions: write
-  pull-requests: read
 
 on:
   push:
@@ -30,11 +29,6 @@ concurrency:
 # by checking if the workflow trigger is 'push' ("github.event_name == 'push'")
 # and if we are in a fork ("github.event.pull_request.head.repo.full_name !=
 # github.repository").
-#
-# The logic for the branch-history check is slightly different, since that check
-# also inspects the PR's labels to allow for overriding failures. In this case,
-# we run on all 'pull_request' triggers, but only run for 'push' triggers that
-# do not correspond to any open PRs.
 
 jobs:
   file-headers-check:
@@ -134,7 +128,10 @@ jobs:
 
   branch-history:
     if: |
-      github.ref != 'refs/heads/next' && github.ref != 'refs/heads/master'
+      github.ref != 'refs/heads/next' &&
+      github.ref != 'refs/heads/master' &&
+      (github.event_name == 'push' ||
+       github.event.pull_request.head.repo.full_name != github.repository)
     runs-on: ubuntu-latest
     steps:
       - name: checkout mfem
@@ -142,27 +139,7 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: check for pull request
-        id: check_pr
-        if: github.event_name == 'push'
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          pr_exists=$(gh pr list --repo "$GITHUB_REPOSITORY" \
-            --head "$GITHUB_REF_NAME" \
-            --state open \
-            --json number \
-            --jq 'length > 0')
-          echo "pr_exists=$pr_exists" >> "$GITHUB_OUTPUT"
-
       - name: branch-history
-        id: branch_history
-        if: |
-          (github.event_name == 'pull_request' ||
-           github.event_name == 'workflow_dispatch' ||
-           steps.check_pr.outputs.pr_exists == 'false')
-        continue-on-error: ${{ contains(github.event.pull_request.labels.*.name,
-                                        'branch-history-override') }}
         run: |
           # We override origin to make sure we point to the main repo.
           # This is to have consistent test results on PRs from forks.
@@ -170,9 +147,3 @@ jobs:
           git remote add origin https://github.com/mfem/mfem.git
           git checkout -b gh-actions-branch-history
           ./config/githooks/pre-push --history
-
-      - name: report branch-history override
-        if: steps.branch_history.outcome == 'failure'
-        run: |
-          echo "::warning::branch-history check failed, but the" \
-            "'branch-history-override' label is set."

.gitignore

@@ -260,7 +260,6 @@ miniapps/meshing/polar-nc
 miniapps/meshing/mesh-quality
 miniapps/meshing/hpref
 miniapps/meshing/phpref
-miniapps/meshing/pref321
 miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
 miniapps/meshing/toroid-*.mesh
@@ -356,11 +355,6 @@ miniapps/performance/refined.mesh
 miniapps/performance/mesh.*
 miniapps/performance/sol.*
 
-miniapps/plasma/g_eqdsk_viewer
-miniapps/plasma/gnuplot_eqdsk.*
-miniapps/plasma/G_EQDSK_Viewer*
-miniapps/plasma/ParaView
-
 miniapps/shifted/distance
 miniapps/shifted/ParaViewDistance
 miniapps/shifted/ParaViewLSF

.gitlab/dane-baseline.yml

@@ -102,14 +102,12 @@ report_baseline:
         mkdir -p ${MACHINE_NAME}
         rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-${BASELINE_TEST}-${CI_COMMIT_REF_SLUG}"
         rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir ${rundir})
-        status=0
-        cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir} || { status=1; }
+        cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
         printf "%s\n" "" "Pipeline URL:" "$CI_PIPELINE_URL" \
           >> ${rundir}/pipeline.txt
         # We create an autotest-email.html file, because that's how we signal
         # that there was an error / diff (temporary).
-        if [[ $status -ne 0 ]] || \
-           [[ -f ${rundir}/${BASELINE_TEST}.err ]] || \
+        if [[ -f ${rundir}/${BASELINE_TEST}.err ]] || \
            [[ -f ${rundir}/${BASELINE_TEST}-${MACHINE_NAME}.diff ]]; then
           cp ${rundir}/pipeline.txt ${rundir}/autotest-email.html
         fi

CHANGELOG

@@ -46,19 +46,8 @@ Discretization improvements
 
 - Extend FindPointsGSLIB to support surface meshes.
 
-- Added support for complex-valued mixed bilinear forms via the new classes
-  MixedSesquilinearForm and ParMixedSesquilinearForm, mirroring the existing
-  SesquilinearForm classes. Rectangular complex operators are now also
-  handled correctly by ComplexSparseMatrix::GetSystemMatrix and
-  ComplexHypreParMatrix::GetSystemMatrix, which previously assumed equal
-  trial and test spaces.
-
 Meshing improvements
 --------------------
-- Added support for nonuniform anisotropic mesh refinement on parallel quad/hex
-  meshes with arbitrary spacing in each direction. This enables in particular
-  3:1 refinement in parallel, as demonstrated in the new meshing miniapp pref321.
-
 - Added option to guarantee mesh validity during TMOP-based r-adaptivity, using
   bounds on the determinant of the mesh transformation Jacobian.
 
@@ -81,15 +70,6 @@ Linear and nonlinear solvers
 
 GPU computing
 -------------
-- Improved partial assembly for VectorDivergenceIntegrator with shared-memory
-  kernels, kernel registration, and transpose support.
-
-- Improved partial-assembly diagonal kernels for VectorMassIntegrator (shared-
-  memory specializations) and ElasticityIntegrator (no scratch Q-vector).
-
-- Added PA gradient and diagonal support for VectorConvectionNLFIntegrator
- (AssembleGradPA, AddMultGradPA, AssembleGradDiagonalPA).
-
 - Added device assembly support for 3D H(curl) VectorFEDomainLFIntegrator.
 
 - Added NVIDIA cuDSS library interface. Implementation examples have been
@@ -98,12 +78,6 @@ GPU computing
 
 - Allow specifying GPU kernel launch bounds for native and RAJA GPU backends.
 
-- Changed VectorFEMassIntegrator to use kernel specialization dispatch for
-  partial assembly.
-
-- Added support for FiniteElement::MapType::INTEGRAL spaces to
-  QuadratureInterpolator.
-
 New and updated examples and miniapps
 -------------------------------------
 - The Lorentz miniapp (in miniapps/electromagnetics) has been updated to

CMakeLists.txt

@@ -239,13 +239,6 @@ else()
   set(MFEM_DEBUG OFF)
 endif()
 
-# Shadow warnings for clang only; GCC's -Wshadow flags more.
-if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
-  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -pedantic -Wall -Wshadow")
-elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -pedantic -Wall")
-endif()
-
 # Shared build on Windows
 if (WIN32 AND BUILD_SHARED_LIBS)
   # CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS works only with MSVC?

config/defaults.mk

@@ -27,13 +27,7 @@ MPICXX = mpicxx
 
 BASE_FLAGS  = -std=c++17
 OPTIM_FLAGS = -O3 $(BASE_FLAGS)
-
-# Shadow warnings for clang only; GCC's -Wshadow flags more.
-SHADOW_WARNING_FLAG = $(if $(findstring clang,\
- $(shell $(MFEM_HOST_CXX) --version 2>/dev/null)),-Wshadow,)
-WARNING_FLAGS = -pedantic -Wall $(SHADOW_WARNING_FLAG)
-
-DEBUG_FLAGS = $(strip -g $(addprefix $(XCOMPILER),$(WARNING_FLAGS)) $(BASE_FLAGS))
+DEBUG_FLAGS = -g $(XCOMPILER)-Wall $(BASE_FLAGS)
 
 # Prefixes for passing flags to the compiler and linker when using CXX or MPICXX
 CXX_XCOMPILER =

config/githooks/README.md

@@ -39,8 +39,3 @@ when a picture was added for documentation.
 If that is the case, make sure the failure is indeed justified, and rerun the
 push command with the `--no-verify` option. This will skip the hooks, allowing
 you to push those changes.
-
-The `branch-history` check is run automatically through GitHub Actions. If a
-branch is known to have a large number of changes that are legitimate, the
-check can be overridden by setting  the label 'branch-history-override' on the
-pull request.

data/tinyzoo-2d.mesh

@@ -1,38 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see fem/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-# PYRAMID     = 7
-
-dimension
-2
-
-elements
-2
-1 3 0 1 4 3
-1 2 1 2 4
-
-boundary
-5
-1 1 0 1
-1 1 1 2
-1 1 2 4
-1 1 4 3
-1 1 3 0
-
-vertices
-5
-2
-0 0
-1 0
-2 0
-0 1
-1 1

doc/CodeDocumentation.dox

@@ -201,7 +201,6 @@ namespace mfem {
  * - <a class="el" href="nurbs__naca__cmesh_8cpp_source.html">NURBS NACA Mesher</a>: generate NURBS based mesh around a NACA foil
  * - <a class="el" href="nurbs__printfunc_8cpp_source.html">NURBS Printer</a>: print the NURBS-basis
  * - <a class="el" href="nurbs__mesh_info_8cpp_source.html">NURBS Mesh info</a>: print the info of a NURBS mesh
- * - <a class="el" href="nurbs__surface_8cpp_source.html">NURBS Surface</a>: interpolate a 3D Surface in a NURBS Patch
  *
  * <H3>Miniapps</H3>
  * - <a class="el" href="volta_8cpp_source.html">Volta</a>: simple electrostatics simulation code
@@ -246,9 +245,6 @@ namespace mfem {
  * - <a class="el" href="pdiffusion_8cpp_source.html">DPG Diffusion example</a>: DPG formulation for the diffusion problem
  * - <a class="el" href="pmaxwell_8cpp_source.html">DPG Maxwell example</a>: DPG formulation for the indefinite Maxwell problem
  * - <a class="el" href="lor__elast_8cpp_source.html">LOR Elasticity</a>: solve linear elasticity with LOR preconditioning on GPUs
- * - <a class="el" href="reflector_8cpp_source.html">Reflector Miniapp</a>: reflect a mesh about a plane
- * - <a class="el" href="ref321_8cpp_source.html">3:1 Refinement Miniapp</a>: perform 3:1 anisotropic mesh refinements
- * - <a class="el" href="pref321_8cpp_source.html">3:1 Refinement Miniapp</a>: parallel 3:1 anisotropic mesh refinements
  *
  * See also the <a class="el" href="https://mfem.org/examples/">examples documentation</a> online.
  */

fem/CMakeLists.txt

@@ -57,8 +57,6 @@ set(SRCS
   integ/lininteg_domain_grad.cpp
   integ/lininteg_domain_vectorfe.cpp
   integ/nonlininteg_vecconvection_pa.cpp
-  integ/nonlininteg_vecconvection_pa_diag.cpp
-  integ/nonlininteg_vecconvection_pa_grad.cpp
   integ/nonlininteg_vecconvection_mf.cpp
   coefficient.cpp
   complex_fem.cpp
@@ -206,11 +204,7 @@ set(HDRS
   integ/bilininteg_mass_kernels.hpp
   integ/bilininteg_mass_pa_simplices.hpp
   integ/bilininteg_vecdiffusion_pa.hpp
-  integ/bilininteg_vecdiv_pa.hpp
   integ/bilininteg_vecmass_pa.hpp
-  integ/nonlininteg_vecconvection_pa.hpp
-  integ/nonlininteg_vecconvection_pa_diag.hpp
-  integ/nonlininteg_vecconvection_pa_grad.hpp
   coefficient.hpp
   complex_fem.hpp
   convergence.hpp

fem/bilininteg.hpp

@@ -2689,22 +2689,14 @@ public:
    void AddMultMF(const Vector &x, Vector &y) const override;
    bool SupportsCeed() const override { return DeviceCanUseCeed(); }
 
-   // PA AddMultPA kernels
    using VectorMassAddMultPAType =
       void(*)(const int,  const int,
               const Array<real_t>&, const Vector&,
               const Vector&, Vector&, const int, const int);
+
    MFEM_REGISTER_KERNELS(VectorMassAddMultPA,
                          VectorMassAddMultPAType,
                          (int, int, int));
-
-   // PA DiagonalPA kernels
-   using VectorMassAssembleDiagonalPAType =
-      void(*)(const int, const int, const int,
-              const real_t*, const real_t*, real_t*);
-   MFEM_REGISTER_KERNELS(VectorMassAssembleDiagonalPA,
-                         VectorMassAssembleDiagonalPAType,
-                         (int /*dim*/, int /*q1d*/));
 };
 
 
@@ -3003,10 +2995,11 @@ public:
     vector (diagonal matrix), or matrix), trial function $u$ is in $H(curl$ or
     $H(div)$, and test function $v$ is in $H(curl$, $H(div)$, or $v=(v_1,\dots,v_n)$, where
     $v_i$ are in $H^1$. */
-class VectorFEMassIntegrator : public BilinearFormIntegrator
+class VectorFEMassIntegrator: public BilinearFormIntegrator
 {
 private:
-   void Init(Coefficient *q, DiagonalMatrixCoefficient *dq, MatrixCoefficient *mq);
+   void Init(Coefficient *q, DiagonalMatrixCoefficient *dq, MatrixCoefficient *mq)
+   { Q = q; DQ = dq; MQ = mq; }
 
 #ifndef MFEM_THREAD_SAFE
    Vector shape;
@@ -3029,8 +3022,7 @@ protected:
    const DofToQuad *mapsOtest;     ///< Not owned. DOF-to-quad map, open.
    const DofToQuad *mapsCtest;     ///< Not owned. DOF-to-quad map, closed.
    const GeometricFactors *geom;   ///< Not owned
-   int dim, ne, nq, dofs1D, dofs1Dtest, quad1D;
-   FiniteElement::DerivType trial_fetype, test_fetype;
+   int dim, ne, nq, dofs1D, dofs1Dtest, quad1D, trial_fetype, test_fetype;
    bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient
 
 public:
@@ -3061,29 +3053,6 @@ public:
                    const bool add) override;
 
    const Coefficient *GetCoefficient() const { return Q; }
-
-   using ApplyKernelType =
-      void (*)(const int NE, bool symmetric, const bool scalar_coeff,
-               const Array<real_t> &trialBO, const Array<real_t> &trialBC,
-               const Array<real_t> &testBOt, const Array<real_t> &testBCt,
-               const Vector &pa_data, const Vector &x, Vector &y,
-               const int triald1d, const int testd1d, const int q1d);
-
-   /// parameters: trial_fetype, test_fetype, ndims, trial_d1d, test_d1d, q1d
-   MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType,
-                         (FiniteElement::DerivType, FiniteElement::DerivType,
-                          int, int, int, int));
-
-   struct Kernels { Kernels(); };
-
-   template <FiniteElement::DerivType TrialType,
-             FiniteElement::DerivType TestType, int DIM, int TRIAL_D1D,
-             int TEST_D1D, int Q1D>
-   static void AddSpecialization()
-   {
-      ApplyPAKernels::Specialization<TrialType, TestType, DIM, TRIAL_D1D,
-                     TEST_D1D, Q1D>::Add();
-   }
 };
 
 /** Integrator for $(Q \nabla \cdot u, v)$ where $u=(u_1,\cdots,u_n)$ and all $u_i$ are in the same
@@ -3129,24 +3098,6 @@ public:
    void AddMultPA(const Vector &x, Vector &y) const override;
    void AddMultTransposePA(const Vector &x, Vector &y) const override;
 
-   using VectorDivergenceAddMultPAType =
-      void (*)(const int ne,
-               const Array<real_t> &b, const Array<real_t> &g, const Array<real_t> &bt,
-               const Vector &op, const Vector &x, Vector &y,
-               const int tr_d1d, const int te_d1d, const int q1d);
-   MFEM_REGISTER_KERNELS(VectorDivergenceAddMultPA,
-                         VectorDivergenceAddMultPAType,
-                         (int, int, int, int));
-
-   using VectorDivergenceAddMultTransposePAType =
-      void (*)(const int ne,
-               const Array<real_t> &bt, const Array<real_t> &gt, const Array<real_t> &b,
-               const Vector &q, const Vector &x, Vector &y,
-               const int tr_d1d, const int te_d1d, const int q1d);
-   MFEM_REGISTER_KERNELS(VectorDivergenceAddMultTransposePA,
-                         VectorDivergenceAddMultTransposePAType,
-                         (int, int, int, int));
-
    static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
                                          const FiniteElement &test_fe,
                                          const ElementTransformation &Trans);

fem/complex_fem.cpp

@@ -718,8 +718,8 @@ SesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
 
    // A = A_r + i A_i
    A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::MFEM_SPARSEMAT) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::MFEM_SPARSEMAT))
+   if ( A_r.Type() == Operator::MFEM_SPARSEMAT ||
+        A_i.Type() == Operator::MFEM_SPARSEMAT )
    {
       ComplexSparseMatrix * A_sp =
          new ComplexSparseMatrix(A_r.As<SparseMatrix>(),
@@ -779,8 +779,8 @@ SesquilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
 
    // A = A_r + i A_i
    A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::MFEM_SPARSEMAT) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::MFEM_SPARSEMAT))
+   if ( A_r.Type() == Operator::MFEM_SPARSEMAT ||
+        A_i.Type() == Operator::MFEM_SPARSEMAT )
    {
       ComplexSparseMatrix * A_sp =
          new ComplexSparseMatrix(A_r.As<SparseMatrix>(),
@@ -843,426 +843,6 @@ SesquilinearForm::Update(FiniteElementSpace *nfes)
    if ( blfi ) { blfi->Update(nfes); }
 }
 
-bool
-MixedSesquilinearForm::RealInteg()
-{
-   int nint = mblfr->GetDBFI()->Size() + mblfr->GetBBFI()->Size() +
-              mblfr->GetFBFI()->Size() + mblfr->GetBFBFI()->Size() +
-              mblfr->GetTFBFI()->Size() + mblfr->GetBTFBFI()->Size();
-   return (nint != 0);
-}
-
-bool
-MixedSesquilinearForm::ImagInteg()
-{
-   int nint = mblfi->GetDBFI()->Size() + mblfi->GetBBFI()->Size() +
-              mblfi->GetFBFI()->Size() + mblfi->GetBFBFI()->Size() +
-              mblfi->GetTFBFI()->Size() + mblfi->GetBTFBFI()->Size();
-   return (nint != 0);
-}
-
-MixedSesquilinearForm::MixedSesquilinearForm(FiniteElementSpace * trial_fes,
-                                             FiniteElementSpace * test_fes,
-                                             ComplexOperator::Convention convention)
-   : conv(convention),
-     mblfr(new mfem::MixedBilinearForm(trial_fes, test_fes)),
-     mblfi(new mfem::MixedBilinearForm(trial_fes, test_fes))
-{
-}
-
-MixedSesquilinearForm::MixedSesquilinearForm(FiniteElementSpace * trial_fes,
-                                             FiniteElementSpace * test_fes,
-                                             MixedBilinearForm * bfr,
-                                             MixedBilinearForm * bfi,
-                                             ComplexOperator::Convention convention)
-   : conv(convention),
-     mblfr(new MixedBilinearForm(trial_fes, test_fes, bfr)),
-     mblfi(new MixedBilinearForm(trial_fes, test_fes, bfi))
-{
-}
-
-MixedSesquilinearForm::~MixedSesquilinearForm()
-{
-   delete mblfr;
-   delete mblfi;
-}
-
-void
-MixedSesquilinearForm::AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                                           BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      mblfr->AddDomainIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddDomainIntegrator(bfi_imag);
-   }
-}
-
-void
-MixedSesquilinearForm::AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                                           BilinearFormIntegrator * bfi_imag,
-                                           Array<int> & elem_marker)
-{
-   if (bfi_real)
-   {
-      mblfr->AddDomainIntegrator(bfi_real, elem_marker);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddDomainIntegrator(bfi_imag, elem_marker);
-   }
-}
-
-void
-MixedSesquilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
-                                             BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      mblfr->AddBoundaryIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddBoundaryIntegrator(bfi_imag);
-   }
-}
-
-void
-MixedSesquilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
-                                             BilinearFormIntegrator * bfi_imag,
-                                             Array<int> & bdr_marker)
-{
-   if (bfi_real)
-   {
-      mblfr->AddBoundaryIntegrator(bfi_real, bdr_marker);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddBoundaryIntegrator(bfi_imag, bdr_marker);
-   }
-}
-
-void
-MixedSesquilinearForm::AddInteriorFaceIntegrator(BilinearFormIntegrator *
-                                                 bfi_real,
-                                                 BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      mblfr->AddInteriorFaceIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddInteriorFaceIntegrator(bfi_imag);
-   }
-}
-
-void
-MixedSesquilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                            BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      mblfr->AddBdrFaceIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddBdrFaceIntegrator(bfi_imag);
-   }
-}
-
-void
-MixedSesquilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                            BilinearFormIntegrator * bfi_imag,
-                                            Array<int> & bdr_marker)
-{
-   if (bfi_real)
-   {
-      mblfr->AddBdrFaceIntegrator(bfi_real, bdr_marker);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddBdrFaceIntegrator(bfi_imag, bdr_marker);
-   }
-}
-
-void MixedSesquilinearForm::AddTraceFaceIntegrator(BilinearFormIntegrator *
-                                                   bfi_real,
-                                                   BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      mblfr->AddTraceFaceIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddTraceFaceIntegrator(bfi_imag);
-   }
-}
-
-void MixedSesquilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator
-                                                      *bfi_real,
-                                                      BilinearFormIntegrator *bfi_imag)
-{
-   if (bfi_real)
-   {
-      mblfr->AddBdrTraceFaceIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddBdrTraceFaceIntegrator(bfi_imag);
-   }
-}
-
-void MixedSesquilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator
-                                                      *bfi_real,
-                                                      BilinearFormIntegrator *bfi_imag,
-                                                      Array<int> &bdr_marker)
-{
-   if (bfi_real)
-   {
-      mblfr->AddBdrTraceFaceIntegrator(bfi_real, bdr_marker);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddBdrTraceFaceIntegrator(bfi_imag, bdr_marker);
-   }
-}
-
-void
-MixedSesquilinearForm::Assemble(int skip_zeros)
-{
-   mblfr->Assemble(skip_zeros);
-   mblfi->Assemble(skip_zeros);
-}
-
-void
-MixedSesquilinearForm::Finalize(int skip_zeros)
-{
-   mblfr->Finalize(skip_zeros);
-   mblfi->Finalize(skip_zeros);
-}
-
-ComplexSparseMatrix *
-MixedSesquilinearForm::AssembleComplexSparseMatrix()
-{
-   return new mfem::ComplexSparseMatrix(
-             &mblfr->SpMat(), &mblfi->SpMat(), false, false, conv);
-}
-
-void
-MixedSesquilinearForm::FormRectangularLinearSystem(const Array<int> &
-                                                   ess_trial_tdof_list,
-                                                   const Array<int> & ess_test_tdof_list,
-                                                   Vector & x,
-                                                   Vector & b,
-                                                   OperatorHandle & A,
-                                                   Vector & X,
-                                                   Vector & B)
-{
-   FiniteElementSpace * fes_trial = mblfr->TrialFESpace();
-   FiniteElementSpace * fes_test = mblfr->TestFESpace();
-   const int vsize_trial = fes_trial->GetVSize();
-   const int vsize_test = fes_test->GetVSize();
-
-   // Allocate temporary Vector
-   Vector b_0;
-   b_0.UseDevice(true);
-   b_0.SetSize(vsize_test);
-   b_0 = 0.0;
-
-   // Extract the real and imaginary parts of the input Vectors
-   MFEM_ASSERT(x.Size() == 2 * vsize_trial,
-               "Input GridFunction of incorrect size!");
-   x.Read();
-   Vector x_r;
-   x_r.MakeRef(x, 0, vsize_trial);
-   Vector x_i;
-   x_i.MakeRef(x, vsize_trial, vsize_trial);
-
-   MFEM_ASSERT(b.Size() == 2 * vsize_test, "Input LinearForm of incorrect size!");
-   b.Read();
-   Vector b_r;
-   b_r.MakeRef(b, 0, vsize_test);
-   Vector b_i;
-   b_i.MakeRef(b, vsize_test, vsize_test);
-
-   if (conv == ComplexOperator::BLOCK_SYMMETRIC)
-   {
-      b_i *= -1.0;
-   }
-
-   const int tvsize_trial = fes_trial->GetTrueVSize();
-   const int tvsize_test = fes_test->GetTrueVSize();
-   OperatorHandle A_r, A_i;
-
-   X.UseDevice(true);
-   X.SetSize(2 * tvsize_trial);
-   X = 0.0;
-
-   B.UseDevice(true);
-   B.SetSize(2 * tvsize_test);
-   B = 0.0;
-
-   Vector X_r;
-   X_r.MakeRef(X, 0, tvsize_trial);
-   Vector X_i;
-   X_i.MakeRef(X, tvsize_trial, tvsize_trial);
-   Vector B_r;
-   B_r.MakeRef(B, 0, tvsize_test);
-   Vector B_i;
-   B_i.MakeRef(B, tvsize_test, tvsize_test);
-
-   Vector X_0, B_0;
-
-   if (RealInteg())
-   {
-      b_0 = b_r;
-      mblfr->FormRectangularLinearSystem(
-         ess_trial_tdof_list, ess_test_tdof_list, x_r, b_0, A_r, X_0, B_0);
-      X_r = X_0;
-      B_r = B_0;
-
-      b_0 = b_i;
-      mblfr->FormRectangularLinearSystem(
-         ess_trial_tdof_list, ess_test_tdof_list, x_i, b_0, A_r, X_0, B_0);
-      X_i = X_0;
-      B_i = B_0;
-
-      if (ImagInteg())
-      {
-         b_0 = 0.0;
-         mblfi->FormRectangularLinearSystem(
-            ess_trial_tdof_list, ess_test_tdof_list, x_i, b_0, A_i, X_0, B_0);
-         B_r -= B_0;
-
-         b_0 = 0.0;
-         mblfi->FormRectangularLinearSystem(
-            ess_trial_tdof_list, ess_test_tdof_list, x_r, b_0, A_i, X_0, B_0);
-         B_i += B_0;
-      }
-   }
-   else if (ImagInteg())
-   {
-      b_0 = b_i;
-      mblfi->FormRectangularLinearSystem(
-         ess_trial_tdof_list, ess_test_tdof_list, x_r, b_0, A_i, X_0, B_0);
-      X_r = X_0;
-      B_i = B_0;
-
-      b_0 = b_r;
-      b_0 *= -1.0;
-      mblfi->FormRectangularLinearSystem(
-         ess_trial_tdof_list, ess_test_tdof_list, x_i, b_0, A_i, X_0, B_0);
-      X_i = X_0;
-      B_r = B_0;
-      B_r *= -1.0;
-   }
-   else
-   {
-      MFEM_ABORT("Real and Imaginary part of the Mixed Sesquilinear form are empty");
-   }
-
-   if (conv == ComplexOperator::BLOCK_SYMMETRIC)
-   {
-      B_i *= -1.0;
-      b_i *= -1.0;
-   }
-
-   x_r.SyncAliasMemory(x);
-   x_i.SyncAliasMemory(x);
-   b_r.SyncAliasMemory(b);
-   b_i.SyncAliasMemory(b);
-
-   X_r.SyncAliasMemory(X);
-   X_i.SyncAliasMemory(X);
-   B_r.SyncAliasMemory(B);
-   B_i.SyncAliasMemory(B);
-
-   // A = A_r + i A_i
-   A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::MFEM_SPARSEMAT) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::MFEM_SPARSEMAT))
-   {
-      ComplexSparseMatrix * A_hyp =
-         new ComplexSparseMatrix(A_r.As<SparseMatrix>(),
-                                 A_i.As<SparseMatrix>(),
-                                 A_r.OwnsOperator(),
-                                 A_i.OwnsOperator(),
-                                 conv);
-      A.Reset<ComplexSparseMatrix>(A_hyp, true);
-   }
-   else
-   {
-      ComplexOperator * A_op = new ComplexOperator(A_r.As<Operator>(),
-                                                   A_i.As<Operator>(),
-                                                   A_r.OwnsOperator(),
-                                                   A_i.OwnsOperator(),
-                                                   conv);
-      A.Reset<ComplexOperator>(A_op, true);
-   }
-   A_r.SetOperatorOwner(false);
-   A_i.SetOperatorOwner(false);
-}
-
-void
-MixedSesquilinearForm::FormRectangularSystemMatrix(const mfem::Array<int> &
-                                                   ess_trial_tdof_list,
-                                                   const mfem::Array<int> & ess_test_tdof_list,
-                                                   mfem::OperatorHandle & A)
-{
-   OperatorHandle A_r, A_i;
-   if (RealInteg())
-   {
-      mblfr->FormRectangularSystemMatrix(ess_trial_tdof_list, ess_test_tdof_list,
-                                         A_r);
-   }
-   if (ImagInteg())
-   {
-      mblfi->FormRectangularSystemMatrix(ess_trial_tdof_list, ess_test_tdof_list,
-                                         A_i);
-   }
-   if (!RealInteg() && !ImagInteg())
-   {
-      MFEM_ABORT("Both Real and Imaginary part of the Mixed Sesquilinear form are empty");
-   }
-
-   // A = A_r + i A_i
-   A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::MFEM_SPARSEMAT) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::MFEM_SPARSEMAT))
-   {
-      ComplexSparseMatrix * A_hyp =
-         new ComplexSparseMatrix(A_r.As<SparseMatrix>(),
-                                 A_i.As<SparseMatrix>(),
-                                 A_r.OwnsOperator(),
-                                 A_i.OwnsOperator(),
-                                 conv);
-      A.Reset<ComplexSparseMatrix>(A_hyp, true);
-   }
-   else
-   {
-      ComplexOperator * A_op = new ComplexOperator(A_r.As<Operator>(),
-                                                   A_i.As<Operator>(),
-                                                   A_r.OwnsOperator(),
-                                                   A_i.OwnsOperator(),
-                                                   conv);
-      A.Reset<ComplexOperator>(A_op, true);
-   }
-   A_r.SetOperatorOwner(false);
-   A_i.SetOperatorOwner(false);
-}
-
-void
-MixedSesquilinearForm::Update()
-{
-   mblfr->Update();
-   mblfi->Update();
-}
-
 
 #ifdef MFEM_USE_MPI
 
@@ -2034,8 +1614,8 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
 
    // A = A_r + i A_i
    A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::Hypre_ParCSR) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::Hypre_ParCSR))
+   if ( A_r.Type() == Operator::Hypre_ParCSR ||
+        A_i.Type() == Operator::Hypre_ParCSR )
    {
       ComplexHypreParMatrix * A_hyp =
          new ComplexHypreParMatrix(A_r.As<HypreParMatrix>(),
@@ -2102,8 +1682,8 @@ ParSesquilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
 
    // A = A_r + i A_i
    A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::Hypre_ParCSR) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::Hypre_ParCSR))
+   if ( A_r.Type() == Operator::Hypre_ParCSR ||
+        A_i.Type() == Operator::Hypre_ParCSR )
    {
       ComplexHypreParMatrix * A_hyp =
          new ComplexHypreParMatrix(A_r.As<HypreParMatrix>(),
@@ -2161,434 +1741,6 @@ ParSesquilinearForm::Update(FiniteElementSpace *nfes)
    if ( pblfi ) { pblfi->Update(nfes); }
 }
 
-bool
-ParMixedSesquilinearForm::RealInteg()
-{
-   int nint = pmblfr->GetDBFI()->Size() + pmblfr->GetBBFI()->Size() +
-              pmblfr->GetFBFI()->Size() + pmblfr->GetBFBFI()->Size() +
-              pmblfr->GetTFBFI()->Size() + pmblfr->GetBTFBFI()->Size();
-   return (nint != 0);
-}
-
-bool
-ParMixedSesquilinearForm::ImagInteg()
-{
-   int nint = pmblfi->GetDBFI()->Size() + pmblfi->GetBBFI()->Size() +
-              pmblfi->GetFBFI()->Size() + pmblfi->GetBFBFI()->Size() +
-              pmblfi->GetTFBFI()->Size() + pmblfi->GetBTFBFI()->Size();
-   return (nint != 0);
-}
-
-ParMixedSesquilinearForm::ParMixedSesquilinearForm(ParFiniteElementSpace *
-                                                   trial_fes,
-                                                   ParFiniteElementSpace * test_fes,
-                                                   ComplexOperator::Convention convention)
-   : conv(convention),
-     pmblfr(new ParMixedBilinearForm(trial_fes, test_fes)),
-     pmblfi(new ParMixedBilinearForm(trial_fes, test_fes))
-{
-}
-
-ParMixedSesquilinearForm::ParMixedSesquilinearForm(ParFiniteElementSpace *
-                                                   trial_fes,
-                                                   ParFiniteElementSpace * test_fes,
-                                                   ParMixedBilinearForm * pbfr,
-                                                   ParMixedBilinearForm * pbfi,
-                                                   ComplexOperator::Convention convention)
-   : conv(convention),
-     pmblfr(new ParMixedBilinearForm(trial_fes, test_fes, pbfr)),
-     pmblfi(new ParMixedBilinearForm(trial_fes, test_fes, pbfi))
-{
-}
-
-ParMixedSesquilinearForm::~ParMixedSesquilinearForm()
-{
-   delete pmblfr;
-   delete pmblfi;
-}
-
-void
-ParMixedSesquilinearForm::AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                                              BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddDomainIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddDomainIntegrator(bfi_imag);
-   }
-}
-
-void
-ParMixedSesquilinearForm::AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                                              BilinearFormIntegrator * bfi_imag,
-                                              Array<int> & elem_marker)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddDomainIntegrator(bfi_real, elem_marker);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddDomainIntegrator(bfi_imag, elem_marker);
-   }
-}
-
-void
-ParMixedSesquilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator *
-                                                bfi_real,
-                                                BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddBoundaryIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddBoundaryIntegrator(bfi_imag);
-   }
-}
-
-void
-ParMixedSesquilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator *
-                                                bfi_real,
-                                                BilinearFormIntegrator * bfi_imag,
-                                                Array<int> & bdr_marker)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddBoundaryIntegrator(bfi_real, bdr_marker);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddBoundaryIntegrator(bfi_imag, bdr_marker);
-   }
-}
-
-void
-ParMixedSesquilinearForm::AddInteriorFaceIntegrator(BilinearFormIntegrator *
-                                                    bfi_real,
-                                                    BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddInteriorFaceIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddInteriorFaceIntegrator(bfi_imag);
-   }
-}
-
-void
-ParMixedSesquilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator *
-                                               bfi_real,
-                                               BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddBdrFaceIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddBdrFaceIntegrator(bfi_imag);
-   }
-}
-
-void
-ParMixedSesquilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator *
-                                               bfi_real,
-                                               BilinearFormIntegrator * bfi_imag,
-                                               Array<int> & bdr_marker)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddBdrFaceIntegrator(bfi_real, bdr_marker);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddBdrFaceIntegrator(bfi_imag, bdr_marker);
-   }
-}
-
-void ParMixedSesquilinearForm::AddTraceFaceIntegrator(BilinearFormIntegrator *
-                                                      bfi_real,
-                                                      BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddTraceFaceIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddTraceFaceIntegrator(bfi_imag);
-   }
-}
-
-void ParMixedSesquilinearForm::AddBdrTraceFaceIntegrator(
-   BilinearFormIntegrator *bfi_real,
-   BilinearFormIntegrator *bfi_imag)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddBdrTraceFaceIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddBdrTraceFaceIntegrator(bfi_imag);
-   }
-}
-
-void ParMixedSesquilinearForm::AddBdrTraceFaceIntegrator(
-   BilinearFormIntegrator *bfi_real,
-   BilinearFormIntegrator *bfi_imag,
-   Array<int> &bdr_marker)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddBdrTraceFaceIntegrator(bfi_real, bdr_marker);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddBdrTraceFaceIntegrator(bfi_imag, bdr_marker);
-   }
-}
-
-void
-ParMixedSesquilinearForm::Assemble(int skip_zeros)
-{
-   pmblfr->Assemble(skip_zeros);
-   pmblfi->Assemble(skip_zeros);
-}
-
-void
-ParMixedSesquilinearForm::Finalize(int skip_zeros)
-{
-   pmblfr->Finalize(skip_zeros);
-   pmblfi->Finalize(skip_zeros);
-}
-
-ComplexHypreParMatrix *
-ParMixedSesquilinearForm::ParallelAssemble()
-{
-   return new ComplexHypreParMatrix(
-             pmblfr->ParallelAssemble(), pmblfi->ParallelAssemble(), true, true, conv);
-}
-
-void
-ParMixedSesquilinearForm::FormRectangularLinearSystem(const Array<int> &
-                                                      ess_trial_tdof_list,
-                                                      const Array<int> & ess_test_tdof_list,
-                                                      Vector & x,
-                                                      Vector & b,
-                                                      OperatorHandle & A,
-                                                      Vector & X,
-                                                      Vector & B)
-{
-   FiniteElementSpace * pfes_trial = pmblfr->TrialFESpace();
-   FiniteElementSpace * pfes_test = pmblfr->TestFESpace();
-   const int vsize_trial = pfes_trial->GetVSize();
-   const int vsize_test = pfes_test->GetVSize();
-
-   // Allocate temporary Vector
-   Vector b_0;
-   b_0.UseDevice(true);
-   b_0.SetSize(vsize_test);
-   b_0 = 0.0;
-
-   // Extract the real and imaginary parts of the input Vectors
-   MFEM_ASSERT(x.Size() == 2 * vsize_trial,
-               "Input GridFunction of incorrect size!");
-   x.Read();
-   Vector x_r;
-   x_r.MakeRef(x, 0, vsize_trial);
-   Vector x_i;
-   x_i.MakeRef(x, vsize_trial, vsize_trial);
-
-   MFEM_ASSERT(b.Size() == 2 * vsize_test, "Input LinearForm of incorrect size!");
-   b.Read();
-   Vector b_r;
-   b_r.MakeRef(b, 0, vsize_test);
-   Vector b_i;
-   b_i.MakeRef(b, vsize_test, vsize_test);
-
-   if (conv == ComplexOperator::BLOCK_SYMMETRIC)
-   {
-      b_i *= -1.0;
-   }
-
-   const int tvsize_trial = pfes_trial->GetTrueVSize();
-   const int tvsize_test = pfes_test->GetTrueVSize();
-   OperatorHandle A_r, A_i;
-
-   X.UseDevice(true);
-   X.SetSize(2 * tvsize_trial);
-   X = 0.0;
-
-   B.UseDevice(true);
-   B.SetSize(2 * tvsize_test);
-   B = 0.0;
-
-   Vector X_r;
-   X_r.MakeRef(X, 0, tvsize_trial);
-   Vector X_i;
-   X_i.MakeRef(X, tvsize_trial, tvsize_trial);
-   Vector B_r;
-   B_r.MakeRef(B, 0, tvsize_test);
-   Vector B_i;
-   B_i.MakeRef(B, tvsize_test, tvsize_test);
-
-   Vector X_0, B_0;
-
-   if (RealInteg())
-   {
-      b_0 = b_r;
-      pmblfr->FormRectangularLinearSystem(
-         ess_trial_tdof_list, ess_test_tdof_list, x_r, b_0, A_r, X_0, B_0);
-      X_r = X_0;
-      B_r = B_0;
-
-      b_0 = b_i;
-      pmblfr->FormRectangularLinearSystem(
-         ess_trial_tdof_list, ess_test_tdof_list, x_i, b_0, A_r, X_0, B_0);
-      X_i = X_0;
-      B_i = B_0;
-
-      if (ImagInteg())
-      {
-         b_0 = 0.0;
-         pmblfi->FormRectangularLinearSystem(
-            ess_trial_tdof_list, ess_test_tdof_list, x_i, b_0, A_i, X_0, B_0);
-         B_r -= B_0;
-
-         b_0 = 0.0;
-         pmblfi->FormRectangularLinearSystem(
-            ess_trial_tdof_list, ess_test_tdof_list, x_r, b_0, A_i, X_0, B_0);
-         B_i += B_0;
-      }
-   }
-   else if (ImagInteg())
-   {
-      b_0 = b_i;
-      pmblfi->FormRectangularLinearSystem(
-         ess_trial_tdof_list, ess_test_tdof_list, x_r, b_0, A_i, X_0, B_0);
-      X_r = X_0;
-      B_i = B_0;
-
-      b_0 = b_r;
-      b_0 *= -1.0;
-      pmblfi->FormRectangularLinearSystem(
-         ess_trial_tdof_list, ess_test_tdof_list, x_i, b_0, A_i, X_0, B_0);
-      X_i = X_0;
-      B_r = B_0;
-      B_r *= -1.0;
-   }
-   else
-   {
-      MFEM_ABORT("Real and Imaginary part of the Mixed Sesquilinear form are empty");
-   }
-
-   if (conv == ComplexOperator::BLOCK_SYMMETRIC)
-   {
-      B_i *= -1.0;
-      b_i *= -1.0;
-   }
-
-   x_r.SyncAliasMemory(x);
-   x_i.SyncAliasMemory(x);
-   b_r.SyncAliasMemory(b);
-   b_i.SyncAliasMemory(b);
-
-   X_r.SyncAliasMemory(X);
-   X_i.SyncAliasMemory(X);
-   B_r.SyncAliasMemory(B);
-   B_i.SyncAliasMemory(B);
-
-   // A = A_r + i A_i
-   A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::Hypre_ParCSR) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::Hypre_ParCSR))
-   {
-      ComplexHypreParMatrix * A_hyp =
-         new ComplexHypreParMatrix(A_r.As<HypreParMatrix>(),
-                                   A_i.As<HypreParMatrix>(),
-                                   A_r.OwnsOperator(),
-                                   A_i.OwnsOperator(),
-                                   conv);
-      A.Reset<ComplexHypreParMatrix>(A_hyp, true);
-   }
-   else
-   {
-      ComplexOperator * A_op = new ComplexOperator(A_r.As<Operator>(),
-                                                   A_i.As<Operator>(),
-                                                   A_r.OwnsOperator(),
-                                                   A_i.OwnsOperator(),
-                                                   conv);
-      A.Reset<ComplexOperator>(A_op, true);
-   }
-   A_r.SetOperatorOwner(false);
-   A_i.SetOperatorOwner(false);
-}
-
-void
-ParMixedSesquilinearForm::FormRectangularSystemMatrix(const Array<int> &
-                                                      ess_trial_tdof_list,
-                                                      const Array<int> & ess_test_tdof_list,
-                                                      OperatorHandle & A)
-{
-   OperatorHandle A_r, A_i;
-   if (RealInteg())
-   {
-      pmblfr->FormRectangularSystemMatrix(ess_trial_tdof_list, ess_test_tdof_list,
-                                          A_r);
-   }
-   if (ImagInteg())
-   {
-      pmblfi->FormRectangularSystemMatrix(ess_trial_tdof_list, ess_test_tdof_list,
-                                          A_i);
-   }
-   if (!RealInteg() && !ImagInteg())
-   {
-      MFEM_ABORT("Both Real and Imaginary part of the Mixed Sesquilinear form are empty");
-   }
-
-   // A = A_r + i A_i
-   A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::Hypre_ParCSR) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::Hypre_ParCSR))
-   {
-      ComplexHypreParMatrix * A_hyp =
-         new ComplexHypreParMatrix(A_r.As<HypreParMatrix>(),
-                                   A_i.As<HypreParMatrix>(),
-                                   A_r.OwnsOperator(),
-                                   A_i.OwnsOperator(),
-                                   conv);
-      A.Reset<ComplexHypreParMatrix>(A_hyp, true);
-   }
-   else
-   {
-      ComplexOperator * A_op = new ComplexOperator(A_r.As<Operator>(),
-                                                   A_i.As<Operator>(),
-                                                   A_r.OwnsOperator(),
-                                                   A_i.OwnsOperator(),
-                                                   conv);
-      A.Reset<ComplexOperator>(A_op, true);
-   }
-   A_r.SetOperatorOwner(false);
-   A_i.SetOperatorOwner(false);
-}
-
-void
-ParMixedSesquilinearForm::Update()
-{
-   pmblfr->Update();
-   pmblfi->Update();
-}
-
-
-
 #endif // MFEM_USE_MPI
 
 }

fem/complex_fem.hpp

@@ -505,186 +505,6 @@ public:
    virtual ~SesquilinearForm();
 };
 
-/** Class for a mixed sesquilinear form
-
-    A mixed sesquilinear form is a generalization of a mixed bilinear form to
-    complex-valued fields. Mixed sesquilinear forms are linear in the second
-    argument but the first argument involves a complex conjugate in the sense
-    that:
-
-                a(alpha u, beta v) = conj(alpha) beta a(u, v)
-
-    The @a convention argument in the class's constructor is documented in the
-    mfem::ComplexOperator class found in linalg/complex_operator.hpp.
-
-    When supplying integrators to the MixedSesquilinearForm either the real or
-    imaginary integrator can be NULL. This indicates that the corresponding
-    portion of the complex-valued material coefficient is equal to zero.
-*/
-class MixedSesquilinearForm
-{
-private:
-   ComplexOperator::Convention conv;
-
-   MixedBilinearForm * mblfr;
-   MixedBilinearForm * mblfi;
-
-   /* These methods check if the real/imag parts of the sesqulinear form are not
-      empty */
-   bool RealInteg();
-   bool ImagInteg();
-
-public:
-   MixedSesquilinearForm(
-      FiniteElementSpace * trial_fes,
-      FiniteElementSpace * test_fes,
-      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
-
-   /** @brief Create a MixedSesquilinearForm on the given trial and test
-       FiniteElementSpaces, using the same integrators as the
-       MixedBilinearForms @a bfr and @a bfi.
-
-       The FiniteElementSpace pointers are not owned by the newly constructed
-       object.
-
-       The integrators are copied as pointers and they are not owned by the
-       newly constructed MixedSesquilinearForm. */
-   MixedSesquilinearForm(
-      FiniteElementSpace * trial_fes,
-      FiniteElementSpace * test_fes,
-      MixedBilinearForm * bfr,
-      MixedBilinearForm * bfi,
-      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
-
-   ComplexOperator::Convention GetConvention() const { return conv; }
-   void SetConvention(const ComplexOperator::Convention & convention) { conv = convention; }
-
-   /// Set the desired assembly level.
-   /** Valid choices are:
-
-       - AssemblyLevel::LEGACY (default)
-       - AssemblyLevel::FULL
-       - AssemblyLevel::PARTIAL
-       - AssemblyLevel::ELEMENT
-       - AssemblyLevel::NONE
-
-       This method must be called before assembly. */
-   void SetAssemblyLevel(AssemblyLevel assembly_level)
-   {
-      mblfr->SetAssemblyLevel(assembly_level);
-      mblfi->SetAssemblyLevel(assembly_level);
-   }
-
-   MixedBilinearForm & real() { return *mblfr; }
-   MixedBilinearForm & imag() { return *mblfi; }
-   const MixedBilinearForm & real() const { return *mblfr; }
-   const MixedBilinearForm & imag() const { return *mblfi; }
-
-   /// Adds new Domain Integrator.
-   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                            BilinearFormIntegrator * bfi_imag);
-
-   /// Adds new Domain Integrator, restricted to specific attributes.
-   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                            BilinearFormIntegrator * bfi_imag,
-                            Array<int> & elem_marker);
-
-   /// Adds new Boundary Integrator.
-   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
-                              BilinearFormIntegrator * bfi_imag);
-
-   /** @brief Adds new boundary Integrator, restricted to specific boundary
-       attributes.
-
-       Assumes ownership of @a bfi.
-
-       The mfem::array @a bdr_marker is stored internally as a pointer to the given
-       mfem::Array<int> object. */
-   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
-                              BilinearFormIntegrator * bfi_imag,
-                              Array<int> & bdr_marker);
-
-   /// Adds new interior Face Integrator. Assumes ownership of @a bfi.
-   void AddInteriorFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag);
-
-   /// Adds new boundary Face Integrator. Assumes ownership of @a bfi.
-   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                             BilinearFormIntegrator * bfi_imag);
-
-   /** @brief Adds new boundary Face Integrator, restricted to specific boundary
-       attributes.
-
-       Assumes ownership of @a bfi.
-
-       The mfem::array @a bdr_marker is stored internally as a pointer to the given
-       mfem::Array<int> object. */
-   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                             BilinearFormIntegrator * bfi_imag,
-                             Array<int> & bdr_marker);
-
-   /** @brief Add a trace face integrator. Assumes ownership of @a bfi.
-
-       This type of integrator assembles terms over all faces of the mesh using
-       the face FE from the trial space and the two adjacent volume FEs from
-       the test space. */
-   void AddTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                               BilinearFormIntegrator * bfi_imag);
-
-   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag);
-
-   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag,
-                                  Array<int> &bdr_marker);
-
-   /// Assemble the local matrix
-   void Assemble(int skip_zeros = 1);
-
-   /// Finalizes the matrix initialization.
-   void Finalize(int skip_zeros = 1);
-
-   /// Updates the internal mixed forms with the new finite element space.
-   virtual void Update();
-
-   /** @brief Return a ComplexSparseMatrix wrapping the local (L-dof) real
-       and imaginary matrices of the form.
-
-       The returned wrapper has to be deleted by the caller, but it does not
-       own the wrapped real and imaginary matrices, which remain owned by
-       this form. */
-   ComplexSparseMatrix *AssembleComplexSparseMatrix();
-
-   /// Return the trial FE space associated with the MixedSesquilinearForm.
-   FiniteElementSpace *TrialFESpace() { return mblfr->TrialFESpace(); }
-
-   /// Read-only access to the associated trial FiniteElementSpace.
-   const FiniteElementSpace *TrialFESpace() const { return mblfr->TrialFESpace(); }
-
-   /// Return the test FE space associated with the MixedSesquilinearForm.
-   FiniteElementSpace *TestFESpace() { return mblfr->TestFESpace(); }
-
-   /// Read-only access to the associated test FiniteElementSpace.
-   const FiniteElementSpace *TestFESpace() const { return mblfr->TestFESpace(); }
-
-
-   void FormRectangularLinearSystem(const Array<int> & ess_trial_tdof_list,
-                                    const Array<int> & ess_test_tdof_list,
-                                    Vector & x,
-                                    Vector & b,
-                                    OperatorHandle & A,
-                                    Vector & X,
-                                    Vector & B);
-
-   void FormRectangularSystemMatrix(const Array<int> & ess_trial_tdof_list,
-                                    const Array<int> & ess_test_tdof_list,
-                                    OperatorHandle & A);
-
-   virtual ~MixedSesquilinearForm();
-};
-
 #ifdef MFEM_USE_MPI
 
 /// Class for parallel complex-valued grid function - real + imaginary part
@@ -1101,169 +921,6 @@ public:
    virtual ~ParSesquilinearForm();
 };
 
-/** Class for a parallel mixed sesquilinear form
-
-    A mixed sesquilinear form is a generalization of a mixed bilinear form to
-    complex-valued fields. Mixed sesquilinear forms are linear in the second
-    argument but the first argument involves a complex conjugate in the sense
-    that:
-
-                a(alpha u, beta v) = conj(alpha) beta a(u, v)
-
-    The @a convention argument in the class's constructor is documented in the
-    mfem::ComplexOperator class found in linalg/complex_operator.hpp.
-
-    When supplying integrators to the ParMixedSesquilinearForm either the real
-    or imaginary integrator can be NULL. This indicates that the corresponding
-    portion of the complex-valued material coefficient is equal to zero.
-*/
-class ParMixedSesquilinearForm
-{
-private:
-   ComplexOperator::Convention conv;
-
-   ParMixedBilinearForm * pmblfr;
-   ParMixedBilinearForm * pmblfi;
-
-   /* These methods check if the real/imag parts of the sesqulinear form are
-    not empty */
-   bool RealInteg();
-   bool ImagInteg();
-
-public:
-   ParMixedSesquilinearForm(
-      ParFiniteElementSpace * trial_fes,
-      ParFiniteElementSpace * test_fes,
-      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
-
-   /** @brief Create a ParMixedSesquilinearForm on the given trial and test
-       ParFiniteElementSpaces, using the same integrators as the
-       ParMixedBilinearForms @a pbfr and @a pbfi.
-
-       The ParFiniteElementSpace pointers are not owned by the newly
-       constructed object.
-
-       The integrators are copied as pointers and they are not owned by the
-       newly constructed ParMixedSesquilinearForm. */
-   ParMixedSesquilinearForm(
-      ParFiniteElementSpace * trial_fes,
-      ParFiniteElementSpace * test_fes,
-      ParMixedBilinearForm * pbfr,
-      ParMixedBilinearForm * pbfi,
-      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
-
-   ComplexOperator::Convention GetConvention() const { return conv; }
-   void SetConvention(const ComplexOperator::Convention & convention) { conv = convention; }
-
-   /// Set the desired assembly level.
-   /** Valid choices are:
-
-       - AssemblyLevel::LEGACY (default)
-       - AssemblyLevel::FULL
-       - AssemblyLevel::PARTIAL
-       - AssemblyLevel::ELEMENT
-       - AssemblyLevel::NONE
-
-       This method must be called before assembly. */
-   void SetAssemblyLevel(AssemblyLevel assembly_level)
-   {
-      pmblfr->SetAssemblyLevel(assembly_level);
-      pmblfi->SetAssemblyLevel(assembly_level);
-   }
-
-   ParMixedBilinearForm & real() { return *pmblfr; }
-   ParMixedBilinearForm & imag() { return *pmblfi; }
-   const ParMixedBilinearForm & real() const { return *pmblfr; }
-   const ParMixedBilinearForm & imag() const { return *pmblfi; }
-
-   /// Adds new Domain Integrator.
-   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                            BilinearFormIntegrator * bfi_imag);
-
-   /// Adds new Domain Integrator, restricted to specific attributes.
-   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                            BilinearFormIntegrator * bfi_imag,
-                            Array<int> & elem_marker);
-
-   /// Adds new Boundary Integrator.
-   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
-                              BilinearFormIntegrator * bfi_imag);
-
-   /** @brief Adds new boundary Integrator, restricted to specific boundary
-       attributes.
-
-       Assumes ownership of @a bfi.
-
-       The mfem::array @a bdr_marker is stored internally as a pointer to the given
-       mfem::Array<int> object. */
-   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
-                              BilinearFormIntegrator * bfi_imag,
-                              Array<int> & bdr_marker);
-
-   /// Adds new interior Face Integrator. Assumes ownership of @a bfi.
-   void AddInteriorFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag);
-
-   /// Adds new boundary Face Integrator. Assumes ownership of @a bfi.
-   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                             BilinearFormIntegrator * bfi_imag);
-
-   /** @brief Adds new boundary Face Integrator, restricted to specific boundary
-       attributes.
-
-       Assumes ownership of @a bfi.
-
-       The mfem::array @a bdr_marker is stored internally as a pointer to the given
-       mfem::Array<int> object. */
-   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                             BilinearFormIntegrator * bfi_imag,
-                             Array<int> & bdr_marker);
-
-   /** @brief Add a trace face integrator. Assumes ownership of @a bfi.
-
-      This type of integrator assembles terms over all faces of the mesh using
-      the face FE from the trial space and the two adjacent volume FEs from
-      the test space. */
-   void AddTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                               BilinearFormIntegrator * bfi_imag);
-
-   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag);
-
-   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag,
-                                  Array<int> &bdr_marker);
-
-   /// Assemble the local matrix
-   void Assemble(int skip_zeros = 1);
-
-   /// Finalizes the matrix initialization.
-   void Finalize(int skip_zeros = 1);
-
-   /// Updates the internal mixed forms with the new finite element space.
-   virtual void Update();
-
-   /// Returns the matrix assembled on the true dofs, i.e. P^t A P.
-   /** The returned matrix has to be deleted by the caller. */
-   ComplexHypreParMatrix * ParallelAssemble();
-
-   void FormRectangularLinearSystem(const Array<int> & ess_trial_tdof_list,
-                                    const Array<int> & ess_test_tdof_list,
-                                    Vector & x,
-                                    Vector & b,
-                                    OperatorHandle & A,
-                                    Vector & X,
-                                    Vector & B);
-
-   void FormRectangularSystemMatrix(const Array<int> & ess_trial_tdof_list,
-                                    const Array<int> & ess_test_tdof_list,
-                                    OperatorHandle & A);
-
-   virtual ~ParMixedSesquilinearForm();
-};
-
 #endif // MFEM_USE_MPI
 
 }

fem/datacollection.cpp

@@ -809,7 +809,7 @@ ParaViewDataCollectionBase::ParaViewDataCollectionBase(
 
 void ParaViewDataCollectionBase::SetLevelsOfDetail(int levels_of_detail_)
 {
-   levels_of_detail = std::max(levels_of_detail_, 1);
+   levels_of_detail = levels_of_detail_;
 }
 
 void ParaViewDataCollectionBase::SetHighOrderOutput(bool high_order_output_)

fem/datacollection.hpp

@@ -25,21 +25,35 @@
 namespace mfem
 {
 
-/// Lightweight adaptor over an std::map from strings to pointer to T
-template<typename T>
-class NamedFieldsMap
+/// Lightweight adaptor over an std::map from type K to type to V
+template<typename K, typename V,
+         typename = typename std::enable_if<std::is_default_constructible<V>::value>::type>
+class GenericFieldMap
 {
+private:
+   static constexpr bool ValueIsPointer = std::is_pointer<V>::value;
+
 public:
-   typedef std::map<std::string, T*> MapType;
+   typedef std::map<K, V> MapType;
    typedef typename MapType::iterator iterator;
    typedef typename MapType::const_iterator const_iterator;
 
-   /// Register field @a field with name @a fname
-   /** Replace existing field associated with @a fname (and optionally
-       delete associated pointer if @a own_data is true) */
-   void Register(const std::string& fname, T* field, bool own_data)
+   /// Register field @a field with name @a key
+   /// Only enabled if the template parameter V is not a pointer
+   template<typename = std::enable_if<!ValueIsPointer, bool>>
+   void Register(const K& key, V field)
    {
-      T*& ref = field_map[fname];
+      field_map[key] = field;
+   }
+
+   /// Register field @a field with name @a key
+   /** Replace existing field associated with @a key (and optionally
+       delete associated pointer if @a own_data is true).
+       Only enabled if the template parameter V is a pointer*/
+   template<typename = std::enable_if<ValueIsPointer, bool>>
+   void Register(const K& key, V field, bool own_data)
+   {
+      V& ref = field_map[key];
       if (own_data)
       {
          delete ref; // if newly allocated -> ref is null -> OK
@@ -47,23 +61,40 @@ public:
       ref = field;
    }
 
-   /// Unregister association between field @a field and name @a fname
-   /** Optionally delete associated pointer if @a own_data is true */
-   void Deregister(const std::string& fname, bool own_data)
+   /// Unregister association between field @a field and name @a key
+   /// Only enabled if the template parameter V is not a pointer
+   template<typename = std::enable_if<!ValueIsPointer, bool>>
+   void Deregister(const K& key)
    {
-      iterator it = field_map.find(fname);
+      iterator it = field_map.find(key);
+      if ( it != field_map.end() )
+      {
+         field_map.erase(it);
+      }
+   }
+
+   /// Unregister association between field @a field and name @a key
+   /** Optionally delete associated pointer if @a own_data is true.
+       Only enabled if the template parameter V is a pointer */
+   template<typename = std::enable_if<ValueIsPointer, bool>>
+   void Deregister(const K& key, bool own_data)
+   {
+      iterator it = field_map.find(key);
       if ( it != field_map.end() )
       {
          if (own_data)
          {
             delete it->second;
+            it->second = nullptr;
          }
          field_map.erase(it);
       }
    }
 
    /// Clear all associations between names and fields
-   /** Delete associated pointers when @a own_data is true */
+   /** Delete associated pointers when @a own_data is true.
+       Only enabled if the template parameter V is a pointer */
+   template<typename = std::enable_if<ValueIsPointer, bool>>
    void DeleteData(bool own_data)
    {
       for (iterator it = field_map.begin(); it != field_map.end(); ++it)
@@ -76,22 +107,37 @@ public:
       }
    }
 
-   /// Predicate to check if a field is associated with name @a fname
-   bool Has(const std::string& fname) const
+   /// Predicate to check if a field is associated with name @a key
+   bool Has(const K& key) const
    {
-      return field_map.find(fname) != field_map.end();
+      return field_map.find(key) != field_map.end();
    }
 
-   /// Get a pointer to the field associated with name @a fname
-   /** @return Pointer to field associated with @a fname or NULL */
-   T* Get(const std::string& fname) const
+   /// Get a pointer to the field associated with name @a key
+   /** @return Field associated with @a key or NULL,
+       if value is pointer and key not found */
+   V Get(const K& key) const
    {
-      const_iterator it = field_map.find(fname);
-      return it != field_map.end() ? it->second : NULL;
+      const_iterator it = field_map.find(key);
+      if (it != field_map.end())
+      {
+         return it->second;
+      }
+      else
+      {
+         if constexpr (ValueIsPointer)
+         {
+            return nullptr;
+         }
+         else
+         {
+            return V(); // Return default-constructed value for non-pointer types
+         }
+      }
    }
 
    /// Returns a const reference to the underlying map
-   const MapType& GetMap() const { return field_map; }
+   const MapType &GetMap() const { return field_map; }
 
    /// Returns the number of registered fields
    int NumFields() const { return field_map.size(); }
@@ -106,21 +152,24 @@ public:
    /// Returns an end const iterator to the registered fields
    const_iterator end() const { return field_map.end(); }
 
-   /// Returns an iterator to the field @a fname
-   iterator find(const std::string& fname)
-   { return field_map.find(fname); }
+   /// Returns an iterator to the field @a key
+   iterator find(const K& key)
+   { return field_map.find(key); }
 
-   /// Returns a const iterator to the field @a fname
-   const_iterator find(const std::string& fname) const
-   { return field_map.find(fname); }
+   /// Returns a const iterator to the field @a key
+   const_iterator find(const K& key) const
+   { return field_map.find(key); }
 
-   /// Clears the map of registered fields without reclaiming memory
+   /// Clears the map of registered fields
    void clear() { field_map.clear(); }
 
 protected:
    MapType field_map;
 };
 
+/// Lightweight adaptor over an std::map from strings to pointer to T
+template<typename T>
+using NamedFieldsMap = GenericFieldMap<std::string, T*>;
 
 /** A class for collecting finite element data that is part of the same
     simulation. Currently, this class groups together grid functions (fields),

fem/dfem/util.hpp

@@ -671,6 +671,20 @@ public:
                     MPI_COMM_WORLD);
    }
 
+   Operator& GetGradient(const Vector &x0) const override
+   {
+      x = x0;
+      f.UseDevice(x.UseDevice());
+      xpev.UseDevice(x.UseDevice());
+
+      op.Mult(x, f);
+      const real_t xnorm_local = x.Norml2();
+      MPI_Allreduce(&xnorm_local, &xnorm, 1, MPITypeMap<real_t>::mpi_type, MPI_SUM,
+                    MPI_COMM_WORLD);
+
+      return const_cast<FDJacobian&>(*this);
+   }
+
    void Mult(const Vector &v, Vector &y) const override
    {
       // See [1] for choice of eps.
@@ -725,11 +739,11 @@ public:
 
 private:
    const Operator &op;
-   Vector x, f;
+   mutable Vector x, f;
    mutable Vector xpev;
    real_t lambda = 1.0e-6;
    real_t fixed_eps;
-   real_t xnorm;
+   mutable real_t xnorm;
 };
 
 /// @brief Find the index of a field descriptor in a vector of field descriptors.

fem/fe/fe_fixed_order.hpp

@@ -307,12 +307,12 @@ public:
 
    /** @brief virtual function which evaluates the values of all
        shape functions at a given point ip and stores
-       them in the vector shape of dimension Dof (6) */
+       them in the vector shape of dimension Dof (4) */
    void CalcShape(const IntegrationPoint &ip, Vector &shape) const override;
 
    /** @brief virtual function which evaluates the values of all
        partial derivatives of all shape functions at a given
-       point ip and stores them in the matrix dshape (Dof x Dim) (6 x 3)
+       point ip and stores them in the matrix dshape (Dof x Dim) (4 x 3)
        so that each row contains the derivatives of one shape function */
    void CalcDShape(const IntegrationPoint &ip,
                    DenseMatrix &dshape) const override;
@@ -336,12 +336,12 @@ public:
 
    /** @brief virtual function which evaluates the values of all
        shape functions at a given point ip and stores
-       them in the vector shape of dimension Dof (5) */
+       them in the vector shape of dimension Dof (4) */
    void CalcShape(const IntegrationPoint &ip, Vector &shape) const override;
 
    /** @brief virtual function which evaluates the values of all
        partial derivatives of all shape functions at a given
-       point ip and stores them in the matrix dshape (Dof x Dim) (5 x 3)
+       point ip and stores them in the matrix dshape (Dof x Dim) (4 x 3)
        so that each row contains the derivatives of one shape function */
    void CalcDShape(const IntegrationPoint &ip,
                    DenseMatrix &dshape) const override;

fem/fe/fe_h1.cpp

@@ -1757,45 +1757,22 @@ H1_BergotPyramidElement::H1_BergotPyramidElement(const int p, const int btype)
       real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
       real_t z = ip.z;
 
-      if (std::abs(z - 1.0) < apex_tol)
-      {
-         // Compute the limit of the basis functions as z->1 with x and y on the
-         // line between the center of the base and the apex
-         o = 0;
-         for (int i = 0; i <= p; i++)
-            for (int j = 0; j <= p; j++)
-            {
-               int maxij = std::max(i, j);
-               for (int k = 0; k <= p - maxij; k++)
-                  if (i == 0 && j == 0)
-                  {
-                     T(o++, m) = ((k + 3.) * k + 2.) / 2.;
-                  }
-                  else
-                  {
-                     T(o++, m) = 0.;
-                  }
-            }
-      }
-      else
-      {
-         poly1d.CalcLegendre(p, x, shape_x.GetData());
-         poly1d.CalcLegendre(p, y, shape_y.GetData());
+      poly1d.CalcLegendre(p, x, shape_x.GetData());
+      poly1d.CalcLegendre(p, y, shape_y.GetData());
 
-         o = 0;
-         for (int i = 0; i <= p; i++)
+      o = 0;
+      for (int i = 0; i <= p; i++)
+      {
+         for (int j = 0; j <= p; j++)
          {
-            for (int j = 0; j <= p; j++)
-            {
-               int maxij = std::max(i, j);
-               FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                                z, 1.0, shape_z);
+            int maxij = std::max(i, j);
+            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
+                                             z, 1.0, shape_z);
 
-               for (int k = 0; k <= p - maxij; k++)
-               {
-                  T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
-                              pow(1.0 - ip.z, maxij);
-               }
+            for (int k = 0; k <= p - maxij; k++)
+            {
+               T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
+                           pow(1.0 - ip.z, maxij);
             }
          }
       }
@@ -1816,44 +1793,25 @@ void H1_BergotPyramidElement::CalcShape(const IntegrationPoint &ip,
    Vector u(dof);
 #endif
 
-   const real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
-   const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
-   const real_t z = ip.z;
+   real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
+   real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
+   real_t z = ip.z;
 
-   if (std::abs(z - 1.0) < apex_tol)
-   {
-      // Compute the limit of the basis functions as z->1 with x and y on the
-      // line between the center of the base and the apex
-      u = 0.;
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            for (int k = 0; k <= p - maxij; k++, o++)
-               if (i == 0 && j == 0)
-               {
-                  u(o) = ((k + 3.) * k + 2.) / 2.;
-               }
-         }
-   }
-   else
-   {
-      poly1d.CalcLegendre(p, x, shape_x.GetData());
-      poly1d.CalcLegendre(p, y, shape_y.GetData());
+   poly1d.CalcLegendre(p, x, shape_x.GetData());
+   poly1d.CalcLegendre(p, y, shape_y.GetData());
+
+   int o = 0;
+   for (int i = 0; i <= p; i++)
+      for (int j = 0; j <= p; j++)
+      {
+         int maxij = std::max(i, j);
+         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
+                                          shape_z);
+         for (int k = 0; k <= p - maxij; k++)
+            u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
+                     pow(1.0 - ip.z, maxij);
+      }
 
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                             z, 1.0, shape_z);
-            for (int k = 0; k <= p - maxij; k++)
-               u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
-                        pow(1.0 - ip.z, maxij);
-         }
-   }
    Ti.Mult(u, shape);
 }
 
@@ -1872,68 +1830,37 @@ void H1_BergotPyramidElement::CalcDShape(const IntegrationPoint &ip,
    Vector dshape_z(order+1);
    Vector dshape_z_dt(order+1);
 #endif
-   const real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
-   const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
-   const real_t z = ip.z;
+   real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
+   real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
+   real_t z = ip.z;
 
-   if (std::abs(z - 1.0) < apex_tol)
-   {
-      // Compute the limit of the gradients of the basis functions as
-      // z->1 with x and y on the line between the center of the base and the
-      // apex
-      du = 0.;
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
+   poly1d.CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
+   poly1d.CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
+
+   int o = 0;
+   for (int i = 0; i <= p; i++)
+      for (int j = 0; j <= p; j++)
+      {
+         int maxij = std::max(i, j);
+         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
+                                          shape_z, dshape_z, dshape_z_dt);
+
+         for (int k = 0; k <= p - maxij; k++, o++)
          {
-            int maxij = std::max(i, j);
-            for (int k = 0; k <= p - maxij; k++, o++)
-            {
-               if (i == 0 && j == 0)
-               {
-                  du(o,2) = (((k + 6.) * k + 11.) * k + 6.) * k / 6.;
-               }
-               else if (i == 1 && j == 0)
-               {
-                  du(o,0) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
-               }
-               else if (i == 0 && j == 1)
-               {
-                  du(o,1) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
-               }
-            }
+            du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
+                      pow(1.0 - ip.z, maxij - 1);
+            du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
+                      pow(1.0 - ip.z, maxij - 1);
+            du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
+                      pow(1.0 - ip.z, maxij) +
+                      (ip.x * dshape_x(i) * shape_y(j) +
+                       ip.y * shape_x(i) * dshape_y(j)) *
+                      shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
+                      maxij * shape_x(i) * shape_y(j) * shape_z(k) *
+                      pow(1.0 - ip.z, maxij - 1);
          }
-   }
-   else
-   {
-      poly1d.CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
-      poly1d.CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
+      }
 
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                             z, 1.0,
-                                             shape_z, dshape_z, dshape_z_dt);
-
-            for (int k = 0; k <= p - maxij; k++, o++)
-            {
-               du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
-                         pow(1.0 - ip.z, maxij - 1);
-               du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
-                         pow(1.0 - ip.z, maxij - 1);
-               du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
-                         pow(1.0 - ip.z, maxij) +
-                         (ip.x * dshape_x(i) * shape_y(j) +
-                          ip.y * shape_x(i) * dshape_y(j)) *
-                         shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
-                         maxij * shape_x(i) * shape_y(j) * shape_z(k) *
-                         (maxij > 0 ? pow(1.0 - ip.z, maxij - 1) : 0.0);
-            }
-         }
-   }
    Ti.Mult(du, dshape);
 }
 

fem/fe/fe_h1.hpp

@@ -208,8 +208,6 @@ private:
 #endif
    DenseMatrixInverse Ti;
 
-   static constexpr real_t apex_tol = 1e-8;
-
 public:
    H1_BergotPyramidElement(const int p,
                            const int btype = BasisType::GaussLobatto);

fem/fe/fe_l2.cpp

@@ -1106,16 +1106,9 @@ L2_BergotPyramidElement::L2_BergotPyramidElement(const int p, const int btype)
          {
             const real_t wik = op[i] + op[k] + op[p-i-k];
             const real_t w = wik * wjk * op[p-k];
-            if (std::abs(w) < apex_tol)
-            {
-               Nodes.IntPoint(o++).Set3(0.,0.,1.);
-            }
-            else
-            {
-               Nodes.IntPoint(o++).Set3(op[i] * (op[j] + op[p-j-k]) / w,
-                                        op[j] * (op[i] + op[p-i-k]) / w,
-                                        op[k] * op[p-k] / w);
-            }
+            Nodes.IntPoint(o++).Set3(op[i] * (op[j] + op[p-j-k]) / w,
+                                     op[j] * (op[j] + op[p-j-k]) / w,
+                                     op[k] * op[p-k] / w);
          }
       }
 
@@ -1132,45 +1125,22 @@ L2_BergotPyramidElement::L2_BergotPyramidElement(const int p, const int btype)
       const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
       const real_t z = ip.z;
 
-      if (std::abs(z - 1.0) < apex_tol)
-      {
-         // Compute the limit of the basis functions as z->1 with x and y on the
-         // line between the center of the base and the apex
-         o = 0;
-         for (int i = 0; i <= p; i++)
-            for (int j = 0; j <= p; j++)
-            {
-               int maxij = std::max(i, j);
-               for (int k = 0; k <= p - maxij; k++)
-                  if (i == 0 && j == 0)
-                  {
-                     T(o++, m) = ((k + 3.) * k + 2.) / 2.;
-                  }
-                  else
-                  {
-                     T(o++, m) = 0.;
-                  }
-            }
-      }
-      else
-      {
-         poly1d.CalcLegendre(p, x, shape_x.GetData());
-         poly1d.CalcLegendre(p, y, shape_y.GetData());
+      poly1d.CalcLegendre(p, x, shape_x.GetData());
+      poly1d.CalcLegendre(p, y, shape_y.GetData());
 
-         o = 0;
-         for (int i = 0; i <= p; i++)
+      o = 0;
+      for (int i = 0; i <= p; i++)
+      {
+         for (int j = 0; j <= p; j++)
          {
-            for (int j = 0; j <= p; j++)
-            {
-               int maxij = std::max(i, j);
-               FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                                z, 1.0, shape_z);
+            int maxij = std::max(i, j);
+            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
+                                             z, 1.0, shape_z);
 
-               for (int k = 0; k <= p - maxij; k++)
-               {
-                  T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
-                              pow(1.0 - ip.z, maxij);
-               }
+            for (int k = 0; k <= p - maxij; k++)
+            {
+               T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
+                           pow(1.0 - ip.z, maxij);
             }
          }
       }
@@ -1195,41 +1165,26 @@ void L2_BergotPyramidElement::CalcShape(const IntegrationPoint &ip,
    const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
    const real_t z = ip.z;
 
-   if (std::abs(z - 1.0) < apex_tol)
-   {
-      // Compute the limit of the basis functions as z->1 with x and y on the
-      // line between the center of the base and the apex
-      u = 0.;
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            for (int k = 0; k <= p - maxij; k++, o++)
-               if (i == 0 && j == 0)
-               {
-                  u(o) = ((k + 3.) * k + 2.) / 2.;
-               }
-         }
-   }
-   else
-   {
-      poly1d.CalcLegendre(p, x, shape_x.GetData());
-      poly1d.CalcLegendre(p, y, shape_y.GetData());
+   poly1d.CalcLegendre(p, x, shape_x.GetData());
+   poly1d.CalcLegendre(p, y, shape_y.GetData());
 
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                             z, 1.0, shape_z);
+   int o = 0;
+   for (int i = 0; i <= p; i++)
+   {
+      for (int j = 0; j <= p; j++)
+      {
+         int maxij = std::max(i, j);
+         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
+                                          shape_z);
 
-            for (int k = 0; k <= p - maxij; k++)
-               u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
-                        pow(1.0 - ip.z, maxij);
+         for (int k = 0; k <= p - maxij; k++)
+         {
+            u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
+                     pow(1.0 - ip.z, maxij);
          }
+      }
    }
+
    Ti.Mult(u, shape);
 }
 
@@ -1253,64 +1208,35 @@ void L2_BergotPyramidElement::CalcDShape(const IntegrationPoint &ip,
    const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
    const real_t z = ip.z;
 
-   if (std::abs(z - 1.0) < apex_tol)
-   {
-      // Compute the limit of the gradients of the basis functions as
-      // z->1 with x and y on the line between the center of the base and the
-      // apex
-      du = 0.;
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            for (int k = 0; k <= p - maxij; k++, o++)
-            {
-               if (i == 0 && j == 0)
-               {
-                  du(o,2) = (((k + 6.) * k + 11.) * k + 6.) * k / 6.;
-               }
-               else if (i == 1 && j == 0)
-               {
-                  du(o,0) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
-               }
-               else if (i == 0 && j == 1)
-               {
-                  du(o,1) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
-               }
-            }
-         }
-   }
-   else
-   {
-      Poly_1D::CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
-      Poly_1D::CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
+   Poly_1D::CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
+   Poly_1D::CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
 
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                             z, 1.0,
-                                             shape_z, dshape_z, dshape_z_dt);
+   int o = 0;
+   for (int i = 0; i <= p; i++)
+   {
+      for (int j = 0; j <= p; j++)
+      {
+         int maxij = std::max(i, j);
+         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
+                                          shape_z, dshape_z, dshape_z_dt);
 
-            for (int k = 0; k <= p - maxij; k++, o++)
-            {
-               du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
-                         pow(1.0 - ip.z, maxij - 1);
-               du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
-                         pow(1.0 - ip.z, maxij - 1);
-               du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
-                         pow(1.0 - ip.z, maxij) +
-                         (ip.x * dshape_x(i) * shape_y(j) +
-                          ip.y * shape_x(i) * dshape_y(j)) *
-                         shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
-                         maxij * shape_x(i) * shape_y(j) * shape_z(k) *
-                         (maxij > 0 ? pow(1.0 - ip.z, maxij - 1) : 0.0);
-            }
+         for (int k = 0; k <= p - maxij; k++, o++)
+         {
+            du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
+                      pow(1.0 - ip.z, maxij - 1);
+            du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
+                      pow(1.0 - ip.z, maxij - 1);
+            du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
+                      pow(1.0 - ip.z, maxij) +
+                      (ip.x * dshape_x(i) * shape_y(j) +
+                       ip.y * shape_x(i) * dshape_y(j)) *
+                      shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
+                      ((maxij > 0) ? (maxij * shape_x(i) * shape_y(j) * shape_z(k) *
+                                      pow(1.0 - ip.z, maxij - 1)) : 0.0);
          }
+      }
    }
+
    Ti.Mult(du, dshape);
 }
 

fem/fe/fe_l2.hpp

@@ -225,8 +225,6 @@ private:
 #endif
    DenseMatrixInverse Ti;
 
-   static constexpr real_t apex_tol = 1e-8;
-
 public:
    /// Construct the L2_PyramidElement of order @a p and BasisType @a btype
    L2_BergotPyramidElement(const int p,

fem/fe/fe_nd.cpp

@@ -1282,49 +1282,12 @@ ND_SegmentElement::ND_SegmentElement(const int p, const int ob_type)
    }
 }
 
-void ND_SegmentElement::CalcShape(const IntegrationPoint &ip,
-                                  Vector &shape) const
-{
-   if (obasis1d.IsIntegratedType()) { obasis1d.ScaleIntegrated(false); }
-   obasis1d.Eval(ip.x, shape);
-}
-
 void ND_SegmentElement::CalcVShape(const IntegrationPoint &ip,
                                    DenseMatrix &shape) const
 {
    Vector vshape(shape.Data(), dof);
 
-   CalcShape(ip, vshape);
-}
-
-void ND_SegmentElement::ProjectIntegrated(VectorCoefficient &vc,
-                                          ElementTransformation &Trans,
-                                          Vector &dofs) const
-{
-   MFEM_ASSERT(obasis1d.IsIntegratedType(), "Not integrated type");
-   real_t vk[Geometry::MaxDim];
-   Vector xk(vk, vc.GetVDim());
-
-   const real_t *cp = poly1d.ClosedPoints(dof, BasisType::GaussLobatto);
-   const IntegrationRule &ir = IntRules.Get(Geometry::SEGMENT, dof);
-   IntegrationPoint ip;
-
-   for (int i = 0; i < dof; i++)
-   {
-      const real_t h = cp[i+1] - cp[i];
-      real_t val = 0.0;
-
-      for (int q = 0; q < ir.GetNPoints(); q++)
-      {
-         const IntegrationPoint &ip1d = ir.IntPoint(q);
-         ip.x = cp[i] + h*ip1d.x;
-         Trans.SetIntPoint(&ip);
-         vc.Eval(xk, Trans, ip);
-         val += ip1d.weight*Trans.Jacobian().InnerProduct(tk, vk);
-      }
-
-      dofs(i) = val*h;
-   }
+   obasis1d.Eval(ip.x, vshape);
 }
 
 const real_t ND_WedgeElement::tk[15] =

fem/fe/fe_nd.hpp

@@ -303,7 +303,8 @@ public:
    /** @brief Construct the ND_SegmentElement of order @a p and open
        BasisType @a ob_type */
    ND_SegmentElement(const int p, const int ob_type = BasisType::GaussLegendre);
-   void CalcShape(const IntegrationPoint &ip, Vector &shape) const override;
+   void CalcShape(const IntegrationPoint &ip, Vector &shape) const override
+   { obasis1d.Eval(ip.x, shape); }
    void CalcVShape(const IntegrationPoint &ip,
                    DenseMatrix &shape) const override;
    void CalcVShape(ElementTransformation &Trans,
@@ -324,10 +325,7 @@ public:
    using FiniteElement::Project;
    void Project(VectorCoefficient &vc,
                 ElementTransformation &Trans, Vector &dofs) const override
-   {
-      if (obasis1d.IsIntegratedType()) { ProjectIntegrated(vc, Trans, dofs); }
-      else { Project_ND(tk, dof2tk, vc, Trans, dofs); }
-   }
+   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
    void ProjectMatrixCoefficient(MatrixCoefficient &mc,
                                  ElementTransformation &T,
                                  Vector &dofs) const override
@@ -340,11 +338,6 @@ public:
                     ElementTransformation &Trans,
                     DenseMatrix &grad) const override
    { ProjectGrad_ND(tk, dof2tk, fe, Trans, grad); }
-
-protected:
-   void ProjectIntegrated(VectorCoefficient &vc,
-                          ElementTransformation &Trans,
-                          Vector &dofs) const;
 };
 
 class ND_WedgeElement : public VectorFiniteElement

fem/fe/fe_pyramid.hpp

@@ -17,12 +17,6 @@
 namespace mfem
 {
 
-struct ScalarPyramid
-{
-   // Default basis type for H1 and L2 pyramids
-   static inline int DefaultType = 1; // Bergot(0) or Fuentes(1)
-};
-
 /** Base class for arbitrary order basis functions on pyramid-shaped elements
 
   This base class provides a common class to store temporary vectors,

fem/fe_coll.cpp

@@ -228,19 +228,7 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name)
    }
    else if (!strncmp(name, "H1_", 3))
    {
-      // Parse pyramid basis type if included in the name
-      const char *pyr = strstr(name, "Pyr");
-      if (pyr == NULL)
-      {
-         // Use default pyramid type elements
-         fec = new H1_FECollection(atoi(name + 7), atoi(name + 3));
-      }
-      else
-      {
-         // Use specific pyramid type elements
-         fec = new H1_FECollection(atoi(name + 7), atoi(name + 3),
-                                   BasisType::GaussLobatto, atoi(pyr + 3));
-      }
+      fec = new H1_FECollection(atoi(name + 7), atoi(name + 3));
    }
    else if (!strncmp(name, "H1Pos_Trace_", 12))
    {
@@ -257,44 +245,26 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name)
    }
    else if (!strncmp(name, "H1@", 3))
    {
-      // Parse pyramid basis type if included in the name
-      const char *pyr = strstr(name, "Pyr");
-      if (pyr == NULL)
-      {
-         // Use default pyramid type elements
-         fec = new H1_FECollection(atoi(name + 9), atoi(name + 5),
-                                   BasisType::GetType(name[3]));
-      }
-      else
-      {
-         // Use specific pyramid type elements
-         fec = new H1_FECollection(atoi(name + 9), atoi(name + 5),
-                                   BasisType::GetType(name[3]),
-                                   atoi(pyr + 3));
-      }
+      fec = new H1_FECollection(atoi(name + 9), atoi(name + 5),
+                                BasisType::GetType(name[3]));
    }
-   else if (!strncmp(name, "L2", 2))
+   else if (!strncmp(name, "L2_T", 4))
+      fec = new L2_FECollection(atoi(name + 10), atoi(name + 6),
+                                atoi(name + 4));
+   else if (!strncmp(name, "L2_", 3))
    {
-      // Parse Map Type
-      const int mtype = strstr(name, "Int") == NULL ?
-                        FiniteElement::VALUE : FiniteElement::INTEGRAL;
-
-      // Parse the base order
-      const int p = atoi(strstr(name, "_P") + 2);
-
-      // Parse the mesh dimension
-      const int dim = atoi(strstr(name, "D") - 1);
-
-      // Parse basis type if specified
-      const char *t = strstr(name, "_T");
-      const int btype = t == NULL ? BasisType::GaussLegendre : atoi(t + 2);
-
-      // Parse the pyramid type if specified
-      const char *pyr = strstr(name, "Pyr");
-      const int ptype = pyr == NULL ? 1 : atoi(pyr + 3);
-
-      // Create collection
-      fec = new L2_FECollection(p, dim, btype, mtype, ptype);
+      fec = new L2_FECollection(atoi(name + 7), atoi(name + 3));
+   }
+   else if (!strncmp(name, "L2Int_T", 7))
+   {
+      fec = new L2_FECollection(atoi(name + 13), atoi(name + 9),
+                                atoi(name + 7), FiniteElement::INTEGRAL);
+   }
+   else if (!strncmp(name, "L2Int_", 6))
+   {
+      fec = new L2_FECollection(atoi(name + 10), atoi(name + 6),
+                                BasisType::GaussLegendre,
+                                FiniteElement::INTEGRAL);
    }
    else if (!strncmp(name, "RT_Trace_", 9))
    {
@@ -1739,10 +1709,9 @@ const int *RT1_3DFECollection::DofOrderForOrientation(Geometry::Type GeomType,
 
 
 H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
-                                 const int pyr_type)
+                                 const int pyrtype)
    : FiniteElementCollection(p)
    , dim(dim)
-   , p_type(pyr_type)
 {
    MFEM_VERIFY(p >= 1, "H1_FECollection requires order >= 1.");
    MFEM_VERIFY(dim >= 0 && dim <= 3, "H1_FECollection requires 0 <= dim <= 3.");
@@ -1755,14 +1724,7 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
    {
       case BasisType::GaussLobatto:
       {
-         if (pyr_type == ScalarPyramid::DefaultType)
-         {
-            snprintf(h1_name, 32, "H1_%dD_P%d", dim, p);
-         }
-         else
-         {
-            snprintf(h1_name, 32, "H1_%dD_P%d_Pyr%d", dim, p, pyr_type);
-         }
+         snprintf(h1_name, 32, "H1_%dD_P%d", dim, p);
          break;
       }
       case BasisType::Positive:
@@ -1948,11 +1910,11 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
          H1_dof[Geometry::TETRAHEDRON] = (TriDof*pm3)/3;
          H1_dof[Geometry::CUBE] = QuadDof*pm1;
          H1_dof[Geometry::PRISM] = TriDof*pm1;
-         if (pyr_type == 0 || b_type == BasisType::Positive)
+         if (pyrtype == 0 || b_type == BasisType::Positive)
          {
             H1_dof[Geometry::PYRAMID] = pm2*pm1*(2*p-3)/6; // Bergot (JSC)
          }
-         else if (pyr_type == 1)
+         else if (pyrtype == 1)
          {
             H1_dof[Geometry::PYRAMID] = pm1*pm1*pm1; // Fuentes
          }
@@ -1973,15 +1935,13 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
                new H1_TetrahedronElement(p, btype);
             H1_Elements[Geometry::CUBE] = new H1_HexahedronElement(p, btype);
             H1_Elements[Geometry::PRISM] = new H1_WedgeElement(p, btype);
-            if (pyr_type == 0)
+            if (pyrtype == 0)
             {
-               H1_Elements[Geometry::PYRAMID] =
-                  new H1_BergotPyramidElement(p, btype);
+               H1_Elements[Geometry::PYRAMID] = new H1_BergotPyramidElement(p, btype);
             }
             else
             {
-               H1_Elements[Geometry::PYRAMID] =
-                  new H1_FuentesPyramidElement(p, btype);
+               H1_Elements[Geometry::PYRAMID] = new H1_FuentesPyramidElement(p, btype);
             }
          }
 
@@ -2188,7 +2148,6 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
    : FiniteElementCollection(p)
    , dim(dim)
    , m_type(map_type)
-   , p_type(pyr_type)
 {
    MFEM_VERIFY(p >= 0, "L2_FECollection requires order >= 0.");
 
@@ -2204,25 +2163,10 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
    switch (btype)
    {
       case BasisType::GaussLegendre:
-         if (pyr_type == ScalarPyramid::DefaultType)
-         {
-            snprintf(d_name, 32, "%s_%dD_P%d", prefix, dim, p);
-         }
-         else
-         {
-            snprintf(d_name, 32, "%s_%dD_P%d_Pyr%d", prefix, dim, p, pyr_type);
-         }
+         snprintf(d_name, 32, "%s_%dD_P%d", prefix, dim, p);
          break;
       default:
-         if (pyr_type == ScalarPyramid::DefaultType)
-         {
-            snprintf(d_name, 32, "%s_T%d_%dD_P%d", prefix, btype, dim, p);
-         }
-         else
-         {
-            snprintf(d_name, 32, "%s_T%d_%dD_P%d_Pyr%d",
-                     prefix, btype, dim, p, pyr_type);
-         }
+         snprintf(d_name, 32, "%s_T%d_%dD_P%d", prefix, btype, dim, p);
    }
 
    for (int g = 0; g < Geometry::NumGeom; g++)
@@ -2341,13 +2285,11 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
          L2_Elements[Geometry::PRISM] = new L2_WedgeElement(p, btype);
          if (pyr_type == 0)
          {
-            L2_Elements[Geometry::PYRAMID] =
-               new L2_BergotPyramidElement(p, btype);
+            L2_Elements[Geometry::PYRAMID] = new L2_BergotPyramidElement(p, btype);
          }
          else
          {
-            L2_Elements[Geometry::PYRAMID] =
-               new L2_FuentesPyramidElement(p, btype);
+            L2_Elements[Geometry::PYRAMID] = new L2_FuentesPyramidElement(p, btype);
          }
       }
 

fem/fe_coll.hpp

@@ -100,10 +100,6 @@ public:
       return FiniteElementForGeometry(GeomType);
    }
 
-   /** @brief Returns a collection of the trace elements.
-
-       @note The collection is owned by the caller and is NOT deleted in the
-       destructor. */
    virtual FiniteElementCollection *GetTraceCollection() const;
 
    virtual ~FiniteElementCollection();
@@ -290,7 +286,7 @@ protected:
 class H1_FECollection : public FiniteElementCollection
 {
 protected:
-   int dim, b_type, p_type;
+   int dim, b_type;
    char h1_name[32];
    FiniteElement *H1_Elements[Geometry::NumGeom];
    int H1_dof[Geometry::NumGeom];
@@ -299,7 +295,7 @@ protected:
 public:
    explicit H1_FECollection(const int p, const int dim = 3,
                             const int btype = BasisType::GaussLobatto,
-                            const int pyr_type = ScalarPyramid::DefaultType);
+                            const int pyrtype = 1);
 
    const FiniteElement *
    FiniteElementForGeometry(Geometry::Type GeomType) const override;
@@ -324,7 +320,7 @@ public:
    const int *GetDofMap(Geometry::Type GeomType, int p) const;
 
    FiniteElementCollection *Clone(int p) const override
-   { return new H1_FECollection(p, dim, b_type, p_type); }
+   { return new H1_FECollection(p, dim, b_type); }
 
    int GetConstructorOrder() const override
    { return base_p; }
@@ -371,7 +367,6 @@ private:
    int dim;
    int b_type; // BasisType
    int m_type; // map type
-   int p_type; // Pyramid type (0 -> Bergot, 1 -> Fuentes)
    char d_name[32];
    ScalarFiniteElement *L2_Elements[Geometry::NumGeom];
    ScalarFiniteElement *Tr_Elements[Geometry::NumGeom];
@@ -384,7 +379,7 @@ public:
    L2_FECollection(const int p, const int dim,
                    const int btype = BasisType::GaussLegendre,
                    const int map_type = FiniteElement::VALUE,
-                   const int pyr_type = ScalarPyramid::DefaultType);
+                   const int pyrtype = 1);
 
    const FiniteElement *
    FiniteElementForGeometry(Geometry::Type GeomType) const override;
@@ -414,7 +409,7 @@ public:
    int GetBasisType() const { return b_type; }
 
    FiniteElementCollection *Clone(int p) const override
-   { return new L2_FECollection(p, dim, b_type, m_type, p_type); }
+   { return new L2_FECollection(p, dim, b_type, m_type); }
 
    int GetConstructorOrder() const override
    { return base_p; }

fem/integ/bilininteg_elasticity_kernels.cpp

@@ -91,15 +91,15 @@ void ElasticityAddMultPA(const int dim, const int nDofs,
 void ElasticityAssembleDiagonalPA(const int dim, const int nDofs,
                                   const CoefficientVector &lambda,
                                   const CoefficientVector &mu, const GeometricFactors &geom,
-                                  const DofToQuad &maps, const IntegrationRule &ir, Vector &diag)
+                                  const DofToQuad &maps, QuadratureFunction &QVec, Vector &diag)
 {
    switch (dim)
    {
       case 2:
-         ElasticityAssembleDiagonalPA_<2>(nDofs, lambda, mu, geom, maps, ir, diag);
+         ElasticityAssembleDiagonalPA_<2>(nDofs, lambda, mu, geom, maps, QVec, diag);
          break;
       case 3:
-         ElasticityAssembleDiagonalPA_<3>(nDofs, lambda, mu, geom, maps, ir, diag);
+         ElasticityAssembleDiagonalPA_<3>(nDofs, lambda, mu, geom, maps, QVec, diag);
          break;
       default:
          MFEM_ABORT("Only dimensions 2 and 3 supported.");

fem/integ/bilininteg_elasticity_kernels.hpp

@@ -38,6 +38,7 @@
 #include "../../linalg/vector.hpp"
 #include "../../linalg/tensor.hpp"
 #include "../quadinterpolator.hpp"
+#include "../bilininteg.hpp"
 #include "../coefficient.hpp"
 #include "../qfunction.hpp"
 
@@ -132,12 +133,12 @@ void ElasticityAssembleEA(const int dim, const int i_block, const int j_block,
 /// @param[in] mu Quadrature function for second Lame param.
 /// @param[in] geom Geometric factors corresponding to fespace.
 /// @param[in] maps DofToQuad maps for one element (assume elements all same).
-/// @param[in] ir Integration rule.
+/// @param QVec Scratch Q-Vector. nQuad x dim x dim x dim x dim x numEls.
 /// @param[out] diag diagonal of A. nDofs x dim x numEls.
 void ElasticityAssembleDiagonalPA(const int dim, const int nDofs,
                                   const CoefficientVector &lambda,
                                   const CoefficientVector &mu, const GeometricFactors &geom,
-                                  const DofToQuad &maps, const IntegrationRule &ir, Vector &diag);
+                                  const DofToQuad &maps, QuadratureFunction &QVec, Vector &diag);
 
 /// Templated implementation of ElasticityAddMultPA.
 template<int dim, int i_block = -1, int j_block = -1>
@@ -279,67 +280,77 @@ void ElasticityAddMultPA_(const int nDofs, const FiniteElementSpace &fespace,
 template<int dim>
 void ElasticityAssembleDiagonalPA_(const int nDofs,
                                    const CoefficientVector &lambda,
-                                   const CoefficientVector &mu,
-                                   const GeometricFactors &geom,
-                                   const DofToQuad &maps,
-                                   const IntegrationRule &ir,
-                                   Vector &diag)
+                                   const CoefficientVector &mu, const GeometricFactors &geom,
+                                   const DofToQuad &maps, QuadratureFunction &QVec, Vector &diag)
 {
+   using future::tensor;
+   using future::make_tensor;
    using future::det;
    using future::inv;
-   using future::make_tensor;
-   using future::tensor;
 
    // Assuming all elements are the same
+   const auto &ir = QVec.GetIntRule(0);
    static constexpr int d = dim;
    const int numPoints = ir.GetNPoints();
-   const int numEls = lambda.Size() / numPoints;
-
+   const int numEls = lambda.Size()/numPoints;
    const auto lamDev = Reshape(lambda.Read(), numPoints, numEls);
    const auto muDev = Reshape(mu.Read(), numPoints, numEls);
    const auto J = Reshape(geom.J.Read(), numPoints, d, d, numEls);
+   auto Q = Reshape(QVec.ReadWrite(), numPoints, d,d, d, numEls);
    const real_t *ipWeights = ir.GetWeights().Read();
-   const auto G = Reshape(maps.G.Read(), numPoints, d, nDofs);
-   auto diagDev = Reshape(diag.Write(), nDofs, d, numEls);
+   mfem::forall_2D(numEls, numPoints,1, [=] MFEM_HOST_DEVICE (int e)
+   {
+      MFEM_FOREACH_THREAD(p, x,numPoints)
+      {
+         auto invJ = inv(make_tensor<d, d>(
+         [&](int i, int j) { return J(p, i, j, e); }));
+         const real_t w = ipWeights[p] /det(invJ);
+         for (int n = 0; n < d; n++)
+         {
+            for (int m = 0; m < d; m++)
+            {
+               for (int q = 0; q < d; q++)
+               {
+                  // compute contraction of 4*sym(grad(u))sym(grad(v)) term.
+                  // this contraction could be made slightly cheaper using Voigt
+                  // notation, but repeated entries are summed for simplicity.
+                  real_t contraction = 0.;
+                  for (int a = 0; a < d; a++)
+                  {
+                     for (int b = 0; b < d; b++)
+                     {
+                        contraction += ((a == q)*invJ(m,b) + (b==q)*invJ(m,a))*((a == q)
+                                                                                *invJ(n, b) + (b==q)*invJ(n,a));
+                     }
+                  }
+                  // lambda*div(u)*div(v) + 2*mu*sym(grad(u))*sym(grad(v))
+                  // contraction = 4*sym(grad(u))sym(grad(v))
+                  Q(p,m,n,q,e) = w*(lamDev(p, e)*invJ(m,q)*invJ(n,q)
+                                    + 0.5*muDev(p, e)*contraction);
+               }
+            }
+         }
+      }
+   });
 
+   // Reduce quadrature function to an E-Vector
+   const auto QRead = Reshape(QVec.Read(), numPoints, d, d, d, numEls);
+   auto diagDev = Reshape(diag.Write(), nDofs, d, numEls);
+   const auto G = Reshape(maps.G.Read(), numPoints, d, nDofs);
    mfem::forall_2D(numEls, d, nDofs, [=] MFEM_HOST_DEVICE (int e)
    {
-      MFEM_FOREACH_THREAD_DIRECT(i, y, nDofs)
+      MFEM_FOREACH_THREAD(i, y, nDofs)
       {
-         MFEM_FOREACH_THREAD_DIRECT(q, x, d)
+         MFEM_FOREACH_THREAD(q, x, d)
          {
-            real_t sum = 0.0;
-            for (int p = 0; p < numPoints; p++)
+            real_t sum = 0.;
+            for (int n = 0; n < d; n++)
             {
-               const auto invJ = inv(make_tensor<d, d>([&](int r, int c)
+               for (int m = 0; m < d; m++)
                {
-                  return J(p, r, c, e);
-               }));
-               const real_t w = ipWeights[p] / det(invJ);
-
-               for (int n = 0; n < d; n++)
-               {
-                  for (int m = 0; m < d; m++)
+                  for (int p = 0; p < numPoints; p++ )
                   {
-                     // compute contraction of 4*sym(grad(u))sym(grad(v)) term.
-                     // this contraction could be made slightly cheaper using Voigt
-                     // notation, but repeated entries are summed for simplicity.
-                     real_t contraction = 0.0;
-                     for (int a = 0; a < d; a++)
-                     {
-                        for (int b = 0; b < d; b++)
-                        {
-                           contraction +=
-                              ((a == q) * invJ(m, b) + (b == q) * invJ(m, a)) *
-                              ((a == q) * invJ(n, b) + (b == q) * invJ(n, a));
-                        }
-                     }
-                     // lambda*div(u)*div(v) + 2*mu*sym(grad(u))*sym(grad(v))
-                     // contraction = 4*sym(grad(u))sym(grad(v))
-                     const real_t Q =
-                        w * (lamDev(p, e) * invJ(m, q) * invJ(n, q)
-                             + 0.5 * muDev(p, e) * contraction);
-                     sum += Q * G(p, m, i) * G(p, n, i);
+                     sum += QRead(p,m,n,q,e)*G(p,m,i)*G(p,n,i);
                   }
                }
             }

fem/integ/bilininteg_elasticity_pa.cpp

@@ -10,6 +10,7 @@
 // CONTRIBUTING.md for details.
 
 #include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
 #include "../qfunction.hpp"
 #include "bilininteg_elasticity_kernels.hpp"
 
@@ -58,8 +59,9 @@ void ElasticityIntegrator::AssemblePA(const FiniteElementSpace &fes)
 
 void ElasticityIntegrator::AssembleDiagonalPA(Vector &diag)
 {
+   q_vec->SetVDim(vdim*vdim*vdim*vdim);
    internal::ElasticityAssembleDiagonalPA(vdim, ndofs, *lambda_quad, *mu_quad,
-                                          *geom, *maps, *IntRule, diag);
+                                          *geom, *maps, *q_vec, diag);
 }
 
 void ElasticityIntegrator::AddMultPA(const Vector &x, Vector &y) const

fem/integ/bilininteg_hcurl_kernels.cpp

@@ -147,16 +147,18 @@ void PAHcurlMassAssembleDiagonal3D(const int D1D,
    }); // end of element loop
 }
 
-void PAHcurlMassApply2D(const int NE, const bool symmetric,
-                        [[maybe_unused]] const bool scalar_coeff,
-                        const Array<real_t> &bo, const Array<real_t> &bc,
-                        const Array<real_t> &bot, const Array<real_t> &bct,
-                        const Vector &pa_data, const Vector &x, Vector &y,
-                        const int D1D, [[maybe_unused]] const int TestD1D,
-                        const int Q1D)
+void PAHcurlMassApply2D(const int D1D,
+                        const int Q1D,
+                        const int NE,
+                        const bool symmetric,
+                        const Array<real_t> &bo,
+                        const Array<real_t> &bc,
+                        const Array<real_t> &bot,
+                        const Array<real_t> &bct,
+                        const Vector &pa_data,
+                        const Vector &x,
+                        Vector &y)
 {
-   MFEM_ASSERT(D1D == TestD1D,
-               "Trial and Test space must have the same number of dofs");
    auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
    auto Bc = Reshape(bc.Read(), Q1D, D1D);
    auto Bot = Reshape(bot.Read(), D1D-1, Q1D);
@@ -275,16 +277,18 @@ void PAHcurlMassApply2D(const int NE, const bool symmetric,
    }); // end of element loop
 }
 
-void PAHcurlMassApply3D(const int NE, const bool symmetric,
-                        [[maybe_unused]] const bool scalar_coeff,
-                        const Array<real_t> &bo, const Array<real_t> &bc,
-                        const Array<real_t> &bot, const Array<real_t> &bct,
-                        const Vector &pa_data, const Vector &x, Vector &y,
-                        const int D1D, [[maybe_unused]] const int TestD1D,
-                        const int Q1D)
+void PAHcurlMassApply3D(const int D1D,
+                        const int Q1D,
+                        const int NE,
+                        const bool symmetric,
+                        const Array<real_t> &bo,
+                        const Array<real_t> &bc,
+                        const Array<real_t> &bot,
+                        const Array<real_t> &bct,
+                        const Vector &pa_data,
+                        const Vector &x,
+                        Vector &y)
 {
-   MFEM_VERIFY(D1D == TestD1D,
-               "Trial and test spaces must have same number of dofs");
    MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
                "Error: D1D > MAX_D1D");
    MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,

fem/integ/bilininteg_hcurl_kernels.hpp

@@ -181,312 +181,228 @@ inline void SmemPAHcurlMassAssembleDiagonal3D(const int d1d,
 }
 
 // PA H(curl) Mass Apply 2D kernel
-void PAHcurlMassApply2D(const int NE, const bool symmetric,
-                        const bool scalar_coeff, const Array<real_t> &bo,
-                        const Array<real_t> &bc, const Array<real_t> &bot,
-                        const Array<real_t> &bct, const Vector &pa_data,
-                        const Vector &x, Vector &y, const int TrialD1D,
-                        const int TestD1D, const int Q1D);
+void PAHcurlMassApply2D(const int D1D,
+                        const int Q1D,
+                        const int NE,
+                        const bool symmetric,
+                        const Array<real_t> &bo,
+                        const Array<real_t> &bc,
+                        const Array<real_t> &bot,
+                        const Array<real_t> &bct,
+                        const Vector &pa_data,
+                        const Vector &x,
+                        Vector &y);
 
 // PA H(curl) Mass Apply 3D kernel
-void PAHcurlMassApply3D(const int NE, const bool symmetric,
-                        [[maybe_unused]] const bool scalar_coeff,
-                        const Array<real_t> &bo, const Array<real_t> &bc,
-                        const Array<real_t> &bot, const Array<real_t> &bct,
-                        const Vector &pa_data, const Vector &x, Vector &y,
-                        const int TrialD1D, [[maybe_unused]] const int TestD1D,
-                        const int Q1D);
+void PAHcurlMassApply3D(const int D1D,
+                        const int Q1D,
+                        const int NE,
+                        const bool symmetric,
+                        const Array<real_t> &bo,
+                        const Array<real_t> &bc,
+                        const Array<real_t> &bot,
+                        const Array<real_t> &bct,
+                        const Vector &pa_data,
+                        const Vector &x,
+                        Vector &y);
 
 // Shared memory PA H(curl) Mass Apply 3D kernel
-template <int T_D1D = 0, int T_Q1D = 0, int TBATCH = 0, bool ACCUMULATE = true>
-inline void SmemPAHcurlMassApply3D(
-   const int NE, const bool symmetric, [[maybe_unused]] const bool scalar_coeff,
-   const Array<real_t> &bo, const Array<real_t> &bc,
-   [[maybe_unused]] const Array<real_t> &bot,
-   [[maybe_unused]] const Array<real_t> &bct, const Vector &pa_data,
-   const Vector &x, Vector &y, const int d1d = 0,
-   [[maybe_unused]] const int test_d1d = 0, const int q1d = 0)
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAHcurlMassApply3D(const int d1d,
+                                   const int q1d,
+                                   const int NE,
+                                   const bool symmetric,
+                                   const Array<real_t> &bo,
+                                   const Array<real_t> &bc,
+                                   const Array<real_t> &bot,
+                                   const Array<real_t> &bct,
+                                   const Vector &pa_data,
+                                   const Vector &x,
+                                   Vector &y)
 {
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
    MFEM_VERIFY(T_D1D || d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
                "Error: d1d > HCURL_MAX_D1D");
    MFEM_VERIFY(T_Q1D || q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
                "Error: q1d > HCURL_MAX_Q1D");
-   MFEM_ASSERT(Q1D >= D1D, "Expected Q1D >= D1D");
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
    const int dataSize = symmetric ? 6 : 9;
 
-   // assume trial space == test space
-   auto Bo = bo.Read();
-   auto Bc = bc.Read();
-   auto op =
-      Reshape(pa_data.Read(), Q1D, Q1D, Q1D, dataSize, NE);
-   auto X_ = Reshape(x.Read(), 3 * (D1D - 1) * D1D * D1D, NE);
-   auto y_ = y.ReadWrite();
+   auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
+   auto Bc = Reshape(bc.Read(), Q1D, D1D);
+   auto op = Reshape(pa_data.Read(), Q1D, Q1D, Q1D, dataSize, NE);
+   auto X = Reshape(x.Read(), 3*(D1D-1)*D1D*D1D, NE);
+   auto Y = Reshape(y.ReadWrite(), 3*(D1D-1)*D1D*D1D, NE);
 
-   constexpr int MD_ = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
-   constexpr int MQ_ = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
-   constexpr int MDQ_ = std::max(MD_, MQ_);
-   constexpr int MB_ = TBATCH ? TBATCH : 1;
-
-   mfem::forall_2D_batch<MDQ_ * MDQ_ * MDQ_ * MB_>(
-      NE, MDQ_ * MDQ_ * MDQ_, 1, MB_, [=] MFEM_HOST_DEVICE(int e)
+   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
    {
-#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
-      constexpr int nbz = TBATCH ? TBATCH : 1;
-      int tidz = MFEM_THREAD_ID(z);
-#else
-      constexpr int nbz = 1;
-      constexpr int tidz = 0;
-#endif
-
       constexpr int VDIM = 3;
       constexpr int MD1D = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
       constexpr int MQ1D = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
-      constexpr int MDQ = std::max(MD1D, MQ1D);
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
 
-      // nvcc limit work-around: can't have Y_ be captured first in
-      // if constexpr, so capture y_ and construct Y_ locally
-      // only works on GPU
-      auto Y = Reshape(y_, VDIM * (D1D - 1) * D1D * D1D, NE);
+      MFEM_SHARED real_t sBo[MQ1D][MD1D];
+      MFEM_SHARED real_t sBc[MQ1D][MD1D];
 
-      MFEM_SHARED real_t sBo[MDQ * (MD1D - 1)];
-      MFEM_SHARED real_t sBc[MDQ * MD1D];
-      auto BO = Reshape(sBo, Q1D, D1D - 1);
-      auto BC = Reshape(sBc, Q1D, D1D);
+      real_t op9[9];
+      MFEM_SHARED real_t sop[9*MQ1D*MQ1D];
+      MFEM_SHARED real_t mass[MQ1D][MQ1D][3];
 
-      MFEM_SHARED real_t sX[nbz * VDIM * (MD1D - 1) * MD1D * MD1D];
-      MFEM_SHARED real_t sm0[nbz * VDIM * MDQ * MDQ * MDQ];
-      MFEM_SHARED real_t sm1[nbz * VDIM * MDQ * MDQ * MDQ];
+      MFEM_SHARED real_t sX[MD1D][MD1D][MD1D];
 
-      real_t(*X)[nbz][(MD1D - 1) * MD1D * MD1D] =
-         (real_t(*)[nbz][(MD1D - 1) * MD1D * MD1D])(sX);
-      // shapes of buffers always use MQ1D to mitigate shared memory bank
-      // conflicts
-      real_t(*DDQ)[nbz][MQ1D][MQ1D][MQ1D] =
-         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm0);
-      real_t(*DQQ)[nbz][MQ1D][MQ1D][MQ1D] =
-         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm1);
-      real_t(*QQQ)[nbz][MQ1D][MQ1D][MQ1D] =
-         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm0);
-      real_t(*QQD)[nbz][MQ1D][MQ1D][MQ1D] =
-         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm1);
-      real_t(*QDD)[nbz][MQ1D][MQ1D][MQ1D] =
-         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm0);
-
-      // load dofs into smem
-      const int offset = (D1D - 1) * D1D * D1D;
-      MFEM_FOREACH_THREAD_DIRECT(ix, x, offset)
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
       {
-         for (int dim = 0; dim < VDIM; ++dim)
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
          {
-            X[dim][tidz][ix] = X_(ix + dim * offset, e);
+            MFEM_FOREACH_THREAD(qz,z,Q1D)
+            {
+               for (int i=0; i<dataSize; ++i)
+               {
+                  op9[i] = op(qx,qy,qz,i,e);
+               }
+            }
          }
       }
-      // load basis functions data
+
+      const int tidx = MFEM_THREAD_ID(x);
+      const int tidy = MFEM_THREAD_ID(y);
+      const int tidz = MFEM_THREAD_ID(z);
+
       if (tidz == 0)
       {
-         MFEM_FOREACH_THREAD_DIRECT(ix, x, D1D * Q1D) { sBc[ix] = Bc[ix]; }
-         MFEM_FOREACH_THREAD_DIRECT(ix, x, (D1D - 1) * Q1D)
+         MFEM_FOREACH_THREAD(d,y,D1D)
          {
-            sBo[ix] = Bo[ix];
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               sBc[q][d] = Bc(q,d);
+               if (d < D1D-1)
+               {
+                  sBo[q][d] = Bo(q,d);
+               }
+            }
          }
       }
+      MFEM_SYNC_THREAD;
 
-      for (int dim0 = 0; dim0 < VDIM; ++dim0)
+      for (int qz=0; qz < Q1D; ++qz)
       {
-         MFEM_SYNC_THREAD;
-         // sum factor to QQQ = Q_{dim0,dim1} B X_{dim1}
-         for (int dim1 = 0; dim1 < VDIM; ++dim1)
+         int osc = 0;
+         for (int c = 0; c < VDIM; ++c)  // loop over x, y, z components
          {
-            const int D1Dz = (dim1 == 2) ? D1D - 1 : D1D;
-            const int D1Dy = (dim1 == 1) ? D1D - 1 : D1D;
-            const int D1Dx = (dim1 == 0) ? D1D - 1 : D1D;
+            const int D1Dz = (c == 2) ? D1D - 1 : D1D;
+            const int D1Dy = (c == 1) ? D1D - 1 : D1D;
+            const int D1Dx = (c == 0) ? D1D - 1 : D1D;
 
-            // threads assigned to mitigate bank conflicts
-            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, Q1D, D1Dy, D1Dz,
-                                                 Q1D, Q1D, Q1D)
+            MFEM_FOREACH_THREAD(dz,z,D1Dz)
             {
-               real_t u = 0;
-               for (int dx = 0; dx < D1Dx; ++dx)
+               MFEM_FOREACH_THREAD(dy,y,D1Dy)
                {
-                  real_t b;
-                  if (dim1 == 0)
+                  MFEM_FOREACH_THREAD(dx,x,D1Dx)
                   {
-                     b = BO(qx, dx);
+                     sX[dz][dy][dx] = X(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e);
                   }
-                  else
-                  {
-                     b = BC(qx, dx);
-                  }
-                  u += X[dim1][tidz][dx + (dy + dz * D1Dy) * D1Dx] * b;
-               }
-               DDQ[dim1][tidz][dz][dy][qx] = u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-         for (int dim1 = 0; dim1 < VDIM; ++dim1)
-         {
-            const int D1Dz = (dim1 == 2) ? D1D - 1 : D1D;
-            const int D1Dy = (dim1 == 1) ? D1D - 1 : D1D;
-            // const int D1Dx = (dim1 == 0) ? D1D - 1 : D1D;
-            // threads assigned to mitigate bank conflicts
-            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, Q1D, Q1D, D1Dz,
-                                                 Q1D, Q1D, Q1D)
-            {
-               real_t u = 0;
-               for (int dy = 0; dy < D1Dy; ++dy)
-               {
-                  real_t b;
-                  if (dim1 == 1)
-                  {
-                     b = BO(qy, dy);
-                  }
-                  else
-                  {
-                     b = BC(qy, dy);
-                  }
-                  u += DDQ[dim1][tidz][dz][dy][qx] * b;
-               }
-               DQQ[dim1][tidz][dz][qy][qx] = u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-         for (int dim1 = 0; dim1 < VDIM; ++dim1)
-         {
-            const int D1Dz = (dim1 == 2) ? D1D - 1 : D1D;
-            // const int D1Dy = (dim1 == 1) ? D1D - 1 : D1D;
-            // const int D1Dx = (dim1 == 0) ? D1D - 1 : D1D;
-            MFEM_FOREACH_THREAD_DIRECT_3D(qx, qy, qz, x, Q1D, Q1D, Q1D)
-            {
-               real_t u = 0;
-               for (int dz = 0; dz < D1Dz; ++dz)
-               {
-                  real_t b;
-                  if (dim1 == 2)
-                  {
-                     b = BO(qz, dz);
-                  }
-                  else
-                  {
-                     b = BC(qz, dz);
-                  }
-                  u += DQQ[dim1][tidz][dz][qy][qx] * b;
-               }
-               // pa_data is row major
-               int idx;
-               if (symmetric)
-               {
-                  int row;
-                  int col;
-                  if (dim0 > dim1)
-                  {
-                     row = dim1;
-                     col = dim0;
-                  }
-                  else
-                  {
-                     row = dim0;
-                     col = dim1;
-                  }
-                  idx = col + VDIM * row - row * (row + 1) / 2;
-               }
-               else
-               {
-                  idx = dim0 * VDIM + dim1;
-               }
-               QQQ[dim1][tidz][qz][qy][qx] = op(qx, qy, qz, idx, e) * u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-         // sum factor back to Y
-         // Assume bot and bct == bo^t and bc^t respectively (i.e. test ==
-         // trial functions), skip loading them again.
-         {
-            const int D1Dz = (dim0 == 2) ? D1D - 1 : D1D;
-            const int D1Dy = (dim0 == 1) ? D1D - 1 : D1D;
-            const int D1Dx = (dim0 == 0) ? D1D - 1 : D1D;
-            // threads assigned to mitigate bank conflicts
-            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, D1Dz, Q1D, Q1D,
-                                                 Q1D, Q1D, Q1D)
-            {
-               for (int dim1 = 0; dim1 < VDIM; ++dim1)
-               {
-                  real_t u = 0;
-                  for (int qz = 0; qz < Q1D; ++qz)
-                  {
-                     real_t b = 0;
-                     if (dim0 == 2)
-                     {
-                        b = BO(qz, dz);
-                     }
-                     else
-                     {
-                        b = BC(qz, dz);
-                     }
-                     u += QQQ[dim1][tidz][qz][qy][qx] * b;
-                  }
-                  QQD[dim1][tidz][qy][qx][dz] = u;
-               }
-            }
-            MFEM_SYNC_THREAD;
-            // threads assigned to mitigate bank conflicts
-            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, D1Dy, D1Dz, Q1D,
-                                                 Q1D, Q1D, Q1D)
-            {
-               for (int dim1 = 0; dim1 < VDIM; ++dim1)
-               {
-                  real_t u = 0;
-                  for (int qy = 0; qy < Q1D; ++qy)
-                  {
-                     real_t b;
-                     if (dim0 == 1)
-                     {
-                        b = BO(qy, dy);
-                     }
-                     else
-                     {
-                        b = BC(qy, dy);
-                     }
-                     u += QQD[dim1][tidz][qy][qx][dz] * b;
-                  }
-                  QDD[dim1][tidz][qx][dz][dy] = u;
                }
             }
             MFEM_SYNC_THREAD;
 
-            MFEM_FOREACH_THREAD_DIRECT_3D(dx, dy, dz, x, D1Dx, D1Dy, D1Dz)
+            if (tidz == qz)
             {
-               int ix = dx + D1Dx * (dy + D1Dy * dz);
-               real_t u = 0;
-               for (int qx = 0; qx < Q1D; ++qx)
+               for (int i=0; i<dataSize; ++i)
                {
-                  real_t b;
-                  if (dim0 == 0)
-                  {
-                     b = BO(qx, dx);
-                  }
-                  else
-                  {
-                     b = BC(qx, dx);
-                  }
-                  for (int dim1 = 0; dim1 < VDIM; ++dim1)
-                  {
-                     u += QDD[dim1][tidz][qx][dz][dy] * b;
-                  }
+                  sop[i + (dataSize*tidx) + (dataSize*Q1D*tidy)] = op9[i];
                }
-               if constexpr (ACCUMULATE)
+
+               MFEM_FOREACH_THREAD(qy,y,Q1D)
                {
-                  Y(ix + dim0 * offset, e) += u;
-               }
-               else
+                  MFEM_FOREACH_THREAD(qx,x,Q1D)
+                  {
+                     real_t u = 0.0;
+
+                     for (int dz = 0; dz < D1Dz; ++dz)
+                     {
+                        const real_t wz = (c == 2) ? sBo[qz][dz] : sBc[qz][dz];
+                        for (int dy = 0; dy < D1Dy; ++dy)
+                        {
+                           const real_t wy = (c == 1) ? sBo[qy][dy] : sBc[qy][dy];
+                           for (int dx = 0; dx < D1Dx; ++dx)
+                           {
+                              const real_t t = sX[dz][dy][dx];
+                              const real_t wx = (c == 0) ? sBo[qx][dx] : sBc[qx][dx];
+                              u += t * wx * wy * wz;
+                           }
+                        }
+                     }
+
+                     mass[qy][qx][c] = u;
+                  } // qx
+               } // qy
+            } // tidz == qz
+
+            osc += D1Dx * D1Dy * D1Dz;
+            MFEM_SYNC_THREAD;
+         } // c
+
+         MFEM_SYNC_THREAD;  // Sync mass[qy][qx][d] and sop
+
+         osc = 0;
+         for (int c = 0; c < VDIM; ++c)  // loop over x, y, z components
+         {
+            const int D1Dz = (c == 2) ? D1D - 1 : D1D;
+            const int D1Dy = (c == 1) ? D1D - 1 : D1D;
+            const int D1Dx = (c == 0) ? D1D - 1 : D1D;
+
+            real_t dxyz = 0.0;
+
+            MFEM_FOREACH_THREAD(dz,z,D1Dz)
+            {
+               const real_t wz = (c == 2) ? sBo[qz][dz] : sBc[qz][dz];
+
+               MFEM_FOREACH_THREAD(dy,y,D1Dy)
                {
-                  Y(ix + dim0 * offset, e) = u;
+                  MFEM_FOREACH_THREAD(dx,x,D1Dx)
+                  {
+                     for (int qy = 0; qy < Q1D; ++qy)
+                     {
+                        const real_t wy = (c == 1) ? sBo[qy][dy] : sBc[qy][dy];
+                        for (int qx = 0; qx < Q1D; ++qx)
+                        {
+                           const int os = (dataSize*qx) + (dataSize*Q1D*qy);
+                           const int id1 = os + ((c == 0) ? 0 : ((c == 1) ? (symmetric ? 1 : 3) :
+                                                                 (symmetric ? 2 : 6))); // O11, O21, O31
+                           const int id2 = os + ((c == 0) ? 1 : ((c == 1) ? (symmetric ? 3 : 4) :
+                                                                 (symmetric ? 4 : 7))); // O12, O22, O32
+                           const int id3 = os + ((c == 0) ? 2 : ((c == 1) ? (symmetric ? 4 : 5) :
+                                                                 (symmetric ? 5 : 8))); // O13, O23, O33
+
+                           const real_t m_c = (sop[id1] * mass[qy][qx][0]) + (sop[id2] * mass[qy][qx][1]) +
+                                              (sop[id3] * mass[qy][qx][2]);
+
+                           const real_t wx = (c == 0) ? sBo[qx][dx] : sBc[qx][dx];
+                           dxyz += m_c * wx * wy * wz;
+                        }
+                     }
+                  }
                }
             }
-         }
-      }
+
+            MFEM_SYNC_THREAD;
+
+            MFEM_FOREACH_THREAD(dz,z,D1Dz)
+            {
+               MFEM_FOREACH_THREAD(dy,y,D1Dy)
+               {
+                  MFEM_FOREACH_THREAD(dx,x,D1Dx)
+                  {
+                     Y(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e) += dxyz;
+                  }
+               }
+            }
+
+            osc += D1Dx * D1Dy * D1Dz;
+         } // c loop
+      } // qz
    }); // end of element loop
 }
 

fem/integ/bilininteg_hcurlhdiv_kernels.hpp

@@ -62,30 +62,6 @@ void PAHcurlHdivMassApply2D(const int D1D,
                             const Vector &x_,
                             Vector &y_);
 
-/// H(curl) test, H(div) trial
-inline void
-PAHcurlHdivMassApply2D(const int NE, const bool, const bool scalarCoeff,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int D1Dtest, const int Q1D)
-{
-   return PAHcurlHdivMassApply2D(D1D, D1Dtest, Q1D, NE, scalarCoeff, false,
-                                 false, Bo_, Bc_, Bot_, Bct_, op_, x_, y_);
-}
-
-/// H(div) test, H(curl) trial
-inline void
-PAHdivHcurlMassApply2D(const int NE, const bool, const bool scalarCoeff,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int D1Dtest, const int Q1D)
-{
-   return PAHcurlHdivMassApply2D(D1D, D1Dtest, Q1D, NE, scalarCoeff, true,
-                                 false, Bo_, Bc_, Bot_, Bct_, op_, x_, y_);
-}
-
 // PA H(curl)-H(div) Mass Apply 3D kernel
 void PAHcurlHdivMassApply3D(const int D1D,
                             const int D1Dtest,
@@ -102,30 +78,6 @@ void PAHcurlHdivMassApply3D(const int D1D,
                             const Vector &x_,
                             Vector &y_);
 
-/// H(curl) test, H(div) trial
-inline void
-PAHcurlHdivMassApply3D(const int NE, const bool, const bool scalarCoeff,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int D1Dtest, const int Q1D)
-{
-   PAHcurlHdivMassApply3D(D1D, D1Dtest, Q1D, NE, scalarCoeff, false, false, Bo_,
-                          Bc_, Bot_, Bct_, op_, x_, y_);
-}
-
-/// H(div) test, H(curl) trial
-inline void
-PAHdivHcurlMassApply3D(const int NE, const bool, const bool scalarCoeff,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int D1Dtest, const int Q1D)
-{
-   PAHcurlHdivMassApply3D(D1D, D1Dtest, Q1D, NE, scalarCoeff, true, false, Bo_,
-                          Bc_, Bot_, Bct_, op_, x_, y_);
-}
-
 // PA H(curl)-H(div) Curl Apply 3D kernel
 template<int T_D1D = 0, int T_D1D_TEST = 0, int T_Q1D = 0>
 inline void PAHcurlHdivApply3D(const int d1d,

fem/integ/bilininteg_hdiv_kernels.cpp

@@ -294,14 +294,61 @@ void PAHdivMassAssembleDiagonal3D(const int D1D,
    }); // end of element loop
 }
 
-void PAHdivMassApply2D(const int NE, const bool symmetric, const bool,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int TestD1D, const int Q1D)
+void PAHdivMassApply(const int dim,
+                     const int D1D,
+                     const int Q1D,
+                     const int NE,
+                     const bool symmetric,
+                     const Array<real_t> &Bo,
+                     const Array<real_t> &Bc,
+                     const Array<real_t> &Bot,
+                     const Array<real_t> &Bct,
+                     const Vector &op,
+                     const Vector &x,
+                     Vector &y)
+{
+   const int id = (D1D << 4) | Q1D;
+
+   if (dim == 2)
+   {
+      switch (id)
+      {
+         case 0x22: return SmemPAHdivMassApply2D<2,2>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x33: return SmemPAHdivMassApply2D<3,3>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x44: return SmemPAHdivMassApply2D<4,4>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x55: return SmemPAHdivMassApply2D<5,5>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         default: // fallback
+            return PAHdivMassApply2D(D1D,Q1D,NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+      }
+   }
+   else if (dim == 3)
+   {
+      switch (id)
+      {
+         case 0x23: return SmemPAHdivMassApply3D<2,3>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x34: return SmemPAHdivMassApply3D<3,4>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x45: return SmemPAHdivMassApply3D<4,5>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x56: return SmemPAHdivMassApply3D<5,6>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x67: return SmemPAHdivMassApply3D<6,7>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x78: return SmemPAHdivMassApply3D<7,8>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         default: // fallback
+            return PAHdivMassApply3D(D1D,Q1D,NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+      }
+   }
+}
+
+void PAHdivMassApply2D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const bool symmetric,
+                       const Array<real_t> &Bo_,
+                       const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_,
+                       const Array<real_t> &Bct_,
+                       const Vector &op_,
+                       const Vector &x_,
+                       Vector &y_)
 {
-   MFEM_VERIFY(D1D == TestD1D,
-               "Trial and test spaces must have same number of dofs");
    auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
    auto Bc = Reshape(Bc_.Read(), Q1D, D1D);
    auto Bot = Reshape(Bot_.Read(), D1D-1, Q1D);
@@ -421,14 +468,18 @@ void PAHdivMassApply2D(const int NE, const bool symmetric, const bool,
    }); // end of element loop
 }
 
-void PAHdivMassApply3D(const int NE, const bool symmetric, const bool,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int TestD1D, const int Q1D)
+void PAHdivMassApply3D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const bool symmetric,
+                       const Array<real_t> &Bo_,
+                       const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_,
+                       const Array<real_t> &Bct_,
+                       const Vector &op_,
+                       const Vector &x_,
+                       Vector &y_)
 {
-   MFEM_VERIFY(D1D == TestD1D,
-               "Trial and test spaces must have same number of dofs");
    MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HDIV_MAX_D1D,
                "Error: D1D > HDIV_MAX_D1D");
    MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HDIV_MAX_Q1D,

fem/integ/bilininteg_hdiv_kernels.hpp

@@ -66,29 +66,58 @@ void PAHdivMassAssembleDiagonal3D(const int D1D,
                                   const Vector &op_,
                                   Vector &diag_);
 
+void PAHdivMassApply(const int dim,
+                     const int D1D,
+                     const int Q1D,
+                     const int NE,
+                     const bool symmetric,
+                     const Array<real_t> &Bo,
+                     const Array<real_t> &Bc,
+                     const Array<real_t> &Bot,
+                     const Array<real_t> &Bct,
+                     const Vector &op,
+                     const Vector &x,
+                     Vector &y);
+
 // PA H(div) Mass Apply 2D kernel
-void PAHdivMassApply2D(const int NE, const bool symmetric,
-                       const bool scalar_coeff, const Array<real_t> &Bo_,
-                       const Array<real_t> &Bc_, const Array<real_t> &Bot_,
-                       const Array<real_t> &Bct_, const Vector &op_,
-                       const Vector &x_, Vector &y_, const int D1D,
-                       const int TestD1D, const int Q1D);
+void PAHdivMassApply2D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const bool symmetric,
+                       const Array<real_t> &Bo_,
+                       const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_,
+                       const Array<real_t> &Bct_,
+                       const Vector &op_,
+                       const Vector &x_,
+                       Vector &y_);
 
 // PA H(div) Mass Apply 3D kernel
-void PAHdivMassApply3D(const int NE, const bool symmetric,
-                       const bool scalar_coeff, const Array<real_t> &Bo_,
-                       const Array<real_t> &Bc_, const Array<real_t> &Bot_,
-                       const Array<real_t> &Bct_, const Vector &op_,
-                       const Vector &x_, Vector &y_, const int D1D,
-                       const int TestD1D, const int Q1D);
+void PAHdivMassApply3D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const bool symmetric,
+                       const Array<real_t> &Bo_,
+                       const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_,
+                       const Array<real_t> &Bct_,
+                       const Vector &op_,
+                       const Vector &x_,
+                       Vector &y_);
 
 // Shared memory PA H(div) Mass Apply 2D kernel
-template <int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAHdivMassApply2D(
-   const int NE, const bool symmetric, const bool, const Array<real_t> &Bo_,
-   const Array<real_t> &Bc_, const Array<real_t> &Bot_,
-   const Array<real_t> &Bct_, const Vector &op_, const Vector &x_, Vector &y_,
-   const int d1d = 0, const int = 0, const int q1d = 0)
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAHdivMassApply2D(const int NE,
+                                  const bool symmetric,
+                                  const Array<real_t> &Bo_,
+                                  const Array<real_t> &Bc_,
+                                  const Array<real_t> &Bot_,
+                                  const Array<real_t> &Bct_,
+                                  const Vector &op_,
+                                  const Vector &x_,
+                                  Vector &y_,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
 {
    MFEM_CONTRACT_VAR(Bot_);
    MFEM_CONTRACT_VAR(Bct_);
@@ -251,13 +280,18 @@ inline void SmemPAHdivMassApply2D(
 }
 
 // Shared memory PA H(div) Mass Apply 3D kernel
-template <int T_D1D = 0, int T_Q1D = 0>
-inline void
-SmemPAHdivMassApply3D(const int NE, const bool symmetric, const bool,
-                      const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                      const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                      const Vector &op_, const Vector &x_, Vector &y_,
-                      const int d1d = 0, const int = 0, const int q1d = 0)
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAHdivMassApply3D(const int NE,
+                                  const bool symmetric,
+                                  const Array<real_t> &Bo_,
+                                  const Array<real_t> &Bc_,
+                                  const Array<real_t> &Bot_,
+                                  const Array<real_t> &Bct_,
+                                  const Vector &op_,
+                                  const Vector &x_,
+                                  Vector &y_,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
 {
    MFEM_CONTRACT_VAR(Bot_);
    MFEM_CONTRACT_VAR(Bct_);

fem/integ/bilininteg_vecdiv_pa.hpp

@@ -1,365 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../config/config.hpp"
-#include "../../general/array.hpp"
-#include "../../general/forall.hpp"
-#include "../../linalg/dtensor.hpp"
-#include "../../linalg/vector.hpp"
-#include "../bilininteg.hpp"
-#include "../kernels.hpp"
-
-namespace mfem
-{
-
-/// \cond DO_NOT_DOCUMENT
-
-namespace internal
-{
-
-// Shared memory PA Divergence Apply 2D kernel
-template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
-inline void SmemPADivergenceApply2D(const int NE,
-                                    const Array<real_t> &b_,
-                                    const Array<real_t> &g_,
-                                    const Array<real_t> &bt_,
-                                    const Vector &q_,
-                                    const Vector &x_,
-                                    Vector &y_,
-                                    const int tr_d1d = 0,
-                                    const int te_d1d = 0,
-                                    const int q1d = 0)
-{
-   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
-   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   const auto B = b_.Read(), G = g_.Read(), Bt = bt_.Read();
-   const auto Q = Reshape(q_.Read(), Q1D, Q1D, 2, 2, NE);
-   const auto X = Reshape(x_.Read(), TR_D1D, TR_D1D, 2, NE);
-   auto Y = Reshape(y_.ReadWrite(), TE_D1D, TE_D1D, 1, NE);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
-
-      kernels::internal::vd_regs2d_t<2, 2, MQ1> g0, g1;
-      kernels::internal::v_regs2d_t<1, MQ1> r0, r1;
-
-      kernels::internal::LoadMatrix(TR_D1D, Q1D, B, sB);
-      kernels::internal::LoadMatrix(TR_D1D, Q1D, G, sG);
-
-      kernels::internal::LoadDofs2d(e, TR_D1D, X, g0);
-      kernels::internal::Grad2d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
-
-      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            r0[0][qy][qx] =
-               g1[0][0][qy][qx] * Q(qx, qy, 0, 0, e) +
-               g1[0][1][qy][qx] * Q(qx, qy, 1, 0, e) +
-               g1[1][0][qy][qx] * Q(qx, qy, 0, 1, e) +
-               g1[1][1][qy][qx] * Q(qx, qy, 1, 1, e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      kernels::internal::LoadMatrix<MQ1,true>(TE_D1D, Q1D, Bt, sB);
-      kernels::internal::EvalTranspose2d(TE_D1D, Q1D, smem, sB, r0, r1);
-      kernels::internal::WriteDofs2d(e, TE_D1D, r1, Y);
-   });
-}
-
-// Shared memory PA Divergence Apply 2D kernel transpose
-template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
-inline void SmemPADivergenceApplyTranspose2D(const int NE,
-                                             const Array<real_t> &bt,
-                                             const Array<real_t> &gt,
-                                             const Array<real_t> &b,
-                                             const Vector &q_,
-                                             const Vector &x_,
-                                             Vector &y_,
-                                             const int tr_d1d = 0,
-                                             const int te_d1d = 0,
-                                             const int q1d = 0)
-{
-   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
-   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   const auto Bt = bt.Read(), Gt = gt.Read(), B = b.Read();
-   const auto Q = Reshape(q_.Read(), Q1D, Q1D, 2, 2, NE);
-   const auto X  = Reshape(x_.Read(), TE_D1D, TE_D1D, 1, NE);
-   auto Y  = Reshape(y_.ReadWrite(), TR_D1D, TR_D1D, 2, NE);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
-
-      kernels::internal::v_regs2d_t<1, MQ1> r0, r1;
-      kernels::internal::vd_regs2d_t<2, 2, MQ1> g0, g1;
-
-      kernels::internal::LoadMatrix(TE_D1D, Q1D, B, sB);
-      kernels::internal::LoadDofs2d(e, TE_D1D, X, r0);
-      kernels::internal::Eval2d(TE_D1D, Q1D, smem, sB, r0, r1);
-
-      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            g0[0][0][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 0, 0, e);
-            g0[0][1][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 1, 0, e);
-            g0[1][0][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 0, 1, e);
-            g0[1][1][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 1, 1, e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Bt, sB);
-      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Gt, sG);
-      kernels::internal::GradTranspose2d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
-      kernels::internal::WriteDofs2d(e, TR_D1D, g1, Y);
-   });
-}
-
-// Shared memory PA Divergence Apply 3D kernel transpose
-template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
-inline void SmemPADivergenceApplyTranspose3D(const int NE,
-                                             const Array<real_t> &bt,
-                                             const Array<real_t> &gt,
-                                             const Array<real_t> &b,
-                                             const Vector &q_,
-                                             const Vector &x_,
-                                             Vector &y_,
-                                             int tr_d1d = 0,
-                                             int te_d1d = 0,
-                                             int q1d = 0)
-{
-   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
-   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   const auto Bt = bt.Read(), Gt = gt.Read(), B = b.Read();
-   const auto Q = Reshape(q_.Read(), Q1D, Q1D, Q1D, 3, 3, NE);
-   const auto X = Reshape(x_.Read(), TE_D1D, TE_D1D, TE_D1D, 1, NE);
-   auto Y = Reshape(y_.ReadWrite(), TR_D1D, TR_D1D, TR_D1D, 3, NE);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
-
-      kernels::internal::v_regs3d_t<1, MQ1> r0, r1;
-      kernels::internal::vd_regs3d_t<3, 3, MQ1> g0, g1;
-
-      kernels::internal::LoadMatrix(TE_D1D, Q1D, B, sB);
-      kernels::internal::LoadDofs3d(e, TE_D1D, X, r0);
-      kernels::internal::Eval3d(TE_D1D, Q1D, smem, sB, r0, r1);
-
-      for (int qz = 0; qz < Q1D; qz++)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-            {
-               const auto r = r1[0][qz][qy][qx];
-               g0[0][0][qz][qy][qx] = r * Q(qx, qy, qz, 0, 0, e);
-               g0[0][1][qz][qy][qx] = r * Q(qx, qy, qz, 1, 0, e);
-               g0[0][2][qz][qy][qx] = r * Q(qx, qy, qz, 2, 0, e);
-
-               g0[1][0][qz][qy][qx] = r * Q(qx, qy, qz, 0, 1, e);
-               g0[1][1][qz][qy][qx] = r * Q(qx, qy, qz, 1, 1, e);
-               g0[1][2][qz][qy][qx] = r * Q(qx, qy, qz, 2, 1, e);
-
-               g0[2][0][qz][qy][qx] = r * Q(qx, qy, qz, 0, 2, e);
-               g0[2][1][qz][qy][qx] = r * Q(qx, qy, qz, 1, 2, e);
-               g0[2][2][qz][qy][qx] = r * Q(qx, qy, qz, 2, 2, e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Bt, sB);
-      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Gt, sG);
-      kernels::internal::GradTranspose3d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
-      kernels::internal::WriteDofs3d(e, TR_D1D, g1, Y);
-   });
-}
-
-// Shared memory PA Divergence Apply 3D kernel
-template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
-inline void SmemPADivergenceApply3D(const int NE,
-                                    const Array<real_t> &b_,
-                                    const Array<real_t> &g_,
-                                    const Array<real_t> &bt_,
-                                    const Vector &q_,
-                                    const Vector &x_,
-                                    Vector &y_,
-                                    const int tr_d1d = 0,
-                                    const int te_d1d = 0,
-                                    const int q1d = 0)
-{
-   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
-   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   const auto B = b_.Read(), G = g_.Read(), Bt = bt_.Read();
-   const auto Q = Reshape(q_.Read(), Q1D, Q1D, Q1D, 3,3, NE);
-   const auto X = Reshape(x_.Read(), TR_D1D, TR_D1D, TR_D1D, 3, NE);
-   auto Y = Reshape(y_.ReadWrite(), TE_D1D, TE_D1D, TE_D1D, 1, NE);
-
-   mfem::forall_2D<T_Q1D*T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
-
-      kernels::internal::vd_regs3d_t<3, 3, MQ1> g0, g1;
-      kernels::internal::v_regs3d_t<1, MQ1> r0, r1;
-
-      kernels::internal::LoadMatrix(TR_D1D, Q1D, B, sB);
-      kernels::internal::LoadMatrix(TR_D1D, Q1D, G, sG);
-
-      kernels::internal::LoadDofs3d(e, TR_D1D, X, g0);
-      kernels::internal::Grad3d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
-
-      for (int qz = 0; qz < Q1D; qz++)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-            {
-               r0[0][qz][qy][qx] =
-                  // c = 0
-                  g1[0][0][qz][qy][qx] * Q(qx, qy, qz, 0, 0, e) +
-                  g1[0][1][qz][qy][qx] * Q(qx, qy, qz, 1, 0, e) +
-                  g1[0][2][qz][qy][qx] * Q(qx, qy, qz, 2, 0, e) +
-                  // c = 1
-                  g1[1][0][qz][qy][qx] * Q(qx, qy, qz, 0, 1, e) +
-                  g1[1][1][qz][qy][qx] * Q(qx, qy, qz, 1, 1, e) +
-                  g1[1][2][qz][qy][qx] * Q(qx, qy, qz, 2, 1, e) +
-                  // c = 2
-                  g1[2][0][qz][qy][qx] * Q(qx, qy, qz, 0, 2, e) +
-                  g1[2][1][qz][qy][qx] * Q(qx, qy, qz, 1, 2, e) +
-                  g1[2][2][qz][qy][qx] * Q(qx, qy, qz, 2, 2, e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      kernels::internal::LoadMatrix<MQ1, true>(TE_D1D, Q1D, Bt, sB);
-      kernels::internal::EvalTranspose3d(TE_D1D, Q1D, smem, sB, r0, r1);
-      kernels::internal::WriteDofs3d(e, TE_D1D, r1, Y);
-   });
-}
-
-} // namespace internal
-
-template<int DIM, int T_TR_D1D, int T_TE_D1D, int T_Q1D>
-VectorDivergenceIntegrator::VectorDivergenceAddMultPAType
-VectorDivergenceIntegrator::VectorDivergenceAddMultPA::Kernel()
-{
-   static_assert(T_TR_D1D <= T_Q1D && T_TE_D1D <= T_Q1D);
-   if constexpr (DIM == 2)
-   {
-      return internal::SmemPADivergenceApply2D<T_TR_D1D, T_TE_D1D, T_Q1D>;
-   }
-   else if constexpr (DIM == 3)
-   {
-      return internal::SmemPADivergenceApply3D<T_TR_D1D, T_TE_D1D, T_Q1D>;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-inline VectorDivergenceIntegrator::VectorDivergenceAddMultPAType
-VectorDivergenceIntegrator::VectorDivergenceAddMultPA::Fallback
-(int dim, int tr_d1d, int te_d1d, int q1d)
-{
-   MFEM_VERIFY(tr_d1d <= q1d && te_d1d <= q1d, "");
-   MFEM_VERIFY(tr_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(te_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   if (dim == 2)
-   {
-      return internal::SmemPADivergenceApply2D;
-   }
-   else if (dim == 3)
-   {
-      return internal::SmemPADivergenceApply3D;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-template<int DIM, int T_TR_D1D, int T_TE_D1D, int T_Q1D>
-VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePAType
-VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePA::Kernel()
-{
-   static_assert(T_TR_D1D <= T_Q1D && T_TE_D1D <= T_Q1D);
-   if constexpr (DIM == 2)
-   {
-      return internal::SmemPADivergenceApplyTranspose2D<T_TR_D1D, T_TE_D1D, T_Q1D>;
-   }
-   else if constexpr (DIM == 3)
-   {
-      return internal::SmemPADivergenceApplyTranspose3D<T_TR_D1D, T_TE_D1D, T_Q1D>;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-inline VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePAType
-VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePA::Fallback
-(int dim, int tr_d1d, int te_d1d, int q1d)
-{
-   MFEM_VERIFY(tr_d1d <= q1d && te_d1d <= q1d, "");
-   MFEM_VERIFY(tr_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(te_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   if (dim == 2)
-   {
-      return internal::SmemPADivergenceApplyTranspose2D;
-   }
-   else if (dim == 3)
-   {
-      return internal::SmemPADivergenceApplyTranspose3D;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
-} // namespace mfem

fem/integ/bilininteg_vecmass_pa.cpp

@@ -205,40 +205,157 @@ void VectorMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 
 }
 
+template <const int T_D1D = 0, const int T_Q1D = 0>
+static void PAVectorMassAssembleDiagonal2D(const int NE,
+                                           const Array<real_t> &b,
+                                           const Vector &pa_data, Vector &diag,
+                                           const int d1d = 0, const int q1d = 0)
+{
+   constexpr int VDIM = 2;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   const auto B = Reshape(b.Read(), Q1D, D1D);
+   const auto D = Reshape(pa_data.Read(), Q1D, Q1D, NE);
+   auto Y = Reshape(diag.ReadWrite(), D1D, D1D, VDIM, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      real_t temp[max_Q1D][max_D1D];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            temp[qx][dy] = 0.0;
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               temp[qx][dy] += B(qy, dy) * B(qy, dy) * D(qx, qy, e);
+            }
+         }
+      }
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            real_t temp1 = 0.0;
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               temp1 += B(qx, dx) * B(qx, dx) * temp[qx][dy];
+            }
+            Y(dx, dy, 0, e) = temp1;
+            Y(dx, dy, 1, e) = temp1;
+         }
+      }
+   });
+}
+
+template <const int T_D1D = 0, const int T_Q1D = 0>
+static void PAVectorMassAssembleDiagonal3D(const int NE,
+                                           const Array<real_t> &B_,
+                                           const Vector &pa_data, Vector &diag,
+                                           const int d1d = 0, const int q1d = 0)
+{
+   constexpr int VDIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   const auto B = Reshape(B_.Read(), Q1D, D1D);
+   MFEM_VERIFY(pa_data.Size() == Q1D * Q1D * Q1D * NE, "pa_data size error");
+   const auto D = Reshape(pa_data.Read(), Q1D, Q1D, Q1D, NE);
+   auto Y = Reshape(diag.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      // the following variables are evaluated at compile time
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      real_t temp[max_Q1D][max_Q1D][max_D1D];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               temp[qx][qy][dz] = 0.0;
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  temp[qx][qy][dz] +=
+                     B(qz, dz) * B(qz, dz) * D(qx, qy, qz, e);
+               }
+            }
+         }
+      }
+      real_t temp2[max_Q1D][max_D1D][max_D1D];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               temp2[qx][dy][dz] = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  temp2[qx][dy][dz] +=
+                     B(qy, dy) * B(qy, dy) * temp[qx][qy][dz];
+               }
+            }
+         }
+      }
+      for (int dz = 0; dz < D1D; ++dz)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               real_t temp3 = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  temp3 += B(qx, dx) * B(qx, dx) * temp2[qx][dy][dz];
+               }
+               Y(dx, dy, dz, 0, e) = temp3;
+               Y(dx, dy, dz, 1, e) = temp3;
+               Y(dx, dy, dz, 2, e) = temp3;
+            }
+         }
+      }
+   });
+}
+
+static void PAVectorMassAssembleDiagonal(const int dim, const int D1D,
+                                         const int Q1D, const int NE,
+                                         const Array<real_t> &B,
+                                         const Vector &pa_data,
+                                         Vector &diag)
+{
+   if (dim == 2)
+   {
+      return PAVectorMassAssembleDiagonal2D(NE, B, pa_data, diag, D1D, Q1D);
+   }
+   else if (dim == 3)
+   {
+      return PAVectorMassAssembleDiagonal3D(NE, B, pa_data, diag, D1D, Q1D);
+   }
+   MFEM_ABORT("Dimension not implemented.");
+}
+
 void VectorMassIntegrator::AssembleDiagonalPA(Vector &diag)
 {
-   if (DeviceCanUseCeed()) { return ceedOp->GetDiagonal(diag); }
-
-   MFEM_VERIFY(coeff_vdim == 1, "coeff_vdim != 1");
-   MFEM_VERIFY(!VQ && !MQ, "VQ and MQ not supported");
-
-   // Add the VectorMassAssembleDiagonalPA specializations
-   static const auto vector_mass_assemble_diagonal_kernel_specializations =
-      ( // 2D
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 2>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 3>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 4>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 5>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 6>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 7>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 8>::Add(),
-         // 3D
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 2>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 3>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 4>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 5>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 6>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 7>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 8>::Add(),
-         true);
-   MFEM_CONTRACT_VAR(vector_mass_assemble_diagonal_kernel_specializations);
-
-   VectorMassAssembleDiagonalPA::Run(dim, quad1D, // templated arguments
-                                     ne, dofs1D, quad1D,
-                                     maps->B.Read(),
-                                     pa_data.Read(),
-                                     diag.ReadWrite());
-
+   if (DeviceCanUseCeed()) { ceedOp->GetDiagonal(diag); }
+   else
+   {
+      MFEM_VERIFY(coeff_vdim == 1, "coeff_vdim != 1");
+      MFEM_VERIFY(!VQ && !MQ, "VQ and MQ not supported");
+      PAVectorMassAssembleDiagonal(dim, dofs1D, quad1D, ne, maps->B, pa_data, diag);
+   }
 }
 
 } // namespace mfem

fem/integ/bilininteg_vecmass_pa.hpp

@@ -176,146 +176,8 @@ void SmemPAVectorMassApply3D(const int NE,
    });
 }
 
-template <int T_Q1D = 0, int T_MDQ = 16>
-static void SmemPAVectorMassAssembleDiagonal2D(const int ne,
-                                               const int d1d,
-                                               const int q1d,
-                                               const real_t *b_r,
-                                               const real_t *d_r,
-                                               real_t *y_rw)
-{
-   constexpr int VDIM = 2;
-
-   const int D1D = d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(Q1D <= T_MDQ && D1D <= Q1D, "");
-
-   const auto B = Reshape(b_r, Q1D, D1D);
-   const auto D = Reshape(d_r, Q1D, Q1D, ne);
-   auto Y = Reshape(y_rw, D1D, D1D, VDIM, ne);
-
-   mfem::forall_2D<T_Q1D*T_Q1D>(
-      ne, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MDQ;
-
-      MFEM_SHARED real_t sm[MQ1][MQ1];
-
-      MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-         {
-            real_t u = 0.0;
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               u += B(qy, dy) * B(qy, dy) * D(qx, qy, e);
-            }
-            sm[qx][dy] = u;
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
-         {
-            real_t u = 0.0;
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               u += B(qx, dx) * B(qx, dx) * sm[qx][dy];
-            }
-            Y(dx, dy, 0, e) += u;
-            Y(dx, dy, 1, e) += u;
-         }
-      }
-   });
-}
-
-// T_MDQ <= 10 so the Q1D^3 thread block stays within the 1024/block GPU limit
-template <int T_Q1D = 0, int T_MDQ = 10>
-static void SmemPAVectorMassAssembleDiagonal3D(const int ne,
-                                               const int d1d,
-                                               const int q1d,
-                                               const real_t *b_r,
-                                               const real_t *d_r,
-                                               real_t *y_rw)
-{
-   constexpr int VDIM = 3;
-
-   const int D1D = d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(Q1D <= T_MDQ && D1D <= Q1D, "");
-
-   const auto B = Reshape(b_r, Q1D, D1D);
-   const auto D = Reshape(d_r, Q1D, Q1D, Q1D, ne);
-   auto Y = Reshape(y_rw, D1D, D1D, D1D, VDIM, ne);
-
-   mfem::forall_3D<T_Q1D*T_Q1D*T_Q1D>(
-      ne, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MDQ;
-
-      MFEM_SHARED real_t sm[2][MQ1][MQ1][MQ1];
-
-      MFEM_FOREACH_THREAD_DIRECT(dz, z, D1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-            {
-               real_t u = 0.0;
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u += B(qz, dz) * B(qz, dz) * D(qx, qy, qz, e);
-               }
-               sm[0][dz][qy][qx] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD_DIRECT(dz, z, D1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-            {
-               real_t u = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  u += B(qy, dy) * B(qy, dy) * sm[0][dz][qy][qx];
-               }
-               sm[1][dz][dy][qx] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD_DIRECT(dz, z, D1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
-            {
-               real_t u = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  u += B(qx, dx) * B(qx, dx) * sm[1][dz][dy][qx];
-               }
-               Y(dx, dy, dz, 0, e) += u;
-               Y(dx, dy, dz, 1, e) += u;
-               Y(dx, dy, dz, 2, e) += u;
-            }
-         }
-      }
-   });
-}
-
 } // namespace internal
 
-// AddMultPA kernels
 template<int DIM, int T_D1D, int T_Q1D>
 VectorMassIntegrator::VectorMassAddMultPAType
 VectorMassIntegrator::VectorMassAddMultPA::Kernel()
@@ -332,7 +194,7 @@ VectorMassIntegrator::VectorMassAddMultPA::Kernel()
 }
 
 inline VectorMassIntegrator::VectorMassAddMultPAType
-VectorMassIntegrator::VectorMassAddMultPA::Fallback(int dim, int, int)
+VectorMassIntegrator::VectorMassAddMultPA::Fallback(int dim, int d1d, int q1d)
 {
    if (dim == 2)
    {
@@ -342,37 +204,7 @@ VectorMassIntegrator::VectorMassAddMultPA::Fallback(int dim, int, int)
    {
       return internal::SmemPAVectorMassApply3D;
    }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-// DiagonalPA kernels
-template<int DIM, int T_Q1D>
-VectorMassIntegrator::VectorMassAssembleDiagonalPAType
-VectorMassIntegrator::VectorMassAssembleDiagonalPA::Kernel()
-{
-   if constexpr (DIM == 2)
-   {
-      return internal::SmemPAVectorMassAssembleDiagonal2D<T_Q1D>;
-   }
-   else if constexpr (DIM == 3)
-   {
-      return internal::SmemPAVectorMassAssembleDiagonal3D<T_Q1D>;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-inline VectorMassIntegrator::VectorMassAssembleDiagonalPAType
-VectorMassIntegrator::VectorMassAssembleDiagonalPA::Fallback(int dim, int)
-{
-   if (dim == 2)
-   {
-      return internal::SmemPAVectorMassAssembleDiagonal2D;
-   }
-   else if (dim == 3)
-   {
-      return internal::SmemPAVectorMassAssembleDiagonal3D;
-   }
-   MFEM_ABORT("Unsupported kernel");
+   else { MFEM_ABORT("Unsupported kernel"); }
 }
 
 /// \endcond DO_NOT_DOCUMENT

fem/integ/bilininteg_vectorfemass_kernels.hpp

@@ -1,113 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_BILININTEG_VECTORFEMASS_KERNELS_HPP
-#define MFEM_BILININTEG_VECTORFEMASS_KERNELS_HPP
-
-#include "../../config/config.hpp"
-#include "../bilininteg.hpp"
-
-#include "bilininteg_diffusion_kernels.hpp"
-#include "bilininteg_hcurl_kernels.hpp"
-#include "bilininteg_hdiv_kernels.hpp"
-#include "bilininteg_hcurlhdiv_kernels.hpp"
-
-namespace mfem
-{
-/// \cond DO_NOT_DOCUMENT
-namespace internal
-{
-namespace hcurlmass
-{
-constexpr int NBZ3D(int d1d, int q1d)
-{
-   if (d1d <= 1 || q1d <= 0)
-   {
-      return 1;
-   }
-   // assume q1d >= d1d
-   // z dimension is capped at 64 on nvidia and amd gpus
-   int tmp = std::min((128 + q1d * q1d * q1d - 1) / (q1d * q1d * q1d), 64);
-   int smem_req =
-      sizeof(mfem::real_t) *
-      (3 * ((d1d - 1) * d1d * d1d + 2 * q1d * q1d * q1d) * tmp +
-       q1d * (d1d - 1) + q1d * d1d);
-   // assume GPU has at least 48k shared memory
-   return std::max(std::min(tmp, (48 * 1024 + smem_req - 1) / smem_req), 1);
-}
-} // namespace hcurlmass
-} // namespace internal
-
-template <FiniteElement::DerivType TrialType, FiniteElement::DerivType TestType,
-          int DIM, int TrialD1D, int TestD1D, int Q1D>
-VectorFEMassIntegrator::ApplyKernelType
-VectorFEMassIntegrator::ApplyPAKernels::Kernel()
-{
-   constexpr bool trial_curl = (TrialType == mfem::FiniteElement::CURL);
-   constexpr bool trial_div = (TrialType == mfem::FiniteElement::DIV);
-   constexpr bool test_curl = (TestType == mfem::FiniteElement::CURL);
-   constexpr bool test_div = (TestType == mfem::FiniteElement::DIV);
-
-   if constexpr (DIM == 3)
-   {
-      if constexpr (trial_curl && test_curl)
-      {
-         if (Device::Allows(Backend::DEVICE_MASK))
-         {
-            // assume TrialD1D == TestD1D
-            return internal::SmemPAHcurlMassApply3D<
-                   TrialD1D, Q1D, internal::hcurlmass::NBZ3D(TrialD1D, Q1D)>;
-         }
-         else
-         {
-            return internal::PAHcurlMassApply3D;
-         }
-      }
-      else if constexpr (trial_div && test_div)
-      {
-         // assumes TrialD1D == TestD1D
-         return internal::SmemPAHdivMassApply3D<TrialD1D, Q1D>;
-      }
-      else if constexpr (trial_curl && test_div)
-      {
-         return internal::PAHdivHcurlMassApply3D;
-      }
-      else if constexpr (trial_div && test_curl)
-      {
-         return internal::PAHcurlHdivMassApply3D;
-      }
-   }
-   else if constexpr (DIM == 2) // 2D
-   {
-      if constexpr (trial_curl && test_curl)
-      {
-         return internal::PAHcurlMassApply2D;
-      }
-      else if constexpr (trial_div && test_div)
-      {
-         // assumes TrialD1D == TestD1D
-         return internal::SmemPAHdivMassApply2D<TrialD1D, Q1D>;
-      }
-      else if constexpr (trial_curl && test_div)
-      {
-         return internal::PAHdivHcurlMassApply2D;
-      }
-      else if constexpr (trial_div && test_curl)
-      {
-         return internal::PAHcurlHdivMassApply2D;
-      }
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-/// \endcond DO_NOT_DOCUMENT
-}
-
-#endif

fem/integ/bilininteg_vectorfemass_pa.cpp

@@ -10,123 +10,15 @@
 // CONTRIBUTING.md for details.
 
 #include "../bilininteg.hpp"
-#include "bilininteg_vectorfemass_kernels.hpp"
+#include "../gridfunc.hpp"
+#include "../qfunction.hpp"
+#include "bilininteg_diffusion_kernels.hpp"
+#include "bilininteg_hcurl_kernels.hpp"
+#include "bilininteg_hdiv_kernels.hpp"
+#include "bilininteg_hcurlhdiv_kernels.hpp"
 
 namespace mfem
 {
-/// \cond DO_NOT_DOCUMENT
-VectorFEMassIntegrator::ApplyKernelType
-VectorFEMassIntegrator::ApplyPAKernels::Fallback(
-   FiniteElement::DerivType TrialType, FiniteElement::DerivType TestType,
-   int dim, int, int, int)
-{
-   const bool trial_curl = (TrialType == mfem::FiniteElement::CURL);
-   const bool trial_div = (TrialType == mfem::FiniteElement::DIV);
-   const bool test_curl = (TestType == mfem::FiniteElement::CURL);
-   const bool test_div = (TestType == mfem::FiniteElement::DIV);
-
-   if (dim == 3)
-   {
-      if (trial_curl && test_curl)
-      {
-         return internal::PAHcurlMassApply3D;
-      }
-      else if (trial_div && test_div)
-      {
-         return internal::PAHdivMassApply3D;
-      }
-      else if (trial_curl && test_div)
-      {
-         return internal::PAHdivHcurlMassApply3D;
-      }
-      else if (trial_div && test_curl)
-      {
-         return internal::PAHcurlHdivMassApply3D;
-      }
-   }
-   else if (dim == 2) // 2D
-   {
-      if (trial_curl && test_curl)
-      {
-         return internal::PAHcurlMassApply2D;
-      }
-      else if (trial_div && test_div)
-      {
-         return internal::PAHdivMassApply2D;
-      }
-      else if (trial_curl && test_div)
-      {
-         return internal::PAHdivHcurlMassApply2D;
-      }
-      else if (trial_div && test_curl)
-      {
-         return internal::PAHcurlHdivMassApply2D;
-      }
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-/// \endcond DO_NOT_DOCUMENT
-
-VectorFEMassIntegrator::Kernels::Kernels()
-{
-   // h(curl), h(curl)
-   // Q = P + 1 (3D)
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 2, 2, 3>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 3, 3, 4>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 4, 4, 5>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 5, 5, 6>();
-   // Q = P + 2 (3D)
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 2, 2, 4>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 3, 3, 5>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 4, 4, 6>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 5, 5, 7>();
-   // Q = P + 4 (3D)
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 2, 2, 6>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 3, 3, 7>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 4, 4, 8>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 5, 5, 9>();
-   // h(div), h(div)
-   // Q = P (2D)
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 2, 2, 2, 2>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 2, 3, 3, 3>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 2, 4, 4, 4>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 2, 5, 5, 5>();
-
-   // Q = P + 1 (3D)
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 3, 2, 2, 3>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 3, 3, 3, 4>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 3, 4, 4, 5>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 3, 5, 5, 6>();
-}
-
-void VectorFEMassIntegrator::Init(Coefficient *q, DiagonalMatrixCoefficient *dq,
-                                  MatrixCoefficient *mq)
-{
-   static Kernels kernels{};
-   Q = q;
-   DQ = dq;
-   MQ = mq;
-}
 
 void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
@@ -175,8 +67,8 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
 
    MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");
 
-   trial_fetype = static_cast<FiniteElement::DerivType>(trial_el->GetDerivType());
-   test_fetype = static_cast<FiniteElement::DerivType>(test_el->GetDerivType());
+   trial_fetype = trial_el->GetDerivType();
+   test_fetype = test_el->GetDerivType();
 
    const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
    const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
@@ -323,34 +215,225 @@ void VectorFEMassIntegrator::AssembleDiagonalPA(Vector& diag)
 
 void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
-   const bool scalar_coeff = !(DQ || MQ);
-   ApplyPAKernels::Run(trial_fetype, test_fetype, dim, dofs1D, dofs1Dtest,
-                       quad1D, ne, symmetric, scalar_coeff, mapsO->B, mapsC->B,
-                       mapsOtest->Bt, mapsCtest->Bt, pa_data, x, y, dofs1D,
-                       dofs1Dtest, quad1D);
+   const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
+   const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
+   const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
+   const bool test_div = (test_fetype == mfem::FiniteElement::DIV);
+
+   if (dim == 3)
+   {
+      if (trial_curl && test_curl)
+      {
+         if (Device::Allows(Backend::DEVICE_MASK))
+         {
+            const int ID = (dofs1D << 4) | quad1D;
+            switch (ID)
+            {
+               case 0x23:
+                  return internal::SmemPAHcurlMassApply3D<2,3>(
+                            dofs1D, quad1D, ne, symmetric,
+                            mapsO->B, mapsC->B, mapsO->Bt,
+                            mapsC->Bt, pa_data, x, y);
+               case 0x34:
+                  return internal::SmemPAHcurlMassApply3D<3,4>(
+                            dofs1D, quad1D, ne, symmetric,
+                            mapsO->B, mapsC->B, mapsO->Bt,
+                            mapsC->Bt, pa_data, x, y);
+               case 0x45:
+                  return internal::SmemPAHcurlMassApply3D<4,5>(
+                            dofs1D, quad1D, ne, symmetric,
+                            mapsO->B, mapsC->B, mapsO->Bt,
+                            mapsC->Bt, pa_data, x, y);
+               case 0x56:
+                  return internal::SmemPAHcurlMassApply3D<5,6>(
+                            dofs1D, quad1D, ne, symmetric,
+                            mapsO->B, mapsC->B, mapsO->Bt,
+                            mapsC->Bt, pa_data, x, y);
+               default:
+                  return internal::SmemPAHcurlMassApply3D(
+                            dofs1D, quad1D, ne, symmetric,
+                            mapsO->B, mapsC->B, mapsO->Bt,
+                            mapsC->Bt, pa_data, x, y);
+            }
+         }
+         else
+         {
+            internal::PAHcurlMassApply3D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
+                                         mapsO->Bt, mapsC->Bt, pa_data, x, y);
+         }
+      }
+      else if (trial_div && test_div)
+      {
+         internal::PAHdivMassApply(3, dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
+                                   mapsO->Bt, mapsC->Bt, pa_data, x, y);
+      }
+      else if (trial_curl && test_div)
+      {
+         const bool scalarCoeff = !(DQ || MQ);
+         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
+                                          true, false, mapsO->B, mapsC->B, mapsOtest->Bt,
+                                          mapsCtest->Bt, pa_data, x, y);
+      }
+      else if (trial_div && test_curl)
+      {
+         const bool scalarCoeff = !(DQ || MQ);
+         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
+                                          false, false, mapsO->B, mapsC->B, mapsOtest->Bt,
+                                          mapsCtest->Bt, pa_data, x, y);
+      }
+      else
+      {
+         MFEM_ABORT("Unknown kernel.");
+      }
+   }
+   else // 2D
+   {
+      if (trial_curl && test_curl)
+      {
+         internal::PAHcurlMassApply2D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
+                                      mapsO->Bt, mapsC->Bt, pa_data, x, y);
+      }
+      else if (trial_div && test_div)
+      {
+         internal::PAHdivMassApply(2, dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
+                                   mapsO->Bt,
+                                   mapsC->Bt, pa_data, x, y);
+      }
+      else if ((trial_curl && test_div) || (trial_div && test_curl))
+      {
+         const bool scalarCoeff = !(DQ || MQ);
+         internal::PAHcurlHdivMassApply2D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
+                                          trial_curl, false, mapsO->B, mapsC->B,
+                                          mapsOtest->Bt, mapsCtest->Bt, pa_data, x, y);
+      }
+      else
+      {
+         MFEM_ABORT("Unknown kernel.");
+      }
+   }
 }
 
 void VectorFEMassIntegrator::AddAbsMultPA(const Vector &x, Vector &y) const
 {
-   const bool scalar_coeff = !(DQ || MQ);
+   const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
+   const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
+   const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
+   const bool test_div = (test_fetype == mfem::FiniteElement::DIV);
 
    Vector abs_pa_data(pa_data);
    abs_pa_data.Abs();
 
    Array<real_t> absBo(mapsO->B);
    Array<real_t> absBc(mapsC->B);
+   Array<real_t> absBto(mapsO->Bt);
+   Array<real_t> absBtc(mapsC->Bt);
    Array<real_t> absBto_t(mapsOtest->Bt);
    Array<real_t> absBtc_t(mapsCtest->Bt);
 
    absBo.Abs();
    absBc.Abs();
+   absBto.Abs();
+   absBtc.Abs();
    absBto_t.Abs();
    absBtc_t.Abs();
 
-   ApplyPAKernels::Run(trial_fetype, test_fetype, dim, dofs1D, dofs1Dtest,
-                       quad1D, ne, symmetric, scalar_coeff, absBo, absBc,
-                       absBto_t, absBtc_t, abs_pa_data, x, y, dofs1D,
-                       dofs1Dtest, quad1D);
+   if (dim == 3)
+   {
+      if (trial_curl && test_curl)
+      {
+         if (Device::Allows(Backend::DEVICE_MASK))
+         {
+            const int ID = (dofs1D << 4) | quad1D;
+            switch (ID)
+            {
+               case 0x23:
+                  return internal::SmemPAHcurlMassApply3D<2,3>(
+                            dofs1D, quad1D, ne, symmetric,
+                            absBo, absBc, absBto, absBtc,
+                            abs_pa_data, x, y);
+               case 0x34:
+                  return internal::SmemPAHcurlMassApply3D<3,4>(
+                            dofs1D, quad1D, ne, symmetric,
+                            absBo, absBc, absBto, absBtc,
+                            abs_pa_data, x, y);
+               case 0x45:
+                  return internal::SmemPAHcurlMassApply3D<4,5>(
+                            dofs1D, quad1D, ne, symmetric,
+                            absBo, absBc, absBto, absBtc,
+                            abs_pa_data, x, y);
+               case 0x56:
+                  return internal::SmemPAHcurlMassApply3D<5,6>(
+                            dofs1D, quad1D, ne, symmetric,
+                            absBo, absBc, absBto, absBtc,
+                            abs_pa_data, x, y);
+               default:
+                  return internal::SmemPAHcurlMassApply3D(
+                            dofs1D, quad1D, ne, symmetric,
+                            absBo, absBc, absBto, absBtc,
+                            abs_pa_data, x, y);
+            }
+         }
+         else
+         {
+            internal::PAHcurlMassApply3D(dofs1D, quad1D, ne, symmetric,
+                                         absBo, absBc, absBto, absBtc,
+                                         abs_pa_data, x, y);
+         }
+      }
+      else if (trial_div && test_div)
+      {
+         internal::PAHdivMassApply(3, dofs1D, quad1D, ne, symmetric,
+                                   absBo, absBc, absBto, absBtc,
+                                   abs_pa_data, x, y);
+      }
+      else if (trial_curl && test_div)
+      {
+         const bool scalarCoeff = !(DQ || MQ);
+         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne,
+                                          scalarCoeff, true, false,
+                                          absBo, absBc, absBto_t, absBtc_t,
+                                          abs_pa_data, x, y);
+      }
+      else if (trial_div && test_curl)
+      {
+         const bool scalarCoeff = !(DQ || MQ);
+         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne,
+                                          scalarCoeff, false, false,
+                                          absBo, absBc, absBto_t, absBtc_t,
+                                          abs_pa_data, x, y);
+      }
+      else
+      {
+         MFEM_ABORT("Unknown kernel.");
+      }
+   }
+   else // 2D
+   {
+      if (trial_curl && test_curl)
+      {
+         internal::PAHcurlMassApply2D(dofs1D, quad1D, ne, symmetric,
+                                      absBo, absBc, absBto, absBtc,
+                                      abs_pa_data, x, y);
+      }
+      else if (trial_div && test_div)
+      {
+         internal::PAHdivMassApply(2, dofs1D, quad1D, ne, symmetric,
+                                   absBo, absBc, absBto, absBtc,
+                                   abs_pa_data, x, y);
+      }
+      else if ((trial_curl && test_div) || (trial_div && test_curl))
+      {
+         const bool scalarCoeff = !(DQ || MQ);
+         internal::PAHcurlHdivMassApply2D(dofs1D, dofs1Dtest, quad1D, ne,
+                                          scalarCoeff, trial_curl, false,
+                                          absBo, absBc, absBto_t, absBtc_t,
+                                          abs_pa_data, x, y);
+      }
+      else
+      {
+         MFEM_ABORT("Unknown kernel.");
+      }
+   }
 }
 
 void VectorFEMassIntegrator::AddMultTransposePA(const Vector &x,

fem/integ/nonlininteg_vecconvection_pa.cpp

@@ -9,51 +9,21 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
+#include "../../general/forall.hpp"
+#include "../nonlininteg.hpp"
 #include "../ceed/integrators/nlconvection/nlconvection.hpp"
-#include "./nonlininteg_vecconvection_pa.hpp" // IWYU pragma: keep
-#include "./nonlininteg_vecconvection_pa_grad.hpp" // IWYU pragma: keep
-#include "./nonlininteg_vecconvection_pa_diag.hpp" // IWYU pragma: keep
 
 namespace mfem
 {
 
-VectorConvectionNLFIntegrator::Kernels::Kernels()
-{
-   // 2D
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 2, 2>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 2, 3>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 3, 4>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 3, 5>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 4, 5>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 4, 6>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 5, 7>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 5, 8>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 6, 8>();
-   // 3D
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 2, 3>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 2, 4>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 2, 5>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 3, 4>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 3, 5>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 3, 6>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 4, 5>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 4, 6>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 4, 7>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 4, 8>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 5, 6>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 5, 7>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 5, 8>();
-}
-
 void VectorConvectionNLFIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
    MFEM_ASSERT(fes.GetOrdering() == Ordering::byNODES,
                "PA Only supports Ordering::byNODES!");
    Mesh *mesh = fes.GetMesh();
    const FiniteElement &el = *fes.GetTypicalFE();
-   ElementTransformation &Tr = *mesh->GetTypicalElementTransformation();
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, Tr);
-
+   ElementTransformation &T = *mesh->GetTypicalElementTransformation();
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, T);
    if (DeviceCanUseCeed())
    {
       delete ceedOp;
@@ -69,124 +39,769 @@ void VectorConvectionNLFIntegrator::AssemblePA(const FiniteElementSpace &fes)
       }
       return;
    }
-
-   ne = mesh->GetNE();
-   nq = ir->GetNPoints();
    dim = mesh->Dimension();
-   MFEM_VERIFY(dim == 2 || dim == 3, "Dimension not supported");
-
-   const MemoryType mt = pa_mt == MemoryType::DEFAULT
-                         ? Device::GetDeviceMemoryType()
-                         : pa_mt;
-   pa_adj.SetSize(ne * nq * dim * dim, mt);
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mt);
+   ne = fes.GetMesh()->GetNE();
+   nq = ir->GetNPoints();
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
    maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
-   d1d = maps->ndof;
-   q1d = maps->nqpt;
-
-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
-
-   const int nq1d = q1d * q1d * (dim==3 ? q1d : 1);
-   MFEM_VERIFY(coeff.Size() == 1 || coeff.Size() == nq1d*ne, "Invalid coeff");
-   MFEM_VERIFY(ir->GetWeights().Size() == nq1d, "Invalid weights size");
-
-   const auto w_r = ir->GetWeights().Read();
-   const bool const_coeff = coeff.Size() == 1;
-
+   pa_data.SetSize(ne * nq * dim * dim, Device::GetMemoryType());
+   real_t COEFF = 1.0;
+   if (Q)
+   {
+      ConstantCoefficient *cQ = dynamic_cast<ConstantCoefficient *>(Q);
+      MFEM_VERIFY(cQ != NULL, "only ConstantCoefficient is supported!");
+      COEFF = cQ->constant;
+   }
+   const int NE = ne;
+   const int NQ = nq;
+   auto W = ir->GetWeights().Read();
+   if (dim == 1)
+   {
+      MFEM_ABORT("dim==1 not supported!");
+   }
    if (dim == 2)
    {
-      const int Q1D = q1d;
-      constexpr int VDIM = 2, DIM = 2;
-      const auto W = Reshape(w_r, Q1D, Q1D);
-      const auto C = const_coeff ?
-                     Reshape(coeff.Read(), 1, 1, 1) :
-                     Reshape(coeff.Read(), Q1D, Q1D, ne);
-      const auto J = Reshape(geom->J.Read(), Q1D, Q1D, VDIM, DIM, ne);
-      auto A = Reshape(pa_adj.Write(), VDIM, DIM, Q1D, Q1D, ne);
-
-      mfem::forall_2D(ne, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+      auto J = Reshape(geom->J.Read(), NQ, 2, 2, NE);
+      auto G = Reshape(pa_data.Write(), NQ, 2, 2, NE);
+      mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
       {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+         for (int q = 0; q < NQ; ++q)
          {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-            {
-               const real_t J11 = J(qx, qy, 0, 0, e), J12 = J(qx, qy, 0, 1, e);
-               const real_t J21 = J(qx, qy, 1, 0, e), J22 = J(qx, qy, 1, 1, e);
-               // adj(J)
-               const real_t A11 = +J22, A12 = -J12;
-               const real_t A21 = -J21, A22 = +J11;
-               // Store w * coeff * adj(J)
-               const real_t w = W(qx, qy);
-               const real_t c = const_coeff ? C(0, 0, 0) : C(qx, qy, e);
-               A(0, 0, qx, qy, e) = w * c * A11;
-               A(1, 0, qx, qy, e) = w * c * A12;
-               A(0, 1, qx, qy, e) = w * c * A21;
-               A(1, 1, qx, qy, e) = w * c * A22;
-            }
+            const real_t J11 = J(q, 0, 0, e);
+            const real_t J12 = J(q, 0, 1, e);
+            const real_t J21 = J(q, 1, 0, e);
+            const real_t J22 = J(q, 1, 1, e);
+            // Store wq * Q * adj(J)
+            G(q, 0, 0, e) = W[q] * COEFF * J22;  // 1,1
+            G(q, 0, 1, e) = W[q] * COEFF * -J12; // 1,2
+            G(q, 1, 0, e) = W[q] * COEFF * -J21; // 2,1
+            G(q, 1, 1, e) = W[q] * COEFF * J11;  // 2,2
          }
       });
    }
-   else if (dim == 3)
+   if (dim == 3)
    {
-      const int Q1D = q1d;
-      constexpr int VDIM = 3, DIM = 3;
-      const auto W = Reshape(w_r, Q1D, Q1D, Q1D);
-      const auto C = const_coeff ?
-                     Reshape(coeff.Read(), 1, 1, 1, 1) :
-                     Reshape(coeff.Read(), Q1D, Q1D, Q1D, ne);
-      const auto J = Reshape(geom->J.Read(), Q1D, Q1D, Q1D, VDIM, DIM, ne);
-      auto A = Reshape(pa_adj.Write(), VDIM, DIM, Q1D, Q1D, Q1D, ne);
-
-      mfem::forall_3D(ne, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+      auto J = Reshape(geom->J.Read(), NQ, 3, 3, NE);
+      auto G = Reshape(pa_data.Write(), NQ, 3, 3, NE);
+      mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
       {
-         MFEM_FOREACH_THREAD_DIRECT(qz, z, Q1D)
+         for (int q = 0; q < NQ; ++q)
          {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+            const real_t J11 = J(q, 0, 0, e);
+            const real_t J21 = J(q, 1, 0, e);
+            const real_t J31 = J(q, 2, 0, e);
+            const real_t J12 = J(q, 0, 1, e);
+            const real_t J22 = J(q, 1, 1, e);
+            const real_t J32 = J(q, 2, 1, e);
+            const real_t J13 = J(q, 0, 2, e);
+            const real_t J23 = J(q, 1, 2, e);
+            const real_t J33 = J(q, 2, 2, e);
+            const real_t cw = W[q] * COEFF;
+            // adj(J)
+            const real_t A11 = (J22 * J33) - (J23 * J32);
+            const real_t A12 = (J32 * J13) - (J12 * J33);
+            const real_t A13 = (J12 * J23) - (J22 * J13);
+            const real_t A21 = (J31 * J23) - (J21 * J33);
+            const real_t A22 = (J11 * J33) - (J13 * J31);
+            const real_t A23 = (J21 * J13) - (J11 * J23);
+            const real_t A31 = (J21 * J32) - (J31 * J22);
+            const real_t A32 = (J31 * J12) - (J11 * J32);
+            const real_t A33 = (J11 * J22) - (J12 * J21);
+            // Store wq * Q * adj(J)
+            G(q, 0, 0, e) = cw * A11; // 1,1
+            G(q, 0, 1, e) = cw * A12; // 1,2
+            G(q, 0, 2, e) = cw * A13; // 1,3
+            G(q, 1, 0, e) = cw * A21; // 2,1
+            G(q, 1, 1, e) = cw * A22; // 2,2
+            G(q, 1, 2, e) = cw * A23; // 2,3
+            G(q, 2, 0, e) = cw * A31; // 3,1
+            G(q, 2, 1, e) = cw * A32; // 3,2
+            G(q, 2, 2, e) = cw * A33; // 3,3
+         }
+      });
+   }
+}
+
+// PA Convection NL 2D kernel
+template<int T_D1D = 0, int T_Q1D = 0>
+static void PAConvectionNLApply2D(const int NE,
+                                  const Array<real_t> &b,
+                                  const Array<real_t> &g,
+                                  const Array<real_t> &bt,
+                                  const Vector &q_,
+                                  const Vector &x_,
+                                  Vector &y_,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto G = Reshape(g.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt.Read(), D1D, Q1D);
+   auto Q = Reshape(q_.Read(), Q1D * Q1D, 2, 2, NE);
+   auto x = Reshape(x_.Read(), D1D, D1D, 2, NE);
+   auto y = Reshape(y_.ReadWrite(), D1D, D1D, 2, NE);
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      real_t data[max_Q1D][max_Q1D][2];
+      real_t grad0[max_Q1D][max_Q1D][2];
+      real_t grad1[max_Q1D][max_Q1D][2];
+      real_t Z[max_Q1D][max_Q1D][2];
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            data[qy][qx][0] = 0.0;
+            data[qy][qx][1] = 0.0;
+            grad0[qy][qx][0] = 0.0;
+            grad0[qy][qx][1] = 0.0;
+            grad1[qy][qx][0] = 0.0;
+            grad1[qy][qx][1] = 0.0;
+         }
+      }
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         real_t dataX[max_Q1D][2];
+         real_t gradX0[max_Q1D][2];
+         real_t gradX1[max_Q1D][2];
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            dataX[qx][0] = 0.0;
+            dataX[qx][1] = 0.0;
+            gradX0[qx][0] = 0.0;
+            gradX0[qx][1] = 0.0;
+            gradX1[qx][0] = 0.0;
+            gradX1[qx][1] = 0.0;
+         }
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            const real_t s0 = x(dx, dy, 0, e);
+            const real_t s1 = x(dx, dy, 1, e);
+            for (int qx = 0; qx < Q1D; ++qx)
             {
-               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+               const real_t Bx = B(qx, dx);
+               const real_t Gx = G(qx, dx);
+               dataX[qx][0] += s0 * Bx;
+               dataX[qx][1] += s1 * Bx;
+               gradX0[qx][0] += s0 * Gx;
+               gradX0[qx][1] += s0 * Bx;
+               gradX1[qx][0] += s1 * Gx;
+               gradX1[qx][1] += s1 * Bx;
+            }
+         }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            const real_t By = B(qy, dy);
+            const real_t Gy = G(qy, dy);
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               data[qy][qx][0] += dataX[qx][0] * By;
+               data[qy][qx][1] += dataX[qx][1] * By;
+               grad0[qy][qx][0] += gradX0[qx][0] * By;
+               grad0[qy][qx][1] += gradX0[qx][1] * Gy;
+               grad1[qy][qx][0] += gradX1[qx][0] * By;
+               grad1[qy][qx][1] += gradX1[qx][1] * Gy;
+            }
+         }
+      }
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            const int q = qx + qy * Q1D;
+            const real_t u1 = data[qy][qx][0];
+            const real_t u2 = data[qy][qx][1];
+            const real_t grad00 = grad0[qy][qx][0];
+            const real_t grad01 = grad0[qy][qx][1];
+            const real_t grad10 = grad1[qy][qx][0];
+            const real_t grad11 = grad1[qy][qx][1];
+            const real_t Dxu1 = grad00 * Q(q, 0, 0, e) + grad01 * Q(q, 1, 0, e);
+            const real_t Dyu1 = grad00 * Q(q, 0, 1, e) + grad01 * Q(q, 1, 1, e);
+            const real_t Dxu2 = grad10 * Q(q, 0, 0, e) + grad11 * Q(q, 1, 0, e);
+            const real_t Dyu2 = grad10 * Q(q, 0, 1, e) + grad11 * Q(q, 1, 1, e);
+            Z[qy][qx][0] = u1 * Dxu1 + u2 * Dyu1;
+            Z[qy][qx][1] = u1 * Dxu2 + u2 * Dyu2;
+         }
+      }
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         real_t Y[max_D1D][2];
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            Y[dx][0] = 0.0;
+            Y[dx][1] = 0.0;
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const real_t Btx = Bt(dx, qx);
+               Y[dx][0] += Btx * Z[qy][qx][0];
+               Y[dx][1] += Btx * Z[qy][qx][1];
+            }
+         }
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const real_t Bty = Bt(dy, qy);
+               y(dx, dy, 0, e) += Bty * Y[dx][0];
+               y(dx, dy, 1, e) += Bty * Y[dx][1];
+            }
+         }
+      }
+   });
+}
+
+// PA Convection NL 3D kernel
+template<int T_D1D = 0, int T_Q1D = 0>
+static void PAConvectionNLApply3D(const int NE,
+                                  const Array<real_t> &b,
+                                  const Array<real_t> &g,
+                                  const Array<real_t> &bt,
+                                  const Vector &q_,
+                                  const Vector &x_,
+                                  Vector &y_,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
+{
+   constexpr int VDIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto G = Reshape(g.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt.Read(), D1D, Q1D);
+   auto Q = Reshape(q_.Read(), Q1D * Q1D * Q1D, VDIM, VDIM, NE);
+   auto x = Reshape(x_.Read(), D1D, D1D, D1D, VDIM, NE);
+   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int VDIM = 3;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      real_t data[max_Q1D][max_Q1D][max_Q1D][VDIM];
+      real_t grad0[max_Q1D][max_Q1D][max_Q1D][VDIM];
+      real_t grad1[max_Q1D][max_Q1D][max_Q1D][VDIM];
+      real_t grad2[max_Q1D][max_Q1D][max_Q1D][VDIM];
+      real_t Z[max_Q1D][max_Q1D][max_Q1D][VDIM];
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               data[qz][qy][qx][0] = 0.0;
+               data[qz][qy][qx][1] = 0.0;
+               data[qz][qy][qx][2] = 0.0;
+
+               grad0[qz][qy][qx][0] = 0.0;
+               grad0[qz][qy][qx][1] = 0.0;
+               grad0[qz][qy][qx][2] = 0.0;
+
+               grad1[qz][qy][qx][0] = 0.0;
+               grad1[qz][qy][qx][1] = 0.0;
+               grad1[qz][qy][qx][2] = 0.0;
+
+               grad2[qz][qy][qx][0] = 0.0;
+               grad2[qz][qy][qx][1] = 0.0;
+               grad2[qz][qy][qx][2] = 0.0;
+            }
+         }
+      }
+      for (int dz = 0; dz < D1D; ++dz)
+      {
+         real_t dataXY[max_Q1D][max_Q1D][VDIM];
+         real_t gradXY0[max_Q1D][max_Q1D][VDIM];
+         real_t gradXY1[max_Q1D][max_Q1D][VDIM];
+         real_t gradXY2[max_Q1D][max_Q1D][VDIM];
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               dataXY[qy][qx][0] = 0.0;
+               dataXY[qy][qx][1] = 0.0;
+               dataXY[qy][qx][2] = 0.0;
+
+               gradXY0[qy][qx][0] = 0.0;
+               gradXY0[qy][qx][1] = 0.0;
+               gradXY0[qy][qx][2] = 0.0;
+
+               gradXY1[qy][qx][0] = 0.0;
+               gradXY1[qy][qx][1] = 0.0;
+               gradXY1[qy][qx][2] = 0.0;
+
+               gradXY2[qy][qx][0] = 0.0;
+               gradXY2[qy][qx][1] = 0.0;
+               gradXY2[qy][qx][2] = 0.0;
+            }
+         }
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            real_t dataX[max_Q1D][VDIM];
+            real_t gradX0[max_Q1D][VDIM];
+            real_t gradX1[max_Q1D][VDIM];
+            real_t gradX2[max_Q1D][VDIM];
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               dataX[qx][0] = 0.0;
+               dataX[qx][1] = 0.0;
+               dataX[qx][2] = 0.0;
+
+               gradX0[qx][0] = 0.0;
+               gradX0[qx][1] = 0.0;
+               gradX0[qx][2] = 0.0;
+
+               gradX1[qx][0] = 0.0;
+               gradX1[qx][1] = 0.0;
+               gradX1[qx][2] = 0.0;
+
+               gradX2[qx][0] = 0.0;
+               gradX2[qx][1] = 0.0;
+               gradX2[qx][2] = 0.0;
+            }
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const real_t s0 = x(dx, dy, dz, 0, e);
+               const real_t s1 = x(dx, dy, dz, 1, e);
+               const real_t s2 = x(dx, dy, dz, 2, e);
+               for (int qx = 0; qx < Q1D; ++qx)
                {
-                  const real_t J11 = J(qx, qy, qz, 0, 0, e),
-                               J12 = J(qx, qy, qz, 0, 1, e),
-                               J13 = J(qx, qy, qz, 0, 2, e);
-                  const real_t J21 = J(qx, qy, qz, 1, 0, e),
-                               J22 = J(qx, qy, qz, 1, 1, e),
-                               J23 = J(qx, qy, qz, 1, 2, e);
-                  const real_t J31 = J(qx, qy, qz, 2, 0, e),
-                               J32 = J(qx, qy, qz, 2, 1, e),
-                               J33 = J(qx, qy, qz, 2, 2, e);
-                  const real_t c =
-                     const_coeff ? C(0, 0, 0, 0) : C(qx, qy, qz, e);
-                  const real_t cw = W(qx, qy, qz) * c;
-                  // adj(J)
-                  const real_t A11 = (J22 * J33) - (J23 * J32);
-                  const real_t A12 = (J32 * J13) - (J12 * J33);
-                  const real_t A13 = (J12 * J23) - (J22 * J13);
-                  const real_t A21 = (J31 * J23) - (J21 * J33);
-                  const real_t A22 = (J11 * J33) - (J13 * J31);
-                  const real_t A23 = (J21 * J13) - (J11 * J23);
-                  const real_t A31 = (J21 * J32) - (J31 * J22);
-                  const real_t A32 = (J31 * J12) - (J11 * J32);
-                  const real_t A33 = (J11 * J22) - (J12 * J21);
-                  // Store wq * coeff * adj(J)
-                  A(0, 0, qx, qy, qz, e) = cw * A11;
-                  A(1, 0, qx, qy, qz, e) = cw * A12;
-                  A(2, 0, qx, qy, qz, e) = cw * A13;
-                  A(0, 1, qx, qy, qz, e) = cw * A21;
-                  A(1, 1, qx, qy, qz, e) = cw * A22;
-                  A(2, 1, qx, qy, qz, e) = cw * A23;
-                  A(0, 2, qx, qy, qz, e) = cw * A31;
-                  A(1, 2, qx, qy, qz, e) = cw * A32;
-                  A(2, 2, qx, qy, qz, e) = cw * A33;
+                  const real_t Bx = B(qx, dx);
+                  const real_t Gx = G(qx, dx);
+
+                  dataX[qx][0] += s0 * Bx;
+                  dataX[qx][1] += s1 * Bx;
+                  dataX[qx][2] += s2 * Bx;
+
+                  gradX0[qx][0] += s0 * Gx;
+                  gradX0[qx][1] += s0 * Bx;
+                  gradX0[qx][2] += s0 * Bx;
+
+                  gradX1[qx][0] += s1 * Gx;
+                  gradX1[qx][1] += s1 * Bx;
+                  gradX1[qx][2] += s1 * Bx;
+
+                  gradX2[qx][0] += s2 * Gx;
+                  gradX2[qx][1] += s2 * Bx;
+                  gradX2[qx][2] += s2 * Bx;
+               }
+            }
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const real_t By = B(qy, dy);
+               const real_t Gy = G(qy, dy);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  dataXY[qy][qx][0] += dataX[qx][0] * By;
+                  dataXY[qy][qx][1] += dataX[qx][1] * By;
+                  dataXY[qy][qx][2] += dataX[qx][2] * By;
+
+                  gradXY0[qy][qx][0] += gradX0[qx][0] * By;
+                  gradXY0[qy][qx][1] += gradX0[qx][1] * Gy;
+                  gradXY0[qy][qx][2] += gradX0[qx][2] * By;
+
+                  gradXY1[qy][qx][0] += gradX1[qx][0] * By;
+                  gradXY1[qy][qx][1] += gradX1[qx][1] * Gy;
+                  gradXY1[qy][qx][2] += gradX1[qx][2] * By;
+
+                  gradXY2[qy][qx][0] += gradX2[qx][0] * By;
+                  gradXY2[qy][qx][1] += gradX2[qx][1] * Gy;
+                  gradXY2[qy][qx][2] += gradX2[qx][2] * By;
                }
             }
          }
-      });
-   }
-   else
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            const real_t Bz = B(qz, dz);
+            const real_t Gz = G(qz, dz);
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  data[qz][qy][qx][0] += dataXY[qy][qx][0] * Bz;
+                  data[qz][qy][qx][1] += dataXY[qy][qx][1] * Bz;
+                  data[qz][qy][qx][2] += dataXY[qy][qx][2] * Bz;
+
+                  grad0[qz][qy][qx][0] += gradXY0[qy][qx][0] * Bz;
+                  grad0[qz][qy][qx][1] += gradXY0[qy][qx][1] * Bz;
+                  grad0[qz][qy][qx][2] += gradXY0[qy][qx][2] * Gz;
+
+                  grad1[qz][qy][qx][0] += gradXY1[qy][qx][0] * Bz;
+                  grad1[qz][qy][qx][1] += gradXY1[qy][qx][1] * Bz;
+                  grad1[qz][qy][qx][2] += gradXY1[qy][qx][2] * Gz;
+
+                  grad2[qz][qy][qx][0] += gradXY2[qy][qx][0] * Bz;
+                  grad2[qz][qy][qx][1] += gradXY2[qy][qx][1] * Bz;
+                  grad2[qz][qy][qx][2] += gradXY2[qy][qx][2] * Gz;
+               }
+            }
+         }
+      }
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const int q = qx + Q1D * (qy + qz * Q1D);
+
+               const real_t u1 = data[qz][qy][qx][0];
+               const real_t u2 = data[qz][qy][qx][1];
+               const real_t u3 = data[qz][qy][qx][2];
+
+               const real_t grad00 = grad0[qz][qy][qx][0];
+               const real_t grad01 = grad0[qz][qy][qx][1];
+               const real_t grad02 = grad0[qz][qy][qx][2];
+
+               const real_t grad10 = grad1[qz][qy][qx][0];
+               const real_t grad11 = grad1[qz][qy][qx][1];
+               const real_t grad12 = grad1[qz][qy][qx][2];
+
+               const real_t grad20 = grad2[qz][qy][qx][0];
+               const real_t grad21 = grad2[qz][qy][qx][1];
+               const real_t grad22 = grad2[qz][qy][qx][2];
+
+               const real_t Dxu1 = grad00 * Q(q, 0, 0, e)
+                                   + grad01 * Q(q, 1, 0, e)
+                                   + grad02 * Q(q, 2, 0, e);
+               const real_t Dyu1 = grad00 * Q(q, 0, 1, e)
+                                   + grad01 * Q(q, 1, 1, e)
+                                   + grad02 * Q(q, 2, 1, e);
+               const real_t Dzu1 = grad00 * Q(q, 0, 2, e)
+                                   + grad01 * Q(q, 1, 2, e)
+                                   + grad02 * Q(q, 2, 2, e);
+
+               const real_t Dxu2 = grad10 * Q(q, 0, 0, e)
+                                   + grad11 * Q(q, 1, 0, e)
+                                   + grad12 * Q(q, 2, 0, e);
+               const real_t Dyu2 = grad10 * Q(q, 0, 1, e)
+                                   + grad11 * Q(q, 1, 1, e)
+                                   + grad12 * Q(q, 2, 1, e);
+               const real_t Dzu2 = grad10 * Q(q, 0, 2, e)
+                                   + grad11 * Q(q, 1, 2, e)
+                                   + grad12 * Q(q, 2, 2, e);
+
+               const real_t Dxu3 = grad20 * Q(q, 0, 0, e)
+                                   + grad21 * Q(q, 1, 0, e)
+                                   + grad22 * Q(q, 2, 0, e);
+               const real_t Dyu3 = grad20 * Q(q, 0, 1, e)
+                                   + grad21 * Q(q, 1, 1, e)
+                                   + grad22 * Q(q, 2, 1, e);
+               const real_t Dzu3 = grad20 * Q(q, 0, 2, e)
+                                   + grad21 * Q(q, 1, 2, e)
+                                   + grad22 * Q(q, 2, 2, e);
+
+               Z[qz][qy][qx][0] = u1 * Dxu1 + u2 * Dyu1 + u3 * Dzu1;
+               Z[qz][qy][qx][1] = u1 * Dxu2 + u2 * Dyu2 + u3 * Dzu2;
+               Z[qz][qy][qx][2] = u1 * Dxu3 + u2 * Dyu3 + u3 * Dzu3;
+            }
+         }
+      }
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         real_t opXY[max_D1D][max_D1D][VDIM];
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               opXY[dy][dx][0] = 0.0;
+               opXY[dy][dx][1] = 0.0;
+               opXY[dy][dx][2] = 0.0;
+            }
+         }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            real_t opX[max_D1D][VDIM];
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               opX[dx][0] = 0.0;
+               opX[dx][1] = 0.0;
+               opX[dx][2] = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  const real_t Btx = Bt(dx, qx);
+                  opX[dx][0] += Btx * Z[qz][qy][qx][0];
+                  opX[dx][1] += Btx * Z[qz][qy][qx][1];
+                  opX[dx][2] += Btx * Z[qz][qy][qx][2];
+               }
+            }
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  const real_t Bty = Bt(dy, qy);
+                  opXY[dy][dx][0] += Bty * opX[dx][0];
+                  opXY[dy][dx][1] += Bty * opX[dx][1];
+                  opXY[dy][dx][2] += Bty * opX[dx][2];
+               }
+            }
+         }
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  const real_t Btz = Bt(dz, qz);
+                  y(dx, dy, dz, 0, e) += Btz * opXY[dy][dx][0];
+                  y(dx, dy, dz, 1, e) += Btz * opXY[dy][dx][1];
+                  y(dx, dy, dz, 2, e) += Btz * opXY[dy][dx][2];
+               }
+            }
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0, int T_MAX_D1D = 0, int T_MAX_Q1D = 0>
+static void SmemPAConvectionNLApply3D(const int NE,
+                                      const Array<real_t> &b_,
+                                      const Array<real_t> &g_,
+                                      const Vector &d_,
+                                      const Vector &x_,
+                                      Vector &y_,
+                                      const int d1d = 0,
+                                      const int q1d = 0)
+{
+   constexpr int VDIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int MD1 = T_D1D ? T_D1D : T_MAX_D1D;
+   constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX_Q1D;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
+
+   auto b = Reshape(b_.Read(), Q1D, D1D);
+   auto g = Reshape(g_.Read(), Q1D, D1D);
+   auto D = Reshape(d_.Read(), Q1D * Q1D * Q1D, VDIM, VDIM, NE);
+   auto x = Reshape(x_.Read(), D1D, D1D, D1D, VDIM, NE);
+   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
+
+   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
    {
-      MFEM_ABORT("dim " << dim << " not supported!");
-   }
+      const int tidz = MFEM_THREAD_ID(z);
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX_Q1D;
+      MFEM_SHARED real_t BG[2][MQ1 * MD1];
+      real_t(*B)[MD1] = (real_t(*)[MD1])(BG + 0);
+      real_t(*G)[MD1] = (real_t(*)[MD1])(BG + 1);
+      real_t(*Bt)[MQ1] = (real_t(*)[MQ1])(BG + 0);
+      MFEM_SHARED real_t U[2][MQ1][MQ1][MQ1];
+      MFEM_SHARED real_t sm0[3][MQ1 * MQ1 * MQ1];
+      MFEM_SHARED real_t sm1[3][MQ1 * MQ1 * MQ1];
+      real_t(*DDQ0)[MD1][MQ1] = (real_t(*)[MD1][MQ1])(sm0 + 0);
+      real_t(*DDQ1)[MD1][MQ1] = (real_t(*)[MD1][MQ1])(sm0 + 1);
+      real_t(*X)[MD1][MD1] = (real_t(*)[MD1][MD1])(sm0 + 2);
+      real_t(*DQQ0)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm1 + 0);
+      real_t(*DQQ1)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm1 + 1);
+      real_t(*DQQ2)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm1 + 2);
+      real_t(*QQQ0)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm0 + 0);
+      real_t(*QQQ1)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm0 + 1);
+      real_t(*QQQ2)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm0 + 2);
+      real_t(*QQD0)[MQ1][MD1] = (real_t(*)[MQ1][MD1])(sm1 + 0);
+      real_t(*QDD0)[MD1][MD1] = (real_t(*)[MD1][MD1])(sm0 + 0);
+      MFEM_SHARED real_t Z[MQ1][MQ1][MQ1];
+
+      for (int cy = 0; cy < VDIM; ++cy)
+      {
+         if (tidz == 0)
+         {
+            MFEM_FOREACH_THREAD(q, x, Q1D)
+            {
+               MFEM_FOREACH_THREAD(d, y, D1D)
+               {
+                  B[q][d] = b(q, d);
+                  G[q][d] = g(q, d);
+               }
+            }
+         }
+         MFEM_FOREACH_THREAD(qz, z, Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy, y, Q1D)
+            {
+               MFEM_FOREACH_THREAD(qx, x, Q1D) { Z[qz][qy][qx] = 0.0; }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         for (int c = 0; c < VDIM; ++c)
+         {
+            MFEM_FOREACH_THREAD(dz, z, D1D)
+            {
+               MFEM_FOREACH_THREAD(dy, y, D1D)
+               {
+                  MFEM_FOREACH_THREAD(dx, x, D1D)
+                  {
+                     X[dz][dy][dx] = x(dx, dy, dz, cy, e);
+                     U[0][dz][dy][dx] = x(dx, dy, dz, c, e);
+                  }
+               }
+            }
+            MFEM_SYNC_THREAD;
+            MFEM_FOREACH_THREAD(dz, z, D1D)
+            {
+               MFEM_FOREACH_THREAD(dy, y, D1D)
+               {
+                  MFEM_FOREACH_THREAD(qx, x, Q1D)
+                  {
+                     real_t u = 0.0;
+                     real_t v = 0.0;
+                     real_t z = 0.0;
+                     for (int dx = 0; dx < D1D; ++dx)
+                     {
+                        const real_t coord = X[dz][dy][dx];
+                        const real_t value = U[0][dz][dy][dx];
+                        u += coord * B[qx][dx];
+                        v += coord * G[qx][dx];
+                        z += value * B[qx][dx];
+                     }
+                     DDQ0[dz][dy][qx] = u;
+                     DDQ1[dz][dy][qx] = v;
+                     U[1][dz][dy][qx] = z;
+                  }
+               }
+            }
+            MFEM_SYNC_THREAD;
+            MFEM_FOREACH_THREAD(dz, z, D1D)
+            {
+               MFEM_FOREACH_THREAD(qy, y, Q1D)
+               {
+                  MFEM_FOREACH_THREAD(qx, x, Q1D)
+                  {
+                     real_t u = 0.0;
+                     real_t v = 0.0;
+                     real_t w = 0.0;
+                     real_t z = 0.0;
+                     for (int dy = 0; dy < D1D; ++dy)
+                     {
+                        u += DDQ1[dz][dy][qx] * B[qy][dy];
+                        v += DDQ0[dz][dy][qx] * G[qy][dy];
+                        w += DDQ0[dz][dy][qx] * B[qy][dy];
+                        z += U[1][dz][dy][qx] * B[qy][dy];
+                     }
+                     DQQ0[dz][qy][qx] = u;
+                     DQQ1[dz][qy][qx] = v;
+                     DQQ2[dz][qy][qx] = w;
+                     U[0][dz][qy][qx] = z;
+                  }
+               }
+            }
+            MFEM_SYNC_THREAD;
+            MFEM_FOREACH_THREAD(qz, z, Q1D)
+            {
+               MFEM_FOREACH_THREAD(qy, y, Q1D)
+               {
+                  MFEM_FOREACH_THREAD(qx, x, Q1D)
+                  {
+                     real_t u = 0.0;
+                     real_t v = 0.0;
+                     real_t w = 0.0;
+                     real_t z = 0.0;
+                     for (int dz = 0; dz < D1D; ++dz)
+                     {
+                        u += DQQ0[dz][qy][qx] * B[qz][dz];
+                        v += DQQ1[dz][qy][qx] * B[qz][dz];
+                        w += DQQ2[dz][qy][qx] * G[qz][dz];
+                        z += U[0][dz][qy][qx] * B[qz][dz];
+                     }
+                     QQQ0[qz][qy][qx] = u;
+                     QQQ1[qz][qy][qx] = v;
+                     QQQ2[qz][qy][qx] = w;
+                     U[1][qz][qy][qx] = z;
+                  }
+               }
+            }
+            MFEM_SYNC_THREAD;
+            MFEM_FOREACH_THREAD(qz, z, Q1D)
+            {
+               MFEM_FOREACH_THREAD(qy, y, Q1D)
+               {
+                  MFEM_FOREACH_THREAD(qx, x, Q1D)
+                  {
+                     const int q = qx + (qy + qz * Q1D) * Q1D;
+                     const real_t z = U[1][qz][qy][qx];
+                     const real_t gX = QQQ0[qz][qy][qx];
+                     const real_t gY = QQQ1[qz][qy][qx];
+                     const real_t gZ = QQQ2[qz][qy][qx];
+                     const real_t d = gX * D(q, 0, c, e) + gY * D(q, 1, c, e)
+                                      + gZ * D(q, 2, c, e);
+                     Z[qz][qy][qx] += z * d;
+                  }
+               }
+            }
+            MFEM_SYNC_THREAD;
+         } // for each conv component
+         if (tidz == 0)
+         {
+            MFEM_FOREACH_THREAD(d, y, D1D)
+            {
+               MFEM_FOREACH_THREAD(q, x, Q1D) { Bt[d][q] = b(q, d); }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(qz, z, Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy, y, Q1D)
+            {
+               MFEM_FOREACH_THREAD(dx, x, D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     u += Z[qz][qy][qx] * Bt[dx][qx];
+                  }
+                  QQD0[qz][qy][dx] = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(qz, z, Q1D)
+         {
+            MFEM_FOREACH_THREAD(dy, y, D1D)
+            {
+               MFEM_FOREACH_THREAD(dx, x, D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qy = 0; qy < Q1D; ++qy)
+                  {
+                     u += QQD0[qz][qy][dx] * Bt[dy][qy];
+                  }
+                  QDD0[qz][dy][dx] = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(dz, z, D1D)
+         {
+            MFEM_FOREACH_THREAD(dy, y, D1D)
+            {
+               MFEM_FOREACH_THREAD(dx, x, D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     u += QDD0[qz][dy][dx] * Bt[dz][qz];
+                  }
+                  Y(dx, dy, dz, cy, e) += u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
 }
 
 void VectorConvectionNLFIntegrator::AddMultPA(const Vector &x, Vector &y) const
@@ -197,13 +812,26 @@ void VectorConvectionNLFIntegrator::AddMultPA(const Vector &x, Vector &y) const
    }
    else
    {
-      AddMultPAKernels::Run(dim, d1d, q1d, ne,
-                            maps->B.Read(),
-                            maps->G.Read(),
-                            pa_adj.Read(),
-                            x.Read(),
-                            y.ReadWrite(),
-                            d1d, q1d);
+      const int NE = ne;
+      const int D1D = maps->ndof;
+      const int Q1D = maps->nqpt;
+      const Vector &QV = pa_data;
+      const Array<real_t> &B = maps->B;
+      const Array<real_t> &G = maps->G;
+      const Array<real_t> &Bt = maps->Bt;
+      if (dim == 2)
+      {
+         return PAConvectionNLApply2D(NE, B, G, Bt, QV, x, y, D1D, Q1D);
+      }
+      if (dim == 3)
+      {
+         constexpr int T_MAX_D1D = 8;
+         constexpr int T_MAX_Q1D = 8;
+         MFEM_VERIFY(D1D <= T_MAX_D1D && Q1D <= T_MAX_Q1D, "Not yet implemented!");
+         return SmemPAConvectionNLApply3D<0, 0, T_MAX_D1D, T_MAX_Q1D>
+                (NE, B, G, QV, x, y, D1D, Q1D);
+      }
+      MFEM_ABORT("Not yet implemented!");
    }
 }
 

fem/integ/nonlininteg_vecconvection_pa.hpp

@@ -1,209 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../config/config.hpp"
-#include "../../general/forall.hpp"
-#include "../../linalg/dtensor.hpp"
-#include "../kernels.hpp"
-#include "../nonlininteg.hpp"
-
-namespace mfem
-{
-
-/// \cond DO_NOT_DOCUMENT
-
-namespace internal
-{
-
-// PA Convection NL 2D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAConvectionNLApply2D(const int NE,
-                                      const real_t *b,
-                                      const real_t *g,
-                                      const real_t *a,
-                                      const real_t *x,
-                                      real_t *y,
-                                      const int d1d = 0,
-                                      const int q1d = 0)
-{
-   static constexpr int VDIM = 2, DIM = 2;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto B = Reshape(b, Q1D, D1D);
-   const auto G = Reshape(g, Q1D, D1D);
-   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, NE);
-   const auto X = Reshape(x, D1D, D1D, VDIM, NE);
-   auto Y = Reshape(y, D1D, D1D, VDIM, NE);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1], sB[MD1][MQ1], sG[MD1][MQ1];
-
-      kernels::internal::vd_regs2d_t<VDIM, DIM, MQ1> g0, g1;
-      kernels::internal::v_regs2d_t<VDIM, MQ1> r0, r1;
-      kernels::internal::v_regs2d_t<VDIM, MQ1> s0, s1;
-
-      kernels::internal::LoadMatrix(D1D, Q1D, B, sB);
-      kernels::internal::LoadMatrix(D1D, Q1D, G, sG);
-
-      kernels::internal::LoadDofs2d(e, D1D, X, r0);
-      kernels::internal::Eval2d(D1D, Q1D, smem, sB, r0, r1); // u vector-value
-      kernels::internal::LoadDofs2d(e, D1D, X, g0);
-      kernels::internal::Grad2d(D1D, Q1D, smem, sB, sG, g0, g1); // u vector-gradient
-
-      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            const future::tensor<real_t, 2> U =
-            {
-               r1[0][qy][qx], r1[1][qy][qx]
-            };
-            const future::tensor<real_t, 2,2> gradU = {{
-                  {g1[0][0][qy][qx], g1[1][0][qy][qx]},
-                  {g1[0][1][qy][qx], g1[1][1][qy][qx]},
-               }
-            };
-            const future::tensor<real_t, 2,2> Q = {{
-                  {A(0,0,qx,qy,e), A(1,0,qx,qy,e)},
-                  {A(0,1,qx,qy,e), A(1,1,qx,qy,e)},
-               }
-            };
-            const future::tensor<real_t, 2> conv = transpose(gradU) * (Q * U);
-            s0[0][qy][qx] = conv[0];
-            s0[1][qy][qx] = conv[1];
-         }
-      }
-      MFEM_SYNC_THREAD;
-      kernels::internal::EvalTranspose2d(D1D, Q1D, smem, sB, s0, s1);
-      kernels::internal::WriteDofs2d(e, D1D, s1, Y);
-   });
-}
-
-// PA Convection NL 3D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAConvectionNLApply3D(const int NE,
-                                      const real_t *b,
-                                      const real_t *g,
-                                      const real_t *a,
-                                      const real_t *x,
-                                      real_t *y,
-                                      const int d1d = 0,
-                                      const int q1d = 0)
-{
-   static constexpr int VDIM = 3, DIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto B = Reshape(b, Q1D, D1D);
-   const auto G = Reshape(g, Q1D, D1D);
-   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, Q1D, NE);
-   const auto X = Reshape(x, D1D, D1D, D1D, VDIM, NE);
-   auto Y = Reshape(y, D1D, D1D, D1D, VDIM, NE);
-
-   mfem::forall_2D<T_Q1D*T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1], sB[MD1][MQ1], sG[MD1][MQ1];
-
-      kernels::internal::vd_regs3d_t<VDIM, DIM, MQ1> g0, g1;
-      kernels::internal::v_regs3d_t<VDIM, MQ1> r0, r1;
-      kernels::internal::v_regs3d_t<VDIM, MQ1> s0, s1;
-
-      kernels::internal::LoadMatrix(D1D, Q1D, B, sB);
-      kernels::internal::LoadMatrix(D1D, Q1D, G, sG);
-
-      kernels::internal::LoadDofs3d(e, D1D, X, r0);
-      kernels::internal::Eval3d(D1D, Q1D, smem, sB, r0, r1); // u vector-value
-      kernels::internal::LoadDofs3d(e, D1D, X, g0);
-      kernels::internal::Grad3d(D1D, Q1D, smem, sB, sG, g0, g1); // u vector-gradient
-
-      for (int qz = 0; qz < Q1D; qz++)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-            {
-               const future::tensor<real_t, 3> U =
-               {
-                  r1[0][qz][qy][qx], r1[1][qz][qy][qx], r1[2][qz][qy][qx]
-               };
-               const future::tensor<real_t, 3,3> gradU = {{
-                     {g1[0][0][qz][qy][qx], g1[1][0][qz][qy][qx], g1[2][0][qz][qy][qx]},
-                     {g1[0][1][qz][qy][qx], g1[1][1][qz][qy][qx], g1[2][1][qz][qy][qx]},
-                     {g1[0][2][qz][qy][qx], g1[1][2][qz][qy][qx], g1[2][2][qz][qy][qx]}
-                  }
-               };
-               const future::tensor<real_t, 3,3> Q = {{
-                     {A(0,0,qx,qy,qz,e), A(1,0,qx,qy,qz,e), A(2,0,qx,qy,qz,e)},
-                     {A(0,1,qx,qy,qz,e), A(1,1,qx,qy,qz,e), A(2,1,qx,qy,qz,e)},
-                     {A(0,2,qx,qy,qz,e), A(1,2,qx,qy,qz,e), A(2,2,qx,qy,qz,e)}
-                  }
-               };
-               const future::tensor<real_t, 3> conv = transpose(gradU) * (Q * U);
-               s0[0][qz][qy][qx] = conv[0];
-               s0[1][qz][qy][qx] = conv[1];
-               s0[2][qz][qy][qx] = conv[2];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      kernels::internal::EvalTranspose3d(D1D, Q1D, smem, sB, s0, s1);
-      kernels::internal::WriteDofs3d(e, D1D, s1, Y);
-   });
-}
-
-} // namespace internal
-
-template<int DIM, int T_D1D, int T_Q1D>
-VectorConvectionNLFIntegrator::AddMultPAType
-VectorConvectionNLFIntegrator::AddMultPAKernels::Kernel()
-{
-   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
-   if constexpr (DIM == 2)
-   {
-      return internal::SmemPAConvectionNLApply2D<T_D1D, T_Q1D>;
-   }
-   else if constexpr (DIM == 3)
-   {
-      return internal::SmemPAConvectionNLApply3D<T_D1D, T_Q1D>;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-inline VectorConvectionNLFIntegrator::AddMultPAType
-VectorConvectionNLFIntegrator::AddMultPAKernels::Fallback
-(int dim, int d1d, int q1d)
-{
-   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
-   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   if (dim == 2)
-   {
-      return internal::SmemPAConvectionNLApply2D<>;
-   }
-   else if (dim == 3)
-   {
-      return internal::SmemPAConvectionNLApply3D<>;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
-} // namespace mfem

fem/integ/nonlininteg_vecconvection_pa_diag.cpp

@@ -1,50 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../ceed/interface/util.hpp"
-#include "./nonlininteg_vecconvection_pa_diag.hpp" // IWYU pragma: keep
-
-namespace mfem
-{
-
-void VectorConvectionNLFIntegrator::AssembleGradDiagonalPA(Vector &de) const
-{
-   MFEM_VERIFY(!DeviceCanUseCeed(),
-               "VectorConvectionNLFIntegrator PA gradients are not supported "
-               "with the libCEED backend");
-
-   if (dim == 2)
-   {
-      GradDiagPA2D::Run(d1d, q1d, ne,
-                        maps->B.Read(),
-                        maps->G.Read(),
-                        pa_adj.Read(),
-                        pa_u.Read(),
-                        de.ReadWrite(),
-                        d1d, q1d);
-   }
-   else if (dim == 3)
-   {
-      GradDiagPA3D::Run(d1d, q1d, ne,
-                        maps->B.Read(),
-                        maps->G.Read(),
-                        pa_adj.Read(),
-                        pa_u.Read(),
-                        de.ReadWrite(),
-                        d1d, q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-   }
-}
-
-} // namespace mfem

fem/integ/nonlininteg_vecconvection_pa_diag.hpp

@@ -1,302 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../config/config.hpp"
-#include "../../general/forall.hpp"
-#include "../../linalg/dtensor.hpp"
-#include "../kernels.hpp"
-#include "../nonlininteg.hpp"
-
-namespace mfem
-{
-
-/// \cond DO_NOT_DOCUMENT
-
-namespace internal
-{
-
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAConvectionNLGradDiagonal2D(const int NE,
-                                             const real_t *b,
-                                             const real_t *g,
-                                             const real_t *a,
-                                             const real_t *u,
-                                             real_t *de,
-                                             const int d1d,
-                                             const int q1d)
-{
-   static constexpr int VDIM = 2, DIM = 2;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, NE);
-   const auto U = Reshape(u, D1D, D1D, VDIM, NE);
-   auto D = Reshape(de, D1D, D1D, VDIM, NE);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t sM[3][MQ1][MQ1], sQ[3][MQ1][MQ1];
-      MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1];
-
-      kernels::internal::v_regs2d_t<VDIM, MQ1> r0, r1;
-      kernels::internal::vd_regs2d_t<VDIM, DIM, MQ1> g0, g1;
-
-      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
-      kernels::internal::LoadMatrix(D1D, Q1D, g, sG);
-
-      kernels::internal::LoadDofs2d(e, D1D, U, r0);
-      kernels::internal::Eval2d(D1D, Q1D, sM[0], sB, r0, r1);
-
-      kernels::internal::LoadDofs2d(e, D1D, U, g0);
-      kernels::internal::Grad2d(D1D, Q1D, sM[0], sB, sG, g0, g1);
-
-      for (int v = 0; v < VDIM; ++v)
-      {
-         future::tensor<real_t, VDIM> e_v = {};
-         e_v[v] = real_t(1);
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-            {
-               const future::tensor<real_t, VDIM> u_val =
-               {
-                  r1[0][qy][qx], r1[1][qy][qx]
-               };
-               const future::tensor<real_t, VDIM, DIM> Q_adj =
-               {
-                  {  { A(0, 0, qx, qy, e), A(1, 0, qx, qy, e) },
-                     { A(0, 1, qx, qy, e), A(1, 1, qx, qy, e) }
-                  }
-               };
-               const future::tensor<real_t, VDIM, DIM> grad_U =
-               {
-                  {  { g1[0][0][qy][qx], g1[1][0][qy][qx] },
-                     { g1[0][1][qy][qx], g1[1][1][qy][qx] }
-                  }
-               };
-               const auto one = Q_adj * u_val;
-               const auto two = transpose(grad_U) * (Q_adj * e_v);
-               sQ[0][qx][qy] = one[0];
-               sQ[1][qx][qy] = one[1];
-               sQ[2][qx][qy] = two[v];
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-            {
-               real_t s[3] = {};
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  const real_t By = sB[dy][qy], Gy = sG[dy][qy];
-                  s[0] += By * By * sQ[0][qx][qy];
-                  s[1] += Gy * By * sQ[1][qx][qy];
-                  s[2] += By * By * sQ[2][qx][qy];
-               }
-               sM[0][qx][dy] = s[0];
-               sM[1][qx][dy] = s[1];
-               sM[2][qx][dy] = s[2];
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
-            {
-               real_t d = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  const real_t Bx = sB[dx][qx], Gx = sG[dx][qx];
-                  d += Gx * Bx * sM[0][qx][dy] +
-                       Bx * Bx * sM[1][qx][dy] +
-                       Bx * Bx * sM[2][qx][dy];
-               }
-               D(dx, dy, v, e) += d;
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAConvectionNLGradDiagonal3D(const int NE,
-                                             const real_t *b,
-                                             const real_t *g,
-                                             const real_t *a,
-                                             const real_t *u,
-                                             real_t *de,
-                                             const int d1d,
-                                             const int q1d)
-{
-   static constexpr int VDIM = 3, DIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, Q1D, NE);
-   const auto U = Reshape(u, D1D, D1D, D1D, VDIM, NE);
-   auto D = Reshape(de, D1D, D1D, D1D, VDIM, NE);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t sM[4][MQ1][MQ1], sQ[4][MQ1][MQ1];
-      MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1];
-
-      kernels::internal::v_regs3d_t<VDIM, MQ1> r0, r1;
-      kernels::internal::vd_regs3d_t<VDIM, DIM, MQ1> g0, g1;
-
-      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
-      kernels::internal::LoadMatrix(D1D, Q1D, g, sG);
-
-      kernels::internal::LoadDofs3d(e, D1D, U, r0);
-      kernels::internal::Eval3d(D1D, Q1D, sM[0], sB, r0, r1);
-
-      kernels::internal::LoadDofs3d(e, D1D, U, g0);
-      kernels::internal::Grad3d(D1D, Q1D, sM[0], sB, sG, g0, g1);
-
-      for (int v = 0; v < VDIM; ++v)
-      {
-         future::tensor<real_t, VDIM> e_v = {};
-         e_v[v] = real_t(1);
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-               {
-                  real_t s[4] = {};
-                  for (int qz = 0; qz < Q1D; ++qz)
-                  {
-                     const future::tensor<real_t, VDIM> u_val =
-                     {
-                        r1[0][qz][qy][qx], r1[1][qz][qy][qx], r1[2][qz][qy][qx]
-                     };
-                     const future::tensor<real_t, VDIM, DIM> Q_adj = {{
-                           {A(0,0,qx,qy,qz,e), A(1,0,qx,qy,qz,e), A(2,0,qx,qy,qz,e)},
-                           {A(0,1,qx,qy,qz,e), A(1,1,qx,qy,qz,e), A(2,1,qx,qy,qz,e)},
-                           {A(0,2,qx,qy,qz,e), A(1,2,qx,qy,qz,e), A(2,2,qx,qy,qz,e)}
-                        }
-                     };
-                     const future::tensor<real_t, VDIM, DIM> grad_U = {{
-                           {g1[0][0][qz][qy][qx], g1[1][0][qz][qy][qx], g1[2][0][qz][qy][qx]},
-                           {g1[0][1][qz][qy][qx], g1[1][1][qz][qy][qx], g1[2][1][qz][qy][qx]},
-                           {g1[0][2][qz][qy][qx], g1[1][2][qz][qy][qx], g1[2][2][qz][qy][qx]}
-                        }
-                     };
-                     const auto one = Q_adj * u_val;
-                     const auto two = transpose(grad_U) * (Q_adj * e_v);
-
-                     const real_t Bz = sB[dz][qz], Gz = sG[dz][qz];
-                     s[0] += one[0] * Bz * Bz;
-                     s[1] += one[1] * Bz * Bz;
-                     s[2] += one[2] * Bz * Gz;
-                     s[3] += two[v] * Bz * Bz;
-                  }
-                  sQ[0][qx][qy] = s[0];
-                  sQ[1][qx][qy] = s[1];
-                  sQ[2][qx][qy] = s[2];
-                  sQ[3][qx][qy] = s[3];
-               }
-            }
-            MFEM_SYNC_THREAD;
-
-            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-               {
-                  real_t s[4] = {};
-                  for (int qy = 0; qy < Q1D; ++qy)
-                  {
-                     const real_t By = sB[dy][qy], Gy = sG[dy][qy];
-                     s[0] += By * By * sQ[0][qx][qy];
-                     s[1] += Gy * By * sQ[1][qx][qy];
-                     s[2] += By * By * sQ[2][qx][qy];
-                     s[3] += By * By * sQ[3][qx][qy];
-                  }
-                  sM[0][dy][qx] = s[0];
-                  sM[1][dy][qx] = s[1];
-                  sM[2][dy][qx] = s[2];
-                  sM[3][dy][qx] = s[3];
-               }
-            }
-            MFEM_SYNC_THREAD;
-
-            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
-               {
-                  real_t d = 0.0;
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     const real_t Bx = sB[dx][qx], Gx = sG[dx][qx];
-                     d += Gx * Bx * sM[0][dy][qx];
-                     d += Bx * Bx * sM[1][dy][qx];
-                     d += Bx * Bx * sM[2][dy][qx];
-                     d += Bx * Bx * sM[3][dy][qx];
-                  }
-                  D(dx, dy, dz, v, e) += d;
-               }
-            }
-            MFEM_SYNC_THREAD;
-         }
-      }
-   });
-}
-
-} // namespace internal
-
-template<int T_D1D, int T_Q1D>
-VectorConvectionNLFIntegrator::GradDiagPAType
-VectorConvectionNLFIntegrator::GradDiagPA2D::Kernel()
-{
-   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
-   return internal::SmemPAConvectionNLGradDiagonal2D<T_D1D, T_Q1D>;
-}
-
-inline VectorConvectionNLFIntegrator::GradDiagPAType
-VectorConvectionNLFIntegrator::GradDiagPA2D::Fallback(int d1d, int q1d)
-{
-   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
-   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   return internal::SmemPAConvectionNLGradDiagonal2D<>;
-}
-
-template<int T_D1D, int T_Q1D>
-VectorConvectionNLFIntegrator::GradDiagPAType
-VectorConvectionNLFIntegrator::GradDiagPA3D::Kernel()
-{
-   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
-   return internal::SmemPAConvectionNLGradDiagonal3D<T_D1D, T_Q1D>;
-}
-
-inline VectorConvectionNLFIntegrator::GradDiagPAType
-VectorConvectionNLFIntegrator::GradDiagPA3D::Fallback(int d1d, int q1d)
-{
-   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
-   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   return internal::SmemPAConvectionNLGradDiagonal3D<>;
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
-} // namespace mfem

fem/integ/nonlininteg_vecconvection_pa_grad.cpp

@@ -1,64 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../ceed/interface/util.hpp"
-#include "./nonlininteg_vecconvection_pa_grad.hpp" // IWYU pragma: keep
-
-namespace mfem
-{
-
-void VectorConvectionNLFIntegrator::AssembleGradPA(
-   const Vector &u, const FiniteElementSpace &fes)
-{
-   MFEM_VERIFY(!DeviceCanUseCeed(),
-               "VectorConvectionNLFIntegrator PA gradients are not supported "
-               "with the libCEED backend");
-
-   this->pa_u = u;
-   AssemblePA(fes);
-}
-
-void VectorConvectionNLFIntegrator::AddMultGradPA(const Vector &x,
-                                                  Vector &y) const
-{
-   MFEM_VERIFY(!DeviceCanUseCeed(),
-               "VectorConvectionNLFIntegrator PA gradients are not supported "
-               "with the libCEED backend");
-
-   if (dim == 2)
-   {
-      AddMultGradPA2D::Run(d1d, q1d, ne,
-                           maps->B.Read(),
-                           maps->G.Read(),
-                           pa_adj.Read(),
-                           pa_u.Read(),
-                           x.Read(),
-                           y.ReadWrite(),
-                           d1d, q1d);
-   }
-   else if (dim == 3)
-   {
-      AddMultGradPA3D::Run(d1d, q1d, ne,
-                           maps->B.Read(),
-                           maps->G.Read(),
-                           pa_adj.Read(),
-                           pa_u.Read(),
-                           x.Read(),
-                           y.ReadWrite(),
-                           d1d, q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-   }
-}
-
-} // namespace mfem

fem/integ/nonlininteg_vecconvection_pa_grad.hpp

@@ -1,257 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../config/config.hpp"
-#include "../../general/forall.hpp"
-#include "../../linalg/dtensor.hpp"
-#include "../kernels.hpp"
-#include "../nonlininteg.hpp"
-
-namespace mfem
-{
-
-/// \cond DO_NOT_DOCUMENT
-
-namespace internal
-{
-
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAConvectionNLGradApply2D(const int ne,
-                                          const real_t *b,
-                                          const real_t *g,
-                                          const real_t *a,
-                                          const real_t *u,
-                                          const real_t *du,
-                                          real_t *y,
-                                          const int d1d,
-                                          const int q1d)
-{
-   static constexpr int VDIM = 2, DIM = 2;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, ne);
-   const auto U = Reshape(u, D1D, D1D, VDIM, ne);
-   const auto dU = Reshape(du, D1D, D1D, VDIM, ne);
-   auto Y = Reshape(y, D1D, D1D, VDIM, ne);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(ne, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1];
-
-      kernels::internal::vd_regs2d_t<VDIM, DIM, MQ1> g0, g1, g2;
-      kernels::internal::v_regs2d_t<DIM, MQ1> r0, r1, r2;
-
-      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
-      kernels::internal::LoadMatrix(D1D, Q1D, g, sG);
-
-      kernels::internal::LoadDofs2d(e, D1D, dU, g0);
-      kernels::internal::Grad2d(D1D, Q1D, smem, sB, sG, g0, g1); // δu gradient
-
-      kernels::internal::LoadDofs2d(e, D1D, U, r0);
-      kernels::internal::Eval2d(D1D, Q1D, smem, sB, r0, r2); // u value
-
-      kernels::internal::LoadDofs2d(e, D1D, dU, r0);
-      kernels::internal::Eval2d(D1D, Q1D, smem, sB, r0, r1); // δu value
-
-      kernels::internal::LoadDofs2d(e, D1D, U, g0);
-      kernels::internal::Grad2d(D1D, Q1D, smem, sB, sG, g0, g2); // u gradient
-
-      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            // First part of the Jacobian: u·∇δu
-            const future::tensor<real_t, DIM> u_val =
-            {
-               r2[0][qy][qx], r2[1][qy][qx]
-            };
-            const future::tensor<real_t, VDIM, DIM> Q_adj =
-            {
-               {  { A(0, 0, qx, qy, e), A(1, 0, qx, qy, e) },
-                  { A(0, 1, qx, qy, e), A(1, 1, qx, qy, e) }
-               }
-            };
-            const future::tensor<real_t, VDIM, DIM> grad_dU =
-            {
-               {  { g1[0][0][qy][qx], g1[1][0][qy][qx] },
-                  { g1[0][1][qy][qx], g1[1][1][qy][qx] }
-               }
-            };
-            const auto one = transpose(grad_dU) * (Q_adj * u_val);
-
-            // Second part of the Jacobian: δu·∇u
-            const future::tensor<real_t, DIM> du_val =
-            {
-               r1[0][qy][qx], r1[1][qy][qx]
-            };
-            const future::tensor<real_t, VDIM, DIM> grad_U =
-            {
-               {  { g2[0][0][qy][qx], g2[1][0][qy][qx] },
-                  { g2[0][1][qy][qx], g2[1][1][qy][qx] }
-               }
-            };
-            const auto two = transpose(grad_U) * (Q_adj * du_val);
-
-            // u⋅∇δu + δu⋅∇u
-            r0[0][qy][qx] = one[0] + two[0];
-            r0[1][qy][qx] = one[1] + two[1];
-         }
-      }
-      MFEM_SYNC_THREAD;
-      kernels::internal::EvalTranspose2d(D1D, Q1D, smem, sB, r0, r1);
-      kernels::internal::WriteDofs2d(e, D1D, r1, Y);
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAConvectionNLGradApply3D(const int ne,
-                                          const real_t *b,
-                                          const real_t *g,
-                                          const real_t *a,
-                                          const real_t *u,
-                                          const real_t *du,
-                                          real_t *y,
-                                          const int d1d,
-                                          const int q1d)
-{
-   static constexpr int VDIM = 3, DIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, Q1D, ne);
-   const auto U = Reshape(u, D1D, D1D, D1D, VDIM, ne);
-   const auto dU = Reshape(du, D1D, D1D, D1D, VDIM, ne);
-   auto Y = Reshape(y, D1D, D1D, D1D, VDIM, ne);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(ne, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1];
-
-      kernels::internal::v_regs3d_t<VDIM, MQ1> r0, r1, r2;
-      kernels::internal::vd_regs3d_t<VDIM, DIM, MQ1> g0, g1, g2;
-
-      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
-      kernels::internal::LoadMatrix(D1D, Q1D, g, sG);
-
-      kernels::internal::LoadDofs3d(e, D1D, dU, g0);
-      kernels::internal::Grad3d(D1D, Q1D, smem, sB, sG, g0, g1); // δu gradient
-
-      kernels::internal::LoadDofs3d(e, D1D, U, r0);
-      kernels::internal::Eval3d(D1D, Q1D, smem, sB, r0, r2); // u value
-
-      kernels::internal::LoadDofs3d(e, D1D, dU, r0);
-      kernels::internal::Eval3d(D1D, Q1D, smem, sB, r0, r1); // δu value
-
-      kernels::internal::LoadDofs3d(e, D1D, U, g0);
-      kernels::internal::Grad3d(D1D, Q1D, smem, sB, sG, g0, g2); // u gradient
-
-      for (int qz = 0; qz < Q1D; qz++)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-            {
-               // First part of the Jacobian: u·∇δu
-               const future::tensor<real_t, DIM> u_val =
-               {
-                  r2[0][qz][qy][qx],
-                  r2[1][qz][qy][qx],
-                  r2[2][qz][qy][qx]
-               };
-               const future::tensor<real_t, VDIM, DIM> Q_adj = {{
-                     {A(0,0,qx,qy,qz,e), A(1,0,qx,qy,qz,e), A(2,0,qx,qy,qz,e)},
-                     {A(0,1,qx,qy,qz,e), A(1,1,qx,qy,qz,e), A(2,1,qx,qy,qz,e)},
-                     {A(0,2,qx,qy,qz,e), A(1,2,qx,qy,qz,e), A(2,2,qx,qy,qz,e)}
-                  }
-               };
-               const future::tensor<real_t, DIM, DIM> grad_dU = {{
-                     {g1[0][0][qz][qy][qx], g1[1][0][qz][qy][qx], g1[2][0][qz][qy][qx]},
-                     {g1[0][1][qz][qy][qx], g1[1][1][qz][qy][qx], g1[2][1][qz][qy][qx]},
-                     {g1[0][2][qz][qy][qx], g1[1][2][qz][qy][qx], g1[2][2][qz][qy][qx]}
-                  }
-               };
-               const auto one = transpose(grad_dU) * (Q_adj * u_val);
-
-               // Second part of the Jacobian: δu·∇u
-               const future::tensor<real_t, DIM> du_val =
-               {
-                  r1[0][qz][qy][qx], r1[1][qz][qy][qx], r1[2][qz][qy][qx]
-               };
-               const future::tensor<real_t, VDIM, DIM> grad_U = {{
-                     {g2[0][0][qz][qy][qx], g2[1][0][qz][qy][qx], g2[2][0][qz][qy][qx]},
-                     {g2[0][1][qz][qy][qx], g2[1][1][qz][qy][qx], g2[2][1][qz][qy][qx]},
-                     {g2[0][2][qz][qy][qx], g2[1][2][qz][qy][qx], g2[2][2][qz][qy][qx]}
-                  }
-               };
-               const auto two = transpose(grad_U) * (Q_adj * du_val);
-
-               // u⋅∇δu + δu⋅∇u
-               r0[0][qz][qy][qx] = one[0] + two[0];
-               r0[1][qz][qy][qx] = one[1] + two[1];
-               r0[2][qz][qy][qx] = one[2] + two[2];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      kernels::internal::EvalTranspose3d(D1D, Q1D, smem, sB, r0, r1);
-      kernels::internal::WriteDofs3d(e, D1D, r1, Y);
-   });
-}
-
-} // namespace internal
-
-template<int T_D1D, int T_Q1D>
-VectorConvectionNLFIntegrator::AddMultGradPAType
-VectorConvectionNLFIntegrator::AddMultGradPA2D::Kernel()
-{
-   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
-   return internal::SmemPAConvectionNLGradApply2D<T_D1D, T_Q1D>;
-}
-
-inline VectorConvectionNLFIntegrator::AddMultGradPAType
-VectorConvectionNLFIntegrator::AddMultGradPA2D::Fallback(int d1d, int q1d)
-{
-   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
-   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   return internal::SmemPAConvectionNLGradApply2D<>;
-}
-
-template<int T_D1D, int T_Q1D>
-VectorConvectionNLFIntegrator::AddMultGradPAType
-VectorConvectionNLFIntegrator::AddMultGradPA3D::Kernel()
-{
-   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
-   return internal::SmemPAConvectionNLGradApply3D<T_D1D, T_Q1D>;
-}
-
-inline VectorConvectionNLFIntegrator::AddMultGradPAType
-VectorConvectionNLFIntegrator::AddMultGradPA3D::Fallback(int d1d, int q1d)
-{
-   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
-   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   return internal::SmemPAConvectionNLGradApply3D<>;
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
-} // namespace mfem

fem/intrules.cpp

@@ -542,10 +542,7 @@ void QuadratureFunctions1D::GaussJacobi(const int np, const real_t alpha,
          return;
    }
 
-#ifdef MFEM_USE_MPFR
-   MFEM_WARNING("MPFR implementation of Gauss-Jacobi quadrature not implemented yet. Falling "
-                "back to double precision implementation...");
-#endif
+#ifndef MFEM_USE_MPFR
 
    const int n = np;
    // common constants for Jacobi polynomials
@@ -614,6 +611,13 @@ void QuadratureFunctions1D::GaussJacobi(const int np, const real_t alpha,
                                                 ab + 1) / ((1.0 - xi*xi)*pp*pp) / pow(2, ab);
       // map nodes and weights to the interval [0,1]
    }
+
+#else // MFEM_USE_MPFR is defined
+
+   MFEM_ABORT("MPFR implementation of Gauss-Jacobi quadrature not defined yet");
+
+#endif // MFEM_USE_MPFR
+
 }
 
 

fem/kernel_dispatch.hpp

@@ -61,7 +61,7 @@ namespace mfem
 #define MFEM_REGISTER_KERNELS_1(KernelName, KernelType, Params)                \
    MFEM_REGISTER_KERNELS_(KernelName, KernelType, Params, (), Params)
 
-// Version of MFEM_REGISTER_KERNELS with optional (non-dispatch)
+// Version of MFEM_REGISTER_KERNELS without any optional (non-dispatch)
 // parameters (e.g. NBZ).
 #define MFEM_REGISTER_KERNELS_2(KernelName, KernelType, Params, OptParams)     \
    MFEM_REGISTER_KERNELS_(KernelName, KernelType, Params, OptParams,           \

fem/kernels.hpp

@@ -83,7 +83,7 @@ constexpr int SetMaxOf(int n) { return NextMultipleOf<4>(n); }
 #endif // CUDA/HIP && DEVICE_COMPILE
 
 /// Load 2D matrix into shared memory
-template <int MQ1, bool TRANSPOSE = false>
+template <int MQ1>
 inline MFEM_HOST_DEVICE void LoadMatrix(const int d1d, const int q1d,
                                         const real_t *M, real_t (*N)[MQ1])
 {
@@ -91,14 +91,7 @@ inline MFEM_HOST_DEVICE void LoadMatrix(const int d1d, const int q1d,
    {
       MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
       {
-         if constexpr (TRANSPOSE)
-         {
-            N[dy][qx] = M[qx * d1d + dy];
-         }
-         else
-         {
-            N[dy][qx] = M[dy * q1d + qx];
-         }
+         N[dy][qx] = M[dy * q1d + qx];
       }
    }
    MFEM_SYNC_THREAD;

fem/nonlinearform.hpp

@@ -224,6 +224,9 @@ public:
    /** @see GetGradient(const Vector &) */
    Operator &GetGradient(const Vector &x, bool finalize) const;
 
+   /// Suppress a warning about hiding overloaded virtual function.
+   using Operator::GetGradient;
+
    /// Update the NonlinearForm to propagate updates of the associated FE space.
    /** After calling this method, the essential boundary conditions need to be
        set again. */

fem/nonlinearform_ext.cpp

@@ -100,17 +100,6 @@ PANonlinearFormExtension::Gradient::Gradient(const PANonlinearFormExtension &e):
 
 void PANonlinearFormExtension::Gradient::AssembleGrad(const Vector &g)
 {
-   if (DeviceCanUseCeed())
-   {
-      for (int i = 0; i < ext.dnfi.Size(); ++i)
-      {
-         MFEM_VERIFY(dynamic_cast<VectorConvectionNLFIntegrator *>
-                     (ext.dnfi[i]) == nullptr,
-                     "VectorConvectionNLFIntegrator PA gradients are not supported "
-                     "with the libCEED backend");
-      }
-   }
-
    ext.elemR->Mult(g, ext.xe);
    for (int i = 0; i < ext.dnfi.Size(); ++i)
    {

fem/nonlininteg.cpp

@@ -954,74 +954,4 @@ void SkewSymmetricVectorConvectionNLFIntegrator::AssembleElementGrad(
    }
 }
 
-void ConvectiveVectorConvectionNLFIntegrator::AssemblePA(
-   const FiniteElementSpace &)
-{
-   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void ConvectiveVectorConvectionNLFIntegrator::AssembleGradPA(
-   const Vector &, const FiniteElementSpace &)
-{
-   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void ConvectiveVectorConvectionNLFIntegrator::AddMultPA(
-   const Vector &, Vector &) const
-{
-   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void ConvectiveVectorConvectionNLFIntegrator::AddMultGradPA(
-   const Vector &, Vector &) const
-{
-   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void ConvectiveVectorConvectionNLFIntegrator::AssembleGradDiagonalPA(
-   Vector &) const
-{
-   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void SkewSymmetricVectorConvectionNLFIntegrator::AssemblePA(
-   const FiniteElementSpace &)
-{
-   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void SkewSymmetricVectorConvectionNLFIntegrator::AssembleGradPA(
-   const Vector &, const FiniteElementSpace &)
-{
-   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void SkewSymmetricVectorConvectionNLFIntegrator::AddMultPA(
-   const Vector &, Vector &) const
-{
-   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void SkewSymmetricVectorConvectionNLFIntegrator::AddMultGradPA(
-   const Vector &, Vector &) const
-{
-   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void SkewSymmetricVectorConvectionNLFIntegrator::AssembleGradDiagonalPA(
-   Vector &) const
-{
-   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
 }

fem/nonlininteg.hpp

@@ -18,7 +18,6 @@
 #include "fespace.hpp"
 #include "ceed/interface/operator.hpp"
 #include "integrator.hpp"
-#include "kernel_dispatch.hpp"
 
 namespace mfem
 {
@@ -385,17 +384,15 @@ private:
    DenseMatrix dshape, dshapex, EF, gradEF, ELV, elmat_comp;
    Vector shape;
    // PA extension
-   int dim, ne, nq, d1d, q1d;
-   Vector pa_adj, pa_u;
+   Vector pa_data;
    const DofToQuad *maps;         ///< Not owned
    const GeometricFactors *geom;  ///< Not owned
+   int dim, ne, nq;
 
 public:
-   struct Kernels { Kernels(); };
+   VectorConvectionNLFIntegrator(Coefficient &q): Q(&q) { }
 
-   VectorConvectionNLFIntegrator(Coefficient &q): Q(&q) { static Kernels kernels; }
-
-   VectorConvectionNLFIntegrator() { static Kernels kernels; }
+   VectorConvectionNLFIntegrator() = default;
 
    static const IntegrationRule &GetRule(const FiniteElement &fe,
                                          const ElementTransformation &T);
@@ -414,56 +411,13 @@ public:
 
    void AssemblePA(const FiniteElementSpace &fes) override;
 
-   void AssembleGradPA(const Vector &x, const FiniteElementSpace &fes) override;
+   void AssembleMF(const FiniteElementSpace &fes) override;
 
    void AddMultPA(const Vector &x, Vector &y) const override;
 
-   using AddMultPAType =
-      void(*)(const int ne, const real_t *B, const real_t *G, const real_t *A,
-              const real_t *x, real_t *y,
-              const int d1d, const int q1d);
-   MFEM_REGISTER_KERNELS(AddMultPAKernels, AddMultPAType, (int, int, int));
-
-   void AddMultGradPA(const Vector &x, Vector &y) const override;
-
-   using AddMultGradPAType =
-      void(*)(const int ne, const real_t *B, const real_t *G, const real_t *A,
-              const real_t *u, const real_t *x, real_t *y,
-              const int d1d, const int q1d);
-
-   MFEM_REGISTER_KERNELS(AddMultGradPA2D, AddMultGradPAType, (int, int));
-   MFEM_REGISTER_KERNELS(AddMultGradPA3D, AddMultGradPAType, (int, int));
-
-   void AssembleGradDiagonalPA(Vector &) const override;
-
-   using GradDiagPAType =
-      void (*)(const int ne, const real_t *B, const real_t *G, const real_t *A,
-               const real_t *u, real_t *y,
-               const int d1d, const int q1d);
-
-   MFEM_REGISTER_KERNELS(GradDiagPA2D, GradDiagPAType, (int, int));
-   MFEM_REGISTER_KERNELS(GradDiagPA3D, GradDiagPAType, (int, int));
-
-   template <int DIM, int D1D, int Q1D>
-   static void AddSpecialization()
-   {
-      AddMultPAKernels::Specialization<DIM, D1D, Q1D>::Add();
-      if constexpr (DIM == 2)
-      {
-         AddMultGradPA2D::Specialization<D1D, Q1D>::Add();
-         GradDiagPA2D::Specialization<D1D, Q1D>::Add();
-      }
-      else if constexpr (DIM == 3)
-      {
-         AddMultGradPA3D::Specialization<D1D, Q1D>::Add();
-         GradDiagPA3D::Specialization<D1D, Q1D>::Add();
-      }
-   }
-
-   void AssembleMF(const FiniteElementSpace &fes) override;
-
    void AddMultMF(const Vector &x, Vector &y) const override;
 
+
 protected:
    const IntegrationRule* GetDefaultIntegrationRule(
       const FiniteElement& trial_fe,
@@ -476,8 +430,7 @@ protected:
 
 
 /** This class is used to assemble the convective form of the nonlinear term
-    arising in the Navier-Stokes equations $(u \cdot \nabla v, w )$.
-    Partial assembly is not supported; use VectorConvectionNLFIntegrator. */
+    arising in the Navier-Stokes equations $(u \cdot \nabla v, w )$ */
 class ConvectiveVectorConvectionNLFIntegrator :
    public VectorConvectionNLFIntegrator
 {
@@ -495,20 +448,12 @@ public:
                             ElementTransformation &trans,
                             const Vector &elfun,
                             DenseMatrix &elmat) override;
-
-   using NonlinearFormIntegrator::AssemblePA;
-   void AssemblePA(const FiniteElementSpace &fes) override;
-   void AssembleGradPA(const Vector &x, const FiniteElementSpace &fes) override;
-   void AddMultPA(const Vector &x, Vector &y) const override;
-   void AddMultGradPA(const Vector &x, Vector &y) const override;
-   void AssembleGradDiagonalPA(Vector &diag) const override;
 };
 
 
 /** This class is used to assemble the skew-symmetric form of the nonlinear term
     arising in the Navier-Stokes equations
-    $.5*(u \cdot \nabla v, w ) - .5*(u \cdot \nabla w, v )$.
-    Partial assembly is not supported; use VectorConvectionNLFIntegrator. */
+    $.5*(u \cdot \nabla v, w ) - .5*(u \cdot \nabla w, v )$ */
 class SkewSymmetricVectorConvectionNLFIntegrator :
    public VectorConvectionNLFIntegrator
 {
@@ -526,13 +471,6 @@ public:
                             ElementTransformation &trans,
                             const Vector &elfun,
                             DenseMatrix &elmat) override;
-
-   using NonlinearFormIntegrator::AssemblePA;
-   void AssemblePA(const FiniteElementSpace &fes) override;
-   void AssembleGradPA(const Vector &x, const FiniteElementSpace &fes) override;
-   void AddMultPA(const Vector &x, Vector &y) const override;
-   void AddMultGradPA(const Vector &x, Vector &y) const override;
-   void AssembleGradDiagonalPA(Vector &diag) const override;
 };
 
 }

fem/pgridfunc.cpp

@@ -22,20 +22,10 @@ using namespace std;
 namespace mfem
 {
 
-ParGridFunction::ParGridFunction(ParFiniteElementSpace *pf, GridFunction *gf,
-                                 bool preserve)
+ParGridFunction::ParGridFunction(ParFiniteElementSpace *pf, GridFunction *gf)
 {
    fes = pfes = pf;
    SetDataAndSize(gf->GetData(), gf->Size());
-
-   if (pfes->HaveDofSigns())
-   {
-      MFEM_VERIFY(!preserve, "Differing sign conventions for the serial and "
-                  "parallel grid functions will prevent preserving the serial "
-                  "GridFunctions in this context.");
-
-      pfes->ApplyDofSigns(HostReadWrite());
-   }
 }
 
 ParGridFunction::ParGridFunction(ParFiniteElementSpace *pf, HypreParVector *tv)

fem/pgridfunc.hpp

@@ -100,12 +100,8 @@ public:
    /// Construct a ParGridFunction using a GridFunction as external data.
    /** The parallel space @a *pf and the space used by @a *gf should match. The
        data from @a *gf is used as the local data of the ParGridFunction on each
-       processor. The ParGridFunction does not assume ownership of the data.
-       The boolean, @a preserve, indicates that the data stored in @a *gf should
-       remain unchanged. An error will occur if @a preserve is true and
-       construction of a valid ParGridFunction requires the data to change. */
-   ParGridFunction(ParFiniteElementSpace *pf, GridFunction *gf,
-                   bool preserve = true);
+       processor. The ParGridFunction does not assume ownership of the data. */
+   ParGridFunction(ParFiniteElementSpace *pf, GridFunction *gf);
 
    /** @brief Creates grid function on (all) dofs from a given vector on the
        true dofs, i.e. P tv. */

fem/qinterp/det.cpp

@@ -21,23 +21,24 @@ namespace quadrature_interpolator
 
 void InitDetKernels()
 {
+   using k = QuadratureInterpolator::DetKernels;
    // 2D
-   QuadratureInterpolator::AddDetSpecializations<2,2,2,2>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,2,3>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,2,4>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,2,6>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,3,4>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,3,6>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,4,4>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,4,6>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,5,6>();
+   k::Specialization<2,2,2,2>::Add();
+   k::Specialization<2,2,2,3>::Add();
+   k::Specialization<2,2,2,4>::Add();
+   k::Specialization<2,2,2,6>::Add();
+   k::Specialization<2,2,3,4>::Add();
+   k::Specialization<2,2,3,6>::Add();
+   k::Specialization<2,2,4,4>::Add();
+   k::Specialization<2,2,4,6>::Add();
+   k::Specialization<2,2,5,6>::Add();
    // 3D
-   QuadratureInterpolator::AddDetSpecializations<3,3,2,4>();
-   QuadratureInterpolator::AddDetSpecializations<3,3,3,3>();
-   QuadratureInterpolator::AddDetSpecializations<3,3,3,5>();
-   QuadratureInterpolator::AddDetSpecializations<3,3,3,6>();
-   QuadratureInterpolator::AddDetSpecializations<3,3,4,6>();
-   QuadratureInterpolator::AddDetSpecializations<3,3,3,4>();
+   k::Specialization<3,3,2,4>::Add();
+   k::Specialization<3,3,3,3>::Add();
+   k::Specialization<3,3,3,5>::Add();
+   k::Specialization<3,3,3,6>::Add();
+   k::Specialization<3,3,4,6>::Add();
+   k::Specialization<3,3,3,4>::Add();
 }
 
 } // namespace quadrature_interpolator
@@ -46,8 +47,8 @@ void InitDetKernels()
 /// @cond Suppress_Doxygen_warnings
 
 QuadratureInterpolator::DetKernelType
-QuadratureInterpolator::DetKernels::Fallback(int DIM, int SDIM, int D1D,
-                                             int Q1D)
+QuadratureInterpolator::DetKernels::Fallback(
+   int DIM, int SDIM, int D1D, int Q1D)
 {
    if (DIM == 1)
    {

fem/qinterp/eval.hpp

@@ -30,18 +30,23 @@ namespace internal
 namespace quadrature_interpolator
 {
 
-template <QVectorLayout Q_LAYOUT, bool Integral>
-static void ImplValues1D(const int NE, const real_t *b_, const real_t *detJ_,
-                         const real_t *x_, real_t *y_, const int vdim,
-                         const int d1d, const int q1d)
+template<QVectorLayout Q_LAYOUT>
+static void Values1D(const int NE,
+                     const real_t *b_,
+                     const real_t *x_,
+                     real_t *y_,
+                     const int vdim,
+                     const int d1d,
+                     const int q1d)
 {
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
+   const auto b = Reshape(b_, q1d, d1d);
+   const auto x = Reshape(x_, d1d, vdim, NE);
+   auto y = Q_LAYOUT == QVectorLayout::byNODES ?
+            Reshape(y_, q1d, vdim, NE):
+            Reshape(y_, vdim, q1d, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      const auto b = Reshape(b_, q1d, d1d);
-      const auto x = Reshape(x_, d1d, vdim, NE);
-      const auto detJ = Reshape(detJ_, q1d, NE);
-      auto y = Q_LAYOUT == QVectorLayout::byNODES ? Reshape(y_, q1d, vdim, NE)
-               : Reshape(y_, vdim, q1d, NE);
       for (int c = 0; c < vdim; c++)
       {
          for (int q = 0; q < q1d; q++)
@@ -51,36 +56,24 @@ static void ImplValues1D(const int NE, const real_t *b_, const real_t *detJ_,
             {
                u += b(q, d) * x(d, c, e);
             }
-            if constexpr (Integral)
-            {
-               u /= detJ(q, e);
-            }
-            if constexpr (Q_LAYOUT == QVectorLayout::byVDIM)
-            {
-               y(c, q, e) = u;
-            }
-            if constexpr (Q_LAYOUT == QVectorLayout::byNODES)
-            {
-               y(q, c, e) = u;
-            }
+            if (Q_LAYOUT == QVectorLayout::byVDIM)  { y(c, q, e) = u; }
+            if (Q_LAYOUT == QVectorLayout::byNODES) { y(q, c, e) = u; }
          }
       }
    });
 }
 
-template <QVectorLayout Q_LAYOUT>
-static void Values1D(const int NE, const real_t *b_, const real_t *x_,
-                     real_t *y_, const int vdim, const int d1d, const int q1d)
-{
-   ImplValues1D<Q_LAYOUT, false>(NE, b_, nullptr, x_, y_, vdim, d1d, q1d);
-}
-
 // Template compute kernel for Values in 2D: tensor product version.
-template <QVectorLayout Q_LAYOUT, bool Integral, int T_VDIM = 0, int T_D1D = 0,
-          int T_Q1D = 0, int T_NBZ = 1>
-static void ImplValues2D(const int NE, const real_t *b_, const real_t *detJ_,
-                         const real_t *x_, real_t *y_, const int vdim = 0,
-                         const int d1d = 0, const int q1d = 0)
+template<QVectorLayout Q_LAYOUT,
+         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
+         int T_NBZ = 1>
+static void Values2D(const int NE,
+                     const real_t *b_,
+                     const real_t *x_,
+                     real_t *y_,
+                     const int vdim = 0,
+                     const int d1d = 0,
+                     const int q1d = 0)
 {
    static constexpr int NBZ = T_NBZ ? T_NBZ : 1;
 
@@ -89,14 +82,13 @@ static void ImplValues2D(const int NE, const real_t *b_, const real_t *detJ_,
    const int VDIM = T_VDIM ? T_VDIM : vdim;
 
    const auto b = Reshape(b_, Q1D, D1D);
+   const auto x = Reshape(x_, D1D, D1D, VDIM, NE);
+   auto y = Q_LAYOUT == QVectorLayout::byNODES ?
+            Reshape(y_, Q1D, Q1D, VDIM, NE):
+            Reshape(y_, VDIM, Q1D, Q1D, NE);
 
-   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE(int e)
+   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int e)
    {
-      const auto x = Reshape(x_, D1D, D1D, VDIM, NE);
-      const auto detJ = Reshape(detJ_, Q1D, Q1D, NE);
-      auto y = Q_LAYOUT == QVectorLayout::byNODES
-               ? Reshape(y_, Q1D, Q1D, VDIM, NE)
-               : Reshape(y_, VDIM, Q1D, Q1D, NE);
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       const int VDIM = T_VDIM ? T_VDIM : vdim;
@@ -118,33 +110,16 @@ static void ImplValues2D(const int NE, const real_t *b_, const real_t *detJ_,
 
       for (int c = 0; c < VDIM; c++)
       {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx, x, D1D)
-            {
-               DD(dx, dy) = x(dx, dy, c, e);
-            }
-         }
-         MFEM_SYNC_THREAD;
+         kernels::internal::LoadX(e,D1D,c,x,DD);
          kernels::internal::EvalX(D1D,Q1D,B,DD,DQ);
          kernels::internal::EvalY(D1D,Q1D,B,DQ,QQ);
          MFEM_FOREACH_THREAD(qy,y,Q1D)
          {
             MFEM_FOREACH_THREAD(qx,x,Q1D)
             {
-               real_t u = QQ(qx, qy);
-               if constexpr (Integral)
-               {
-                  u /= detJ(qx, qy, e);
-               }
-               if constexpr (Q_LAYOUT == QVectorLayout::byVDIM)
-               {
-                  y(c, qx, qy, e) = u;
-               }
-               if constexpr (Q_LAYOUT == QVectorLayout::byNODES)
-               {
-                  y(qx, qy, c, e) = u;
-               }
+               real_t u = QQ(qx,qy);
+               if (Q_LAYOUT == QVectorLayout::byVDIM) { y(c,qx,qy,e) = u; }
+               if (Q_LAYOUT == QVectorLayout::byNODES) { y(qx,qy,c,e) = u; }
             }
          }
          MFEM_SYNC_THREAD;
@@ -152,37 +127,29 @@ static void ImplValues2D(const int NE, const real_t *b_, const real_t *detJ_,
    });
 }
 
-// Template compute kernel for Values in 2D: tensor product version.
-template <QVectorLayout Q_LAYOUT, int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
-          int T_NBZ = 1>
-static void Values2D(const int NE, const real_t *b_, const real_t *x_,
-                     real_t *y_, const int vdim = 0, const int d1d = 0,
-                     const int q1d = 0)
-{
-   return ImplValues2D<Q_LAYOUT, false, T_VDIM, T_D1D, T_Q1D, T_NBZ>(
-             NE, b_, nullptr, x_, y_, vdim, d1d, q1d);
-}
-
 // Template compute kernel for Values in 3D: tensor product version.
-template <QVectorLayout Q_LAYOUT, bool Integral, int T_VDIM = 0, int T_D1D = 0,
-          int T_Q1D = 0>
-static void ImplValues3D(const int NE, const real_t *b_, const real_t *detJ_,
-                         const real_t *x_, real_t *y_, const int vdim = 0,
-                         const int d1d = 0, const int q1d = 0)
+template<QVectorLayout Q_LAYOUT,
+         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0>
+static void Values3D(const int NE,
+                     const real_t *b_,
+                     const real_t *x_,
+                     real_t *y_,
+                     const int vdim = 0,
+                     const int d1d = 0,
+                     const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
    const int VDIM = T_VDIM ? T_VDIM : vdim;
 
    const auto b = Reshape(b_, Q1D, D1D);
+   const auto x = Reshape(x_, D1D, D1D, D1D, VDIM, NE);
+   auto y = Q_LAYOUT == QVectorLayout:: byNODES ?
+            Reshape(y_, Q1D, Q1D, Q1D, VDIM, NE):
+            Reshape(y_, VDIM, Q1D, Q1D, Q1D, NE);
 
-   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
    {
-      const auto x = Reshape(x_, D1D, D1D, D1D, VDIM, NE);
-      const auto detJ = Reshape(detJ_, Q1D, Q1D, Q1D, NE);
-      auto y = Q_LAYOUT == QVectorLayout::byNODES
-               ? Reshape(y_, Q1D, Q1D, Q1D, VDIM, NE)
-               : Reshape(y_, VDIM, Q1D, Q1D, Q1D, NE);
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       const int VDIM = T_VDIM ? T_VDIM : vdim;
@@ -204,17 +171,7 @@ static void ImplValues3D(const int NE, const real_t *b_, const real_t *detJ_,
 
       for (int c = 0; c < VDIM; c++)
       {
-         MFEM_FOREACH_THREAD(dz, z, D1D)
-         {
-            MFEM_FOREACH_THREAD(dy, y, D1D)
-            {
-               MFEM_FOREACH_THREAD(dx, x, D1D)
-               {
-                  DDD(dx, dy, dz) = x(dx, dy, dz, c, e);
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
+         kernels::internal::LoadX(e,D1D,c,x,DDD);
          kernels::internal::EvalX(D1D,Q1D,B,DDD,DDQ);
          kernels::internal::EvalY(D1D,Q1D,B,DDQ,DQQ);
          kernels::internal::EvalZ(D1D,Q1D,B,DQQ,QQQ);
@@ -224,19 +181,9 @@ static void ImplValues3D(const int NE, const real_t *b_, const real_t *detJ_,
             {
                MFEM_FOREACH_THREAD(qx,x,Q1D)
                {
-                  real_t u = QQQ(qz,qy,qx);
-                  if constexpr (Integral)
-                  {
-                     u /= detJ(qx, qy, qz, e);
-                  }
-                  if constexpr (Q_LAYOUT == QVectorLayout::byVDIM)
-                  {
-                     y(c, qx, qy, qz, e) = u;
-                  }
-                  if constexpr (Q_LAYOUT == QVectorLayout::byNODES)
-                  {
-                     y(qx, qy, qz, c, e) = u;
-                  }
+                  const real_t u = QQQ(qz,qy,qx);
+                  if (Q_LAYOUT == QVectorLayout::byVDIM) { y(c,qx,qy,qz,e) = u; }
+                  if (Q_LAYOUT == QVectorLayout::byNODES) { y(qx,qy,qz,c,e) = u; }
                }
             }
          }
@@ -245,431 +192,14 @@ static void ImplValues3D(const int NE, const real_t *b_, const real_t *detJ_,
    });
 }
 
-// Template compute kernel for Values in 3D: tensor product version.
-template <QVectorLayout Q_LAYOUT, int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0>
-static void Values3D(const int NE, const real_t *b_, const real_t *x_,
-                     real_t *y_, const int vdim = 0, const int d1d = 0,
-                     const int q1d = 0)
-{
-   return ImplValues3D<Q_LAYOUT, false, T_VDIM, T_D1D, T_Q1D>(
-             NE, b_, nullptr, x_, y_, vdim, d1d, q1d);
-}
-
-template <bool Integral>
-void ImplEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
-                const real_t *detJ, const GeometricFactors *geom,
-                const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
-                Vector &q_der, Vector &q_det, const int eval_flags);
-
-inline void Eval1D(const int NE, const int vdim, const QVectorLayout q_layout,
-                   const GeometricFactors *geom, const DofToQuad &maps,
-                   const Vector &e_vec, Vector &q_val, Vector &q_der, Vector &q_det,
-                   const int eval_flags)
-{
-   ImplEval1D<false>(NE, vdim, q_layout, nullptr, geom, maps, e_vec, q_val,
-                     q_der, q_det, eval_flags);
-}
-
-// Template compute kernel for 2D quadrature interpolation:
-// * non-tensor product version,
-// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
-// * assumes 'maps.mode == FULL'.
-template <bool Integral, const int T_VDIM, const int T_ND, const int T_NQ>
-static void ImplEval2D(const int NE, const int vdim,
-                       const QVectorLayout q_layout, const real_t *detJ_,
-                       const GeometricFactors *geom, const DofToQuad &maps,
-                       const Vector &e_vec, Vector &q_val, Vector &q_der,
-                       Vector &q_det, const int eval_flags)
-{
-   using QI = QuadratureInterpolator;
-
-   const int nd = maps.ndof;
-   const int nq = maps.nqpt;
-   const int ND = T_ND ? T_ND : nd;
-   const int NQ = T_NQ ? T_NQ : nq;
-   const int NMAX = NQ > ND ? NQ : ND;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-   MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
-   MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 2, "");
-   MFEM_VERIFY(ND <= QI::MAX_ND2D, "");
-   MFEM_VERIFY(NQ <= QI::MAX_NQ2D, "");
-   if constexpr(Integral)
-   {
-      MFEM_VERIFY(!(eval_flags & (QI::DERIVATIVES | QI::PHYSICAL_DERIVATIVES |
-                                  QI::DETERMINANTS)),
-                  "Integral FE does not support computing derivatives");
-   }
-   const auto B = Reshape(maps.B.Read(), NQ, ND);
-   const auto G = Reshape(maps.G.Read(), NQ, 2, ND);
-   const auto J = Reshape(geom ? geom->J.Read() : nullptr, NQ, 2, 2, NE);
-   const auto E_ = e_vec.Read();
-   auto val = q_layout == QVectorLayout::byNODES ?
-              Reshape(q_val.Write(), NQ, VDIM, NE):
-              Reshape(q_val.Write(), VDIM, NQ, NE);
-   auto der = q_layout == QVectorLayout::byNODES ?
-              Reshape(q_der.Write(), NQ, VDIM, 2, NE):
-              Reshape(q_der.Write(), VDIM, 2, NQ, NE);
-   auto det = Reshape(q_det.Write(), NQ, NE);
-   mfem::forall_2D(NE, NMAX, 1, [=] MFEM_HOST_DEVICE(int e)
-   {
-      const auto E = Reshape(E_, ND, VDIM, NE);
-      const auto detJ = Reshape(detJ_, NQ, NE);
-      const int ND = T_ND ? T_ND : nd;
-      const int NQ = T_NQ ? T_NQ : nq;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int max_ND = T_ND ? T_ND : QI::MAX_ND2D;
-      constexpr int max_VDIM = T_VDIM ? T_VDIM : QI::MAX_VDIM2D;
-      MFEM_SHARED real_t s_E[max_VDIM*max_ND];
-      MFEM_FOREACH_THREAD(d, x, ND)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            s_E[c + d * VDIM] = E(d, c, e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(q, x, NQ)
-      {
-         if (eval_flags & (QI::VALUES | QI::PHYSICAL_VALUES))
-         {
-            real_t ed[max_VDIM];
-            for (int c = 0; c < VDIM; c++)
-            {
-               ed[c] = 0.0;
-            }
-            for (int d = 0; d < ND; ++d)
-            {
-               const real_t b = B(q,d);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  ed[c] += b * s_E[c + d * VDIM];
-               }
-            }
-            for (int c = 0; c < VDIM; c++)
-            {
-               if constexpr (Integral)
-               {
-                  ed[c] /= detJ(q, e);
-               }
-               if (q_layout == QVectorLayout::byVDIM)
-               {
-                  val(c, q, e) = ed[c];
-               }
-               if (q_layout == QVectorLayout::byNODES)
-               {
-                  val(q, c, e) = ed[c];
-               }
-            }
-         }
-         if ((eval_flags & QI::DERIVATIVES) ||
-             (eval_flags & QI::PHYSICAL_DERIVATIVES) ||
-             (eval_flags & QI::DETERMINANTS))
-         {
-            // use MAX_VDIM2D to avoid "subscript out of range" warnings
-            real_t D[QI::MAX_VDIM2D*2];
-            for (int i = 0; i < 2*VDIM; i++)
-            {
-               D[i] = 0.0;
-            }
-            for (int d = 0; d < ND; ++d)
-            {
-               const real_t wx = G(q,0,d);
-               const real_t wy = G(q,1,d);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  real_t s_e = s_E[c+d*VDIM];
-                  D[c+VDIM*0] += s_e * wx;
-                  D[c+VDIM*1] += s_e * wy;
-               }
-            }
-            if (eval_flags & QI::DERIVATIVES)
-            {
-               for (int c = 0; c < VDIM; c++)
-               {
-                  if (q_layout == QVectorLayout::byVDIM)
-                  {
-                     der(c,0,q,e) = D[c+VDIM*0];
-                     der(c,1,q,e) = D[c+VDIM*1];
-                  }
-                  if (q_layout == QVectorLayout::byNODES)
-                  {
-                     der(q,c,0,e) = D[c+VDIM*0];
-                     der(q,c,1,e) = D[c+VDIM*1];
-                  }
-               }
-            }
-            if (eval_flags & QI::PHYSICAL_DERIVATIVES)
-            {
-               real_t Jloc[4], Jinv[4];
-               Jloc[0] = J(q,0,0,e);
-               Jloc[1] = J(q,1,0,e);
-               Jloc[2] = J(q,0,1,e);
-               Jloc[3] = J(q,1,1,e);
-               kernels::CalcInverse<2>(Jloc, Jinv);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  const real_t u = D[c+VDIM*0];
-                  const real_t v = D[c+VDIM*1];
-                  const real_t JiU = Jinv[0]*u + Jinv[1]*v;
-                  const real_t JiV = Jinv[2]*u + Jinv[3]*v;
-                  if (q_layout == QVectorLayout::byVDIM)
-                  {
-                     der(c,0,q,e) = JiU;
-                     der(c,1,q,e) = JiV;
-                  }
-                  if (q_layout == QVectorLayout::byNODES)
-                  {
-                     der(q,c,0,e) = JiU;
-                     der(q,c,1,e) = JiV;
-                  }
-               }
-            }
-            if (eval_flags & QI::DETERMINANTS)
-            {
-               if (VDIM == 2)
-               {
-                  det(q, e) = kernels::Det<2>(D);
-               }
-               else
-               {
-                  DeviceTensor<2> j(D, 3, 2);
-                  const real_t dE = j(0,0)*j(0,0) + j(1,0)*j(1,0) + j(2,0)*j(2,0);
-                  const real_t dF = j(0,0)*j(0,1) + j(1,0)*j(1,1) + j(2,0)*j(2,1);
-                  const real_t dG = j(0,1)*j(0,1) + j(1,1)*j(1,1) + j(2,1)*j(2,1);
-                  det(q,e) = std::sqrt(dE*dG - dF*dF);
-               }
-            }
-         }
-      }
-   });
-}
-
-// Template compute kernel for 2D quadrature interpolation:
-// * non-tensor product version,
-// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
-// * assumes 'maps.mode == FULL'.
-template <const int T_VDIM, const int T_ND, const int T_NQ>
-static void Eval2D(const int NE, const int vdim, const QVectorLayout q_layout,
-                   const GeometricFactors *geom, const DofToQuad &maps,
-                   const Vector &e_vec, Vector &q_val, Vector &q_der,
-                   Vector &q_det, const int eval_flags)
-{
-   ImplEval2D<false, T_VDIM, T_ND, T_NQ>(NE, vdim, q_layout, nullptr, geom,
-                                         maps, e_vec, q_val, q_der, q_det,
-                                         eval_flags);
-}
-
-// Template compute kernel for 3D quadrature interpolation:
-// * non-tensor product version,
-// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
-// * assumes 'maps.mode == FULL'.
-template <bool Integral, const int T_VDIM, const int T_ND, const int T_NQ>
-static void ImplEval3D(const int NE, const int vdim,
-                       const QVectorLayout q_layout, const real_t *detJ_,
-                       const GeometricFactors *geom, const DofToQuad &maps,
-                       const Vector &e_vec, Vector &q_val, Vector &q_der,
-                       Vector &q_det, const int eval_flags)
-{
-   using QI = QuadratureInterpolator;
-
-   const int nd = maps.ndof;
-   const int nq = maps.nqpt;
-   const int ND = T_ND ? T_ND : nd;
-   const int NQ = T_NQ ? T_NQ : nq;
-   const int NMAX = NQ > ND ? NQ : ND;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-   MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
-   MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 3, "");
-   MFEM_VERIFY(ND <= QI::MAX_ND3D, "");
-   MFEM_VERIFY(NQ <= QI::MAX_NQ3D, "");
-   MFEM_VERIFY(VDIM == 3 || !(eval_flags & QI::DETERMINANTS), "");
-   if constexpr(Integral)
-   {
-      MFEM_VERIFY(!(eval_flags & (QI::DERIVATIVES | QI::PHYSICAL_DERIVATIVES |
-                                  QI::DETERMINANTS)),
-                  "Integral FE does not support computing derivatives");
-   }
-   const auto B = Reshape(maps.B.Read(), NQ, ND);
-   const auto G = Reshape(maps.G.Read(), NQ, 3, ND);
-   const auto J = Reshape(geom ? geom->J.Read() : nullptr, NQ, 3, 3, NE);
-   auto E_ = e_vec.Read();
-   auto val = q_layout == QVectorLayout::byNODES ?
-              Reshape(q_val.Write(), NQ, VDIM, NE):
-              Reshape(q_val.Write(), VDIM, NQ, NE);
-   auto der = q_layout == QVectorLayout::byNODES ?
-              Reshape(q_der.Write(), NQ, VDIM, 3, NE):
-              Reshape(q_der.Write(), VDIM, 3, NQ, NE);
-   auto det = Reshape(q_det.Write(), NQ, NE);
-   mfem::forall_2D(NE, NMAX, 1, [=] MFEM_HOST_DEVICE(int e)
-   {
-      const auto E = Reshape(E_, ND, VDIM, NE);
-      const auto detJ = Reshape(detJ_, NQ, NE);
-      const int ND = T_ND ? T_ND : nd;
-      const int NQ = T_NQ ? T_NQ : nq;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int max_ND = T_ND ? T_ND : QI::MAX_ND3D;
-      constexpr int max_VDIM = T_VDIM ? T_VDIM : QI::MAX_VDIM3D;
-      MFEM_SHARED real_t s_E[max_VDIM*max_ND];
-      MFEM_FOREACH_THREAD(d, x, ND)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            s_E[c + d * VDIM] = E(d, c, e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(q, x, NQ)
-      {
-         if (eval_flags & (QI::VALUES | QI::PHYSICAL_VALUES))
-         {
-            real_t ed[max_VDIM];
-            for (int c = 0; c < VDIM; c++)
-            {
-               ed[c] = 0.0;
-            }
-            for (int d = 0; d < ND; ++d)
-            {
-               const real_t b = B(q,d);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  ed[c] += b * s_E[c + d * VDIM];
-               }
-            }
-            for (int c = 0; c < VDIM; c++)
-            {
-               if constexpr (Integral)
-               {
-                  ed[c] /= detJ(q, e);
-               }
-               if (q_layout == QVectorLayout::byVDIM)
-               {
-                  val(c, q, e) = ed[c];
-               }
-               if (q_layout == QVectorLayout::byNODES)
-               {
-                  val(q, c, e) = ed[c];
-               }
-            }
-         }
-         if ((eval_flags & QI::DERIVATIVES) ||
-             (eval_flags & QI::PHYSICAL_DERIVATIVES) ||
-             (eval_flags & QI::DETERMINANTS))
-         {
-            // use MAX_VDIM3D to avoid "subscript out of range" warnings
-            real_t D[QI::MAX_VDIM3D*3];
-            for (int i = 0; i < 3*VDIM; i++)
-            {
-               D[i] = 0.0;
-            }
-            for (int d = 0; d < ND; ++d)
-            {
-               const real_t wx = G(q,0,d);
-               const real_t wy = G(q,1,d);
-               const real_t wz = G(q,2,d);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  real_t s_e = s_E[c+d*VDIM];
-                  D[c+VDIM*0] += s_e * wx;
-                  D[c+VDIM*1] += s_e * wy;
-                  D[c+VDIM*2] += s_e * wz;
-               }
-            }
-            if (eval_flags & QI::DERIVATIVES)
-            {
-               for (int c = 0; c < VDIM; c++)
-               {
-                  if (q_layout == QVectorLayout::byVDIM)
-                  {
-                     der(c,0,q,e) = D[c+VDIM*0];
-                     der(c,1,q,e) = D[c+VDIM*1];
-                     der(c,2,q,e) = D[c+VDIM*2];
-                  }
-                  if (q_layout == QVectorLayout::byNODES)
-                  {
-                     der(q,c,0,e) = D[c+VDIM*0];
-                     der(q,c,1,e) = D[c+VDIM*1];
-                     der(q,c,2,e) = D[c+VDIM*2];
-                  }
-               }
-            }
-            if (eval_flags & QI::PHYSICAL_DERIVATIVES)
-            {
-               real_t Jloc[9], Jinv[9];
-               for (int col = 0; col < 3; col++)
-               {
-                  for (int row = 0; row < 3; row++)
-                  {
-                     Jloc[row+3*col] = J(q,row,col,e);
-                  }
-               }
-               kernels::CalcInverse<3>(Jloc, Jinv);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  const real_t u = D[c+VDIM*0];
-                  const real_t v = D[c+VDIM*1];
-                  const real_t w = D[c+VDIM*2];
-                  const real_t JiU = Jinv[0]*u + Jinv[1]*v + Jinv[2]*w;
-                  const real_t JiV = Jinv[3]*u + Jinv[4]*v + Jinv[5]*w;
-                  const real_t JiW = Jinv[6]*u + Jinv[7]*v + Jinv[8]*w;
-                  if (q_layout == QVectorLayout::byVDIM)
-                  {
-                     der(c,0,q,e) = JiU;
-                     der(c,1,q,e) = JiV;
-                     der(c,2,q,e) = JiW;
-                  }
-                  if (q_layout == QVectorLayout::byNODES)
-                  {
-                     der(q,c,0,e) = JiU;
-                     der(q,c,1,e) = JiV;
-                     der(q,c,2,e) = JiW;
-                  }
-               }
-            }
-            if (VDIM == 3 && (eval_flags & QI::DETERMINANTS))
-            {
-               // The check (VDIM == 3) should eliminate this block when VDIM is
-               // known at compile time and (VDIM != 3).
-               det(q,e) = kernels::Det<3>(D);
-            }
-         }
-      }
-   });
-}
-
-// Template compute kernel for 3D quadrature interpolation:
-// * non-tensor product version,
-// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
-// * assumes 'maps.mode == FULL'.
-template <const int T_VDIM, const int T_ND, const int T_NQ>
-static void Eval3D(const int NE, const int vdim, const QVectorLayout q_layout,
-                   const GeometricFactors *geom, const DofToQuad &maps,
-                   const Vector &e_vec, Vector &q_val, Vector &q_der,
-                   Vector &q_det, const int eval_flags)
-{
-   ImplEval3D<false, T_VDIM, T_ND, T_NQ>(NE, vdim, q_layout, nullptr, geom,
-                                         maps, e_vec, q_val, q_der, q_det,
-                                         eval_flags);
-}
-
 } // namespace quadrature_interpolator
 
 } // namespace internal
 
 /// @cond Suppress_Doxygen_warnings
 
-template <int DIM, QVectorLayout Q_LAYOUT, int VDIM, int D1D, int Q1D, int NBZ>
-QuadratureInterpolator::IntTensorEvalKernelType
-QuadratureInterpolator::IntTensorEvalKernels::Kernel()
-{
-   if constexpr (DIM == 1) { return internal::quadrature_interpolator::ImplValues1D<Q_LAYOUT, true>; }
-   else if constexpr (DIM == 2) { return internal::quadrature_interpolator::ImplValues2D<Q_LAYOUT, true, VDIM, D1D, Q1D, NBZ>; }
-   else if constexpr (DIM == 3) { return internal::quadrature_interpolator::ImplValues3D<Q_LAYOUT, true, VDIM, D1D, Q1D>; }
-   MFEM_ABORT("");
-}
-
-template <int DIM, QVectorLayout Q_LAYOUT, int VDIM, int D1D, int Q1D, int NBZ>
+template<int DIM, QVectorLayout Q_LAYOUT,
+         int VDIM, int D1D, int Q1D, int NBZ>
 QuadratureInterpolator::TensorEvalKernelType
 QuadratureInterpolator::TensorEvalKernels::Kernel()
 {
@@ -679,28 +209,6 @@ QuadratureInterpolator::TensorEvalKernels::Kernel()
    MFEM_ABORT("");
 }
 
-template <int DIM, int VDIM, int ND, int NQ>
-QuadratureInterpolator::IntEvalKernelType
-QuadratureInterpolator::IntEvalKernels::Kernel()
-{
-   using namespace internal::quadrature_interpolator;
-   if constexpr (DIM == 1) { return ImplEval1D<true>; }
-   else if constexpr (DIM == 2) { return ImplEval2D<true,VDIM,ND,NQ>; }
-   else if constexpr (DIM == 3) { return ImplEval3D<true,VDIM,ND,NQ>; }
-   MFEM_ABORT("");
-}
-
-template <int DIM, int VDIM, int ND, int NQ>
-QuadratureInterpolator::EvalKernelType
-QuadratureInterpolator::EvalKernels::Kernel()
-{
-   using namespace internal::quadrature_interpolator;
-   if constexpr (DIM == 1) { return Eval1D; }
-   else if constexpr (DIM == 2) { return Eval2D<VDIM,ND,NQ>; }
-   else if constexpr (DIM == 3) { return Eval3D<VDIM,ND,NQ>; }
-   MFEM_ABORT("");
-}
-
 /// @endcond
 
 } // namespace mfem

fem/qinterp/eval_by_nodes.cpp

@@ -21,105 +21,63 @@ namespace quadrature_interpolator
 
 void InitEvalByNodesKernels()
 {
+   using k = QuadratureInterpolator::TensorEvalKernels;
+
    // 2D
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 3, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 2, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 3, 2, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 3, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 3, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 4, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 4, 4, 1>();
+   k::Specialization<2,QVectorLayout::byNODES,1,3,3>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,1,2,4>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,1,3,2>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,1,3,4>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,1,3,6>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,1,4,3>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,1,4,4>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 2, 2, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 2, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 2, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 2, 5, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 2, 6, 1>();
+   k::Specialization<2,QVectorLayout::byNODES,2,2,2>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,2,3>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,2,4>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,2,5>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,2,6>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 3, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 3, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 3, 6, 1>();
+   k::Specialization<2,QVectorLayout::byNODES,2,3,3>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,3,4>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,3,6>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 4, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 4, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 4, 5, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 4, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 4, 7, 1>();
+   k::Specialization<2,QVectorLayout::byNODES,2,4,3>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,4,4>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,4,5>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,4,6>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,4,7>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 5, 6, 1>();
+   k::Specialization<2,QVectorLayout::byNODES,2,5,6>::Opt<1>::Add();
 
    // 3D
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 2, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 3, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 3, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 3, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 4, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 4, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 4, 8, 1>();
+   k::Specialization<3,QVectorLayout::byNODES,1,2,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,1,3,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,1,3,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,1,3,6>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,1,4,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,1,4,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,1,4,8>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 2, 2, 2, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 2, 2, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 2, 3, 4, 1>();
+   k::Specialization<3,QVectorLayout::byNODES,2,2,2>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,2,2,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,2,3,4>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 2, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 2, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 2, 5, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 2, 6, 1>();
+   k::Specialization<3,QVectorLayout::byNODES,3,2,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,2,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,2,5>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,2,6>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 3, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 3, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 3, 5, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 3, 6, 1>();
+   k::Specialization<3,QVectorLayout::byNODES,3,3,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,3,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,3,5>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,3,6>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 4, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 4, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 4, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 4, 7, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 4, 8, 1>();
+   k::Specialization<3,QVectorLayout::byNODES,3,4,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,4,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,4,6>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,4,7>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,4,8>::Opt<1>::Add();
 }
 
 } // namespace quadrature_interpolator

fem/qinterp/eval_by_vdim.cpp

@@ -21,59 +21,36 @@ namespace quadrature_interpolator
 
 void InitEvalByVDimKernels()
 {
+   using k = QuadratureInterpolator::TensorEvalKernels;
    // 2D
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 1, 2, 4, 8>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 1, 3, 6, 4>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 1, 4, 8, 2>();
+   k::Specialization<2,QVectorLayout::byVDIM,1,2,4>::Opt<8>::Add();
+   k::Specialization<2,QVectorLayout::byVDIM,1,3,6>::Opt<4>::Add();
+   k::Specialization<2,QVectorLayout::byVDIM,1,4,8>::Opt<2>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 2, 2, 4, 8>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 2, 3, 4, 8>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 2, 3, 6, 4>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 2, 4, 6, 2>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 2, 4, 8, 2>();
+   k::Specialization<2,QVectorLayout::byVDIM,2,2,4>::Opt<8>::Add();
+   k::Specialization<2,QVectorLayout::byVDIM,2,3,4>::Opt<8>::Add();
+   k::Specialization<2,QVectorLayout::byVDIM,2,3,6>::Opt<4>::Add();
+   k::Specialization<2,QVectorLayout::byVDIM,2,4,6>::Opt<2>::Add();
+   k::Specialization<2,QVectorLayout::byVDIM,2,4,8>::Opt<2>::Add();
    // 3D
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 1, 2, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 1, 3, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 1, 4, 8, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 2, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 3, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 4, 8, 1>();
+   k::Specialization<3,QVectorLayout::byVDIM,1,2,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,1,3,6>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,1,4,8>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,2,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,3,6>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,4,8>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 2, 2, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 3, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 4, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 5, 5, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 6, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 7, 7, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 8, 8, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 9, 9, 1>();
+   k::Specialization<3,QVectorLayout::byVDIM,3,2,2>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,3,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,4,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,5,5>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,6,6>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,7,7>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,8,8>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,9,9>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 4, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 3, 4, 1>();
+   k::Specialization<3,QVectorLayout::byVDIM,3,4,6>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,3,4>::Opt<1>::Add();
 }
 
 } // namespace quadrature_interpolator

fem/qinterp/grad.hpp

@@ -268,9 +268,8 @@ static void Derivatives3D(const int NE,
       DeviceMatrix B(BG[0], D1D, Q1D);
       DeviceMatrix G(BG[1], D1D, Q1D);
 
-      constexpr int MDQ = MD1 > MQ1 ? MD1 : MQ1;
-      MFEM_SHARED real_t sm0[3][MD1*MD1*MDQ];
-      MFEM_SHARED real_t sm1[3][MD1*MQ1*MQ1];
+      MFEM_SHARED real_t sm0[3][MQ1*MQ1*MQ1];
+      MFEM_SHARED real_t sm1[3][MQ1*MQ1*MQ1];
       DeviceTensor<3> X(sm0[2], D1D, D1D, D1D);
       DeviceTensor<3> DDQ0(sm0[0], D1D, D1D, Q1D);
       DeviceTensor<3> DDQ1(sm0[1], D1D, D1D, Q1D);

fem/qinterp/grad_by_nodes.cpp

@@ -22,71 +22,74 @@ namespace quadrature_interpolator
 template <bool P>
 void InitGradByNodesKernels()
 {
+   using k = QuadratureInterpolator::GradKernels;
    constexpr auto L = QVectorLayout::byNODES;
    // 2D
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,3,3,16>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,3,4,16>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,4,3,16>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,4,4,16>();
+   k::Specialization<2,L,P,1,3,3>::template Opt<16>::Add();
+   k::Specialization<2,L,P,1,3,4>::template Opt<16>::Add();
+   k::Specialization<2,L,P,1,4,3>::template Opt<16>::Add();
+   k::Specialization<2,L,P,1,4,4>::template Opt<16>::Add();
 
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,2,2,16>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,2,3,8>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,2,4,4>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,2,5,4>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,2,6,2>();
+   k::Specialization<2,L,P,2,2,2>::template Opt<16>::Add();
+   k::Specialization<2,L,P,2,2,3>::template Opt<8>::Add();
+   k::Specialization<2,L,P,2,2,4>::template Opt<4>::Add();
+   k::Specialization<2,L,P,2,2,5>::template Opt<4>::Add();
+   k::Specialization<2,L,P,2,2,6>::template Opt<2>::Add();
 
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,3,3,2>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,3,4,4>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,3,4>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,3,6,2>();
+   k::Specialization<2,L,P,2,3,3>::template Opt<2>::Add();
+   k::Specialization<2,L,P,2,3,4>::template Opt<4>::Add();
+   k::Specialization<2,L,P,2,4,3>::template Opt<4>::Add();
+   k::Specialization<2,L,P,2,3,6>::template Opt<2>::Add();
 
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,4,2>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,5,2>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,6,2>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,7,2>();
+   k::Specialization<2,L,P,2,4,4>::template Opt<2>::Add();
+   k::Specialization<2,L,P,2,4,5>::template Opt<2>::Add();
+   k::Specialization<2,L,P,2,4,6>::template Opt<2>::Add();
+   k::Specialization<2,L,P,2,4,7>::template Opt<2>::Add();
 
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,5,6,2>();
+   k::Specialization<2,L,P,2,5,6>::template Opt<2>::Add();
    // 3D
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,2,4>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,3,3>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,3,4>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,3,6>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,4,4>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,4,8>();
+   k::Specialization<3,L,P,1,2,4>::Add();
+   k::Specialization<3,L,P,1,3,3>::Add();
+   k::Specialization<3,L,P,1,3,4>::Add();
+   k::Specialization<3,L,P,1,3,6>::Add();
+   k::Specialization<3,L,P,1,4,4>::Add();
+   k::Specialization<3,L,P,1,4,8>::Add();
 
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,2,3>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,2,4>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,2,5>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,2,6>();
+   k::Specialization<3,L,P,3,2,3>::Add();
+   k::Specialization<3,L,P,3,2,4>::Add();
+   k::Specialization<3,L,P,3,2,5>::Add();
+   k::Specialization<3,L,P,3,2,6>::Add();
 
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,3,3>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,3,4>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,3,5>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,3,6>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,4,4>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,4,6>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,4,7>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,4,8>();
+   k::Specialization<3,L,P,3,3,3>::Add();
+   k::Specialization<3,L,P,3,3,4>::Add();
+   k::Specialization<3,L,P,3,3,5>::Add();
+   k::Specialization<3,L,P,3,3,6>::Add();
+   k::Specialization<3,L,P,3,4,4>::Add();
+   k::Specialization<3,L,P,3,4,6>::Add();
+   k::Specialization<3,L,P,3,4,7>::Add();
+   k::Specialization<3,L,P,3,4,8>::Add();
+
+   using k2 = QuadratureInterpolator::CollocatedGradKernels;
 
    // 2D
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,2,16>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,3,16>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,4,16>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,2,16>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,3,4>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,4,2>();
+   k2::Specialization<2,L,P,1,2>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,1,3>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,1,4>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,2,2>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,2,3>::template Opt<4>::Add();
+   k2::Specialization<2,L,P,2,4>::template Opt<2>::Add();
 
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,2>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,3>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,4>();
+   k2::Specialization<3,L,P,1,2>::Add();
+   k2::Specialization<3,L,P,1,3>::Add();
+   k2::Specialization<3,L,P,1,4>::Add();
 
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,2>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,3>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,4>();
+   k2::Specialization<3,L,P,2,2>::Add();
+   k2::Specialization<3,L,P,2,3>::Add();
+   k2::Specialization<3,L,P,2,4>::Add();
 
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,2>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,3>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,4>();
+   k2::Specialization<3,L,P,3,2>::Add();
+   k2::Specialization<3,L,P,3,3>::Add();
+   k2::Specialization<3,L,P,3,4>::Add();
 }
 
 template void InitGradByNodesKernels<true>();

fem/qinterp/grad_by_vdim.cpp

@@ -22,45 +22,47 @@ namespace quadrature_interpolator
 template <bool P>
 void InitGradByVDimKernels()
 {
+   using k = QuadratureInterpolator::GradKernels;
    constexpr auto L = QVectorLayout::byVDIM;
    // 2D
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,3,4,8>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,4,6,4>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,5,8,2>();
+   k::Specialization<2,L,P,1,3,4>::template Opt<8>::Add();
+   k::Specialization<2,L,P,1,4,6>::template Opt<4>::Add();
+   k::Specialization<2,L,P,1,5,8>::template Opt<2>::Add();
 
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,3,3,8>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,3,4,8>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,6,4>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,5,8,2>();
+   k::Specialization<2,L,P,2,3,3>::template Opt<8>::Add();
+   k::Specialization<2,L,P,2,3,4>::template Opt<8>::Add();
+   k::Specialization<2,L,P,2,4,6>::template Opt<4>::Add();
+   k::Specialization<2,L,P,2,5,8>::template Opt<2>::Add();
    // 3D
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,3,4>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,4,6>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,5,8>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,3,4>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,4,6>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,5,8>();
+   k::Specialization<3,L,P,1,3,4>::Add();
+   k::Specialization<3,L,P,1,4,6>::Add();
+   k::Specialization<3,L,P,1,5,8>::Add();
+   k::Specialization<3,L,P,3,3,4>::Add();
+   k::Specialization<3,L,P,3,4,6>::Add();
+   k::Specialization<3,L,P,3,5,8>::Add();
 
+   using k2 = QuadratureInterpolator::CollocatedGradKernels;
    // 2D
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,2,16>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,3,16>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,4,16>();
+   k2::Specialization<2,L,P,1,2>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,1,3>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,1,4>::template Opt<16>::Add();
 
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,2,16>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,3,4>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,4,2>();
+   k2::Specialization<2,L,P,2,2>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,2,3>::template Opt<4>::Add();
+   k2::Specialization<2,L,P,2,4>::template Opt<2>::Add();
 
    // 3D
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,2>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,3>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,4>();
+   k2::Specialization<3,L,P,1,2>::Add();
+   k2::Specialization<3,L,P,1,3>::Add();
+   k2::Specialization<3,L,P,1,4>::Add();
 
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,2>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,3>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,4>();
+   k2::Specialization<3,L,P,2,2>::Add();
+   k2::Specialization<3,L,P,2,3>::Add();
+   k2::Specialization<3,L,P,2,4>::Add();
 
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,2>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,3>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,4>();
+   k2::Specialization<3,L,P,3,2>::Add();
+   k2::Specialization<3,L,P,3,3>::Add();
+   k2::Specialization<3,L,P,3,4>::Add();
 }
 
 template void InitGradByVDimKernels<true>();

fem/quadinterpolator.cpp

@@ -69,9 +69,8 @@ QuadratureInterpolator::QuadratureInterpolator(const FiniteElementSpace &fes,
 
    d_buffer.UseDevice(true);
    if (fespace->GetNE() == 0) { return; }
-   MFEM_VERIFY(
-      SupportsFESpace(fes),
-      "Only elements with MapType VALUE, INTEGRAL, or H_DIV are supported!");
+   MFEM_VERIFY(SupportsFESpace(fes),
+               "Only elements with MapType VALUE and H_DIV are supported!");
 }
 
 QuadratureInterpolator::QuadratureInterpolator(const FiniteElementSpace &fes,
@@ -85,9 +84,8 @@ QuadratureInterpolator::QuadratureInterpolator(const FiniteElementSpace &fes,
 {
    d_buffer.UseDevice(true);
    if (fespace->GetNE() == 0) { return; }
-   MFEM_VERIFY(
-      SupportsFESpace(fes),
-      "Only elements with MapType VALUE, INTEGRAL, or H_DIV are supported!");
+   MFEM_VERIFY(SupportsFESpace(fes),
+               "Only elements with MapType VALUE and H_DIV are supported!");
 }
 
 bool QuadratureInterpolator::SupportsFESpace(const FiniteElementSpace &fespace)
@@ -95,9 +93,9 @@ bool QuadratureInterpolator::SupportsFESpace(const FiniteElementSpace &fespace)
    const FiniteElement *fe = fespace.GetTypicalFE();
    const Mesh &mesh = *fespace.GetMesh();
    return (fe->GetMapType() == FiniteElement::MapType::VALUE ||
-           fe->GetMapType() == FiniteElement::MapType::INTEGRAL ||
-           fe->GetMapType() == FiniteElement::MapType::H_DIV) &&
-          (!fespace.IsVariableOrder()) && (!mesh.IsMixedMesh());
+           fe->GetMapType() == FiniteElement::MapType::H_DIV)
+          && (!fespace.IsVariableOrder())
+          && (!mesh.IsMixedMesh());
 }
 
 namespace internal
@@ -110,11 +108,16 @@ namespace quadrature_interpolator
 // * non-tensor product version,
 // * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
 // * assumes 'maps.mode == FULL'.
-template <bool Integral>
-void ImplEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
-                const real_t *detJ_, const GeometricFactors *geom,
-                const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
-                Vector &q_der, Vector &q_det, const int eval_flags)
+static void Eval1D(const int NE,
+                   const int vdim,
+                   const QVectorLayout q_layout,
+                   const GeometricFactors *geom,
+                   const DofToQuad &maps,
+                   const Vector &e_vec,
+                   Vector &q_val,
+                   Vector &q_der,
+                   Vector &q_det,
+                   const int eval_flags)
 {
    using QI = QuadratureInterpolator;
 
@@ -123,16 +126,13 @@ void ImplEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
    MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
    MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 1, "");
    MFEM_VERIFY(vdim == 1 || !(eval_flags & QI::DETERMINANTS), "");
-   if constexpr(Integral)
-   {
-      MFEM_VERIFY(!(eval_flags & (QI::DERIVATIVES | QI::PHYSICAL_DERIVATIVES |
-                                  QI::DETERMINANTS)),
-                  "Integral FE does not support computing derivatives");
-   }
-   const auto B_ = maps.B.Read();
-   const auto G_ = maps.G.Read();
+   MFEM_VERIFY(bool(geom) == bool(eval_flags & QI::PHYSICAL_DERIVATIVES),
+               "'geom' must be given (non-null) only when evaluating physical"
+               " derivatives");
+   const auto B = Reshape(maps.B.Read(), nq, nd);
+   const auto G = Reshape(maps.G.Read(), nq, nd);
    const auto J = Reshape(geom ? geom->J.Read() : nullptr, nq, NE);
-   const auto E_ = e_vec.Read();
+   const auto E = Reshape(e_vec.Read(), nd, vdim, NE);
    auto val = q_layout == QVectorLayout::byNODES ?
               Reshape(q_val.Write(), nq, vdim, NE):
               Reshape(q_val.Write(), vdim, nq, NE);
@@ -140,12 +140,8 @@ void ImplEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
               Reshape(q_der.Write(), nq, vdim, NE):
               Reshape(q_der.Write(), vdim, nq, NE);
    auto det = Reshape(q_det.Write(), nq, NE);
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      const auto B = Reshape(B_, nq, nd);
-      const auto G = Reshape(G_, nq, nd);
-      const auto E = Reshape(E_, nd, vdim, NE);
-      const auto detJ = Reshape(detJ_, nq, NE);
       for (int q = 0; q < nq; ++q)
       {
          if (eval_flags & (QI::VALUES | QI::PHYSICAL_VALUES))
@@ -155,20 +151,10 @@ void ImplEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
                real_t q_val = 0.0;
                for (int d = 0; d < nd; ++d)
                {
-                  q_val += B(q, d) * E(d, c, e);
-               }
-               if constexpr (Integral)
-               {
-                  q_val /= detJ(q, e);
-               }
-               if (q_layout == QVectorLayout::byVDIM)
-               {
-                  val(c, q, e) = q_val;
-               }
-               if (q_layout == QVectorLayout::byNODES)
-               {
-                  val(q, c, e) = q_val;
+                  q_val += B(q,d)*E(d,c,e);
                }
+               if (q_layout == QVectorLayout::byVDIM)  { val(c,q,e) = q_val; }
+               if (q_layout == QVectorLayout::byNODES) { val(q,c,e) = q_val; }
             }
          }
          if ((eval_flags & QI::DERIVATIVES) ||
@@ -180,7 +166,7 @@ void ImplEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
                real_t q_d = 0.0;
                for (int d = 0; d < nd; ++d)
                {
-                  q_d += G(q, d) * E(d, c, e);
+                  q_d += G(q,d)*E(d,c,e);
                }
                if (eval_flags & QI::PHYSICAL_DERIVATIVES)
                {
@@ -188,14 +174,8 @@ void ImplEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
                }
                if (eval_flags & QI::DERIVATIVES || eval_flags & QI::PHYSICAL_DERIVATIVES)
                {
-                  if (q_layout == QVectorLayout::byVDIM)
-                  {
-                     der(c, q, e) = q_d;
-                  }
-                  if (q_layout == QVectorLayout::byNODES)
-                  {
-                     der(q, c, e) = q_d;
-                  }
+                  if (q_layout == QVectorLayout::byVDIM) { der(c,q,e) = q_d; }
+                  if (q_layout == QVectorLayout::byNODES) { der(q,c,e) = q_d; }
                }
                if (vdim == 1 && (eval_flags & QI::DETERMINANTS))
                {
@@ -207,17 +187,317 @@ void ImplEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
    });
 }
 
-template void
-ImplEval1D<true>(const int NE, const int vdim, const QVectorLayout q_layout,
-                 const real_t *detJ, const GeometricFactors *geom,
-                 const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
-                 Vector &q_der, Vector &q_det, const int eval_flags);
+// Template compute kernel for 2D quadrature interpolation:
+// * non-tensor product version,
+// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
+// * assumes 'maps.mode == FULL'.
+template<const int T_VDIM, const int T_ND, const int T_NQ>
+static void Eval2D(const int NE,
+                   const int vdim,
+                   const QVectorLayout q_layout,
+                   const GeometricFactors *geom,
+                   const DofToQuad &maps,
+                   const Vector &e_vec,
+                   Vector &q_val,
+                   Vector &q_der,
+                   Vector &q_det,
+                   const int eval_flags)
+{
+   using QI = QuadratureInterpolator;
 
-template void
-ImplEval1D<false>(const int NE, const int vdim, const QVectorLayout q_layout,
-                  const real_t *detJ, const GeometricFactors *geom,
-                  const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
-                  Vector &q_der, Vector &q_det, const int eval_flags);
+   const int nd = maps.ndof;
+   const int nq = maps.nqpt;
+   const int ND = T_ND ? T_ND : nd;
+   const int NQ = T_NQ ? T_NQ : nq;
+   const int NMAX = NQ > ND ? NQ : ND;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+   MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
+   MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 2, "");
+   MFEM_VERIFY(ND <= QI::MAX_ND2D, "");
+   MFEM_VERIFY(NQ <= QI::MAX_NQ2D, "");
+   MFEM_VERIFY(bool(geom) == bool(eval_flags & QI::PHYSICAL_DERIVATIVES),
+               "'geom' must be given (non-null) only when evaluating physical"
+               " derivatives");
+   const auto B = Reshape(maps.B.Read(), NQ, ND);
+   const auto G = Reshape(maps.G.Read(), NQ, 2, ND);
+   const auto J = Reshape(geom ? geom->J.Read() : nullptr, NQ, 2, 2, NE);
+   const auto E = Reshape(e_vec.Read(), ND, VDIM, NE);
+   auto val = q_layout == QVectorLayout::byNODES ?
+              Reshape(q_val.Write(), NQ, VDIM, NE):
+              Reshape(q_val.Write(), VDIM, NQ, NE);
+   auto der = q_layout == QVectorLayout::byNODES ?
+              Reshape(q_der.Write(), NQ, VDIM, 2, NE):
+              Reshape(q_der.Write(), VDIM, 2, NQ, NE);
+   auto det = Reshape(q_det.Write(), NQ, NE);
+   mfem::forall_2D(NE, NMAX, 1, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int ND = T_ND ? T_ND : nd;
+      const int NQ = T_NQ ? T_NQ : nq;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int max_ND = T_ND ? T_ND : QI::MAX_ND2D;
+      constexpr int max_VDIM = T_VDIM ? T_VDIM : QI::MAX_VDIM2D;
+      MFEM_SHARED real_t s_E[max_VDIM*max_ND];
+      MFEM_FOREACH_THREAD(d, x, ND)
+      {
+         for (int c = 0; c < VDIM; c++)
+         {
+            s_E[c+d*VDIM] = E(d,c,e);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(q, x, NQ)
+      {
+         if (eval_flags & (QI::VALUES | QI::PHYSICAL_VALUES))
+         {
+            real_t ed[max_VDIM];
+            for (int c = 0; c < VDIM; c++) { ed[c] = 0.0; }
+            for (int d = 0; d < ND; ++d)
+            {
+               const real_t b = B(q,d);
+               for (int c = 0; c < VDIM; c++) { ed[c] += b*s_E[c+d*VDIM]; }
+            }
+            for (int c = 0; c < VDIM; c++)
+            {
+               if (q_layout == QVectorLayout::byVDIM)  { val(c,q,e) = ed[c]; }
+               if (q_layout == QVectorLayout::byNODES) { val(q,c,e) = ed[c]; }
+            }
+         }
+         if ((eval_flags & QI::DERIVATIVES) ||
+             (eval_flags & QI::PHYSICAL_DERIVATIVES) ||
+             (eval_flags & QI::DETERMINANTS))
+         {
+            // use MAX_VDIM2D to avoid "subscript out of range" warnings
+            real_t D[QI::MAX_VDIM2D*2];
+            for (int i = 0; i < 2*VDIM; i++) { D[i] = 0.0; }
+            for (int d = 0; d < ND; ++d)
+            {
+               const real_t wx = G(q,0,d);
+               const real_t wy = G(q,1,d);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  real_t s_e = s_E[c+d*VDIM];
+                  D[c+VDIM*0] += s_e * wx;
+                  D[c+VDIM*1] += s_e * wy;
+               }
+            }
+            if (eval_flags & QI::DERIVATIVES)
+            {
+               for (int c = 0; c < VDIM; c++)
+               {
+                  if (q_layout == QVectorLayout::byVDIM)
+                  {
+                     der(c,0,q,e) = D[c+VDIM*0];
+                     der(c,1,q,e) = D[c+VDIM*1];
+                  }
+                  if (q_layout == QVectorLayout::byNODES)
+                  {
+                     der(q,c,0,e) = D[c+VDIM*0];
+                     der(q,c,1,e) = D[c+VDIM*1];
+                  }
+               }
+            }
+            if (eval_flags & QI::PHYSICAL_DERIVATIVES)
+            {
+               real_t Jloc[4], Jinv[4];
+               Jloc[0] = J(q,0,0,e);
+               Jloc[1] = J(q,1,0,e);
+               Jloc[2] = J(q,0,1,e);
+               Jloc[3] = J(q,1,1,e);
+               kernels::CalcInverse<2>(Jloc, Jinv);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  const real_t u = D[c+VDIM*0];
+                  const real_t v = D[c+VDIM*1];
+                  const real_t JiU = Jinv[0]*u + Jinv[1]*v;
+                  const real_t JiV = Jinv[2]*u + Jinv[3]*v;
+                  if (q_layout == QVectorLayout::byVDIM)
+                  {
+                     der(c,0,q,e) = JiU;
+                     der(c,1,q,e) = JiV;
+                  }
+                  if (q_layout == QVectorLayout::byNODES)
+                  {
+                     der(q,c,0,e) = JiU;
+                     der(q,c,1,e) = JiV;
+                  }
+               }
+            }
+            if (eval_flags & QI::DETERMINANTS)
+            {
+               if (VDIM == 2) { det(q,e) = kernels::Det<2>(D); }
+               else
+               {
+                  DeviceTensor<2> j(D, 3, 2);
+                  const double E = j(0,0)*j(0,0) + j(1,0)*j(1,0) + j(2,0)*j(2,0);
+                  const double F = j(0,0)*j(0,1) + j(1,0)*j(1,1) + j(2,0)*j(2,1);
+                  const double G = j(0,1)*j(0,1) + j(1,1)*j(1,1) + j(2,1)*j(2,1);
+                  det(q,e) = std::sqrt(E*G - F*F);
+               }
+            }
+         }
+      }
+   });
+}
+
+// Template compute kernel for 3D quadrature interpolation:
+// * non-tensor product version,
+// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
+// * assumes 'maps.mode == FULL'.
+template<const int T_VDIM, const int T_ND, const int T_NQ>
+static void Eval3D(const int NE,
+                   const int vdim,
+                   const QVectorLayout q_layout,
+                   const GeometricFactors *geom,
+                   const DofToQuad &maps,
+                   const Vector &e_vec,
+                   Vector &q_val,
+                   Vector &q_der,
+                   Vector &q_det,
+                   const int eval_flags)
+{
+   using QI = QuadratureInterpolator;
+
+   const int nd = maps.ndof;
+   const int nq = maps.nqpt;
+   const int ND = T_ND ? T_ND : nd;
+   const int NQ = T_NQ ? T_NQ : nq;
+   const int NMAX = NQ > ND ? NQ : ND;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+   MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
+   MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 3, "");
+   MFEM_VERIFY(ND <= QI::MAX_ND3D, "");
+   MFEM_VERIFY(NQ <= QI::MAX_NQ3D, "");
+   MFEM_VERIFY(VDIM == 3 || !(eval_flags & QI::DETERMINANTS), "");
+   MFEM_VERIFY(bool(geom) == bool(eval_flags & QI::PHYSICAL_DERIVATIVES),
+               "'geom' must be given (non-null) only when evaluating physical"
+               " derivatives");
+   const auto B = Reshape(maps.B.Read(), NQ, ND);
+   const auto G = Reshape(maps.G.Read(), NQ, 3, ND);
+   const auto J = Reshape(geom ? geom->J.Read() : nullptr, NQ, 3, 3, NE);
+   const auto E = Reshape(e_vec.Read(), ND, VDIM, NE);
+   auto val = q_layout == QVectorLayout::byNODES ?
+              Reshape(q_val.Write(), NQ, VDIM, NE):
+              Reshape(q_val.Write(), VDIM, NQ, NE);
+   auto der = q_layout == QVectorLayout::byNODES ?
+              Reshape(q_der.Write(), NQ, VDIM, 3, NE):
+              Reshape(q_der.Write(), VDIM, 3, NQ, NE);
+   auto det = Reshape(q_det.Write(), NQ, NE);
+   mfem::forall_2D(NE, NMAX, 1, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int ND = T_ND ? T_ND : nd;
+      const int NQ = T_NQ ? T_NQ : nq;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int max_ND = T_ND ? T_ND : QI::MAX_ND3D;
+      constexpr int max_VDIM = T_VDIM ? T_VDIM : QI::MAX_VDIM3D;
+      MFEM_SHARED real_t s_E[max_VDIM*max_ND];
+      MFEM_FOREACH_THREAD(d, x, ND)
+      {
+         for (int c = 0; c < VDIM; c++)
+         {
+            s_E[c+d*VDIM] = E(d,c,e);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(q, x, NQ)
+      {
+         if (eval_flags & (QI::VALUES | QI::PHYSICAL_VALUES))
+         {
+            real_t ed[max_VDIM];
+            for (int c = 0; c < VDIM; c++) { ed[c] = 0.0; }
+            for (int d = 0; d < ND; ++d)
+            {
+               const real_t b = B(q,d);
+               for (int c = 0; c < VDIM; c++) { ed[c] += b*s_E[c+d*VDIM]; }
+            }
+            for (int c = 0; c < VDIM; c++)
+            {
+               if (q_layout == QVectorLayout::byVDIM)  { val(c,q,e) = ed[c]; }
+               if (q_layout == QVectorLayout::byNODES) { val(q,c,e) = ed[c]; }
+            }
+         }
+         if ((eval_flags & QI::DERIVATIVES) ||
+             (eval_flags & QI::PHYSICAL_DERIVATIVES) ||
+             (eval_flags & QI::DETERMINANTS))
+         {
+            // use MAX_VDIM3D to avoid "subscript out of range" warnings
+            real_t D[QI::MAX_VDIM3D*3];
+            for (int i = 0; i < 3*VDIM; i++) { D[i] = 0.0; }
+            for (int d = 0; d < ND; ++d)
+            {
+               const real_t wx = G(q,0,d);
+               const real_t wy = G(q,1,d);
+               const real_t wz = G(q,2,d);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  real_t s_e = s_E[c+d*VDIM];
+                  D[c+VDIM*0] += s_e * wx;
+                  D[c+VDIM*1] += s_e * wy;
+                  D[c+VDIM*2] += s_e * wz;
+               }
+            }
+            if (eval_flags & QI::DERIVATIVES)
+            {
+               for (int c = 0; c < VDIM; c++)
+               {
+                  if (q_layout == QVectorLayout::byVDIM)
+                  {
+                     der(c,0,q,e) = D[c+VDIM*0];
+                     der(c,1,q,e) = D[c+VDIM*1];
+                     der(c,2,q,e) = D[c+VDIM*2];
+                  }
+                  if (q_layout == QVectorLayout::byNODES)
+                  {
+                     der(q,c,0,e) = D[c+VDIM*0];
+                     der(q,c,1,e) = D[c+VDIM*1];
+                     der(q,c,2,e) = D[c+VDIM*2];
+                  }
+               }
+            }
+            if (eval_flags & QI::PHYSICAL_DERIVATIVES)
+            {
+               real_t Jloc[9], Jinv[9];
+               for (int col = 0; col < 3; col++)
+               {
+                  for (int row = 0; row < 3; row++)
+                  {
+                     Jloc[row+3*col] = J(q,row,col,e);
+                  }
+               }
+               kernels::CalcInverse<3>(Jloc, Jinv);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  const real_t u = D[c+VDIM*0];
+                  const real_t v = D[c+VDIM*1];
+                  const real_t w = D[c+VDIM*2];
+                  const real_t JiU = Jinv[0]*u + Jinv[1]*v + Jinv[2]*w;
+                  const real_t JiV = Jinv[3]*u + Jinv[4]*v + Jinv[5]*w;
+                  const real_t JiW = Jinv[6]*u + Jinv[7]*v + Jinv[8]*w;
+                  if (q_layout == QVectorLayout::byVDIM)
+                  {
+                     der(c,0,q,e) = JiU;
+                     der(c,1,q,e) = JiV;
+                     der(c,2,q,e) = JiW;
+                  }
+                  if (q_layout == QVectorLayout::byNODES)
+                  {
+                     der(q,c,0,e) = JiU;
+                     der(q,c,1,e) = JiV;
+                     der(q,c,2,e) = JiW;
+                  }
+               }
+            }
+            if (VDIM == 3 && (eval_flags & QI::DETERMINANTS))
+            {
+               // The check (VDIM == 3) should eliminate this block when VDIM is
+               // known at compile time and (VDIM != 3).
+               det(q,e) = kernels::Det<3>(D);
+            }
+         }
+      }
+   });
+}
 
 } // namespace quadrature_interpolator
 
@@ -255,20 +535,10 @@ void QuadratureInterpolator::Mult(const Vector &e_vec,
    const int nd = maps.ndof;
    const int nq = maps.nqpt;
    const GeometricFactors *geom = nullptr;
+   if (eval_flags & PHYSICAL_DERIVATIVES)
    {
-      int jac_factors = 0;
-      if (eval_flags & PHYSICAL_DERIVATIVES)
-      {
-         jac_factors = GeometricFactors::JACOBIANS;
-      }
-      if (fe->GetMapType() == FiniteElement::MapType::INTEGRAL)
-      {
-         jac_factors |= GeometricFactors::DETERMINANTS;
-      }
-      if (jac_factors)
-      {
-         geom = fespace->GetMesh()->GetGeometricFactors(*ir, jac_factors);
-      }
+      const int jacobians = GeometricFactors::JACOBIANS;
+      geom = fespace->GetMesh()->GetGeometricFactors(*ir, jacobians);
    }
 
    MFEM_ASSERT(!(eval_flags & DETERMINANTS) || dim == vdim ||
@@ -282,61 +552,29 @@ void QuadratureInterpolator::Mult(const Vector &e_vec,
    {
       if (eval_flags & (VALUES | PHYSICAL_VALUES))
       {
-         if (fe->GetMapType() == FiniteElement::MapType::INTEGRAL)
-         {
-            IntTensorEvalKernels::Run(dim, q_layout, vdim, nd, nq, ne,
-                                      maps.B.Read(), geom->detJ.Read(),
-                                      e_vec.Read(), q_val.Write(), vdim, nd, nq);
-         }
-         else
-         {
-            TensorEvalKernels::Run(dim, q_layout, vdim, nd, nq, ne,
-                                   maps.B.Read(), e_vec.Read(), q_val.Write(),
-                                   vdim, nd, nq);
-         }
+         TensorEvalKernels::Run(dim, q_layout, vdim, nd, nq, ne, maps.B.Read(),
+                                e_vec.Read(), q_val.Write(), vdim, nd, nq);
       }
       if (eval_flags & (DERIVATIVES | PHYSICAL_DERIVATIVES))
       {
          const bool phys = (eval_flags & PHYSICAL_DERIVATIVES);
          const real_t *J = phys ? geom->J.Read() : nullptr;
          const int s_dim = phys ? sdim : dim;
-         if (fe->GetMapType() == FiniteElement::MapType::INTEGRAL)
-         {
-            MFEM_ABORT("");
-         }
-         else
-         {
-            GradKernels::Run(dim, q_layout, phys, vdim, nd, nq, ne,
-                             maps.B.Read(), maps.G.Read(), J, e_vec.Read(),
-                             q_der.Write(), s_dim, vdim, nd, nq);
-         }
+         GradKernels::Run(dim, q_layout, phys, vdim, nd, nq, ne,
+                          maps.B.Read(), maps.G.Read(), J, e_vec.Read(),
+                          q_der.Write(), s_dim, vdim, nd, nq);
       }
       if (eval_flags & DETERMINANTS)
       {
-         if (fe->GetMapType() == FiniteElement::MapType::INTEGRAL)
-         {
-            MFEM_ABORT("");
-         }
-         else
-         {
-            DetKernels::Run(dim, vdim, nd, nq, ne, maps.B.Read(), maps.G.Read(),
-                            e_vec.Read(), q_det.Write(), nd, nq, &d_buffer);
-         }
+         DetKernels::Run(dim, vdim, nd, nq, ne, maps.B.Read(),
+                         maps.G.Read(), e_vec.Read(), q_det.Write(), nd,
+                         nq, &d_buffer);
       }
    }
    else // use_tensor_eval == false
    {
-      if (fe->GetMapType() == FiniteElement::MapType::INTEGRAL)
-      {
-         IntEvalKernels::Run(dim, vdim, maps.ndof, maps.nqpt, ne, vdim,
-                             q_layout, geom->detJ.Read(), geom, maps, e_vec,
-                             q_val, q_der, q_det, eval_flags);
-      }
-      else
-      {
-         EvalKernels::Run(dim, vdim, maps.ndof, maps.nqpt, ne, vdim, q_layout,
-                          geom, maps, e_vec, q_val, q_der, q_det, eval_flags);
-      }
+      EvalKernels::Run(dim, vdim, maps.ndof, maps.nqpt, ne,vdim, q_layout,
+                       geom, maps, e_vec, q_val, q_der, q_det, eval_flags);
    }
 }
 
@@ -462,41 +700,22 @@ namespace
 
 using namespace internal::quadrature_interpolator;
 
-template <QVectorLayout Q_LAYOUT> auto IntFallbackTensorEvalKernel(int DIM)
+using EvalKernel = QuadratureInterpolator::EvalKernelType;
+using TensorEvalKernel = QuadratureInterpolator::TensorEvalKernelType;
+using GradKernel = QuadratureInterpolator::GradKernelType;
+using CollocatedGradKernel = QuadratureInterpolator::CollocatedGradKernelType;
+
+template <QVectorLayout Q_LAYOUT>
+TensorEvalKernel FallbackTensorEvalKernel(int DIM)
 {
-   if (DIM == 1)
-   {
-      return ImplValues1D<Q_LAYOUT, true>;
-   }
-   else if (DIM == 2)
-   {
-      return ImplValues2D<Q_LAYOUT, true>;
-   }
-   else if (DIM == 3)
-   {
-      return ImplValues3D<Q_LAYOUT, true>;
-   }
-   MFEM_ABORT("");
+   if (DIM == 1) { return Values1D<Q_LAYOUT>; }
+   else if (DIM == 2) { return Values2D<Q_LAYOUT>; }
+   else if (DIM == 3) { return Values3D<Q_LAYOUT>; }
+   else { MFEM_ABORT(""); }
 }
 
-template <QVectorLayout Q_LAYOUT> auto FallbackTensorEvalKernel(int DIM)
-{
-   if (DIM == 1)
-   {
-      return Values1D<Q_LAYOUT>;
-   }
-   else if (DIM == 2)
-   {
-      return Values2D<Q_LAYOUT>;
-   }
-   else if (DIM == 3)
-   {
-      return Values3D<Q_LAYOUT>;
-   }
-   MFEM_ABORT("");
-}
-
-template <QVectorLayout Q_LAYOUT, bool GRAD_PHYS> auto GetGradKernel(int DIM)
+template<QVectorLayout Q_LAYOUT, bool GRAD_PHYS>
+GradKernel GetGradKernel(int DIM)
 {
    if (DIM == 1) { return Derivatives1D<Q_LAYOUT, GRAD_PHYS>; }
    else if (DIM == 2) { return Derivatives2D<Q_LAYOUT, GRAD_PHYS>; }
@@ -504,185 +723,79 @@ template <QVectorLayout Q_LAYOUT, bool GRAD_PHYS> auto GetGradKernel(int DIM)
    else { MFEM_ABORT(""); }
 }
 
-template <QVectorLayout Q_LAYOUT> auto GetGradKernel(int DIM, bool GRAD_PHYS)
+
+template<QVectorLayout Q_LAYOUT>
+GradKernel GetGradKernel(int DIM, bool GRAD_PHYS)
 {
    if (GRAD_PHYS) { return GetGradKernel<Q_LAYOUT, true>(DIM); }
    else { return GetGradKernel<Q_LAYOUT, false>(DIM); }
 }
 
 template<QVectorLayout Q_LAYOUT, bool GRAD_PHYS>
-auto GetCollocatedGradKernel(int DIM)
+CollocatedGradKernel GetCollocatedGradKernel(int DIM)
 {
-   if (DIM == 1)
-   {
-      return CollocatedDerivatives1D<Q_LAYOUT, GRAD_PHYS>;
-   }
-   else if (DIM == 2)
-   {
-      return CollocatedDerivatives2D<Q_LAYOUT, GRAD_PHYS>;
-   }
-   else if (DIM == 3)
-   {
-      return CollocatedDerivatives3D<Q_LAYOUT, GRAD_PHYS>;
-   }
-   MFEM_ABORT("");
+   if (DIM == 1) { return CollocatedDerivatives1D<Q_LAYOUT, GRAD_PHYS>; }
+   else if (DIM == 2) { return CollocatedDerivatives2D<Q_LAYOUT, GRAD_PHYS>; }
+   else if (DIM == 3) { return CollocatedDerivatives3D<Q_LAYOUT, GRAD_PHYS>; }
+   else { MFEM_ABORT(""); }
 }
 
-template <QVectorLayout Q_LAYOUT>
-auto GetCollocatedGradKernel(int DIM, bool GRAD_PHYS)
+template<QVectorLayout Q_LAYOUT>
+CollocatedGradKernel GetCollocatedGradKernel(int DIM, bool GRAD_PHYS)
 {
    if (GRAD_PHYS) { return GetCollocatedGradKernel<Q_LAYOUT, true>(DIM); }
    else { return GetCollocatedGradKernel<Q_LAYOUT, false>(DIM); }
 }
-
-auto GetCollocatedGradKernel(int DIM, bool GRAD_PHYS, QVectorLayout Q_LAYOUT)
-{
-   if (Q_LAYOUT == QVectorLayout::byNODES)
-   {
-      return GetCollocatedGradKernel<QVectorLayout::byNODES>(
-                DIM, GRAD_PHYS);
-   }
-   else
-   {
-      return GetCollocatedGradKernel<QVectorLayout::byVDIM>(
-                DIM, GRAD_PHYS);
-   }
-}
 } // namespace
 
-template <int DIM, bool Integral>
-auto GetEvalKernelVDimFallback(int VDIM)
+template <int DIM, int VDIM, int ND, int NQ>
+EvalKernel QuadratureInterpolator::EvalKernels::Kernel()
 {
-   if constexpr (Integral)
-   {
-      using EvalKernels = QuadratureInterpolator::IntEvalKernels;
-      if (VDIM == 1)
-      {
-         return EvalKernels::Kernel<DIM, 1, 0, 0>();
-      }
-      else if (VDIM == 2)
-      {
-         return EvalKernels::Kernel<DIM, 2, 0, 0>();
-      }
-      else if (VDIM == 3)
-      {
-         return EvalKernels::Kernel<DIM, 3, 0, 0>();
-      }
-   }
-   if constexpr (!Integral)
-   {
-      using EvalKernels = QuadratureInterpolator::EvalKernels;
-      if (VDIM == 1)
-      {
-         return EvalKernels::Kernel<DIM, 1, 0, 0>();
-      }
-      else if (VDIM == 2)
-      {
-         return EvalKernels::Kernel<DIM, 2, 0, 0>();
-      }
-      else if (VDIM == 3)
-      {
-         return EvalKernels::Kernel<DIM, 3, 0, 0>();
-      }
-   }
+   using namespace internal::quadrature_interpolator;
+   if constexpr (DIM == 1) { return Eval1D; }
+   else if constexpr (DIM == 2) { return Eval2D<VDIM,ND,NQ>; }
+   else if constexpr (DIM == 3) { return Eval3D<VDIM,ND,NQ>; }
    MFEM_ABORT("");
 }
 
-template auto GetEvalKernelVDimFallback<1, true>(int VDIM);
-template auto GetEvalKernelVDimFallback<1, false>(int VDIM);
-template auto GetEvalKernelVDimFallback<2, true>(int VDIM);
-template auto GetEvalKernelVDimFallback<2, false>(int VDIM);
-template auto GetEvalKernelVDimFallback<3, true>(int VDIM);
-template auto GetEvalKernelVDimFallback<3, false>(int VDIM);
-
-QuadratureInterpolator::IntEvalKernelType
-QuadratureInterpolator::IntEvalKernels::Fallback(int DIM, int VDIM, int ND,
-                                                 int NQ)
+template <int DIM>
+EvalKernel GetEvalKernelVDimFallback(int VDIM)
 {
-   if (DIM == 1)
-   {
-      return GetEvalKernelVDimFallback<1, true>(VDIM);
-   }
-   else if (DIM == 2)
-   {
-      return GetEvalKernelVDimFallback<2, true>(VDIM);
-   }
-   else if (DIM == 3)
-   {
-      return GetEvalKernelVDimFallback<3, true>(VDIM);
-   }
-   else
-   {
-      MFEM_ABORT("");
-   }
+   using EvalKernels = QuadratureInterpolator::EvalKernels;
+   if (VDIM == 1) { return EvalKernels::Kernel<DIM,1,0,0>(); }
+   else if (VDIM == 2) { return EvalKernels::Kernel<DIM,2,0,0>(); }
+   else if (VDIM == 3) { return EvalKernels::Kernel<DIM,3,0,0>(); }
+   else { MFEM_ABORT(""); }
 }
 
-QuadratureInterpolator::EvalKernelType
-QuadratureInterpolator::EvalKernels::Fallback(int DIM, int VDIM, int ND, int NQ)
+EvalKernel QuadratureInterpolator::EvalKernels::Fallback(
+   int DIM, int VDIM, int ND, int NQ)
 {
-   if (DIM == 1)
-   {
-      return GetEvalKernelVDimFallback<1, false>(VDIM);
-   }
-   else if (DIM == 2)
-   {
-      return GetEvalKernelVDimFallback<2, false>(VDIM);
-   }
-   else if (DIM == 3)
-   {
-      return GetEvalKernelVDimFallback<3, false>(VDIM);
-   }
-   else
-   {
-      MFEM_ABORT("");
-   }
+   if (DIM == 1) { return GetEvalKernelVDimFallback<1>(VDIM); }
+   else if (DIM == 2) { return GetEvalKernelVDimFallback<2>(VDIM); }
+   else if (DIM == 3) { return GetEvalKernelVDimFallback<3>(VDIM); }
+   else { MFEM_ABORT(""); }
 }
 
-QuadratureInterpolator::IntTensorEvalKernelType
-QuadratureInterpolator::IntTensorEvalKernels::Fallback(int DIM,
-                                                       QVectorLayout Q_LAYOUT,
-                                                       int, int, int)
+TensorEvalKernel QuadratureInterpolator::TensorEvalKernels::Fallback(
+   int DIM, QVectorLayout Q_LAYOUT, int, int, int)
 {
-   if (Q_LAYOUT == QVectorLayout::byNODES)
-   {
-      return IntFallbackTensorEvalKernel<QVectorLayout::byNODES>(DIM);
-   }
-   else
-   {
-      return IntFallbackTensorEvalKernel<QVectorLayout::byVDIM>(DIM);
-   }
+   if (Q_LAYOUT == QVectorLayout::byNODES) { return FallbackTensorEvalKernel<QVectorLayout::byNODES>(DIM); }
+   else { return FallbackTensorEvalKernel<QVectorLayout::byVDIM>(DIM); }
 }
 
-QuadratureInterpolator::TensorEvalKernelType
-QuadratureInterpolator::TensorEvalKernels::Fallback(int DIM,
-                                                    QVectorLayout Q_LAYOUT, int,
-                                                    int, int)
-{
-   if (Q_LAYOUT == QVectorLayout::byNODES)
-   {
-      return FallbackTensorEvalKernel<QVectorLayout::byNODES>(DIM);
-   }
-   else
-   {
-      return FallbackTensorEvalKernel<QVectorLayout::byVDIM>(DIM);
-   }
-}
-
-QuadratureInterpolator::GradKernelType
-QuadratureInterpolator::GradKernels::Fallback(int DIM, QVectorLayout Q_LAYOUT,
-                                              bool GRAD_PHYS, int, int, int)
+GradKernel QuadratureInterpolator::GradKernels::Fallback(
+   int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int, int, int)
 {
    if (Q_LAYOUT == QVectorLayout::byNODES) { return GetGradKernel<QVectorLayout::byNODES>(DIM, GRAD_PHYS); }
    else { return GetGradKernel<QVectorLayout::byVDIM>(DIM, GRAD_PHYS); }
 }
 
-QuadratureInterpolator::CollocatedGradKernelType
-QuadratureInterpolator::CollocatedGradKernels::Fallback(int DIM,
-                                                        QVectorLayout Q_LAYOUT,
-                                                        bool GRAD_PHYS, int,
-                                                        int)
+CollocatedGradKernel QuadratureInterpolator::CollocatedGradKernels::Fallback(
+   int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int, int)
 {
-   return GetCollocatedGradKernel(DIM, GRAD_PHYS, Q_LAYOUT);
+   if (Q_LAYOUT == QVectorLayout::byNODES) { return GetCollocatedGradKernel<QVectorLayout::byNODES>(DIM, GRAD_PHYS); }
+   else { return GetCollocatedGradKernel<QVectorLayout::byVDIM>(DIM, GRAD_PHYS); }
 }
 
 /// @endcond
@@ -693,97 +806,98 @@ namespace quadrature_interpolator
 {
 void InitEvalKernels()
 {
+   using k = QuadratureInterpolator::EvalKernels;
    // 2D, VDIM = 1
-   QuadratureInterpolator::AddEvalSpecializations<2,1,1,1>();
-   QuadratureInterpolator::AddEvalSpecializations<2,1,1,4>();
+   k::Specialization<2,1,1,1>::Add();
+   k::Specialization<2,1,1,4>::Add();
    // Q1
-   QuadratureInterpolator::AddEvalSpecializations<2,1,4,4>();
-   QuadratureInterpolator::AddEvalSpecializations<2,1,4,9>();
+   k::Specialization<2,1,4,4>::Add();
+   k::Specialization<2,1,4,9>::Add();
    // Q2
-   QuadratureInterpolator::AddEvalSpecializations<2,1,9,9>();
-   QuadratureInterpolator::AddEvalSpecializations<2,1,9,16>();
+   k::Specialization<2,1,9,9>::Add();
+   k::Specialization<2,1,9,16>::Add();
    // Q3
-   QuadratureInterpolator::AddEvalSpecializations<2,1,16,16>();
-   QuadratureInterpolator::AddEvalSpecializations<2,1,16,25>();
-   QuadratureInterpolator::AddEvalSpecializations<2,1,16,36>();
+   k::Specialization<2,1,16,16>::Add();
+   k::Specialization<2,1,16,25>::Add();
+   k::Specialization<2,1,16,36>::Add();
    // Q4
-   QuadratureInterpolator::AddEvalSpecializations<2,1,25,25>();
-   QuadratureInterpolator::AddEvalSpecializations<2,1,25,36>();
-   QuadratureInterpolator::AddEvalSpecializations<2,1,25,49>();
-   QuadratureInterpolator::AddEvalSpecializations<2,1,25,64>();
+   k::Specialization<2,1,25,25>::Add();
+   k::Specialization<2,1,25,36>::Add();
+   k::Specialization<2,1,25,49>::Add();
+   k::Specialization<2,1,25,64>::Add();
 
    // 3D, VDIM = 1
    // Q0
-   QuadratureInterpolator::AddEvalSpecializations<3,1,1,1>();
-   QuadratureInterpolator::AddEvalSpecializations<3,1,1,8>();
+   k::Specialization<3,1,1,1>::Add();
+   k::Specialization<3,1,1,8>::Add();
    // Q1
-   QuadratureInterpolator::AddEvalSpecializations<3,1,8,8>();
-   QuadratureInterpolator::AddEvalSpecializations<3,1,8,27>();
+   k::Specialization<3,1,8,8>::Add();
+   k::Specialization<3,1,8,27>::Add();
    // Q2
-   QuadratureInterpolator::AddEvalSpecializations<3,1,27,27>();
-   QuadratureInterpolator::AddEvalSpecializations<3,1,27,64>();
+   k::Specialization<3,1,27,27>::Add();
+   k::Specialization<3,1,27,64>::Add();
    // Q3
-   QuadratureInterpolator::AddEvalSpecializations<3,1,64,64>();
-   QuadratureInterpolator::AddEvalSpecializations<3,1,64,125>();
-   QuadratureInterpolator::AddEvalSpecializations<3,1,64,216>();
+   k::Specialization<3,1,64,64>::Add();
+   k::Specialization<3,1,64,125>::Add();
+   k::Specialization<3,1,64,216>::Add();
    // Q4
-   QuadratureInterpolator::AddEvalSpecializations<3,1,125,125>();
-   QuadratureInterpolator::AddEvalSpecializations<3,1,125,216>();
+   k::Specialization<3,1,125,125>::Add();
+   k::Specialization<3,1,125,216>::Add();
 
    // 2D, VDIM = 3
    // Q0
-   QuadratureInterpolator::AddEvalSpecializations<2,3,1,1>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,1,4>();
+   k::Specialization<2,3,1,1>::Add();
+   k::Specialization<2,3,1,4>::Add();
    // Q1
-   QuadratureInterpolator::AddEvalSpecializations<2,3,4,4>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,4,9>();
+   k::Specialization<2,3,4,4>::Add();
+   k::Specialization<2,3,4,9>::Add();
    // Q2
-   QuadratureInterpolator::AddEvalSpecializations<2,3,9,4>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,9,9>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,9,16>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,9,25>();
+   k::Specialization<2,3,9,4>::Add();
+   k::Specialization<2,3,9,9>::Add();
+   k::Specialization<2,3,9,16>::Add();
+   k::Specialization<2,3,9,25>::Add();
    // Q3
-   QuadratureInterpolator::AddEvalSpecializations<2,3,16,16>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,16,25>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,16,36>();
+   k::Specialization<2,3,16,16>::Add();
+   k::Specialization<2,3,16,25>::Add();
+   k::Specialization<2,3,16,36>::Add();
    // Q4
-   QuadratureInterpolator::AddEvalSpecializations<2,3,25,25>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,25,36>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,25,49>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,25,64>();
+   k::Specialization<2,3,25,25>::Add();
+   k::Specialization<2,3,25,36>::Add();
+   k::Specialization<2,3,25,49>::Add();
+   k::Specialization<2,3,25,64>::Add();
 
    // 2D, VDIM = 2
    // Q1
-   QuadratureInterpolator::AddEvalSpecializations<2,2,4,4>();
-   QuadratureInterpolator::AddEvalSpecializations<2,2,4,9>();
+   k::Specialization<2,2,4,4>::Add();
+   k::Specialization<2,2,4,9>::Add();
    // Q2
-   QuadratureInterpolator::AddEvalSpecializations<2,2,9,9>();
-   QuadratureInterpolator::AddEvalSpecializations<2,2,9,16>();
+   k::Specialization<2,2,9,9>::Add();
+   k::Specialization<2,2,9,16>::Add();
    // Q3
-   QuadratureInterpolator::AddEvalSpecializations<2,2,16,16>();
-   QuadratureInterpolator::AddEvalSpecializations<2,2,16,25>();
-   QuadratureInterpolator::AddEvalSpecializations<2,2,16,36>();
+   k::Specialization<2,2,16,16>::Add();
+   k::Specialization<2,2,16,25>::Add();
+   k::Specialization<2,2,16,36>::Add();
    // Q4
-   QuadratureInterpolator::AddEvalSpecializations<2,2,25,25>();
-   QuadratureInterpolator::AddEvalSpecializations<2,2,25,36>();
-   QuadratureInterpolator::AddEvalSpecializations<2,2,25,49>();
-   QuadratureInterpolator::AddEvalSpecializations<2,2,25,64>();
+   k::Specialization<2,2,25,25>::Add();
+   k::Specialization<2,2,25,36>::Add();
+   k::Specialization<2,2,25,49>::Add();
+   k::Specialization<2,2,25,64>::Add();
 
    // 3D, VDIM = 3
    // Q1
-   QuadratureInterpolator::AddEvalSpecializations<3,3,8,8>();
-   QuadratureInterpolator::AddEvalSpecializations<3,3,8,27>();
+   k::Specialization<3,3,8,8>::Add();
+   k::Specialization<3,3,8,27>::Add();
    // Q2
-   QuadratureInterpolator::AddEvalSpecializations<3,3,27,27>();
-   QuadratureInterpolator::AddEvalSpecializations<3,3,27,64>();
-   QuadratureInterpolator::AddEvalSpecializations<3,3,27,125>();
+   k::Specialization<3,3,27,27>::Add();
+   k::Specialization<3,3,27,64>::Add();
+   k::Specialization<3,3,27,125>::Add();
    // Q3
-   QuadratureInterpolator::AddEvalSpecializations<3,3,64,64>();
-   QuadratureInterpolator::AddEvalSpecializations<3,3,64,125>();
-   QuadratureInterpolator::AddEvalSpecializations<3,3,64,216>();
+   k::Specialization<3,3,64,64>::Add();
+   k::Specialization<3,3,64,125>::Add();
+   k::Specialization<3,3,64,216>::Add();
    // Q4
-   QuadratureInterpolator::AddEvalSpecializations<3,3,125,125>();
-   QuadratureInterpolator::AddEvalSpecializations<3,3,125,216>();
+   k::Specialization<3,3,125,125>::Add();
+   k::Specialization<3,3,125,216>::Add();
 }
 
 } // namespace quadrature_Interpolator

fem/quadinterpolator.hpp

@@ -117,10 +117,6 @@ public:
        FiniteElementSpace is a vector space) and their determinants are computed
        and stored in @a q_det.
 
-       For Integral spaces, the flags VALUES requests the computation of the
-       scalar field values. The result is stored in @a q_val. Derivative types
-       are not supported.
-
        For H(div)-conforming spaces, the flags VALUES / PHYSICAL_VALUES request
        the computation of the vector field values in reference or physical
        space, respectively. The flag PHYSICAL_MAGNITUDES requests the
@@ -163,49 +159,26 @@ public:
    /// QuadratureInterpolator.
    static bool SupportsFESpace(const FiniteElementSpace &fespace);
 
-   // value map types
-   using TensorEvalKernelType = void (*)(const int ne, const real_t *B,
-                                         const real_t *e_vec, real_t *q_val,
-                                         const int vdim, const int nd,
-                                         const int nq);
-   using GradKernelType = void (*)(const int ne, const real_t *B,
-                                   const real_t *G, const real_t *J,
-                                   const real_t *e_vec, real_t *q_der,
-                                   const int s_dim, const int v_dim,
-                                   const int nd, const int nq);
-   using CollocatedGradKernelType = void (*)(const int ne, const real_t *G,
-                                             const real_t *J,
-                                             const real_t *e_vec, real_t *q_der,
-                                             const int sdim, const int vdim,
-                                             const int d1d);
-   using DetKernelType = void (*)(const int NE, const real_t *B,
-                                  const real_t *G, const real_t *e_vec,
-                                  real_t *q_det, const int nd, const int nq,
-                                  Vector *d_buffer);
-   using EvalKernelType = void (*)(const int NE, const int vdim,
-                                   const QVectorLayout q_layout,
-                                   const GeometricFactors *geom,
-                                   const DofToQuad &maps, const Vector &e_vec,
-                                   Vector &q_val, Vector &q_der, Vector &q_det,
-                                   const int eval_flags);
-
-   // integral map types
-   using IntTensorEvalKernelType = void (*)(const int ne, const real_t *B,
-                                            const real_t *detJ,
-                                            const real_t *e_vec, real_t *q_val,
-                                            const int vdim, const int nd,
-                                            const int nq);
-   using IntEvalKernelType =
-      void (*)(const int NE, const int vdim, const QVectorLayout q_layout,
-               const real_t *detJ, const GeometricFactors *geom,
-               const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
-               Vector &q_der, Vector &q_det, const int eval_flags);
-
+   using TensorEvalKernelType = void(*)(const int, const real_t *, const real_t *,
+                                        real_t *, const int, const int, const int);
+   using GradKernelType = void(*)(const int, const real_t *, const real_t *,
+                                  const real_t *, const real_t *, real_t *,
+                                  const int, const int, const int, const int);
+   using CollocatedGradKernelType = void(*)(const int, const real_t *,
+                                            const real_t *, const real_t *,
+                                            real_t *, const int, const int,
+                                            const int);
+   using DetKernelType = void(*)(const int NE, const real_t *, const real_t *,
+                                 const real_t *, real_t *, const int, const int,
+                                 Vector *);
+   using EvalKernelType = void(*)(const int, const int, const QVectorLayout,
+                                  const GeometricFactors *, const DofToQuad &,
+                                  const Vector &, Vector &, Vector &, Vector &,
+                                  const int);
    using TensorEvalHDivKernelType =
       void(*)(const int, const real_t *, const real_t *, const real_t *,
               const real_t *, real_t *, const int, const int);
 
-   // value-type mapping
    MFEM_REGISTER_KERNELS(TensorEvalKernels, TensorEvalKernelType,
                          (int, QVectorLayout, int, int, int), (int));
    MFEM_REGISTER_KERNELS(GradKernels, GradKernelType,
@@ -214,84 +187,8 @@ public:
    MFEM_REGISTER_KERNELS(EvalKernels, EvalKernelType, (int, int, int, int));
    MFEM_REGISTER_KERNELS(CollocatedGradKernels, CollocatedGradKernelType,
                          (int, QVectorLayout, bool, int, int), (int));
-
-   // integral-type mapping
-   MFEM_REGISTER_KERNELS(IntTensorEvalKernels, IntTensorEvalKernelType,
-                         (int, QVectorLayout, int, int, int), (int));
-   MFEM_REGISTER_KERNELS(IntEvalKernels, IntEvalKernelType, (int, int, int, int));
-
    MFEM_REGISTER_KERNELS(TensorEvalHDivKernels, TensorEvalHDivKernelType,
                          (int, QVectorLayout, unsigned, int, int));
-
-   /// Adds specializations for TensorEvalKernels
-   template <int DIM, QVectorLayout Q_LAYOUT, int VDIM, int D1D, int Q1D,
-             int NBZ = 0>
-   static void AddTensorEvalSpecializations()
-   {
-      if constexpr (NBZ)
-      {
-         IntTensorEvalKernels::Specialization<DIM, Q_LAYOUT, VDIM, D1D,
-                              Q1D>::template Opt<NBZ>::Add();
-         TensorEvalKernels::Specialization<DIM, Q_LAYOUT, VDIM, D1D,
-                           Q1D>::template Opt<NBZ>::Add();
-      }
-      else if constexpr (NBZ == 0)
-      {
-         IntTensorEvalKernels::Specialization<DIM, Q_LAYOUT, VDIM, D1D,
-                              Q1D>::Add();
-         TensorEvalKernels::Specialization<DIM, Q_LAYOUT, VDIM, D1D,
-                           Q1D>::Add();
-      }
-   }
-
-   /// Adds specializations for EvalKernels
-   template <int DIM, int VDIM, int ND, int NQ>
-   static void AddEvalSpecializations()
-   {
-      IntEvalKernels::Specialization<DIM, VDIM, ND, NQ>::Add();
-      EvalKernels::Specialization<DIM, VDIM, ND, NQ>::Add();
-   }
-
-   /// Adds specializations for GradKernels
-   template <int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int VDIM, int D1D,
-             int Q1D, int NBZ = 0>
-   static void AddGradSpecializations()
-   {
-      if constexpr (NBZ)
-      {
-         GradKernels::Specialization<DIM, Q_LAYOUT, GRAD_PHYS, VDIM, D1D,
-                     Q1D>::template Opt<NBZ>::Add();
-      }
-      else if constexpr (NBZ == 0)
-      {
-         GradKernels::Specialization<DIM, Q_LAYOUT, GRAD_PHYS, VDIM, D1D,
-                     Q1D>::Add();
-      }
-   }
-
-   /// Adds specializations for CollocatedGradKernels
-   template <int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int VDIM, int D1D,
-             int NBZ = 0>
-   static void AddCollocatedGradSpecializations()
-   {
-      if constexpr (NBZ)
-      {
-         CollocatedGradKernels::Specialization<DIM, Q_LAYOUT, GRAD_PHYS, VDIM,
-                               D1D>::template Opt<NBZ>::Add();
-      }
-      else if constexpr (NBZ == 0)
-      {
-         CollocatedGradKernels::Specialization<DIM, Q_LAYOUT, GRAD_PHYS, VDIM,
-                               D1D>::Add();
-      }
-   }
-
-   /// Adds specializations for DetKernels
-   template <int DIM, int SDIM, int D1D, int Q1D>
-   static void AddDetSpecializations()
-   {
-      DetKernels::Specialization<DIM, SDIM, D1D, Q1D>::Add();
-   }
 };
 
 }

general/backends.hpp

@@ -66,23 +66,6 @@ constexpr bool mfem_use_gpu = false;
 #define MFEM_THREAD_SIZE(k) 1
 #define MFEM_FOREACH_THREAD(i,k,N) for(int i=0; i<N; i++)
 #define MFEM_FOREACH_THREAD_DIRECT(i,k,N) MFEM_FOREACH_THREAD(i,k,N)
-// Assigns a thread block shaped (SX,SY,SZ) contiguous in x.
-// Example (3,2,1) block:
-// 0 (0,0), 1 (1,0), 2 (2,0)
-// 3 (1,0), 4 (1,1), 5 (2,1)
-#define MFEM_FOREACH_THREAD_DIRECT_3D(ix, iy, iz, k, SX, SY, SZ)               \
-   for (int iz = 0; iz < SZ; ++iz)                                             \
-      for (int iy = 0; iy < SY; ++iy)                                          \
-         for (int ix = 0; ix < SX; ++ix)
-// Assigns a thread block shaped (OX,OY,OZ) to work on items (SX,SY,SZ),
-// contiguous in x. This intentionally offsets threads within the block to avoid
-// shared memory bank conflicts.
-// Example (3,2,1) block assigned to work on (2,2,1) items:
-// 0 (0,0), 1 (1,0), 2 (N/A)
-// 3 (1,0), 4 (1,1), 5 (N/A)
-#define MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(ix, iy, iz, k, SX, SY, SZ, OX,    \
-                                             OY, OZ)                           \
-   MFEM_FOREACH_THREAD_DIRECT_3D(ix, iy, iz, k, SX, SY, SZ)
 #endif
 
 // 'double' and 'float' atomicAdd implementation for previous versions of CUDA

general/cuda.hpp

@@ -49,23 +49,6 @@ constexpr bool mfem_use_gpu = true;
 #define MFEM_THREAD_SIZE(k) blockDim.k
 #define MFEM_FOREACH_THREAD(i,k,N) for(int i=threadIdx.k; i<N; i+=blockDim.k)
 #define MFEM_FOREACH_THREAD_DIRECT(i,k,N) if(const int i=threadIdx.k; i<N)
-// Assigns a thread block shaped (SX,SY,SZ) contiguous in x.
-// Example (3,2,1) block:
-// 0 (0,0), 1 (1,0), 2 (2,0)
-// 3 (1,0), 4 (1,1), 5 (2,1)
-#define MFEM_FOREACH_THREAD_DIRECT_3D(ix, iy, iz, k, SX, SY, SZ)               \
-   if (int ix = threadIdx.k % (SX), iy = threadIdx.k / (SX), iz = iy / (SY);   \
-       (iy %= (SY)), (threadIdx.k < (SX) * (SY) * (SZ)))
-// Assigns a thread block shaped (OX,OY,OZ) to work on items (SX,SY,SZ),
-// contiguous in x. This intentionally offsets threads within the block to avoid
-// shared memory bank conflicts.
-// Example (3,2,1) block assigned to work on (2,2,1) items:
-// 0 (0,0), 1 (1,0), 2 (N/A)
-// 3 (1,0), 4 (1,1), 5 (N/A)
-#define MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(ix, iy, iz, k, SX, SY, SZ, OX,    \
-                                             OY, OZ)                           \
-   if (int ix = threadIdx.k % (OX), iy = threadIdx.k / (OX), iz = iy / (OY);   \
-       (ix < (SX)) && ((iy %= (OY)) < (SY)) && (iz < (SZ)))
 #endif // defined(__CUDA_ARCH__)
 #endif // defined(MFEM_USE_CUDA) && defined(__CUDACC__)
 

general/forall.hpp

@@ -480,8 +480,8 @@ template <typename DBODY>
 void RajaHipWrap1D(const int N, DBODY &&d_body)
 {
    //true denotes asynchronous kernel
-   RAJA::forall<RAJA::hip_exec<MFEM_HIP_BLOCKS, true> >(
-      Device::GetRajaResource(), RAJA::RangeSegment(0, N), d_body);
+   RAJA::forall<RAJA::hip_exec<MFEM_HIP_BLOCKS,true> >(RAJA::RangeSegment(0,N),
+                                                       d_body);
 }
 
 template <typename DBODY>

general/hip.hpp

@@ -51,25 +51,6 @@ constexpr bool mfem_use_gpu = true;
    for(int i=hipThreadIdx_ ##k; i<N; i+=hipBlockDim_ ##k)
 #define MFEM_FOREACH_THREAD_DIRECT(i,k,N) \
    if(const int i=hipThreadIdx_ ##k; i<N)
-// Assigns a thread block shaped (SX,SY,SZ) contiguous in x.
-// Example (3,2,1) block:
-// 0 (0,0), 1 (1,0), 2 (2,0)
-// 3 (1,0), 4 (1,1), 5 (2,1)
-#define MFEM_FOREACH_THREAD_DIRECT_3D(ix, iy, iz, k, SX, SY, SZ)               \
-   if (int ix = hipThreadIdx_##k % (SX), iy = hipThreadIdx_##k / (SX),         \
-       iz = iy / (SY);                                                         \
-       (iy %= (SY)), (hipThreadIdx_##k < (SX) * (SY) * (SZ)))
-// Assigns a thread block shaped (OX,OY,OZ) to work on items (SX,SY,SZ),
-// contiguous in x. This intentionally offsets threads within the block to avoid
-// shared memory bank conflicts.
-// Example (3,2,1) block assigned to work on (2,2,1) items:
-// 0 (0,0), 1 (1,0), 2 (N/A)
-// 3 (1,0), 4 (1,1), 5 (N/A)
-#define MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(ix, iy, iz, k, SX, SY, SZ, OX,    \
-                                             OY, OZ)                           \
-   if (int ix = hipThreadIdx_##k % (OX), iy = hipThreadIdx_##k / (OX),         \
-       iz = iy / (OY);                                                         \
-       (ix < (SX)) && ((iy %= (OY)) < (SY)) && (iz < (SZ)))
 #endif // defined(__HIP_DEVICE_COMPILE__)
 #endif // defined(MFEM_USE_HIP) && defined(__HIP__)
 

linalg/CMakeLists.txt

@@ -27,6 +27,7 @@ list(APPEND SRCS
   handle.cpp
   matrix.cpp
   mma.cpp
+  multivector.cpp
   ode.cpp
   operator.cpp
   ordering.cpp
@@ -63,6 +64,7 @@ list(APPEND HDRS
   linalg.hpp
   matrix.hpp
   mma.hpp
+  multivector.hpp
   ode.hpp
   operator.hpp
   ordering.hpp

linalg/batched/batched.cpp

@@ -13,7 +13,6 @@
 #include "native.hpp"
 #include "gpu_blas.hpp"
 #include "magma.hpp"
-#include "../../general/reducers.hpp"
 
 namespace mfem
 {
@@ -120,16 +119,4 @@ void BatchedLinAlgBase::MultTranspose(const DenseTensor &A, const Vector &x,
    AddMult(A, x, y, 1.0, 0.0, Op::T);
 }
 
-void VerifyBatchedLUInfo(const Array<int> &info_array, const char *message)
-{
-   static Array<int> workspace;
-   int status = 0;
-   const int *d_info = info_array.Read();
-   mfem::reduce(
-      info_array.Size(), status,
-   [=] MFEM_HOST_DEVICE (int i, int &r) { r |= d_info[i]; },
-   BOrReducer<int> {}, true, workspace);
-   MFEM_VERIFY(status == 0, message);
-}
-
 }

linalg/batched/batched.hpp

@@ -141,9 +141,6 @@ public:
    virtual ~BatchedLinAlgBase() { }
 };
 
-/// Check that all batched LU info values are zero.
-void VerifyBatchedLUInfo(const Array<int> &info_array, const char *message);
-
 } // namespace mfem
 
 #endif

linalg/batched/gpu_blas.cpp

@@ -126,8 +126,7 @@ void GPUBlasBatchedLinAlg::LUFactor(DenseTensor &A, Array<int> &P) const
    const blasStatus_t status = MFEM_GPUBLAS_PREFIX(getrfBatched)(
                                   GPUBlas::Handle(), n, d_A_ptrs, n, P.Write(),
                                   info_array.Write(), n_mat);
-   MFEM_VERIFY(status == MFEM_BLAS_SUCCESS, "GPU BLAS error.");
-   VerifyBatchedLUInfo(info_array, "Batch LU factorization failed");
+   MFEM_VERIFY(status == MFEM_BLAS_SUCCESS, "");
 }
 
 void GPUBlasBatchedLinAlg::LUSolve(
@@ -190,14 +189,12 @@ void GPUBlasBatchedLinAlg::Invert(DenseTensor &A) const
    status = MFEM_GPUBLAS_PREFIX(getrfBatched)(
                GPUBlas::Handle(), n, d_LU_ptrs, n, P.Write(),
                info_array.Write(), n_mat);
-   MFEM_VERIFY(status == MFEM_BLAS_SUCCESS, "GPU BLAS error.");
-   VerifyBatchedLUInfo(info_array, "Batch LU factorization failed");
+   MFEM_VERIFY(status == MFEM_BLAS_SUCCESS, "");
 
    status = MFEM_GPUBLAS_PREFIX(getriBatched)(
                GPUBlas::Handle(), n, d_LU_ptrs, n, P.ReadWrite(), d_A_ptrs, n,
                info_array.Write(), n_mat);
-   MFEM_VERIFY(status == MFEM_BLAS_SUCCESS, "GPU BLAS error.");
-   VerifyBatchedLUInfo(info_array, "Batch matrix inversion failed");
+   MFEM_VERIFY(status == MFEM_BLAS_SUCCESS, "");
 }
 
 #endif

linalg/batched/magma.cpp

@@ -99,8 +99,7 @@ void MagmaBatchedLinAlg::LUFactor(DenseTensor &A, Array<int> &P) const
    const magma_int_t status = MFEM_MAGMA_PREFIX(getrf_batched)(
                                  n, n, d_A_ptrs, n, d_P_ptrs,
                                  info_array.Write(), n_mat, Magma::Queue());
-   MFEM_VERIFY(status == MAGMA_SUCCESS, "MAGMA error.");
-   VerifyBatchedLUInfo(info_array, "Batch LU factorization failed");
+   MFEM_VERIFY(status == MAGMA_SUCCESS, "");
 }
 
 void MagmaBatchedLinAlg::LUSolve(
@@ -170,14 +169,12 @@ void MagmaBatchedLinAlg::Invert(DenseTensor &A) const
    status = MFEM_MAGMA_PREFIX(getrf_batched)(
                n, n, d_LU_ptrs, n, d_P_ptrs, info_array.Write(), n_mat,
                Magma::Queue());
-   MFEM_VERIFY(status == MAGMA_SUCCESS, "MAGMA error.");
-   VerifyBatchedLUInfo(info_array, "Batch LU factorization failed");
+   MFEM_VERIFY(status == MAGMA_SUCCESS, "");
 
    status = MFEM_MAGMA_PREFIX(getri_outofplace_batched)(
                n, d_LU_ptrs, n, d_P_ptrs, d_A_ptrs, n, info_array.Write(),
                n_mat, Magma::Queue());
-   MFEM_VERIFY(status == MAGMA_SUCCESS, "MAGMA error.");
-   VerifyBatchedLUInfo(info_array, "Batch matrix inversion failed");
+   MFEM_VERIFY(status == MAGMA_SUCCESS, "");
 }
 
 } // namespace mfem

linalg/complex_operator.cpp

@@ -246,10 +246,6 @@ SparseMatrix * ComplexSparseMatrix::GetSystemMatrix() const
    const int  nrows_i = (A_i)?A_i->Height():0;
    const int    nrows = std::max(nrows_r, nrows_i);
 
-   const int  ncols_r = (A_r)?A_r->Width():0;
-   const int  ncols_i = (A_i)?A_i->Width():0;
-   const int    ncols = std::max(ncols_r, ncols_i);
-
    const int     *I_r = (A_r)?A_r->GetI():NULL;
    const int     *I_i = (A_i)?A_i->GetI():NULL;
 
@@ -284,7 +280,7 @@ SparseMatrix * ComplexSparseMatrix::GetSystemMatrix() const
             J[I[i] + j] = J_r[I_r[i] + j];
             D[I[i] + j] = D_r[I_r[i] + j];
 
-            J[I[i+nrows] + off_i + j] = J_r[I_r[i] + j] + ncols;
+            J[I[i+nrows] + off_i + j] = J_r[I_r[i] + j] + nrows;
             D[I[i+nrows] + off_i + j] = factor*D_r[I_r[i] + j];
          }
       }
@@ -293,7 +289,7 @@ SparseMatrix * ComplexSparseMatrix::GetSystemMatrix() const
          const int off_r = (I_r)?(I_r[i+1] - I_r[i]):0;
          for (int j=0; j<I_i[i+1] - I_i[i]; j++)
          {
-            J[I[i] + off_r + j] =  J_i[I_i[i] + j] + ncols;
+            J[I[i] + off_r + j] =  J_i[I_i[i] + j] + nrows;
             D[I[i] + off_r + j] = -D_i[I_i[i] + j];
 
             J[I[i+nrows] + j] = J_i[I_i[i] + j];
@@ -896,12 +892,12 @@ ComplexHypreParMatrix::getColStartStop(const HypreParMatrix * A_r,
    HYPRE_BigInt loc_start_stop[2];
    offd_col_start_stop = new HYPRE_BigInt[2 * num_recv_procs];
 
-   const HYPRE_BigInt * col_part = (A_r) ? A_r->ColPart() :
-                                   ((A_i) ? A_i->ColPart() : NULL);
+   const HYPRE_BigInt * row_part = (A_r) ? A_r->RowPart() :
+                                   ((A_i) ? A_i->RowPart() : NULL);
 
-   int col_part_ind = (HYPRE_AssumedPartitionCheck()) ? 0 : myid_;
-   loc_start_stop[0] = col_part[col_part_ind];
-   loc_start_stop[1] = col_part[col_part_ind+1];
+   int row_part_ind = (HYPRE_AssumedPartitionCheck()) ? 0 : myid_;
+   loc_start_stop[0] = row_part[row_part_ind];
+   loc_start_stop[1] = row_part[row_part_ind+1];
 
    MPI_Request * req = new MPI_Request[send_procs.size()+recv_procs.size()];
    MPI_Status * stat = new MPI_Status[send_procs.size()+recv_procs.size()];

linalg/linalg.hpp

@@ -15,6 +15,7 @@
 // Linear algebra header file
 
 #include "vector.hpp"
+#include "multivector.hpp"
 #include "operator.hpp"
 #include "matrix.hpp"
 #include "sparsemat.hpp"

linalg/multivector.cpp

@@ -0,0 +1,60 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "multivector.hpp"
+
+namespace mfem
+{
+
+MultiVector::MultiVector(const Array<int> &vector_sizes)
+{
+   SetSizes(vector_sizes);
+}
+
+MultiVector::MultiVector(const Array<int> &vector_sizes, MemoryType mt)
+{
+   SetSizes(vector_sizes, mt);
+}
+
+MultiVector::MultiVector(Vector &base, const Array<int> &vector_sizes)
+{
+   MakeRef(base, vector_sizes);
+}
+
+void MultiVector::SetSizes(const Array<int> &vector_sizes)
+{
+   blocks.resize(vector_sizes.Size());
+   for (int i = 0; i < vector_sizes.Size(); i++)
+   {
+      operator[](i).SetSize(vector_sizes[i]);
+   }
+}
+
+void MultiVector::SetSizes(const Array<int> &vector_sizes, MemoryType mt)
+{
+   blocks.resize(vector_sizes.Size());
+   for (int i = 0; i < vector_sizes.Size(); i++)
+   {
+      operator[](i).SetSize(vector_sizes[i], mt);
+   }
+}
+
+void MultiVector::MakeRef(Vector &base, const Array<int> &vector_sizes)
+{
+   blocks.resize(vector_sizes.Size());
+   for (int offset = 0, i = 0; i < vector_sizes.Size(); i++)
+   {
+      blocks[i].emplace<0>(base, offset, vector_sizes[i]);
+      offset += vector_sizes[i];
+   }
+}
+
+} // namespace mfem

linalg/multivector.hpp

@@ -0,0 +1,198 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_MULTIVECTOR_HPP
+#define MFEM_MULTIVECTOR_HPP
+
+#include "../general/array.hpp"
+#include "vector.hpp"
+#include <vector>
+#include <array>
+#include <variant>
+
+namespace mfem
+{
+
+/// Class representing an array of Vectors with generally different sizes.
+/** This class is similar to BlockVector with the following two main
+    differences:
+    - the data for the individual Vector blocks does not need to be part of one
+      big contiguous memory allocation;
+    - this class does not inherit from class Vector (as a consequence of the
+      first bullet).
+
+    Internally, each Vector block is represented as either:
+    - (default) a Vector object constructed and owned by this class; this
+      object, in turn, as any Vector object, can own its Memory allocation or
+      refer to a sub-Memory of another Memory object; or
+    - a pointer to an externally allocated Vector or classes derived from
+      Vector. */
+class MultiVector
+{
+private:
+   std::vector<std::variant<Vector,Vector*>> blocks;
+
+public:
+   /// Create an empty MultiVector with zero blocks.
+   MultiVector() = default;
+
+   /** @brief Create a MultiVector with @a num_blocks blocks. The individual
+       Vector blocks are default initialized, i.e. they all have size zero. */
+   MultiVector(int num_blocks)
+      : blocks(num_blocks) { }
+
+   /** @brief Construct a MultiVector with number of blocks and individual block
+       Vector sizes given by @a vector_sizes.
+
+       @note The memory of the individual Vector blocks is NOT initialized. */
+   MultiVector(const Array<int> &vector_sizes);
+
+   /** @brief Construct a MultiVector with number of blocks and individual block
+       Vector sizes given by @a vector_sizes. All Vector blocks use the
+       MemoryType @a mt.
+
+       @note The memory of the individual Vector blocks is NOT initialized. */
+   MultiVector(const Array<int> &vector_sizes, MemoryType mt);
+
+   /** @brief Construct a MultiVector referencing data within a given monolithic
+       Vector @a base.
+
+       With this constructor, the Memory flags of @a base and of the individual
+       Vector blocks may need to be explicitly synchronized when data is moved
+       between host and device. */
+   MultiVector(Vector &base, const Array<int> &vector_sizes);
+
+   /** @brief Construct a MultiVector referencing multiple Vectors given as
+       arguments.
+
+       The VectorTypes reference arguments are expected to be static_cast-able
+       to (Vector &) which is the case if the types are derived from Vector,
+       e.g. HypreParVector, GridFunction, etc.
+
+       With this constructor, operations on individual Vector blocks are
+       performed directly on the objects @a vs. In particular, there is no need
+       to synchronize the Memory flags of @a vs and the ones of the individual
+       Vector blocks when data is moved between host and device. */
+   template <typename... VectorTypes,
+             std::enable_if_t<
+                std::conjunction_v<
+                   std::is_convertible<VectorTypes&,Vector&>...>, bool> = true>
+   MultiVector(VectorTypes &...vs) { MakeRef(vs...); }
+
+   /// Return the number of Vectors in the MultiVector.
+   int NumBlocks() const { return blocks.size(); }
+
+   /** @brief Set the number of Vectors in the MultiVector. Existing Vector
+       blocks will remain unmodified. New Vector blocks will be default
+       initialized, i.e. they all have size zero. */
+   void SetNumBlocks(int num_blocks) { blocks.resize(num_blocks); }
+
+   /// Read-write access to the i-th Vector.
+   inline Vector &operator[](int i);
+
+   /// Read-only access to the i-th Vector.
+   inline const Vector &operator[](int i) const;
+
+   /** @brief Update the MultiVector according to the given @a vector_sizes.
+
+       This method can be used to add or remove blocks. The individual Vector
+       sizes are updated using the method Vector::SetSize(int). */
+   void SetSizes(const Array<int> &vector_sizes);
+
+   /** @brief Update the MultiVector according to the given @a vector_sizes and
+       MemoryType @a mt.
+
+       This method can be used to add or remove blocks. The individual Vector
+       sizes and MemoryType are updated using the method
+       Vector::SetSize(int, MemoryType). */
+   void SetSizes(const Array<int> &vector_sizes, MemoryType mt);
+
+   /** @brief Update the MultiVector to reference data within a given monolithic
+       Vector @a base.
+
+       After calling this method, the Memory flags of @a base and of the
+       individual Vector blocks may need to be explicitly synchronized when data
+       is moved between host and device.*/
+   void MakeRef(Vector &base, const Array<int> &vector_sizes);
+
+   /** @brief Update the @a i-th MultiVector block to reference data within the
+       given monolithic Vector @a base at the given @a offset and with the given
+       @a size.
+
+       After calling this method, the Memory flags of @a base and of the @a i-th
+       Vector block may need to be explicitly synchronized when data is moved
+       between host and device.*/
+   inline void MakeRef(int i, Vector &base, int offset, int size)
+   {
+      blocks[i].emplace<0>(base, offset, size);
+   }
+
+   /** @brief Update the MultiVector to reference multiple Vectors given as
+       arguments.
+
+       The VectorTypes reference arguments are expected to be static_cast-able
+       to (Vector &) which is the case if the types are derived from Vector,
+       e.g. HypreParVector, GridFunction, etc.
+
+       After calling this method, operations on individual Vector blocks are
+       performed directly on the objects @a vs. In particular, there is no need
+       to synchronize the Memory flags of @a vs and the ones of the individual
+       Vector blocks when data is moved between host and device. */
+   template <typename... VectorTypes,
+             std::enable_if_t<
+                std::conjunction_v<
+                   std::is_convertible<VectorTypes&,Vector&>...>, bool> = true>
+   inline void MakeRef(VectorTypes &...vs);
+
+   /** @brief Update the @a i-th MultiVector block to reference the given
+       Vector @a v.
+
+       After calling this method, operations on the @a i-th Vector block are
+       performed directly on the Vector @a v. In particular, there is no need
+       to synchronize the Memory flags of @a v and the ones of the @a i-th
+       Vector blocks when data is moved between host and device. */
+   inline void MakeRef(int i, Vector &v) { blocks[i] = &v; }
+};
+
+// Inline and template methods
+
+inline Vector &MultiVector::operator[](int i)
+{
+   auto &bi = blocks[i];
+   return (bi.index() == 0) ? std::get<0>(bi) : *std::get<1>(bi);
+}
+
+inline const Vector &MultiVector::operator[](int i) const
+{
+   auto &bi = blocks[i];
+   return (bi.index() == 0) ? std::get<0>(bi) : *std::get<1>(bi);
+}
+
+template <typename... VectorTypes,
+          std::enable_if_t<
+             std::conjunction_v<
+                std::is_convertible<VectorTypes&,Vector&>...>, bool>>
+inline void MultiVector::MakeRef(VectorTypes &...vs)
+{
+   blocks.resize(sizeof...(vs));
+   if constexpr (sizeof...(vs) > 0)
+   {
+      const std::array vs_p{&static_cast<Vector&>(vs)...};
+      for (std::size_t i = 0; i < sizeof...(vs); i++)
+      {
+         blocks[i] = vs_p[i];
+      }
+   }
+}
+
+} // namespace mfem
+
+#endif // MFEM_MULTIVECTOR_HPP

linalg/operator.cpp

@@ -111,6 +111,16 @@ void Operator::ArrayAddMultTranspose(const Array<const Vector *> &X,
    }
 }
 
+void Operator::Mult(const MultiVector &, MultiVector &)
+{
+   MFEM_ABORT("this method is not overriden for this class!");
+}
+
+Operator &Operator::GetGradient(const MultiVector &) const
+{
+   MFEM_ABORT("this method is not overriden for this class!");
+}
+
 void Operator::FormLinearSystem(const Array<int> &ess_tdof_list,
                                 Vector &x, Vector &b,
                                 Operator* &Aout, Vector &X, Vector &B,

linalg/operator.hpp

@@ -13,6 +13,7 @@
 #define MFEM_OPERATOR
 
 #include "vector.hpp"
+#include "multivector.hpp"
 
 namespace mfem
 {
@@ -129,6 +130,16 @@ public:
    virtual void ArrayAddMultTranspose(const Array<const Vector *> &X,
                                       Array<Vector *> &Y, const real_t a = 1.0) const;
 
+   /** @brief Operator application, y = A(x), where the input @a x and the
+       output @a y are MultiVector objects, i.e. they generally use
+       non-contiguous memory representation.
+
+       The typical use case for this method are block operators like
+       DifferentiableOperator.
+
+       The base class implementation for the method is to generate an error. */
+   virtual void Mult(const MultiVector &x, MultiVector &y);
+
    /** @brief Evaluate the gradient operator at the point @a x. The default
        behavior in class Operator is to generate an error. */
    virtual Operator &GetGradient(const Vector &x) const
@@ -137,6 +148,16 @@ public:
       return const_cast<Operator &>(*this);
    }
 
+   /** @brief Evaluate the gradient operator at the point @a x. The input @a x
+       is provided as a MultiVector, i.e. it generally uses non-contiguous
+       memory representation.
+
+       The typical use case for this method are block operators like
+       DifferentiableOperator.
+
+       The base class implementation for the method is to generate an error. */
+   virtual Operator &GetGradient(const MultiVector &x) const;
+
    /** @brief Computes the diagonal entries into @a diag. Typically, this
        operation only makes sense for linear Operator%s. In some cases, only an
        approximation of the diagonal is computed. */

makefile

@@ -810,7 +810,6 @@ MINIAPPS_SUBDIRS = dpg/util hooke/operators hooke/preconditioners \
    hooke/materials hooke/kernels
 FORMAT_FILES += $(foreach dir,$(TESTS_SUBDIRS),tests/$(dir)/*.?pp)
 FORMAT_FILES += $(foreach dir,$(UNIT_TESTS_SUBDIRS),tests/unit/$(dir)/*.?pp)
-FORMAT_FILES += tests/unit/fem/specializations/*.?pp
 FORMAT_FILES += $(foreach dir,$(MINIAPPS_SUBDIRS),miniapps/$(dir)/*.?pp)
 FORMAT_FILES += config/cmake/config.hpp.in config/config.hpp.in mfem*.hpp
 FORMAT_EXCLUDE = general/tinyxml2.cpp tests/unit/catch.hpp

mesh/mesh.cpp

@@ -667,84 +667,9 @@ void Mesh::GetEdgeTransformation(int EdgeNo,
          }
          EdTr->SetFE(edge_el);
       }
-      else // L2 Nodes (e.g., periodic mesh), go through the face containing the edge
+      else
       {
-         // Search for a face that contains this edge
-         GetEdgeFaceTable();
-
-         Array<int> faces_e;
-         edge_face->GetRow(EdgeNo, faces_e);
-
-         MFEM_VERIFY(faces_e.Size() > 0, "Edge not found in any face!");
-         const int face_no = faces_e[0];
-
-         // Get edge local index and orientation
-         Array<int> edges_f, oris_f;
-         GetFaceEdges(face_no, edges_f, oris_f);
-         const int local_idx = edges_f.Find(EdgeNo);
-         MFEM_ASSERT(local_idx >= 0, "Edge not found on the face!");
-         const int edge_ori = oris_f[local_idx] > 0 ? 0 : 1;
-
-         // Get face information
-         const FaceInfo &face_info = faces_info[face_no];
-
-         // Get transformation from face to edge
-         IntegrationPointTransformation LocEdge;
-         int edge_info = EncodeFaceInfo(local_idx, edge_ori);
-         Element::Type face_type = GetFaceElementType(face_no);
-
-         switch (face_type)
-         {
-            case Element::TRIANGLE:
-               GetLocalSegToTriTransformation(LocEdge.Transf, edge_info);
-               break;
-            case Element::QUADRILATERAL:
-               GetLocalSegToQuadTransformation(LocEdge.Transf, edge_info);
-               break;
-            default:
-               MFEM_ABORT("Unsupported face type for edge transformation!");
-         }
-
-         // Get edge element
-         const int order = Nodes->FESpace()->GetElementOrder(face_info.Elem1No);
-         const L2_FECollection *l2_fec = dynamic_cast<const L2_FECollection*>
-                                         (Nodes->FESpace()->FEColl());
-         if (l2_fec)
-         {
-            // L2 elements do not have a defined trace space
-            if (!EdgeTransfElement || EdgeTransfElement->GetOrder() != order
-                || EdgeTransfElement->GetBasisType() != l2_fec->GetBasisType())
-            {
-               EdgeTransfElement = make_unique<L2_SegmentElement>(
-                                      order, l2_fec->GetBasisType());
-            }
-            edge_el = EdgeTransfElement.get();
-         }
-         else
-         {
-            MFEM_ABORT("Unsupported finite element collection.");
-         }
-
-         // Map edge nodes to face reference space
-         IntegrationRule face_ir(edge_el->GetDof());
-         LocEdge.Transform(edge_el->GetNodes(), face_ir);
-
-         // Then, map from face to element
-         IntegrationPointTransformation Loc1;
-         GetLocalFaceTransformation(face_type,
-                                    GetElementType(face_info.Elem1No),
-                                    Loc1.Transf, face_info.Elem1Inf);
-
-         IntegrationRule elem_ir(edge_el->GetDof());
-         Loc1.Transf.ElementNo = face_info.Elem1No;
-         Loc1.Transf.ElementType = ElementTransformation::ELEMENT;
-         Loc1.Transf.mesh = this;
-         Loc1.Transform(face_ir, elem_ir);
-
-         // Finally, get the physical coordinates
-         Nodes->GetVectorValues(Loc1.Transf, elem_ir, pm);
-
-         EdTr->SetFE(edge_el);
+         MFEM_ABORT("Not implemented.");
       }
    }
 }
@@ -1899,8 +1824,8 @@ void Mesh::Init()
 
 void Mesh::InitTables()
 {
-   el_to_edge = el_to_face = el_to_el = bel_to_edge = NULL;
-   face_edge = edge_face = edge_vertex = NULL;
+   el_to_edge =
+      el_to_face = el_to_el = bel_to_edge = face_edge = edge_vertex = NULL;
    face_to_elem = NULL;
 }
 
@@ -1923,7 +1848,6 @@ void Mesh::DestroyTables()
    }
 
    delete face_edge;
-   delete edge_face;
    delete edge_vertex;
 
    delete face_to_elem;
@@ -1997,7 +1921,6 @@ void Mesh::ResetLazyData()
 {
    delete el_to_el;     el_to_el = NULL;
    delete face_edge;    face_edge = NULL;
-   delete edge_face;    edge_face = NULL;
    delete face_to_elem;    face_to_elem = NULL;
    delete edge_vertex;  edge_vertex = NULL;
    DeleteGeometricFactors();
@@ -2922,7 +2845,6 @@ void Mesh::ReorderElements(const Array<int> &ordering, bool reorder_vertices)
    //                 boundary element ordering
    // - el_to_el    - no need to rebuild
    // - face_edge   - no need to rebuild
-   // - edge_face   - no need to rebuild
    // - edge_vertex - no need to rebuild
    // - geom_factors - no need to rebuild
 
@@ -3405,25 +3327,11 @@ void Mesh::DoNodeReorder(DSTable *old_v_to_v, Table *old_elem_vert)
       // loop over all elements
       for (int i = 0; i < GetNE(); i++)
       {
-         fes->GetElementInteriorDofs(i, old_dofs);
-         // No need to permute the dofs if there are fewer than two
-         if (old_dofs.Size() < 2)
-         {
-            offset += old_dofs.Size();
-            continue;
-         }
-
          const int *old_v = old_elem_vert->GetRow(i);
          const int *new_v = elements[i]->GetVertices();
          const int *dof_ord;
          int new_or;
          const Geometry::Type geom = elements[i]->GetGeometryType();
-         if (geom == Geometry::CUBE || geom == Geometry::PRISM ||
-             geom == Geometry::PYRAMID)
-         {
-            offset += old_dofs.Size();
-            continue;
-         }
          switch (geom)
          {
             case Geometry::SEGMENT:
@@ -3447,8 +3355,9 @@ void Mesh::DoNodeReorder(DSTable *old_v_to_v, Table *old_elem_vert)
          dof_ord = fec->DofOrderForOrientation(geom, new_or);
          MFEM_VERIFY(dof_ord != NULL,
                      "FE collection '" << fec->Name()
-                     << "' does not define reordering (" << new_or << ") for "
+                     << "' does not define reordering for "
                      << Geometry::Name[geom] << " elements!");
+         fes->GetElementInteriorDofs(i, old_dofs);
          new_dofs.SetSize(old_dofs.Size());
          for (int j = 0; j < new_dofs.Size(); j++)
          {
@@ -4676,9 +4585,8 @@ Mesh::Mesh(const Mesh &mesh, bool copy_nodes)
    // Do NOT copy the element-to-element Table, el_to_el
    el_to_el = NULL;
 
-   // Do NOT copy the face-to-edge Table, face_edge and edge_face
+   // Do NOT copy the face-to-edge Table, face_edge
    face_edge = NULL;
-   edge_face = NULL;
    face_to_elem = NULL;
 
    // Copy the edge-to-vertex Table, edge_vertex
@@ -7208,8 +7116,7 @@ const FiniteElementSpace *Mesh::GetNodalFESpace() const
    return ((Nodes) ? Nodes->FESpace() : NULL);
 }
 
-void Mesh::SetCurvature(int order, bool discont, int space_dim, int ordering,
-                        int pyr_type)
+void Mesh::SetCurvature(int order, bool discont, int space_dim, int ordering)
 {
    if (order <= 0)
    {
@@ -7222,12 +7129,11 @@ void Mesh::SetCurvature(int order, bool discont, int space_dim, int ordering,
    if (discont)
    {
       const int type = 1; // Gauss-Lobatto points
-      nfec = new L2_FECollection(order, Dim, type, FiniteElement::VALUE,
-                                 pyr_type);
+      nfec = new L2_FECollection(order, Dim, type);
    }
    else
    {
-      nfec = new H1_FECollection(order, Dim, BasisType::GaussLobatto, pyr_type);
+      nfec = new H1_FECollection(order, Dim);
    }
    FiniteElementSpace* nfes = new FiniteElementSpace(this, nfec, space_dim,
                                                      ordering);
@@ -8173,22 +8079,6 @@ Table *Mesh::GetFaceEdgeTable() const
    return (face_edge);
 }
 
-Table *Mesh::GetEdgeFaceTable() const
-{
-   if (edge_face)
-   {
-      return edge_face;
-   }
-
-   if (Dim != 3)
-   {
-      return NULL;
-   }
-
-   edge_face = Transpose(*GetFaceEdgeTable());
-   return edge_face;
-}
-
 Table *Mesh::GetEdgeVertexTable() const
 {
    if (edge_vertex)
@@ -11547,7 +11437,6 @@ void Mesh::Swap(Mesh& other, bool non_geometry)
    mfem::Swap(bel_to_edge, other.bel_to_edge);
    mfem::Swap(be_to_face, other.be_to_face);
    mfem::Swap(face_edge, other.face_edge);
-   mfem::Swap(edge_face, other.edge_face);
    mfem::Swap(face_to_elem, other.face_to_elem);
    mfem::Swap(edge_vertex, other.edge_vertex);
 

mesh/mesh.hpp

@@ -250,18 +250,16 @@ protected:
    Table *bel_to_edge;    // for 3D only
 
    // Note that the following tables are owned by this class and should not be
-   // deleted by the caller. Of these four tables, only face_edge, edge_face and
+   // deleted by the caller. Of these three tables, only face_edge and
    // edge_vertex are returned by access functions.
    mutable Table *face_to_elem;  // Used by FindFaceNeighbors, not returned.
    mutable Table *face_edge;     // Returned by GetFaceEdgeTable().
-   mutable Table *edge_face;     // Returned by GetEdgeFaceTable().
    mutable Table *edge_vertex;   // Returned by GetEdgeVertexTable().
 
    IsoparametricTransformation Transformation, Transformation2;
    IsoparametricTransformation BdrTransformation;
    IsoparametricTransformation FaceTransformation, EdgeTransformation;
    FaceElementTransformations FaceElemTr;
-   mutable std::unique_ptr<L2_SegmentElement> EdgeTransfElement;
 
    // refinement embeddings for forward compatibility with NCMesh
    mutable CoarseFineTransformations CoarseFineTr;
@@ -1733,11 +1731,6 @@ public:
    /// @note The returned object should NOT be deleted by the caller.
    Table *GetFaceEdgeTable() const;
 
-   /// Returns the edge-to-face Table (3D)
-   ///
-   /// @note The returned object should NOT be deleted by the caller.
-   Table *GetEdgeFaceTable() const;
-
    /// Returns the edge-to-vertex Table (3D)
    ///
    /// @note The returned object should NOT be deleted by the caller.
@@ -2432,13 +2425,9 @@ public:
                                finite element space (continuous is default).
        @param[in]  space_dim   The space dimension (optional).
        @param[in]  ordering    The Ordering of the finite element space
-                               (Ordering::byVDIM is the default).
-       @param[in]  pyr_type    Select Bergot (pyr_type = 0) or Fuentes
-                               (pyr_type = 1) basis functions for pyramid
-                               shaped elements. */
-   virtual void SetCurvature(int order, bool discont = false,
-                             int space_dim = -1, int ordering = 1,
-                             int pyr_type = 1);
+                               (Ordering::byVDIM is the default). */
+   virtual void SetCurvature(int order, bool discont = false, int space_dim = -1,
+                             int ordering = 1);
 
    /// @}
 

mesh/nurbs.cpp

@@ -1354,27 +1354,22 @@ NURBSPatch::NURBSPatch(std::istream &input)
    int pdim, dim, size = 1;
    string ident;
 
-   skip_comment_lines(input, '#');
    input >> ws >> ident >> pdim; // knotvectors
    kv.SetSize(pdim);
    for (int i = 0; i < pdim; i++)
    {
-      skip_comment_lines(input, '#');
       kv[i] = new KnotVector(input);
       size *= kv[i]->GetNCP();
    }
 
-   skip_comment_lines(input, '#');
    input >> ws >> ident >> dim; // dimension
    init(dim + 1);
 
-   skip_comment_lines(input, '#');
    input >> ws >> ident; // controlpoints (homogeneous coordinates)
    if (ident == "controlpoints" || ident == "controlpoints_homogeneous")
    {
       for (int j = 0, i = 0; i < size; i++)
       {
-         skip_comment_lines(input, '#');
          for (int d = 0; d <= dim; d++, j++)
          {
             input >> data[j];
@@ -1385,7 +1380,6 @@ NURBSPatch::NURBSPatch(std::istream &input)
    {
       for (int j = 0, i = 0; i < size; i++)
       {
-         skip_comment_lines(input, '#');
          for (int d = 0; d <= dim; d++)
          {
             input >> data[j+d];

mesh/pmesh.cpp

@@ -2031,20 +2031,18 @@ std::unique_ptr<ParGridFunction> ParMesh::GetJacobianDeterminantGF() const
    return detgf;
 }
 
-void ParMesh::SetCurvature(int order, bool discont, int space_dim, int ordering,
-                           int pyrtype)
+void ParMesh::SetCurvature(int order, bool discont, int space_dim, int ordering)
 {
    DeleteFaceNbrData();
    space_dim = (space_dim == -1) ? spaceDim : space_dim;
    FiniteElementCollection* nfec;
    if (discont)
    {
-      nfec = new L2_FECollection(order, Dim, BasisType::GaussLobatto,
-                                 FiniteElement::VALUE, pyrtype);
+      nfec = new L2_FECollection(order, Dim, BasisType::GaussLobatto);
    }
    else
    {
-      nfec = new H1_FECollection(order, Dim, BasisType::GaussLobatto, pyrtype);
+      nfec = new H1_FECollection(order, Dim);
    }
    ParFiniteElementSpace* nfes = new ParFiniteElementSpace(this, nfec, space_dim,
                                                            ordering);
@@ -4866,13 +4864,6 @@ void ParMesh::Print(std::ostream &os, const std::string &comments) const
       return;
    }
 
-   if (pncmesh && pncmesh->using_scaling)
-   {
-      // For nodes scaling, we write the file in the format MFEM NC mesh v1.1.
-      Printer(os, "", comments);
-      return;
-   }
-
    const Array<int>* s2l_face;
    if (!pncmesh)
    {

mesh/pmesh.hpp

@@ -563,7 +563,7 @@ public:
    void ExchangeFaceNbrNodes();
 
    void SetCurvature(int order, bool discont = false, int space_dim = -1,
-                     int ordering = 1, int pyrtype = 1) override;
+                     int ordering = 1) override;
 
    std::unique_ptr<ParGridFunction> GetJacobianDeterminantGF() const;
 

mesh/pncmesh.cpp

@@ -28,48 +28,6 @@ namespace mfem
 
 using namespace bin_io;
 
-static int GetHexEdgeSplit(const int* nodes, int v1, int v2);
-
-static bool SameSplitScale(real_t a, real_t b)
-{
-#ifdef MFEM_USE_DOUBLE
-   constexpr real_t rel_tol = 1.0e-8;
-#else
-   constexpr real_t rel_tol = 1.0e-5;
-#endif
-   return std::abs(a - b) <= rel_tol *
-          std::max(real_t(1.0), std::max(std::abs(a), std::abs(b)));
-}
-
-static real_t DirectedHexEdgeScale(const int* nodes, const Refinement &ref,
-                                   int v0, int v1)
-{
-   const int dir = GetHexEdgeSplit(nodes, v0, v1);
-   static const int split_edges[3][4][2] =
-   {
-      {{0, 1}, {3, 2}, {4, 5}, {7, 6}},
-      {{1, 2}, {0, 3}, {5, 6}, {4, 7}},
-      {{0, 4}, {1, 5}, {2, 6}, {3, 7}}
-   };
-
-   for (int i = 0; i < 4; i++)
-   {
-      const int a = nodes[split_edges[dir][i][0]];
-      const int b = nodes[split_edges[dir][i][1]];
-      if (a == v0 && b == v1)
-      {
-         return ref.s[dir];
-      }
-      if (a == v1 && b == v0)
-      {
-         return 1.0 - ref.s[dir];
-      }
-   }
-
-   MFEM_ABORT("Shared face edge does not match the refinement direction.");
-   return 0.0;
-}
-
 ParNCMesh::ParNCMesh(MPI_Comm comm, const NCMesh &ncmesh,
                      const int *partitioning)
    : NCMesh(ncmesh)
@@ -1597,7 +1555,7 @@ bool ParNCMesh::AnisotropicConflict(const Array<Refinement> &refinements,
       ElementNeighborProcessors(elem, ranks);
       for (int j = 0; j < ranks.Size(); j++)
       {
-         send_ref[ranks[j]].AddRefinement(elem, ref);
+         send_ref[ranks[j]].AddRefinement(elem, ref.GetType());
       }
    }
 
@@ -1618,8 +1576,8 @@ bool ParNCMesh::AnisotropicConflict(const Array<Refinement> &refinements,
    for (int i = 0; i < refinements.Size(); i++)
    {
       const Refinement &ref = refinements[i];
-      CheckRefinement(leaf_elements[ref.index], ref, refinements, elemToRef,
-                      conflicts);
+      CheckRefinement(leaf_elements[ref.index], ref.GetType(), refinements,
+                      elemToRef, conflicts);
    }
 
    // Receive (ghost layer) refinements from all neighbors
@@ -1635,9 +1593,7 @@ bool ParNCMesh::AnisotropicConflict(const Array<Refinement> &refinements,
       // check the ghost refinements
       for (int i = 0; i < msg.Size(); i++)
       {
-         Refinement ghost_ref(msg.elements[i], msg.values[i].ref_type);
-         ghost_ref.SetScaleForType(msg.values[i].scale);
-         CheckRefinement(msg.elements[i], ghost_ref, refinements, elemToRef,
+         CheckRefinement(msg.elements[i], msg.values[i], refinements, elemToRef,
                          conflicts);
       }
    }
@@ -1793,7 +1749,7 @@ int FindHexFace(const int* no, int vn1, int vn2, int vn3, int vn4)
 
 // Assumption: v1 and v2 are indices of hex vertices connected by an edge.
 // The return value is {0,1,2} denoting split {X,Y,Z}.
-static int GetHexEdgeSplit(const int* nodes, int v1, int v2)
+int GetHexEdgeSplit(const int* nodes, int v1, int v2)
 {
    Array<int> v(2);
    v[0] = v1;
@@ -1824,8 +1780,7 @@ static int GetHexEdgeSplit(const int* nodes, int v1, int v2)
    return edgeDir[edge];
 }
 
-void ParNCMesh::CheckRefAnisoFace(const Refinement &ref, int elem,
-                                  int vn1, int vn2, int vn3, int vn4,
+void ParNCMesh::CheckRefAnisoFace(int elem, int vn1, int vn2, int vn3, int vn4,
                                   const Array<Refinement> &refinements,
                                   const std::map<int, int> &elemToRef,
                                   std::set<int> &conflicts)
@@ -1843,11 +1798,11 @@ void ParNCMesh::CheckRefAnisoFace(const Refinement &ref, int elem,
    if (elemToRef.count(nghbIndex) > 0)
    {
       const int refIndex = elemToRef.at(nghbIndex);
-      const Refinement& nghb_ref = refinements[refIndex];
+      const Refinement& ref = refinements[refIndex];
 
       bool refDir[3];
       for (int i=0; i<3; ++i)
-         refDir[i] = nghb_ref.s[i] > real_t{0};
+         refDir[i] = ref.s[i] > real_t{0};
 
       const int localFace = FindHexFace(nghb.node, vn1, vn2, vn3, vn4);
       const int faceDir = GetHexFaceDir(localFace);
@@ -1879,50 +1834,30 @@ void ParNCMesh::CheckRefAnisoFace(const Refinement &ref, int elem,
          MFEM_ASSERT(cnt == 2 && hexSplitOnFace >= 0, "");
 
          const int edgeSplit = GetHexEdgeSplit(nghb.node, vn1, vn2);
-         if (edgeSplit != hexSplitOnFace)
-         {
-            conflicts.insert(refIndex);
-         }
-         else
-         {
-            const real_t elem_scale =
-               DirectedHexEdgeScale(elements[elem].node, ref, vn1, vn2);
-            const real_t nghb_scale =
-               DirectedHexEdgeScale(nghb.node, nghb_ref, vn1, vn2);
-            if (!SameSplitScale(elem_scale, nghb_scale))
-            {
-               conflicts.insert(refIndex);
-            }
-         }
+         if (edgeSplit != hexSplitOnFace) { conflicts.insert(refIndex); }
       }
    }
    // The else case is that the neighbor is not refined, so there is no need to
    // check for conflicts.
 }
 
-void ParNCMesh::CheckRefIsoFace(const Refinement &ref, int elem,
-                                int vn1, int vn2, int vn3, int vn4,
+void ParNCMesh::CheckRefIsoFace(int elem, int vn1, int vn2, int vn3, int vn4,
                                 int en1, int en2, int en3, int en4,
                                 const Array<Refinement> &refinements,
                                 const std::map<int, int> &elemToRef,
                                 std::set<int> &conflicts)
 {
-   CheckRefAnisoFace(ref, elem, vn1, vn2, en2, en4, refinements, elemToRef,
-                     conflicts);
-   CheckRefAnisoFace(ref, elem, en4, en2, vn3, vn4, refinements, elemToRef,
-                     conflicts);
-   CheckRefAnisoFace(ref, elem, vn4, vn1, en1, en3, refinements, elemToRef,
-                     conflicts);
-   CheckRefAnisoFace(ref, elem, en3, en1, vn2, vn3, refinements, elemToRef,
-                     conflicts);
+   CheckRefAnisoFace(elem, vn1, vn2, en2, en4, refinements, elemToRef, conflicts);
+   CheckRefAnisoFace(elem, en4, en2, vn3, vn4, refinements, elemToRef, conflicts);
+   CheckRefAnisoFace(elem, vn4, vn1, en1, en3, refinements, elemToRef, conflicts);
+   CheckRefAnisoFace(elem, en3, en1, vn2, vn3, refinements, elemToRef, conflicts);
 }
 
-void ParNCMesh::CheckRefinement(int elem, const Refinement &ref,
+void ParNCMesh::CheckRefinement(int elem, char ref_type,
                                 const Array<Refinement> &refinements,
                                 const std::map<int, int> &elemToRef,
                                 std::set<int> &conflicts)
 {
-   const char ref_type = ref.GetType();
    const Element &el = elements[elem];
    MFEM_ASSERT(el.geom == Geometry::CUBE && el.ref_type == 0,
                "Element must be an unrefined hexahedron");
@@ -1933,46 +1868,46 @@ void ParNCMesh::CheckRefinement(int elem, const Refinement &ref,
    // This follows the logic of NCMesh::RefineElement().
    if (ref_type == Refinement::X) // split along X axis
    {
-      CheckRefAnisoFace(ref, elem, no[0], no[1], no[5], no[4], refinements,
+      CheckRefAnisoFace(elem, no[0], no[1], no[5], no[4], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(ref, elem, no[2], no[3], no[7], no[6], refinements,
+      CheckRefAnisoFace(elem, no[2], no[3], no[7], no[6], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(ref, elem, no[4], no[5], no[6], no[7], refinements,
+      CheckRefAnisoFace(elem, no[4], no[5], no[6], no[7], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(ref, elem, no[3], no[2], no[1], no[0], refinements,
+      CheckRefAnisoFace(elem, no[3], no[2], no[1], no[0], refinements,
                         elemToRef, conflicts);
    }
    else if (ref_type == Refinement::Y) // split along Y axis
    {
-      CheckRefAnisoFace(ref, elem, no[1], no[2], no[6], no[5], refinements,
+      CheckRefAnisoFace(elem, no[1], no[2], no[6], no[5], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(ref, elem, no[3], no[0], no[4], no[7], refinements,
+      CheckRefAnisoFace(elem, no[3], no[0], no[4], no[7], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(ref, elem, no[5], no[6], no[7], no[4], refinements,
+      CheckRefAnisoFace(elem, no[5], no[6], no[7], no[4], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(ref, elem, no[0], no[3], no[2], no[1], refinements,
+      CheckRefAnisoFace(elem, no[0], no[3], no[2], no[1], refinements,
                         elemToRef, conflicts);
    }
    else if (ref_type == Refinement::Z) // split along Z axis
    {
-      CheckRefAnisoFace(ref, elem, no[4], no[0], no[1], no[5], refinements,
+      CheckRefAnisoFace(elem, no[4], no[0], no[1], no[5], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(ref, elem, no[5], no[1], no[2], no[6], refinements,
+      CheckRefAnisoFace(elem, no[5], no[1], no[2], no[6], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(ref, elem, no[6], no[2], no[3], no[7], refinements,
+      CheckRefAnisoFace(elem, no[6], no[2], no[3], no[7], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(ref, elem, no[7], no[3], no[0], no[4], refinements,
+      CheckRefAnisoFace(elem, no[7], no[3], no[0], no[4], refinements,
                         elemToRef, conflicts);
    }
    else if (ref_type == Refinement::XY) // XY split
    {
-      CheckRefAnisoFace(ref, elem, no[0], no[1], no[5], no[4], refinements,
+      CheckRefAnisoFace(elem, no[0], no[1], no[5], no[4], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(ref, elem, no[1], no[2], no[6], no[5], refinements,
+      CheckRefAnisoFace(elem, no[1], no[2], no[6], no[5], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(ref, elem, no[2], no[3], no[7], no[6], refinements,
+      CheckRefAnisoFace(elem, no[2], no[3], no[7], no[6], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(ref, elem, no[3], no[0], no[4], no[7], refinements,
+      CheckRefAnisoFace(elem, no[3], no[0], no[4], no[7], refinements,
                         elemToRef, conflicts);
 
       const int mid01 = GetMidEdgeNode(no[0], no[1]);
@@ -1985,20 +1920,20 @@ void ParNCMesh::CheckRefinement(int elem, const Refinement &ref,
       const int mid67 = GetMidEdgeNode(no[6], no[7]);
       const int mid74 = GetMidEdgeNode(no[7], no[4]);
 
-      CheckRefIsoFace(ref, elem, no[3], no[2], no[1], no[0], mid23, mid12, mid01,
+      CheckRefIsoFace(elem, no[3], no[2], no[1], no[0], mid23, mid12, mid01,
                       mid30, refinements, elemToRef, conflicts);
-      CheckRefIsoFace(ref, elem, no[4], no[5], no[6], no[7], mid45, mid56, mid67,
+      CheckRefIsoFace(elem, no[4], no[5], no[6], no[7], mid45, mid56, mid67,
                       mid74, refinements, elemToRef, conflicts);
    }
    else if (ref_type == Refinement::XZ) // XZ split
    {
-      CheckRefAnisoFace(ref, elem, no[3], no[2], no[1], no[0], refinements,
+      CheckRefAnisoFace(elem, no[3], no[2], no[1], no[0], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(ref, elem, no[2], no[6], no[5], no[1], refinements,
+      CheckRefAnisoFace(elem, no[2], no[6], no[5], no[1], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(ref, elem, no[6], no[7], no[4], no[5], refinements,
+      CheckRefAnisoFace(elem, no[6], no[7], no[4], no[5], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(ref, elem, no[7], no[3], no[0], no[4], refinements,
+      CheckRefAnisoFace(elem, no[7], no[3], no[0], no[4], refinements,
                         elemToRef, conflicts);
 
       const int mid01 = GetMidEdgeNode(no[0], no[1]);
@@ -2011,9 +1946,9 @@ void ParNCMesh::CheckRefinement(int elem, const Refinement &ref,
       const int mid26 = GetMidEdgeNode(no[2], no[6]);
       const int mid37 = GetMidEdgeNode(no[3], no[7]);
 
-      CheckRefIsoFace(ref, elem, no[0], no[1], no[5], no[4], mid01, mid15, mid45,
+      CheckRefIsoFace(elem, no[0], no[1], no[5], no[4], mid01, mid15, mid45,
                       mid04, refinements, elemToRef, conflicts);
-      CheckRefIsoFace(ref, elem, no[2], no[3], no[7], no[6], mid23, mid37, mid67,
+      CheckRefIsoFace(elem, no[2], no[3], no[7], no[6], mid23, mid37, mid67,
                       mid26, refinements, elemToRef, conflicts);
    }
    else if (ref_type == Refinement::YZ) // YZ split
@@ -2028,18 +1963,18 @@ void ParNCMesh::CheckRefinement(int elem, const Refinement &ref,
       const int mid26 = GetMidEdgeNode(no[2], no[6]);
       const int mid37 = GetMidEdgeNode(no[3], no[7]);
 
-      CheckRefAnisoFace(ref, elem, no[4], no[0], no[1], no[5], refinements,
+      CheckRefAnisoFace(elem, no[4], no[0], no[1], no[5], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(ref, elem, no[0], no[3], no[2], no[1], refinements,
+      CheckRefAnisoFace(elem, no[0], no[3], no[2], no[1], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(ref, elem, no[3], no[7], no[6], no[2], refinements,
+      CheckRefAnisoFace(elem, no[3], no[7], no[6], no[2], refinements,
                         elemToRef, conflicts);
-      CheckRefAnisoFace(ref, elem, no[7], no[4], no[5], no[6], refinements,
+      CheckRefAnisoFace(elem, no[7], no[4], no[5], no[6], refinements,
                         elemToRef, conflicts);
 
-      CheckRefIsoFace(ref, elem, no[1], no[2], no[6], no[5], mid12, mid26, mid56,
+      CheckRefIsoFace(elem, no[1], no[2], no[6], no[5], mid12, mid26, mid56,
                       mid15, refinements, elemToRef, conflicts);
-      CheckRefIsoFace(ref, elem, no[3], no[0], no[4], no[7], mid30, mid04, mid74,
+      CheckRefIsoFace(elem, no[3], no[0], no[4], no[7], mid30, mid04, mid74,
                       mid37, refinements, elemToRef, conflicts);
    }
    else if (ref_type == Refinement::XYZ) // XYZ split
@@ -2059,17 +1994,17 @@ void ParNCMesh::CheckRefinement(int elem, const Refinement &ref,
       const int mid26 = GetMidEdgeNode(no[2], no[6]);
       const int mid37 = GetMidEdgeNode(no[3], no[7]);
 
-      CheckRefIsoFace(ref, elem, no[3], no[2], no[1], no[0], mid23, mid12, mid01,
+      CheckRefIsoFace(elem, no[3], no[2], no[1], no[0], mid23, mid12, mid01,
                       mid30, refinements, elemToRef, conflicts);
-      CheckRefIsoFace(ref, elem, no[0], no[1], no[5], no[4], mid01, mid15, mid45,
+      CheckRefIsoFace(elem, no[0], no[1], no[5], no[4], mid01, mid15, mid45,
                       mid04, refinements, elemToRef, conflicts);
-      CheckRefIsoFace(ref, elem, no[1], no[2], no[6], no[5], mid12, mid26, mid56,
+      CheckRefIsoFace(elem, no[1], no[2], no[6], no[5], mid12, mid26, mid56,
                       mid15, refinements, elemToRef, conflicts);
-      CheckRefIsoFace(ref, elem, no[2], no[3], no[7], no[6], mid23, mid37, mid67,
+      CheckRefIsoFace(elem, no[2], no[3], no[7], no[6], mid23, mid37, mid67,
                       mid26, refinements, elemToRef, conflicts);
-      CheckRefIsoFace(ref, elem, no[3], no[0], no[4], no[7], mid30, mid04, mid74,
+      CheckRefIsoFace(elem, no[3], no[0], no[4], no[7], mid30, mid04, mid74,
                       mid37, refinements, elemToRef, conflicts);
-      CheckRefIsoFace(ref, elem, no[4], no[5], no[6], no[7], mid45, mid56, mid67,
+      CheckRefIsoFace(elem, no[4], no[5], no[6], no[7], mid45, mid56, mid67,
                       mid74, refinements, elemToRef, conflicts);
    }
    else
@@ -2118,7 +2053,7 @@ void ParNCMesh::Refine(const Array<Refinement> &refinements)
       ElementNeighborProcessors(elem, ranks);
       for (int j = 0; j < ranks.Size(); j++)
       {
-         send_ref[ranks[j]].AddRefinement(elem, ref);
+         send_ref[ranks[j]].AddRefinement(elem, ref.GetType());
       }
    }
 
@@ -2128,9 +2063,8 @@ void ParNCMesh::Refine(const Array<Refinement> &refinements)
    // do local refinements
    for (int i = 0; i < refinements.Size(); i++)
    {
-      Refinement ref_i = refinements[i];
-      ref_i.index = leaf_elements[refinements[i].index];
-      NCMesh::RefineElement(ref_i);
+      const Refinement &ref = refinements[i];
+      NCMesh::RefineElement(leaf_elements[ref.index], ref.GetType());
    }
 
    // receive (ghost layer) refinements from all neighbors
@@ -2146,9 +2080,7 @@ void ParNCMesh::Refine(const Array<Refinement> &refinements)
       // do the ghost refinements
       for (int i = 0; i < msg.Size(); i++)
       {
-         Refinement ghost_ref(msg.elements[i], msg.values[i].ref_type);
-         ghost_ref.SetScaleForType(msg.values[i].scale);
-         NCMesh::RefineElement(ghost_ref);
+         NCMesh::RefineElement(msg.elements[i], msg.values[i]);
       }
    }
 

mesh/pncmesh.hpp

@@ -497,27 +497,11 @@ protected: // implementation
    /** Used by ParNCMesh::Refine() to inform neighbors about refinements at
     *  the processor boundary. This keeps their ghost layers synchronized.
     */
-   struct NeighborRefinement
-   {
-      char ref_type;
-      real_t scale[3];
-   };
-
-   class NeighborRefinementMessage
-      : public ElementValueMessage<NeighborRefinement, false,
-        VarMessageTag::NEIGHBOR_REFINEMENT_VM>
+   class NeighborRefinementMessage : public ElementValueMessage<char, false,
+      VarMessageTag::NEIGHBOR_REFINEMENT_VM>
    {
    public:
-      void AddRefinement(int elem, const Refinement &ref)
-      {
-         NeighborRefinement data{};
-         data.ref_type = ref.GetType();
-         for (int i = 0; i < 3; i++)
-         {
-            data.scale[i] = ref.s[i];
-         }
-         Add(elem, data);
-      }
+      void AddRefinement(int elem, char ref_type) { Add(elem, ref_type); }
       typedef std::map<int, NeighborRefinementMessage> Map;
    };
 
@@ -618,8 +602,7 @@ protected: // implementation
    /** For the face with ordered vertices vn* and neighboring element @a elem,
        check whether the other neighboring element (if it exists) is marked for
        a horizontal refinement conflicting with a vertical split. */
-   void CheckRefAnisoFace(const Refinement &ref, int elem,
-                          int vn1, int vn2, int vn3, int vn4,
+   void CheckRefAnisoFace(int elem, int vn1, int vn2, int vn3, int vn4,
                           const Array<Refinement> &refinements,
                           const std::map<int, int> &elemToRef,
                           std::set<int> &conflicts);
@@ -628,8 +611,7 @@ protected: // implementation
        neighboring element @a elem, check whether the other neighboring element
        (if it exists) is marked for a refinement conflicting with an isotropic
        refinement of the face. */
-   void CheckRefIsoFace(const Refinement &ref, int elem,
-                        int vn1, int vn2, int vn3, int vn4,
+   void CheckRefIsoFace(int elem, int vn1, int vn2, int vn3, int vn4,
                         int en1, int en2, int en3, int en4,
                         const Array<Refinement> &refinements,
                         const std::map<int, int> &elemToRef,
@@ -640,8 +622,9 @@ protected: // implementation
                               const std::map<int, int> &elemToRef,
                               std::set<int> &conflicts);
 
-   /// Check whether the input refinement would cause a conflict.
-   void CheckRefinement(int elem, const Refinement &ref,
+   /** Check whether the refinement of the element with index @a elem and type
+       @a ref_type would cause a conflict. */
+   void CheckRefinement(int elem, char ref_type,
                         const Array<Refinement> &refinements,
                         const std::map<int, int> &elemToRef,
                         std::set<int> &conflicts);

miniapps/CMakeLists.txt

@@ -22,6 +22,7 @@ add_subdirectory(common)
 add_subdirectory(contact)
 add_subdirectory(dfem)
 add_subdirectory(diag-smoothers)
+add_subdirectory(multiapp)
 add_subdirectory(dpg)
 add_subdirectory(electromagnetics)
 add_subdirectory(fluids/navier)

miniapps/hdiv-linear-solver/change_basis.hpp

@@ -80,6 +80,8 @@ public:
    // limitations
    void MultRT_2D(const Vector &x, Vector &y, Mode mode) const;
    void MultRT_3D(const Vector &x, Vector &y, Mode mode) const;
+   // suppress warning about hiding overloaded virtual function:
+   using Operator::Mult;
 };
 
 } // namespace mfem

miniapps/meshing/CMakeLists.txt

@@ -151,10 +151,6 @@ if (MFEM_USE_MPI)
     MAIN phpref.cpp
     LIBRARIES mfem)
 
-  add_mfem_miniapp(pref321
-    MAIN pref321.cpp
-    LIBRARIES mfem)
-
   # Add parallel tests.
   if (MFEM_ENABLE_TESTING)
     set(PARALLEL_TESTS
@@ -164,7 +160,6 @@ if (MFEM_USE_MPI)
       fit-node-position
       pminimal-surface
       phpref
-      pref321
     )
     # Meshing miniapps that return MFEM_SKIP_RETURN_VALUE in some cases:
     set(SKIP_TESTS)

miniapps/meshing/makefile

@@ -24,7 +24,7 @@ SEQ_MINIAPPS = mobius-strip klein-bottle toroid trimmer twist mesh-explorer\
 	shaper extruder mesh-optimizer minimal-surface polar-nc reflector\
 	ref321 mesh-quality hpref
 PAR_MINIAPPS = pmesh-optimizer pminimal-surface pmesh-fitting fit-node-position\
-	phpref pref321 mesh-bounding-boxes
+	phpref mesh-bounding-boxes
 ifeq ($(MFEM_USE_MPI),NO)
    MINIAPPS = $(SEQ_MINIAPPS)
 else
@@ -99,8 +99,6 @@ hpref-test-seq: hpref
 	@$(call mfem-test,$<,, Serial hp-refinement)
 phpref-test-par: phpref
 	@$(call mfem-test,$<, $(RUN_MPI), Parallel hp-refinement)
-pref321-test-par: pref321
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel 3:1 refinement)
 mesh-bounding-boxes-test-par: mesh-bounding-boxes
 	@$(call mfem-test,$<, $(RUN_MPI), Parallel bounding boxes)
 ref321-test-seq: ref321

miniapps/meshing/pref321.cpp

@@ -1,336 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-//
-//      -----------------------------------------------------------------
-//      3:1 Refinement Miniapp: Parallel 3:1 anisotropic mesh refinements
-//      -----------------------------------------------------------------
-//
-// This miniapp performs random 3:1 refinements of a quadrilateral or hexahedral
-// mesh. A diffusion equation is solved in an H1 finite element space defined on
-// the refined mesh, and its continuity is verified across local and shared
-// faces.
-//
-// Compile with: make pref321
-//
-// Sample runs:  mpirun -np 4 pref321 -mm -dim 2 -o 2 -r 100
-//               mpirun -np 4 pref321 -mm -dim 3 -o 2 -r 100
-//               mpirun -np 4 pref321 -m ../../data/star.mesh -o 2 -r 100
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-real_t CheckH1Continuity(ParGridFunction &x);
-
-// Find the two children of parent element `elem` after its refinement in one
-// direction.
-void FindChildren(const Mesh &mesh, int elem, Array<int> &children)
-{
-   const CoarseFineTransformations &cf = mesh.ncmesh->GetRefinementTransforms();
-   MFEM_ASSERT(mesh.GetNE() == cf.embeddings.Size(), "");
-
-   // Note that row `elem` of the table constructed by cf.MakeCoarseToFineTable
-   // is an alternative to this global loop, but constructing the table is also
-   // a global operation with global storage.
-   for (int i = 0; i < mesh.GetNE(); i++)
-   {
-      const int p = cf.embeddings[i].parent;
-      if (p == elem)
-      {
-         children.Append(i);
-      }
-   }
-}
-
-// Refine 3:1 via 2 refinements with scalings 2/3 and 1/2.
-void Refine31(Mesh &mesh, int elem, char type)
-{
-   Array<Refinement> refs; // Refinement is defined in ncmesh.hpp
-   refs.Append(Refinement(elem, type, 2.0 / 3.0));
-   mesh.GeneralRefinement(refs);
-
-   // Find the elements with parent `elem`
-   Array<int> children;
-   FindChildren(mesh, elem, children);
-   MFEM_ASSERT(children.Size() == 2, "");
-
-   const int elem1 = children[0];
-
-   refs.SetSize(0);
-   refs.Append(Refinement(elem1, type)); // Default scaling of 0.5
-   mesh.GeneralRefinement(refs);
-}
-
-// Randomly select elements for 3:1 refinements in random directions.
-void TestAnisoRefRandom(int num_refs, int dim, ParMesh &mesh, int myid,
-                        int seed = 0)
-{
-   std::mt19937 gen(seed);
-   for (int i = 0; i < num_refs; i++)
-   {
-      const int elem = gen() % mesh.GetNE();
-      const int t = gen() % dim;
-      auto type = t == 0 ? Refinement::X :
-                  (t == 1 ? Refinement::Y : Refinement::Z);
-
-      // In 3D, check for conflicts in the parallel refinements.
-      if (dim == 3)
-      {
-         std::set<int> conflicts; // Indices in refs of conflicting elements
-         Array<Refinement> refs;
-         refs.Append(Refinement(elem, type));
-         const bool conflict = mesh.AnisotropicConflict(refs, conflicts);
-         if (conflict)
-         {
-            if (myid == 0)
-               cout << "Anisotropic conflict on iteration " << i
-                    << ", retrying\n";
-            i--;
-            continue;
-         }
-      }
-
-      Refine31(mesh, elem, type);
-   }
-
-   mesh.EnsureNodes();
-   mesh.SetScaledNCMesh();
-}
-
-int main(int argc, char *argv[])
-{
-   Mpi::Init(argc, argv);
-   Hypre::Init();
-
-   const int num_procs = Mpi::WorldSize();
-   const int myid = Mpi::WorldRank();
-
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../data/star.mesh";
-   int order = 1;
-   bool visualization = true;
-   bool makeMesh = false;
-   int num_refs = 1;
-   int tdim = 2;  // Mesh dimension for Cartesian meshes.
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&makeMesh, "-mm", "--make-mesh", "-no-mm",
-                  "--no-make-mesh", "Create Cartesian mesh");
-   args.AddOption(&tdim, "-dim", "--dimension", "Dimension for Cartesian mesh");
-   args.AddOption(&num_refs, "-r", "--refs", "Number of 3:1 refinements");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 2. Create or read the serial mesh on all ranks, then apply the same
-   //    deterministic 3:1 refinement sequence before partitioning it.
-   Mesh mesh;
-   if (makeMesh)
-   {
-      mesh = tdim == 3 ? Mesh::MakeCartesian3D(2, 2, 2, Element::HEXAHEDRON) :
-             Mesh::MakeCartesian2D(2, 2, Element::QUADRILATERAL);
-   }
-   else
-   {
-      mesh = Mesh::LoadFromFile(mesh_file, 1, 1);
-   }
-
-   const int dim = mesh.Dimension();
-
-   mesh.EnsureNCMesh();
-   mesh.SetScaledNCMesh();
-
-   // 3. Partition the refined serial mesh.
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
-
-   TestAnisoRefRandom(num_refs, dim, pmesh, myid, myid);
-
-   // 4. Define a parallel H1 finite element space and report its global size.
-   H1_FECollection fec(order, dim);
-   ParFiniteElementSpace fespace(&pmesh, &fec);
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: "
-           << fespace.GlobalTrueVSize() << endl;
-   }
-
-   // 5. Assemble and solve the Poisson problem, following ex1p.
-   ParGridFunction x(&fespace);
-   x = 0.0;
-
-   ParLinearForm b(&fespace);
-   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
-
-   ParBilinearForm a(&fespace);
-   a.AddDomainIntegrator(new DiffusionIntegrator());
-   a.Assemble();
-
-   OperatorPtr A;
-   Vector B, X;
-   Array<int> ess_tdof_list;
-   if (pmesh.bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
-      ess_bdr = 0;
-      pmesh.MarkExternalBoundaries(ess_bdr);
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-
-   HypreBoomerAMG M;
-   CGSolver cg(MPI_COMM_WORLD);
-   cg.SetPreconditioner(M);
-   cg.SetOperator(*A);
-   cg.SetRelTol(1e-12);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(1);
-   cg.Mult(B, X);
-
-   a.RecoverFEMSolution(X, b, x);
-
-   // 6. Verify the continuity of the solution in H1 over local and shared
-   //    faces and compute the global maximum jump.
-   const real_t h1err = CheckH1Continuity(x);
-   if (myid == 0)
-   {
-      cout << "Error of H1 continuity: " << h1err << endl;
-   }
-   MFEM_VERIFY(h1err < 1.0e-7, "H1 discontinuity found");
-
-   // 7. Save the refined mesh and the solution in parallel. This output can
-   //    be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh.Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 8. Send the parallel solution to GLVis.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int visport = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << pmesh << x << flush;
-   }
-
-   return 0;
-}
-
-real_t CheckH1Continuity(ParGridFunction &x)
-{
-   const ParFiniteElementSpace *pfes = x.ParFESpace();
-   ParMesh *pmesh = pfes->GetParMesh();
-   const int dim = pmesh->Dimension();
-
-   real_t errorMax = 0.0;
-
-   // Shared-face values require face-neighbor data.
-   x.ExchangeFaceNbrData();
-
-   // First handle faces for which both elements are local to this rank.
-   for (int f = 0; f < pmesh->GetNumFaces(); f++)
-   {
-      const auto info = pmesh->GetFaceInformation(f);
-      if (!info.IsLocal())
-      {
-         continue;
-      }
-
-      FaceElementTransformations *FT = pmesh->GetFaceElementTransformations(f);
-      const int faceOrder = dim == 3 ? pfes->GetFaceOrder(f) :
-                            pfes->GetEdgeOrder(f);
-      const IntegrationRule &ir = IntRules.Get(FT->FaceGeom, 2 * faceOrder);
-
-      for (int i = 0; i < ir.GetNPoints(); i++)
-      {
-         const IntegrationPoint &fip = ir.IntPoint(i);
-         IntegrationPoint ip1, ip2;
-
-         FT->Loc1.Transform(fip, ip1);
-         FT->Loc2.Transform(fip, ip2);
-
-         const real_t v1 = x.GetValue(*FT->Elem1, ip1);
-         const real_t v2 = x.GetValue(*FT->Elem2, ip2);
-         errorMax = std::max(errorMax, std::abs(v1 - v2));
-      }
-   }
-
-   // Then check partition interfaces. Conforming shared faces are handled on
-   // the lower-rank side, while shared slave nonconforming faces are handled
-   // only on the slave side and therefore do not need additional filtering.
-   for (int sf = 0; sf < pmesh->GetNSharedFaces(); sf++)
-   {
-      const int f = pmesh->GetSharedFace(sf);
-      const auto info = pmesh->GetFaceInformation(f);
-      if (!info.IsShared())
-      {
-         continue;
-      }
-
-      FaceElementTransformations *FT = pmesh->GetSharedFaceTransformations(sf);
-      const int faceOrder = dim == 3 ? pfes->GetFaceOrder(f) :
-                            pfes->GetEdgeOrder(f);
-      const IntegrationRule &ir = IntRules.Get(FT->FaceGeom, 2 * faceOrder);
-
-      for (int i = 0; i < ir.GetNPoints(); i++)
-      {
-         const IntegrationPoint &fip = ir.IntPoint(i);
-         IntegrationPoint ip1, ip2;
-
-         FT->Loc1.Transform(fip, ip1);
-         FT->Loc2.Transform(fip, ip2);
-
-         const real_t v1 = x.GetValue(*FT->Elem1, ip1);
-         const real_t v2 = x.GetValue(*FT->Elem2, ip2);
-         errorMax = std::max(errorMax, std::abs(v1 - v2));
-      }
-   }
-
-   MPI_Allreduce(MPI_IN_PLACE, &errorMax, 1, MFEM_MPI_REAL_T, MPI_MAX,
-                 pmesh->GetComm());
-
-   return errorMax;
-}

miniapps/meshing/ref321.cpp

@@ -71,14 +71,22 @@ void Refine31(Mesh & mesh, int elem, char type)
    mesh.GeneralRefinement(refs);
 }
 
-// Randomly select elements for 3:1 refinements in random directions.
-void TestAnisoRefRandom(int num_refs, int dim, Mesh & mesh)
+// Deterministic, somewhat random integer generator
+int MyRand(int & s)
 {
-   std::mt19937 gen(1);
-   for (int i = 0; i < num_refs; i++)
+   s++;
+   const double a = 1000 * sin(s * 1.1234 * M_PI);
+   return int(std::abs(a));
+}
+
+// Randomly select elements for 3:1 refinements in random directions.
+void TestAnisoRefRandom(int iter, int dim, Mesh & mesh)
+{
+   int seed = 0;
+   for (int i = 0; i < iter; i++)
    {
-      const auto elem = gen() % mesh.GetNE();
-      const auto t = gen() % dim;
+      const int elem = MyRand(seed) % mesh.GetNE();
+      const int t = MyRand(seed) % dim;
       auto type = t == 0 ? Refinement::X :
                   (t == 1 ? Refinement::Y : Refinement::Z);
       Refine31(mesh, elem, type);

miniapps/multiapp/CMakeLists.txt

@@ -0,0 +1,52 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+set(MESH_FILES
+  backward-facing-step.msh
+  channel-cylinder.msh
+)
+
+# Add a target to copy the mesh files from the source directory; used by sample
+# runs.
+set(SRC_MESH_FILES)
+foreach(MESH_FILE ${MESH_FILES})
+  list(APPEND SRC_MESH_FILES ${CMAKE_CURRENT_SOURCE_DIR}/${MESH_FILE})
+endforeach()
+add_custom_command(OUTPUT data_is_copied
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different ${SRC_MESH_FILES} .
+  COMMAND ${CMAKE_COMMAND} -E touch data_is_copied
+  COMMENT "Copying multiapp miniapps data files ...")
+add_custom_target(copy_miniapps_multiapp_data DEPENDS data_is_copied)
+
+
+list(APPEND MULTIAPP_COMMON_SOURCES
+	multiapp.cpp)
+
+list(APPEND MULTIAPP_COMMON_HEADERS
+	multiapp.hpp)
+
+set(MULTIAPP_COMMON_FILES
+	EXTRA_SOURCES ${MULTIAPP_COMMON_SOURCES}
+	EXTRA_HEADERS ${MULTIAPP_COMMON_HEADERS})
+
+
+# Parallel apps.
+if (MFEM_USE_MPI)
+  add_mfem_miniapp(coupled-diffusion
+    MAIN coupled-diffusion.cpp
+    ${MFEM_MINIAPPS_COMMON_HEADERS} ${MULTIAPP_COMMON_FILES}
+    LIBRARIES mfem-common)
+  add_dependencies(coupled-diffusion copy_miniapps_multiapp_data)
+
+# Add parallel tests.
+  # if (MFEM_ENABLE_TESTING)
+  # endif()
+endif()

miniapps/multiapp/coupled-diffusion.cpp

@@ -0,0 +1,895 @@
+#include "mfem.hpp"
+#include "multiapp.hpp"
+#include <fstream>
+using namespace mfem;
+using namespace std;
+
+struct CaseContext
+{
+   int ser_ref = 1;         // Serial mesh refinement
+   int order = 3;           // Finite element order
+   bool visualization = true;// Visualization on/off
+   int grad_mode = 1;       // Gradient mode for the coupled operator - 0: finite difference,
+                            //                                          1: back/forward propagation
+   bool coupled = true;     // Coupled (true) vs. uncoupled (false) solves
+   int nl_iter = 50;        // Maximum number of nonlinear iterations
+   int lin_iter = 2000;     // Maximum number of linear iterations
+
+#if defined(MFEM_USE_DOUBLE)
+   real_t tol_nsolve = 1e-4;
+   real_t tol_lsolve = 1e-6;
+#elif defined(MFEM_USE_SINGLE)
+   real_t tol_nsolve = 1e-3;
+   real_t tol_lsolve = 1e-3;
+#else
+#error "Only single and double precision are supported!"
+   real_t tol_nsolve = 0;
+   real_t tol_lsolve = 0;
+#endif
+} ctx;
+
+void SetSolverParameters(IterativeSolver *solver, real_t rtol, real_t atol , int max_it,
+                         int print_level, bool iterative_mode);
+
+
+/// A functional diffusion coefficient (i.e., k(T))
+class FunctionalCoefficient : public Coefficient
+{
+public:
+   enum Mode { FUNC = 0, GRAD = 1};
+
+protected:
+   ParGridFunction *T_gf = nullptr;
+   real_t kref = 1.0;
+   real_t a0 = 0.0, a1 = 0.0, a2 = 0.0;
+   int findex = 0;
+   Mode mode = Mode::FUNC; // otherwise, grad
+
+public:
+   FunctionalCoefficient(ParGridFunction *T_gf, real_t kref): 
+                         T_gf(T_gf), kref(kref) { }
+
+   FunctionalCoefficient(ParGridFunction *T_gf, real_t kref, real_t a0):
+                         T_gf(T_gf), kref(kref), a0(a0) { findex = 1; }
+
+   FunctionalCoefficient(ParGridFunction *T_gf, real_t kref,
+                         real_t a0, real_t a1, real_t a2): T_gf(T_gf),
+                         kref(kref), a0(a0), a1(a1), a2(a2) { findex = 2; }
+
+    real_t Exponential(real_t x, bool eval_f) const
+    { 
+      real_t f = kref*exp(a0*x);
+      return (eval_f ? f : a0*f);
+    }
+    real_t Polynomial(real_t x, bool eval_f) const
+    { 
+      return (eval_f ? kref*(a0 + a1*x + a2*x*x) : kref*(a1 + 2*a2*x));
+    }
+
+    void SetMode(Mode mode) { this->mode = mode; }
+    Mode GetMode() const { return mode; }
+    
+    void UpdateGridFunction(ParGridFunction *gf) { T_gf = gf; }
+
+    real_t Eval(real_t x, bool eval_f) const
+    {
+        switch (findex)
+        {
+        case 1:
+         return Exponential(x, eval_f);
+        case 2:
+         return Polynomial(x, eval_f);
+        default:
+         return kref;
+        }
+    }
+
+    real_t Eval(ElementTransformation &Tr,
+              const IntegrationPoint &ip) override
+    {
+        real_t T = T_gf ? T_gf->GetValue(Tr, ip) : 0.0;
+        bool eval_f = (mode == Mode::FUNC);
+        return Eval(T, eval_f);
+    }
+};
+
+
+/// A coefficient defined by the product of grid functions, e.g. k(T) = prod_i x_i
+class GridFunctionProductCoefficient : public Coefficient
+{
+protected:
+   std::vector<ParGridFunction*> &x;
+
+public:
+   GridFunctionProductCoefficient(std::vector<ParGridFunction*> &x) : x(x) { }
+
+   real_t Eval(ElementTransformation &Tr, const IntegrationPoint &ip) override
+   {
+      real_t prod = 1.0;
+      for(size_t i = 0; i < x.size(); i++)
+      {
+         real_t val = x[i]->GetValue(Tr, ip);
+         prod *= val;
+      }
+      return prod;
+   }
+};
+
+class CoefficientIntegrator : public NonlinearFormIntegrator
+{
+protected:
+   FunctionalCoefficient *func = nullptr;
+   Vector shape;
+
+public:
+   CoefficientIntegrator(FunctionalCoefficient *func) : func(func) { }
+
+
+   void SetCoefficient(FunctionalCoefficient *f) { func = f; }
+
+   void AssembleElementVector(const FiniteElement &el,
+                              ElementTransformation &Tr,
+                              const Vector &elfun, Vector &elvect)
+   {
+      int dof = el.GetDof();
+      shape.SetSize(dof);
+      elvect.SetSize(dof);
+      elvect = 0.0;
+
+      const IntegrationRule *ir = &el.GetNodes();
+
+      for (int i = 0; i < ir->GetNPoints(); i++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(i);
+         el.CalcShape(ip, shape);
+         Tr.SetIntPoint(&ip);
+         real_t x = elfun * shape; // Evaluate the function at the integration point
+         real_t fval = func->Eval(x, true);
+         for (int j = 0; j < dof; j++)
+         {
+            elvect(j) += fval * shape(j);
+         }
+      }
+   }
+
+   void AssembleElementGrad(const FiniteElement &el, ElementTransformation &Tr,
+                            const Vector &elfun, DenseMatrix &elmat)
+   {
+      int dof = el.GetDof();
+      shape.SetSize(dof);
+      elmat.SetSize(dof);
+      elmat = 0.0;
+
+      const IntegrationRule *ir = &el.GetNodes();
+
+      for (int i = 0; i < ir->GetNPoints(); i++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(i);
+         el.CalcShape(ip, shape);
+         Tr.SetIntPoint(&ip);
+         real_t x = elfun * shape; // Evaluate the function at the integration point
+         real_t dfdx = func->Eval(x, false); // Evaluate the derivative of the function at the integration point
+         for (int j = 0; j < dof; j++)
+         {
+            elmat(j,j) += dfdx * shape(j); // Diagonal contribution to the Jacobian
+         }
+      }
+   }
+};
+
+
+class NonlinearDiffusionIntegrator : public NonlinearFormIntegrator
+{
+protected:
+   Coefficient *k;
+   Coefficient *dk;
+
+   Vector u, vec, shape;
+   DenseMatrix dshape, dshapedxt, adjJ;
+public:
+   NonlinearDiffusionIntegrator(Coefficient *kappa, Coefficient *dkappa) :
+                                k(kappa), dk(dkappa) { }
+
+   virtual void AssembleElementVector(const FiniteElement &el,
+                                      ElementTransformation &Tr,
+                                      const Vector &elfun, Vector &elvec)
+   {
+      int dim = el.GetDim();
+      int dof = el.GetDof();
+      real_t w;
+
+      elvec.SetSize(dof);
+      elvec = 0.0;
+
+      const IntegrationRule *ir = IntRule ? IntRule : &IntRules.Get(el.GetGeomType(), 2*el.GetOrder());
+      u.SetSize(dim);
+      vec.SetSize(dim);
+      dshape.SetSize(dof, dim);
+      adjJ.SetSize(dim, dim);
+
+      for (int i = 0; i < ir->GetNPoints(); i++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(i);
+         el.CalcDShape(ip, dshape);
+
+         Tr.SetIntPoint(&ip);
+         CalcAdjugate(Tr.Jacobian(), adjJ);
+         w = ip.weight / Tr.Weight();
+
+         dshape.MultTranspose(elfun, u);
+         adjJ.MultTranspose(u, vec);
+         if(k)
+         {
+            w *= k->Eval(Tr, ip);
+         }
+
+         vec *= w;
+         adjJ.Mult(vec, u);
+         dshape.AddMult(u, elvec);
+      }
+   }
+
+   void AssembleElementGrad(const FiniteElement &el, ElementTransformation &Tr,
+                            const Vector &elfun, DenseMatrix &elmat)
+   {
+      int dim = el.GetDim();
+      int dof = el.GetDof();
+      real_t w, k0 = 0.0, dk0 = 0.0;
+
+      elmat.SetSize(dof);
+      elmat = 0.0;
+
+      const IntegrationRule *ir = IntRule ? IntRule : &IntRules.Get(el.GetGeomType(), 2*el.GetOrder());
+      u.SetSize(dim);
+      shape.SetSize(dof);
+      vec.SetSize(dof);
+      dshape.SetSize(dof, dim);
+      dshapedxt.SetSize(dof, dim);
+
+      // f = grad(psi) * k(u) * grad(T)
+      // df/dT = grad(psi) ( k(u0) * grad(T) + k'(u0) * grad(u0) * T )
+      for (int i = 0; i < ir->GetNPoints(); i++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(i);
+         el.CalcShape(ip, shape);
+         el.CalcDShape(ip, dshape);
+
+         Tr.SetIntPoint(&ip);
+         w = ip.weight / Tr.Weight();
+
+         Mult(dshape, Tr.AdjugateJacobian(), dshapedxt);
+
+         k0  = k ? k->Eval(Tr, ip) : 0.0;
+         dk0 = dk ? dk->Eval(Tr, ip) : 0.0;
+
+         if(k0 != 0.0) // grad(psi) * k(u0) * grad(T)
+         {
+            real_t kdT = w*k0;
+            AddMult_a_AAt(kdT, dshapedxt, elmat);
+         }
+
+         if(dk0 != 0.0) // grad(psi) * (k'(T0) * grad(T0)) * T
+         {
+            dk0 = w*dk->Eval(Tr, ip);
+            dshapedxt.MultTranspose(elfun, u); // grad(T0) in physical space
+            u *= dk0; // k'(T0) * grad(T0)
+            dshapedxt.Mult(u, vec); // grad(psi) * k'(T0) * grad(T0)
+            AddMultVWt(vec, shape, elmat); // grad(psi) * k'(T0) * grad(T0) * T
+         }
+      }
+   }
+};
+
+/// An application that takes an input field T, and computes an output field k(T)
+// represented by the FunctionalCoefficient class.
+class DiffusionCoefficient : public GraphNode
+{
+public:
+   using Mode = FunctionalCoefficient::Mode;
+
+protected:
+   ParFiniteElementSpace &fes;
+   mutable ParGridFunction T, k;
+   mutable FunctionalCoefficient *kc;
+   // mutable Vector tdof, kdof, dk_dof, dT_dof;
+   mutable Mode mode = Mode::FUNC;
+
+   mutable ParNonlinearForm Nform;
+   mutable Operator *J = nullptr; // Jacobian for the nonlinear form
+
+   CoefficientIntegrator *coeff_integrator = nullptr;
+
+public:
+   DiffusionCoefficient(ParFiniteElementSpace &fes) :
+                        GraphNode(fes.GetTrueVSize()), fes(fes), T(&fes), k(&fes),
+                        kc(new FunctionalCoefficient(&T, 1.0, 5.0e-2)),
+                        Nform(&fes),
+                        coeff_integrator(new CoefficientIntegrator(kc))
+   {
+      k = 0.0;
+      T = 0.0;
+      k.ProjectCoefficient(*kc);
+
+      // Testing with the nonlinear form framework to compute k(T) and dk/dT
+      Nform.AddDomainIntegrator(coeff_integrator); // Transfer ownership
+      Nform.SetGradientType(Operator::Type::Hypre_ParCSR);
+      Nform.Setup();
+
+      SetInputOffsets(Array<int>({0, fes.GetTrueVSize()}));
+      SetOutputOffsets(Array<int>({0, fes.GetTrueVSize()}));
+   }
+
+   void SetMode(Mode mode) { this->mode = mode; }
+
+   FunctionalCoefficient* GetCoefficient() { return kc; }
+
+   void SetCoefficient(FunctionalCoefficient *fc)
+   {
+      if(kc) delete kc;
+      kc = fc;
+      kc->SetMode(mode);
+      kc->UpdateGridFunction(&T);
+      coeff_integrator->SetCoefficient(kc);
+   }
+
+   void Mult(const Vector &x, Vector &y) const override
+   {
+      BlockVector xb(x.GetData(), InputOffsets());
+      BlockVector yb(y.GetData(), OutputOffsets());
+
+      MultiVector xmv(1), ymv(1);
+      xmv.MakeRef(0, xb.GetBlock(0));
+      ymv.MakeRef(0, yb.GetBlock(0));
+
+      const_cast<DiffusionCoefficient*>(this)->Mult(xmv, ymv);
+   }
+
+   void Mult(const MultiVector &x, MultiVector &y) override
+   {
+      const Vector &tdof = x[0];
+      Vector &kdof = y[0];
+
+      Nform.Mult(tdof, kdof);
+      if(exec_mode == GraphNode::GRADIENT_MODE)
+      {
+         J = &Nform.GetGradient(tdof); // Store jacobian for JVP
+      }
+      else
+      {
+         J = nullptr; // Clear the Jacobian if not in gradient mode
+      }
+   }
+
+   // TODO: Possibly delete and only support MultiVector version of GradientMult
+   void GradientMult(const Vector &x, const Vector &dx, Vector &dy) const override
+   {
+      BlockVector xb(x.GetData(), InputOffsets());
+      BlockVector dxb(dx.GetData(), InputOffsets());
+      BlockVector dyb(dy.GetData(), OutputOffsets());
+
+      MultiVector xmv(1), dxmv(1), dymv(1);
+      xmv.MakeRef(0, xb.GetBlock(0));
+      dxmv.MakeRef(0, dxb.GetBlock(0));
+      dymv.MakeRef(0, dyb.GetBlock(0));
+      const_cast<DiffusionCoefficient*>(this)->GradientMult(xmv, dxmv, dymv);
+   }
+
+   void GradientMult(const MultiVector &x, const MultiVector &dx, MultiVector &dy) const override
+   {
+      const Vector &tdof = x[0];
+      const Vector &xadj = dx[0];
+      Vector &yadj = dy[0];
+
+      if(J)
+      {
+         J->Mult(xadj, yadj);
+      }
+      else
+      {
+         J = &Nform.GetGradient(tdof); // Store jacobian for JVP
+         J->Mult(xadj, yadj);
+      }
+   }
+
+   ~DiffusionCoefficient() override
+   {
+      if(kc) delete kc;
+   }
+};
+
+/// An application that takes n input fields x_i, and computes an output 
+/// field prod(x) := y = prod_i x_i.
+/// Also provides the derivative dy/dx_i = prod_{j!=i} x_j * dx_i/dx for i = 0,...,n-1.
+class ProductGridFunctions : public GraphNode
+{
+protected:
+
+   ParFiniteElementSpace &fes;
+   mutable std::vector<ParGridFunction*> x_gf;
+   mutable Vector dfdx;
+   mutable ParGridFunction y_gf;
+   mutable GridFunctionProductCoefficient prod_coeff;
+
+public:
+   ProductGridFunctions(ParFiniteElementSpace &fes, int n) :
+                     //   GraphNode(fes.GetTrueVSize()),
+                       GraphNode(fes.GetTrueVSize(), fes.GetTrueVSize() * n),
+                       fes(fes), x_gf(n),
+                       y_gf(&fes), prod_coeff(x_gf)
+   {
+      Array<int> offsets(n+1);
+      offsets[0] = 0;
+      for (int i = 0; i < n; i++)
+      {
+         x_gf[i]  = new ParGridFunction(&fes);
+         *x_gf[i] = 0.0;
+         offsets[i+1] = offsets[i] + fes.GetTrueVSize();
+      }
+      y_gf = 0.0;
+      y_gf.ProjectCoefficient(prod_coeff);
+
+      SetInputOffsets(offsets);
+      SetOutputOffsets(Array<int>({0, fes.GetTrueVSize()}));
+   }
+
+   void Mult(const Vector &x, Vector &y) const override
+   {
+      BlockVector xb(x.GetData(), InputOffsets());
+      BlockVector yb(y.GetData(), OutputOffsets());
+
+      MultiVector xmv(x_gf.size()), ymv(1);
+      for (size_t i = 0; i < x_gf.size(); i++)
+      {
+         xmv.MakeRef(i, xb.GetBlock(i));
+      }
+      ymv.MakeRef(0, yb.GetBlock(0));
+      const_cast<ProductGridFunctions*>(this)->Mult(xmv, ymv);
+   }
+
+   void Mult(const MultiVector &x, MultiVector &y) override
+   {
+      for (size_t i = 0; i < x_gf.size(); i++)
+      {
+         const Vector &x_dof = x[i];
+         x_gf[i]->SetFromTrueDofs(x_dof);
+      }
+
+      Field *out_field = OutputField(0);
+      Vector &y_dof = y[0];
+      y_gf.ProjectCoefficient(prod_coeff);
+      y_gf.GetTrueDofs(y_dof);
+   }
+
+   // TODO: Possibly delete and only support MultiVector version of GradientMult
+   void GradientMult(const Vector &x, const Vector &dx, Vector &dy) const override
+   {
+      BlockVector xb(x.GetData(), InputOffsets());
+      BlockVector dxb(dx.GetData(), InputOffsets());
+      BlockVector dyb(dy.GetData(), OutputOffsets());
+
+      MultiVector xmv(x_gf.size()), dxmv(x_gf.size()), dymv(1);
+      for (size_t i = 0; i < x_gf.size(); i++)
+      {
+         xmv.MakeRef(i, xb.GetBlock(i));
+         dxmv.MakeRef(i, dxb.GetBlock(i));
+      }
+      dymv.MakeRef(0, dyb.GetBlock(0));
+      const_cast<ProductGridFunctions*>(this)->GradientMult(xmv, dxmv, dymv);
+   }
+
+   void GradientMult(const MultiVector &x, const MultiVector &dx, MultiVector &dy) const override
+   {
+      // Jacobian vector product for y = prod_i x_i is:
+      // dy/dx = sum_i (prod_{j!=i} x_j * dx_i/dx)
+      for (size_t i = 0; i < x_gf.size(); i++)
+      {
+         const Vector &x_dof = x[i];
+         x_gf[i]->SetFromTrueDofs(x_dof); // Set all x_i
+      }
+
+      Vector &jvp = dy[0];
+      jvp = 0.0;
+      for (size_t i = 0; i < x_gf.size(); i++)
+      {
+         const Vector &x_dof = x[i];
+         const Vector &dx_dof = dx[i]; // Get dx_i/dx
+
+         x_gf[i]->SetFromTrueDofs(dx_dof); // Set x_i = dx_i/dx for i-th term in the product
+         y_gf.ProjectCoefficient(prod_coeff); // Recompute product with x_i replaced by dx_i/dx
+         y_gf.GetTrueDofs(dfdx); // Get prod_{j!=i} x_j * dx_i/dx for i-th term
+         jvp += dfdx; // Accumulate contribution from i-th term
+         x_gf[i]->SetFromTrueDofs(x_dof); // reset to original value for next iteration
+      }
+   }
+
+   ~ProductGridFunctions() override
+   {
+      for (size_t i = 0; i < x_gf.size(); i++)
+      {
+         if(x_gf[i]) delete x_gf[i];
+      }
+   }
+};
+
+
+/// An application that represents the nonlinear diffusion operator: f(T) = -Div(k(u) grad(T)) 
+/// with input field T and k, and output field f(T).
+class DiffusionOperator : public GraphNode
+{
+public:
+
+   // Mesh and finite element space
+   ParMesh &mesh;
+   ParFiniteElementSpace &fes;
+
+   /// Essential dof array.
+   Array<int> ess_tdofs;
+
+   /// Grid functions for the temperature and heat flux
+   mutable ParGridFunction T, k, dk;
+   mutable GridFunctionCoefficient k_gfc, dk_gfc;
+   mutable ParNonlinearForm Nform;
+   mutable ParLinearForm bform;
+   mutable Vector b;
+
+   ConstantCoefficient zero_coeff, one_coeff;
+
+   mutable FunctionalCoefficient *kc = nullptr;
+   mutable HypreParMatrix *dfdk_mat = nullptr, *dfdT_mat = nullptr;
+
+public:
+
+   DiffusionOperator(ParFiniteElementSpace &fes_) :
+                     // GraphNode(fes_.GetTrueVSize()),
+                     GraphNode(fes_.GetTrueVSize(),2*fes_.GetTrueVSize()),
+                     mesh(*fes_.GetParMesh()), fes(fes_),
+                     T(&fes), k(&fes), dk(&fes),
+                     k_gfc(&k), dk_gfc(&dk),
+                     Nform(&fes), bform(&fes),
+                     zero_coeff(0.0), one_coeff(1.0)
+   {
+      fes.GetBoundaryTrueDofs(ess_tdofs);
+      T = 0.0;
+      k = 0.0;
+      dk = 0.0;
+
+      bform.AddDomainIntegrator(new DomainLFIntegrator(one_coeff));
+      Nform.AddDomainIntegrator(new NonlinearDiffusionIntegrator(&k_gfc, &dk_gfc));
+      Nform.SetGradientType(Operator::Type::Hypre_ParCSR);
+
+      b.SetSize(fes.GetTrueVSize()); b = 0.0;
+      Assemble();
+
+      SetInputOffsets(Array<int>({0, fes.GetTrueVSize(), 2*fes.GetTrueVSize()}));
+      SetOutputOffsets(Array<int>({0, fes.GetTrueVSize()}));
+   }
+
+   void SetCoefficient(FunctionalCoefficient *fc) { kc = fc; }
+
+   void Assemble()
+   {
+      AssembleLinearForms();
+      AssembleBilinearForms();
+      AssembleNonlinearForms();
+   }
+
+   void AssembleBilinearForms()
+   {}
+
+   void AssembleNonlinearForms()
+   {
+      Nform.SetEssentialTrueDofs(ess_tdofs);
+      Nform.Setup();
+   }
+
+   void AssembleLinearForms()
+   {
+      bform.Assemble();
+      bform.ParallelAssemble(b);
+   }
+
+   void Mult(const Vector &x, Vector &y) const override
+   {
+      BlockVector xb(x.GetData(), InputOffsets());
+      BlockVector yb(y.GetData(), OutputOffsets());
+
+      MultiVector xmv(2), ymv(1);
+      xmv.MakeRef(0, xb.GetBlock(0));
+      xmv.MakeRef(1, xb.GetBlock(1));
+      ymv.MakeRef(0, yb.GetBlock(0));
+
+      const_cast<DiffusionOperator*>(this)->Mult(xmv, ymv);
+   }
+
+   void Mult(const MultiVector &x, MultiVector &y) override
+   {
+      const Vector &tdofs = x[0];
+      const Vector &kdofs = x[1];
+      Vector &fdofs = y[0];
+
+      k.SetFromTrueDofs(kdofs); // update for use in k_gfc
+
+      if(exec_mode == GraphNode::GRADIENT_MODE)
+      {
+         if(dfdT_mat) delete dfdT_mat;
+         if(dfdk_mat) delete dfdk_mat;
+
+         dk = 0.0;
+         k.SetFromTrueDofs(kdofs);
+         Operator* grad = &Nform.GetGradient(tdofs);
+         dfdT_mat = new HypreParMatrix(dynamic_cast<const HypreParMatrix&>(*grad)); // deep copy
+
+         dk = 1.0;
+         k  = 0.0;
+         grad = &Nform.GetGradient(tdofs);
+         dfdk_mat = new HypreParMatrix(dynamic_cast<const HypreParMatrix&>(*grad)); // deep copy
+      }
+      else
+      {
+         if(dfdT_mat) { delete dfdT_mat; dfdT_mat = nullptr; }
+         if(dfdk_mat) { delete dfdk_mat; dfdk_mat = nullptr; }
+      }
+
+      Nform.Mult(tdofs, fdofs);
+      fdofs.SetSubVector(ess_tdofs, 0.0);
+   }
+
+   // Exact block jacobian [df/dT, df/dk]
+   Operator& GetGradient(const Vector &x) const override
+   {
+      MFEM_ABORT("GetGradient not implemented for DiffusionOperator");
+   }
+
+   // TODO: Possibly delete and only support MultiVector version of GradientMult
+   void GradientMult(const Vector &x, const Vector &dx, Vector &dy) const override
+   {
+      BlockVector xb(x.GetData(), InputOffsets());
+      BlockVector dxb(dx.GetData(), InputOffsets());
+      BlockVector dyb(dy.GetData(), OutputOffsets());
+
+      Vector &Tadj = dxb.GetBlock(0);
+      Vector &kadj = dxb.GetBlock(1);
+      Vector &yadj = dyb.GetBlock(0);
+
+      Vector &tdofs = xb.GetBlock(0);
+      Vector &kdofs = xb.GetBlock(1);
+
+      dfdT_mat->Mult(Tadj, yadj);
+      dfdk_mat->AddMult(kadj, yadj);
+   }
+
+   void GradientMult(const MultiVector &x, const MultiVector &dx, MultiVector &dy) const override
+   {
+      const Vector &Tadj = dx[0];
+      const Vector &kadj = dx[1];
+      Vector &yadj = dy[0];
+
+      const Vector &tdofs = x[0];
+      const Vector &kdofs = x[1];
+
+      dfdT_mat->Mult(Tadj, yadj);
+      dfdk_mat->AddMult(kadj, yadj);
+   }
+
+   /// @brief Destroy the DiffusionOperator object
+   ~DiffusionOperator() override
+   {
+      if(dfdT_mat) delete dfdT_mat;
+      if(dfdk_mat) delete dfdk_mat;
+   }
+};
+
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   Hypre::Init();
+
+   using GradMode = DAGraph::GradMode;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&ctx.order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&ctx.visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&ctx.ser_ref, "-rs", "--serial-refine",
+                  "Number of times to refine the mesh in serial.");
+
+   args.AddOption(&ctx.grad_mode, "-gm", "--grad-mode",
+                  "Gradient mode for the coupled operator (0: exact, 1: finite difference, 2: algorithmic differentiation)");
+   args.AddOption(&ctx.coupled, "-cp", "--coupled", "-ucp", "--uncoupled",
+                  "Coupled (true) vs. uncoupled (false) solves.");
+   args.ParseCheck();
+
+
+   int order = ctx.order;
+   std::string mesh_file = "../../data/star.mesh";
+   Mesh *serial_mesh = new Mesh(mesh_file);
+   int dim = serial_mesh->Dimension();
+
+   for (int i = 0; i < ctx.ser_ref; ++i) { serial_mesh->UniformRefinement(); }
+   serial_mesh->SetCurvature(order, false, dim, Ordering::byNODES);
+
+   ParMesh pmesh(MPI_COMM_WORLD, *serial_mesh);
+   delete serial_mesh;
+   pmesh.UniformRefinement();
+
+   // Finite element spaces
+   H1_FECollection fec(order, dim);
+   ParFiniteElementSpace fes(&pmesh, &fec);
+
+   // Build all operator nodes
+   DiffusionCoefficient diff_coeff_1(fes);
+   diff_coeff_1.SetName("k(T1)");
+   diff_coeff_1.SetCoefficient(new FunctionalCoefficient(nullptr, 1.0, 3.5e-2));
+   // diff_coeff_1.SetCoefficient(new FunctionalCoefficient(nullptr, 1.0, 1.0, 0.1, 0.0));
+
+   DiffusionCoefficient diff_coeff_2(fes);
+   diff_coeff_2.SetName("k(T2)");
+   diff_coeff_2.SetCoefficient(new FunctionalCoefficient(nullptr, 1.0, 1.0, 2.0, 0.0));
+   // diff_coeff_2.SetCoefficient(new FunctionalCoefficient(nullptr, 1.5, 2.5e-2));
+
+   ProductGridFunctions prod_coeff(fes, 2);
+   prod_coeff.SetName("k(T1,T2)");
+
+   DiffusionOperator diff_op1(fes);
+   diff_op1.SetName("Div(k(T1,T2) grad(T1))");
+   diff_op1.SetCoefficient(diff_coeff_1.GetCoefficient());
+
+   DiffusionOperator diff_op2(fes);
+   diff_op2.SetName("Div(k(T1,T2) grad(T2))");
+   diff_op2.SetCoefficient(diff_coeff_2.GetCoefficient());
+
+
+   // Build the DAG in any order, and then sort it to ensure the correct execution order
+   DAGraph dag(5);
+   dag.AddOperator(&diff_coeff_1);
+   dag.AddOperator(&diff_op1, fes.GetTrueVSize());
+   dag.AddOperator(&diff_op2, fes.GetTrueVSize());
+   dag.AddOperator(&diff_coeff_2);
+   dag.AddOperator(&prod_coeff);
+
+   Vector k1vec(fes.GetTrueVSize()); k1vec = 0.0;
+   Vector k2vec(fes.GetTrueVSize()); k2vec = 0.0;
+   Vector kpvec(fes.GetTrueVSize()); kpvec = 0.0;
+
+   Vector k1adj(fes.GetTrueVSize()); k1adj = 0.0;
+   Vector k2adj(fes.GetTrueVSize()); k2adj = 0.0;
+   // Vector kpadj(fes.GetTrueVSize()); kpadj = 0.0;
+
+   // Input fields get data from 'x' in DAGraph::Mult(x, y)
+   Field T1_field(nullptr, nullptr);
+   Field T2_field(nullptr, nullptr);
+
+   // Write space for data and adjoint only needed
+   // for the intermediate fields k1, k2, and k_prod
+   Field k1_field(&k1vec, &k1adj);
+   Field k2_field(&k2vec, &k2adj);
+   Field kp_field(&kpvec, &kpvec); // can use same space for data & adjoint
+
+   // Output fields get data from 'y' in DAGraph::Mult(x, y)
+   Field f1_field(nullptr, nullptr);
+   Field f2_field(nullptr, nullptr);
+
+
+   // Add input and output to the DAG
+   int sz = fes.GetTrueVSize();
+   dag.AddInput(&T1_field, sz);
+   dag.AddInput(&T2_field, sz);
+   dag.AddOutput(&f1_field, sz);
+   dag.AddOutput(&f2_field, sz);
+
+   // Form connections between the nodes in the DAG
+   diff_coeff_1.AddInput(&T1_field);
+   diff_coeff_1.AddOutput(&k1_field);
+
+   diff_coeff_2.AddInput(&T2_field);
+   diff_coeff_2.AddOutput(&k2_field);
+
+   prod_coeff.AddInputs(&k1_field, &k2_field);
+   prod_coeff.AddOutput(&kp_field);
+
+   diff_op1.AddInput(&T1_field);
+   diff_op1.AddOutput(&f1_field);
+
+   diff_op2.AddInput(&T2_field);
+   diff_op2.AddOutput(&f2_field);
+
+   if(ctx.coupled)
+   {
+      diff_op1.AddInput(&kp_field); // kp_field
+      diff_op2.AddInput(prod_coeff.OutputField(0)); // Can also use kp_field directly
+   }
+   else
+   {
+      diff_op1.AddInput(&k1_field); // Can also use diff_coeff_1.OutputField(0)
+      diff_op2.AddInput(&k2_field); // Can also use diff_coeff_2.OutputField(0)
+   }
+
+   // Assemble DAG: topological sort, validate nodes, etc.
+   dag.Assemble();
+
+   std::string output_prefix = ctx.coupled ? "Coupled_Diffusion" : "Uncoupled_Diffusion";
+
+   if(Mpi::Root())
+   {
+      std::ofstream fout(output_prefix+"-dag.txt");
+      fout << "{\n";
+      dag.Save(fout);
+      fout << "}\n";
+      fout << std::flush;
+      fout.close();
+   }
+
+   // Set initial guess and boundary conditions for T1 and T2
+   Array<int> ess_tdofs;
+   fes.GetBoundaryTrueDofs(ess_tdofs);
+
+   int T1_idx = 0;
+   int T2_idx = 1;
+
+   BlockVector xb(dag.InputOffsets());
+   BlockVector yb(dag.OutputOffsets());
+
+   xb.GetBlock(T1_idx).Randomize();
+   xb.GetBlock(T2_idx).Randomize();
+   xb.GetBlock(T1_idx).SetSubVector(ess_tdofs, 0.0);
+   xb.GetBlock(T2_idx).SetSubVector(ess_tdofs, 0.0);
+
+   // Build the nonlinear solver and linear solver for the DAG
+   NewtonSolver newton_solver(pmesh.GetComm());
+   GMRESSolver linear_solver(pmesh.GetComm());
+   linear_solver.SetKDim(500);
+   SetSolverParameters(&newton_solver, ctx.tol_nsolve, 0.0, ctx.nl_iter, 1, true);
+   SetSolverParameters(&linear_solver, ctx.tol_lsolve, 0.0, ctx.lin_iter, 1, false);
+
+   newton_solver.SetPreconditioner(linear_solver);
+   linear_solver.SetPrintLevel(1);
+
+   // Set the gradient mode for the DAG and solve the coupled system
+   GradMode gm = static_cast<GradMode>(ctx.grad_mode);
+   dag.SetGradientMode(gm);
+   newton_solver.SetOperator(dag);
+   newton_solver.Mult(xb, yb);
+
+   ParaViewDataCollection *pv = nullptr;
+   if (ctx.visualization)
+   {
+      std::string pv_prefix;
+      switch (ctx.grad_mode)
+      {
+         case 0: pv_prefix = "FD"; break;
+         case 1: pv_prefix = "MF"; break;
+         default: pv_prefix = "Unknown"; break;
+      }
+
+      pv = new ParaViewDataCollection(output_prefix+"-"+pv_prefix, &pmesh);
+      pv->SetLevelsOfDetail(order);
+      pv->SetDataFormat(VTKFormat::BINARY);
+      pv->SetHighOrderOutput(true);
+
+      ParGridFunction T1_gf(&fes);
+      ParGridFunction T2_gf(&fes);
+      T1_gf.SetFromTrueDofs(yb.GetBlock(T1_idx));
+      T2_gf.SetFromTrueDofs(yb.GetBlock(T2_idx));
+
+      pv->RegisterField("T1", &T1_gf);
+      pv->RegisterField("T2", &T2_gf);
+      pv->Save();
+      delete pv;
+   }
+
+   std::cout << "Finished solving the coupled diffusion problem." << std::endl;
+   return 0;
+}
+
+void SetSolverParameters(IterativeSolver *solver, real_t rtol, real_t atol,
+                         int max_it, int print_level, bool iterative_mode)
+{
+    solver->SetRelTol(rtol);
+    solver->SetAbsTol(atol);
+    solver->SetMaxIter(max_it);
+    solver->SetPrintLevel(print_level);
+    solver->iterative_mode = iterative_mode;
+}

miniapps/multiapp/multiapp.cpp

@@ -0,0 +1,873 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "multiapp.hpp"
+
+namespace mfem
+{
+
+DAGraph::~DAGraph()
+{
+    for(int i=0; i < nnodes; i++)
+    {
+        if(node_owned[i] && nodes[i]) delete nodes[i];
+    }
+    if(grad) delete grad;
+}
+
+void DAGraph::Assemble()
+{
+    // Sort graph nodes topologically to ensure correct execution order
+    // Ordering is not unique, hence, id->index maps are needed
+    TopologicalSort();
+
+    // Collect all fields from the nodes into the field map
+    CollectFieldMaps();
+
+    // Compute depth of the graph nodes
+    ComputeDepth();
+
+    // Validate each node
+    for (auto &node : nodes)
+    {
+        ValidateNode(*node);
+    }
+
+    // Update width and height of the DAG from offsets
+    // Check that the input and output offsets are consistent
+    ValidateOffsets();
+    width  = input_offsets.Last();
+    height = output_offsets.Last();
+
+    // Delete any existing gradient operator as node ordering may have changed
+    if (grad) delete grad;
+
+    assembled = true;
+}
+
+void DAGraph::ValidateOffsets()
+{
+    // Check that the input and output offsets are consistent
+    // with the number of inputs and outputs
+    if(InputFields().Size() > 1)
+    {
+        MFEM_ASSERT(input_offsets.Size() == InputFields().Size() + 1,
+                    "Input offsets size inconsistent with number of input fields");
+    }
+    else
+    {
+        input_offsets = Array<int>({0, nodes[0]->Width()});
+    }
+
+    if(OutputFields().Size() > 1)
+    {
+        MFEM_ASSERT(output_offsets.Size() == OutputFields().Size() + 1,
+                    "Output offsets size inconsistent with number of output fields");
+    }
+    else
+    {
+        output_offsets = Array<int>({0, nodes.Last()->Height()});
+    }
+}
+
+void DAGraph::ValidateNode(GraphNode &node)
+{
+    // Validate that the node's input and output fields are consistent with the graph's field map
+    auto inputs = node.InputFields();
+    auto outputs = node.OutputFields();
+
+    // Check offsets match width and height of the node
+    MFEM_ASSERT(node.InputOffsets().Last() == node.Width(),
+                "Node ID: " << node.ID() << " input offsets do not match node width.");
+    MFEM_ASSERT(node.OutputOffsets().Last() == node.Height(),
+                "Node ID: " << node.ID() << " output offsets do not match node height.");
+
+    // Check number of input and output fields match the offsets
+    MFEM_ASSERT(node.InputOffsets().Size() == inputs.Size() + 1,
+                "Node input offsets size inconsistent with number of input fields");
+    MFEM_ASSERT(node.OutputOffsets().Size() == outputs.Size() + 1,
+                "Node output offsets size inconsistent with number of output fields");
+
+    // Check that all input and output fields are registered in the graph's field map
+    for(auto input_field : inputs)
+    {
+        MFEM_ASSERT(fid_to_index.Has(input_field->ID()),
+                    "Input field ID " << input_field->ID() << " not found in graph's field map");
+    }
+    for(auto output_field : outputs)
+    {
+        MFEM_ASSERT(fid_to_index.Has(output_field->ID()),
+                    "Output field ID " << output_field->ID() << " not found in graph's field map");
+    }
+}
+
+void DAGraph::TopologicalSort()
+{
+    Array<int> sorted_indices;
+    sorted_indices.Reserve(nnodes);
+
+    Array<bool> visited(nnodes);
+    visited = false; // Initialize all nodes as unvisited
+
+    // Perform a depth-first search to sort the nodes topologically
+    std::function<void(int)> DepthFirstSearch = [&](int node_index)
+    {
+        if(visited[node_index]) return;
+        visited[node_index] = true;
+        auto node = nodes[node_index];
+        // Visit all nodes that this node depends on
+        for(auto input_field : node->InputFields())
+        {
+            for(int j=0; j < nnodes; j++)
+            {
+                auto other_node = nodes[j];
+                if(other_node == node) continue;
+                for(auto output_field : other_node->OutputFields())
+                {
+                    if(input_field->ID() == output_field->ID()) // Compare by unique ID
+                    {
+                        DepthFirstSearch(j);
+                    }
+                }
+            }
+        }
+        sorted_indices.push_back(node_index);
+    };
+
+    for(int i=0; i < nnodes; i++)
+    {
+        DepthFirstSearch(i);
+    }
+
+    nodes.Permute(sorted_indices);
+    node_owned.Permute(sorted_indices);
+
+    // Update the node indices after sorting
+    for(int i=0; i < nnodes; i++)
+    {
+        nodes[i]->SetNodeIndex(i);
+    }
+
+    sorted = true;
+}
+
+void DAGraph::ComputeDepth()
+{
+    // Compute depth of ordered nodes
+    node_depth.SetSize(nnodes);
+    node_depth = 0;
+    for(int i=0; i < nnodes; i++)
+    {
+        int max_depth = 0;
+        auto node = nodes[i];
+        for(auto input_field : node->InputFields())
+        {
+            for(int j=0; j < i; j++)
+            {
+                auto other_node = nodes[j];
+                if(other_node == node) continue;
+                for(auto output_field : other_node->OutputFields())
+                {
+                    if(input_field->ID() == output_field->ID()) // Compare by unique ID
+                    {
+                        max_depth = std::max(max_depth, node_depth[j] + 1);
+                    }
+                }
+            }
+        }
+        node_depth[i] = max_depth;
+    }
+}
+
+void DAGraph::CollectFieldMaps()
+{
+    MFEM_ASSERT(sorted, "DAGraph must be topologically sorted before collecting fields");
+
+    fid_to_index.clear();
+    fid_to_field.clear();
+
+    int nfields = 0;
+    for (auto f : InputFields())
+    {
+        fid_to_index.Register(f->ID(), nfields++);
+        fid_to_field.Register(f->ID(), f);
+    }
+
+    for (auto &node : nodes)
+    {
+        for (auto f : node->OutputFields())
+        {
+            if (!fid_to_index.Has(f->ID()))
+            {
+                fid_to_index.Register(f->ID(), nfields++);
+            }
+            if (!fid_to_field.Has(f->ID()))
+            {
+                fid_to_field.Register(f->ID(), f);
+            }
+        }
+    }
+    // TODO: Possibly add all intermediate fields from nodes to the graph's FieldCollection
+}
+
+void DAGraph::Mult(const Vector &x, Vector &y) const
+{
+    MFEM_ASSERT(width == x.Size(), "Input vector size (" << x.Size()
+                << ") must match matrix width (" << width << ")");
+
+    MFEM_ASSERT(height == y.Size(), "Output vector size (" << y.Size()
+                << ") must match matrix height (" << height << ")");
+
+    auto inputs  = InputFields();
+    auto outputs = OutputFields();
+
+    BlockVector xb(x.GetData(), input_offsets);
+    BlockVector yb(y.GetData(), output_offsets);
+    MultiVector xmv(inputs.Size()), ymv(outputs.Size());
+
+    // Set the data pointers of the input and output fields
+    // of the graph to point to the corresponding blocks of
+    // the input and output vectors
+    for(int i=0; i < inputs.Size(); i++)
+    {
+        xmv.MakeRef(i, xb.GetBlock(i));
+    }
+
+    for(int i=0; i < outputs.Size(); i++)
+    {
+        ymv.MakeRef(i, yb.GetBlock(i));
+    }
+
+    const_cast<DAGraph*>(this)->Mult(xmv, ymv);
+}
+
+void DAGraph::Mult(const MultiVector &x, MultiVector &y)
+{
+    auto inputs  = InputFields();
+    auto outputs = OutputFields();
+
+    MFEM_ASSERT(inputs.Size() == x.NumBlocks(), "Number of input blocks (" << x.NumBlocks()
+                << ") must match number of input fields (" << inputs.Size() << ")");
+
+    MFEM_ASSERT(outputs.Size() == y.NumBlocks(), "Number of output blocks (" << y.NumBlocks()
+                << ") must match number of output fields (" << outputs.Size() << ")");
+
+    for(int i=0; i < inputs.Size(); i++)
+    {
+        inputs[i]->SetData(const_cast<Vector*>(&x[i]));
+    }
+    for (int i=0; i < outputs.Size(); i++)
+    {
+        outputs[i]->SetData(&y[i]);
+    }
+
+    auto index_map = GetFieldIdToIndexMap();
+    auto fld_map = GetFieldIdToFieldMap();
+    int nfields = index_map.NumFields();
+    MultiVector ymv(nfields); // TODO: Should this be a member function?
+
+    // Assemble the multivector from the individual fields based on their IDs
+    // This multivector contains all input, output, and intermediate fields in the graph
+    for (auto const& [id, idx] : index_map)
+    {
+        if (fld_map.Has(id))
+        {
+            auto field = fld_map.Get(id);
+            ymv.MakeRef(idx, *field->Data());
+        }
+        else
+        {
+            MFEM_ABORT("Field ID " << id << " not found in field map");
+        }
+    }
+    
+    Execute(x, ymv);
+
+    for(auto &f : inputs)
+    {
+        f->SetData(nullptr);
+    }
+    for(auto &f : outputs)
+    {
+        f->SetData(nullptr);
+    }
+}
+
+void DAGraph::Execute(const MultiVector &x, MultiVector &y) const
+{
+    MFEM_ASSERT(assembled, "DAGraph must be assembled before calling Execute()");
+
+    MFEM_ASSERT(x.NumBlocks() == InputFields().Size(),
+                "Number of input blocks (" << x.NumBlocks()
+                << ") must match number of input fields (" << InputFields().Size() << ")");
+    
+    auto index_map = GetFieldIdToIndexMap();
+    
+    MFEM_ASSERT(y.NumBlocks() == index_map.NumFields(),
+                "Number of output blocks (" << y.NumBlocks()
+                << ") must match number of fields (" << index_map.NumFields() << ")");
+
+    auto inputs  = InputFields();
+    for(int i=0; i < inputs.Size(); i++)
+    {
+        int idx = index_map.Get(inputs[i]->ID());
+        if(&y[idx] != &x[i]) // copy data, if address is different
+        {
+            y[idx] = x[i];
+        }
+    }
+
+    if(input_type == InputType::VECTOR)
+    {
+        x_node.SetSize(MaxWidth());
+        y_node.SetSize(MaxHeight());
+
+        for (auto node : nodes)
+        {
+            x_node.SetSize(node->Width());
+            y_node.SetSize(node->Height());
+
+            // Assemble input fields into a single vector for the node
+            auto node_inputs = node->InputFields();
+            auto ioffsets = node->InputOffsets();
+            for (int i=0; i < node_inputs.Size(); i++)
+            {
+                auto in_field = node_inputs[i];
+                int idx = index_map.Get(in_field->ID());
+                x_node.SetVector(y[idx],ioffsets[i]);
+            }
+
+            node->Mult(x_node, y_node);
+
+            // Disassemble output vector back
+            auto node_outputs = node->OutputFields();
+            BlockVector ynb(y_node.GetData(), node->OutputOffsets());
+            for (int i=0; i < node_outputs.Size(); i++)
+            {
+                auto out_field = node_outputs[i];
+                int idx = index_map.Get(out_field->ID());
+                y[idx] = ynb.GetBlock(i);
+            }
+        }
+    }
+    else if(input_type == InputType::MULTIVECTOR)
+    {
+        for (auto node : nodes)
+        {
+            auto node_inputs = node->InputFields();
+            auto node_outputs = node->OutputFields();
+            xmv_node.SetNumBlocks(node_inputs.Size());
+            ymv_node.SetNumBlocks(node_outputs.Size());
+
+            for (int i=0; i < node_inputs.Size(); i++)
+            {
+                int idx = index_map.Get(node_inputs[i]->ID());
+                xmv_node.MakeRef(i, y[idx]);
+            }
+            for (int i=0; i < node_outputs.Size(); i++)
+            {
+                int idx = index_map.Get(node_outputs[i]->ID());
+                ymv_node.MakeRef(i, y[idx]);
+            }
+            node->Mult(xmv_node, ymv_node);
+        }
+    }
+    else if(input_type == InputType::NONE)
+    {
+        Vector x_unused, y_unused;
+        for (auto node : nodes)
+        {
+            node->Mult(x_unused, y_unused);
+        }
+    }
+    else
+    {
+        MFEM_ABORT("DAGraph::Execute() not implemented for input type: "
+                    << static_cast<int>(input_type));
+    }
+}
+
+Operator& DAGraph::GetGradient(const Vector &x) const
+{
+    // TODO: Should/could be removed
+    if(grad_mode == GradMode::FINITE_DIFF)
+    {
+        if(!grad)
+        {
+            grad = new future::FDJacobian(*this, x, 1e-6);
+        }
+        else
+        {
+            grad->GetGradient(x); // Update the FDJacobian with new point x
+        }
+        return *grad;
+    }
+
+    MFEM_ASSERT(static_cast<int>(grad_mode) < static_cast<int>(GradMode::NONE),
+                "DAGraph::GetGradient() called with invalid grad_mode: "
+                << static_cast<int>(grad_mode));
+
+    if(!grad)
+    {
+        grad = new GraphGradient(const_cast<DAGraph&>(*this));
+    }
+
+    if(grad_mode == GradMode::ASSEMBLED)
+    {
+        return grad->GetGradient(x); // Assemble the Jacobian matrix
+    }
+    else // GradMode::MATRIX_FREE
+    {
+        dynamic_cast<GraphGradient*>(grad)->Update(x); // Update the GraphGradient with new point x
+    }
+
+    return *grad;
+}
+
+GraphGradient::GraphGradient(DAGraph &dag) : Operator(dag.Height(), dag.Width()),
+                                             graph(&dag)
+{
+    MFEM_ASSERT(graph->IsAssembled(), "GraphGradient requires an assembled DAGraph.");
+    MFEM_ASSERT(graph->IsSorted(), "GraphGradient requires a topologically sorted DAGraph.");
+
+    auto index_map = graph->GetFieldIdToIndexMap();
+    auto field_map = graph->GetFieldIdToFieldMap();
+
+    MFEM_ASSERT(index_map.NumFields() == field_map.NumFields(),
+                "Mismatch in number of fields between index_map and field_map");
+
+    x_work.DeleteAll(); // Clear any existing pointers
+    x_work.SetSize(index_map.NumFields());
+    x_work = nullptr; // Initialize all pointers to nullptr
+    xlin.SetNumBlocks(index_map.NumFields());
+
+    for (auto const& [id, idx] : index_map)
+    {
+        MFEM_ASSERT(idx >= 0 && idx < x_work.Size(), "Index out of bounds for field ID: " << id);
+        MFEM_ASSERT(field_map.Has(id), "Field ID not found in field_map: " << id);
+
+        if(x_work[idx] == nullptr)
+        {
+            x_work[idx] = new Vector(); // Allocate a new Vector for this field
+        }
+        xlin.MakeRef(idx, *x_work[idx]); // Make xlin refer to the allocated Vector
+    }
+
+}
+
+void GraphGradient::Update(const Vector &x)
+{
+    MFEM_ASSERT(graph != nullptr, "GraphGradient operator requires a non-null DAGraph pointer.");
+
+    auto set_exec_mode = [&](DAGraph::ExecutionMode mode)
+    {
+        for (auto &node : graph->Nodes())
+        {
+            node->SetExecutionMode(mode);
+        }
+    };
+
+    auto inputs = graph->InputFields();
+    BlockVector xb(x.GetData(), graph->InputOffsets());
+    MultiVector xmv(inputs.Size());
+    for(int i=0; i < inputs.Size(); i++)
+    {
+        xmv.MakeRef(i, xb.GetBlock(i));
+    }
+
+    set_exec_mode(DAGraph::ExecutionMode::GRADIENT_MODE);
+    graph->Execute(xmv, xlin); // Forward pass to populate fields for gradient computations
+    set_exec_mode(DAGraph::ExecutionMode::DEFAULT_MODE); // Reset execution mode for forward pass
+}
+
+void GraphGradient::Mult(const Vector &x, Vector &y) const
+{
+    MFEM_ASSERT(x.Size() == graph->Width(), "Input vector size (" << x.Size()
+                << ") must match graph width (" << graph->Width() << ")");
+
+    MFEM_ASSERT(y.Size() == graph->Height(), "Output vector size (" << y.Size()
+                << ") must match graph height (" << graph->Height() << ")");
+
+    auto in_offsets  = graph->InputOffsets();
+    auto out_offsets = graph->OutputOffsets();
+
+    auto inputs  = graph->InputFields();
+    auto outputs = graph->OutputFields();
+
+    BlockVector xb(x.GetData(), in_offsets);
+    BlockVector yb(y.GetData(), out_offsets);
+    MultiVector xmv(inputs.Size()), ymv(outputs.Size());
+
+    for(int i=0; i < inputs.Size(); i++)
+    {
+        xmv.MakeRef(i, xb.GetBlock(i));
+    }
+
+    for(int i=0; i < outputs.Size(); i++)
+    {
+        ymv.MakeRef(i, yb.GetBlock(i));
+    }
+
+    const_cast<GraphGradient*>(this)->Mult(xmv, ymv); // Forward mode: compute JVP, y = J(z) * x
+}
+
+void GraphGradient::Mult(const MultiVector &x, MultiVector &y)
+{
+    auto inputs  = graph->InputFields();
+    auto outputs = graph->OutputFields();
+
+    MFEM_ASSERT(inputs.Size() == x.NumBlocks(), "Number of input blocks (" << x.NumBlocks()
+                << ") must match number of input fields (" << inputs.Size() << ")");
+
+    MFEM_ASSERT(outputs.Size() == y.NumBlocks(), "Number of output blocks (" << y.NumBlocks()
+                << ") must match number of output fields (" << outputs.Size() << ")");
+
+    for(int i=0; i < inputs.Size(); i++)
+    {
+        inputs[i]->SetAdjoint(const_cast<Vector*>(&x[i]));
+    }
+    for (int i=0; i < outputs.Size(); i++)
+    {
+        outputs[i]->SetAdjoint(&y[i]);
+    }
+
+    auto index_map = graph->GetFieldIdToIndexMap();
+    auto fld_map = graph->GetFieldIdToFieldMap();
+    int nfields = index_map.NumFields();
+    MultiVector ymv(nfields); // TODO: Should this be a member function?
+
+    // Assemble the multivector from the individual fields based on their IDs
+    // This multivector contains all input, output, and intermediate fields in the graph
+    for (auto const& [id, idx] : index_map)
+    {
+        if (fld_map.Has(id))
+        {
+            auto field = fld_map.Get(id);
+            ymv.MakeRef(idx, *field->Adjoint());
+        }
+        else
+        {
+            MFEM_ABORT("Field ID " << id << " not found in field map");
+        }
+    }
+
+    Forward(x, ymv); // Forward mode: compute JVP, y = J(z) * x
+
+    for (auto &f : inputs)
+    {
+        f->SetAdjoint(nullptr);
+    }
+    for (auto &f : outputs)
+    {
+        f->SetAdjoint(nullptr);
+    }
+}
+
+void GraphGradient::MultTranspose(const Vector &x, Vector &y) const
+{
+    MFEM_ASSERT(x.Size() == graph->Height(), "Input vector size (" << x.Size()
+                << ") must match graph height (" << graph->Height() << ")");
+    MFEM_ASSERT(y.Size() == graph->Width(), "Output vector size (" << y.Size()
+                << ") must match graph width (" << graph->Width() << ")");
+
+    auto in_offsets  = graph->InputOffsets();
+    auto out_offsets = graph->OutputOffsets();
+
+    auto inputs  = graph->InputFields();
+    auto outputs = graph->OutputFields();
+
+    BlockVector xb(x.GetData(), out_offsets);
+    BlockVector yb(y.GetData(), in_offsets);
+    MultiVector xmv(outputs.Size()), ymv(inputs.Size());
+
+    for(int i=0; i < inputs.Size(); i++)
+    {
+        xmv.MakeRef(i, xb.GetBlock(i));
+    }
+
+    for(int i=0; i < outputs.Size(); i++)
+    {
+        ymv.MakeRef(i, yb.GetBlock(i));
+    }
+    const_cast<GraphGradient*>(this)->MultTranspose(xmv, ymv); // Reverse mode: compute VJP, y = J(z)^T * x
+}
+
+void GraphGradient::MultTranspose(const MultiVector &x, MultiVector &y)
+{
+    auto inputs  = graph->InputFields();
+    auto outputs = graph->OutputFields();
+
+    MFEM_ASSERT(outputs.Size() == x.NumBlocks(), "Number of input blocks (" << x.NumBlocks()
+                << ") must match number of output fields (" << outputs.Size() << ")");
+
+    MFEM_ASSERT(inputs.Size() == y.NumBlocks(), "Number of output blocks (" << y.NumBlocks()
+                << ") must match number of input fields (" << inputs.Size() << ")");
+
+    for(int i=0; i < outputs.Size(); i++)
+    {
+        outputs[i]->SetAdjoint(const_cast<Vector*>(&x[i]));
+    }
+    for (int i=0; i < inputs.Size(); i++)
+    {
+        inputs[i]->SetAdjoint(&y[i]);
+    }
+
+    auto index_map = graph->GetFieldIdToIndexMap();
+    auto fld_map = graph->GetFieldIdToFieldMap();
+    int nfields = index_map.NumFields();
+    MultiVector ymv(nfields); // TODO: Should this be a member function?
+
+    for(auto const& [id, idx] : index_map)
+    {
+        if (fld_map.Has(id))
+        {
+            auto field = fld_map.Get(id);
+            ymv.MakeRef(idx, *field->Adjoint());
+        }
+        else
+        {
+            MFEM_ABORT("Field ID " << id << " not found in field map");
+        }
+    }
+
+    Reverse(x, ymv); // Reverse mode: compute VJP, y = J(z)^T * x
+
+    for (auto &f : outputs)
+    {
+        f->SetAdjoint(nullptr);
+    }
+    for (auto &f : inputs)
+    {
+        f->SetAdjoint(nullptr);
+    }
+}
+
+void GraphGradient::Forward(const MultiVector &x, MultiVector &y) const
+{
+    MFEM_ASSERT(x.NumBlocks() == graph->InputFields().Size(),
+                "Number of input blocks (" << x.NumBlocks()
+                << ") must match number of input fields (" << graph->InputFields().Size() << ")");
+    
+    auto in_type = graph->GetInputType();
+    auto index_map  = graph->GetFieldIdToIndexMap();
+    auto field_map = graph->GetFieldIdToFieldMap();
+
+    MFEM_ASSERT(y.NumBlocks() == index_map.NumFields(),
+                "Number of output blocks (" << y.NumBlocks()
+                << ") must match number of fields (" << index_map.NumFields() << ")");
+
+    auto inputs  = graph->InputFields();
+    for(int i=0; i < inputs.Size(); i++)
+    {
+        int idx = index_map.Get(inputs[i]->ID());
+        if(&y[idx] != &x[i]) // copy data, if address is different
+        {
+            y[idx] = x[i];
+        }
+    }
+
+    if(in_type == InputType::VECTOR)
+    {
+        x0.SetSize(graph->MaxWidth());
+        dx.SetSize(graph->MaxWidth());
+        dy.SetSize(graph->MaxHeight());
+
+        auto nodes = graph->Nodes();
+        for (auto node : nodes)
+        {
+            x0.SetSize(node->Width());
+            dx.SetSize(node->Width());
+            dy.SetSize(node->Height());
+
+            // Assemble input fields into a single vector for the node
+            auto node_inputs = node->InputFields();
+            auto ioffsets = node->InputOffsets();
+            for(int i=0; i < node_inputs.Size(); i++)
+            {
+                auto in_field = node_inputs[i];
+                MFEM_ASSERT(index_map.Has(in_field->ID()), "Input field ID not found in index_map");
+                int idx = index_map.Get(in_field->ID());
+                x0.SetVector(xlin[idx], ioffsets[i]);
+                dx.SetVector(y[idx], ioffsets[i]);
+            }
+
+            node->GradientMult(x0, dx, dy); // Compute JVP for the node
+
+            // Disassemble output vector back
+            auto node_outputs = node->OutputFields();
+            BlockVector ynb(dy.GetData(), node->OutputOffsets());
+            for(int i=0; i < node_outputs.Size(); i++)
+            {
+                auto out_field = node_outputs[i];
+                MFEM_ASSERT(index_map.Has(out_field->ID()), "Output field ID not found in index_map");
+                int idx = index_map.Get(out_field->ID());
+                y[idx] = ynb.GetBlock(i);
+            }
+        }
+    }
+    else if(in_type == InputType::MULTIVECTOR)
+    {
+        auto nodes = graph->Nodes();
+        for (auto node : nodes)
+        {
+            auto node_inputs = node->InputFields();
+            auto node_outputs = node->OutputFields();
+            x0_mv.SetNumBlocks(node_inputs.Size());
+            dx_mv.SetNumBlocks(node_inputs.Size());
+            dy_mv.SetNumBlocks(node_outputs.Size());
+
+            for(int i=0; i < node_inputs.Size(); i++)
+            {
+                int idx = index_map.Get(node_inputs[i]->ID());
+                x0_mv.MakeRef(i, xlin[idx]);
+                dx_mv.MakeRef(i, y[idx]);
+            }
+            for(int i=0; i < node_outputs.Size(); i++)
+            {
+                int idx = index_map.Get(node_outputs[i]->ID());
+                dy_mv.MakeRef(i, y[idx]);
+            }
+            node->GradientMult(x0_mv, dx_mv, dy_mv); // Compute JVP for the node
+        }
+    }
+    else if(in_type == InputType::NONE)
+    {
+        Vector x_unused, dx_unused, dy_unused;
+        auto nodes = graph->Nodes();
+        for (auto node : nodes)
+        {
+            node->GradientMult(x_unused, dx_unused, dy_unused);
+        }
+    }
+    else
+    {
+        MFEM_ABORT("GraphGradient::Forward() not implemented for input type: "
+                    << static_cast<int>(in_type));
+    }
+}
+
+void GraphGradient::Reverse(const MultiVector &x, MultiVector &y) const
+{
+    MFEM_ASSERT(x.NumBlocks() == graph->OutputFields().Size(),
+                "Number of input blocks (" << x.NumBlocks()
+                << ") must match number of output fields (" << graph->OutputFields().Size() << ")");
+
+    auto in_type = graph->GetInputType();
+    auto index_map  = graph->GetFieldIdToIndexMap();
+    auto field_map = graph->GetFieldIdToFieldMap();
+    int nnodes  = graph->Size();
+
+    MFEM_ASSERT(y.NumBlocks() == index_map.NumFields(),
+                "Number of output blocks (" << y.NumBlocks()
+                << ") must match number of fields (" << index_map.NumFields() << ")");
+
+    auto outputs  = graph->OutputFields();
+    for(int i=0; i < outputs.Size(); i++)
+    {
+        int idx = index_map.Get(outputs[i]->ID());
+        if(&y[idx] != &x[i]) // copy data, if address is different
+        {
+            y[idx] = x[i];
+        }
+    }
+
+    if(in_type == InputType::VECTOR)
+    {
+        x0.SetSize(graph->MaxWidth());
+        dx.SetSize(graph->MaxHeight());
+        dy.SetSize(graph->MaxWidth());
+
+        for (int i=nnodes-1; i >= 0; i--)
+        {
+            auto node = graph->GetNode(i);
+            x0.SetSize(node->Width());
+            dx.SetSize(node->Height());
+            dy.SetSize(node->Width());
+
+            auto node_inputs = node->InputFields();
+            auto ioffsets = node->InputOffsets();
+            for(int i=0; i < node_inputs.Size(); i++)
+            {
+                auto in_field = node_inputs[i];
+                MFEM_ASSERT(index_map.Has(in_field->ID()), "Input field ID not found in index_map");
+                int idx = index_map.Get(in_field->ID());
+                x0.SetVector(xlin[idx], ioffsets[i]);
+            }
+
+            auto node_outputs = node->OutputFields();
+            auto ooffsets = node->OutputOffsets();
+            for(int i=0; i < node_outputs.Size(); i++)
+            {
+                auto out_field = node_outputs[i];
+                MFEM_ASSERT(index_map.Has(out_field->ID()), "Output field ID not found in index_map");
+                int idx = index_map.Get(out_field->ID());
+                dx.SetVector(y[idx], ooffsets[i]);
+            }
+
+            node->GradientMultTranspose(x0, dx, dy); // Compute JVP for the node
+
+            BlockVector dynb(dy.GetData(), node->InputOffsets());
+            for(int i=0; i < node_inputs.Size(); i++)
+            {
+                int idx = index_map.Get(node_inputs[i]->ID());
+                y[idx] = dynb.GetBlock(i);
+            }
+        }
+    }
+    else if(in_type == InputType::MULTIVECTOR)
+    {
+        for (int i=nnodes-1; i >= 0; i--)
+        {
+            auto node = graph->GetNode(i);
+            auto node_inputs = node->InputFields();
+            auto node_outputs = node->OutputFields();
+            x0_mv.SetNumBlocks(node_inputs.Size());
+            dx_mv.SetNumBlocks(node_outputs.Size());
+            dy_mv.SetNumBlocks(node_inputs.Size());
+
+            for(int i=0; i < node_inputs.Size(); i++)
+            {
+                int idx = index_map.Get(node_inputs[i]->ID());
+                x0_mv.MakeRef(i, xlin[idx]);
+                dy_mv.MakeRef(i, y[idx]);
+            }
+            for(int i=0; i < node_outputs.Size(); i++)
+            {
+                int idx = index_map.Get(node_outputs[i]->ID());
+                dx_mv.MakeRef(i, y[idx]);
+            }
+            node->GradientMultTranspose(x0_mv, dx_mv, dy_mv); // Compute JVP for the node
+        }
+    }
+    else if(in_type == InputType::NONE)
+    {
+        Vector x_unused, dx_unused, dy_unused;
+        for (int i=nnodes-1; i >= 0; i--)
+        {
+            auto node = graph->GetNode(i);
+            node->GradientMultTranspose(x_unused, dx_unused, dy_unused); // Compute VJP for the node
+        }
+    }
+    else
+    {
+        MFEM_ABORT("GraphGradient::Reverse() not implemented for input type: "
+                    << static_cast<int>(in_type));
+    }
+}
+
+Operator& GraphGradient::GetGradient(const Vector &x) const
+{
+    // Used to build Jacobian matrix
+    MFEM_ABORT("GraphGradient::GetGradient() not implemented");
+}
+
+
+} // namespace mfem

miniapps/multiapp/multiapp.hpp

@@ -0,0 +1,838 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+
+#ifndef MFEM_MULTIAPP_HPP
+#define MFEM_MULTIAPP_HPP
+
+#include "mfem.hpp"
+
+namespace mfem
+{
+
+/// Forward declarations needed below
+class Field;
+class FieldCollection;
+class GraphNode;
+class DAGraph;
+class GraphGradient;
+
+
+/// @brief Base class for storing data (Vector) and distinguishing
+/// fields variables
+class Field
+{
+public:
+    enum Type ///< Not used for now, but could be used to distinguish between input/output fields
+    {
+        INPUT , ///< Input field
+        OUTPUT, ///< Output field
+        DEFAULT ///< Any field
+    };
+
+    friend class GraphNode;
+
+private:
+    Type type = Type::DEFAULT;
+    inline static int next_id = 0;
+
+protected:
+    Vector *data = nullptr;
+    Vector *adjoint = nullptr; // For storing derivative info
+    int id = -1; // initialized to invalid id
+
+    std::string name; // Optional name for the field
+    Operator *oper  = nullptr; // Operator that outputs this field
+
+    int GetValidID(int id_, int lb=0, int ub = std::numeric_limits<int>::max())
+    {
+        return (id_ >= lb && id_ <= ub) ? id_ : next_id++;
+    }
+
+public:
+
+    ///@brief Constructor for a Field of type Type with optional ID
+    Field(Vector *field, Vector *adjoint, Type type, int id_ = -1) :
+          type(type), data(field), adjoint(adjoint), id(GetValidID(id_)),
+          name("Field_" + std::to_string(id)) { }
+
+    ///@brief Constructor for a Field of Default type with optional ID
+    Field(Vector *field, Vector *adjoint, int id_ = -1) : 
+          Field(field, adjoint, Type::DEFAULT, id_) { }
+
+    ///@brief Constructor for an input field
+    Field(Vector *field, int id_ = -1) :
+          Field(field, nullptr, Type::DEFAULT, id_) { }
+
+    ///@brief Constructor for a Field of type Type
+    Field(Vector *field, Type type, int id_ = -1) :
+          Field(field, nullptr, type, id_) { }
+
+    ///@brief Get the stored internally stored data pointer
+    Vector* Data() const { return data; }
+    Vector* Adjoint() const { return adjoint; }
+    Operator* GetOperator() const { return oper; }
+
+    ///@brief Set the internally stored data pointer
+    virtual void SetData(Vector *field) { data = field; }
+    virtual void SetAdjoint(Vector *adj) { adjoint = adj; }
+    virtual void SetOperator(Operator *op) { oper = op; }
+
+    virtual void GetData(Vector &field) const { field = *data; }
+    virtual void GetAdjoint(Vector &adj) const { adj = *adjoint; }
+
+    std::string Name() const { return name; }
+    void SetName(const std::string &n) { name = n; }
+    int ID() const { return id; }
+
+    void SetID(int i)
+    {
+        MFEM_ASSERT(i >= 0, "ID must be non-negative.");
+        id = i;
+    }
+
+    bool IsInput() const {return (type == Type::INPUT);}
+    bool IsOutput() const {return (type == Type::OUTPUT);}
+    bool IsDefault() const {return (type == Type::DEFAULT);}
+
+    virtual ~Field() = default;
+
+protected:
+
+    ///@brief Set the type of the field (prevents changing type of input/output fields)
+    void SetType(Type t)
+    {
+        type = t;
+    }
+};
+
+/// @brief A collection of Fields, each identified by a name
+class FieldCollection
+{
+public:
+    using FieldMap = GenericFieldMap<std::string, Field*>;
+    using IndexMap = GenericFieldMap<std::string, int>;
+
+private:
+    std::string name; /// Name of the collection
+    Operator *oper = nullptr; /// Operator associated with this collection (not owned)
+    FieldMap fields; /// Map from field name to Field pointer
+    IndexMap index_map; /// Map from field name to index in input/output vectors
+
+    Array<Field*> input_fields;  // Input fields for this node
+    Array<Field*> output_fields; // Output fields for this node
+
+public:
+
+    FieldCollection() = default;
+
+    /// @brief Constructor with collection name and optional associated operator
+    FieldCollection(std::string collection_name, Operator *op = nullptr):
+                    name(collection_name), oper(op) {}
+
+    /// @brief Constructor with associated operator and default collection name
+    FieldCollection(Operator *op) : name("FieldCollection"), oper(op) {}
+
+    /// @brief Get the number of fields in the collection
+    int Size() const { return fields.NumFields(); }
+
+    /// @brief Set the name of the collection
+    void SetName(const std::string &collection_name) { name = collection_name;}
+
+    /// @brief Get the name of the collection
+    std::string Name() const { return name; }
+
+    /// @brief Set the operator associated with this collection
+    void SetOperator(Operator *op){ oper = op; }
+
+    /// @brief Get the operator associated with this collection
+    const Operator* GetOperator() const { return oper; }
+
+    /// @brief Get the field associated with the given name, or nullptr if not found
+    Field* GetField(const std::string &field_name) const
+    {
+        return fields.Get(field_name);
+    }
+
+    /// @brief Add a field to the collection with a given name and ownership flag
+    void AddField(const std::string &field_name, Field *field, bool own = false)
+    {
+        if(fields.Has(field_name))
+        {
+            MFEM_WARNING("FieldCollection::AddField: Field with name "
+                         << field_name << " already exists. Replacing existing field.");
+        }
+        fields.Register(field_name, field, own);
+    }
+
+    void AddInput(const std::string &field_name,
+                  Field *field, bool own = false)
+    {
+        bool has_field = fields.Has(field_name);
+        bool has_index = index_map.Has(field_name);
+        if(has_field && has_index)
+        {
+            int i = index_map.Get(field_name);
+            input_fields[i] = field;
+        }
+        else
+        {
+            input_fields.push_back(field);
+            index_map.Register(field_name, input_fields.Size() - 1);
+        }
+        AddField(field_name, field, own);
+    }
+
+    void AddOutput(const std::string &field_name,
+                   Field *field, bool own = false)
+    {
+        bool has_field = fields.Has(field_name);
+        bool has_index = index_map.Has(field_name);
+        if(has_field && has_index)
+        {
+            int i = index_map.Get(field_name);
+            output_fields[i] = field;
+        }
+        else
+        {
+            output_fields.push_back(field);
+            index_map.Register(field_name, output_fields.Size() - 1);
+        }
+
+        AddField(field_name, field, own);
+        if(field->GetOperator() == nullptr)
+        {
+            field->SetOperator(oper);
+        }
+    }
+
+    Array<Field*>& InputFields() { return input_fields; }
+    Array<Field*>& OutputFields() { return output_fields; }
+
+    Field* InputField(int i) const { return input_fields[i]; }
+    Field *InputField(const std::string &field_name) const
+    {
+        bool has_index = index_map.Has(field_name);
+        if(!has_index)
+        {
+            MFEM_WARNING("FieldCollection::InputField: Field with name "
+                         << field_name << " does not exist in the collection.");
+            return nullptr;
+        }
+
+        int index = index_map.Get(field_name);
+        MFEM_VERIFY(index >= 0 && index < input_fields.Size(),
+                    "FieldCollection::InputField: Invalid index for field name: "
+                    << field_name << ".");
+        return input_fields[index];
+    }
+
+    Field* OutputField(int i) const { return output_fields[i]; }
+    Field *OutputField(const std::string &field_name) const
+    {
+        bool has_index = index_map.Has(field_name);
+        if(!has_index)
+        {
+            MFEM_WARNING("FieldCollection::OutputField: Field with name "
+                         << field_name << " does not exist in the collection.");
+            return nullptr;
+        }
+
+        int index = index_map.Get(field_name);
+        MFEM_VERIFY(index >= 0 && index < output_fields.Size(),
+                    "FieldCollection::OutputField: Invalid index for field name: "
+                    << field_name << ".");
+        return output_fields[index];
+    }
+
+    FieldMap &Fields() { return fields; }
+    FieldMap Fields() const { return fields; }
+
+    virtual void Save (std::ostream &out) const
+    {
+        out << "\"Fields\":\n";
+        out << "{\n";
+        for (auto f = fields.begin(); f != fields.end(); ++f)
+        {
+            std::string f_name = f->first;
+            Field *f_obj = f->second;
+            // out << "  " << f_name << ": ID " << f_obj->ID() << ",\n";
+            // out << f_obj->ID() << ": " << f_name << ",\n";
+            out << '\"' << f_obj->ID() << "\": \"" << f_name << "\"";
+            if(f != std::prev(fields.end())) out << ",";
+            out << "\n";
+        }
+        out << "},\n";
+
+        out << "\"Inputs\":\n";
+        out << "{\n";
+        for (int i = 0; i < input_fields.Size(); ++i)
+        {
+            Field *f_obj = input_fields[i];
+            out << '\"' << f_obj->ID() << "\": \"" << f_obj->Name() << "\"";
+            if(i != input_fields.Size() - 1) out << ",";
+            out << "\n";
+        }
+        out << "},\n";
+
+        out << "\"Outputs\":\n";
+        out << "{\n";
+        for (int i = 0; i < output_fields.Size(); ++i)
+        {
+            Field *f_obj = output_fields[i];
+            out << '\"' << f_obj->ID() << "\": \"" << f_obj->Name() << "\"";
+            if(i != output_fields.Size() - 1) out << ",";
+            out << "\n";
+        }
+        out << "}\n";
+    }
+
+    Field* HasField(const Field &field) const
+    {
+        for (auto f = fields.begin(); f != fields.end(); ++f)
+        {
+            if(f->second == &field)
+            {
+                return f->second;
+            }
+        }
+        return nullptr;
+    }
+
+    Field* HasField(const std::string &field_name) const
+    {
+        return fields.Get(field_name);
+    }
+
+    Field* HasField(const int id) const
+    {
+        for (auto f = fields.begin(); f != fields.end(); ++f)
+        {
+            if(f->second->ID() == id)
+            {
+                return f->second;
+            }
+        }
+        return nullptr;
+    }
+
+    ~FieldCollection(){}
+
+};
+
+
+class GraphNode : public Operator
+{
+public:
+    enum ExecutionMode
+    {
+        GRADIENT_MODE, ///< Node is being executed as part of a gradient evaluation
+        DEFAULT_MODE   ///< Node is being executed as default, operator evaluation
+    };
+
+private:
+    inline static int next_id = 0;
+
+protected:
+    int id = -1;
+    int node_index = -1;
+    mutable ExecutionMode exec_mode = DEFAULT_MODE;
+
+    std::string name;
+    mutable FieldCollection field_collection; ///< Collection of fields associated with this node
+
+    // Offsets to be used for operation on BlockVector
+    Array<int> input_offsets;  ///< Offsets for input fields
+    Array<int> output_offsets; ///< Offsets for output fields
+
+    int GetValidID(int id_, int lb=0, int ub = std::numeric_limits<int>::max())
+    {
+        return (id_ >= lb && id_ <= ub) ? id_ : next_id++;
+    }
+
+public:
+
+    GraphNode(int h, int w) : Operator(h,w), id(GetValidID(-1)),
+                              name("Node_" + std::to_string(id)),
+                              field_collection(this) { }
+
+    GraphNode(int s = 0) : GraphNode(s, s) { }
+
+    void SetNodeIndex(int index){ node_index = index; }
+    int GetNodeIndex() const { return node_index; }
+
+    void SetExecutionMode(ExecutionMode mode) { exec_mode = mode; }
+    ExecutionMode GetExecutionMode() const { return exec_mode; }
+
+    void SetName(const std::string &name_) { name = name_; }
+    std::string Name() const { return name; }
+
+    void SetID(int id_) { id = id_; }
+    int ID() const { return id; }
+
+    FieldCollection::FieldMap& Fields() { return field_collection.Fields(); }
+    Field* Fields(const std::string &f) { return field_collection.GetField(f); }
+
+    FieldCollection::FieldMap Fields() const { return field_collection.Fields(); }
+    Field* Fields(const std::string &f) const { return field_collection.GetField(f); }
+
+    Array<Field*>& InputFields() const { return field_collection.InputFields(); }
+    Array<Field*>& OutputFields() const { return field_collection.OutputFields(); }
+    Field* InputField(int i) const { return field_collection.InputField(i); }
+    Field* OutputField(int i) const { return field_collection.OutputField(i); }
+
+
+    virtual void AddInput(const std::string &field_name,
+                          Field *field, bool own = false)
+    { field_collection.AddInput(field_name, field, own); }
+
+    virtual void AddInput(Field *field, bool own = false)
+    { AddInput(field->Name(), field, own); }
+
+    template<bool OwnInputs = false,
+             typename... Args,
+             bool AreFields = std::conjunction<std::is_base_of<Field, std::remove_pointer_t<Args>> ...>::value,
+             typename std::enable_if<AreFields, bool>::type = true >
+    void AddInputs(Args... args)
+    {
+        ((AddInput(std::forward<Args>(args), OwnInputs)), ...);
+    }
+
+    virtual void AddOutput(const std::string &field_name,
+                           Field *field, bool own = false)
+    { field_collection.AddOutput(field_name, field, own); }
+
+    virtual void AddOutput(Field *field, bool own = false)
+    { AddOutput(field->Name(), field, own); }
+
+    template<bool OwnOutputs = false,
+             typename... Args,
+             bool AreFields = std::conjunction<std::is_base_of<Field, std::remove_pointer_t<Args>> ...>::value,
+             typename std::enable_if<AreFields, bool>::type = true >
+    void AddOutputs(Args... args)
+    {
+        ((AddOutput(std::forward<Args>(args), OwnOutputs)), ...);
+    }
+
+    virtual void Save (std::ostream &out) const
+    {
+        out << "\"Node-" << id << "\" : " << std::endl;
+        out << "{\n";
+        out << "\"Name\": \"" << name << "\",\n";
+        field_collection.Save(out);
+        out << "}";
+    }
+
+    virtual void Mult(const Vector &x, Vector &y) const override
+    {
+        MFEM_ABORT("GraphNode::Mult() not implemented");
+    }
+
+    virtual void Mult(const MultiVector &x, MultiVector &y) override
+    {
+        MFEM_ABORT("GraphNode::Mult(MultiVector) not implemented");
+    }
+
+    using Operator::GetGradient;
+
+    // TODO: Possibly remove this and only support MultiVector version of GradientMult
+    virtual void GradientMult(const Vector &x, const Vector &dx, Vector &dy) const
+    {
+        MFEM_ABORT("GraphNode::GradientMult() not implemented");
+        GetGradient(x).Mult(dx, dy);
+    }
+
+    virtual void GradientMult(const MultiVector &x, const MultiVector &dx, MultiVector &dy) const
+    {
+        MFEM_ABORT("GraphNode::GradientMult() not implemented");
+        GetGradient(x).Mult(dx, dy);
+    }
+
+    // TODO: Possibly remove this and only support MultiVector version of GradientMultTranspose
+    virtual void GradientMultTranspose(const Vector &x, const Vector &dx, Vector &dy) const
+    {
+        MFEM_ABORT("GraphNode::GradientMultTranspose() not implemented");
+        GetGradient(x).MultTranspose(dx, dy);
+    }
+
+    virtual void GradientMultTranspose(const MultiVector &x, const MultiVector &dx, MultiVector &dy) const
+    {
+        MFEM_ABORT("GraphNode::GradientMultTranspose() not implemented");
+        // GetGradient(x).MultTranspose(dx, dy); // Not yet implemented
+    }
+
+    /// @brief Return the input offsets for block starts.
+    Array<int>& InputOffsets() { return input_offsets; }
+
+    /// @brief Read only access to the input offsets for block starts.
+    const Array<int>& InputOffsets() const { return input_offsets; }
+
+    void SetInputOffsets(const Array<int> &offsets) { input_offsets = offsets; }
+
+    /// @brief Return the output offsets for block starts.
+    Array<int>& OutputOffsets() { return output_offsets; }
+
+    /// @brief Read only access to the output offsets for block starts.
+    const Array<int>& OutputOffsets() const { return output_offsets; }
+
+    void SetOutputOffsets(const Array<int> &offsets) { output_offsets = offsets; }
+
+    virtual ~GraphNode() = default;
+};
+
+
+/**
+   @brief An abstract, type-erased class to define the interface for 
+   operators, not inherited from @a GraphNode. It performs SFINAE 
+   checks for stored operator's member functions and override the Mult
+   to call the stored object's functions.
+ */
+template <typename OpType>
+class AbstractOperator : public GraphNode
+{
+protected:
+    /// Define a template class 'check' to test for the existence of member functions
+    template <typename C>
+    class CheckMember{
+        private:
+
+        /// @brief A type trait to check if the erased class has the function Mult
+        /// with the needed signatures.
+        template<class T>
+        using Mult = decltype(std::declval<T&>().Mult(std::declval<const Vector&>(),
+                                                      std::declval<Vector&>()));
+
+        template<class T>
+        using MultPtr = decltype(std::declval<T&>().Mult(std::declval<const int>(),
+                                                         std::declval<const real_t*>(),
+                                                         std::declval<const int>(),
+                                                         std::declval<real_t*>()));
+        // ---------------------------------------------------------------------
+        
+        template <typename T, template<typename> typename Func, typename R>
+        static constexpr auto Check(T*) -> typename std::is_same< Func<T>, R>::type;
+
+        template <typename, template<typename> typename, typename >
+        static constexpr std::false_type Check(...);
+
+        // --- Check for the existence of the member functions
+        typedef decltype(Check<C,Mult,void>(0)) Has_Mult;
+        typedef decltype(Check<C,MultPtr,void>(0)) Has_MultPtr;
+    public:
+        static constexpr bool HasMult  = Has_Mult::value;
+        static constexpr bool HasMultPtr  = Has_MultPtr::value;
+    };
+
+    OpType *op;  ///< Pointer to the operator
+
+public:
+
+    constexpr bool HasExecute(){return CheckMember<OpType>::HasStep;}
+    constexpr bool HasMult(){return CheckMember<OpType>::HasMult;}
+
+
+    /// @brief Constructor for the type-erased AbstractOperator class
+    AbstractOperator(OpType *op_, int h, int w) : GraphNode(h,w), op(op_)
+    { }
+
+    /// @brief Constructor for the type-erased AbstractOperator class.
+    AbstractOperator(OpType *op_, int s = 0) : AbstractOperator(op_,s,s) {}
+
+    /**
+       @brief Perform Mult operation with the stored operator, if it exists.
+     */
+    void Mult(const Vector &x, Vector &y) const override
+    {
+        if constexpr (CheckMember<OpType>::HasMult)
+        {
+            op->Mult(x,y);
+        }
+        else if constexpr (CheckMember<OpType>::HasMultPtr)
+        {
+            op->Mult(x.Size(), x.GetData(), y.Size(), y.GetData());
+        }
+        else
+        {
+            MFEM_ABORT("The AbstractOperator does not have the function, "
+                       "Mult(const Vector&, Vector&) or "
+                       "Mult(int, double*, int, double*).");
+        }
+    }
+};
+
+
+/**
+   @brief A class to store and coupled multiple operators together.
+ */
+class DAGraph : public GraphNode
+{
+public:
+
+    using IntToIntMap = GenericFieldMap<int, int>;
+    using IntToFieldMap = GenericFieldMap<int, Field*>;
+
+    enum class GradMode
+    {
+        FINITE_DIFF = 0,  ///< Finite difference Jacobian
+        MATRIX_FREE = 1,  ///< Matrix-free Jacobian
+        ASSEMBLED = 2,    ///< Assembled Jacobian
+        NONE = 3          ///< Not implemented
+    };
+
+    enum InputType
+    {
+        VECTOR,      ///< Asemble the input blockvector from individual fields
+        MULTIVECTOR, ///< Asemble the multivector from individual fields
+        NONE         ///< No input
+    };
+
+protected:
+    Array<GraphNode*> nodes; ///< Vector of individual operators
+    Array<bool> node_owned; ///< Whether the operators are owned
+    Array<int> node_depth; ///< Depth of each operator in the graph
+
+    int max_width  = 0;     ///< Largest operator width
+    int max_height = 0;     ///< Largest operator height
+    int nnodes     = 0;     ///< The number of nodes
+    bool sorted    = false; ///< True if the nodes are topologically sorted
+    bool assembled = false; ///< True if the graph is assembled
+
+    GradMode grad_mode = GradMode::MATRIX_FREE; ///< Gradient mode for the graph
+    mutable Operator *grad = nullptr; ///< Gradient operator
+
+    InputType input_type = InputType::MULTIVECTOR; ///< Input type for the graph
+    mutable Vector x_node, y_node; ///< Temporary vectors for evaluating nodes
+    mutable MultiVector xmv_node, ymv_node; ///< Temporary multivectors for evaluating nodes
+
+    IntToFieldMap fid_to_field; ///< Map from Field ID to Field pointer
+    IntToIntMap fid_to_index; ///< Map from ID to index in an array; needed since ordering is not unique
+
+    friend class GraphGradient;
+
+public:
+    /**
+       @brief Construct a new CoupledOperator object.
+       @param nop Total number of operators to couple
+     */
+    DAGraph(const int nop) : GraphNode()
+    {
+        nodes.Reserve(nop);
+        node_owned.Reserve(nop);
+    }
+
+    /**
+       @brief Construct a new CoupledOperator object for an 
+       abstract non/mfem operator.
+     */
+    template <class OpType>
+    DAGraph(const OpType &op) : DAGraph(1)
+    {
+        AddOperator(op);
+    }
+
+    /**
+       @brief Add an operator to the list of coupled operator and
+       return pointer to it. Not owned unless it's not derived from GraphNode.
+     */
+    template <class OpType>
+    GraphNode* AddOperator(OpType *op_, int h, int w)
+    {
+        // Add operator to list of operators
+        if constexpr(std::is_base_of<GraphNode, OpType>::value)
+        {
+            nodes.push_back(op_);
+            node_owned.Append(false);
+        } 
+        else
+        {
+            nodes.push_back(new AbstractOperator<OpType>(op_,h,w));
+            node_owned.Append(true);
+        }
+        nnodes++;
+
+        // Update size of the coupled operator and the block offsets
+        GraphNode* op = nodes.Last();
+        op->SetNodeIndex(nnodes-1); // Set the index of the operator
+
+        int ht = op->Height();
+        int wt = op->Width();
+
+        max_width = std::max(max_width, wt);
+        max_height = std::max(max_height, ht);
+        sorted = false;
+
+        return op;
+    }
+
+    /// @brief Add an operator to the list of coupled operator and return pointer to it.
+    template <class OpType>
+    GraphNode* AddOperator(OpType *op_, int s = 0) { return AddOperator(op_,s,s);}
+
+    /// @brief Get the number of coupled operators
+    int Size(){return nnodes;}
+
+    /// @brief Get the size of the largest operator
+    int MaxWidth() const {return max_width;}
+    int MaxHeight() const {return max_height;}
+
+    IntToIntMap &GetFieldIdToIndexMap() { return fid_to_index; }
+    IntToIntMap GetFieldIdToIndexMap() const { return fid_to_index; }
+
+    IntToFieldMap &GetFieldIdToFieldMap() { return fid_to_field; }
+    IntToFieldMap GetFieldIdToFieldMap() const { return fid_to_field; }
+
+    /// @brief Get the operator at index @a i
+    GraphNode* GetNode(const int i)
+    {
+        MFEM_ASSERT(i >= 0 && i < nnodes,
+               "index [" << i << "] is out of range [0," << nnodes << ")");
+        return nodes[i];
+    }
+
+    Array<GraphNode*>& Nodes() { return nodes; }
+
+    /// @brief Specify whether the operator at index @a i is owned.
+    void OwnNode(const int i, bool own = true)
+    {
+        MFEM_ASSERT(i >= 0 && i < nnodes,
+               "index [" << i << "] is out of range [0," << nnodes << ")");
+        node_owned[i] = own;
+    }
+
+    void Assemble();
+    bool IsAssembled() const { return assembled; }
+
+    void TopologicalSort();
+    bool IsSorted() const { return sorted; }
+
+    void ComputeDepth();
+
+    void ValidateOffsets();
+
+    void ValidateNode(GraphNode &node);
+
+    void CollectFieldMaps();
+
+    using GraphNode::AddInput;
+    void AddInput(Field *field, int sz, bool own = false)
+    {
+        if(input_offsets.Size() == 0)
+        {   // First entry
+            input_offsets.Append(0);
+        }
+        input_offsets.Append(input_offsets.Last() + sz);
+        AddInput(field, own);
+    }
+
+    using GraphNode::AddOutput;
+    void AddOutput(Field *field, int sz, bool own = false)
+    {
+        if(output_offsets.Size() == 0)
+        {   // First entry
+            output_offsets.Append(0);
+        }
+        output_offsets.Append(output_offsets.Last() + sz);
+        AddOutput(field, own);
+    }
+
+    /// @brief Set the gradient mode for the coupled operator
+    void SetGradientMode(GradMode mode)
+    {
+        if(mode != grad_mode)
+        {
+            if(grad) { delete grad; grad = nullptr; }
+            grad_mode = mode;
+        }
+    }
+
+    void SetInputType(InputType type) { input_type = type; }
+    InputType GetInputType() const { return input_type; }
+
+    /**
+       @brief Apply the operator to the vector @a x 
+       and return the result in @a y.
+     */
+    virtual void Mult(const Vector &x, Vector &y) const override;
+
+    virtual void Mult(const MultiVector &x, MultiVector &y) override;
+
+    virtual void Execute(const MultiVector &x, MultiVector &y) const;
+
+    virtual void Save (std::ostream &out) const
+    {
+        out << "\"DAGraph\":\n";
+        out << "{\n";
+        // out << "\"nodes\" : " << nnodes << ",\n";
+        out << "\"Nodes\":\n";
+        out << "{\n";
+        for (int i = 0; i < nodes.Size(); i++)
+        {
+            nodes[i]->Save(out);
+            if(i != nodes.Size()-1) out << ",";
+            out << "\n";
+        }
+        out << "},\n"; // End of Nodes
+        field_collection.Save(out);
+        out << "}\n";
+    }
+
+    Operator& GetGradient(const Vector &x) const override;
+
+    /// @brief Destroy the Coupled Application object
+    ~DAGraph();
+};
+
+
+
+class GraphGradient : public Operator
+{
+public:
+    using InputType = DAGraph::InputType;
+
+protected:
+    mutable DAGraph *graph = nullptr; ///< Pointer to the DAGraph for which this is the gradient operator
+    Array<Vector*> x_work; ///< Array to store linearization point (intermediate fields)
+    mutable MultiVector xlin;
+    mutable Vector x0, dx, dy;
+    mutable MultiVector x0_mv, dx_mv, dy_mv;
+
+public:
+    GraphGradient(DAGraph &dag);
+
+    void Update(const Vector &x);
+
+    void Mult(const Vector &x, Vector &y) const override;
+
+    void Mult(const MultiVector &x, MultiVector &y) override;
+
+    void MultTranspose(const Vector &x, Vector &y) const override;
+
+    void MultTranspose(const MultiVector &x, MultiVector &y);
+
+    Operator &GetGradient(const Vector &x) const override;
+
+    void Forward(const MultiVector &x, MultiVector &y) const;
+
+    void Reverse(const MultiVector &x, MultiVector &y) const;
+
+    ~GraphGradient()
+    {
+        for (auto &v : x_work)
+        {
+            if(v) { delete v; v = nullptr; }
+        }
+        x_work.DeleteAll();
+    }
+};
+
+} //mfem namespace
+
+#endif

miniapps/plasma/CMakeLists.txt

@@ -9,12 +9,6 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-add_mfem_miniapp(g_eqdsk_viewer
-  MAIN g_eqdsk_viewer.cpp
-  EXTRA_SOURCES g_eqdsk_data.cpp
-  EXTRA_HEADERS g_eqdsk_data.hpp plasma.hpp ${MFEM_MINIAPPS_COMMON_HEADERS}
-  LIBRARIES mfem mfem-common)
-
 if (MFEM_USE_MPI)
   list(APPEND PLASMA_COMMON_SOURCES)