initial design

Add support for cuDSS 0.8.0

.github/workflows/README.md

@@ -29,16 +29,12 @@ Runs a number of static repository-level sanity checks.
 
 - `branch-history` guards against accidental commits of large files using the `--history` option of the `config/githooks/pre-push` script.
 
-## `mfem-analysis.yml` (`build-analysis`)
-
-Checks if the code builds and satisfies minimal requirements.
-
-- `gitignore` builds hypre, METIS, and MFEM using `mfem/github-actions/build-hypre`, `mfem/github-actions/build-metis`, and `mfem/github-actions/build-mfem` and checks for correct `.gitignore` settings by running the `tests/scripts/gitignore` script.
-
 ## `builds-and-tests.yml`
 
 Runs a matrix of builds and tests runs with different compilers, OS, mfem/hypre settings, etc. Also processes and upload Codecov reports.
 
+One matrix job runs `tests/scripts/gitignore` after `make test-noclean` to check generated artifacts against `.gitignore`.
+
 Uses the following GitHub Actions from <https://github.com/mfem/github-actions>:
 
 - `mfem/github-actions/build-hypre`

.github/workflows/builds-and-tests.yml

@@ -111,6 +111,7 @@ jobs:
             build-system: make
             hypre-target: int64
             precision: fp64
+            gitignore-check: YES
           - os: ubuntu-latest
             target: opt
             codecov: NO
@@ -317,7 +318,13 @@ jobs:
       - name: tests
         if: matrix.build-system == 'make' && (matrix.target == 'opt' || matrix.os == 'ubuntu-latest')
         run: |
-          cd ${{ env.MFEM_TOP_DIR }} && make test
+          cd ${{ env.MFEM_TOP_DIR }}
+          if [[ "${{ matrix.gitignore-check }}" == "YES" ]]; then
+            make test-noclean
+          else
+            make test
+          fi
+        shell: bash
 
       - name: cmake checks
         if: matrix.build-system == 'cmake' && matrix.target == 'dbg'
@@ -369,3 +376,9 @@ jobs:
           directories: "fem general linalg mesh"
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+      - name: gitignore
+        if: matrix.gitignore-check == 'YES'
+        run: |
+          cd ${{ env.MFEM_TOP_DIR }}/tests/scripts
+          ./runtest gitignore

.github/workflows/codeql-analysis.yml

@@ -14,9 +14,19 @@ name: "Static Analysis"
 on:
   push:
     branches: ["master", "next"]
+    paths-ignore: &docs-only-paths
+      - "**/*.md"
+      - "doc/**"
+      - ".binder/**"
+      - "CITATION.cff"
+      - "LICENSE"
+      - "NOTICE"
+      - "CHANGELOG"
+      - "INSTALL"
   pull_request:
     # The branches below must be a subset of the branches above
     branches: ["master"]
+    paths-ignore: *docs-only-paths
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

.github/workflows/mfem-analysis.yml

@@ -1,102 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-name: "Build Analysis"
-
-permissions:
-  actions: write
-
-on:
-  push:
-    branches:
-      - master
-      - next
-  pull_request:
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-env:
-  HYPRE_ARCHIVE: v2.19.0.tar.gz
-  HYPRE_TOP_DIR: hypre-2.19.0
-  METIS_ARCHIVE: metis-4.0.3.tar.gz
-  METIS_TOP_DIR: metis-4.0.3
-  COVERAGE_ENV: mfem-coverage
-  MFEM_ACTIONS_VERSION: v2.7
-
-jobs:
-  gitignore:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: checkout MFEM
-        uses: actions/checkout@v6
-        with:
-          path: mfem
-
-      - name: Get MPI (Linux)
-        run: |
-          sudo apt-get install openmpi-bin libopenmpi-dev
-          export OMPI_MCA_rmaps_base_oversubscribe=1
-
-      - name: Cache Hypre Install
-        id: hypre-cache
-        uses: actions/cache@v5
-        with:
-          path: ${{ env.HYPRE_TOP_DIR }}
-          key: ${{ runner.os }}-ompi-build-${{ env.HYPRE_TOP_DIR }}-int32-fp64-${{ env.MFEM_ACTIONS_VERSION }}
-
-      - name: Get Hypre
-        if: steps.hypre-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-hypre@v2.7
-        with:
-          archive: ${{ env.HYPRE_ARCHIVE }}
-          dir: ${{ env.HYPRE_TOP_DIR }}
-          target: int32
-          precision: fp64
-
-      - name: Cache Metis Install
-        id: metis-cache
-        uses: actions/cache@v5
-        with:
-          path: ${{ env.METIS_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-${{ env.MFEM_ACTIONS_VERSION }}
-
-      - name: Install Metis
-        if: steps.metis-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.7
-        with:
-          archive: ${{ env.METIS_ARCHIVE }}
-          dir: ${{ env.METIS_TOP_DIR }}
-
-      # MFEM build and test
-      - name: build-mfem
-        uses: mfem/github-actions/build-mfem@v2.7
-        with:
-          os: ${{ runner.os }}
-          target: opt
-          codecov: NO
-          mpi: par
-          build-system: make
-          hypre-dir: ${{ env.HYPRE_TOP_DIR }}
-          metis-dir: ${{ env.METIS_TOP_DIR }}
-          mfem-dir: mfem
-
-      - name: test (no clean)
-        run: |
-          cd mfem && make test-noclean
-
-      - name: gitignore
-        run: |
-          cd mfem/tests/scripts
-          ./runtest gitignore

.github/workflows/sanitizers.yml

@@ -17,7 +17,17 @@ permissions:
 on:
   push:
     branches: ["master", "next"]
+    paths-ignore: &docs-only-paths
+      - "**/*.md"
+      - "doc/**"
+      - ".binder/**"
+      - "CITATION.cff"
+      - "LICENSE"
+      - "NOTICE"
+      - "CHANGELOG"
+      - "INSTALL"
   pull_request:
+    paths-ignore: *docs-only-paths
   workflow_dispatch:
 
 concurrency:

CHANGELOG

@@ -15,6 +15,10 @@ Version 4.9.1 (development)
 
 Discretization improvements
 ---------------------------
+- Improved FindPointsGSLIB surface mesh capability with support for simplices
+  and an option to specify axis-aligned bounding box padding for near-surface
+  point queries.
+
 - Added GPU-enabled partial assembly for simplicial Bernstein H1 basis based on
   ragged tensor algorithms (see DOI: 10.1137/11082539X) for mass and diffusion
   integrators.
@@ -73,8 +77,6 @@ GPU computing
   details. Supported versions >= 0.6.0.
 
 - Allow specifying GPU kernel launch bounds for native and RAJA GPU backends.
-- Added support for FiniteElement::MapType::INTEGRAL spaces to
-  QuadratureInterpolator.
 
 New and updated examples and miniapps
 -------------------------------------

CONTRIBUTING.md

@@ -6,7 +6,6 @@
 <a href="https://github.com/mfem/mfem/blob/master/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-BSD-blue.svg"></a>
 <a href="https://github.com/mfem/mfem/releases/latest"><img alt="GitHub release" src="https://img.shields.io/github/v/release/mfem/mfem"></a>
 <a href="https://github.com/mfem/mfem/actions/workflows/repo-check.yml?query=branch%3Amaster"><img alt="Repo check" src="https://github.com/mfem/mfem/actions/workflows/repo-check.yml/badge.svg?branch=master"></a>
-<a href="https://github.com/mfem/mfem/actions/workflows/mfem-analysis.yml?query=branch%3Amaster"><img alt="Build Analysis" src="https://github.com/mfem/mfem/actions/workflows/mfem-analysis.yml/badge.svg?branch=master"></a>
 <a href="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml?query=branch%3Amaster"><img alt="Builds and Tests" src="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml/badge.svg?branch=master"></a>
 <a href="https://ci.appveyor.com/project/mfem/mfem"><img alt="Build Status" src="https://ci.appveyor.com/api/projects/status/19non9sqm6msi2wy?svg=true"></a>
 <a href="https://docs.mfem.org/html/index.html"><img alt="Documentation" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>

config/cmake/modules/FindSuiteSparse.cmake

@@ -22,15 +22,15 @@ include(MfemCmakeUtilities)
 mfem_find_package(SuiteSparse SuiteSparse SuiteSparse_DIR "" "" "" ""
   "Paths to headers required by SuiteSparse."
   "Libraries required by SuiteSparse."
-  ADD_COMPONENT "UMFPACK" "include;suitesparse" umfpack.h "lib" umfpack
-  ADD_COMPONENT "KLU" "include;suitesparse" klu.h "lib" klu
-  ADD_COMPONENT "AMD" "include;suitesparse" amd.h "lib" amd
-  ADD_COMPONENT "BTF" "include;suitesparse" btf.h "lib" btf
-  ADD_COMPONENT "CHOLMOD" "include;suitesparse" cholmod.h "lib" cholmod
-  ADD_COMPONENT "COLAMD" "include;suitesparse" colamd.h "lib" colamd
-  ADD_COMPONENT "CAMD" "include;suitesparse" camd.h "lib" camd
-  ADD_COMPONENT "CCOLAMD" "include;suitesparse" ccolamd.h "lib" ccolamd
-  ADD_COMPONENT "config" "include;suitesparse" SuiteSparse_config.h "lib"
+  ADD_COMPONENT "UMFPACK" "include;include/suitesparse;suitesparse" umfpack.h "lib" umfpack
+  ADD_COMPONENT "KLU" "include;include/suitesparse;suitesparse" klu.h "lib" klu
+  ADD_COMPONENT "AMD" "include;include/suitesparse;suitesparse" amd.h "lib" amd
+  ADD_COMPONENT "BTF" "include;include/suitesparse;suitesparse" btf.h "lib" btf
+  ADD_COMPONENT "CHOLMOD" "include;include/suitesparse;suitesparse" cholmod.h "lib" cholmod
+  ADD_COMPONENT "COLAMD" "include;include/suitesparse;suitesparse" colamd.h "lib" colamd
+  ADD_COMPONENT "CAMD" "include;include/suitesparse;suitesparse" camd.h "lib" camd
+  ADD_COMPONENT "CCOLAMD" "include;include/suitesparse;suitesparse" ccolamd.h "lib" ccolamd
+  ADD_COMPONENT "config" "include;include/suitesparse;suitesparse" SuiteSparse_config.h "lib"
     suitesparseconfig)
 
 if (SuiteSparse_FOUND AND METIS_VERSION_5)

fem/CMakeLists.txt

@@ -133,7 +133,7 @@ set(SRCS
   tmop/assemble/diag2.cpp
   tmop/assemble/grad2_limit.cpp
   tmop/assemble/grad2.cpp
-  tmop/assemble/diag3_limit.cpp  
+  tmop/assemble/diag3_limit.cpp
   tmop/assemble/diag3.cpp
   tmop/assemble/grad3_limit.cpp
   tmop/assemble/grad3.cpp
@@ -311,6 +311,7 @@ set(HDRS
   tmop_tools.hpp
   tmop_amr.hpp
   gslib.hpp
+  gslib/gslib_kernel_helpers.hpp
   transfer.hpp
   hyperbolic.hpp
   integrator.hpp

fem/bilinearform.cpp

@@ -1255,31 +1255,6 @@ void BilinearForm::Mult(const Vector &x, Vector &y) const
    }
 }
 
-void BilinearForm::AddMult(const Vector &x, Vector &y, const real_t a) const
-{
-   if (ext)
-   {
-      ext->AddMult(x, y, a);
-   }
-   else
-   {
-      mat->AddMult(x, y, a);
-   }
-}
-
-void BilinearForm::AddMultTranspose(const Vector &x, Vector &y,
-                                    const real_t a) const
-{
-   if (ext)
-   {
-      ext->AddMultTranspose(x, y, a);
-   }
-   else
-   {
-      mat->AddMultTranspose(x, y, a);
-   }
-}
-
 void BilinearForm::MultTranspose(const Vector & x, Vector & y) const
 {
    if (ext)

fem/bilinearform.hpp

@@ -307,8 +307,8 @@ public:
    { mat->Mult(x, y); mat_e->AddMult(x, y); }
 
    /// Add the matrix vector multiple to a vector:  $ y += a M x $
-   void AddMult(const Vector &x, Vector &y,
-                const real_t a = 1.0) const override;
+   void AddMult(const Vector &x, Vector &y, const real_t a = 1.0) const override
+   { mat -> AddMult (x, y, a); }
 
    /** @brief Add the original uneliminated matrix vector multiple to a vector.
        The original matrix is $ M + Me $ so we have:
@@ -318,7 +318,8 @@ public:
 
    /// Add the matrix transpose vector multiplication:  $ y += a M^T x $
    void AddMultTranspose(const Vector & x, Vector & y,
-                         const real_t a = 1.0) const override;
+                         const real_t a = 1.0) const override
+   { mat->AddMultTranspose(x, y, a); }
 
    /** @brief Add the original uneliminated matrix transpose vector
        multiple to a vector. The original matrix is $ M + M_e $

fem/bilininteg.hpp

@@ -1110,36 +1110,20 @@ public:
    MixedDotProductIntegrator(VectorCoefficient &vq)
       : MixedScalarVectorIntegrator(vq, true) {}
 
-   inline bool VerifyFiniteElementTypes(
+   inline virtual bool VerifyFiniteElementTypes(
       const FiniteElement & trial_fe,
-      const FiniteElement & test_fe) const override
+      const FiniteElement & test_fe) const
    {
       return (trial_fe.GetRangeType() == mfem::FiniteElement::VECTOR &&
               test_fe.GetRangeType()  == mfem::FiniteElement::SCALAR );
    }
 
-   inline const char * FiniteElementTypeFailureMessage() const override
+   inline virtual const char * FiniteElementTypeFailureMessage() const
    {
       return "MixedDotProductIntegrator:  "
              "Trial space must be a vector field "
              "and the test space must be a scalar field";
    }
-
-   using BilinearFormIntegrator::AssemblePA;
-   void AssemblePA(const FiniteElementSpace &trial_fes,
-                   const FiniteElementSpace &test_fes) override;
-
-   void AddMultPA(const Vector&, Vector&) const override;
-   void AddMultTransposePA(const Vector&, Vector&) const override;
-
-private:
-   Vector pa_data;
-   const DofToQuad *mapsO = nullptr;      ///< Not owned. Trial open map.
-   const DofToQuad *mapsC = nullptr;      ///< Not owned. Trial closed map.
-   const DofToQuad *mapsTest = nullptr;   ///< Not owned. Scalar test map.
-   const GeometricFactors *geom = nullptr;///< Not owned.
-   int dim = 0, ne = 0, dofs1D = 0, dofs1Dtest = 0, quad1D = 0;
-   bool test_map_integral = false;
 };
 
 /** Class for integrating the bilinear form $a(u,v) := (-\vec{V} \cdot u, \nabla \cdot v)$ in 2D or
@@ -3011,10 +2995,11 @@ public:
     vector (diagonal matrix), or matrix), trial function $u$ is in $H(curl$ or
     $H(div)$, and test function $v$ is in $H(curl$, $H(div)$, or $v=(v_1,\dots,v_n)$, where
     $v_i$ are in $H^1$. */
-class VectorFEMassIntegrator : public BilinearFormIntegrator
+class VectorFEMassIntegrator: public BilinearFormIntegrator
 {
 private:
-   void Init(Coefficient *q, DiagonalMatrixCoefficient *dq, MatrixCoefficient *mq);
+   void Init(Coefficient *q, DiagonalMatrixCoefficient *dq, MatrixCoefficient *mq)
+   { Q = q; DQ = dq; MQ = mq; }
 
 #ifndef MFEM_THREAD_SAFE
    Vector shape;
@@ -3037,8 +3022,7 @@ protected:
    const DofToQuad *mapsOtest;     ///< Not owned. DOF-to-quad map, open.
    const DofToQuad *mapsCtest;     ///< Not owned. DOF-to-quad map, closed.
    const GeometricFactors *geom;   ///< Not owned
-   int dim, ne, nq, dofs1D, dofs1Dtest, quad1D;
-   FiniteElement::DerivType trial_fetype, test_fetype;
+   int dim, ne, nq, dofs1D, dofs1Dtest, quad1D, trial_fetype, test_fetype;
    bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient
 
 public:
@@ -3069,29 +3053,6 @@ public:
                    const bool add) override;
 
    const Coefficient *GetCoefficient() const { return Q; }
-
-   using ApplyKernelType =
-      void (*)(const int NE, bool symmetric, const bool scalar_coeff,
-               const Array<real_t> &trialBO, const Array<real_t> &trialBC,
-               const Array<real_t> &testBOt, const Array<real_t> &testBCt,
-               const Vector &pa_data, const Vector &x, Vector &y,
-               const int triald1d, const int testd1d, const int q1d);
-
-   /// parameters: trial_fetype, test_fetype, ndims, trial_d1d, test_d1d, q1d
-   MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType,
-                         (FiniteElement::DerivType, FiniteElement::DerivType,
-                          int, int, int, int));
-
-   struct Kernels { Kernels(); };
-
-   template <FiniteElement::DerivType TrialType,
-             FiniteElement::DerivType TestType, int DIM, int TRIAL_D1D,
-             int TEST_D1D, int Q1D>
-   static void AddSpecialization()
-   {
-      ApplyPAKernels::Specialization<TrialType, TestType, DIM, TRIAL_D1D,
-                     TEST_D1D, Q1D>::Add();
-   }
 };
 
 /** Integrator for $(Q \nabla \cdot u, v)$ where $u=(u_1,\cdots,u_n)$ and all $u_i$ are in the same
@@ -3945,7 +3906,7 @@ class DiscreteInterpolator : public BilinearFormIntegrator { };
 
 
 /** Class for constructing the gradient as a DiscreteLinearOperator from an
-    $H^1$-conforming space to an $H(curl)$-conforming space. The range space can be
+    $H^1$-conforming space to an $H(curl$-conforming space. The range space can be
     vector $L_2$ space as well. */
 class GradientInterpolator : public DiscreteInterpolator
 {
@@ -4054,48 +4015,12 @@ public:
     discrete curl matrix. */
 class CurlInterpolator : public DiscreteInterpolator
 {
-   int dim, ne;
-   // "dof" are the domain fespace dof counts
-   int ndof_o;
-   // "quads" are the range fespace dof counts
-   int nquad_o;
-
-   Vector pa_data;
-
 public:
-   CurlInterpolator();
-
    void AssembleElementMatrix2(const FiniteElement &dom_fe,
                                const FiniteElement &ran_fe,
                                ElementTransformation &Trans,
                                DenseMatrix &elmat) override
    { ran_fe.ProjectCurl(dom_fe, Trans, elmat); }
-
-   void AssemblePA(const FiniteElementSpace &dom_fes,
-                   const FiniteElementSpace &ran_fes) override;
-   void AssemblePA(const FiniteElementSpace &fes) override
-   {
-      AssemblePA(fes, fes);
-   }
-   void AddMultPA(const Vector &x, Vector &y) const override;
-   void AddMultTransposePA(const Vector &x, Vector &y) const override;
-
-   using ApplyKernelType = void (*)(const int ne, const int ndof_o,
-                                    const int nquad_o, const Vector &pa,
-                                    const Vector &x, Vector &y);
-
-   /// arguments: DIM, ndof_o, nquad_o
-   MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int));
-   /// arguments: DIM, ndof_o, nquad_o
-   MFEM_REGISTER_KERNELS(ApplyTPAKernels, ApplyKernelType, (int, int, int));
-
-   template <int DIM, int NDOF_O, int NQUAD_O> static void AddSpecialization()
-   {
-      ApplyPAKernels::Specialization<DIM, NDOF_O, NQUAD_O>::Add();
-      ApplyTPAKernels::Specialization<DIM, NDOF_O, NQUAD_O>::Add();
-   }
-
-   struct Kernels { Kernels(); };
 };
 
 

fem/fe/fe_base.hpp

@@ -1446,8 +1446,6 @@ public:
                                 dof2quad_array_open);
    }
 
-   const Poly_1D::Basis &GetOpenBasis1D() const { return obasis1d; }
-
    virtual ~VectorTensorFiniteElement();
 };
 

fem/gslib.hpp

@@ -12,6 +12,9 @@
 #ifndef MFEM_GSLIB
 #define MFEM_GSLIB
 
+#include <map>
+#include <vector>
+
 #include "../config/config.hpp"
 #ifdef MFEM_USE_MPI
 #include "pgridfunc.hpp"
@@ -119,6 +122,11 @@ protected:
    // IntegrationRules for simplex->Quad/Hex and to project to p_max in-case of
    // p-refinement.
    Array<IntegrationRule *> ir_split;
+   /// Integration rules built at the field polynomial order (only for surface
+   /// meshes when mesh order is not the same as gridfunction order).
+   Array<IntegrationRule *> ir_split_sol;
+   /// Order at which #ir_split_sol was built; -1 means not built.
+   int ir_split_sol_order = -1;
    Array<FiniteElementSpace *> fes_rst_map; //FESpaces to map Quad/Hex->Simplex
    Array<GridFunction *> gf_rst_map; // GridFunctions to map Quad/Hex->Simplex
    FiniteElementCollection *fec_map_lin;
@@ -134,6 +142,8 @@ protected:
    AvgType avgtype;             // average type used for L2 functions
    Array<int> split_element_map;
    Array<int> split_element_index;
+   // Geometry::Type (as int) of the original element for each split quad.
+   Array<int> split_element_geom;
    int        NE_split_total;   // total number of elements after mesh splitting
    int        mesh_points_cnt;  // number of mesh nodes
    // Tolerance to ignore points found beyond the mesh boundary.
@@ -141,6 +151,12 @@ protected:
    double     bdr_tol;
    // Use CPU functions for Mesh/GridFunction on device for gslib1.0.7
    bool       gpu_to_cpu_fallback = false;
+   // Check if a point is inside the oriented bounding box of an
+   // element before the Newton iteration.
+   // Note: only used in MFEM implementation (not in gslib) which currently
+   // supports GPU kernels for area meshes in 2D, volume meshes in 3D,
+   // and surface meshes in 1D/2D/3D.
+   bool       obb_check = true;
 
    // Device specific data used for FindPoints
    struct DEV_STRUCT
@@ -162,11 +178,16 @@ protected:
       mutable double surf_dist_tol;
    } DEV;
 
-   /// Use GSLIB for communication and interpolation
+   // Helper function to setup and free gslib's crystal router.
+   void SetupCrystal(); // Called inside Setup and SetupSurf_base
+   void FreeCrystal();  // Called inside FreeData
+
+   /// Use GSLIB for communication and interpolation. Updates field_out on
+   /// host.
    virtual void InterpolateH1(const GridFunction &field_in, Vector &field_out,
                               const int field_out_ordering);
    /// Uses GSLIB Crystal Router for communication followed by MFEM's
-   /// interpolation functions
+   /// interpolation functions. Updates field_out on host.
    virtual void InterpolateGeneral(const GridFunction &field_in,
                                    Vector &field_out,
                                    const int field_out_ordering);
@@ -181,12 +202,26 @@ protected:
                                                  IntegrationRule *irule,
                                                  int order);
 
+   /** @brief Build integration rules at the given @a order for each split mesh
+    *  and store them in @a ir_out. Requires that \ref SetupSplitMeshes has
+    *  already been called. */
+   virtual void SetupIntegrationRules(const int order,
+                                      Array<IntegrationRule *> &ir_out);
+
    /** @brief Helper function that calls \ref SetupSplitMeshes and
-    * \ref SetupIntegrationRuleForSplitMesh. */
+    * \ref SetupIntegrationRules. */
    virtual void SetupSplitMeshesAndIntegrationRules(const int order);
 
-   /// Get GridFunction value at the points expected by GSLIB.
-   virtual void GetNodalValues(const GridFunction *gf_in, Vector &node_vals) const;
+   /** @brief Get GridFunction value at the points expected by GSLIB.
+    *  @param[in]  gf_in       Grid function to evaluate.
+    *  @param[out] node_vals   Output values.
+    *  @param[in]  ir_in       If non-null, use these rules instead of #ir_split.
+    *  @param[in]  by_element  If true, output has element-major layout
+    *                          [nel][vdim][ndofs]; otherwise component-major
+    *                          layout [vdim][total_pts]. */
+   virtual void GetNodalValues(const GridFunction *gf_in, Vector &node_vals,
+                               const Array<IntegrationRule *> *ir_in = nullptr,
+                               bool by_element = false) const;
 
    /** @brief Map {r,s,t} coordinates from [-1,1] to [0,1] for MFEM. For
     *  simplices, find the original element number (that was split into
@@ -293,9 +328,10 @@ protected:
                             const unsigned n,
                             const uint nel,
                             const unsigned m,
-                            const double bbox_tol,
+                            const double bbox_rel_size_inc,
                             const uint local_hash_size,
-                            const uint global_hash_size);
+                            const uint global_hash_size,
+                            const Vector *aabb_sz_inc);
 
    /// Preprocess 3D surface mesh needed for FindPoints.
    void findptssurf_setup_3(DEV_STRUCT &devs,
@@ -303,17 +339,47 @@ protected:
                             const unsigned n,
                             const uint nel,
                             const unsigned m,
-                            const double bbox_tol,
+                            const double bbox_rel_size_inc,
                             const uint local_hash_size,
                             const uint global_hash_size,
-                            const int rD);
+                            const int rD,
+                            const Vector *aabb_sz_inc);
 
+   /** @brief Shared implementation for the public surface-setup methods.
+    *
+    *  @details Initializes the surface-search data structures, builds the
+    *  split-element representation expected by gslib, and constructs the
+    *  element bounding boxes used by the MFEM surface kernels.
+    *
+    *  If @a aabb_sz_inc is null, the setup stores the default oriented
+    *  bounding boxes and uses @a bbox_rel_size_inc as their relative size
+    *  increase factor.
+    *
+    *  If @a aabb_sz_inc is non-null, the setup stores axis-aligned bounding
+    *  boxes only, applies the requested absolute AABB expansion in each
+    *  physical direction, and adjusts the tolerance @a bdr_tol so points
+    *  found in the expanded region are classified as border points.
+    *
+    *  @param[in] m                  Input surface mesh.
+    *  @param[in] bbox_rel_size_inc  Relative size increase applied when
+    *                                expanding each element bounding box during
+    *                                setup.
+    *  @param[in] aabb_sz_inc        Optional total absolute AABB expansion
+    *                                applied to the stored axis-aligned
+    *                                bounding boxes after construction.
+    *  @param[in] newt_tol           Newton tolerance for the point-search
+    *                                kernels.
+    */
+   void SetupSurf_Base(Mesh &m,
+                       const double bbox_rel_size_inc,
+                       const Vector *aabb_sz_inc,
+                       const double newt_tol);
 public:
    /// Serial constructor
    FindPointsGSLIB();
 
    /// Serial constructor + setup with given Mesh (see \ref Setup)
-   FindPointsGSLIB(Mesh &mesh_in, const double bb_t = 0.1,
+   FindPointsGSLIB(Mesh &mesh_in, const double bbox_rel_size_inc = 0.1,
                    const double newt_tol = 1.0e-12,
                    const int npt_max = 256);
 
@@ -322,7 +388,7 @@ public:
    FindPointsGSLIB(MPI_Comm comm_);
 
    /// Constructor + setup with given ParMesh (see \ref Setup)
-   FindPointsGSLIB(ParMesh &mesh_in, const double bb_t = 0.1,
+   FindPointsGSLIB(ParMesh &mesh_in, const double bbox_rel_size_inc = 0.1,
                    const double newt_tol = 1.0e-12,
                    const int npt_max = 256);
 #endif
@@ -338,23 +404,59 @@ public:
        Note: not tested with periodic (L2).
        Note: the input mesh \p m must have Nodes set.
 
-       @param[in] m         Input mesh.
-       @param[in] bb_t      (Optional) Relative size of bounding box around
-                            each element.
-       @param[in] newt_tol  (Optional) Newton tolerance for the gslib
-                            search methods.
-       @param[in] npt_max   (Optional) Number of points for simultaneous
-                            iteration. This alters performance and
-                            memory footprint.
+       @param[in] m                  Input mesh.
+       @param[in] bbox_rel_size_inc  (Optional) Relative size increase applied
+                                     when expanding each element bounding box.
+       @param[in] newt_tol           (Optional) Newton tolerance for the gslib
+                                     search methods.
+       @param[in] npt_max            (Optional) Number of points for
+                                     simultaneous iteration. This alters
+                                     performance and memory footprint.
    */
-   void Setup(Mesh &m, const double bb_t = 0.1, const double newt_tol = 1.0e-12,
+   void Setup(Mesh &m, const double bbox_rel_size_inc = 0.1,
+              const double newt_tol = 1.0e-12,
               const int npt_max = 256);
 
    /// Preprocess the surface mesh to compute data for FindPoints.
    void SetupSurf(Mesh &m,
-                  const double bb_t = 0.1,
-                  const double newt_tol = 1.0e-12,
-                  const int npt_max = 256);
+                  const double bbox_rel_size_inc = 0.1,
+                  const double newt_tol = 1.0e-12);
+
+   /** @brief Preprocess the surface mesh to compute data for FindPoints using
+    *  absolute AABB expansion.
+    *
+    *  @details This method computes only axis-aligned bounding boxes and
+    *  increases their total length by a user-specified amount in each
+    *  physical direction. The absolute AABB expansion is applied
+    *  symmetrically to the lower and upper bounds.
+    *
+    *  The size of @a aabb_sz_inc determines how the expansion values are
+    *  interpreted:
+    *  - `1`: one expansion value used in every direction for every element
+    *  - `NElements`: one expansion value per element, reused in x/y/z
+    *    directions
+    *  - `SpaceDim`: one expansion value per physical direction, reused for
+    *    every element
+    *  - `NElements*SpaceDim`: one expansion value per element and direction,
+    *    ordered as `(dx1,dy1,dz1, ... dxN,dyN,dzN)`
+    *
+    *  This method disables the oriented bounding-box precheck because the
+    *  stored boxes are modified only in their axis-aligned representation.
+    *
+    *  @param[in] m                  Input surface mesh.
+    *  @param[in] aabb_sz_inc        Total absolute AABB expansion applied in
+    *                                each physical direction to the stored
+    *                                axis-aligned bounding boxes.
+    *  @param[in] newt_tol           Newton tolerance for the point-search
+    *                                kernels.
+    *
+    *  @note We disable the oriented bounding box check with this setup.
+    *        @a bdr_tol is also adjusted so that all points in the AABBs can
+    *        be found.
+    */
+   void SetupSurfWithAABBExpansion(Mesh &m, const Vector &aabb_sz_inc,
+                                   const double newt_tol = 1.0e-12);
+
 
    /** @brief Searches positions given in physical space by \p point_pos.
 
@@ -401,7 +503,8 @@ public:
    /// Setup FindPoints and search positions
    void FindPoints(Mesh &m, const Vector &point_pos,
                    const int point_pos_ordering = Ordering::byNODES,
-                   const double bb_t = 0.1, const double newt_tol = 1.0e-12,
+                   const double bbox_rel_size_inc = 0.1,
+                   const double newt_tol = 1.0e-12,
                    const int npt_max = 256);
 
    /** @brief Interpolation of field values at prescribed reference space
@@ -413,7 +516,11 @@ public:
                               mesh that was given to Setup().
        @param[out] field_out  Interpolated values. For points that are not found
                               the value is set to #default_interp_value.
-                              The output ordering is determined from field_in.*/
+                              The output ordering is determined from field_in.
+
+       @note: field_out is moved to device if field_in is on device. Otherwise,
+              field_out memory allocation is not changed.
+   */
    virtual void Interpolate(const GridFunction &field_in, Vector &field_out);
 
    /// Interpolation of field values, with output ordering specification.
@@ -468,7 +575,12 @@ public:
     *  @details When using FindPoints, gslib may return points as found on the
     *  boundary even when they are slightly outside the domain. This tolerance
     *  is used to filter such points based on the distance^2 value and mark them
-    *  as not found.*/
+    *  as not found.
+    *
+    *  @note When the SetupSurfWithAABBExpansion method is used for surface
+    *  meshes, this tolerance is automatically computed based on the size of
+    *  expanded AABBs. Using this method will override that computed tolerance.
+    *  */
    virtual void SetDistanceToleranceForPointsFoundOnBoundary(double bdr_tol_)
    {
       bdr_tol = bdr_tol_;
@@ -603,25 +715,28 @@ public:
        Note: not tested with periodic meshes (L2).
        Note: the input mesh \p m must have Nodes set.
 
-       @param[in] m         Input mesh.
-       @param[in] meshid    A unique # for each overlapping mesh. This id is
-                            used to make sure that points being searched are not
-                            looked for in the mesh that they belong to.
-       @param[in] gfmax     (Optional) GridFunction in H1 that is used as a
-                            discriminator when one point is located in multiple
-                            meshes. The mesh that maximizes gfmax is chosen.
-                            For example, using the distance field based on the
-                            overlapping boundaries is helpful for convergence
-                            during Schwarz iterations.
-       @param[in] bb_t      (Optional) Relative size of bounding box around
-                            each element.
-       @param[in] newt_tol  (Optional) Newton tolerance for the gslib
-                            search methods.
-       @param[in] npt_max   (Optional) Number of points for simultaneous
-                            iteration. This alters performance and
-                            memory footprint.*/
-   void Setup(Mesh &m, const int meshid, GridFunction *gfmax = NULL,
-              const double bb_t = 0.1, const double newt_tol = 1.0e-12,
+       @param[in] m                  Input mesh.
+       @param[in] meshid             A unique # for each overlapping mesh.
+                                     This id is used to make sure that points
+                                     being searched are not looked for in the
+                                     mesh that they belong to.
+       @param[in] gfmax              (Optional) GridFunction in H1 that is used
+                                     as a discriminator when one point is
+                                     located in multiple meshes. The mesh that
+                                     maximizes gfmax is chosen. For example,
+                                     using the distance field based on the
+                                     overlapping boundaries is helpful for
+                                     convergence during Schwarz iterations.
+       @param[in] bbox_rel_size_inc  (Optional) Relative size increase applied
+                                     when expanding each element bounding box.
+       @param[in] newt_tol           (Optional) Newton tolerance for the gslib
+                                     search methods.
+       @param[in] npt_max            (Optional) Number of points for
+                                     simultaneous iteration. This alters
+                                     performance and memory footprint.*/
+   void Setup(Mesh &m, const int meshid, GridFunction *gfmax = nullptr,
+              const double bbox_rel_size_inc = 0.1,
+              const double newt_tol = 1.0e-12,
               const int npt_max = 256);
 
    /** Searches positions given in physical space by \p point_pos. All output
@@ -677,7 +792,7 @@ class GSOPGSLIB
 protected:
    struct gslib::crystal *cr;               // gslib's internal data
    struct gslib::comm *gsl_comm;            // gslib's internal data
-   struct gslib::gs_data *gsl_data = NULL;
+   struct gslib::gs_data *gsl_data = nullptr;
    int num_ids;
 
 public:

fem/gslib/findpts_local_2.cpp

@@ -11,7 +11,7 @@
 
 #include "../gslib.hpp"
 #include "../../general/forall.hpp"
-#include "../../linalg/kernels.hpp"
+#include "gslib_kernel_helpers.hpp"
 
 #ifdef MFEM_USE_GSLIB
 
@@ -27,8 +27,6 @@
 #pragma GCC diagnostic pop
 #endif
 
-#include <climits>
-
 namespace mfem
 {
 #if GSLIB_RELEASE_VERSION >= 10009
@@ -54,127 +52,14 @@ struct findptsElementGPT_t
    double x[DIM], jac[DIM * DIM], hes[4];
 };
 
-struct dbl_range_t
-{
-   double min, max;
-};
-struct obbox_t
-{
-   double c0[DIM], A[DIM * DIM];
-   dbl_range_t x[DIM];
-};
-
-struct findptsLocalHashData_t
-{
-   int hash_n;
-   dbl_range_t bnd[DIM];
-   double fac[DIM];
-   unsigned int *offset;
-   int max;
-};
-
-// Eval the ith Lagrange interpolant and its first derivative at x.
-// Note: lCoeff stores pre-computed coefficients for fast evaluation.
-static MFEM_HOST_DEVICE inline void lag_eval_first_der(double *p0, double x,
-                                                       int i, const double *z,
-                                                       const double *lCoeff,
-                                                       int pN)
-{
-   double u0 = 1, u1 = 0;
-   for (int j = 0; j < pN; ++j)
-   {
-      if (i != j)
-      {
-         double d_j = 2 * (x - z[j]);
-         u1 = d_j * u1 + u0;
-         u0 = d_j * u0;
-      }
-   }
-   p0[i] = lCoeff[i] * u0;
-   p0[pN+i] = 2.0 * lCoeff[i] * u1;
-}
-
-// Eval the ith Lagrange interpolant and its first and second derivative at x.
-// Note: lCoeff stores pre-computed coefficients for fast evaluation.
-static MFEM_HOST_DEVICE inline void lag_eval_second_der(double *p0, double x,
-                                                        int i, const double *z,
-                                                        const double *lCoeff,
-                                                        int pN)
-{
-   double u0 = 1, u1 = 0, u2 = 0;
-   for (int j = 0; j < pN; ++j)
-   {
-      if (i != j)
-      {
-         double d_j = 2 * (x - z[j]);
-         u2 = d_j * u2 + u1;
-         u1 = d_j * u1 + u0;
-         u0 = d_j * u0;
-      }
-   }
-   p0[i] = lCoeff[i] * u0;
-   p0[pN+i] = 2.0 * lCoeff[i] * u1;
-   p0[2*pN+i] = 8.0 * lCoeff[i] * u2;
-}
-
-// Axis-aligned bounding box test.
-static MFEM_HOST_DEVICE inline double AABB_test(const obbox_t *const b,
-                                                const double x[2])
-{
-   double test = 1;
-   for (int d = 0; d < 2; ++d)
-   {
-      double b_d = (x[d] - b->x[d].min) * (b->x[d].max - x[d]);
-      test = test < 0 ? test : b_d;
-   }
-   return test;
-}
-
-// Axis-aligned bounding box test followed by oriented bounding-box test.
-static MFEM_HOST_DEVICE inline double bbox_test(const obbox_t *const b,
-                                                const double x[2])
-{
-   const double bxyz = AABB_test(b, x);
-   if (bxyz < 0)
-   {
-      return bxyz;
-   }
-   else
-   {
-      double dxyz[2];
-      for (int d = 0; d < 2; ++d)
-      {
-         dxyz[d] = x[d] - b->c0[d];
-      }
-      double test = 1;
-      for (int d = 0; d < 2; ++d)
-      {
-         double rst = 0;
-         for (int e = 0; e < 2; ++e)
-         {
-            rst += b->A[d * 2 + e] * dxyz[e];
-         }
-         double brst = (rst + 1) * (1 - rst);
-         test = test < 0 ? test : brst;
-      }
-      return test;
-   }
-}
-
-// Element index corresponding to hash mesh that the point is located in.
-static MFEM_HOST_DEVICE inline int hash_index(const findptsLocalHashData_t *p,
-                                              const double x[2])
-{
-   const int n = p->hash_n;
-   int sum = 0;
-   for (int d = 2 - 1; d >= 0; --d)
-   {
-      sum *= n;
-      int i = (int)floor((x[d] - p->bnd[d].min) * p->fac[d]);
-      sum += i < 0 ? 0 : (n - 1 < i ? n - 1 : i);
-   }
-   return sum;
-}
+using dbl_range_t = gslib::dbl_range_t;
+using obbox_t = gslib::obbox_t<DIM>;
+using findptsLocalHashData_t = gslib::findptsLocalHashData_t<DIM>;
+using gslib::bbox_test;
+using gslib::hash_index;
+using gslib::l2norm2;
+using gslib::lag_eval_first_der;
+using gslib::lag_eval_second_der;
 
 /*Solve Ax=y. A is row-major */
 static MFEM_HOST_DEVICE inline void lin_solve_2(double x[2], const double A[4],
@@ -185,12 +70,6 @@ static MFEM_HOST_DEVICE inline void lin_solve_2(double x[2], const double A[4],
    x[1] = idet*(A[0]*y[1] - A[2]*y[0]);
 }
 
-/* L2 norm squared. */
-static MFEM_HOST_DEVICE inline double l2norm2(const double x[2])
-{
-   return x[0] * x[0] + x[1] * x[1];
-}
-
 /* the bit structure of flags is CSSRR
    the C bit --- 1<<4 --- is set when the point is converged
    RR is 0 = 00b if r is unconstrained,
@@ -352,7 +231,7 @@ static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsElementPoint_t *res,
                                                  const findptsElementPoint_t *p,
                                                  const double tol)
 {
-   const double dist2 = l2norm2(resid);
+   const double dist2 = l2norm2<2>(resid);
    const double decr = p->dist2 - dist2;
    const double pred = p->dist2p;
    for (int d = 0; d < 2; ++d)
@@ -695,25 +574,25 @@ static MFEM_HOST_DEVICE double tensor_ig2_j(double *g_partials,
 }
 
 template<int T_D1D = 0>
-static void FindPointsLocal2D_Kernel(const int npt,
-                                     const double tol,
-                                     const double *x,
-                                     const int point_pos_ordering,
-                                     const double *xElemCoord,
-                                     const int nel,
-                                     const double *wtend,
-                                     const double *boxinfo,
-                                     const int hash_n,
-                                     const double *hashMin,
-                                     const double *hashFac,
-                                     unsigned int *hashOffset,
-                                     unsigned int *const code_base,
-                                     unsigned int *const el_base,
-                                     double *const r_base,
-                                     double *const dist2_base,
-                                     const double *gll1D,
-                                     const double *lagcoeff,
-                                     const int pN = 0)
+static void FindPointsLocal2DKernel(const int npt,
+                                    const double tol,
+                                    const double *x,
+                                    const int point_pos_ordering,
+                                    const double *xElemCoord,
+                                    const int nel,
+                                    const double *wtend,
+                                    const double *boxinfo,
+                                    const int hash_n,
+                                    const double *hashMin,
+                                    const double *hashFac,
+                                    unsigned int *hashOffset,
+                                    unsigned int *const code_base,
+                                    unsigned int *const el_base,
+                                    double *const r_base,
+                                    double *const dist2_base,
+                                    const double *gll1D,
+                                    const double *lagcoeff,
+                                    const int pN = 0)
 {
    const int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
    const int D1D = T_D1D ? T_D1D : pN;
@@ -1175,30 +1054,45 @@ void FindPointsGSLIB::FindPointsLocal2(const Vector &point_pos,
    switch (DEV.dof1d)
    {
       case 2:
-         return FindPointsLocal2D_Kernel<2>(
-                   npt, DEV.newt_tol, pp, point_pos_ordering, pgslm, NE_split_total, pwt,
-                   pbb, DEV.lh_nx, plhm, plhf, plho, pcode, pelem, pref, pdist,
-                   pgll1d, plc);
+         FindPointsLocal2DKernel<2>(npt, DEV.newt_tol, pp,
+                                    point_pos_ordering, pgslm,
+                                    NE_split_total, pwt, pbb,
+                                    DEV.lh_nx, plhm, plhf, plho,
+                                    pcode, pelem, pref, pdist,
+                                    pgll1d, plc);
+         break;
       case 3:
-         return FindPointsLocal2D_Kernel<3>(
-                   npt, DEV.newt_tol, pp, point_pos_ordering, pgslm, NE_split_total, pwt,
-                   pbb, DEV.lh_nx, plhm, plhf, plho, pcode, pelem, pref, pdist,
-                   pgll1d, plc);
+         FindPointsLocal2DKernel<3>(npt, DEV.newt_tol, pp,
+                                    point_pos_ordering, pgslm,
+                                    NE_split_total, pwt, pbb,
+                                    DEV.lh_nx, plhm, plhf, plho,
+                                    pcode, pelem, pref, pdist,
+                                    pgll1d, plc);
+         break;
       case 4:
-         return FindPointsLocal2D_Kernel<4>(
-                   npt, DEV.newt_tol, pp, point_pos_ordering, pgslm, NE_split_total, pwt,
-                   pbb, DEV.lh_nx, plhm, plhf, plho, pcode, pelem, pref, pdist,
-                   pgll1d, plc);
+         FindPointsLocal2DKernel<4>(npt, DEV.newt_tol, pp,
+                                    point_pos_ordering, pgslm,
+                                    NE_split_total, pwt, pbb,
+                                    DEV.lh_nx, plhm, plhf, plho,
+                                    pcode, pelem, pref, pdist,
+                                    pgll1d, plc);
+         break;
       case 5:
-         return FindPointsLocal2D_Kernel<5>(
-                   npt, DEV.newt_tol, pp, point_pos_ordering, pgslm, NE_split_total, pwt,
-                   pbb, DEV.lh_nx, plhm, plhf, plho, pcode, pelem, pref, pdist,
-                   pgll1d, plc);
+         FindPointsLocal2DKernel<5>(npt, DEV.newt_tol, pp,
+                                    point_pos_ordering, pgslm,
+                                    NE_split_total, pwt, pbb,
+                                    DEV.lh_nx, plhm, plhf, plho,
+                                    pcode, pelem, pref, pdist,
+                                    pgll1d, plc);
+         break;
       default:
-         return FindPointsLocal2D_Kernel(npt, DEV.newt_tol, pp, point_pos_ordering,
-                                         pgslm, NE_split_total, pwt, pbb, DEV.lh_nx,
-                                         plhm, plhf, plho, pcode, pelem,
-                                         pref, pdist, pgll1d, plc, DEV.dof1d);
+         FindPointsLocal2DKernel(npt, DEV.newt_tol, pp,
+                                 point_pos_ordering, pgslm,
+                                 NE_split_total, pwt, pbb,
+                                 DEV.lh_nx, plhm, plhf, plho,
+                                 pcode, pelem, pref, pdist,
+                                 pgll1d, plc, DEV.dof1d);
+         break;
    }
 }
 #undef DIM2

fem/gslib/findpts_local_3.cpp

@@ -11,9 +11,7 @@
 
 #include "../gslib.hpp"
 #include "../../general/forall.hpp"
-#include "../../linalg/kernels.hpp"
-
-#include <climits>
+#include "gslib_kernel_helpers.hpp"
 
 #ifdef MFEM_USE_GSLIB
 
@@ -59,128 +57,15 @@ struct findptsElemPt
    double x[DIM], jac[DIM * DIM], hes[18];
 };
 
-struct dbl_range_t
-{
-   double min, max;
-};
-
-struct obbox_t
-{
-   double c0[DIM], A[DIM * DIM];
-   dbl_range_t x[DIM];
-};
-
-struct findptsLocalHashData_t
-{
-   int hash_n;
-   dbl_range_t bnd[DIM];
-   double fac[DIM];
-   unsigned int *offset;
-   // int max;
-};
-
-// Eval the ith Lagrange interpolant and its first derivative at x.
-// Note: lCoeff stores pre-computed coefficients for fast evaluation.
-static MFEM_HOST_DEVICE inline void lag_eval_first_der(double *p0, double x,
-                                                       int i, const double *z,
-                                                       const double *lCoeff,
-                                                       int pN)
-{
-   double u0 = 1, u1 = 0;
-   for (int j = 0; j < pN; ++j)
-   {
-      if (i != j)
-      {
-         double d_j = 2*(x-z[j]);
-         u1 = d_j*u1+u0;
-         u0 = d_j*u0;
-      }
-   }
-   p0[i] = lCoeff[i]*u0;
-   p0[pN+i] = 2.0*lCoeff[i]*u1;
-}
-
-// Eval the ith Lagrange interpolant and its first and second derivative at x.
-// Note: lCoeff stores pre-computed coefficients for fast evaluation.
-static MFEM_HOST_DEVICE inline void lag_eval_second_der(double *p0, double x,
-                                                        int i, const double *z,
-                                                        const double *lCoeff,
-                                                        int pN)
-{
-   double u0 = 1, u1 = 0, u2 = 0;
-   for (int j = 0; j < pN; ++j)
-   {
-      if (i != j)
-      {
-         double d_j = 2*(x-z[j]);
-         u2 = d_j*u2+u1;
-         u1 = d_j*u1+u0;
-         u0 = d_j*u0;
-      }
-   }
-   p0[i] = lCoeff[i]*u0;
-   p0[pN+i] = 2.0*lCoeff[i]*u1;
-   p0[2*pN+i] = 8.0*lCoeff[i]*u2;
-}
-
-// Axis-aligned bounding box test.
-static MFEM_HOST_DEVICE inline double AABB_test(const obbox_t *const b,
-                                                const double x[3])
-{
-   double b_d;
-   for (int d = 0; d < 3; ++d)
-   {
-      b_d = (x[d]-b->x[d].min)*(b->x[d].max-x[d]);
-      if (b_d < 0) { return b_d; }
-   }
-   return b_d;
-}
-
-// Axis-aligned bounding box test followed by oriented bounding-box test.
-static MFEM_HOST_DEVICE inline double bbox_test(const obbox_t *const b,
-                                                const double x[3])
-{
-   const double bxyz = AABB_test(b, x);
-   if (bxyz < 0)
-   {
-      return bxyz;
-   }
-   else
-   {
-      double dxyz[3];
-      for (int d = 0; d < 3; ++d)
-      {
-         dxyz[d] = x[d]-b->c0[d];
-      }
-      double test = 1;
-      for (int d = 0; d < 3; ++d)
-      {
-         double rst = 0;
-         for (int e = 0; e < 3; ++e)
-         {
-            rst += b->A[d*3+e]*dxyz[e];
-         }
-         double brst = (rst+1)*(1-rst);
-         test = test < 0 ? test : brst;
-      }
-      return test;
-   }
-}
-
-// Element index corresponding to hash mesh that the point is located in.
-static MFEM_HOST_DEVICE inline int hash_index(const findptsLocalHashData_t *p,
-                                              const double x[3])
-{
-   const int n = p->hash_n;
-   int sum = 0;
-   for (int d = 3-1; d >= 0; --d)
-   {
-      sum *= n;
-      int i = (int)floor((x[d]-p->bnd[d].min)*p->fac[d]);
-      sum += i < 0 ? 0 : (n-1 < i ? n-1 : i);
-   }
-   return sum;
-}
+using dbl_range_t = gslib::dbl_range_t;
+using obbox_t = gslib::obbox_t<DIM>;
+using findptsLocalHashData_t = gslib::findptsLocalHashData_t<DIM>;
+using gslib::bbox_test;
+using gslib::hash_index;
+using gslib::l2norm2;
+using gslib::lag_eval_first_der;
+using gslib::lag_eval_second_der;
+using gslib::lin_solve_sym_2;
 
 // Solve Ax=y. A is row-major.
 static MFEM_HOST_DEVICE inline void lin_solve_3(double x[3], const double A[9],
@@ -199,22 +84,6 @@ static MFEM_HOST_DEVICE inline void lin_solve_3(double x[3], const double A[9],
    x[2] = idet*(inv6*y[0]+inv7*y[1]+inv8*y[2]);
 }
 
-// Solve Ax=y. A is a symmetric 2x2 matrix.
-static MFEM_HOST_DEVICE inline void lin_solve_sym_2(double x[2],
-                                                    const double A[3],
-                                                    const double y[2])
-{
-   const double idet = 1 / (A[0]*A[2]-A[1]*A[1]);
-   x[0] = idet*(A[2]*y[0]-A[1]*y[1]);
-   x[1] = idet*(A[0]*y[1]-A[1]*y[0]);
-}
-
-// L2 norm.
-static MFEM_HOST_DEVICE inline double l2norm2(const double x[3])
-{
-   return x[0]*x[0]+x[1]*x[1]+x[2]*x[2];
-}
-
 /* the bit structure of flags is CTTSSRR
    the C bit --- 1<<6 --- is set when the point is converged
    RR is 0 = 00b if r is unconstrained,
@@ -459,7 +328,7 @@ static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsPt *res,
                                                  const findptsPt *p,
                                                  const double tol)
 {
-   const double dist2 = l2norm2(resid);
+   const double dist2 = l2norm2<3>(resid);
    const double decr = p->dist2-dist2;
    const double pred = p->dist2p;
    for (int d = 0; d < 3; ++d)
@@ -1809,33 +1678,36 @@ void FindPointsGSLIB::FindPointsLocal3(const Vector &point_pos,
    {
       case 2:
          FindPointsLocal3DKernel<2>(npt, DEV.newt_tol, pp, point_pos_ordering,
-                                    pgslm, NE_split_total, pwt, pbb, DEV.lh_nx, plhm,
-                                    plhf, plho, pcode, pelem, pref, pdist, pgll1d,
-                                    plc);
+                                    pgslm, NE_split_total, pwt, pbb,
+                                    DEV.lh_nx, plhm, plhf, plho,
+                                    pcode, pelem, pref, pdist, pgll1d, plc);
          break;
       case 3:
          FindPointsLocal3DKernel<3>(npt, DEV.newt_tol, pp, point_pos_ordering,
-                                    pgslm, NE_split_total, pwt, pbb, DEV.lh_nx, plhm,
-                                    plhf, plho, pcode, pelem, pref, pdist, pgll1d,
-                                    plc);
+                                    pgslm, NE_split_total, pwt, pbb,
+                                    DEV.lh_nx, plhm, plhf, plho,
+                                    pcode, pelem, pref, pdist, pgll1d, plc);
          break;
       case 4:
          FindPointsLocal3DKernel<4>(npt, DEV.newt_tol, pp, point_pos_ordering,
-                                    pgslm, NE_split_total, pwt, pbb, DEV.lh_nx, plhm,
-                                    plhf, plho, pcode, pelem, pref, pdist, pgll1d,
-                                    plc);
+                                    pgslm, NE_split_total, pwt, pbb,
+                                    DEV.lh_nx, plhm, plhf, plho,
+                                    pcode, pelem, pref, pdist, pgll1d, plc);
          break;
       case 5:
          FindPointsLocal3DKernel<5>(npt, DEV.newt_tol, pp, point_pos_ordering,
-                                    pgslm, NE_split_total, pwt, pbb, DEV.lh_nx, plhm,
-                                    plhf, plho, pcode, pelem, pref, pdist, pgll1d,
-                                    plc);
+                                    pgslm, NE_split_total, pwt, pbb,
+                                    DEV.lh_nx, plhm, plhf, plho,
+                                    pcode, pelem, pref, pdist, pgll1d, plc);
          break;
       default:
-         FindPointsLocal3DKernel(npt, DEV.newt_tol, pp, point_pos_ordering, pgslm,
-                                 NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
-                                 plho, pcode, pelem, pref, pdist, pgll1d, plc,
+         FindPointsLocal3DKernel(npt, DEV.newt_tol, pp,
+                                 point_pos_ordering, pgslm,
+                                 NE_split_total, pwt, pbb,
+                                 DEV.lh_nx, plhm, plhf, plho,
+                                 pcode, pelem, pref, pdist, pgll1d, plc,
                                  DEV.dof1d);
+         break;
    }
 }
 #undef pMax

fem/gslib/findptsedge_local_2.cpp

@@ -11,6 +11,7 @@
 
 #include "../gslib.hpp"
 #include "../../general/forall.hpp"
+#include "gslib_kernel_helpers.hpp"
 
 #ifdef MFEM_USE_GSLIB
 
@@ -52,113 +53,14 @@ struct findptsElementGPT_t
    double x[sDIM], jac[sDIM*rDIM], hes[sDIM*rDIM];
 };
 
-struct dbl_range_t
-{
-   double min, max;
-};
-
-struct obbox_t
-{
-   double c0[sDIM], A[sDIM*sDIM];
-   dbl_range_t x[sDIM];
-};
-
-struct findptsLocalHashData_t
-{
-   int hash_n;
-   dbl_range_t bnd[sDIM];
-   double fac[sDIM];
-   unsigned int *offset;
-};
-
-static MFEM_HOST_DEVICE inline void lag_eval_second_der(double *p0, double x,
-                                                        int i, const double *z,
-                                                        const double *lCoeff,
-                                                        int pN)
-{
-   double u0 = 1, u1 = 0, u2 = 0;
-   for (int j = 0; j < pN; ++j)
-   {
-      if (i != j)
-      {
-         double d_j = 2 * (x-z[j]);
-         u2 = d_j * u2 + u1;
-         u1 = d_j * u1 + u0;
-         u0 = d_j * u0;
-      }
-   }
-   double *p1 = p0 + pN, *p2 = p0 + 2 * pN;
-   p0[i] = lCoeff[i] * u0;
-   p1[i] = 2.0 * lCoeff[i] * u1;
-   p2[i] = 8.0 * lCoeff[i] * u2;
-}
-
-/* positive when possibly inside */
-static MFEM_HOST_DEVICE inline double obbox_axis_test(const obbox_t *const b,
-                                                      const double x[sDIM])
-{
-   double b_d;
-   for (int d=0; d<sDIM; ++d)
-   {
-      b_d = (x[d] - b->x[d].min) * (b->x[d].max - x[d]);
-      if (b_d < 0)   // if outside in any dimension
-      {
-         return b_d;
-      }
-   }
-   return b_d;       // only positive if inside
-}
-
-/* positive when given point is possibly inside given obbox b */
-static MFEM_HOST_DEVICE inline double obbox_test(const obbox_t *const b,
-                                                 const double x[sDIM])
-{
-   const double bxyz = obbox_axis_test(b,x);
-   if (bxyz<0)   // test if point is in AABB
-   {
-      return bxyz;
-   }
-   else   // test OBB only if inside AABB
-   {
-      double dxyz[sDIM];
-      for (int d=0; d<sDIM; ++d)
-      {
-         dxyz[d] = x[d] - b->c0[d];
-      }
-      double test = 1;
-      for (int d=0; d<sDIM; ++d)
-      {
-         double rst = 0;
-         for (int e=0; e<sDIM; ++e)
-         {
-            rst += b->A[d*2 + e] * dxyz[e];
-         }
-         double brst = (rst+1)*(1-rst);
-         test = test<0 ? test : brst;
-      }
-      return test;
-   }
-}
-
-/* Hash index in the hash table to the elements that possibly contain the point x */
-static MFEM_HOST_DEVICE inline int hash_index(const findptsLocalHashData_t *p,
-                                              const double x[2])
-{
-   const int n = p->hash_n;
-   int sum = 0;
-   for (int d=sDIM-1; d>=0; --d)
-   {
-      sum *= n;
-      int i = (int)floor((x[d] - p->bnd[d].min) * p->fac[d]);
-      sum += i<0 ? 0 : (n-1 < i ? n-1 : i);
-   }
-   return sum;
-}
-
-static MFEM_HOST_DEVICE inline double l2norm2(const double x[2])
-{
-   return x[0] * x[0] + x[1] * x[1];
-}
+using dbl_range_t = gslib::dbl_range_t;
+using obbox_t = gslib::obbox_t<sDIM>;
+using findptsLocalHashData_t = gslib::findptsLocalHashData_t<sDIM>;
+using gslib::AABB_test;
+using gslib::bbox_test;
+using gslib::hash_index;
+using gslib::l2norm2;
+using gslib::lag_eval_second_der;
 
 /* the bit structure of flags is CRR
    the C bit --- 1<<2 --- is set when the point is converged
@@ -187,29 +89,29 @@ static MFEM_HOST_DEVICE inline int point_index(const int x)
 /* check reduction in objective against prediction, and adjust
    trust region radius (p->tr) accordingly;
    may reject the prior step, returning 1; otherwise returns 0
-   sets out->dist2, out->index, out->x, out->oldr in any event,
-   leaving out->r, out->dr, out->flags to be set when returning 0 */
-static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsElementPoint_t *out,
+   sets out_pt->dist2, out_pt->index, out_pt->x, out_pt->oldr in any event,
+   leaving out_pt->r, out_pt->dr, out_pt->flags to be set when returning 0 */
+static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsElementPoint_t *out_pt,
                                                  const double resid[2],
                                                  const findptsElementPoint_t *p,
                                                  const double tol)
 {
-   const double dist2 = l2norm2(resid);
+   const double dist2 = l2norm2<2>(resid);
    const double decr = p->dist2 - dist2;
    const double pred = p->dist2p;
-   out->x[0] = p->x[0];
-   out->x[1] = p->x[1];
-   out->oldr = p->r;
-   out->dist2 = dist2;
+   out_pt->x[0] = p->x[0];
+   out_pt->x[1] = p->x[1];
+   out_pt->oldr = p->r;
+   out_pt->dist2 = dist2;
    if (decr >= 0.01*pred)
    {
       if (decr >= 0.9*pred)   // very good iteration
       {
-         out->tr = p->tr*2;
+         out_pt->tr = p->tr*2;
       }
       else                    // somewhat good iteration
       {
-         out->tr = p->tr;
+         out_pt->tr = p->tr;
       }
       return false;
    }
@@ -220,21 +122,21 @@ static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsElementPoint_t *out,
          "very good iteration" --- this doubles the trust radius,
          which is why we divide by 4 below */
       double v0 = fabs(p->r - p->oldr);
-      out->tr = v0/4.0;
-      out->dist2 = p->dist2;
-      out->r = p->oldr;
-      out->flags = p->flags>>3;
-      out->dist2p = -HUGE_VAL;
+      out_pt->tr = v0/4.0;
+      out_pt->dist2 = p->dist2;
+      out_pt->r = p->oldr;
+      out_pt->flags = p->flags>>3;
+      out_pt->dist2p = -HUGE_VAL;
       if (pred < dist2*tol)
       {
-         out->flags |= CONVERGED_FLAG;
+         out_pt->flags |= CONVERGED_FLAG;
       }
       return true;
    }
 }
 
 static MFEM_HOST_DEVICE inline void newton_edge( findptsElementPoint_t *const
-                                                 out,
+                                                 out_pt,
                                                  const double jac[2],
                                                  const double rhess,
                                                  const double resid[2],
@@ -304,9 +206,9 @@ newton_edge_fin:
    {
       new_flags |= CONVERGED_FLAG;
    }
-   out->r = newr;
-   out->dist2p = -v;
-   out->flags = flags | new_flags | ((p->flags & FLAG_MASK)<<3);
+   out_pt->r = newr;
+   out_pt->dist2p = -v;
+   out_pt->flags = flags | new_flags | ((p->flags & FLAG_MASK)<<3);
 }
 
 static MFEM_HOST_DEVICE void seed_j( const double *elx[sDIM],
@@ -332,26 +234,27 @@ static MFEM_HOST_DEVICE void seed_j( const double *elx[sDIM],
 }
 
 template<int T_D1D = 0>
-static void FindPointsEdgeLocal2D_Kernel( const int     npt,
-                                          const double  tol,
-                                          const double  dist2tol,
-                                          const double  *x,
-                                          const int     point_pos_ordering,
-                                          const double  *xElemCoord,
-                                          const int     nel,
-                                          const double  *wtend,
-                                          const double  *boxinfo,
-                                          const int     hash_n,
-                                          const double  *hashMin,
-                                          const double  *hashFac,
-                                          unsigned int  *hashOffset,
-                                          unsigned int *const code_base,
-                                          unsigned int *const el_base,
-                                          double *const r_base,
-                                          double *const dist2_base,
-                                          const double  *gll1D,
-                                          const double  *lagcoeff,
-                                          const int     pN = 0 )
+static void FindPointsEdgeLocal2DKernel( const int     npt,
+                                         const double  tol,
+                                         const double  dist2tol,
+                                         const double  *x,
+                                         const int     point_pos_ordering,
+                                         const double  *xElemCoord,
+                                         const int     nel,
+                                         const double  *wtend,
+                                         const double  *boxinfo,
+                                         const bool    obb_check,
+                                         const int     hash_n,
+                                         const double  *hashMin,
+                                         const double  *hashFac,
+                                         unsigned int  *hashOffset,
+                                         unsigned int *const code_base,
+                                         unsigned int *const el_base,
+                                         double *const r_base,
+                                         double *const dist2_base,
+                                         const double  *gll1D,
+                                         const double  *lagcoeff,
+                                         const int     pN = 0 )
 {
    const int MD1   = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
    const int D1D   = T_D1D ? T_D1D : pN;
@@ -412,22 +315,34 @@ static void FindPointsEdgeLocal2D_Kernel( const int     npt,
       {
          const unsigned int el = *elp;
 
+         const int n_box_ents = obb_check ? (3*sDIM + sDIM2) : (2*sDIM);
+         bool pass_bb = true;
          obbox_t box;
-         int n_box_ents = 3*sDIM + sDIM2;
-
-         for (int idx = 0; idx < sDIM; ++idx)
+         if (obb_check)
          {
-            box.c0[idx] = boxinfo[n_box_ents*el + idx];
-            box.x[idx].min = boxinfo[n_box_ents*el + sDIM + idx];
-            box.x[idx].max = boxinfo[n_box_ents*el + 2*sDIM + idx];
+            for (int idx = 0; idx < sDIM; ++idx)
+            {
+               box.c0[idx] = boxinfo[n_box_ents*el + idx];
+               box.x[idx].min = boxinfo[n_box_ents*el + sDIM + idx];
+               box.x[idx].max = boxinfo[n_box_ents*el + 2*sDIM + idx];
+            }
+            for (int idx = 0; idx < sDIM2; ++idx)
+            {
+               box.A[idx] = boxinfo[n_box_ents*el + 3*sDIM + idx];
+            }
+            pass_bb = (bbox_test(&box, x_i) >= 0);
+         }
+         else
+         {
+            for (int d = 0; d < sDIM; ++d)
+            {
+               box.x[d].min = boxinfo[n_box_ents*el + d];
+               box.x[d].max = boxinfo[n_box_ents*el + sDIM + d];
+            }
+            pass_bb = (AABB_test(&box, x_i) >= 0);
          }
 
-         for (int idx = 0; idx < sDIM2; ++idx)
-         {
-            box.A[idx] = boxinfo[n_box_ents*el + 3*sDIM + idx];
-         }
-
-         if (obbox_test(&box,x_i)>=0)
+         if (pass_bb)
          {
             //------------ findpts_local ------------------
             {
@@ -516,11 +431,14 @@ static void FindPointsEdgeLocal2D_Kernel( const int     npt,
                            double *hess = jac + sDIM*rDIM;
 
                            findptsElementGEdge_t edge;
+                           for (int d=0; d<sDIM; ++d)
+                           {
+                              edge.x[d] = constraint_workspace + d*D1D;
+                           }
                            MFEM_FOREACH_THREAD(j,x,D1D)
                            {
                               for (int d=0; d<sDIM; ++d)
                               {
-                                 edge.x[d] = constraint_workspace + d*D1D;
                                  edge.x[d][j] = elx[d][j];
                               }
                            }
@@ -681,28 +599,41 @@ void FindPointsGSLIB::FindPointsEdgeLocal2( const Vector &point_pos,
    auto pgll1d = DEV.gll1d.ReadWrite(use_dev);
    auto plc = DEV.lagcoeff.Read(use_dev);
    double dist2tol = DEV.surf_dist_tol;
+   const bool obb_chk = obb_check;
    switch (DEV.dof1d)
    {
       case 2:
-         return FindPointsEdgeLocal2D_Kernel<2>(
-                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
-                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
-                   plho, pcode, pelem, pref, pdist, pgll1d, plc);
+         FindPointsEdgeLocal2DKernel<2>(npt, DEV.newt_tol, dist2tol,
+                                        pp, point_pos_ordering, pgslm,
+                                        NE_split_total, pwt, pbb, obb_chk,
+                                        DEV.lh_nx, plhm, plhf, plho,
+                                        pcode, pelem, pref, pdist,
+                                        pgll1d, plc);
+         break;
       case 3:
-         return FindPointsEdgeLocal2D_Kernel<3>(
-                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
-                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
-                   plho, pcode, pelem, pref, pdist, pgll1d, plc);
+         FindPointsEdgeLocal2DKernel<3>(npt, DEV.newt_tol, dist2tol,
+                                        pp, point_pos_ordering, pgslm,
+                                        NE_split_total, pwt, pbb, obb_chk,
+                                        DEV.lh_nx, plhm, plhf, plho,
+                                        pcode, pelem, pref, pdist,
+                                        pgll1d, plc);
+         break;
       case 4:
-         return FindPointsEdgeLocal2D_Kernel<4>(
-                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
-                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
-                   plho, pcode, pelem, pref, pdist, pgll1d, plc);
+         FindPointsEdgeLocal2DKernel<4>(npt, DEV.newt_tol, dist2tol,
+                                        pp, point_pos_ordering, pgslm,
+                                        NE_split_total, pwt, pbb, obb_chk,
+                                        DEV.lh_nx, plhm, plhf, plho,
+                                        pcode, pelem, pref, pdist,
+                                        pgll1d, plc);
+         break;
       default:
-         return FindPointsEdgeLocal2D_Kernel(
-                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
-                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
-                   plho, pcode, pelem, pref, pdist, pgll1d, plc, DEV.dof1d);
+         FindPointsEdgeLocal2DKernel(npt, DEV.newt_tol, dist2tol, pp,
+                                     point_pos_ordering, pgslm,
+                                     NE_split_total, pwt, pbb, obb_chk,
+                                     DEV.lh_nx, plhm, plhf, plho,
+                                     pcode, pelem, pref, pdist,
+                                     pgll1d, plc, DEV.dof1d);
+         break;
    }
 }
 #undef sDIM

fem/gslib/findptsedge_local_3.cpp

@@ -11,6 +11,7 @@
 
 #include "../gslib.hpp"
 #include "../../general/forall.hpp"
+#include "gslib_kernel_helpers.hpp"
 
 #ifdef MFEM_USE_GSLIB
 
@@ -54,117 +55,14 @@ struct findptsElementGPT_t
    double x[sDIM], jac[sDIM], hes[sDIM*(1+1)];
 };
 
-struct dbl_range_t
-{
-   double min, max;
-};
-
-struct obbox_t
-{
-   double c0[sDIM], A[sDIM*sDIM];
-   dbl_range_t x[sDIM];
-};
-
-struct findptsLocalHashData_t
-{
-   int hash_n;
-   dbl_range_t bnd[sDIM];
-   double fac[sDIM];
-   unsigned int *offset;
-};
-
-static MFEM_HOST_DEVICE inline void lag_eval_second_der(double *p0, double x,
-                                                        int i, const double *z,
-                                                        const double *lCoeff,
-                                                        int pN)
-{
-   double u0 = 1, u1 = 0, u2 = 0;
-   for (int j=0; j<pN; ++j)
-   {
-      if (i!=j)
-      {
-         double d_j = 2 * (x-z[j]);
-         u2 = d_j * u2 + u1;
-         u1 = d_j * u1 + u0;
-         u0 = d_j * u0;
-      }
-   }
-   double *p1 = p0 + pN, *p2 = p0 + 2 * pN;
-   p0[i] = lCoeff[i] * u0;
-   p1[i] = 2.0 * lCoeff[i] * u1;
-   p2[i] = 8.0 * lCoeff[i] * u2;
-}
-
-/* positive when possibly inside */
-static MFEM_HOST_DEVICE inline double obbox_axis_test(const obbox_t *const b,
-                                                      const double x[sDIM])
-{
-   double b_d;
-   for (int d=0; d<sDIM; ++d)
-   {
-      b_d = (x[d] - b->x[d].min) * (b->x[d].max - x[d]);
-      if (b_d < 0)   // if outside in any dimension
-      {
-         return b_d;
-      }
-   }
-   return b_d;       // only positive if inside in all dimensions
-}
-
-/* positive when possibly inside */
-static MFEM_HOST_DEVICE inline double obbox_test(const obbox_t *const b,
-                                                 const double x[sDIM])
-{
-   const double bxyz = obbox_axis_test(b, x);
-   if (bxyz<0)
-   {
-      return bxyz;
-   }
-   else
-   {
-      double dxyz[3];
-      // dxyz: distance of the point from the center of the OBB
-      for (int d=0; d<sDIM; ++d)
-      {
-         dxyz[d] = x[d] - b->c0[d];
-      }
-      // transform dxyz to the local coordinate system of the OBB,
-      // and check if the point is inside the OBB [-1,1]^sDIM
-      double test = 1;
-      for (int d=0; d<sDIM; ++d)
-      {
-         double rst = 0;
-         for (int e=0; e<sDIM; ++e)
-         {
-            rst += b->A[d*sDIM + e] * dxyz[e];
-         }
-         double brst = (rst+1)*(1-rst);
-         test = test<0 ? test : brst;
-      }
-      return test;
-   }
-}
-
-/* Hash index in the hash table to the elements that possibly contain the point x */
-static MFEM_HOST_DEVICE inline int hash_index(const findptsLocalHashData_t *p,
-                                              const double x[sDIM])
-{
-   const int n = p->hash_n;
-   int sum = 0;
-   for (int d=sDIM-1; d>=0; --d)
-   {
-      sum *= n;
-      int i = (int)floor((x[d] - p->bnd[d].min) * p->fac[d]);
-      sum += i<0 ? 0 : (n-1 < i ? n-1 : i);
-   }
-   return sum;
-}
-
-
-static MFEM_HOST_DEVICE inline double norm2(const double x[sDIM])
-{
-   return ( x[0]*x[0] + x[1]*x[1] + x[2]*x[2] );
-}
+using dbl_range_t = gslib::dbl_range_t;
+using obbox_t = gslib::obbox_t<sDIM>;
+using findptsLocalHashData_t = gslib::findptsLocalHashData_t<sDIM>;
+using gslib::AABB_test;
+using gslib::bbox_test;
+using gslib::hash_index;
+using gslib::l2norm2;
+using gslib::lag_eval_second_der;
 
 /* the bit structure of flags is CRR
    the C bit --- 1<<2 --- is set when the point is converged
@@ -175,47 +73,46 @@ static MFEM_HOST_DEVICE inline double norm2(const double x[sDIM])
 #define CONVERGED_FLAG (1u<<2)
 #define FLAG_MASK 0x07u
 
-/* returns the number of constrained reference coordinates, max 2
+/* returns the number of constrained reference coordinates, max 1
 */
 static MFEM_HOST_DEVICE inline int num_constrained(const int flags)
 {
-   const int y = (flags | flags>>1);
-   return (y & 1u) + (y>>2 & 1u);
+   return ((flags | flags>>1) & 1u);
 }
 
 static MFEM_HOST_DEVICE inline int point_index(const int x)
 {
-   return ((x>>1)&1u) | ((x>>2)&2u);
+   return ((x>>1)&1u);
 }
 
 /* check reduction in objective against prediction, and adjust
    trust region radius (p->tr) accordingly;
    may reject the prior step, returning 1; otherwise returns 0
-   sets out->dist2, out->index, out->x, out->oldr in any event,
-   leaving out->r, out->dr, out->flags to be set when returning 0 */
-static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsElementPoint_t *out,
+   sets out_pt->dist2, out_pt->index, out_pt->x, out_pt->oldr in any event,
+   leaving out_pt->r, out_pt->dr, out_pt->flags to be set when returning 0 */
+static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsElementPoint_t *out_pt,
                                                  const double resid[3],
                                                  const findptsElementPoint_t *p,
                                                  const double tol)
 {
-   const double dist2 = norm2(resid);
+   const double dist2 = l2norm2<sDIM>(resid);
    const double decr  = p->dist2 - dist2;
    const double pred  = p->dist2p;
    for (int d=0; d<sDIM; ++d)
    {
-      out->x[d] = p->x[d];
+      out_pt->x[d] = p->x[d];
    }
-   out->oldr = p->r;
-   out->dist2 = dist2;
+   out_pt->oldr = p->r;
+   out_pt->dist2 = dist2;
    if (decr>=0.01*pred)
    {
       if (decr>=0.9*pred)   // very good iteration
       {
-         out->tr = 2*p->tr;
+         out_pt->tr = 2*p->tr;
       }
       else   // good iteration
       {
-         out->tr = p->tr;
+         out_pt->tr = p->tr;
       }
       return false;
    }
@@ -226,21 +123,21 @@ static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsElementPoint_t *out,
          "very good iteration" --- this doubles the trust radius,
          which is why we divide by 4 below */
       double v0 = fabs(p->r - p->oldr);
-      out->tr = v0/4.0;
-      out->dist2 = p->dist2;
-      out->r = p->oldr;
-      out->flags = p->flags>>3;
-      out->dist2p = -HUGE_VAL;
+      out_pt->tr = v0/4.0;
+      out_pt->dist2 = p->dist2;
+      out_pt->r = p->oldr;
+      out_pt->flags = p->flags>>3;
+      out_pt->dist2p = -HUGE_VAL;
       if (pred<dist2*tol)
       {
-         out->flags |= CONVERGED_FLAG;
+         out_pt->flags |= CONVERGED_FLAG;
       }
       return true;
    }
 }
 
 static MFEM_HOST_DEVICE inline void newton_edge(findptsElementPoint_t *const
-                                                out,
+                                                out_pt,
                                                 const double jac[sDIM*rDIM],
                                                 const double rhes,
                                                 const double resid[sDIM],
@@ -314,9 +211,9 @@ newton_edge_fin:
    {
       new_flags |= CONVERGED_FLAG;
    }
-   out->r = nr;
-   out->dist2p = -v;
-   out->flags = flags | new_flags | ((p->flags & FLAG_MASK)<<3);
+   out_pt->r = nr;
+   out_pt->dist2p = -v;
+   out_pt->flags = flags | new_flags | ((p->flags & FLAG_MASK)<<3);
 #undef EVAL
 }
 
@@ -338,31 +235,32 @@ static MFEM_HOST_DEVICE void seed_j(const double *elx[sDIM],
    {
       dx[d] = x[d] - elx[d][ir];
    }
-   dist2[ir] = norm2(dx);;
+   dist2[ir] = l2norm2(dx);
    r[ir] = z[ir];
 }
 
 template<int T_D1D = 0>
-static void FindPointsEdgeLocal3D_Kernel(const int npt,
-                                         const double tol,
-                                         const double dist2tol,
-                                         const double *x,
-                                         const int point_pos_ordering,
-                                         const double *xElemCoord,
-                                         const int nel,
-                                         const double *wtend,
-                                         const double *boxinfo,
-                                         const int hash_n,
-                                         const double *hashMin,
-                                         const double *hashFac,
-                                         unsigned int *hashOffset,
-                                         unsigned int *const code_base,
-                                         unsigned int *const el_base,
-                                         double *const r_base,
-                                         double *const dist2_base,
-                                         const double *gll1D,
-                                         const double *lagcoeff,
-                                         const int pN = 0)
+static void FindPointsEdgeLocal3DKernel(const int npt,
+                                        const double tol,
+                                        const double dist2tol,
+                                        const double *x,
+                                        const int point_pos_ordering,
+                                        const double *xElemCoord,
+                                        const int nel,
+                                        const double *wtend,
+                                        const double *boxinfo,
+                                        const bool obb_check,
+                                        const int hash_n,
+                                        const double *hashMin,
+                                        const double *hashFac,
+                                        unsigned int *hashOffset,
+                                        unsigned int *const code_base,
+                                        unsigned int *const el_base,
+                                        double *const r_base,
+                                        double *const dist2_base,
+                                        const double *gll1D,
+                                        const double *lagcoeff,
+                                        const int pN = 0)
 {
    const int MD1   = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
    const int D1D   = T_D1D ? T_D1D : pN;
@@ -419,21 +317,35 @@ static void FindPointsEdgeLocal3D_Kernel(const int npt,
       for (; elp!=ele; ++elp)
       {
          const unsigned int el = *elp;
+
+         const int n_box_ents = obb_check ? (3*sDIM + sDIM2) : (2*sDIM);
+         bool pass_bb = true;
          obbox_t box;
-         int n_box_ents = 3*sDIM + sDIM2;
-
-         for (int idx = 0; idx < sDIM; ++idx)
+         if (obb_check)
          {
-            box.c0[idx] = boxinfo[n_box_ents*el + idx];
-            box.x[idx].min = boxinfo[n_box_ents*el + sDIM + idx];
-            box.x[idx].max = boxinfo[n_box_ents*el + 2*sDIM + idx];
+            for (int idx = 0; idx < sDIM; ++idx)
+            {
+               box.c0[idx] = boxinfo[n_box_ents*el + idx];
+               box.x[idx].min = boxinfo[n_box_ents*el + sDIM + idx];
+               box.x[idx].max = boxinfo[n_box_ents*el + 2*sDIM + idx];
+            }
+            for (int idx = 0; idx < sDIM2; ++idx)
+            {
+               box.A[idx] = boxinfo[n_box_ents*el + 3*sDIM + idx];
+            }
+            pass_bb = (bbox_test(&box, x_i) >= 0);
          }
-         for (int idx = 0; idx < sDIM2; ++idx)
+         else
          {
-            box.A[idx] = boxinfo[n_box_ents*el + 3*sDIM + idx];
+            for (int d = 0; d < sDIM; ++d)
+            {
+               box.x[d].min = boxinfo[n_box_ents*el + d];
+               box.x[d].max = boxinfo[n_box_ents*el + sDIM + d];
+            }
+            pass_bb = (AABB_test(&box, x_i) >= 0);
          }
 
-         if (obbox_test(&box, x_i)>=0)
+         if (pass_bb)
          {
             //// findpts_local ////
             {
@@ -521,11 +433,14 @@ static void FindPointsEdgeLocal3D_Kernel(const int npt,
                            double *hess = jac + sDIM*rDIM;
 
                            findptsElementGEdge_t edge;
+                           for (int d=0; d<sDIM; ++d)
+                           {
+                              edge.x[d] = constraint_workspace + d*D1D;
+                           }
                            MFEM_FOREACH_THREAD(j,x,D1D)
                            {
                               for (int d=0; d<sDIM; ++d)
                               {
-                                 edge.x[d] = constraint_workspace + d*D1D;
                                  edge.x[d][j] = elx[d][j];
                               }
                            }
@@ -688,28 +603,41 @@ void FindPointsGSLIB::FindPointsEdgeLocal3(const Vector &point_pos,
    auto pgll1d = DEV.gll1d.ReadWrite(use_dev);
    auto plc = DEV.lagcoeff.Read(use_dev);
    double dist2tol = DEV.surf_dist_tol;
+   const bool obb_chk = obb_check;
    switch (DEV.dof1d)
    {
       case 2:
-         return FindPointsEdgeLocal3D_Kernel<2>(
-                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
-                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
-                   plho, pcode, pelem, pref, pdist, pgll1d, plc);
+         FindPointsEdgeLocal3DKernel<2>(npt, DEV.newt_tol, dist2tol,
+                                        pp, point_pos_ordering, pgslm,
+                                        NE_split_total, pwt, pbb, obb_chk,
+                                        DEV.lh_nx, plhm, plhf, plho,
+                                        pcode, pelem, pref, pdist,
+                                        pgll1d, plc);
+         break;
       case 3:
-         return FindPointsEdgeLocal3D_Kernel<3>(
-                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
-                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
-                   plho, pcode, pelem, pref, pdist, pgll1d, plc);
+         FindPointsEdgeLocal3DKernel<3>(npt, DEV.newt_tol, dist2tol,
+                                        pp, point_pos_ordering, pgslm,
+                                        NE_split_total, pwt, pbb, obb_chk,
+                                        DEV.lh_nx, plhm, plhf, plho,
+                                        pcode, pelem, pref, pdist,
+                                        pgll1d, plc);
+         break;
       case 4:
-         return FindPointsEdgeLocal3D_Kernel<4>(
-                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
-                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
-                   plho, pcode, pelem, pref, pdist, pgll1d, plc);
+         FindPointsEdgeLocal3DKernel<4>(npt, DEV.newt_tol, dist2tol,
+                                        pp, point_pos_ordering, pgslm,
+                                        NE_split_total, pwt, pbb, obb_chk,
+                                        DEV.lh_nx, plhm, plhf, plho,
+                                        pcode, pelem, pref, pdist,
+                                        pgll1d, plc);
+         break;
       default:
-         return FindPointsEdgeLocal3D_Kernel(
-                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
-                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
-                   plho, pcode, pelem, pref, pdist, pgll1d, plc, DEV.dof1d);
+         FindPointsEdgeLocal3DKernel(npt, DEV.newt_tol, dist2tol, pp,
+                                     point_pos_ordering, pgslm,
+                                     NE_split_total, pwt, pbb, obb_chk,
+                                     DEV.lh_nx, plhm, plhf, plho,
+                                     pcode, pelem, pref, pdist,
+                                     pgll1d, plc, DEV.dof1d);
+         break;
    }
 }
 #undef rDIM2

fem/gslib/findptssurf_local_3.cpp

@@ -11,6 +11,7 @@
 
 #include "../gslib.hpp"
 #include "../../general/forall.hpp"
+#include "gslib_kernel_helpers.hpp"
 #ifdef MFEM_USE_GSLIB
 
 #ifdef MFEM_HAVE_GCC_PRAGMA_DIAGNOSTIC
@@ -51,124 +52,15 @@ struct findptsElementGPT_t
    double x[sDIM], jac[sDIM*rDIM], hes[sDIM*(rDIM+1)];
 };
 
-struct dbl_range_t
-{
-   double min, max;
-};
-
-struct obbox_t
-{
-   double c0[sDIM], A[sDIM*sDIM];
-   dbl_range_t x[sDIM];
-};
-
-struct findptsLocalHashData_t
-{
-   int hash_n;
-   dbl_range_t bnd[sDIM];
-   double fac[sDIM];
-   unsigned int *offset;
-};
-
-static MFEM_HOST_DEVICE inline void lag_eval_second_der(double *p0, double x,
-                                                        int i, const double *z,
-                                                        const double *lCoeff,
-                                                        int pN)
-{
-   double u0 = 1, u1 = 0, u2 = 0;
-   for (int j = 0; j < pN; ++j)
-   {
-      if (i != j)
-      {
-         double d_j = 2 * (x - z[j]);
-         u2 = d_j * u2 + u1;
-         u1 = d_j * u1 + u0;
-         u0 = d_j * u0;
-      }
-   }
-   p0[i] = lCoeff[i] * u0;
-   p0[pN+i] = 2.0 * lCoeff[i] * u1;
-   p0[2*pN+i] = 8.0 * lCoeff[i] * u2;
-}
-
-/* positive when possibly inside */
-static MFEM_HOST_DEVICE inline double AABB_test(const obbox_t *const b,
-                                                const double x[sDIM])
-{
-   double b_d;
-   for (int d=0; d<sDIM; ++d)
-   {
-      b_d = (x[d] - b->x[d].min) * (b->x[d].max - x[d]);
-      if (b_d < 0)   // if outside in any dimension
-      {
-         return b_d;
-      }
-   }
-   return b_d;       // only positive if inside in all dimensions
-}
-
-/* positive when possibly inside */
-static MFEM_HOST_DEVICE inline double bbox_test(const obbox_t *const b,
-                                                const double x[sDIM])
-{
-   const double bxyz = AABB_test(b, x);
-   if (bxyz<0)
-   {
-      return bxyz;
-   }
-   else
-   {
-      double dxyz[3];
-      // dxyz: distance of the point from the center of the OBB
-      for (int d=0; d<sDIM; ++d)
-      {
-         dxyz[d] = x[d] - b->c0[d];
-      }
-      // tranform dxyz to the local coordinate system of the OBB,
-      // and check if the point is inside the OBB [-1,1]^sDIM
-      double test = 1;
-      for (int d=0; d<sDIM; ++d)
-      {
-         double rst = 0;
-         for (int e=0; e<sDIM; ++e)
-         {
-            rst += b->A[d*sDIM + e] * dxyz[e];
-         }
-         double brst = (rst+1)*(1-rst);
-         test = test<0 ? test : brst;
-      }
-      return test;
-   }
-}
-
-/* Hash index in the hash table to the elements that possibly contain the point x */
-static MFEM_HOST_DEVICE inline int hash_index(const findptsLocalHashData_t *p,
-                                              const double x[sDIM])
-{
-   const int n = p->hash_n;
-   int sum = 0;
-   for (int d=sDIM-1; d>=0; --d)
-   {
-      sum *= n;
-      int i = (int)floor((x[d] - p->bnd[d].min) * p->fac[d]);
-      sum += i<0 ? 0 : (n-1 < i ? n-1 : i);
-   }
-   return sum;
-}
-
-static MFEM_HOST_DEVICE inline void lin_solve_sym_2(double x[2],
-                                                    const double A[3],
-                                                    const double y[2])
-{
-   const double idet = 1 / (A[0] * A[2] - A[1] * A[1]);
-   x[0] = idet * (A[2] * y[0] - A[1] * y[1]);
-   x[1] = idet * (A[0] * y[1] - A[1] * y[0]);
-}
-
-static MFEM_HOST_DEVICE inline double l2norm2(const double x[sDIM])
-{
-   return ( x[0]*x[0] + x[1]*x[1] + x[2]*x[2]);
-}
+using dbl_range_t = gslib::dbl_range_t;
+using obbox_t = gslib::obbox_t<sDIM>;
+using findptsLocalHashData_t = gslib::findptsLocalHashData_t<sDIM>;
+using gslib::AABB_test;
+using gslib::bbox_test;
+using gslib::hash_index;
+using gslib::l2norm2;
+using gslib::lag_eval_second_der;
+using gslib::lin_solve_sym_2;
 
 /* the bit structure of flags is CSSRR
    the C bit --- 1<<4 --- is set when the point is converged
@@ -219,18 +111,10 @@ static MFEM_HOST_DEVICE inline int point_index(const int x)
    return ((x>>1)&1u) | ((x>>2)&2u);
 }
 
-static MFEM_HOST_DEVICE inline findptsElementGEdge_t
+static MFEM_HOST_DEVICE inline void
 get_edge(const double *elx[3], const double *wtend, int ei,
-         double *workspace, int &side_init, int jidx, int pN)
+         int &side_init, int jidx, int pN, findptsElementGEdge_t &edge)
 {
-   findptsElementGEdge_t edge;
-   for (int d=0; d<sDIM; ++d)
-   {
-      edge.x[d]     = workspace             + d*pN;
-      edge.dxdn[d]  = workspace +   sDIM*pN + d*pN;
-      edge.d2xdn[d] = workspace + 2*sDIM*pN + d*pN;
-   }
-
    // given edge index, compute normal and tangential directions
    const int dn = ei>>1, //0 for rmin/rmax, 1 for smin/smax
              de = plus_1_mod_2(dn); // 1 for rmin/rmax, 0 for smin/smax
@@ -256,7 +140,6 @@ get_edge(const double *elx[3], const double *wtend, int ei,
       edge.d2xdn[dd][jj] = sums_k[1];
 #undef ELX
    }
-   return edge;
 }
 
 static MFEM_HOST_DEVICE inline findptsElementGPT_t get_pt(const double *elx[3],
@@ -312,34 +195,34 @@ static MFEM_HOST_DEVICE inline findptsElementGPT_t get_pt(const double *elx[3],
 /* check reduction in objective against prediction, and adjust
    trust region radius (p->tr) accordingly;
    may reject the prior step, returning 1; otherwise returns 0
-   sets out->dist2, out->index, out->x, out->oldr in any event,
-   leaving out->r, out->dr, out->flags to be set when returning 0 */
-static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsElementPoint_t *out,
+   sets out_pt->dist2, out_pt->index, out_pt->x, out_pt->oldr in any event,
+   leaving out_pt->r, out_pt->dr, out_pt->flags to be set when returning 0 */
+static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsElementPoint_t *out_pt,
                                                  const double resid[3],
                                                  const findptsElementPoint_t *p,
                                                  const double tol)
 {
-   const double dist2 = l2norm2(resid);
+   const double dist2 = l2norm2<sDIM>(resid);
    const double decr  = p->dist2 - dist2;
    const double pred  = p->dist2p;
    for (int d=0; d<sDIM; ++d)
    {
-      out->x[d] = p->x[d];
+      out_pt->x[d] = p->x[d];
    }
    for (int d=0; d<rDIM; ++d)
    {
-      out->oldr[d] = p->r[d];
+      out_pt->oldr[d] = p->r[d];
    }
-   out->dist2 = dist2;
+   out_pt->dist2 = dist2;
    if (decr>=0.01*pred)
    {
       if (decr>=0.9*pred)   // very good iteration
       {
-         out->tr = 2*p->tr;
+         out_pt->tr = 2*p->tr;
       }
       else   // good iteration
       {
-         out->tr = p->tr;
+         out_pt->tr = p->tr;
       }
       return false;
    }
@@ -351,17 +234,17 @@ static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsElementPoint_t *out,
          which is why we divide by 4 below */
       double v0 = fabs(p->r[0] - p->oldr[0]),
              v1 = fabs(p->r[1] - p->oldr[1]);
-      out->tr     = ( v0>v1 ? v0 : v1 )/4;
-      out->dist2  = p->dist2;
-      out->flags   = p->flags >> 5;
-      out->dist2p = -HUGE_VAL;
+      out_pt->tr     = ( v0>v1 ? v0 : v1 )/4;
+      out_pt->dist2  = p->dist2;
+      out_pt->flags   = p->flags >> 5;
+      out_pt->dist2p = -HUGE_VAL;
       for (int d=0; d<rDIM; ++d)
       {
-         out->r[d] = p->oldr[d];
+         out_pt->r[d] = p->oldr[d];
       }
       if (pred<dist2*tol)
       {
-         out->flags |= CONVERGED_FLAG;
+         out_pt->flags |= CONVERGED_FLAG;
       }
       return true;
    }
@@ -369,7 +252,7 @@ static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsElementPoint_t *out,
 
 /* minimize ||resid - jac * dr||_2, with |dr| <= tr, |r0+dr|<=1
    (exact solution of trust region problem) */
-static MFEM_HOST_DEVICE void newton_face( findptsElementPoint_t *const out,
+static MFEM_HOST_DEVICE void newton_face( findptsElementPoint_t *const out_pt,
                                           const double jac[sDIM*rDIM],
                                           const double rhes[3],
                                           const double resid[sDIM],
@@ -540,19 +423,19 @@ newton_face_constrained:
    }
 
 newton_face_fin:
-   out->dist2p = -2*v;
+   out_pt->dist2p = -2*v;
    dr[0] = r[0] - p->r[0];
    dr[1] = r[1] - p->r[1];
    if ( fabs(dr[0])+fabs(dr[1]) < tol)
    {
       new_flags |= CONVERGED_FLAG;
    }
-   out->r[0] = r[0], out->r[1] = r[1];
-   out->flags = new_flags | ((p->flags & FLAG_MASK)<<5);
+   out_pt->r[0] = r[0], out_pt->r[1] = r[1];
+   out_pt->flags = new_flags | ((p->flags & FLAG_MASK)<<5);
 }
 
 static MFEM_HOST_DEVICE inline void newton_edge(findptsElementPoint_t *const
-                                                out,
+                                                out_pt,
                                                 const double jac[sDIM*rDIM],
                                                 const double rhes,
                                                 const double resid[sDIM],
@@ -637,10 +520,10 @@ newton_edge_fin:
    {
       new_flags |= CONVERGED_FLAG;
    }
-   out->r[de] = nr;
-   out->r[dn] = p->r[dn];
-   out->dist2p = -v;
-   out->flags = flags | new_flags | ((p->flags & FLAG_MASK)<<5);
+   out_pt->r[de] = nr;
+   out_pt->r[dn] = p->r[dn];
+   out_pt->dist2p = -v;
+   out_pt->flags = flags | new_flags | ((p->flags & FLAG_MASK)<<5);
 #undef EVAL
 }
 
@@ -676,26 +559,27 @@ static MFEM_HOST_DEVICE void seed_j(const double *elx[sDIM],
 // global memory access of element coordinates.
 // Are the structs being stored in "local memory" or registers?
 template<int T_D1D = 0>
-static void FindPointsSurfLocal3D_Kernel(const int npt,
-                                         const double tol,
-                                         const double dist2tol,
-                                         const double *x,
-                                         const int point_pos_ordering,
-                                         const double *xElemCoord,
-                                         const int nel,
-                                         const double *wtend,
-                                         const double *boxinfo,
-                                         const int hash_n,
-                                         const double *hashMin,
-                                         const double *hashFac,
-                                         unsigned int *hashOffset,
-                                         unsigned int *const code_base,
-                                         unsigned int *const el_base,
-                                         double *const r_base,
-                                         double *const dist2_base,
-                                         const double *gll1D,
-                                         const double *lagcoeff,
-                                         const int pN = 0)
+static void FindPointsSurfLocal3DKernel(const int npt,
+                                        const double tol,
+                                        const double dist2tol,
+                                        const double *x,
+                                        const int point_pos_ordering,
+                                        const double *xElemCoord,
+                                        const int nel,
+                                        const double *wtend,
+                                        const double *boxinfo,
+                                        const bool obb_check,
+                                        const int hash_n,
+                                        const double *hashMin,
+                                        const double *hashFac,
+                                        unsigned int *hashOffset,
+                                        unsigned int *const code_base,
+                                        unsigned int *const el_base,
+                                        double *const r_base,
+                                        double *const dist2_base,
+                                        const double *gll1D,
+                                        const double *lagcoeff,
+                                        const int pN = 0)
 {
    const int MD1   = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
    const int D1D   = T_D1D ? T_D1D : pN;
@@ -753,22 +637,36 @@ static void FindPointsSurfLocal3D_Kernel(const int npt,
       {
          const unsigned int el = *elp;
 
-         // construct obbox on the fly
+         const int n_box_ents = obb_check ? (3*sDIM + sDIM2) : (2*sDIM);
+         bool pass_bb = true;
          obbox_t box;
-         int n_box_ents = 3*sDIM + sDIM2;
-         for (int idx = 0; idx < sDIM; ++idx)
+         if (obb_check)
          {
-            box.c0[idx] = boxinfo[n_box_ents*el + idx];
-            box.x[idx].min = boxinfo[n_box_ents*el + sDIM + idx];
-            box.x[idx].max = boxinfo[n_box_ents*el + 2*sDIM + idx];
+            // construct obbox on the fly
+            for (int idx = 0; idx < sDIM; ++idx)
+            {
+               box.c0[idx] = boxinfo[n_box_ents*el + idx];
+               box.x[idx].min = boxinfo[n_box_ents*el + sDIM + idx];
+               box.x[idx].max = boxinfo[n_box_ents*el + 2*sDIM + idx];
+            }
+
+            for (int idx = 0; idx < sDIM2; ++idx)
+            {
+               box.A[idx] = boxinfo[n_box_ents*el + 3*sDIM + idx];
+            }
+            pass_bb = (bbox_test(&box, x_i) >= 0);
+         }
+         else
+         {
+            for (int d = 0; d < sDIM; ++d)
+            {
+               box.x[d].min = boxinfo[n_box_ents*el + d];
+               box.x[d].max = boxinfo[n_box_ents*el + sDIM + d];
+            }
+            pass_bb = (AABB_test(&box, x_i) >= 0);
          }
 
-         for (int idx = 0; idx < sDIM2; ++idx)
-         {
-            box.A[idx] = boxinfo[n_box_ents*el + 3*sDIM + idx];
-         }
-
-         if (bbox_test(&box, x_i) < 0) { continue; }
+         if (!pass_bb) { continue; }
 
          //// findpts_local ////
          {
@@ -968,13 +866,19 @@ static void FindPointsSurfLocal3D_Kernel(const int npt,
                         double *hes_T = jac + sDIM*rDIM;
                         double *hes   = hes_T + hes_count*sDIM;
                         findptsElementGEdge_t edge;
+                        for (int d=0; d<sDIM; ++d)
+                        {
+                           edge.x[d]     = constraint_workspace + d*D1D;
+                           edge.dxdn[d]  = constraint_workspace + d*D1D
+                                           + sDIM*D1D;
+                           edge.d2xdn[d] = constraint_workspace + d*D1D
+                                           + 2*sDIM*D1D;
+                        }
 
                         MFEM_FOREACH_THREAD(j,x,D1D*sDIM)
                         {
-                           // utilized first D1D threads
-                           edge = get_edge(elx, wtend, ei,
-                                           constraint_workspace, edge_init, j,
-                                           D1D);
+                           // One thread per physical component and edge DOF.
+                           get_edge(elx, wtend, ei, edge_init, j, D1D, edge);
                         }
                         MFEM_SYNC_THREAD;
 
@@ -1045,7 +949,15 @@ static void FindPointsSurfLocal3D_Kernel(const int npt,
                               steep *= tmp->r[dn];
                               if (steep<0)
                               {
-                                 newton_face( fpt,jac,hes,resid,tmp->flags&CONVERGED_FLAG,tmp,tol);
+                                 double face_hes[3] =
+                                 {
+                                    dn == 0 ? hes[2] : hes[0],
+                                    hes[1],
+                                    dn == 0 ? hes[0] : hes[2]
+                                 };
+                                 newton_face(fpt, jac, face_hes, resid,
+                                             tmp->flags & CONVERGED_FLAG,
+                                             tmp, tol);
                               }
                               else
                               {
@@ -1211,29 +1123,42 @@ void FindPointsGSLIB::FindPointsSurfLocal3(const Vector &point_pos,
    auto pgll1d = DEV.gll1d.ReadWrite(use_dev);
    auto plc = DEV.lagcoeff.Read(use_dev);
    double dist2tol = DEV.surf_dist_tol;
+   const bool obb_chk = obb_check;
 
    switch (DEV.dof1d)
    {
       case 2:
-         return FindPointsSurfLocal3D_Kernel<2>(
-                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
-                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
-                   plho, pcode, pelem, pref, pdist, pgll1d, plc);
+         FindPointsSurfLocal3DKernel<2>(npt, DEV.newt_tol, dist2tol,
+                                        pp, point_pos_ordering, pgslm,
+                                        NE_split_total, pwt, pbb, obb_chk,
+                                        DEV.lh_nx, plhm, plhf, plho,
+                                        pcode, pelem, pref, pdist,
+                                        pgll1d, plc);
+         break;
       case 3:
-         return FindPointsSurfLocal3D_Kernel<3>(
-                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
-                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
-                   plho, pcode, pelem, pref, pdist, pgll1d, plc);
+         FindPointsSurfLocal3DKernel<3>(npt, DEV.newt_tol, dist2tol,
+                                        pp, point_pos_ordering, pgslm,
+                                        NE_split_total, pwt, pbb, obb_chk,
+                                        DEV.lh_nx, plhm, plhf, plho,
+                                        pcode, pelem, pref, pdist,
+                                        pgll1d, plc);
+         break;
       case 4:
-         return FindPointsSurfLocal3D_Kernel<4>(
-                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
-                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
-                   plho, pcode, pelem, pref, pdist, pgll1d, plc);
+         FindPointsSurfLocal3DKernel<4>(npt, DEV.newt_tol, dist2tol,
+                                        pp, point_pos_ordering, pgslm,
+                                        NE_split_total, pwt, pbb, obb_chk,
+                                        DEV.lh_nx, plhm, plhf, plho,
+                                        pcode, pelem, pref, pdist,
+                                        pgll1d, plc);
+         break;
       default:
-         return FindPointsSurfLocal3D_Kernel(
-                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
-                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
-                   plho, pcode, pelem, pref, pdist, pgll1d, plc, DEV.dof1d);
+         FindPointsSurfLocal3DKernel(npt, DEV.newt_tol, dist2tol, pp,
+                                     point_pos_ordering, pgslm,
+                                     NE_split_total, pwt, pbb, obb_chk,
+                                     DEV.lh_nx, plhm, plhf, plho,
+                                     pcode, pelem, pref, pdist,
+                                     pgll1d, plc, DEV.dof1d);
+         break;
    }
 }
 

fem/gslib/gslib_kernel_helpers.hpp

@@ -0,0 +1,190 @@
+#ifndef MFEM_GSLIB_KERNEL_HELPERS_HPP
+#define MFEM_GSLIB_KERNEL_HELPERS_HPP
+
+#include "../../config/config.hpp"
+
+#include <cmath>
+
+namespace mfem
+{
+
+namespace gslib
+{
+
+struct dbl_range_t
+{
+   double min, max;
+};
+
+template <int SDIM>
+struct obbox_t
+{
+   double c0[SDIM], A[SDIM * SDIM];
+   dbl_range_t x[SDIM];
+};
+
+template <int SDIM>
+struct findptsLocalHashData_t
+{
+   int hash_n;
+   dbl_range_t bnd[SDIM];
+   double fac[SDIM];
+   unsigned int *offset;
+};
+
+// Eval the ith Lagrange interpolant at x.
+MFEM_HOST_DEVICE inline void lagrange_eval(double *p0, double x,
+                                           int i, int p_Nq,
+                                           double *z, double *lagrangeCoeff)
+{
+   double p_i = (1 << (p_Nq - 1));
+   for (int j = 0; j < p_Nq; ++j)
+   {
+      const double d_j = x - z[j];
+      p_i *= j == i ? 1 : d_j;
+   }
+   p0[i] = lagrangeCoeff[i] * p_i;
+}
+
+// Eval the ith Lagrange interpolant and its first derivative at x.
+MFEM_HOST_DEVICE inline void lag_eval_first_der(double *p0, double x,
+                                                int i, const double *z,
+                                                const double *lCoeff,
+                                                int pN)
+{
+   double u0 = 1, u1 = 0;
+   for (int j = 0; j < pN; ++j)
+   {
+      if (i != j)
+      {
+         const double d_j = 2 * (x - z[j]);
+         u1 = d_j * u1 + u0;
+         u0 = d_j * u0;
+      }
+   }
+   p0[i] = lCoeff[i] * u0;
+   p0[pN + i] = 2.0 * lCoeff[i] * u1;
+}
+
+// Eval the ith Lagrange interpolant and its first and second derivative at x.
+MFEM_HOST_DEVICE inline void lag_eval_second_der(double *p0, double x,
+                                                 int i, const double *z,
+                                                 const double *lCoeff,
+                                                 int pN)
+{
+   double u0 = 1, u1 = 0, u2 = 0;
+   for (int j = 0; j < pN; ++j)
+   {
+      if (i != j)
+      {
+         const double d_j = 2 * (x - z[j]);
+         u2 = d_j * u2 + u1;
+         u1 = d_j * u1 + u0;
+         u0 = d_j * u0;
+      }
+   }
+   p0[i] = lCoeff[i] * u0;
+   p0[pN + i] = 2.0 * lCoeff[i] * u1;
+   p0[2 * pN + i] = 8.0 * lCoeff[i] * u2;
+}
+
+// Solve Ax=y where A is a symmetric 2x2 matrix packed as {a00, a01, a11}.
+MFEM_HOST_DEVICE inline void lin_solve_sym_2(double x[2],
+                                             const double A[3],
+                                             const double y[2])
+{
+   const double idet = 1 / (A[0] * A[2] - A[1] * A[1]);
+   x[0] = idet * (A[2] * y[0] - A[1] * y[1]);
+   x[1] = idet * (A[0] * y[1] - A[1] * y[0]);
+}
+
+// Positive when the point is inside the axis-aligned bounding box.
+template <int SDIM>
+MFEM_HOST_DEVICE inline double AABB_test(const obbox_t<SDIM> *const b,
+                                         const double (&x)[SDIM])
+{
+   double test = 1.0;
+   for (int d = 0; d < SDIM; ++d)
+   {
+      const double b_d = (x[d] - b->x[d].min) * (b->x[d].max - x[d]);
+      test = test < 0.0 ? test : b_d;
+   }
+   return test;
+}
+
+// Positive when the point is inside the oriented bounding box.
+template <int SDIM>
+MFEM_HOST_DEVICE inline double bbox_test(const obbox_t<SDIM> *const b,
+                                         const double (&x)[SDIM])
+{
+   const double bxyz = AABB_test(b, x);
+   if (bxyz < 0.0)
+   {
+      return bxyz;
+   }
+
+   double dxyz[SDIM];
+   for (int d = 0; d < SDIM; ++d)
+   {
+      dxyz[d] = x[d] - b->c0[d];
+   }
+
+   double test = 1.0;
+   for (int d = 0; d < SDIM; ++d)
+   {
+      double rst = 0.0;
+      for (int e = 0; e < SDIM; ++e)
+      {
+         rst += b->A[d * SDIM + e] * dxyz[e];
+      }
+      const double brst = (rst + 1.0) * (1.0 - rst);
+      test = test < 0.0 ? test : brst;
+   }
+   return test;
+}
+
+// Hash index in the hash table for the point x.
+template <int SDIM>
+MFEM_HOST_DEVICE inline int hash_index(
+   const findptsLocalHashData_t<SDIM> *const p,
+   const double (&x)[SDIM])
+{
+   const int n = p->hash_n;
+   int sum = 0;
+   for (int d = SDIM - 1; d >= 0; --d)
+   {
+      sum *= n;
+      const int i = (int)floor((x[d] - p->bnd[d].min) * p->fac[d]);
+      sum += i < 0 ? 0 : (n - 1 < i ? n - 1 : i);
+   }
+   return sum;
+}
+
+// Squared Euclidean norm.
+template <int SDIM>
+MFEM_HOST_DEVICE inline double l2norm2(const double (&x)[SDIM])
+{
+   double sum = 0.0;
+   for (int d = 0; d < SDIM; ++d)
+   {
+      sum += x[d] * x[d];
+   }
+   return sum;
+}
+
+template <int SDIM>
+MFEM_HOST_DEVICE inline double l2norm2(const double *x)
+{
+   double sum = 0.0;
+   for (int d = 0; d < SDIM; ++d)
+   {
+      sum += x[d] * x[d];
+   }
+   return sum;
+}
+
+} // namespace gslib
+
+} // namespace mfem
+
+#endif

fem/gslib/interpolate_local_1.cpp

@@ -11,7 +11,7 @@
 
 #include "../gslib.hpp"
 #include "../../general/forall.hpp"
-#include "../../linalg/kernels.hpp"
+#include "gslib_kernel_helpers.hpp"
 
 #ifdef MFEM_USE_GSLIB
 
@@ -33,17 +33,7 @@ namespace mfem
 #define CODE_BORDER 1
 #define CODE_NOT_FOUND 2
 
-static MFEM_HOST_DEVICE void lagrange_eval(double *p0, double x,
-                                           int i, int p_Nq,
-                                           double *z, double *lagrangeCoeff)
-{
-   double p_i = (1 << (p_Nq - 1));
-   for (int j=0; j<p_Nq; ++j)
-   {
-      p_i *= j==i ? 1 : x-z[j];
-   }
-   p0[i] = lagrangeCoeff[i] * p_i;
-}
+using gslib::lagrange_eval;
 
 template<int T_D1D = 0>
 static void InterpolateLocal1DKernel(const double *const gf_in,
@@ -123,21 +113,26 @@ void FindPointsGSLIB::InterpolateLocal1( const Vector &field_in,
    auto plcf = DEV.lagcoeff_sol.ReadWrite(use_dev);
    switch (dof1Dsol)
    {
-      case 2: return InterpolateLocal1DKernel<2>(pfin, pgsl, pgslr, pfout,
-                                                    npt, ncomp,
-                                                    pgll, plcf);
-      case 3: return InterpolateLocal1DKernel<3>(pfin, pgsl, pgslr, pfout,
-                                                    npt, ncomp,
-                                                    pgll, plcf);
-      case 4: return InterpolateLocal1DKernel<4>(pfin, pgsl, pgslr, pfout,
-                                                    npt, ncomp,
-                                                    pgll, plcf);
-      case 5: return InterpolateLocal1DKernel<5>(pfin, pgsl, pgslr, pfout,
-                                                    npt, ncomp,
-                                                    pgll, plcf);
-      default: return InterpolateLocal1DKernel(pfin, pgsl, pgslr, pfout,
-                                                  npt, ncomp,
-                                                  pgll, plcf, dof1Dsol);
+      case 2:
+         InterpolateLocal1DKernel<2>(pfin, pgsl, pgslr, pfout,
+                                     npt, ncomp, pgll, plcf);
+         break;
+      case 3:
+         InterpolateLocal1DKernel<3>(pfin, pgsl, pgslr, pfout,
+                                     npt, ncomp, pgll, plcf);
+         break;
+      case 4:
+         InterpolateLocal1DKernel<4>(pfin, pgsl, pgslr, pfout,
+                                     npt, ncomp, pgll, plcf);
+         break;
+      case 5:
+         InterpolateLocal1DKernel<5>(pfin, pgsl, pgslr, pfout,
+                                     npt, ncomp, pgll, plcf);
+         break;
+      default:
+         InterpolateLocal1DKernel(pfin, pgsl, pgslr, pfout,
+                                  npt, ncomp, pgll, plcf, dof1Dsol);
+         break;
    }
 }
 #undef CODE_INTERNAL

fem/gslib/interpolate_local_2.cpp

@@ -11,6 +11,7 @@
 
 #include "../gslib.hpp"
 #include "../../general/forall.hpp"
+#include "gslib_kernel_helpers.hpp"
 
 #ifdef MFEM_USE_GSLIB
 
@@ -32,18 +33,7 @@ namespace mfem
 #define CODE_BORDER 1
 #define CODE_NOT_FOUND 2
 
-static MFEM_HOST_DEVICE void lagrange_eval(double *p0, double x,
-                                           int i, int p_Nq,
-                                           double *z, double *lagrangeCoeff)
-{
-   double p_i = (1 << (p_Nq - 1));
-   for (int j = 0; j < p_Nq; ++j)
-   {
-      double d_j = x - z[j];
-      p_i *= j == i ? 1 : d_j;
-   }
-   p0[i] = lagrangeCoeff[i] * p_i;
-}
+using gslib::lagrange_eval;
 
 template<int T_D1D = 0>
 static void InterpolateLocal2DKernel(const double *const gf_in,
@@ -132,21 +122,26 @@ void FindPointsGSLIB::InterpolateLocal2(const Vector &field_in,
    auto plcf = DEV.lagcoeff_sol.ReadWrite(use_dev);
    switch (dof1Dsol)
    {
-      case 2: return InterpolateLocal2DKernel<2>(pfin, pgsl, pgslr, pfout,
-                                                    npt, ncomp,
-                                                    pgll, plcf);
-      case 3: return InterpolateLocal2DKernel<3>(pfin, pgsl, pgslr, pfout,
-                                                    npt, ncomp,
-                                                    pgll, plcf);
-      case 4: return InterpolateLocal2DKernel<4>(pfin, pgsl, pgslr, pfout,
-                                                    npt, ncomp,
-                                                    pgll, plcf);
-      case 5: return InterpolateLocal2DKernel<5>(pfin, pgsl, pgslr, pfout,
-                                                    npt, ncomp,
-                                                    pgll, plcf);
-      default: return InterpolateLocal2DKernel(pfin, pgsl, pgslr, pfout,
-                                                  npt, ncomp,
-                                                  pgll, plcf, dof1Dsol);
+      case 2:
+         InterpolateLocal2DKernel<2>(pfin, pgsl, pgslr, pfout,
+                                     npt, ncomp, pgll, plcf);
+         break;
+      case 3:
+         InterpolateLocal2DKernel<3>(pfin, pgsl, pgslr, pfout,
+                                     npt, ncomp, pgll, plcf);
+         break;
+      case 4:
+         InterpolateLocal2DKernel<4>(pfin, pgsl, pgslr, pfout,
+                                     npt, ncomp, pgll, plcf);
+         break;
+      case 5:
+         InterpolateLocal2DKernel<5>(pfin, pgsl, pgslr, pfout,
+                                     npt, ncomp, pgll, plcf);
+         break;
+      default:
+         InterpolateLocal2DKernel(pfin, pgsl, pgslr, pfout,
+                                  npt, ncomp, pgll, plcf, dof1Dsol);
+         break;
    }
 }
 

fem/gslib/interpolate_local_3.cpp

@@ -11,6 +11,7 @@
 
 #include "../gslib.hpp"
 #include "../../general/forall.hpp"
+#include "gslib_kernel_helpers.hpp"
 
 #ifdef MFEM_USE_GSLIB
 
@@ -32,18 +33,7 @@ namespace mfem
 #define CODE_BORDER 1
 #define CODE_NOT_FOUND 2
 
-static MFEM_HOST_DEVICE void lagrange_eval(double *p0, double x,
-                                           int i, int p_Nq,
-                                           double *z, double *lagrangeCoeff)
-{
-   double p_i = (1 << (p_Nq - 1));
-   for (int j = 0; j < p_Nq; ++j)
-   {
-      double d_j = x - z[j];
-      p_i *= j == i ? 1 : d_j;
-   }
-   p0[i] = lagrangeCoeff[i] * p_i;
-}
+using gslib::lagrange_eval;
 
 template<int T_D1D = 0>
 static void InterpolateLocal3DKernel(const double *const gf_in,
@@ -135,21 +125,26 @@ void FindPointsGSLIB::InterpolateLocal3(const Vector &field_in,
    auto plcf = DEV.lagcoeff_sol.ReadWrite(use_dev);
    switch (dof1Dsol)
    {
-      case 2: return InterpolateLocal3DKernel<2>(pfin, pgsle, pgslr, pfout,
-                                                    npt, ncomp,
-                                                    pgll, plcf);
-      case 3: return InterpolateLocal3DKernel<3>(pfin, pgsle, pgslr, pfout,
-                                                    npt, ncomp,
-                                                    pgll, plcf);
-      case 4: return InterpolateLocal3DKernel<4>(pfin, pgsle, pgslr, pfout,
-                                                    npt, ncomp,
-                                                    pgll, plcf);
-      case 5: return InterpolateLocal3DKernel<5>(pfin, pgsle, pgslr, pfout,
-                                                    npt, ncomp,
-                                                    pgll, plcf);
-      default: return InterpolateLocal3DKernel(pfin, pgsle, pgslr, pfout,
-                                                  npt, ncomp,
-                                                  pgll, plcf, dof1Dsol);
+      case 2:
+         InterpolateLocal3DKernel<2>(pfin, pgsle, pgslr, pfout,
+                                     npt, ncomp, pgll, plcf);
+         break;
+      case 3:
+         InterpolateLocal3DKernel<3>(pfin, pgsle, pgslr, pfout,
+                                     npt, ncomp, pgll, plcf);
+         break;
+      case 4:
+         InterpolateLocal3DKernel<4>(pfin, pgsle, pgslr, pfout,
+                                     npt, ncomp, pgll, plcf);
+         break;
+      case 5:
+         InterpolateLocal3DKernel<5>(pfin, pgsle, pgslr, pfout,
+                                     npt, ncomp, pgll, plcf);
+         break;
+      default:
+         InterpolateLocal3DKernel(pfin, pgsle, pgslr, pfout,
+                                  npt, ncomp, pgll, plcf, dof1Dsol);
+         break;
    }
 }
 

fem/integ/bilininteg_hcurl_kernels.cpp

@@ -147,15 +147,18 @@ void PAHcurlMassAssembleDiagonal3D(const int D1D,
    }); // end of element loop
 }
 
-void PAHcurlMassApply2D(const int NE, const bool symmetric,
-                        const bool scalar_coeff, const Array<real_t> &bo,
-                        const Array<real_t> &bc, const Array<real_t> &bot,
-                        const Array<real_t> &bct, const Vector &pa_data,
-                        const Vector &x, Vector &y, const int D1D,
-                        const int TestD1D, const int Q1D)
+void PAHcurlMassApply2D(const int D1D,
+                        const int Q1D,
+                        const int NE,
+                        const bool symmetric,
+                        const Array<real_t> &bo,
+                        const Array<real_t> &bc,
+                        const Array<real_t> &bot,
+                        const Array<real_t> &bct,
+                        const Vector &pa_data,
+                        const Vector &x,
+                        Vector &y)
 {
-   MFEM_ASSERT(D1D == TestD1D,
-               "Trial and Test space must have the same number of dofs");
    auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
    auto Bc = Reshape(bc.Read(), Q1D, D1D);
    auto Bot = Reshape(bot.Read(), D1D-1, Q1D);
@@ -274,15 +277,18 @@ void PAHcurlMassApply2D(const int NE, const bool symmetric,
    }); // end of element loop
 }
 
-void PAHcurlMassApply3D(const int NE, const bool symmetric,
-                        const bool scalar_coeff, const Array<real_t> &bo,
-                        const Array<real_t> &bc, const Array<real_t> &bot,
-                        const Array<real_t> &bct, const Vector &pa_data,
-                        const Vector &x, Vector &y, const int D1D,
-                        const int TestD1D, const int Q1D)
+void PAHcurlMassApply3D(const int D1D,
+                        const int Q1D,
+                        const int NE,
+                        const bool symmetric,
+                        const Array<real_t> &bo,
+                        const Array<real_t> &bc,
+                        const Array<real_t> &bot,
+                        const Array<real_t> &bct,
+                        const Vector &pa_data,
+                        const Vector &x,
+                        Vector &y)
 {
-   MFEM_VERIFY(D1D == TestD1D,
-               "Trial and test spaces must have same number of dofs");
    MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
                "Error: D1D > MAX_D1D");
    MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,

fem/integ/bilininteg_hcurl_kernels.hpp

@@ -181,309 +181,228 @@ inline void SmemPAHcurlMassAssembleDiagonal3D(const int d1d,
 }
 
 // PA H(curl) Mass Apply 2D kernel
-void PAHcurlMassApply2D(const int NE, const bool symmetric,
-                        const bool scalar_coeff, const Array<real_t> &bo,
-                        const Array<real_t> &bc, const Array<real_t> &bot,
-                        const Array<real_t> &bct, const Vector &pa_data,
-                        const Vector &x, Vector &y, const int TrialD1D,
-                        const int TestD1D, const int Q1D);
+void PAHcurlMassApply2D(const int D1D,
+                        const int Q1D,
+                        const int NE,
+                        const bool symmetric,
+                        const Array<real_t> &bo,
+                        const Array<real_t> &bc,
+                        const Array<real_t> &bot,
+                        const Array<real_t> &bct,
+                        const Vector &pa_data,
+                        const Vector &x,
+                        Vector &y);
 
 // PA H(curl) Mass Apply 3D kernel
-void PAHcurlMassApply3D(const int NE, const bool symmetric,
-                        const bool scalar_coeff, const Array<real_t> &bo,
-                        const Array<real_t> &bc, const Array<real_t> &bot,
-                        const Array<real_t> &bct, const Vector &pa_data,
-                        const Vector &x, Vector &y, const int TrialD1D,
-                        const int TestD1D, const int Q1D);
+void PAHcurlMassApply3D(const int D1D,
+                        const int Q1D,
+                        const int NE,
+                        const bool symmetric,
+                        const Array<real_t> &bo,
+                        const Array<real_t> &bc,
+                        const Array<real_t> &bot,
+                        const Array<real_t> &bct,
+                        const Vector &pa_data,
+                        const Vector &x,
+                        Vector &y);
 
 // Shared memory PA H(curl) Mass Apply 3D kernel
-template <int T_D1D = 0, int T_Q1D = 0, int TBATCH = 0, bool ACCUMULATE = true>
-inline void SmemPAHcurlMassApply3D(
-   const int NE, const bool symmetric, const bool scalar_coeff,
-   const Array<real_t> &bo, const Array<real_t> &bc, const Array<real_t> &bot,
-   const Array<real_t> &bct, const Vector &pa_data, const Vector &x, Vector &y,
-   const int d1d = 0, const int = 0, const int q1d = 0)
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAHcurlMassApply3D(const int d1d,
+                                   const int q1d,
+                                   const int NE,
+                                   const bool symmetric,
+                                   const Array<real_t> &bo,
+                                   const Array<real_t> &bc,
+                                   const Array<real_t> &bot,
+                                   const Array<real_t> &bct,
+                                   const Vector &pa_data,
+                                   const Vector &x,
+                                   Vector &y)
 {
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
    MFEM_VERIFY(T_D1D || d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
                "Error: d1d > HCURL_MAX_D1D");
    MFEM_VERIFY(T_Q1D || q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
                "Error: q1d > HCURL_MAX_Q1D");
-   MFEM_ASSERT(Q1D >= D1D, "Expected Q1D >= D1D");
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
    const int dataSize = symmetric ? 6 : 9;
 
-   // assume trial space == test space
-   auto Bo = bo.Read();
-   auto Bc = bc.Read();
-   auto op =
-      Reshape(pa_data.Read(), Q1D, Q1D, Q1D, dataSize, NE);
-   auto X_ = Reshape(x.Read(), 3 * (D1D - 1) * D1D * D1D, NE);
-   auto y_ = y.ReadWrite();
+   auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
+   auto Bc = Reshape(bc.Read(), Q1D, D1D);
+   auto op = Reshape(pa_data.Read(), Q1D, Q1D, Q1D, dataSize, NE);
+   auto X = Reshape(x.Read(), 3*(D1D-1)*D1D*D1D, NE);
+   auto Y = Reshape(y.ReadWrite(), 3*(D1D-1)*D1D*D1D, NE);
 
-   constexpr int MD_ = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
-   constexpr int MQ_ = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
-   constexpr int MDQ_ = std::max(MD_, MQ_);
-   constexpr int MB_ = TBATCH ? TBATCH : 1;
-
-   mfem::forall_2D_batch<MDQ_ * MDQ_ * MDQ_ * MB_>(
-      NE, MDQ_ * MDQ_ * MDQ_, 1, MB_, [=] MFEM_HOST_DEVICE(int e)
+   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
    {
-#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
-      constexpr int nbz = TBATCH ? TBATCH : 1;
-      int tidz = MFEM_THREAD_ID(z);
-#else
-      constexpr int nbz = 1;
-      constexpr int tidz = 0;
-#endif
-
       constexpr int VDIM = 3;
       constexpr int MD1D = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
       constexpr int MQ1D = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
-      constexpr int MDQ = std::max(MD1D, MQ1D);
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
 
-      // nvcc limit work-around: can't have Y_ be captured first in
-      // if constexpr, so capture y_ and construct Y_ locally
-      // only works on GPU
-      auto Y = Reshape(y_, VDIM * (D1D - 1) * D1D * D1D, NE);
+      MFEM_SHARED real_t sBo[MQ1D][MD1D];
+      MFEM_SHARED real_t sBc[MQ1D][MD1D];
 
-      MFEM_SHARED real_t sBo[MDQ * (MD1D - 1)];
-      MFEM_SHARED real_t sBc[MDQ * MD1D];
-      auto BO = Reshape(sBo, Q1D, D1D - 1);
-      auto BC = Reshape(sBc, Q1D, D1D);
+      real_t op9[9];
+      MFEM_SHARED real_t sop[9*MQ1D*MQ1D];
+      MFEM_SHARED real_t mass[MQ1D][MQ1D][3];
 
-      MFEM_SHARED real_t sX[nbz * VDIM * (MD1D - 1) * MD1D * MD1D];
-      MFEM_SHARED real_t sm0[nbz * VDIM * MDQ * MDQ * MDQ];
-      MFEM_SHARED real_t sm1[nbz * VDIM * MDQ * MDQ * MDQ];
+      MFEM_SHARED real_t sX[MD1D][MD1D][MD1D];
 
-      real_t(*X)[nbz][(MD1D - 1) * MD1D * MD1D] =
-         (real_t(*)[nbz][(MD1D - 1) * MD1D * MD1D])(sX);
-      // shapes of buffers always use MQ1D to mitigate shared memory bank
-      // conflicts
-      real_t(*DDQ)[nbz][MQ1D][MQ1D][MQ1D] =
-         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm0);
-      real_t(*DQQ)[nbz][MQ1D][MQ1D][MQ1D] =
-         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm1);
-      real_t(*QQQ)[nbz][MQ1D][MQ1D][MQ1D] =
-         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm0);
-      real_t(*QQD)[nbz][MQ1D][MQ1D][MQ1D] =
-         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm1);
-      real_t(*QDD)[nbz][MQ1D][MQ1D][MQ1D] =
-         (real_t(*)[nbz][MQ1D][MQ1D][MQ1D])(sm0);
-
-      // load dofs into smem
-      const int offset = (D1D - 1) * D1D * D1D;
-      MFEM_FOREACH_THREAD_DIRECT(ix, x, offset)
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
       {
-         for (int dim = 0; dim < VDIM; ++dim)
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
          {
-            X[dim][tidz][ix] = X_(ix + dim * offset, e);
+            MFEM_FOREACH_THREAD(qz,z,Q1D)
+            {
+               for (int i=0; i<dataSize; ++i)
+               {
+                  op9[i] = op(qx,qy,qz,i,e);
+               }
+            }
          }
       }
-      // load basis functions data
+
+      const int tidx = MFEM_THREAD_ID(x);
+      const int tidy = MFEM_THREAD_ID(y);
+      const int tidz = MFEM_THREAD_ID(z);
+
       if (tidz == 0)
       {
-         MFEM_FOREACH_THREAD_DIRECT(ix, x, D1D * Q1D) { sBc[ix] = Bc[ix]; }
-         MFEM_FOREACH_THREAD_DIRECT(ix, x, (D1D - 1) * Q1D)
+         MFEM_FOREACH_THREAD(d,y,D1D)
          {
-            sBo[ix] = Bo[ix];
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               sBc[q][d] = Bc(q,d);
+               if (d < D1D-1)
+               {
+                  sBo[q][d] = Bo(q,d);
+               }
+            }
          }
       }
+      MFEM_SYNC_THREAD;
 
-      for (int dim0 = 0; dim0 < VDIM; ++dim0)
+      for (int qz=0; qz < Q1D; ++qz)
       {
-         MFEM_SYNC_THREAD;
-         // sum factor to QQQ = Q_{dim0,dim1} B X_{dim1}
-         for (int dim1 = 0; dim1 < VDIM; ++dim1)
+         int osc = 0;
+         for (int c = 0; c < VDIM; ++c)  // loop over x, y, z components
          {
-            const int D1Dz = (dim1 == 2) ? D1D - 1 : D1D;
-            const int D1Dy = (dim1 == 1) ? D1D - 1 : D1D;
-            const int D1Dx = (dim1 == 0) ? D1D - 1 : D1D;
+            const int D1Dz = (c == 2) ? D1D - 1 : D1D;
+            const int D1Dy = (c == 1) ? D1D - 1 : D1D;
+            const int D1Dx = (c == 0) ? D1D - 1 : D1D;
 
-            // threads assigned to mitigate bank conflicts
-            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, Q1D, D1Dy, D1Dz,
-                                                 Q1D, Q1D, Q1D)
+            MFEM_FOREACH_THREAD(dz,z,D1Dz)
             {
-               real_t u = 0;
-               for (int dx = 0; dx < D1Dx; ++dx)
+               MFEM_FOREACH_THREAD(dy,y,D1Dy)
                {
-                  real_t b;
-                  if (dim1 == 0)
+                  MFEM_FOREACH_THREAD(dx,x,D1Dx)
                   {
-                     b = BO(qx, dx);
+                     sX[dz][dy][dx] = X(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e);
                   }
-                  else
-                  {
-                     b = BC(qx, dx);
-                  }
-                  u += X[dim1][tidz][dx + (dy + dz * D1Dy) * D1Dx] * b;
-               }
-               DDQ[dim1][tidz][dz][dy][qx] = u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-         for (int dim1 = 0; dim1 < VDIM; ++dim1)
-         {
-            const int D1Dz = (dim1 == 2) ? D1D - 1 : D1D;
-            const int D1Dy = (dim1 == 1) ? D1D - 1 : D1D;
-            // const int D1Dx = (dim1 == 0) ? D1D - 1 : D1D;
-            // threads assigned to mitigate bank conflicts
-            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, Q1D, Q1D, D1Dz,
-                                                 Q1D, Q1D, Q1D)
-            {
-               real_t u = 0;
-               for (int dy = 0; dy < D1Dy; ++dy)
-               {
-                  real_t b;
-                  if (dim1 == 1)
-                  {
-                     b = BO(qy, dy);
-                  }
-                  else
-                  {
-                     b = BC(qy, dy);
-                  }
-                  u += DDQ[dim1][tidz][dz][dy][qx] * b;
-               }
-               DQQ[dim1][tidz][dz][qy][qx] = u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-         for (int dim1 = 0; dim1 < VDIM; ++dim1)
-         {
-            const int D1Dz = (dim1 == 2) ? D1D - 1 : D1D;
-            // const int D1Dy = (dim1 == 1) ? D1D - 1 : D1D;
-            // const int D1Dx = (dim1 == 0) ? D1D - 1 : D1D;
-            MFEM_FOREACH_THREAD_DIRECT_3D(qx, qy, qz, x, Q1D, Q1D, Q1D)
-            {
-               real_t u = 0;
-               for (int dz = 0; dz < D1Dz; ++dz)
-               {
-                  real_t b;
-                  if (dim1 == 2)
-                  {
-                     b = BO(qz, dz);
-                  }
-                  else
-                  {
-                     b = BC(qz, dz);
-                  }
-                  u += DQQ[dim1][tidz][dz][qy][qx] * b;
-               }
-               // pa_data is row major
-               int idx;
-               if (symmetric)
-               {
-                  int row;
-                  int col;
-                  if (dim0 > dim1)
-                  {
-                     row = dim1;
-                     col = dim0;
-                  }
-                  else
-                  {
-                     row = dim0;
-                     col = dim1;
-                  }
-                  idx = col + VDIM * row - row * (row + 1) / 2;
-               }
-               else
-               {
-                  idx = dim0 * VDIM + dim1;
-               }
-               QQQ[dim1][tidz][qz][qy][qx] = op(qx, qy, qz, idx, e) * u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-         // sum factor back to Y
-         // Assume bot and bct == bo^t and bc^t respectively (i.e. test ==
-         // trial functions), skip loading them again.
-         {
-            const int D1Dz = (dim0 == 2) ? D1D - 1 : D1D;
-            const int D1Dy = (dim0 == 1) ? D1D - 1 : D1D;
-            const int D1Dx = (dim0 == 0) ? D1D - 1 : D1D;
-            // threads assigned to mitigate bank conflicts
-            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, D1Dz, Q1D, Q1D,
-                                                 Q1D, Q1D, Q1D)
-            {
-               for (int dim1 = 0; dim1 < VDIM; ++dim1)
-               {
-                  real_t u = 0;
-                  for (int qz = 0; qz < Q1D; ++qz)
-                  {
-                     real_t b = 0;
-                     if (dim0 == 2)
-                     {
-                        b = BO(qz, dz);
-                     }
-                     else
-                     {
-                        b = BC(qz, dz);
-                     }
-                     u += QQQ[dim1][tidz][qz][qy][qx] * b;
-                  }
-                  QQD[dim1][tidz][qy][qx][dz] = u;
-               }
-            }
-            MFEM_SYNC_THREAD;
-            // threads assigned to mitigate bank conflicts
-            MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, D1Dy, D1Dz, Q1D,
-                                                 Q1D, Q1D, Q1D)
-            {
-               for (int dim1 = 0; dim1 < VDIM; ++dim1)
-               {
-                  real_t u = 0;
-                  for (int qy = 0; qy < Q1D; ++qy)
-                  {
-                     real_t b;
-                     if (dim0 == 1)
-                     {
-                        b = BO(qy, dy);
-                     }
-                     else
-                     {
-                        b = BC(qy, dy);
-                     }
-                     u += QQD[dim1][tidz][qy][qx][dz] * b;
-                  }
-                  QDD[dim1][tidz][qx][dz][dy] = u;
                }
             }
             MFEM_SYNC_THREAD;
 
-            MFEM_FOREACH_THREAD_DIRECT_3D(dx, dy, dz, x, D1Dx, D1Dy, D1Dz)
+            if (tidz == qz)
             {
-               int ix = dx + D1Dx * (dy + D1Dy * dz);
-               real_t u = 0;
-               for (int qx = 0; qx < Q1D; ++qx)
+               for (int i=0; i<dataSize; ++i)
                {
-                  real_t b;
-                  if (dim0 == 0)
-                  {
-                     b = BO(qx, dx);
-                  }
-                  else
-                  {
-                     b = BC(qx, dx);
-                  }
-                  for (int dim1 = 0; dim1 < VDIM; ++dim1)
-                  {
-                     u += QDD[dim1][tidz][qx][dz][dy] * b;
-                  }
+                  sop[i + (dataSize*tidx) + (dataSize*Q1D*tidy)] = op9[i];
                }
-               if constexpr (ACCUMULATE)
+
+               MFEM_FOREACH_THREAD(qy,y,Q1D)
                {
-                  Y(ix + dim0 * offset, e) += u;
-               }
-               else
+                  MFEM_FOREACH_THREAD(qx,x,Q1D)
+                  {
+                     real_t u = 0.0;
+
+                     for (int dz = 0; dz < D1Dz; ++dz)
+                     {
+                        const real_t wz = (c == 2) ? sBo[qz][dz] : sBc[qz][dz];
+                        for (int dy = 0; dy < D1Dy; ++dy)
+                        {
+                           const real_t wy = (c == 1) ? sBo[qy][dy] : sBc[qy][dy];
+                           for (int dx = 0; dx < D1Dx; ++dx)
+                           {
+                              const real_t t = sX[dz][dy][dx];
+                              const real_t wx = (c == 0) ? sBo[qx][dx] : sBc[qx][dx];
+                              u += t * wx * wy * wz;
+                           }
+                        }
+                     }
+
+                     mass[qy][qx][c] = u;
+                  } // qx
+               } // qy
+            } // tidz == qz
+
+            osc += D1Dx * D1Dy * D1Dz;
+            MFEM_SYNC_THREAD;
+         } // c
+
+         MFEM_SYNC_THREAD;  // Sync mass[qy][qx][d] and sop
+
+         osc = 0;
+         for (int c = 0; c < VDIM; ++c)  // loop over x, y, z components
+         {
+            const int D1Dz = (c == 2) ? D1D - 1 : D1D;
+            const int D1Dy = (c == 1) ? D1D - 1 : D1D;
+            const int D1Dx = (c == 0) ? D1D - 1 : D1D;
+
+            real_t dxyz = 0.0;
+
+            MFEM_FOREACH_THREAD(dz,z,D1Dz)
+            {
+               const real_t wz = (c == 2) ? sBo[qz][dz] : sBc[qz][dz];
+
+               MFEM_FOREACH_THREAD(dy,y,D1Dy)
                {
-                  Y(ix + dim0 * offset, e) = u;
+                  MFEM_FOREACH_THREAD(dx,x,D1Dx)
+                  {
+                     for (int qy = 0; qy < Q1D; ++qy)
+                     {
+                        const real_t wy = (c == 1) ? sBo[qy][dy] : sBc[qy][dy];
+                        for (int qx = 0; qx < Q1D; ++qx)
+                        {
+                           const int os = (dataSize*qx) + (dataSize*Q1D*qy);
+                           const int id1 = os + ((c == 0) ? 0 : ((c == 1) ? (symmetric ? 1 : 3) :
+                                                                 (symmetric ? 2 : 6))); // O11, O21, O31
+                           const int id2 = os + ((c == 0) ? 1 : ((c == 1) ? (symmetric ? 3 : 4) :
+                                                                 (symmetric ? 4 : 7))); // O12, O22, O32
+                           const int id3 = os + ((c == 0) ? 2 : ((c == 1) ? (symmetric ? 4 : 5) :
+                                                                 (symmetric ? 5 : 8))); // O13, O23, O33
+
+                           const real_t m_c = (sop[id1] * mass[qy][qx][0]) + (sop[id2] * mass[qy][qx][1]) +
+                                              (sop[id3] * mass[qy][qx][2]);
+
+                           const real_t wx = (c == 0) ? sBo[qx][dx] : sBc[qx][dx];
+                           dxyz += m_c * wx * wy * wz;
+                        }
+                     }
+                  }
                }
             }
-         }
-      }
+
+            MFEM_SYNC_THREAD;
+
+            MFEM_FOREACH_THREAD(dz,z,D1Dz)
+            {
+               MFEM_FOREACH_THREAD(dy,y,D1Dy)
+               {
+                  MFEM_FOREACH_THREAD(dx,x,D1Dx)
+                  {
+                     Y(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e) += dxyz;
+                  }
+               }
+            }
+
+            osc += D1Dx * D1Dy * D1Dz;
+         } // c loop
+      } // qz
    }); // end of element loop
 }
 

fem/integ/bilininteg_hcurlhdiv_kernels.hpp

@@ -62,30 +62,6 @@ void PAHcurlHdivMassApply2D(const int D1D,
                             const Vector &x_,
                             Vector &y_);
 
-/// H(curl) test, H(div) trial
-inline void
-PAHcurlHdivMassApply2D(const int NE, const bool, const bool scalarCoeff,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int D1Dtest, const int Q1D)
-{
-   return PAHcurlHdivMassApply2D(D1D, D1Dtest, Q1D, NE, scalarCoeff, false,
-                                 false, Bo_, Bc_, Bot_, Bct_, op_, x_, y_);
-}
-
-/// H(div) test, H(curl) trial
-inline void
-PAHdivHcurlMassApply2D(const int NE, const bool, const bool scalarCoeff,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int D1Dtest, const int Q1D)
-{
-   return PAHcurlHdivMassApply2D(D1D, D1Dtest, Q1D, NE, scalarCoeff, true,
-                                 false, Bo_, Bc_, Bot_, Bct_, op_, x_, y_);
-}
-
 // PA H(curl)-H(div) Mass Apply 3D kernel
 void PAHcurlHdivMassApply3D(const int D1D,
                             const int D1Dtest,
@@ -102,30 +78,6 @@ void PAHcurlHdivMassApply3D(const int D1D,
                             const Vector &x_,
                             Vector &y_);
 
-/// H(curl) test, H(div) trial
-inline void
-PAHcurlHdivMassApply3D(const int NE, const bool, const bool scalarCoeff,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int D1Dtest, const int Q1D)
-{
-   PAHcurlHdivMassApply3D(D1D, D1Dtest, Q1D, NE, scalarCoeff, false, false, Bo_,
-                          Bc_, Bot_, Bct_, op_, x_, y_);
-}
-
-/// H(div) test, H(curl) trial
-inline void
-PAHdivHcurlMassApply3D(const int NE, const bool, const bool scalarCoeff,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int D1Dtest, const int Q1D)
-{
-   PAHcurlHdivMassApply3D(D1D, D1Dtest, Q1D, NE, scalarCoeff, true, false, Bo_,
-                          Bc_, Bot_, Bct_, op_, x_, y_);
-}
-
 // PA H(curl)-H(div) Curl Apply 3D kernel
 template<int T_D1D = 0, int T_D1D_TEST = 0, int T_Q1D = 0>
 inline void PAHcurlHdivApply3D(const int d1d,
@@ -864,656 +816,8 @@ inline void PAHcurlHdivApplyTranspose3D(const int d1d,
    }); // end of element loop
 }
 
-namespace curlinterp
-{
-constexpr int NBZ3D(int ndof_o, int nquad_o, int mdq)
-{
-   if (ndof_o <= 0 || nquad_o <= 0)
-   {
-      return 1;
-   }
-   int ndof_c = ndof_o + 1;
-   int nquad_c = nquad_o + 1;
-   // z dimension is capped at 64 on nvidia and amd gpus
-   int tmp =
-      std::min((128 + mdq * mdq * (mdq - 1) - 1) / (mdq * mdq * (mdq - 1)), 64);
-   int smem_req =
-      sizeof(mfem::real_t) *
-      ((3 * ndof_c * ndof_c * ndof_o + 2 * 2 * mdq * mdq * mdq) * tmp +
-       ndof_c * nquad_o + ndof_c * nquad_c + ndof_o * nquad_o);
-   // assume GPU has at least 48k shared memory
-   return std::max(std::min(tmp, (48 * 1024 + smem_req - 1) / smem_req), 1);
-}
-}
-
-template <int T_NDOF_O, int T_NQUAD_O>
-void CurlInterpolatorApply3DSmem(const int ne, const int ndof_o,
-                                 const int nquad_o, const Vector &pa,
-                                 const Vector &x_, Vector &y_)
-{
-   constexpr int mnd_o = T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
-   constexpr int mnq_o =
-      T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
-   constexpr int mndq = std::max(mnd_o + 1, mnq_o + 1);
-   constexpr int tbatch = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, mndq);
-   MFEM_VERIFY(ndof_o <= mnd_o, "Error: H(curl) order larger than supported");
-   MFEM_VERIFY(nquad_o <= mnq_o, "Error: H(div) order larger than supported");
-   int mnq = std::max(ndof_o + 1, nquad_o + 1);
-   auto pa_data = pa.Read();
-   auto x_d = x_.Read();
-   auto y_d = y_.ReadWrite();
-   mfem::forall_2D_batch<mndq * mndq * (mndq - 1) * tbatch>(
-      ne, mnq * mnq * (mnq - 1), 1, tbatch, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MND_O =
-         T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
-      constexpr int MNQ_O =
-         T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
-      constexpr int MNDQ = std::max(MND_O + 1, MNQ_O + 1);
-#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
-      constexpr int nbz = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, MNDQ);
-      int tidz = MFEM_THREAD_ID(z);
-      // Make mnq a local variable since capturing would result in different
-      // captures between host/device versions, and spuriously fails
-      int mnq = std::max(ndof_o + 1, nquad_o + 1);
-#else
-      constexpr int nbz = 1;
-      constexpr int tidz = 0;
-#endif
-      const int NDOF_O = T_NDOF_O ? T_NDOF_O : ndof_o;
-      const int NQUAD_O = T_NQUAD_O ? T_NQUAD_O : nquad_o;
-      const int NDOF_C = NDOF_O + 1;
-      const int NQUAD_C = NQUAD_O + 1;
-      MFEM_SHARED real_t
-      sBG[(MND_O + 1) * MNQ_O + (MND_O + 1) * (MNQ_O + 1) + MND_O * MNQ_O];
-      auto X_ = Reshape(x_d, 3 * NDOF_C * NDOF_C * NDOF_O, ne);
-      auto Y = Reshape(y_d, 3 * NQUAD_C * NQUAD_O * NQUAD_O, ne);
-      auto Gco = Reshape(sBG, NQUAD_O, NDOF_C);
-      auto Bcc = Reshape(sBG + NDOF_C * NQUAD_O, NQUAD_C, NDOF_C);
-      auto Boo =
-         Reshape(sBG + NDOF_C * NQUAD_O + NDOF_C * NQUAD_C, NQUAD_O, NDOF_O);
-      MFEM_SHARED real_t X[3][nbz][MND_O * (MND_O + 1) * (MND_O + 1)];
-      MFEM_SHARED real_t sm0[nbz * 2 * MNDQ * MNDQ * MNDQ];
-      MFEM_SHARED real_t sm1[nbz * 2 * MNDQ * MNDQ * MNDQ];
-
-      // shapes of buffers always use MNDQ to mitigate shared memory bank
-      // conflicts
-      real_t(*DDQ)[nbz][MNDQ][MNDQ][MNDQ] =
-         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
-      real_t(*DQQ)[nbz][MNDQ][MNDQ][MNDQ] =
-         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm1);
-      real_t(*QQQ)[nbz][MNDQ][MNDQ][MNDQ] =
-         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
-      const int offset = NDOF_O * NDOF_C * NDOF_C;
-      const int offsetq = NQUAD_C * NQUAD_O * NQUAD_O;
-      MFEM_FOREACH_THREAD_DIRECT(ix, x, offset)
-      {
-         for (int dim = 0; dim < 3; ++dim)
-         {
-            X[dim][tidz][ix] = X_(ix + dim * offset, e);
-         }
-      }
-      // load basis functions data
-      if (tidz == 0)
-      {
-         auto npts = NDOF_C * NQUAD_O + NDOF_C * NQUAD_C + NDOF_O * NQUAD_O;
-         MFEM_FOREACH_THREAD(ix, x, npts) { sBG[ix] = pa_data[ix]; }
-      }
-      MFEM_SYNC_THREAD;
-
-      // x: Vz Bcc Gco Boo - Vy Bcc Boo Gco
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_C, NDOF_C,
-                                           NDOF_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int dx = 0; dx < NDOF_C; ++dx)
-         {
-            u += X[2][tidz][dx + (dy + dz * NDOF_C) * NDOF_C] * Bcc(qx, dx);
-         }
-         DDQ[0][tidz][dz][dy][qx] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_C, NDOF_O,
-                                           NDOF_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int dx = 0; dx < NDOF_C; ++dx)
-         {
-            u += X[1][tidz][dx + (dy + dz * NDOF_O) * NDOF_C] * Bcc(qx, dx);
-         }
-         DDQ[1][tidz][dz][dy][qx] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_C, NQUAD_O,
-                                           NDOF_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int dy = 0; dy < NDOF_C; ++dy)
-         {
-            u += DDQ[0][tidz][dz][dy][qx] * Gco(qy, dy);
-         }
-         DQQ[0][tidz][dz][qy][qx] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_C, NQUAD_O,
-                                           NDOF_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int dy = 0; dy < NDOF_O; ++dy)
-         {
-            u += DDQ[1][tidz][dz][dy][qx] * Boo(qy, dy);
-         }
-         DQQ[1][tidz][dz][qy][qx] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_C, NQUAD_O,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int dz = 0; dz < NDOF_O; ++dz)
-         {
-            u += DQQ[0][tidz][dz][qy][qx] * Boo(qz, dz);
-         }
-         QQQ[0][tidz][qz][qy][qx] = u;
-      }
-      MFEM_SYNC_THREAD;
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_C, NQUAD_O,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int dz = 0; dz < NDOF_C; ++dz)
-         {
-            u += DQQ[1][tidz][dz][qy][qx] * Gco(qz, dz);
-         }
-         Y(qx + (qy + qz * NQUAD_O) * NQUAD_C, e) =
-            QQQ[0][tidz][qz][qy][qx] - u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // y: Vx Boo Bcc Gco - Vz Gco Bcc Boo
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_C,
-                                           NDOF_C, mnq - 1, mnq, mnq)
-      {
-         real_t u = 0;
-         for (int dx = 0; dx < NDOF_O; ++dx)
-         {
-            u += X[0][tidz][dx + (dy + dz * NDOF_C) * NDOF_O] * Boo(qx, dx);
-         }
-         DDQ[0][tidz][dz][dy][qx] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_C,
-                                           NDOF_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int dx = 0; dx < NDOF_C; ++dx)
-         {
-            u += X[2][tidz][dx + (dy + dz * NDOF_C) * NDOF_C] * Gco(qx, dx);
-         }
-         DDQ[1][tidz][dz][dy][qx] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_C,
-                                           NDOF_C, mnq - 1, mnq, mnq)
-      {
-         real_t u = 0;
-         for (int dy = 0; dy < NDOF_C; ++dy)
-         {
-            u += DDQ[0][tidz][dz][dy][qx] * Bcc(qy, dy);
-         }
-         DQQ[0][tidz][dz][qy][qx] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_C,
-                                           NDOF_O, mnq - 1, mnq, mnq)
-      {
-         real_t u = 0;
-         for (int dy = 0; dy < NDOF_C; ++dy)
-         {
-            u += DDQ[1][tidz][dz][dy][qx] * Bcc(qy, dy);
-         }
-         DQQ[1][tidz][dz][qy][qx] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_C,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int dz = 0; dz < NDOF_C; ++dz)
-         {
-            u += DQQ[0][tidz][dz][qy][qx] * Gco(qz, dz);
-         }
-         QQQ[0][tidz][qz][qy][qx] = u;
-      }
-      MFEM_SYNC_THREAD;
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_C,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int dz = 0; dz < NDOF_O; ++dz)
-         {
-            u += DQQ[1][tidz][dz][qy][qx] * Boo(qz, dz);
-         }
-         Y(qx + (qy + qz * NQUAD_C) * NQUAD_O + offsetq, e) =
-            QQQ[0][tidz][qz][qy][qx] - u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // z: Vy Gco Boo Bcc - Vx Boo Gco Bcc
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_O,
-                                           NDOF_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int dx = 0; dx < NDOF_C; ++dx)
-         {
-            u += X[1][tidz][dx + (dy + dz * NDOF_O) * NDOF_C] * Gco(qx, dx);
-         }
-         DDQ[0][tidz][dz][dy][qx] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_C,
-                                           NDOF_C, mnq - 1, mnq, mnq)
-      {
-         real_t u = 0;
-         for (int dx = 0; dx < NDOF_O; ++dx)
-         {
-            u += X[0][tidz][dx + (dy + dz * NDOF_C) * NDOF_O] * Boo(qx, dx);
-         }
-         DDQ[1][tidz][dz][dy][qx] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_O,
-                                           NDOF_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int dy = 0; dy < NDOF_O; ++dy)
-         {
-            u += DDQ[0][tidz][dz][dy][qx] * Boo(qy, dy);
-         }
-         DQQ[0][tidz][dz][qy][qx] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_O,
-                                           NDOF_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int dy = 0; dy < NDOF_C; ++dy)
-         {
-            u += DDQ[1][tidz][dz][dy][qx] * Gco(qy, dy);
-         }
-         DQQ[1][tidz][dz][qy][qx] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_O,
-                                           NQUAD_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int dz = 0; dz < NDOF_C; ++dz)
-         {
-            u += DQQ[0][tidz][dz][qy][qx] * Bcc(qz, dz);
-         }
-         QQQ[0][tidz][qz][qy][qx] = u;
-      }
-      MFEM_SYNC_THREAD;
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_O,
-                                           NQUAD_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int dz = 0; dz < NDOF_C; ++dz)
-         {
-            u += DQQ[1][tidz][dz][qy][qx] * Bcc(qz, dz);
-         }
-         Y(qx + (qy + qz * NQUAD_O) * NQUAD_O + 2 * offsetq, e) =
-            QQQ[0][tidz][qz][qy][qx] - u;
-      }
-      MFEM_SYNC_THREAD;
-   });
-}
-
-template <int T_NDOF_O, int T_NQUAD_O>
-void CurlInterpolatorTApply3DSmem(const int ne, const int ndof_o,
-                                  const int nquad_o, const Vector &pa,
-                                  const Vector &x_, Vector &y_)
-{
-   constexpr int mnd_o = T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
-   constexpr int mnq_o =
-      T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
-   constexpr int mndq = std::max(mnd_o + 1, mnq_o + 1);
-   constexpr int tbatch = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, mndq);
-   MFEM_VERIFY(ndof_o <= mnd_o, "Error: H(curl) order larger than supported");
-   MFEM_VERIFY(nquad_o <= mnq_o, "Error: H(div) order larger than supported");
-   int mnq = std::max(ndof_o + 1, nquad_o + 1);
-   auto pa_data = pa.Read();
-   auto x_d = x_.Read();
-   auto y_d = y_.ReadWrite();
-   mfem::forall_2D_batch<mndq * mndq * (mndq - 1) * tbatch>(
-      ne, mnq * mnq * (mnq - 1), 1, tbatch, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MND_O =
-         T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
-      constexpr int MNQ_O =
-         T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
-      constexpr int MNDQ = std::max(MND_O + 1, MNQ_O + 1);
-#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
-      constexpr int nbz = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, MNDQ);
-      int tidz = MFEM_THREAD_ID(z);
-      // Make mnq a local variable since capturing would result in different
-      // captures between host/device versions, and spuriously fails
-      int mnq = std::max(ndof_o + 1, nquad_o + 1);
-#else
-      constexpr int nbz = 1;
-      constexpr int tidz = 0;
-#endif
-      const int NDOF_O = T_NDOF_O ? T_NDOF_O : ndof_o;
-      const int NQUAD_O = T_NQUAD_O ? T_NQUAD_O : nquad_o;
-      const int NDOF_C = NDOF_O + 1;
-      const int NQUAD_C = NQUAD_O + 1;
-      MFEM_SHARED real_t
-      sBG[(MND_O + 1) * MNQ_O + (MND_O + 1) * (MNQ_O + 1) + MND_O * MNQ_O];
-      auto X_ = Reshape(x_d, 3 * NQUAD_C * NQUAD_O * NQUAD_O, ne);
-      auto Y = Reshape(y_d, 3 * NDOF_C * NDOF_C * NDOF_O, ne);
-      auto Gco = Reshape(sBG, NQUAD_O, NDOF_C);
-      auto Bcc = Reshape(sBG + NDOF_C * NQUAD_O, NQUAD_C, NDOF_C);
-      auto Boo =
-         Reshape(sBG + NDOF_C * NQUAD_O + NDOF_C * NQUAD_C, NQUAD_O, NDOF_O);
-      MFEM_SHARED real_t X[3][nbz][MNQ_O * MNQ_O * (MNQ_O + 1)];
-      MFEM_SHARED real_t sm0[nbz * 2 * MNDQ * MNDQ * MNDQ];
-      MFEM_SHARED real_t sm1[nbz * 2 * MNDQ * MNDQ * MNDQ];
-
-      // shapes of buffers always use MNDQ to mitigate shared memory bank
-      // conflicts
-      real_t(*QQD)[nbz][MNDQ][MNDQ][MNDQ] =
-         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
-      real_t(*QDD)[nbz][MNDQ][MNDQ][MNDQ] =
-         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm1);
-      real_t(*DDD)[nbz][MNDQ][MNDQ][MNDQ] =
-         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
-      const int offset = NDOF_O * NDOF_C * NDOF_C;
-      const int offsetq = NQUAD_C * NQUAD_O * NQUAD_O;
-      MFEM_FOREACH_THREAD_DIRECT(ix, x, offsetq)
-      {
-         for (int dim = 0; dim < 3; ++dim)
-         {
-            X[dim][tidz][ix] = X_(ix + dim * offsetq, e);
-         }
-      }
-      // load basis functions data
-      if (tidz == 0)
-      {
-         auto npts = NDOF_C * NQUAD_O + NDOF_C * NQUAD_C + NDOF_O * NQUAD_O;
-         MFEM_FOREACH_THREAD(ix, x, npts) { sBG[ix] = pa_data[ix]; }
-      }
-      MFEM_SYNC_THREAD;
-
-      // x: Vy Boo Bcc Gco - Vz Boo Gco Bcc
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_O,
-                                           NQUAD_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int qz = 0; qz < NQUAD_O; ++qz)
-         {
-            u += X[1][tidz][qx + (qy + qz * NQUAD_C) * NQUAD_O] * Gco(qz, dz);
-         }
-         QQD[0][tidz][qy][qx][dz] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_O,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qz = 0; qz < NQUAD_C; ++qz)
-         {
-            u += X[2][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_O] * Bcc(qz, dz);
-         }
-         QQD[1][tidz][qy][qx][dz] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_C,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qy = 0; qy < NQUAD_C; ++qy)
-         {
-            u += QQD[0][tidz][qy][qx][dz] * Bcc(qy, dy);
-         }
-         QDD[0][tidz][qx][dz][dy] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_C,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qy = 0; qy < NQUAD_O; ++qy)
-         {
-            u += QQD[1][tidz][qy][qx][dz] * Gco(qy, dy);
-         }
-         QDD[1][tidz][qx][dz][dy] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_O, NDOF_C,
-                                           NDOF_C, mnq - 1, mnq, mnq)
-      {
-         real_t u = 0;
-         for (int qx = 0; qx < NQUAD_O; ++qx)
-         {
-            u += QDD[0][tidz][qx][dz][dy] * Boo(qx, dx);
-         }
-         DDD[0][tidz][dz][dy][dx] = u;
-      }
-      MFEM_SYNC_THREAD;
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_O, NDOF_C,
-                                           NDOF_C, mnq - 1, mnq, mnq)
-      {
-         real_t u = 0;
-         for (int qx = 0; qx < NQUAD_O; ++qx)
-         {
-            u += QDD[1][tidz][qx][dz][dy] * Boo(qx, dx);
-         }
-         Y(dx + (dy + dz * NDOF_C) * NDOF_O, e) =
-            DDD[0][tidz][dz][dy][dx] - u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // y: Vz Gco Boo Bcc - Vx Bcc Boo Gco
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_O,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qz = 0; qz < NQUAD_C; ++qz)
-         {
-            u += X[2][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_O] * Bcc(qz, dz);
-         }
-         QQD[0][tidz][qy][qx][dz] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_C,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qz = 0; qz < NQUAD_O; ++qz)
-         {
-            u += X[0][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_C] * Gco(qz, dz);
-         }
-         QQD[1][tidz][qy][qx][dz] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_O, NDOF_C,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qy = 0; qy < NQUAD_O; ++qy)
-         {
-            u += QQD[0][tidz][qy][qx][dz] * Boo(qy, dy);
-         }
-         QDD[0][tidz][qx][dz][dy] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_O, NDOF_C,
-                                           NQUAD_C, mnq - 1, mnq, mnq)
-      {
-         real_t u = 0;
-         for (int qy = 0; qy < NQUAD_O; ++qy)
-         {
-            u += QQD[1][tidz][qy][qx][dz] * Boo(qy, dy);
-         }
-         QDD[1][tidz][qx][dz][dy] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_O,
-                                           NDOF_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int qx = 0; qx < NQUAD_O; ++qx)
-         {
-            u += QDD[0][tidz][qx][dz][dy] * Gco(qx, dx);
-         }
-         DDD[0][tidz][dz][dy][dx] = u;
-      }
-      MFEM_SYNC_THREAD;
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_O,
-                                           NDOF_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int qx = 0; qx < NQUAD_C; ++qx)
-         {
-            u += QDD[1][tidz][qx][dz][dy] * Bcc(qx, dx);
-         }
-         Y(dx + (dy + dz * NDOF_O) * NDOF_C + offset, e) =
-            DDD[0][tidz][dz][dy][dx] - u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // z: Vx Bcc Gco Boo - Vy Gco Bcc Boo
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_O, NQUAD_C,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qz = 0; qz < NQUAD_O; ++qz)
-         {
-            u += X[0][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_C] * Boo(qz, dz);
-         }
-         QQD[0][tidz][qy][qx][dz] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_O, NQUAD_O,
-                                           NQUAD_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int qz = 0; qz < NQUAD_O; ++qz)
-         {
-            u += X[1][tidz][qx + (qy + qz * NQUAD_C) * NQUAD_O] * Boo(qz, dz);
-         }
-         QQD[1][tidz][qy][qx][dz] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_O,
-                                           NQUAD_C, mnq, mnq - 1, mnq)
-      {
-         real_t u = 0;
-         for (int qy = 0; qy < NQUAD_O; ++qy)
-         {
-            u += QQD[0][tidz][qy][qx][dz] * Gco(qy, dy);
-         }
-         QDD[0][tidz][qx][dz][dy] = u;
-      }
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_O,
-                                           NQUAD_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qy = 0; qy < NQUAD_C; ++qy)
-         {
-            u += QQD[1][tidz][qy][qx][dz] * Bcc(qy, dy);
-         }
-         QDD[1][tidz][qx][dz][dy] = u;
-      }
-      MFEM_SYNC_THREAD;
-
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_C,
-                                           NDOF_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qx = 0; qx < NQUAD_C; ++qx)
-         {
-            u += QDD[0][tidz][qx][dz][dy] * Bcc(qx, dx);
-         }
-         DDD[0][tidz][dz][dy][dx] = u;
-      }
-      MFEM_SYNC_THREAD;
-      // threads assigned to mitigate bank conflicts
-      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_C,
-                                           NDOF_O, mnq, mnq, mnq - 1)
-      {
-         real_t u = 0;
-         for (int qx = 0; qx < NQUAD_O; ++qx)
-         {
-            u += QDD[1][tidz][qx][dz][dy] * Gco(qx, dx);
-         }
-         Y(dx + (dy + dz * NDOF_C) * NDOF_C + 2 * offset, e) =
-            DDD[0][tidz][dz][dy][dx] - u;
-      }
-      MFEM_SYNC_THREAD;
-   });
-}
-
 } // namespace internal
 
-template <int DIM, int NDOF_O, int NQUAD_O>
-CurlInterpolator::ApplyKernelType
-CurlInterpolator::ApplyPAKernels::Kernel()
-{
-   if constexpr (DIM == 3)
-   {
-      return internal::CurlInterpolatorApply3DSmem<NDOF_O, NQUAD_O>;
-   }
-   MFEM_ABORT("Bad dimension!");
-}
-
-template <int DIM, int NDOF_O, int NQUAD_O>
-CurlInterpolator::ApplyKernelType
-CurlInterpolator::ApplyTPAKernels::Kernel()
-{
-   if constexpr (DIM == 3)
-   {
-      return internal::CurlInterpolatorTApply3DSmem<NDOF_O, NQUAD_O>;
-   }
-   MFEM_ABORT("Bad dimension!");
-}
-
 } // namespace mfem
 
 /// \endcond DO_NOT_DOCUMENT

fem/integ/bilininteg_hdiv_kernels.cpp

@@ -294,14 +294,61 @@ void PAHdivMassAssembleDiagonal3D(const int D1D,
    }); // end of element loop
 }
 
-void PAHdivMassApply2D(const int NE, const bool symmetric, const bool,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int TestD1D, const int Q1D)
+void PAHdivMassApply(const int dim,
+                     const int D1D,
+                     const int Q1D,
+                     const int NE,
+                     const bool symmetric,
+                     const Array<real_t> &Bo,
+                     const Array<real_t> &Bc,
+                     const Array<real_t> &Bot,
+                     const Array<real_t> &Bct,
+                     const Vector &op,
+                     const Vector &x,
+                     Vector &y)
+{
+   const int id = (D1D << 4) | Q1D;
+
+   if (dim == 2)
+   {
+      switch (id)
+      {
+         case 0x22: return SmemPAHdivMassApply2D<2,2>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x33: return SmemPAHdivMassApply2D<3,3>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x44: return SmemPAHdivMassApply2D<4,4>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x55: return SmemPAHdivMassApply2D<5,5>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         default: // fallback
+            return PAHdivMassApply2D(D1D,Q1D,NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+      }
+   }
+   else if (dim == 3)
+   {
+      switch (id)
+      {
+         case 0x23: return SmemPAHdivMassApply3D<2,3>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x34: return SmemPAHdivMassApply3D<3,4>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x45: return SmemPAHdivMassApply3D<4,5>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x56: return SmemPAHdivMassApply3D<5,6>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x67: return SmemPAHdivMassApply3D<6,7>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x78: return SmemPAHdivMassApply3D<7,8>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         default: // fallback
+            return PAHdivMassApply3D(D1D,Q1D,NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+      }
+   }
+}
+
+void PAHdivMassApply2D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const bool symmetric,
+                       const Array<real_t> &Bo_,
+                       const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_,
+                       const Array<real_t> &Bct_,
+                       const Vector &op_,
+                       const Vector &x_,
+                       Vector &y_)
 {
-   MFEM_VERIFY(D1D == TestD1D,
-               "Trial and test spaces must have same number of dofs");
    auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
    auto Bc = Reshape(Bc_.Read(), Q1D, D1D);
    auto Bot = Reshape(Bot_.Read(), D1D-1, Q1D);
@@ -421,14 +468,18 @@ void PAHdivMassApply2D(const int NE, const bool symmetric, const bool,
    }); // end of element loop
 }
 
-void PAHdivMassApply3D(const int NE, const bool symmetric, const bool,
-                       const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                       const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                       const Vector &op_, const Vector &x_, Vector &y_,
-                       const int D1D, const int TestD1D, const int Q1D)
+void PAHdivMassApply3D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const bool symmetric,
+                       const Array<real_t> &Bo_,
+                       const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_,
+                       const Array<real_t> &Bct_,
+                       const Vector &op_,
+                       const Vector &x_,
+                       Vector &y_)
 {
-   MFEM_VERIFY(D1D == TestD1D,
-               "Trial and test spaces must have same number of dofs");
    MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HDIV_MAX_D1D,
                "Error: D1D > HDIV_MAX_D1D");
    MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HDIV_MAX_Q1D,

fem/integ/bilininteg_hdiv_kernels.hpp

@@ -66,29 +66,58 @@ void PAHdivMassAssembleDiagonal3D(const int D1D,
                                   const Vector &op_,
                                   Vector &diag_);
 
+void PAHdivMassApply(const int dim,
+                     const int D1D,
+                     const int Q1D,
+                     const int NE,
+                     const bool symmetric,
+                     const Array<real_t> &Bo,
+                     const Array<real_t> &Bc,
+                     const Array<real_t> &Bot,
+                     const Array<real_t> &Bct,
+                     const Vector &op,
+                     const Vector &x,
+                     Vector &y);
+
 // PA H(div) Mass Apply 2D kernel
-void PAHdivMassApply2D(const int NE, const bool symmetric,
-                       const bool scalar_coeff, const Array<real_t> &Bo_,
-                       const Array<real_t> &Bc_, const Array<real_t> &Bot_,
-                       const Array<real_t> &Bct_, const Vector &op_,
-                       const Vector &x_, Vector &y_, const int D1D,
-                       const int TestD1D, const int Q1D);
+void PAHdivMassApply2D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const bool symmetric,
+                       const Array<real_t> &Bo_,
+                       const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_,
+                       const Array<real_t> &Bct_,
+                       const Vector &op_,
+                       const Vector &x_,
+                       Vector &y_);
 
 // PA H(div) Mass Apply 3D kernel
-void PAHdivMassApply3D(const int NE, const bool symmetric,
-                       const bool scalar_coeff, const Array<real_t> &Bo_,
-                       const Array<real_t> &Bc_, const Array<real_t> &Bot_,
-                       const Array<real_t> &Bct_, const Vector &op_,
-                       const Vector &x_, Vector &y_, const int D1D,
-                       const int TestD1D, const int Q1D);
+void PAHdivMassApply3D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const bool symmetric,
+                       const Array<real_t> &Bo_,
+                       const Array<real_t> &Bc_,
+                       const Array<real_t> &Bot_,
+                       const Array<real_t> &Bct_,
+                       const Vector &op_,
+                       const Vector &x_,
+                       Vector &y_);
 
 // Shared memory PA H(div) Mass Apply 2D kernel
-template <int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAHdivMassApply2D(
-   const int NE, const bool symmetric, const bool, const Array<real_t> &Bo_,
-   const Array<real_t> &Bc_, const Array<real_t> &Bot_,
-   const Array<real_t> &Bct_, const Vector &op_, const Vector &x_, Vector &y_,
-   const int d1d = 0, const int = 0, const int q1d = 0)
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAHdivMassApply2D(const int NE,
+                                  const bool symmetric,
+                                  const Array<real_t> &Bo_,
+                                  const Array<real_t> &Bc_,
+                                  const Array<real_t> &Bot_,
+                                  const Array<real_t> &Bct_,
+                                  const Vector &op_,
+                                  const Vector &x_,
+                                  Vector &y_,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
 {
    MFEM_CONTRACT_VAR(Bot_);
    MFEM_CONTRACT_VAR(Bct_);
@@ -251,13 +280,18 @@ inline void SmemPAHdivMassApply2D(
 }
 
 // Shared memory PA H(div) Mass Apply 3D kernel
-template <int T_D1D = 0, int T_Q1D = 0>
-inline void
-SmemPAHdivMassApply3D(const int NE, const bool symmetric, const bool,
-                      const Array<real_t> &Bo_, const Array<real_t> &Bc_,
-                      const Array<real_t> &Bot_, const Array<real_t> &Bct_,
-                      const Vector &op_, const Vector &x_, Vector &y_,
-                      const int d1d = 0, const int = 0, const int q1d = 0)
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAHdivMassApply3D(const int NE,
+                                  const bool symmetric,
+                                  const Array<real_t> &Bo_,
+                                  const Array<real_t> &Bc_,
+                                  const Array<real_t> &Bot_,
+                                  const Array<real_t> &Bct_,
+                                  const Vector &op_,
+                                  const Vector &x_,
+                                  Vector &y_,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
 {
    MFEM_CONTRACT_VAR(Bot_);
    MFEM_CONTRACT_VAR(Bct_);

fem/integ/bilininteg_interp_pa.cpp

@@ -14,8 +14,6 @@
 #include "../gridfunc.hpp"
 #include "../qfunction.hpp"
 
-#include "bilininteg_hcurlhdiv_kernels.hpp"
-
 namespace mfem
 {
 
@@ -1952,138 +1950,4 @@ void IdentityInterpolator::AddMultTransposePA(const Vector &x, Vector &y) const
    }
 }
 
-void CurlInterpolator::AssemblePA(const FiniteElementSpace &dom_fes,
-                                  const FiniteElementSpace &ran_fes)
-{
-   // TODO: 1D and 2D meshes
-   Mesh *mesh = dom_fes.GetMesh();
-   const VectorTensorFiniteElement *dom_el =
-      dynamic_cast<const VectorTensorFiniteElement *>(dom_fes.GetTypicalFE());
-   const VectorTensorFiniteElement *ran_el =
-      dynamic_cast<const VectorTensorFiniteElement *>(ran_fes.GetTypicalFE());
-   MFEM_VERIFY(dom_el != NULL, "Only VectorTensorFiniteElement is supported!");
-   MFEM_VERIFY(ran_el != NULL, "Only VectorTensorFiniteElement is supported!");
-   // only supports H(curl) -> H(div) because of discontinuity requirements
-   MFEM_VERIFY(dom_el->GetDerivType() == FiniteElement::CURL,
-               "Domain space must be H(curl)");
-   MFEM_VERIFY(ran_el->GetDerivType() == FiniteElement::DIV,
-               "Range space must be H(div)");
-
-   const int dims = dom_el->GetDim();
-   MFEM_VERIFY(dims == 3, "");
-   dim = mesh->Dimension();
-
-   ne = dom_fes.GetNE();
-   ndof_o = dom_el->GetOrder();
-   int ndof_c = ndof_o + 1;
-   nquad_o = ran_el->GetOrder();
-   int nquad_c = nquad_o + 1;
-
-   // extract the tensor product range dof locations
-   std::vector<real_t> qc(nquad_c);
-   std::vector<real_t> qo(nquad_o);
-   {
-      const IntegrationRule &ran_nodes = ran_el->GetNodes();
-      const Array<int> &quad_map = ran_el->GetDofMap();
-      for (int i = 0; i < nquad_c; ++i)
-      {
-         int idx = UnsignIndex(quad_map[i]);
-         qc[i] = ran_nodes.IntPoint(idx).x;
-      }
-      int offset = ndof_c * ndof_o * ndof_o;
-      for (int i = 0; i < nquad_o; ++i)
-      {
-         int idx = UnsignIndex(quad_map[i + offset]);
-         qo[i] = ran_nodes.IntPoint(idx).x;
-      }
-   }
-
-   // evaluate closed/open 1D basis (and their derivatives) at closed and
-   // open quads
-   // storage order: GCO, BCC, BOO
-   pa_data.SetSize(ndof_c * nquad_o + ndof_c * nquad_c + ndof_o * nquad_o);
-   auto ptr = pa_data.HostWrite();
-   auto &cbasis1d = dom_el->GetBasis1D();
-   auto &obasis1d = dom_el->GetOpenBasis1D();
-   Vector b, g;
-   b.SetSize(ndof_c);
-   g.SetSize(ndof_c);
-   for (int j = 0; j < nquad_o; ++j)
-   {
-      cbasis1d.Eval(qo[j], b, g);
-      for (int i = 0; i < ndof_c; ++i)
-      {
-         ptr[j + i * nquad_o] = g[i];
-      }
-   }
-   ptr += nquad_o * ndof_c;
-
-   for (int j = 0; j < nquad_c; ++j)
-   {
-      cbasis1d.Eval(qc[j], b);
-      for (int i = 0; i < ndof_c; ++i)
-      {
-         ptr[j + i * nquad_c] = b[i];
-      }
-   }
-   ptr += ndof_c * nquad_c;
-
-   b.SetSize(ndof_o);
-   for (int j = 0; j < nquad_o; ++j)
-   {
-      obasis1d.Eval(qo[j], b);
-      for (int i = 0; i < ndof_o; ++i)
-      {
-         ptr[j + i * nquad_o] = b[i];
-      }
-   }
-}
-
-CurlInterpolator::Kernels::Kernels()
-{
-   CurlInterpolator::AddSpecialization<3, 1, 1>();
-   CurlInterpolator::AddSpecialization<3, 2, 2>();
-   CurlInterpolator::AddSpecialization<3, 3, 3>();
-   CurlInterpolator::AddSpecialization<3, 4, 4>();
-   CurlInterpolator::AddSpecialization<3, 5, 5>();
-}
-
-CurlInterpolator::CurlInterpolator() { static Kernels kernels{}; }
-
-void CurlInterpolator::AddMultPA(const Vector &x, Vector &y) const
-{
-   ApplyPAKernels::Run(dim, ndof_o, nquad_o, ne, ndof_o, nquad_o, pa_data, x,
-                       y);
-}
-
-void CurlInterpolator::AddMultTransposePA(const Vector &x, Vector &y) const
-{
-   ApplyTPAKernels::Run(dim, ndof_o, nquad_o, ne, ndof_o, nquad_o, pa_data, x,
-                        y);
-}
-
-/// \cond DO_NOT_DOCUMENT
-
-CurlInterpolator::ApplyKernelType
-CurlInterpolator::ApplyPAKernels::Fallback(int DIM, int, int)
-{
-   if (DIM == 3)
-   {
-      return internal::CurlInterpolatorApply3DSmem<0, 0>;
-   }
-   MFEM_ABORT("Bad dimension!");
-}
-
-CurlInterpolator::ApplyKernelType
-CurlInterpolator::ApplyTPAKernels::Fallback(int DIM, int, int)
-{
-   if (DIM == 3)
-   {
-      return internal::CurlInterpolatorTApply3DSmem<0, 0>;
-   }
-   MFEM_ABORT("Bad dimension!");
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
 } // namespace mfem

fem/integ/bilininteg_mixedcurl_pa.cpp

@@ -18,779 +18,6 @@
 namespace mfem
 {
 
-namespace
-{
-
-void PAHcurlDotSetup2D(const int q1d,
-                       const int ne,
-                       const bool test_map_integral,
-                       const Array<real_t> &w,
-                       const Vector &jacobians,
-                       const Vector &coeff,
-                       Vector &op)
-{
-   auto W = Reshape(w.Read(), q1d, q1d);
-   auto J = Reshape(jacobians.Read(), q1d, q1d, 2, 2, ne);
-   auto C = Reshape(coeff.Read(), 2, q1d, q1d, ne);
-   auto O = Reshape(op.Write(), 2, q1d, q1d, ne);
-
-   mfem::forall_2D(ne, q1d, q1d, [=] MFEM_HOST_DEVICE (int e)
-   {
-      MFEM_FOREACH_THREAD(qy, y, q1d)
-      {
-         MFEM_FOREACH_THREAD(qx, x, q1d)
-         {
-            const real_t J11 = J(qx, qy, 0, 0, e);
-            const real_t J12 = J(qx, qy, 1, 0, e);
-            const real_t J21 = J(qx, qy, 0, 1, e);
-            const real_t J22 = J(qx, qy, 1, 1, e);
-            const real_t detJ = (J11 * J22) - (J21 * J12);
-            const real_t scale = W(qx, qy) * (test_map_integral ? 1.0 / detJ : 1.0);
-            const real_t Vx = C(0, qx, qy, e);
-            const real_t Vy = C(1, qx, qy, e);
-
-            O(0, qx, qy, e) = scale * ( J22 * Vx - J12 * Vy);
-            O(1, qx, qy, e) = scale * (-J21 * Vx + J11 * Vy);
-         }
-      }
-   });
-}
-
-void PAHcurlDotSetup3D(const int q1d,
-                       const int ne,
-                       const bool test_map_integral,
-                       const Array<real_t> &w,
-                       const Vector &jacobians,
-                       const Vector &coeff,
-                       Vector &op)
-{
-   auto W = Reshape(w.Read(), q1d, q1d, q1d);
-   auto J = Reshape(jacobians.Read(), q1d, q1d, q1d, 3, 3, ne);
-   auto C = Reshape(coeff.Read(), 3, q1d, q1d, q1d, ne);
-   auto O = Reshape(op.Write(), 3, q1d, q1d, q1d, ne);
-
-   mfem::forall_3D(ne, q1d, q1d, q1d, [=] MFEM_HOST_DEVICE (int e)
-   {
-      MFEM_FOREACH_THREAD(qz, z, q1d)
-      {
-         MFEM_FOREACH_THREAD(qy, y, q1d)
-         {
-            MFEM_FOREACH_THREAD(qx, x, q1d)
-            {
-               const real_t J11 = J(qx, qy, qz, 0, 0, e);
-               const real_t J12 = J(qx, qy, qz, 0, 1, e);
-               const real_t J13 = J(qx, qy, qz, 0, 2, e);
-               const real_t J21 = J(qx, qy, qz, 1, 0, e);
-               const real_t J22 = J(qx, qy, qz, 1, 1, e);
-               const real_t J23 = J(qx, qy, qz, 1, 2, e);
-               const real_t J31 = J(qx, qy, qz, 2, 0, e);
-               const real_t J32 = J(qx, qy, qz, 2, 1, e);
-               const real_t J33 = J(qx, qy, qz, 2, 2, e);
-               const real_t detJ = J11 * (J22 * J33 - J32 * J23)
-                                   - J21 * (J12 * J33 - J32 * J13)
-                                   + J31 * (J12 * J23 - J22 * J13);
-               const real_t scale = W(qx, qy, qz) *
-                                    (test_map_integral ? 1.0 / detJ : 1.0);
-               const real_t Vx = C(0, qx, qy, qz, e);
-               const real_t Vy = C(1, qx, qy, qz, e);
-               const real_t Vz = C(2, qx, qy, qz, e);
-
-               O(0, qx, qy, qz, e) = scale *
-                                     ((J22 * J33 - J23 * J32) * Vx +
-                                      (J13 * J32 - J12 * J33) * Vy +
-                                      (J12 * J23 - J13 * J22) * Vz);
-               O(1, qx, qy, qz, e) = scale *
-                                     ((J23 * J31 - J21 * J33) * Vx +
-                                      (J11 * J33 - J13 * J31) * Vy +
-                                      (J13 * J21 - J11 * J23) * Vz);
-               O(2, qx, qy, qz, e) = scale *
-                                     ((J21 * J32 - J22 * J31) * Vx +
-                                      (J12 * J31 - J11 * J32) * Vy +
-                                      (J11 * J22 - J12 * J21) * Vz);
-            }
-         }
-      }
-   });
-}
-
-void PAHcurlDotApply2D(const int d1d,
-                       const int d1d_test,
-                       const int q1d,
-                       const int ne,
-                       const Array<real_t> &bo,
-                       const Array<real_t> &bc,
-                       const Array<real_t> &bt,
-                       const Vector &pa_data,
-                       const Vector &x,
-                       Vector &y)
-{
-   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D, "");
-   MFEM_VERIFY(d1d_test <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D, "");
-
-   auto Bo = Reshape(bo.Read(), q1d, d1d - 1);
-   auto Bc = Reshape(bc.Read(), q1d, d1d);
-   auto Bt = Reshape(bt.Read(), d1d_test, q1d);
-   auto O = Reshape(pa_data.Read(), 2, q1d, q1d, ne);
-   auto X = Reshape(x.Read(), 2 * (d1d - 1) * d1d, ne);
-   auto Y = Reshape(y.ReadWrite(), d1d_test, d1d_test, ne);
-
-   mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int MAX_D1D = DofQuadLimits::MAX_D1D;
-      constexpr int MAX_Q1D = DofQuadLimits::HCURL_MAX_Q1D;
-
-      real_t u0[MAX_Q1D][MAX_Q1D];
-      real_t u1[MAX_Q1D][MAX_Q1D];
-
-      for (int qy = 0; qy < q1d; ++qy)
-      {
-         for (int qx = 0; qx < q1d; ++qx)
-         {
-            u0[qy][qx] = 0.0;
-            u1[qy][qx] = 0.0;
-         }
-      }
-
-      int osc = 0;
-      for (int dy = 0; dy < d1d; ++dy)
-      {
-         real_t mass_x[MAX_Q1D];
-         for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] = 0.0; }
-         for (int dx = 0; dx < d1d - 1; ++dx)
-         {
-            const real_t t = X(dx + (dy * (d1d - 1)) + osc, e);
-            for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] += t * Bo(qx, dx); }
-         }
-         for (int qy = 0; qy < q1d; ++qy)
-         {
-            const real_t wy = Bc(qy, dy);
-            for (int qx = 0; qx < q1d; ++qx) { u0[qy][qx] += mass_x[qx] * wy; }
-         }
-      }
-
-      osc += (d1d - 1) * d1d;
-      for (int dy = 0; dy < d1d - 1; ++dy)
-      {
-         real_t mass_x[MAX_Q1D];
-         for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] = 0.0; }
-         for (int dx = 0; dx < d1d; ++dx)
-         {
-            const real_t t = X(dx + (dy * d1d) + osc, e);
-            for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] += t * Bc(qx, dx); }
-         }
-         for (int qy = 0; qy < q1d; ++qy)
-         {
-            const real_t wy = Bo(qy, dy);
-            for (int qx = 0; qx < q1d; ++qx) { u1[qy][qx] += mass_x[qx] * wy; }
-         }
-      }
-
-      for (int qy = 0; qy < q1d; ++qy)
-      {
-         real_t sol_x[MAX_D1D];
-         for (int dx = 0; dx < d1d_test; ++dx) { sol_x[dx] = 0.0; }
-         for (int qx = 0; qx < q1d; ++qx)
-         {
-            const real_t s = O(0, qx, qy, e) * u0[qy][qx]
-                             + O(1, qx, qy, e) * u1[qy][qx];
-            for (int dx = 0; dx < d1d_test; ++dx)
-            {
-               sol_x[dx] += s * Bt(dx, qx);
-            }
-         }
-         for (int dy = 0; dy < d1d_test; ++dy)
-         {
-            const real_t wy = Bt(dy, qy);
-            for (int dx = 0; dx < d1d_test; ++dx)
-            {
-               Y(dx, dy, e) += sol_x[dx] * wy;
-            }
-         }
-      }
-   });
-}
-
-void PAHcurlDotApplyTranspose2D(const int d1d,
-                                const int d1d_test,
-                                const int q1d,
-                                const int ne,
-                                const Array<real_t> &bo,
-                                const Array<real_t> &bc,
-                                const Array<real_t> &b,
-                                const Vector &pa_data,
-                                const Vector &x,
-                                Vector &y)
-{
-   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D, "");
-   MFEM_VERIFY(d1d_test <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D, "");
-
-   auto Bo = Reshape(bo.Read(), q1d, d1d - 1);
-   auto Bc = Reshape(bc.Read(), q1d, d1d);
-   auto B = Reshape(b.Read(), q1d, d1d_test);
-   auto O = Reshape(pa_data.Read(), 2, q1d, q1d, ne);
-   auto X = Reshape(x.Read(), d1d_test, d1d_test, ne);
-   auto Y = Reshape(y.ReadWrite(), 2 * (d1d - 1) * d1d, ne);
-
-   mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int MAX_D1D = DofQuadLimits::MAX_D1D;
-      constexpr int MAX_Q1D = DofQuadLimits::HCURL_MAX_Q1D;
-
-      real_t mass[MAX_Q1D][MAX_Q1D];
-      for (int qy = 0; qy < q1d; ++qy)
-      {
-         for (int qx = 0; qx < q1d; ++qx)
-         {
-            mass[qy][qx] = 0.0;
-         }
-      }
-
-      for (int dy = 0; dy < d1d_test; ++dy)
-      {
-         real_t sol_x[MAX_Q1D];
-         for (int qx = 0; qx < q1d; ++qx) { sol_x[qx] = 0.0; }
-         for (int dx = 0; dx < d1d_test; ++dx)
-         {
-            const real_t t = X(dx, dy, e);
-            for (int qx = 0; qx < q1d; ++qx) { sol_x[qx] += t * B(qx, dx); }
-         }
-         for (int qy = 0; qy < q1d; ++qy)
-         {
-            const real_t wy = B(qy, dy);
-            for (int qx = 0; qx < q1d; ++qx) { mass[qy][qx] += sol_x[qx] * wy; }
-         }
-      }
-
-      int osc = 0;
-      for (int qy = 0; qy < q1d; ++qy)
-      {
-         real_t mass_x[MAX_D1D];
-         for (int dx = 0; dx < d1d - 1; ++dx) { mass_x[dx] = 0.0; }
-         for (int qx = 0; qx < q1d; ++qx)
-         {
-            const real_t s = O(0, qx, qy, e) * mass[qy][qx];
-            for (int dx = 0; dx < d1d - 1; ++dx) { mass_x[dx] += s * Bo(qx, dx); }
-         }
-         for (int dy = 0; dy < d1d; ++dy)
-         {
-            const real_t wy = Bc(qy, dy);
-            for (int dx = 0; dx < d1d - 1; ++dx)
-            {
-               Y(dx + (dy * (d1d - 1)) + osc, e) += mass_x[dx] * wy;
-            }
-         }
-      }
-
-      osc += (d1d - 1) * d1d;
-      for (int qy = 0; qy < q1d; ++qy)
-      {
-         real_t mass_x[MAX_D1D];
-         for (int dx = 0; dx < d1d; ++dx) { mass_x[dx] = 0.0; }
-         for (int qx = 0; qx < q1d; ++qx)
-         {
-            const real_t s = O(1, qx, qy, e) * mass[qy][qx];
-            for (int dx = 0; dx < d1d; ++dx) { mass_x[dx] += s * Bc(qx, dx); }
-         }
-         for (int dy = 0; dy < d1d - 1; ++dy)
-         {
-            const real_t wy = Bo(qy, dy);
-            for (int dx = 0; dx < d1d; ++dx)
-            {
-               Y(dx + (dy * d1d) + osc, e) += mass_x[dx] * wy;
-            }
-         }
-      }
-   });
-}
-
-void PAHcurlDotApply3D(const int d1d,
-                       const int d1d_test,
-                       const int q1d,
-                       const int ne,
-                       const Array<real_t> &bo,
-                       const Array<real_t> &bc,
-                       const Array<real_t> &bt,
-                       const Vector &pa_data,
-                       const Vector &x,
-                       Vector &y)
-{
-   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D, "");
-   MFEM_VERIFY(d1d_test <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D, "");
-
-   auto Bo = Reshape(bo.Read(), q1d, d1d - 1);
-   auto Bc = Reshape(bc.Read(), q1d, d1d);
-   auto Bt = Reshape(bt.Read(), d1d_test, q1d);
-   auto O = Reshape(pa_data.Read(), 3, q1d, q1d, q1d, ne);
-   auto X = Reshape(x.Read(), 3 * (d1d - 1) * d1d * d1d, ne);
-   auto Y = Reshape(y.ReadWrite(), d1d_test, d1d_test, d1d_test, ne);
-
-   mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int MAX_D1D = DofQuadLimits::MAX_D1D;
-      constexpr int MAX_Q1D = DofQuadLimits::HCURL_MAX_Q1D;
-
-      real_t u[MAX_Q1D][MAX_Q1D][MAX_Q1D][3];
-      for (int qz = 0; qz < q1d; ++qz)
-      {
-         for (int qy = 0; qy < q1d; ++qy)
-         {
-            for (int qx = 0; qx < q1d; ++qx)
-            {
-               for (int c = 0; c < 3; ++c) { u[qz][qy][qx][c] = 0.0; }
-            }
-         }
-      }
-
-      int osc = 0;
-      for (int dz = 0; dz < d1d; ++dz)
-      {
-         real_t mass_xy[MAX_Q1D][MAX_Q1D];
-         for (int qy = 0; qy < q1d; ++qy)
-         {
-            for (int qx = 0; qx < q1d; ++qx) { mass_xy[qy][qx] = 0.0; }
-         }
-
-         for (int dy = 0; dy < d1d; ++dy)
-         {
-            real_t mass_x[MAX_Q1D];
-            for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] = 0.0; }
-            for (int dx = 0; dx < d1d - 1; ++dx)
-            {
-               const real_t t = X(dx + ((dy + (dz * d1d)) * (d1d - 1)) + osc, e);
-               for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] += t * Bo(qx, dx); }
-            }
-            for (int qy = 0; qy < q1d; ++qy)
-            {
-               const real_t wy = Bc(qy, dy);
-               for (int qx = 0; qx < q1d; ++qx) { mass_xy[qy][qx] += mass_x[qx] * wy; }
-            }
-         }
-
-         for (int qz = 0; qz < q1d; ++qz)
-         {
-            const real_t wz = Bc(qz, dz);
-            for (int qy = 0; qy < q1d; ++qy)
-            {
-               for (int qx = 0; qx < q1d; ++qx) { u[qz][qy][qx][0] += mass_xy[qy][qx] * wz; }
-            }
-         }
-      }
-
-      osc += (d1d - 1) * d1d * d1d;
-      for (int dz = 0; dz < d1d; ++dz)
-      {
-         real_t mass_xy[MAX_Q1D][MAX_Q1D];
-         for (int qy = 0; qy < q1d; ++qy)
-         {
-            for (int qx = 0; qx < q1d; ++qx) { mass_xy[qy][qx] = 0.0; }
-         }
-
-         for (int dy = 0; dy < d1d - 1; ++dy)
-         {
-            real_t mass_x[MAX_Q1D];
-            for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] = 0.0; }
-            for (int dx = 0; dx < d1d; ++dx)
-            {
-               const real_t t = X(dx + ((dy + (dz * (d1d - 1))) * d1d) + osc, e);
-               for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] += t * Bc(qx, dx); }
-            }
-            for (int qy = 0; qy < q1d; ++qy)
-            {
-               const real_t wy = Bo(qy, dy);
-               for (int qx = 0; qx < q1d; ++qx) { mass_xy[qy][qx] += mass_x[qx] * wy; }
-            }
-         }
-
-         for (int qz = 0; qz < q1d; ++qz)
-         {
-            const real_t wz = Bc(qz, dz);
-            for (int qy = 0; qy < q1d; ++qy)
-            {
-               for (int qx = 0; qx < q1d; ++qx) { u[qz][qy][qx][1] += mass_xy[qy][qx] * wz; }
-            }
-         }
-      }
-
-      osc += (d1d - 1) * d1d * d1d;
-      for (int dz = 0; dz < d1d - 1; ++dz)
-      {
-         real_t mass_xy[MAX_Q1D][MAX_Q1D];
-         for (int qy = 0; qy < q1d; ++qy)
-         {
-            for (int qx = 0; qx < q1d; ++qx) { mass_xy[qy][qx] = 0.0; }
-         }
-
-         for (int dy = 0; dy < d1d; ++dy)
-         {
-            real_t mass_x[MAX_Q1D];
-            for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] = 0.0; }
-            for (int dx = 0; dx < d1d; ++dx)
-            {
-               const real_t t = X(dx + ((dy + (dz * d1d)) * d1d) + osc, e);
-               for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] += t * Bc(qx, dx); }
-            }
-            for (int qy = 0; qy < q1d; ++qy)
-            {
-               const real_t wy = Bc(qy, dy);
-               for (int qx = 0; qx < q1d; ++qx) { mass_xy[qy][qx] += mass_x[qx] * wy; }
-            }
-         }
-
-         for (int qz = 0; qz < q1d; ++qz)
-         {
-            const real_t wz = Bo(qz, dz);
-            for (int qy = 0; qy < q1d; ++qy)
-            {
-               for (int qx = 0; qx < q1d; ++qx) { u[qz][qy][qx][2] += mass_xy[qy][qx] * wz; }
-            }
-         }
-      }
-
-      for (int qz = 0; qz < q1d; ++qz)
-      {
-         real_t mass_xy[MAX_D1D][MAX_D1D];
-         for (int dy = 0; dy < d1d_test; ++dy)
-         {
-            for (int dx = 0; dx < d1d_test; ++dx) { mass_xy[dy][dx] = 0.0; }
-         }
-
-         for (int qy = 0; qy < q1d; ++qy)
-         {
-            real_t mass_x[MAX_D1D];
-            for (int dx = 0; dx < d1d_test; ++dx) { mass_x[dx] = 0.0; }
-            for (int qx = 0; qx < q1d; ++qx)
-            {
-               const real_t s = O(0, qx, qy, qz, e) * u[qz][qy][qx][0]
-                                + O(1, qx, qy, qz, e) * u[qz][qy][qx][1]
-                                + O(2, qx, qy, qz, e) * u[qz][qy][qx][2];
-               for (int dx = 0; dx < d1d_test; ++dx) { mass_x[dx] += s * Bt(dx, qx); }
-            }
-            for (int dy = 0; dy < d1d_test; ++dy)
-            {
-               const real_t wy = Bt(dy, qy);
-               for (int dx = 0; dx < d1d_test; ++dx) { mass_xy[dy][dx] += mass_x[dx] * wy; }
-            }
-         }
-
-         for (int dz = 0; dz < d1d_test; ++dz)
-         {
-            const real_t wz = Bt(dz, qz);
-            for (int dy = 0; dy < d1d_test; ++dy)
-            {
-               for (int dx = 0; dx < d1d_test; ++dx)
-               {
-                  Y(dx, dy, dz, e) += mass_xy[dy][dx] * wz;
-               }
-            }
-         }
-      }
-   });
-}
-
-void PAHcurlDotApplyTranspose3D(const int d1d,
-                                const int d1d_test,
-                                const int q1d,
-                                const int ne,
-                                const Array<real_t> &bo,
-                                const Array<real_t> &bc,
-                                const Array<real_t> &b,
-                                const Vector &pa_data,
-                                const Vector &x,
-                                Vector &y)
-{
-   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D, "");
-   MFEM_VERIFY(d1d_test <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D, "");
-
-   auto Bo = Reshape(bo.Read(), q1d, d1d - 1);
-   auto Bc = Reshape(bc.Read(), q1d, d1d);
-   auto B = Reshape(b.Read(), q1d, d1d_test);
-   auto O = Reshape(pa_data.Read(), 3, q1d, q1d, q1d, ne);
-   auto X = Reshape(x.Read(), d1d_test, d1d_test, d1d_test, ne);
-   auto Y = Reshape(y.ReadWrite(), 3 * (d1d - 1) * d1d * d1d, ne);
-
-   mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int MAX_D1D = DofQuadLimits::MAX_D1D;
-      constexpr int MAX_Q1D = DofQuadLimits::HCURL_MAX_Q1D;
-
-      real_t mass[MAX_Q1D][MAX_Q1D][MAX_Q1D];
-      for (int qz = 0; qz < q1d; ++qz)
-      {
-         for (int qy = 0; qy < q1d; ++qy)
-         {
-            for (int qx = 0; qx < q1d; ++qx) { mass[qz][qy][qx] = 0.0; }
-         }
-      }
-
-      for (int dz = 0; dz < d1d_test; ++dz)
-      {
-         real_t mass_xy[MAX_Q1D][MAX_Q1D];
-         for (int qy = 0; qy < q1d; ++qy)
-         {
-            for (int qx = 0; qx < q1d; ++qx) { mass_xy[qy][qx] = 0.0; }
-         }
-
-         for (int dy = 0; dy < d1d_test; ++dy)
-         {
-            real_t mass_x[MAX_Q1D];
-            for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] = 0.0; }
-            for (int dx = 0; dx < d1d_test; ++dx)
-            {
-               const real_t t = X(dx, dy, dz, e);
-               for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] += t * B(qx, dx); }
-            }
-            for (int qy = 0; qy < q1d; ++qy)
-            {
-               const real_t wy = B(qy, dy);
-               for (int qx = 0; qx < q1d; ++qx) { mass_xy[qy][qx] += mass_x[qx] * wy; }
-            }
-         }
-
-         for (int qz = 0; qz < q1d; ++qz)
-         {
-            const real_t wz = B(qz, dz);
-            for (int qy = 0; qy < q1d; ++qy)
-            {
-               for (int qx = 0; qx < q1d; ++qx) { mass[qz][qy][qx] += mass_xy[qy][qx] * wz; }
-            }
-         }
-      }
-
-      int osc = 0;
-      for (int qz = 0; qz < q1d; ++qz)
-      {
-         real_t mass_xy[MAX_D1D][MAX_D1D];
-         for (int dy = 0; dy < d1d; ++dy)
-         {
-            for (int dx = 0; dx < d1d - 1; ++dx) { mass_xy[dy][dx] = 0.0; }
-         }
-
-         for (int qy = 0; qy < q1d; ++qy)
-         {
-            real_t mass_x[MAX_D1D];
-            for (int dx = 0; dx < d1d - 1; ++dx) { mass_x[dx] = 0.0; }
-            for (int qx = 0; qx < q1d; ++qx)
-            {
-               const real_t s = O(0, qx, qy, qz, e) * mass[qz][qy][qx];
-               for (int dx = 0; dx < d1d - 1; ++dx) { mass_x[dx] += s * Bo(qx, dx); }
-            }
-            for (int dy = 0; dy < d1d; ++dy)
-            {
-               const real_t wy = Bc(qy, dy);
-               for (int dx = 0; dx < d1d - 1; ++dx) { mass_xy[dy][dx] += mass_x[dx] * wy; }
-            }
-         }
-
-         for (int dz = 0; dz < d1d; ++dz)
-         {
-            const real_t wz = Bc(qz, dz);
-            for (int dy = 0; dy < d1d; ++dy)
-            {
-               for (int dx = 0; dx < d1d - 1; ++dx)
-               {
-                  Y(dx + ((dy + (dz * d1d)) * (d1d - 1)) + osc, e) += mass_xy[dy][dx] * wz;
-               }
-            }
-         }
-      }
-
-      osc += (d1d - 1) * d1d * d1d;
-      for (int qz = 0; qz < q1d; ++qz)
-      {
-         real_t mass_xy[MAX_D1D][MAX_D1D];
-         for (int dy = 0; dy < d1d - 1; ++dy)
-         {
-            for (int dx = 0; dx < d1d; ++dx) { mass_xy[dy][dx] = 0.0; }
-         }
-
-         for (int qy = 0; qy < q1d; ++qy)
-         {
-            real_t mass_x[MAX_D1D];
-            for (int dx = 0; dx < d1d; ++dx) { mass_x[dx] = 0.0; }
-            for (int qx = 0; qx < q1d; ++qx)
-            {
-               const real_t s = O(1, qx, qy, qz, e) * mass[qz][qy][qx];
-               for (int dx = 0; dx < d1d; ++dx) { mass_x[dx] += s * Bc(qx, dx); }
-            }
-            for (int dy = 0; dy < d1d - 1; ++dy)
-            {
-               const real_t wy = Bo(qy, dy);
-               for (int dx = 0; dx < d1d; ++dx) { mass_xy[dy][dx] += mass_x[dx] * wy; }
-            }
-         }
-
-         for (int dz = 0; dz < d1d; ++dz)
-         {
-            const real_t wz = Bc(qz, dz);
-            for (int dy = 0; dy < d1d - 1; ++dy)
-            {
-               for (int dx = 0; dx < d1d; ++dx)
-               {
-                  Y(dx + ((dy + (dz * (d1d - 1))) * d1d) + osc, e) += mass_xy[dy][dx] * wz;
-               }
-            }
-         }
-      }
-
-      osc += (d1d - 1) * d1d * d1d;
-      for (int qz = 0; qz < q1d; ++qz)
-      {
-         real_t mass_xy[MAX_D1D][MAX_D1D];
-         for (int dy = 0; dy < d1d; ++dy)
-         {
-            for (int dx = 0; dx < d1d; ++dx) { mass_xy[dy][dx] = 0.0; }
-         }
-
-         for (int qy = 0; qy < q1d; ++qy)
-         {
-            real_t mass_x[MAX_D1D];
-            for (int dx = 0; dx < d1d; ++dx) { mass_x[dx] = 0.0; }
-            for (int qx = 0; qx < q1d; ++qx)
-            {
-               const real_t s = O(2, qx, qy, qz, e) * mass[qz][qy][qx];
-               for (int dx = 0; dx < d1d; ++dx) { mass_x[dx] += s * Bc(qx, dx); }
-            }
-            for (int dy = 0; dy < d1d; ++dy)
-            {
-               const real_t wy = Bc(qy, dy);
-               for (int dx = 0; dx < d1d; ++dx) { mass_xy[dy][dx] += mass_x[dx] * wy; }
-            }
-         }
-
-         for (int dz = 0; dz < d1d - 1; ++dz)
-         {
-            const real_t wz = Bo(qz, dz);
-            for (int dy = 0; dy < d1d; ++dy)
-            {
-               for (int dx = 0; dx < d1d; ++dx)
-               {
-                  Y(dx + ((dy + (dz * d1d)) * d1d) + osc, e) += mass_xy[dy][dx] * wz;
-               }
-            }
-         }
-      }
-   });
-}
-
-} // namespace
-
-void MixedDotProductIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
-                                           const FiniteElementSpace &test_fes)
-{
-   Mesh *mesh = trial_fes.GetMesh();
-   const FiniteElement *trial_fel = trial_fes.GetTypicalFE();
-   const FiniteElement *test_fel = test_fes.GetTypicalFE();
-
-   const VectorTensorFiniteElement *trial_el =
-      dynamic_cast<const VectorTensorFiniteElement*>(trial_fel);
-   MFEM_VERIFY(trial_el != NULL, "Only VectorTensorFiniteElement is supported!");
-
-   const TensorBasisElement *test_tensor_el =
-      dynamic_cast<const TensorBasisElement*>(test_fel);
-   MFEM_VERIFY(test_tensor_el != NULL,
-               "Only tensor-product scalar test elements are supported!");
-
-   MFEM_VERIFY(trial_el->GetDerivType() == mfem::FiniteElement::CURL,
-               "Only H(curl) trial spaces are supported!");
-
-   const IntegrationRule *ir = IntRule;
-   if (ir == nullptr)
-   {
-      const int order = trial_fel->GetOrder() + test_fel->GetOrder()
-                        + mesh->GetTypicalElementTransformation()->OrderW();
-      ir = &IntRules.Get(trial_fel->GetGeomType(), order);
-   }
-
-   dim = mesh->Dimension();
-   MFEM_VERIFY(dim == 2 || dim == 3, "Unsupported dimension!");
-   MFEM_VERIFY(trial_el->GetDim() == dim && test_fel->GetDim() == dim,
-               "Trial/test dimension mismatch.");
-
-   ne = trial_fes.GetNE();
-   MFEM_VERIFY(ne == test_fes.GetNE(),
-               "Different meshes for test and trial spaces");
-
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
-   mapsC = &trial_el->GetDofToQuad(*ir, DofToQuad::TENSOR);
-   mapsO = &trial_el->GetDofToQuadOpen(*ir, DofToQuad::TENSOR);
-   mapsTest = &test_fel->GetDofToQuad(*ir, DofToQuad::TENSOR);
-
-   dofs1D = mapsC->ndof;
-   dofs1Dtest = mapsTest->ndof;
-   quad1D = mapsC->nqpt;
-   test_map_integral = (test_fel->GetMapType() == FiniteElement::INTEGRAL);
-
-   MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");
-   MFEM_VERIFY(quad1D == mapsTest->nqpt, "Trial/test quadrature mismatch");
-   MFEM_VERIFY(dofs1D <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D, "");
-   MFEM_VERIFY(dofs1Dtest <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(quad1D <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D, "");
-
-   const int nq = ir->GetNPoints();
-   if (dim == 2) { MFEM_VERIFY(nq == quad1D * quad1D, ""); }
-   else { MFEM_VERIFY(nq == quad1D * quad1D * quad1D, ""); }
-
-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(*VQ, qs, CoefficientStorage::FULL);
-   MFEM_VERIFY(coeff.GetVDim() == dim, "Vector coefficient dimension mismatch.");
-
-   pa_data.SetSize(dim * nq * ne, Device::GetMemoryType());
-
-   if (dim == 2)
-   {
-      PAHcurlDotSetup2D(quad1D, ne, test_map_integral, ir->GetWeights(),
-                        geom->J, coeff, pa_data);
-   }
-   else
-   {
-      PAHcurlDotSetup3D(quad1D, ne, test_map_integral, ir->GetWeights(),
-                        geom->J, coeff, pa_data);
-   }
-}
-
-void MixedDotProductIntegrator::AddMultPA(const Vector &x, Vector &y) const
-{
-   if (dim == 2)
-   {
-      PAHcurlDotApply2D(dofs1D, dofs1Dtest, quad1D, ne,
-                        mapsO->B, mapsC->B, mapsTest->Bt, pa_data, x, y);
-   }
-   else if (dim == 3)
-   {
-      PAHcurlDotApply3D(dofs1D, dofs1Dtest, quad1D, ne,
-                        mapsO->B, mapsC->B, mapsTest->Bt, pa_data, x, y);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension!");
-   }
-}
-
-void MixedDotProductIntegrator::AddMultTransposePA(const Vector &x,
-                                                   Vector &y) const
-{
-   if (dim == 2)
-   {
-      PAHcurlDotApplyTranspose2D(dofs1D, dofs1Dtest, quad1D, ne,
-                                 mapsO->B, mapsC->B, mapsTest->B,
-                                 pa_data, x, y);
-   }
-   else if (dim == 3)
-   {
-      PAHcurlDotApplyTranspose3D(dofs1D, dofs1Dtest, quad1D, ne,
-                                 mapsO->B, mapsC->B, mapsTest->B,
-                                 pa_data, x, y);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension!");
-   }
-}
-
 void MixedScalarCurlIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
                                            const FiniteElementSpace &test_fes)
 {
@@ -1093,7 +320,6 @@ void MixedVectorWeakCurlIntegrator::AssemblePA(const FiniteElementSpace
    CoefficientVector coeff(qs, CoefficientStorage::FULL);
    if (Q) { coeff.Project(*Q); }
    else if (DQ) { coeff.Project(*DQ); }
-   else if (MQ) { MFEM_ABORT("Not implemented."); }
    else { coeff.SetConstant(1.0); }
 
    if (trialType == mfem::FiniteElement::CURL && dim == 3)

fem/integ/bilininteg_vectorfemass_kernels.hpp

@@ -1,131 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_BILININTEG_VECTORFEMASS_KERNELS_HPP
-#define MFEM_BILININTEG_VECTORFEMASS_KERNELS_HPP
-
-#include "../../config/config.hpp"
-#include "../../general/array.hpp"
-#include "../../general/forall.hpp"
-#include "../../linalg/dtensor.hpp"
-#include "../../linalg/vector.hpp"
-#include "../bilininteg.hpp"
-
-#include "bilininteg_diffusion_kernels.hpp"
-#include "bilininteg_hcurl_kernels.hpp"
-#include "bilininteg_hdiv_kernels.hpp"
-#include "bilininteg_hcurlhdiv_kernels.hpp"
-
-namespace mfem
-{
-/// \cond DO_NOT_DOCUMENT
-namespace internal
-{
-namespace hcurlmass
-{
-constexpr int NBZ3D(int d1d, int q1d)
-{
-   if (d1d <= 1 || q1d <= 0)
-   {
-      return 1;
-   }
-   // assume q1d >= d1d
-   // z dimension is capped at 64 on nvidia and amd gpus
-   int tmp = std::min((128 + q1d * q1d * q1d - 1) / (q1d * q1d * q1d), 64);
-   int smem_req =
-      sizeof(mfem::real_t) *
-      (3 * ((d1d - 1) * d1d * d1d + 2 * q1d * q1d * q1d) * tmp +
-       q1d * (d1d - 1) + q1d * d1d);
-   // assume GPU has at least 48k shared memory
-   return std::max(std::min(tmp, (48 * 1024 + smem_req - 1) / smem_req), 1);
-}
-} // namespace hcurlmass
-} // namespace internal
-
-template <FiniteElement::DerivType TrialType, FiniteElement::DerivType TestType,
-          int DIM, int TrialD1D, int TestD1D, int Q1D>
-VectorFEMassIntegrator::ApplyKernelType
-VectorFEMassIntegrator::ApplyPAKernels::Kernel()
-{
-   constexpr bool trial_curl = (TrialType == mfem::FiniteElement::CURL);
-   constexpr bool trial_div = (TrialType == mfem::FiniteElement::DIV);
-   constexpr bool test_curl = (TestType == mfem::FiniteElement::CURL);
-   constexpr bool test_div = (TestType == mfem::FiniteElement::DIV);
-
-   if constexpr (DIM == 3)
-   {
-      if constexpr (trial_curl && test_curl)
-      {
-         if (Device::Allows(Backend::DEVICE_MASK))
-         {
-            // assume TrialD1D == TestD1D
-            return internal::SmemPAHcurlMassApply3D<
-                   TrialD1D, Q1D, internal::hcurlmass::NBZ3D(TrialD1D, Q1D)>;
-         }
-         else
-         {
-            return internal::PAHcurlMassApply3D;
-         }
-      }
-      else if constexpr (trial_div && test_div)
-      {
-         if (Device::Allows(Backend::DEVICE_MASK))
-         {
-            // assumes TrialD1D == TestD1D
-            return internal::SmemPAHdivMassApply3D<TrialD1D, Q1D>;
-         }
-         else
-         {
-            return internal::PAHdivMassApply3D;
-         }
-      }
-      else if constexpr (trial_curl && test_div)
-      {
-         return internal::PAHdivHcurlMassApply3D;
-      }
-      else if constexpr (trial_div && test_curl)
-      {
-         return internal::PAHcurlHdivMassApply3D;
-      }
-   }
-   else if constexpr (DIM == 2) // 2D
-   {
-      if constexpr (trial_curl && test_curl)
-      {
-         return internal::PAHcurlMassApply2D;
-      }
-      else if constexpr (trial_div && test_div)
-      {
-         if (Device::Allows(Backend::DEVICE_MASK))
-         {
-            // assumes TrialD1D == TestD1D
-            return internal::SmemPAHdivMassApply2D<TrialD1D, Q1D>;
-         }
-         else
-         {
-            return internal::PAHdivMassApply2D;
-         }
-      }
-      else if constexpr (trial_curl && test_div)
-      {
-         return internal::PAHdivHcurlMassApply2D;
-      }
-      else if constexpr (trial_div && test_curl)
-      {
-         return internal::PAHcurlHdivMassApply2D;
-      }
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-/// \endcond DO_NOT_DOCUMENT
-}
-
-#endif

fem/integ/bilininteg_vectorfemass_pa.cpp

@@ -12,115 +12,13 @@
 #include "../bilininteg.hpp"
 #include "../gridfunc.hpp"
 #include "../qfunction.hpp"
-#include "bilininteg_vectorfemass_kernels.hpp"
+#include "bilininteg_diffusion_kernels.hpp"
+#include "bilininteg_hcurl_kernels.hpp"
+#include "bilininteg_hdiv_kernels.hpp"
+#include "bilininteg_hcurlhdiv_kernels.hpp"
 
 namespace mfem
 {
-/// \cond DO_NOT_DOCUMENT
-VectorFEMassIntegrator::ApplyKernelType
-VectorFEMassIntegrator::ApplyPAKernels::Fallback(
-   FiniteElement::DerivType TrialType, FiniteElement::DerivType TestType,
-   int dim, int, int, int)
-{
-   const bool trial_curl = (TrialType == mfem::FiniteElement::CURL);
-   const bool trial_div = (TrialType == mfem::FiniteElement::DIV);
-   const bool test_curl = (TestType == mfem::FiniteElement::CURL);
-   const bool test_div = (TestType == mfem::FiniteElement::DIV);
-
-   if (dim == 3)
-   {
-      if (trial_curl && test_curl)
-      {
-         return internal::PAHcurlMassApply3D;
-      }
-      else if (trial_div && test_div)
-      {
-         return internal::PAHdivMassApply3D;
-      }
-      else if (trial_curl && test_div)
-      {
-         return internal::PAHdivHcurlMassApply3D;
-      }
-      else if (trial_div && test_curl)
-      {
-         return internal::PAHcurlHdivMassApply3D;
-      }
-   }
-   else if (dim == 2) // 2D
-   {
-      if (trial_curl && test_curl)
-      {
-         return internal::PAHcurlMassApply2D;
-      }
-      else if (trial_div && test_div)
-      {
-         return internal::PAHdivMassApply2D;
-      }
-      else if (trial_curl && test_div)
-      {
-         return internal::PAHdivHcurlMassApply2D;
-      }
-      else if (trial_div && test_curl)
-      {
-         return internal::PAHcurlHdivMassApply2D;
-      }
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-/// \endcond DO_NOT_DOCUMENT
-
-VectorFEMassIntegrator::Kernels::Kernels()
-{
-   // h(curl), h(curl)
-   // P = Q (3D)
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 2, 2, 2>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 3, 3, 3>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 4, 4, 4>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 5, 5, 5>();
-
-   // P = Q + 1 (3D)
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 2, 2, 3>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 3, 3, 4>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 4, 4, 5>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 5, 5, 6>();
-   // h(div), h(div)
-   // P = Q (2D)
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 2, 2, 2, 2>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 2, 3, 3, 3>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 2, 4, 4, 4>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 2, 5, 5, 5>();
-
-   // P = Q + 1 (3D)
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 3, 2, 2, 3>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 3, 3, 3, 4>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 3, 4, 4, 5>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
-                          FiniteElement::DIV, 3, 5, 5, 6>();
-}
-
-void VectorFEMassIntegrator::Init(Coefficient *q, DiagonalMatrixCoefficient *dq,
-                                  MatrixCoefficient *mq)
-{
-   static Kernels kernels{};
-   Q = q;
-   DQ = dq;
-   MQ = mq;
-}
 
 void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
@@ -169,8 +67,8 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
 
    MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");
 
-   trial_fetype = static_cast<FiniteElement::DerivType>(trial_el->GetDerivType());
-   test_fetype = static_cast<FiniteElement::DerivType>(test_el->GetDerivType());
+   trial_fetype = trial_el->GetDerivType();
+   test_fetype = test_el->GetDerivType();
 
    const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
    const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
@@ -317,36 +215,225 @@ void VectorFEMassIntegrator::AssembleDiagonalPA(Vector& diag)
 
 void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
-   const bool scalar_coeff = !(DQ || MQ);
-   ApplyPAKernels::Run(trial_fetype, test_fetype, dim, dofs1D, dofs1Dtest,
-                       quad1D, ne, symmetric, scalar_coeff, mapsO->B, mapsC->B,
-                       mapsOtest->Bt, mapsCtest->Bt, pa_data, x, y, dofs1D,
-                       dofs1Dtest, quad1D);
+   const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
+   const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
+   const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
+   const bool test_div = (test_fetype == mfem::FiniteElement::DIV);
+
+   if (dim == 3)
+   {
+      if (trial_curl && test_curl)
+      {
+         if (Device::Allows(Backend::DEVICE_MASK))
+         {
+            const int ID = (dofs1D << 4) | quad1D;
+            switch (ID)
+            {
+               case 0x23:
+                  return internal::SmemPAHcurlMassApply3D<2,3>(
+                            dofs1D, quad1D, ne, symmetric,
+                            mapsO->B, mapsC->B, mapsO->Bt,
+                            mapsC->Bt, pa_data, x, y);
+               case 0x34:
+                  return internal::SmemPAHcurlMassApply3D<3,4>(
+                            dofs1D, quad1D, ne, symmetric,
+                            mapsO->B, mapsC->B, mapsO->Bt,
+                            mapsC->Bt, pa_data, x, y);
+               case 0x45:
+                  return internal::SmemPAHcurlMassApply3D<4,5>(
+                            dofs1D, quad1D, ne, symmetric,
+                            mapsO->B, mapsC->B, mapsO->Bt,
+                            mapsC->Bt, pa_data, x, y);
+               case 0x56:
+                  return internal::SmemPAHcurlMassApply3D<5,6>(
+                            dofs1D, quad1D, ne, symmetric,
+                            mapsO->B, mapsC->B, mapsO->Bt,
+                            mapsC->Bt, pa_data, x, y);
+               default:
+                  return internal::SmemPAHcurlMassApply3D(
+                            dofs1D, quad1D, ne, symmetric,
+                            mapsO->B, mapsC->B, mapsO->Bt,
+                            mapsC->Bt, pa_data, x, y);
+            }
+         }
+         else
+         {
+            internal::PAHcurlMassApply3D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
+                                         mapsO->Bt, mapsC->Bt, pa_data, x, y);
+         }
+      }
+      else if (trial_div && test_div)
+      {
+         internal::PAHdivMassApply(3, dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
+                                   mapsO->Bt, mapsC->Bt, pa_data, x, y);
+      }
+      else if (trial_curl && test_div)
+      {
+         const bool scalarCoeff = !(DQ || MQ);
+         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
+                                          true, false, mapsO->B, mapsC->B, mapsOtest->Bt,
+                                          mapsCtest->Bt, pa_data, x, y);
+      }
+      else if (trial_div && test_curl)
+      {
+         const bool scalarCoeff = !(DQ || MQ);
+         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
+                                          false, false, mapsO->B, mapsC->B, mapsOtest->Bt,
+                                          mapsCtest->Bt, pa_data, x, y);
+      }
+      else
+      {
+         MFEM_ABORT("Unknown kernel.");
+      }
+   }
+   else // 2D
+   {
+      if (trial_curl && test_curl)
+      {
+         internal::PAHcurlMassApply2D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
+                                      mapsO->Bt, mapsC->Bt, pa_data, x, y);
+      }
+      else if (trial_div && test_div)
+      {
+         internal::PAHdivMassApply(2, dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
+                                   mapsO->Bt,
+                                   mapsC->Bt, pa_data, x, y);
+      }
+      else if ((trial_curl && test_div) || (trial_div && test_curl))
+      {
+         const bool scalarCoeff = !(DQ || MQ);
+         internal::PAHcurlHdivMassApply2D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
+                                          trial_curl, false, mapsO->B, mapsC->B,
+                                          mapsOtest->Bt, mapsCtest->Bt, pa_data, x, y);
+      }
+      else
+      {
+         MFEM_ABORT("Unknown kernel.");
+      }
+   }
 }
 
 void VectorFEMassIntegrator::AddAbsMultPA(const Vector &x, Vector &y) const
 {
-   const bool scalar_coeff = !(DQ || MQ);
+   const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
+   const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
+   const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
+   const bool test_div = (test_fetype == mfem::FiniteElement::DIV);
 
    Vector abs_pa_data(pa_data);
    abs_pa_data.Abs();
 
    Array<real_t> absBo(mapsO->B);
    Array<real_t> absBc(mapsC->B);
-   // Array<real_t> absBto(mapsO->Bt);
-   // Array<real_t> absBtc(mapsC->Bt);
+   Array<real_t> absBto(mapsO->Bt);
+   Array<real_t> absBtc(mapsC->Bt);
    Array<real_t> absBto_t(mapsOtest->Bt);
    Array<real_t> absBtc_t(mapsCtest->Bt);
 
    absBo.Abs();
    absBc.Abs();
+   absBto.Abs();
+   absBtc.Abs();
    absBto_t.Abs();
    absBtc_t.Abs();
 
-   ApplyPAKernels::Run(trial_fetype, test_fetype, dim, dofs1D, dofs1Dtest,
-                       quad1D, ne, symmetric, scalar_coeff, absBo, absBc,
-                       absBto_t, absBtc_t, abs_pa_data, x, y, dofs1D,
-                       dofs1Dtest, quad1D);
+   if (dim == 3)
+   {
+      if (trial_curl && test_curl)
+      {
+         if (Device::Allows(Backend::DEVICE_MASK))
+         {
+            const int ID = (dofs1D << 4) | quad1D;
+            switch (ID)
+            {
+               case 0x23:
+                  return internal::SmemPAHcurlMassApply3D<2,3>(
+                            dofs1D, quad1D, ne, symmetric,
+                            absBo, absBc, absBto, absBtc,
+                            abs_pa_data, x, y);
+               case 0x34:
+                  return internal::SmemPAHcurlMassApply3D<3,4>(
+                            dofs1D, quad1D, ne, symmetric,
+                            absBo, absBc, absBto, absBtc,
+                            abs_pa_data, x, y);
+               case 0x45:
+                  return internal::SmemPAHcurlMassApply3D<4,5>(
+                            dofs1D, quad1D, ne, symmetric,
+                            absBo, absBc, absBto, absBtc,
+                            abs_pa_data, x, y);
+               case 0x56:
+                  return internal::SmemPAHcurlMassApply3D<5,6>(
+                            dofs1D, quad1D, ne, symmetric,
+                            absBo, absBc, absBto, absBtc,
+                            abs_pa_data, x, y);
+               default:
+                  return internal::SmemPAHcurlMassApply3D(
+                            dofs1D, quad1D, ne, symmetric,
+                            absBo, absBc, absBto, absBtc,
+                            abs_pa_data, x, y);
+            }
+         }
+         else
+         {
+            internal::PAHcurlMassApply3D(dofs1D, quad1D, ne, symmetric,
+                                         absBo, absBc, absBto, absBtc,
+                                         abs_pa_data, x, y);
+         }
+      }
+      else if (trial_div && test_div)
+      {
+         internal::PAHdivMassApply(3, dofs1D, quad1D, ne, symmetric,
+                                   absBo, absBc, absBto, absBtc,
+                                   abs_pa_data, x, y);
+      }
+      else if (trial_curl && test_div)
+      {
+         const bool scalarCoeff = !(DQ || MQ);
+         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne,
+                                          scalarCoeff, true, false,
+                                          absBo, absBc, absBto_t, absBtc_t,
+                                          abs_pa_data, x, y);
+      }
+      else if (trial_div && test_curl)
+      {
+         const bool scalarCoeff = !(DQ || MQ);
+         internal::PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne,
+                                          scalarCoeff, false, false,
+                                          absBo, absBc, absBto_t, absBtc_t,
+                                          abs_pa_data, x, y);
+      }
+      else
+      {
+         MFEM_ABORT("Unknown kernel.");
+      }
+   }
+   else // 2D
+   {
+      if (trial_curl && test_curl)
+      {
+         internal::PAHcurlMassApply2D(dofs1D, quad1D, ne, symmetric,
+                                      absBo, absBc, absBto, absBtc,
+                                      abs_pa_data, x, y);
+      }
+      else if (trial_div && test_div)
+      {
+         internal::PAHdivMassApply(2, dofs1D, quad1D, ne, symmetric,
+                                   absBo, absBc, absBto, absBtc,
+                                   abs_pa_data, x, y);
+      }
+      else if ((trial_curl && test_div) || (trial_div && test_curl))
+      {
+         const bool scalarCoeff = !(DQ || MQ);
+         internal::PAHcurlHdivMassApply2D(dofs1D, dofs1Dtest, quad1D, ne,
+                                          scalarCoeff, trial_curl, false,
+                                          absBo, absBc, absBto_t, absBtc_t,
+                                          abs_pa_data, x, y);
+      }
+      else
+      {
+         MFEM_ABORT("Unknown kernel.");
+      }
+   }
 }
 
 void VectorFEMassIntegrator::AddMultTransposePA(const Vector &x,

fem/kernel_dispatch.hpp

@@ -61,7 +61,7 @@ namespace mfem
 #define MFEM_REGISTER_KERNELS_1(KernelName, KernelType, Params)                \
    MFEM_REGISTER_KERNELS_(KernelName, KernelType, Params, (), Params)
 
-// Version of MFEM_REGISTER_KERNELS with optional (non-dispatch)
+// Version of MFEM_REGISTER_KERNELS without any optional (non-dispatch)
 // parameters (e.g. NBZ).
 #define MFEM_REGISTER_KERNELS_2(KernelName, KernelType, Params, OptParams)     \
    MFEM_REGISTER_KERNELS_(KernelName, KernelType, Params, OptParams,           \

fem/lor/lor_dg_impl.hpp

@@ -258,8 +258,8 @@ void BatchedLOR_DG::Assemble2D()
                   const int w_idx = (n_idx == 0) ? iy : ix;
                   const int x_idx = (n_idx == 0) ? i_0 : j_0;
 
-                  const real_t J1 = J(ix, iy, n_idx, (int)!n_idx, iel_ho);
-                  const real_t J2 = J(ix, iy, (int)!n_idx, (int)!n_idx, iel_ho);
+                  const real_t J1 = J(ix, iy, n_idx, !n_idx, iel_ho);
+                  const real_t J2 = J(ix, iy, !n_idx, !n_idx, iel_ho);
                   const real_t Jh = (J1*J1 + J2*J2) / detJ(ix, iy, iel_ho);
 
                   V(v_idx, ix, iy, iel_ho) =

fem/particleset.cpp

@@ -10,6 +10,7 @@
 // CONTRIBUTING.md for details.
 
 #include "particleset.hpp"
+#include "../general/forall.hpp"
 
 #if defined(MFEM_USE_MPI) && defined(MFEM_USE_GSLIB)
 
@@ -225,6 +226,7 @@ void ParticleSet::AddParticles(const Array<IDType> &new_ids,
       }
    }
    // Add new ids
+   ids.HostReadWrite();
    ids.Append(new_ids);
 
    // Update data
@@ -244,6 +246,102 @@ void ParticleSet::AddParticles(const Array<IDType> &new_ids,
 #if defined(MFEM_USE_MPI) && defined(MFEM_USE_GSLIB)
 
 /// \cond DO_NOT_DOCUMENT
+// Static helper: gather selected particle-vector entries into a compact buffer.
+// nvcc does not allow extended host/device lambdas in non-public members.
+static void GatherParticleVectorDevice(const ParticleVector &pv,
+                                       const Array<int> &send_idxs,
+                                       Vector &send_data,
+                                       int nsend)
+{
+   const int vdim = pv.GetVDim();
+   const int ordering = pv.GetOrdering();
+   const int num_particles = pv.GetNumParticles();
+   const MemoryClass device_mc = Device::GetDeviceMemoryClass();
+   send_data.SetSize(nsend*vdim);
+   real_t *d_send_data =
+      send_data.GetMemory().Write(device_mc, send_data.Size());
+   const real_t *d_src = pv.GetMemory().Read(device_mc, pv.Size());
+   const int *d_send_idxs = send_idxs.GetMemory().Read(device_mc, nsend);
+
+   mfem::forall(nsend, [=] MFEM_HOST_DEVICE (int i)
+   {
+      const int p = d_send_idxs[i];
+      const int offset = (ordering == Ordering::byVDIM) ? p * vdim : p;
+      const int stride = (ordering == Ordering::byVDIM) ? 1 : num_particles;
+
+      for (int c = 0; c < vdim; c++)
+      {
+         d_send_data[i*vdim + c] = d_src[offset + c*stride];
+      }
+   });
+}
+
+// Static helper: gather selected tag values into a compact buffer.
+// nvcc does not allow extended host/device lambdas in non-public members.
+static void GatherParticleTagsDevice(const Array<int> &tag,
+                                     const Array<int> &send_idxs,
+                                     Array<int> &send_tag,
+                                     int nsend)
+{
+   const MemoryClass device_mc = Device::GetDeviceMemoryClass();
+   send_tag.SetSize(nsend);
+   int *d_send_tag = send_tag.GetMemory().Write(device_mc, nsend);
+   const int *d_tag = tag.GetMemory().Read(device_mc, tag.Size());
+   const int *d_send_idxs = send_idxs.GetMemory().Read(device_mc, nsend);
+
+   mfem::forall(nsend, [=] MFEM_HOST_DEVICE (int i)
+   {
+      d_send_tag[i] = d_tag[d_send_idxs[i]];
+   });
+}
+
+// Static helper: scatter compact particle-vector entries to particle storage.
+// nvcc does not allow extended host/device lambdas in non-public members.
+static void ScatterParticleVectorDevice(ParticleVector &pv,
+                                        const Vector &recv_data,
+                                        const Array<int> &recv_locs,
+                                        int nrecv)
+{
+   const int vdim = pv.GetVDim();
+   const int ordering = pv.GetOrdering();
+   const int num_particles = pv.GetNumParticles();
+   const MemoryClass device_mc = Device::GetDeviceMemoryClass();
+   const real_t *d_recv_data =
+      recv_data.GetMemory().Read(device_mc, recv_data.Size());
+   const int *d_recv_locs = recv_locs.GetMemory().Read(device_mc, nrecv);
+   real_t *d_dst = pv.GetMemory().ReadWrite(device_mc, pv.Size());
+
+   mfem::forall(nrecv, [=] MFEM_HOST_DEVICE (int i)
+   {
+      const int p = d_recv_locs[i];
+      const int offset = (ordering == Ordering::byVDIM) ? p * vdim : p;
+      const int stride = (ordering == Ordering::byVDIM) ? 1 : num_particles;
+
+      for (int c = 0; c < vdim; c++)
+      {
+         d_dst[offset + c*stride] = d_recv_data[i*vdim + c];
+      }
+   });
+}
+
+// Static helper: scatter compact tag values to particle storage.
+// nvcc does not allow extended host/device lambdas in non-public members.
+static void ScatterParticleTagsDevice(Array<int> &tag,
+                                      const Array<int> &recv_tag,
+                                      const Array<int> &recv_locs,
+                                      int nrecv)
+{
+   const MemoryClass device_mc = Device::GetDeviceMemoryClass();
+   const int *d_recv_tag = recv_tag.GetMemory().Read(device_mc, nrecv);
+   const int *d_recv_locs = recv_locs.GetMemory().Read(device_mc, nrecv);
+   int *d_tag = tag.GetMemory().ReadWrite(device_mc, tag.Size());
+
+   mfem::forall(nrecv, [=] MFEM_HOST_DEVICE (int i)
+   {
+      d_tag[d_recv_locs[i]] = d_recv_tag[i];
+   });
+}
+
 template<size_t NBytes>
 void ParticleSet::TransferParticlesImpl(ParticleSet &pset,
                                         const Array<int> &send_idxs,
@@ -266,37 +364,108 @@ void ParticleSet::TransferParticlesImpl(ParticleSet &pset,
    array_init(parr_t, &gsl_arr, send_idxs.Size());
    pdata_arr = (parr_t*) gsl_arr.ptr;
 
+   int nparticles = pset.GetNParticles();
+   int nsend      = send_idxs.Size();
    gsl_arr.n = send_idxs.Size();
+
+   const int *h_send_idxs_initial = send_idxs.HostRead();
+   const IDType *h_ids = pset.GetIDs().HostRead();
    for (int i = 0; i < send_idxs.Size(); i++)
    {
       parr_t &pdata = pdata_arr[i];
-      pdata.id = pset.GetIDs()[send_idxs[i]];
+      pdata.id = h_ids[h_send_idxs_initial[i]];
+   }
 
-      // Copy particle data directly into pdata
-      size_t counter = 0;
-      for (int f = -1; f < pset.GetNFields(); f++)
+   // Pack coords and fields into the GSLIB send buffer. Device-resident data
+   // is first gathered into a compact device buffer so that only selected
+   // particles are copied back to host. Host-resident data is packed directly.
+   int max_vdim = pset.Coords().GetVDim();
+   for (int f = 0; f < pset.GetNFields(); f++)
+   {
+      int f_vdim = pset.Field(f).GetVDim();
+      if (f_vdim > max_vdim) { max_vdim = f_vdim; }
+   }
+   Vector send_data;
+   Array<int> send_tag;
+   if (Device::IsEnabled())
+   {
+      send_data.SetSize(nsend * max_vdim); // allocate max size over all fields
+      send_tag.SetSize(nsend);
+   }
+
+   size_t counter = 0;
+   for (int f = -1; f < pset.GetNFields(); f++)
+   {
+      const ParticleVector &pv = f == -1 ? pset.Coords() : pset.Field(f);
+      const int vdim = pv.GetVDim();
+      const int ordering = pv.GetOrdering();
+      const int num_particles = pv.GetNumParticles();
+      const bool use_dev = Device::IsEnabled() && pv.UseDevice();
+
+      if (use_dev)
       {
-         ParticleVector &pv = (f == -1 ? pset.Coords() : pset.Field(f));
-         for (int c = 0; c < pv.GetVDim(); c++)
+         GatherParticleVectorDevice(pv, send_idxs, send_data, nsend);
+
+         const real_t *h_send_data = send_data.HostRead();
+         for (int i = 0; i < nsend; i++)
          {
-            std::memcpy(pdata.data.data() + counter, &pv(send_idxs[i], c),
-                        sizeof(real_t));
-            counter += sizeof(real_t);
+            std::memcpy(pdata_arr[i].data.data() + counter,
+                        h_send_data + i*vdim, vdim * sizeof(real_t));
+         }
+      }
+      else
+      {
+         const real_t *h_src = pv.HostRead();
+         const int *h_send_idxs = send_idxs.HostRead();
+         for (int i = 0; i < nsend; i++)
+         {
+            parr_t &pdata = pdata_arr[i];
+            const int p = h_send_idxs[i];
+            const int offset = (ordering == Ordering::byVDIM) ? p * vdim : p;
+            const int stride = (ordering == Ordering::byVDIM) ? 1 :
+                               num_particles;
+
+            for (int c = 0; c < vdim; c++)
+            {
+               std::memcpy(pdata.data.data() + counter + c*sizeof(real_t),
+                           h_src + offset + c*stride, sizeof(real_t));
+            }
          }
       }
 
-      // Copy tags
-      for (int t = 0; t < pset.GetNTags(); t++)
-      {
-         Array<int> &tag_arr = pset.Tag(t);
-         std::memcpy(pdata.data.data() + counter, &tag_arr[send_idxs[i]],
-                     sizeof(int));
-         counter += sizeof(int);
-      }
+      counter += vdim*sizeof(real_t);
    }
 
-   int nparticles = pset.GetNParticles();
-   int nsend      = send_idxs.Size();
+   // Pack tags after all real_t data. Each tag uses the same selective
+   // device gather path when its Array is device-resident.
+   for (int t = 0; t < pset.GetNTags(); t++)
+   {
+      const Array<int> &tag = pset.Tag(t);
+      const size_t tag_counter = counter + t*sizeof(int);
+      const bool use_dev = Device::IsEnabled() && tag.UseDevice();
+
+      if (use_dev)
+      {
+         GatherParticleTagsDevice(tag, send_idxs, send_tag, nsend);
+
+         const int *h_send_tag = send_tag.HostRead();
+         for (int i = 0; i < nsend; i++)
+         {
+            std::memcpy(pdata_arr[i].data.data() + tag_counter,
+                        h_send_tag + i, sizeof(int));
+         }
+      }
+      else
+      {
+         const int *h_tag = tag.HostRead();
+         const int *h_send_idxs = send_idxs.HostRead();
+         for (int i = 0; i < nsend; i++)
+         {
+            std::memcpy(pdata_arr[i].data.data() + tag_counter,
+                        h_tag + h_send_idxs[i], sizeof(int));
+         }
+      }
+   }
 
    // Transfer particles
    sarray_transfer_ext(parr_t, &gsl_arr, send_ranks.GetData(),
@@ -304,11 +473,20 @@ void ParticleSet::TransferParticlesImpl(ParticleSet &pset,
 
    // Make sure we have enough space for received particles
    int nrecv = (int) gsl_arr.n;
+
+   Vector recv_data;
+   Array<int> recv_tag;
+   if (Device::IsEnabled())
+   {
+      recv_data.SetSize(nrecv * max_vdim);
+      recv_tag.SetSize(nrecv);
+   }
+
    int ndelete = nsend - nrecv;
    if (ndelete > 0)
    {
       // Remove unneeded particles
-      auto datap = const_cast<int*>(send_idxs.GetData());
+      auto datap = const_cast<int*>(send_idxs.HostRead());
       Array<int> delete_idxs(datap + nrecv, ndelete);
       pset.RemoveParticles(delete_idxs);
    }
@@ -319,47 +497,133 @@ void ParticleSet::TransferParticlesImpl(ParticleSet &pset,
 
    pdata_arr = (parr_t*) gsl_arr.ptr;
 
-   // Add newly-recvd data directly to active state
+   // Make a list of new IDs to add
+   int num_new = nrecv > nsend ? nrecv - nsend : 0;
+   Array<IDType> new_ids(num_new);
+   for (int i = 0; i < num_new; i++)
+   {
+      new_ids[i] = pdata_arr[nsend + i].id;
+   }
+
+   // Add particles in batch
+   Array<int> new_indices;
+   if (num_new > 0)
+   {
+      pset.AddParticles(new_ids, &new_indices);
+   }
+
+   // Map each received packet to the local particle slot it updates.
+   Array<int> recv_locs(nrecv);
+   int *h_recv_locs = recv_locs.HostWrite();
+   const int *h_send_idxs_recv = send_idxs.HostRead();
    for (int i = 0; i < nrecv; i++)
    {
       parr_t &pdata = pdata_arr[i];
-      IDType id = pdata.id;
-
-      int new_loc_idx;
       if (i < nsend) // update existing particle
       {
-         new_loc_idx = send_idxs[i];
-         pset.UpdateID(new_loc_idx, id);
+         h_recv_locs[i] = h_send_idxs_recv[i];
+         pset.UpdateID(h_recv_locs[i], pdata.id);
       }
       else
       {
-         // add new particle
-         Array<int> idx_temp;
-         pset.AddParticles(Array<IDType>({id}), &idx_temp);
-         new_loc_idx = idx_temp[0]; // Get index of newly-added particle
+         h_recv_locs[i] = new_indices[i - nsend];
       }
+   }
 
-      size_t counter = 0;
-      for (int f = -1; f < pset.GetNFields(); f++)
+   // Unpack coords and fields from GSLIB host packets. Device-resident
+   // destinations use a compact host buffer followed by a device scatter.
+   size_t recv_counter = 0;
+   for (int f = -1; f < pset.GetNFields(); f++)
+   {
+      ParticleVector &pv = (f == -1 ? pset.Coords() : pset.Field(f));
+      const int vdim = pv.GetVDim();
+      const int ordering = pv.GetOrdering();
+      const int num_particles = pv.GetNumParticles();
+      const bool use_dev = Device::IsEnabled() && pv.UseDevice();
+
+      if (use_dev)
       {
-         ParticleVector &pv = (f == -1 ? pset.Coords() : pset.Field(f));
-         for (int c = 0; c < pv.GetVDim(); c++)
+         recv_data.SetSize(nrecv*vdim);
+         real_t *h_recv_data = recv_data.HostWrite();
+
+         for (int i = 0; i < nrecv; i++)
          {
-            real_t& val = pv(new_loc_idx, c);
-            std::memcpy(&val, pdata.data.data() + counter, sizeof(real_t));
-            counter += sizeof(real_t);
+            std::memcpy(h_recv_data + i*vdim,
+                        pdata_arr[i].data.data() + recv_counter,
+                        vdim*sizeof(real_t));
+         }
+
+         ScatterParticleVectorDevice(pv, recv_data, recv_locs, nrecv);
+      }
+      else
+      {
+         real_t *h_dst = pv.HostReadWrite();
+         const int *h_recv_locs_read = recv_locs.HostRead();
+         for (int i = 0; i < nrecv; i++)
+         {
+            parr_t &pdata = pdata_arr[i];
+            const int p = h_recv_locs_read[i];
+            const int offset = (ordering == Ordering::byVDIM) ? p * vdim : p;
+            const int stride = (ordering == Ordering::byVDIM) ? 1 :
+                               num_particles;
+
+            for (int c = 0; c < vdim; c++)
+            {
+               std::memcpy(h_dst + offset + c*stride,
+                           pdata.data.data() + recv_counter + c*sizeof(real_t),
+                           sizeof(real_t));
+            }
          }
       }
 
-      for (int t = 0; t < pset.GetNTags(); t++)
+      recv_counter += vdim*sizeof(real_t);
+   }
+
+   // Unpack tags after all real_t data, using the same compact scatter path
+   // for device-resident tag arrays.
+   for (int t = 0; t < pset.GetNTags(); t++)
+   {
+      Array<int> &tag = pset.Tag(t);
+      const size_t tag_counter = recv_counter + t*sizeof(int);
+      const bool use_dev = Device::IsEnabled() && tag.UseDevice();
+
+      if (use_dev)
       {
-         Array<int> &tag_arr = pset.Tag(t);
-         std::memcpy(&tag_arr[new_loc_idx],
-                     pdata.data.data() + counter, sizeof(int));
-         counter += sizeof(int);
+         recv_tag.SetSize(nrecv);
+         int *h_recv_tag = recv_tag.HostWrite();
+
+         for (int i = 0; i < nrecv; i++)
+         {
+            std::memcpy(h_recv_tag + i,
+                        pdata_arr[i].data.data() + tag_counter, sizeof(int));
+         }
+
+         ScatterParticleTagsDevice(tag, recv_tag, recv_locs, nrecv);
+      }
+      else
+      {
+         int *h_tag = tag.HostReadWrite();
+         const int *h_recv_locs_read = recv_locs.HostRead();
+         for (int i = 0; i < nrecv; i++)
+         {
+            std::memcpy(h_tag + h_recv_locs_read[i],
+                        pdata_arr[i].data.data() + tag_counter, sizeof(int));
+         }
       }
    }
    array_free(&gsl_arr);
+
+   // Restore Device validity if needed
+   for (int f = -1; f < pset.GetNFields(); f++)
+   {
+      ParticleVector &pv = (f == -1 ? pset.Coords() : pset.Field(f));
+      pv.ReadWrite(pv.UseDevice());
+   }
+   for (int t = 0; t < pset.GetNTags(); t++)
+   {
+      Array<int> &tag_arr = pset.Tag(t);
+      if (tag_arr.UseDevice()) { tag_arr.ReadWrite(true); }
+   }
 }
 
 template<size_t NBytes>
@@ -526,11 +790,14 @@ ParticleSet::ParticleSet(int id_stride_, IDType id_counter_, int num_particles,
                          int dim, Ordering::Type coords_ordering, const Array<int> &field_vdims,
                          const Array<Ordering::Type> &field_orderings,
                          const Array<const char*> &field_names_, int num_tags,
-                         const Array<const char*> &tag_names_)
+                         const Array<const char*> &tag_names_,
+                         bool use_device)
    : id_stride(id_stride_),
      id_counter(id_counter_),
      coords(dim, coords_ordering)
 {
+   if (use_device) { coords.UseDevice(true); }
+
    // Initialize fields
    for (int f = 0; f < field_vdims.Size(); f++)
    {
@@ -580,21 +847,22 @@ bool ParticleSet::IsValidParticle(const Particle &p) const
 }
 
 ParticleSet::ParticleSet(int num_particles, int dim,
-                         Ordering::Type coords_ordering)
+                         Ordering::Type coords_ordering,
+                         bool use_device)
    : ParticleSet(1, 0, num_particles, dim, coords_ordering, Array<int>(),
                  Array<Ordering::Type>(), Array<const char*>(), 0,
-                 Array<const char*>())
+                 Array<const char*>(), use_device)
 {
 
 }
 
 ParticleSet::ParticleSet(int num_particles, int dim,
                          const Array<int> &field_vdims, int num_tags,
-                         Ordering::Type all_ordering)
+                         Ordering::Type all_ordering, bool use_device)
    : ParticleSet(1, 0, num_particles, dim, all_ordering, field_vdims,
                  GetOrderingArray(all_ordering, field_vdims.Size()),
                  GetEmptyNameArray(field_vdims.Size()), num_tags,
-                 GetEmptyNameArray(num_tags))
+                 GetEmptyNameArray(num_tags), use_device)
 {
 }
 
@@ -602,11 +870,11 @@ ParticleSet::ParticleSet(int num_particles, int dim,
                          const Array<int> &field_vdims, const Array<const
                          char*> &field_names_, int num_tags,
                          const Array<const char*> &tag_names_,
-                         Ordering::Type all_ordering)
+                         Ordering::Type all_ordering, bool use_device)
    : ParticleSet(1, 0, num_particles, dim, all_ordering, field_vdims,
                  GetOrderingArray(all_ordering, field_vdims.Size()),
                  field_names_, num_tags,
-                 tag_names_)
+                 tag_names_, use_device)
 {
 
 }
@@ -616,9 +884,9 @@ ParticleSet::ParticleSet(int num_particles, int dim,
                          const Array<int> &field_vdims,
                          const Array<Ordering::Type> &field_orderings,
                          const Array<const char*> &field_names_, int num_tags,
-                         const Array<const char*> &tag_names_)
+                         const Array<const char*> &tag_names_, bool use_device)
    : ParticleSet(1, 0, num_particles, dim, coords_ordering, field_vdims,
-                 field_orderings, field_names_, num_tags, tag_names_)
+                 field_orderings, field_names_, num_tags, tag_names_, use_device)
 {
 
 }
@@ -627,21 +895,21 @@ ParticleSet::ParticleSet(int num_particles, int dim,
 
 #ifdef MFEM_USE_MPI
 ParticleSet::ParticleSet(MPI_Comm comm_, int rank_num_particles, int dim,
-                         Ordering::Type coords_ordering)
+                         Ordering::Type coords_ordering, bool use_device)
    : ParticleSet(comm_, rank_num_particles, dim, coords_ordering, Array<int>(),
                  Array<Ordering::Type>(), Array<const char*>(), 0,
-                 Array<const char*>())
+                 Array<const char*>(), use_device)
 {
 
 };
 
 ParticleSet::ParticleSet(MPI_Comm comm_, int rank_num_particles, int dim,
                          const Array<int> &field_vdims, int num_tags,
-                         Ordering::Type all_ordering)
+                         Ordering::Type all_ordering, bool use_device)
    : ParticleSet(comm_, rank_num_particles, dim, all_ordering, field_vdims,
                  GetOrderingArray(all_ordering, field_vdims.Size()),
                  GetEmptyNameArray(field_vdims.Size()), num_tags,
-                 GetEmptyNameArray(num_tags))
+                 GetEmptyNameArray(num_tags), use_device)
 {
 
 }
@@ -650,11 +918,11 @@ ParticleSet::ParticleSet(MPI_Comm comm_, int rank_num_particles, int dim,
                          const Array<int> &field_vdims, const Array<const
                          char*> &field_names_,
                          int num_tags, const Array<const char*> &tag_names_,
-                         Ordering::Type all_ordering)
+                         Ordering::Type all_ordering, bool use_device)
    : ParticleSet(comm_, rank_num_particles, dim, all_ordering, field_vdims,
                  GetOrderingArray(all_ordering, field_vdims.Size()),
                  field_names_, num_tags,
-                 tag_names_)
+                 tag_names_, use_device)
 {
 
 }
@@ -664,7 +932,7 @@ ParticleSet::ParticleSet(MPI_Comm comm_, int rank_num_particles, int dim,
                          const Array<int> &field_vdims,
                          const Array<Ordering::Type> &field_orderings,
                          const Array<const char*> &field_names_, int num_tags,
-                         const Array<const char*> &tag_names_)
+                         const Array<const char*> &tag_names_, bool use_device)
    : ParticleSet(GetSize(comm_), (IDType)GetRank(comm_),
                  rank_num_particles,
                  dim,
@@ -673,7 +941,7 @@ ParticleSet::ParticleSet(MPI_Comm comm_, int rank_num_particles, int dim,
                  field_orderings,
                  field_names_,
                  num_tags,
-                 tag_names_)
+                 tag_names_, use_device)
 {
    comm = comm_;
 #ifdef MFEM_USE_GSLIB
@@ -705,6 +973,7 @@ int ParticleSet::AddField(int vdim, Ordering::Type field_ordering,
    }
    fields.emplace_back(std::make_unique<ParticleVector>(vdim, field_ordering,
                                                         GetNParticles()));
+   if (coords.UseDevice()) { fields.back()->UseDevice(true); }
    field_names.emplace_back(field_name_str);
 
    return GetNFields() - 1;
@@ -718,6 +987,7 @@ int ParticleSet::AddTag(const char* tag_name)
       tag_name_str = GetDefaultTagName(tag_names.size());
    }
    tags.emplace_back(std::make_unique<Array<int>>(GetNParticles()));
+   if (coords.UseDevice()) { tags.back()->GetMemory().UseDevice(true); }
    tag_names.emplace_back(tag_name_str);
 
    return GetNTags() - 1;
@@ -782,7 +1052,7 @@ Particle ParticleSet::GetParticle(int i) const
 
    for (int t = 0; t < GetNTags(); t++)
    {
-      p.Tag(t) = Tag(t)[i];
+      p.Tag(t) = Tag(t).HostRead()[i];
    }
 
    return p;
@@ -790,13 +1060,21 @@ Particle ParticleSet::GetParticle(int i) const
 
 bool ParticleSet::IsParticleRefValid() const
 {
-   if (coords.GetOrdering() == Ordering::byNODES)
+   if (coords.GetOrdering() == Ordering::byNODES || coords.UseDevice())
    {
       return false;
    }
    for (int f = 0; f < GetNFields(); f++)
    {
-      if (fields[f]->GetOrdering() == Ordering::byNODES)
+      if (fields[f]->GetOrdering() == Ordering::byNODES ||
+          fields[f]->UseDevice())
+      {
+         return false;
+      }
+   }
+   for (int t = 0; t < GetNTags(); t++)
+   {
+      if (tags[t]->UseDevice())
       {
          return false;
       }
@@ -806,6 +1084,10 @@ bool ParticleSet::IsParticleRefValid() const
 
 Particle ParticleSet::GetParticleRef(int i)
 {
+   MFEM_ASSERT(IsParticleRefValid(),
+               "GetParticleRef is only valid when coordinates and fields are "
+               "ordered byVDIM and particle data is host-resident.");
+
    Particle p = CreateParticle();
 
    Coords().GetValuesRef(i, p.Coords());
@@ -839,7 +1121,7 @@ void ParticleSet::SetParticle(int i, const Particle &p)
 
    for (int t = 0; t < GetNTags(); t++)
    {
-      Tag(t)[i] = p.Tag(t);
+      Tag(t).HostReadWrite()[i] = p.Tag(t);
    }
 }
 
@@ -900,6 +1182,15 @@ void ParticleSet::PrintCSV(const char *fname, const Array<int> &field_idxs,
 #ifdef MFEM_USE_MPI
    int rank = GetRank(comm);
 #endif // MFEM_USE_MPI
+   // make sure we can read tag data on host. fields and coords will be read as
+   // needed in the loop below, so we don't need to pre-read them here.
+   for (int i = 0; i < GetNTags(); i++)
+   {
+      tags[i]->HostRead();
+   }
+   ids.HostRead();
+
+   // Write particle data
    for (int i = 0; i < GetNParticles(); i++)
    {
       ss_data << ids[i];

fem/particleset.hpp

@@ -211,6 +211,12 @@ public:
  *  byVDIM). The unique_ptrs to all the ParticleVectors are stored in the
  *  std::vector \ref fields.
  *
+ *  @par Device Behavior:
+ *  When a ParticleSet is constructed with \p use_device=true, \ref coords and
+ *  all ParticleVector fields are marked to use device memory. Fields added
+ *  later through \ref AddField inherit the current device mode (through
+ *  \ref coords).
+ *
  *  @par Tags:
  *  Tags represent integers associated with each particle. For a given tag,
  *  all particle data are stored in a single Array<int>. The unique_ptrs to all
@@ -369,7 +375,10 @@ protected:
     *  ID of a particle.
     */
    void UpdateID(int local_idx, IDType new_global_id)
-   { ids[local_idx] = new_global_id; }
+   {
+      ids.HostReadWrite();
+      ids[local_idx] = new_global_id;
+   }
 
    /** @brief Create a Particle object with the same spatial dimension,
     *  number of fields and field vdims, and number of tags as this ParticleSet.
@@ -399,12 +408,14 @@ protected:
     *  @param[in] field_names_        Array of field names.
     *  @param[in] num_tags            Number of tags to register.
     *  @param[in] tag_names_          Array of tag names.
+    *  @param[in] use_device          Use device memory for particle fields.
     */
    ParticleSet(int id_stride_, IDType id_counter_, int num_particles, int dim,
                Ordering::Type coords_ordering, const Array<int> &field_vdims,
                const Array<Ordering::Type> &field_orderings,
                const Array<const char*> &field_names_, int num_tags,
-               const Array<const char*> &tag_names_);
+               const Array<const char*> &tag_names_,
+               bool use_device);
 
 public:
 
@@ -413,9 +424,12 @@ public:
     *  @param[in] num_particles       Number of particles to initialize.
     *  @param[in] dim                 Particle spatial dimension.
     *  @param[in] coords_ordering     Ordering of coordinates.
+    *  @param[in] use_device          (Optional) Use device memory for particle
+    *                                 fields.
     */
    ParticleSet(int num_particles, int dim,
-               Ordering::Type coords_ordering=Ordering::byVDIM);
+               Ordering::Type coords_ordering=Ordering::byVDIM,
+               bool use_device=false);
 
    /** @brief Construct a serial ParticleSet with specified fields and tags at
     *  construction.
@@ -426,9 +440,12 @@ public:
     *  @param[in] num_tags            Number of tags to register.
     *  @param[in] all_ordering        (Optional) Ordering of coordinates and
     *                                 field ParticleVector.
+    *  @param[in] use_device          (Optional) Use device memory for particle
+    *                                 fields.
     */
    ParticleSet(int num_particles, int dim, const Array<int> &field_vdims,
-               int num_tags, Ordering::Type all_ordering=Ordering::byVDIM);
+               int num_tags, Ordering::Type all_ordering=Ordering::byVDIM,
+               bool use_device=false);
 
    /** @brief Construct a serial ParticleSet with specified fields and tags at
     *  construction, with names.
@@ -441,11 +458,14 @@ public:
     *  @param[in] tag_names_          Array of tag names.
     *  @param[in] all_ordering        (Optional) Ordering of coordinates and
     *                                 field ParticleVector.
+    *  @param[in] use_device          (Optional) Use device memory for particle
+    *                                 fields.
     */
    ParticleSet(int num_particles, int dim, const Array<int> &field_vdims,
                const Array<const char*> &field_names_, int num_tags,
                const Array<const char*> &tag_names_,
-               Ordering::Type all_ordering=Ordering::byVDIM);
+               Ordering::Type all_ordering=Ordering::byVDIM,
+               bool use_device=false);
 
    /** @brief Comprehensive serial constructor of ParticleSet.
     *
@@ -457,12 +477,15 @@ public:
     *  @param[in] field_names_        Array of field names.
     *  @param[in] num_tags            Number of tags to register.
     *  @param[in] tag_names_          Array of tag names.
+    *  @param[in] use_device          (Optional) Use device memory for particle
+    *                                 fields.
     */
    ParticleSet(int num_particles, int dim, Ordering::Type coords_ordering,
                const Array<int> &field_vdims,
                const Array<Ordering::Type> &field_orderings,
                const Array<const char*> &field_names_, int num_tags,
-               const Array<const char*> &tag_names_);
+               const Array<const char*> &tag_names_,
+               bool use_device=false);
 
 #ifdef MFEM_USE_MPI
    /** @brief Construct a parallel ParticleSet.
@@ -471,9 +494,12 @@ public:
     *  @param[in] rank_num_particles  Number of particles to initialize.
     *  @param[in] dim                 Particle spatial dimension.
     *  @param[in] coords_ordering     (Optional) Ordering of coordinates.
+    *  @param[in] use_device          (Optional) Use device memory for particle
+    *                                 fields.
     */
    ParticleSet(MPI_Comm comm_, int rank_num_particles, int dim,
-               Ordering::Type coords_ordering=Ordering::byVDIM);
+               Ordering::Type coords_ordering=Ordering::byVDIM,
+               bool use_device=false);
 
    /** @brief Construct a parallel ParticleSet with specified fields and tags
     *  at construction.
@@ -485,10 +511,13 @@ public:
     *  @param[in] num_tags            Number of tags to register.
     *  @param[in] all_ordering        (Optional) Ordering of coordinates and
     *                                 field ParticleVector.
+    *  @param[in] use_device          (Optional) Use device memory for particle
+    *                                 fields.
     */
    ParticleSet(MPI_Comm comm_, int rank_num_particles, int dim,
                const Array<int> &field_vdims, int num_tags,
-               Ordering::Type all_ordering=Ordering::byVDIM);
+               Ordering::Type all_ordering=Ordering::byVDIM,
+               bool use_device=false);
 
    /** @brief Construct a parallel ParticleSet with specified fields and tags
     *  at construction, with names (for PrintCSV()).
@@ -502,12 +531,15 @@ public:
     *  @param[in] tag_names_          Array of tag names.
     *  @param[in] all_ordering        (Optional) Ordering of coordinates and
     *                                 field ParticleVector.
+    *  @param[in] use_device          (Optional) Use device memory for particle
+    *                                 fields.
     */
    ParticleSet(MPI_Comm comm_, int rank_num_particles, int dim,
                const Array<int> &field_vdims,
                const Array<const char*> &field_names_,
                int num_tags, const Array<const char*> &tag_names_,
-               Ordering::Type all_ordering=Ordering::byVDIM);
+               Ordering::Type all_ordering=Ordering::byVDIM,
+               bool use_device=false);
 
    /** @brief Comprehensive parallel constructor of ParticleSet.
     *
@@ -520,12 +552,15 @@ public:
     *  @param[in] field_names_        Array of field names.
     *  @param[in] num_tags            Number of tags to register.
     *  @param[in] tag_names_          Array of tag names.
+    *  @param[in] use_device          (Optional) Use device memory for particle
+    *                                 fields.
     */
    ParticleSet(MPI_Comm comm_, int rank_num_particles, int dim,
                Ordering::Type coords_ordering, const Array<int> &field_vdims,
                const Array<Ordering::Type> &field_orderings,
                const Array<const char*> &field_names_, int num_tags,
-               const Array<const char*> &tag_names_);
+               const Array<const char*> &tag_names_,
+               bool use_device=false);
 
    /// Get the MPI communicator for this ParticleSet.
    MPI_Comm GetComm() const { return comm; };
@@ -545,6 +580,8 @@ public:
     *  @param[in] field_ordering   (Optional) Ordering::Type of the field.
     *  @param[in] field_name       (Optional) Name of the field.
     *
+    *  @note New fields inherit the current device mode of \ref coords.
+    *
     *  @return Index of the newly-added field.
     */
    int AddField(int vdim, Ordering::Type field_ordering=Ordering::byVDIM,
@@ -637,8 +674,8 @@ public:
 
    /** @brief Determine if GetParticleRef is valid.
     *
-    * If coordinates and all fields are ordered byVDIM, then returns true.
-    * Otherwise, false.
+    * Returns true when coordinates and all fields are ordered byVDIM and
+    * particle data is host-resident. Otherwise, false.
     */
    bool IsParticleRefValid() const;
 

fem/qinterp/det.cpp

@@ -21,23 +21,24 @@ namespace quadrature_interpolator
 
 void InitDetKernels()
 {
+   using k = QuadratureInterpolator::DetKernels;
    // 2D
-   QuadratureInterpolator::AddDetSpecializations<2,2,2,2>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,2,3>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,2,4>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,2,6>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,3,4>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,3,6>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,4,4>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,4,6>();
-   QuadratureInterpolator::AddDetSpecializations<2,2,5,6>();
+   k::Specialization<2,2,2,2>::Add();
+   k::Specialization<2,2,2,3>::Add();
+   k::Specialization<2,2,2,4>::Add();
+   k::Specialization<2,2,2,6>::Add();
+   k::Specialization<2,2,3,4>::Add();
+   k::Specialization<2,2,3,6>::Add();
+   k::Specialization<2,2,4,4>::Add();
+   k::Specialization<2,2,4,6>::Add();
+   k::Specialization<2,2,5,6>::Add();
    // 3D
-   QuadratureInterpolator::AddDetSpecializations<3,3,2,4>();
-   QuadratureInterpolator::AddDetSpecializations<3,3,3,3>();
-   QuadratureInterpolator::AddDetSpecializations<3,3,3,5>();
-   QuadratureInterpolator::AddDetSpecializations<3,3,3,6>();
-   QuadratureInterpolator::AddDetSpecializations<3,3,4,6>();
-   QuadratureInterpolator::AddDetSpecializations<3,3,3,4>();
+   k::Specialization<3,3,2,4>::Add();
+   k::Specialization<3,3,3,3>::Add();
+   k::Specialization<3,3,3,5>::Add();
+   k::Specialization<3,3,3,6>::Add();
+   k::Specialization<3,3,4,6>::Add();
+   k::Specialization<3,3,3,4>::Add();
 }
 
 } // namespace quadrature_interpolator
@@ -46,53 +47,26 @@ void InitDetKernels()
 /// @cond Suppress_Doxygen_warnings
 
 QuadratureInterpolator::DetKernelType
-QuadratureInterpolator::DetKernels::Fallback(int DIM, int SDIM, int D1D,
-                                             int Q1D)
+QuadratureInterpolator::DetKernels::Fallback(
+   int DIM, int SDIM, int D1D, int Q1D)
 {
    if (DIM == 1)
    {
-      if (SDIM == 1)
-      {
-         return internal::quadrature_interpolator::Det1D;
-      }
-      else if (SDIM == 2)
-      {
-         return internal::quadrature_interpolator::Det1DSurface<0, 0, 2>;
-      }
-      else if (SDIM == 3)
-      {
-         return internal::quadrature_interpolator::Det1DSurface<0, 0, 3>;
-      }
-      else
-      {
-         MFEM_ABORT("");
-      }
-   }
-   else if (DIM == 2 && SDIM == 2)
-   {
-      return internal::quadrature_interpolator::Det2D<>;
-   }
-   else if (DIM == 2 && SDIM == 3)
-   {
-      return internal::quadrature_interpolator::Det2DSurface<>;
+      if (SDIM == 1) { return internal::quadrature_interpolator::Det1D; }
+      else if (SDIM == 2) { return internal::quadrature_interpolator::Det1DSurface<0,0,2>; }
+      else if (SDIM == 3) { return internal::quadrature_interpolator::Det1DSurface<0,0,3>; }
+      else { MFEM_ABORT(""); }
    }
+   else if (DIM == 2 && SDIM == 2) { return internal::quadrature_interpolator::Det2D; }
+   else if (DIM == 2 && SDIM == 3) { return internal::quadrature_interpolator::Det2DSurface; }
    else if (DIM == 3)
    {
       const int MD = DeviceDofQuadLimits::Get().MAX_DET_1D;
       const int MQ = DeviceDofQuadLimits::Get().MAX_DET_1D;
-      if (D1D <= MD && Q1D <= MQ)
-      {
-         return internal::quadrature_interpolator::Det3D<0, 0, true>;
-      }
-      else
-      {
-         return internal::quadrature_interpolator::Det3D<0, 0, false>;
-      }
-   }
-   else
-   {
-      MFEM_ABORT("");
+      if (D1D <= MD && Q1D <= MQ) { return internal::quadrature_interpolator::Det3D<0,0,true>; }
+      else { return internal::quadrature_interpolator::Det3D<0,0,false>; }
    }
+   else { MFEM_ABORT(""); }
 }
 
 /// @endcond

fem/qinterp/eval.hpp

@@ -30,18 +30,23 @@ namespace internal
 namespace quadrature_interpolator
 {
 
-template <QVectorLayout Q_LAYOUT, bool Integral>
-static void IntValues1D(const int NE, const real_t *b_, const real_t *detJ_,
-                        const real_t *x_, real_t *y_, const int vdim,
-                        const int d1d, const int q1d)
+template<QVectorLayout Q_LAYOUT>
+static void Values1D(const int NE,
+                     const real_t *b_,
+                     const real_t *x_,
+                     real_t *y_,
+                     const int vdim,
+                     const int d1d,
+                     const int q1d)
 {
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
+   const auto b = Reshape(b_, q1d, d1d);
+   const auto x = Reshape(x_, d1d, vdim, NE);
+   auto y = Q_LAYOUT == QVectorLayout::byNODES ?
+            Reshape(y_, q1d, vdim, NE):
+            Reshape(y_, vdim, q1d, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      const auto b = Reshape(b_, q1d, d1d);
-      const auto x = Reshape(x_, d1d, vdim, NE);
-      const auto detJ = Reshape(detJ_, q1d, NE);
-      auto y = Q_LAYOUT == QVectorLayout::byNODES ? Reshape(y_, q1d, vdim, NE)
-               : Reshape(y_, vdim, q1d, NE);
       for (int c = 0; c < vdim; c++)
       {
          for (int q = 0; q < q1d; q++)
@@ -51,36 +56,24 @@ static void IntValues1D(const int NE, const real_t *b_, const real_t *detJ_,
             {
                u += b(q, d) * x(d, c, e);
             }
-            if constexpr (Integral)
-            {
-               u /= detJ(q, e);
-            }
-            if constexpr (Q_LAYOUT == QVectorLayout::byVDIM)
-            {
-               y(c, q, e) = u;
-            }
-            if constexpr (Q_LAYOUT == QVectorLayout::byNODES)
-            {
-               y(q, c, e) = u;
-            }
+            if (Q_LAYOUT == QVectorLayout::byVDIM)  { y(c, q, e) = u; }
+            if (Q_LAYOUT == QVectorLayout::byNODES) { y(q, c, e) = u; }
          }
       }
    });
 }
 
-template <QVectorLayout Q_LAYOUT>
-static void Values1D(const int NE, const real_t *b_, const real_t *x_,
-                     real_t *y_, const int vdim, const int d1d, const int q1d)
-{
-   IntValues1D<Q_LAYOUT, false>(NE, b_, nullptr, x_, y_, vdim, d1d, q1d);
-}
-
 // Template compute kernel for Values in 2D: tensor product version.
-template <QVectorLayout Q_LAYOUT, bool Integral, int T_VDIM = 0, int T_D1D = 0,
-          int T_Q1D = 0, int T_NBZ = 1>
-static void IntValues2D(const int NE, const real_t *b_, const real_t *detJ_,
-                        const real_t *x_, real_t *y_, const int vdim = 0,
-                        const int d1d = 0, const int q1d = 0)
+template<QVectorLayout Q_LAYOUT,
+         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
+         int T_NBZ = 1>
+static void Values2D(const int NE,
+                     const real_t *b_,
+                     const real_t *x_,
+                     real_t *y_,
+                     const int vdim = 0,
+                     const int d1d = 0,
+                     const int q1d = 0)
 {
    static constexpr int NBZ = T_NBZ ? T_NBZ : 1;
 
@@ -89,14 +82,13 @@ static void IntValues2D(const int NE, const real_t *b_, const real_t *detJ_,
    const int VDIM = T_VDIM ? T_VDIM : vdim;
 
    const auto b = Reshape(b_, Q1D, D1D);
+   const auto x = Reshape(x_, D1D, D1D, VDIM, NE);
+   auto y = Q_LAYOUT == QVectorLayout::byNODES ?
+            Reshape(y_, Q1D, Q1D, VDIM, NE):
+            Reshape(y_, VDIM, Q1D, Q1D, NE);
 
-   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE(int e)
+   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int e)
    {
-      const auto x = Reshape(x_, D1D, D1D, VDIM, NE);
-      const auto detJ = Reshape(detJ_, Q1D, Q1D, NE);
-      auto y = Q_LAYOUT == QVectorLayout::byNODES
-               ? Reshape(y_, Q1D, Q1D, VDIM, NE)
-               : Reshape(y_, VDIM, Q1D, Q1D, NE);
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       const int VDIM = T_VDIM ? T_VDIM : vdim;
@@ -118,33 +110,16 @@ static void IntValues2D(const int NE, const real_t *b_, const real_t *detJ_,
 
       for (int c = 0; c < VDIM; c++)
       {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx, x, D1D)
-            {
-               DD(dx, dy) = x(dx, dy, c, e);
-            }
-         }
-         MFEM_SYNC_THREAD;
+         kernels::internal::LoadX(e,D1D,c,x,DD);
          kernels::internal::EvalX(D1D,Q1D,B,DD,DQ);
          kernels::internal::EvalY(D1D,Q1D,B,DQ,QQ);
          MFEM_FOREACH_THREAD(qy,y,Q1D)
          {
             MFEM_FOREACH_THREAD(qx,x,Q1D)
             {
-               real_t u = QQ(qx, qy);
-               if constexpr (Integral)
-               {
-                  u /= detJ(qx, qy, e);
-               }
-               if constexpr (Q_LAYOUT == QVectorLayout::byVDIM)
-               {
-                  y(c, qx, qy, e) = u;
-               }
-               if constexpr (Q_LAYOUT == QVectorLayout::byNODES)
-               {
-                  y(qx, qy, c, e) = u;
-               }
+               real_t u = QQ(qx,qy);
+               if (Q_LAYOUT == QVectorLayout::byVDIM) { y(c,qx,qy,e) = u; }
+               if (Q_LAYOUT == QVectorLayout::byNODES) { y(qx,qy,c,e) = u; }
             }
          }
          MFEM_SYNC_THREAD;
@@ -152,37 +127,29 @@ static void IntValues2D(const int NE, const real_t *b_, const real_t *detJ_,
    });
 }
 
-// Template compute kernel for Values in 2D: tensor product version.
-template <QVectorLayout Q_LAYOUT, int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0,
-          int T_NBZ = 1>
-static void Values2D(const int NE, const real_t *b_, const real_t *x_,
-                     real_t *y_, const int vdim = 0, const int d1d = 0,
-                     const int q1d = 0)
-{
-   return IntValues2D<Q_LAYOUT, false, T_VDIM, T_D1D, T_Q1D, T_NBZ>(
-             NE, b_, nullptr, x_, y_, vdim, d1d, q1d);
-}
-
 // Template compute kernel for Values in 3D: tensor product version.
-template <QVectorLayout Q_LAYOUT, bool Integral, int T_VDIM = 0, int T_D1D = 0,
-          int T_Q1D = 0>
-static void IntValues3D(const int NE, const real_t *b_, const real_t *detJ_,
-                        const real_t *x_, real_t *y_, const int vdim = 0,
-                        const int d1d = 0, const int q1d = 0)
+template<QVectorLayout Q_LAYOUT,
+         int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0>
+static void Values3D(const int NE,
+                     const real_t *b_,
+                     const real_t *x_,
+                     real_t *y_,
+                     const int vdim = 0,
+                     const int d1d = 0,
+                     const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
    const int VDIM = T_VDIM ? T_VDIM : vdim;
 
    const auto b = Reshape(b_, Q1D, D1D);
+   const auto x = Reshape(x_, D1D, D1D, D1D, VDIM, NE);
+   auto y = Q_LAYOUT == QVectorLayout:: byNODES ?
+            Reshape(y_, Q1D, Q1D, Q1D, VDIM, NE):
+            Reshape(y_, VDIM, Q1D, Q1D, Q1D, NE);
 
-   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
    {
-      const auto x = Reshape(x_, D1D, D1D, D1D, VDIM, NE);
-      const auto detJ = Reshape(detJ_, Q1D, Q1D, Q1D, NE);
-      auto y = Q_LAYOUT == QVectorLayout::byNODES
-               ? Reshape(y_, Q1D, Q1D, Q1D, VDIM, NE)
-               : Reshape(y_, VDIM, Q1D, Q1D, Q1D, NE);
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       const int VDIM = T_VDIM ? T_VDIM : vdim;
@@ -204,17 +171,7 @@ static void IntValues3D(const int NE, const real_t *b_, const real_t *detJ_,
 
       for (int c = 0; c < VDIM; c++)
       {
-         MFEM_FOREACH_THREAD(dz, z, D1D)
-         {
-            MFEM_FOREACH_THREAD(dy, y, D1D)
-            {
-               MFEM_FOREACH_THREAD(dx, x, D1D)
-               {
-                  DDD(dx, dy, dz) = x(dx, dy, dz, c, e);
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
+         kernels::internal::LoadX(e,D1D,c,x,DDD);
          kernels::internal::EvalX(D1D,Q1D,B,DDD,DDQ);
          kernels::internal::EvalY(D1D,Q1D,B,DDQ,DQQ);
          kernels::internal::EvalZ(D1D,Q1D,B,DQQ,QQQ);
@@ -224,19 +181,9 @@ static void IntValues3D(const int NE, const real_t *b_, const real_t *detJ_,
             {
                MFEM_FOREACH_THREAD(qx,x,Q1D)
                {
-                  real_t u = QQQ(qz,qy,qx);
-                  if constexpr (Integral)
-                  {
-                     u /= detJ(qx, qy, qz, e);
-                  }
-                  if constexpr (Q_LAYOUT == QVectorLayout::byVDIM)
-                  {
-                     y(c, qx, qy, qz, e) = u;
-                  }
-                  if constexpr (Q_LAYOUT == QVectorLayout::byNODES)
-                  {
-                     y(qx, qy, qz, c, e) = u;
-                  }
+                  const real_t u = QQQ(qz,qy,qx);
+                  if (Q_LAYOUT == QVectorLayout::byVDIM) { y(c,qx,qy,qz,e) = u; }
+                  if (Q_LAYOUT == QVectorLayout::byNODES) { y(qx,qy,qz,c,e) = u; }
                }
             }
          }
@@ -245,417 +192,14 @@ static void IntValues3D(const int NE, const real_t *b_, const real_t *detJ_,
    });
 }
 
-// Template compute kernel for Values in 3D: tensor product version.
-template <QVectorLayout Q_LAYOUT, int T_VDIM = 0, int T_D1D = 0, int T_Q1D = 0>
-static void Values3D(const int NE, const real_t *b_, const real_t *x_,
-                     real_t *y_, const int vdim = 0, const int d1d = 0,
-                     const int q1d = 0)
-{
-   return IntValues3D<Q_LAYOUT, false, T_VDIM, T_D1D, T_Q1D>(
-             NE, b_, nullptr, x_, y_, vdim, d1d, q1d);
-}
-
-template <bool Integral>
-void IntEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
-               const real_t *detJ, const GeometricFactors *geom,
-               const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
-               Vector &q_der, Vector &q_det, const int eval_flags);
-
-inline void Eval1D(const int NE, const int vdim, const QVectorLayout q_layout,
-                   const GeometricFactors *geom, const DofToQuad &maps,
-                   const Vector &e_vec, Vector &q_val, Vector &q_der, Vector &q_det,
-                   const int eval_flags)
-{
-   IntEval1D<false>(NE, vdim, q_layout, nullptr, geom, maps, e_vec, q_val,
-                    q_der, q_det, eval_flags);
-}
-
-// Template compute kernel for 2D quadrature interpolation:
-// * non-tensor product version,
-// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
-// * assumes 'maps.mode == FULL'.
-template <bool Integral, const int T_VDIM, const int T_ND, const int T_NQ>
-static void
-IntEval2D(const int NE, const int vdim, const QVectorLayout q_layout,
-          const real_t *detJ_, const GeometricFactors *geom,
-          const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
-          Vector &q_der, Vector &q_det, const int eval_flags)
-{
-   using QI = QuadratureInterpolator;
-
-   const int nd = maps.ndof;
-   const int nq = maps.nqpt;
-   const int ND = T_ND ? T_ND : nd;
-   const int NQ = T_NQ ? T_NQ : nq;
-   const int NMAX = NQ > ND ? NQ : ND;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-   MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
-   MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 2, "");
-   MFEM_VERIFY(ND <= QI::MAX_ND2D, "");
-   MFEM_VERIFY(NQ <= QI::MAX_NQ2D, "");
-   const auto B = Reshape(maps.B.Read(), NQ, ND);
-   const auto G = Reshape(maps.G.Read(), NQ, 2, ND);
-   const auto J = Reshape(geom ? geom->J.Read() : nullptr, NQ, 2, 2, NE);
-   const auto E_ = e_vec.Read();
-   auto val = q_layout == QVectorLayout::byNODES ?
-              Reshape(q_val.Write(), NQ, VDIM, NE):
-              Reshape(q_val.Write(), VDIM, NQ, NE);
-   auto der = q_layout == QVectorLayout::byNODES ?
-              Reshape(q_der.Write(), NQ, VDIM, 2, NE):
-              Reshape(q_der.Write(), VDIM, 2, NQ, NE);
-   auto det = Reshape(q_det.Write(), NQ, NE);
-   mfem::forall_2D(NE, NMAX, 1, [=] MFEM_HOST_DEVICE(int e)
-   {
-      const auto E = Reshape(E_, ND, VDIM, NE);
-      const auto detJ = Reshape(detJ_, NQ, NE);
-      const int ND = T_ND ? T_ND : nd;
-      const int NQ = T_NQ ? T_NQ : nq;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int max_ND = T_ND ? T_ND : QI::MAX_ND2D;
-      constexpr int max_VDIM = T_VDIM ? T_VDIM : QI::MAX_VDIM2D;
-      MFEM_SHARED real_t s_E[max_VDIM*max_ND];
-      MFEM_FOREACH_THREAD(d, x, ND)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            s_E[c + d * VDIM] = E(d, c, e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(q, x, NQ)
-      {
-         if (eval_flags & (QI::VALUES | QI::PHYSICAL_VALUES))
-         {
-            real_t ed[max_VDIM];
-            for (int c = 0; c < VDIM; c++)
-            {
-               ed[c] = 0.0;
-            }
-            for (int d = 0; d < ND; ++d)
-            {
-               const real_t b = B(q,d);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  ed[c] += b * s_E[c + d * VDIM];
-               }
-            }
-            for (int c = 0; c < VDIM; c++)
-            {
-               if constexpr (Integral)
-               {
-                  ed[c] /= detJ(q, e);
-               }
-               if (q_layout == QVectorLayout::byVDIM)
-               {
-                  val(c, q, e) = ed[c];
-               }
-               if (q_layout == QVectorLayout::byNODES)
-               {
-                  val(q, c, e) = ed[c];
-               }
-            }
-         }
-         if ((eval_flags & QI::DERIVATIVES) ||
-             (eval_flags & QI::PHYSICAL_DERIVATIVES) ||
-             (eval_flags & QI::DETERMINANTS))
-         {
-            // use MAX_VDIM2D to avoid "subscript out of range" warnings
-            real_t D[QI::MAX_VDIM2D*2];
-            for (int i = 0; i < 2*VDIM; i++)
-            {
-               D[i] = 0.0;
-            }
-            for (int d = 0; d < ND; ++d)
-            {
-               const real_t wx = G(q,0,d);
-               const real_t wy = G(q,1,d);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  real_t s_e = s_E[c+d*VDIM];
-                  D[c+VDIM*0] += s_e * wx;
-                  D[c+VDIM*1] += s_e * wy;
-               }
-            }
-            if (eval_flags & QI::DERIVATIVES)
-            {
-               for (int c = 0; c < VDIM; c++)
-               {
-                  if (q_layout == QVectorLayout::byVDIM)
-                  {
-                     der(c,0,q,e) = D[c+VDIM*0];
-                     der(c,1,q,e) = D[c+VDIM*1];
-                  }
-                  if (q_layout == QVectorLayout::byNODES)
-                  {
-                     der(q,c,0,e) = D[c+VDIM*0];
-                     der(q,c,1,e) = D[c+VDIM*1];
-                  }
-               }
-            }
-            if (eval_flags & QI::PHYSICAL_DERIVATIVES)
-            {
-               real_t Jloc[4], Jinv[4];
-               Jloc[0] = J(q,0,0,e);
-               Jloc[1] = J(q,1,0,e);
-               Jloc[2] = J(q,0,1,e);
-               Jloc[3] = J(q,1,1,e);
-               kernels::CalcInverse<2>(Jloc, Jinv);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  const real_t u = D[c+VDIM*0];
-                  const real_t v = D[c+VDIM*1];
-                  const real_t JiU = Jinv[0]*u + Jinv[1]*v;
-                  const real_t JiV = Jinv[2]*u + Jinv[3]*v;
-                  if (q_layout == QVectorLayout::byVDIM)
-                  {
-                     der(c,0,q,e) = JiU;
-                     der(c,1,q,e) = JiV;
-                  }
-                  if (q_layout == QVectorLayout::byNODES)
-                  {
-                     der(q,c,0,e) = JiU;
-                     der(q,c,1,e) = JiV;
-                  }
-               }
-            }
-            if (eval_flags & QI::DETERMINANTS)
-            {
-               if (VDIM == 2)
-               {
-                  det(q, e) = kernels::Det<2>(D);
-               }
-               else
-               {
-                  DeviceTensor<2> j(D, 3, 2);
-                  const real_t dE = j(0,0)*j(0,0) + j(1,0)*j(1,0) + j(2,0)*j(2,0);
-                  const real_t dF = j(0,0)*j(0,1) + j(1,0)*j(1,1) + j(2,0)*j(2,1);
-                  const real_t dG = j(0,1)*j(0,1) + j(1,1)*j(1,1) + j(2,1)*j(2,1);
-                  det(q,e) = std::sqrt(dE*dG - dF*dF);
-               }
-            }
-         }
-      }
-   });
-}
-
-// Template compute kernel for 2D quadrature interpolation:
-// * non-tensor product version,
-// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
-// * assumes 'maps.mode == FULL'.
-template <const int T_VDIM, const int T_ND, const int T_NQ>
-static void Eval2D(const int NE, const int vdim, const QVectorLayout q_layout,
-                   const GeometricFactors *geom, const DofToQuad &maps,
-                   const Vector &e_vec, Vector &q_val, Vector &q_der,
-                   Vector &q_det, const int eval_flags)
-{
-   IntEval2D<false, T_VDIM, T_ND, T_NQ>(NE, vdim, q_layout, nullptr, geom, maps,
-                                        e_vec, q_val, q_der, q_det, eval_flags);
-}
-
-// Template compute kernel for 3D quadrature interpolation:
-// * non-tensor product version,
-// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
-// * assumes 'maps.mode == FULL'.
-template <bool Integral, const int T_VDIM, const int T_ND, const int T_NQ>
-static void
-IntEval3D(const int NE, const int vdim, const QVectorLayout q_layout,
-          const real_t *detJ_, const GeometricFactors *geom,
-          const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
-          Vector &q_der, Vector &q_det, const int eval_flags)
-{
-   using QI = QuadratureInterpolator;
-
-   const int nd = maps.ndof;
-   const int nq = maps.nqpt;
-   const int ND = T_ND ? T_ND : nd;
-   const int NQ = T_NQ ? T_NQ : nq;
-   const int NMAX = NQ > ND ? NQ : ND;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-   MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
-   MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 3, "");
-   MFEM_VERIFY(ND <= QI::MAX_ND3D, "");
-   MFEM_VERIFY(NQ <= QI::MAX_NQ3D, "");
-   MFEM_VERIFY(VDIM == 3 || !(eval_flags & QI::DETERMINANTS), "");
-   const auto B = Reshape(maps.B.Read(), NQ, ND);
-   const auto G = Reshape(maps.G.Read(), NQ, 3, ND);
-   const auto J = Reshape(geom ? geom->J.Read() : nullptr, NQ, 3, 3, NE);
-   auto E_ = e_vec.Read();
-   auto val = q_layout == QVectorLayout::byNODES ?
-              Reshape(q_val.Write(), NQ, VDIM, NE):
-              Reshape(q_val.Write(), VDIM, NQ, NE);
-   auto der = q_layout == QVectorLayout::byNODES ?
-              Reshape(q_der.Write(), NQ, VDIM, 3, NE):
-              Reshape(q_der.Write(), VDIM, 3, NQ, NE);
-   auto det = Reshape(q_det.Write(), NQ, NE);
-   mfem::forall_2D(NE, NMAX, 1, [=] MFEM_HOST_DEVICE(int e)
-   {
-      const auto E = Reshape(E_, ND, VDIM, NE);
-      const auto detJ = Reshape(detJ_, NQ, NE);
-      const int ND = T_ND ? T_ND : nd;
-      const int NQ = T_NQ ? T_NQ : nq;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int max_ND = T_ND ? T_ND : QI::MAX_ND3D;
-      constexpr int max_VDIM = T_VDIM ? T_VDIM : QI::MAX_VDIM3D;
-      MFEM_SHARED real_t s_E[max_VDIM*max_ND];
-      MFEM_FOREACH_THREAD(d, x, ND)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            s_E[c + d * VDIM] = E(d, c, e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(q, x, NQ)
-      {
-         if (eval_flags & (QI::VALUES | QI::PHYSICAL_VALUES))
-         {
-            real_t ed[max_VDIM];
-            for (int c = 0; c < VDIM; c++)
-            {
-               ed[c] = 0.0;
-            }
-            for (int d = 0; d < ND; ++d)
-            {
-               const real_t b = B(q,d);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  ed[c] += b * s_E[c + d * VDIM];
-               }
-            }
-            for (int c = 0; c < VDIM; c++)
-            {
-               if constexpr (Integral)
-               {
-                  ed[c] /= detJ(q, e);
-               }
-               if (q_layout == QVectorLayout::byVDIM)
-               {
-                  val(c, q, e) = ed[c];
-               }
-               if (q_layout == QVectorLayout::byNODES)
-               {
-                  val(q, c, e) = ed[c];
-               }
-            }
-         }
-         if ((eval_flags & QI::DERIVATIVES) ||
-             (eval_flags & QI::PHYSICAL_DERIVATIVES) ||
-             (eval_flags & QI::DETERMINANTS))
-         {
-            // use MAX_VDIM3D to avoid "subscript out of range" warnings
-            real_t D[QI::MAX_VDIM3D*3];
-            for (int i = 0; i < 3*VDIM; i++)
-            {
-               D[i] = 0.0;
-            }
-            for (int d = 0; d < ND; ++d)
-            {
-               const real_t wx = G(q,0,d);
-               const real_t wy = G(q,1,d);
-               const real_t wz = G(q,2,d);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  real_t s_e = s_E[c+d*VDIM];
-                  D[c+VDIM*0] += s_e * wx;
-                  D[c+VDIM*1] += s_e * wy;
-                  D[c+VDIM*2] += s_e * wz;
-               }
-            }
-            if (eval_flags & QI::DERIVATIVES)
-            {
-               for (int c = 0; c < VDIM; c++)
-               {
-                  if (q_layout == QVectorLayout::byVDIM)
-                  {
-                     der(c,0,q,e) = D[c+VDIM*0];
-                     der(c,1,q,e) = D[c+VDIM*1];
-                     der(c,2,q,e) = D[c+VDIM*2];
-                  }
-                  if (q_layout == QVectorLayout::byNODES)
-                  {
-                     der(q,c,0,e) = D[c+VDIM*0];
-                     der(q,c,1,e) = D[c+VDIM*1];
-                     der(q,c,2,e) = D[c+VDIM*2];
-                  }
-               }
-            }
-            if (eval_flags & QI::PHYSICAL_DERIVATIVES)
-            {
-               real_t Jloc[9], Jinv[9];
-               for (int col = 0; col < 3; col++)
-               {
-                  for (int row = 0; row < 3; row++)
-                  {
-                     Jloc[row+3*col] = J(q,row,col,e);
-                  }
-               }
-               kernels::CalcInverse<3>(Jloc, Jinv);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  const real_t u = D[c+VDIM*0];
-                  const real_t v = D[c+VDIM*1];
-                  const real_t w = D[c+VDIM*2];
-                  const real_t JiU = Jinv[0]*u + Jinv[1]*v + Jinv[2]*w;
-                  const real_t JiV = Jinv[3]*u + Jinv[4]*v + Jinv[5]*w;
-                  const real_t JiW = Jinv[6]*u + Jinv[7]*v + Jinv[8]*w;
-                  if (q_layout == QVectorLayout::byVDIM)
-                  {
-                     der(c,0,q,e) = JiU;
-                     der(c,1,q,e) = JiV;
-                     der(c,2,q,e) = JiW;
-                  }
-                  if (q_layout == QVectorLayout::byNODES)
-                  {
-                     der(q,c,0,e) = JiU;
-                     der(q,c,1,e) = JiV;
-                     der(q,c,2,e) = JiW;
-                  }
-               }
-            }
-            if (VDIM == 3 && (eval_flags & QI::DETERMINANTS))
-            {
-               // The check (VDIM == 3) should eliminate this block when VDIM is
-               // known at compile time and (VDIM != 3).
-               det(q,e) = kernels::Det<3>(D);
-            }
-         }
-      }
-   });
-}
-
-// Template compute kernel for 3D quadrature interpolation:
-// * non-tensor product version,
-// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
-// * assumes 'maps.mode == FULL'.
-template <const int T_VDIM, const int T_ND, const int T_NQ>
-static void Eval3D(const int NE, const int vdim, const QVectorLayout q_layout,
-                   const GeometricFactors *geom, const DofToQuad &maps,
-                   const Vector &e_vec, Vector &q_val, Vector &q_der,
-                   Vector &q_det, const int eval_flags)
-{
-   IntEval3D<false, T_VDIM, T_ND, T_NQ>(NE, vdim, q_layout, nullptr, geom, maps,
-                                        e_vec, q_val, q_der, q_det, eval_flags);
-}
-
 } // namespace quadrature_interpolator
 
 } // namespace internal
 
 /// @cond Suppress_Doxygen_warnings
 
-template <int DIM, QVectorLayout Q_LAYOUT, int VDIM, int D1D, int Q1D, int NBZ>
-QuadratureInterpolator::IntTensorEvalKernelType
-QuadratureInterpolator::IntTensorEvalKernels::Kernel()
-{
-   if constexpr (DIM == 1) { return internal::quadrature_interpolator::IntValues1D<Q_LAYOUT, true>; }
-   else if constexpr (DIM == 2) { return internal::quadrature_interpolator::IntValues2D<Q_LAYOUT, true, VDIM, D1D, Q1D, NBZ>; }
-   else if constexpr (DIM == 3) { return internal::quadrature_interpolator::IntValues3D<Q_LAYOUT, true, VDIM, D1D, Q1D>; }
-   MFEM_ABORT("");
-}
-
-template <int DIM, QVectorLayout Q_LAYOUT, int VDIM, int D1D, int Q1D, int NBZ>
+template<int DIM, QVectorLayout Q_LAYOUT,
+         int VDIM, int D1D, int Q1D, int NBZ>
 QuadratureInterpolator::TensorEvalKernelType
 QuadratureInterpolator::TensorEvalKernels::Kernel()
 {
@@ -665,28 +209,6 @@ QuadratureInterpolator::TensorEvalKernels::Kernel()
    MFEM_ABORT("");
 }
 
-template <int DIM, int VDIM, int ND, int NQ>
-QuadratureInterpolator::IntEvalKernelType
-QuadratureInterpolator::IntEvalKernels::Kernel()
-{
-   using namespace internal::quadrature_interpolator;
-   if constexpr (DIM == 1) { return IntEval1D<true>; }
-   else if constexpr (DIM == 2) { return IntEval2D<true,VDIM,ND,NQ>; }
-   else if constexpr (DIM == 3) { return IntEval3D<true,VDIM,ND,NQ>; }
-   MFEM_ABORT("");
-}
-
-template <int DIM, int VDIM, int ND, int NQ>
-QuadratureInterpolator::EvalKernelType
-QuadratureInterpolator::EvalKernels::Kernel()
-{
-   using namespace internal::quadrature_interpolator;
-   if constexpr (DIM == 1) { return Eval1D; }
-   else if constexpr (DIM == 2) { return Eval2D<VDIM,ND,NQ>; }
-   else if constexpr (DIM == 3) { return Eval3D<VDIM,ND,NQ>; }
-   MFEM_ABORT("");
-}
-
 /// @endcond
 
 } // namespace mfem

fem/qinterp/eval_by_nodes.cpp

@@ -21,105 +21,63 @@ namespace quadrature_interpolator
 
 void InitEvalByNodesKernels()
 {
+   using k = QuadratureInterpolator::TensorEvalKernels;
+
    // 2D
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 3, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 2, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 3, 2, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 3, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 3, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 4, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 1, 4, 4, 1>();
+   k::Specialization<2,QVectorLayout::byNODES,1,3,3>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,1,2,4>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,1,3,2>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,1,3,4>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,1,3,6>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,1,4,3>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,1,4,4>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 2, 2, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 2, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 2, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 2, 5, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 2, 6, 1>();
+   k::Specialization<2,QVectorLayout::byNODES,2,2,2>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,2,3>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,2,4>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,2,5>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,2,6>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 3, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 3, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 3, 6, 1>();
+   k::Specialization<2,QVectorLayout::byNODES,2,3,3>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,3,4>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,3,6>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 4, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 4, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 4, 5, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 4, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 4, 7, 1>();
+   k::Specialization<2,QVectorLayout::byNODES,2,4,3>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,4,4>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,4,5>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,4,6>::Opt<1>::Add();
+   k::Specialization<2,QVectorLayout::byNODES,2,4,7>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byNODES, 2, 5, 6, 1>();
+   k::Specialization<2,QVectorLayout::byNODES,2,5,6>::Opt<1>::Add();
 
    // 3D
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 2, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 3, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 3, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 3, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 4, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 4, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 1, 4, 8, 1>();
+   k::Specialization<3,QVectorLayout::byNODES,1,2,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,1,3,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,1,3,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,1,3,6>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,1,4,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,1,4,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,1,4,8>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 2, 2, 2, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 2, 2, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 2, 3, 4, 1>();
+   k::Specialization<3,QVectorLayout::byNODES,2,2,2>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,2,2,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,2,3,4>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 2, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 2, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 2, 5, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 2, 6, 1>();
+   k::Specialization<3,QVectorLayout::byNODES,3,2,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,2,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,2,5>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,2,6>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 3, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 3, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 3, 5, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 3, 6, 1>();
+   k::Specialization<3,QVectorLayout::byNODES,3,3,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,3,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,3,5>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,3,6>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 4, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 4, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 4, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 4, 7, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byNODES, 3, 4, 8, 1>();
+   k::Specialization<3,QVectorLayout::byNODES,3,4,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,4,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,4,6>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,4,7>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byNODES,3,4,8>::Opt<1>::Add();
 }
 
 } // namespace quadrature_interpolator

fem/qinterp/eval_by_vdim.cpp

@@ -21,59 +21,36 @@ namespace quadrature_interpolator
 
 void InitEvalByVDimKernels()
 {
+   using k = QuadratureInterpolator::TensorEvalKernels;
    // 2D
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 1, 2, 4, 8>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 1, 3, 6, 4>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 1, 4, 8, 2>();
+   k::Specialization<2,QVectorLayout::byVDIM,1,2,4>::Opt<8>::Add();
+   k::Specialization<2,QVectorLayout::byVDIM,1,3,6>::Opt<4>::Add();
+   k::Specialization<2,QVectorLayout::byVDIM,1,4,8>::Opt<2>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 2, 2, 4, 8>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 2, 3, 4, 8>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 2, 3, 6, 4>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 2, 4, 6, 2>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   2, QVectorLayout::byVDIM, 2, 4, 8, 2>();
+   k::Specialization<2,QVectorLayout::byVDIM,2,2,4>::Opt<8>::Add();
+   k::Specialization<2,QVectorLayout::byVDIM,2,3,4>::Opt<8>::Add();
+   k::Specialization<2,QVectorLayout::byVDIM,2,3,6>::Opt<4>::Add();
+   k::Specialization<2,QVectorLayout::byVDIM,2,4,6>::Opt<2>::Add();
+   k::Specialization<2,QVectorLayout::byVDIM,2,4,8>::Opt<2>::Add();
    // 3D
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 1, 2, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 1, 3, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 1, 4, 8, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 2, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 3, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 4, 8, 1>();
+   k::Specialization<3,QVectorLayout::byVDIM,1,2,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,1,3,6>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,1,4,8>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,2,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,3,6>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,4,8>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 2, 2, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 3, 3, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 4, 4, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 5, 5, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 6, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 7, 7, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 8, 8, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 9, 9, 1>();
+   k::Specialization<3,QVectorLayout::byVDIM,3,2,2>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,3,3>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,4,4>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,5,5>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,6,6>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,7,7>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,8,8>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,9,9>::Opt<1>::Add();
 
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 4, 6, 1>();
-   QuadratureInterpolator::AddTensorEvalSpecializations<
-   3, QVectorLayout::byVDIM, 3, 3, 4, 1>();
+   k::Specialization<3,QVectorLayout::byVDIM,3,4,6>::Opt<1>::Add();
+   k::Specialization<3,QVectorLayout::byVDIM,3,3,4>::Opt<1>::Add();
 }
 
 } // namespace quadrature_interpolator

fem/qinterp/eval_hdiv.hpp

@@ -136,8 +136,8 @@ inline void EvalHDiv2D(const int NE,
                {
                   qq += QD(qx,dy,vd) * By(dy,qy);
                }
-               if ((FLAGS & (QuadratureInterpolator::PHYSICAL_VALUES |
-                             QuadratureInterpolator::PHYSICAL_MAGNITUDES)) != 0)
+               if (FLAGS & (QuadratureInterpolator::PHYSICAL_VALUES |
+                            QuadratureInterpolator::PHYSICAL_MAGNITUDES))
                {
                   QQ(qx,qy,vd) = qq;
                }
@@ -153,8 +153,8 @@ inline void EvalHDiv2D(const int NE,
          }
       }
       MFEM_SYNC_THREAD;
-      if ((FLAGS & (QuadratureInterpolator::PHYSICAL_VALUES |
-                    QuadratureInterpolator::PHYSICAL_MAGNITUDES)) != 0)
+      if (FLAGS & (QuadratureInterpolator::PHYSICAL_VALUES |
+                   QuadratureInterpolator::PHYSICAL_MAGNITUDES))
       {
          if (tidz == 0)
          {
@@ -178,7 +178,7 @@ inline void EvalHDiv2D(const int NE,
                   const real_t detJ = kernels::Det<DIM>(J_loc);
                   kernels::Mult(DIM, DIM, J_loc, u_ref, u_phys);
                   kernels::Set(DIM, 1, 1_r/detJ, u_phys, u_phys);
-                  if ((FLAGS & QuadratureInterpolator::PHYSICAL_VALUES) != 0)
+                  if (FLAGS & QuadratureInterpolator::PHYSICAL_VALUES)
                   {
                      MFEM_UNROLL(DIM)
                      for (int sd = 0; sd < DIM; sd++)
@@ -193,8 +193,7 @@ inline void EvalHDiv2D(const int NE,
                         }
                      }
                   }
-                  else if ((FLAGS &
-                            QuadratureInterpolator::PHYSICAL_MAGNITUDES) != 0)
+                  else if (FLAGS & QuadratureInterpolator::PHYSICAL_MAGNITUDES)
                   {
                      y(qx,qy,0,e) = kernels::Norml2(DIM, u_phys);
                   }
@@ -373,9 +372,8 @@ inline void EvalHDiv3D(const int NE,
                MFEM_UNROLL(MQ1)
                for (int qz = 0; qz < Q1D; ++qz)
                {
-                  if ((FLAGS & (QuadratureInterpolator::PHYSICAL_VALUES |
-                                QuadratureInterpolator::PHYSICAL_MAGNITUDES)) !=
-                      0)
+                  if (FLAGS & (QuadratureInterpolator::PHYSICAL_VALUES |
+                               QuadratureInterpolator::PHYSICAL_MAGNITUDES))
                   {
                      QQQ(qx,qy,qz,vd) = u[qz];
                   }
@@ -392,8 +390,8 @@ inline void EvalHDiv3D(const int NE,
          }
       }
       MFEM_SYNC_THREAD;
-      if ((FLAGS & (QuadratureInterpolator::PHYSICAL_VALUES |
-                    QuadratureInterpolator::PHYSICAL_MAGNITUDES)) != 0)
+      if (FLAGS & (QuadratureInterpolator::PHYSICAL_VALUES |
+                   QuadratureInterpolator::PHYSICAL_MAGNITUDES))
       {
          MFEM_FOREACH_THREAD(qz,z,Q1D)
          {
@@ -417,7 +415,7 @@ inline void EvalHDiv3D(const int NE,
                   const real_t detJ = kernels::Det<DIM>(J_loc);
                   kernels::Mult(DIM, DIM, J_loc, u_ref, u_phys);
                   kernels::Set(DIM, 1, 1_r/detJ, u_phys, u_phys);
-                  if ((FLAGS & QuadratureInterpolator::PHYSICAL_VALUES) != 0)
+                  if (FLAGS & QuadratureInterpolator::PHYSICAL_VALUES)
                   {
                      MFEM_UNROLL(DIM)
                      for (int sd = 0; sd < DIM; sd++)
@@ -432,8 +430,7 @@ inline void EvalHDiv3D(const int NE,
                         }
                      }
                   }
-                  else if ((FLAGS &
-                            QuadratureInterpolator::PHYSICAL_MAGNITUDES) != 0)
+                  else if (FLAGS & QuadratureInterpolator::PHYSICAL_MAGNITUDES)
                   {
                      y(qx,qy,qz,0,e) = kernels::Norml2(DIM, u_phys);
                   }

fem/qinterp/grad_by_nodes.cpp

@@ -22,71 +22,74 @@ namespace quadrature_interpolator
 template <bool P>
 void InitGradByNodesKernels()
 {
+   using k = QuadratureInterpolator::GradKernels;
    constexpr auto L = QVectorLayout::byNODES;
    // 2D
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,3,3,16>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,3,4,16>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,4,3,16>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,4,4,16>();
+   k::Specialization<2,L,P,1,3,3>::template Opt<16>::Add();
+   k::Specialization<2,L,P,1,3,4>::template Opt<16>::Add();
+   k::Specialization<2,L,P,1,4,3>::template Opt<16>::Add();
+   k::Specialization<2,L,P,1,4,4>::template Opt<16>::Add();
 
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,2,2,16>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,2,3,8>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,2,4,4>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,2,5,4>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,2,6,2>();
+   k::Specialization<2,L,P,2,2,2>::template Opt<16>::Add();
+   k::Specialization<2,L,P,2,2,3>::template Opt<8>::Add();
+   k::Specialization<2,L,P,2,2,4>::template Opt<4>::Add();
+   k::Specialization<2,L,P,2,2,5>::template Opt<4>::Add();
+   k::Specialization<2,L,P,2,2,6>::template Opt<2>::Add();
 
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,3,3,2>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,3,4,4>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,3,4>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,3,6,2>();
+   k::Specialization<2,L,P,2,3,3>::template Opt<2>::Add();
+   k::Specialization<2,L,P,2,3,4>::template Opt<4>::Add();
+   k::Specialization<2,L,P,2,4,3>::template Opt<4>::Add();
+   k::Specialization<2,L,P,2,3,6>::template Opt<2>::Add();
 
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,4,2>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,5,2>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,6,2>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,7,2>();
+   k::Specialization<2,L,P,2,4,4>::template Opt<2>::Add();
+   k::Specialization<2,L,P,2,4,5>::template Opt<2>::Add();
+   k::Specialization<2,L,P,2,4,6>::template Opt<2>::Add();
+   k::Specialization<2,L,P,2,4,7>::template Opt<2>::Add();
 
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,5,6,2>();
+   k::Specialization<2,L,P,2,5,6>::template Opt<2>::Add();
    // 3D
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,2,4>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,3,3>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,3,4>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,3,6>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,4,4>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,4,8>();
+   k::Specialization<3,L,P,1,2,4>::Add();
+   k::Specialization<3,L,P,1,3,3>::Add();
+   k::Specialization<3,L,P,1,3,4>::Add();
+   k::Specialization<3,L,P,1,3,6>::Add();
+   k::Specialization<3,L,P,1,4,4>::Add();
+   k::Specialization<3,L,P,1,4,8>::Add();
 
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,2,3>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,2,4>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,2,5>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,2,6>();
+   k::Specialization<3,L,P,3,2,3>::Add();
+   k::Specialization<3,L,P,3,2,4>::Add();
+   k::Specialization<3,L,P,3,2,5>::Add();
+   k::Specialization<3,L,P,3,2,6>::Add();
 
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,3,3>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,3,4>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,3,5>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,3,6>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,4,4>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,4,6>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,4,7>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,4,8>();
+   k::Specialization<3,L,P,3,3,3>::Add();
+   k::Specialization<3,L,P,3,3,4>::Add();
+   k::Specialization<3,L,P,3,3,5>::Add();
+   k::Specialization<3,L,P,3,3,6>::Add();
+   k::Specialization<3,L,P,3,4,4>::Add();
+   k::Specialization<3,L,P,3,4,6>::Add();
+   k::Specialization<3,L,P,3,4,7>::Add();
+   k::Specialization<3,L,P,3,4,8>::Add();
+
+   using k2 = QuadratureInterpolator::CollocatedGradKernels;
 
    // 2D
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,2,16>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,3,16>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,4,16>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,2,16>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,3,4>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,4,2>();
+   k2::Specialization<2,L,P,1,2>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,1,3>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,1,4>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,2,2>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,2,3>::template Opt<4>::Add();
+   k2::Specialization<2,L,P,2,4>::template Opt<2>::Add();
 
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,2>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,3>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,4>();
+   k2::Specialization<3,L,P,1,2>::Add();
+   k2::Specialization<3,L,P,1,3>::Add();
+   k2::Specialization<3,L,P,1,4>::Add();
 
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,2>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,3>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,4>();
+   k2::Specialization<3,L,P,2,2>::Add();
+   k2::Specialization<3,L,P,2,3>::Add();
+   k2::Specialization<3,L,P,2,4>::Add();
 
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,2>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,3>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,4>();
+   k2::Specialization<3,L,P,3,2>::Add();
+   k2::Specialization<3,L,P,3,3>::Add();
+   k2::Specialization<3,L,P,3,4>::Add();
 }
 
 template void InitGradByNodesKernels<true>();

fem/qinterp/grad_by_vdim.cpp

@@ -22,45 +22,47 @@ namespace quadrature_interpolator
 template <bool P>
 void InitGradByVDimKernels()
 {
+   using k = QuadratureInterpolator::GradKernels;
    constexpr auto L = QVectorLayout::byVDIM;
    // 2D
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,3,4,8>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,4,6,4>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,1,5,8,2>();
+   k::Specialization<2,L,P,1,3,4>::template Opt<8>::Add();
+   k::Specialization<2,L,P,1,4,6>::template Opt<4>::Add();
+   k::Specialization<2,L,P,1,5,8>::template Opt<2>::Add();
 
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,3,3,8>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,3,4,8>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,4,6,4>();
-   QuadratureInterpolator::AddGradSpecializations<2,L,P,2,5,8,2>();
+   k::Specialization<2,L,P,2,3,3>::template Opt<8>::Add();
+   k::Specialization<2,L,P,2,3,4>::template Opt<8>::Add();
+   k::Specialization<2,L,P,2,4,6>::template Opt<4>::Add();
+   k::Specialization<2,L,P,2,5,8>::template Opt<2>::Add();
    // 3D
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,3,4>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,4,6>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,1,5,8>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,3,4>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,4,6>();
-   QuadratureInterpolator::AddGradSpecializations<3,L,P,3,5,8>();
+   k::Specialization<3,L,P,1,3,4>::Add();
+   k::Specialization<3,L,P,1,4,6>::Add();
+   k::Specialization<3,L,P,1,5,8>::Add();
+   k::Specialization<3,L,P,3,3,4>::Add();
+   k::Specialization<3,L,P,3,4,6>::Add();
+   k::Specialization<3,L,P,3,5,8>::Add();
 
+   using k2 = QuadratureInterpolator::CollocatedGradKernels;
    // 2D
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,2,16>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,3,16>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,1,4,16>();
+   k2::Specialization<2,L,P,1,2>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,1,3>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,1,4>::template Opt<16>::Add();
 
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,2,16>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,3,4>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<2,L,P,2,4,2>();
+   k2::Specialization<2,L,P,2,2>::template Opt<16>::Add();
+   k2::Specialization<2,L,P,2,3>::template Opt<4>::Add();
+   k2::Specialization<2,L,P,2,4>::template Opt<2>::Add();
 
    // 3D
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,2>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,3>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,1,4>();
+   k2::Specialization<3,L,P,1,2>::Add();
+   k2::Specialization<3,L,P,1,3>::Add();
+   k2::Specialization<3,L,P,1,4>::Add();
 
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,2>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,3>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,2,4>();
+   k2::Specialization<3,L,P,2,2>::Add();
+   k2::Specialization<3,L,P,2,3>::Add();
+   k2::Specialization<3,L,P,2,4>::Add();
 
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,2>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,3>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<3,L,P,3,4>();
+   k2::Specialization<3,L,P,3,2>::Add();
+   k2::Specialization<3,L,P,3,3>::Add();
+   k2::Specialization<3,L,P,3,4>::Add();
 }
 
 template void InitGradByVDimKernels<true>();

fem/quadinterpolator.cpp

@@ -69,9 +69,8 @@ QuadratureInterpolator::QuadratureInterpolator(const FiniteElementSpace &fes,
 
    d_buffer.UseDevice(true);
    if (fespace->GetNE() == 0) { return; }
-   MFEM_VERIFY(
-      SupportsFESpace(fes),
-      "Only elements with MapType VALUE, INTEGRAL, or H_DIV are supported!");
+   MFEM_VERIFY(SupportsFESpace(fes),
+               "Only elements with MapType VALUE and H_DIV are supported!");
 }
 
 QuadratureInterpolator::QuadratureInterpolator(const FiniteElementSpace &fes,
@@ -85,9 +84,8 @@ QuadratureInterpolator::QuadratureInterpolator(const FiniteElementSpace &fes,
 {
    d_buffer.UseDevice(true);
    if (fespace->GetNE() == 0) { return; }
-   MFEM_VERIFY(
-      SupportsFESpace(fes),
-      "Only elements with MapType VALUE, INTEGRAL, or H_DIV are supported!");
+   MFEM_VERIFY(SupportsFESpace(fes),
+               "Only elements with MapType VALUE and H_DIV are supported!");
 }
 
 bool QuadratureInterpolator::SupportsFESpace(const FiniteElementSpace &fespace)
@@ -95,9 +93,9 @@ bool QuadratureInterpolator::SupportsFESpace(const FiniteElementSpace &fespace)
    const FiniteElement *fe = fespace.GetTypicalFE();
    const Mesh &mesh = *fespace.GetMesh();
    return (fe->GetMapType() == FiniteElement::MapType::VALUE ||
-           fe->GetMapType() == FiniteElement::MapType::INTEGRAL ||
-           fe->GetMapType() == FiniteElement::MapType::H_DIV) &&
-          (!fespace.IsVariableOrder()) && (!mesh.IsMixedMesh());
+           fe->GetMapType() == FiniteElement::MapType::H_DIV)
+          && (!fespace.IsVariableOrder())
+          && (!mesh.IsMixedMesh());
 }
 
 namespace internal
@@ -110,11 +108,16 @@ namespace quadrature_interpolator
 // * non-tensor product version,
 // * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
 // * assumes 'maps.mode == FULL'.
-template <bool Integral>
-void IntEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
-               const real_t *detJ_, const GeometricFactors *geom,
-               const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
-               Vector &q_der, Vector &q_det, const int eval_flags)
+static void Eval1D(const int NE,
+                   const int vdim,
+                   const QVectorLayout q_layout,
+                   const GeometricFactors *geom,
+                   const DofToQuad &maps,
+                   const Vector &e_vec,
+                   Vector &q_val,
+                   Vector &q_der,
+                   Vector &q_det,
+                   const int eval_flags)
 {
    using QI = QuadratureInterpolator;
 
@@ -123,10 +126,13 @@ void IntEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
    MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
    MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 1, "");
    MFEM_VERIFY(vdim == 1 || !(eval_flags & QI::DETERMINANTS), "");
-   const auto B_ = maps.B.Read();
-   const auto G_ = maps.G.Read();
+   MFEM_VERIFY(bool(geom) == bool(eval_flags & QI::PHYSICAL_DERIVATIVES),
+               "'geom' must be given (non-null) only when evaluating physical"
+               " derivatives");
+   const auto B = Reshape(maps.B.Read(), nq, nd);
+   const auto G = Reshape(maps.G.Read(), nq, nd);
    const auto J = Reshape(geom ? geom->J.Read() : nullptr, nq, NE);
-   const auto E_ = e_vec.Read();
+   const auto E = Reshape(e_vec.Read(), nd, vdim, NE);
    auto val = q_layout == QVectorLayout::byNODES ?
               Reshape(q_val.Write(), nq, vdim, NE):
               Reshape(q_val.Write(), vdim, nq, NE);
@@ -134,12 +140,8 @@ void IntEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
               Reshape(q_der.Write(), nq, vdim, NE):
               Reshape(q_der.Write(), vdim, nq, NE);
    auto det = Reshape(q_det.Write(), nq, NE);
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      const auto B = Reshape(B_, nq, nd);
-      const auto G = Reshape(G_, nq, nd);
-      const auto E = Reshape(E_, nd, vdim, NE);
-      const auto detJ = Reshape(detJ_, nq, NE);
       for (int q = 0; q < nq; ++q)
       {
          if (eval_flags & (QI::VALUES | QI::PHYSICAL_VALUES))
@@ -149,20 +151,10 @@ void IntEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
                real_t q_val = 0.0;
                for (int d = 0; d < nd; ++d)
                {
-                  q_val += B(q, d) * E(d, c, e);
-               }
-               if constexpr (Integral)
-               {
-                  q_val /= detJ(q, e);
-               }
-               if (q_layout == QVectorLayout::byVDIM)
-               {
-                  val(c, q, e) = q_val;
-               }
-               if (q_layout == QVectorLayout::byNODES)
-               {
-                  val(q, c, e) = q_val;
+                  q_val += B(q,d)*E(d,c,e);
                }
+               if (q_layout == QVectorLayout::byVDIM)  { val(c,q,e) = q_val; }
+               if (q_layout == QVectorLayout::byNODES) { val(q,c,e) = q_val; }
             }
          }
          if ((eval_flags & QI::DERIVATIVES) ||
@@ -174,7 +166,7 @@ void IntEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
                real_t q_d = 0.0;
                for (int d = 0; d < nd; ++d)
                {
-                  q_d += G(q, d) * E(d, c, e);
+                  q_d += G(q,d)*E(d,c,e);
                }
                if (eval_flags & QI::PHYSICAL_DERIVATIVES)
                {
@@ -182,14 +174,8 @@ void IntEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
                }
                if (eval_flags & QI::DERIVATIVES || eval_flags & QI::PHYSICAL_DERIVATIVES)
                {
-                  if (q_layout == QVectorLayout::byVDIM)
-                  {
-                     der(c, q, e) = q_d;
-                  }
-                  if (q_layout == QVectorLayout::byNODES)
-                  {
-                     der(q, c, e) = q_d;
-                  }
+                  if (q_layout == QVectorLayout::byVDIM) { der(c,q,e) = q_d; }
+                  if (q_layout == QVectorLayout::byNODES) { der(q,c,e) = q_d; }
                }
                if (vdim == 1 && (eval_flags & QI::DETERMINANTS))
                {
@@ -201,17 +187,317 @@ void IntEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
    });
 }
 
-template void
-IntEval1D<true>(const int NE, const int vdim, const QVectorLayout q_layout,
-                const real_t *detJ, const GeometricFactors *geom,
-                const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
-                Vector &q_der, Vector &q_det, const int eval_flags);
+// Template compute kernel for 2D quadrature interpolation:
+// * non-tensor product version,
+// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
+// * assumes 'maps.mode == FULL'.
+template<const int T_VDIM, const int T_ND, const int T_NQ>
+static void Eval2D(const int NE,
+                   const int vdim,
+                   const QVectorLayout q_layout,
+                   const GeometricFactors *geom,
+                   const DofToQuad &maps,
+                   const Vector &e_vec,
+                   Vector &q_val,
+                   Vector &q_der,
+                   Vector &q_det,
+                   const int eval_flags)
+{
+   using QI = QuadratureInterpolator;
 
-template void
-IntEval1D<false>(const int NE, const int vdim, const QVectorLayout q_layout,
-                 const real_t *detJ, const GeometricFactors *geom,
-                 const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
-                 Vector &q_der, Vector &q_det, const int eval_flags);
+   const int nd = maps.ndof;
+   const int nq = maps.nqpt;
+   const int ND = T_ND ? T_ND : nd;
+   const int NQ = T_NQ ? T_NQ : nq;
+   const int NMAX = NQ > ND ? NQ : ND;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+   MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
+   MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 2, "");
+   MFEM_VERIFY(ND <= QI::MAX_ND2D, "");
+   MFEM_VERIFY(NQ <= QI::MAX_NQ2D, "");
+   MFEM_VERIFY(bool(geom) == bool(eval_flags & QI::PHYSICAL_DERIVATIVES),
+               "'geom' must be given (non-null) only when evaluating physical"
+               " derivatives");
+   const auto B = Reshape(maps.B.Read(), NQ, ND);
+   const auto G = Reshape(maps.G.Read(), NQ, 2, ND);
+   const auto J = Reshape(geom ? geom->J.Read() : nullptr, NQ, 2, 2, NE);
+   const auto E = Reshape(e_vec.Read(), ND, VDIM, NE);
+   auto val = q_layout == QVectorLayout::byNODES ?
+              Reshape(q_val.Write(), NQ, VDIM, NE):
+              Reshape(q_val.Write(), VDIM, NQ, NE);
+   auto der = q_layout == QVectorLayout::byNODES ?
+              Reshape(q_der.Write(), NQ, VDIM, 2, NE):
+              Reshape(q_der.Write(), VDIM, 2, NQ, NE);
+   auto det = Reshape(q_det.Write(), NQ, NE);
+   mfem::forall_2D(NE, NMAX, 1, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int ND = T_ND ? T_ND : nd;
+      const int NQ = T_NQ ? T_NQ : nq;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int max_ND = T_ND ? T_ND : QI::MAX_ND2D;
+      constexpr int max_VDIM = T_VDIM ? T_VDIM : QI::MAX_VDIM2D;
+      MFEM_SHARED real_t s_E[max_VDIM*max_ND];
+      MFEM_FOREACH_THREAD(d, x, ND)
+      {
+         for (int c = 0; c < VDIM; c++)
+         {
+            s_E[c+d*VDIM] = E(d,c,e);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(q, x, NQ)
+      {
+         if (eval_flags & (QI::VALUES | QI::PHYSICAL_VALUES))
+         {
+            real_t ed[max_VDIM];
+            for (int c = 0; c < VDIM; c++) { ed[c] = 0.0; }
+            for (int d = 0; d < ND; ++d)
+            {
+               const real_t b = B(q,d);
+               for (int c = 0; c < VDIM; c++) { ed[c] += b*s_E[c+d*VDIM]; }
+            }
+            for (int c = 0; c < VDIM; c++)
+            {
+               if (q_layout == QVectorLayout::byVDIM)  { val(c,q,e) = ed[c]; }
+               if (q_layout == QVectorLayout::byNODES) { val(q,c,e) = ed[c]; }
+            }
+         }
+         if ((eval_flags & QI::DERIVATIVES) ||
+             (eval_flags & QI::PHYSICAL_DERIVATIVES) ||
+             (eval_flags & QI::DETERMINANTS))
+         {
+            // use MAX_VDIM2D to avoid "subscript out of range" warnings
+            real_t D[QI::MAX_VDIM2D*2];
+            for (int i = 0; i < 2*VDIM; i++) { D[i] = 0.0; }
+            for (int d = 0; d < ND; ++d)
+            {
+               const real_t wx = G(q,0,d);
+               const real_t wy = G(q,1,d);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  real_t s_e = s_E[c+d*VDIM];
+                  D[c+VDIM*0] += s_e * wx;
+                  D[c+VDIM*1] += s_e * wy;
+               }
+            }
+            if (eval_flags & QI::DERIVATIVES)
+            {
+               for (int c = 0; c < VDIM; c++)
+               {
+                  if (q_layout == QVectorLayout::byVDIM)
+                  {
+                     der(c,0,q,e) = D[c+VDIM*0];
+                     der(c,1,q,e) = D[c+VDIM*1];
+                  }
+                  if (q_layout == QVectorLayout::byNODES)
+                  {
+                     der(q,c,0,e) = D[c+VDIM*0];
+                     der(q,c,1,e) = D[c+VDIM*1];
+                  }
+               }
+            }
+            if (eval_flags & QI::PHYSICAL_DERIVATIVES)
+            {
+               real_t Jloc[4], Jinv[4];
+               Jloc[0] = J(q,0,0,e);
+               Jloc[1] = J(q,1,0,e);
+               Jloc[2] = J(q,0,1,e);
+               Jloc[3] = J(q,1,1,e);
+               kernels::CalcInverse<2>(Jloc, Jinv);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  const real_t u = D[c+VDIM*0];
+                  const real_t v = D[c+VDIM*1];
+                  const real_t JiU = Jinv[0]*u + Jinv[1]*v;
+                  const real_t JiV = Jinv[2]*u + Jinv[3]*v;
+                  if (q_layout == QVectorLayout::byVDIM)
+                  {
+                     der(c,0,q,e) = JiU;
+                     der(c,1,q,e) = JiV;
+                  }
+                  if (q_layout == QVectorLayout::byNODES)
+                  {
+                     der(q,c,0,e) = JiU;
+                     der(q,c,1,e) = JiV;
+                  }
+               }
+            }
+            if (eval_flags & QI::DETERMINANTS)
+            {
+               if (VDIM == 2) { det(q,e) = kernels::Det<2>(D); }
+               else
+               {
+                  DeviceTensor<2> j(D, 3, 2);
+                  const double E = j(0,0)*j(0,0) + j(1,0)*j(1,0) + j(2,0)*j(2,0);
+                  const double F = j(0,0)*j(0,1) + j(1,0)*j(1,1) + j(2,0)*j(2,1);
+                  const double G = j(0,1)*j(0,1) + j(1,1)*j(1,1) + j(2,1)*j(2,1);
+                  det(q,e) = std::sqrt(E*G - F*F);
+               }
+            }
+         }
+      }
+   });
+}
+
+// Template compute kernel for 3D quadrature interpolation:
+// * non-tensor product version,
+// * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
+// * assumes 'maps.mode == FULL'.
+template<const int T_VDIM, const int T_ND, const int T_NQ>
+static void Eval3D(const int NE,
+                   const int vdim,
+                   const QVectorLayout q_layout,
+                   const GeometricFactors *geom,
+                   const DofToQuad &maps,
+                   const Vector &e_vec,
+                   Vector &q_val,
+                   Vector &q_der,
+                   Vector &q_det,
+                   const int eval_flags)
+{
+   using QI = QuadratureInterpolator;
+
+   const int nd = maps.ndof;
+   const int nq = maps.nqpt;
+   const int ND = T_ND ? T_ND : nd;
+   const int NQ = T_NQ ? T_NQ : nq;
+   const int NMAX = NQ > ND ? NQ : ND;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+   MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
+   MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 3, "");
+   MFEM_VERIFY(ND <= QI::MAX_ND3D, "");
+   MFEM_VERIFY(NQ <= QI::MAX_NQ3D, "");
+   MFEM_VERIFY(VDIM == 3 || !(eval_flags & QI::DETERMINANTS), "");
+   MFEM_VERIFY(bool(geom) == bool(eval_flags & QI::PHYSICAL_DERIVATIVES),
+               "'geom' must be given (non-null) only when evaluating physical"
+               " derivatives");
+   const auto B = Reshape(maps.B.Read(), NQ, ND);
+   const auto G = Reshape(maps.G.Read(), NQ, 3, ND);
+   const auto J = Reshape(geom ? geom->J.Read() : nullptr, NQ, 3, 3, NE);
+   const auto E = Reshape(e_vec.Read(), ND, VDIM, NE);
+   auto val = q_layout == QVectorLayout::byNODES ?
+              Reshape(q_val.Write(), NQ, VDIM, NE):
+              Reshape(q_val.Write(), VDIM, NQ, NE);
+   auto der = q_layout == QVectorLayout::byNODES ?
+              Reshape(q_der.Write(), NQ, VDIM, 3, NE):
+              Reshape(q_der.Write(), VDIM, 3, NQ, NE);
+   auto det = Reshape(q_det.Write(), NQ, NE);
+   mfem::forall_2D(NE, NMAX, 1, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int ND = T_ND ? T_ND : nd;
+      const int NQ = T_NQ ? T_NQ : nq;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int max_ND = T_ND ? T_ND : QI::MAX_ND3D;
+      constexpr int max_VDIM = T_VDIM ? T_VDIM : QI::MAX_VDIM3D;
+      MFEM_SHARED real_t s_E[max_VDIM*max_ND];
+      MFEM_FOREACH_THREAD(d, x, ND)
+      {
+         for (int c = 0; c < VDIM; c++)
+         {
+            s_E[c+d*VDIM] = E(d,c,e);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(q, x, NQ)
+      {
+         if (eval_flags & (QI::VALUES | QI::PHYSICAL_VALUES))
+         {
+            real_t ed[max_VDIM];
+            for (int c = 0; c < VDIM; c++) { ed[c] = 0.0; }
+            for (int d = 0; d < ND; ++d)
+            {
+               const real_t b = B(q,d);
+               for (int c = 0; c < VDIM; c++) { ed[c] += b*s_E[c+d*VDIM]; }
+            }
+            for (int c = 0; c < VDIM; c++)
+            {
+               if (q_layout == QVectorLayout::byVDIM)  { val(c,q,e) = ed[c]; }
+               if (q_layout == QVectorLayout::byNODES) { val(q,c,e) = ed[c]; }
+            }
+         }
+         if ((eval_flags & QI::DERIVATIVES) ||
+             (eval_flags & QI::PHYSICAL_DERIVATIVES) ||
+             (eval_flags & QI::DETERMINANTS))
+         {
+            // use MAX_VDIM3D to avoid "subscript out of range" warnings
+            real_t D[QI::MAX_VDIM3D*3];
+            for (int i = 0; i < 3*VDIM; i++) { D[i] = 0.0; }
+            for (int d = 0; d < ND; ++d)
+            {
+               const real_t wx = G(q,0,d);
+               const real_t wy = G(q,1,d);
+               const real_t wz = G(q,2,d);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  real_t s_e = s_E[c+d*VDIM];
+                  D[c+VDIM*0] += s_e * wx;
+                  D[c+VDIM*1] += s_e * wy;
+                  D[c+VDIM*2] += s_e * wz;
+               }
+            }
+            if (eval_flags & QI::DERIVATIVES)
+            {
+               for (int c = 0; c < VDIM; c++)
+               {
+                  if (q_layout == QVectorLayout::byVDIM)
+                  {
+                     der(c,0,q,e) = D[c+VDIM*0];
+                     der(c,1,q,e) = D[c+VDIM*1];
+                     der(c,2,q,e) = D[c+VDIM*2];
+                  }
+                  if (q_layout == QVectorLayout::byNODES)
+                  {
+                     der(q,c,0,e) = D[c+VDIM*0];
+                     der(q,c,1,e) = D[c+VDIM*1];
+                     der(q,c,2,e) = D[c+VDIM*2];
+                  }
+               }
+            }
+            if (eval_flags & QI::PHYSICAL_DERIVATIVES)
+            {
+               real_t Jloc[9], Jinv[9];
+               for (int col = 0; col < 3; col++)
+               {
+                  for (int row = 0; row < 3; row++)
+                  {
+                     Jloc[row+3*col] = J(q,row,col,e);
+                  }
+               }
+               kernels::CalcInverse<3>(Jloc, Jinv);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  const real_t u = D[c+VDIM*0];
+                  const real_t v = D[c+VDIM*1];
+                  const real_t w = D[c+VDIM*2];
+                  const real_t JiU = Jinv[0]*u + Jinv[1]*v + Jinv[2]*w;
+                  const real_t JiV = Jinv[3]*u + Jinv[4]*v + Jinv[5]*w;
+                  const real_t JiW = Jinv[6]*u + Jinv[7]*v + Jinv[8]*w;
+                  if (q_layout == QVectorLayout::byVDIM)
+                  {
+                     der(c,0,q,e) = JiU;
+                     der(c,1,q,e) = JiV;
+                     der(c,2,q,e) = JiW;
+                  }
+                  if (q_layout == QVectorLayout::byNODES)
+                  {
+                     der(q,c,0,e) = JiU;
+                     der(q,c,1,e) = JiV;
+                     der(q,c,2,e) = JiW;
+                  }
+               }
+            }
+            if (VDIM == 3 && (eval_flags & QI::DETERMINANTS))
+            {
+               // The check (VDIM == 3) should eliminate this block when VDIM is
+               // known at compile time and (VDIM != 3).
+               det(q,e) = kernels::Det<3>(D);
+            }
+         }
+      }
+   });
+}
 
 } // namespace quadrature_interpolator
 
@@ -249,20 +535,10 @@ void QuadratureInterpolator::Mult(const Vector &e_vec,
    const int nd = maps.ndof;
    const int nq = maps.nqpt;
    const GeometricFactors *geom = nullptr;
+   if (eval_flags & PHYSICAL_DERIVATIVES)
    {
-      int jac_factors = 0;
-      if (eval_flags & PHYSICAL_DERIVATIVES)
-      {
-         jac_factors = GeometricFactors::JACOBIANS;
-      }
-      if (fe->GetMapType() == FiniteElement::MapType::INTEGRAL)
-      {
-         jac_factors |= GeometricFactors::DETERMINANTS;
-      }
-      if (jac_factors)
-      {
-         geom = fespace->GetMesh()->GetGeometricFactors(*ir, jac_factors);
-      }
+      const int jacobians = GeometricFactors::JACOBIANS;
+      geom = fespace->GetMesh()->GetGeometricFactors(*ir, jacobians);
    }
 
    MFEM_ASSERT(!(eval_flags & DETERMINANTS) || dim == vdim ||
@@ -276,61 +552,29 @@ void QuadratureInterpolator::Mult(const Vector &e_vec,
    {
       if (eval_flags & (VALUES | PHYSICAL_VALUES))
       {
-         if (fe->GetMapType() == FiniteElement::MapType::INTEGRAL)
-         {
-            IntTensorEvalKernels::Run(dim, q_layout, vdim, nd, nq, ne,
-                                      maps.B.Read(), geom->detJ.Read(),
-                                      e_vec.Read(), q_val.Write(), vdim, nd, nq);
-         }
-         else
-         {
-            TensorEvalKernels::Run(dim, q_layout, vdim, nd, nq, ne,
-                                   maps.B.Read(), e_vec.Read(), q_val.Write(),
-                                   vdim, nd, nq);
-         }
+         TensorEvalKernels::Run(dim, q_layout, vdim, nd, nq, ne, maps.B.Read(),
+                                e_vec.Read(), q_val.Write(), vdim, nd, nq);
       }
       if (eval_flags & (DERIVATIVES | PHYSICAL_DERIVATIVES))
       {
          const bool phys = (eval_flags & PHYSICAL_DERIVATIVES);
          const real_t *J = phys ? geom->J.Read() : nullptr;
          const int s_dim = phys ? sdim : dim;
-         if (fe->GetMapType() == FiniteElement::MapType::INTEGRAL)
-         {
-            MFEM_ABORT("");
-         }
-         else
-         {
-            GradKernels::Run(dim, q_layout, phys, vdim, nd, nq, ne,
-                             maps.B.Read(), maps.G.Read(), J, e_vec.Read(),
-                             q_der.Write(), s_dim, vdim, nd, nq);
-         }
+         GradKernels::Run(dim, q_layout, phys, vdim, nd, nq, ne,
+                          maps.B.Read(), maps.G.Read(), J, e_vec.Read(),
+                          q_der.Write(), s_dim, vdim, nd, nq);
       }
       if (eval_flags & DETERMINANTS)
       {
-         if (fe->GetMapType() == FiniteElement::MapType::INTEGRAL)
-         {
-            MFEM_ABORT("");
-         }
-         else
-         {
-            DetKernels::Run(dim, vdim, nd, nq, ne, maps.B.Read(), maps.G.Read(),
-                            e_vec.Read(), q_det.Write(), nd, nq, &d_buffer);
-         }
+         DetKernels::Run(dim, vdim, nd, nq, ne, maps.B.Read(),
+                         maps.G.Read(), e_vec.Read(), q_det.Write(), nd,
+                         nq, &d_buffer);
       }
    }
    else // use_tensor_eval == false
    {
-      if (fe->GetMapType() == FiniteElement::MapType::INTEGRAL)
-      {
-         IntEvalKernels::Run(dim, vdim, maps.ndof, maps.nqpt, ne, vdim,
-                             q_layout, geom->detJ.Read(), geom, maps, e_vec,
-                             q_val, q_der, q_det, eval_flags);
-      }
-      else
-      {
-         EvalKernels::Run(dim, vdim, maps.ndof, maps.nqpt, ne, vdim, q_layout,
-                          geom, maps, e_vec, q_val, q_der, q_det, eval_flags);
-      }
+      EvalKernels::Run(dim, vdim, maps.ndof, maps.nqpt, ne,vdim, q_layout,
+                       geom, maps, e_vec, q_val, q_der, q_det, eval_flags);
    }
 }
 
@@ -456,41 +700,22 @@ namespace
 
 using namespace internal::quadrature_interpolator;
 
-template <QVectorLayout Q_LAYOUT> auto IntFallbackTensorEvalKernel(int DIM)
+using EvalKernel = QuadratureInterpolator::EvalKernelType;
+using TensorEvalKernel = QuadratureInterpolator::TensorEvalKernelType;
+using GradKernel = QuadratureInterpolator::GradKernelType;
+using CollocatedGradKernel = QuadratureInterpolator::CollocatedGradKernelType;
+
+template <QVectorLayout Q_LAYOUT>
+TensorEvalKernel FallbackTensorEvalKernel(int DIM)
 {
-   if (DIM == 1)
-   {
-      return IntValues1D<Q_LAYOUT, true>;
-   }
-   else if (DIM == 2)
-   {
-      return IntValues2D<Q_LAYOUT, true>;
-   }
-   else if (DIM == 3)
-   {
-      return IntValues3D<Q_LAYOUT, true>;
-   }
-   MFEM_ABORT("");
+   if (DIM == 1) { return Values1D<Q_LAYOUT>; }
+   else if (DIM == 2) { return Values2D<Q_LAYOUT>; }
+   else if (DIM == 3) { return Values3D<Q_LAYOUT>; }
+   else { MFEM_ABORT(""); }
 }
 
-template <QVectorLayout Q_LAYOUT> auto FallbackTensorEvalKernel(int DIM)
-{
-   if (DIM == 1)
-   {
-      return Values1D<Q_LAYOUT>;
-   }
-   else if (DIM == 2)
-   {
-      return Values2D<Q_LAYOUT>;
-   }
-   else if (DIM == 3)
-   {
-      return Values3D<Q_LAYOUT>;
-   }
-   MFEM_ABORT("");
-}
-
-template <QVectorLayout Q_LAYOUT, bool GRAD_PHYS> auto GetGradKernel(int DIM)
+template<QVectorLayout Q_LAYOUT, bool GRAD_PHYS>
+GradKernel GetGradKernel(int DIM)
 {
    if (DIM == 1) { return Derivatives1D<Q_LAYOUT, GRAD_PHYS>; }
    else if (DIM == 2) { return Derivatives2D<Q_LAYOUT, GRAD_PHYS>; }
@@ -498,185 +723,79 @@ template <QVectorLayout Q_LAYOUT, bool GRAD_PHYS> auto GetGradKernel(int DIM)
    else { MFEM_ABORT(""); }
 }
 
-template <QVectorLayout Q_LAYOUT> auto GetGradKernel(int DIM, bool GRAD_PHYS)
+
+template<QVectorLayout Q_LAYOUT>
+GradKernel GetGradKernel(int DIM, bool GRAD_PHYS)
 {
    if (GRAD_PHYS) { return GetGradKernel<Q_LAYOUT, true>(DIM); }
    else { return GetGradKernel<Q_LAYOUT, false>(DIM); }
 }
 
 template<QVectorLayout Q_LAYOUT, bool GRAD_PHYS>
-auto GetCollocatedGradKernel(int DIM)
+CollocatedGradKernel GetCollocatedGradKernel(int DIM)
 {
-   if (DIM == 1)
-   {
-      return CollocatedDerivatives1D<Q_LAYOUT, GRAD_PHYS>;
-   }
-   else if (DIM == 2)
-   {
-      return CollocatedDerivatives2D<Q_LAYOUT, GRAD_PHYS>;
-   }
-   else if (DIM == 3)
-   {
-      return CollocatedDerivatives3D<Q_LAYOUT, GRAD_PHYS>;
-   }
-   MFEM_ABORT("");
+   if (DIM == 1) { return CollocatedDerivatives1D<Q_LAYOUT, GRAD_PHYS>; }
+   else if (DIM == 2) { return CollocatedDerivatives2D<Q_LAYOUT, GRAD_PHYS>; }
+   else if (DIM == 3) { return CollocatedDerivatives3D<Q_LAYOUT, GRAD_PHYS>; }
+   else { MFEM_ABORT(""); }
 }
 
-template <QVectorLayout Q_LAYOUT>
-auto GetCollocatedGradKernel(int DIM, bool GRAD_PHYS)
+template<QVectorLayout Q_LAYOUT>
+CollocatedGradKernel GetCollocatedGradKernel(int DIM, bool GRAD_PHYS)
 {
    if (GRAD_PHYS) { return GetCollocatedGradKernel<Q_LAYOUT, true>(DIM); }
    else { return GetCollocatedGradKernel<Q_LAYOUT, false>(DIM); }
 }
-
-auto GetCollocatedGradKernel(int DIM, bool GRAD_PHYS, QVectorLayout Q_LAYOUT)
-{
-   if (Q_LAYOUT == QVectorLayout::byNODES)
-   {
-      return GetCollocatedGradKernel<QVectorLayout::byNODES>(
-                DIM, GRAD_PHYS);
-   }
-   else
-   {
-      return GetCollocatedGradKernel<QVectorLayout::byVDIM>(
-                DIM, GRAD_PHYS);
-   }
-}
 } // namespace
 
-template <int DIM, bool Integral>
-auto GetEvalKernelVDimFallback(int VDIM)
+template <int DIM, int VDIM, int ND, int NQ>
+EvalKernel QuadratureInterpolator::EvalKernels::Kernel()
 {
-   if constexpr (Integral)
-   {
-      using EvalKernels = QuadratureInterpolator::IntEvalKernels;
-      if (VDIM == 1)
-      {
-         return EvalKernels::Kernel<DIM, 1, 0, 0>();
-      }
-      else if (VDIM == 2)
-      {
-         return EvalKernels::Kernel<DIM, 2, 0, 0>();
-      }
-      else if (VDIM == 3)
-      {
-         return EvalKernels::Kernel<DIM, 3, 0, 0>();
-      }
-   }
-   if constexpr (!Integral)
-   {
-      using EvalKernels = QuadratureInterpolator::EvalKernels;
-      if (VDIM == 1)
-      {
-         return EvalKernels::Kernel<DIM, 1, 0, 0>();
-      }
-      else if (VDIM == 2)
-      {
-         return EvalKernels::Kernel<DIM, 2, 0, 0>();
-      }
-      else if (VDIM == 3)
-      {
-         return EvalKernels::Kernel<DIM, 3, 0, 0>();
-      }
-   }
+   using namespace internal::quadrature_interpolator;
+   if constexpr (DIM == 1) { return Eval1D; }
+   else if constexpr (DIM == 2) { return Eval2D<VDIM,ND,NQ>; }
+   else if constexpr (DIM == 3) { return Eval3D<VDIM,ND,NQ>; }
    MFEM_ABORT("");
 }
 
-template auto GetEvalKernelVDimFallback<1, true>(int VDIM);
-template auto GetEvalKernelVDimFallback<1, false>(int VDIM);
-template auto GetEvalKernelVDimFallback<2, true>(int VDIM);
-template auto GetEvalKernelVDimFallback<2, false>(int VDIM);
-template auto GetEvalKernelVDimFallback<3, true>(int VDIM);
-template auto GetEvalKernelVDimFallback<3, false>(int VDIM);
-
-QuadratureInterpolator::IntEvalKernelType
-QuadratureInterpolator::IntEvalKernels::Fallback(int DIM, int VDIM, int ND,
-                                                 int NQ)
+template <int DIM>
+EvalKernel GetEvalKernelVDimFallback(int VDIM)
 {
-   if (DIM == 1)
-   {
-      return GetEvalKernelVDimFallback<1, true>(VDIM);
-   }
-   else if (DIM == 2)
-   {
-      return GetEvalKernelVDimFallback<2, true>(VDIM);
-   }
-   else if (DIM == 3)
-   {
-      return GetEvalKernelVDimFallback<3, true>(VDIM);
-   }
-   else
-   {
-      MFEM_ABORT("");
-   }
+   using EvalKernels = QuadratureInterpolator::EvalKernels;
+   if (VDIM == 1) { return EvalKernels::Kernel<DIM,1,0,0>(); }
+   else if (VDIM == 2) { return EvalKernels::Kernel<DIM,2,0,0>(); }
+   else if (VDIM == 3) { return EvalKernels::Kernel<DIM,3,0,0>(); }
+   else { MFEM_ABORT(""); }
 }
 
-QuadratureInterpolator::EvalKernelType
-QuadratureInterpolator::EvalKernels::Fallback(int DIM, int VDIM, int ND, int NQ)
+EvalKernel QuadratureInterpolator::EvalKernels::Fallback(
+   int DIM, int VDIM, int ND, int NQ)
 {
-   if (DIM == 1)
-   {
-      return GetEvalKernelVDimFallback<1, false>(VDIM);
-   }
-   else if (DIM == 2)
-   {
-      return GetEvalKernelVDimFallback<2, false>(VDIM);
-   }
-   else if (DIM == 3)
-   {
-      return GetEvalKernelVDimFallback<3, false>(VDIM);
-   }
-   else
-   {
-      MFEM_ABORT("");
-   }
+   if (DIM == 1) { return GetEvalKernelVDimFallback<1>(VDIM); }
+   else if (DIM == 2) { return GetEvalKernelVDimFallback<2>(VDIM); }
+   else if (DIM == 3) { return GetEvalKernelVDimFallback<3>(VDIM); }
+   else { MFEM_ABORT(""); }
 }
 
-QuadratureInterpolator::IntTensorEvalKernelType
-QuadratureInterpolator::IntTensorEvalKernels::Fallback(int DIM,
-                                                       QVectorLayout Q_LAYOUT,
-                                                       int, int, int)
+TensorEvalKernel QuadratureInterpolator::TensorEvalKernels::Fallback(
+   int DIM, QVectorLayout Q_LAYOUT, int, int, int)
 {
-   if (Q_LAYOUT == QVectorLayout::byNODES)
-   {
-      return IntFallbackTensorEvalKernel<QVectorLayout::byNODES>(DIM);
-   }
-   else
-   {
-      return IntFallbackTensorEvalKernel<QVectorLayout::byVDIM>(DIM);
-   }
+   if (Q_LAYOUT == QVectorLayout::byNODES) { return FallbackTensorEvalKernel<QVectorLayout::byNODES>(DIM); }
+   else { return FallbackTensorEvalKernel<QVectorLayout::byVDIM>(DIM); }
 }
 
-QuadratureInterpolator::TensorEvalKernelType
-QuadratureInterpolator::TensorEvalKernels::Fallback(int DIM,
-                                                    QVectorLayout Q_LAYOUT, int,
-                                                    int, int)
-{
-   if (Q_LAYOUT == QVectorLayout::byNODES)
-   {
-      return FallbackTensorEvalKernel<QVectorLayout::byNODES>(DIM);
-   }
-   else
-   {
-      return FallbackTensorEvalKernel<QVectorLayout::byVDIM>(DIM);
-   }
-}
-
-QuadratureInterpolator::GradKernelType
-QuadratureInterpolator::GradKernels::Fallback(int DIM, QVectorLayout Q_LAYOUT,
-                                              bool GRAD_PHYS, int, int, int)
+GradKernel QuadratureInterpolator::GradKernels::Fallback(
+   int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int, int, int)
 {
    if (Q_LAYOUT == QVectorLayout::byNODES) { return GetGradKernel<QVectorLayout::byNODES>(DIM, GRAD_PHYS); }
    else { return GetGradKernel<QVectorLayout::byVDIM>(DIM, GRAD_PHYS); }
 }
 
-QuadratureInterpolator::CollocatedGradKernelType
-QuadratureInterpolator::CollocatedGradKernels::Fallback(int DIM,
-                                                        QVectorLayout Q_LAYOUT,
-                                                        bool GRAD_PHYS, int,
-                                                        int)
+CollocatedGradKernel QuadratureInterpolator::CollocatedGradKernels::Fallback(
+   int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int, int)
 {
-   return GetCollocatedGradKernel(DIM, GRAD_PHYS, Q_LAYOUT);
+   if (Q_LAYOUT == QVectorLayout::byNODES) { return GetCollocatedGradKernel<QVectorLayout::byNODES>(DIM, GRAD_PHYS); }
+   else { return GetCollocatedGradKernel<QVectorLayout::byVDIM>(DIM, GRAD_PHYS); }
 }
 
 /// @endcond
@@ -687,97 +806,98 @@ namespace quadrature_interpolator
 {
 void InitEvalKernels()
 {
+   using k = QuadratureInterpolator::EvalKernels;
    // 2D, VDIM = 1
-   QuadratureInterpolator::AddEvalSpecializations<2,1,1,1>();
-   QuadratureInterpolator::AddEvalSpecializations<2,1,1,4>();
+   k::Specialization<2,1,1,1>::Add();
+   k::Specialization<2,1,1,4>::Add();
    // Q1
-   QuadratureInterpolator::AddEvalSpecializations<2,1,4,4>();
-   QuadratureInterpolator::AddEvalSpecializations<2,1,4,9>();
+   k::Specialization<2,1,4,4>::Add();
+   k::Specialization<2,1,4,9>::Add();
    // Q2
-   QuadratureInterpolator::AddEvalSpecializations<2,1,9,9>();
-   QuadratureInterpolator::AddEvalSpecializations<2,1,9,16>();
+   k::Specialization<2,1,9,9>::Add();
+   k::Specialization<2,1,9,16>::Add();
    // Q3
-   QuadratureInterpolator::AddEvalSpecializations<2,1,16,16>();
-   QuadratureInterpolator::AddEvalSpecializations<2,1,16,25>();
-   QuadratureInterpolator::AddEvalSpecializations<2,1,16,36>();
+   k::Specialization<2,1,16,16>::Add();
+   k::Specialization<2,1,16,25>::Add();
+   k::Specialization<2,1,16,36>::Add();
    // Q4
-   QuadratureInterpolator::AddEvalSpecializations<2,1,25,25>();
-   QuadratureInterpolator::AddEvalSpecializations<2,1,25,36>();
-   QuadratureInterpolator::AddEvalSpecializations<2,1,25,49>();
-   QuadratureInterpolator::AddEvalSpecializations<2,1,25,64>();
+   k::Specialization<2,1,25,25>::Add();
+   k::Specialization<2,1,25,36>::Add();
+   k::Specialization<2,1,25,49>::Add();
+   k::Specialization<2,1,25,64>::Add();
 
    // 3D, VDIM = 1
    // Q0
-   QuadratureInterpolator::AddEvalSpecializations<3,1,1,1>();
-   QuadratureInterpolator::AddEvalSpecializations<3,1,1,8>();
+   k::Specialization<3,1,1,1>::Add();
+   k::Specialization<3,1,1,8>::Add();
    // Q1
-   QuadratureInterpolator::AddEvalSpecializations<3,1,8,8>();
-   QuadratureInterpolator::AddEvalSpecializations<3,1,8,27>();
+   k::Specialization<3,1,8,8>::Add();
+   k::Specialization<3,1,8,27>::Add();
    // Q2
-   QuadratureInterpolator::AddEvalSpecializations<3,1,27,27>();
-   QuadratureInterpolator::AddEvalSpecializations<3,1,27,64>();
+   k::Specialization<3,1,27,27>::Add();
+   k::Specialization<3,1,27,64>::Add();
    // Q3
-   QuadratureInterpolator::AddEvalSpecializations<3,1,64,64>();
-   QuadratureInterpolator::AddEvalSpecializations<3,1,64,125>();
-   QuadratureInterpolator::AddEvalSpecializations<3,1,64,216>();
+   k::Specialization<3,1,64,64>::Add();
+   k::Specialization<3,1,64,125>::Add();
+   k::Specialization<3,1,64,216>::Add();
    // Q4
-   QuadratureInterpolator::AddEvalSpecializations<3,1,125,125>();
-   QuadratureInterpolator::AddEvalSpecializations<3,1,125,216>();
+   k::Specialization<3,1,125,125>::Add();
+   k::Specialization<3,1,125,216>::Add();
 
    // 2D, VDIM = 3
    // Q0
-   QuadratureInterpolator::AddEvalSpecializations<2,3,1,1>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,1,4>();
+   k::Specialization<2,3,1,1>::Add();
+   k::Specialization<2,3,1,4>::Add();
    // Q1
-   QuadratureInterpolator::AddEvalSpecializations<2,3,4,4>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,4,9>();
+   k::Specialization<2,3,4,4>::Add();
+   k::Specialization<2,3,4,9>::Add();
    // Q2
-   QuadratureInterpolator::AddEvalSpecializations<2,3,9,4>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,9,9>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,9,16>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,9,25>();
+   k::Specialization<2,3,9,4>::Add();
+   k::Specialization<2,3,9,9>::Add();
+   k::Specialization<2,3,9,16>::Add();
+   k::Specialization<2,3,9,25>::Add();
    // Q3
-   QuadratureInterpolator::AddEvalSpecializations<2,3,16,16>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,16,25>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,16,36>();
+   k::Specialization<2,3,16,16>::Add();
+   k::Specialization<2,3,16,25>::Add();
+   k::Specialization<2,3,16,36>::Add();
    // Q4
-   QuadratureInterpolator::AddEvalSpecializations<2,3,25,25>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,25,36>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,25,49>();
-   QuadratureInterpolator::AddEvalSpecializations<2,3,25,64>();
+   k::Specialization<2,3,25,25>::Add();
+   k::Specialization<2,3,25,36>::Add();
+   k::Specialization<2,3,25,49>::Add();
+   k::Specialization<2,3,25,64>::Add();
 
    // 2D, VDIM = 2
    // Q1
-   QuadratureInterpolator::AddEvalSpecializations<2,2,4,4>();
-   QuadratureInterpolator::AddEvalSpecializations<2,2,4,9>();
+   k::Specialization<2,2,4,4>::Add();
+   k::Specialization<2,2,4,9>::Add();
    // Q2
-   QuadratureInterpolator::AddEvalSpecializations<2,2,9,9>();
-   QuadratureInterpolator::AddEvalSpecializations<2,2,9,16>();
+   k::Specialization<2,2,9,9>::Add();
+   k::Specialization<2,2,9,16>::Add();
    // Q3
-   QuadratureInterpolator::AddEvalSpecializations<2,2,16,16>();
-   QuadratureInterpolator::AddEvalSpecializations<2,2,16,25>();
-   QuadratureInterpolator::AddEvalSpecializations<2,2,16,36>();
+   k::Specialization<2,2,16,16>::Add();
+   k::Specialization<2,2,16,25>::Add();
+   k::Specialization<2,2,16,36>::Add();
    // Q4
-   QuadratureInterpolator::AddEvalSpecializations<2,2,25,25>();
-   QuadratureInterpolator::AddEvalSpecializations<2,2,25,36>();
-   QuadratureInterpolator::AddEvalSpecializations<2,2,25,49>();
-   QuadratureInterpolator::AddEvalSpecializations<2,2,25,64>();
+   k::Specialization<2,2,25,25>::Add();
+   k::Specialization<2,2,25,36>::Add();
+   k::Specialization<2,2,25,49>::Add();
+   k::Specialization<2,2,25,64>::Add();
 
    // 3D, VDIM = 3
    // Q1
-   QuadratureInterpolator::AddEvalSpecializations<3,3,8,8>();
-   QuadratureInterpolator::AddEvalSpecializations<3,3,8,27>();
+   k::Specialization<3,3,8,8>::Add();
+   k::Specialization<3,3,8,27>::Add();
    // Q2
-   QuadratureInterpolator::AddEvalSpecializations<3,3,27,27>();
-   QuadratureInterpolator::AddEvalSpecializations<3,3,27,64>();
-   QuadratureInterpolator::AddEvalSpecializations<3,3,27,125>();
+   k::Specialization<3,3,27,27>::Add();
+   k::Specialization<3,3,27,64>::Add();
+   k::Specialization<3,3,27,125>::Add();
    // Q3
-   QuadratureInterpolator::AddEvalSpecializations<3,3,64,64>();
-   QuadratureInterpolator::AddEvalSpecializations<3,3,64,125>();
-   QuadratureInterpolator::AddEvalSpecializations<3,3,64,216>();
+   k::Specialization<3,3,64,64>::Add();
+   k::Specialization<3,3,64,125>::Add();
+   k::Specialization<3,3,64,216>::Add();
    // Q4
-   QuadratureInterpolator::AddEvalSpecializations<3,3,125,125>();
-   QuadratureInterpolator::AddEvalSpecializations<3,3,125,216>();
+   k::Specialization<3,3,125,125>::Add();
+   k::Specialization<3,3,125,216>::Add();
 }
 
 } // namespace quadrature_Interpolator

fem/quadinterpolator.hpp

@@ -159,49 +159,26 @@ public:
    /// QuadratureInterpolator.
    static bool SupportsFESpace(const FiniteElementSpace &fespace);
 
-   // value map types
-   using TensorEvalKernelType = void (*)(const int ne, const real_t *B,
-                                         const real_t *e_vec, real_t *q_val,
-                                         const int vdim, const int nd,
-                                         const int nq);
-   using GradKernelType = void (*)(const int ne, const real_t *B,
-                                   const real_t *G, const real_t *J,
-                                   const real_t *e_vec, real_t *q_der,
-                                   const int s_dim, const int v_dim,
-                                   const int nd, const int nq);
-   using CollocatedGradKernelType = void (*)(const int ne, const real_t *G,
-                                             const real_t *J,
-                                             const real_t *e_vec, real_t *q_der,
-                                             const int sdim, const int vdim,
-                                             const int d1d);
-   using DetKernelType = void (*)(const int NE, const real_t *B,
-                                  const real_t *G, const real_t *e_vec,
-                                  real_t *q_det, const int nd, const int nq,
-                                  Vector *d_buffer);
-   using EvalKernelType = void (*)(const int NE, const int vdim,
-                                   const QVectorLayout q_layout,
-                                   const GeometricFactors *geom,
-                                   const DofToQuad &maps, const Vector &e_vec,
-                                   Vector &q_val, Vector &q_der, Vector &q_det,
-                                   const int eval_flags);
-
-   // integral map types
-   using IntTensorEvalKernelType = void (*)(const int ne, const real_t *B,
-                                            const real_t *detJ,
-                                            const real_t *e_vec, real_t *q_val,
-                                            const int vdim, const int nd,
-                                            const int nq);
-   using IntEvalKernelType =
-      void (*)(const int NE, const int vdim, const QVectorLayout q_layout,
-               const real_t *detJ, const GeometricFactors *geom,
-               const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
-               Vector &q_der, Vector &q_det, const int eval_flags);
-
+   using TensorEvalKernelType = void(*)(const int, const real_t *, const real_t *,
+                                        real_t *, const int, const int, const int);
+   using GradKernelType = void(*)(const int, const real_t *, const real_t *,
+                                  const real_t *, const real_t *, real_t *,
+                                  const int, const int, const int, const int);
+   using CollocatedGradKernelType = void(*)(const int, const real_t *,
+                                            const real_t *, const real_t *,
+                                            real_t *, const int, const int,
+                                            const int);
+   using DetKernelType = void(*)(const int NE, const real_t *, const real_t *,
+                                 const real_t *, real_t *, const int, const int,
+                                 Vector *);
+   using EvalKernelType = void(*)(const int, const int, const QVectorLayout,
+                                  const GeometricFactors *, const DofToQuad &,
+                                  const Vector &, Vector &, Vector &, Vector &,
+                                  const int);
    using TensorEvalHDivKernelType =
       void(*)(const int, const real_t *, const real_t *, const real_t *,
               const real_t *, real_t *, const int, const int);
 
-   // value-type mapping
    MFEM_REGISTER_KERNELS(TensorEvalKernels, TensorEvalKernelType,
                          (int, QVectorLayout, int, int, int), (int));
    MFEM_REGISTER_KERNELS(GradKernels, GradKernelType,
@@ -210,84 +187,8 @@ public:
    MFEM_REGISTER_KERNELS(EvalKernels, EvalKernelType, (int, int, int, int));
    MFEM_REGISTER_KERNELS(CollocatedGradKernels, CollocatedGradKernelType,
                          (int, QVectorLayout, bool, int, int), (int));
-
-   // integral-type mapping
-   MFEM_REGISTER_KERNELS(IntTensorEvalKernels, IntTensorEvalKernelType,
-                         (int, QVectorLayout, int, int, int), (int));
-   MFEM_REGISTER_KERNELS(IntEvalKernels, IntEvalKernelType, (int, int, int, int));
-
    MFEM_REGISTER_KERNELS(TensorEvalHDivKernels, TensorEvalHDivKernelType,
                          (int, QVectorLayout, unsigned, int, int));
-
-   /// Adds specializations for TensorEvalKernels
-   template <int DIM, QVectorLayout Q_LAYOUT, int VDIM, int D1D, int Q1D,
-             int NBZ = 0>
-   static void AddTensorEvalSpecializations()
-   {
-      if constexpr (NBZ)
-      {
-         IntTensorEvalKernels::Specialization<DIM, Q_LAYOUT, VDIM, D1D,
-                              Q1D>::template Opt<NBZ>::Add();
-         TensorEvalKernels::Specialization<DIM, Q_LAYOUT, VDIM, D1D,
-                           Q1D>::template Opt<NBZ>::Add();
-      }
-      else if constexpr (NBZ == 0)
-      {
-         IntTensorEvalKernels::Specialization<DIM, Q_LAYOUT, VDIM, D1D,
-                              Q1D>::Add();
-         TensorEvalKernels::Specialization<DIM, Q_LAYOUT, VDIM, D1D,
-                           Q1D>::Add();
-      }
-   }
-
-   /// Adds specializations for EvalKernels
-   template <int DIM, int VDIM, int ND, int NQ>
-   static void AddEvalSpecializations()
-   {
-      IntEvalKernels::Specialization<DIM, VDIM, ND, NQ>::Add();
-      EvalKernels::Specialization<DIM, VDIM, ND, NQ>::Add();
-   }
-
-   /// Adds specializations for GradKernels
-   template <int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int VDIM, int D1D,
-             int Q1D, int NBZ = 0>
-   static void AddGradSpecializations()
-   {
-      if constexpr (NBZ)
-      {
-         GradKernels::Specialization<DIM, Q_LAYOUT, GRAD_PHYS, VDIM, D1D,
-                     Q1D>::template Opt<NBZ>::Add();
-      }
-      else if constexpr (NBZ == 0)
-      {
-         GradKernels::Specialization<DIM, Q_LAYOUT, GRAD_PHYS, VDIM, D1D,
-                     Q1D>::Add();
-      }
-   }
-
-   /// Adds specializations for CollocatedGradKernels
-   template <int DIM, QVectorLayout Q_LAYOUT, bool GRAD_PHYS, int VDIM, int D1D,
-             int NBZ = 0>
-   static void AddCollocatedGradSpecializations()
-   {
-      if constexpr (NBZ)
-      {
-         CollocatedGradKernels::Specialization<DIM, Q_LAYOUT, GRAD_PHYS, VDIM,
-                               D1D>::template Opt<NBZ>::Add();
-      }
-      else if constexpr (NBZ == 0)
-      {
-         CollocatedGradKernels::Specialization<DIM, Q_LAYOUT, GRAD_PHYS, VDIM,
-                               D1D>::Add();
-      }
-   }
-
-   /// Adds specializations for DetKernels
-   template <int DIM, int SDIM, int D1D, int Q1D>
-   static void AddDetSpecializations()
-   {
-      DetKernels::Specialization<DIM, SDIM, D1D, Q1D>::Add();
-   }
 };
 
 }

general/backends.hpp

@@ -38,15 +38,6 @@
 #define CUB_IGNORE_DEPRECATED_CPP_DIALECT
 #define THRUST_IGNORE_DEPRECATED_CPP_DIALECT
 
-// MFEM only supports using RAJA/CAMP backends in default stream mode because
-// memory calls are performed outside of the RAJA ecosystem
-#ifndef CAMP_USE_PLATFORM_DEFAULT_STREAM
-#define CAMP_USE_PLATFORM_DEFAULT_STREAM 1
-#else
-#if !CAMP_USE_PLATFORM_DEFAULT_STREAM
-#error "MFEM only supports RAJA/CAMP with the default platform stream."
-#endif
-#endif
 #include "RAJA/RAJA.hpp"
 #if defined(RAJA_ENABLE_CUDA) && !defined(MFEM_USE_CUDA)
 #error When RAJA is built with CUDA, MFEM_USE_CUDA=YES is required
@@ -75,16 +66,6 @@ constexpr bool mfem_use_gpu = false;
 #define MFEM_THREAD_SIZE(k) 1
 #define MFEM_FOREACH_THREAD(i,k,N) for(int i=0; i<N; i++)
 #define MFEM_FOREACH_THREAD_DIRECT(i,k,N) MFEM_FOREACH_THREAD(i,k,N)
-// Assigns a thread block shaped (SX,SY,SZ) contiguous in x.
-#define MFEM_FOREACH_THREAD_DIRECT_3D(ix, iy, iz, k, SX, SY, SZ)               \
-   for (int iz = 0; iz < SZ; ++iz)                                             \
-      for (int iy = 0; iy < SY; ++iy)                                          \
-         for (int ix = 0; ix < SX; ++ix)
-// Assigns a thread block shaped (OX,OY,OZ) to work on items (SX,SY,SZ),
-// contiguous in x. This intentionally offsets threads
-#define MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(ix, iy, iz, k, SX, SY, SZ, OX,    \
-                                             OY, OZ)                           \
-   MFEM_FOREACH_THREAD_DIRECT_3D(ix, iy, iz, k, SX, SY, SZ)
 #endif
 
 // 'double' and 'float' atomicAdd implementation for previous versions of CUDA

general/cuda.hpp

@@ -49,16 +49,6 @@ constexpr bool mfem_use_gpu = true;
 #define MFEM_THREAD_SIZE(k) blockDim.k
 #define MFEM_FOREACH_THREAD(i,k,N) for(int i=threadIdx.k; i<N; i+=blockDim.k)
 #define MFEM_FOREACH_THREAD_DIRECT(i,k,N) if(const int i=threadIdx.k; i<N)
-// Assigns a thread block shaped (SX,SY,SZ) contiguous in x.
-#define MFEM_FOREACH_THREAD_DIRECT_3D(ix, iy, iz, k, SX, SY, SZ)               \
-   if (int ix = threadIdx.k % (SX), iy = threadIdx.k / (SX), iz = iy / (SY);   \
-       (iy %= (SY)), (threadIdx.k < (SX) * (SY) * (SZ)))
-// Assigns a thread block shaped (OX,OY,OZ) to work on items (SX,SY,SZ),
-// contiguous in x. This intentionally offsets threads
-#define MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(ix, iy, iz, k, SX, SY, SZ, OX,    \
-                                             OY, OZ)                           \
-   if (int ix = threadIdx.k % (OX), iy = threadIdx.k / (OX), iz = iy / (OY);   \
-       (ix < (SX)) && ((iy %= (OY)) < (SY)) && (iz < (SZ)))
 #endif // defined(__CUDA_ARCH__)
 #endif // defined(MFEM_USE_CUDA) && defined(__CUDACC__)
 

general/device.cpp

@@ -581,7 +581,9 @@ void Device::Setup(const std::string &device_option, const int device_id)
    if (Allows(Backend::CUDA)) { CudaDeviceSetup(dev, ngpu); }
    if (Allows(Backend::HIP)) { HipDeviceSetup(dev, ngpu); }
    if (Allows(Backend::RAJA_CUDA) || Allows(Backend::RAJA_HIP))
-   { RajaDeviceSetup(dev, ngpu); }
+   {
+      RajaDeviceSetup(dev, ngpu);
+   }
    // The check for MFEM_USE_OCCA is in the function OccaDeviceSetup().
    if (Allows(Backend::OCCA_MASK)) { OccaDeviceSetup(dev); }
    if (Allows(Backend::CEED_MASK))

general/device.hpp

@@ -16,6 +16,11 @@
 #include "globals.hpp"
 #include "mem_manager.hpp"
 
+#ifdef MFEM_USE_RAJA
+#include "RAJA/RAJA.hpp"
+#endif
+
+#include <memory>
 #include <string>
 
 namespace mfem
@@ -266,6 +271,18 @@ public:
    static inline bool Allows(unsigned long b_mask)
    { return Get().backends & b_mask; }
 
+#if defined(MFEM_USE_RAJA) &&                                                  \
+   (defined(RAJA_ENABLE_CUDA) || defined(RAJA_ENABLE_HIP))
+   static inline auto GetRajaResource()
+   {
+#if defined(RAJA_ENABLE_CUDA)
+      return RAJA::resources::Cuda::CudaFromStream(0, Get().GetId());
+#elif defined(RAJA_ENABLE_HIP)
+      return RAJA::resources::Hip::HipFromStream(0, Get().GetId());
+#endif
+   }
+#endif
+
    /** @brief Get the current Host MemoryType. This is the MemoryType used by
        most MFEM classes when allocating memory used on the host.
    */

general/error.cpp

@@ -171,14 +171,7 @@ void mfem_error(const char *msg)
 #ifdef MFEM_USE_EXCEPTIONS
    if (mfem_error_action == MFEM_ERROR_THROW)
    {
-      if (msg)
-      {
-         throw ErrorException(msg);
-      }
-      else
-      {
-         throw ErrorException("");
-      }
+      throw ErrorException(msg);
    }
 #endif
 

general/forall.hpp

@@ -317,8 +317,8 @@ template <typename DBODY>
 void RajaCuWrap1D(const int N, DBODY &&d_body)
 {
    //true denotes asynchronous kernel
-   RAJA::forall<RAJA::cuda_exec<MFEM_CUDA_BLOCKS,true>>(RAJA::RangeSegment(0,N),
-                                                        d_body);
+   RAJA::forall<RAJA::cuda_exec<MFEM_CUDA_BLOCKS, true> >(
+      Device::GetRajaResource(), RAJA::RangeSegment(0, N), d_body);
 }
 
 template <typename DBODY>
@@ -331,9 +331,9 @@ void RajaCuWrap2D(const int N, DBODY &&d_body,
    using namespace RAJA;
    using RAJA::RangeSegment;
 
-   launch<cuda_launch_policy>
-   (LaunchParams(Teams(G), Threads(X, Y, BZ)),
-    [=] RAJA_DEVICE (LaunchContext ctx)
+   launch<cuda_launch_policy>(Device::GetRajaResource(),
+                              LaunchParams(Teams(G), Threads(X, Y, BZ)),
+                              [=] RAJA_DEVICE(LaunchContext ctx)
    {
 
       loop<cuda_teams_x>(ctx, RangeSegment(0, G), [&] (const int n)
@@ -349,7 +349,6 @@ void RajaCuWrap2D(const int N, DBODY &&d_body,
          });
 
       });
-
    });
 
    MFEM_GPU_CHECK(cudaGetLastError());
@@ -365,9 +364,9 @@ void RajaCuWrap2DLaunchBounds(const int N, DBODY &&d_body, const int X,
    using namespace RAJA;
    using RAJA::RangeSegment;
 
-   launch<cuda_launch_bounds_policy<LB> >
-   (LaunchParams(Teams(G), Threads(X, Y, BZ)),
-    [=] RAJA_DEVICE(LaunchContext ctx)
+   launch<cuda_launch_bounds_policy<LB> >(
+      Device::GetRajaResource(), LaunchParams(Teams(G), Threads(X, Y, BZ)),
+      [=] RAJA_DEVICE(LaunchContext ctx)
    {
       loop<cuda_teams_x>(ctx, RangeSegment(0, G), [&] (const int n)
       {
@@ -390,13 +389,12 @@ void RajaCuWrap3D(const int N, DBODY &&d_body,
    using namespace RAJA;
    using RAJA::RangeSegment;
 
-   launch<cuda_launch_policy>
-   (LaunchParams(Teams(GRID), Threads(X, Y, Z)),
-    [=] RAJA_DEVICE (LaunchContext ctx)
+   launch<cuda_launch_policy>(Device::GetRajaResource(),
+                              LaunchParams(Teams(GRID), Threads(X, Y, Z)),
+                              [=] RAJA_DEVICE(LaunchContext ctx)
    {
 
       loop<cuda_teams_x>(ctx, RangeSegment(0, N), d_body);
-
    });
 
    MFEM_GPU_CHECK(cudaGetLastError());
@@ -410,12 +408,10 @@ void RajaCuWrap3DLaunchBounds(const int N, DBODY &&d_body,
    using namespace RAJA;
    using RAJA::RangeSegment;
 
-   launch<cuda_launch_bounds_policy<LB> >
-   (LaunchParams(Teams(GRID), Threads(X, Y, Z)),
-    [=] RAJA_DEVICE(LaunchContext ctx)
-   {
-      loop<cuda_teams_x>(ctx, RangeSegment(0, N), d_body);
-   });
+   launch<cuda_launch_bounds_policy<LB> >(
+      Device::GetRajaResource(), LaunchParams(Teams(GRID), Threads(X, Y, Z)),
+      [=] RAJA_DEVICE(LaunchContext ctx)
+   { loop<cuda_teams_x>(ctx, RangeSegment(0, N), d_body); });
    MFEM_GPU_CHECK(cudaGetLastError());
 }
 
@@ -484,8 +480,8 @@ template <typename DBODY>
 void RajaHipWrap1D(const int N, DBODY &&d_body)
 {
    //true denotes asynchronous kernel
-   RAJA::forall<RAJA::hip_exec<MFEM_HIP_BLOCKS,true>>(RAJA::RangeSegment(0,N),
-                                                      d_body);
+   RAJA::forall<RAJA::hip_exec<MFEM_HIP_BLOCKS,true> >(RAJA::RangeSegment(0,N),
+                                                       d_body);
 }
 
 template <typename DBODY>
@@ -498,9 +494,9 @@ void RajaHipWrap2D(const int N, DBODY &&d_body,
    using namespace RAJA;
    using RAJA::RangeSegment;
 
-   launch<hip_launch_policy>
-   (LaunchParams(Teams(G), Threads(X, Y, BZ)),
-    [=] RAJA_DEVICE (LaunchContext ctx)
+   launch<hip_launch_policy>(Device::GetRajaResource(),
+                             LaunchParams(Teams(G), Threads(X, Y, BZ)),
+                             [=] RAJA_DEVICE(LaunchContext ctx)
    {
 
       loop<hip_teams_x>(ctx, RangeSegment(0, G), [&] (const int n)
@@ -516,7 +512,6 @@ void RajaHipWrap2D(const int N, DBODY &&d_body,
          });
 
       });
-
    });
 
    MFEM_GPU_CHECK(hipGetLastError());
@@ -532,9 +527,9 @@ void RajaHipWrap2DLaunchBounds(const int N, DBODY &&d_body, const int X,
    using namespace RAJA;
    using RAJA::RangeSegment;
 
-   launch<hip_launch_bounds_policy<LB> >
-   (LaunchParams(Teams(G), Threads(X, Y, BZ)),
-    [=] RAJA_DEVICE(LaunchContext ctx)
+   launch<hip_launch_bounds_policy<LB> >(
+      Device::GetRajaResource(), LaunchParams(Teams(G), Threads(X, Y, BZ)),
+      [=] RAJA_DEVICE(LaunchContext ctx)
    {
       loop<hip_teams_x>(ctx, RangeSegment(0, G), [&] (const int n)
       {
@@ -557,13 +552,12 @@ void RajaHipWrap3D(const int N, DBODY &&d_body,
    using namespace RAJA;
    using RAJA::RangeSegment;
 
-   launch<hip_launch_policy>
-   (LaunchParams(Teams(GRID), Threads(X, Y, Z)),
-    [=] RAJA_DEVICE (LaunchContext ctx)
+   launch<hip_launch_policy>(Device::GetRajaResource(),
+                             LaunchParams(Teams(GRID), Threads(X, Y, Z)),
+                             [=] RAJA_DEVICE(LaunchContext ctx)
    {
 
       loop<hip_teams_x>(ctx, RangeSegment(0, N), d_body);
-
    });
 
    MFEM_GPU_CHECK(hipGetLastError());
@@ -577,12 +571,10 @@ void RajaHipWrap3DLaunchBounds(const int N, DBODY &&d_body, const int X,
    using namespace RAJA;
    using RAJA::RangeSegment;
 
-   launch<hip_launch_bounds_policy<LB> >
-   (LaunchParams(Teams(GRID), Threads(X, Y, Z)),
-    [=] RAJA_DEVICE(LaunchContext ctx)
-   {
-      loop<hip_teams_x>(ctx, RangeSegment(0, N), d_body);
-   });
+   launch<hip_launch_bounds_policy<LB> >(
+      Device::GetRajaResource(), LaunchParams(Teams(GRID), Threads(X, Y, Z)),
+      [=] RAJA_DEVICE(LaunchContext ctx)
+   { loop<hip_teams_x>(ctx, RangeSegment(0, N), d_body); });
    MFEM_GPU_CHECK(hipGetLastError());
 }
 

general/hip.hpp

@@ -51,18 +51,6 @@ constexpr bool mfem_use_gpu = true;
    for(int i=hipThreadIdx_ ##k; i<N; i+=hipBlockDim_ ##k)
 #define MFEM_FOREACH_THREAD_DIRECT(i,k,N) \
    if(const int i=hipThreadIdx_ ##k; i<N)
-// Assigns a thread block shaped (SX,SY,SZ) contiguous in x.
-#define MFEM_FOREACH_THREAD_DIRECT_3D(ix, iy, iz, k, SX, SY, SZ)               \
-   if (int ix = hipThreadIdx_##k % (SX), iy = hipThreadIdx_##k / (SX),         \
-       iz = iy / (SY);                                                         \
-       (iy %= (SY)), (hipThreadIdx_##k < (SX) * (SY) * (SZ)))
-// Assigns a thread block shaped (OX,OY,OZ) to work on items (SX,SY,SZ),
-// contiguous in x. This intentionally offsets threads
-#define MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(ix, iy, iz, k, SX, SY, SZ, OX,    \
-                                             OY, OZ)                           \
-   if (int ix = hipThreadIdx_##k % (OX), iy = hipThreadIdx_##k / (OX),         \
-       iz = iy / (OY);                                                         \
-       (ix < (SX)) && ((iy %= (OY)) < (SY)) && (iz < (SZ)))
 #endif // defined(__HIP_DEVICE_COMPILE__)
 #endif // defined(MFEM_USE_HIP) && defined(__HIP__)
 

general/optparser.cpp

@@ -148,26 +148,9 @@ void parseVector(char * str, Vector & var)
    }
 }
 
-bool parseEnumOption(const char *str, size_t &var,
-                     const std::vector<std::string> &options)
-{
-   for (size_t i = 0; i < options.size(); ++i)
-   {
-      auto &v = options[i];
-      if (v == str)
-      {
-         var = i;
-         // success
-         return true;
-      }
-   }
-   // failure
-   return false;
-}
-
 void OptionsParser::Parse()
 {
-   option_check.SetSize(options.size());
+   option_check.SetSize(options.Size());
    option_check = 0;
    for (int i = 1; i < argc; )
    {
@@ -178,9 +161,9 @@ void OptionsParser::Parse()
          return;
       }
 
-      for (size_t j = 0; true; j++)
+      for (int j = 0; true; j++)
       {
-         if (j >= options.size())
+         if (j >= options.Size())
          {
             // unrecognized option
             error_type = 2;
@@ -188,8 +171,8 @@ void OptionsParser::Parse()
             return;
          }
 
-         if (strcmp(argv[i], options[j].short_name.c_str()) == 0 ||
-             strcmp(argv[i], options[j].long_name.c_str()) == 0)
+         if (strcmp(argv[i], options[j].short_name) == 0 ||
+             strcmp(argv[i], options[j].long_name) == 0)
          {
             OptionType type = options[j].type;
 
@@ -241,11 +224,6 @@ void OptionsParser::Parse()
                case VECTOR:
                   parseVector(argv[i++], *(Vector*)(options[j].var_ptr) );
                   break;
-               case ENUM_OPTION:
-                  isValid =
-                     parseEnumOption(argv[i++], *(size_t *)(options[j].var_ptr),
-                                     options[j].options);
-                  break;
             }
 
             if (!isValid)
@@ -261,7 +239,7 @@ void OptionsParser::Parse()
    }
 
    // check for missing required options
-   for (size_t i = 0; i < options.size(); i++)
+   for (int i = 0; i < options.Size(); i++)
       if (options[i].required &&
           (option_check[i] == 0 ||
            (options[i].type == ENABLE && option_check[++i] == 0)))
@@ -345,12 +323,6 @@ void OptionsParser::WriteValue(const Option &opt, std::ostream &os)
          break;
       }
 
-      case ENUM_OPTION:
-      {
-         os << opt.options.at(*((size_t *)opt.var_ptr));
-         break;
-      }
-
       default: // provide a default to suppress warning
          break;
    }
@@ -361,7 +333,7 @@ void OptionsParser::PrintOptions(ostream &os) const
    static const char *indent = "   ";
 
    os << "Options used:\n";
-   for (size_t j = 0; j < options.size(); j++)
+   for (int j = 0; j < options.Size(); j++)
    {
       OptionType type = options[j].type;
 
@@ -383,7 +355,7 @@ void OptionsParser::PrintOptions(ostream &os) const
          os << options[j].long_name << " ";
          WriteValue(options[j], os);
       }
-      os << std::endl;
+      os << '\n';
    }
 }
 
@@ -438,13 +410,12 @@ void OptionsParser::PrintHelp(ostream &os) const
    static const char *line_sep = "";
    static const char *types[] = { " <int>", " <double>", " <string>",
                                   " <string>", "", "", " '<int>...'",
-                                  " '<double>...'",
-                                  " <string>",
+                                  " '<double>...'"
                                 };
 
    os << indent << "-h" << seprtr << "--help" << descr_sep
       << "Print this help message and exit.\n" << line_sep;
-   for (size_t j = 0; j < options.size(); j++)
+   for (int j = 0; j < options.Size(); j++)
    {
       OptionType type = options[j].type;
 
@@ -480,22 +451,9 @@ void OptionsParser::PrintHelp(ostream &os) const
       }
       os << descr_sep;
 
-      if (options[j].description.size())
+      if (options[j].description)
       {
-         os << options[j].description << std::endl;
-      }
-      if (options[j].type == ENUM_OPTION)
-      {
-         os << "\tavailable choices: ";
-         for (size_t i = 0; i < options[j].options.size(); ++i)
-         {
-            os << '"' << options[j].options[i] << '"';
-            if (i + 1 < options[j].options.size())
-            {
-               os << ", ";
-            }
-         }
-         os << std::endl;
+         os << options[j].description << '\n';
       }
       os << line_sep;
    }

general/optparser.hpp

@@ -15,9 +15,6 @@
 #include "../config/config.hpp"
 #include "array.hpp"
 
-#include <string>
-#include <vector>
-
 namespace mfem
 {
 
@@ -34,18 +31,17 @@ class Vector;
 class OptionsParser
 {
 public:
-   enum OptionType { INT, DOUBLE, STRING, STD_STRING, ENABLE, DISABLE, ARRAY, VECTOR, ENUM_OPTION };
+   enum OptionType { INT, DOUBLE, STRING, STD_STRING, ENABLE, DISABLE, ARRAY, VECTOR };
 
 private:
    struct Option
    {
       OptionType type;
       void *var_ptr;
-      std::string short_name;
-      std::string long_name;
-      std::string description;
+      const char *short_name;
+      const char *long_name;
+      const char *description;
       bool required;
-      std::vector<std::string> options;
 
       Option() = default;
 
@@ -53,19 +49,11 @@ private:
              const char *long_name_, const char *description_, bool req)
          : type(type_), var_ptr(var_ptr_), short_name(short_name_),
            long_name(long_name_), description(description_), required(req) { }
-
-      Option(size_t *var_ptr_, const char *short_name_, const char *long_name_,
-             const char *description_, std::vector<std::string> &&options_,
-             bool req)
-         : type(ENUM_OPTION), var_ptr(var_ptr_), short_name(short_name_),
-           long_name(long_name_), description(description_),
-           required(req), options(std::move(options_))
-      {}
    };
 
    int argc;
    char **argv;
-   std::vector<Option> options;
+   Array<Option> options;
    Array<int> option_check;
    // error_type can be:
    //  0 - no error
@@ -96,26 +84,26 @@ public:
                   const char *disable_long_name, const char *description,
                   bool required = false)
    {
-      options.emplace_back(ENABLE, var, enable_short_name, enable_long_name,
-                           description, required);
-      options.emplace_back(DISABLE, var, disable_short_name, disable_long_name,
-                           description, required);
+      options.Append(Option(ENABLE, var, enable_short_name, enable_long_name,
+                            description, required));
+      options.Append(Option(DISABLE, var, disable_short_name, disable_long_name,
+                            description, required));
    }
 
    /// Add an integer option and set 'var' to receive the value.
    void AddOption(int *var, const char *short_name, const char *long_name,
                   const char *description, bool required = false)
    {
-      options.emplace_back(INT, var, short_name, long_name, description,
-                           required);
+      options.Append(Option(INT, var, short_name, long_name, description,
+                            required));
    }
 
    /// Add a double option and set 'var' to receive the value.
    void AddOption(real_t *var, const char *short_name, const char *long_name,
                   const char *description, bool required = false)
    {
-      options.emplace_back(DOUBLE, var, short_name, long_name, description,
-                           required);
+      options.Append(Option(DOUBLE, var, short_name, long_name, description,
+                            required));
    }
 
    /// Add a string (char*) option and set 'var' to receive the value.
@@ -123,8 +111,8 @@ public:
                   const char *long_name, const char *description,
                   bool required = false)
    {
-      options.emplace_back(STRING, var, short_name, long_name, description,
-                           required);
+      options.Append(Option(STRING, var, short_name, long_name, description,
+                            required));
    }
 
    /// Add a string (std::string) option and set 'var' to receive the value.
@@ -132,8 +120,8 @@ public:
                   const char *long_name, const char *description,
                   bool required = false)
    {
-      options.emplace_back(STD_STRING, var, short_name, long_name, description,
-                           required);
+      options.Append(Option(STD_STRING, var, short_name, long_name, description,
+                            required));
    }
 
    /** Add an integer array (separated by spaces) option and set 'var' to
@@ -142,8 +130,8 @@ public:
                   const char *long_name, const char *description,
                   bool required = false)
    {
-      options.emplace_back(ARRAY, var, short_name, long_name, description,
-                           required);
+      options.Append(Option(ARRAY, var, short_name, long_name, description,
+                            required));
    }
 
    /** Add a vector (doubles separated by spaces) option and set 'var' to
@@ -152,20 +140,8 @@ public:
                   const char *long_name, const char *description,
                   bool required = false)
    {
-      options.emplace_back(VECTOR, var, short_name, long_name, description,
-                           required);
-   }
-
-   /**
-    * Add an option which must be one of a given list of options
-    */
-   void AddOptionChoice(size_t *index, const char *short_name,
-                        const char *long_name, const char *description,
-                        std::vector<std::string> options_,
-                        bool required = false)
-   {
-      options.emplace_back(index, short_name, long_name, description,
-                           std::move(options_), required);
+      options.Append(Option(VECTOR, var, short_name, long_name, description,
+                            required));
    }
 
    /** @brief Parse the command-line options.

linalg/cudss.cpp

@@ -15,10 +15,20 @@
 
 #ifdef MFEM_USE_CUDSS
 
+#if CUDSS_VERSION >= 800
 #ifdef MFEM_USE_SINGLE
-#define CUDA_REAL_T CUDA_R_32F
+#define CUDSS_REAL_T CUDSS_R_32F
 #else
-#define CUDA_REAL_T CUDA_R_64F
+#define CUDSS_REAL_T CUDSS_R_64F
+#endif
+#define CUDSS_INT_T CUDSS_R_32I
+#else
+#ifdef MFEM_USE_SINGLE
+#define CUDSS_REAL_T CUDA_R_32F
+#else
+#define CUDSS_REAL_T CUDA_R_64F
+#endif
+#define CUDSS_INT_T CUDA_R_32I
 #endif
 
 // Define a cuDSS error check macro, MFEM_CUDSS_CHECK(x), where x returns/is of
@@ -65,8 +75,13 @@ CuDSSSolver::CuDSSSolver(MPI_Comm comm_) : mpi_comm(comm_)
 #endif
    MFEM_CUDSS_CHECK(cudssSetCommLayer(handle, comm_lib));
 
+#if CUDSS_VERSION >= 800
+   MFEM_CUDSS_CHECK(cudssDataSet(handle, solverData, CUDSS_DATA_COMM_HOST,
+                                 &mpi_comm, sizeof(MPI_Comm *)));
+#else
    MFEM_CUDSS_CHECK(cudssDataSet(handle, solverData, CUDSS_DATA_COMM,
                                  &mpi_comm, sizeof(MPI_Comm *)));
+#endif
 }
 #endif // MFEM_USE_MPI
 
@@ -257,11 +272,19 @@ void CuDSSSolver::SetMatrixCuDSS(int *csr_offsets, int *csr_columns,
          CuMemcpyDtoD(csr_offsets_d, csr_offsets, (n_loc + 1) * sizeof(int));
          CuMemcpyDtoD(csr_columns_d, csr_columns, nnz * sizeof(int));
 
+#if CUDSS_VERSION >= 800
          MFEM_CUDSS_CHECK(
             cudssMatrixCreateCsr(
                Ac.get(), n_global, n_global, nnz, csr_offsets_d, NULL,
-               csr_columns_d, csr_values_d, CUDA_R_32I, CUDA_REAL_T, mat_type, mview,
-               CUDSS_BASE_ZERO));
+               csr_columns_d, csr_values_d, CUDSS_INT_T, CUDSS_INT_T, CUDSS_REAL_T,
+               mat_type, mview, CUDSS_BASE_ZERO));
+#else
+         MFEM_CUDSS_CHECK(
+            cudssMatrixCreateCsr(
+               Ac.get(), n_global, n_global, nnz, csr_offsets_d, NULL,
+               csr_columns_d, csr_values_d, CUDSS_INT_T, CUDSS_REAL_T,
+               mat_type, mview, CUDSS_BASE_ZERO));
+#endif
       }
       else    // !reorder_reuse
       {
@@ -269,11 +292,19 @@ void CuDSSSolver::SetMatrixCuDSS(int *csr_offsets, int *csr_columns,
          {
             MFEM_CUDSS_CHECK(cudssMatrixDestroy(*Ac));
          }
+#if CUDSS_VERSION >= 800
          MFEM_CUDSS_CHECK(
             cudssMatrixCreateCsr(
-               Ac.get(), n_global, n_global, nnz, csr_offsets, NULL, csr_columns,
-               csr_values_d, CUDA_R_32I, CUDA_REAL_T, mat_type, mview,
-               CUDSS_BASE_ZERO));
+               Ac.get(), n_global, n_global, nnz, csr_offsets, NULL,
+               csr_columns, csr_values_d, CUDSS_INT_T, CUDSS_INT_T, CUDSS_REAL_T,
+               mat_type, mview, CUDSS_BASE_ZERO));
+#else
+         MFEM_CUDSS_CHECK(
+            cudssMatrixCreateCsr(
+               Ac.get(), n_global, n_global, nnz, csr_offsets, NULL,
+               csr_columns, csr_values_d, CUDSS_INT_T, CUDSS_REAL_T,
+               mat_type, mview, CUDSS_BASE_ZERO));
+#endif
       }
 #ifdef MFEM_USE_MPI
       if (Mpi::IsInitialized())
@@ -334,10 +365,10 @@ void CuDSSSolver::SetNumRHS(int nrhs_) const
       }
       // Create empty RHS and solution vectors
       MFEM_CUDSS_CHECK(cudssMatrixCreateDn(&xc, n_global, nrhs_, n_global, NULL,
-                                           CUDA_REAL_T, CUDSS_LAYOUT_COL_MAJOR));
+                                           CUDSS_REAL_T, CUDSS_LAYOUT_COL_MAJOR));
 
       MFEM_CUDSS_CHECK(cudssMatrixCreateDn(&yc, n_global, nrhs_, n_global, NULL,
-                                           CUDA_REAL_T, CUDSS_LAYOUT_COL_MAJOR));
+                                           CUDSS_REAL_T, CUDSS_LAYOUT_COL_MAJOR));
 
 #ifdef MFEM_USE_MPI
       MFEM_CUDSS_CHECK(cudssMatrixSetDistributionRow1d(xc, row_start, row_end));

linalg/ginkgo.hpp

@@ -39,6 +39,7 @@ namespace Ginkgo
 {
 
 template <typename T> using gko_array = gko::array<T>;
+#if defined(MFEM_USE_MPI) && GINKGO_BUILD_MPI
 // for inter-operability with hypre integer types
 using gko_hypre_int =
    std::conditional_t<sizeof(HYPRE_Int) == sizeof(std::int32_t), std::int32_t,
@@ -50,6 +51,7 @@ static_assert(!std::is_void_v<gko_hypre_int>,
               "HYPRE_Int type is incompatible with Ginkgo");
 static_assert(!std::is_void_v<gko_hypre_bigint>,
               "HYPRE_BigInt type is incompatible with Ginkgo");
+#endif
 
 /**
 * Helper class for a case where a wrapped MFEM Vector

linalg/hypre.cpp

@@ -2872,8 +2872,8 @@ void HypreParMatrix::Destroy()
    if (HypreUsingGPU() && ParCSROwner && (diagOwner < 0 || offdOwner < 0))
    {
       // Put the "host" or "hypre" pointers in {i,j,data} of A->{diag,offd}, so
-      // that they can be destroyed by hypre when hypre_ParCSRMatrixDestroy(A)
-      // is called below.
+      // that they can be destroyed by mfem_hypre_TFree_host() or hypre when
+      // hypre_ParCSRMatrixDestroy(A) is called below, respectively.
 
       // Check that if both diagOwner and offdOwner are negative then they have
       // the same value.
@@ -2882,7 +2882,33 @@ void HypreParMatrix::Destroy()
 
       MemoryClass mc = (diagOwner == -1 || offdOwner == -1) ?
                        Device::GetHostMemoryClass() : GetHypreMemoryClass();
-      Write(mc, diagOwner < 0, offdOwner <0);
+      Write(mc, diagOwner < 0, offdOwner < 0);
+      if (diagOwner == -1)
+      {
+         // Note: mfem_hypre_TFree_host() sets the pointer to NULL.
+         mfem_hypre_TFree_host(hypre_CSRMatrixI(A->diag));
+         if (hypre_CSRMatrixOwnsData(A->diag))
+         {
+            mfem_hypre_TFree_host(hypre_CSRMatrixJ(A->diag));
+            mfem_hypre_TFree_host(hypre_CSRMatrixData(A->diag));
+         }
+#if MFEM_HYPRE_VERSION >= 21800
+         hypre_CSRMatrixMemoryLocation(A->diag) = GetHypreMemoryLocation();
+#endif
+      }
+      if (offdOwner == -1)
+      {
+         // Note: mfem_hypre_TFree_host() sets the pointer to NULL.
+         mfem_hypre_TFree_host(hypre_CSRMatrixI(A->offd));
+         if (hypre_CSRMatrixOwnsData(A->offd))
+         {
+            mfem_hypre_TFree_host(hypre_CSRMatrixJ(A->offd));
+            mfem_hypre_TFree_host(hypre_CSRMatrixData(A->offd));
+         }
+#if MFEM_HYPRE_VERSION >= 21800
+         hypre_CSRMatrixMemoryLocation(A->offd) = GetHypreMemoryLocation();
+#endif
+      }
    }
 #endif
 

linalg/hypre.hpp

@@ -432,10 +432,11 @@ private:
    // and A->col_map_offd.
    // The possible values for diagOwner are:
    //  -1: no special treatment of A->diag (default)
-   //      when hypre is built with CUDA support, A->diag owns the "host"
-   //      pointers (according to A->diag->owns_data)
-   //  -2: used when hypre is built with CUDA support, A->diag owns the "hypre"
-   //      pointers (according to A->diag->owns_data)
+   //      when hypre is using GPU, A->diag owns the "host" pointers (according
+   //      to A->diag->owns_data); these host pointers are freed by MFEM using
+   //      hypre's host deallocation macros
+   //  -2: used when hypre is using GPU, A->diag owns the "hypre" pointers
+   //      (according to A->diag->owns_data)
    //   0: prevent hypre from destroying A->diag->{i,j,data}
    //   1: same as 0, plus own the "host" A->diag->{i,j}
    //   2: same as 0, plus own the "host" A->diag->data

linalg/particlevector.cpp

@@ -10,6 +10,7 @@
 // CONTRIBUTING.md for details.
 
 #include "particlevector.hpp"
+#include "../general/forall.hpp"
 
 namespace mfem
 {
@@ -46,20 +47,38 @@ void ParticleVector::GetValues(int i, Vector &nvals) const
 {
    nvals.SetSize(vdim);
 
-   if (ordering == Ordering::byNODES)
+   const bool nvals_use_dev = nvals.UseDevice();
+   // Use ParticleVector's device flag to minimize movement from large source
+   const bool use_dev = UseDevice();
+   const auto d_src = Read(use_dev);
+   auto d_dest = nvals.Write(use_dev);
+
+   const int vdim_ = vdim;
+   const int ordering_ = (int)ordering;
+   const int nv = (ordering == Ordering::byNODES) ? size / vdim : 0;
+
+   mfem::forall_switch(use_dev, vdim_, [=] MFEM_HOST_DEVICE (int c)
    {
-      int nv = GetNumParticles();
-      for (int c = 0; c < vdim; c++)
+      if (ordering_ == Ordering::byNODES)
       {
-         nvals[c] = Vector::operator[](i+nv*c);
+         d_dest[c] = d_src[i + nv*c];
       }
+      else
+      {
+         d_dest[c] = d_src[c + vdim_*i];
+      }
+   });
+
+   // If nvals was not using device but ParticleVector is, copy back to host
+   if (!nvals_use_dev && use_dev)
+   {
+      nvals.HostRead();
+      nvals.UseDevice(false);
    }
-   else
+   // If nvals was using device but ParticleVector is not, copy back to device
+   if (!use_dev && nvals_use_dev)
    {
-      for (int c = 0; c < vdim; c++)
-      {
-         nvals[c] = Vector::operator[](c+vdim*i);
-      }
+      nvals.Read();
    }
 }
 
@@ -99,21 +118,27 @@ void ParticleVector::GetComponentsRef(int vd, Vector &nref)
 
 void ParticleVector::SetValues(int i, const Vector &nvals)
 {
-   if (ordering == Ordering::byNODES)
+   const bool use_dev = UseDevice(); // use ParticleVector's device flag
+   const auto mc = use_dev ? Device::GetDeviceMemoryClass()
+                   : Device::GetHostMemoryClass();
+   auto d_dest = ReadWrite(use_dev);
+   const auto d_src = nvals.GetMemory().Read(mc, nvals.Size());
+
+   const int vdim_ = vdim;
+   const int ordering_ = (int)ordering;
+   const int nv = (ordering == Ordering::byNODES) ? size / vdim : 0;
+
+   mfem::forall_switch(use_dev, vdim_, [=] MFEM_HOST_DEVICE (int c)
    {
-      int nv = GetNumParticles();
-      for (int c = 0; c < vdim; c++)
+      if (ordering_ == Ordering::byNODES)
       {
-         Vector::operator[](i + c*nv) = nvals[c];
+         d_dest[i + c*nv] = d_src[c];
       }
-   }
-   else
-   {
-      for (int c = 0; c < vdim; c++)
+      else
       {
-         Vector::operator[](c + i*vdim) = nvals[c];
+         d_dest[c + i*vdim_] = d_src[c];
       }
-   }
+   });
 }
 
 void ParticleVector::SetComponents(int vd, const Vector &comp)
@@ -144,6 +169,9 @@ real_t& ParticleVector::operator()(int i, int comp)
                "Component index " << comp <<
                " is invalid for vector dimension " << vdim);
 
+   // non-const so we make host flag valid in case user modifies data
+   HostReadWrite();
+
    if (ordering == Ordering::byNODES)
    {
       return Vector::operator[](i + comp*GetNumParticles());
@@ -163,6 +191,8 @@ const real_t& ParticleVector::operator()(int i, int comp) const
                "Component index " << comp <<
                " is invalid for vector dimension " << vdim);
 
+   HostRead();
+
    if (ordering == Ordering::byNODES)
    {
       return Vector::operator[](i + comp*GetNumParticles());
@@ -240,9 +270,37 @@ void ParticleVector::SetVDim(int vdim_, bool keep_data)
 
 void ParticleVector::SetOrdering(Ordering::Type ordering_, bool keep_data)
 {
-   if (keep_data)
+   if (keep_data && ordering != ordering_)
    {
-      Ordering::Reorder(*this, vdim, ordering, ordering_);
+      int num_particles = GetNumParticles();
+      // create deep copy of old data that will be copied
+      Vector old_data(*this);
+
+      const bool use_dev = UseDevice();
+      const auto d_src = old_data.Read(use_dev);
+      auto d_dest = Write(use_dev);
+
+      const int vdim_ = vdim;
+      const int size_ = size;
+
+      if (ordering_ == Ordering::byNODES) // byVDIM -> byNODES
+      {
+         mfem::forall_switch(use_dev, size_, [=] MFEM_HOST_DEVICE (int k)
+         {
+            int i = k / vdim_; // src particle index
+            int d = k % vdim_; // src component index
+            d_dest[i + d * num_particles] = d_src[k];
+         });
+      }
+      else // byNODES -> byVDIM
+      {
+         mfem::forall_switch(use_dev, size_, [=] MFEM_HOST_DEVICE (int k)
+         {
+            int d = k / num_particles; // src component index
+            int i = k % num_particles; // src particle index
+            d_dest[d + i * vdim_] = d_src[k];
+         });
+      }
    }
    ordering = ordering_;
 }
@@ -270,32 +328,47 @@ void ParticleVector::SetNumParticles(int num_vectors, bool keep_data)
 
       if (!keep_data) { return; }
 
+      const bool use_dev = UseDevice();
+      auto d_dest = this->ReadWrite(use_dev);
+
       if (ordering == Ordering::byNODES)
       {
-         // Shift entries for byNODES
-         for (int c = vdim-1; c > 0; c--)
-         {
-            for (int i = old_nv-1; i >= 0; i--)
-            {
-               Vector::operator[](i+c*num_vectors) = Vector::operator[](i+c*old_nv);
-            }
-         }
+         // create deep copy of old data that will be copied
+         Vector old_slice;
+         old_slice.MakeRef(*this, 0, old_nv * vdim);
+         Vector old_copy(old_slice);
 
-         // Zero-out data now associated with new Vectors
-         for (int c = 0; c < vdim; c++)
+         const auto d_src = old_copy.Read(use_dev);
+         const int vdim_ = vdim;
+
+         // Shift entries for byNODES
+         mfem::forall_switch(use_dev, old_nv * vdim_,
+                             [=] MFEM_HOST_DEVICE (int k)
          {
-            for (int i = old_nv; i < num_vectors; i++)
-            {
-               Vector::operator[](i+c*num_vectors) = 0.0;
-            }
-         }
+            const int d = k / old_nv;
+            const int i = k % old_nv;
+            d_dest[i + d*num_vectors] = d_src[k];
+         });
+
+         // Zero-out new data slots
+         const int diff = num_vectors - old_nv;
+         mfem::forall_switch(use_dev, diff * vdim,
+                             [=] MFEM_HOST_DEVICE (int k)
+         {
+            const int d = k / diff;
+            const int i = k % diff;
+            d_dest[d * num_vectors + old_nv + i] = 0.0;
+         });
       }
       else // byVDIM
       {
-         for (int i = old_nv*vdim; i < num_vectors*vdim; i++)
+         const int start_idx = old_nv * vdim;
+         const int end_idx = num_vectors * vdim;
+         const int diff = end_idx - start_idx;
+         mfem::forall_switch(use_dev, diff, [=] MFEM_HOST_DEVICE (int i)
          {
-            data[i] = 0.0;
-         }
+            d_dest[start_idx + i] = 0.0;
+         });
       }
    }
    else // Else just remove the trailing vector data

miniapps/CMakeLists.txt

@@ -33,6 +33,7 @@ add_subdirectory(meshing)
 add_subdirectory(mtop)
 add_subdirectory(multidomain)
 add_subdirectory(nurbs)
+add_subdirectory(optprob)
 add_subdirectory(parelag)
 add_subdirectory(performance)
 add_subdirectory(plasma)

miniapps/common/particles_extras.cpp

@@ -126,6 +126,7 @@ void VisualizeParticles(socketstream &sock, const char* vishost, int visport,
    {
       Vector pcoords;
       pset.Coords().GetValues(i, pcoords);
+      pcoords.HostRead();
       if (dim == 2)
       {
          Add2DPoint(pcoords, particles_mesh, psize);
@@ -139,6 +140,7 @@ void VisualizeParticles(socketstream &sock, const char* vishost, int visport,
 
    FiniteElementSpace fes(&particles_mesh, &l2fec, 1);
    GridFunction gf(&fes);
+   gf.HostWrite();
 
    for (int i = 0; i < pset.GetNParticles(); i++)
    {
@@ -193,6 +195,7 @@ void ParticleTrajectories::AddSegmentStart()
    {
       Vector pcoords;
       pset.Coords().GetValues(i, pcoords);
+      pcoords.HostRead();
       segment_meshes.front().AddVertex(pcoords);
    }
 }
@@ -213,6 +216,7 @@ void ParticleTrajectories::SetSegmentEnd()
       {
          Vector pcoords;
          pset.Coords().GetValues(pidx, pcoords);
+         pcoords.HostRead();
          segment_meshes.front().AddVertex(pcoords);
       }
       else // Otherwise set its end vertex == start vertex

miniapps/electromagnetics/CMakeLists.txt

@@ -34,11 +34,6 @@ if (MFEM_USE_MPI)
    EXTRA_HEADERS maxwell_solver.hpp ${MFEM_MINIAPPS_COMMON_HEADERS}
    LIBRARIES mfem-common)
 
- add_mfem_miniapp(maxwell-gpu
-   MAIN maxwell_gpu.cpp
-   EXTRA_HEADERS ${MFEM_MINIAPPS_COMMON_HEADERS}
-   LIBRARIES mfem-common)
-
  if (MFEM_USE_GSLIB)
   add_mfem_miniapp(lorentz
     MAIN lorentz.cpp

miniapps/electromagnetics/electromagnetics.hpp

@@ -12,8 +12,6 @@
 #ifndef MFEM_ELECTROMAGNETICS_HPP
 #define MFEM_ELECTROMAGNETICS_HPP
 
-#include <cmath>
-
 namespace mfem
 {
 

miniapps/electromagnetics/lorentz.cpp

@@ -91,6 +91,7 @@ struct LorentzContext
    int nt = 1000;                  // number of timesteps
    int redist_interval = 5;        // redistribution interval
    int redist_mesh = 0;            // redistribution mesh: 0: E mesh, 1: B mesh
+   std::string device_config = "cpu";
 } ctx;
 
 /// This class implements the Boris algorithm as described in the article
@@ -130,7 +131,7 @@ protected:
 public:
 
    Boris(MPI_Comm comm, GridFunction *E_gf_, GridFunction *B_gf_,
-         int nparticles, Ordering::Type pdata_ordering);
+         int nparticles, Ordering::Type pdata_ordering, bool use_device);
 
    /// Find Particles in mesh corresponding to E and B fields
    void FindParticles();
@@ -139,9 +140,12 @@ public:
    /// right after FindParticles has been called.
    void EvaluateFieldsAtParticles();
 
-   /// Advance particles one time step using Boris algorithm
+   /// Advance particles one time step using Boris algorithm. Host version.
    void Step(real_t &t, real_t &dt);
 
+   /// Advance particles one time step using Boris algorithm. Device version.
+   void StepDevice(real_t &t, real_t &dt);
+
    /// Remove lost particles and return their indices
    Array<int> RemoveLostParticles();
 
@@ -235,6 +239,8 @@ int main(int argc, char *argv[])
    args.AddOption(&vis_interval, "-vf", "--vis-interval",
                   "GLVis visualization update after this many timesteps. "
                   "0 means no visualization.");
+   args.AddOption(&ctx.device_config, "-d", "--device",
+                  "Device configuration definition string.");
 
    args.Parse();
    if (!args.Good())
@@ -251,6 +257,10 @@ int main(int argc, char *argv[])
       args.PrintOptions(cout);
    }
 
+   Device device(ctx.device_config);
+   if (Mpi::Root()) { device.Print(); }
+   bool use_device = (ctx.device_config != "cpu") && Device::IsEnabled();
+
    std::unique_ptr<VisItDataCollection> E_dc, B_dc;
    ParGridFunction *E_gf = nullptr, *B_gf = nullptr;
    Vector bb_xmin, bb_xmax;
@@ -266,6 +276,7 @@ int main(int argc, char *argv[])
          return 1;
       }
       E_gf->ParFESpace()->GetParMesh()->GetBoundingBox(bb_xmin, bb_xmax, 2);
+      E_gf->UseDevice(use_device);
    }
 
    // Read B field if provided
@@ -280,6 +291,7 @@ int main(int argc, char *argv[])
       }
       Vector bb_xmint, bb_xmaxt;
       B_gf->ParFESpace()->GetParMesh()->GetBoundingBox(bb_xmint, bb_xmaxt, 2);
+      B_gf->UseDevice(use_device);
       if (ctx.E.coll_name != "")
       {
          // compute intersection of bounding boxes
@@ -302,10 +314,14 @@ int main(int argc, char *argv[])
    // Initialize particles
    int num_particles = ctx.npt/num_ranks +
                        (rank < (ctx.npt % num_ranks) ? 1 : 0);
-   Boris boris(MPI_COMM_WORLD, E_gf, B_gf, num_particles, ordering_type);
+   Boris boris(MPI_COMM_WORLD, E_gf, B_gf, num_particles, ordering_type,
+               use_device);
    InitializeChargedParticles(boris.GetParticles(), ctx.x_min, ctx.x_max,
                               ctx.p_min, ctx.p_max, ctx.m, ctx.q);
+
+   Array<int> removed_idxs_dummy;
    boris.FindParticles();
+   boris.Redistribute(ctx.redist_mesh, removed_idxs_dummy);
    boris.EvaluateFieldsAtParticles();
 
    real_t t = 0.0;
@@ -329,7 +345,14 @@ int main(int argc, char *argv[])
    for (int step = 1; step <= ctx.nt; step++)
    {
       // Step the Boris algorithm
-      boris.Step(t, dt);
+      if (use_device)
+      {
+         boris.StepDevice(t, dt);
+      }
+      else
+      {
+         boris.Step(t, dt);
+      }
       if (Mpi::Root())
       {
          mfem::out << "Step: " << step << " | Time: " << t << endl;
@@ -397,7 +420,7 @@ void Boris::ParticleStep(Particle &part, real_t &dt)
 }
 
 Boris::Boris(MPI_Comm comm, GridFunction *E_gf_, GridFunction *B_gf_,
-             int nparticles, Ordering::Type pdata_ordering)
+             int nparticles, Ordering::Type pdata_ordering, bool use_device)
    : E_gf(E_gf_),
      B_gf(B_gf_),
      E_finder(comm),
@@ -426,6 +449,7 @@ Boris::Boris(MPI_Comm comm, GridFunction *E_gf_, GridFunction *B_gf_,
    }
 
    int dim = E_mesh ? E_mesh->SpaceDimension() : B_mesh->SpaceDimension();
+   MFEM_VERIFY(dim == 3, "Only 3D meshes are currently supported.");
 
    pxB_.SetSize(dim); pm_.SetSize(dim); pp_.SetSize(dim);
 
@@ -435,7 +459,8 @@ Boris::Boris(MPI_Comm comm, GridFunction *E_gf_, GridFunction *B_gf_,
    Array<int> field_vdims({1, 1, dim, dim, dim});
 
    charged_particles = std::make_unique<ParticleSet>
-                       (comm, nparticles, dim, field_vdims, 0, pdata_ordering);
+                       (comm, nparticles, dim, field_vdims, 0, pdata_ordering,
+                        use_device);
 }
 
 void Boris::FindParticles()
@@ -481,7 +506,6 @@ void Boris::Step(real_t &t, real_t &dt)
 {
    // Interpolate E and B fields onto particles
    EvaluateFieldsAtParticles();
-
    // Individually step each particle. If all ParticleSet fields are ordered
    // byVDIM, we can use GetParticleRef for better performance.
    if (charged_particles->IsParticleRefValid())
@@ -509,6 +533,112 @@ void Boris::Step(real_t &t, real_t &dt)
    t += dt;
 }
 
+void Boris::StepDevice(real_t &t, real_t &dt)
+{
+   // Interpolate E and B fields onto particles
+   EvaluateFieldsAtParticles();
+   const int N = charged_particles->GetNParticles();
+   auto &X = charged_particles->Coords();
+   auto &M = charged_particles->Field(MASS);
+   auto &Q = charged_particles->Field(CHARGE);
+   auto &P = charged_particles->Field(MOM);
+   auto &E = charged_particles->Field(EFIELD);
+   auto &B = charged_particles->Field(BFIELD);
+
+   const int dim = X.GetVDim();
+
+   // Capture orderings for each field to ensure correct access
+   const bool byVDIM_X = (X.GetOrdering() == Ordering::byVDIM);
+   const bool byVDIM_P = (P.GetOrdering() == Ordering::byVDIM);
+   const bool byVDIM_E = (E.GetOrdering() == Ordering::byVDIM);
+   const bool byVDIM_B = (B.GetOrdering() == Ordering::byVDIM);
+
+   auto d_x = X.ReadWrite();
+   auto d_m = M.Read();
+   auto d_q = Q.Read();
+   auto d_p = P.ReadWrite();
+   auto d_e = E.Read();
+   auto d_b = B.Read();
+
+   mfem::forall(N, [=] MFEM_HOST_DEVICE (int i)
+   {
+      const real_t m = d_m[i];
+      const real_t q = d_q[i];
+
+      real_t x[3], p[3], e[3], b[3];
+      // Load data
+      for (int d = 0; d < dim; d++)
+      {
+         x[d] = d_x[byVDIM_X ? i * dim + d : i + d * N];
+         p[d] = d_p[byVDIM_P ? i * dim + d : i + d * N];
+         e[d] = d_e[byVDIM_E ? i * dim + d : i + d * N];
+         b[d] = d_b[byVDIM_B ? i * dim + d : i + d * N];
+      }
+
+      // Boris algorithm implementation
+      real_t pm[3], pxB[3], pp[3];
+
+      // Compute half of the contribution from q E
+      // pm = p + 0.5 * dt * q * e
+      for (int d = 0; d < dim; d++)
+      {
+         pm[d] = p[d] + (0.5 * dt * q) * e[d];
+      }
+
+      // Compute the contribution from q p x B
+      real_t B2 = 0.0;
+      for (int d = 0; d < dim; d++) { B2 += b[d] * b[d]; }
+
+      // ... along pm x B
+      // pxB = pm x b
+      pxB[0] = pm[1] * b[2] - pm[2] * b[1];
+      pxB[1] = pm[2] * b[0] - pm[0] * b[2];
+      pxB[2] = pm[0] * b[1] - pm[1] * b[0];
+
+      // pp = a1 * pxB
+      const real_t a1 = 4.0 * dt * q * m;
+      for (int d = 0; d < dim; d++) { pp[d] = a1 * pxB[d]; }
+
+      // ... along pm
+      // pp += a2 * pm
+      const real_t a2 = 4.0 * m * m - dt * dt * q * q * B2;
+      for (int d = 0; d < dim; d++) { pp[d] += a2 * pm[d]; }
+
+      // ... along B
+      real_t b_dot_pm = 0.0;
+      for (int d = 0; d < dim; d++) { b_dot_pm += b[d] * pm[d]; }
+      const real_t a3 = 2.0 * dt * dt * q * q * b_dot_pm;
+      // pp += a3 * b
+      for (int d = 0; d < dim; d++) { pp[d] += a3 * b[d]; }
+
+      // scale by common denominator
+      const real_t a4 = 4.0 * m * m + dt * dt * q * q * B2;
+      for (int d = 0; d < dim; d++) { pp[d] /= a4; }
+
+      // Update the momentum
+      // p = pp + 0.5 * dt * q * e
+      for (int d = 0; d < dim; d++)
+      {
+         p[d] = pp[d] + (0.5 * dt * q) * e[d];
+      }
+
+      // Update the position
+      // x += (dt / m) * p
+      // Store back to global arrays
+      for (int d = 0; d < dim; d++)
+      {
+         d_p[byVDIM_P ? i * dim + d : i + d * N] = p[d];
+         d_x[byVDIM_X ? i * dim + d : i + d * N] = x[d] + (dt / m) * p[d];
+      }
+   });
+
+   // Find updated particle locations in E and B field meshes
+   FindParticles();
+
+   // Update time
+   t += dt;
+}
+
 Array<int> Boris::RemoveLostParticles()
 {
    Array<int> lost_idxs;
@@ -617,6 +747,11 @@ void InitializeChargedParticles(ParticleSet &charged_particles,
    ParticleVector &M = charged_particles.Field(Boris::MASS);
    ParticleVector &Q = charged_particles.Field(Boris::CHARGE);
 
+   X.HostWrite();
+   P.HostWrite();
+   M.HostWrite();
+   Q.HostWrite();
+
    for (int i = 0; i < charged_particles.GetNParticles(); i++)
    {
       for (int d = 0; d < dim; d++)
@@ -643,4 +778,9 @@ void InitializeChargedParticles(ParticleSet &charged_particles,
       M(i) = m;
       Q(i) = q;
    }
+
+   X.Read();
+   P.Read();
+   M.Read();
+   Q.Read();
 }

miniapps/electromagnetics/maxwell_solver.cpp

@@ -308,7 +308,7 @@ MaxwellSolver::~MaxwellSolver()
    {
       int i = mit1->first;
       delete pcg_[i];
-      // delete diagScale_[i];
+      delete diagScale_[i];
       delete A1_[i];
       delete a1_[i];
    }
@@ -377,7 +377,7 @@ MaxwellSolver::Mult(const Vector &B, Vector &dEdt) const
 void
 MaxwellSolver::ImplicitSolve(real_t dt, const Vector &B, Vector &dEdt)
 {
-   const_cast<const MaxwellSolver *>(this)->implicitSolve(dt, B, dEdt);
+   implicitSolve(dt, B, dEdt);
 }
 
 void
@@ -420,12 +420,12 @@ MaxwellSolver::setupSolver(const int idt, const real_t dt) const
       a1_[idt]->Finalize();
       A1_[idt] = a1_[idt]->ParallelAssemble();
 
-      // diagScale_[idt] = new HypreDiagScale(*A1_[idt]);
+      diagScale_[idt] = new HypreDiagScale(*A1_[idt]);
       pcg_[idt] = new HyprePCG(*A1_[idt]);
       pcg_[idt]->SetTol(1.0e-12);
       pcg_[idt]->SetMaxIter(200);
       pcg_[idt]->SetPrintLevel(0);
-      // pcg_[idt]->SetPreconditioner(*diagScale_[idt]);
+      pcg_[idt]->SetPreconditioner(*diagScale_[idt]);
    }
 }
 

miniapps/gslib/findpts.cpp

@@ -49,6 +49,9 @@
 //    findpts -m ../../data/ref-square.mesh -o 2 -mo 1 -random 1 -surf
 //    findpts -m ../../data/ref-cube.mesh -o 2 -mo 1 -random 1 -surf
 //    findpts -m ../../data/square-disc-p2.mesh -o 4 -mo 2 -random 1 -surf
+// Surface meshes + bounding box size increase:
+//    findpts -m ../../data/square-disc-p2.mesh -o 4 -mo 2 -random 1 -surf -sabs 0.1
+//    findpts -m ../../data/tinyzoo-3d.mesh -o 4 -mo 2 -random 1 -surf -sabs 0.1
 
 #include "mfem.hpp"
 #include "../common/mfem-common.hpp"
@@ -109,6 +112,7 @@ int main (int argc, char *argv[])
    int randomization     = 0;
    int npt               = 100;
    bool surface          = false;
+   double surf_aabb_sz_inc = 0.0;
 
    // Parse command-line options.
    OptionsParser args(argc, argv);
@@ -150,6 +154,9 @@ int main (int argc, char *argv[])
    args.AddOption(&surface, "-surf", "--surface", "-no-surf",
                   "--no-surface",
                   "Extract surface mesh from volume mesh.");
+   args.AddOption(&surf_aabb_sz_inc, "-sabs", "--surface-aabb-size-inc",
+                  "Absolute AABB expansion applied to surface-search "
+                  "axis-aligned bounding boxes in FindPointsGSLIB surface meshes.");
 
    args.Parse();
    if (!args.Good())
@@ -384,8 +391,17 @@ int main (int argc, char *argv[])
 
    // Find and Interpolate FE function values on the desired points.
    Vector interp_vals(pts_cnt*vec_dim);
-   FindPointsGSLIB finder(*mesh);
-   finder.SetDistanceToleranceForPointsFoundOnBoundary(10);
+   FindPointsGSLIB finder;
+   if (surface && surf_aabb_sz_inc > 0.0)
+   {
+      Vector bb_size({surf_aabb_sz_inc});
+      finder.SetupSurfWithAABBExpansion(*mesh, bb_size);
+   }
+   else
+   {
+      finder.Setup(*mesh);
+      // finder.SetDistanceToleranceForPointsFoundOnBoundary(10);
+   }
    finder.SetL2AvgType(FindPointsGSLIB::NONE);
    finder.Interpolate(vxyz, field_vals, interp_vals, point_ordering);
    Array<unsigned int> code_out    = finder.GetCode();
@@ -424,7 +440,7 @@ int main (int argc, char *argv[])
         << "Searched points:     "   << pts_cnt
         << "\nFound points:        " << found
         << "\nMax interp error:    " << max_err
-        << "\nMax dist (of found): " << max_dist
+        << "\nMax dist^2 (of found): " << max_dist
         << "\nPoints not found:    " << not_found;
    if (randomization == 1)
    {

miniapps/gslib/pfindpts.cpp

@@ -48,11 +48,14 @@
 // Device runs:
 //    mpirun -np 2 pfindpts -m ../../data/inline-quad.mesh -o 3 -mo 2 -random 1 -d debug
 //    mpirun -np 2 pfindpts -m ../../data/amr-quad.mesh -rs 1 -o 4 -mo 2 -random 1 -npt 100 -d debug
-//    mpirun -np 2 pfindpts -m ../../data/inline-hex.mesh -o 3 -mo 2 -random 1 -d debug
+//    mpirun -np 2 pfindpts -m ../../data/inline-hex.mesh -o 3 -mo 2 -random 1 -d debug -ft 1
 // Surface meshes:
 //    mpirun -np 4 pfindpts -m ../../data/square-disc-p2.mesh -o 4 -mo 2 -vis -random 1 -surf
 //    mpirun -np 4 pfindpts -m ../../data/star-q3.mesh -o 6 -mo 3 -vis -random 1 -surf
 //    mpirun -np 4 pfindpts -m ../../data/fichera-q2.mesh -o 6 -mo 3 -vis -random 1 -surf
+// Surface meshes + bounding box size increase:
+//    mpirun -np 4 pfindpts -m ../../data/square-disc-p2.mesh -o 4 -mo 2 -vis -random 1 -surf -sabs 0.1
+//    mpirun -np 4 pfindpts -m ../../data/tinyzoo-3d.mesh -o 4 -mo 2 -vis -random 1 -surf -sabs 0.1
 
 #include "mfem.hpp"
 #include "../common/mfem-common.hpp"
@@ -102,6 +105,7 @@ int main (int argc, char *argv[])
    int randomization     = 0;
    int npt               = 100; //points per proc
    bool surface          = false;
+   double surf_aabb_sz_inc = 0.0;
 
    // Parse command-line options.
    OptionsParser args(argc, argv);
@@ -145,7 +149,9 @@ int main (int argc, char *argv[])
    args.AddOption(&surface, "-surf", "--surface", "-no-surf",
                   "--no-surface",
                   "Extract surface mesh from volume mesh.");
-
+   args.AddOption(&surf_aabb_sz_inc, "-sabs", "--surface-aabb-size-inc",
+                  "Absolute AABB expansion applied to surface-search "
+                  "axis-aligned bounding boxes in FindPointsGSLIB surface meshes.");
    args.Parse();
    if (!args.Good())
    {
@@ -343,7 +349,7 @@ int main (int argc, char *argv[])
             Geometry::GetRandomPoint(geom, ip);
             if (j < npt_face_per_elem)
             {
-               ip.x = 0.0; // force point to be on the face
+               ip.x = 0.0; // force point to be on a face
                npt_total_face++;
             }
             Vector pos_i(sdim);
@@ -373,8 +379,17 @@ int main (int argc, char *argv[])
 
    // Find and Interpolate FE function values on the desired points.
    Vector interp_vals(pts_cnt*vec_dim);
-   FindPointsGSLIB finder(pmesh);
-   finder.SetDistanceToleranceForPointsFoundOnBoundary(10);
+   FindPointsGSLIB finder;
+   if (surface && surf_aabb_sz_inc > 0.0)
+   {
+      Vector bb_size({surf_aabb_sz_inc});
+      finder.SetupSurfWithAABBExpansion(pmesh, bb_size);
+   }
+   else
+   {
+      finder.Setup(pmesh);
+   }
+   // finder.SetDistanceToleranceForPointsFoundOnBoundary(1e-10);
    // Enable GPU to CPU fallback for GPUData only if you are using an older
    // version of GSLIB.
    // finder.SetGPUtoCPUFallback(true);
@@ -456,10 +471,11 @@ int main (int argc, char *argv[])
            << "\nPoints on faces:      " << face_pts << " out of "
            << npt_total_face
            << "\nMax interp error:     " << max_error
-           << "\nMax dist (of found):  " << max_dist
+           << "\nMax dist^2 (of found): " << max_dist
            << endl;
    }
 
+
    delete fec;
 
    if (randomization != 0)

miniapps/optprob/problem.hpp

@@ -0,0 +1,204 @@
+#include "mfem.hpp"
+
+namespace mfem
+{
+
+class StackedOperator : public Operator
+{
+public:
+   StackedOperator(int m=0): Operator(0, m), offset{0} {}
+
+   virtual int AddOperator(Operator &op)
+   {
+      MFEM_VERIFY(!finalized, "Operator is finalized");
+      MFEM_VERIFY(op.Width() == width, "Operator width inconsistent");
+      offset.Append(op.Height());
+      ops.Append(&op);
+      return ops.Size()-1;
+   }
+
+   void Finalize()
+   {
+      MFEM_VERIFY(!finalized, "Operator already been finalized");
+      offset.PartialSum();
+      Array<int> col_offset({0, width});
+      blk_op.reset(new BlockOperator(offset, col_offset));
+      for (int i=0; i<ops.Size(); i++)
+      {
+         blk_op->SetBlock(i, 0, ops[i]);
+      }
+   }
+
+   bool IsFinalized() const { return finalized; }
+
+   BlockOperator &AsBlockOperator() const
+   {
+      MFEM_VERIFY(finalized, "Operator not finalized");
+      return *blk_op;
+   }
+
+   void Mult(const Vector &x, Vector &y) const override
+   {
+      MFEM_VERIFY(finalized, "Operator not finalized");
+      blk_op->Mult(x, y);
+   }
+
+   Operator &GetGradient(const Vector &x) const override
+   {
+      MFEM_VERIFY(finalized, "Operator not finalized");
+      if (!grad_op) { grad_op.reset(new ProblemGradient(*this)); }
+      grad_op->SetPoint(x);
+      return *grad_op;
+   }
+
+   Operator &GetGradient(const int i, const Vector &x) const
+   {
+      MFEM_VERIFY(finalized, "Operator not finalized");
+      return ops[i]->GetGradient(x);
+   }
+
+private:
+
+   class ProblemGradient : public Operator
+   {
+   public:
+      ProblemGradient(const StackedOperator &prob)
+         : Operator(prob.Width(), prob.Width())
+         , prob(prob)
+      {}
+      void SetPoint(const Vector &x) { x_ = x; }
+      void Mult(const Vector &x, Vector &y) const override
+      {
+         //
+      }
+   private:
+      const StackedOperator &prob;
+      Vector x_;
+   };
+
+protected:
+   bool finalized = false;
+   Array<int> offset;
+   Array<Operator *> ops;
+   std::unique_ptr<BlockOperator> blk_op;
+   mutable std::unique_ptr<ProblemGradient> grad_op;
+};
+
+class OptimProblem : public StackedOperator
+{
+   enum class ConstType
+   {
+      EQ, // equality constraint
+      LE, // less than or equal constraint
+   };
+
+   int AddOperator(Operator &op) override
+   {
+      MFEM_ABORT("Use SetObjective or AddConstraint to add operators to the optimization problem");
+      return -1;
+   }
+
+   int SetObjective(Operator &obj, int obj_idx=0)
+   {
+      MFEM_VERIFY(!finalized, "Operator is finalized");
+      MFEM_VERIFY(obj_blk_idx == -1, "Objective already set");
+      MFEM_VERIFY(obj_idx >= 0, "Objective index must be non-negative");
+      MFEM_VERIFY(obj_idx < ops.Size(), "Objective index out of bounds");
+      obj_loc_idx = obj_idx;
+      obj_blk_idx = StackedOperator::AddOperator(obj);
+      return obj_blk_idx;
+   }
+
+   int AddConstraint(Operator &con, ConstType type, int con_idx=0)
+   {
+      MFEM_VERIFY(!finalized, "Operator is finalized");
+      MFEM_VERIFY(con_idx >= 0, "Constraint index must be non-negative");
+      MFEM_VERIFY(con_idx < ops.Size(), "Constraint index out of bounds");
+      constraint_types.Append(type);
+      return StackedOperator::AddOperator(con);
+   }
+
+   void UpdateObjectiveIndex(int obj_block, int obj_loc_idx_=0)
+   {
+      MFEM_VERIFY(obj_block >= 0 && obj_block < ops.Size(),
+                  "Objective block index out of bounds");
+      MFEM_VERIFY(obj_loc_idx_ >= 0, "Objective index must be non-negative");
+      MFEM_VERIFY(obj_loc_idx_ < ops[obj_block]->Height(),
+                  "Objective index out of bounds");
+      obj_blk_idx = obj_block;
+      obj_loc_idx = obj_loc_idx_;
+   }
+
+   real_t GetEnergy(const Vector &x) const
+   {
+      MFEM_VERIFY(finalized, "Operator not finalized");
+      aux_y.SetSize(ops[obj_blk_idx]->Height());
+      ops[obj_blk_idx]->Mult(x, aux_y);
+      return aux_y(obj_loc_idx);
+   }
+
+   real_t Objective(const Vector &x) const
+   {
+      return GetEnergy(x);
+   }
+
+   void Mult(const Vector &x, Vector &y) const override
+   {
+      MFEM_VERIFY(finalized, "Operator not finalized");
+      blk_op->Mult(x, y);
+   }
+
+   ConstType GetConstraintType(int con_block) const
+   {
+      MFEM_VERIFY(con_block >= 0 && con_block < constraint_types.Size(),
+                  "Constraint block index out of bounds");
+      return constraint_types[con_block];
+   }
+
+   // @brief Set the lower bound for the optimization variables (dof)
+   // @param lb The lower bound vector (will be copied)
+   void SetDofLowerBound(const Vector &lb)
+   {
+      MFEM_VERIFY(lb.Size() == width, "Lower bound size mismatch");
+      dof_lb.UseDevice(true);
+      dof_lb.SetSize(width);
+      dof_lb = lb;
+   }
+
+   // @brief Set the upper bound for the optimization variables (dof)
+   // @param ub The upper bound vector (will be copied)
+   void SetDofUpperBound(const Vector &ub)
+   {
+      MFEM_VERIFY(ub.Size() == width, "Upper bound size mismatch");
+      dof_ub.SetSize(width);
+      dof_ub = ub;
+   }
+
+   // @brief Set the upper and lower bounds for the optimization variables (dof)
+   // @param lb The lower bound vector (will be copied)
+   // @param ub The upper bound vector (will be copied)
+   void SetDofBounds(const Vector &lb, const Vector &ub)
+   {
+      MFEM_VERIFY(lb.Size() == width, "Lower bound size mismatch");
+      MFEM_VERIFY(ub.Size() == width, "Upper bound size mismatch");
+      dof_lb.SetSize(width);
+      dof_lb = lb;
+      dof_ub.SetSize(width);
+      dof_ub = ub;
+   }
+   bool HasDofLowerBound() const { return dof_lb.Size() > 0; }
+   bool HasDofUpperBound() const { return dof_ub.Size() > 0; }
+   bool HasDofBounds() const { return dof_lb.Size() > 0 && dof_ub.Size() > 0; }
+
+private:
+   Array<ConstType> constraint_types;
+   int obj_blk_idx = -1;
+   int obj_loc_idx = -1;
+   mutable Vector aux_y;
+
+   Vector dof_lb;
+   Vector dof_ub;
+};
+
+}
+

miniapps/spde/spde_solver.hpp

@@ -83,7 +83,7 @@ real_t IntegrateBC(const ParGridFunction &x, const Array<int> &bdr,
 /// where A is
 /// A = div ( Theta(x) grad + Id ) u(x)
 /// and alpha is given as
-/// alpha = (2 nu + dim) / 2.
+/// alpha = (2 nu + dim) / 4.
 /// Theta (anisotropy tensor) and nu (smoothness) can be specified in the
 /// constructor. Traditionally, the SPDE method requires the specification of
 /// a white noise right hands side. SPDESolver accepts arbitrary right hand

tests/unit/fem/test_3d_bilininteg.cpp

@@ -5920,206 +5920,4 @@ TEST_CASE("3D Bilinear Div Div Integrators",
    }
 }
 
-TEST_CASE("3D Bilinear VectorFE Integrators PartialAssembly",
-          "[BilinearFormIntegrator]"
-          "[GPU]")
-{
-   auto order = GENERATE(1, 3);
-   CAPTURE(order);
-   int dim = 3;
-
-   FunctionCoefficient q3_coeff(q3);
-   VectorFunctionCoefficient F3_coeff(dim, F3);
-   MatrixFunctionCoefficient M3_coeff(dim, M3);
-
-   auto mesh_fname =
-      GENERATE("../../data/fichera-amr.mesh", "../../data/ball-nurbs.mesh");
-   CAPTURE(mesh_fname);
-   Mesh mesh(mesh_fname);
-   REQUIRE(mesh.Dimension() == dim);
-   REQUIRE(mesh.SpaceDimension() == dim);
-
-   // convert nurbs into piecewise-quadratic curved mesh
-   if (mesh.NURBSext)
-   {
-      mesh.UniformRefinement();
-      mesh.SetCurvature(2);
-   }
-
-   SECTION("H(curl) H(curl) Scalar Coeff")
-   {
-      ND_FECollection fec_nd(order, dim);
-      FiniteElementSpace fespace_nd(&mesh, &fec_nd);
-
-      BilinearForm bfa(&fespace_nd);
-      bfa.AddDomainIntegrator(new VectorFEMassIntegrator(q3_coeff));
-      bfa.Assemble();
-      bfa.Finalize();
-
-      BilinearForm bpa(&fespace_nd);
-      bpa.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      bpa.AddDomainIntegrator(new VectorFEMassIntegrator(q3_coeff));
-      bpa.Assemble();
-
-      GridFunction x(&fespace_nd), y_fa(&fespace_nd), y_pa(&fespace_nd);
-      x.Randomize(1234);
-      bfa.Mult(x, y_fa);
-      bpa.Mult(x, y_pa);
-      y_pa -= y_fa;
-      REQUIRE( y_pa.Normlinf() == MFEM_Approx(0_r) );
-   }
-
-   SECTION("H(curl) H(curl) Diagonal Matrix Coeff")
-   {
-      ND_FECollection fec_nd(order, dim);
-      FiniteElementSpace fespace_nd(&mesh, &fec_nd);
-
-      BilinearForm bfa(&fespace_nd);
-      bfa.AddDomainIntegrator(new VectorFEMassIntegrator(F3_coeff));
-      bfa.Assemble();
-      bfa.Finalize();
-
-      BilinearForm bpa(&fespace_nd);
-      bpa.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      bpa.AddDomainIntegrator(new VectorFEMassIntegrator(F3_coeff));
-      bpa.Assemble();
-
-      GridFunction x(&fespace_nd), y_fa(&fespace_nd), y_pa(&fespace_nd);
-      x.Randomize(1234);
-      bfa.Mult(x, y_fa);
-      bpa.Mult(x, y_pa);
-      y_pa -= y_fa;
-      REQUIRE( y_pa.Normlinf() == MFEM_Approx(0_r) );
-   }
-
-   SECTION("H(curl) H(curl) Matrix Coeff")
-   {
-      ND_FECollection fec_nd(order, dim);
-      FiniteElementSpace fespace_nd(&mesh, &fec_nd);
-
-      BilinearForm bfa(&fespace_nd);
-      bfa.AddDomainIntegrator(new VectorFEMassIntegrator(M3_coeff));
-      bfa.Assemble();
-      bfa.Finalize();
-
-      BilinearForm bpa(&fespace_nd);
-      bpa.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      bpa.AddDomainIntegrator(new VectorFEMassIntegrator(M3_coeff));
-      bpa.Assemble();
-
-      GridFunction x(&fespace_nd), y_fa(&fespace_nd), y_pa(&fespace_nd);
-      x.Randomize(1234);
-      bfa.Mult(x, y_fa);
-      bpa.Mult(x, y_pa);
-      y_pa -= y_fa;
-      REQUIRE( y_pa.Normlinf() == MFEM_Approx(0_r) );
-   }
-}
-
-TEST_CASE("3D Bilinear Weak Curl Integrators Partial Assembly",
-          "[MixedVectorWeakCurlIntegrator]"
-          "[BilinearFormIntegrator]"
-          "[GPU]")
-{
-   auto order = GENERATE(1, 3);
-   CAPTURE(order);
-   int dim = 3;
-
-   FunctionCoefficient q3_coeff(q3);
-   VectorFunctionCoefficient F3_coeff(dim, F3);
-
-   auto mesh_fname =
-      GENERATE("../../data/fichera-amr.mesh", "../../data/ball-nurbs.mesh");
-   CAPTURE(mesh_fname);
-   Mesh mesh(mesh_fname);
-   REQUIRE(mesh.Dimension() == dim);
-   REQUIRE(mesh.SpaceDimension() == dim);
-
-   // convert nurbs into piecewise-quadratic curved mesh
-   if (mesh.NURBSext)
-   {
-      mesh.UniformRefinement();
-      mesh.SetCurvature(2);
-   }
-
-   SECTION("H(div) H(curl) No Coeff")
-   {
-      ND_FECollection fec_nd(order, dim);
-      FiniteElementSpace fespace_nd(&mesh, &fec_nd);
-      RT_FECollection fec_rt(order - 1, dim);
-      FiniteElementSpace fespace_rt(&mesh, &fec_rt);
-
-      MixedBilinearForm bfa(&fespace_rt, &fespace_nd);
-      bfa.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator);
-      bfa.Assemble();
-      bfa.Finalize();
-
-      MixedBilinearForm bpa(&fespace_rt, &fespace_nd);
-      bpa.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      bpa.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator);
-      bpa.Assemble();
-
-      GridFunction x(&fespace_rt), y_fa(&fespace_nd), y_pa(&fespace_nd);
-      x.Randomize(1234);
-      REQUIRE(bfa.Height() == y_fa.Size());
-      REQUIRE(bfa.Width() == x.Size());
-      REQUIRE(bpa.Height() == y_fa.Size());
-      REQUIRE(bpa.Width() == x.Size());
-      bfa.Mult(x, y_fa);
-      bpa.Mult(x, y_pa);
-      y_pa -= y_fa;
-      REQUIRE( y_pa.Normlinf() == MFEM_Approx(0_r) );
-   }
-
-   SECTION("H(div) H(curl) Scalar Coeff")
-   {
-      ND_FECollection fec_nd(order, dim);
-      FiniteElementSpace fespace_nd(&mesh, &fec_nd);
-      RT_FECollection fec_rt(order - 1, dim);
-      FiniteElementSpace fespace_rt(&mesh, &fec_rt);
-
-      MixedBilinearForm bfa(&fespace_rt, &fespace_nd);
-      bfa.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(q3_coeff));
-      bfa.Assemble();
-      bfa.Finalize();
-
-      MixedBilinearForm bpa(&fespace_rt, &fespace_nd);
-      bpa.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      bpa.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(q3_coeff));
-      bpa.Assemble();
-
-      GridFunction x(&fespace_rt), y_fa(&fespace_nd), y_pa(&fespace_nd);
-      x.Randomize(1234);
-      bfa.Mult(x, y_fa);
-      bpa.Mult(x, y_pa);
-      y_pa -= y_fa;
-      REQUIRE( y_pa.Normlinf() == MFEM_Approx(0_r) );
-   }
-
-   SECTION("H(div) H(curl) Diagonal Matrix Coeff")
-   {
-      ND_FECollection fec_nd(order, dim);
-      FiniteElementSpace fespace_nd(&mesh, &fec_nd);
-      RT_FECollection fec_rt(order - 1, dim);
-      FiniteElementSpace fespace_rt(&mesh, &fec_rt);
-
-      MixedBilinearForm bfa(&fespace_rt, &fespace_nd);
-      bfa.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(F3_coeff));
-      bfa.Assemble();
-      bfa.Finalize();
-
-      MixedBilinearForm bpa(&fespace_rt, &fespace_nd);
-      bpa.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      bpa.AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(F3_coeff));
-      bpa.Assemble();
-
-      GridFunction x(&fespace_rt), y_fa(&fespace_nd), y_pa(&fespace_nd);
-      x.Randomize(1234);
-      bfa.Mult(x, y_fa);
-      bpa.Mult(x, y_pa);
-      y_pa -= y_fa;
-      REQUIRE( y_pa.Normlinf() == MFEM_Approx(0_r) );
-   }
-}
-
 } // namespace bilininteg_3d

tests/unit/fem/test_col_lag_der.cpp

@@ -39,15 +39,15 @@ TEST_CASE("Collocated Derivative Kernels", "[QuadratureInterpolator]")
 {
    // Add some specializations for the kernels
    // DIM, LAYOUT, PHYS, VDIM, D1D, Q1D
-   QuadratureInterpolator::AddGradSpecializations<1, QVectorLayout::byNODES,
-                          false, 1, 2, 2>();
-   QuadratureInterpolator::AddGradSpecializations<1, QVectorLayout::byNODES,
-                          true, 1, 2, 2>();
+   QuadratureInterpolator::GradKernels::Specialization
+   <1, QVectorLayout::byNODES, false, 1, 2, 2>::Add();
+   QuadratureInterpolator::GradKernels::Specialization
+   <1, QVectorLayout::byNODES, true, 1, 2, 2>::Add();
 
-   QuadratureInterpolator::AddCollocatedGradSpecializations<
-   1, QVectorLayout::byNODES, false, 1, 2>();
-   QuadratureInterpolator::AddCollocatedGradSpecializations<
-   1, QVectorLayout::byNODES, true, 1, 2>();
+   QuadratureInterpolator::CollocatedGradKernels::Specialization
+   <1, QVectorLayout::byNODES, false, 1, 2>::Add();
+   QuadratureInterpolator::CollocatedGradKernels::Specialization
+   <1, QVectorLayout::byNODES, true, 1, 2>::Add();
 
    const auto mesh_fname = GENERATE(
                               "../../data/inline-segment.mesh",
@@ -157,9 +157,9 @@ TEST_CASE("Collocated Derivative Kernels", "[QuadratureInterpolator]")
       const int nq = maps.nqpt;
 
       Vector qp_der(nelem*vdim*nqp*(P ? sdim : dim));
-      GK::Run(dim, L, P, vdim, nd, nq, nelem, maps.B.Read(), maps.G.Read(),
-              geom->J.Read(), evec_values.Read(), qp_der.Write(), sdim, vdim,
-              nd, nq);
+      GK::Run(dim, L, P, vdim, nd, nq, nelem, maps.B.Read(),
+              maps.G.Read(), geom->J.Read(), evec_values.Read(),
+              qp_der.Write(), sdim, vdim, nd, nq);
 
       Vector col_der(nelem*vdim*nqp*(P ? sdim : dim));
       CGK::Run(dim, L, P, vdim, nd, nelem, maps.G.Read(), geom->J.Read(),

tests/unit/fem/test_gslib.cpp

@@ -12,6 +12,8 @@
 #include "unit_tests.hpp"
 #include "mfem.hpp"
 
+#include <random>
+
 using namespace mfem;
 #ifdef MFEM_USE_GSLIB
 namespace gslib_test
@@ -36,6 +38,165 @@ void F_exact(const Vector &p, Vector &F)
 
 enum class Space { H1, L2 };
 
+enum class SurfaceMeshType { Segment2D, Segment3D, Quad3D, Tri3D };
+
+const char *SurfaceMeshName(const SurfaceMeshType type)
+{
+   switch (type)
+   {
+      case SurfaceMeshType::Segment2D: return "segment-2d";
+      case SurfaceMeshType::Segment3D: return "segment-3d";
+      case SurfaceMeshType::Quad3D:    return "quad-3d";
+      case SurfaceMeshType::Tri3D:     return "tri-3d";
+   }
+   return "unknown";
+}
+
+int SurfaceSpaceDim(const SurfaceMeshType type)
+{
+   switch (type)
+   {
+      case SurfaceMeshType::Segment2D: return 2;
+      case SurfaceMeshType::Segment3D: return 3;
+      case SurfaceMeshType::Quad3D:    return 3;
+      case SurfaceMeshType::Tri3D:     return 3;
+   }
+   return -1;
+}
+
+Mesh MakeSurfaceMesh(const SurfaceMeshType type, const int ne)
+{
+   switch (type)
+   {
+      case SurfaceMeshType::Segment2D:
+         return Mesh::MakeCartesian1D(ne);
+      case SurfaceMeshType::Segment3D:
+         return Mesh::MakeCartesian1D(ne);
+      case SurfaceMeshType::Quad3D:
+         return Mesh::MakeCartesian2D(ne, ne, Element::QUADRILATERAL);
+      case SurfaceMeshType::Tri3D:
+         return Mesh::MakeCartesian2D(ne, ne, Element::TRIANGLE);
+   }
+   MFEM_ABORT("Unknown surface mesh type.");
+   return Mesh();
+}
+
+void GetSurfaceInteriorPoints(Mesh &mesh, const int npt_per_el,
+                              const int ordering, Vector &xyz,
+                              const int p0 = 0)
+{
+   MFEM_VERIFY(mesh.GetNodes() != nullptr, "Mesh nodes are required.");
+
+   const int sdim = mesh.SpaceDimension();
+   const int npt = xyz.Size()/sdim;
+   MFEM_VERIFY((p0 + mesh.GetNE()*npt_per_el)*sdim <= xyz.Size(),
+               "Output vector is too small.");
+   Vector point(sdim);
+   std::mt19937 gen(123);
+   std::uniform_real_distribution<double> uni(0.01, 0.99);
+   int p = p0;
+
+   for (int e = 0; e < mesh.GetNE(); e++)
+   {
+      ElementTransformation *T = mesh.GetElementTransformation(e);
+      const Geometry::Type geom = mesh.GetElementBaseGeometry(e);
+      for (int j = 0; j < npt_per_el; j++)
+      {
+         IntegrationPoint ip;
+         real_t xv = uni(gen);
+         if (geom == Geometry::SEGMENT)
+         {
+            ip.x = xv;
+         }
+         else if (geom == Geometry::SQUARE)
+         {
+            ip.Set2(xv, uni(gen));
+         }
+         else
+         {
+            MFEM_VERIFY(geom == Geometry::TRIANGLE,
+                        "Unsupported surface element geometry.");
+            ip.Set2(xv, uni(gen)*(1.0 - xv));
+         }
+         T->Transform(ip, point);
+         for (int d = 0; d < sdim; d++)
+         {
+            const int idx = (ordering == Ordering::byNODES) ?
+                            d*npt + p :
+                            p*sdim + d;
+            xyz(idx) = point(d);
+         }
+         p++;
+      }
+   }
+}
+
+void GetSurfaceBoundaryPoints(Mesh &mesh, const int npt_per_el,
+                              const int ordering, Vector &xyz,
+                              const int p0 = 0)
+{
+   MFEM_VERIFY(mesh.GetNodes() != nullptr, "Mesh nodes are required.");
+
+   const int sdim = mesh.SpaceDimension();
+   const int npt = xyz.Size()/sdim;
+   MFEM_VERIFY((p0 + mesh.GetNE()*npt_per_el)*sdim <= xyz.Size(),
+               "Output vector is too small.");
+   Vector point(sdim);
+   std::mt19937 gen(246);
+   std::uniform_real_distribution<double> uni(0.01, 0.99);
+   int p = p0;
+
+   for (int e = 0; e < mesh.GetNE(); e++)
+   {
+      ElementTransformation *T = mesh.GetElementTransformation(e);
+      const Geometry::Type geom = mesh.GetElementBaseGeometry(e);
+      for (int j = 0; j < npt_per_el; j++)
+      {
+         IntegrationPoint ip;
+         if (geom == Geometry::SEGMENT)
+         {
+            MFEM_VERIFY(npt_per_el == 2,
+                        "Segment boundary sampling requires npt_per_el = 2.");
+            ip.x = (j == 0) ? 0.0 : 1.0;
+         }
+         else
+         {
+            const double t = uni(gen);
+            if (geom == Geometry::SQUARE)
+            {
+               switch (j % 4)
+               {
+                  case 0: ip.Set2(t, 0.0);     break;
+                  case 1: ip.Set2(1.0, t);     break;
+                  case 2: ip.Set2(t, 1.0);     break;
+                  case 3: ip.Set2(0.0, t);     break;
+               }
+            }
+            else
+            {
+               MFEM_VERIFY(geom == Geometry::TRIANGLE,
+                           "Unsupported surface element geometry.");
+               switch (j % 3)
+               {
+                  case 0: ip.Set2(t, 0.0);     break;
+                  case 1: ip.Set2(t, 1.0 - t); break;
+                  case 2: ip.Set2(0.0, t);     break;
+               }
+            }
+         }
+         T->Transform(ip, point);
+         for (int d = 0; d < sdim; d++)
+         {
+            const int idx = (ordering == Ordering::byNODES) ?
+                            d*npt + p :
+                            p*sdim + d;
+            xyz(idx) = point(d);
+         }
+         p++;
+      }
+   }
+}
+
 TEST_CASE("GSLIBInterpolate", "[GSLIBInterpolate][GSLIB]")
 {
    auto space               = GENERATE(Space::H1, Space::L2);
@@ -190,6 +351,92 @@ TEST_CASE("GSLIBInterpolate", "[GSLIBInterpolate][GSLIB]")
    delete c_fec;
 }
 
+TEST_CASE("GSLIBSurfInterpolate", "[GSLIBSurfInterpolate][GSLIB]")
+{
+   auto surface_mesh_type   = GENERATE(SurfaceMeshType::Segment2D,
+                                       SurfaceMeshType::Segment3D,
+                                       SurfaceMeshType::Quad3D,
+                                       SurfaceMeshType::Tri3D);
+   func_order               = GENERATE(1, 2);
+   int mesh_order           = GENERATE(1, 2);
+   int mesh_node_ordering   = GENERATE(0, 1);
+   int point_ordering       = GENERATE(0, 1);
+   int ncomp                = GENERATE(1, 2);
+   int gf_ordering          = GENERATE(0, 1);
+   int func_out_ordering    = GENERATE(0, 1);
+
+   const char *mesh_name = SurfaceMeshName(surface_mesh_type);
+   CAPTURE(mesh_name, func_order, mesh_order, mesh_node_ordering,
+           point_ordering, ncomp, gf_ordering, func_out_ordering);
+
+   if (ncomp == 1 && gf_ordering == 1)
+   {
+      return;
+   }
+
+   Mesh mesh = MakeSurfaceMesh(surface_mesh_type, 4);
+   const int sdim = SurfaceSpaceDim(surface_mesh_type);
+   mesh.SetCurvature(mesh_order, false, sdim, mesh_node_ordering);
+
+   H1_FECollection c_fec(func_order, mesh.Dimension());
+   FiniteElementSpace c_fespace(&mesh, &c_fec, ncomp, gf_ordering);
+   GridFunction field_vals(&c_fespace);
+
+   VectorFunctionCoefficient F(ncomp, F_exact);
+   field_vals.ProjectCoefficient(F);
+
+   const int npt_per_el = 8;
+   const int pts_cnt = mesh.GetNE()*npt_per_el;
+   Vector vxyz(pts_cnt*sdim);
+   GetSurfaceInteriorPoints(mesh, npt_per_el, point_ordering, vxyz);
+
+   Vector interp_vals(pts_cnt*ncomp);
+   FindPointsGSLIB finder;
+   finder.SetupSurf(mesh);
+   finder.SetL2AvgType(FindPointsGSLIB::NONE);
+   finder.Interpolate(vxyz, field_vals, interp_vals, point_ordering,
+                      func_out_ordering);
+
+   Array<unsigned int> code_out = finder.GetCode();
+   Vector dist_p_out = finder.GetDist();
+
+   int not_found = 0;
+   double err = 0.0, max_err = 0.0, max_dist = 0.0;
+   Vector pos(sdim);
+   Vector exact_val(ncomp);
+
+   for (int i = 0; i < pts_cnt; i++)
+   {
+      max_dist = std::max(max_dist, dist_p_out(i));
+      for (int d = 0; d < sdim; d++)
+      {
+         const int idx = (point_ordering == Ordering::byNODES) ?
+                         d*pts_cnt + i :
+                         i*sdim + d;
+         pos(d) = vxyz(idx);
+      }
+      F_exact(pos, exact_val);
+      for (int j = 0; j < ncomp; j++)
+      {
+         if (code_out[i] < 2)
+         {
+            err = func_out_ordering == Ordering::byNODES ?
+                  fabs(exact_val(j) - interp_vals[i + j*pts_cnt]) :
+                  fabs(exact_val(j) - interp_vals[i*ncomp + j]);
+            max_err = std::max(max_err, err);
+         }
+         else if (j == 0)
+         {
+            not_found++;
+         }
+      }
+   }
+
+   REQUIRE(max_err < 1e-12);
+   REQUIRE(max_dist < 1e-10);
+   REQUIRE(not_found == 0);
+}
+
 // Generates meshes with different element types, followed by points at
 // element faces and interior, and finally checks to see if these points are
 // correctly detected at element boundary or not.
@@ -257,9 +504,8 @@ TEST_CASE("GSLIBFindAtElementBoundary",
       int nptface = xyz.Size()/dim;
 
       // Generate points inside each element
-      FiniteElementCollection *l2_fec = new L2_FECollection(l2_order, dim);
-      FiniteElementSpace l2_fespace =
-         FiniteElementSpace(&mesh, l2_fec, 1);
+      L2_FECollection l2_fec(l2_order, dim);
+      FiniteElementSpace l2_fespace(&mesh, &l2_fec, 1);
       DenseMatrix vals;
       DenseMatrix tr;
       for (int e = 0; e < mesh.GetNE(); e++)
@@ -295,7 +541,92 @@ TEST_CASE("GSLIBFindAtElementBoundary",
          cmax = std::max(code_out[i], cmax);
       }
       REQUIRE((cmin == 0 && cmax == 0)); // should be found inside element
-      delete l2_fec;
+   }
+}
+
+TEST_CASE("GSLIBSurfFindAtElementBoundary",
+          "[GSLIBSurfFindAtElementBoundary][GSLIB]")
+{
+   auto surface_mesh_type = GENERATE(SurfaceMeshType::Segment2D,
+                                     SurfaceMeshType::Segment3D,
+                                     SurfaceMeshType::Quad3D,
+                                     SurfaceMeshType::Tri3D);
+   const char *mesh_name = SurfaceMeshName(surface_mesh_type);
+   CAPTURE(mesh_name);
+
+   Mesh mesh = MakeSurfaceMesh(surface_mesh_type, 4);
+   const int sdim = SurfaceSpaceDim(surface_mesh_type);
+   mesh.SetCurvature(2, false, sdim);
+
+   const int nptface_per_el = (mesh.Dimension() == 1) ? 2 : 8;
+   const int nptint_per_el = 8;
+   const int nptface = mesh.GetNE()*nptface_per_el;
+   const int nptint = mesh.GetNE()*nptint_per_el;
+   Vector xyz((nptface + nptint)*sdim);
+   GetSurfaceBoundaryPoints(mesh, nptface_per_el, Ordering::byVDIM, xyz, 0);
+   GetSurfaceInteriorPoints(mesh, nptint_per_el, Ordering::byVDIM, xyz,
+                            nptface);
+
+   FindPointsGSLIB finder;
+   finder.SetupSurf(mesh);
+   finder.FindPoints(xyz, Ordering::byVDIM);
+   Array<unsigned int> code_out = finder.GetCode();
+
+   for (int i = 0; i < nptface; i++)
+   {
+      REQUIRE(code_out[i] == 1);
+   }
+   for (int i = nptface; i < nptface + nptint; i++)
+   {
+      REQUIRE(code_out[i] == 0);
+   }
+}
+
+TEST_CASE("GSLIBSurfAABBExpansion", "[GSLIBSurfAABBExpansion][GSLIB]")
+{
+   auto surface_mesh_type = GENERATE(SurfaceMeshType::Segment2D,
+                                     SurfaceMeshType::Segment3D,
+                                     SurfaceMeshType::Quad3D,
+                                     SurfaceMeshType::Tri3D);
+   const char *mesh_name = SurfaceMeshName(surface_mesh_type);
+   CAPTURE(mesh_name);
+
+   constexpr double offset = 1.0e-3;
+   const int npt_per_el = 8;
+
+   Mesh mesh = MakeSurfaceMesh(surface_mesh_type, 4);
+   const int sdim = SurfaceSpaceDim(surface_mesh_type);
+   mesh.SetCurvature(2, false, sdim);
+
+   const int npt = mesh.GetNE()*npt_per_el;
+   Vector xyz(npt*sdim);
+   GetSurfaceInteriorPoints(mesh, npt_per_el, Ordering::byVDIM, xyz);
+   // offset them to move away from the surface
+   const int off_d = (surface_mesh_type == SurfaceMeshType::Segment2D) ? 1 : 2;
+   for (int i = 0; i < npt; i++)
+   {
+      xyz(i*sdim + off_d) += offset;
+   }
+
+   FindPointsGSLIB finder;
+   finder.SetupSurf(mesh, 0.0);
+   finder.FindPoints(xyz, Ordering::byVDIM);
+   Array<unsigned int> code_no_pad = finder.GetCode();
+
+   for (int i = 0; i < code_no_pad.Size(); i++)
+   {
+      REQUIRE(code_no_pad[i] == 2);
+   }
+
+   // make aabb at least big enough to include the offset points
+   Vector aabb_sz_inc({2.1*offset});
+   finder.SetupSurfWithAABBExpansion(mesh, aabb_sz_inc);
+   finder.FindPoints(xyz, Ordering::byVDIM);
+   Array<unsigned int> code_with_pad = finder.GetCode();
+
+   for (int i = 0; i < npt; i++)
+   {
+      REQUIRE(code_with_pad[i] == 1);
    }
 }
 
@@ -322,9 +653,8 @@ TEST_CASE("GSLIBInterpolateL2ElementBoundary",
    mesh.SetCurvature(mesh_order);
 
    // Set GridFunction to be interpolated
-   FiniteElementCollection *c_fec = new L2_FECollection(3, dim);
-   FiniteElementSpace c_fespace =
-      FiniteElementSpace(&mesh, c_fec, 1);
+   L2_FECollection c_fec(3, dim);
+   FiniteElementSpace c_fespace(&mesh, &c_fec, 1);
    GridFunction field_vals(&c_fespace);
    Array<int> dofs;
    double leftval = 1.0;
@@ -366,7 +696,6 @@ TEST_CASE("GSLIBInterpolateL2ElementBoundary",
    REQUIRE(interp_vals(0) == MFEM_Approx(0.5*(leftval+rightval)));
 
    finder.FreeData();
-   delete c_fec;
 }
 
 #ifdef MFEM_USE_MPI

tests/unit/fem/test_lin_interp.cpp

@@ -1069,75 +1069,4 @@ TEST_CASE("Exact Sequence Properties: d(df)=0",
    }
 }
 
-TEST_CASE("Partial Assemble Linear Interpolator",
-          "[CurlInterpolator]"
-          "[GPU]")
-{
-   const int maxOrder = 3;
-   auto order = GENERATE_COPY(range(1, maxOrder + 1));
-   CAPTURE(order);
-
-   int n = 3, dim = -1;
-   real_t tol = 1e-10;
-
-   auto type = Element::HEXAHEDRON;
-   CAPTURE(type);
-
-   Mesh mesh;
-
-   {
-      dim = 3;
-      mesh = Mesh::MakeCartesian3D(n, n, n, (Element::Type)type,
-                                   2.0, 3.0, 5.0);
-   }
-
-   ND_FECollection    fec_nd(order, dim);
-   RT_FECollection    fec_rt(order - 1, dim);
-
-   FiniteElementSpace fespace_nd(&mesh, &fec_nd);
-   FiniteElementSpace fespace_rt(&mesh, &fec_rt);
-
-   // 3D
-   {
-      DiscreteLinearOperator CurlFA(&fespace_nd, &fespace_rt);
-      CurlFA.AddDomainInterpolator(new CurlInterpolator());
-      CurlFA.Assemble();
-      CurlFA.Finalize();
-      DiscreteLinearOperator CurlPA(&fespace_nd, &fespace_rt);
-      CurlPA.AddDomainInterpolator(new CurlInterpolator());
-      CurlPA.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      CurlPA.Assemble();
-
-      SECTION("Curl (3D)")
-      {
-         SparseMatrix &Curl = CurlFA.SpMat();
-         GridFunction x(&fespace_nd), y_fa(&fespace_rt), y_pa(&fespace_rt);
-         VectorFunctionCoefficient coeff(3, [](const Vector &x, Vector &y)
-         {
-            y.SetSize(3);
-            y[0] = sin(2 * M_PI * x[2] / 5) - cos(2 * M_PI * x[1] / 3);
-            y[1] = sin(2 * M_PI * x[0] / 2) - cos(2 * M_PI * x[2] / 5);
-            y[2] = sin(2 * M_PI * x[1] / 3) - cos(2 * M_PI * x[0] / 2);
-         });
-         x.ProjectCoefficient(coeff);
-         REQUIRE(x.Size() == Curl.Width());
-         REQUIRE(y_fa.Size() == Curl.Height());
-         REQUIRE(x.Size() == CurlPA.Width());
-         REQUIRE(y_pa.Size() == CurlPA.Height());
-         Curl.Mult(x, y_fa);
-         CurlPA.Mult(x, y_pa);
-         y_pa -= y_fa;
-         REQUIRE(y_pa.Normlinf() < tol);
-
-         // transpose
-         y_fa.ProjectCoefficient(coeff);
-         GridFunction x_fa(&fespace_nd), x_pa(&fespace_nd);
-         Curl.MultTranspose(y_fa, x_fa);
-         CurlPA.MultTranspose(y_fa, x_pa);
-         x_pa -= x_fa;
-         REQUIRE(x_pa.Normlinf() < tol);
-      }
-   }
-}
-
 } // namespace lin_interp

tests/unit/fem/test_particleset.cpp

@@ -200,10 +200,71 @@ int CheckArrayEquality(const Array<T> &arr1, const Array<T> &arr2)
    return wrong_ct;
 }
 
+// Apply a deterministic perturbation to particle data on host.
+void PerturbParticleDataOnHost(std::vector<Particle> &particles)
+{
+   for (auto &p : particles)
+   {
+      for (int f = -1; f < p.GetNFields(); f++)
+      {
+         Vector &field = f == -1 ? p.Coords() : p.Field(f);
+         field.HostReadWrite();
+         const real_t scale = (f == -1) ? 0.001 : 1.0;
+         for (int c = 0; c < field.Size(); c++)
+         {
+            field(c) += scale * (f + c + 2);
+         }
+      }
+
+      for (int t = 0; t < p.GetNTags(); t++)
+      {
+         p.Tag(t) += t + 1;
+      }
+   }
+}
+
+// Apply a deterministic perturbation to particle data on device.
+void PerturbParticleDataOnDevice(ParticleSet &pset)
+{
+   const int np = pset.GetNParticles();
+
+   // Shift coordinates and fields using the same per-component formula while
+   // honoring the ParticleVector ordering selected by the test.
+   for (int f = -1; f < pset.GetNFields(); f++)
+   {
+      ParticleVector &field = f == -1 ? pset.Coords() : pset.Field(f);
+      const int vdim = field.GetVDim();
+      const bool by_vdim = (field.GetOrdering() == Ordering::byVDIM);
+      const real_t scale = (f == -1) ? 0.001 : 1.0;
+      auto d_field = field.ReadWrite();
+
+      mfem::forall(np, [=] MFEM_HOST_DEVICE (int i)
+      {
+         for (int c = 0; c < vdim; c++)
+         {
+            const int idx = by_vdim ? i * vdim + c : i + c * np;
+            d_field[idx] += scale * (f + c + 2);
+         }
+      });
+   }
+
+   for (int t = 0; t < pset.GetNTags(); t++)
+   {
+      Array<int> &tag = pset.Tag(t);
+      auto d_tag = tag.ReadWrite();
+
+      mfem::forall(np, [=] MFEM_HOST_DEVICE (int i)
+      {
+         d_tag[i] += t + 1;
+      });
+   }
+}
+
 void TestRedistribute(Ordering::Type ordering)
 {
    int size = Mpi::WorldSize();
    int rank = Mpi::WorldRank();
+   const bool use_device = Device::IsEnabled();
 
    // Create a 3D hex mesh
    Mesh m = Mesh::MakeCartesian3D(N_e, N_e, N_e, Element::Type::HEXAHEDRON);
@@ -252,15 +313,22 @@ void TestRedistribute(Ordering::Type ordering)
    SECTION(std::string("Ordering: ") +
            (ordering == Ordering::byNODES ? "byNODES" : "byVDIM"))
    {
-      // Add the particles uniquely to each rank particleset
       ParticleSet pset(MPI_COMM_WORLD, 0, SpaceDim, FieldVDims,
-                       NumTags, ordering);
+                       NumTags, ordering, use_device);
+      CHECK(pset.IsParticleRefValid() ==
+            (!use_device && ordering == Ordering::byVDIM));
 
       for (int i = 0; i < N_rank; i++)
       {
          pset.AddParticle(all_particles[i*size+rank]);
       }
 
+      if (use_device)
+      {
+         PerturbParticleDataOnDevice(pset);
+         PerturbParticleDataOnHost(all_particles);
+      }
+
       // Find points
       FindPointsGSLIB finder(MPI_COMM_WORLD);
       finder.Setup(pmesh);
@@ -270,6 +338,7 @@ void TestRedistribute(Ordering::Type ordering)
       int code_1_count = 0;
       int code_2_count = 0;
       const Array<unsigned int> &code = finder.GetCode();
+      code.HostRead();
       for (int i = 0; i < code.Size(); i++)
       {
          if (code[i] == 1)
@@ -292,6 +361,7 @@ void TestRedistribute(Ordering::Type ordering)
       finder.FindPoints(pset.Coords(), ordering);
 
       const Array<unsigned int> &procs = finder.GetProc();
+      procs.HostRead();
 
       int wrong_proc_count = 0;
       for (int i = 0; i < procs.Size(); i++)
@@ -307,6 +377,11 @@ void TestRedistribute(Ordering::Type ordering)
 
       // Check that coordinates + fields + tags are all still correct
       int wrong_particle_count = 0;
+      pset.GetIDs().HostRead();
+      for (int t = 0; t < pset.GetNTags(); t++)
+      {
+         pset.Tag(t).HostRead();
+      }
       for (int i = 0; i < pset.GetNParticles(); i++)
       {
          Particle &actual_p = all_particles[pset.GetIDs()[i]];
@@ -317,13 +392,13 @@ void TestRedistribute(Ordering::Type ordering)
             wrong_particle_count++;
          }
       }
-      MPI_Allreduce(MPI_IN_PLACE, &wrong_proc_count, 1, MPI_INT, MPI_SUM,
+      MPI_Allreduce(MPI_IN_PLACE, &wrong_particle_count, 1, MPI_INT, MPI_SUM,
                     MPI_COMM_WORLD);
       CHECK(wrong_particle_count == 0);
    }
 }
 
-TEST_CASE("Particle Redistribution", "[ParticleSet][Parallel]")
+TEST_CASE("Particle Redistribution", "[ParticleSet][Parallel][GPU]")
 {
    TestRedistribute(Ordering::byNODES);
    TestRedistribute(Ordering::byVDIM);

tests/unit/fem/test_quadf_coef.cpp

@@ -361,57 +361,3 @@ TEST_CASE("QuadratureFunction::ProjectGridFunction",
       compare_qf_to_coeff(qf, coeff);
    }
 }
-
-TEST_CASE("QuadratureFunction::ProjectGridFunction Integral",
-          "[Coefficient][QuadratureFunction]")
-{
-   const int order = GENERATE(1, 2);
-   const auto mesh_fname =
-      GENERATE("../../data/star.mesh", "../../data/star-mixed.mesh",
-               "../../data/fichera.mesh", "../../data/fichera-mixed.mesh",
-               "../../data/inline-tri.mesh", "../../data/inline-tet.mesh",
-               "../../data/inline-wedge.mesh", "../../data/inline-pyramid.mesh",
-               "../../data/ball-nurbs.mesh");
-   CAPTURE(order, mesh_fname);
-
-   Mesh mesh(mesh_fname);
-   if (mesh.NURBSext)
-   {
-      mesh.UniformRefinement();
-      mesh.SetCurvature(2);
-   }
-   L2_FECollection fec(order, mesh.Dimension(), BasisType::GaussLegendre,
-                       FiniteElement::INTEGRAL);
-   FiniteElementSpace fes(&mesh, &fec);
-
-   GridFunction gf(&fes);
-   gf.Randomize(1);
-   GridFunctionCoefficient coeff(&gf);
-
-   auto compare_qf_to_coeff = [](QuadratureFunction &qf, Coefficient &coeff)
-   {
-      auto &qs = *qf.GetSpace();
-      for (int i = 0; i < qs.GetNE(); ++i)
-      {
-         const IntegrationRule &ir = qs.GetIntRule(i);
-         ElementTransformation &T = *qs.GetTransformation(i);
-         Vector values;
-         qf.GetValues(i, values);
-         for (int iq = 0; iq < ir.Size(); ++iq)
-         {
-            const int iq_p = qs.GetPermutedIndex(i, iq);
-            const IntegrationPoint &ip = ir[iq];
-            T.SetIntPoint(&ip);
-            REQUIRE(coeff.Eval(T, ip) == MFEM_Approx(values[iq_p]));
-         }
-      }
-   };
-
-   SECTION("QuadratureSpace")
-   {
-      QuadratureSpace qs(&mesh, order + 1);
-      QuadratureFunction qf(qs);
-      coeff.Project(qf);
-      compare_qf_to_coeff(qf, coeff);
-   }
-}

tests/unit/fem/test_quadinterpolator.cpp

@@ -14,16 +14,16 @@
 
 using namespace mfem;
 
-static bool H1testQuadratureInterpolator(const int dim, const int p,
-                                         const int qpts,
-                                         const QVectorLayout q_layout,
-                                         const int nx, const int ny,
-                                         const int nz)
+static bool testQuadratureInterpolator(const int dim,
+                                       const int p,
+                                       const int qpts,
+                                       const QVectorLayout q_layout,
+                                       const int nx, const int ny, const int nz)
 {
    // Keep for debugging purposes:
    if (verbose_tests)
    {
-      std::cout << "H1testQuadratureInterpolator(dim=" << dim
+      std::cout << "testQuadratureInterpolator(dim=" << dim
                 << ",p=" << p
                 << ",q=" << qpts
                 << ",l=" << (q_layout == QVectorLayout::byNODES ?
@@ -236,188 +236,6 @@ static bool H1testQuadratureInterpolator(const int dim, const int p,
    return true;
 }
 
-static bool L2testQuadratureInterpolator(const int dim, const int p,
-                                         const int qpts,
-                                         const QVectorLayout q_layout,
-                                         const int nx, const int ny,
-                                         const int nz)
-{
-   // Keep for debugging purposes:
-   if (verbose_tests)
-   {
-      std::cout << "L2testQuadratureInterpolator(dim=" << dim
-                << ",p=" << p
-                << ",q=" << qpts
-                << ",l=" << (q_layout == QVectorLayout::byNODES ?
-                             "by_nodes" : "by_vdim")
-                << ",nx=" << nx
-                << ",ny=" << ny
-                << ",nz=" << nz
-                << ")" << std::endl;
-   }
-
-   const int vdim = dim;
-   const int ordering = Ordering::byNODES;
-
-   Mesh mesh = dim == 1 ? Mesh::MakeCartesian1D(nx, Element::SEGMENT) :
-               dim == 2 ? Mesh::MakeCartesian2D(nx,ny, Element::QUADRILATERAL) :
-               Mesh::MakeCartesian3D(nx,nx,nz, Element::HEXAHEDRON);
-   mesh.SetCurvature(2);
-   switch (dim)
-   {
-      case 1:
-         mesh.Transform([](const Vector &x, Vector &y) { y[0] = x[0] * x[0]; });
-         break;
-      case 2:
-         mesh.Transform([](const Vector &x, Vector &y)
-         {
-            y[0] = x[0] + 0.1 * sin(2 * M_PI * x[0]);
-            y[1] = x[1] + 0.1 * cos(2 * M_PI * x[1]);
-         });
-         break;
-      case 3:
-         mesh.Transform([](const Vector &x, Vector &y)
-         {
-            y[0] = x[0] + 0.1 * sin(2 * M_PI * x[0]);
-            y[1] = x[1] + 0.1 * cos(2 * M_PI * x[1]);
-            y[2] = x[2] + 0.1 * cos(2 * M_PI * x[2]);
-         });
-   }
-
-   const L2_FECollection fec(p, dim);
-   const L2_FECollection ifec(p, dim, BasisType::GaussLegendre,
-                              FiniteElement::INTEGRAL);
-   FiniteElementSpace sfes(&mesh, &fec, 1, ordering);
-   FiniteElementSpace vfes(&mesh, &fec, vdim, ordering);
-   FiniteElementSpace isfes(&mesh, &ifec, 1, ordering);
-   FiniteElementSpace ivfes(&mesh, &ifec, vdim, ordering);
-
-   FunctionCoefficient coeff([](const Vector &x)
-   {
-      real_t res = 0;
-      for (int dim = 0; dim < x.Size(); ++dim)
-      {
-         res += cos(x[dim]);
-      }
-      return res;
-   });
-
-   VectorFunctionCoefficient vcoeff(vdim, [vdim](const Vector &x, Vector &y)
-   {
-      y.SetSize(vdim);
-      for (int v = 0; v < vdim; ++v)
-      {
-         real_t res = 0;
-         for (int dim = 0; dim < x.Size(); ++dim)
-         {
-            res += cos(x[dim] * (v + 1));
-         }
-         y[v] = res;
-      }
-   });
-
-   GridFunction ix(&isfes);
-   ix.ProjectCoefficient(coeff);
-
-   GridFunction inodes(&ivfes);
-   inodes.ProjectCoefficient(vcoeff);
-
-   const Geometry::Type GeomType = mesh.GetTypicalElementGeometry();
-   const IntegrationRule &ir = IntRules.Get(GeomType, 2*qpts-1);
-   const QuadratureInterpolator *isqi(isfes.GetQuadratureInterpolator(ir));
-   const QuadratureInterpolator *ivqi(ivfes.GetQuadratureInterpolator(ir));
-
-   const int NE(mesh.GetNE());
-   const int NQ(ir.GetNPoints());
-   const int ND(sfes.GetTypicalFE()->GetDof());
-   REQUIRE(ND == vfes.GetTypicalFE()->GetDof());
-
-   const ElementDofOrdering nat_ordering = ElementDofOrdering::NATIVE;
-   const ElementDofOrdering lex_ordering = ElementDofOrdering::LEXICOGRAPHIC;
-   const Operator *iSRN(isfes.GetElementRestriction(nat_ordering));
-   const Operator *iSRL(isfes.GetElementRestriction(lex_ordering));
-   const Operator *iVRN(ivfes.GetElementRestriction(nat_ordering));
-   const Operator *iVRL(ivfes.GetElementRestriction(lex_ordering));
-   MFEM_VERIFY(iSRN, "No element sn-restriction operator found!");
-   MFEM_VERIFY(iSRL, "No element sl-restriction operator found!");
-   MFEM_VERIFY(iVRN, "No element vn-restriction operator found!");
-   MFEM_VERIFY(iVRL, "No element vl-restriction operator found!");
-
-   const real_t rel_tol = 1e-12;
-
-   {
-      // Scalar
-      isqi->SetOutputLayout(q_layout);
-      Vector xe(1*ND*NE);
-      REQUIRE(xe.Size() == iSRN->Height());
-      REQUIRE(iSRN->Height() == iSRL->Height());
-
-      // Full results
-      Vector isq_val_f(NQ*NE);
-      // Tensor results
-      Vector isq_val_t(NQ*NE);
-      {
-         // Full
-         iSRN->Mult(ix, xe);
-         isqi->DisableTensorProducts();
-
-         isqi->Values(xe, isq_val_f);
-      }
-      {
-         // Tensor
-         iSRL->Mult(ix, xe);
-         isqi->EnableTensorProducts();
-
-         isqi->Values(xe, isq_val_t);
-      }
-      real_t norm, rel_error;
-
-      norm = isq_val_f.Normlinf();
-      isq_val_f -= isq_val_t;
-      rel_error = isq_val_f.Normlinf()/norm;
-      if (verbose_tests)
-      { std::cout << "isq_val rel. error = " << rel_error << std::endl; }
-      REQUIRE(rel_error <= rel_tol);
-   }
-
-   {
-      // Vector
-      ivqi->SetOutputLayout(q_layout);
-      Vector ne(vdim*ND*NE);
-      REQUIRE(ne.Size() == iVRN->Height());
-      REQUIRE(iVRN->Height() == iVRL->Height());
-
-      // Full results
-      Vector ivq_val_f(dim*NQ*NE);
-      // Tensor results
-      Vector ivq_val_t(dim*NQ*NE);
-      {
-         // Full
-         iVRN->Mult(inodes, ne);
-         ivqi->DisableTensorProducts();
-
-         ivqi->Values(ne, ivq_val_f);
-      }
-      {
-         // Tensor
-         iVRL->Mult(inodes, ne);
-         ivqi->EnableTensorProducts();
-
-         ivqi->Values(ne, ivq_val_t);
-      }
-      real_t norm, rel_error;
-
-      norm = ivq_val_f.Normlinf();
-      ivq_val_f -= ivq_val_t;
-      rel_error = ivq_val_f.Normlinf()/norm;
-      if (verbose_tests)
-      { std::cout << "ivq_val rel. error = " << rel_error << std::endl; }
-      REQUIRE(rel_error <= rel_tol);
-   }
-
-   return true;
-}
-
 TEST_CASE("QuadratureInterpolator", "[QuadratureInterpolator][GPU]")
 {
    SECTION("H1 tensor elements: compare tensor and non-tensor evaluations")
@@ -429,21 +247,7 @@ TEST_CASE("QuadratureInterpolator", "[QuadratureInterpolator][GPU]")
       const auto nx = 3; // number of element in x
       const auto ny = 3; // number of element in y
       const auto nz = 3; // number of element in z
-      CAPTURE(dim, p, q, l);
-      H1testQuadratureInterpolator(dim, p, q, l, nx, ny, nz);
-   }
-
-   SECTION("L2 tensor elements: compare tensor and non-tensor evaluations")
-   {
-      const auto dim = GENERATE(1,2,3); // dimension
-      const auto p = GENERATE(range(1,7)); // element order, 1 <= p < 7
-      const auto q = GENERATE_COPY(p+1,p+2); // 1D quadrature points
-      const auto l = GENERATE(QVectorLayout::byNODES, QVectorLayout::byVDIM);
-      const auto nx = 3; // number of element in x
-      const auto ny = 3; // number of element in y
-      const auto nz = 3; // number of element in z
-      CAPTURE(dim, p, q, l);
-      L2testQuadratureInterpolator(dim, p, q, l, nx, ny, nz);
+      testQuadratureInterpolator(dim, p, q, l, nx, ny, nz);
    }
 
    SECTION("H1 elements: values and physical derivatives")

tests/unit/linalg/test_complex_operator.cpp

@@ -113,3 +113,65 @@ TEST_CASE("ComplexOperator Quaternion Tests", "[ComplexOperator]")
       REQUIRE(qikx.Normlinf() < tol);
    }
 }
+
+#ifdef MFEM_USE_MPI
+
+TEST_CASE("ComplexHypreParMatrix GetSystemMatrix",
+          "[ComplexOperator][Parallel][GPU]")
+{
+   // This test reproduces the issue described in PR #5200 on GitHub. See also
+   // the follow up PR #5346.
+
+   // 1. Construct ComplexHypreParMatrix similar to ex25p.
+   const char mesh_file[] = "../../data/inline-quad.mesh";
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+   int ref_levels = 1;
+   for (int l = 0; l < ref_levels; l++)
+   {
+      mesh->UniformRefinement();
+   }
+   ParMesh pmesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   int par_ref_levels = 1;
+   for (int l = 0; l < par_ref_levels; l++)
+   {
+      pmesh.UniformRefinement();
+   }
+   int order = 1;
+   ND_FECollection fec(order, dim);
+   ParFiniteElementSpace fespace(&pmesh, &fec);
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+   }
+   fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   ComplexOperator::Convention conv = ComplexOperator::HERMITIAN;
+   VectorConstantCoefficient f(Vector{1_r, 2_r});
+   ParComplexLinearForm b(&fespace, conv);
+   b.AddDomainIntegrator(NULL, new VectorFEDomainLFIntegrator(f));
+   b = 0.0;
+   b.Assemble();
+   ParComplexGridFunction x(&fespace);
+   x = 0.0;
+   ConstantCoefficient one(1_r);
+   ParSesquilinearForm a(&fespace, conv);
+   a.AddDomainIntegrator(new CurlCurlIntegrator(one),
+                         new CurlCurlIntegrator(one));
+   a.AddDomainIntegrator(new VectorFEMassIntegrator(one),
+                         new VectorFEMassIntegrator(one));
+   a.Assemble();
+   OperatorPtr Ah;
+   Vector B, X;
+   a.FormLinearSystem(ess_tdof_list, x, b, Ah, X, B);
+
+   // 2. Test the call to ComplexHypreParMatrix::GetSystemMatrix and destroying
+   //    the returned matrix.
+   HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
+   delete A;
+}
+
+#endif // MFEM_USE_MPI

tests/unit/miniapps/test_tmop_pa.cpp

@@ -935,35 +935,27 @@ static void tmop_tests(int id = 0, bool all = false)
 
 #ifndef _WIN32
    {
-      QuadratureInterpolator::AddDetSpecializations<2, 2, 3, 3>();
-      QuadratureInterpolator::AddDetSpecializations<2, 2, 5, 5>();
-      QuadratureInterpolator::AddDetSpecializations<3, 3, 2, 3>();
-      QuadratureInterpolator::AddDetSpecializations<3, 3, 3, 4>();
-      QuadratureInterpolator::AddDetSpecializations<3, 3, 4, 6>();
+      using Det = QuadratureInterpolator::DetKernels;
+      Det::Specialization<2, 2, 3, 3>::Add();
+      Det::Specialization<2, 2, 5, 5>::Add();
+      Det::Specialization<3, 3, 2, 3>::Add();
+      Det::Specialization<3, 3, 3, 4>::Add();
+      Det::Specialization<3, 3, 4, 6>::Add();
 
-      QuadratureInterpolator::AddGradSpecializations<2, QVectorLayout::byNODES,
-                             false, 2, 3, 5>();
-      QuadratureInterpolator::AddGradSpecializations<2, QVectorLayout::byNODES,
-                             false, 2, 5, 5>();
-      QuadratureInterpolator::AddGradSpecializations<2, QVectorLayout::byNODES,
-                             false, 2, 6, 6>();
-      QuadratureInterpolator::AddGradSpecializations<3, QVectorLayout::byNODES,
-                             false, 3, 4, 5>();
+      using Grad = QuadratureInterpolator::GradKernels;
+      Grad::Specialization<2, QVectorLayout::byNODES, false, 2, 3, 5>::Add();
+      Grad::Specialization<2, QVectorLayout::byNODES, false, 2, 5, 5>::Add();
+      Grad::Specialization<2, QVectorLayout::byNODES, false, 2, 6, 6>::Add();
+      Grad::Specialization<3, QVectorLayout::byNODES, false, 3, 4, 5>::Add();
 
-      QuadratureInterpolator::AddTensorEvalSpecializations<
-      2, QVectorLayout::byVDIM, 2, 2, 2, 4>();
-      QuadratureInterpolator::AddTensorEvalSpecializations<
-      2, QVectorLayout::byVDIM, 2, 3, 3, 4>();
-      QuadratureInterpolator::AddTensorEvalSpecializations<
-      2, QVectorLayout::byVDIM, 2, 4, 4, 2>();
-      QuadratureInterpolator::AddTensorEvalSpecializations<
-      2, QVectorLayout::byVDIM, 2, 5, 5, 2>();
-      QuadratureInterpolator::AddTensorEvalSpecializations<
-      3, QVectorLayout::byVDIM, 3, 2, 3, 2>();
-      QuadratureInterpolator::AddTensorEvalSpecializations<
-      3, QVectorLayout::byVDIM, 3, 3, 4, 1>();
-      QuadratureInterpolator::AddTensorEvalSpecializations<
-      3, QVectorLayout::byVDIM, 3, 4, 6, 1>();
+      using TensorEval = QuadratureInterpolator::TensorEvalKernels;
+      TensorEval::Specialization<2, QVectorLayout::byVDIM, 2, 2, 2>::Opt<4>::Add();
+      TensorEval::Specialization<2, QVectorLayout::byVDIM, 2, 3, 3>::Opt<4>::Add();
+      TensorEval::Specialization<2, QVectorLayout::byVDIM, 2, 4, 4>::Opt<2>::Add();
+      TensorEval::Specialization<2, QVectorLayout::byVDIM, 2, 5, 5>::Opt<2>::Add();
+      TensorEval::Specialization<3, QVectorLayout::byVDIM, 3, 2, 3>::Opt<2>::Add();
+      TensorEval::Specialization<3, QVectorLayout::byVDIM, 3, 3, 4>::Opt<1>::Add();
+      TensorEval::Specialization<3, QVectorLayout::byVDIM, 3, 4, 6>::Opt<1>::Add();
 
       using MassDiagonal = MassIntegrator::DiagonalPAKernels;
       MassDiagonal::Specialization<2, 2, 3>::Add();