Use external PA data

version 1: PA new
version 2: PA new (layout by vdim)
version 3: MF ∂fem
version 4: PA ∂fem
version 5: PA ∂fem new
version 6: Auto PA ∂fem
version 7: Auto PA ∂fem new

.github/workflows/builds-and-tests.yml

@@ -132,14 +132,12 @@ jobs:
             hypre-target: int32
             precision: fp64
             enzyme: true
-            config-opts: MFEM_USE_ENZYME=YES ENZYME_DIR=$(brew --prefix enzyme) LDFLAGS=-L$LLVM_PREFIX/lib/c++
+            config-opts: MFEM_USE_ENZYME=YES ENZYME_DIR=$(brew --prefix enzyme)
 
     name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}${{ matrix.enzyme && '-enzyme' || '' }}
 
     runs-on: ${{ matrix.os }}
 
-    continue-on-error: ${{ matrix.enzyme && true || false }}
-
     steps:
       # Fix 'No space left on device' errors for Ubuntu builds.
       - name: Run Actions Cleaner
@@ -170,13 +168,10 @@ jobs:
           env
         shell: bash
 
-      # For info on Xcode see:
-      #  - https://github.com/actions/runner-images/issues/12541
-      #  - https://github.com/actions/runner-images/blob/releases/macos-15-arm64/20250811/images/macos/macos-15-arm64-Readme.md#xcode
       - name: Xcode version setup (MacOS)
         if: matrix.os == 'macos-latest'
         run: |
-          XCODE_PATH="/Applications/Xcode_16.4.app"
+          XCODE_PATH="/Applications/Xcode_15.3.app"
           echo "> sudo xcode-select -s ${XCODE_PATH}"
           sudo xcode-select -s ${XCODE_PATH}
           echo "> g++ -v"
@@ -294,12 +289,10 @@ jobs:
         run: |
           export HOMEBREW_NO_INSTALL_CLEANUP=1
           brew update
-          brew install enzyme
-          ENZYME_LLVM=$(brew info enzyme | sed -n 's/^Required:.*\(llvm[^ ]*\).*/\1/p')
-          LLVM_PREFIX=$(brew --prefix $ENZYME_LLVM)
-          echo "LLVM_PREFIX=$LLVM_PREFIX" >> $GITHUB_ENV
-          echo "OMPI_CC=$LLVM_PREFIX/bin/clang" >> $GITHUB_ENV
-          echo "OMPI_CXX=$LLVM_PREFIX/bin/clang++" >> $GITHUB_ENV
+          brew install llvm@19 enzyme
+          echo "LLVM_PREFIX=$(brew --prefix llvm@19)" >> $GITHUB_ENV
+          echo "OMPI_CC=$(brew --prefix llvm@19)/bin/clang" >> $GITHUB_ENV
+          echo "OMPI_CXX=$(brew --prefix llvm@19)/bin/clang++" >> $GITHUB_ENV
 
       # MFEM build and test
       - name: build

.github/workflows/stale.yml

@@ -29,47 +29,3 @@ jobs:
         operations-per-run: 500
         exempt-issue-labels: "bug,WIP,ready-for-review,in-review,in-next"
         exempt-pr-labels: "bug,WIP,ready-for-review,in-review,in-next"
-
-  # Stale action for PRs with "in-review" label.
-  stale-in-review-pr:
-
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-      pull-requests: write
-      actions: write
-
-    steps:
-    - uses: actions/stale@v9
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        stale-pr-message: ':warning: This PR has been automatically marked as stale because it has not had any activity in the last 150 days. *If no activity occurs in the next 30 days, it will be automatically closed.* Thank you for your contributions.'
-        only-pr-labels: "in-review"
-        days-before-pr-stale: 150
-        days-before-pr-close: 30
-        days-before-issue-stale: -1
-        days-before-issue-close: -1
-        stale-pr-label: 'stale'
-        operations-per-run: 500
-
-  # Stale action for PRs with "WIP" label.
-  stale-wip-pr:
-
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-      pull-requests: write
-      actions: write
-
-    steps:
-    - uses: actions/stale@v9
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        stale-pr-message: ':warning: This PR has been automatically marked as stale because it has not had any activity in the last 300 days. *If no activity occurs in the next 30 days, it will be automatically closed.* Thank you for your contributions.'
-        only-pr-labels: "WIP"
-        days-before-pr-stale: 300
-        days-before-pr-close: 30
-        days-before-issue-stale: -1
-        days-before-issue-close: -1
-        stale-pr-label: 'stale'
-        operations-per-run: 500

.gitignore

@@ -208,13 +208,10 @@ miniapps/electromagnetics/volta
 miniapps/electromagnetics/tesla
 miniapps/electromagnetics/maxwell
 miniapps/electromagnetics/joule
-miniapps/electromagnetics/lorentz
 miniapps/electromagnetics/Volta-AMR*
 miniapps/electromagnetics/Tesla-AMR*
 miniapps/electromagnetics/Maxwell-Parallel*
-miniapps/electromagnetics/Joule_[0-9]*
-miniapps/electromagnetics/Lorentz_[0-9]*
-miniapps/electromagnetics/Lorentz.dat
+miniapps/electromagnetics/Joule_*
 
 miniapps/gslib/field-diff
 miniapps/gslib/field-interp
@@ -270,9 +267,9 @@ miniapps/meshing/bounding-box*
 miniapps/meshing/jacobian-determinant*
 
 miniapps/mtop/parheat
-miniapps/mtop/ParHeat/*
+miniapps/mtop/ParHeat*
 miniapps/mtop/seqheat
-miniapps/mtop/SeqHeat/*
+miniapps/mtop/SeqHeat*
 
 miniapps/autodiff/paradiff
 miniapps/autodiff/seqadiff
@@ -280,7 +277,7 @@ miniapps/autodiff/seqtest
 miniapps/autodiff/par_example
 miniapps/autodiff/seq_example
 miniapps/autodiff/seq_test
-miniapps/autodiff/Example/*
+miniapps/autodiff/Exampl*
 
 miniapps/navier/navier_mms
 miniapps/navier/navier_kovasznay
@@ -303,7 +300,6 @@ miniapps/nurbs/nurbs_solenoidal
 miniapps/nurbs/nurbs_printfunc
 miniapps/nurbs/nurbs_patch_ex1
 miniapps/nurbs/nurbs_curveint
-miniapps/nurbs/nurbs_surface
 miniapps/nurbs/refined.mesh
 miniapps/nurbs/mesh.*
 miniapps/nurbs/sol_?.gf
@@ -322,7 +318,6 @@ miniapps/nurbs/nurbs_naca_cmesh
 miniapps/nurbs/naca-cmesh.mesh
 miniapps/nurbs/glvis_naca-cmesh.mesh
 miniapps/nurbs/Naca_cmesh
-miniapps/nurbs/*-Surface.mesh
 
 miniapps/performance/ex1
 miniapps/performance/ex1p
@@ -418,8 +413,8 @@ miniapps/diag-smoothers/mg-abs-l1-jacobi
 tests/unit/output_meshes
 tests/unit/unit_tests
 tests/unit/punit_tests
-tests/unit/gpu_unit_tests
-tests/unit/pgpu_unit_tests
+tests/unit/cunit_tests
+tests/unit/pcunit_tests
 tests/unit/sedov_tests_*
 tests/unit/psedov_tests_*
 tests/unit/tmop_pa_tests_*

.gitlab-ci.yml

@@ -22,7 +22,7 @@ include:
 # the "needs" keyword and express the DAG of jobs for more efficiency.
 # - We use setup and setup_baseline phases to download content outside of mfem
 #   directory.
-# - Allocate/Release is where Dane resource are allocated/released once for all.
+# - Allocate/Release is where ruby resource are allocated/released once for all.
 # - Build and Test is where we build and MFEM for multiple toolchains.
 # - Baseline_checks gathers baseline-type test suites execution
 # - Baseline_publish, only available on master, allows to update baseline
@@ -53,7 +53,7 @@ variables:
   AUTOTEST_COMMIT: "YES"
 
 # Trigger subpipelines:
-dane-build-and-test:
+ruby-build-and-test:
   stage: sub-pipelines
   variables:
     # Explicitly pass down values that we want to be able to set when triggering
@@ -61,10 +61,10 @@ dane-build-and-test:
     AUTOTEST: "${AUTOTEST}"
     AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
   trigger:
-    include: .gitlab/dane-build-and-test.yml
+    include: .gitlab/ruby-build-and-test.yml
     strategy: depend
 
-dane-baseline:
+ruby-baseline:
   stage: sub-pipelines
   variables:
     # Explicitly pass down values that we want to be able to set when triggering
@@ -73,7 +73,7 @@ dane-baseline:
     AUTOTEST: "${AUTOTEST}"
     AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
   trigger:
-    include: .gitlab/dane-baseline.yml
+    include: .gitlab/ruby-baseline.yml
     strategy: depend
 
 lassen-build-and-test:

.gitlab/README.md

@@ -24,7 +24,7 @@ and `test type`.
 
 Machines typically include:
 
-* Dane: Intel Sapphire Rapids
+* Ruby: 2nd Gen Intel Xeon (Cascade Lake)
 * Lassen: Power9 + Nvidia GPU
 * Corona: AMD GPU
 
@@ -76,13 +76,13 @@ with a spack spec of MFEM, within the limits permitted by the MFEM spack
 package.
 
 In any build-and-test sub-pipeline a job basically consists in defining the
-spack spec to use. Adding a job on Dane for example resumes to:
+spack spec to use. Adding a job on ruby for example resumes to:
 
 ```yaml
 <job_name>:
   variables:
     SPEC: "<spack_spec>"
-  extends: .build_and_test_on_dane
+  extends: .build_and_test_on_ruby
 ```
 
 The remaining and non trivial work is to make sure this spec is working. To

.gitlab/configs/common.yml

@@ -24,7 +24,7 @@ variables:
 # TODO: add a clean-up mechanism
   BUILD_ROOT: ${USER_CI_TOP_DIR}/${CI_PROJECT_NAME}-${MACHINE_NAME}-pipeline-${CI_PIPELINE_ID}
 
-# On LLNL's Dane, there is only one allocation shared among jobs in order to
+# On LLNL's ruby, there is only one allocation shared among jobs in order to
 # save time and resource. This allocation has to be uniquely named so that we
 # are sure to retrieve it.
   ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}

.gitlab/configs/ruby-config.yml

@@ -9,17 +9,17 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-# GitLab pipelines configurations for the Dane machine at LLNL
+# GitLab pipelines configurations for the Ruby machine at LLNL
 variables:
-  MACHINE_NAME: dane
+  MACHINE_NAME: ruby
 
-.on_dane:
+.on_ruby:
   tags:
     - shell
-    - dane
+    - ruby
   rules:
-    # Don't run dane jobs if...
-    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_DANE == "OFF"'
+    # Don't run ruby jobs if...
+    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_RUBY == "OFF"'
       when: never
     # Don't run autotest update if...
     - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
@@ -40,17 +40,16 @@ variables:
     - when: on_success
 
 # Spack helped builds
-# Generic dane build job, extending build script
-.build_and_test_on_dane:
-  extends: [.on_dane]
+# Generic ruby build job, extending build script
+.build_and_test_on_ruby:
+  extends: [.on_ruby]
   stage: build_and_test
   script:
     # THREADS is used by 'tests/gitlab/build_and_test', run below
-    # Dane has 224 threads/node and we run 7 separate jobs: 224=7*32
-    - export THREADS=28
+    - export THREADS=16
     - echo ${ALLOC_NAME}
     - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
     - echo ${JOBID}
     - echo ${MFEM_DATA_DIR}
     - echo ${SPEC}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) --reservation=ci -t 60 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
+    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) --reservation=ci -t 45 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data

.gitlab/configs/setup-baseline.yml

@@ -18,7 +18,7 @@
 setup_baseline:
   tags:
     - shell
-    - dane
+    - ruby
   stage: setup
   variables:
     GIT_STRATEGY: none

.gitlab/configs/setup-build-and-test.yml

@@ -16,7 +16,7 @@
 setup:
   tags:
     - shell
-    - dane
+    - ruby
   stage: setup
   variables:
     GIT_STRATEGY: none

.gitlab/ruby-baseline.yml

@@ -19,8 +19,8 @@ stages:
   - cleanup
   - baseline_publish
 
-baselinecheck_mfem_intel_dane:
-  extends: [.on_dane]
+baselinecheck_mfem_intel_ruby:
+  extends: [.on_ruby]
   stage: baseline_check
   variables:
     # TPLS_DIR is used in .gitlab/scripts/baseline to provide the tpls location
@@ -31,8 +31,8 @@ baselinecheck_mfem_intel_dane:
   script:
     - echo ${BUILD_ROOT}
     - echo ${TPLS_DIR}
-    # Used by the tests in MFEM/tests, dane has 224 threads/node:
-    - export MFEM_TEST_NP=192
+    # Used by the tests in MFEM/tests:
+    - export MFEM_TEST_NP=48
     # The next script uses the following environment variables:
     # * BASELINE_TEST, SYS_TYPE, CI_PROJECT_DIR, ARTIFACTS_DIR,
     # * BUILD_ROOT, TPLS_DIR, MACHINE_NAME
@@ -44,7 +44,7 @@ baselinecheck_mfem_intel_dane:
   allow_failure: true
 
 cleanup:
-  extends: .on_dane
+  extends: .on_ruby
   stage: cleanup
   variables:
     GIT_STRATEGY: none
@@ -53,7 +53,7 @@ cleanup:
     - rm -rf "${BUILD_ROOT}" || true
 
 report_baseline:
-  extends: [.on_dane]
+  extends: [.on_ruby]
   stage: baseline_report
   script:
     - echo ${MACHINE_NAME}
@@ -113,8 +113,8 @@ report_baseline:
         exit $err
       ) 9> autotest.lock
 
-baselinepublish_mfem_dane:
-  extends: [.on_dane]
+baselinepublish_mfem_ruby:
+  extends: [.on_ruby]
   stage: baseline_publish
   rules:
     # - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "YES"'
@@ -129,5 +129,5 @@ baselinepublish_mfem_dane:
 
 include:
   - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/dane-config.yml
+  - local: .gitlab/configs/ruby-config.yml
   - local: .gitlab/configs/setup-baseline.yml

.gitlab/ruby-build-and-test.yml

@@ -19,54 +19,54 @@ stages:
 allocate_resource:
   variables:
     GIT_STRATEGY: none
-  extends: .on_dane
+  extends: .on_ruby
   stage: allocate_resource
   script:
     - echo ${ALLOC_NAME}
     - salloc --exclusive --nodes=1 --reservation=ci --time=60 --no-shell --job-name=${ALLOC_NAME}
   timeout: 6h
 
-# GitLab jobs for the Dane machine at LLNL
+# GitLab jobs for the Ruby machine at LLNL
 debug_ser_gcc_10:
   variables:
     SPEC: "%gcc@10.3.1 +debug~mpi"
-  extends: .build_and_test_on_dane
+  extends: .build_and_test_on_ruby
 
 debug_par_gcc_10:
   variables:
     SPEC: "%gcc@10.3.1 +debug+mpi"
-  extends: .build_and_test_on_dane
+  extends: .build_and_test_on_ruby
 
 opt_ser_gcc_10:
   variables:
     SPEC: "%gcc@10.3.1 ~mpi"
-  extends: .build_and_test_on_dane
+  extends: .build_and_test_on_ruby
 
 opt_par_gcc_10:
   variables:
     SPEC: "%gcc@10.3.1"
-  extends: .build_and_test_on_dane
+  extends: .build_and_test_on_ruby
 
 opt_par_gcc_10_sundials:
   variables:
     SPEC: "%gcc@10.3.1 +sundials"
-  extends: .build_and_test_on_dane
+  extends: .build_and_test_on_ruby
 
 opt_par_gcc_10_petsc:
   variables:
     SPEC: "%gcc@10.3.1 +petsc ^petsc+mumps~superlu-dist"
-  extends: .build_and_test_on_dane
+  extends: .build_and_test_on_ruby
 
 opt_par_gcc_10_pumi:
   variables:
     SPEC: "%gcc@10.3.1 +pumi"
-  extends: .build_and_test_on_dane
+  extends: .build_and_test_on_ruby
 
 # Release
 release_resource:
   variables:
     GIT_STRATEGY: none
-  extends: .on_dane
+  extends: .on_ruby
   stage: release_resource_and_report
   script:
     - echo ${ALLOC_NAME}
@@ -78,17 +78,17 @@ release_resource:
 report_job_success:
   stage: release_resource_and_report
   extends:
-    - .on_dane
+    - .on_ruby
     - .report_job_success
 
 report_job_failure:
   stage: release_resource_and_report
   extends:
-    - .on_dane
+    - .on_ruby
     - .report_job_failure
 
 include:
   - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/dane-config.yml
+  - local: .gitlab/configs/ruby-config.yml
   - local: .gitlab/configs/setup-build-and-test.yml
   - local: .gitlab/configs/report-build-and-test.yml

.gitlab/scripts/baseline

@@ -14,7 +14,7 @@
 # locals
 glob_err=${BASELINE_TEST}.err
 base=${BASELINE_TEST}-${SYS_TYPE}
-if [[ "${MACHINE_NAME}" == "dane" ]]; then
+if [[ "${MACHINE_NAME}" == "ruby" ]]; then
   base="${BASELINE_TEST}-${MACHINE_NAME}"
 fi
 base_diff=${base}.diff
@@ -31,7 +31,7 @@ cd tests
 mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
 
 # run
-if [[ "${MACHINE_NAME}" == "dane" ]]; then
+if [[ "${MACHINE_NAME}" == "ruby" ]]; then
   salloc --nodes=1 --exclusive --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "corona" ]]; then
   salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"

.gitlab/scripts/rebaseline

@@ -11,7 +11,7 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-# There will be collision between corona and dane baselines.
+# There will be collision between corona and ruby baselines.
 # Once the corresponding files have been generated, we can switch to machine
 # specific ref.
 ARTIFACT_PATH=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}
@@ -21,7 +21,7 @@ PATCH_FILE=${ARTIFACT_PATH}.patch
 FULL_FILE=${ARTIFACT_PATH}.out
 DIFF_FILE=${ARTIFACT_PATH}.diff
 
-# There will be collision between corona and dane baselines.
+# There will be collision between corona and ruby baselines.
 # Once the corresponding files have been generated, we can switch to machine
 # specific ref.
 SAVED_NAME=baseline-${SYS_TYPE}.saved

CHANGELOG

@@ -27,34 +27,11 @@ Discretization improvements
 - In the ParMoonolith integration, added support for variational resampling of
   H1 vector fields.
 
-- Added support for boundary integration to the hyperbolic framework. In this
-  regard, new classes `BdrHyperbolicDirichletIntegrator` and
-  `BoundaryHyperbolicFlowIntegrator` have been introduced for implementation
-  of weak Dirichlet boundary conditions with a general flux or for the linear
-  case respectively.
-
-- Added method to compute piecewise linear bounds on high-order functions on
-  tensor-product elements.
-
-- Parallel anisotropic refinement of hexahedral meshes is now supported,
-  provided that neighboring hexahedra are not refined in conflicting directions.
-  A new ParMesh method is added to check for such conflicts, before refinement.
-
 Meshing improvements
 --------------------
-
-- Introduced NC-patch NURBS meshes, which are conforming element-wise but allow
-  for nonconforming patch topology. This new mesh format supports element
-  spacing formulas for refinement, as well as local refinement factors for a
-  subset of knot vectors.
-
 - Added support for higher order meshes in Mesh::MakeSimplicial and
   ParMesh::MakeSimplicial.
 
-- Added a new miniapp for interpolating a surface grid of points in 3D using a
-  smooth NURBS surface, that can then be sampled at arbitrary resolution while
-  staying close to the original geometry. See miniapps/nurbs/nurbs_surface.
-
 GPU computing
 -------------
 - The function Vector::SetSubVector(const Array<int> &, const real_t) now
@@ -62,20 +39,9 @@ GPU computing
   set. This is most often used for setting constant essential boundary
   conditions. A new function Vector::SetSubVectorHost has been added in cases
   where host execution is always needed (e.g. when the DOFs array is small).
-
 - Introduced MFEM_FOREACH_THREAD_DIRECT, which directly maps loop tasks to GPU
   threads, assigning one task per thread.
 
-- Implemented a GPU-accelerated matrix-free AMR derefinement `GridFunction`
-  update operator. This supports mixed geometry meshes and variable order
-  spaces, and is the default derefinement operator constructed by
-  `FiniteElementSpace::Update` and `ParFiniteElementSpace::Update`.
-  The operator requires `FiniteElementSpace::Nonconforming() == true`.
-- Added new method: GridFunction::GetGradients, with GPU support, for computing
-  the gradients of a GridFunction on all elements.
-- Added GPU support in GradientGridFunctionCoefficient and
-  InnerProductCoefficient by implementing their Project methods.
-
 New and updated examples and miniapps
 -------------------------------------
 - Added miniapps to demonstrate an implementation of the absolute-value
@@ -84,22 +50,10 @@ New and updated examples and miniapps
   operators as smoothers.
   These miniapps can be found in `miniapps/diag-smoothers`.
 
-- Added a new miniapp (meshing/mesh-bounding-boxes) that computes the bounding
-  boxes for each element of a given mesh, and the bounds on the determinant of
-  the Jacobian of the transformation.
-
-- Added a new miniapp (tools/gridfunction-bounds) to compute piecewise linear
-  bounds on a given high-order grid function.
-
-- Added a new miniapp (electromagnetics/lorentz) which computes the trajectory
-  of a charged particle, subject to Lorentz forces, in electrostatic and/or
-  magnetostatic fields as computed by the volta or tesla miniapps.
-
-API changes:
+API changes
 -----------
 - mfem::internal::tensor and mfem::internal::dual have been moved to
   mfem::future::tensor and mfem::future::dual.
-
 - API addition: in class `Operator`, added virtual functions: `AbsMult`, and
   `AbsMultTranspose`; in class `Vector`, added `Abs` and `Pow`.
 
@@ -107,25 +61,16 @@ Miscellaneous
 -------------
 - Added the "gpu", "raja-gpu", and "ceed-gpu" backend aliases/shortcuts which
   automatically select between CUDA or HIP.
-
 - The CUDA-specific names used by some of the unit tests like 'cunit_tests' and
   'pcunit_tests' were replaced by names using 'gpu' instead of 'c' (short for
   CUDA) or 'cuda'. These tests automatically run the CUDA/HIP tests based on the
   MFEM build configuration.
-
 - Added the option to enable GPU-aware MPI in MFEM using the environment
   variable 'MFEM_GPU_AWARE_MPI' set to any value. Setting this environment
   variable is an alternative to calling 'Device::SetGPUAwareMPI(true)'.
-
 - Added parallel Address Sanitizer, serial and parallel Undefined Behavior
   Sanitizer and serial Memory Sanitizer GitHub actions tests on Ubuntu.
 
-- FindPointsGSLIB has a new constructor that accepts the mesh object and
-  internally calls the Setup() method so that the user does not have to.
-  The FreeData() method has also been moved to the destructor so the user does
-  not need to manually free-up the memory if the destructor is called before
-  MPI_Finalize().
-
 Version 4.8, released on Apr 9, 2025
 ====================================
 

CMakeLists.txt

@@ -278,11 +278,6 @@ if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
   endif()
 endif()
 
-# Umpire (must be included before hypre, so hypre can use it if needed)
-if (MFEM_USE_UMPIRE)
-   find_package(UMPIRE REQUIRED)
-endif()
-
 # MPI -> hypre; PETSc (optional)
 if (MFEM_USE_MPI)
   find_package(MPI REQUIRED)
@@ -500,13 +495,14 @@ endif()
 
 # RAJA
 if (MFEM_USE_RAJA)
-   # RAJA uses FindCUDA, which needs CMP0146=OLD in CMake >= 3.27
-   if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.27.0)
-      cmake_policy(SET CMP0146 OLD)
-   endif()
    find_package(RAJA REQUIRED)
 endif()
 
+# UMPIRE
+if (MFEM_USE_UMPIRE)
+   find_package(UMPIRE REQUIRED)
+endif()
+
 # GOOGLE-BENCHMARK
 if (MFEM_USE_BENCHMARK)
    find_package(Benchmark REQUIRED)
@@ -600,25 +596,18 @@ set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
     NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
     ADIOS2 MKL_CPARDISO MKL_PARDISO AMGX MAGMA CUSPARSE CUBLAS CALIPER CODIPACK
     BENCHMARK PARELAG TRIBOL MPI_CXX HIP HIPBLAS HIPSPARSE MOONOLITH BLITZ
-    ALGOIM ENZYME CUDA::cudart)
+    ALGOIM ENZYME)
 
-# Add all created targets and *_FOUND libraries in the variables TPL_TARGETS and
-# TPL_LIBRARIES, respectively.
-set(TPL_TARGETS)
+# Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")
 foreach(TPL IN LISTS MFEM_TPLS)
-  if (${TPL}_FOUND OR TARGET ${TPL})
+  if (${TPL}_FOUND)
     message(STATUS "MFEM: using package ${TPL}")
-    if (TARGET ${TPL})
-      list(APPEND TPL_TARGETS ${TPL})
-    else()
-      list(APPEND TPL_LIBRARIES ${${TPL}_LIBRARIES})
-      list(APPEND TPL_INCLUDE_DIRS ${${TPL}_INCLUDE_DIRS})
-    endif()
+    list(APPEND TPL_LIBRARIES ${${TPL}_LIBRARIES})
+    list(APPEND TPL_INCLUDE_DIRS ${${TPL}_INCLUDE_DIRS})
   endif()
 endforeach(TPL)
-
 list(REVERSE TPL_LIBRARIES)
 list(REMOVE_DUPLICATES TPL_LIBRARIES)
 list(REVERSE TPL_LIBRARIES)
@@ -691,10 +680,7 @@ set(MFEM_INSTALL_DIR ${CMAKE_INSTALL_PREFIX})
 # Declaring the library
 mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
 # message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
-target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES} ${TPL_TARGETS})
-if (TPL_TARGETS)
-  add_dependencies(mfem ${TPL_TARGETS})
-endif()
+target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES})
 if (MINGW)
   target_link_libraries(mfem PRIVATE ws2_32)
 endif()

INSTALL

@@ -121,11 +121,6 @@ Parallel build:
    make -j 4
 (For METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)
 
-Parallel build with fetching of hypre and METIS:
-   mkdir <mfem-buil-dir> ; cd <mfem-build-dir>
-   cmake <mfem-source-dir> -DMFEM_USE_MPI=YES -DFETCH_TPLS=YES
-   make -j 4
-
 CUDA build:
    (this build requires CMake 3.17 or newer)
    mkdir <mfem-build-dir> ; cd <mfem-build-dir>
@@ -668,7 +663,6 @@ The specific libraries and their options are:
 - OpenMP (optional), usually part of compiler, used when either MFEM_USE_OPENMP
   or MFEM_USE_LEGACY_OPENMP is set to YES.
   Options: OPENMP_OPT, OPENMP_LIB.
-  Versions: OpenMP >= 3.1 when MFEM_USE_OPENMP=YES.
 
 - High-resolution POSIX clocks: when using MFEM_TIMER_TYPE = 2, it may be
   necessary to link with a system library (e.g. librt.so).
@@ -848,7 +842,6 @@ The specific libraries and their options are:
 - HIP (optional), used when MFEM_USE_HIP = YES.
   URL: https://rocmdocs.amd.com
   Options: HIP_CXX, HIP_ARCH, HIP_OPT, HIP_LIB.
-  Versions: ROCm >= 5.6.1.
 
 - OCCA (optional), used when MFEM_USE_OCCA = YES.
   URL: https://libocca.org
@@ -859,7 +852,7 @@ The specific libraries and their options are:
   URL: https://github.com/CEED/libCEED
        https://ceed.exascaleproject.org/libceed
   Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED >= 0.12.0.
+  Versions: libCEED >= 0.12.
 
 - RAJA (optional), used when MFEM_USE_RAJA = YES.
   Beginning with MFEM v4.5.1, only RAJA v2022.10.3+ is supported.
@@ -1081,9 +1074,6 @@ The following options are CMake specific:
 MFEM_ENABLE_TESTING  - Enable the ctest framework for testing.
 MFEM_ENABLE_EXAMPLES - Build all of the examples by default.
 MFEM_ENABLE_MINIAPPS - Build all of the miniapps by default.
-FETCH_TPLS           - Enable fetching of all supported third-party libraries.
-HYPRE_FETCH          - Enable fetching of hypre.
-METIS_FETCH          - Enable fetching of metis.
 
 External libraries (CMake):
 ---------------------------
@@ -1145,12 +1135,6 @@ The following built-in CMake packages are also used:
       set the <LIBNAME>_LIBRARIES option directly; the configuration option
       <LIBNAME>_DIR is not supported.
 
-The MFEM CMake build system also provides fetching (automated building) for the
-packages/libraries listed below. Note that when fetching is enabled, any related
-auto-detection functionality is disabled.
-
- - HYPRE
- - METIS
 
 Building without GNU make or CMake
 ==================================

config/cmake/MFEMConfig.cmake.in

@@ -84,31 +84,6 @@ set_and_check(MFEM_LIBRARY_DIR "@PACKAGE_LIB_INSTALL_DIR@")
 
 check_required_components(MFEM)
 
-include(CMakeFindDependencyMacro)
-
-if (MFEM_USE_CUDA)
-  # required for projects linking to MFEM+CUDA, even if they don't use CUDA directly
-  find_dependency(CUDAToolkit)
-endif (MFEM_USE_CUDA)
-
-if (MFEM_USE_HIP)
-  # hip/rocm uses the modern MFEM way of linking to targets, need to find dependencies
-  find_dependency(HIP)
-  find_dependency(HIPBLAS)
-  find_dependency(HIPSPARSE)
-  if (MFEM_USE_MPI)
-    # assume HYPRE uses HIP
-    # alternatively could check HYPRE_USING_HIP
-    find_dependency(rocsparse)
-    find_dependency(rocrand)
-    find_dependency(rocsolver)
-  endif (MFEM_USE_MPI)
-endif (MFEM_USE_HIP)
-
-if (MFEM_USE_RAJA)
-   find_dependency(RAJA)
-endif()
-
 if (NOT TARGET mfem)
   include(${CMAKE_CURRENT_LIST_DIR}/MFEMTargets.cmake)
 endif (NOT TARGET mfem)

config/cmake/modules/FindHYPRE.cmake

@@ -9,25 +9,21 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-# Defines the following variables if fetching of TPLs is disabled (default):
+# Defines the following variables:
 #   - HYPRE_FOUND
 #   - HYPRE_LIBRARIES
 #   - HYPRE_INCLUDE_DIRS
 #   - HYPRE_VERSION
 #   - HYPRE_USING_CUDA (internal)
 #   - HYPRE_USING_HIP (internal)
-# otherwise, the following are defined:
-#   - HYPRE (imported library target)
-#   - HYPRE_VERSION (cache variable)
 
-if (HYPRE_FOUND OR TARGET HYPRE)
+if (HYPRE_FOUND)
   if (HYPRE_USING_CUDA)
     find_package(CUDAToolkit REQUIRED)
   endif()
   if (HYPRE_USING_HIP)
     find_package(rocsparse REQUIRED)
     find_package(rocrand REQUIRED)
-    find_package(rocsolver REQUIRED)
   endif()
   if (HYPRE_LIBRARIES AND HYPRE_INCLUDE_DIRS AND HYPRE_VERSION)
     find_package_handle_standard_args(HYPRE
@@ -37,95 +33,6 @@ if (HYPRE_FOUND OR TARGET HYPRE)
   endif()
 endif()
 
-if (HYPRE_FETCH OR FETCH_TPLS)
-  # Collect all HYPRE_ENABLE variables and pass them to hypre, assuming they are BOOL.
-  set(HYPRE_CMAKE_OPTIONS "")
-  get_cmake_property(all_vars VARIABLES)
-  foreach(var ${all_vars})
-    if(var MATCHES "^HYPRE_ENABLE")
-      list(APPEND HYPRE_CMAKE_OPTIONS "-D${var}:BOOL=${${var}}")
-    endif()
-  endforeach()
-
-  set(HYPRE_FETCH_VERSION 2.33.0)
-  set(HYPRE_FETCH_TAG "v${HYPRE_FETCH_VERSION}" CACHE STRING "Tag, branch, or commit for HYPRE")
-  add_library(HYPRE STATIC IMPORTED)
-  # set options and associated dependencies
-  list(APPEND HYPRE_CMAKE_OPTIONS -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE})
-  if (MFEM_USE_CUDA)
-    list(APPEND HYPRE_CMAKE_OPTIONS -DHYPRE_ENABLE_CUDA:BOOL=ON -DCMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES})
-    find_package(CUDAToolkit REQUIRED)
-    target_link_libraries(HYPRE INTERFACE CUDA::cusparse CUDA::curand CUDA::cublas)
-  elseif (MFEM_USE_HIP)
-    list(APPEND HYPRE_CMAKE_OPTIONS -DHYPRE_ENABLE_HIP:BOOL=ON)
-    find_package(rocsparse REQUIRED)
-    find_package(rocrand REQUIRED)
-    target_link_libraries(HYPRE INTERFACE rocsparse rocrand)
-  endif()
-  if (MFEM_USE_CUDA OR MFEM_USE_HIP)
-    if (MFEM_USE_UMPIRE)
-      if (EXISTS ${umpire_DIR})
-        list(APPEND HYPRE_CMAKE_OPTIONS -DHYPRE_ENABLE_UMPIRE:BOOL=ON -Dumpire_DIR:PATH=${umpire_DIR})
-      else()
-        message(FATAL_ERROR "MFEM_USE_UMPIRE=ON, however umpire_DIR isn't visible to HYPRE")
-      endif()
-    else()
-      list(APPEND HYPRE_CMAKE_OPTIONS -DHYPRE_ENABLE_UMPIRE:BOOL=OFF)
-      message(WARNING
-"================================================================================
- Umpire is disabled while building HYPRE with GPU support.
- This is not recommended for performance reasons!
- Consider enabling Umpire with -DMFEM_USE_UMPIRE=ON and providing -DUMPIRE_DIR.
-================================================================================")
-    endif()
-  endif()
-  if (MFEM_USE_SINGLE)
-    list(APPEND HYPRE_CMAKE_OPTIONS -DHYPRE_ENABLE_SINGLE:BOOL=ON)
-  endif()
-  # define external project and create future include directory so it is present
-  # to pass CMake checks at end of MFEM configuration step
-  message(STATUS "Will fetch HYPRE ${HYPRE_FETCH_TAG} to be built with ${HYPRE_CMAKE_OPTIONS}")
-  set(HYPRE_INSTALL ${CMAKE_BINARY_DIR}/fetch/hypre)
-  include(ExternalProject)
-  ExternalProject_Add(hypre
-    GIT_REPOSITORY https://github.com/hypre-space/hypre.git
-    GIT_TAG ${HYPRE_FETCH_TAG}
-    GIT_SHALLOW TRUE
-    GIT_PROGRESS TRUE
-    UPDATE_DISCONNECTED TRUE
-    SOURCE_SUBDIR src
-    PREFIX ${HYPRE_INSTALL}
-    BUILD_COMMAND ${CMAKE_COMMAND} --build . -- -j${CMAKE_BUILD_PARALLEL_LEVEL}
-    CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${HYPRE_INSTALL} -DCMAKE_INSTALL_LIBDIR:PATH=lib ${HYPRE_CMAKE_OPTIONS})
-  file(MAKE_DIRECTORY ${HYPRE_INSTALL}/include)
-  # set imported library target properties
-  add_dependencies(HYPRE hypre)
-  set_target_properties(HYPRE PROPERTIES
-    IMPORTED_LOCATION ${HYPRE_INSTALL}/lib/libHYPRE.a
-    INTERFACE_INCLUDE_DIRECTORIES ${HYPRE_INSTALL}/include)
-  # convert HYPRE version to integer
-  if (HYPRE_FETCH_TAG MATCHES "^v?([0-9]+)\\.([0-9]+)\\.([0-9]+)$")
-    # Exact release tag X.Y.Z
-    string(REGEX MATCHALL "[0-9]+" HYPRE_SPLIT_VERSION "${HYPRE_FETCH_TAG}")
-  elseif (HYPRE_FETCH_VERSION MATCHES "([0-9]+)\\.([0-9]+)(\\.([0-9]+))?")
-    string(REGEX MATCHALL "[0-9]+" HYPRE_SPLIT_VERSION "${HYPRE_FETCH_VERSION}")
-  else (NOT DEFINED HYPRE_VERSION)
-    message(FATAL_ERROR "Unable to find HYPRE release version. Please provide it via -DHYPRE_VERSION")
-  endif()
-  if (HYPRE_SPLIT_VERSION AND NOT DEFINED HYPRE_VERSION)
-    list(GET HYPRE_SPLIT_VERSION 0 HYPRE_MAJOR_VERSION)
-    list(GET HYPRE_SPLIT_VERSION 1 HYPRE_MINOR_VERSION)
-    if (HYPRE_SPLIT_VERSION GREATER 2)
-      list(GET HYPRE_SPLIT_VERSION 2 HYPRE_PATCH_VERSION)
-    else()
-      set(HYPRE_PATCH_VERSION 0)
-    endif()
-    math(EXPR HYPRE_VERSION "10000*${HYPRE_MAJOR_VERSION} + 100*${HYPRE_MINOR_VERSION} + ${HYPRE_PATCH_VERSION}")
-    set(HYPRE_VERSION ${HYPRE_VERSION} CACHE STRING "HYPRE version." FORCE)
-  endif()
-  return()
-endif()
-
 include(MfemCmakeUtilities)
 mfem_find_package(HYPRE HYPRE HYPRE_DIR "include" "HYPRE.h" "lib" "HYPRE"
   "Paths to headers required by HYPRE." "Libraries required by HYPRE."
@@ -190,8 +97,7 @@ endif()
 if (HYPRE_FOUND AND HYPRE_USING_HIP)
   find_package(rocsparse REQUIRED)
   find_package(rocrand REQUIRED)
-  find_package(rocsolver REQUIRED)
-  list(APPEND HYPRE_LIBRARIES ${rocsparse_LIBRARIES} ${rocrand_LIBRARIES} roc::rocsolver roc::rocblas)
+  list(APPEND HYPRE_LIBRARIES ${rocsparse_LIBRARIES} ${rocrand_LIBRARIES})
   set(HYPRE_LIBRARIES ${HYPRE_LIBRARIES} CACHE STRING
       "HYPRE libraries + dependencies." FORCE)
   message(STATUS "Updated HYPRE_LIBRARIES: ${HYPRE_LIBRARIES}")

config/cmake/modules/FindMETIS.cmake

@@ -9,39 +9,10 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-# Defines the following variables if fetching of TPLs is disabled (default):
+# Defines the following variables:
 #   - METIS_FOUND
 #   - METIS_LIBRARIES
 #   - METIS_INCLUDE_DIRS
-#   - METIS_VERSION_5
-# otherwise, the following are defined:
-#   - METIS (imported library target)
-#   - METIS_VERSION_5 (cache variable)
-
-if (METIS_FETCH OR FETCH_TPLS)
-  set(METIS_FETCH_VERSION 4.0.3)
-  add_library(METIS STATIC IMPORTED)
-  # define external project
-  message(STATUS "Will fetch METIS ${METIS_FETCH_VERSION} to be built with default options")
-  set(PREFIX ${CMAKE_BINARY_DIR}/fetch/metis)
-  include(ExternalProject)
-  ExternalProject_Add(metis
-    GIT_REPOSITORY https://github.com/mfem/tpls
-    GIT_TAG b60352fbe9675d374b00828055e55be4584c7995 # tag from 1/16/25
-    GIT_SHALLOW TRUE
-    UPDATE_DISCONNECTED TRUE
-    PREFIX ${PREFIX}
-    CONFIGURE_COMMAND tar -xzf ../metis/metis-${METIS_FETCH_VERSION}-mac.tgz --strip=1
-    BUILD_COMMAND $(MAKE) COPTIONS=-Wno-incompatible-pointer-types
-    INSTALL_COMMAND mkdir -p ${PREFIX}/lib && cp libmetis.a ${PREFIX}/lib/)
-  # set imported library target properties
-  add_dependencies(METIS metis)
-  set_target_properties(METIS PROPERTIES
-    IMPORTED_LOCATION ${PREFIX}/lib/libmetis.a)
-  # set cache variables that would otherwise be set after mfem_find_package call
-  set(METIS_VERSION_5 FALSE CACHE BOOL "Is METIS version 5?")
-  return()
-endif()
 
 include(MfemCmakeUtilities)
 mfem_find_package(METIS METIS METIS_DIR "include;Lib" "metis.h"

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -718,7 +718,7 @@ function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
   get_target_property(IsImported ${tgt} IMPORTED)
   # message(STATUS "${tgt}[IMPORTED]: ${IsImported}")
   # Generally, the possible target types are: STATIC_LIBRARY, MODULE_LIBRARY,
-  # SHARED_LIBRARY, INTERFACE_LIBRARY, UNKNOWN_LIBRARY, EXECUTABLE.
+  # SHARED_LIBRARY, INTERFACE_LIBRARY, EXECUTABLE.
   get_target_property(type ${tgt} TYPE)
   # message(STATUS "${tgt}[TYPE]: ${type}")
   unset(ImportConfig)
@@ -766,7 +766,7 @@ function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
     else()
       message(STATUS " *** Warning: [${tgt}] LOCATION not defined!")
     endif()
-  elseif ("${type}" STREQUAL "SHARED_LIBRARY" OR "${type}" STREQUAL "UNKNOWN_LIBRARY")
+  elseif ("${type}" STREQUAL "SHARED_LIBRARY")
     get_target_property(Location ${tgt} LOCATION)
     if (Location)
       get_filename_component(Dir ${Location} DIRECTORY)
@@ -932,14 +932,12 @@ function(mfem_export_mk_files)
   endif()
   set(MFEM_BUILD_TAG "${CMAKE_SYSTEM}")
   set(MFEM_PREFIX "${CMAKE_INSTALL_PREFIX}")
-  # For the next 4 variables, these are the values for the build-tree version of
+  # For the next 4 variable, these are the values for the build-tree version of
   # 'config.mk'
   set(MFEM_INC_DIR "${PROJECT_BINARY_DIR}")
   set(MFEM_LIB_DIR "${PROJECT_BINARY_DIR}")
   set(MFEM_TEST_MK "${PROJECT_SOURCE_DIR}/config/test.mk")
   set(MFEM_CONFIG_EXTRA "MFEM_BUILD_DIR ?= ${PROJECT_BINARY_DIR}")
-  # TODO: CUDA/HIP support:
-  set(MFEM_XLINKER "${CMAKE_CXX_LINKER_WRAPPER_FLAG}")
   set(MFEM_MPIEXEC ${MPIEXEC})
   if (NOT MFEM_MPIEXEC)
     set(MFEM_MPIEXEC "mpirun")

config/config.hpp

@@ -23,14 +23,11 @@
 #include "_config.hpp"
 #endif
 
-#include <cstdint>
-#include <climits>
-
 namespace mfem
 {
 
 #if (defined(MFEM_USE_CUDA) && defined(__CUDACC__)) || \
-    (defined(MFEM_USE_HIP) && defined(__HIP__))
+    (defined(MFEM_USE_HIP) && defined(__HIPCC__))
 #define MFEM_HOST_DEVICE __host__ __device__
 #else
 #define MFEM_HOST_DEVICE
@@ -64,6 +61,15 @@ constexpr real_t operator""_r(unsigned long long v)
 
 } // namespace mfem
 
+// Defines 'MFEM_THROW' depending if we are compiled with -fno-exceptions
+#if defined(__EXCEPTIONS) || defined(__cpp_exceptions) || defined(_CPPUNWIND)
+#define MFEM_HAS_EXCEPTIONS
+#define MFEM_THROW(exception_type, msg) throw exception_type(msg)
+#else
+#undef MFEM_HAS_EXCEPTIONS
+#define MFEM_THROW(...)
+#endif
+
 // Return value for main function in examples that should be skipped by testing
 // in some case. This return value prevents failures in testing.
 #define MFEM_SKIP_RETURN_VALUE 242

config/config.mk.in

@@ -88,7 +88,6 @@ MFEM_BUILD_TAG = @MFEM_BUILD_TAG@
 MFEM_PREFIX    = @MFEM_PREFIX@
 MFEM_INC_DIR   = @MFEM_INC_DIR@
 MFEM_LIB_DIR   = @MFEM_LIB_DIR@
-MFEM_XLINKER   = @MFEM_XLINKER@
 
 # Location of test.mk
 MFEM_TEST_MK = @MFEM_TEST_MK@

config/defaults.cmake

@@ -89,12 +89,6 @@ option(MFEM_ENABLE_EXAMPLES "Build all of the examples" OFF)
 option(MFEM_ENABLE_MINIAPPS "Build all of the miniapps" OFF)
 option(MFEM_ENABLE_BENCHMARKS "Build all of the benchmarks" OFF)
 
-# Allow a user to specify fetching of certain third-party libraries instead of
-# searching for existing installations.
-option(FETCH_TPLS "Enable fetching of all supported third-party libraries" OFF)
-option(HYPRE_FETCH "Enable fetching of hypre" OFF)
-option(METIS_FETCH "Enable fetching of METIS" OFF)
-
 # Setting CXX/MPICXX on the command line or in user.cmake will overwrite the
 # autodetected C++ compiler.
 # set(CXX g++)

config/defaults.mk

@@ -57,7 +57,7 @@ CUDA_DIR = $(or $(CUDA_HOME),$(patsubst %/,%,$(dir \
 CLANG_CUDA_FLAGS = -xcuda --cuda-path=$(CUDA_DIR) --cuda-gpu-arch=$(CUDA_ARCH)
 # flags for nvcc
 NVCC_FLAGS = -x=cu --expt-extended-lambda --expt-relaxed-constexpr \
- -arch=$(CUDA_ARCH) -isystem "$(CUDA_DIR)/include"
+ -arch=$(CUDA_ARCH)
 # Prefixes for passing flags to the host compiler and linker when using
 # CUDA_CXX=nvcc
 CUDA_XCOMPILER = -Xcompiler=

data/mfem.mesh

@@ -1,593 +0,0 @@
-MFEM mesh v1.0
-
-# Created by: Pointwise
-
-# MFEM Geometry Types:
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-
-dimension
-2
-
-elements
-160
-1 3 1 164 163 0
-1 3 164 165 162 163
-1 3 2 166 164 1
-1 3 166 132 165 164
-1 3 3 167 166 2
-1 3 167 131 132 166
-1 3 4 168 167 3
-1 3 168 130 131 167
-1 3 5 169 168 4
-1 3 169 129 130 168
-1 3 6 170 169 5
-1 3 170 128 129 169
-1 3 171 172 170 6
-1 3 172 127 128 170
-1 3 124 125 172 171
-1 3 125 126 127 172
-1 3 162 165 173 161
-1 3 165 132 133 173
-1 3 161 173 174 160
-1 3 173 133 134 174
-1 3 160 174 175 159
-1 3 174 134 135 175
-1 3 6 7 176 171
-1 3 7 8 177 176
-1 3 171 176 123 124
-1 3 176 177 122 123
-1 3 159 175 178 158
-1 3 175 135 136 178
-1 3 158 178 179 157
-1 3 178 136 137 179
-1 3 157 179 180 156
-1 3 179 137 138 180
-1 3 122 177 181 121
-1 3 177 8 182 181
-1 3 8 9 183 182
-1 3 9 10 184 183
-1 3 10 11 185 184
-1 3 11 12 186 185
-1 3 12 13 187 186
-1 3 13 14 15 187
-1 3 121 181 119 120
-1 3 181 182 118 119
-1 3 182 183 117 118
-1 3 183 184 188 117
-1 3 184 185 109 188
-1 3 185 186 108 109
-1 3 186 187 189 108
-1 3 187 15 16 189
-1 3 109 110 190 188
-1 3 110 111 191 190
-1 3 111 112 113 191
-1 3 188 190 116 117
-1 3 190 191 115 116
-1 3 191 113 114 115
-1 3 189 192 107 108
-1 3 192 193 106 107
-1 3 193 194 105 106
-1 3 194 195 104 105
-1 3 195 196 103 104
-1 3 16 17 192 189
-1 3 17 18 193 192
-1 3 18 19 194 193
-1 3 19 20 195 194
-1 3 20 21 196 195
-1 3 97 98 197 96
-1 3 98 99 198 197
-1 3 99 100 199 198
-1 3 100 101 200 199
-1 3 101 102 201 200
-1 3 102 103 202 201
-1 3 103 196 203 202
-1 3 196 21 22 203
-1 3 96 197 204 95
-1 3 197 198 39 204
-1 3 198 199 38 39
-1 3 199 200 205 38
-1 3 200 201 32 205
-1 3 201 202 31 32
-1 3 202 203 206 31
-1 3 203 22 23 206
-1 3 32 33 207 205
-1 3 33 34 35 207
-1 3 205 207 37 38
-1 3 207 35 36 37
-1 3 39 40 208 204
-1 3 40 41 209 208
-1 3 41 42 210 209
-1 3 42 43 211 210
-1 3 43 44 212 211
-1 3 204 208 94 95
-1 3 208 209 93 94
-1 3 209 210 92 93
-1 3 210 211 91 92
-1 3 211 212 90 91
-1 3 90 212 213 89
-1 3 212 44 214 213
-1 3 44 45 215 214
-1 3 45 46 216 215
-1 3 46 47 217 216
-1 3 47 48 218 217
-1 3 48 49 219 218
-1 3 49 50 51 219
-1 3 89 213 87 88
-1 3 213 214 86 87
-1 3 214 215 85 86
-1 3 215 216 84 85
-1 3 216 217 83 84
-1 3 217 218 82 83
-1 3 218 219 220 82
-1 3 219 51 52 220
-1 3 53 221 220 52
-1 3 221 81 82 220
-1 3 54 222 221 53
-1 3 222 80 81 221
-1 3 55 223 222 54
-1 3 223 79 80 222
-1 3 26 27 224 25
-1 3 27 28 29 224
-1 3 25 224 225 24
-1 3 224 29 30 225
-1 3 24 225 206 23
-1 3 225 30 31 206
-1 3 154 155 226 153
-1 3 155 156 180 226
-1 3 153 226 227 152
-1 3 226 180 138 227
-1 3 152 227 228 151
-1 3 227 138 139 228
-1 3 151 228 229 150
-1 3 228 139 140 229
-1 3 150 229 230 149
-1 3 229 140 141 230
-1 3 149 230 231 148
-1 3 230 141 142 231
-1 3 148 231 232 147
-1 3 231 142 143 232
-1 3 147 232 145 146
-1 3 232 143 144 145
-1 3 56 233 223 55
-1 3 233 78 79 223
-1 3 57 234 233 56
-1 3 234 77 78 233
-1 3 58 235 234 57
-1 3 235 76 77 234
-1 3 61 236 59 60
-1 3 236 235 58 59
-1 3 62 237 236 61
-1 3 237 76 235 236
-1 3 63 238 237 62
-1 3 238 75 76 237
-1 3 64 239 238 63
-1 3 239 74 75 238
-1 3 65 240 239 64
-1 3 240 73 74 239
-1 3 66 241 240 65
-1 3 241 72 73 240
-1 3 67 242 241 66
-1 3 242 71 72 241
-1 3 68 69 242 67
-1 3 69 70 71 242
-
-boundary
-164
-3 1 0 1
-3 1 1 2
-3 1 2 3
-3 1 3 4
-3 1 4 5
-3 1 5 6
-3 1 6 7
-3 1 7 8
-3 1 8 9
-3 1 9 10
-3 1 10 11
-3 1 11 12
-3 1 12 13
-3 1 13 14
-3 1 16 17
-3 1 17 18
-3 1 18 19
-3 1 19 20
-3 1 20 21
-3 1 21 22
-3 1 22 23
-3 1 23 24
-3 1 24 25
-3 1 25 26
-3 1 26 27
-3 1 27 28
-3 1 28 29
-3 1 29 30
-3 1 30 31
-3 1 31 32
-3 1 32 33
-3 1 33 34
-3 1 34 35
-3 1 35 36
-3 1 36 37
-3 1 37 38
-3 1 38 39
-3 1 39 40
-3 1 40 41
-3 1 41 42
-3 1 42 43
-3 1 43 44
-3 1 49 50
-3 1 48 49
-3 1 47 48
-3 1 46 47
-3 1 45 46
-3 1 44 45
-3 1 52 53
-3 1 53 54
-3 1 54 55
-3 1 57 58
-3 1 56 57
-3 1 55 56
-3 1 60 61
-3 1 61 62
-3 1 62 63
-3 1 63 64
-3 1 64 65
-3 1 65 66
-3 1 66 67
-3 1 67 68
-3 1 75 76
-3 1 74 75
-3 1 73 74
-3 1 72 73
-3 1 71 72
-3 1 70 71
-3 1 76 77
-3 1 77 78
-3 1 78 79
-3 1 81 82
-3 1 80 81
-3 1 79 80
-3 1 82 83
-3 1 83 84
-3 1 84 85
-3 1 85 86
-3 1 86 87
-3 1 87 88
-3 1 94 95
-3 1 93 94
-3 1 92 93
-3 1 91 92
-3 1 90 91
-3 1 96 97
-3 1 95 96
-3 1 97 98
-3 1 98 99
-3 1 99 100
-3 1 100 101
-3 1 101 102
-3 1 102 103
-3 1 107 108
-3 1 106 107
-3 1 105 106
-3 1 104 105
-3 1 103 104
-3 1 108 109
-3 1 109 110
-3 1 110 111
-3 1 111 112
-3 1 112 113
-3 1 113 114
-3 1 114 115
-3 1 115 116
-3 1 116 117
-3 1 119 120
-3 1 118 119
-3 1 117 118
-3 1 131 132
-3 1 130 131
-3 1 129 130
-3 1 128 129
-3 1 127 128
-3 1 126 127
-3 1 132 133
-3 1 133 134
-3 1 134 135
-3 1 137 138
-3 1 136 137
-3 1 135 136
-3 1 138 139
-3 1 139 140
-3 1 140 141
-3 1 141 142
-3 1 142 143
-3 1 143 144
-3 1 147 148
-3 1 146 147
-3 1 153 154
-3 1 152 153
-3 1 151 152
-3 1 150 151
-3 1 149 150
-3 1 148 149
-3 1 156 157
-3 1 157 158
-3 1 158 159
-3 1 161 162
-3 1 160 161
-3 1 159 160
-2 1 69 70
-2 1 68 69
-3 1 88 89
-3 1 89 90
-3 1 121 122
-3 1 120 121
-3 1 123 124
-3 1 122 123
-3 1 125 126
-3 1 124 125
-1 1 144 145
-1 1 145 146
-3 1 15 16
-3 1 14 15
-3 1 50 51
-3 1 51 52
-3 1 59 60
-3 1 58 59
-3 1 154 155
-3 1 155 156
-3 1 163 0
-3 1 162 163
-
-vertices
-243
-2
-4 4
-4 3.5
-4 3
-4 2.5
-4 2
-4 1.5
-4 1
-4.5 1
-5 1
-5 1.5
-5 2
-5 2.5
-5 3
-5 3.5
-5 4
-5.500 4
-6 4
-6.500 4
-7 4
-7.5 4
-8 4
-8.5 4
-9 4
-9.5 4
-10 4
-10.5 4
-11 4
-11 3.5
-11 3
-10.5 3
-10 3
-9.5 3
-9.5 2.5
-10 2.5
-10.5 2.5
-10.5 2
-10.5 1.5
-10 1.5
-9.5 1.5
-9.5 1
-10 1
-10.5 1
-11 1
-11.5 1
-12 1
-12 1.5
-12 2
-12 2.5
-12 3
-12 3.5
-12 4
-12.5 4
-13 4
-13.333 3.75
-13.666 3.5
-14.000 3.25
-14.333 3.5
-14.666 3.75
-15.000 4
-15.500 4
-16.000 4
-16.000 3.5
-16.000 3
-16.000 2.5
-16.000 2
-16.000 1.5
-16.000 1
-16.000 0.5
-16.000 0
-15.500 0
-15.000 0
-15.000 0.5000000000000002
-15.000 1
-15.000 1.5
-15.000 2
-15.000 2.5
-15.000 3
-14.666 2.75
-14.333 2.5
-14.000 2.25
-13.666 2.5
-13.333 2.75
-13 3
-13 2.5
-13 2
-13 1.5
-13 1
-13 0.500
-13 0
-12.5 0
-12 0
-11.5 0
-11 0
-10.5 0
-10 0
-9.5 0
-9 0
-8.5 0
-8.5 0.5
-8.5 1
-8.5 1.5
-8.5 2
-8.5 2.5
-8.5 3
-8 3
-7.5 3
-7 3
-6.500 3
-6 3
-6 2.5
-6.5 2.5
-7 2.5
-7.5 2.5
-7.5 2
-7.5 1.5
-7.000 1.5
-6.5 1.5
-6 1.5
-6 1
-6 0.5
-6 0
-5.5 0
-5 0
-4.5 0
-4 0
-3.5 0
-3 0
-3 0.500
-3 1
-3 1.5
-3 2
-3 2.5
-3 3
-2.666 2.75
-2.333 2.5
-2.000 2.25
-1.666 2.5
-1.333 2.75
-1.000 3
-1.000 2.5
-1.000 2
-1.000 1.5
-1.000 1
-1.000 0.5000
-1.000 0
-0.5000 0
-0.0000 0
-0.0000 0.5
-0.0000 1
-0.0000 1.5
-0.0000 2
-0.0000 2.5
-0.0000 3
-0.0000 3.5
-0.0000 4
-0.5000 4
-1.000 4
-1.333 3.75
-1.666 3.5
-2.000 3.25
-2.333 3.5
-2.666 3.75
-3 4
-3.5 4
-3.5 3.5
-3 3.5
-3.5 3
-3.5 2.5
-3.5 2
-3.5 1.5
-3.5 1
-4 0.5
-3.5 0.5
-2.666 3.25
-2.333 3
-2.000 2.75
-4.5 0.5
-5 0.5
-1.666 3
-1.333 3.25
-1.000 3.5
-5.5 0.5
-5.500 1
-5.500 1.5
-5.500 2
-5.500 2.5
-5.500 3
-5.500 3.5
-6 2
-6 3.5
-6.5 2
-7 2
-6.5 3.5
-7 3.5
-7.5 3.5
-8 3.5
-8.5 3.5
-9 0.5
-9 1
-9 1.5
-9 2
-9 2.5
-9 3
-9 3.5
-9.5 0.5
-9.5 2
-9.5 3.5
-10 2
-10 0.5
-10.5 0.5
-11 0.5
-11.5 0.5
-12 0.5
-12.5 0.500
-12.5 1
-12.5 1.5
-12.5 2
-12.5 2.5
-12.5 3
-12.5 3.5
-13 3.5
-13.333 3.250
-13.666 3
-14.000 2.75
-10.5 3.5
-10 3.5
-0.500 3.5
-0.500 3
-0.500 2.5
-0.500 2
-0.500 1.5
-0.500 1
-0.500 0.5
-14.333 3
-14.666 3.25
-15.000 3.5
-15.500 3.5
-15.500 3
-15.500 2.5
-15.500 2
-15.500 1.5
-15.500 1
-15.500 0.5

data/nc-nurbs3d.mesh

@@ -1,342 +0,0 @@
-MFEM NURBS NC-patch mesh v1.0
-dimension
-3
-
-elements
-13
-0 1 5 0 8 10 11 9 4 6 7 5
-0 1 5 0 18 8 24 32 30 23 36 38
-0 1 5 0 0 18 32 14 12 30 38 29
-0 1 5 0 32 24 10 20 38 36 26 35
-0 1 5 0 14 32 20 2 29 38 35 16
-0 1 5 0 30 23 36 38 31 22 37 39
-0 1 5 0 12 30 38 29 13 31 39 28
-0 1 5 0 38 36 26 35 39 37 27 34
-0 1 5 0 29 38 35 16 28 39 34 17
-0 1 5 0 31 22 37 39 19 9 25 33
-0 1 5 0 13 31 39 28 1 19 33 15
-0 1 5 0 39 37 27 34 33 25 11 21
-0 1 5 0 28 39 34 17 15 33 21 3
-
-boundary
-31
-9999 3 8 10 6 4
-9999 3 10 11 7 6
-9999 3 11 9 5 7
-9999 3 9 8 4 5
-9999 3 4 6 7 5
-9999 3 32 24 8 18
-9999 3 18 8 23 30
-9999 3 14 32 18 0
-9999 3 0 18 30 12
-9999 3 14 0 12 29
-9999 3 20 10 24 32
-9999 3 10 20 35 26
-9999 3 2 20 32 14
-9999 3 20 2 16 35
-9999 3 2 14 29 16
-9999 3 30 23 22 31
-9999 3 12 30 31 13
-9999 3 29 12 13 28
-9999 3 26 35 34 27
-9999 3 35 16 17 34
-9999 3 16 29 28 17
-9999 3 31 22 9 19
-9999 3 19 9 25 33
-9999 3 13 31 19 1
-9999 3 28 13 1 15
-9999 3 1 19 33 15
-9999 3 27 34 21 11
-9999 3 33 25 11 21
-9999 3 34 17 3 21
-9999 3 17 28 15 3
-9999 3 15 33 21 3
-
-vertex_to_knotspan
-8
-23 0 1 8 10 11 9
-22 0 2 8 10 11 9
-24 1 0 8 10 11 9
-36 1 1 8 10 11 9
-37 1 2 8 10 11 9
-25 1 3 8 10 11 9
-26 2 1 8 10 11 9
-27 2 2 8 10 11 9
-
-coordinates
-40
-3
-0 0 0
-0 1 0
-4 0 0
-4 1 0
-0 0 4
-0 1 4
-4 0 4
-4 1 4
-0 0 2
-0 1 2
-4 0 2
-4 1 2
-0 0.333333333333333 0
-0 0.666666666666667 0
-2 0 0
-2 1 0
-4 0.333333333333334 0
-4 0.666666666666667 0
-0 0 1
-0 1 1
-4 0 1
-4 1 1
-0 0.666666666666667 2
-0 0.333333333333333 2
-2 0 2
-2 1 2
-4 0.333333333333333 2
-4 0.666666666666667 2
-2 0.666666666666667 0
-2 0.333333333333333 0
-0 0.333333333333333 1
-0 0.666666666666667 1
-1.81325211007895 0 1
-1.81325211007895 1 1
-4 0.666666666666667 1
-4 0.333333333333333 1
-2 0.333333333333333 2
-2 0.666666666666667 2
-1.81325211007895 0.333333333333333 1
-1.81325211007895 0.666666666666667 1
-
-edges
-87
-0 8 10
-1 10 11
-0 9 11
-1 8 9
-0 4 6
-1 6 7
-0 5 7
-1 4 5
-2 4 8
-2 6 10
-2 7 11
-2 5 9
-9 18 8
-7 8 24
-9 32 24
-7 18 32
-9 30 23
-7 23 36
-9 38 36
-7 30 38
-3 18 30
-3 8 23
-3 24 36
-3 32 38
-8 0 18
-8 14 32
-7 0 14
-8 12 30
-8 29 38
-7 12 29
-3 0 12
-3 14 29
-6 24 10
-9 20 10
-6 32 20
-6 36 26
-9 35 26
-6 38 35
-3 10 26
-3 20 35
-8 2 20
-6 14 2
-8 16 35
-6 29 16
-3 2 16
-9 31 22
-7 22 37
-9 39 37
-7 31 39
-4 30 31
-4 23 22
-4 36 37
-4 38 39
-8 13 31
-8 28 39
-7 13 28
-4 12 13
-4 29 28
-6 37 27
-9 34 27
-6 39 34
-4 26 27
-4 35 34
-8 17 34
-6 28 17
-4 16 17
-9 19 9
-7 9 25
-9 33 25
-7 19 33
-5 31 19
-5 22 9
-5 37 25
-5 39 33
-8 1 19
-8 15 33
-7 1 15
-5 13 1
-5 28 15
-6 25 11
-9 21 11
-6 33 21
-5 27 11
-5 34 21
-8 3 21
-6 15 3
-5 17 3
-
-knotvectors
-10
-1 3 0 0 0.5 1 1
-1 4 0 0 0.333333333333333 0.666666666666667 1 1
-1 3 0 0 0.5 1 1
-1 2 0 0 1 1
-1 2 0 0 1 1
-1 2 0 0 1 1
-1 2 0 0 1 1
-1 2 0 0 1 1
-1 2 0 0 1 1
-1 2 0 0 1 1
-
-spacing
-0
-
-weights
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-
-FiniteElementSpace
-FiniteElementCollection: NURBS1
-VDim: 3
-Ordering: 1
-
-0 0 0
-0 1 0
-4 0 0
-4 1 0
-0 0 4
-0 1 4
-4 0 4
-4 1 4
-0 0 2
-0 1 2
-4 0 2
-4 1 2
-0 0.333333333333333 0
-0 0.666666666666667 0
-2 0 0
-2 1 0
-4 0.333333333333334 0
-4 0.666666666666667 0
-0 0 1
-0 1 1
-4 0 1
-4 1 1
-0 0.666666666666667 2
-0 0.333333333333333 2
-2 0 2
-2 1 2
-4 0.333333333333333 2
-4 0.666666666666667 2
-2 0.666666666666667 0
-2 0.333333333333333 0
-0 0.333333333333333 1
-0 0.666666666666667 1
-1.81325211007895 0 1
-1.81325211007895 1 1
-4 0.666666666666667 1
-4 0.333333333333333 1
-2 0.333333333333333 2
-2 0.666666666666667 2
-1.81325211007895 0.333333333333333 1
-1.81325211007895 0.666666666666667 1
-2 0 4
-4 0.333333333333333 4
-4 0.666666666666667 4
-2 1 4
-0 0.333333333333333 4
-0 0.666666666666667 4
-0 0 3
-4 0 3
-4 1 3
-0 1 3
-2 0 3
-4 0.333333333333333 3
-4 0.666666666666667 3
-2 1 3
-0 0.666666666666667 3
-0 0.333333333333333 3
-2 0.333333333333333 4
-2 0.666666666666667 4
-2 0.333333333333333 3
-2 0.666666666666667 3

data/nc3-nurbs.mesh

@@ -1,96 +0,0 @@
-MFEM NURBS NC-patch mesh v1.0
-dimension
-2
-
-# rank attr geom ref_type nodes/children
-elements
-3
-0 1 3 0 0 4 5 1
-0 1 3 0 6 7 4 2
-0 1 3 0 6 3 5 7
-
-# attr geom nodes
-boundary
-7
-1 1 0 4
-1 1 5 1
-1 1 1 0
-1 1 2 6
-1 1 6 3
-1 1 4 2
-1 1 5 3
-
-vertex_to_knotspan
-1
-7 1 4 5
-
-# top-level node coordinates
-coordinates
-8
-2
-0 0
-0 1
-2 0
-2 1
-1 0
-1 1
-2 0.5
-1 0.5
-
-edges
-11
-0 0 4
-1 4 5
-0 1 5
-1 0 1
-2 6 7
-4 7 4
-2 2 4
-4 6 2
-3 6 3
-2 3 5
-3 7 5
-
-knotvectors
-5
-1 3 0 0 0.5 1 1
-1 3 0 0 0.5 1 1
-1 2 0 0 1 1
-1 2 0 0 1 1
-1 2 0 0 1 1
-
-spacing
-0
-
-weights
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-
-FiniteElementSpace
-FiniteElementCollection: NURBS1
-VDim: 2
-Ordering: 1
-
-0 0
-0 1
-2 0
-2 1
-1 0
-1 1
-2 0.5
-1 0.5
-0.5 0
-0.5 1
-0 0.5
-0.5 0.5
-mfem_mesh_end

doc/CodeDocumentation.dox

@@ -202,7 +202,6 @@ namespace mfem {
  * - <a class="el" href="tesla_8cpp_source.html">Tesla</a>: simple magnetostatics simulation code
  * - <a class="el" href="maxwell_8cpp_source.html">Maxwell</a>: simple transient full-wave electromagnetics simulation code
  * - <a class="el" href="joule_8cpp_source.html">Joule</a>: transient magnetics and Joule heating miniapp
- * - <a class="el" href="lorentz_8cpp_source.html">Lorentz</a>: simple particle tracking code based on the Lorentz force
  * - <a class="el" href="classmfem_1_1navier_1_1NavierSolver.html">Navier</a>: solve the transient incompressible Navier-Stokes equations
  * - <a class="el" href="mobius-strip_8cpp_source.html">Mobius Strip</a>: generate various Mobius strip-like meshes
  * - <a class="el" href="klein-bottle_8cpp_source.html">Klein Bottle</a>: generate three types of Klein bottle surfaces

examples/CMakeLists.txt

@@ -205,15 +205,6 @@ if (MFEM_ENABLE_TESTING)
       $<TARGET_FILE:ex25p> "-no-vis" "--mumps-solver"
       ${MPIEXEC_POSTFLAGS})
   endif()
-
-  # Parallel libCEED example
-  if (MFEM_USE_CEED AND MFEM_USE_MPI)
-    add_test(NAME ex1p_ceed_np=${MFEM_MPI_NP}
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:ex1p> "-no-vis" "-d ceed-cpu" "-pa" "-a"
-      ${MPIEXEC_POSTFLAGS})
-  endif()
 endif()
 
 # Include the examples/amgx directory if AmgX is enabled

examples/ex1.cpp

@@ -27,7 +27,6 @@
 //               ex1 -m ../data/fichera-amr.mesh
 //               ex1 -m ../data/mobius-strip.mesh
 //               ex1 -m ../data/mobius-strip.mesh -o -1 -sc
-//               ex1 -m ../data/nc3-nurbs.mesh -o -1
 //
 // Device sample runs:
 //               ex1 -pa -d cuda

examples/makefile

@@ -173,12 +173,6 @@ ex11p-test-cpardiso: ex11p
 	@$(call mfem-test,$<, $(RUN_MPI), MKL_CPARDISO example,--cpardiso)
 test-par-YES: ex11p-test-cpardiso
 endif
-ifeq ($(MFEM_USE_CEED),YES)
-ex1p-test-ceed: ex1p
-	@$(call mfem-test,$<, $(RUN_MPI),\
-	   Parallel libCEED example,-d ceed-cpu -pa -a)
-test-par-YES: ex1p-test-ceed
-endif
 
 # Testing: "test" target and mfem-test* variables are defined in config/test.mk
 

fem/CMakeLists.txt

@@ -82,8 +82,6 @@ set(SRCS
   fe/fe_ser.cpp
   fe_coll.cpp
   fespace.cpp
-  derefmat_op.cpp
-  pderefmat_op.cpp
   geom.cpp
   gridfunc.cpp
   hybridization.cpp
@@ -171,11 +169,6 @@ set(HDRS
   bilinearform.hpp
   bilinearform_ext.hpp
   bilininteg.hpp
-  integ/lininteg_domain_kernels.hpp
-  integ/bilininteg_dgdiffusion_kernels.hpp
-  integ/bilininteg_dgtrace_kernels.hpp
-  integ/bilininteg_vecdiffusion_kernels.hpp
-  integ/bilininteg_convection_kernels.hpp
   integ/bilininteg_diffusion_kernels.hpp
   integ/bilininteg_elasticity_kernels.hpp
   integ/bilininteg_hcurl_kernels.hpp
@@ -246,13 +239,8 @@ set(HDRS
   lor/lor_ams.hpp
   lor/lor_batched.hpp
   lor/lor_h1.hpp
-  lor/lor_dg.hpp
   lor/lor_nd.hpp
   lor/lor_rt.hpp
-  lor/lor_h1_impl.hpp
-  lor/lor_dg_impl.hpp
-  lor/lor_nd_impl.hpp
-  lor/lor_rt_impl.hpp
   lor/lor_util.hpp
   multigrid.hpp
   nonlinearform.hpp

fem/bilinearform_ext.cpp

@@ -266,7 +266,11 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
 
       // Gather the attributes on the host from all the elements
       const Mesh &mesh = *trial_fes->GetMesh();
-      elem_attributes = &mesh.GetElementAttributes();
+      elem_attributes.SetSize(mesh.GetNE());
+      for (int i = 0; i < mesh.GetNE(); ++i)
+      {
+         elem_attributes[i] = mesh.GetAttribute(i);
+      }
    }
 
    // Construct face restriction operators only if the bilinear form has
@@ -325,7 +329,45 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
          bdr_face_dYdn.SetSize(bdr_face_restrict_lex->Height());
       }
 
-      bdr_face_attributes = &trial_fes->GetMesh()->GetBdrFaceAttributes();
+      const Mesh &mesh = *trial_fes->GetMesh();
+      // See LinearFormExtension::Update for explanation of f_to_be logic.
+      std::unordered_map<int,int> f_to_be;
+      for (int i = 0; i < mesh.GetNBE(); ++i)
+      {
+         const int f = mesh.GetBdrElementFaceIndex(i);
+         f_to_be[f] = i;
+      }
+      const int nf_bdr = trial_fes->GetNFbyType(FaceType::Boundary);
+      bdr_attributes.SetSize(nf_bdr);
+      int f_ind = 0;
+      int missing_bdr_elems = 0;
+      for (int f = 0; f < mesh.GetNumFaces(); ++f)
+      {
+         if (!mesh.GetFaceInformation(f).IsOfFaceType(FaceType::Boundary))
+         {
+            continue;
+         }
+         int attribute = 1; // default value
+         if (f_to_be.find(f) != f_to_be.end())
+         {
+            const int be = f_to_be[f];
+            attribute = mesh.GetBdrAttribute(be);
+         }
+         else
+         {
+            // If a boundary face does not correspond to the a boundary element,
+            // we assign it the default attribute of 1. We also generate a
+            // warning at runtime with the number of such missing elements.
+            ++missing_bdr_elems;
+         }
+         bdr_attributes[f_ind] = attribute;
+         ++f_ind;
+      }
+      if (missing_bdr_elems)
+      {
+         MFEM_WARNING("Missing " << missing_bdr_elems << " boundary elements "
+                      "for boundary faces.");
+      }
    }
 }
 
@@ -387,7 +429,7 @@ void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
          mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e)
          {
             const int attr = d_attr[e];
-            if (attr <= 0 || d_m[attr - 1] == 0)
+            if (d_m[attr - 1] == 0)
             {
                for (int i = 0; i < nd; ++i)
                {
@@ -408,7 +450,7 @@ void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
          for (int i = 0; i < iSz; ++i)
          {
             assemble_diagonal_with_markers(*integrators[i], elem_markers[i],
-                                           *elem_attributes, localY);
+                                           elem_attributes, localY);
          }
          const ElementRestriction* H1elem_restrict =
             dynamic_cast<const ElementRestriction*>(elem_restrict);
@@ -434,7 +476,7 @@ void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
       for (int i = 0; i < iSz; ++i)
       {
          assemble_diagonal_with_markers(*integrators[i], elem_markers[i],
-                                        *elem_attributes, y);
+                                        elem_attributes, y);
       }
    }
 
@@ -447,7 +489,7 @@ void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
       for (int i = 0; i < n_bdr_integs; ++i)
       {
          assemble_diagonal_with_markers(*bdr_integs[i], bdr_markers[i],
-                                        *bdr_face_attributes, bdr_face_Y);
+                                        bdr_attributes, bdr_face_Y);
       }
       bdr_face_restrict_lex->AddAbsMultTranspose(bdr_face_Y, y);
    }
@@ -546,7 +588,7 @@ void PABilinearFormExtension::MultInternal(const Vector &x, Vector &y,
          for (int i = 0; i < iSz; ++i)
          {
             AddMultWithMarkers(*integrators[i], localX, elem_markers[i],
-                               *elem_attributes, false, localY, useAbs);
+                               elem_attributes, false, localY, useAbs);
          }
          if (H1elem_restrict && useAbs)
          {
@@ -648,8 +690,8 @@ void PABilinearFormExtension::MultInternal(const Vector &x, Vector &y,
          }
          for (int i = 0; i < n_bdr_integs; ++i)
          {
-            AddMultWithMarkers(*bdr_integs[i], bdr_face_X, bdr_markers[i],
-                               *bdr_face_attributes, false, bdr_face_Y);
+            AddMultWithMarkers(*bdr_integs[i], bdr_face_X, bdr_markers[i], bdr_attributes,
+                               false, bdr_face_Y);
          }
          for (int i = 0; i < n_bdr_face_integs; ++i)
          {
@@ -657,14 +699,12 @@ void PABilinearFormExtension::MultInternal(const Vector &x, Vector &y,
             {
                AddMultNormalDerivativesWithMarkers(
                   *bdr_face_integs[i], bdr_face_X, bdr_face_dXdn,
-                  bdr_face_markers[i], *bdr_face_attributes, bdr_face_Y,
-                  bdr_face_dYdn);
+                  bdr_face_markers[i], bdr_attributes, bdr_face_Y, bdr_face_dYdn);
             }
             else
             {
-               AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X,
-                                  bdr_face_markers[i], *bdr_face_attributes, false,
-                                  bdr_face_Y);
+               AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X, bdr_face_markers[i],
+                                  bdr_attributes, false, bdr_face_Y);
             }
          }
          bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
@@ -687,7 +727,7 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
       localY = 0.0;
       for (int i = 0; i < iSz; ++i)
       {
-         AddMultWithMarkers(*integrators[i], localX, elem_markers[i], *elem_attributes,
+         AddMultWithMarkers(*integrators[i], localX, elem_markers[i], elem_attributes,
                             true, localY);
       }
       elem_restrict->MultTranspose(localY, y);
@@ -734,14 +774,13 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
          bdr_face_Y = 0.0;
          for (int i = 0; i < n_bdr_integs; ++i)
          {
-            AddMultWithMarkers(*bdr_integs[i], bdr_face_X, bdr_markers[i],
-                               *bdr_face_attributes, true, bdr_face_Y);
+            AddMultWithMarkers(*bdr_integs[i], bdr_face_X, bdr_markers[i], bdr_attributes,
+                               true, bdr_face_Y);
          }
          for (int i = 0; i < n_bdr_face_integs; ++i)
          {
-            AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X,
-                               bdr_face_markers[i], *bdr_face_attributes, true,
-                               bdr_face_Y);
+            AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X, bdr_face_markers[i],
+                               bdr_attributes, true, bdr_face_Y);
          }
          bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
       }
@@ -765,7 +804,7 @@ static void AddWithMarkers_(
    mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e)
    {
       const int attr = d_attr[e];
-      if (attr <= 0 || d_m[attr - 1] == 0) { return; }
+      if (d_m[attr - 1] == 0) { return; }
       for (int i = 0; i < nd; ++i)
       {
          d_y(i, e) += d_x(i, e);
@@ -881,8 +920,7 @@ void EABilinearFormExtension::Assemble()
       {
          const int i = idx % sz;
          const int e = idx / sz;
-         const real_t val =
-            d_a[e] > 0 ? (d_m[d_a[e] - 1] ? d_ea_1(i, e) : 0) : 0;
+         const real_t val = d_m[d_a[e] - 1] ? d_ea_1(i, e) : 0.0;
          if (add)
          {
             d_ea_2(i, e) += val;
@@ -915,7 +953,7 @@ void EABilinearFormExtension::Assemble()
             ea_data_tmp.SetSize(ea_data.Size());
             integrators[i]->AssembleEA(*a->FESpace(), ea_data_tmp, false);
             add_with_markers(ea_data_tmp, ea_data, ne, *markers,
-                             *elem_attributes, add);
+                             elem_attributes, add);
          }
       }
    }
@@ -944,7 +982,7 @@ void EABilinearFormExtension::Assemble()
             ea_data_tmp.SetSize(ea_data_bdr.Size());
             bdr_integs[i]->AssembleEABoundary(*a->FESpace(), ea_data_tmp, add);
             add_with_markers(ea_data_tmp, ea_data_bdr, nf_bdr, *markers,
-                             *bdr_face_attributes, add);
+                             bdr_attributes, add);
          }
       }
    }
@@ -993,7 +1031,7 @@ void EABilinearFormExtension::Assemble()
                                                         ea_data_tmp,
                                                         add);
             add_with_markers(ea_data_tmp, ea_data_bdr, nf_bdr, *markers,
-                             *bdr_face_attributes, add);
+                             bdr_attributes, add);
          }
       }
    }

fem/bilinearform_ext.hpp

@@ -69,8 +69,7 @@ class PABilinearFormExtension : public BilinearFormExtension
 protected:
    const FiniteElementSpace *trial_fes, *test_fes; // Not owned
    /// Attributes of all mesh elements.
-   const Array<int> *elem_attributes; // Not owned
-   const Array<int> *bdr_face_attributes; // Not owned
+   Array<int> elem_attributes, bdr_attributes;
    mutable Vector tmp_evec; // Work array
    mutable Vector localX, localY;
    mutable Vector int_face_X, int_face_Y;

fem/bilininteg.hpp

@@ -23,8 +23,6 @@
 
 namespace mfem
 {
-class QuadratureSpace;
-class FaceQuadratureSpace;
 
 /// Abstract base class BilinearFormIntegrator
 class BilinearFormIntegrator : public NonlinearFormIntegrator
@@ -814,7 +812,7 @@ protected:
       const FiniteElement & test_fe) const
    {
       return (trial_fe.GetDim() == 1 && test_fe.GetDim() == 1 &&
-              trial_fe.GetDerivType() == mfem::FiniteElement::GRAD &&
+              trial_fe.GetDerivType() == mfem::FiniteElement::GRAD  &&
               test_fe.GetRangeType()  == mfem::FiniteElement::SCALAR );
    }
 
@@ -886,7 +884,7 @@ protected:
       const FiniteElement & trial_fe,
       const FiniteElement & test_fe) const
    {
-      return (trial_fe.GetDerivType() == mfem::FiniteElement::DIV &&
+      return (trial_fe.GetDerivType() == mfem::FiniteElement::DIV  &&
               test_fe.GetRangeType()  == mfem::FiniteElement::SCALAR );
    }
 
@@ -921,7 +919,7 @@ protected:
       const FiniteElement & trial_fe,
       const FiniteElement & test_fe) const
    {
-      return (trial_fe.GetDerivType() == mfem::FiniteElement::DIV &&
+      return (trial_fe.GetDerivType() == mfem::FiniteElement::DIV  &&
               test_fe.GetRangeType()  == mfem::FiniteElement::VECTOR );
    }
 
@@ -1602,7 +1600,7 @@ public:
    {
       return (trial_fe.GetCurlDim() == 3 && test_fe.GetRangeDim() == 3 &&
               trial_fe.GetRangeType() == mfem::FiniteElement::VECTOR &&
-              trial_fe.GetDerivType() == mfem::FiniteElement::CURL  &&
+              trial_fe.GetDerivType() == mfem::FiniteElement::CURL   &&
               test_fe.GetRangeType()  == mfem::FiniteElement::VECTOR );
    }
 
@@ -1637,7 +1635,7 @@ public:
    {
       return (trial_fe.GetDim() == 2 && test_fe.GetDim() == 2 &&
               trial_fe.GetRangeType() == mfem::FiniteElement::VECTOR &&
-              trial_fe.GetDerivType() == mfem::FiniteElement::CURL  &&
+              trial_fe.GetDerivType() == mfem::FiniteElement::CURL   &&
               test_fe.GetRangeType()  == mfem::FiniteElement::VECTOR );
    }
 
@@ -1671,7 +1669,7 @@ public:
    {
       return (trial_fe.GetDim() == 2 && test_fe.GetDim() == 2 &&
               trial_fe.GetRangeType() == mfem::FiniteElement::SCALAR &&
-              trial_fe.GetDerivType() == mfem::FiniteElement::GRAD  &&
+              trial_fe.GetDerivType() == mfem::FiniteElement::GRAD   &&
               test_fe.GetRangeType()  == mfem::FiniteElement::SCALAR );
    }
 
@@ -1762,7 +1760,7 @@ public:
       const FiniteElement & test_fe) const
    {
       return (trial_fe.GetRangeType() == mfem::FiniteElement::SCALAR &&
-              trial_fe.GetDerivType() == mfem::FiniteElement::GRAD  &&
+              trial_fe.GetDerivType() == mfem::FiniteElement::GRAD   &&
               test_fe.GetRangeType()  == mfem::FiniteElement::SCALAR );
    }
 
@@ -1795,7 +1793,7 @@ public:
       const FiniteElement & test_fe) const
    {
       return (trial_fe.GetRangeType() == mfem::FiniteElement::SCALAR &&
-              trial_fe.GetDerivType() == mfem::FiniteElement::GRAD  &&
+              trial_fe.GetDerivType() == mfem::FiniteElement::GRAD   &&
               test_fe.GetRangeType()  == mfem::FiniteElement::VECTOR &&
               test_fe.GetDerivType()  == mfem::FiniteElement::DIV   );
    }
@@ -1834,7 +1832,7 @@ public:
       const FiniteElement & test_fe) const
    {
       return (trial_fe.GetRangeType() == mfem::FiniteElement::VECTOR &&
-              trial_fe.GetDerivType() == mfem::FiniteElement::DIV   &&
+              trial_fe.GetDerivType() == mfem::FiniteElement::DIV    &&
               test_fe.GetRangeType()  == mfem::FiniteElement::SCALAR &&
               test_fe.GetDerivType()  == mfem::FiniteElement::GRAD
              );
@@ -1975,7 +1973,7 @@ protected:
       const FiniteElement & test_fe) const override
    {
       return (trial_fe.GetCurlDim() == 3 && test_fe.GetRangeDim() == 3 &&
-              trial_fe.GetDerivType() == mfem::FiniteElement::CURL &&
+              trial_fe.GetDerivType() == mfem::FiniteElement::CURL  &&
               test_fe.GetRangeType()  == mfem::FiniteElement::VECTOR );
    }
 
@@ -2178,18 +2176,22 @@ class DiffusionIntegrator: public BilinearFormIntegrator
 {
 public:
 
-   using ApplyKernelType = void(*)(const int, const bool, const Array<real_t>&,
-                                   const Array<real_t>&, const Array<real_t>&,
-                                   const Array<real_t>&,
-                                   const Vector&, const Vector&,
-                                   Vector&, const int, const int);
+   using DiffusionApplyKernelType = void(*)(const int, const bool,
+                                            const Array<real_t>&,
+                                            const Array<real_t>&, const Array<real_t>&,
+                                            const Array<real_t>&,
+                                            const Vector&, const Vector&,
+                                            Vector&, const int, const int);
 
-   using DiagonalKernelType = void(*)(const int, const bool, const Array<real_t>&,
-                                      const Array<real_t>&, const Vector&, Vector&,
-                                      const int, const int);
+   using DiffusionDiagonalKernelType = void(*)(const int, const bool,
+                                               const Array<real_t>&,
+                                               const Array<real_t>&, const Vector&, Vector&,
+                                               const int, const int);
 
-   MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int));
-   MFEM_REGISTER_KERNELS(DiagonalPAKernels, DiagonalKernelType, (int, int, int));
+   MFEM_REGISTER_KERNELS(DiffusionApplyPAKernel, DiffusionApplyKernelType,
+                         (int, int, int));
+   MFEM_REGISTER_KERNELS(DiffusionDiagonalPAKernel, DiffusionDiagonalKernelType,
+                         (int, int, int));
    struct Kernels { Kernels(); };
 
 protected:
@@ -2209,6 +2211,7 @@ private:
    const FiniteElementSpace *fespace;
    const DofToQuad *maps;         ///< Not owned
    const GeometricFactors *geom;  ///< Not owned
+public:
    int dim, ne, dofs1D, quad1D;
    Vector pa_data;
    bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient
@@ -2350,8 +2353,8 @@ public:
    template <int DIM, int D1D, int Q1D>
    static void AddSpecialization()
    {
-      ApplyPAKernels::Specialization<DIM,D1D,Q1D>::Add();
-      DiagonalPAKernels::Specialization<DIM,D1D,Q1D>::Add();
+      DiffusionApplyPAKernel::Specialization<DIM,D1D,Q1D>::Add();
+      DiffusionDiagonalPAKernel::Specialization<DIM,D1D,Q1D>::Add();
    }
 protected:
    const IntegrationRule* GetDefaultIntegrationRule(
@@ -2496,7 +2499,8 @@ private:
 #endif
 
 public:
-   ConvectionIntegrator(VectorCoefficient &q, real_t a = 1.0);
+   ConvectionIntegrator(VectorCoefficient &q, real_t a = 1.0)
+      : Q(&q) { alpha = a; }
 
    void AssembleElementMatrix(const FiniteElement &,
                               ElementTransformation &,
@@ -2529,28 +2533,6 @@ public:
 
    bool SupportsCeed() const override { return DeviceCanUseCeed(); }
 
-   /// arguments: NE, B, G, Bt, Gt, pa_data, x, y, D1D, Q1D
-   using ApplyKernelType = void (*)(const int, const Array<real_t> &,
-                                    const Array<real_t> &,
-                                    const Array<real_t> &,
-                                    const Array<real_t> &, const Vector &,
-                                    const Vector &, Vector &, const int,
-                                    const int);
-
-   /// arguments: DIMS, D1D, Q1D
-   MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int));
-   /// arguments: DIMS, D1D, Q1D
-   MFEM_REGISTER_KERNELS(ApplyPATKernels, ApplyKernelType, (int, int, int));
-
-   template <int DIM, int D1D, int Q1D>
-   static void AddSpecialization()
-   {
-      ApplyPAKernels::Specialization<DIM, D1D, Q1D>::Add();
-      ApplyPATKernels::Specialization<DIM, D1D, Q1D>::Add();
-   }
-
-   struct Kernels { Kernels(); };
-
 protected:
    const IntegrationRule* GetDefaultIntegrationRule(
       const FiniteElement& trial_fe,
@@ -2821,13 +2803,15 @@ protected:
    bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient
 
 public:
-   CurlCurlIntegrator();
+   CurlCurlIntegrator() { Q = NULL; DQ = NULL; MQ = NULL; }
    /// Construct a bilinear form integrator for Nedelec elements
-   CurlCurlIntegrator(Coefficient &q, const IntegrationRule *ir = nullptr);
+   CurlCurlIntegrator(Coefficient &q, const IntegrationRule *ir = NULL) :
+      BilinearFormIntegrator(ir), Q(&q), DQ(NULL), MQ(NULL) { }
    CurlCurlIntegrator(DiagonalMatrixCoefficient &dq,
-                      const IntegrationRule *ir = nullptr);
-   CurlCurlIntegrator(MatrixCoefficient &mq,
-                      const IntegrationRule *ir = nullptr);
+                      const IntegrationRule *ir = NULL) :
+      BilinearFormIntegrator(ir), Q(NULL), DQ(&dq), MQ(NULL) { }
+   CurlCurlIntegrator(MatrixCoefficient &mq, const IntegrationRule *ir = NULL) :
+      BilinearFormIntegrator(ir), Q(NULL), DQ(NULL), MQ(&mq) { }
 
    /* Given a particular Finite Element, compute the
       element curl-curl matrix elmat */
@@ -2857,34 +2841,6 @@ public:
    void AssembleDiagonalPA(Vector& diag) override;
 
    const Coefficient *GetCoefficient() const { return Q; }
-
-   /// arguments: d1d, q1d, symmetric, NE, bo, bc, bot, bct, gc, gct, pa_data,
-   /// x, y, useAbs
-   using ApplyKernelType = void (*)(
-                              const int, const int, const bool, const int, const Array<real_t> &,
-                              const Array<real_t> &, const Array<real_t> &, const Array<real_t> &,
-                              const Array<real_t> &, const Array<real_t> &, const Vector &,
-                              const Vector &, Vector &, const bool);
-
-   /// arguments: d1d, q1d, symmetric, ne, Bo, Bc, Go, Gc, pa_data, diag
-   using DiagonalKernelType = void (*)(const int, const int, const bool,
-                                       const int, const Array<real_t> &,
-                                       const Array<real_t> &,
-                                       const Array<real_t> &,
-                                       const Array<real_t> &, const Vector &,
-                                       Vector &);
-
-   /// parameters: dim, d1d, q1d
-   MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int));
-   /// parameters: dim, d1d, q1d
-   MFEM_REGISTER_KERNELS(DiagonalPAKernels, DiagonalKernelType, (int, int, int));
-   struct Kernels { Kernels(); };
-
-   template <int DIM, int D1D, int Q1D> static void AddSpecialization()
-   {
-      ApplyPAKernels::Specialization<DIM, D1D, Q1D>::Add();
-      DiagonalPAKernels::Specialization<DIM, D1D, Q1D>::Add();
-   }
 };
 
 /** Integrator for $(\mathrm{curl}(u), \mathrm{curl}(v))$ for FE spaces defined by 'dim' copies of a
@@ -3138,18 +3094,21 @@ private:
    Vector vcoeff;
 
 public:
-   VectorDiffusionIntegrator(const IntegrationRule *ir = nullptr);
+   VectorDiffusionIntegrator() { }
 
    /** \brief Integrator with unit coefficient for caller-specified vector
        dimension.
 
        If the vector dimension does not match the true dimension of the space,
        the resulting element matrix will be mathematically invalid. */
-   VectorDiffusionIntegrator(int vector_dimension);
+   VectorDiffusionIntegrator(int vector_dimension)
+      : vdim(vector_dimension) { }
 
-   VectorDiffusionIntegrator(Coefficient &q);
+   VectorDiffusionIntegrator(Coefficient &q)
+      : Q(&q) { }
 
-   VectorDiffusionIntegrator(Coefficient &q, const IntegrationRule *ir);
+   VectorDiffusionIntegrator(Coefficient &q, const IntegrationRule *ir)
+      : BilinearFormIntegrator(ir), Q(&q) { }
 
    /** \brief Integrator with scalar coefficient for caller-specified vector
        dimension.
@@ -3159,7 +3118,8 @@ public:
 
        If the vector dimension does not match the true dimension of the space,
        the resulting element matrix will be mathematically invalid. */
-   VectorDiffusionIntegrator(Coefficient &q, int vector_dimension);
+   VectorDiffusionIntegrator(Coefficient &q, int vector_dimension)
+      : Q(&q), vdim(vector_dimension) { }
 
    /** \brief Integrator with \c VectorCoefficient. The vector dimension of the
        \c FiniteElementSpace is assumed to be the same as the dimension of the
@@ -3170,7 +3130,8 @@ public:
 
        If the vector dimension does not match the true dimension of the space,
        the resulting element matrix will be mathematically invalid. */
-   VectorDiffusionIntegrator(VectorCoefficient &vq);
+   VectorDiffusionIntegrator(VectorCoefficient &vq)
+      : VQ(&vq), vdim(vq.GetVDim()) { }
 
    /** \brief Integrator with \c MatrixCoefficient. The vector dimension of the
        \c FiniteElementSpace is assumed to be the same as the dimension of the
@@ -3181,7 +3142,8 @@ public:
 
        If the vector dimension does not match the true dimension of the space,
        the resulting element matrix will be mathematically invalid. */
-   VectorDiffusionIntegrator(MatrixCoefficient& mq);
+   VectorDiffusionIntegrator(MatrixCoefficient& mq)
+      : MQ(&mq), vdim(mq.GetVDim()) { }
 
    void AssembleElementMatrix(const FiniteElement &el,
                               ElementTransformation &Trans,
@@ -3197,28 +3159,6 @@ public:
    void AddMultPA(const Vector &x, Vector &y) const override;
    void AddMultMF(const Vector &x, Vector &y) const override;
    bool SupportsCeed() const override { return DeviceCanUseCeed(); }
-
-   /// arguments: ne, B, G, Bt, Gt, pa_data, x, y, d1d, q1d, vdim
-   using ApplyKernelType = void (*)(const int, const Array<real_t> &,
-                                    const Array<real_t> &,
-                                    const Array<real_t> &,
-                                    const Array<real_t> &, const Vector &,
-                                    const Vector &, Vector &, const int,
-                                    const int, const int);
-
-   /// arguments: dim, vdim, d1d, q1d
-   MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int, int));
-
-   template <int DIM, int VDIM, int D1D, int Q1D>
-   static void AddSpecialization()
-   {
-      ApplyPAKernels::Specialization<DIM, VDIM, D1D, Q1D>::Add();
-   }
-
-   struct Kernels
-   {
-      Kernels();
-   };
 };
 
 /** Integrator for the linear elasticity form:
@@ -3372,8 +3312,8 @@ public:
 class DGTraceIntegrator : public BilinearFormIntegrator
 {
 protected:
-   Coefficient *rho = nullptr;
-   VectorCoefficient *u = nullptr;
+   Coefficient *rho;
+   VectorCoefficient *u;
    real_t alpha, beta;
    // PA extension
    Vector pa_data;
@@ -3386,16 +3326,17 @@ private:
    Vector tr_shape1, te_shape1, tr_shape2, te_shape2;
 
 public:
-   DGTraceIntegrator(real_t a, real_t b);
-
    /// Construct integrator with $\rho = 1$, $\beta = \alpha/2$.
-   DGTraceIntegrator(VectorCoefficient &u_, real_t a);
+   DGTraceIntegrator(VectorCoefficient &u_, real_t a)
+   { rho = NULL; u = &u_; alpha = a; beta = 0.5*a; }
 
    /// Construct integrator with $\rho = 1$.
-   DGTraceIntegrator(VectorCoefficient &u_, real_t a, real_t b);
+   DGTraceIntegrator(VectorCoefficient &u_, real_t a, real_t b)
+   { rho = NULL; u = &u_; alpha = a; beta = b; }
 
    DGTraceIntegrator(Coefficient &rho_, VectorCoefficient &u_,
-                     real_t a, real_t b);
+                     real_t a, real_t b)
+   { rho = &rho_; u = &u_; alpha = a; beta = b; }
 
    using BilinearFormIntegrator::AssembleFaceMatrix;
    void AssembleFaceMatrix(const FiniteElement &el1,
@@ -3434,26 +3375,6 @@ public:
    static const IntegrationRule &GetRule(Geometry::Type geom, int order,
                                          const ElementTransformation &T);
 
-   /// arguments: nf, B, Bt, pa_data, x, y, dofs1D, quad1D
-   using ApplyKernelType = void (*)(const int, const Array<real_t> &,
-                                    const Array<real_t> &, const Vector &,
-                                    const Vector &, Vector &, const int,
-                                    const int);
-
-   /// arguments: DIM, d1d, q1d
-   MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int));
-   /// arguments: DIM, d1d, q1d
-   MFEM_REGISTER_KERNELS(ApplyPATKernels, ApplyKernelType, (int, int, int));
-
-   template <int DIM, int D1D, int Q1D> static void AddSpecialization()
-   {
-      ApplyPAKernels::Specialization<DIM, D1D, Q1D>::Add();
-      ApplyPATKernels::Specialization<DIM, D1D, Q1D>::Add();
-   }
-
-   struct Kernels { Kernels(); };
-
-
 private:
    void SetupPA(const FiniteElementSpace &fes, FaceType type);
 };
@@ -3500,8 +3421,8 @@ public:
 class DGDiffusionIntegrator : public BilinearFormIntegrator
 {
 protected:
-   Coefficient *Q = nullptr;
-   MatrixCoefficient *MQ = nullptr;
+   Coefficient *Q;
+   MatrixCoefficient *MQ;
    real_t sigma, kappa;
 
    // these are not thread-safe!
@@ -3516,11 +3437,15 @@ protected:
    IntegrationRules irs{0, Quadrature1D::GaussLobatto};
 
 public:
-   DGDiffusionIntegrator(const real_t s, const real_t k);
-   DGDiffusionIntegrator(Coefficient &q, const real_t s, const real_t k);
-   DGDiffusionIntegrator(MatrixCoefficient &q, const real_t s, const real_t k);
+   DGDiffusionIntegrator(const real_t s, const real_t k)
+      : Q(NULL), MQ(NULL), sigma(s), kappa(k) { }
+   DGDiffusionIntegrator(Coefficient &q, const real_t s, const real_t k)
+      : Q(&q), MQ(NULL), sigma(s), kappa(k) { }
+   DGDiffusionIntegrator(MatrixCoefficient &q, const real_t s, const real_t k)
+      : Q(NULL), MQ(&q), sigma(s), kappa(k) { }
    using BilinearFormIntegrator::AssembleFaceMatrix;
-   void AssembleFaceMatrix(const FiniteElement &el1, const FiniteElement &el2,
+   void AssembleFaceMatrix(const FiniteElement &el1,
+                           const FiniteElement &el2,
                            FaceElementTransformations &Trans,
                            DenseMatrix &elmat) override;
 
@@ -3539,28 +3464,6 @@ public:
 
    const IntegrationRule &GetRule(int order, Geometry::Type geom);
 
-   real_t GetPenaltyParameter() const { return kappa; }
-
-   /// arguments: nf, B, Bt, G, Gt, sigma, pa_data, x, dxdn, y, dydn, dofs1D,
-   /// quad1D
-   using ApplyKernelType = void (*)(const int, const Array<real_t> &,
-                                    const Array<real_t> &,
-                                    const Array<real_t> &,
-                                    const Array<real_t> &, const real_t,
-                                    const Vector &, const Vector &_,
-                                    const Vector &, Vector &, Vector &,
-                                    const int, const int);
-
-   /// arguments: DIM, d1d, q1d
-   MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int));
-
-   template <int DIM, int D1D, int Q1D> static void AddSpecialization()
-   {
-      ApplyPAKernels::Specialization<DIM, D1D, Q1D>::Add();
-   }
-
-   struct Kernels { Kernels(); };
-
 private:
    void SetupPA(const FiniteElementSpace &fes, FaceType type);
 };

fem/ceed/integrators/convection/convection_qf.h

@@ -8,7 +8,6 @@
 // MFEM is free software; you can redistribute it and/or modify it under the
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
-#include <ceed/types.h>
 
 /// A structure used to pass additional data to f_build_conv and f_apply_conv
 struct ConvectionContext {
@@ -92,7 +91,7 @@ CEED_QFUNCTION(f_build_conv_const)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 /// libCEED Q-function for building quadrature data for a convection operator
@@ -168,7 +167,7 @@ CEED_QFUNCTION(f_build_conv_quad)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 /// libCEED Q-function for applying a conv operator
@@ -234,7 +233,7 @@ CEED_QFUNCTION(f_apply_conv)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 /// libCEED Q-function for applying a conv operator
@@ -382,7 +381,7 @@ CEED_QFUNCTION(f_apply_conv_mf_const)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 CEED_QFUNCTION(f_apply_conv_mf_quad)(void *ctx, CeedInt Q,
@@ -526,5 +525,5 @@ CEED_QFUNCTION(f_apply_conv_mf_quad)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }

fem/ceed/integrators/diffusion/diffusion_qf.h

@@ -8,7 +8,7 @@
 // MFEM is free software; you can redistribute it and/or modify it under the
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
-#include <ceed/types.h>
+
 
 /// A structure used to pass additional data to f_build_diff and f_apply_diff
 struct DiffusionContext { CeedInt dim, space_dim, vdim; CeedScalar coeff; };
@@ -85,7 +85,7 @@ CEED_QFUNCTION(f_build_diff_const)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 /// libCEED Q-function for building quadrature data for a diffusion operator
@@ -161,7 +161,7 @@ CEED_QFUNCTION(f_build_diff_quad)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 /// libCEED Q-function for applying a diff operator
@@ -241,7 +241,7 @@ CEED_QFUNCTION(f_apply_diff)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 /// libCEED Q-function for applying a diff operator
@@ -394,7 +394,7 @@ CEED_QFUNCTION(f_apply_diff_mf_const)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 CEED_QFUNCTION(f_apply_diff_mf_quad)(void *ctx, CeedInt Q,
@@ -549,5 +549,5 @@ CEED_QFUNCTION(f_apply_diff_mf_quad)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }

fem/ceed/integrators/mass/mass_qf.h

@@ -8,7 +8,7 @@
 // MFEM is free software; you can redistribute it and/or modify it under the
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
-#include <ceed/types.h>
+
 
 /// A structure used to pass additional data to f_build_diff and f_apply_diff
 struct MassContext { CeedInt dim, space_dim, vdim; CeedScalar coeff; };
@@ -53,7 +53,7 @@ CEED_QFUNCTION(f_build_mass_const)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 /// libCEED Q-function for building quadrature data for a mass operator with a
@@ -95,7 +95,7 @@ CEED_QFUNCTION(f_build_mass_quad)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 /// libCEED Q-function for applying a mass operator
@@ -135,7 +135,7 @@ CEED_QFUNCTION(f_apply_mass)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 /// libCEED Q-function for applying a diff operator
@@ -199,7 +199,7 @@ CEED_QFUNCTION(f_apply_mass_mf_const)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 CEED_QFUNCTION(f_apply_mass_mf_quad)(void *ctx, CeedInt Q,
@@ -266,5 +266,5 @@ CEED_QFUNCTION(f_apply_mass_mf_quad)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }

fem/ceed/integrators/nlconvection/nlconvection_qf.h

@@ -8,7 +8,6 @@
 // MFEM is free software; you can redistribute it and/or modify it under the
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
-#include <ceed/types.h>
 
 /// A structure used to pass additional data to f_build_conv and f_apply_conv
 struct NLConvectionContext { CeedInt dim, space_dim, vdim; CeedScalar coeff; };
@@ -88,7 +87,7 @@ CEED_QFUNCTION(f_build_conv_const)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 /// libCEED Q-function for building quadrature data for a convection operator
@@ -168,7 +167,7 @@ CEED_QFUNCTION(f_build_conv_quad)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 /// libCEED Q-function for applying a conv operator
@@ -248,7 +247,7 @@ CEED_QFUNCTION(f_apply_conv)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 /// libCEED Q-function for applying a conv operator
@@ -363,7 +362,7 @@ CEED_QFUNCTION(f_apply_conv_mf_const)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 CEED_QFUNCTION(f_apply_conv_mf_quad)(void *ctx, CeedInt Q,
@@ -476,5 +475,5 @@ CEED_QFUNCTION(f_apply_conv_mf_quad)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }

fem/ceed/interface/ceed.hpp

@@ -18,21 +18,10 @@
 
 #include <ceed.h>
 
-#if !CEED_VERSION_GE(0, 12, 0)
+#if !CEED_VERSION_GE(0,12,0)
 #error MFEM requires a libCEED version >= 0.12.0
 #endif
 
-#if !CEED_VERSION_GE(0, 13, 0)
-#define CeedOperatorCreateComposite(ceed, op)                                  \
-    CeedCompositeOperatorCreate((ceed), (op))
-#define CeedOperatorCompositeAddSub(op, sub)                                   \
-    CeedCompositeOperatorAddSub((op), (sub))
-#define CeedOperatorCompositeGetNumSub(op, num)                                \
-    CeedCompositeOperatorGetNumSub((op), (num))
-#define CeedOperatorCompositeGetSubList(op, list)                              \
-    CeedCompositeOperatorGetSubList((op), (list))
-#endif
-
 namespace mfem
 {
 

fem/ceed/interface/mixed_integrator.hpp

@@ -83,7 +83,7 @@ public:
       }
 
       // Create composite CeedOperator
-      CeedOperatorCreateComposite(internal::ceed, &oper);
+      CeedCompositeOperatorCreate(internal::ceed, &oper);
 
       // Create each sub-CeedOperator
       sub_ops.reserve(element_indices.size());
@@ -101,7 +101,7 @@ public:
          int nelem = *count[value.first];
          sub_op->Assemble(info, fes, ir, nelem, indices, Q);
          sub_ops.push_back(sub_op);
-         CeedOperatorCompositeAddSub(oper, sub_op->GetCeedOperator());
+         CeedCompositeOperatorAddSub(oper, sub_op->GetCeedOperator());
       }
 
       const int ndofs = fes.GetVDim() * fes.GetNDofs();

fem/ceed/interface/util.cpp

@@ -140,7 +140,11 @@ int CeedOperatorGetActiveField(CeedOperator oper, CeedOperatorField *field)
    CeedOperator *subops;
    if (isComposite)
    {
-      ierr = CeedOperatorCompositeGetSubList(oper, &subops); PCeedChk(ierr);
+#if CEED_VERSION_GE(0, 10, 2)
+      ierr = CeedCompositeOperatorGetSubList(oper, &subops); PCeedChk(ierr);
+#else
+      ierr = CeedOperatorGetSubList(oper, &subops); PCeedChk(ierr);
+#endif
       ierr = CeedOperatorGetQFunction(subops[0], &qf); PCeedChk(ierr);
    }
    else
@@ -167,11 +171,7 @@ int CeedOperatorGetActiveField(CeedOperator oper, CeedOperatorField *field)
    for (int i = 0; i < numinputfields; ++i)
    {
       ierr = CeedOperatorFieldGetVector(inputfields[i], &if_vector); PCeedChk(ierr);
-      bool is_active = if_vector == CEED_VECTOR_ACTIVE;
-#if CEED_VERSION_GE(0, 13, 0)
-      ierr = CeedVectorDestroy(&if_vector); PCeedChk(ierr);
-#endif
-      if (is_active)
+      if (if_vector == CEED_VECTOR_ACTIVE)
       {
          if (found)
          {

fem/ceed/solvers/algebraic.cpp

@@ -228,7 +228,7 @@ void AddToCompositeOperator(BilinearFormIntegrator *integ, CeedOperator op)
 {
    if (integ->SupportsCeed())
    {
-      CeedOperatorCompositeAddSub(op, integ->GetCeedOp().GetCeedOperator());
+      CeedCompositeOperatorAddSub(op, integ->GetCeedOp().GetCeedOperator());
    }
    else
    {
@@ -240,7 +240,7 @@ CeedOperator CreateCeedCompositeOperatorFromBilinearForm(BilinearForm &form)
 {
    int ierr;
    CeedOperator op;
-   ierr = CeedOperatorCreateComposite(internal::ceed, &op); PCeedChk(ierr);
+   ierr = CeedCompositeOperatorCreate(internal::ceed, &op); PCeedChk(ierr);
 
    MFEM_VERIFY(form.GetBBFI()->Size() == 0,
                "Not implemented for this integrator!");
@@ -271,13 +271,18 @@ CeedOperator CoarsenCeedCompositeOperator(
    MFEM_ASSERT(isComposite, "");
 
    CeedOperator op_coarse;
-   ierr = CeedOperatorCreateComposite(internal::ceed,
+   ierr = CeedCompositeOperatorCreate(internal::ceed,
                                       &op_coarse); PCeedChk(ierr);
 
    int nsub;
    CeedOperator *subops;
-   ierr = CeedOperatorCompositeGetNumSub(op, &nsub); PCeedChk(ierr);
-   ierr = CeedOperatorCompositeGetSubList(op, &subops); PCeedChk(ierr);
+#if CEED_VERSION_GE(0, 10, 2)
+   ierr = CeedCompositeOperatorGetNumSub(op, &nsub); PCeedChk(ierr);
+   ierr = CeedCompositeOperatorGetSubList(op, &subops); PCeedChk(ierr);
+#else
+   ierr = CeedOperatorGetNumSub(op, &nsub); PCeedChk(ierr);
+   ierr = CeedOperatorGetSubList(op, &subops); PCeedChk(ierr);
+#endif
    for (int isub=0; isub<nsub; ++isub)
    {
       CeedOperator subop = subops[isub];
@@ -289,7 +294,7 @@ CeedOperator CoarsenCeedCompositeOperator(
       // refcounted by existing objects
       ierr = CeedBasisDestroy(&basis_coarse); PCeedChk(ierr);
       ierr = CeedBasisDestroy(&basis_c2f); PCeedChk(ierr);
-      ierr = CeedOperatorCompositeAddSub(op_coarse, subop_coarse);
+      ierr = CeedCompositeOperatorAddSub(op_coarse, subop_coarse);
       PCeedChk(ierr);
       ierr = CeedOperatorDestroy(&subop_coarse); PCeedChk(ierr);
    }

fem/ceed/solvers/full-assembly.cpp

@@ -81,27 +81,12 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
       ierr = CeedOperatorFieldGetVector(input_fields[i], &vec); PCeedChk(ierr);
       if (vec == CEED_VECTOR_ACTIVE)
       {
-         CeedBasis basis;
-         ierr = CeedOperatorFieldGetBasis(input_fields[i], &basis); PCeedChk(ierr);
-         if (!basisin)
-         {
-            ierr = CeedBasisReferenceCopy(basis, &basisin); PCeedChk(ierr);
-         }
-#if CEED_VERSION_GE(0, 13, 0)
-         ierr = CeedBasisDestroy(&basis); PCeedChk(ierr);
-#endif
+         ierr = CeedOperatorFieldGetBasis(input_fields[i], &basisin);
+         PCeedChk(ierr);
          ierr = CeedBasisGetNumComponents(basisin, &ncomp); PCeedChk(ierr);
          ierr = CeedBasisGetDimension(basisin, &dim); PCeedChk(ierr);
-         CeedElemRestriction rstr;
-         ierr = CeedOperatorFieldGetElemRestriction(input_fields[i], &rstr);
+         ierr = CeedOperatorFieldGetElemRestriction(input_fields[i], &rstrin);
          PCeedChk(ierr);
-         if (!rstrin)
-         {
-            ierr = CeedElemRestrictionReferenceCopy(rstr, &rstrin); PCeedChk(ierr);
-         }
-#if CEED_VERSION_GE(0, 13, 0)
-         ierr = CeedElemRestrictionDestroy(&rstr); PCeedChk(ierr);
-#endif
          CeedEvalMode emode;
          ierr = CeedQFunctionFieldGetEvalMode(qffields[i], &emode);
          PCeedChk(ierr);
@@ -127,9 +112,6 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
                break; // Caught by QF Assembly
          }
       }
-#if CEED_VERSION_GE(0, 13, 0)
-      ierr = CeedVectorDestroy(&vec); PCeedChk(ierr);
-#endif
    }
 
    // Determine active output basis
@@ -145,25 +127,11 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
       ierr = CeedOperatorFieldGetVector(output_fields[i], &vec); PCeedChk(ierr);
       if (vec == CEED_VECTOR_ACTIVE)
       {
-         CeedBasis basis;
-         ierr = CeedOperatorFieldGetBasis(output_fields[i], &basis); PCeedChk(ierr);
-         if (!basisout)
-         {
-            ierr = CeedBasisReferenceCopy(basis, &basisout); PCeedChk(ierr);
-         }
-#if CEED_VERSION_GE(0, 13, 0)
-         ierr = CeedBasisDestroy(&basis); PCeedChk(ierr);
-#endif
-         CeedElemRestriction rstr;
-         ierr = CeedOperatorFieldGetElemRestriction(output_fields[i], &rstr);
+         ierr = CeedOperatorFieldGetBasis(output_fields[i], &basisout);
+         PCeedChk(ierr);
+         ierr = CeedOperatorFieldGetElemRestriction(output_fields[i], &rstrout);
+         PCeedChk(ierr);
          PCeedChk(ierr);
-         if (!rstrout)
-         {
-            ierr = CeedElemRestrictionReferenceCopy(rstr, &rstrout); PCeedChk(ierr);
-         }
-#if CEED_VERSION_GE(0, 13, 0)
-         ierr = CeedElemRestrictionDestroy(&rstr); PCeedChk(ierr);
-#endif
          CeedEvalMode emode;
          ierr = CeedQFunctionFieldGetEvalMode(qffields[i], &emode);
          PCeedChk(ierr);
@@ -189,9 +157,6 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
                break; // Caught by QF Assembly
          }
       }
-#if CEED_VERSION_GE(0, 13, 0)
-      ierr = CeedVectorDestroy(&vec); PCeedChk(ierr);
-#endif
    }
 
    CeedInt nelem, elemsize, nqpts;
@@ -235,11 +200,7 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
    PCeedChk(ierr);
 
    CeedInt layout[3];
-#if CEED_VERSION_GE(0, 13, 0)
-   ierr = CeedElemRestrictionGetELayout(rstr_q, layout); PCeedChk(ierr);
-#else
    ierr = CeedElemRestrictionGetELayout(rstr_q, &layout); PCeedChk(ierr);
-#endif
    ierr = CeedElemRestrictionDestroy(&rstr_q); PCeedChk(ierr);
 
    // enforce structurally symmetric for later elimination
@@ -324,10 +285,6 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
    ierr = CeedVectorRestoreArrayRead(assembledqf, &assembledqfarray);
    PCeedChk(ierr);
    ierr = CeedVectorDestroy(&assembledqf); PCeedChk(ierr);
-   ierr = CeedElemRestrictionDestroy(&rstrin); PCeedChk(ierr);
-   ierr = CeedElemRestrictionDestroy(&rstrout); PCeedChk(ierr);
-   ierr = CeedBasisDestroy(&basisin); PCeedChk(ierr);
-   ierr = CeedBasisDestroy(&basisout); PCeedChk(ierr);
    ierr = CeedHackFree(&emodein); PCeedChk(ierr);
    ierr = CeedHackFree(&emodeout); PCeedChk(ierr);
 
@@ -353,8 +310,13 @@ int CeedOperatorFullAssemble(CeedOperator op, SparseMatrix **mat)
    {
       CeedInt numsub;
       CeedOperator *subops;
-      ierr = CeedOperatorCompositeGetNumSub(op, &numsub); PCeedChk(ierr);
-      ierr = CeedOperatorCompositeGetSubList(op, &subops); PCeedChk(ierr);
+#if CEED_VERSION_GE(0, 10, 2)
+      CeedCompositeOperatorGetNumSub(op, &numsub);
+      ierr = CeedCompositeOperatorGetSubList(op, &subops); PCeedChk(ierr);
+#else
+      CeedOperatorGetNumSub(op, &numsub);
+      ierr = CeedOperatorGetSubList(op, &subops); PCeedChk(ierr);
+#endif
       for (int i = 0; i < numsub; ++i)
       {
          ierr = CeedSingleOperatorFullAssemble(subops[i], out); PCeedChk(ierr);

fem/ceed/solvers/solvers-atpmg.cpp

@@ -120,11 +120,7 @@ int CeedATPMGElemRestriction(int order,
    }
    ierr = CeedVectorRestoreArray(in_lvec, &lvec_data); PCeedChk(ierr);
    CeedInt in_layout[3];
-#if CEED_VERSION_GE(0, 13, 0)
-   ierr = CeedElemRestrictionGetELayout(er_in, in_layout); PCeedChk(ierr);
-#else
    ierr = CeedElemRestrictionGetELayout(er_in, &in_layout); PCeedChk(ierr);
-#endif
    if (in_layout[0] == 0 && in_layout[1] == 0 && in_layout[2] == 0)
    {
       return CeedError(ceed, 1, "Cannot interpret e-vector ordering of given"
@@ -668,11 +664,7 @@ int CeedATPMGOperator(CeedOperator oper, int order_reduction,
 
    for (int i = 0; i < numinputfields; ++i)
    {
-#if CEED_VERSION_GE(0, 13, 0)
-      const char * fieldname;
-#else
       char * fieldname;
-#endif
       ierr = CeedQFunctionFieldGetName(inputqfields[i], &fieldname); PCeedChk(ierr);
       if (if_vector[i] == CEED_VECTOR_ACTIVE)
       {
@@ -684,19 +676,10 @@ int CeedATPMGOperator(CeedOperator oper, int order_reduction,
          ierr = CeedOperatorSetField(coper, fieldname, er_input[i], basis_input[i],
                                      if_vector[i]); PCeedChk(ierr);
       }
-#if CEED_VERSION_GE(0, 13, 0)
-      ierr = CeedVectorDestroy(&if_vector[i]); PCeedChk(ierr);
-      ierr = CeedElemRestrictionDestroy(&er_input[i]); PCeedChk(ierr);
-      ierr = CeedBasisDestroy(&basis_input[i]); PCeedChk(ierr);
-#endif
    }
    for (int i = 0; i < numoutputfields; ++i)
    {
-#if CEED_VERSION_GE(0, 13, 0)
-      const char * fieldname;
-#else
       char * fieldname;
-#endif
       ierr = CeedQFunctionFieldGetName(outputqfields[i], &fieldname); PCeedChk(ierr);
       if (of_vector[i] == CEED_VECTOR_ACTIVE)
       {
@@ -708,11 +691,6 @@ int CeedATPMGOperator(CeedOperator oper, int order_reduction,
          ierr = CeedOperatorSetField(coper, fieldname, er_output[i], basis_output[i],
                                      of_vector[i]); PCeedChk(ierr);
       }
-#if CEED_VERSION_GE(0, 13, 0)
-      ierr = CeedVectorDestroy(&of_vector[i]); PCeedChk(ierr);
-      ierr = CeedElemRestrictionDestroy(&er_output[i]); PCeedChk(ierr);
-      ierr = CeedBasisDestroy(&basis_output[i]); PCeedChk(ierr);
-#endif
    }
    delete [] er_input;
    delete [] er_output;
@@ -763,9 +741,7 @@ int CeedOperatorGetOrder(CeedOperator oper, CeedInt * order)
    int P1d;
    ierr = CeedBasisGetNumNodes1D(basis, &P1d); PCeedChk(ierr);
    *order = P1d - 1;
-#if CEED_VERSION_GE(0, 13, 0)
-   ierr = CeedBasisDestroy(&basis); PCeedChk(ierr);
-#endif
+
    return 0;
 }
 

fem/coefficient.cpp

@@ -12,7 +12,6 @@
 // Implementation of Coefficient class
 
 #include "fem.hpp"
-#include "../general/forall.hpp"
 
 #include <cmath>
 #include <limits>
@@ -81,49 +80,6 @@ real_t PWConstCoefficient::Eval(ElementTransformation & T,
    return (constants(att-1));
 }
 
-void PWConstCoefficient::Project(QuadratureFunction &qf)
-{
-   auto &qs = *qf.GetSpace();
-
-   const bool compressed =
-      qs.Offsets(QSpaceOffsetStorage::COMPRESSED).Size() == 1;
-   const int *offsets = qs.Offsets(QSpaceOffsetStorage::COMPRESSED).Read();
-   const int ne = qs.GetNE();
-
-   const int *attributes = [&]()
-   {
-      if (dynamic_cast<QuadratureSpace*>(&qs) != nullptr)
-      {
-         return qs.GetMesh()->GetElementAttributes().Read();
-      }
-      else if (auto *qs_f = dynamic_cast<FaceQuadratureSpace*>(&qs))
-      {
-         MFEM_VERIFY(qs_f->GetFaceType() == FaceType::Boundary,
-                     "Interior faces do not have attributes.");
-         return qs.GetMesh()->GetBdrFaceAttributes().Read();
-      }
-      else
-      {
-         MFEM_ABORT("Unsupported case.");
-      }
-   }();
-
-   const real_t *d_c = constants.Read();
-   real_t *d_qf = qf.Write();
-
-   mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e)
-   {
-      const int a = attributes[e];
-      const real_t elementConstant = d_c[a - 1];
-      const int begin = compressed ? e*offsets[0] : offsets[e];
-      const int end = compressed ? (e+1)*offsets[0] : offsets[e+1];
-      for (int i = begin; i < end; ++i)
-      {
-         d_qf[i] = elementConstant;
-      }
-   });
-}
-
 void PWCoefficient::InitMap(const Array<int> & attr,
                             const Array<Coefficient*> & coefs)
 {
@@ -563,26 +519,6 @@ void GradientGridFunctionCoefficient::Eval(
    }
 }
 
-void GradientGridFunctionCoefficient::Project(QuadratureFunction &qf)
-{
-   const FiniteElementSpace &fes = *GridFunc->FESpace();
-   const Mesh &mesh = *fes.GetMesh();
-   const int sdim = mesh.SpaceDimension();
-   const int gf_vdim = fes.GetVDim(); // assumed to be 1 in this class
-   qf.SetVDim(sdim*gf_vdim);
-   if (mesh.GetNE() == 0) { return; }
-   // All mesh element must be the same type:
-   MFEM_VERIFY(mesh.GetNumGeometries(mesh.Dimension()) == 1,
-               "All mesh elements must be the same type!");
-   const IntegrationRule &ir = qf.GetIntRule(0);
-   // All elements must use the same quadrature rule:
-   MFEM_VERIFY(qf.Size() == sdim*gf_vdim*ir.GetNPoints()*mesh.GetNE(),
-               "All mesh elements must use the same quadrature rule!");
-   // QuadratureFunction uses the layout qf_vdim x nq x ne, i.e.
-   // gf_vdim x sdim x nq x nq, so we need to request QVectorLayout::byVDIM:
-   GridFunc->GetGradients(ir, qf, QVectorLayout::byVDIM);
-}
-
 CurlGridFunctionCoefficient::CurlGridFunctionCoefficient(
    const GridFunction *gf)
    : VectorCoefficient(0)
@@ -1129,41 +1065,6 @@ real_t InnerProductCoefficient::Eval(ElementTransformation &T,
    return va * vb;
 }
 
-void InnerProductCoefficient::Project(QuadratureFunction &qf)
-{
-   MFEM_VERIFY(a->GetVDim() == b->GetVDim(),
-               "Incompatible vector coefficients: a->GetVDim(): "
-               << a->GetVDim() << ", b->GetVDim(): " << b->GetVDim());
-
-   const int vdim = a->GetVDim();
-   MFEM_VERIFY(vdim >= 1, "invalid vdim: " << vdim);
-
-   // When running on device, make sure the output data is allocated before any
-   // local temporary data to reduce potential heap fragmentation:
-   auto dot_d = qf.Write();
-
-   QuadratureFunction qf_a(qf.GetSpace(), vdim);
-   QuadratureFunction qf_b(qf.GetSpace(), vdim);
-
-   a->Project(qf_a);
-   b->Project(qf_b);
-
-   auto a_d = qf_a.Read();
-   auto b_d = qf_b.Read();
-
-   mfem::forall(qf.GetSpace()->GetSize(), [=] MFEM_HOST_DEVICE (int i)
-   {
-      const real_t *ai = a_d + i*vdim;
-      const real_t *bi = b_d + i*vdim;
-      real_t dot = ai[0]*bi[0];
-      for (int d = 1; d < vdim; d++)
-      {
-         dot += ai[d]*bi[d];
-      }
-      dot_d[i] = dot;
-   });
-}
-
 VectorRotProductCoefficient::VectorRotProductCoefficient(VectorCoefficient &A,
                                                          VectorCoefficient &B)
    : a(&A), b(&B), va(A.GetVDim()), vb(B.GetVDim())

fem/coefficient.hpp

@@ -132,9 +132,6 @@ public:
    /// Evaluate the coefficient.
    real_t Eval(ElementTransformation &T,
                const IntegrationPoint &ip) override;
-
-   /// Fill the QuadratureFunction @a qf with the piecewise constant values.
-   void Project(QuadratureFunction &qf) override;
 };
 
 /** @brief A piecewise coefficient with the pieces keyed off the element
@@ -897,9 +894,6 @@ public:
    void Eval(DenseMatrix &M, ElementTransformation &T,
              const IntegrationRule &ir) override;
 
-   /// @copydoc VectorCoefficient::Project(QuadratureFunction &)
-   void Project(QuadratureFunction &qf) override;
-
    virtual ~GradientGridFunctionCoefficient() { }
 };
 
@@ -1777,9 +1771,6 @@ public:
    /// Evaluate the coefficient at @a ip.
    real_t Eval(ElementTransformation &T,
                const IntegrationPoint &ip) override;
-
-   /// @copydoc Coefficient::Project(QuadratureFunction &)
-   void Project(QuadratureFunction &qf) override;
 };
 
 /// Scalar coefficient defined as a cross product of two vectors in the xy-plane.

fem/conduitdatacollection.cpp

@@ -912,7 +912,7 @@ ConduitDataCollection::GridFunctionToBlueprintField(mfem::GridFunction *gf,
 
    if (vdim == 1) // scalar case
    {
-      n_field["values"].set_external(const_cast<real_t *>(gf->HostRead()),
+      n_field["values"].set_external(gf->GetData(),
                                      ndofs);
    }
    else // vector case
@@ -925,18 +925,18 @@ ConduitDataCollection::GridFunctionToBlueprintField(mfem::GridFunction *gf,
       int vdim_stride  = (ordering == Ordering::byNODES ? ndofs : 1);
 
       index_t offset = 0;
-      index_t stride = sizeof(real_t) * entry_stride;
+      index_t stride = sizeof(double) * entry_stride;
 
       for (int d = 0;  d < vdim; d++)
       {
          std::ostringstream oss;
          oss << "v" << d;
          std::string comp_name = oss.str();
-         n_field["values"][comp_name].set_external(const_cast<real_t *>(gf->HostRead()),
+         n_field["values"][comp_name].set_external(gf->GetData(),
                                                    ndofs,
                                                    offset,
                                                    stride);
-         offset +=  sizeof(real_t) * vdim_stride;
+         offset +=  sizeof(double) * vdim_stride;
       }
    }
 

fem/derefmat_op.cpp

@@ -1,266 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "derefmat_op.hpp"
-#include "fes_kernels.hpp"
-
-/// \cond DO_NOT_DOCUMENT
-namespace mfem
-{
-
-namespace internal
-{
-template <Ordering::Type Order, bool Atomic>
-static void DerefMultKernelImpl(const DerefineMatrixOp &op, const Vector &x,
-                                Vector &y)
-{
-   DerefineMatrixOpMultFunctor<Order, Atomic> func;
-   func.xptr = x.Read();
-   y.UseDevice();
-   y = 0.;
-   func.yptr = y.ReadWrite();
-   func.bsptr = op.block_storage.Read();
-   func.boptr = op.block_offsets.Read();
-   func.brptr = op.block_row_idcs_offsets.Read();
-   func.bcptr = op.block_col_idcs_offsets.Read();
-   func.rptr = op.row_idcs.Read();
-   func.cptr = op.col_idcs.Read();
-   func.vdims = op.fespace->GetVDim();
-   func.nblocks = op.block_offsets.Size();
-   func.width = op.Width() / func.vdims;
-   func.height = op.Height() / func.vdims;
-   func.Run(op.max_rows);
-}
-
-} // namespace internal
-
-DerefineMatrixOp::DerefineMatrixOp(FiniteElementSpace &fespace_, int old_ndofs,
-                                   const Table *old_elem_dof,
-                                   const Table *old_elem_fos)
-   : Operator(fespace_.GetVSize(), old_ndofs * fespace_.GetVDim()),
-     fespace(&fespace_)
-{
-   static Kernels kernels;
-   constexpr int max_team_size = 256;
-   /// TODO: Implement DofTransformation support
-
-   MFEM_VERIFY(fespace->Nonconforming(),
-               "Not implemented for conforming meshes.");
-   MFEM_VERIFY(old_ndofs, "Missing previous (finer) space.");
-   MFEM_VERIFY(fespace->GetNDofs() <= old_ndofs,
-               "Previous space is not finer.");
-
-   const CoarseFineTransformations &dtrans =
-      fespace->GetMesh()->ncmesh->GetDerefinementTransforms();
-
-   MFEM_ASSERT(dtrans.embeddings.Size() == old_elem_dof->Size(), "");
-
-   const bool is_dg = fespace->FEColl()->GetContType()
-                      == FiniteElementCollection::DISCONTINUOUS;
-   DenseMatrix localRVO; // for variable-order only
-
-   DenseTensor localR[Geometry::NumGeom];
-   int total_rows = 0;
-   int total_cols = 0;
-   block_offsets.SetSize(dtrans.embeddings.Size());
-   block_offsets.HostWrite();
-   if (fespace->IsVariableOrder())
-   {
-      // TODO: any potential for some compression here?
-      // determine storage size and offsets
-      block_offsets[0] = 0;
-      int total_size = 0;
-      for (int k = 0; k < dtrans.embeddings.Size(); ++k)
-      {
-         const Embedding &emb = dtrans.embeddings[k];
-         const FiniteElement *fe = fespace->GetFE(emb.parent);
-         const int ldof = fe->GetDof();
-         if (k + 1 < dtrans.embeddings.Size())
-         {
-            block_offsets[k + 1] = block_offsets[k] + ldof * ldof;
-         }
-         total_rows += ldof;
-         total_cols += ldof;
-         total_size += ldof * ldof;
-      }
-      block_storage.SetSize(total_size);
-   }
-   else
-   {
-      // compression scheme:
-      // block_offsets is the start of each block, potentially repeated
-      // only need to store localR for used shapes
-      Mesh::GeometryList elem_geoms(*fespace->GetMesh());
-
-      int geom_offsets[Geometry::NumGeom];
-      {
-         int size = 0;
-         for (int i = 0; i < elem_geoms.Size(); ++i)
-         {
-            fespace->GetLocalDerefinementMatrices(elem_geoms[i],
-                                                  localR[elem_geoms[i]]);
-            geom_offsets[elem_geoms[i]] = size;
-            size += localR[elem_geoms[i]].TotalSize();
-         }
-         block_storage.SetSize(size);
-         // copy blocks into block_storage
-         auto bs_ptr = block_storage.HostWrite();
-         for (int i = 0; i < elem_geoms.Size(); ++i)
-         {
-            std::copy(localR[elem_geoms[i]].Data(),
-                      localR[elem_geoms[i]].Data()
-                      + localR[elem_geoms[i]].TotalSize(),
-                      bs_ptr);
-            bs_ptr += localR[elem_geoms[i]].TotalSize();
-         }
-      }
-      for (int k = 0; k < dtrans.embeddings.Size(); ++k)
-      {
-         const Embedding &emb = dtrans.embeddings[k];
-         Geometry::Type geom =
-            fespace->GetMesh()->GetElementBaseGeometry(emb.parent);
-
-         auto size = localR[geom].SizeI() * localR[geom].SizeJ();
-         total_rows += localR[geom].SizeI();
-         total_cols += localR[geom].SizeJ();
-         // set block offsets and sizes
-         block_offsets[k] = geom_offsets[geom] + size * emb.matrix;
-      }
-   }
-   row_idcs.SetSize(total_rows);
-   row_idcs.HostWrite();
-   col_idcs.SetSize(total_cols);
-   col_idcs.HostWrite();
-   block_row_idcs_offsets.SetSize(dtrans.embeddings.Size() + 1);
-   block_row_idcs_offsets.HostWrite();
-   block_col_idcs_offsets.SetSize(dtrans.embeddings.Size() + 1);
-   block_col_idcs_offsets.HostWrite();
-   block_row_idcs_offsets[0] = 0;
-   block_col_idcs_offsets[0] = 0;
-
-   // compute index information
-   Array<int> dofs, old_dofs;
-   max_rows = 1;
-
-   {
-      Array<int> mark(fespace->GetNDofs());
-      mark = 0;
-      auto bs_ptr = block_storage.HostWrite();
-      int ridx = 0;
-      int cidx = 0;
-      int num_marked = 0;
-      for (int k = 0; k < dtrans.embeddings.Size(); k++)
-      {
-         const Embedding &emb = dtrans.embeddings[k];
-         Geometry::Type geom =
-            fespace->GetMesh()->GetElementBaseGeometry(emb.parent);
-
-         if (fespace->IsVariableOrder())
-         {
-            const FiniteElement *fe = fespace->GetFE(emb.parent);
-            const DenseTensor &pmats = dtrans.point_matrices[geom];
-            const int ldof = fe->GetDof();
-
-            IsoparametricTransformation isotr;
-            isotr.SetIdentityTransformation(geom);
-
-            localRVO.SetSize(ldof, ldof);
-            isotr.SetPointMat(pmats(emb.matrix));
-            // Local restriction is size ldofxldof assuming that the parent
-            // and child are of same polynomial order.
-            fe->GetLocalRestriction(isotr, localRVO);
-            // copy block
-            auto size = localRVO.Height() * localRVO.Width();
-            std::copy(localRVO.Data(), localRVO.Data() + size, bs_ptr);
-            bs_ptr += size;
-         }
-         DenseMatrix &lR =
-            fespace->IsVariableOrder() ? localRVO : localR[geom](emb.matrix);
-         block_row_idcs_offsets[k + 1] =
-            block_row_idcs_offsets[k] + lR.Height();
-         block_col_idcs_offsets[k + 1] = block_col_idcs_offsets[k] + lR.Width();
-         max_rows = std::max(lR.Height(), max_rows);
-         // index information
-         fespace->elem_dof->GetRow(emb.parent, dofs);
-         old_elem_dof->GetRow(k, old_dofs);
-         MFEM_VERIFY(old_dofs.Size() == dofs.Size(),
-                     "Parent and child must have same #dofs.");
-         for (int i = 0; i < lR.Height(); ++i, ++ridx)
-         {
-            if (!std::isfinite(lR(i, 0)))
-            {
-               row_idcs[ridx] = INT_MAX;
-               continue;
-            }
-            int r = dofs[i];
-            int m = (r >= 0) ? r : (-1 - r);
-            if (is_dg || !mark[m])
-            {
-               row_idcs[ridx] = r;
-               mark[m] = 1;
-               ++num_marked;
-            }
-            else
-            {
-               row_idcs[ridx] = INT_MAX;
-            }
-         }
-         for (int i = 0; i < lR.Width(); ++i, ++cidx)
-         {
-            col_idcs[cidx] = old_dofs[i];
-         }
-      }
-      if (!is_dg && !fespace->IsVariableOrder())
-      {
-         MFEM_VERIFY(num_marked * fespace->GetVDim() == Height(),
-                     "internal error: not all rows were set.");
-      }
-   }
-   // if not using GPU, set max_rows/max_cols to zero
-   if (Device::Allows(Backend::DEVICE_MASK))
-   {
-      max_rows = std::min(max_rows, max_team_size);
-   }
-   else
-   {
-      max_rows = 1;
-   }
-}
-
-void DerefineMatrixOp::Mult(const Vector &x, Vector &y) const
-{
-   const bool is_dg = fespace->FEColl()->GetContType()
-                      == FiniteElementCollection::DISCONTINUOUS;
-   // DG needs atomic summation
-   MultKernel::Run(fespace->GetOrdering(), is_dg, *this, x, y);
-}
-
-DerefineMatrixOp::Kernels::Kernels()
-{
-   MultKernel::Specialization<Ordering::byNODES, false>::Add();
-   MultKernel::Specialization<Ordering::byVDIM, false>::Add();
-   MultKernel::Specialization<Ordering::byNODES, true>::Add();
-   MultKernel::Specialization<Ordering::byVDIM, true>::Add();
-}
-
-template <Ordering::Type Order, bool Atomic>
-DerefineMatrixOp::MultKernelType DerefineMatrixOp::MultKernel::Kernel()
-{
-   return internal::DerefMultKernelImpl<Order, Atomic>;
-}
-
-DerefineMatrixOp::MultKernelType
-DerefineMatrixOp::MultKernel::Fallback(Ordering::Type, bool)
-{
-   MFEM_ABORT("invalid MultKernel parameters");
-}
-} // namespace mfem
-/// \endcond DO_NOT_DOCUMENT

fem/derefmat_op.hpp

@@ -1,65 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_DEREFMAT_OP
-#define MFEM_DEREFMAT_OP
-
-#include "fespace.hpp"
-
-#include "kernel_dispatch.hpp"
-
-namespace mfem
-{
-
-/// \cond DO_NOT_DOCUMENT
-
-struct DerefineMatrixOp : public Operator
-{
-   FiniteElementSpace *fespace;
-   /// offsets into block_storage
-   Array<int> block_offsets;
-   /// offsets into row_idcs
-   Array<int> block_row_idcs_offsets;
-   /// offsets into col_idcs
-   Array<int> block_col_idcs_offsets;
-   /// mapping for row dofs, INT_MAX indicates the block row should be ignored.
-   /// negative means the row data should be negated.
-   Array<int> row_idcs;
-   /// mapping for col dofs, negative means the col data should be negated.
-   Array<int> col_idcs;
-   /// dense block matrices which can be reused to construct the full matrix
-   /// operation. These are stored contiguously and blocks have no restrictions
-   /// on shape (can be rectangle and differ from block to block).
-   Vector block_storage;
-   /// maximum height of any block in block_storage for GPU
-   /// parallelization, or 1 for CPU runs.
-   int max_rows;
-
-   using MultKernelType = void (*)(const DerefineMatrixOp &, const Vector &,
-                                   Vector &);
-   /// template args: ordering, atomic
-   MFEM_REGISTER_KERNELS(MultKernel, MultKernelType, (Ordering::Type, bool));
-
-   struct Kernels
-   {
-      Kernels();
-   };
-
-   void Mult(const Vector &x, Vector &y) const;
-
-   DerefineMatrixOp(FiniteElementSpace &fespace_, int old_ndofs,
-                    const Table *old_elem_dof, const Table *old_elem_fos);
-};
-
-/// \endcond DO_NOT_DOCUMENT
-
-} // namespace mfem
-#endif

fem/dfem/action.hpp

@@ -0,0 +1,488 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include <cstddef>
+#include <utility>
+
+#include "fem/kernels.hpp"
+#include "fem/kernel_dispatch.hpp"
+
+#include "util.hpp"
+
+#if defined(__has_include) && __has_include("general/nvtx.hpp") && !defined(_WIN32)
+#undef NVTX_COLOR
+#define NVTX_COLOR ::nvtx::kOrchid
+#include "general/nvtx.hpp"
+#else
+#define dbg(...)
+#endif
+
+using restriction_callback_t =
+   std::function<void(std::vector<mfem::Vector> &,
+                      const std::vector<mfem::Vector> &,
+                      std::vector<mfem::Vector> &)>;
+
+template<typename T, typename = void>
+struct GetTensorDim
+{
+   static constexpr int ndim = 0;
+};
+
+template<typename T>
+struct GetTensorDim<T, std::void_t<decltype(T::ndim)>>
+{
+   static constexpr int ndim = T::ndim;
+};
+
+template<typename T>
+using TensorDim = GetTensorDim<std::remove_cv_t<T>>;
+
+namespace mfem::future
+{
+
+template <std::size_t num_fields>
+inline MFEM_ALWAYS_INLINE MFEM_HOST_DEVICE
+std::array<DeviceTensor<1>, num_fields>
+load_field_address(const std::array<int, num_fields> &sizes,
+                   const std::array<DeviceTensor<2>, num_fields> &fields_e,
+                   const int &entity_idx)
+{
+   std::array<DeviceTensor<1>, num_fields> f;
+   for_constexpr<num_fields>([&](auto field_idx)
+   {
+      f[field_idx] =
+         DeviceTensor<1>(&fields_e[field_idx](0, entity_idx), sizes[field_idx]);
+   });
+   return f;
+}
+
+template <std::size_t N>
+inline MFEM_ALWAYS_INLINE MFEM_HOST_DEVICE
+std::array<real_t*, N>
+load_field_e_ptr(const std::array<DeviceTensor<2>, N> &fields_e,
+                 const int e)
+{
+   std::array<real_t*, N> f;
+   for_constexpr<N>([&](auto i) { f[i] = &fields_e[i](0, e); });
+   return f;
+}
+
+namespace qf
+{
+
+template <typename reg_t, typename T, int n>
+inline MFEM_ALWAYS_INLINE MFEM_HOST_DEVICE
+void process_qf_result_from_reg(reg_t &r0,
+                                const int qx, const int qy, const int qz,
+                                const tensor<T, n> &v)
+{
+   r0[0][qz][qy][qx] = v[0];
+   r0[1][qz][qy][qx] = v[1];
+   r0[2][qz][qy][qx] = v[2];
+}
+
+template <int T_Q1D,
+          size_t num_args,
+          typename reg_t,
+          typename qfunc_t,
+          typename args_ts>
+inline MFEM_ALWAYS_INLINE MFEM_HOST_DEVICE
+void apply_kernel(reg_t &r0, reg_t &r1,
+                  real_t *r2, const int Q1D,
+                  const int qx, const int qy, const int qz,
+                  const qfunc_t &qfunc,
+                  args_ts &args)
+{
+   if constexpr (num_args == 2)
+   {
+      // ∇u
+      tensor<real_t, 3> &arg_0 = get<0>(args);
+      arg_0[0] = r1[0][qz][qy][qx];
+      arg_0[1] = r1[1][qz][qy][qx];
+      arg_0[2] = r1[2][qz][qy][qx];
+
+      // D (PA data)
+      alignas(64) tensor<real_t, 3, 3> &arg_1 = get<1>(args);
+
+      if constexpr (T_Q1D > 0)
+      {
+         auto *D = (real_t (*)[T_Q1D][T_Q1D][3][3]) r2;
+         for (int j = 0; j < 3; j++)
+         {
+            for (int k = 0; k < 3; k++)
+            {
+               arg_1[k][j] = D[qx][qy][qz][k][j];
+            }
+         }
+      }
+      else
+      {
+         // dbg("Q1D:{}", Q1D);
+         const auto D = Reshape(r2, 3, 3, Q1D, Q1D, Q1D);
+         for (int j = 0; j < 3; j++)
+         {
+            for (int k = 0; k < 3; k++)
+            {
+               arg_1[k][j] = D(j, k, qz, qy, qx);
+            }
+         }
+      }
+   }
+
+   const auto r = get<0>(apply(qfunc, args));
+
+
+   if constexpr (decltype(r)::ndim == 1)
+      // if constexpr (TensorDim<decltype(r)>::ndim == 1)
+   {
+      // process_qf_result_from_reg(r0, qx, qy, qz, r);
+      r0[0][qz][qy][qx] = r[0];
+      r0[1][qz][qy][qx] = r[1];
+      r0[2][qz][qy][qx] = r[2];
+   }
+   if constexpr (TensorDim<decltype(r)>::ndim == 0)
+   {
+      MFEM_ABORT("qfunc returned a scalar, expected a vector");
+   }
+}
+
+} // namespace qf
+
+template<size_t num_fields,
+         size_t num_inputs,
+         size_t num_outputs,
+         typename qfunc_t,
+         typename input_t,
+         typename output_fop_t>
+class NewActionCallback
+{
+   const restriction_callback_t restriction_cb;
+   const qfunc_t qfunc;
+   input_t &inputs;
+   const std::array<int, num_inputs> input_to_field;
+   const std::array<DofToQuadMap, num_inputs> input_dtq_maps;
+   const std::array<DofToQuadMap, num_outputs> output_dtq_maps;
+   const int num_entities;
+   const int test_vdim;
+   const int num_test_dof;
+   const int dimension;
+   const int q1d_;
+   const ThreadBlocks thread_blocks;
+   SharedMemoryInfo<num_fields, num_inputs, num_outputs> shmem_info;
+   Array<int> &elem_attributes;
+   const output_fop_t &output_fop;
+   const Array<int> &domain_attributes;
+   // refs
+   std::vector<Vector> &fields_e;
+   Vector &residual_e;
+   std::function<void(Vector &, Vector &)> output_restriction_transpose;
+   // args
+   std::vector<Vector> &solutions_l;
+   const std::vector<Vector> &parameters_l;
+   Vector &residual_l;
+
+public:
+   inline MFEM_ALWAYS_INLINE
+   NewActionCallback(const bool use_kernels_specialization,
+                     const restriction_callback_t restriction_cb,
+                     const qfunc_t qfunc,
+                     input_t &inputs,
+                     const std::array<int, num_inputs> input_to_field,
+                     const std::array<DofToQuadMap, num_inputs> input_dtq_maps,
+                     const std::array<DofToQuadMap, num_outputs> output_dtq_maps,
+                     const int num_entities,
+                     const int test_vdim,
+                     const int num_test_dof,
+                     const int dimension,
+                     const int q1d,
+                     const ThreadBlocks thread_blocks,
+                     SharedMemoryInfo<num_fields, num_inputs, num_outputs> shmem_info,
+                     Array<int> &elem_attributes,
+                     const output_fop_t &output_fop,
+                     const Array<int> &domain_attributes,
+                     // refs
+                     std::vector<Vector> &fields_e,
+                     Vector &residual_e,
+                     std::function<void(Vector &, Vector &)> output_restriction_transpose,
+                     // args
+                     std::vector<Vector> &solutions_l,
+                     const std::vector<Vector> &parameters_l,
+                     Vector &residual_l):
+      restriction_cb(restriction_cb),
+      qfunc(qfunc),
+      inputs(inputs),
+      input_to_field(input_to_field),
+      input_dtq_maps(input_dtq_maps),
+      output_dtq_maps(output_dtq_maps),
+      num_entities(num_entities),
+      test_vdim(test_vdim),
+      num_test_dof(num_test_dof),
+      dimension(dimension),
+      q1d_(q1d),
+      thread_blocks(thread_blocks),
+      shmem_info(shmem_info),
+      elem_attributes(elem_attributes),
+      output_fop(output_fop),
+      domain_attributes(domain_attributes),
+      fields_e(fields_e),
+      residual_e(residual_e),
+      output_restriction_transpose(std::move(output_restriction_transpose)),
+      solutions_l(solutions_l),
+      parameters_l(parameters_l),
+      residual_l(residual_l)
+   {
+      if (!use_kernels_specialization) { return; }
+      NewActionCallbackKernels::template Specialization<3,4>::Add();
+      // NewActionCallbackKernels::template Specialization<4,5>::Add();
+      NewActionCallbackKernels::template Specialization<5,6>::Add();
+      // NewActionCallbackKernels::template Specialization<6,7>::Add();
+      NewActionCallbackKernels::template Specialization<7,8>::Add();
+      // NewActionCallbackKernels::template Specialization<8,9>::Add();
+      // NewActionCallbackKernels::template Specialization<9,10>::Add();
+   }
+
+   template<int T_D1D = 0, int T_Q1D = 0>
+   static inline MFEM_ALWAYS_INLINE
+   void action_callback_new(const restriction_callback_t &restriction_cb,
+                            const qfunc_t &qfunc,
+                            input_t &inputs,
+                            const std::array<int, num_inputs> &input_to_field,
+                            const std::array<DofToQuadMap, num_inputs> &input_dtq_maps,
+                            const std::array<DofToQuadMap, num_outputs> &output_dtq_maps,
+                            const int &ne,
+                            const int &test_vdim,
+                            const int &num_test_dof,
+                            const int &dimension,
+                            //   const int q1d,
+                            const ThreadBlocks &thread_blocks,
+                            SharedMemoryInfo<num_fields, num_inputs, num_outputs> &shmem_info,
+                            Array<int> &elem_attributes,
+                            const output_fop_t &output_fop,
+                            const Array<int> &domain_attributes,
+                            // refs
+                            std::vector<Vector> &fields_e,
+                            Vector &residual_e,
+                            std::function<void(Vector &, Vector &)> output_restriction_transpose,
+                            // args
+                            std::vector<Vector> &solutions_l,
+                            const std::vector<Vector> &parameters_l,
+                            Vector &residual_l,
+                            // fallback arguments
+                            const int d1d, const int q1d)
+   {
+      db1();
+      assert(dimension == 3);
+
+      // constexpr int D1D = T_D1D;// ? T_D1D : d1d; // 🔥
+      // constexpr int Q1D = T_Q1D;// ? T_Q1D : q1d; // 🔥
+
+      constexpr int DIM = 3;
+      constexpr int MQ1 = T_Q1D > 0 ? kernels::internal::SetMaxOf(T_Q1D) : 32;
+      constexpr int MD1 = T_D1D > 0 ? kernels::internal::SetMaxOf(T_D1D) : 32;
+      // constexpr int MQ1 = T_Q1D ? T_Q1D : 32;
+      // constexpr int MD1 = T_D1D ? T_D1D : 32;
+
+      // types
+      using qf_signature =
+         typename create_function_signature<decltype(&qfunc_t::operator())>::type;
+      using qf_param_ts = typename qf_signature::parameter_ts;
+
+      // #warning 0xDEADBEEF
+      // auto solutions_l_pi = solutions_l;
+      // solutions_l_pi[0].Randomize(0xDEADBEEF);
+
+      restriction_cb(solutions_l, parameters_l, fields_e); // 🔥 to inline
+      /*for (size_t i = 0; i < 1; i++)
+      {
+         dbl("fields_e: size:{} dot:{}\n", fields_e[i].Size(), fields_e[i]*fields_e[i]);
+         for (int j = 0; j < fields_e[i].Size(); j++)
+         {
+            dba("{} ", fields_e[i][j]);
+         }
+         dbc();
+      }*/
+
+      residual_e = 0.0;
+      auto ye = Reshape(residual_e.ReadWrite(), num_test_dof, test_vdim, ne);
+
+      // std::array<DeviceTensor<2>, num_fields>: (field_sizes, num_entities)
+      auto wrapped_fields_e = wrap_fields(fields_e,
+                                          shmem_info.field_sizes,
+                                          ne);
+
+      const bool has_attr = domain_attributes.Size() > 0;
+      const auto d_domain_attr = domain_attributes.Read();
+      const auto d_elem_attr = elem_attributes.Read();
+
+      // db1("forall");
+      forall([=] MFEM_HOST_DEVICE (int e, void *)
+      {
+         const int D1D = T_D1D ? T_D1D : d1d;
+         const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+         // this could be optimized out
+         if (has_attr && !d_domain_attr[d_elem_attr[e] - 1]) { return; }
+
+         alignas(64) MFEM_SHARED real_t smem[MQ1][MQ1];
+         alignas(64) MFEM_SHARED real_t sB[MD1][MQ1];
+         alignas(64) MFEM_SHARED real_t sG[MD1][MQ1];
+         alignas(64) kernels::internal::d_regs3d_t<DIM, MQ1> r0, r1;
+         alignas(64) real_t *r2;
+
+         const auto fields_e_ptr = load_field_e_ptr(wrapped_fields_e, e);
+
+         // Interpolate
+         // const auto dummy_field_weight = DeviceTensor<1>(nullptr, 0);
+         for_constexpr<num_inputs>([&](auto i)
+         {
+            const auto input = get<i>(inputs);
+            using field_operator_t = std::decay_t<decltype(input)>;
+
+            if constexpr (is_gradient_fop<field_operator_t>::value) // Grad
+            {
+               // const int vdim = input.vdim;
+               constexpr int VDIM = 1; // 🔥
+               const real_t *field_e_r = fields_e_ptr[input_to_field[i]];
+               const auto field = Reshape(field_e_r, D1D, D1D, D1D, VDIM);
+               const auto dtq = input_dtq_maps[i];
+               const auto B = dtq.B, G = dtq.G;
+               // db1("B:{} {} {} {} {} {}", B[0], B[1], B[2], B[3], B[4], B[5]);
+               // db1("G:{} {} {} {} {} {}", G[0], G[1], G[2], G[3], G[4], G[5]);
+               kernels::internal::LoadMatrix(D1D, Q1D, B, sB);
+               kernels::internal::LoadMatrix(D1D, Q1D, G, sG);
+               // const auto B = reinterpret_cast<const real_t (*)[MQ1]>(Bi[i]);
+               // const auto G = reinterpret_cast<const real_t (*)[MQ1]>(Gi[i]);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  kernels::internal::LoadDofs3d(D1D, c, field, r0);
+                  kernels::internal::Grad3d(D1D, Q1D, smem, sB, sG, r0, r1, c);
+               }
+            }
+
+            if constexpr (is_identity_fop<field_operator_t>::value) // Identity
+            {
+               // db1("Identity");
+               r2 = fields_e_ptr[input_to_field[i]];
+            }
+         }); // for_constexpr<num_inputs>
+
+         // db1("Now calling qfunction");
+         auto qf_args = decay_tuple<qf_param_ts> {};
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            // MFEM_UNROLL(MQ1)
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+            {
+               // MFEM_UNROLL(MQ1)
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+               {
+                  // dbg("r1: ({},{},{}) {} {} {}", qx, qy, qz,
+                  //     r1[0][qz][qy][qx], r1[1][qz][qy][qx], r1[2][qz][qy][qx]);
+                  qf::apply_kernel<T_Q1D, num_inputs>(r0, r1, r2, Q1D, qx, qy, qz,
+                                                      qfunc, qf_args);
+                  // dbg("r0: ({},{},{}) {} {} {}", qx, qy, qz,
+                  //     r0[0][qz][qy][qx], r0[1][qz][qy][qx], r0[2][qz][qy][qx]);
+               }
+            }
+         }
+
+         // db1("Integrate");
+         auto y = Reshape(&ye(0, 0, e), num_test_dof, test_vdim);
+         if constexpr (is_gradient_fop<std::decay_t<output_fop_t>>::value) // GradientT
+         {
+            // const int vdim = output_fop.vdim;
+            constexpr int VDIM = 1; // 🔥
+            auto yd = Reshape(&y(0, 0), D1D, D1D, D1D, VDIM);
+            // const auto B = reinterpret_cast<const real_t (*)[MQ1]>(Bo);
+            // const auto G = reinterpret_cast<const real_t (*)[MQ1]>(Go);
+            for (int c = 0; c < VDIM; c++)
+            {
+               kernels::internal::GradTranspose3d(D1D, Q1D, smem, sB, sG, r0, r1, c);
+               kernels::internal::WriteDofs3d(D1D, c, r1, yd);
+            }
+         }
+      },
+      ne,
+      thread_blocks,
+      0,
+      nullptr);
+
+      // dbg("residual_e: size:{} dot:{}",
+      //     residual_e.Size(), residual_e * residual_e);
+      output_restriction_transpose(residual_e, residual_l);  // 🔥 to inline
+   }
+
+   using NewActionCallbackType = decltype(
+                                    &NewActionCallback::action_callback_new<>);
+   MFEM_REGISTER_KERNELS(NewActionCallbackKernels, NewActionCallbackType, (int,
+                                                                           int));
+
+   inline MFEM_ALWAYS_INLINE MFEM_HOST_DEVICE
+   void Apply(const int d1d, const int q1d)
+   {
+      NewActionCallbackKernels::Run(d1d, q1d,
+                                    // args
+                                    restriction_cb,
+                                    qfunc,
+                                    inputs,
+                                    input_to_field,
+                                    input_dtq_maps,
+                                    output_dtq_maps,
+                                    num_entities,
+                                    test_vdim,
+                                    num_test_dof,
+                                    dimension,
+                                    thread_blocks,
+                                    shmem_info,
+                                    elem_attributes,
+                                    output_fop,
+                                    domain_attributes,
+                                    fields_e,
+                                    residual_e,
+                                    output_restriction_transpose,
+                                    solutions_l,
+                                    parameters_l,
+                                    residual_l,
+                                    // fallback arguments
+                                    d1d, q1d);
+   }
+};
+
+template<size_t num_fields,
+         size_t num_inputs,
+         size_t num_outputs,
+         typename qfunc_t,
+         typename input_t,
+         typename output_fop_t>
+template<int D1D, int Q1D>
+inline MFEM_ALWAYS_INLINE
+typename NewActionCallback<num_fields, num_inputs, num_outputs, qfunc_t, input_t, output_fop_t>::NewActionCallbackType
+NewActionCallback<num_fields, num_inputs, num_outputs, qfunc_t, input_t, output_fop_t>::NewActionCallbackKernels::Kernel()
+{
+   return action_callback_new<D1D, Q1D>;
+}
+
+template<size_t num_fields,
+         size_t num_inputs,
+         size_t num_outputs,
+         typename qfunc_t,
+         typename input_t,
+         typename output_fop_t>
+inline MFEM_ALWAYS_INLINE
+typename NewActionCallback<num_fields, num_inputs, num_outputs, qfunc_t, input_t, output_fop_t>::NewActionCallbackType
+NewActionCallback<num_fields, num_inputs, num_outputs, qfunc_t, input_t, output_fop_t>::NewActionCallbackKernels::Fallback
+(int d1d, int q1d)
+{
+   return action_callback_new<>;
+}
+
+} // namespace mfem::future

fem/dfem/autopa.hpp

@@ -0,0 +1,406 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include <cstddef>
+
+#include "fem/kernels.hpp"
+#include "fem/kernel_dispatch.hpp"
+
+#include "util.hpp"
+
+#if defined(__has_include) && __has_include("general/nvtx.hpp") && !defined(_WIN32)
+#undef NVTX_COLOR
+#define NVTX_COLOR ::nvtx::kSandyBrown
+#include "general/nvtx.hpp"
+#else
+#define dbg(...)
+#endif
+
+namespace mfem::future
+{
+
+template<size_t num_inputs,
+         size_t num_outputs,
+         typename input_t,
+         size_t num_fields,
+         typename output_fop_t>
+class NewAutoActionCallback
+{
+   input_t &inputs;
+   const std::array<DofToQuadMap, num_inputs> &input_dtq_maps;
+   const std::array<DofToQuadMap, num_outputs> &output_dtq_maps;
+   const int num_entities;
+   const int test_vdim;
+   const int num_test_dof;
+   const int dimension;
+   const FieldDescriptor direction;
+   const int test_op_dim;
+   const int trial_vdim;
+   const int total_trial_op_dim;
+   const int num_qp;
+   Vector &dependent_inputs_trial_op_dim;
+   const int num_dependent_inputs;
+   const ElementDofOrdering element_dof_ordering;
+   const int q1d_;
+   const ThreadBlocks &thread_blocks;
+   Vector &shmem_cache;
+   SharedMemoryInfo<num_fields, num_inputs, num_outputs> &shmem_info;
+   Array<int> &elem_attributes;
+   const output_fop_t &output_fop;
+   const Array<int> &domain_attributes;
+   const DeviceTensor<1, const real_t> &ir_weights;
+   const bool use_sum_factorization;
+   const std::array<bool, num_inputs> &input_is_dependent;
+   Vector &direction_e;
+   Vector &derivative_action_e;
+   const real_t *pa_data; // pointer to the PA data
+   // refs
+   Vector &qpdc_mem; // derivative_qp_caches
+   std::function<void(Vector &, Vector &)> &output_restriction_transpose;
+   // args
+   std::vector<Vector> &f_e;
+   const Vector &dir_l;
+   Vector &der_action_l;
+
+public:
+   inline MFEM_ALWAYS_INLINE
+   NewAutoActionCallback(const bool &use_kernels_specialization,
+                         input_t &inputs,
+                         const std::array<DofToQuadMap, num_inputs> &input_dtq_maps,
+                         const std::array<DofToQuadMap, num_outputs> &output_dtq_maps,
+                         const int dimension,
+                         const int num_entities,
+                         const int test_vdim,
+                         const int num_test_dof,
+                         const FieldDescriptor &direction,
+                         const int test_op_dim,
+                         const int trial_vdim,
+                         const int total_trial_op_dim,
+                         const int num_qp,
+                         Vector &dependent_inputs_trial_op_dim,
+                         const int num_dependent_inputs,
+                         const ElementDofOrdering element_dof_ordering,
+                         const int q1d,
+                         const ThreadBlocks &thread_blocks,
+                         Vector &shmem_cache,
+                         SharedMemoryInfo<num_fields, num_inputs, num_outputs> &shmem_info,
+                         Array<int> &elem_attributes,
+                         const output_fop_t &output_fop,
+                         const Array<int> &domain_attributes,
+                         const DeviceTensor<1, const real_t> &ir_weights,
+                         const bool use_sum_factorization,
+                         const std::array<bool, num_inputs> &input_is_dependent,
+                         Vector &direction_e,
+                         Vector &derivative_action_e,
+                         const real_t *pa_data,
+                         // refs
+                         Vector &qpdc_mem,
+                         std::function<void(Vector &, Vector &)> &output_restriction_transpose,
+                         // args
+                         std::vector<Vector> &f_e,
+                         const Vector &dir_l,
+                         Vector &der_action_l):
+      inputs(inputs),
+      input_dtq_maps(input_dtq_maps),
+      output_dtq_maps(output_dtq_maps),
+      num_entities(num_entities),
+      test_vdim(test_vdim),
+      num_test_dof(num_test_dof),
+      dimension(dimension),
+      direction(direction),
+      test_op_dim(test_op_dim),
+      trial_vdim(trial_vdim),
+      total_trial_op_dim(total_trial_op_dim),
+      num_qp(num_qp),
+      dependent_inputs_trial_op_dim(dependent_inputs_trial_op_dim),
+      num_dependent_inputs(num_dependent_inputs),
+      element_dof_ordering(element_dof_ordering),
+      q1d_(q1d),
+      thread_blocks(thread_blocks),
+      shmem_cache(shmem_cache),
+      shmem_info(shmem_info),
+      elem_attributes(elem_attributes),
+      output_fop(output_fop),
+      domain_attributes(domain_attributes),
+      ir_weights(ir_weights),
+      use_sum_factorization(use_sum_factorization),
+      input_is_dependent(input_is_dependent),
+      direction_e(direction_e),
+      derivative_action_e(derivative_action_e),
+      pa_data(pa_data),
+      qpdc_mem(qpdc_mem),
+      output_restriction_transpose(output_restriction_transpose),
+      f_e(f_e),
+      dir_l(dir_l),
+      der_action_l(der_action_l)
+   {
+      if (!use_kernels_specialization) { return; }
+      NewAutoActionCallbackKernels::template Specialization<3,4>::Add();
+      // NewAutoActionCallbackKernels::template Specialization<4,5>::Add();
+      NewAutoActionCallbackKernels::template Specialization<5,6>::Add();
+      // NewAutoActionCallbackKernels::template Specialization<6,7>::Add();
+      NewAutoActionCallbackKernels::template Specialization<7,8>::Add();
+      // NewAutoActionCallbackKernels::template Specialization<8,9>::Add();
+      // NewAutoActionCallbackKernels::template Specialization<9,10>::Add();
+   }
+
+   template<int T_D1D = 0, int T_Q1D = 0>
+   static inline MFEM_ALWAYS_INLINE
+   void auto_pa_action_callback(const input_t &inputs,
+                                const std::array<DofToQuadMap, num_inputs> input_dtq_maps,
+                                const int ne,
+                                const int test_vdim,
+                                const int num_test_dof,
+                                const int dimension,
+                                const FieldDescriptor &direction,
+                                const int test_op_dim,
+                                const int trial_vdim,
+                                const int total_trial_op_dim,
+                                const int num_qp,
+                                const int num_dependent_inputs,
+                                const ElementDofOrdering ordering,
+                                const ThreadBlocks thread_blocks,
+                                Array<int> elem_attributes,
+                                const output_fop_t output_fop,
+                                const Array<int> domain_attributes,
+                                const bool use_sum_factorization,
+                                Vector &direction_e,
+                                Vector &derivative_action_e,
+                                const real_t *pa_data,
+                                // refs
+                                Vector &qpdc_mem,
+                                std::function<void(Vector &, Vector &)> &or_transpose,
+                                // args
+                                // std::vector<Vector> &f_e, // unused
+                                const Vector &direction_l,
+                                Vector &der_action_l,
+                                // fallback arguments
+                                const int d1d, const int q1d)
+   {
+      db1("d1d:{} q1d:{}", d1d, q1d);
+      db1("T_D1D:{} T_Q1D:{}", T_D1D, T_Q1D);
+      db1("num_qp: {}, ne: {}", num_qp, ne);
+      db1("test_op_dim: {}, test_vdim: {}", test_op_dim, test_vdim);
+      db1("total_trial_op_dim: {}, trial_vdim: {}", total_trial_op_dim, trial_vdim);
+      db1("num_inputs:{} num_fields:{} num_outputs:{}", num_inputs, num_fields,
+          num_outputs);
+      assert(dimension == 3);
+
+      constexpr int DIM = 3;
+      constexpr int MQ1 = T_Q1D > 0 ? T_Q1D : 32;
+      constexpr int MD1 = T_D1D > 0 ? T_D1D : 32;
+
+      assert(ordering == ElementDofOrdering::LEXICOGRAPHIC);
+      // db1("direction_l: size:{} dot:{}", direction_l.Size(),
+      //     direction_l*direction_l);
+      restriction<Entity::Element>(direction, direction_l, direction_e, ordering);
+      // db1("direction_e: size:{} dot:{}", direction_e.Size(),
+      //     direction_e*direction_e);
+      const auto dir_e = Reshape(direction_e.Read(), d1d,d1d,d1d, 1, ne);
+
+      assert(pa_data);
+      // const auto qpdc = Reshape(pa_data ? nullptr :qpdc_mem.Read(),
+      //                           test_vdim, test_op_dim,
+      //                           trial_vdim, total_trial_op_dim,
+      //                           num_qp, ne);
+      const auto DX = Reshape(pa_data, 3, 3, q1d, q1d, q1d, ne);
+
+      // db1("qpdc: size:{} dot:{}", qpdc_mem.Size(), qpdc_mem*qpdc_mem);
+
+      // const auto d_elem_attr = elem_attributes.Read();
+      // const bool has_attr = domain_attributes.Size() > 0;
+      // const auto d_domain_attr = domain_attributes.Read();
+
+      derivative_action_e = 0.0;
+      assert(test_vdim == 1);
+      assert(num_test_dof == d1d*d1d*d1d);
+      auto ye = Reshape(derivative_action_e.ReadWrite(), num_test_dof, test_vdim, ne);
+
+      forall([=] MFEM_HOST_DEVICE (int e, real_t *)
+      {
+         const int D1D = T_D1D ? T_D1D : d1d;
+         const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+         // if (has_attr && !d_domain_attr[d_elem_attr[e] - 1]) { return; }
+
+         MFEM_SHARED real_t smem[MQ1][MQ1];
+         MFEM_SHARED real_t sB[MD1][MQ1];
+         MFEM_SHARED real_t sG[MD1][MQ1];
+         kernels::internal::d_regs3d_t<DIM, MQ1> r0, r1;
+
+         const auto dir_fop = get<0>(inputs);
+         // const int vdim = dir_fop.vdim;
+         const int vd = 0;
+
+         // Interpolate
+         const auto input = get<0>(inputs);
+         using field_operator_t = std::decay_t<decltype(input)>;
+         if constexpr (is_gradient_fop<field_operator_t>::value) // Grad
+         {
+            constexpr int VDIM = 1; // 🔥
+            // const auto dtq = input_dtq_maps[0];
+            const auto dtq = get<0>(input_dtq_maps);
+            const auto B = dtq.B, G = dtq.G;
+            // db1("B:{} {} {} {} {} {}", B[0], B[1], B[2], B[3], B[4], B[5]);
+            // db1("G:{} {} {} {} {} {}", G[0], G[1], G[2], G[3], G[4], G[5]);
+            kernels::internal::LoadMatrix(D1D, Q1D, B, sB);
+            kernels::internal::LoadMatrix(D1D, Q1D, G, sG);
+            for (int c = 0; c < VDIM; c++)
+            {
+               kernels::internal::LoadDofs3d(e, D1D, c, dir_e, r0);
+               kernels::internal::Grad3d(D1D, Q1D, smem, sB, sG, r0, r1, c);
+            }
+         }
+         else if constexpr (is_identity_fop<field_operator_t>::value) // Identity
+         {
+            // db1("Id");
+            assert(false); // not here
+         }
+         else { assert(false); }
+
+
+         // db1("Qfunction");
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+               {
+                  // const int q = qx + q1d * (qy + q1d * qz);
+
+                  const real_t u = r1[0][qz][qy][qx];
+                  const real_t v = r1[1][qz][qy][qx];
+                  const real_t w = r1[2][qz][qy][qx];
+
+                  for (int k = 0; k < test_op_dim; k++)
+                  {
+                     // const auto trial_op_dim = dpitod(0, 1);
+
+                     // size_t m_offset = 0;
+                     for (int j = 0; j < trial_vdim; j++)
+                     {
+                        // const real_t val = qpdc(vd, k, j, 0 + m_offset, q, e) * u
+                        //                    + qpdc(vd, k, j, 1 + m_offset, q, e) * v
+                        //                    + qpdc(vd, k, j, 2 + m_offset, q, e) * w;
+                        const real_t val = DX(k, 0, qx, qy, qz, e) * u +
+                                           DX(k, 1, qx, qy, qz, e) * v +
+                                           DX(k, 2, qx, qy, qz, e) * w;
+                        r0[k][qz][qy][qx] = val;
+                     }
+                     // m_offset += 3;//trial_op_dim;
+                  }
+               }
+            }
+         }
+
+         // db1("Integrate");
+         auto y = Reshape(&ye(0, 0, e), num_test_dof, test_vdim);
+         if constexpr (is_gradient_fop<std::decay_t<output_fop_t>>::value) // GradientT
+         {
+            // db1("GradTranspose3d");
+            // const int vdim = output_fop.vdim;
+            constexpr int VDIM = 1; // 🔥
+            auto yd = Reshape(&y(0, 0), D1D, D1D, D1D, VDIM);
+            // const auto B = reinterpret_cast<const real_t (*)[MQ1]>(Bo);
+            // const auto G = reinterpret_cast<const real_t (*)[MQ1]>(Go);
+            for (int c = 0; c < VDIM; c++)
+            {
+               kernels::internal::GradTranspose3d(D1D, Q1D, smem, sB, sG, r0, r1, c);
+               kernels::internal::WriteDofs3d(D1D, c, r1, yd);
+            }
+         }
+         else
+         {
+            assert(false);
+         }
+
+      },
+      ne,
+      thread_blocks,
+      0,
+      nullptr);
+
+      // dbg("derivative_action_e: size:{} dot:{}",
+      //     derivative_action_e.Size(), derivative_action_e * derivative_action_e);
+      or_transpose(derivative_action_e, der_action_l);
+   }
+
+   using NewAutoActionCallbackType =
+      decltype(&NewAutoActionCallback::auto_pa_action_callback<>);
+   MFEM_REGISTER_KERNELS(NewAutoActionCallbackKernels,
+                         NewAutoActionCallbackType, (int, int));
+
+
+   inline MFEM_ALWAYS_INLINE MFEM_HOST_DEVICE
+   void Apply(const int d1d, const int q1d)
+   {
+      NewAutoActionCallbackKernels::Run(d1d, q1d,
+                                        // args
+                                        inputs,
+                                        input_dtq_maps,
+                                        num_entities,
+                                        test_vdim,
+                                        num_test_dof,
+                                        dimension,
+                                        direction,
+                                        test_op_dim,
+                                        trial_vdim,
+                                        total_trial_op_dim,
+                                        num_qp,
+                                        num_dependent_inputs,
+                                        element_dof_ordering,
+                                        thread_blocks,
+                                        elem_attributes,
+                                        output_fop,
+                                        domain_attributes,
+                                        use_sum_factorization,
+                                        direction_e,
+                                        derivative_action_e,
+                                        pa_data,
+                                        // refs
+                                        qpdc_mem,
+                                        output_restriction_transpose,
+                                        // args
+                                        dir_l,
+                                        der_action_l,
+                                        // fallback arguments
+                                        d1d, q1d);
+   }
+};
+
+template<size_t num_inputs,
+         size_t num_outputs,
+         typename input_t,
+         size_t num_fields,
+         typename output_fop_t>
+template<int D1D, int Q1D>
+inline MFEM_ALWAYS_INLINE
+typename NewAutoActionCallback<num_inputs, num_outputs, input_t, num_fields, output_fop_t>::NewAutoActionCallbackType
+NewAutoActionCallback<num_inputs, num_outputs, input_t, num_fields, output_fop_t>::NewAutoActionCallbackKernels::Kernel()
+{
+   return auto_pa_action_callback<D1D, Q1D>;
+}
+
+template<size_t num_inputs,
+         size_t num_outputs,
+         typename input_t,
+         size_t num_fields,
+         typename output_fop_t>
+inline MFEM_ALWAYS_INLINE
+typename NewAutoActionCallback<num_inputs, num_outputs, input_t, num_fields, output_fop_t>::NewAutoActionCallbackType
+NewAutoActionCallback<num_inputs, num_outputs, input_t, num_fields, output_fop_t>::NewAutoActionCallbackKernels::Fallback
+(int d1d, int q1d)
+{
+   return auto_pa_action_callback<>;
+}
+
+} // namespace mfem::future

fem/dfem/integrate.hpp

@@ -102,7 +102,7 @@ void map_quadrature_data_to_fields_tensor_impl_2d(
    const DeviceTensor<3, real_t> &f,
    const output_t &output,
    const DofToQuadMap &dtq,
-   std::array<DeviceTensor<1>, 6> &scratch_mem)
+   const std::array<DeviceTensor<1>, 6> &scratch_mem)
 {
    [[maybe_unused]] auto B = dtq.B;
    [[maybe_unused]] auto G = dtq.G;
@@ -239,7 +239,7 @@ void map_quadrature_data_to_fields_tensor_impl_3d(
    const DeviceTensor<3, real_t> &f,
    const output_t &output,
    const DofToQuadMap &dtq,
-   std::array<DeviceTensor<1>, 6> &scratch_mem)
+   const std::array<DeviceTensor<1>, 6> &scratch_mem)
 {
    [[maybe_unused]] auto B = dtq.B;
    [[maybe_unused]] auto G = dtq.G;
@@ -258,11 +258,11 @@ void map_quadrature_data_to_fields_tensor_impl_3d(
 
       for (int vd = 0; vd < vdim; vd++)
       {
-         MFEM_FOREACH_THREAD(qy, y, q1d)
+         MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
          {
-            MFEM_FOREACH_THREAD(dx, x, d1d)
+            MFEM_FOREACH_THREAD_DIRECT(dx, x, d1d)
             {
-               MFEM_FOREACH_THREAD(qz, z, q1d)
+               MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
                {
                   real_t acc = 0.0;
                   for (int qx = 0; qx < q1d; qx++)
@@ -275,11 +275,11 @@ void map_quadrature_data_to_fields_tensor_impl_3d(
          }
          MFEM_SYNC_THREAD;
 
-         MFEM_FOREACH_THREAD(dy, y, d1d)
+         MFEM_FOREACH_THREAD_DIRECT(dy, y, d1d)
          {
-            MFEM_FOREACH_THREAD(dx, x, d1d)
+            MFEM_FOREACH_THREAD_DIRECT(dx, x, d1d)
             {
-               MFEM_FOREACH_THREAD(qz, z, q1d)
+               MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
                {
                   real_t acc = 0.0;
                   for (int qy = 0; qy < q1d; qy++)
@@ -293,11 +293,11 @@ void map_quadrature_data_to_fields_tensor_impl_3d(
          MFEM_SYNC_THREAD;
 
 
-         MFEM_FOREACH_THREAD(dy, y, d1d)
+         MFEM_FOREACH_THREAD_DIRECT(dy, y, d1d)
          {
-            MFEM_FOREACH_THREAD(dx, x, d1d)
+            MFEM_FOREACH_THREAD_DIRECT(dx, x, d1d)
             {
-               MFEM_FOREACH_THREAD(dz, z, d1d)
+               MFEM_FOREACH_THREAD_DIRECT(dz, z, d1d)
                {
                   real_t acc = 0.0;
                   for (int qz = 0; qz < q1d; qz++)
@@ -328,62 +328,69 @@ void map_quadrature_data_to_fields_tensor_impl_3d(
 
       for (int vd = 0; vd < vdim; vd++)
       {
-         MFEM_FOREACH_THREAD(qz, z, q1d)
+         MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
          {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
             {
-               MFEM_FOREACH_THREAD(dx, x, d1d)
+               MFEM_FOREACH_THREAD_DIRECT(dx, x, d1d)
                {
-                  real_t uvw[3] = {0.0, 0.0, 0.0};
+                  real_t u = 0.0, v = 0.0, w = 0.0;
                   for (int qx = 0; qx < q1d; qx++)
                   {
-                     uvw[0] += fqp(vd, 0, qx, qy, qz) * G(qx, 0, dx);
-                     uvw[1] += fqp(vd, 1, qx, qy, qz) * B(qx, 0, dx);
-                     uvw[2] += fqp(vd, 2, qx, qy, qz) * B(qx, 0, dx);
+                     const real_t b = B(qx, 0, dx);
+                     const real_t g = G(qx, 0, dx);
+
+                     u += fqp(vd, 0, qx, qy, qz) * g;
+                     v += fqp(vd, 1, qx, qy, qz) * b;
+                     w += fqp(vd, 2, qx, qy, qz) * b;
                   }
-                  s0(qz, qy, dx) = uvw[0];
-                  s1(qz, qy, dx) = uvw[1];
-                  s2(qz, qy, dx) = uvw[2];
+                  s0(qz, qy, dx) = u;
+                  s1(qz, qy, dx) = v;
+                  s2(qz, qy, dx) = w;
                }
             }
          }
          MFEM_SYNC_THREAD;
 
-         MFEM_FOREACH_THREAD(qz, z, q1d)
+         MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
          {
-            MFEM_FOREACH_THREAD(dy, y, d1d)
+            MFEM_FOREACH_THREAD_DIRECT(dy, y, d1d)
             {
-               MFEM_FOREACH_THREAD(dx, x, d1d)
+               MFEM_FOREACH_THREAD_DIRECT(dx, x, d1d)
                {
-                  real_t uvw[3] = {0.0, 0.0, 0.0};
+                  real_t u = 0.0, v = 0.0, w = 0.0;
                   for (int qy = 0; qy < q1d; qy++)
                   {
-                     uvw[0] += s0(qz, qy, dx) * B(qy, 0, dy);
-                     uvw[1] += s1(qz, qy, dx) * G(qy, 0, dy);
-                     uvw[2] += s2(qz, qy, dx) * B(qy, 0, dy);
+                     const real_t b = B(qy, 0, dy);
+                     const real_t g = G(qy, 0, dy);
+                     u += s0(qz, qy, dx) * b;
+                     v += s1(qz, qy, dx) * g;
+                     w += s2(qz, qy, dx) * b;
                   }
-                  s3(qz, dy, dx) = uvw[0];
-                  s4(qz, dy, dx) = uvw[1];
-                  s5(qz, dy, dx) = uvw[2];
+                  s3(qz, dy, dx) = u;
+                  s4(qz, dy, dx) = v;
+                  s5(qz, dy, dx) = w;
                }
             }
          }
          MFEM_SYNC_THREAD;
 
-         MFEM_FOREACH_THREAD(dz, z, d1d)
+         MFEM_FOREACH_THREAD_DIRECT(dz, z, d1d)
          {
-            MFEM_FOREACH_THREAD(dy, y, d1d)
+            MFEM_FOREACH_THREAD_DIRECT(dy, y, d1d)
             {
-               MFEM_FOREACH_THREAD(dx, x, d1d)
+               MFEM_FOREACH_THREAD_DIRECT(dx, x, d1d)
                {
-                  real_t uvw[3] = {0.0, 0.0, 0.0};
+                  real_t u = 0.0, v = 0.0, w = 0.0;
                   for (int qz = 0; qz < q1d; qz++)
                   {
-                     uvw[0] += s3(qz, dy, dx) * B(qz, 0, dz);
-                     uvw[1] += s4(qz, dy, dx) * B(qz, 0, dz);
-                     uvw[2] += s5(qz, dy, dx) * G(qz, 0, dz);
+                     const real_t b = B(qz, 0, dz);
+                     const real_t g = G(qz, 0, dz);
+                     u += s3(qz, dy, dx) * b;
+                     v += s4(qz, dy, dx) * b;
+                     w += s5(qz, dy, dx) * g;
                   }
-                  yd(dx, dy, dz, vd) += uvw[0] + uvw[1] + uvw[2];
+                  yd(dx, dy, dz, vd) += u + v + w;
                }
             }
          }
@@ -398,11 +405,11 @@ void map_quadrature_data_to_fields_tensor_impl_3d(
 
       for (int sq = 0; sq < output.size_on_qp; sq++)
       {
-         MFEM_FOREACH_THREAD(qx, x, q1d)
+         MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
          {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
             {
-               MFEM_FOREACH_THREAD(qz, z, q1d)
+               MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
                {
                   yqp(sq, qx, qy, qz) = fqp(sq, qx, qy, qz);
                }
@@ -425,7 +432,7 @@ void map_quadrature_data_to_fields(
    const DeviceTensor<3, real_t> &f,
    const output_t &output,
    const DofToQuadMap &dtq,
-   std::array<DeviceTensor<1>, 6> &scratch_mem,
+   const std::array<DeviceTensor<1>, 6> &scratch_mem,
    const int &dimension,
    const bool &use_sum_factorization)
 {

fem/dfem/interpolate.hpp

@@ -11,6 +11,23 @@
 #pragma once
 
 #include "util.hpp"
+#include "fem/kernels.hpp"
+
+///////////////////////////////////////////////////////////////////////////////
+template <class T>
+inline std::enable_if_t<!std::numeric_limits<T>::is_integer, bool>
+AlmostEq(T x, T y, T tolerance = 15.0 * std::numeric_limits<T>::epsilon())
+{
+   const T neg = std::abs(x - y);
+   constexpr T min = std::numeric_limits<T>::min();
+   constexpr T eps = std::numeric_limits<T>::epsilon();
+   const T min_abs = std::min(std::abs(x), std::abs(y));
+   if (std::abs(min_abs) == 0.0) { return neg < eps; }
+   return (neg / (1.0 + std::max(min, min_abs))) < tolerance;
+}
+#undef NVTX_COLOR
+#define NVTX_COLOR nvtx::kPeru
+#include "general/nvtx.hpp"
 
 namespace mfem::future
 {
@@ -30,6 +47,7 @@ void map_field_to_quadrature_data_tensor_product_3d(
 
    if constexpr (is_value_fop<std::decay_t<field_operator_t>>::value)
    {
+      dbg("Value");
       auto [q1d, unused, d1d] = B.GetShape();
       const int vdim = input.vdim;
       const auto field = Reshape(&field_e[0], d1d, d1d, d1d, vdim);
@@ -39,11 +57,11 @@ void map_field_to_quadrature_data_tensor_product_3d(
 
       for (int vd = 0; vd < vdim; vd++)
       {
-         MFEM_FOREACH_THREAD(dz, z, d1d)
+         MFEM_FOREACH_THREAD_DIRECT(dz, z, d1d)
          {
-            MFEM_FOREACH_THREAD(dy, y, d1d)
+            MFEM_FOREACH_THREAD_DIRECT(dy, y, d1d)
             {
-               MFEM_FOREACH_THREAD(qx, x, q1d)
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
                {
                   real_t acc = 0.0;
                   for (int dx = 0; dx < d1d; dx++)
@@ -56,11 +74,11 @@ void map_field_to_quadrature_data_tensor_product_3d(
          }
          MFEM_SYNC_THREAD;
 
-         MFEM_FOREACH_THREAD(dz, z, d1d)
+         MFEM_FOREACH_THREAD_DIRECT(dz, z, d1d)
          {
-            MFEM_FOREACH_THREAD(qx, x, q1d)
+            MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
             {
-               MFEM_FOREACH_THREAD(qy, y, q1d)
+               MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
                {
                   real_t acc = 0.0;
                   for (int dy = 0; dy < d1d; dy++)
@@ -73,11 +91,11 @@ void map_field_to_quadrature_data_tensor_product_3d(
          }
          MFEM_SYNC_THREAD;
 
-         MFEM_FOREACH_THREAD(qz, z, q1d)
+         MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
          {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
             {
-               MFEM_FOREACH_THREAD(qx, x, q1d)
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
                {
                   real_t acc = 0.0;
                   for (int dz = 0; dz < d1d; dz++)
@@ -94,10 +112,11 @@ void map_field_to_quadrature_data_tensor_product_3d(
    else if constexpr (
       is_gradient_fop<std::decay_t<field_operator_t>>::value)
    {
-      const auto [q1d, unused, d1d] = B.GetShape();
+      // dbg("Gradient");
+      const auto [q1d, B_dim, d1d] = B.GetShape();
       const int vdim = input.vdim;
       const int dim = input.dim;
-      const auto field = Reshape(&field_e[0], d1d, d1d, d1d, vdim);
+      const auto field = Reshape(&std::as_const(field_e[0]), d1d, d1d, d1d, vdim);
       auto fqp = Reshape(&field_qp[0], vdim, dim, q1d, q1d, q1d);
 
       auto s0 = Reshape(&scratch_mem[0](0), d1d, d1d, q1d);
@@ -106,18 +125,41 @@ void map_field_to_quadrature_data_tensor_product_3d(
       auto s3 = Reshape(&scratch_mem[3](0), d1d, q1d, q1d);
       auto s4 = Reshape(&scratch_mem[4](0), d1d, q1d, q1d);
 
-      for (int vd = 0; vd < vdim; vd++)
+      // constexpr int MQ1 = T_Q1D > 0 ? T_Q1D : 8;
+      // static constexpr int DIM = 3;
+      // MFEM_VERIFY(q1d <= MQ1, "q1d > MQ1");
+      // MFEM_SHARED real_t smem[MQ1][MQ1];
+
+      // kernels::internal::d_regs3d_t<DIM, MQ1> r0, r1;
+      // real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
+
+      /*
       {
-         MFEM_FOREACH_THREAD(dz, z, d1d)
+         assert(B_dim == 1 && "1D B required!");
+         kernels::internal::LoadMatrix(d1d, q1d, B, sB);
+         kernels::internal::LoadMatrix(d1d, q1d, G, sG);
+         for (int qx = 0; qx < q1d; qx++)
          {
-            MFEM_FOREACH_THREAD(dy, y, d1d)
+            for (int dx = 0; dx < d1d; dx++)
             {
-               MFEM_FOREACH_THREAD(qx, x, q1d)
+               assert(AlmostEq(B(qx, 0, dx), sB[dx][qx]));
+               assert(AlmostEq(G(qx, 0, dx), sG[dx][qx]));
+            }
+         }
+      }*/
+
+      for (int c = 0; c < vdim; c++)
+      {
+         MFEM_FOREACH_THREAD_DIRECT(dz, z, d1d)
+         {
+            MFEM_FOREACH_THREAD_DIRECT(dy, y, d1d)
+            {
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
                {
                   real_t uv[2] = {0.0, 0.0};
                   for (int dx = 0; dx < d1d; dx++)
                   {
-                     const real_t f = field(dx, dy, dz, vd);
+                     const real_t f = field(dx, dy, dz, c);
                      uv[0] += f * B(qx, 0, dx);
                      uv[1] += f * G(qx, 0, dx);
                   }
@@ -128,11 +170,11 @@ void map_field_to_quadrature_data_tensor_product_3d(
          }
          MFEM_SYNC_THREAD;
 
-         MFEM_FOREACH_THREAD(dz, z, d1d)
+         MFEM_FOREACH_THREAD_DIRECT(dz, z, d1d)
          {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
             {
-               MFEM_FOREACH_THREAD(qx, x, q1d)
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
                {
                   real_t uvw[3] = {0.0, 0.0, 0.0};
                   for (int dy = 0; dy < d1d; dy++)
@@ -150,11 +192,11 @@ void map_field_to_quadrature_data_tensor_product_3d(
          }
          MFEM_SYNC_THREAD;
 
-         MFEM_FOREACH_THREAD(qz, z, q1d)
+         MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
          {
-            MFEM_FOREACH_THREAD(qy, y, q1d)
+            MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
             {
-               MFEM_FOREACH_THREAD(qx, x, q1d)
+               MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
                {
                   real_t uvw[3] = {0.0, 0.0, 0.0};
                   for (int dz = 0; dz < d1d; dz++)
@@ -163,19 +205,59 @@ void map_field_to_quadrature_data_tensor_product_3d(
                      uvw[1] += s3(dz, qy, qx) * B(qz, 0, dz);
                      uvw[2] += s4(dz, qy, qx) * G(qz, 0, dz);
                   }
-                  fqp(vd, 0, qx, qy, qz) = uvw[0];
-                  fqp(vd, 1, qx, qy, qz) = uvw[1];
-                  fqp(vd, 2, qx, qy, qz) = uvw[2];
+                  fqp(c, 0, qx, qy, qz) = uvw[0];
+                  fqp(c, 1, qx, qy, qz) = uvw[1];
+                  fqp(c, 2, qx, qy, qz) = uvw[2];
                }
             }
          }
          MFEM_SYNC_THREAD;
       }
+
+      /*
+      {
+         for (int c = 0; c < vdim; c++)
+         {
+            kernels::internal::LoadDofs3d(d1d, c, field, r0);
+            for (int d = 0; d < DIM; d++)
+            {
+               for (int dz = 0; dz < d1d; dz++)
+               {
+                  for (int dy = 0; dy < d1d; dy++)
+                  {
+                     for (int dx = 0; dx < d1d; dx++)
+                     {
+                        const real_t f = field(dx, dy, dz, c);
+                        assert(AlmostEq(f, r0[d][dz][dy][dx]));
+                     }
+                  }
+               }
+            }
+
+            kernels::internal::Grad3d(d1d, q1d, smem, sB, sG, r0, r1, c);
+            for (int qz = 0; qz < q1d; qz++)
+            {
+               for (int qy = 0; qy < q1d; qy++)
+               {
+                  for (int qx = 0; qx < q1d; qx++)
+                  {
+                     if (!AlmostEq(fqp(c, d, qx, qy, qz), r1[d][qz][qy][qx]))
+                     {
+                        dbg("\x1b[31m[{}:d] {} {}", c, fqp(c, d, qx, qy, qz), r1[d][qz][qy][qx]);
+                        dbg("❌❌❌"), std::exit(EXIT_FAILURE);
+                     }
+                  }
+               }
+            }
+         }
+         // dbg("✅✅✅✅✅✅✅✅✅✅✅✅✅✅✅");//, std::exit(EXIT_SUCCESS);
+      }*/
    }
    // TODO: Create separate function for clarity
    else if constexpr (
       std::is_same_v<std::decay_t<field_operator_t>, Weight>)
    {
+      // dbg("None");
       const int num_qp = integration_weights.GetShape()[0];
       // TODO: eeek
       const int q1d = (int)floor(std::pow(num_qp, 1.0/input.dim) + 0.5);
@@ -432,6 +514,9 @@ void map_fields_to_quadrature_data(
    const int &dimension,
    const bool &use_sum_factorization = false)
 {
+   // dbg();
+   assert(use_sum_factorization && "❌ use_sum_factorization required");
+
    // When the input_to_field map returns -1, this means the requested input
    // is the integration weight. Weights don't have a user defined field
    // attached to them and we create a dummy field which is not accessed
@@ -485,18 +570,19 @@ void map_field_to_quadrature_data_conditional(
    const int &dimension,
    const bool &use_sum_factorization = false)
 {
+   assert(false && "❌ condition not implemented");
    if (condition)
    {
       if (use_sum_factorization)
       {
          if (dimension == 2)
          {
-            map_field_to_quadrature_data_tensor_product_3d(
+            map_field_to_quadrature_data_tensor_product_2d(
                field_qp, dtqmap, field_e, fop, integration_weights, scratch_mem);
          }
          else if (dimension == 3)
          {
-            map_field_to_quadrature_data_tensor_product_2d(
+            map_field_to_quadrature_data_tensor_product_3d(
                field_qp, dtqmap, field_e, fop, integration_weights, scratch_mem);
          }
       }
@@ -520,6 +606,7 @@ void map_fields_to_quadrature_data_conditional(
    const std::array<bool, num_inputs> &conditions,
    const bool &use_sum_factorization = false)
 {
+   assert(false && "❌ condition not implemented");
    for_constexpr<num_inputs>([&](auto i)
    {
       map_field_to_quadrature_data_conditional(

fem/dfem/qfunction_transform.hpp

@@ -235,6 +235,8 @@ void process_qf_arg(
    }
 }
 
+// const tensor<real_t, DIM>       ∇u
+// const tensor<real_t, DIM, DIM>  D (PA_DATA)
 template <typename arg_type>
 MFEM_HOST_DEVICE inline
 void process_qf_arg(const DeviceTensor<2> &u, arg_type &arg, int qp)

fem/dfem/qp.hpp

@@ -0,0 +1,76 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+#pragma once
+
+#include "tuple.hpp"
+#include "../linalg/tensor.hpp"
+
+using namespace mfem::future;
+using mfem::future::tensor;
+
+// Helper to add dimension to tensor type
+template<typename T, int qp>
+struct AddQPDimension;
+
+// Specialization for tensor<real_t, dim>
+template<typename real_t, int dim, int qp>
+struct AddQPDimension<tensor<real_t, dim>, qp>
+{
+   using type = tensor<real_t, dim, qp>;
+};
+
+// Specialization for tensor<real_t, dim, dim>
+template<typename real_t, int dim, int qp>
+struct AddQPDimension<tensor<real_t, dim, dim>, qp>
+{
+   using type = tensor<real_t, dim, dim, qp>;
+};
+
+// Specialization for real_t (transforms to tensor<real_t, qp>)
+template<typename real_t, int qp>
+struct AddQPDimension
+{
+   using type = tensor<real_t, qp>;
+};
+
+// Helper to transform tuple
+template<typename Tuple, int qp>
+struct TransformTupleQP {};
+
+// Specialization for mfem::future::tuple
+template<int qp, typename... Types>
+struct TransformTupleQP<mfem::future::tuple<Types...>, qp>
+{
+   using type = mfem::future::tuple<typename AddQPDimension<Types, qp>::type...>;
+};
+
+template<int qp, typename... Types>
+struct TransformTupleQP<std::tuple<Types...>, qp>
+{
+   using type = std::tuple<typename AddQPDimension<Types, qp>::type...>;
+};
+
+// Function to transform tuple type with qp dimension
+template<int qp, typename qf_param_ts>
+struct add_qp_dimension
+{
+   using type = typename TransformTupleQP<qf_param_ts, qp>::type;
+};
+
+// Helper alias template for cleaner usage
+template<int qp, typename qf_param_ts>
+using add_qp_dimension_t = typename add_qp_dimension<qp, qf_param_ts>::type;
+
+// ...AddDomainIntegrator...
+// {
+//    constexpr int Q1D = 4;
+//    using qf_param_augmentd_ts = add_qp_dimension_t<Q1D, decay_tuple<qf_param_ts>>;
+// }

fem/dfem/util.hpp

@@ -34,10 +34,19 @@
 #include "parameterspace.hpp"
 #include "tuple.hpp"
 
+#if defined(__has_include) && __has_include("general/nvtx.hpp") && !defined(_WIN32)
+#undef NVTX_COLOR
+#define NVTX_COLOR ::nvtx::kSeashell
+#include "general/nvtx.hpp"
+#else
+#define dbg(...)
+#endif
+
 namespace mfem::future
 {
 
 template<typename... Ts>
+inline MFEM_ALWAYS_INLINE
 constexpr auto to_array(const std::tuple<Ts...>& tuple)
 {
    constexpr auto get_array = [](const Ts&... x) { return std::array<typename std::common_type<Ts...>::type, sizeof...(Ts)> { x... }; };
@@ -48,6 +57,7 @@ namespace detail
 {
 
 template <typename lambda, std::size_t... i>
+inline MFEM_ALWAYS_INLINE
 constexpr void for_constexpr(lambda&& f,
                              std::integral_constant<std::size_t, i>... Is)
 {
@@ -56,6 +66,7 @@ constexpr void for_constexpr(lambda&& f,
 
 
 template <std::size_t... n, typename lambda, typename... arg_types>
+inline MFEM_ALWAYS_INLINE
 constexpr void for_constexpr(lambda&& f,
                              std::integer_sequence<std::size_t, n...>,
                              arg_types... args)
@@ -67,6 +78,7 @@ constexpr void for_constexpr(lambda&& f,
 }  // namespace detail
 
 template <typename lambda, std::size_t... i>
+inline MFEM_ALWAYS_INLINE
 constexpr void for_constexpr(lambda&& f,
                              std::integer_sequence<std::size_t, i ... >)
 {
@@ -74,15 +86,18 @@ constexpr void for_constexpr(lambda&& f,
 }
 
 template <typename lambda>
+inline MFEM_ALWAYS_INLINE
 constexpr void for_constexpr(lambda&& f, std::integer_sequence<std::size_t>) {}
 
 template <int... n, typename lambda>
+inline MFEM_ALWAYS_INLINE
 constexpr void for_constexpr(lambda&& f)
 {
    detail::for_constexpr(f, std::make_integer_sequence<std::size_t, n> {}...);
 }
 
 template <typename lambda, typename arg_t>
+inline MFEM_ALWAYS_INLINE
 constexpr void for_constexpr_with_arg(lambda&& f, arg_t&& arg,
                                       std::integer_sequence<std::size_t>)
 {
@@ -90,6 +105,7 @@ constexpr void for_constexpr_with_arg(lambda&& f, arg_t&& arg,
 }
 
 template <typename lambda, typename arg_t, std::size_t i, std::size_t... Is>
+inline MFEM_ALWAYS_INLINE
 constexpr void for_constexpr_with_arg(lambda&& f, arg_t&& arg,
                                       std::integer_sequence<std::size_t, i, Is...>)
 {
@@ -99,6 +115,7 @@ constexpr void for_constexpr_with_arg(lambda&& f, arg_t&& arg,
 }
 
 template <typename lambda, typename arg_t>
+inline MFEM_ALWAYS_INLINE
 constexpr void for_constexpr_with_arg(lambda&& f, arg_t&& arg)
 {
    using indices =
@@ -108,6 +125,7 @@ constexpr void for_constexpr_with_arg(lambda&& f, arg_t&& arg)
 }
 
 template <typename... input_ts, std::size_t... Is>
+inline MFEM_ALWAYS_INLINE
 auto make_dependency_map_impl(
    tuple<input_ts...> inputs,
    std::index_sequence<Is...>)
@@ -136,6 +154,7 @@ auto make_dependency_map_impl(
 // @returns an unordered_map where the keys are the field IDs and the values
 // are arrays of booleans indicating which inputs depend on each field ID.
 template <typename... input_ts>
+inline MFEM_ALWAYS_INLINE
 auto make_dependency_map(tuple<input_ts...> inputs)
 {
    return make_dependency_map_impl(inputs, std::index_sequence_for<input_ts...> {});
@@ -150,6 +169,7 @@ auto make_dependency_map(tuple<input_ts...> inputs)
 // ```
 // prints "int".
 template <typename T>
+inline MFEM_ALWAYS_INLINE
 constexpr auto get_type_name() -> std::string_view
 {
 #if defined(__clang__)
@@ -253,6 +273,7 @@ void pretty_print(const mfem::Vector& v)
 ///
 /// @param v vector of vectors to print
 template <typename T>
+inline
 void pretty_print(const mfem::Array<T>& v)
 {
    out << "[";
@@ -275,6 +296,7 @@ void pretty_print(const mfem::Array<T>& v)
 /// @tparam T type of array elements
 /// @tparam N size of array
 template<typename K, typename T, std::size_t N>
+inline
 void pretty_print(const std::unordered_map<K,std::array<T,N>>& map)
 {
    out << "{";
@@ -327,8 +349,8 @@ void print_mpi_sync(const std::string& msg)
    // First gather string lengths
    size_t msg_len = msg.length();
    std::vector<size_t> lengths(nranks);
-   MPI_Gather(&msg_len, 1, MPITypeMap<size_t>::mpi_type,
-              lengths.data(), 1, MPITypeMap<size_t>::mpi_type,
+   MPI_Gather(&msg_len, 1, MPI_INT,
+              lengths.data(), 1, MPI_INT,
               0, MPI_COMM_WORLD);
 
    if (myrank == 0)
@@ -384,6 +406,7 @@ void pretty_print_mpi(const mfem::Vector& v)
 
 
 template <typename ... Ts>
+inline MFEM_ALWAYS_INLINE
 constexpr auto decay_types(tuple<Ts...> const &)
 -> tuple<std::remove_cv_t<std::remove_reference_t<Ts>>...>;
 
@@ -416,12 +439,14 @@ struct create_function_signature<output_t (*)(input_ts...)>
 };
 
 template <typename T>
+inline MFEM_ALWAYS_INLINE
 constexpr int GetFieldId()
 {
    return T::GetFieldId();
 }
 
 template <typename Tuple, std::size_t... Is>
+inline MFEM_ALWAYS_INLINE
 constexpr auto extract_field_ids_impl(Tuple&& t, std::index_sequence<Is...>)
 {
    return std::array<int, sizeof...(Is)>
@@ -435,6 +460,7 @@ constexpr auto extract_field_ids_impl(Tuple&& t, std::index_sequence<Is...>)
 /// @param t the tuple to extract field IDs from.
 /// @returns an array of field IDs.
 template <typename... Ts>
+inline MFEM_ALWAYS_INLINE
 constexpr auto extract_field_ids(const std::tuple<Ts...>& t)
 {
    return extract_field_ids_impl(t, std::index_sequence_for<Ts...> {});
@@ -446,6 +472,7 @@ constexpr auto extract_field_ids(const std::tuple<Ts...>& t)
 /// @param size the size of the array.
 /// @param value the value to search for.
 /// @returns true if the value is found, false otherwise.
+inline MFEM_ALWAYS_INLINE
 constexpr bool contains(const int* arr, std::size_t size, int value)
 {
    for (std::size_t i = 0; i < size; ++i)
@@ -463,6 +490,7 @@ constexpr bool contains(const int* arr, std::size_t size, int value)
 /// @param t the tuple to count unique field IDs from.
 /// @returns the number of unique field IDs.
 template <typename... Ts>
+inline MFEM_ALWAYS_INLINE
 constexpr std::size_t count_unique_field_ids(const std::tuple<Ts...>& t)
 {
    auto ids = extract_field_ids(t);
@@ -489,6 +517,7 @@ constexpr std::size_t count_unique_field_ids(const std::tuple<Ts...>& t)
 /// @param marker the marker std::array indicating which entries to get.
 /// @returns a std::vector containing the marked entries.
 template <typename T, std::size_t N>
+inline MFEM_ALWAYS_INLINE
 auto get_marked_entries(
    const std::array<T, N> &a,
    const std::array<bool, N> &marker)
@@ -509,6 +538,7 @@ auto get_marked_entries(
 /// @param t the tuple to filter fields from.
 /// @returns a tuple containing only the fields with field IDs not equal to -1.
 template <typename... Ts>
+inline MFEM_ALWAYS_INLINE
 constexpr auto filter_fields(const std::tuple<Ts...>& t)
 {
    return std::tuple_cat(
@@ -532,11 +562,13 @@ struct FieldDescriptor
    data_variant_t data;
 
    /// Default constructor
+   inline MFEM_ALWAYS_INLINE
    FieldDescriptor() :
       id(SIZE_MAX), data(data_variant_t{}) {}
 
    /// Constructor
    template <typename T>
+   inline MFEM_ALWAYS_INLINE
    FieldDescriptor(std::size_t field_id, const T* v) :
       id(field_id), data(v) {}
 };
@@ -568,11 +600,26 @@ struct ThreadBlocks
    int z = 1;
 };
 
-#if defined(MFEM_USE_CUDA_OR_HIP)
+#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP)
+/*template <typename tag_t>
+struct ForallKernel
+{
+   template <typename func_t>
+   __global__ static void run(func_t f, int n)
+   {
+      int i = blockIdx.x;
+      extern __shared__ real_t shmem[];
+      if (i < n)
+      {
+         f(i, shmem);
+      }
+   }
+};*/
+
 template <typename func_t>
 __global__ void forall_kernel_shmem(func_t f, int n)
 {
-   int i = blockIdx.x;
+   int i = MFEM_BLOCK_ID(x);
    extern __shared__ real_t shmem[];
    if (i < n)
    {
@@ -581,27 +628,38 @@ __global__ void forall_kernel_shmem(func_t f, int n)
 }
 #endif
 
-template <typename func_t>
+template </*typename kernel_tag,*/ typename func_t>
+inline MFEM_ALWAYS_INLINE
 void forall(func_t f,
             const int &N,
             const ThreadBlocks &blocks,
             int num_shmem = 0,
             real_t *shmem = nullptr)
 {
+   // dbg("num_shmem: {}", num_shmem);
+   // dbg("blocks: {}x{}x{}", blocks.x, blocks.y, blocks.z);
+
    if (Device::Allows(Backend::CUDA_MASK) ||
        Device::Allows(Backend::HIP_MASK))
    {
-#if defined(MFEM_USE_CUDA_OR_HIP)
-      // int gridsize = (N + Z - 1) / Z;
+#if (defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
       int num_bytes = num_shmem * sizeof(decltype(shmem));
       dim3 block_size(blocks.x, blocks.y, blocks.z);
-      forall_kernel_shmem<<<N, block_size, num_bytes>>>(f, N);
+      // ForallKernel<kernel_tag>::run<<<N, block_size, num_bytes>>>(f, N);
+      if (num_bytes > 0)
+      {
+         forall_kernel_shmem<<<N, block_size, num_bytes>>>(f, N);
+      }
+      else
+      {
+         forall_kernel_shmem<<<N, block_size>>>(f, N);
+      }
 #if defined(MFEM_USE_CUDA)
       MFEM_GPU_CHECK(cudaGetLastError());
 #elif defined(MFEM_USE_HIP)
       MFEM_GPU_CHECK(hipGetLastError());
 #endif
-      MFEM_DEVICE_SYNC;
+      // MFEM_DEVICE_SYNC;
 #endif
    }
    else if (Device::Allows(Backend::CPU_MASK))
@@ -709,7 +767,7 @@ private:
 /// @param fields the vector of field descriptors.
 /// @returns the index of the field descriptor with the given ID,
 /// or SIZE_MAX if not found.
-inline
+inline MFEM_ALWAYS_INLINE
 std::size_t FindIdx(const std::size_t& id,
                     const std::vector<FieldDescriptor>& fields)
 {
@@ -727,7 +785,7 @@ std::size_t FindIdx(const std::size_t& id,
 ///
 /// @param f the field descriptor.
 /// @returns the vdof size of the field descriptor.
-inline
+inline MFEM_ALWAYS_INLINE
 int GetVSize(const FieldDescriptor &f)
 {
    return std::visit([](auto arg)
@@ -762,7 +820,7 @@ int GetVSize(const FieldDescriptor &f)
 /// @param f the field descriptor.
 /// @param el the element index.
 /// @param vdofs the array to store the element vdofs.
-inline
+inline MFEM_ALWAYS_INLINE
 void GetElementVDofs(const FieldDescriptor &f, int el, Array<int> &vdofs)
 {
    return std::visit([&](auto arg)
@@ -796,7 +854,7 @@ void GetElementVDofs(const FieldDescriptor &f, int el, Array<int> &vdofs)
 ///
 /// @param f the field descriptor.
 /// @returns the true dof size of the field descriptor.
-inline
+inline MFEM_ALWAYS_INLINE
 int GetTrueVSize(const FieldDescriptor &f)
 {
    return std::visit([](auto arg)
@@ -831,7 +889,7 @@ int GetTrueVSize(const FieldDescriptor &f)
 ///
 /// @param f the field descriptor.
 /// @returns the vdim of the field descriptor.
-inline
+inline MFEM_ALWAYS_INLINE
 int GetVDim(const FieldDescriptor &f)
 {
    return std::visit([](auto && arg)
@@ -863,6 +921,7 @@ int GetVDim(const FieldDescriptor &f)
 /// @tparam entity_t the entity type (see Entity).
 /// @returns the spatial dimension of the field descriptor.
 template <typename entity_t>
+inline MFEM_ALWAYS_INLINE
 int GetDimension(const FieldDescriptor &f)
 {
    return std::visit([](auto && arg)
@@ -897,7 +956,7 @@ int GetDimension(const FieldDescriptor &f)
 ///
 /// @param f the field descriptor.
 /// @returns the prolongation operator for the field descriptor.
-inline
+inline MFEM_ALWAYS_INLINE
 const Operator *get_prolongation(const FieldDescriptor &f)
 {
    return std::visit([](auto&& arg) -> const Operator*
@@ -926,7 +985,7 @@ const Operator *get_prolongation(const FieldDescriptor &f)
 /// @param o the element dof ordering.
 /// @returns the element restriction operator for the field descriptor in
 /// specified ordering.
-inline
+inline MFEM_ALWAYS_INLINE
 const Operator *get_element_restriction(const FieldDescriptor &f,
                                         ElementDofOrdering o)
 {
@@ -957,7 +1016,7 @@ const Operator *get_element_restriction(const FieldDescriptor &f,
 /// @returns the restriction operator for the field descriptor in
 /// specified ordering.
 template <typename entity_t>
-inline
+inline MFEM_ALWAYS_INLINE
 const Operator *get_restriction(const FieldDescriptor &f,
                                 const ElementDofOrdering &o)
 {
@@ -977,7 +1036,8 @@ const Operator *get_restriction(const FieldDescriptor &f,
 /// @returns a tuple containting a std::function with the transpose
 /// restriction callback and it's height.
 template <typename entity_t, typename fop_t>
-inline std::tuple<std::function<void(const Vector&, Vector&)>, int>
+inline MFEM_ALWAYS_INLINE
+std::tuple<std::function<void(const Vector&, Vector&)>, int>
 get_restriction_transpose(
    const FieldDescriptor &f,
    const ElementDofOrdering &o,
@@ -985,18 +1045,19 @@ get_restriction_transpose(
 {
    if constexpr (is_sum_fop<fop_t>::value)
    {
-      auto RT = [=](const Vector &v_e, Vector &v_l)
+      const auto RT = [=](const Vector &v_e, Vector &v_l)
       {
-         v_l += v_e;
+         v_l = v_e;
       };
       return std::make_tuple(RT, 1);
    }
    else
    {
       const Operator *R = get_restriction<entity_t>(f, o);
-      std::function<void(const Vector&, Vector&)> RT = [=](const Vector &x, Vector &y)
+      const std::function<void(const Vector&, Vector&)> RT =
+         [=](const Vector &x, Vector &y)
       {
-         R->AddMultTranspose(x, y);
+         R->MultTranspose(x, y);
       };
       return std::make_tuple(RT, R->Height());
    }
@@ -1012,7 +1073,7 @@ get_restriction_transpose(
 /// @param field the field descriptor.
 /// @param x the input vector in tdofs.
 /// @param field_l the output vector in vdofs.
-inline
+inline MFEM_ALWAYS_INLINE
 void prolongation(const FieldDescriptor field, const Vector &x, Vector &field_l)
 {
    const auto P = get_prolongation(field);
@@ -1032,6 +1093,7 @@ void prolongation(const FieldDescriptor field, const Vector &x, Vector &field_l)
 /// @tparam N the number of fields.
 /// @tparam M the number of output fields.
 template <std::size_t N, std::size_t M>
+inline MFEM_ALWAYS_INLINE
 void prolongation(const std::array<FieldDescriptor, N> fields,
                   const Vector &x,
                   std::array<Vector, M> &fields_l)
@@ -1059,7 +1121,7 @@ void prolongation(const std::array<FieldDescriptor, N> fields,
 /// @param fields the array of field descriptors.
 /// @param x the input vector in tdofs.
 /// @param fields_l the array of output vectors in vdofs.
-inline
+inline MFEM_ALWAYS_INLINE
 void prolongation(const std::vector<FieldDescriptor> fields,
                   const Vector &x,
                   std::vector<Vector> &fields_l)
@@ -1076,24 +1138,6 @@ void prolongation(const std::vector<FieldDescriptor> fields,
    }
 }
 
-inline
-void get_lvectors(const std::vector<FieldDescriptor> fields,
-                  const Vector &x,
-                  std::vector<Vector> &fields_l)
-{
-   int data_offset = 0;
-   for (std::size_t i = 0; i < fields.size(); i++)
-   {
-      const int sz = GetVSize(fields[i]);
-      fields_l[i].SetSize(sz);
-
-      const Vector x_i(const_cast<Vector&>(x), data_offset, sz);
-      fields_l[i] = x_i;
-
-      data_offset += sz;
-   }
-}
-
 /// @brief Get a transpose prolongation callback for a field descriptor.
 ///
 /// In the special case of a one field operator, the transpose prolongation
@@ -1104,7 +1148,7 @@ void get_lvectors(const std::vector<FieldDescriptor> fields,
 /// @param mpi_comm the MPI communicator.
 /// @tparam fop_t the field operator type.
 template <typename fop_t>
-inline
+inline MFEM_ALWAYS_INLINE
 std::function<void(const Vector&, Vector&)> get_prolongation_transpose(
    const FieldDescriptor &f,
    const fop_t &fop,
@@ -1144,6 +1188,7 @@ std::function<void(const Vector&, Vector&)> get_prolongation_transpose(
 /// @param ordering the element dof ordering.
 /// @tparam entity_t the entity type (see Entity).
 template <typename entity_t>
+inline MFEM_ALWAYS_INLINE
 void restriction(const FieldDescriptor u,
                  const Vector &u_l,
                  Vector &field_e,
@@ -1166,6 +1211,7 @@ void restriction(const FieldDescriptor u,
 /// @param offset the array index offset to start writing in fields_e.
 /// @tparam entity_t the entity type (see Entity).
 template <typename entity_t>
+inline MFEM_ALWAYS_INLINE
 void restriction(const std::vector<FieldDescriptor> u,
                  const std::vector<Vector> &u_l,
                  std::vector<Vector> &fields_e,
@@ -1185,6 +1231,7 @@ void restriction(const std::vector<FieldDescriptor> u,
 
 // TODO: keep this temporarily
 template <std::size_t N, std::size_t M>
+inline MFEM_ALWAYS_INLINE
 void element_restriction(const std::array<FieldDescriptor, N> u,
                          const std::array<Vector, N> &u_l,
                          std::array<Vector, M> &fields_e,
@@ -1208,6 +1255,7 @@ void element_restriction(const std::array<FieldDescriptor, N> u,
 /// @tparam entity_t the entity type (see Entity).
 /// @returns the number of entities of the given type.
 template <typename entity_t>
+inline MFEM_ALWAYS_INLINE
 int GetNumEntities(const mfem::Mesh &mesh)
 {
    if constexpr (std::is_same_v<entity_t, Entity::Element>)
@@ -1235,7 +1283,7 @@ int GetNumEntities(const mfem::Mesh &mesh)
 /// @param mode the mode of the DofToQuad object.
 /// @tparam entity_t the entity type (see Entity).
 template <typename entity_t>
-inline
+inline MFEM_ALWAYS_INLINE
 const DofToQuad *GetDofToQuad(const FieldDescriptor &f,
                               const IntegrationRule &ir,
                               DofToQuad::Mode mode)
@@ -1710,7 +1758,7 @@ void print_shared_memory_info(shmem_info_t &shmem_info)
 }
 
 template <std::size_t N>
-MFEM_HOST_DEVICE inline
+MFEM_HOST_DEVICE inline MFEM_ALWAYS_INLINE
 std::array<DofToQuadMap, N> load_dtq_mem(
    void *mem,
    int offset,
@@ -1773,7 +1821,7 @@ std::array<DofToQuadMap, N> load_dtq_mem(
 }
 
 template <std::size_t num_fields>
-MFEM_HOST_DEVICE inline
+MFEM_HOST_DEVICE inline MFEM_ALWAYS_INLINE
 std::array<DeviceTensor<1>, num_fields>
 load_field_mem(
    void *mem,
@@ -1807,12 +1855,12 @@ load_field_mem(
    return f;
 }
 
-MFEM_HOST_DEVICE inline
+MFEM_HOST_DEVICE inline MFEM_ALWAYS_INLINE
 DeviceTensor<1> load_direction_mem(
    void *mem,
    int offset,
    const int &size,
-   const DeviceTensor<2> &direction,
+   const DeviceTensor<2, const real_t> &direction,
    const int &entity_idx)
 {
    int block_size = MFEM_THREAD_SIZE(x) *
@@ -1832,7 +1880,7 @@ DeviceTensor<1> load_direction_mem(
 }
 
 template <std::size_t N>
-MFEM_HOST_DEVICE inline
+MFEM_HOST_DEVICE inline MFEM_ALWAYS_INLINE
 std::array<DeviceTensor<2>, N> load_input_mem(
    void *mem,
    int offset,
@@ -1850,7 +1898,7 @@ std::array<DeviceTensor<2>, N> load_input_mem(
    return f;
 }
 
-MFEM_HOST_DEVICE inline
+MFEM_HOST_DEVICE inline MFEM_ALWAYS_INLINE
 DeviceTensor<2> load_residual_mem(
    void *mem,
    int offset,
@@ -1862,7 +1910,7 @@ DeviceTensor<2> load_residual_mem(
 }
 
 template <std::size_t N>
-MFEM_HOST_DEVICE inline
+MFEM_HOST_DEVICE inline MFEM_ALWAYS_INLINE
 std::array<DeviceTensor<1>, 6> load_scratch_mem(
    void *mem,
    int offset,
@@ -1878,7 +1926,7 @@ std::array<DeviceTensor<1>, 6> load_scratch_mem(
 }
 
 template <typename shared_mem_info_t, std::size_t num_inputs, std::size_t num_outputs, std::size_t num_fields>
-MFEM_HOST_DEVICE inline
+MFEM_HOST_DEVICE inline MFEM_ALWAYS_INLINE
 auto unpack_shmem(
    void *shmem,
    const shared_mem_info_t &shmem_info,
@@ -1941,14 +1989,14 @@ auto unpack_shmem(
 }
 
 template <typename shared_mem_info_t, std::size_t num_inputs, std::size_t num_outputs, std::size_t num_fields>
-MFEM_HOST_DEVICE inline
+MFEM_HOST_DEVICE inline MFEM_ALWAYS_INLINE
 auto unpack_shmem(
    void *shmem,
    const shared_mem_info_t &shmem_info,
    const std::array<DofToQuadMap, num_inputs> &input_dtq_maps,
    const std::array<DofToQuadMap, num_outputs> &output_dtq_maps,
    const std::array<DeviceTensor<2>, num_fields> &wrapped_fields_e,
-   const DeviceTensor<2> &wrapped_direction_e,
+   const DeviceTensor<2, const real_t> &wrapped_direction_e,
    const int &num_qp,
    const int &e)
 {
@@ -2021,7 +2069,7 @@ auto unpack_shmem(
 }
 
 template <std::size_t... i>
-MFEM_HOST_DEVICE inline
+MFEM_HOST_DEVICE inline MFEM_ALWAYS_INLINE
 std::array<DeviceTensor<2>, sizeof...(i)> get_local_input_qp(
    const std::array<DeviceTensor<3>, sizeof...(i)> &input_qp_global, int e,
    std::index_sequence<i...>)
@@ -2036,7 +2084,7 @@ std::array<DeviceTensor<2>, sizeof...(i)> get_local_input_qp(
 }
 
 template <std::size_t N>
-MFEM_HOST_DEVICE inline
+MFEM_HOST_DEVICE inline MFEM_ALWAYS_INLINE
 void set_zero(std::array<DeviceTensor<2>, N> &v)
 {
    for (std::size_t i = 0; i < N; i++)
@@ -2051,7 +2099,7 @@ void set_zero(std::array<DeviceTensor<2>, N> &v)
 }
 
 template <std::size_t n>
-MFEM_HOST_DEVICE inline
+MFEM_HOST_DEVICE inline MFEM_ALWAYS_INLINE
 void set_zero(DeviceTensor<n> &u)
 {
    int s = 1;
@@ -2072,7 +2120,7 @@ void set_zero(DeviceTensor<n> &u)
 /// @param v destination DeviceTensor
 /// @tparam n DeviceTensor rank
 template <int n>
-MFEM_HOST_DEVICE inline
+MFEM_HOST_DEVICE inline MFEM_ALWAYS_INLINE
 void copy(DeviceTensor<n> &u, DeviceTensor<n> &v)
 {
    int s = 1;
@@ -2095,7 +2143,7 @@ void copy(DeviceTensor<n> &u, DeviceTensor<n> &v)
 /// @tparam n DeviceTensor rank
 /// @tparam m number of DeviceTensors
 template <int n, std::size_t m>
-MFEM_HOST_DEVICE inline
+MFEM_HOST_DEVICE inline MFEM_ALWAYS_INLINE
 void copy(std::array<DeviceTensor<n>, m> &u,
           std::array<DeviceTensor<n>, m> &v)
 {
@@ -2113,6 +2161,7 @@ void copy(std::array<DeviceTensor<n>, m> &u,
 /// @tparam num_fields number of fields
 /// @return array of field data wrapped in DeviceTensors
 template <std::size_t num_fields>
+inline MFEM_ALWAYS_INLINE
 std::array<DeviceTensor<2>, num_fields> wrap_fields(
    std::vector<Vector> &fields,
    std::array<int, num_fields> &field_sizes,
@@ -2149,6 +2198,7 @@ std::array<DeviceTensor<2>, num_fields> wrap_fields(
 /// size required on quadrature points using GetSizeOnQP() and adds it to the
 /// total. Non-dependent inputs contribute zero to the total size.
 template <typename input_t, std::size_t num_fields, std::size_t... i>
+inline MFEM_ALWAYS_INLINE
 int accumulate_sizes_on_qp(
    const input_t &inputs,
    std::array<bool, sizeof...(i)> &kinput_is_dependent,
@@ -2175,6 +2225,7 @@ template <
    typename field_operator_ts,
    std::size_t N = tuple_size<field_operator_ts>::value,
    std::size_t... Is>
+inline MFEM_ALWAYS_INLINE
 std::array<DofToQuadMap, N> create_dtq_maps_impl(
    field_operator_ts &fops,
    std::vector<const DofToQuad*> &dtqs,
@@ -2260,6 +2311,7 @@ template <
    typename entity_t,
    typename field_operator_ts,
    std::size_t num_fields>
+inline MFEM_ALWAYS_INLINE
 std::array<DofToQuadMap, num_fields> create_dtq_maps(
    field_operator_ts &fops,
    std::vector<const DofToQuad*> &dtqmaps,

fem/fe/face_map_utils.hpp

@@ -259,30 +259,6 @@ inline void FaceIdxToVolIdx3D(const int index, const int size1d,
    i = yz_plane ? level : _i;
 }
 
-MFEM_HOST_DEVICE
-inline int FaceIdxToVolIdx(int dim, int i, int size1d, int face0, int face1,
-                           int side, int orientation)
-{
-   if (dim == 2)
-   {
-      int ix, iy;
-      internal::FaceIdxToVolIdx2D(i, size1d, face0, face1, side, ix, iy);
-      return ix + iy*size1d;
-   }
-   else if (dim == 3)
-   {
-      int ix, iy, iz;
-      internal::FaceIdxToVolIdx3D(i, size1d, face0, face1, side, orientation,
-                                  ix, iy, iz);
-      return ix + size1d*iy + size1d*size1d*iz;
-   }
-   else
-   {
-      MFEM_ABORT_KERNEL("Invalid dimension");
-      return -1;
-   }
-};
-
 } // namespace internal
 
 } // namespace mfem

fem/fe_coll.cpp

@@ -401,6 +401,9 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name)
    {
       MFEM_ABORT("unknown FiniteElementCollection: " << name);
    }
+   MFEM_VERIFY(!strcmp(fec->Name(), name), "input name: \"" << name
+               << "\" does not match the created collection name: \""
+               << fec->Name() << '"');
 
    return fec;
 }
@@ -2456,7 +2459,8 @@ RT_FECollection::RT_FECollection(const int order, const int dim,
       const char *cb_name = BasisType::Name(cb_type); // this may abort
       MFEM_ABORT("unknown closed BasisType: " << cb_name);
    }
-   if (Quadrature1D::CheckOpen(op_type) == Quadrature1D::Invalid)
+   if (Quadrature1D::CheckOpen(op_type) == Quadrature1D::Invalid &&
+       ob_type != BasisType::IntegratedGLL)
    {
       const char *ob_name = BasisType::Name(ob_type); // this may abort
       MFEM_ABORT("unknown open BasisType: " << ob_name);
@@ -2514,7 +2518,6 @@ RT_FECollection::RT_FECollection(const int p, const int dim,
                                  const int map_type, const bool signs,
                                  const int ob_type)
    : FiniteElementCollection(p + 1)
-   , dim(dim)
    , ob_type(ob_type)
 {
    if (Quadrature1D::CheckOpen(BasisType::GetQuadrature1D(ob_type)) ==
@@ -2783,7 +2786,8 @@ ND_FECollection::ND_FECollection(const int p, const int dim,
    int cp_type = BasisType::GetQuadrature1D(cb_type);
 
    // Error checking
-   if (Quadrature1D::CheckOpen(op_type) == Quadrature1D::Invalid)
+   if (Quadrature1D::CheckOpen(op_type) == Quadrature1D::Invalid &&
+       ob_type != BasisType::IntegratedGLL)
    {
       const char *ob_name = BasisType::Name(ob_type);
       MFEM_ABORT("Invalid open basis point type: " << ob_name);

fem/fe_coll.hpp

@@ -464,13 +464,6 @@ public:
    RT_Trace_FECollection(const int p, const int dim,
                          const int map_type = FiniteElement::INTEGRAL,
                          const int ob_type = BasisType::GaussLegendre);
-
-   FiniteElementCollection *Clone(int p) const override
-   {
-      const int map_type = (strncmp(rt_name, "RT_Trace", 8) == 0)?
-                           (FiniteElement::INTEGRAL):(FiniteElement::VALUE);
-      return new RT_Trace_FECollection(p, dim, map_type, ob_type);
-   }
 };
 
 /** Arbitrary order discontinuous finite elements defined on the interface
@@ -482,13 +475,6 @@ public:
    DG_Interface_FECollection(const int p, const int dim,
                              const int map_type = FiniteElement::VALUE,
                              const int ob_type = BasisType::GaussLegendre);
-
-   FiniteElementCollection *Clone(int p) const override
-   {
-      const int map_type = (strncmp(rt_name, "DG_Iface", 8) == 0)?
-                           (FiniteElement::VALUE):(FiniteElement::INTEGRAL);
-      return new DG_Interface_FECollection(p, dim, map_type, ob_type);
-   }
 };
 
 /// Arbitrary order H(curl)-conforming Nedelec finite elements.

fem/fem.hpp

@@ -51,7 +51,7 @@
 #include "hyperbolic.hpp"
 #include "bounds.hpp"
 
-#include "dfem/doperator.hpp"
+// #include "dfem/doperator.hpp"
 
 #ifdef MFEM_USE_MPI
 #include "pfespace.hpp"

fem/fes_kernels.hpp

@@ -1,249 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_FES_KERNELS_HPP
-#define MFEM_FES_KERNELS_HPP
-
-#include "../general/forall.hpp"
-
-#include <climits>
-
-namespace mfem
-{
-/// \cond DO_NOT_DOCUMENT
-namespace internal
-{
-
-///
-/// Implements matrix-vector multiply $y = A x$ for a sparse matrix composed of
-/// a sum of smaller dense blocks. There is additional permutation/sign
-/// information associated with each block. The base class only implements
-/// helper routines such as computing block widths, index into x, index into y,
-/// and column in A given sub-block information.
-/// @sa DerefineMatrixOpMultFunctor
-///
-/// @tparam Order vdim ordering for x and y. Note that for Diag = false this is
-/// ignored for x as x has a special interleaved order.
-/// @tparam Base used for the curious recurring template pattern (CRTP) so the
-/// base class can access child class fields without virtual functions
-/// @tparam Diag true if this corresponds to the diagonal block (coarse element
-/// and fine element are on our rank), false otherwise (coarse element is on our
-/// rank, fine element is on a different rank).
-///
-template <Ordering::Type Order, class Base, bool Diag = true>
-struct DerefineMatrixOpFunctorBase;
-
-template <class Base>
-struct DerefineMatrixOpFunctorBase<Ordering::byNODES, Base, true>
-{
-   /// block column indices offsets
-   const int *bcptr;
-   /// column indices
-   const int *cptr;
-
-   int MFEM_HOST_DEVICE BlockWidth(int k) const
-   {
-      return bcptr[k + 1] - bcptr[k];
-   }
-
-   void MFEM_HOST_DEVICE Col(int j, int k, int &col, int &sign) const
-   {
-      col = cptr[bcptr[k] + j];
-      if (col < 0)
-      {
-         col = -1 - col;
-         sign = -sign;
-      }
-   }
-
-   int MFEM_HOST_DEVICE IndexX(int col, int vdim, int) const
-   {
-      return col + vdim * static_cast<const Base *>(this)->width;
-   }
-   int MFEM_HOST_DEVICE IndexY(int row, int vdim) const
-   {
-      return row + vdim * static_cast<const Base *>(this)->height;
-   }
-};
-
-template <class Base>
-struct DerefineMatrixOpFunctorBase<Ordering::byVDIM, Base, true>
-{
-   /// block column indices offsets
-   const int *bcptr;
-   /// column indices
-   const int *cptr;
-
-   int MFEM_HOST_DEVICE BlockWidth(int k) const
-   {
-      return bcptr[k + 1] - bcptr[k];
-   }
-
-   void MFEM_HOST_DEVICE Col(int j, int k, int &col, int &sign) const
-   {
-      col = cptr[bcptr[k] + j];
-      if (col < 0)
-      {
-         col = -1 - col;
-         sign = -sign;
-      }
-   }
-
-   int MFEM_HOST_DEVICE IndexX(int col, int vdim, int) const
-   {
-      return vdim + col * static_cast<const Base *>(this)->vdims;
-   }
-   int MFEM_HOST_DEVICE IndexY(int row, int vdim) const
-   {
-      return vdim + row * static_cast<const Base *>(this)->vdims;
-   }
-};
-
-template <class Base>
-struct DerefineMatrixOpFunctorBase<Ordering::byNODES, Base, false>
-{
-   /// receive segment offsets
-   const int *segptr;
-   /// receive segment index
-   const int *rsptr;
-   /// off-diagonal block column offsets
-   const int *coptr;
-   /// off-diagonal block widths
-   const int *bwptr;
-
-   int MFEM_HOST_DEVICE BlockWidth(int k) const { return bwptr[k]; }
-
-   void MFEM_HOST_DEVICE Col(int j, int k, int &col, int &sign) const
-   {
-      col = coptr[k] + j;
-   }
-
-   int MFEM_HOST_DEVICE IndexX(int col, int vdim, int k) const
-   {
-      int tmp = rsptr[k];
-      int segwidth = segptr[tmp + 1] - segptr[tmp];
-      return segptr[tmp] * static_cast<const Base *>(this)->vdims + col +
-             vdim * segwidth;
-   }
-   int MFEM_HOST_DEVICE IndexY(int row, int vdim) const
-   {
-      return row + vdim * static_cast<const Base *>(this)->height;
-   }
-};
-
-template <class Base>
-struct DerefineMatrixOpFunctorBase<Ordering::byVDIM, Base, false>
-{
-   /// receive segment offsets
-   const int *segptr;
-   /// receive segment index
-   const int *rsptr;
-   /// off-diagonal block column offsets
-   const int *coptr;
-   /// off-diagonal block widths
-   const int *bwptr;
-
-   int MFEM_HOST_DEVICE BlockWidth(int k) const { return bwptr[k]; }
-
-   void MFEM_HOST_DEVICE Col(int j, int k, int &col, int &sign) const
-   {
-      col = coptr[k] + j;
-   }
-
-   int MFEM_HOST_DEVICE IndexX(int col, int vdim, int k) const
-   {
-      int tmp = rsptr[k];
-      int segwidth = segptr[tmp + 1] - segptr[tmp];
-      return segptr[tmp] * static_cast<const Base *>(this)->vdims + col +
-             vdim * segwidth;
-   }
-   int MFEM_HOST_DEVICE IndexY(int row, int vdim) const
-   {
-      return vdim + row * static_cast<const Base *>(this)->vdims;
-   }
-};
-
-/// internally used to implement the derefinement operator Mult diagonal
-/// block
-template <Ordering::Type Order, bool Atomic, bool Diag = true>
-struct DerefineMatrixOpMultFunctor
-   : public DerefineMatrixOpFunctorBase<
-     Order, DerefineMatrixOpMultFunctor<Order, Atomic, Diag>, Diag>
-{
-   const real_t *xptr;
-   real_t *yptr;
-   /// block storage
-   const real_t *bsptr;
-   /// block offsets
-   const int *boptr;
-   /// block row index offsets
-   const int *brptr;
-   /// row indices
-   const int *rptr;
-
-   // number of blocks
-   int nblocks;
-   // number of components
-   int vdims;
-   /// overall operator height (for vdim = 1)
-   int height;
-   /// overall operator width (for vdim = 1)
-   int width;
-   void MFEM_HOST_DEVICE operator()(int kidx) const
-   {
-      int k = kidx % nblocks;
-      int vdim = kidx / nblocks;
-
-      int block_height = brptr[k + 1] - brptr[k];
-      int block_width = this->BlockWidth(k);
-      MFEM_FOREACH_THREAD(i, x, block_height)
-      {
-         int row = rptr[brptr[k] + i];
-         int rsign = 1;
-         if (row < 0)
-         {
-            row = -1 - row;
-            rsign = -1;
-         }
-         if (row < INT_MAX)
-         {
-            // row not marked as unused
-            real_t sum = 0;
-            for (int j = 0; j < block_width; ++j)
-            {
-               int col, sign = rsign;
-               this->Col(j, k, col, sign);
-               sum += sign * bsptr[boptr[k] + i + j * block_height] *
-                      xptr[this->IndexX(col, vdim, k)];
-            }
-#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
-            if (Atomic)
-            {
-               atomicAdd(yptr + this->IndexY(row, vdim), sum);
-            }
-            else
-#endif
-            {
-               yptr[this->IndexY(row, vdim)] += sum;
-            }
-         }
-      }
-   }
-
-   /// N is the max block row size (doesn't have to be a power of 2)
-   void Run(int N) const { forall_2D(nblocks * vdims, N, 1, *this); }
-};
-
-} // namespace internal
-/// \endcond DO_NOT_DOCUMENT
-} // namespace mfem
-
-#endif

fem/fespace.cpp

@@ -17,9 +17,6 @@
 #include "fem.hpp"
 #include "ceed/interface/util.hpp"
 
-#include "derefmat_op.hpp"
-
-#include <algorithm>
 #include <cmath>
 #include <cstdarg>
 
@@ -27,9 +24,9 @@ using namespace std;
 
 namespace mfem
 {
-template <>
-void Ordering::DofsToVDofs<Ordering::byNODES>(int ndofs, int vdim,
-                                              Array<int> &dofs)
+
+template <> void Ordering::
+DofsToVDofs<Ordering::byNODES>(int ndofs, int vdim, Array<int> &dofs)
 {
    // static method
    int size = dofs.Size();
@@ -43,9 +40,8 @@ void Ordering::DofsToVDofs<Ordering::byNODES>(int ndofs, int vdim,
    }
 }
 
-template <>
-void Ordering::DofsToVDofs<Ordering::byVDIM>(int ndofs, int vdim,
-                                             Array<int> &dofs)
+template <> void Ordering::
+DofsToVDofs<Ordering::byVDIM>(int ndofs, int vdim, Array<int> &dofs)
 {
    // static method
    int size = dofs.Size();
@@ -59,6 +55,7 @@ void Ordering::DofsToVDofs<Ordering::byVDIM>(int ndofs, int vdim,
    }
 }
 
+
 FiniteElementSpace::FiniteElementSpace()
    : mesh(NULL), fec(NULL), vdim(0), ordering(Ordering::byNODES),
      ndofs(0), nvdofs(0), nedofs(0), nfdofs(0), nbdofs(0),
@@ -4247,11 +4244,7 @@ void FiniteElementSpace::Update(bool want_transform)
          case Mesh::DEREFINE:
          {
             BuildConformingInterpolation();
-#if 0
             Th.Reset(DerefinementMatrix(old_ndofs, old_elem_dof, old_elem_fos));
-#else
-            Th.Reset(new DerefineMatrixOp(*this, old_ndofs, old_elem_dof, old_elem_fos));
-#endif
             if (IsVariableOrder())
             {
                if (cP && cR_hp)

fem/fespace.hpp

@@ -113,7 +113,7 @@ class QuadratureSpace;
 class QuadratureInterpolator;
 class FaceQuadratureInterpolator;
 class PRefinementTransferOperator;
-struct DerefineMatrixOp;
+
 
 /** @brief Class FiniteElementSpace - responsible for providing FEM view of the
     mesh, mainly managing the set of degrees of freedom.
@@ -246,7 +246,6 @@ class FiniteElementSpace
    friend class PRefinementTransferOperator;
    friend void Mesh::Swap(Mesh &, bool);
    friend class LORBase;
-   friend struct DerefineMatrixOp;
 
 protected:
    /// The mesh that FE space lives on (not owned).
@@ -683,12 +682,8 @@ public:
    NURBSExtension *GetNURBSext() { return NURBSext; }
    NURBSExtension *StealNURBSext();
 
-   bool Conforming() const
-   {
-      return NURBSext != NULL ||
-             (mesh->Conforming() && cP == NULL);
-   }
-   bool Nonconforming() const { return !Conforming(); }
+   bool Conforming() const { return mesh->Conforming() && cP == NULL; }
+   bool Nonconforming() const { return mesh->Nonconforming() || cP != NULL; }
 
    /** Set the prolongation operator of the space to an arbitrary sparse matrix,
        creating a copy of the argument. */
@@ -926,9 +921,6 @@ public:
    { return mesh->GetBdrElementType(i); }
 
    /// Returns ElementTransformation for the @a i-th element.
-   /// @note The returned pointer references an object owned by the associated
-   /// @a Mesh that will be modified by other calls to `GetElementTransformation`.
-   /// As such, this pointer should @b not be deleted by the caller.
    ElementTransformation *GetElementTransformation(int i) const
    { return mesh->GetElementTransformation(i); }
 

fem/gridfunc.cpp

@@ -68,7 +68,7 @@ GridFunction::GridFunction(Mesh *m, std::istream &input)
       Vector::Load(input, fes->GetVSize());
 
       // if the mesh is a legacy (v1.1) NC mesh, it has old vertex ordering
-      if (fes->Nonconforming() && fes->GetMesh()->ncmesh &&
+      if (fes->Nonconforming() &&
           fes->GetMesh()->ncmesh->IsLegacyLoaded())
       {
          LegacyNCReorder();
@@ -1374,50 +1374,6 @@ void GridFunction::GetVectorGradientHat(
    MultAtB(loc_data_mat, dshape, gh);
 }
 
-void GridFunction::GetGradients(const IntegrationRule &ir, Vector &grad,
-                                QVectorLayout ql, MemoryType d_mt) const
-{
-   const FiniteElement &fe = *fes->GetTypicalFE();
-   const int dim  = fe.GetDim();
-   const int vdim = fes->GetVDim();
-   const int NE   = fes->GetNE();
-   const int ND   = fe.GetDof();
-   const int NQ   = ir.GetNPoints();
-
-   MemoryType my_d_mt = (d_mt != MemoryType::DEFAULT) ? d_mt :
-                        Device::GetDeviceMemoryType();
-
-   // ql == QVectorLayout::byNODES :   NQ x VDIM x  DIM x NE
-   // ql == QVectorLayout::byVDIM  : VDIM x  DIM x NQPT x NE
-   grad.SetSize(dim*vdim*NQ*NE, my_d_mt);
-
-   const QuadratureInterpolator &qi = *fes->GetQuadratureInterpolator(ir);
-   qi.SetOutputLayout(ql);
-
-   const bool use_tensor_products = UsesTensorBasis(*fes);
-   qi.DisableTensorProducts(!use_tensor_products);
-   const ElementDofOrdering e_ordering = use_tensor_products ?
-                                         ElementDofOrdering::LEXICOGRAPHIC :
-                                         ElementDofOrdering::NATIVE;
-   const Operator *elem_restr = fes->GetElementRestriction(e_ordering);
-
-   // Pre-compute the geometric factors in order to set the desired MemoryType
-   // they use:
-   fes->GetMesh()->GetGeometricFactors(
-      ir, GeometricFactors::JACOBIANS, my_d_mt);
-
-   if (elem_restr) // currently, always true
-   {
-      Vector f_e(vdim*ND*NE, my_d_mt);
-      elem_restr->Mult(*this, f_e);
-      qi.PhysDerivatives(f_e, grad);
-   }
-   else
-   {
-      qi.PhysDerivatives(*this, grad);
-   }
-}
-
 real_t GridFunction::GetDivergence(ElementTransformation &T) const
 {
    DofTransformation doftrans;
@@ -2668,7 +2624,7 @@ void GridFunction::ProjectBdrCoefficient(Coefficient *coeff[],
    }
    for (int i = 0; i < values_counter.Size(); i++)
    {
-      MFEM_ASSERT(bool(values_counter[i]) == bool(ess_vdofs_marker[i]),
+      MFEM_ASSERT(bool(values_counter[i]) == ess_vdofs_marker[i],
                   "internal error");
    }
 #endif

fem/gridfunc.hpp

@@ -153,8 +153,7 @@ public:
    /// Shortcut for calling SetFromTrueDofs() with GetTrueVector() as argument.
    void SetFromTrueVector() { SetFromTrueDofs(GetTrueVector()); }
 
-   /** @brief Returns the values at the vertices of element @a i for the 1-based
-       dimension vdim. */
+   /// Returns the values in the vertices of i'th element for dimension vdim.
    void GetNodalValues(int i, Array<real_t> &nval, int vdim = 1) const;
 
    /** @name Element index Get Value Methods
@@ -309,8 +308,7 @@ public:
    /// For a vector grid function, makes sure that the ordering is byNODES.
    void ReorderByNodes();
 
-   /** @brief Returns the values as a vector at mesh vertices, for the 1-based
-       dimension vdim. */
+   /// Return the values as a vector on mesh vertices for dimension vdim.
    void GetNodalValues(Vector &nval, int vdim = 1) const;
 
    void GetVectorFieldNodalValues(Vector &val, int comp) const;
@@ -361,33 +359,6 @@ public:
        variable. */
    void GetVectorGradientHat(ElementTransformation &T, DenseMatrix &gh) const;
 
-   /** @brief Evaluate the gradients of the GridFunction at the given quadrature
-       points, @a ir, in all mesh elements. */
-   /** This method assumes that all mesh elements are the same type and that the
-       IntegrationRule @a ir is consistent with that type of element.
-
-       @param[in] ir     Quadrature points at which the gradients are to be
-                         evaluated.
-       @param[out] grad  Output vector of size `SDIM*VDIM*NQ*NE` where `SDIM` is
-                         the spatial dimention of the mesh, `VDIM` is the vector
-                         dimension of the GridFunction, `NQ` is the number of
-                         quadrature points in @a ir, and `NE` is the number of
-                         elements in the mesh. The layout of @a grad is
-                         determined by the parameter @a ql: when @a ql is
-                         QVectorLayout::byNODES, the layout is
-                         `NQ x VDIM x SDIM x NE`; when @a ql is
-                         QVectorLayout::byVDIM, the layout is
-                         `VDIM x SDIM x NQ x NE`.
-       @param[in] ql     Determines the layout of the output vector @a grad; see
-                         the description of @a grad for details.
-       @param[in] d_mt   MemoryType to use for allocating the output vector
-                         @a grad, as well the GeometricFactors and temporary
-                         vector used by the method. By default, the current
-                         device memory type is used. */
-   void GetGradients(const IntegrationRule &ir, Vector &grad,
-                     QVectorLayout ql = QVectorLayout::byNODES,
-                     MemoryType d_mt = MemoryType::DEFAULT) const;
-
    /** Compute $ (\int_{\Omega} (*this) \psi_i)/(\int_{\Omega} \psi_i) $,
        where $ \psi_i $ are the basis functions for the FE space of avgs.
        Both FE spaces should be scalar and on the same mesh. */

fem/gslib.cpp

@@ -85,9 +85,9 @@ namespace mfem
 {
 
 FindPointsGSLIB::FindPointsGSLIB()
-   : mesh(nullptr),
-     fec_map_lin(nullptr),
-     fdataD(nullptr), cr(nullptr), gsl_comm(nullptr),
+   : mesh(NULL),
+     fec_map_lin(NULL),
+     fdataD(NULL), cr(NULL), gsl_comm(NULL),
      dim(-1), points_cnt(-1), setupflag(false), default_interp_value(0),
      avgtype(AvgType::ARITHMETIC), bdr_tol(1e-8)
 {
@@ -97,10 +97,10 @@ FindPointsGSLIB::FindPointsGSLIB()
    gf_rst_map.SetSize(4);
    for (int i = 0; i < mesh_split.Size(); i++)
    {
-      mesh_split[i] = nullptr;
-      ir_split[i] = nullptr;
-      fes_rst_map[i] = nullptr;
-      gf_rst_map[i] = nullptr;
+      mesh_split[i] = NULL;
+      ir_split[i] = NULL;
+      fes_rst_map[i] = NULL;
+      gf_rst_map[i] = NULL;
    }
 
    gsl_comm = new gslib::comm;
@@ -117,40 +117,27 @@ FindPointsGSLIB::FindPointsGSLIB()
    crystal_init(cr, gsl_comm);
 }
 
-FindPointsGSLIB::FindPointsGSLIB(Mesh &mesh_in, const double bb_t,
-                                 const double newt_tol, const int npt_max)
-   : FindPointsGSLIB()
-{
-   Setup(mesh_in, bb_t, newt_tol, npt_max);
-}
-
 FindPointsGSLIB::~FindPointsGSLIB()
 {
-   FreeData();
-#ifdef MFEM_USE_MPI
-   if (!Mpi::IsFinalized())  // currently segfaults inside gslib otherwise
-#endif
+   crystal_free(cr);
+   comm_free(gsl_comm);
+   delete gsl_comm;
+   delete cr;
+   for (int i = 0; i < 4; i++)
    {
-      crystal_free(cr);
-      comm_free(gsl_comm);
-      delete gsl_comm;
-      delete cr;
+      if (mesh_split[i]) { delete mesh_split[i]; mesh_split[i] = NULL; }
+      if (ir_split[i]) { delete ir_split[i]; ir_split[i] = NULL; }
+      if (fes_rst_map[i]) { delete fes_rst_map[i]; fes_rst_map[i] = NULL; }
+      if (gf_rst_map[i]) { delete gf_rst_map[i]; gf_rst_map[i] = NULL; }
    }
-   for (int i = 0; i < mesh_split.Size(); i++)
-   {
-      if (mesh_split[i]) { delete mesh_split[i]; mesh_split[i] = nullptr; }
-      if (ir_split[i]) { delete ir_split[i]; ir_split[i] = nullptr; }
-      if (fes_rst_map[i]) { delete fes_rst_map[i]; fes_rst_map[i] = nullptr; }
-      if (gf_rst_map[i]) { delete gf_rst_map[i]; gf_rst_map[i] = nullptr; }
-   }
-   if (fec_map_lin) { delete fec_map_lin; fec_map_lin = nullptr; }
+   if (fec_map_lin) { delete fec_map_lin; fec_map_lin = NULL; }
 }
 
 #ifdef MFEM_USE_MPI
 FindPointsGSLIB::FindPointsGSLIB(MPI_Comm comm_)
-   : mesh(nullptr),
-     fec_map_lin(nullptr),
-     fdataD(nullptr), cr(nullptr), gsl_comm(nullptr),
+   : mesh(NULL),
+     fec_map_lin(NULL),
+     fdataD(NULL), cr(NULL), gsl_comm(NULL),
      dim(-1), points_cnt(-1), setupflag(false), default_interp_value(0),
      avgtype(AvgType::ARITHMETIC), bdr_tol(1e-8)
 {
@@ -160,10 +147,10 @@ FindPointsGSLIB::FindPointsGSLIB(MPI_Comm comm_)
    gf_rst_map.SetSize(4);
    for (int i = 0; i < mesh_split.Size(); i++)
    {
-      mesh_split[i] = nullptr;
-      ir_split[i] = nullptr;
-      fes_rst_map[i] = nullptr;
-      gf_rst_map[i] = nullptr;
+      mesh_split[i] = NULL;
+      ir_split[i] = NULL;
+      fes_rst_map[i] = NULL;
+      gf_rst_map[i] = NULL;
    }
 
    gsl_comm = new gslib::comm;
@@ -171,21 +158,12 @@ FindPointsGSLIB::FindPointsGSLIB(MPI_Comm comm_)
    comm_init(gsl_comm, comm_);
    crystal_init(cr, gsl_comm);
 }
-
-FindPointsGSLIB::FindPointsGSLIB(ParMesh &mesh_in, const double bb_t,
-                                 const double newt_tol, const int npt_max)
-   : FindPointsGSLIB(mesh_in.GetComm())
-{
-   Setup(mesh_in, bb_t, newt_tol, npt_max);
-}
 #endif
 
 void FindPointsGSLIB::Setup(Mesh &m, const double bb_t, const double newt_tol,
                             const int npt_max)
 {
    MFEM_VERIFY(m.GetNodes() != NULL, "Mesh nodes are required.");
-   MFEM_VERIFY(m.SpaceDimension() == m.Dimension(),
-               "Mesh spatial dimension and reference element dimension must be the same");
    const int meshOrder = m.GetNodes()->FESpace()->GetMaxElementOrder();
 
    // call FreeData if FindPointsGSLIB::Setup has been called already
@@ -193,9 +171,37 @@ void FindPointsGSLIB::Setup(Mesh &m, const double bb_t, const double newt_tol,
 
    mesh = &m;
    dim  = mesh->Dimension();
-   const unsigned int dof1D = meshOrder+1;
+   unsigned dof1D = meshOrder + 1;
 
-   SetupSplitMeshesAndIntegrationRules(meshOrder);
+   SetupSplitMeshes();
+   if (dim == 2)
+   {
+      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
+      ir_split[0] = new IntegrationRule(3*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder);
+
+      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
+      ir_split[1] = new IntegrationRule(pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder);
+   }
+   else if (dim == 3)
+   {
+      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
+      ir_split[0] = new IntegrationRule(pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder);
+
+      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
+      ir_split[1] = new IntegrationRule(4*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder);
+
+      if (ir_split[2]) { delete ir_split[2]; ir_split[2] = NULL; }
+      ir_split[2] = new IntegrationRule(3*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[2], ir_split[2], meshOrder);
+
+      if (ir_split[3]) { delete ir_split[3]; ir_split[3] = NULL; }
+      ir_split[3] = new IntegrationRule(8*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[3], ir_split[3], meshOrder);
+   }
 
    GetNodalValues(mesh->GetNodes(), gsl_mesh);
 
@@ -1122,18 +1128,13 @@ void FindPointsGSLIB::Interpolate(Mesh &m, const Vector &point_pos,
 void FindPointsGSLIB::FreeData()
 {
    if (!setupflag) { return; }
-#ifdef MFEM_USE_MPI
-   if (!Mpi::IsFinalized())  // currently segfaults inside gslib otherwise
-#endif
+   if (dim == 2)
    {
-      if (dim == 2)
-      {
-         findpts_free_2((gslib::findpts_data_2 *)this->fdataD);
-      }
-      else
-      {
-         findpts_free_3((gslib::findpts_data_3 *)this->fdataD);
-      }
+      findpts_free_2((gslib::findpts_data_2 *)this->fdataD);
+   }
+   else
+   {
+      findpts_free_3((gslib::findpts_data_3 *)this->fdataD);
    }
    gsl_code.DeleteAll();
    gsl_proc.DeleteAll();
@@ -1157,8 +1158,8 @@ void FindPointsGSLIB::FreeData()
 
 void FindPointsGSLIB::SetupSplitMeshes()
 {
-   if (fec_map_lin == nullptr) { fec_map_lin = new H1_FECollection(1, dim); }
-   if (dim == 2)
+   fec_map_lin = new H1_FECollection(1, dim);
+   if (mesh->Dimension() == 2)
    {
       int Nvert = 7;
       int NEsplit = 3;
@@ -1200,7 +1201,7 @@ void FindPointsGSLIB::SetupSplitMeshes()
       mesh_split[1] = new Mesh(Mesh::MakeCartesian2D(1, 1,
                                                      Element::QUADRILATERAL));
    }
-   else if (dim == 3)
+   else if (mesh->Dimension() == 3)
    {
       mesh_split[0] = new Mesh(Mesh::MakeCartesian3D(1, 1, 1,
                                                      Element::HEXAHEDRON));
@@ -1345,6 +1346,41 @@ void FindPointsGSLIB::SetupSplitMeshes()
          }
       }
    }
+
+   NE_split_total = 0;
+   split_element_map.SetSize(0);
+   split_element_index.SetSize(0);
+   int NEsplit = 0;
+   for (int e = 0; e < mesh->GetNE(); e++)
+   {
+      const Geometry::Type gt   = mesh->GetElement(e)->GetGeometryType();
+      if (gt == Geometry::TRIANGLE || gt == Geometry::PRISM)
+      {
+         NEsplit = 3;
+      }
+      else if (gt == Geometry::TETRAHEDRON)
+      {
+         NEsplit = 4;
+      }
+      else if (gt == Geometry::PYRAMID)
+      {
+         NEsplit = 8;
+      }
+      else if (gt == Geometry::SQUARE || gt == Geometry::CUBE)
+      {
+         NEsplit = 1;
+      }
+      else
+      {
+         MFEM_ABORT("Unsupported geometry type.");
+      }
+      NE_split_total += NEsplit;
+      for (int i = 0; i < NEsplit; i++)
+      {
+         split_element_map.Append(e);
+         split_element_index.Append(i);
+      }
+   }
 }
 
 void FindPointsGSLIB::SetupIntegrationRuleForSplitMesh(Mesh *meshin,
@@ -1395,79 +1431,6 @@ void FindPointsGSLIB::SetupIntegrationRuleForSplitMesh(Mesh *meshin,
    }
 }
 
-void FindPointsGSLIB::SetupSplitMeshesAndIntegrationRules(const int order)
-{
-   MFEM_VERIFY(mesh, "Setup FindPointsGSLIB with mesh first.");
-   const int dof1D = order+1;
-   const int dim = mesh->Dimension();
-
-   SetupSplitMeshes();
-   if (dim == 2)
-   {
-      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
-      ir_split[0] = new IntegrationRule(3*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], order);
-
-      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
-      ir_split[1] = new IntegrationRule(pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], order);
-   }
-   else if (dim == 3)
-   {
-      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
-      ir_split[0] = new IntegrationRule(pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], order);
-
-      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
-      ir_split[1] = new IntegrationRule(4*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], order);
-
-      if (ir_split[2]) { delete ir_split[2]; ir_split[2] = NULL; }
-      ir_split[2] = new IntegrationRule(3*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[2], ir_split[2], order);
-
-      if (ir_split[3]) { delete ir_split[3]; ir_split[3] = NULL; }
-      ir_split[3] = new IntegrationRule(8*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[3], ir_split[3], order);
-   }
-
-   // Setup map for non tensor-product elements
-   NE_split_total = 0;
-   split_element_map.SetSize(0);
-   split_element_index.SetSize(0);
-   int NEsplit = 0;
-   for (int e = 0; e < mesh->GetNE(); e++)
-   {
-      const Geometry::Type gt   = mesh->GetElement(e)->GetGeometryType();
-      if (gt == Geometry::TRIANGLE || gt == Geometry::PRISM)
-      {
-         NEsplit = 3;
-      }
-      else if (gt == Geometry::TETRAHEDRON)
-      {
-         NEsplit = 4;
-      }
-      else if (gt == Geometry::PYRAMID)
-      {
-         NEsplit = 8;
-      }
-      else if (gt == Geometry::SQUARE || gt == Geometry::CUBE)
-      {
-         NEsplit = 1;
-      }
-      else
-      {
-         MFEM_ABORT("Unsupported geometry type.");
-      }
-      NE_split_total += NEsplit;
-      for (int i = 0; i < NEsplit; i++)
-      {
-         split_element_map.Append(e);
-         split_element_index.Append(i);
-      }
-   }
-}
-
 void FindPointsGSLIB::GetNodalValues(const GridFunction *gf_in,
                                      Vector &node_vals)
 {
@@ -2118,19 +2081,6 @@ void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
    } // parallel
 }
 
-Array<unsigned int> FindPointsGSLIB::GetPointsNotFoundIndices() const
-{
-   Array<unsigned int> nf_idxs;
-   for (int i = 0; i < gsl_code.Size(); i++)
-   {
-      if (gsl_code[i] == 2)
-      {
-         nf_idxs.Append(i);
-      }
-   }
-   return nf_idxs;
-}
-
 void FindPointsGSLIB::DistributePointInfoToOwningMPIRanks(
    Array<unsigned int> &recv_elem, Vector &recv_ref,
    Array<unsigned int> &recv_code)
@@ -2436,10 +2386,6 @@ void OversetFindPointsGSLIB::Setup(Mesh &m, const int meshid,
 {
    MFEM_VERIFY(m.GetNodes() != NULL, "Mesh nodes are required.");
    const int meshOrder = m.GetNodes()->FESpace()->GetMaxElementOrder();
-   const int gfOrder = gfmax ? gfmax->FESpace()->GetMaxElementOrder() :
-                       meshOrder;
-   MFEM_VERIFY(meshOrder == gfOrder,
-               "Mesh order must match gfmax order in OversetFindPointsGSLIB.");
 
    // FreeData if OversetFindPointsGSLIB::Setup has been called already
    if (setupflag) { FreeData(); }
@@ -2449,7 +2395,35 @@ void OversetFindPointsGSLIB::Setup(Mesh &m, const int meshid,
    const FiniteElement *fe = mesh->GetNodalFESpace()->GetTypicalFE();
    unsigned dof1D = fe->GetOrder() + 1;
 
-   SetupSplitMeshesAndIntegrationRules(meshOrder);
+   SetupSplitMeshes();
+   if (dim == 2)
+   {
+      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
+      ir_split[0] = new IntegrationRule(3*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder);
+
+      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
+      ir_split[1] = new IntegrationRule(pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder);
+   }
+   else if (dim == 3)
+   {
+      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
+      ir_split[0] = new IntegrationRule(pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder);
+
+      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
+      ir_split[1] = new IntegrationRule(4*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder);
+
+      if (ir_split[2]) { delete ir_split[2]; ir_split[2] = NULL; }
+      ir_split[2] = new IntegrationRule(3*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[2], ir_split[2], meshOrder);
+
+      if (ir_split[3]) { delete ir_split[3]; ir_split[3] = NULL; }
+      ir_split[3] = new IntegrationRule(8*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[3], ir_split[3], meshOrder);
+   }
 
    GetNodalValues(mesh->GetNodes(), gsl_mesh);
 
@@ -2506,7 +2480,7 @@ void OversetFindPointsGSLIB::FindPoints(const Vector &point_pos,
 {
    MFEM_VERIFY(setupflag, "Use OversetFindPointsGSLIB::Setup before "
                "finding points.");
-   MFEM_VERIFY(overset, "Please use OversetFindPoints for overlapping grids.");
+   MFEM_VERIFY(overset, "Please setup FindPoints for overlapping grids.");
    points_cnt = point_pos.Size() / dim;
    unsigned int match = 0; // Don't find points in the mesh if point_id=mesh_id
 

fem/gslib.hpp

@@ -13,11 +13,7 @@
 #define MFEM_GSLIB
 
 #include "../config/config.hpp"
-#ifdef MFEM_USE_MPI
-#include "pgridfunc.hpp"
-#else
 #include "gridfunc.hpp"
-#endif
 
 #ifdef MFEM_USE_GSLIB
 
@@ -135,10 +131,6 @@ protected:
                                                  IntegrationRule *irule,
                                                  int order);
 
-   /// Helper function that calls \ref SetupSplitMeshes and
-   /// \ref SetupIntegrationRuleForSplitMesh.
-   virtual void SetupSplitMeshesAndIntegrationRules(const int order);
-
    /// Get GridFunction value at the points expected by GSLIB.
    virtual void GetNodalValues(const GridFunction *gf_in, Vector &node_vals);
 
@@ -198,23 +190,14 @@ protected:
    void InterpolateOnDevice(const Vector &field_in_evec, Vector &field_out,
                             const int nel, const int ncomp,
                             const int dof1dsol, const int ordering);
-
 public:
    FindPointsGSLIB();
-   FindPointsGSLIB(Mesh &mesh_in, const double bb_t = 0.1,
-                   const double newt_tol = 1.0e-12,
-                   const int npt_max = 256);
 
 #ifdef MFEM_USE_MPI
    FindPointsGSLIB(MPI_Comm comm_);
-   FindPointsGSLIB(ParMesh &mesh_in, const double bb_t = 0.1,
-                   const double newt_tol = 1.0e-12,
-                   const int npt_max = 256);
 #endif
 
    virtual ~FindPointsGSLIB();
-   FindPointsGSLIB(const FindPointsGSLIB&) = delete;
-   FindPointsGSLIB& operator=(const FindPointsGSLIB&) = delete;
 
    /** Initializes the internal mesh in gslib, by sending the positions of the
        Gauss-Lobatto nodes of the input Mesh object \p m.
@@ -229,8 +212,8 @@ public:
        @param[in] npt_max   (Optional) Number of points for simultaneous
                             iteration. This alters performance and
                             memory footprint.*/
-
-   void Setup(Mesh &m, const double bb_t = 0.1, const double newt_tol = 1.0e-12,
+   void Setup(Mesh &m, const double bb_t = 0.1,
+              const double newt_tol = 1.0e-12,
               const int npt_max = 256);
    /** Searches positions given in physical space by \p point_pos.
        These positions can be ordered byNodes: (XXX...,YYY...,ZZZ) or
@@ -306,12 +289,7 @@ public:
 
    /** Cleans up memory allocated internally by gslib.
        Note that in parallel, this must be called before MPI_Finalize(), as it
-       calls MPI_Comm_free() for internal gslib communicators. FreeData is
-       also called by the class destructor and there are no memory leaks if the
-       destructor is called before MPI_Finalize(). If the destructor is called
-       after MPI_Finalize(), there will be an error because gslib will try to
-       invoke some MPI functions.
-   */
+       calls MPI_Comm_free() for internal gslib communicators. */
    virtual void FreeData();
 
    /// Return code for each point searched by FindPoints: inside element (0), on
@@ -334,9 +312,6 @@ public:
    /// point found by FindPoints.
    virtual const Vector &GetGSLIBReferencePosition() const { return gsl_ref; }
 
-   /// Get array of indices of not-found points.
-   Array<unsigned int> GetPointsNotFoundIndices() const;
-
    /** @name Methods to support a custom interpolation procedure.
        \brief The physical-space point that the user seeks to interpolate at
        could be located inside an element on another mpi rank.

fem/hyperbolic.cpp

@@ -181,7 +181,7 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
    // current elements' the number of degrees of freedom
    // does not consider the number of equations
    const int dof1 = el1.GetDof();
-   const int dof2 = (Tr.Elem2No >= 0)?(el2.GetDof()):(0);
+   const int dof2 = el2.GetDof();
 
 #ifdef MFEM_THREAD_SAFE
    // Local storage for element integration
@@ -219,9 +219,7 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
    const IntegrationRule *ir = IntRule;
    if (!ir)
    {
-      const int max_el_order = dof2 ? std::max(el1.GetOrder(),
-                                               el2.GetOrder()) : el1.GetOrder();
-      const int order = 2*max_el_order + IntOrderOffset;
+      const int order = 2*std::max(el1.GetOrder(), el2.GetOrder()) + IntOrderOffset;
       ir = &IntRules.Get(Tr.GetGeometryType(), order);
    }
    // loop over integration points
@@ -233,22 +231,18 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
 
       // Calculate basis functions on both elements at the face
       el1.CalcShape(Tr.GetElement1IntPoint(), shape1);
+      el2.CalcShape(Tr.GetElement2IntPoint(), shape2);
 
       // Interpolate elfun at the point
       elfun1_mat.MultTranspose(shape1, state1);
-
-      if (dof2)
-      {
-         // Calculate basis functions on both elements at the face
-         el2.CalcShape(Tr.GetElement2IntPoint(), shape2);
-         // Interpolate elfun at the point
-         elfun2_mat.MultTranspose(shape2, state2);
-      }
+      elfun2_mat.MultTranspose(shape2, state2);
 
       // Get the normal vector and the flux on the face
       if (nor.Size() == 1)  // if 1D, use 1 or -1.
       {
-         nor(0) = 2*Tr.GetElement1IntPoint().x - 1.;
+         // This assume the 1D integration point is in (0,1). This may not work
+         // if this changes.
+         nor(0) = (Tr.GetElement1IntPoint().x - 0.5) * 2.0;
       }
       else
       {
@@ -256,18 +250,14 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
       }
       // Compute F(u+, x) and F(u-, x) with maximum characteristic speed
       // Compute hat(F) using evaluated quantities
-      const real_t speed = (dof2) ? numFlux.Eval(state1, state2, nor, Tr, fluxN):
-                           fluxFunction.ComputeFluxDotN(state1, nor, Tr, fluxN);
+      const real_t speed = numFlux.Eval(state1, state2, nor, Tr, fluxN);
 
       // Update the global max char speed
       max_char_speed = std::max(speed, max_char_speed);
 
       // pre-multiply integration weight to flux
       AddMult_a_VWt(-ip.weight*sign, shape1, fluxN, elvect1_mat);
-      if (dof2)
-      {
-         AddMult_a_VWt(+ip.weight*sign, shape2, fluxN, elvect2_mat);
-      }
+      AddMult_a_VWt(+ip.weight*sign, shape2, fluxN, elvect2_mat);
    }
 }
 
@@ -278,7 +268,7 @@ void HyperbolicFormIntegrator::AssembleFaceGrad(
    // current elements' the number of degrees of freedom
    // does not consider the number of equations
    const int dof1 = el1.GetDof();
-   const int dof2 = (Tr.Elem2No >= 0)?(el2.GetDof()):(0);
+   const int dof2 = el2.GetDof();
 
 #ifdef MFEM_THREAD_SAFE
    // Local storage for element integration
@@ -312,9 +302,7 @@ void HyperbolicFormIntegrator::AssembleFaceGrad(
    const IntegrationRule *ir = IntRule;
    if (!ir)
    {
-      const int max_el_order = dof2 ? std::max(el1.GetOrder(),
-                                               el2.GetOrder()) : el1.GetOrder();
-      const int order = 2*max_el_order + IntOrderOffset;
+      const int order = 2*std::max(el1.GetOrder(), el2.GetOrder()) + IntOrderOffset;
       ir = &IntRules.Get(Tr.GetGeometryType(), order);
    }
    // loop over integration points
@@ -324,25 +312,20 @@ void HyperbolicFormIntegrator::AssembleFaceGrad(
 
       Tr.SetAllIntPoints(&ip); // set face and element int. points
 
-      // Calculate basis functions of the first element at the face
+      // Calculate basis functions on both elements at the face
       el1.CalcShape(Tr.GetElement1IntPoint(), shape1);
+      el2.CalcShape(Tr.GetElement2IntPoint(), shape2);
 
       // Interpolate elfun at the point
       elfun1_mat.MultTranspose(shape1, state1);
-
-      if (dof2)
-      {
-         // Calculate basis function of the second element at the face
-         el2.CalcShape(Tr.GetElement2IntPoint(), shape2);
-
-         // Interpolate elfun at the point
-         elfun2_mat.MultTranspose(shape2, state2);
-      }
+      elfun2_mat.MultTranspose(shape2, state2);
 
       // Get the normal vector and the flux on the face
       if (nor.Size() == 1)  // if 1D, use 1 or -1.
       {
-         nor(0) = 2*Tr.GetElement1IntPoint().x - 1.;
+         // This assume the 1D integration point is in (0,1). This may not work
+         // if this changes.
+         nor(0) = (Tr.GetElement1IntPoint().x - 0.5) * 2.0;
       }
       else
       {
@@ -352,14 +335,7 @@ void HyperbolicFormIntegrator::AssembleFaceGrad(
       // Trial side 1
 
       // Compute hat(J) using evaluated quantities
-      if (dof2)
-      {
-         numFlux.Grad(1, state1, state2, nor, Tr, JDotN);
-      }
-      else
-      {
-         fluxFunction.ComputeFluxJacobianDotN(state1, nor, Tr, JDotN);
-      }
+      numFlux.Grad(1, state1, state2, nor, Tr, JDotN);
 
       const int ioff = fluxFunction.num_equations * dof1;
 
@@ -384,325 +360,36 @@ void HyperbolicFormIntegrator::AssembleFaceGrad(
             }
          }
 
-      if (dof2)
-      {
-         // Trial side 2
-
-         // Compute hat(J) using evaluated quantities
-         numFlux.Grad(2, state1, state2, nor, Tr, JDotN);
-
-         const int joff = ioff;
-
-         for (int di = 0; di < fluxFunction.num_equations; di++)
-            for (int dj = 0; dj < fluxFunction.num_equations; dj++)
-            {
-               // pre-multiply integration weight to Jacobian
-               const real_t w = +ip.weight * sign * JDotN(di,dj);
-               for (int j = 0; j < dof2; j++)
-               {
-                  // Test side 1
-                  for (int i = 0; i < dof1; i++)
-                  {
-                     elmat(i+dof1*di, joff+j+dof2*dj) += w * shape1(i) * shape2(j);
-                  }
-
-                  // Test side 2
-                  for (int i = 0; i < dof2; i++)
-                  {
-                     elmat(ioff+i+dof2*di, joff+j+dof2*dj) -= w * shape2(i) * shape2(j);
-                  }
-               }
-            }
-      }
-   }
-}
-
-BdrHyperbolicDirichletIntegrator::BdrHyperbolicDirichletIntegrator(
-   const NumericalFlux &numFlux,
-   VectorCoefficient &bdrState,
-   const int IntOrderOffset,
-   real_t sign)
-   : NonlinearFormIntegrator(),
-     numFlux(numFlux),
-     fluxFunction(numFlux.GetFluxFunction()),
-     u_vcoeff(bdrState),
-     IntOrderOffset(IntOrderOffset),
-     sign(sign),
-     num_equations(fluxFunction.num_equations)
-{
-   MFEM_VERIFY(fluxFunction.num_equations == bdrState.GetVDim(),
-               "Flux function does not match the vector dimension of the coefficient!");
-#ifndef MFEM_THREAD_SAFE
-   state_in.SetSize(num_equations);
-   state_out.SetSize(num_equations);
-   fluxN.SetSize(num_equations);
-   JDotN.SetSize(num_equations);
-   nor.SetSize(fluxFunction.dim);
-#endif
-   ResetMaxCharSpeed();
-}
-
-void BdrHyperbolicDirichletIntegrator::AssembleFaceVector(
-   const FiniteElement &el, const FiniteElement &,
-   FaceElementTransformations &Tr, const Vector &elfun, Vector &elvect)
-{
-   MFEM_ASSERT(Tr.Elem2No < 0, "Not a boundary face!");
-
-   // current elements' the number of degrees of freedom
-   // does not consider the number of equations
-   const int dof = el.GetDof();
-
-#ifdef MFEM_THREAD_SAFE
-   // Local storage for element integration
-
-   // shape function value at an integration point
-   Vector shape(dof);
-   // normal vector (usually not a unit vector)
-   Vector nor(Tr.GetSpaceDim());
-   // state value at an integration point - interior
-   Vector state_in(num_equations);
-   // state value at an integration point - boundary
-   Vector state_out(num_equations);
-   // hat(F)(u,x)
-   Vector fluxN(num_equations);
-#else
-   shape.SetSize(dof);
-#endif
-
-   elvect.SetSize(dof * num_equations);
-   elvect = 0.0;
-
-   const DenseMatrix elfun_mat(elfun.GetData(), dof, num_equations);
-
-   DenseMatrix elvect_mat(elvect.GetData(), dof, num_equations);
-
-   // Obtain integration rule. If integration is rule is given, then use it.
-   // Otherwise, get (2*p + IntOrderOffset) order integration rule
-   const IntegrationRule *ir = IntRule;
-   if (!ir)
-   {
-      const int order = 2*el.GetOrder() + IntOrderOffset;
-      ir = &IntRules.Get(Tr.GetGeometryType(), order);
-   }
-   // loop over integration points
-   for (int i = 0; i < ir->GetNPoints(); i++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(i);
-
-      Tr.SetAllIntPoints(&ip); // set face and element int. points
-
-      // Calculate basis functions at the face
-      el.CalcShape(Tr.GetElement1IntPoint(), shape);
-
-      // Interpolate elfun at the point
-      elfun_mat.MultTranspose(shape, state_in);
-
-      // Evaluate boundary state at the point
-      u_vcoeff.Eval(state_out, Tr, ip);
-
-      // Get the normal vector and the flux on the face
-      if (nor.Size() == 1)  // if 1D, use 1 or -1.
-      {
-         nor(0) = 2*Tr.GetElement1IntPoint().x - 1.;
-      }
-      else
-      {
-         CalcOrtho(Tr.Jacobian(), nor);
-      }
-      // Compute F(u+, x) and F(u_b, x) with maximum characteristic speed
-      // Compute hat(F) using evaluated quantities
-      const real_t speed = numFlux.Eval(state_in, state_out, nor, Tr, fluxN);
-
-      // Update the global max char speed
-      max_char_speed = std::max(speed, max_char_speed);
-
-      // pre-multiply integration weight to flux
-      AddMult_a_VWt(-ip.weight*sign, shape, fluxN, elvect_mat);
-   }
-}
-
-void BdrHyperbolicDirichletIntegrator::AssembleFaceGrad(
-   const FiniteElement &el, const FiniteElement &,
-   FaceElementTransformations &Tr, const Vector &elfun, DenseMatrix &elmat)
-{
-   // current elements' the number of degrees of freedom
-   // does not consider the number of equations
-   const int dof = el.GetDof();
-
-#ifdef MFEM_THREAD_SAFE
-   // Local storage for element integration
-
-   // shape function value at an integration point
-   Vector shape(dof);
-   // normal vector (usually not a unit vector)
-   Vector nor(Tr.GetSpaceDim());
-   // state value at an integration point - interior
-   Vector state_in(num_equations);
-   // state value at an integration point - boundary
-   Vector state_out(num_equations);
-   // hat(J)(u,x)
-   DenseMatrix JDotN(num_equations);
-#else
-   shape.SetSize(dof);
-#endif
-
-   elmat.SetSize(dof * num_equations);
-   elmat = 0.0;
-
-   const DenseMatrix elfun_mat(elfun.GetData(), dof, num_equations);
-
-   // Obtain integration rule. If integration is rule is given, then use it.
-   // Otherwise, get (2*p + IntOrderOffset) order integration rule
-   const IntegrationRule *ir = IntRule;
-   if (!ir)
-   {
-      const int order = 2*el.GetOrder() + IntOrderOffset;
-      ir = &IntRules.Get(Tr.GetGeometryType(), order);
-   }
-   // loop over integration points
-   for (int q = 0; q < ir->GetNPoints(); q++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(q);
-
-      Tr.SetAllIntPoints(&ip); // set face and element int. points
-
-      // Calculate basis functions at the face
-      el.CalcShape(Tr.GetElement1IntPoint(), shape);
-
-      // Interpolate elfun at the point
-      elfun_mat.MultTranspose(shape, state_in);
-
-      // Evaluate boundary state at the point
-      u_vcoeff.Eval(state_out, Tr, ip);
-
-      // Get the normal vector and the flux on the face
-      if (nor.Size() == 1)  // if 1D, use 1 or -1.
-      {
-         nor(0) = 2*Tr.GetElement1IntPoint().x - 1.;
-      }
-      else
-      {
-         CalcOrtho(Tr.Jacobian(), nor);
-      }
+      // Trial side 2
 
       // Compute hat(J) using evaluated quantities
-      numFlux.Grad(1, state_in, state_out, nor, Tr, JDotN);
+      numFlux.Grad(2, state1, state2, nor, Tr, JDotN);
+
+      const int joff = ioff;
 
       for (int di = 0; di < fluxFunction.num_equations; di++)
          for (int dj = 0; dj < fluxFunction.num_equations; dj++)
          {
             // pre-multiply integration weight to Jacobian
-            const real_t w = -ip.weight * sign * JDotN(di,dj);
-            for (int j = 0; j < dof; j++)
-               for (int i = 0; i < dof; i++)
+            const real_t w = +ip.weight * sign * JDotN(di,dj);
+            for (int j = 0; j < dof2; j++)
+            {
+               // Test side 1
+               for (int i = 0; i < dof1; i++)
                {
-                  elmat(i+dof*di, j+dof*dj) += w * shape(i) * shape(j);
+                  elmat(i+dof1*di, joff+j+dof2*dj) += w * shape1(i) * shape2(j);
                }
+
+               // Test side 2
+               for (int i = 0; i < dof2; i++)
+               {
+                  elmat(ioff+i+dof2*di, joff+j+dof2*dj) -= w * shape2(i) * shape2(j);
+               }
+            }
          }
    }
 }
 
-BoundaryHyperbolicFlowIntegrator::BoundaryHyperbolicFlowIntegrator(
-   const FluxFunction &flux, VectorCoefficient &u, real_t alpha_, real_t beta_,
-   const int IntOrderOffset_)
-   : fluxFunction(flux), u_vcoeff(u), alpha(alpha_), beta(beta_),
-     IntOrderOffset(IntOrderOffset_)
-{
-   MFEM_VERIFY(fluxFunction.num_equations == u_vcoeff.GetVDim(),
-               "Flux function does not match the vector dimension of the coefficient!");
-#ifndef MFEM_THREAD_SAFE
-   state.SetSize(fluxFunction.num_equations);
-   nor.SetSize(fluxFunction.dim);
-   fluxN.SetSize(fluxFunction.num_equations);
-#endif
-   ResetMaxCharSpeed();
-}
-
-void BoundaryHyperbolicFlowIntegrator::AssembleRHSElementVect(
-   const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
-{
-   mfem_error("BoundaryHyperbolicFlowIntegrator::AssembleRHSElementVect\n"
-              "  is not implemented as boundary integrator!\n"
-              "  Use LinearForm::AddBdrFaceIntegrator instead of\n"
-              "  LinearForm::AddBoundaryIntegrator.");
-}
-
-void BoundaryHyperbolicFlowIntegrator::AssembleRHSElementVect(
-   const FiniteElement &el, FaceElementTransformations &Tr, Vector &elvect)
-{
-   // current elements' the number of degrees of freedom
-   // does not consider the number of equations
-   const int dof = el.GetDof();
-
-#ifdef MFEM_THREAD_SAFE
-   // Local storage for element integration
-
-   // shape function value at an integration point
-   Vector shape(dof);
-   // state value at an integration point
-   Vector state(fluxFunction.num_equations);
-   // normal vector (usually not a unit vector)
-   Vector nor(Tr.GetSpaceDim());
-   // hat(F)(u,x)
-   Vector fluxN(fluxFunction.num_equations);
-#else
-   shape.SetSize(dof);
-#endif
-
-   elvect.SetSize(dof * fluxFunction.num_equations);
-   elvect = 0.0;
-
-   DenseMatrix elvect_mat(elvect.GetData(), dof, fluxFunction.num_equations);
-
-   // Obtain integration rule. If integration is rule is given, then use it.
-   // Otherwise, get (2*p + IntOrderOffset) order integration rule
-   const IntegrationRule *ir = IntRule;
-   if (!ir)
-   {
-      const int order = 2*el.GetOrder() + IntOrderOffset;
-      ir = &IntRules.Get(Tr.GetGeometryType(), order);
-   }
-   // loop over integration points
-   for (int i = 0; i < ir->GetNPoints(); i++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(i);
-
-      Tr.SetAllIntPoints(&ip); // set face and element int. points
-
-      // Calculate basis functions on both elements at the face
-      el.CalcShape(Tr.GetElement1IntPoint(), shape);
-
-      // Evaluate the coefficient at the point
-      u_vcoeff.Eval(state, Tr, ip);
-
-      // Get the normal vector and the flux on the face
-      if (nor.Size() == 1)  // if 1D, use 1 or -1.
-      {
-         nor(0) = 2*Tr.GetElement1IntPoint().x - 1.;
-      }
-      else
-      {
-         CalcOrtho(Tr.Jacobian(), nor);
-      }
-      // Compute F(u, x) with maximum characteristic speed
-      const real_t speed = fluxFunction.ComputeFluxDotN(state, nor, Tr, fluxN);
-
-      // Update the global max char speed
-      max_char_speed = std::max(speed, max_char_speed);
-
-      // pre-multiply integration weight to flux
-      const real_t a = 0.5 * alpha  * ip.weight;
-      const real_t b = beta * ip.weight;
-
-      for (int n = 0; n < fluxFunction.num_equations; n++)
-      {
-         fluxN(n) = a * fluxN(n) - b * fabs(fluxN(n));
-      }
-
-      AddMultVWt(shape, fluxN, elvect_mat);
-   }
-}
-
 real_t FluxFunction::ComputeFluxDotN(const Vector &U,
                                      const Vector &normal,
                                      FaceElementTransformations &Tr,

fem/hyperbolic.hpp

@@ -306,14 +306,12 @@ MFEM_DEPRECATED typedef NumericalFlux RiemannSolver;
 class HyperbolicFormIntegrator : public NonlinearFormIntegrator
 {
 private:
+   // The maximum characteristic speed, updated during element/face vector assembly
+   real_t max_char_speed;
    const NumericalFlux &numFlux;   // Numerical flux that maps F(u±,x) to F̂
    const FluxFunction &fluxFunction;
    const int IntOrderOffset; // integration order offset, 2*p + IntOrderOffset.
    const real_t sign;
-
-   // The maximum characteristic speed, updated during element/face vector assembly
-   real_t max_char_speed;
-
 #ifndef MFEM_THREAD_SAFE
    // Local storage for element integration
    Vector shape;              // shape function value at an integration point
@@ -333,9 +331,8 @@ private:
 
 public:
    const int num_equations;  // the number of equations
-
    /**
-    * @brief Construct a new HyperbolicFormIntegrator object
+    * @brief Construct a new Hyperbolic Form Integrator object
     *
     * @param[in] numFlux numerical flux
     * @param[in] IntOrderOffset integration order offset
@@ -346,14 +343,21 @@ public:
       const int IntOrderOffset = 0,
       const real_t sign = 1.);
 
-   /// Reset the maximum characteristic speed to zero
-   void ResetMaxCharSpeed() { max_char_speed = 0.0; }
+   /**
+    * @brief Reset the Max Char Speed 0
+    *
+    */
+   void ResetMaxCharSpeed()
+   {
+      max_char_speed = 0.0;
+   }
 
-   /// Get the maximum characteristic speed
-   real_t GetMaxCharSpeed() const { return max_char_speed; }
+   real_t GetMaxCharSpeed()
+   {
+      return max_char_speed;
+   }
 
-   /// Get the associated flux function
-   const FluxFunction &GetFluxFunction() const { return fluxFunction; }
+   const FluxFunction &GetFluxFunction() { return fluxFunction; }
 
    /**
     * @brief Implements (F(u), ∇v) with abstract F computed by
@@ -412,182 +416,6 @@ public:
                          const Vector &elfun, DenseMatrix &elmat) override;
 };
 
-/**
- * @brief Abstract boundary hyperbolic form integrator, assembling
- * <F̂(u⁻,u_b,x) n, [v]> term for scalar finite elements at the boundary.
- *
- * This form integrator is coupled with a NumericalFlux that implements the
- * numerical flux F̂ at the boundary faces. The flux F is obtained from the
- * FluxFunction assigned to the aforementioned NumericalFlux with the given
- * boundary coefficient for the state u_b.
- *
- * Note the class can be used for imposing conditions on interior interfaces.
- */
-class BdrHyperbolicDirichletIntegrator : public NonlinearFormIntegrator
-{
-private:
-   const NumericalFlux &numFlux;    // Numerical flux that maps F to F̂
-   const FluxFunction &fluxFunction;
-   VectorCoefficient &u_vcoeff;     // Boundary state vector coefficient
-   const int IntOrderOffset; // integration order offset, 2*p + IntOrderOffset.
-   const real_t sign;
-
-   // The maximum characteristic speed, updated during element/face vector assembly
-   real_t max_char_speed;
-
-#ifndef MFEM_THREAD_SAFE
-   // Local storage for element integration
-   Vector shape;  // shape function value at an integration point
-   Vector state_in;  // state value at an integration point - interior
-   Vector state_out;  // state value at an integration point - boundary
-   Vector nor;     // normal vector, see mfem::CalcOrtho()
-   Vector fluxN;   // F̂(u⁻,u_b,x) n
-   DenseMatrix JDotN;   // Ĵ(u⁻,u_b,x) n
-#endif
-
-public:
-   const int num_equations;  // the number of equations
-
-   /**
-    * @brief Construct a new BdrHyperbolicDirichletIntegrator object
-    *
-    * @param[in] numFlux numerical flux
-    * @param[in] bdrState boundary state coefficient
-    * @param[in] IntOrderOffset integration order offset
-    * @param[in] sign sign of the convection term
-    */
-   BdrHyperbolicDirichletIntegrator(
-      const NumericalFlux &numFlux,
-      VectorCoefficient &bdrState,
-      const int IntOrderOffset = 0,
-      const real_t sign = 1.);
-
-   /// Reset the maximum characteristic speed to zero
-   void ResetMaxCharSpeed() { max_char_speed = 0.0; }
-
-   /// Get the maximum characteristic speed
-   real_t GetMaxCharSpeed() const { return max_char_speed; }
-
-   /// Get the associated flux function
-   const FluxFunction &GetFluxFunction() const { return fluxFunction; }
-
-   /**
-    * @brief Implements <-F̂(u⁻,u_b,x) n, [v]> with abstract F̂ computed by
-    * NumericalFlux::Eval() of the numerical flux object
-    *
-    * @param[in] el1 finite element of the interior element
-    * @param[in] el2 not used
-    * @param[in] Tr face element transformations
-    * @param[in] elfun local coefficient of basis for the interior element
-    * @param[out] elvect evaluated dual vector <-F̂(u⁻,u_b,x) n, [v]>
-    */
-   void AssembleFaceVector(const FiniteElement &el1,
-                           const FiniteElement &el2,
-                           FaceElementTransformations &Tr,
-                           const Vector &elfun, Vector &elvect) override;
-
-   /**
-    * @brief Implements <-Ĵ(u⁻,u_b,x) n, [v]> with abstract Ĵ computed by
-    * NumericalFlux::Grad() of the numerical flux object
-    *
-    * @param[in] el1 finite element of the interior element
-    * @param[in] el2 not used
-    * @param[in] Tr face element transformations
-    * @param[in] elfun local coefficient of basis for the interior element
-    * @param[out] elmat evaluated Jacobian matrix <-Ĵ(u⁻,u_b,x) n, [v]>
-    */
-   void AssembleFaceGrad(const FiniteElement &el1,
-                         const FiniteElement &el2,
-                         FaceElementTransformations &Tr,
-                         const Vector &elfun, DenseMatrix &elmat) override;
-};
-
-/**
- * @brief Abstract boundary hyperbolic linear form integrator, assembling
- * <ɑ/2 F(u,x) n - β |F(u,x) n|, v> terms for scalar finite elements.
- *
- * This form integrator is coupled with a FluxFunction that evaluates the
- * flux F at the boundary.
- *
- * Note the upwinding is performed component-wise. For general boundary
- * integration with a numerical flux, see BdrHyperbolicDirichletIntegrator.
- */
-class BoundaryHyperbolicFlowIntegrator : public LinearFormIntegrator
-{
-   const FluxFunction &fluxFunction;
-   VectorCoefficient &u_vcoeff;
-   const real_t alpha, beta;
-   const int IntOrderOffset; // integration order offset, 2*p + IntOrderOffset.
-
-   // The maximum characteristic speed, updated during face vector assembly
-   real_t max_char_speed;
-
-#ifndef MFEM_THREAD_SAFE
-   // Local storage for element integration
-   Vector shape;   // shape function value at an integration point
-   Vector state;   // state value at an integration point
-   Vector nor;     // normal vector, see mfem::CalcOrtho()
-   Vector fluxN;   // F(u,x) n
-#endif
-
-public:
-   /**
-    * @brief Construct a new BoundaryHyperbolicFlowIntegrator object
-    *
-    * @param[in] flux flux function
-    * @param[in] u vector state coefficient
-    * @param[in] alpha ɑ coefficient (β = ɑ/2)
-    * @param[in] IntOrderOffset integration order offset
-    */
-   BoundaryHyperbolicFlowIntegrator(const FluxFunction &flux, VectorCoefficient &u,
-                                    real_t alpha = -1., int IntOrderOffset = 0)
-      : BoundaryHyperbolicFlowIntegrator(flux, u, alpha, alpha/2., IntOrderOffset) { }
-
-   /**
-    * @brief Construct a new BoundaryHyperbolicFlowIntegrator object
-    *
-    * @param[in] flux flux function
-    * @param[in] u vector state coefficient
-    * @param[in] alpha ɑ coefficient
-    * @param[in] beta β coefficient
-    * @param[in] IntOrderOffset integration order offset
-    */
-   BoundaryHyperbolicFlowIntegrator(const FluxFunction &flux, VectorCoefficient &u,
-                                    real_t alpha, real_t beta, int IntOrderOffset = 0);
-
-   /// Reset the maximum characteristic speed to zero
-   void ResetMaxCharSpeed() { max_char_speed = 0.0; }
-
-   /// Get the maximum characteristic speed
-   real_t GetMaxCharSpeed() const { return max_char_speed; }
-
-   /// Get the associated flux function
-   const FluxFunction &GetFluxFunction() const { return fluxFunction; }
-
-   using LinearFormIntegrator::AssembleRHSElementVect;
-
-   /**
-    * @warning Boundary element integration not implemented, use
-    * AssembleRHSElementVect(const FiniteElement&,
-    * FaceElementTransformations &, Vector &) instead
-    */
-   void AssembleRHSElementVect(const FiniteElement &el,
-                               ElementTransformation &Tr,
-                               Vector &elvect) override;
-
-   /**
-    * @brief Implements <-F(u,x) n, v> with abstract F computed by
-    * FluxFunction::ComputeFluxDotN() of the flux function object
-    *
-    * @param[in] el finite element
-    * @param[in] Tr face element transformations
-    * @param[out] elvect evaluated dual vector <F(u,x) n, v>
-    */
-   void AssembleRHSElementVect(const FiniteElement &el,
-                               FaceElementTransformations &Tr,
-                               Vector &elvect) override;
-};
-
 
 /**
  * @brief Rusanov flux, also known as local Lax-Friedrichs,

fem/integ/bilininteg_curlcurl_pa.cpp

@@ -15,86 +15,6 @@
 namespace mfem
 {
 
-CurlCurlIntegrator::CurlCurlIntegrator() : Q(nullptr), DQ(nullptr), MQ(nullptr)
-{
-   static Kernels kernels;
-}
-
-CurlCurlIntegrator::CurlCurlIntegrator(Coefficient &q,
-                                       const IntegrationRule *ir)
-   : BilinearFormIntegrator(ir), Q(&q), DQ(nullptr), MQ(nullptr)
-{
-   static Kernels kernels;
-}
-
-CurlCurlIntegrator::CurlCurlIntegrator(DiagonalMatrixCoefficient &dq,
-                                       const IntegrationRule *ir)
-   : BilinearFormIntegrator(ir), Q(nullptr), DQ(&dq), MQ(nullptr)
-{
-   static Kernels kernels;
-}
-
-CurlCurlIntegrator::CurlCurlIntegrator(MatrixCoefficient &mq,
-                                       const IntegrationRule *ir)
-   : BilinearFormIntegrator(ir), Q(nullptr), DQ(nullptr), MQ(&mq)
-{
-   static Kernels kernels;
-}
-
-/// \cond DO_NOT_DOCUMENT
-
-CurlCurlIntegrator::Kernels::Kernels()
-{
-   CurlCurlIntegrator::AddSpecialization<3, 2, 3>();
-   CurlCurlIntegrator::AddSpecialization<3, 3, 4>();
-   CurlCurlIntegrator::AddSpecialization<3, 4, 5>();
-   CurlCurlIntegrator::AddSpecialization<3, 5, 6>();
-}
-
-CurlCurlIntegrator::ApplyKernelType
-CurlCurlIntegrator::ApplyPAKernels::Fallback(int DIM, int, int)
-{
-   if (DIM == 2) { return internal::PACurlCurlApply2D; }
-   else if (DIM == 3)
-   {
-      if (Device::Allows(Backend::DEVICE_MASK))
-      {
-         return internal::SmemPACurlCurlApply3D;
-      }
-      else
-      {
-         return internal::PACurlCurlApply3D;
-      }
-   }
-   else { MFEM_ABORT(""); }
-}
-
-CurlCurlIntegrator::DiagonalKernelType
-CurlCurlIntegrator::DiagonalPAKernels::Fallback(int DIM, int, int)
-{
-   if (DIM == 2)
-   {
-      return internal::PACurlCurlAssembleDiagonal2D;
-   }
-   else if (DIM == 3)
-   {
-      if (Device::Allows(Backend::DEVICE_MASK))
-      {
-         return internal::SmemPACurlCurlAssembleDiagonal3D;
-      }
-      else
-      {
-         return internal::PACurlCurlAssembleDiagonal3D;
-      }
-   }
-   else
-   {
-      MFEM_ABORT("");
-   }
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
 void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
    // Assumes tensor-product elements
@@ -157,16 +77,129 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
 
 void CurlCurlIntegrator::AssembleDiagonalPA(Vector& diag)
 {
-   DiagonalPAKernels::Run(dim, dofs1D, quad1D, dofs1D, quad1D, symmetric, ne,
-                          mapsO->B, mapsC->B, mapsO->G, mapsC->G, pa_data,
-                          diag);
+   if (dim == 3)
+   {
+      if (Device::Allows(Backend::DEVICE_MASK))
+      {
+         const int ID = (dofs1D << 4) | quad1D;
+         switch (ID)
+         {
+            case 0x23:
+               return internal::SmemPACurlCurlAssembleDiagonal3D<2,3>(
+                         dofs1D,
+                         quad1D,
+                         symmetric, ne,
+                         mapsO->B, mapsC->B,
+                         mapsO->G, mapsC->G,
+                         pa_data, diag);
+            case 0x34:
+               return internal::SmemPACurlCurlAssembleDiagonal3D<3,4>(
+                         dofs1D,
+                         quad1D,
+                         symmetric, ne,
+                         mapsO->B, mapsC->B,
+                         mapsO->G, mapsC->G,
+                         pa_data, diag);
+            case 0x45:
+               return internal::SmemPACurlCurlAssembleDiagonal3D<4,5>(
+                         dofs1D,
+                         quad1D,
+                         symmetric, ne,
+                         mapsO->B, mapsC->B,
+                         mapsO->G, mapsC->G,
+                         pa_data, diag);
+            case 0x56:
+               return internal::SmemPACurlCurlAssembleDiagonal3D<5,6>(
+                         dofs1D,
+                         quad1D,
+                         symmetric, ne,
+                         mapsO->B, mapsC->B,
+                         mapsO->G, mapsC->G,
+                         pa_data, diag);
+            default:
+               return internal::SmemPACurlCurlAssembleDiagonal3D(
+                         dofs1D, quad1D,
+                         symmetric, ne,
+                         mapsO->B, mapsC->B,
+                         mapsO->G, mapsC->G,
+                         pa_data, diag);
+         }
+      }
+      else
+      {
+         internal::PACurlCurlAssembleDiagonal3D(dofs1D, quad1D, symmetric, ne,
+                                                mapsO->B, mapsC->B,
+                                                mapsO->G, mapsC->G,
+                                                pa_data, diag);
+      }
+   }
+   else if (dim == 2)
+   {
+      internal::PACurlCurlAssembleDiagonal2D(dofs1D, quad1D, ne,
+                                             mapsO->B, mapsC->G, pa_data, diag);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension!");
+   }
 }
 
 void CurlCurlIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
-   ApplyPAKernels::Run(dim, dofs1D, quad1D, dofs1D, quad1D, symmetric, ne,
-                       mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt, mapsC->G,
-                       mapsC->Gt, pa_data, x, y, false);
+   if (dim == 3)
+   {
+      if (Device::Allows(Backend::DEVICE_MASK))
+      {
+         const int ID = (dofs1D << 4) | quad1D;
+         switch (ID)
+         {
+            case 0x23:
+               return internal::SmemPACurlCurlApply3D<2,3>(
+                         dofs1D, quad1D,
+                         symmetric, ne,
+                         mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt,
+                         mapsC->G, mapsC->Gt, pa_data, x, y);
+            case 0x34:
+               return internal::SmemPACurlCurlApply3D<3,4>(
+                         dofs1D, quad1D,
+                         symmetric, ne,
+                         mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt,
+                         mapsC->G, mapsC->Gt, pa_data, x, y);
+            case 0x45:
+               return internal::SmemPACurlCurlApply3D<4,5>(
+                         dofs1D, quad1D,
+                         symmetric, ne,
+                         mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt,
+                         mapsC->G, mapsC->Gt, pa_data, x, y);
+            case 0x56:
+               return internal::SmemPACurlCurlApply3D<5,6>(
+                         dofs1D, quad1D,
+                         symmetric, ne,
+                         mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt,
+                         mapsC->G, mapsC->Gt, pa_data, x, y);
+            default:
+               return internal::SmemPACurlCurlApply3D(
+                         dofs1D, quad1D, symmetric, ne,
+                         mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt,
+                         mapsC->G, mapsC->Gt, pa_data, x, y);
+         }
+      }
+      else
+      {
+         internal::PACurlCurlApply3D(dofs1D, quad1D, symmetric, ne, mapsO->B, mapsC->B,
+                                     mapsO->Bt, mapsC->Bt, mapsC->G, mapsC->Gt,
+                                     pa_data, x, y);
+      }
+   }
+   else if (dim == 2)
+   {
+      internal::PACurlCurlApply2D(dofs1D, quad1D, ne, mapsO->B, mapsO->Bt,
+                                  mapsC->G, mapsC->Gt, pa_data, x, y);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension!");
+   }
 }
 
 void CurlCurlIntegrator::AddAbsMultPA(const Vector &x, Vector &y) const
@@ -176,9 +209,61 @@ void CurlCurlIntegrator::AddAbsMultPA(const Vector &x, Vector &y) const
    auto absO = mapsO->Abs();
    auto absC = mapsC->Abs();
 
-   ApplyPAKernels::Run(dim, dofs1D, quad1D, dofs1D, quad1D, symmetric, ne,
-                       absO.B, absC.B, absO.Bt, absC.Bt, absC.G, absC.Gt,
-                       abs_pa_data, x, y, true);
+   if (dim == 3)
+   {
+      if (Device::Allows(Backend::DEVICE_MASK))
+      {
+         const int ID = (dofs1D << 4) | quad1D;
+         switch (ID)
+         {
+            case 0x23:
+               return internal::SmemPACurlCurlApply3D<2,3>(
+                         dofs1D, quad1D,
+                         symmetric, ne,
+                         absO.B, absC.B, absO.Bt, absC.Bt,
+                         absC.G, absC.Gt, abs_pa_data, x, y, true);
+            case 0x34:
+               return internal::SmemPACurlCurlApply3D<3,4>(
+                         dofs1D, quad1D,
+                         symmetric, ne,
+                         absO.B, absC.B, absO.Bt, absC.Bt,
+                         absC.G, absC.Gt, abs_pa_data, x, y, true);
+            case 0x45:
+               return internal::SmemPACurlCurlApply3D<4,5>(
+                         dofs1D, quad1D,
+                         symmetric, ne,
+                         absO.B, absC.B, absO.Bt, absC.Bt,
+                         absC.G, absC.Gt, abs_pa_data, x, y, true);
+            case 0x56:
+               return internal::SmemPACurlCurlApply3D<5,6>(
+                         dofs1D, quad1D,
+                         symmetric, ne,
+                         absO.B, absC.B, absO.Bt, absC.Bt,
+                         absC.G, absC.Gt, abs_pa_data, x, y, true);
+            default:
+               return internal::SmemPACurlCurlApply3D<0,0>(
+                         dofs1D, quad1D, symmetric, ne,
+                         absO.B, absC.B, absO.Bt, absC.Bt,
+                         absC.G, absC.Gt, abs_pa_data, x, y, true);
+         }
+      }
+      else
+      {
+         internal::PACurlCurlApply3D<0,0>(
+            dofs1D, quad1D, symmetric, ne,
+            absO.B, absC.B, absO.Bt, absC.Bt, absC.G, absC.Gt,
+            abs_pa_data, x, y, true);
+      }
+   }
+   else if (dim == 2)
+   {
+      internal::PACurlCurlApply2D(dofs1D, quad1D, ne, absO.B, absO.Bt,
+                                  absC.G, absC.Gt, abs_pa_data, x, y, true);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension!");
+   }
 }
 
 } // namespace mfem

fem/integ/bilininteg_dgdiffusion_kernels.hpp

@@ -1,500 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_BILININTEG_DGDIFFUSION_KERNELS_HPP
-#define MFEM_BILININTEG_DGDIFFUSION_KERNELS_HPP
-
-#include "../../general/forall.hpp"
-#include "../../mesh/face_nbr_geom.hpp"
-#include "../fe/face_map_utils.hpp"
-#include "../gridfunc.hpp"
-#include "../qfunction.hpp"
-
-/// \cond DO_NOT_DOCUMENT
-namespace mfem
-{
-
-namespace internal
-{
-
-template <int T_D1D = 0, int T_Q1D = 0>
-static void PADGDiffusionApply2D(const int NF, const Array<real_t> &b,
-                                 const Array<real_t> &bt,
-                                 const Array<real_t> &g,
-                                 const Array<real_t> &gt, const real_t sigma,
-                                 const Vector &pa_data, const Vector &x_,
-                                 const Vector &dxdn_, Vector &y_, Vector &dydn_,
-                                 const int d1d = 0, const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   auto B_ = Reshape(b.Read(), Q1D, D1D);
-   auto G_ = Reshape(g.Read(), Q1D, D1D);
-
-   auto pa =
-      Reshape(pa_data.Read(), 6, Q1D, NF); // (q, 1/h, J00, J01, J10, J11)
-
-   auto x = Reshape(x_.Read(), D1D, 2, NF);
-   auto y = Reshape(y_.ReadWrite(), D1D, 2, NF);
-   auto dxdn = Reshape(dxdn_.Read(), D1D, 2, NF);
-   auto dydn = Reshape(dydn_.ReadWrite(), D1D, 2, NF);
-
-   const int NBX = std::max(D1D, Q1D);
-
-   mfem::forall_2D(NF, NBX, 2, [=] MFEM_HOST_DEVICE(int f) -> void
-   {
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t u0[max_D1D];
-      MFEM_SHARED real_t u1[max_D1D];
-      MFEM_SHARED real_t du0[max_D1D];
-      MFEM_SHARED real_t du1[max_D1D];
-
-      MFEM_SHARED real_t Bu0[max_Q1D];
-      MFEM_SHARED real_t Bu1[max_Q1D];
-      MFEM_SHARED real_t Bdu0[max_Q1D];
-      MFEM_SHARED real_t Bdu1[max_Q1D];
-
-      MFEM_SHARED real_t r[max_Q1D];
-
-      MFEM_SHARED real_t BG[2 * max_D1D * max_Q1D];
-      DeviceMatrix B(BG, Q1D, D1D);
-      DeviceMatrix G(BG + D1D * Q1D, Q1D, D1D);
-
-      if (MFEM_THREAD_ID(y) == 0)
-      {
-         MFEM_FOREACH_THREAD(p, x, Q1D)
-         {
-            for (int d = 0; d < D1D; ++d)
-            {
-               B(p, d) = B_(p, d);
-               G(p, d) = G_(p, d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // copy edge values to u0, u1 and copy edge normals to du0, du1
-      MFEM_FOREACH_THREAD(side, y, 2)
-      {
-         real_t *u = (side == 0) ? u0 : u1;
-         real_t *du = (side == 0) ? du0 : du1;
-         MFEM_FOREACH_THREAD(d, x, D1D)
-         {
-            u[d] = x(d, side, f);
-            du[d] = dxdn(d, side, f);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // eval @ quad points
-      MFEM_FOREACH_THREAD(side, y, 2)
-      {
-         real_t *u = (side == 0) ? u0 : u1;
-         real_t *du = (side == 0) ? du0 : du1;
-         real_t *Bu = (side == 0) ? Bu0 : Bu1;
-         real_t *Bdu = (side == 0) ? Bdu0 : Bdu1;
-
-         MFEM_FOREACH_THREAD(p, x, Q1D)
-         {
-            const real_t Je_side[] = {pa(2 + 2 * side, p, f),
-                                      pa(2 + 2 * side + 1, p, f)
-                                     };
-
-            Bu[p] = 0.0;
-            Bdu[p] = 0.0;
-
-            for (int d = 0; d < D1D; ++d)
-            {
-               const real_t b = B(p, d);
-               const real_t g = G(p, d);
-
-               Bu[p] += b * u[d];
-               Bdu[p] += Je_side[0] * b * du[d] + Je_side[1] * g * u[d];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // term - < {Q du/dn}, [v] > +  kappa * < {Q/h} [u], [v] >:
-      if (MFEM_THREAD_ID(y) == 0)
-      {
-         MFEM_FOREACH_THREAD(p, x, Q1D)
-         {
-            const real_t q = pa(0, p, f);
-            const real_t hi = pa(1, p, f);
-            const real_t jump = Bu0[p] - Bu1[p];
-            const real_t avg = Bdu0[p] + Bdu1[p]; // = {Q du/dn} * w * det(J)
-            r[p] = -avg + hi * q * jump;
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(d, x, D1D)
-      {
-         real_t Br = 0.0;
-
-         for (int p = 0; p < Q1D; ++p)
-         {
-            Br += B(p, d) * r[p];
-         }
-
-         u0[d] = Br; // overwrite u0, u1
-         u1[d] = -Br;
-      } // for d
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(side, y, 2)
-      {
-         real_t *du = (side == 0) ? du0 : du1;
-         MFEM_FOREACH_THREAD(d, x, D1D) { du[d] = 0.0; }
-      }
-      MFEM_SYNC_THREAD;
-
-      // term sigma * < [u], {Q dv/dn} >
-      MFEM_FOREACH_THREAD(side, y, 2)
-      {
-         real_t *const du = (side == 0) ? du0 : du1;
-         real_t *const u = (side == 0) ? u0 : u1;
-
-         MFEM_FOREACH_THREAD(d, x, D1D)
-         {
-            for (int p = 0; p < Q1D; ++p)
-            {
-               const real_t Je[] = {pa(2 + 2 * side, p, f),
-                                    pa(2 + 2 * side + 1, p, f)
-                                   };
-               const real_t jump = Bu0[p] - Bu1[p];
-               const real_t r_p = Je[0] * jump; // normal
-               const real_t w_p = Je[1] * jump; // tangential
-               du[d] += sigma * B(p, d) * r_p;
-               u[d] += sigma * G(p, d) * w_p;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(side, y, 2)
-      {
-         real_t *u = (side == 0) ? u0 : u1;
-         real_t *du = (side == 0) ? du0 : du1;
-         MFEM_FOREACH_THREAD(d, x, D1D)
-         {
-            y(d, side, f) += u[d];
-            dydn(d, side, f) += du[d];
-         }
-      }
-   }); // mfem::forall
-}
-
-template <int T_D1D = 0, int T_Q1D = 0>
-static void PADGDiffusionApply3D(const int NF, const Array<real_t> &b,
-                                 const Array<real_t> &bt,
-                                 const Array<real_t> &g,
-                                 const Array<real_t> &gt, const real_t sigma,
-                                 const Vector &pa_data, const Vector &x_,
-                                 const Vector &dxdn_, Vector &y_, Vector &dydn_,
-                                 const int d1d = 0, const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   auto B_ = Reshape(b.Read(), Q1D, D1D);
-   auto G_ = Reshape(g.Read(), Q1D, D1D);
-
-   // (J0[0], J0[1], J0[2], J1[0], J1[1], J1[2], q/h)
-   auto pa = Reshape(pa_data.Read(), 7, Q1D, Q1D, NF);
-
-   auto x = Reshape(x_.Read(), D1D, D1D, 2, NF);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, 2, NF);
-   auto dxdn = Reshape(dxdn_.Read(), D1D, D1D, 2, NF);
-   auto dydn = Reshape(dydn_.ReadWrite(), D1D, D1D, 2, NF);
-
-   const int NBX = std::max(D1D, Q1D);
-
-   mfem::forall_3D(NF, NBX, NBX, 2, [=] MFEM_HOST_DEVICE(int f) -> void
-   {
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t u0[max_Q1D][max_Q1D];
-      MFEM_SHARED real_t u1[max_Q1D][max_Q1D];
-
-      MFEM_SHARED real_t du0[max_Q1D][max_Q1D];
-      MFEM_SHARED real_t du1[max_Q1D][max_Q1D];
-
-      MFEM_SHARED real_t Gu0[max_Q1D][max_Q1D];
-      MFEM_SHARED real_t Gu1[max_Q1D][max_Q1D];
-
-      MFEM_SHARED real_t Bu0[max_Q1D][max_Q1D];
-      MFEM_SHARED real_t Bu1[max_Q1D][max_Q1D];
-
-      MFEM_SHARED real_t Bdu0[max_Q1D][max_Q1D];
-      MFEM_SHARED real_t Bdu1[max_Q1D][max_Q1D];
-
-      MFEM_SHARED real_t kappa_Qh[max_Q1D][max_Q1D];
-
-      MFEM_SHARED real_t nJe[2][max_Q1D][max_Q1D][3];
-      MFEM_SHARED real_t BG[2 * max_D1D * max_Q1D];
-
-      // some buffers are reused multiple times, but for clarity have new names:
-      real_t(*Bj0)[max_Q1D] = Bu0;
-      real_t(*Bj1)[max_Q1D] = Bu1;
-      real_t(*Bjn0)[max_Q1D] = Bdu0;
-      real_t(*Bjn1)[max_Q1D] = Bdu1;
-      real_t(*Gj0)[max_Q1D] = Gu0;
-      real_t(*Gj1)[max_Q1D] = Gu1;
-
-      DeviceMatrix B(BG, Q1D, D1D);
-      DeviceMatrix G(BG + D1D * Q1D, Q1D, D1D);
-
-      // copy face values to u0, u1 and copy normals to du0, du1
-      MFEM_FOREACH_THREAD(side, z, 2)
-      {
-         real_t(*u)[max_Q1D] = (side == 0) ? u0 : u1;
-         real_t(*du)[max_Q1D] = (side == 0) ? du0 : du1;
-
-         MFEM_FOREACH_THREAD(d2, x, D1D)
-         {
-            MFEM_FOREACH_THREAD(d1, y, D1D)
-            {
-               u[d2][d1] = x(d1, d2, side,
-                             f); // copy transposed for better memory access
-               du[d2][d1] = dxdn(d1, d2, side, f);
-            }
-         }
-
-         MFEM_FOREACH_THREAD(p1, x, Q1D)
-         {
-            MFEM_FOREACH_THREAD(p2, y, Q1D)
-            {
-               for (int l = 0; l < 3; ++l)
-               {
-                  nJe[side][p2][p1][l] = pa(3 * side + l, p1, p2, f);
-               }
-
-               if (side == 0)
-               {
-                  kappa_Qh[p2][p1] = pa(6, p1, p2, f);
-               }
-            }
-         }
-
-         if (side == 0)
-         {
-            MFEM_FOREACH_THREAD(p, x, Q1D)
-            {
-               MFEM_FOREACH_THREAD(d, y, D1D)
-               {
-                  B(p, d) = B_(p, d);
-                  G(p, d) = G_(p, d);
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // eval u and normal derivative @ quad points
-      MFEM_FOREACH_THREAD(side, z, 2)
-      {
-         real_t(*u)[max_Q1D] = (side == 0) ? u0 : u1;
-         real_t(*du)[max_Q1D] = (side == 0) ? du0 : du1;
-         real_t(*Bu)[max_Q1D] = (side == 0) ? Bu0 : Bu1;
-         real_t(*Bdu)[max_Q1D] = (side == 0) ? Bdu0 : Bdu1;
-         real_t(*Gu)[max_Q1D] = (side == 0) ? Gu0 : Gu1;
-
-         MFEM_FOREACH_THREAD(p1, x, Q1D)
-         {
-            MFEM_FOREACH_THREAD(d2, y, D1D)
-            {
-               real_t bu = 0.0;
-               real_t bdu = 0.0;
-               real_t gu = 0.0;
-
-               for (int d1 = 0; d1 < D1D; ++d1)
-               {
-                  const real_t b = B(p1, d1);
-                  const real_t g = G(p1, d1);
-
-                  bu += b * u[d2][d1];
-                  bdu += b * du[d2][d1];
-                  gu += g * u[d2][d1];
-               }
-
-               Bu[p1][d2] = bu;
-               Bdu[p1][d2] = bdu;
-               Gu[p1][d2] = gu;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(side, z, 2)
-      {
-         real_t(*u)[max_Q1D] = (side == 0) ? u0 : u1;
-         real_t(*du)[max_Q1D] = (side == 0) ? du0 : du1;
-         real_t(*Bu)[max_Q1D] = (side == 0) ? Bu0 : Bu1;
-         real_t(*Gu)[max_Q1D] = (side == 0) ? Gu0 : Gu1;
-         real_t(*Bdu)[max_Q1D] = (side == 0) ? Bdu0 : Bdu1;
-
-         MFEM_FOREACH_THREAD(p2, x, Q1D)
-         {
-            MFEM_FOREACH_THREAD(p1, y, Q1D)
-            {
-               const real_t *Je = nJe[side][p2][p1];
-
-               real_t bbu = 0.0;
-               real_t bgu = 0.0;
-               real_t gbu = 0.0;
-               real_t bbdu = 0.0;
-
-               for (int d2 = 0; d2 < D1D; ++d2)
-               {
-                  const real_t b = B(p2, d2);
-                  const real_t g = G(p2, d2);
-                  bbu += b * Bu[p1][d2];
-                  gbu += g * Bu[p1][d2];
-                  bgu += b * Gu[p1][d2];
-                  bbdu += b * Bdu[p1][d2];
-               }
-
-               u[p2][p1] = bbu;
-               // du <- Q du/dn * w * det(J)
-               du[p2][p1] = Je[0] * bbdu + Je[1] * bgu + Je[2] * gbu;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(side, z, 2)
-      {
-         real_t(*Bj)[max_Q1D] = (side == 0) ? Bj0 : Bj1;
-         real_t(*Bjn)[max_Q1D] = (side == 0) ? Bjn0 : Bjn1;
-         real_t(*Gj)[max_Q1D] = (side == 0) ? Gj0 : Gj1;
-
-         MFEM_FOREACH_THREAD(d1, x, D1D)
-         {
-            MFEM_FOREACH_THREAD(p2, y, Q1D)
-            {
-               real_t bj = 0.0;
-               real_t bjn = 0.0;
-               real_t gj = 0.0;
-               real_t br = 0.0;
-
-               for (int p1 = 0; p1 < Q1D; ++p1)
-               {
-                  const real_t b = B(p1, d1);
-                  const real_t g = G(p1, d1);
-
-                  const real_t *Je = nJe[side][p2][p1];
-
-                  const real_t jump = u0[p2][p1] - u1[p2][p1];
-                  const real_t avg = du0[p2][p1] + du1[p2][p1];
-
-                  // r = - < {Q du/dn}, [v] > + kappa * < {Q/h} [u], [v] >
-                  const real_t r = -avg + kappa_Qh[p2][p1] * jump;
-
-                  // bj, gj, bjn contribute to sigma term
-                  bj += b * Je[0] * jump;
-                  gj += g * Je[1] * jump;
-                  bjn += b * Je[2] * jump;
-
-                  br += b * r;
-               }
-
-               Bj[d1][p2] = sigma * bj;
-               Bjn[d1][p2] = sigma * bjn;
-
-               // group br and gj together since we will multiply them both by B
-               // and then sum
-               const real_t sgn = (side == 0) ? 1.0 : -1.0;
-               Gj[d1][p2] = sgn * br + sigma * gj;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(side, z, 2)
-      {
-         real_t(*u)[max_Q1D] = (side == 0) ? u0 : u1;
-         real_t(*du)[max_Q1D] = (side == 0) ? du0 : du1;
-         real_t(*Bj)[max_Q1D] = (side == 0) ? Bj0 : Bj1;
-         real_t(*Bjn)[max_Q1D] = (side == 0) ? Bjn0 : Bjn1;
-         real_t(*Gj)[max_Q1D] = (side == 0) ? Gj0 : Gj1;
-
-         MFEM_FOREACH_THREAD(d2, x, D1D)
-         {
-            MFEM_FOREACH_THREAD(d1, y, D1D)
-            {
-               real_t bbj = 0.0;
-               real_t gbj = 0.0;
-               real_t bgj = 0.0;
-
-               for (int p2 = 0; p2 < Q1D; ++p2)
-               {
-                  const real_t b = B(p2, d2);
-                  const real_t g = G(p2, d2);
-
-                  bbj += b * Bj[d1][p2];
-                  bgj += b * Gj[d1][p2];
-                  gbj += g * Bjn[d1][p2];
-               }
-
-               du[d2][d1] = bbj;
-               u[d2][d1] = bgj + gbj;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // map back to y and dydn
-      MFEM_FOREACH_THREAD(side, z, 2)
-      {
-         const real_t(*u)[max_Q1D] = (side == 0) ? u0 : u1;
-         const real_t(*du)[max_Q1D] = (side == 0) ? du0 : du1;
-
-         MFEM_FOREACH_THREAD(d2, x, D1D)
-         {
-            MFEM_FOREACH_THREAD(d1, y, D1D)
-            {
-               y(d1, d2, side, f) += u[d2][d1];
-               dydn(d1, d2, side, f) += du[d2][d1];
-            }
-         }
-      }
-   });
-}
-
-} // namespace internal
-
-template <int DIM, int D1D, int Q1D>
-DGDiffusionIntegrator::ApplyKernelType
-DGDiffusionIntegrator::ApplyPAKernels::Kernel()
-{
-   if constexpr (DIM == 2)
-   {
-      return internal::PADGDiffusionApply2D<D1D, Q1D>;
-   }
-   else if constexpr (DIM == 3)
-   {
-      return internal::PADGDiffusionApply3D<D1D, Q1D>;
-   }
-   MFEM_ABORT("");
-}
-} // namespace mfem
-/// \endcond DO_NOT_DOCUMENT
-#endif

fem/integ/bilininteg_dgdiffusion_pa.cpp

@@ -11,39 +11,42 @@
 
 #include "../../general/forall.hpp"
 #include "../../mesh/face_nbr_geom.hpp"
-#include "../fe/face_map_utils.hpp"
 #include "../gridfunc.hpp"
 #include "../qfunction.hpp"
+#include "../fe/face_map_utils.hpp"
 
-#include "bilininteg_dgdiffusion_kernels.hpp"
+using namespace std;
 
 namespace mfem
 {
 
-static void PADGDiffusionSetup2D(const int Q1D, const int NE, const int NF,
+static void PADGDiffusionSetup2D(const int Q1D,
+                                 const int NE,
+                                 const int NF,
                                  const Array<real_t> &w,
                                  const GeometricFactors &el_geom,
                                  const FaceGeometricFactors &face_geom,
                                  const FaceNeighborGeometricFactors *nbr_geom,
-                                 const Vector &q, const real_t sigma,
-                                 const real_t kappa, Vector &pa_data,
+                                 const Vector &q,
+                                 const real_t sigma,
+                                 const real_t kappa,
+                                 Vector &pa_data,
                                  const Array<int> &face_info_)
 {
    const auto J_loc = Reshape(el_geom.J.Read(), Q1D, Q1D, 2, 2, NE);
    const auto detJe_loc = Reshape(el_geom.detJ.Read(), Q1D, Q1D, NE);
 
    const int n_nbr = nbr_geom ? nbr_geom->num_neighbor_elems : 0;
-   const auto J_shared =
-      Reshape(nbr_geom ? nbr_geom->J.Read() : nullptr, Q1D, Q1D, 2, 2, n_nbr);
-   const auto detJ_shared =
-      Reshape(nbr_geom ? nbr_geom->detJ.Read() : nullptr, Q1D, Q1D, n_nbr);
+   const auto J_shared = Reshape(nbr_geom ? nbr_geom->J.Read() : nullptr,
+                                 Q1D, Q1D, 2, 2, n_nbr);
+   const auto detJ_shared = Reshape(nbr_geom ? nbr_geom->detJ.Read() : nullptr,
+                                    Q1D, Q1D, n_nbr);
 
    const auto detJf = Reshape(face_geom.detJ.Read(), Q1D, NF);
    const auto n = Reshape(face_geom.normal.Read(), Q1D, 2, NF);
 
    const bool const_q = (q.Size() == 1);
-   const auto Q =
-      const_q ? Reshape(q.Read(), 1, 1) : Reshape(q.Read(), Q1D, NF);
+   const auto Q = const_q ? Reshape(q.Read(), 1,1) : Reshape(q.Read(), Q1D,NF);
 
    const auto W = w.Read();
 
@@ -53,7 +56,7 @@ static void PADGDiffusionSetup2D(const int Q1D, const int NE, const int NF,
    // (q, 1/h, J0_0, J0_1, J1_0, J1_1)
    auto pa = Reshape(pa_data.Write(), 6, Q1D, NF);
 
-   mfem::forall(NF, [=] MFEM_HOST_DEVICE(int f) -> void
+   mfem::forall(NF, [=] MFEM_HOST_DEVICE (int f) -> void
    {
       const int normal_dir[] = {face_info(0, f), face_info(1, f)};
       const int fid[] = {face_info(4, f), face_info(5, f)};
@@ -71,7 +74,7 @@ static void PADGDiffusionSetup2D(const int Q1D, const int NE, const int NF,
 
       for (int p = 0; p < Q1D; ++p)
       {
-         const real_t Qp = const_q ? Q(0, 0) : Q(p, f);
+         const real_t Qp = const_q ? Q(0,0) : Q(p, f);
          pa(0, p, f) = kappa * Qp * W[p] * detJf(p, f);
 
          real_t hi = 0.0;
@@ -82,19 +85,17 @@ static void PADGDiffusionSetup2D(const int Q1D, const int NE, const int NF,
 
             // Always opposite direction in "native" ordering
             // Need to multiply the native=>lex0 with native=>lex1 and negate
-            const int sgn = (side == 1) ? -1 * sgn0 * sgn1 : 1;
+            const int sgn = (side == 1) ? -1*sgn0*sgn1 : 1;
 
             const int e = el[side];
             const auto &J = (side == 1 && shared) ? J_shared : J_loc;
             const auto &detJ = (side == 1 && shared) ? detJ_shared : detJe_loc;
 
             real_t nJi[2];
-            nJi[0] =
-               n(p, 0, f) * J(i, j, 1, 1, e) - n(p, 1, f) * J(i, j, 0, 1, e);
-            nJi[1] =
-               -n(p, 0, f) * J(i, j, 1, 0, e) + n(p, 1, f) * J(i, j, 0, 0, e);
+            nJi[0] = n(p,0,f)*J(i,j, 1,1, e) - n(p,1,f)*J(i,j,0,1,e);
+            nJi[1] = -n(p,0,f)*J(i,j,1,0, e) + n(p,1,f)*J(i,j,0,0,e);
 
-            const real_t dJe = detJ(i, j, e);
+            const real_t dJe = detJ(i,j,e);
             const real_t dJf = detJf(p, f);
 
             const real_t w = factor * Qp * W[p] * dJf / dJe;
@@ -103,9 +104,9 @@ static void PADGDiffusionSetup2D(const int Q1D, const int NE, const int NF,
             const int ti = 1 - ni;
 
             // Normal
-            pa(2 + 2 * side + 0, p, f) = w * nJi[ni];
+            pa(2 + 2*side + 0, p, f) = w * nJi[ni];
             // Tangential
-            pa(2 + 2 * side + 1, p, f) = sgn * w * nJi[ti];
+            pa(2 + 2*side + 1, p, f) = sgn * w * nJi[ti];
 
             hi += factor * dJf / dJe;
          }
@@ -121,43 +122,47 @@ static void PADGDiffusionSetup2D(const int Q1D, const int NE, const int NF,
    });
 }
 
-static void PADGDiffusionSetup3D(const int Q1D, const int NE, const int NF,
+static void PADGDiffusionSetup3D(const int Q1D,
+                                 const int NE,
+                                 const int NF,
                                  const Array<real_t> &w,
                                  const GeometricFactors &el_geom,
                                  const FaceGeometricFactors &face_geom,
                                  const FaceNeighborGeometricFactors *nbr_geom,
-                                 const Vector &q, const real_t sigma,
-                                 const real_t kappa, Vector &pa_data,
+                                 const Vector &q,
+                                 const real_t sigma,
+                                 const real_t kappa,
+                                 Vector &pa_data,
                                  const Array<int> &face_info_)
 {
    const auto J_loc = Reshape(el_geom.J.Read(), Q1D, Q1D, Q1D, 3, 3, NE);
    const auto detJe_loc = Reshape(el_geom.detJ.Read(), Q1D, Q1D, Q1D, NE);
 
    const int n_nbr = nbr_geom ? nbr_geom->num_neighbor_elems : 0;
-   const auto J_shared = Reshape(nbr_geom ? nbr_geom->J.Read() : nullptr, Q1D,
-                                 Q1D, Q1D, 3, 3, n_nbr);
-   const auto detJ_shared =
-      Reshape(nbr_geom ? nbr_geom->detJ.Read() : nullptr, Q1D, Q1D, Q1D, n_nbr);
+   const auto J_shared = Reshape(nbr_geom ? nbr_geom->J.Read() : nullptr,
+                                 Q1D, Q1D, Q1D, 3, 3, n_nbr);
+   const auto detJ_shared = Reshape(nbr_geom ? nbr_geom->detJ.Read() : nullptr,
+                                    Q1D, Q1D, Q1D, n_nbr);
 
    const auto detJf = Reshape(face_geom.detJ.Read(), Q1D, Q1D, NF);
    const auto n = Reshape(face_geom.normal.Read(), Q1D, Q1D, 3, NF);
 
    const bool const_q = (q.Size() == 1);
-   const auto Q =
-      const_q ? Reshape(q.Read(), 1, 1, 1) : Reshape(q.Read(), Q1D, Q1D, NF);
+   const auto Q = const_q ? Reshape(q.Read(), 1, 1, 1)
+                  : Reshape(q.Read(), Q1D, Q1D, NF);
 
    const auto W = Reshape(w.Read(), Q1D, Q1D);
 
    // (perm[0], perm[1], perm[2], element_index, local_face_id, orientation)
    const auto face_info = Reshape(face_info_.Read(), 6, 2, NF);
-   constexpr int _el_ = 3;  // offset in face_info for element index
+   constexpr int _el_ = 3; // offset in face_info for element index
    constexpr int _fid_ = 4; // offset in face_info for local face id
-   constexpr int _or_ = 5;  // offset in face_info for orientation
+   constexpr int _or_ = 5; // offset in face_info for orientation
 
    // (J00, J01, J02, J10, J11, J12, q/h)
    const auto pa = Reshape(pa_data.Write(), 7, Q1D, Q1D, NF);
 
-   mfem::forall_2D(NF, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int f) -> void
+   mfem::forall_2D(NF, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int f) -> void
    {
       MFEM_SHARED int perm[2][3];
       MFEM_SHARED int el[2];
@@ -167,7 +172,10 @@ static void PADGDiffusionSetup3D(const int Q1D, const int NE, const int NF,
 
       MFEM_FOREACH_THREAD(side, x, 2)
       {
-         MFEM_FOREACH_THREAD(i, y, 3) { perm[side][i] = face_info(i, side, f); }
+         MFEM_FOREACH_THREAD(i, y, 3)
+         {
+            perm[side][i] = face_info(i, side, f);
+         }
 
          if (MFEM_THREAD_ID(y) == 0)
          {
@@ -192,16 +200,16 @@ static void PADGDiffusionSetup3D(const int Q1D, const int NE, const int NF,
       {
          MFEM_FOREACH_THREAD(p2, y, Q1D)
          {
-            const real_t Qp = const_q ? Q(0, 0, 0) : Q(p1, p2, f);
-            const real_t dJf = detJf(p1, p2, f);
+            const real_t Qp = const_q ? Q(0,0,0) : Q(p1, p2, f);
+            const real_t dJf = detJf(p1,p2,f);
 
             real_t hi = 0.0;
 
             for (int side = 0; side < nsides; ++side)
             {
                int i, j, k;
-               internal::FaceIdxToVolIdx3D(p1 + Q1D * p2, Q1D, fid[0], fid[1],
-                                           side, ortn[1], i, j, k);
+               internal::FaceIdxToVolIdx3D(
+                  p1 + Q1D*p2, Q1D, fid[0], fid[1], side, ortn[1], i, j, k);
 
                const int e = el[side];
                const auto &J = shared[side] ? J_shared : J_loc;
@@ -209,45 +217,27 @@ static void PADGDiffusionSetup3D(const int Q1D, const int NE, const int NF,
 
                // *INDENT-OFF*
                real_t nJi[3];
-               nJi[0] = (-J(i, j, k, 1, 2, e) * J(i, j, k, 2, 1, e) +
-                         J(i, j, k, 1, 1, e) * J(i, j, k, 2, 2, e)) *
-                           n(p1, p2, 0, f) +
-                        (J(i, j, k, 0, 2, e) * J(i, j, k, 2, 1, e) -
-                         J(i, j, k, 0, 1, e) * J(i, j, k, 2, 2, e)) *
-                           n(p1, p2, 1, f) +
-                        (-J(i, j, k, 0, 2, e) * J(i, j, k, 1, 1, e) +
-                         J(i, j, k, 0, 1, e) * J(i, j, k, 1, 2, e)) *
-                           n(p1, p2, 2, f);
+               nJi[0] = (  -J(i,j,k, 1,2, e)*J(i,j,k, 2,1, e) + J(i,j,k, 1,1, e)*J(i,j,k, 2,2, e)) * n(p1,p2, 0, f)
+                        + ( J(i,j,k, 0,2, e)*J(i,j,k, 2,1, e) - J(i,j,k, 0,1, e)*J(i,j,k, 2,2, e)) * n(p1,p2, 1, f)
+                        + (-J(i,j,k, 0,2, e)*J(i,j,k, 1,1, e) + J(i,j,k, 0,1, e)*J(i,j,k, 1,2, e)) * n(p1,p2, 2, f);
 
-               nJi[1] = (J(i, j, k, 1, 2, e) * J(i, j, k, 2, 0, e) -
-                         J(i, j, k, 1, 0, e) * J(i, j, k, 2, 2, e)) *
-                           n(p1, p2, 0, f) +
-                        (-J(i, j, k, 0, 2, e) * J(i, j, k, 2, 0, e) +
-                         J(i, j, k, 0, 0, e) * J(i, j, k, 2, 2, e)) *
-                           n(p1, p2, 1, f) +
-                        (J(i, j, k, 0, 2, e) * J(i, j, k, 1, 0, e) -
-                         J(i, j, k, 0, 0, e) * J(i, j, k, 1, 2, e)) *
-                           n(p1, p2, 2, f);
+               nJi[1] = (   J(i,j,k, 1,2, e)*J(i,j,k, 2,0, e) - J(i,j,k, 1,0, e)*J(i,j,k, 2,2, e)) * n(p1,p2, 0, f)
+                        + (-J(i,j,k, 0,2, e)*J(i,j,k, 2,0, e) + J(i,j,k, 0,0, e)*J(i,j,k, 2,2, e)) * n(p1,p2, 1, f)
+                        + ( J(i,j,k, 0,2, e)*J(i,j,k, 1,0, e) - J(i,j,k, 0,0, e)*J(i,j,k, 1,2, e)) * n(p1,p2, 2, f);
 
-               nJi[2] = (-J(i, j, k, 1, 1, e) * J(i, j, k, 2, 0, e) +
-                         J(i, j, k, 1, 0, e) * J(i, j, k, 2, 1, e)) *
-                           n(p1, p2, 0, f) +
-                        (J(i, j, k, 0, 1, e) * J(i, j, k, 2, 0, e) -
-                         J(i, j, k, 0, 0, e) * J(i, j, k, 2, 1, e)) *
-                           n(p1, p2, 1, f) +
-                        (-J(i, j, k, 0, 1, e) * J(i, j, k, 1, 0, e) +
-                         J(i, j, k, 0, 0, e) * J(i, j, k, 1, 1, e)) *
-                           n(p1, p2, 2, f);
+               nJi[2] = (  -J(i,j,k, 1,1, e)*J(i,j,k, 2,0, e) + J(i,j,k, 1,0, e)*J(i,j,k, 2,1, e)) * n(p1,p2, 0, f)
+                        + ( J(i,j,k, 0,1, e)*J(i,j,k, 2,0, e) - J(i,j,k, 0,0, e)*J(i,j,k, 2,1, e)) * n(p1,p2, 1, f)
+                        + (-J(i,j,k, 0,1, e)*J(i,j,k, 1,0, e) + J(i,j,k, 0,0, e)*J(i,j,k, 1,1, e)) * n(p1,p2, 2, f);
                // *INDENT-ON*
 
-               const real_t dJe = detJe(i, j, k, e);
+               const real_t dJe = detJe(i,j,k,e);
                const real_t val = factor * Qp * W(p1, p2) * dJf / dJe;
 
                for (int d = 0; d < 3; ++d)
                {
                   const int idx = std::abs(perm[side][d]) - 1;
                   const int sgn = (perm[side][d] < 0) ? -1 : 1;
-                  pa(3 * side + d, p1, p2, f) = sgn * val * nJi[idx];
+                  pa(3*side + d, p1, p2, f) = sgn * val * nJi[idx];
                }
 
                hi += factor * dJf / dJe;
@@ -267,8 +257,7 @@ static void PADGDiffusionSetup3D(const int Q1D, const int NE, const int NF,
 }
 
 static void PADGDiffusionSetupFaceInfo2D(const int nf, const Mesh &mesh,
-                                         const FaceType type,
-                                         Array<int> &face_info_)
+                                         const FaceType type, Array<int> &face_info_)
 {
    const int ne = mesh.GetNE();
 
@@ -337,7 +326,8 @@ inline void FaceNormalPermutation(int perm[3], const int face_id)
 
 // Assigns to perm the permutation as in FaceNormalPermutation for the second
 // element on the face but signed to indicate the sign of the normal derivative.
-inline void SignedFaceNormalPermutation(int perm[3], const int face_id1,
+inline void SignedFaceNormalPermutation(int perm[3],
+                                        const int face_id1,
                                         const int face_id2,
                                         const int orientation)
 {
@@ -396,19 +386,17 @@ inline void SignedFaceNormalPermutation(int perm[3], const int face_id1,
 }
 
 static void PADGDiffusionSetupFaceInfo3D(const int nf, const Mesh &mesh,
-                                         const FaceType type,
-                                         Array<int> &face_info_)
+                                         const FaceType type, Array<int> &face_info_)
 {
    const int ne = mesh.GetNE();
 
    int fidx = 0;
-   // face_info array has 12 entries per face, 6 for each of the adjacent
-   // elements: (perm[0], perm[1], perm[2], element_index, local_face_id,
-   // orientation)
+   // face_info array has 12 entries per face, 6 for each of the adjacent elements:
+   // (perm[0], perm[1], perm[2], element_index, local_face_id, orientation)
    face_info_.SetSize(nf * 12);
-   constexpr int _e_ = 3;   // offset for element index
+   constexpr int _e_ = 3; // offset for element index
    constexpr int _fid_ = 4; // offset for local face id
-   constexpr int _or_ = 5;  // offset for orientation
+   constexpr int _or_ = 5; // offset for orientation
 
    auto face_info = Reshape(face_info_.HostWrite(), 6, 2, nf);
    for (int f = 0; f < mesh.GetNumFaces(); ++f)
@@ -420,9 +408,9 @@ static void PADGDiffusionSetupFaceInfo3D(const int nf, const Mesh &mesh,
          const int fid0 = f_info.element[0].local_face_id;
          const int or0 = f_info.element[0].orientation;
 
-         face_info(_e_, 0, fidx) = f_info.element[0].index;
+         face_info(  _e_, 0, fidx) = f_info.element[0].index;
          face_info(_fid_, 0, fidx) = fid0;
-         face_info(_or_, 0, fidx) = or0;
+         face_info( _or_, 0, fidx) = or0;
 
          FaceNormalPermutation(&face_info(0, 0, fidx), fid0);
 
@@ -433,17 +421,16 @@ static void PADGDiffusionSetupFaceInfo3D(const int nf, const Mesh &mesh,
 
             if (f_info.IsShared())
             {
-               face_info(_e_, 1, fidx) = ne + f_info.element[1].index;
+               face_info(  _e_, 1, fidx) = ne + f_info.element[1].index;
             }
             else
             {
-               face_info(_e_, 1, fidx) = f_info.element[1].index;
+               face_info(  _e_, 1, fidx) = f_info.element[1].index;
             }
             face_info(_fid_, 1, fidx) = fid1;
-            face_info(_or_, 1, fidx) = or1;
+            face_info( _or_, 1, fidx) = or1;
 
-            SignedFaceNormalPermutation(&face_info(0, 1, fidx), fid0, fid1,
-                                        or1);
+            SignedFaceNormalPermutation(&face_info(0, 1, fidx), fid0, fid1, or1);
          }
          else
          {
@@ -461,8 +448,8 @@ static void PADGDiffusionSetupFaceInfo3D(const int nf, const Mesh &mesh,
 void DGDiffusionIntegrator::SetupPA(const FiniteElementSpace &fes,
                                     FaceType type)
 {
-   const MemoryType mt =
-      (pa_mt == MemoryType::DEFAULT) ? Device::GetDeviceMemoryType() : pa_mt;
+   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
+                         Device::GetDeviceMemoryType() : pa_mt;
 
    const int ne = fes.GetNE();
    nf = fes.GetNFbyType(type);
@@ -471,17 +458,16 @@ void DGDiffusionIntegrator::SetupPA(const FiniteElementSpace &fes,
    Mesh &mesh = *fes.GetMesh();
    const Geometry::Type face_geom_type = mesh.GetTypicalFaceGeometry();
    const FiniteElement &el = *fes.GetTypicalTraceElement();
-   const int ir_order = IntRule
-                        ? IntRule->GetOrder()
+   const int ir_order = IntRule ? IntRule->GetOrder()
                         : GetRule(el.GetOrder(), face_geom_type).GetOrder();
    const IntegrationRule &ir = irs.Get(face_geom_type, ir_order);
    dim = mesh.Dimension();
-   const int q1d = (ir.GetOrder() + 3) / 2;
-   MFEM_ASSERT(q1d == pow(real_t(ir.Size()), 1.0 / (dim - 1)), "");
+   const int q1d = (ir.GetOrder() + 3)/2;
+   MFEM_ASSERT(q1d == pow(real_t(ir.Size()), 1.0/(dim - 1)), "");
 
    const auto vol_ir = irs.Get(mesh.GetTypicalElementGeometry(), ir_order);
-   const auto geom_flags =
-      GeometricFactors::JACOBIANS | GeometricFactors::DETERMINANTS;
+   const auto geom_flags = GeometricFactors::JACOBIANS |
+                           GeometricFactors::DETERMINANTS;
    const auto el_geom = mesh.GetGeometricFactors(vol_ir, geom_flags, mt);
 
    std::unique_ptr<FaceNeighborGeometricFactors> nbr_geom;
@@ -490,8 +476,8 @@ void DGDiffusionIntegrator::SetupPA(const FiniteElementSpace &fes,
       nbr_geom.reset(new FaceNeighborGeometricFactors(*el_geom));
    }
 
-   const auto face_geom_flags =
-      FaceGeometricFactors::DETERMINANTS | FaceGeometricFactors::NORMALS;
+   const auto face_geom_flags = FaceGeometricFactors::DETERMINANTS |
+                                FaceGeometricFactors::NORMALS;
    auto face_geom = mesh.GetFaceGeometricFactors(ir, face_geom_flags, type, mt);
    maps = &el.GetDofToQuad(ir, DofToQuad::TENSOR);
    dofs1D = maps->ndof;
@@ -503,18 +489,9 @@ void DGDiffusionIntegrator::SetupPA(const FiniteElementSpace &fes,
    // Evaluate the coefficient at the face quadrature points.
    FaceQuadratureSpace fqs(mesh, ir, type);
    CoefficientVector q(fqs, CoefficientStorage::COMPRESSED);
-   if (Q)
-   {
-      q.Project(*Q);
-   }
-   else if (MQ)
-   {
-      MFEM_ABORT("Not yet implemented"); /* q.Project(*MQ); */
-   }
-   else
-   {
-      q.SetConstant(1.0);
-   }
+   if (Q) { q.Project(*Q); }
+   else if (MQ) { MFEM_ABORT("Not yet implemented"); /* q.Project(*MQ); */ }
+   else { q.SetConstant(1.0); }
 
    Array<int> face_info;
    if (dim == 1)
@@ -524,16 +501,14 @@ void DGDiffusionIntegrator::SetupPA(const FiniteElementSpace &fes,
    else if (dim == 2)
    {
       PADGDiffusionSetupFaceInfo2D(nf, mesh, type, face_info);
-      PADGDiffusionSetup2D(quad1D, ne, nf, ir.GetWeights(), *el_geom,
-                           *face_geom, nbr_geom.get(), q, sigma, kappa, pa_data,
-                           face_info);
+      PADGDiffusionSetup2D(quad1D, ne, nf, ir.GetWeights(), *el_geom, *face_geom,
+                           nbr_geom.get(), q, sigma, kappa, pa_data, face_info);
    }
    else if (dim == 3)
    {
       PADGDiffusionSetupFaceInfo3D(nf, mesh, type, face_info);
-      PADGDiffusionSetup3D(quad1D, ne, nf, ir.GetWeights(), *el_geom,
-                           *face_geom, nbr_geom.get(), q, sigma, kappa, pa_data,
-                           face_info);
+      PADGDiffusionSetup3D(quad1D, ne, nf, ir.GetWeights(), *el_geom, *face_geom,
+                           nbr_geom.get(), q, sigma, kappa, pa_data, face_info);
    }
 }
 
@@ -549,76 +524,529 @@ void DGDiffusionIntegrator::AssemblePABoundaryFaces(
    SetupPA(fes, FaceType::Boundary);
 }
 
-void DGDiffusionIntegrator::AddMultPAFaceNormalDerivatives(const Vector &x,
-                                                           const Vector &dxdn,
-                                                           Vector &y,
-                                                           Vector &dydn) const
+template<int T_D1D = 0, int T_Q1D = 0> static
+void PADGDiffusionApply2D(const int NF,
+                          const Array<real_t> &b,
+                          const Array<real_t> &bt,
+                          const Array<real_t>& g,
+                          const Array<real_t>& gt,
+                          const real_t sigma,
+                          const Vector &pa_data,
+                          const Vector &x_,
+                          const Vector &dxdn_,
+                          Vector &y_,
+                          Vector &dydn_,
+                          const int d1d = 0,
+                          const int q1d = 0)
 {
-   ApplyPAKernels::Run(dim, dofs1D, quad1D, nf, maps->B, maps->Bt, maps->G,
-                       maps->Gt, sigma, pa_data, x, dxdn, y, dydn, dofs1D,
-                       quad1D);
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+
+   auto B_ = Reshape(b.Read(), Q1D, D1D);
+   auto G_ = Reshape(g.Read(), Q1D, D1D);
+
+   auto pa = Reshape(pa_data.Read(), 6, Q1D, NF); // (q, 1/h, J00, J01, J10, J11)
+
+   auto x =    Reshape(x_.Read(),         D1D, 2, NF);
+   auto y =    Reshape(y_.ReadWrite(),    D1D, 2, NF);
+   auto dxdn = Reshape(dxdn_.Read(),      D1D, 2, NF);
+   auto dydn = Reshape(dydn_.ReadWrite(), D1D, 2, NF);
+
+   const int NBX = std::max(D1D, Q1D);
+
+   mfem::forall_2D(NF, NBX, 2, [=] MFEM_HOST_DEVICE (int f) -> void
+   {
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      MFEM_SHARED real_t u0[max_D1D];
+      MFEM_SHARED real_t u1[max_D1D];
+      MFEM_SHARED real_t du0[max_D1D];
+      MFEM_SHARED real_t du1[max_D1D];
+
+      MFEM_SHARED real_t Bu0[max_Q1D];
+      MFEM_SHARED real_t Bu1[max_Q1D];
+      MFEM_SHARED real_t Bdu0[max_Q1D];
+      MFEM_SHARED real_t Bdu1[max_Q1D];
+
+      MFEM_SHARED real_t r[max_Q1D];
+
+      MFEM_SHARED real_t BG[2*max_D1D*max_Q1D];
+      DeviceMatrix B(BG, Q1D, D1D);
+      DeviceMatrix G(BG + D1D*Q1D, Q1D, D1D);
+
+      if (MFEM_THREAD_ID(y) == 0)
+      {
+         MFEM_FOREACH_THREAD(p,x,Q1D)
+         {
+            for (int d = 0; d < D1D; ++d)
+            {
+               B(p,d) = B_(p,d);
+               G(p,d) = G_(p,d);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // copy edge values to u0, u1 and copy edge normals to du0, du1
+      MFEM_FOREACH_THREAD(side,y,2)
+      {
+         real_t *u = (side == 0) ? u0 : u1;
+         real_t *du = (side == 0) ? du0 : du1;
+         MFEM_FOREACH_THREAD(d,x,D1D)
+         {
+            u[d] = x(d, side, f);
+            du[d] = dxdn(d, side, f);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // eval @ quad points
+      MFEM_FOREACH_THREAD(side,y,2)
+      {
+         real_t *u = (side == 0) ? u0 : u1;
+         real_t *du = (side == 0) ? du0 : du1;
+         real_t *Bu = (side == 0) ? Bu0 : Bu1;
+         real_t *Bdu = (side == 0) ? Bdu0 : Bdu1;
+
+         MFEM_FOREACH_THREAD(p,x,Q1D)
+         {
+            const real_t Je_side[] = {pa(2 + 2*side, p, f), pa(2 + 2*side + 1, p, f)};
+
+            Bu[p] = 0.0;
+            Bdu[p] = 0.0;
+
+            for (int d = 0; d < D1D; ++d)
+            {
+               const real_t b = B(p,d);
+               const real_t g = G(p,d);
+
+               Bu[p] += b*u[d];
+               Bdu[p] += Je_side[0] * b * du[d] + Je_side[1] * g * u[d];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // term - < {Q du/dn}, [v] > +  kappa * < {Q/h} [u], [v] >:
+      if (MFEM_THREAD_ID(y) == 0)
+      {
+         MFEM_FOREACH_THREAD(p,x,Q1D)
+         {
+            const real_t q = pa(0, p, f);
+            const real_t hi = pa(1, p, f);
+            const real_t jump = Bu0[p] - Bu1[p];
+            const real_t avg = Bdu0[p] + Bdu1[p]; // = {Q du/dn} * w * det(J)
+            r[p] = -avg + hi * q * jump;
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(d,x,D1D)
+      {
+         real_t Br = 0.0;
+
+         for (int p = 0; p < Q1D; ++p)
+         {
+            Br += B(p, d) * r[p];
+         }
+
+         u0[d] =  Br; // overwrite u0, u1
+         u1[d] = -Br;
+      } // for d
+      MFEM_SYNC_THREAD;
+
+
+      MFEM_FOREACH_THREAD(side,y,2)
+      {
+         real_t *du = (side == 0) ? du0 : du1;
+         MFEM_FOREACH_THREAD(d,x,D1D)
+         {
+            du[d] = 0.0;
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // term sigma * < [u], {Q dv/dn} >
+      MFEM_FOREACH_THREAD(side,y,2)
+      {
+         real_t * const du = (side == 0) ? du0 : du1;
+         real_t * const u = (side == 0) ? u0 : u1;
+
+         MFEM_FOREACH_THREAD(d,x,D1D)
+         {
+            for (int p = 0; p < Q1D; ++p)
+            {
+               const real_t Je[] = {pa(2 + 2*side, p, f), pa(2 + 2*side + 1, p, f)};
+               const real_t jump = Bu0[p] - Bu1[p];
+               const real_t r_p = Je[0] * jump; // normal
+               const real_t w_p = Je[1] * jump; // tangential
+               du[d] += sigma * B(p, d) * r_p;
+               u[d] += sigma * G(p, d) * w_p;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(side,y,2)
+      {
+         real_t *u = (side == 0) ? u0 : u1;
+         real_t *du = (side == 0) ? du0 : du1;
+         MFEM_FOREACH_THREAD(d,x,D1D)
+         {
+            y(d, side, f) += u[d];
+            dydn(d, side, f) += du[d];
+         }
+      }
+   }); // mfem::forall
 }
 
-DGDiffusionIntegrator::DGDiffusionIntegrator(const real_t s, const real_t k)
-   : sigma(s), kappa(k)
+template <int T_D1D = 0, int T_Q1D = 0>
+static void PADGDiffusionApply3D(const int NF,
+                                 const Array<real_t>& b,
+                                 const Array<real_t>& bt,
+                                 const Array<real_t>& g,
+                                 const Array<real_t>& gt,
+                                 const real_t sigma,
+                                 const Vector& pa_data,
+                                 const Vector& x_,
+                                 const Vector& dxdn_,
+                                 Vector& y_,
+                                 Vector& dydn_,
+                                 const int d1d = 0,
+                                 const int q1d = 0)
 {
-   static Kernels kernels;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+
+   auto B_ = Reshape(b.Read(), Q1D, D1D);
+   auto G_ = Reshape(g.Read(), Q1D, D1D);
+
+   // (J0[0], J0[1], J0[2], J1[0], J1[1], J1[2], q/h)
+   auto pa = Reshape(pa_data.Read(), 7, Q1D, Q1D, NF);
+
+   auto x =    Reshape(x_.Read(),         D1D, D1D, 2, NF);
+   auto y =    Reshape(y_.ReadWrite(),    D1D, D1D, 2, NF);
+   auto dxdn = Reshape(dxdn_.Read(),      D1D, D1D, 2, NF);
+   auto dydn = Reshape(dydn_.ReadWrite(), D1D, D1D, 2, NF);
+
+   const int NBX = std::max(D1D, Q1D);
+
+   mfem::forall_3D(NF, NBX, NBX, 2, [=] MFEM_HOST_DEVICE (int f) -> void
+   {
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      MFEM_SHARED real_t u0[max_Q1D][max_Q1D];
+      MFEM_SHARED real_t u1[max_Q1D][max_Q1D];
+
+      MFEM_SHARED real_t du0[max_Q1D][max_Q1D];
+      MFEM_SHARED real_t du1[max_Q1D][max_Q1D];
+
+      MFEM_SHARED real_t Gu0[max_Q1D][max_Q1D];
+      MFEM_SHARED real_t Gu1[max_Q1D][max_Q1D];
+
+      MFEM_SHARED real_t Bu0[max_Q1D][max_Q1D];
+      MFEM_SHARED real_t Bu1[max_Q1D][max_Q1D];
+
+      MFEM_SHARED real_t Bdu0[max_Q1D][max_Q1D];
+      MFEM_SHARED real_t Bdu1[max_Q1D][max_Q1D];
+
+      MFEM_SHARED real_t kappa_Qh[max_Q1D][max_Q1D];
+
+      MFEM_SHARED real_t nJe[2][max_Q1D][max_Q1D][3];
+      MFEM_SHARED real_t BG[2*max_D1D*max_Q1D];
+
+      // some buffers are reused multiple times, but for clarity have new names:
+      real_t (*Bj0)[max_Q1D] = Bu0;
+      real_t (*Bj1)[max_Q1D] = Bu1;
+      real_t (*Bjn0)[max_Q1D] = Bdu0;
+      real_t (*Bjn1)[max_Q1D] = Bdu1;
+      real_t (*Gj0)[max_Q1D] = Gu0;
+      real_t (*Gj1)[max_Q1D] = Gu1;
+
+      DeviceMatrix B(BG, Q1D, D1D);
+      DeviceMatrix G(BG + D1D*Q1D, Q1D, D1D);
+
+      // copy face values to u0, u1 and copy normals to du0, du1
+      MFEM_FOREACH_THREAD(side, z, 2)
+      {
+         real_t (*u)[max_Q1D] = (side == 0) ? u0 : u1;
+         real_t (*du)[max_Q1D] = (side == 0) ? du0 : du1;
+
+         MFEM_FOREACH_THREAD(d2, x, D1D)
+         {
+            MFEM_FOREACH_THREAD(d1, y, D1D)
+            {
+               u[d2][d1] = x(d1, d2, side, f); // copy transposed for better memory access
+               du[d2][d1] = dxdn(d1, d2, side, f);
+            }
+         }
+
+         MFEM_FOREACH_THREAD(p1, x, Q1D)
+         {
+            MFEM_FOREACH_THREAD(p2, y, Q1D)
+            {
+               for (int l=0; l < 3; ++l)
+               {
+                  nJe[side][p2][p1][l] = pa(3*side + l, p1, p2, f);
+               }
+
+               if (side == 0)
+               {
+                  kappa_Qh[p2][p1] = pa(6, p1, p2, f);
+               }
+            }
+         }
+
+         if (side == 0)
+         {
+            MFEM_FOREACH_THREAD(p, x, Q1D)
+            {
+               MFEM_FOREACH_THREAD(d, y, D1D)
+               {
+                  B(p, d) = B_(p, d);
+                  G(p, d) = G_(p, d);
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // eval u and normal derivative @ quad points
+      MFEM_FOREACH_THREAD(side, z, 2)
+      {
+         real_t (*u)[max_Q1D] = (side == 0) ? u0 : u1;
+         real_t (*du)[max_Q1D] = (side == 0) ? du0 : du1;
+         real_t (*Bu)[max_Q1D] = (side == 0) ? Bu0 : Bu1;
+         real_t (*Bdu)[max_Q1D] = (side == 0) ? Bdu0 : Bdu1;
+         real_t (*Gu)[max_Q1D] = (side == 0) ? Gu0 : Gu1;
+
+         MFEM_FOREACH_THREAD(p1, x, Q1D)
+         {
+            MFEM_FOREACH_THREAD(d2, y, D1D)
+            {
+               real_t bu = 0.0;
+               real_t bdu = 0.0;
+               real_t gu = 0.0;
+
+               for (int d1=0; d1 < D1D; ++d1)
+               {
+                  const real_t b = B(p1, d1);
+                  const real_t g = G(p1, d1);
+
+                  bu += b * u[d2][d1];
+                  bdu += b * du[d2][d1];
+                  gu += g * u[d2][d1];
+               }
+
+               Bu[p1][d2] = bu;
+               Bdu[p1][d2] = bdu;
+               Gu[p1][d2] = gu;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(side, z, 2)
+      {
+         real_t (*u)[max_Q1D] = (side == 0) ? u0 : u1;
+         real_t (*du)[max_Q1D] = (side == 0) ? du0 : du1;
+         real_t (*Bu)[max_Q1D] = (side == 0) ? Bu0 : Bu1;
+         real_t (*Gu)[max_Q1D] = (side == 0) ? Gu0 : Gu1;
+         real_t (*Bdu)[max_Q1D] = (side == 0) ? Bdu0 : Bdu1;
+
+         MFEM_FOREACH_THREAD(p2, x, Q1D)
+         {
+            MFEM_FOREACH_THREAD(p1, y, Q1D)
+            {
+               const real_t * Je = nJe[side][p2][p1];
+
+               real_t bbu = 0.0;
+               real_t bgu = 0.0;
+               real_t gbu = 0.0;
+               real_t bbdu = 0.0;
+
+               for (int d2 = 0; d2 < D1D; ++d2)
+               {
+                  const real_t b = B(p2, d2);
+                  const real_t g = G(p2, d2);
+                  bbu += b * Bu[p1][d2];
+                  gbu += g * Bu[p1][d2];
+                  bgu += b * Gu[p1][d2];
+                  bbdu += b * Bdu[p1][d2];
+               }
+
+               u[p2][p1] = bbu;
+               // du <- Q du/dn * w * det(J)
+               du[p2][p1] = Je[0] * bbdu + Je[1] * bgu + Je[2] * gbu;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(side, z, 2)
+      {
+         real_t (*Bj)[max_Q1D] = (side == 0) ? Bj0 : Bj1;
+         real_t (*Bjn)[max_Q1D] = (side == 0) ? Bjn0 : Bjn1;
+         real_t (*Gj)[max_Q1D] = (side == 0) ? Gj0 : Gj1;
+
+         MFEM_FOREACH_THREAD(d1, x, D1D)
+         {
+            MFEM_FOREACH_THREAD(p2, y, Q1D)
+            {
+               real_t bj = 0.0;
+               real_t bjn = 0.0;
+               real_t gj = 0.0;
+               real_t br = 0.0;
+
+               for (int p1 = 0; p1 < Q1D; ++p1)
+               {
+                  const real_t b = B(p1, d1);
+                  const real_t g = G(p1, d1);
+
+                  const real_t * Je = nJe[side][p2][p1];
+
+                  const real_t jump = u0[p2][p1] - u1[p2][p1];
+                  const real_t avg = du0[p2][p1] + du1[p2][p1];
+
+                  // r = - < {Q du/dn}, [v] > + kappa * < {Q/h} [u], [v] >
+                  const real_t r = -avg + kappa_Qh[p2][p1] * jump;
+
+                  // bj, gj, bjn contribute to sigma term
+                  bj += b * Je[0] * jump;
+                  gj += g * Je[1] * jump;
+                  bjn += b * Je[2] * jump;
+
+                  br += b * r;
+               }
+
+               Bj[d1][p2] = sigma * bj;
+               Bjn[d1][p2] = sigma * bjn;
+
+               // group br and gj together since we will multiply them both by B
+               // and then sum
+               const real_t sgn = (side == 0) ? 1.0 : -1.0;
+               Gj[d1][p2] = sgn * br + sigma * gj;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(side, z, 2)
+      {
+         real_t (*u)[max_Q1D] = (side == 0) ? u0 : u1;
+         real_t (*du)[max_Q1D] = (side == 0) ? du0 : du1;
+         real_t (*Bj)[max_Q1D] = (side == 0) ? Bj0 : Bj1;
+         real_t (*Bjn)[max_Q1D] = (side == 0) ? Bjn0 : Bjn1;
+         real_t (*Gj)[max_Q1D] = (side == 0) ? Gj0 : Gj1;
+
+         MFEM_FOREACH_THREAD(d2, x, D1D)
+         {
+            MFEM_FOREACH_THREAD(d1, y, D1D)
+            {
+               real_t bbj = 0.0;
+               real_t gbj = 0.0;
+               real_t bgj = 0.0;
+
+               for (int p2 = 0; p2 < Q1D; ++p2)
+               {
+                  const real_t b = B(p2, d2);
+                  const real_t g = G(p2, d2);
+
+                  bbj += b * Bj[d1][p2];
+                  bgj += b * Gj[d1][p2];
+                  gbj += g * Bjn[d1][p2];
+               }
+
+               du[d2][d1] = bbj;
+               u[d2][d1] = bgj + gbj;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // map back to y and dydn
+      MFEM_FOREACH_THREAD(side, z, 2)
+      {
+         const real_t (*u)[max_Q1D] = (side == 0) ? u0 : u1;
+         const real_t (*du)[max_Q1D] = (side == 0) ? du0 : du1;
+
+         MFEM_FOREACH_THREAD(d2, x, D1D)
+         {
+            MFEM_FOREACH_THREAD(d1, y, D1D)
+            {
+               y(d1, d2, side, f) += u[d2][d1];
+               dydn(d1, d2, side, f) += du[d2][d1];
+            }
+         }
+      }
+   });
 }
 
-DGDiffusionIntegrator::DGDiffusionIntegrator(Coefficient &q, const real_t s,
-                                             const real_t k)
-   : DGDiffusionIntegrator(s, k)
-{
-   Q = &q;
-}
-
-DGDiffusionIntegrator::DGDiffusionIntegrator(MatrixCoefficient &q,
-                                             const real_t s, const real_t k)
-   : DGDiffusionIntegrator(s, k)
-{
-   MQ = &q;
-}
-
-/// \cond DO_NOT_DOCUMENT
-
-DGDiffusionIntegrator::ApplyKernelType
-DGDiffusionIntegrator::ApplyPAKernels::Fallback(int dim, int, int)
+static void PADGDiffusionApply(const int dim,
+                               const int D1D,
+                               const int Q1D,
+                               const int NF,
+                               const Array<real_t> &B,
+                               const Array<real_t> &Bt,
+                               const Array<real_t> &G,
+                               const Array<real_t> &Gt,
+                               const real_t sigma,
+                               const Vector &pa_data,
+                               const Vector &x,
+                               const Vector &dxdn,
+                               Vector &y,
+                               Vector &dydn)
 {
    if (dim == 2)
    {
-      return internal::PADGDiffusionApply2D;
+      auto kernel = PADGDiffusionApply2D<0,0>;
+      switch ((D1D << 4 ) | Q1D)
+      {
+         case 0x23: kernel = PADGDiffusionApply2D<2,3>; break;
+         case 0x34: kernel = PADGDiffusionApply2D<3,4>; break;
+         case 0x45: kernel = PADGDiffusionApply2D<4,5>; break;
+         case 0x56: kernel = PADGDiffusionApply2D<5,6>; break;
+         case 0x67: kernel = PADGDiffusionApply2D<6,7>; break;
+         case 0x78: kernel = PADGDiffusionApply2D<7,8>; break;
+         case 0x89: kernel = PADGDiffusionApply2D<8,9>; break;
+         case 0x9A: kernel = PADGDiffusionApply2D<9,10>; break;
+      }
+      kernel(NF, B, Bt, G, Gt, sigma, pa_data, x, dxdn, y, dydn, D1D, Q1D);
    }
    else if (dim == 3)
    {
-      return internal::PADGDiffusionApply3D;
+      auto kernel = PADGDiffusionApply3D<0,0>;
+      switch ((D1D << 4) | Q1D)
+      {
+         case 0x24: kernel = PADGDiffusionApply3D<2,4>; break;
+         case 0x35: kernel = PADGDiffusionApply3D<3,5>; break;
+         case 0x46: kernel = PADGDiffusionApply3D<4,6>; break;
+         case 0x57: kernel = PADGDiffusionApply3D<5,7>; break;
+         case 0x68: kernel = PADGDiffusionApply3D<6,8>; break;
+         case 0x79: kernel = PADGDiffusionApply3D<7,9>; break;
+         case 0x8A: kernel = PADGDiffusionApply3D<8,10>; break;
+         case 0x9B: kernel = PADGDiffusionApply3D<9,11>; break;
+      }
+      kernel(NF, B, Bt, G, Gt, sigma, pa_data, x, dxdn, y, dydn, D1D, Q1D);
    }
    else
    {
-      MFEM_ABORT("");
+      MFEM_ABORT("Unsupported dimension");
    }
 }
 
-DGDiffusionIntegrator::Kernels::Kernels()
+void DGDiffusionIntegrator::AddMultPAFaceNormalDerivatives(
+   const Vector &x, const Vector &dxdn, Vector &y, Vector &dydn) const
 {
-   DGDiffusionIntegrator::AddSpecialization<2, 2, 3>();
-   DGDiffusionIntegrator::AddSpecialization<2, 3, 4>();
-   DGDiffusionIntegrator::AddSpecialization<2, 4, 5>();
-   DGDiffusionIntegrator::AddSpecialization<2, 5, 6>();
-   DGDiffusionIntegrator::AddSpecialization<2, 6, 7>();
-   DGDiffusionIntegrator::AddSpecialization<2, 7, 8>();
-   DGDiffusionIntegrator::AddSpecialization<2, 8, 9>();
-   DGDiffusionIntegrator::AddSpecialization<2, 9, 10>();
-
-   DGDiffusionIntegrator::AddSpecialization<3, 2, 4>();
-   DGDiffusionIntegrator::AddSpecialization<3, 3, 5>();
-   DGDiffusionIntegrator::AddSpecialization<3, 4, 6>();
-   DGDiffusionIntegrator::AddSpecialization<3, 5, 7>();
-   DGDiffusionIntegrator::AddSpecialization<3, 6, 8>();
-   DGDiffusionIntegrator::AddSpecialization<3, 7, 9>();
-   DGDiffusionIntegrator::AddSpecialization<3, 8, 10>();
-   DGDiffusionIntegrator::AddSpecialization<3, 9, 11>();
+   PADGDiffusionApply(dim, dofs1D, quad1D, nf,
+                      maps->B, maps->Bt, maps->G, maps->Gt,
+                      sigma, pa_data, x, dxdn, y, dydn);
 }
 
-/// \endcond DO_NOT_DOCUMENT
-
 } // namespace mfem

fem/integ/bilininteg_dgtrace_kernels.hpp

@@ -1,793 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef BILININTEG_DGTRACE_KERNELS_HPP
-#define BILININTEG_DGTRACE_KERNELS_HPP
-
-#include "../../general/forall.hpp"
-#include "../bilininteg.hpp"
-#include "../gridfunc.hpp"
-#include "../qfunction.hpp"
-#include "../restriction.hpp"
-
-/// \cond DO_NOT_DOCUMENT
-namespace mfem
-{
-
-namespace internal
-{
-
-// PA DGTrace Apply 2D kernel for Gauss-Lobatto/Bernstein
-template <int T_D1D = 0, int T_Q1D = 0>
-static void PADGTraceApply2D(const int NF, const Array<real_t> &b,
-                             const Array<real_t> &bt, const Vector &op_,
-                             const Vector &x_, Vector &y_, const int d1d = 0,
-                             const int q1d = 0)
-{
-   const int VDIM = 1;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto op = Reshape(op_.Read(), Q1D, 2, 2, NF);
-   auto x = Reshape(x_.Read(), D1D, VDIM, 2, NF);
-   auto y = Reshape(y_.ReadWrite(), D1D, VDIM, 2, NF);
-
-   mfem::forall(NF, [=] MFEM_HOST_DEVICE(int f)
-   {
-      const int VDIM = 1;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      real_t u0[max_D1D][VDIM];
-      real_t u1[max_D1D][VDIM];
-      for (int d = 0; d < D1D; d++)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            u0[d][c] = x(d, c, 0, f);
-            u1[d][c] = x(d, c, 1, f);
-         }
-      }
-      real_t Bu0[max_Q1D][VDIM];
-      real_t Bu1[max_Q1D][VDIM];
-      for (int q = 0; q < Q1D; ++q)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            Bu0[q][c] = 0.0;
-            Bu1[q][c] = 0.0;
-         }
-         for (int d = 0; d < D1D; ++d)
-         {
-            const real_t b = B(q, d);
-            for (int c = 0; c < VDIM; c++)
-            {
-               Bu0[q][c] += b * u0[d][c];
-               Bu1[q][c] += b * u1[d][c];
-            }
-         }
-      }
-      real_t DBu[max_Q1D][VDIM];
-      for (int q = 0; q < Q1D; ++q)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            DBu[q][c] = op(q, 0, 0, f) * Bu0[q][c] + op(q, 1, 0, f) * Bu1[q][c];
-         }
-      }
-      real_t BDBu[max_D1D][VDIM];
-      for (int d = 0; d < D1D; ++d)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            BDBu[d][c] = 0.0;
-         }
-         for (int q = 0; q < Q1D; ++q)
-         {
-            const real_t b = Bt(d, q);
-            for (int c = 0; c < VDIM; c++)
-            {
-               BDBu[d][c] += b * DBu[q][c];
-            }
-         }
-         for (int c = 0; c < VDIM; c++)
-         {
-            y(d, c, 0, f) += BDBu[d][c];
-            y(d, c, 1, f) += -BDBu[d][c];
-         }
-      }
-   });
-}
-
-// PA DGTrace Apply 3D kernel for Gauss-Lobatto/Bernstein
-template <int T_D1D = 0, int T_Q1D = 0>
-static void PADGTraceApply3D(const int NF, const Array<real_t> &b,
-                             const Array<real_t> &bt, const Vector &op_,
-                             const Vector &x_, Vector &y_, const int d1d = 0,
-                             const int q1d = 0)
-{
-   const int VDIM = 1;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, 2, 2, NF);
-   auto x = Reshape(x_.Read(), D1D, D1D, VDIM, 2, NF);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, VDIM, 2, NF);
-
-   mfem::forall(NF, [=] MFEM_HOST_DEVICE(int f)
-   {
-      const int VDIM = 1;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      real_t u0[max_D1D][max_D1D][VDIM];
-      real_t u1[max_D1D][max_D1D][VDIM];
-      for (int d1 = 0; d1 < D1D; d1++)
-      {
-         for (int d2 = 0; d2 < D1D; d2++)
-         {
-            for (int c = 0; c < VDIM; c++)
-            {
-               u0[d1][d2][c] = x(d1, d2, c, 0, f);
-               u1[d1][d2][c] = x(d1, d2, c, 1, f);
-            }
-         }
-      }
-      real_t Bu0[max_Q1D][max_D1D][VDIM];
-      real_t Bu1[max_Q1D][max_D1D][VDIM];
-      for (int q = 0; q < Q1D; ++q)
-      {
-         for (int d2 = 0; d2 < D1D; d2++)
-         {
-            for (int c = 0; c < VDIM; c++)
-            {
-               Bu0[q][d2][c] = 0.0;
-               Bu1[q][d2][c] = 0.0;
-            }
-            for (int d1 = 0; d1 < D1D; ++d1)
-            {
-               const real_t b = B(q, d1);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  Bu0[q][d2][c] += b * u0[d1][d2][c];
-                  Bu1[q][d2][c] += b * u1[d1][d2][c];
-               }
-            }
-         }
-      }
-      real_t BBu0[max_Q1D][max_Q1D][VDIM];
-      real_t BBu1[max_Q1D][max_Q1D][VDIM];
-      for (int q1 = 0; q1 < Q1D; ++q1)
-      {
-         for (int q2 = 0; q2 < Q1D; q2++)
-         {
-            for (int c = 0; c < VDIM; c++)
-            {
-               BBu0[q1][q2][c] = 0.0;
-               BBu1[q1][q2][c] = 0.0;
-            }
-            for (int d2 = 0; d2 < D1D; ++d2)
-            {
-               const real_t b = B(q2, d2);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  BBu0[q1][q2][c] += b * Bu0[q1][d2][c];
-                  BBu1[q1][q2][c] += b * Bu1[q1][d2][c];
-               }
-            }
-         }
-      }
-      real_t DBBu[max_Q1D][max_Q1D][VDIM];
-      for (int q1 = 0; q1 < Q1D; ++q1)
-      {
-         for (int q2 = 0; q2 < Q1D; q2++)
-         {
-            for (int c = 0; c < VDIM; c++)
-            {
-               DBBu[q1][q2][c] = op(q1, q2, 0, 0, f) * BBu0[q1][q2][c] +
-                                 op(q1, q2, 1, 0, f) * BBu1[q1][q2][c];
-            }
-         }
-      }
-      real_t BDBBu[max_Q1D][max_D1D][VDIM];
-      for (int q1 = 0; q1 < Q1D; ++q1)
-      {
-         for (int d2 = 0; d2 < D1D; d2++)
-         {
-            for (int c = 0; c < VDIM; c++)
-            {
-               BDBBu[q1][d2][c] = 0.0;
-            }
-            for (int q2 = 0; q2 < Q1D; ++q2)
-            {
-               const real_t b = Bt(d2, q2);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  BDBBu[q1][d2][c] += b * DBBu[q1][q2][c];
-               }
-            }
-         }
-      }
-      real_t BBDBBu[max_D1D][max_D1D][VDIM];
-      for (int d1 = 0; d1 < D1D; ++d1)
-      {
-         for (int d2 = 0; d2 < D1D; d2++)
-         {
-            for (int c = 0; c < VDIM; c++)
-            {
-               BBDBBu[d1][d2][c] = 0.0;
-            }
-            for (int q1 = 0; q1 < Q1D; ++q1)
-            {
-               const real_t b = Bt(d1, q1);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  BBDBBu[d1][d2][c] += b * BDBBu[q1][d2][c];
-               }
-            }
-            for (int c = 0; c < VDIM; c++)
-            {
-               y(d1, d2, c, 0, f) += BBDBBu[d1][d2][c];
-               y(d1, d2, c, 1, f) += -BBDBBu[d1][d2][c];
-            }
-         }
-      }
-   });
-}
-
-// Optimized PA DGTrace Apply 3D kernel for Gauss-Lobatto/Bernstein
-template <int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
-static void SmemPADGTraceApply3D(const int NF, const Array<real_t> &b,
-                                 const Array<real_t> &bt, const Vector &op_,
-                                 const Vector &x_, Vector &y_,
-                                 const int d1d = 0, const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, 2, 2, NF);
-   auto x = Reshape(x_.Read(), D1D, D1D, 2, NF);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, 2, NF);
-
-   mfem::forall_2D_batch(NF, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE(int f)
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      // the following variables are evaluated at compile time
-      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      MFEM_SHARED real_t u0[NBZ][max_D1D][max_D1D];
-      MFEM_SHARED real_t u1[NBZ][max_D1D][max_D1D];
-      MFEM_FOREACH_THREAD(d1, x, D1D)
-      {
-         MFEM_FOREACH_THREAD(d2, y, D1D)
-         {
-            u0[tidz][d1][d2] = x(d1, d2, 0, f);
-            u1[tidz][d1][d2] = x(d1, d2, 1, f);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_SHARED real_t Bu0[NBZ][max_Q1D][max_D1D];
-      MFEM_SHARED real_t Bu1[NBZ][max_Q1D][max_D1D];
-      MFEM_FOREACH_THREAD(q1, x, Q1D)
-      {
-         MFEM_FOREACH_THREAD(d2, y, D1D)
-         {
-            real_t Bu0_ = 0.0;
-            real_t Bu1_ = 0.0;
-            for (int d1 = 0; d1 < D1D; ++d1)
-            {
-               const real_t b = B(q1, d1);
-               Bu0_ += b * u0[tidz][d1][d2];
-               Bu1_ += b * u1[tidz][d1][d2];
-            }
-            Bu0[tidz][q1][d2] = Bu0_;
-            Bu1[tidz][q1][d2] = Bu1_;
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_SHARED real_t BBu0[NBZ][max_Q1D][max_Q1D];
-      MFEM_SHARED real_t BBu1[NBZ][max_Q1D][max_Q1D];
-      MFEM_FOREACH_THREAD(q1, x, Q1D)
-      {
-         MFEM_FOREACH_THREAD(q2, y, Q1D)
-         {
-            real_t BBu0_ = 0.0;
-            real_t BBu1_ = 0.0;
-            for (int d2 = 0; d2 < D1D; ++d2)
-            {
-               const real_t b = B(q2, d2);
-               BBu0_ += b * Bu0[tidz][q1][d2];
-               BBu1_ += b * Bu1[tidz][q1][d2];
-            }
-            BBu0[tidz][q1][q2] = BBu0_;
-            BBu1[tidz][q1][q2] = BBu1_;
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_SHARED real_t DBBu[NBZ][max_Q1D][max_Q1D];
-      MFEM_FOREACH_THREAD(q1, x, Q1D)
-      {
-         MFEM_FOREACH_THREAD(q2, y, Q1D)
-         {
-            DBBu[tidz][q1][q2] = op(q1, q2, 0, 0, f) * BBu0[tidz][q1][q2] +
-                                 op(q1, q2, 1, 0, f) * BBu1[tidz][q1][q2];
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_SHARED real_t BDBBu[NBZ][max_Q1D][max_D1D];
-      MFEM_FOREACH_THREAD(q1, x, Q1D)
-      {
-         MFEM_FOREACH_THREAD(d2, y, D1D)
-         {
-            real_t BDBBu_ = 0.0;
-            for (int q2 = 0; q2 < Q1D; ++q2)
-            {
-               const real_t b = Bt(d2, q2);
-               BDBBu_ += b * DBBu[tidz][q1][q2];
-            }
-            BDBBu[tidz][q1][d2] = BDBBu_;
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(d1, x, D1D)
-      {
-         MFEM_FOREACH_THREAD(d2, y, D1D)
-         {
-            real_t BBDBBu_ = 0.0;
-            for (int q1 = 0; q1 < Q1D; ++q1)
-            {
-               const real_t b = Bt(d1, q1);
-               BBDBBu_ += b * BDBBu[tidz][q1][d2];
-            }
-            y(d1, d2, 0, f) += BBDBBu_;
-            y(d1, d2, 1, f) += -BBDBBu_;
-         }
-      }
-   });
-}
-
-// PA DGTrace Apply 2D kernel for Gauss-Lobatto/Bernstein
-template <int T_D1D = 0, int T_Q1D = 0>
-static void PADGTraceApplyTranspose2D(const int NF, const Array<real_t> &b,
-                                      const Array<real_t> &bt,
-                                      const Vector &op_, const Vector &x_,
-                                      Vector &y_, const int d1d = 0,
-                                      const int q1d = 0)
-{
-   const int VDIM = 1;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto op = Reshape(op_.Read(), Q1D, 2, 2, NF);
-   auto x = Reshape(x_.Read(), D1D, VDIM, 2, NF);
-   auto y = Reshape(y_.ReadWrite(), D1D, VDIM, 2, NF);
-
-   mfem::forall(NF, [=] MFEM_HOST_DEVICE(int f)
-   {
-      const int VDIM = 1;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      real_t u0[max_D1D][VDIM];
-      real_t u1[max_D1D][VDIM];
-      for (int d = 0; d < D1D; d++)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            u0[d][c] = x(d, c, 0, f);
-            u1[d][c] = x(d, c, 1, f);
-         }
-      }
-      real_t Bu0[max_Q1D][VDIM];
-      real_t Bu1[max_Q1D][VDIM];
-      for (int q = 0; q < Q1D; ++q)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            Bu0[q][c] = 0.0;
-            Bu1[q][c] = 0.0;
-         }
-         for (int d = 0; d < D1D; ++d)
-         {
-            const real_t b = B(q, d);
-            for (int c = 0; c < VDIM; c++)
-            {
-               Bu0[q][c] += b * u0[d][c];
-               Bu1[q][c] += b * u1[d][c];
-            }
-         }
-      }
-      real_t DBu0[max_Q1D][VDIM];
-      real_t DBu1[max_Q1D][VDIM];
-      for (int q = 0; q < Q1D; ++q)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            DBu0[q][c] =
-               op(q, 0, 0, f) * Bu0[q][c] + op(q, 0, 1, f) * Bu1[q][c];
-            DBu1[q][c] =
-               op(q, 1, 0, f) * Bu0[q][c] + op(q, 1, 1, f) * Bu1[q][c];
-         }
-      }
-      real_t BDBu0[max_D1D][VDIM];
-      real_t BDBu1[max_D1D][VDIM];
-      for (int d = 0; d < D1D; ++d)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            BDBu0[d][c] = 0.0;
-            BDBu1[d][c] = 0.0;
-         }
-         for (int q = 0; q < Q1D; ++q)
-         {
-            const real_t b = Bt(d, q);
-            for (int c = 0; c < VDIM; c++)
-            {
-               BDBu0[d][c] += b * DBu0[q][c];
-               BDBu1[d][c] += b * DBu1[q][c];
-            }
-         }
-         for (int c = 0; c < VDIM; c++)
-         {
-            y(d, c, 0, f) += BDBu0[d][c];
-            y(d, c, 1, f) += BDBu1[d][c];
-         }
-      }
-   });
-}
-
-// PA DGTrace Apply Transpose 3D kernel for Gauss-Lobatto/Bernstein
-template <int T_D1D = 0, int T_Q1D = 0>
-static void PADGTraceApplyTranspose3D(const int NF, const Array<real_t> &b,
-                                      const Array<real_t> &bt,
-                                      const Vector &op_, const Vector &x_,
-                                      Vector &y_, const int d1d = 0,
-                                      const int q1d = 0)
-{
-   const int VDIM = 1;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, 2, 2, NF);
-   auto x = Reshape(x_.Read(), D1D, D1D, VDIM, 2, NF);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, VDIM, 2, NF);
-
-   mfem::forall(NF, [=] MFEM_HOST_DEVICE(int f)
-   {
-      const int VDIM = 1;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      real_t u0[max_D1D][max_D1D][VDIM];
-      real_t u1[max_D1D][max_D1D][VDIM];
-      for (int d1 = 0; d1 < D1D; d1++)
-      {
-         for (int d2 = 0; d2 < D1D; d2++)
-         {
-            for (int c = 0; c < VDIM; c++)
-            {
-               u0[d1][d2][c] = x(d1, d2, c, 0, f);
-               u1[d1][d2][c] = x(d1, d2, c, 1, f);
-            }
-         }
-      }
-      real_t Bu0[max_Q1D][max_D1D][VDIM];
-      real_t Bu1[max_Q1D][max_D1D][VDIM];
-      for (int q1 = 0; q1 < Q1D; ++q1)
-      {
-         for (int d2 = 0; d2 < D1D; ++d2)
-         {
-            for (int c = 0; c < VDIM; c++)
-            {
-               Bu0[q1][d2][c] = 0.0;
-               Bu1[q1][d2][c] = 0.0;
-            }
-            for (int d1 = 0; d1 < D1D; ++d1)
-            {
-               const real_t b = B(q1, d1);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  Bu0[q1][d2][c] += b * u0[d1][d2][c];
-                  Bu1[q1][d2][c] += b * u1[d1][d2][c];
-               }
-            }
-         }
-      }
-      real_t BBu0[max_Q1D][max_Q1D][VDIM];
-      real_t BBu1[max_Q1D][max_Q1D][VDIM];
-      for (int q1 = 0; q1 < Q1D; ++q1)
-      {
-         for (int q2 = 0; q2 < Q1D; ++q2)
-         {
-            for (int c = 0; c < VDIM; c++)
-            {
-               BBu0[q1][q2][c] = 0.0;
-               BBu1[q1][q2][c] = 0.0;
-            }
-            for (int d2 = 0; d2 < D1D; ++d2)
-            {
-               const real_t b = B(q2, d2);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  BBu0[q1][q2][c] += b * Bu0[q1][d2][c];
-                  BBu1[q1][q2][c] += b * Bu1[q1][d2][c];
-               }
-            }
-         }
-      }
-      real_t DBu0[max_Q1D][max_Q1D][VDIM];
-      real_t DBu1[max_Q1D][max_Q1D][VDIM];
-      for (int q1 = 0; q1 < Q1D; ++q1)
-      {
-         for (int q2 = 0; q2 < Q1D; ++q2)
-         {
-            const real_t D00 = op(q1, q2, 0, 0, f);
-            const real_t D01 = op(q1, q2, 0, 1, f);
-            const real_t D10 = op(q1, q2, 1, 0, f);
-            const real_t D11 = op(q1, q2, 1, 1, f);
-            for (int c = 0; c < VDIM; c++)
-            {
-               DBu0[q1][q2][c] = D00 * BBu0[q1][q2][c] + D01 * BBu1[q1][q2][c];
-               DBu1[q1][q2][c] = D10 * BBu0[q1][q2][c] + D11 * BBu1[q1][q2][c];
-            }
-         }
-      }
-      real_t BDBu0[max_D1D][max_Q1D][VDIM];
-      real_t BDBu1[max_D1D][max_Q1D][VDIM];
-      for (int d1 = 0; d1 < D1D; ++d1)
-      {
-         for (int q2 = 0; q2 < Q1D; ++q2)
-         {
-            for (int c = 0; c < VDIM; c++)
-            {
-               BDBu0[d1][q2][c] = 0.0;
-               BDBu1[d1][q2][c] = 0.0;
-            }
-            for (int q1 = 0; q1 < Q1D; ++q1)
-            {
-               const real_t b = Bt(d1, q1);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  BDBu0[d1][q2][c] += b * DBu0[q1][q2][c];
-                  BDBu1[d1][q2][c] += b * DBu1[q1][q2][c];
-               }
-            }
-         }
-      }
-      real_t BBDBu0[max_D1D][max_D1D][VDIM];
-      real_t BBDBu1[max_D1D][max_D1D][VDIM];
-      for (int d1 = 0; d1 < D1D; ++d1)
-      {
-         for (int d2 = 0; d2 < D1D; ++d2)
-         {
-            for (int c = 0; c < VDIM; c++)
-            {
-               BBDBu0[d1][d2][c] = 0.0;
-               BBDBu1[d1][d2][c] = 0.0;
-            }
-            for (int q2 = 0; q2 < Q1D; ++q2)
-            {
-               const real_t b = Bt(d2, q2);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  BBDBu0[d1][d2][c] += b * BDBu0[d1][q2][c];
-                  BBDBu1[d1][d2][c] += b * BDBu1[d1][q2][c];
-               }
-            }
-            for (int c = 0; c < VDIM; c++)
-            {
-               y(d1, d2, c, 0, f) += BBDBu0[d1][d2][c];
-               y(d1, d2, c, 1, f) += BBDBu1[d1][d2][c];
-            }
-         }
-      }
-   });
-}
-
-// Optimized PA DGTrace Apply Transpose 3D kernel for Gauss-Lobatto/Bernstein
-template <int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
-static void SmemPADGTraceApplyTranspose3D(const int NF, const Array<real_t> &b,
-                                          const Array<real_t> &bt,
-                                          const Vector &op_, const Vector &x_,
-                                          Vector &y_, const int d1d = 0,
-                                          const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, 2, 2, NF);
-   auto x = Reshape(x_.Read(), D1D, D1D, 2, NF);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, 2, NF);
-
-   mfem::forall_2D_batch(NF, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE(int f)
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      // the following variables are evaluated at compile time
-      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      MFEM_SHARED real_t u0[NBZ][max_D1D][max_D1D];
-      MFEM_SHARED real_t u1[NBZ][max_D1D][max_D1D];
-      MFEM_FOREACH_THREAD(d1, x, D1D)
-      {
-         MFEM_FOREACH_THREAD(d2, y, D1D)
-         {
-            u0[tidz][d1][d2] = x(d1, d2, 0, f);
-            u1[tidz][d1][d2] = x(d1, d2, 1, f);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_SHARED real_t Bu0[NBZ][max_Q1D][max_D1D];
-      MFEM_SHARED real_t Bu1[NBZ][max_Q1D][max_D1D];
-      MFEM_FOREACH_THREAD(q1, x, Q1D)
-      {
-         MFEM_FOREACH_THREAD(d2, y, D1D)
-         {
-            real_t Bu0_ = 0.0;
-            real_t Bu1_ = 0.0;
-            for (int d1 = 0; d1 < D1D; ++d1)
-            {
-               const real_t b = B(q1, d1);
-               Bu0_ += b * u0[tidz][d1][d2];
-               Bu1_ += b * u1[tidz][d1][d2];
-            }
-            Bu0[tidz][q1][d2] = Bu0_;
-            Bu1[tidz][q1][d2] = Bu1_;
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_SHARED real_t BBu0[NBZ][max_Q1D][max_Q1D];
-      MFEM_SHARED real_t BBu1[NBZ][max_Q1D][max_Q1D];
-      MFEM_FOREACH_THREAD(q1, x, Q1D)
-      {
-         MFEM_FOREACH_THREAD(q2, y, Q1D)
-         {
-            real_t BBu0_ = 0.0;
-            real_t BBu1_ = 0.0;
-            for (int d2 = 0; d2 < D1D; ++d2)
-            {
-               const real_t b = B(q2, d2);
-               BBu0_ += b * Bu0[tidz][q1][d2];
-               BBu1_ += b * Bu1[tidz][q1][d2];
-            }
-            BBu0[tidz][q1][q2] = BBu0_;
-            BBu1[tidz][q1][q2] = BBu1_;
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_SHARED real_t DBBu0[NBZ][max_Q1D][max_Q1D];
-      MFEM_SHARED real_t DBBu1[NBZ][max_Q1D][max_Q1D];
-      MFEM_FOREACH_THREAD(q1, x, Q1D)
-      {
-         MFEM_FOREACH_THREAD(q2, y, Q1D)
-         {
-            const real_t D00 = op(q1, q2, 0, 0, f);
-            const real_t D01 = op(q1, q2, 0, 1, f);
-            const real_t D10 = op(q1, q2, 1, 0, f);
-            const real_t D11 = op(q1, q2, 1, 1, f);
-            const real_t u0q = BBu0[tidz][q1][q2];
-            const real_t u1q = BBu1[tidz][q1][q2];
-            DBBu0[tidz][q1][q2] = D00 * u0q + D01 * u1q;
-            DBBu1[tidz][q1][q2] = D10 * u0q + D11 * u1q;
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_SHARED real_t BDBBu0[NBZ][max_Q1D][max_D1D];
-      MFEM_SHARED real_t BDBBu1[NBZ][max_Q1D][max_D1D];
-      MFEM_FOREACH_THREAD(q1, x, Q1D)
-      {
-         MFEM_FOREACH_THREAD(d2, y, D1D)
-         {
-            real_t BDBBu0_ = 0.0;
-            real_t BDBBu1_ = 0.0;
-            for (int q2 = 0; q2 < Q1D; ++q2)
-            {
-               const real_t b = Bt(d2, q2);
-               BDBBu0_ += b * DBBu0[tidz][q1][q2];
-               BDBBu1_ += b * DBBu1[tidz][q1][q2];
-            }
-            BDBBu0[tidz][q1][d2] = BDBBu0_;
-            BDBBu1[tidz][q1][d2] = BDBBu1_;
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(d1, x, D1D)
-      {
-         MFEM_FOREACH_THREAD(d2, y, D1D)
-         {
-            real_t BBDBBu0_ = 0.0;
-            real_t BBDBBu1_ = 0.0;
-            for (int q1 = 0; q1 < Q1D; ++q1)
-            {
-               const real_t b = Bt(d1, q1);
-               BBDBBu0_ += b * BDBBu0[tidz][q1][d2];
-               BBDBBu1_ += b * BDBBu1[tidz][q1][d2];
-            }
-            y(d1, d2, 0, f) += BBDBBu0_;
-            y(d1, d2, 1, f) += BBDBBu1_;
-         }
-      }
-   });
-}
-
-} // namespace internal
-
-template <int DIM, int D1D, int Q1D>
-DGTraceIntegrator::ApplyKernelType DGTraceIntegrator::ApplyPAKernels::Kernel()
-{
-   if constexpr (DIM == 2)
-   {
-      return internal::PADGTraceApply2D<D1D, Q1D>;
-   }
-   else if constexpr (DIM == 3)
-   {
-      if constexpr (D1D == 3 || D1D == 4)
-      {
-         return internal::SmemPADGTraceApply3D<D1D, Q1D, 2>;
-      }
-      else
-      {
-         return internal::SmemPADGTraceApply3D<D1D, Q1D>;
-      }
-   }
-   MFEM_ABORT("");
-}
-
-template <int DIM, int D1D, int Q1D>
-DGTraceIntegrator::ApplyKernelType DGTraceIntegrator::ApplyPATKernels::Kernel()
-{
-   if constexpr (DIM == 2)
-   {
-      return internal::PADGTraceApplyTranspose2D<D1D, Q1D>;
-   }
-   else if constexpr (DIM == 3)
-   {
-      return internal::SmemPADGTraceApplyTranspose3D<D1D, Q1D>;
-   }
-   MFEM_ABORT("");
-}
-} // namespace mfem
-
-/// \endcond DO_NOT_DOCUMENT
-#endif

fem/integ/bilininteg_diffusion_kernels.cpp

@@ -19,8 +19,6 @@ namespace mfem
 DiffusionIntegrator::Kernels::Kernels()
 {
    // 2D
-   // Q = P+1
-   DiffusionIntegrator::AddSpecialization<2,1,1>();
    DiffusionIntegrator::AddSpecialization<2,2,2>();
    DiffusionIntegrator::AddSpecialization<2,3,3>();
    DiffusionIntegrator::AddSpecialization<2,4,4>();
@@ -29,39 +27,17 @@ DiffusionIntegrator::Kernels::Kernels()
    DiffusionIntegrator::AddSpecialization<2,7,7>();
    DiffusionIntegrator::AddSpecialization<2,8,8>();
    DiffusionIntegrator::AddSpecialization<2,9,9>();
-   // Q = P+2
-   DiffusionIntegrator::AddSpecialization<2,1,2>();
-   DiffusionIntegrator::AddSpecialization<2,2,3>();
-   DiffusionIntegrator::AddSpecialization<2,3,4>();
-   DiffusionIntegrator::AddSpecialization<2,4,5>();
-   DiffusionIntegrator::AddSpecialization<2,5,6>();
-   DiffusionIntegrator::AddSpecialization<2,6,7>();
-   DiffusionIntegrator::AddSpecialization<2,7,8>();
-   DiffusionIntegrator::AddSpecialization<2,8,9>();
-   DiffusionIntegrator::AddSpecialization<2,9,10>();
-   // others
    // 3D
-   // Q = P+1
-   DiffusionIntegrator::AddSpecialization<3,1,1>();
    DiffusionIntegrator::AddSpecialization<3,2,2>();
-   DiffusionIntegrator::AddSpecialization<3,3,3>();
-   DiffusionIntegrator::AddSpecialization<3,4,4>();
-   DiffusionIntegrator::AddSpecialization<3,5,5>();
-   DiffusionIntegrator::AddSpecialization<3,6,6>();
-   DiffusionIntegrator::AddSpecialization<3,7,7>();
-   DiffusionIntegrator::AddSpecialization<3,8,8>();
-   // Q = P+2
-   DiffusionIntegrator::AddSpecialization<3,1,2>();
    DiffusionIntegrator::AddSpecialization<3,2,3>();
    DiffusionIntegrator::AddSpecialization<3,3,4>();
    DiffusionIntegrator::AddSpecialization<3,4,5>();
+   DiffusionIntegrator::AddSpecialization<3,4,6>();
    DiffusionIntegrator::AddSpecialization<3,5,6>();
+   DiffusionIntegrator::AddSpecialization<3,5,8>();
    DiffusionIntegrator::AddSpecialization<3,6,7>();
    DiffusionIntegrator::AddSpecialization<3,7,8>();
    DiffusionIntegrator::AddSpecialization<3,8,9>();
-   // others
-   DiffusionIntegrator::AddSpecialization<3,4,6>();
-   DiffusionIntegrator::AddSpecialization<3,5,8>();
 }
 
 namespace internal

fem/integ/bilininteg_diffusion_kernels.hpp

@@ -672,12 +672,12 @@ inline void SmemPADiffusionApply2D(const int NE,
       real_t (*Gt)[MQ1] = (real_t (*)[MQ1]) (sBG+1);
       MFEM_SHARED real_t Xz[NBZ][MD1][MD1];
       MFEM_SHARED real_t GD[2][NBZ][MD1][MQ1];
-      MFEM_SHARED real_t GQ[2][NBZ][MQ1][MQ1];
+      MFEM_SHARED real_t GQ[2][NBZ][MD1][MQ1];
       real_t (*X)[MD1] = (real_t (*)[MD1])(Xz + tidz);
-      real_t (*DQ0)[MQ1] = (real_t (*)[MQ1])(GD[0] + tidz);
-      real_t (*DQ1)[MQ1] = (real_t (*)[MQ1])(GD[1] + tidz);
-      real_t (*QQ0)[MQ1] = (real_t (*)[MQ1])(GQ[0] + tidz);
-      real_t (*QQ1)[MQ1] = (real_t (*)[MQ1])(GQ[1] + tidz);
+      real_t (*DQ0)[MD1] = (real_t (*)[MD1])(GD[0] + tidz);
+      real_t (*DQ1)[MD1] = (real_t (*)[MD1])(GD[1] + tidz);
+      real_t (*QQ0)[MD1] = (real_t (*)[MD1])(GQ[0] + tidz);
+      real_t (*QQ1)[MD1] = (real_t (*)[MD1])(GQ[1] + tidz);
       MFEM_FOREACH_THREAD(dy,y,D1D)
       {
          MFEM_FOREACH_THREAD(dx,x,D1D)
@@ -1214,20 +1214,23 @@ inline void SmemPADiffusionApply3D(const int NE,
 
 namespace
 {
-using ApplyKernelType = DiffusionIntegrator::ApplyKernelType;
-using DiagonalKernelType = DiffusionIntegrator::DiagonalKernelType;
+using DiffusionApplyKernelType =
+   DiffusionIntegrator::DiffusionApplyKernelType;
+
+using DiffusionDiagonalKernelType =
+   DiffusionIntegrator::DiffusionDiagonalKernelType;
 }
 
 template<int DIM, int T_D1D, int T_Q1D>
-ApplyKernelType DiffusionIntegrator::ApplyPAKernels::Kernel()
+DiffusionApplyKernelType DiffusionIntegrator::DiffusionApplyPAKernel::Kernel()
 {
-   if constexpr (DIM == 2) { return internal::SmemPADiffusionApply2D<T_D1D,T_Q1D>; }
-   else if constexpr (DIM == 3) { return internal::SmemPADiffusionApply3D<T_D1D, T_Q1D>; }
-   MFEM_ABORT("");
+   if (DIM == 2) { return internal::SmemPADiffusionApply2D<T_D1D,T_Q1D>; }
+   else if (DIM == 3) { return internal::SmemPADiffusionApply3D<T_D1D, T_Q1D>; }
+   else { MFEM_ABORT(""); }
 }
 
-inline
-ApplyKernelType DiffusionIntegrator::ApplyPAKernels::Fallback(int DIM, int, int)
+inline DiffusionApplyKernelType
+DiffusionIntegrator::DiffusionApplyPAKernel::Fallback(int DIM, int, int)
 {
    if (DIM == 2) { return internal::PADiffusionApply2D; }
    else if (DIM == 3) { return internal::PADiffusionApply3D; }
@@ -1235,15 +1238,16 @@ ApplyKernelType DiffusionIntegrator::ApplyPAKernels::Fallback(int DIM, int, int)
 }
 
 template<int DIM, int D1D, int Q1D>
-DiagonalKernelType DiffusionIntegrator::DiagonalPAKernels::Kernel()
+DiffusionDiagonalKernelType
+DiffusionIntegrator::DiffusionDiagonalPAKernel::Kernel()
 {
-   if constexpr (DIM == 2) { return internal::SmemPADiffusionDiagonal2D<D1D,Q1D>; }
-   else if constexpr (DIM == 3) { return internal::SmemPADiffusionDiagonal3D<D1D, Q1D>; }
-   MFEM_ABORT("");
+   if (DIM == 2) { return internal::SmemPADiffusionDiagonal2D<D1D,Q1D>; }
+   else if (DIM == 3) { return internal::SmemPADiffusionDiagonal3D<D1D, Q1D>; }
+   else { MFEM_ABORT(""); }
 }
 
-inline DiagonalKernelType
-DiffusionIntegrator::DiagonalPAKernels::Fallback(int DIM, int, int)
+inline DiffusionDiagonalKernelType
+DiffusionIntegrator::DiffusionDiagonalPAKernel::Fallback(int DIM, int, int)
 {
    if (DIM == 2) { return internal::PADiffusionDiagonal2D; }
    else if (DIM == 3) { return internal::PADiffusionDiagonal3D; }

fem/integ/bilininteg_diffusion_pa.cpp

@@ -16,6 +16,14 @@
 #include "../ceed/integrators/diffusion/diffusion.hpp"
 #include "bilininteg_diffusion_kernels.hpp"
 
+#if defined(__has_include) && __has_include("general/nvtx.hpp") && !defined(_WIN32)
+#undef NVTX_COLOR
+#define NVTX_COLOR ::nvtx::kNvidia
+#include "general/nvtx.hpp"
+#else
+#define dbg(...)
+#endif
+
 namespace mfem
 {
 
@@ -31,8 +39,8 @@ void DiffusionIntegrator::AssembleDiagonalPA(Vector &diag)
       const Array<real_t> &B = maps->B;
       const Array<real_t> &G = maps->G;
       const Vector &Dv = pa_data;
-      DiagonalPAKernels::Run(dim, dofs1D, quad1D, ne, symmetric, B, G, Dv,
-                             diag, dofs1D, quad1D);
+      DiffusionDiagonalPAKernel::Run(dim, dofs1D, quad1D, ne, symmetric, B, G, Dv,
+                                     diag, dofs1D, quad1D);
    }
 }
 
@@ -67,9 +75,9 @@ void DiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
          MFEM_ABORT("OCCA PADiffusionApply unknown kernel!");
       }
 #endif // MFEM_USE_OCCA
-
-      ApplyPAKernels::Run(dim, dofs1D, quad1D, ne, symmetric, B, G, Bt,
-                          Gt, Dv, x, y, dofs1D, quad1D);
+      db1("[DiffusionApplyPAKernel] D1D:{} Q1D:{}", dofs1D, quad1D);
+      DiffusionApplyPAKernel::Run(dim, dofs1D, quad1D, ne, symmetric, B, G, Bt,
+                                  Gt, Dv, x, y, dofs1D, quad1D);
    }
 }
 
@@ -174,9 +182,9 @@ void DiffusionIntegrator::AddAbsMultPA(const Vector &x, Vector &y) const
    abs_pa_data.Abs();
    auto abs_maps = maps->Abs();
 
-   ApplyPAKernels::Run(dim, dofs1D, quad1D, ne, symmetric,
-                       abs_maps.B, abs_maps.G, abs_maps.Bt, abs_maps.Gt,
-                       abs_pa_data, x, y, dofs1D, quad1D);
+   DiffusionApplyPAKernel::Run(dim, dofs1D, quad1D, ne, symmetric,
+                               abs_maps.B, abs_maps.G, abs_maps.Bt, abs_maps.Gt,
+                               abs_pa_data, x, y, dofs1D, quad1D);
 }
 
 void DiffusionIntegrator::AddAbsMultTransposePA(const Vector &x,

fem/integ/bilininteg_hcurl_kernels.cpp

@@ -599,11 +599,13 @@ void PACurlCurlSetup3D(const int Q1D,
    });
 }
 
-void PACurlCurlAssembleDiagonal2D(const int D1D, const int Q1D, const bool,
-                                  const int NE, const Array<real_t> &bo,
-                                  const Array<real_t> &, const Array<real_t> &,
+void PACurlCurlAssembleDiagonal2D(const int D1D,
+                                  const int Q1D,
+                                  const int NE,
+                                  const Array<real_t> &bo,
                                   const Array<real_t> &gc,
-                                  const Vector &pa_data, Vector &diag)
+                                  const Vector &pa_data,
+                                  Vector &diag)
 {
    auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
    auto Gc = Reshape(gc.Read(), Q1D, D1D);
@@ -651,11 +653,16 @@ void PACurlCurlAssembleDiagonal2D(const int D1D, const int Q1D, const bool,
    }); // end of element loop
 }
 
-void PACurlCurlApply2D(const int D1D, const int Q1D, const bool, const int NE,
-                       const Array<real_t> &bo, const Array<real_t> &,
-                       const Array<real_t> &bot, const Array<real_t> &,
-                       const Array<real_t> &gc, const Array<real_t> &gct,
-                       const Vector &pa_data, const Vector &x, Vector &y,
+void PACurlCurlApply2D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const Array<real_t> &bo,
+                       const Array<real_t> &bot,
+                       const Array<real_t> &gc,
+                       const Array<real_t> &gct,
+                       const Vector &pa_data,
+                       const Vector &x,
+                       Vector &y,
                        const bool useAbs)
 {
 

fem/integ/bilininteg_hcurl_kernels.hpp

@@ -24,7 +24,7 @@
 
 namespace mfem
 {
-/// \cond DO_NOT_DOCUMENT
+
 namespace internal
 {
 
@@ -426,11 +426,8 @@ void PACurlCurlSetup3D(const int Q1D,
 // PA H(curl) curl-curl Diagonal 2D kernel
 void PACurlCurlAssembleDiagonal2D(const int D1D,
                                   const int Q1D,
-                                  const bool symmetric, // unused
                                   const int NE,
                                   const Array<real_t> &bo,
-                                  const Array<real_t> &bc, // unused
-                                  const Array<real_t> &go, // unused
                                   const Array<real_t> &gc,
                                   const Vector &pa_data,
                                   Vector &diag);
@@ -834,12 +831,9 @@ inline void SmemPACurlCurlAssembleDiagonal3D(const int d1d,
 // PA H(curl) curl-curl Apply/AbsApply 2D kernel
 void PACurlCurlApply2D(const int D1D,
                        const int Q1D,
-                       const bool symmetric, // unused
                        const int NE,
                        const Array<real_t> &bo,
-                       const Array<real_t> &bc, // unused
                        const Array<real_t> &bot,
-                       const Array<real_t> &bct, // unused
                        const Array<real_t> &gc,
                        const Array<real_t> &gct,
                        const Vector &pa_data,
@@ -3164,49 +3158,6 @@ inline void SmemPAHcurlL2ApplyTranspose3D(const int d1d,
 
 } // namespace internal
 
-template<int DIM, int T_D1D, int T_Q1D>
-CurlCurlIntegrator::ApplyKernelType CurlCurlIntegrator::ApplyPAKernels::Kernel()
-{
-   if constexpr (DIM == 2)
-   {
-      return internal::PACurlCurlApply2D;
-   }
-   else if constexpr (DIM == 3)
-   {
-      if (Device::Allows(Backend::DEVICE_MASK))
-      {
-         return internal::SmemPACurlCurlApply3D<T_D1D, T_Q1D>;
-      }
-      else
-      {
-         return internal::PACurlCurlApply3D;
-      }
-   }
-   MFEM_ABORT("");
-}
-
-template <int DIM, int T_D1D, int T_Q1D>
-CurlCurlIntegrator::DiagonalKernelType
-CurlCurlIntegrator::DiagonalPAKernels::Kernel()
-{
-   if constexpr (DIM == 2)
-   {
-      return internal::PACurlCurlAssembleDiagonal2D;
-   }
-   else if constexpr (DIM == 3)
-   {
-      if (Device::Allows(Backend::DEVICE_MASK))
-      {
-         return internal::SmemPACurlCurlAssembleDiagonal3D<T_D1D, T_Q1D>;
-      }
-      else
-      {
-         return internal::PACurlCurlAssembleDiagonal3D;
-      }
-   }
-   MFEM_ABORT("");
-}
-/// \endcond DO_NOT_DOCUMENT
 } // namespace mfem
 
 #endif

fem/integ/bilininteg_hcurlhdiv_kernels.hpp

@@ -19,7 +19,6 @@
 #include "../../linalg/vector.hpp"
 #include "../bilininteg.hpp"
 
-/// \cond DO_NOT_DOCUMENT
 namespace mfem
 {
 
@@ -820,6 +819,4 @@ inline void PAHcurlHdivApplyTranspose3D(const int d1d,
 
 } // namespace mfem
 
-/// \endcond DO_NOT_DOCUMENT
-
 #endif

fem/integ/bilininteg_mass_kernels.cpp

@@ -17,8 +17,6 @@ namespace mfem
 MassIntegrator::Kernels::Kernels()
 {
    // 2D
-   // Q=P+1
-   MassIntegrator::AddSpecialization<2,1,1>();
    MassIntegrator::AddSpecialization<2,2,2>();
    MassIntegrator::AddSpecialization<2,3,3>();
    MassIntegrator::AddSpecialization<2,4,4>();
@@ -27,45 +25,17 @@ MassIntegrator::Kernels::Kernels()
    MassIntegrator::AddSpecialization<2,7,7>();
    MassIntegrator::AddSpecialization<2,8,8>();
    MassIntegrator::AddSpecialization<2,9,9>();
-   // Q=P+2
-   MassIntegrator::AddSpecialization<2,1,2>();
-   MassIntegrator::AddSpecialization<2,2,3>();
-   MassIntegrator::AddSpecialization<2,3,4>();
-   MassIntegrator::AddSpecialization<2,4,5>();
-   MassIntegrator::AddSpecialization<2,5,6>();
-   MassIntegrator::AddSpecialization<2,6,7>();
-   MassIntegrator::AddSpecialization<2,7,8>();
-   MassIntegrator::AddSpecialization<2,8,9>();
-   MassIntegrator::AddSpecialization<2,9,10>();
-   // others
-   MassIntegrator::AddSpecialization<2,2,4>();
-   MassIntegrator::AddSpecialization<2,3,6>();
-   MassIntegrator::AddSpecialization<2,4,6>();
    // 3D
-   // Q=P+1
-   MassIntegrator::AddSpecialization<3,1,1>();
    MassIntegrator::AddSpecialization<3,2,2>();
-   MassIntegrator::AddSpecialization<3,3,3>();
-   MassIntegrator::AddSpecialization<3,4,4>();
-   MassIntegrator::AddSpecialization<3,5,5>();
-   MassIntegrator::AddSpecialization<3,6,6>();
-   MassIntegrator::AddSpecialization<3,7,7>();
-   MassIntegrator::AddSpecialization<3,8,8>();
-   MassIntegrator::AddSpecialization<3,9,9>();
-   // Q=P+2
-   MassIntegrator::AddSpecialization<3,1,2>();
    MassIntegrator::AddSpecialization<3,2,3>();
    MassIntegrator::AddSpecialization<3,3,4>();
-   MassIntegrator::AddSpecialization<3,3,6>();
    MassIntegrator::AddSpecialization<3,4,5>();
+   MassIntegrator::AddSpecialization<3,4,6>();
    MassIntegrator::AddSpecialization<3,5,6>();
+   MassIntegrator::AddSpecialization<3,5,8>();
    MassIntegrator::AddSpecialization<3,6,7>();
    MassIntegrator::AddSpecialization<3,7,8>();
    MassIntegrator::AddSpecialization<3,8,9>();
-   // others
-   MassIntegrator::AddSpecialization<3,2,4>();
-   MassIntegrator::AddSpecialization<3,4,6>();
-   MassIntegrator::AddSpecialization<3,5,8>();
 }
 
 namespace internal

fem/integ/bilininteg_mass_kernels.hpp

@@ -1392,10 +1392,10 @@ using DiagonalKernelType = MassIntegrator::DiagonalKernelType;
 template<int DIM, int T_D1D, int T_Q1D>
 ApplyKernelType MassIntegrator::ApplyPAKernels::Kernel()
 {
-   if constexpr (DIM == 1) { return internal::PAMassApply1D; }
-   else if constexpr (DIM == 2) { return internal::SmemPAMassApply2D<T_D1D,T_Q1D>; }
-   else if constexpr (DIM == 3) { return internal::SmemPAMassApply3D<T_D1D, T_Q1D>; }
-   MFEM_ABORT("");
+   if (DIM == 1) { return internal::PAMassApply1D; }
+   else if (DIM == 2) { return internal::SmemPAMassApply2D<T_D1D,T_Q1D>; }
+   else if (DIM == 3) { return internal::SmemPAMassApply3D<T_D1D, T_Q1D>; }
+   else { MFEM_ABORT(""); }
 }
 
 inline ApplyKernelType MassIntegrator::ApplyPAKernels::Fallback(
@@ -1410,10 +1410,10 @@ inline ApplyKernelType MassIntegrator::ApplyPAKernels::Fallback(
 template<int DIM, int T_D1D, int T_Q1D>
 DiagonalKernelType MassIntegrator::DiagonalPAKernels::Kernel()
 {
-   if constexpr (DIM == 1) { return internal::PAMassAssembleDiagonal1D; }
-   else if constexpr (DIM == 2) { return internal::SmemPAMassAssembleDiagonal2D<T_D1D,T_Q1D>; }
-   else if constexpr (DIM == 3) { return internal::SmemPAMassAssembleDiagonal3D<T_D1D, T_Q1D>; }
-   MFEM_ABORT("");
+   if (DIM == 1) { return internal::PAMassAssembleDiagonal1D; }
+   else if (DIM == 2) { return internal::SmemPAMassAssembleDiagonal2D<T_D1D,T_Q1D>; }
+   else if (DIM == 3) { return internal::SmemPAMassAssembleDiagonal3D<T_D1D, T_Q1D>; }
+   else { MFEM_ABORT(""); }
 }
 
 inline DiagonalKernelType MassIntegrator::DiagonalPAKernels::Fallback(

fem/integ/bilininteg_mass_pa.cpp

@@ -59,23 +59,26 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 
    QuadratureSpace qs(*mesh, *ir);
    CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
+
+   const int NE = ne;
+   const int Q1D = quad1D;
+   const int NQ = static_cast<int>(std::pow(Q1D, dim));
+   const bool const_c = coeff.Size() == 1;
+   const bool by_val = map_type == FiniteElement::VALUE;
+   const auto W = Reshape(ir->GetWeights().Read(), NQ);
+   const auto J = Reshape(geom->detJ.Read(), NQ, NE);
+   const auto C = const_c ? Reshape(coeff.Read(), 1, 1) :
+                  Reshape(coeff.Read(), NQ,NE);
+   auto v = Reshape(pa_data.Write(), NQ, NE);
+   mfem::forall_2D(NE, NQ, 1, [=] MFEM_HOST_DEVICE (int e)
    {
-      const int NE = ne;
-      const int NQ = nq;
-      const bool const_c = coeff.Size() == 1;
-      const bool by_val = map_type == FiniteElement::VALUE;
-      const auto W = Reshape(ir->GetWeights().Read(), NQ);
-      const auto J = Reshape(geom->detJ.Read(), NQ, NE);
-      const auto C =
-         const_c ? Reshape(coeff.Read(), 1, 1) : Reshape(coeff.Read(), NQ, NE);
-      auto v = Reshape(pa_data.Write(), NQ, NE);
-      mfem::forall(NQ, NE, [=] MFEM_HOST_DEVICE(int q, int e)
+      MFEM_FOREACH_THREAD(i, x, NQ)
       {
-         const real_t detJ = J(q, e);
-         const real_t coeff = const_c ? C(0, 0) : C(q, e);
-         v(q, e) = W(q) * coeff * (by_val ? detJ : 1.0 / detJ);
-      });
-   }
+         const real_t detJ = J(i,e);
+         const real_t coeff = const_c ? C(0,0) : C(i,e);
+         v(i,e) =  W(i) * coeff * (by_val ? detJ : 1.0/detJ);
+      }
+   });
 }
 
 void MassIntegrator::AssemblePABoundary(const FiniteElementSpace &fes)
@@ -106,22 +109,50 @@ void MassIntegrator::AssemblePABoundary(const FiniteElementSpace &fes)
    CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
 
    const int NE = ne;
-   const int NQ = nq;
+   const int Q1D = quad1D;
    const bool const_c = coeff.Size() == 1;
    const bool by_val = map_type == FiniteElement::VALUE;
+   if (dim==1)
    {
-      const auto W = Reshape(ir->GetWeights().Read(), NQ);
-      const auto J = Reshape(face_geom->detJ.Read(), NQ, NE);
-      const auto C = const_c ? Reshape(coeff.Read(), 1, 1)
-                     : Reshape(coeff.Read(), NQ, NE);
-      auto v = Reshape(pa_data.Write(), NQ, NE);
-      mfem::forall(NQ, NE, [=] MFEM_HOST_DEVICE(int q, int e)
+      const auto W = Reshape(ir->GetWeights().Read(), Q1D);
+      const auto J = Reshape(face_geom->detJ.Read(), Q1D, NE);
+      const auto C = const_c ? Reshape(coeff.Read(), 1, 1) :
+                     Reshape(coeff.Read(), Q1D, NE);
+      auto v = Reshape(pa_data.Write(), Q1D, NE);
+      mfem::forall_2D(NE, Q1D, 1, [=] MFEM_HOST_DEVICE (int e)
       {
-         const real_t detJ = J(q, e);
-         const real_t coeff = const_c ? C(0, 0) : C(q, e);
-         v(q, e) = W(q) * coeff * (by_val ? detJ : 1.0 / detJ);
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            const real_t detJ = J(qx,e);
+            const real_t coeff = const_c ? C(0,0) : C(qx,e);
+            v(qx,e) =  W(qx) * coeff * (by_val ? detJ : 1.0/detJ);
+         }
       });
    }
+   else if (dim==2)
+   {
+      const auto W = Reshape(ir->GetWeights().Read(), Q1D,Q1D);
+      const auto J = Reshape(face_geom->detJ.Read(), Q1D,Q1D,NE);
+      const auto C = const_c ? Reshape(coeff.Read(), 1,1,1) :
+                     Reshape(coeff.Read(), Q1D,Q1D,NE);
+      auto v = Reshape(pa_data.Write(), Q1D,Q1D, NE);
+      mfem::forall_2D(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               const real_t detJ = J(qx,qy,e);
+               const real_t coeff = const_c ? C(0,0,0) : C(qx,qy,e);
+               v(qx,qy,e) =  W(qx,qy) * coeff * (by_val ? detJ : 1.0/detJ);
+            }
+         }
+      });
+   }
+   else
+   {
+      MFEM_ABORT("Not supported.");
+   }
 }
 
 void MassIntegrator::AssembleDiagonalPA(Vector &diag)

fem/integ/bilininteg_vecdiffusion_kernels.hpp

@@ -1,355 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_BILININTEG_VECDIFFUSION_KERNELS_HPP
-#define MFEM_BILININTEG_VECDIFFUSION_KERNELS_HPP
-
-#include "../../general/forall.hpp"
-#include "../bilininteg.hpp"
-#include "../ceed/integrators/diffusion/diffusion.hpp"
-#include "../gridfunc.hpp"
-#include "../qfunction.hpp"
-/// \cond DO_NOT_DOCUMENT
-namespace mfem
-{
-
-namespace internal
-{
-
-// PA Diffusion Apply 2D kernel
-template <int T_D1D = 0, int T_Q1D = 0, int T_VDIM = 0>
-static void
-PAVectorDiffusionApply2D(const int NE, const Array<real_t> &b,
-                         const Array<real_t> &g, const Array<real_t> &bt,
-                         const Array<real_t> &gt, const Vector &d_,
-                         const Vector &x_, Vector &y_, const int d1d = 0,
-                         const int q1d = 0, const int vdim = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto G = Reshape(g.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto Gt = Reshape(gt.Read(), D1D, Q1D);
-   auto D = Reshape(d_.Read(), Q1D * Q1D, 3, NE);
-   auto x = Reshape(x_.Read(), D1D, D1D, VDIM, NE);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, VDIM, NE);
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      real_t grad[max_Q1D][max_Q1D][2];
-      for (int c = 0; c < VDIM; c++)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               grad[qy][qx][0] = 0.0;
-               grad[qy][qx][1] = 0.0;
-            }
-         }
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            real_t gradX[max_Q1D][2];
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               gradX[qx][0] = 0.0;
-               gradX[qx][1] = 0.0;
-            }
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               const real_t s = x(dx, dy, c, e);
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  gradX[qx][0] += s * B(qx, dx);
-                  gradX[qx][1] += s * G(qx, dx);
-               }
-            }
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const real_t wy = B(qy, dy);
-               const real_t wDy = G(qy, dy);
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  grad[qy][qx][0] += gradX[qx][1] * wy;
-                  grad[qy][qx][1] += gradX[qx][0] * wDy;
-               }
-            }
-         }
-         // Calculate Dxy, xDy in plane
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const int q = qx + qy * Q1D;
-               const real_t O11 = D(q, 0, e);
-               const real_t O12 = D(q, 1, e);
-               const real_t O22 = D(q, 2, e);
-               const real_t gradX = grad[qy][qx][0];
-               const real_t gradY = grad[qy][qx][1];
-               grad[qy][qx][0] = (O11 * gradX) + (O12 * gradY);
-               grad[qy][qx][1] = (O12 * gradX) + (O22 * gradY);
-            }
-         }
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            real_t gradX[max_D1D][2];
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               gradX[dx][0] = 0.0;
-               gradX[dx][1] = 0.0;
-            }
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const real_t gX = grad[qy][qx][0];
-               const real_t gY = grad[qy][qx][1];
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  const real_t wx = Bt(dx, qx);
-                  const real_t wDx = Gt(dx, qx);
-                  gradX[dx][0] += gX * wDx;
-                  gradX[dx][1] += gY * wx;
-               }
-            }
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               const real_t wy = Bt(dy, qy);
-               const real_t wDy = Gt(dy, qy);
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  y(dx, dy, c, e) +=
-                     ((gradX[dx][0] * wy) + (gradX[dx][1] * wDy));
-               }
-            }
-         }
-      }
-   });
-}
-
-// PA Diffusion Apply 3D kernel
-template <const int T_D1D = 0, const int T_Q1D = 0>
-static void
-PAVectorDiffusionApply3D(const int NE, const Array<real_t> &b,
-                         const Array<real_t> &g, const Array<real_t> &bt,
-                         const Array<real_t> &gt, const Vector &op_,
-                         const Vector &x_, Vector &y_, const int d1d = 0,
-                         const int q1d = 0, const int sdim = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int VDIM = 3;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto G = Reshape(g.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto Gt = Reshape(gt.Read(), D1D, Q1D);
-   auto op = Reshape(op_.Read(), Q1D * Q1D * Q1D, 6, NE);
-   auto x = Reshape(x_.Read(), D1D, D1D, D1D, VDIM, NE);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      for (int c = 0; c < VDIM; ++c)
-      {
-         real_t grad[max_Q1D][max_Q1D][max_Q1D][3];
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  grad[qz][qy][qx][0] = 0.0;
-                  grad[qz][qy][qx][1] = 0.0;
-                  grad[qz][qy][qx][2] = 0.0;
-               }
-            }
-         }
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            real_t gradXY[max_Q1D][max_Q1D][3];
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  gradXY[qy][qx][0] = 0.0;
-                  gradXY[qy][qx][1] = 0.0;
-                  gradXY[qy][qx][2] = 0.0;
-               }
-            }
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               real_t gradX[max_Q1D][2];
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  gradX[qx][0] = 0.0;
-                  gradX[qx][1] = 0.0;
-               }
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  const real_t s = x(dx, dy, dz, c, e);
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     gradX[qx][0] += s * B(qx, dx);
-                     gradX[qx][1] += s * G(qx, dx);
-                  }
-               }
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  const real_t wy = B(qy, dy);
-                  const real_t wDy = G(qy, dy);
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     const real_t wx = gradX[qx][0];
-                     const real_t wDx = gradX[qx][1];
-                     gradXY[qy][qx][0] += wDx * wy;
-                     gradXY[qy][qx][1] += wx * wDy;
-                     gradXY[qy][qx][2] += wx * wy;
-                  }
-               }
-            }
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               const real_t wz = B(qz, dz);
-               const real_t wDz = G(qz, dz);
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     grad[qz][qy][qx][0] += gradXY[qy][qx][0] * wz;
-                     grad[qz][qy][qx][1] += gradXY[qy][qx][1] * wz;
-                     grad[qz][qy][qx][2] += gradXY[qy][qx][2] * wDz;
-                  }
-               }
-            }
-         }
-         // Calculate Dxyz, xDyz, xyDz in plane
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  const int q = qx + (qy + qz * Q1D) * Q1D;
-                  const real_t O11 = op(q, 0, e);
-                  const real_t O12 = op(q, 1, e);
-                  const real_t O13 = op(q, 2, e);
-                  const real_t O22 = op(q, 3, e);
-                  const real_t O23 = op(q, 4, e);
-                  const real_t O33 = op(q, 5, e);
-                  const real_t gradX = grad[qz][qy][qx][0];
-                  const real_t gradY = grad[qz][qy][qx][1];
-                  const real_t gradZ = grad[qz][qy][qx][2];
-                  grad[qz][qy][qx][0] =
-                     (O11 * gradX) + (O12 * gradY) + (O13 * gradZ);
-                  grad[qz][qy][qx][1] =
-                     (O12 * gradX) + (O22 * gradY) + (O23 * gradZ);
-                  grad[qz][qy][qx][2] =
-                     (O13 * gradX) + (O23 * gradY) + (O33 * gradZ);
-               }
-            }
-         }
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            real_t gradXY[max_D1D][max_D1D][3];
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  gradXY[dy][dx][0] = 0;
-                  gradXY[dy][dx][1] = 0;
-                  gradXY[dy][dx][2] = 0;
-               }
-            }
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               real_t gradX[max_D1D][3];
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  gradX[dx][0] = 0;
-                  gradX[dx][1] = 0;
-                  gradX[dx][2] = 0;
-               }
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  const real_t gX = grad[qz][qy][qx][0];
-                  const real_t gY = grad[qz][qy][qx][1];
-                  const real_t gZ = grad[qz][qy][qx][2];
-                  for (int dx = 0; dx < D1D; ++dx)
-                  {
-                     const real_t wx = Bt(dx, qx);
-                     const real_t wDx = Gt(dx, qx);
-                     gradX[dx][0] += gX * wDx;
-                     gradX[dx][1] += gY * wx;
-                     gradX[dx][2] += gZ * wx;
-                  }
-               }
-               for (int dy = 0; dy < D1D; ++dy)
-               {
-                  const real_t wy = Bt(dy, qy);
-                  const real_t wDy = Gt(dy, qy);
-                  for (int dx = 0; dx < D1D; ++dx)
-                  {
-                     gradXY[dy][dx][0] += gradX[dx][0] * wy;
-                     gradXY[dy][dx][1] += gradX[dx][1] * wDy;
-                     gradXY[dy][dx][2] += gradX[dx][2] * wy;
-                  }
-               }
-            }
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               const real_t wz = Bt(dz, qz);
-               const real_t wDz = Gt(dz, qz);
-               for (int dy = 0; dy < D1D; ++dy)
-               {
-                  for (int dx = 0; dx < D1D; ++dx)
-                  {
-                     y(dx, dy, dz, c, e) +=
-                        ((gradXY[dy][dx][0] * wz) + (gradXY[dy][dx][1] * wz) +
-                         (gradXY[dy][dx][2] * wDz));
-                  }
-               }
-            }
-         }
-      }
-   });
-}
-} // namespace internal
-
-template <int DIM, int VDIM, int T_D1D, int T_Q1D>
-VectorDiffusionIntegrator::ApplyKernelType
-VectorDiffusionIntegrator::ApplyPAKernels::Kernel()
-{
-   if constexpr (DIM == 2)
-   {
-      return internal::PAVectorDiffusionApply2D<T_D1D, T_Q1D, VDIM>;
-   }
-   else if constexpr (DIM == 3)
-   {
-      return internal::PAVectorDiffusionApply3D;
-   }
-   MFEM_ABORT("");
-}
-
-} // namespace mfem
-/// \endcond DO_NOT_DOCUMENT
-#endif

fem/integ/bilininteg_vecdiffusion_pa.cpp

@@ -15,58 +15,9 @@
 #include "../qfunction.hpp"
 #include "../ceed/integrators/diffusion/diffusion.hpp"
 
-#include "bilininteg_vecdiffusion_kernels.hpp"
-
 namespace mfem
 {
 
-VectorDiffusionIntegrator::VectorDiffusionIntegrator(const IntegrationRule *ir)
-   : BilinearFormIntegrator(ir)
-{
-   static Kernels kernels;
-}
-
-VectorDiffusionIntegrator::VectorDiffusionIntegrator(Coefficient &q)
-   : VectorDiffusionIntegrator()
-{
-   Q = &q;
-}
-
-VectorDiffusionIntegrator::VectorDiffusionIntegrator(int vector_dimension)
-   : VectorDiffusionIntegrator()
-{
-   vdim = vector_dimension;
-}
-
-VectorDiffusionIntegrator::VectorDiffusionIntegrator(Coefficient &q,
-                                                     const IntegrationRule *ir)
-   : VectorDiffusionIntegrator(ir)
-{
-   Q = &q;
-}
-
-VectorDiffusionIntegrator::VectorDiffusionIntegrator(Coefficient &q,
-                                                     int vector_dimension)
-   : VectorDiffusionIntegrator()
-{
-   Q = &q;
-   vdim = vector_dimension;
-}
-
-VectorDiffusionIntegrator::VectorDiffusionIntegrator(VectorCoefficient &vq)
-   : VectorDiffusionIntegrator()
-{
-   VQ = &vq;
-   vdim = vq.GetVDim();
-}
-
-VectorDiffusionIntegrator::VectorDiffusionIntegrator(MatrixCoefficient &mq)
-   : VectorDiffusionIntegrator()
-{
-   MQ = &mq;
-   vdim = mq.GetVDim();
-}
-
 // PA Diffusion Assemble 2D kernel
 static void PAVectorDiffusionSetup2D(const int Q1D,
                                      const int NE,
@@ -474,6 +425,322 @@ void VectorDiffusionIntegrator::AssembleDiagonalPA(Vector &diag)
    }
 }
 
+// PA Diffusion Apply 2D kernel
+template<int T_D1D = 0, int T_Q1D = 0, int T_VDIM = 0> static
+void PAVectorDiffusionApply2D(const int NE,
+                              const Array<real_t> &b,
+                              const Array<real_t> &g,
+                              const Array<real_t> &bt,
+                              const Array<real_t> &gt,
+                              const Vector &d_,
+                              const Vector &x_,
+                              Vector &y_,
+                              const int d1d = 0,
+                              const int q1d = 0,
+                              const int vdim = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto G = Reshape(g.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt.Read(), D1D, Q1D);
+   auto Gt = Reshape(gt.Read(), D1D, Q1D);
+   auto D = Reshape(d_.Read(), Q1D*Q1D, 3, NE);
+   auto x = Reshape(x_.Read(), D1D, D1D, VDIM, NE);
+   auto y = Reshape(y_.ReadWrite(), D1D, D1D, VDIM, NE);
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      real_t grad[max_Q1D][max_Q1D][2];
+      for (int c = 0; c < VDIM; c++)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               grad[qy][qx][0] = 0.0;
+               grad[qy][qx][1] = 0.0;
+            }
+         }
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            real_t gradX[max_Q1D][2];
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               gradX[qx][0] = 0.0;
+               gradX[qx][1] = 0.0;
+            }
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const real_t s = x(dx,dy,c,e);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  gradX[qx][0] += s * B(qx,dx);
+                  gradX[qx][1] += s * G(qx,dx);
+               }
+            }
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const real_t wy  = B(qy,dy);
+               const real_t wDy = G(qy,dy);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  grad[qy][qx][0] += gradX[qx][1] * wy;
+                  grad[qy][qx][1] += gradX[qx][0] * wDy;
+               }
+            }
+         }
+         // Calculate Dxy, xDy in plane
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const int q = qx + qy * Q1D;
+               const real_t O11 = D(q,0,e);
+               const real_t O12 = D(q,1,e);
+               const real_t O22 = D(q,2,e);
+               const real_t gradX = grad[qy][qx][0];
+               const real_t gradY = grad[qy][qx][1];
+               grad[qy][qx][0] = (O11 * gradX) + (O12 * gradY);
+               grad[qy][qx][1] = (O12 * gradX) + (O22 * gradY);
+            }
+         }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            real_t gradX[max_D1D][2];
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               gradX[dx][0] = 0.0;
+               gradX[dx][1] = 0.0;
+            }
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const real_t gX = grad[qy][qx][0];
+               const real_t gY = grad[qy][qx][1];
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  const real_t wx  = Bt(dx,qx);
+                  const real_t wDx = Gt(dx,qx);
+                  gradX[dx][0] += gX * wDx;
+                  gradX[dx][1] += gY * wx;
+               }
+            }
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               const real_t wy  = Bt(dy,qy);
+               const real_t wDy = Gt(dy,qy);
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  y(dx,dy,c,e) += ((gradX[dx][0] * wy) + (gradX[dx][1] * wDy));
+               }
+            }
+         }
+      }
+   });
+}
+
+// PA Diffusion Apply 3D kernel
+template<const int T_D1D = 0,
+         const int T_Q1D = 0> static
+void PAVectorDiffusionApply3D(const int NE,
+                              const Array<real_t> &b,
+                              const Array<real_t> &g,
+                              const Array<real_t> &bt,
+                              const Array<real_t> &gt,
+                              const Vector &op_,
+                              const Vector &x_,
+                              Vector &y_,
+                              int d1d = 0, int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int VDIM = 3;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto G = Reshape(g.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt.Read(), D1D, Q1D);
+   auto Gt = Reshape(gt.Read(), D1D, Q1D);
+   auto op = Reshape(op_.Read(), Q1D*Q1D*Q1D, 6, NE);
+   auto x = Reshape(x_.Read(), D1D, D1D, D1D, VDIM, NE);
+   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      for (int c = 0; c < VDIM; ++ c)
+      {
+         real_t grad[max_Q1D][max_Q1D][max_Q1D][3];
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  grad[qz][qy][qx][0] = 0.0;
+                  grad[qz][qy][qx][1] = 0.0;
+                  grad[qz][qy][qx][2] = 0.0;
+               }
+            }
+         }
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            real_t gradXY[max_Q1D][max_Q1D][3];
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  gradXY[qy][qx][0] = 0.0;
+                  gradXY[qy][qx][1] = 0.0;
+                  gradXY[qy][qx][2] = 0.0;
+               }
+            }
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               real_t gradX[max_Q1D][2];
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  gradX[qx][0] = 0.0;
+                  gradX[qx][1] = 0.0;
+               }
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  const real_t s = x(dx,dy,dz,c,e);
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     gradX[qx][0] += s * B(qx,dx);
+                     gradX[qx][1] += s * G(qx,dx);
+                  }
+               }
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  const real_t wy  = B(qy,dy);
+                  const real_t wDy = G(qy,dy);
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     const real_t wx  = gradX[qx][0];
+                     const real_t wDx = gradX[qx][1];
+                     gradXY[qy][qx][0] += wDx * wy;
+                     gradXY[qy][qx][1] += wx  * wDy;
+                     gradXY[qy][qx][2] += wx  * wy;
+                  }
+               }
+            }
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               const real_t wz  = B(qz,dz);
+               const real_t wDz = G(qz,dz);
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     grad[qz][qy][qx][0] += gradXY[qy][qx][0] * wz;
+                     grad[qz][qy][qx][1] += gradXY[qy][qx][1] * wz;
+                     grad[qz][qy][qx][2] += gradXY[qy][qx][2] * wDz;
+                  }
+               }
+            }
+         }
+         // Calculate Dxyz, xDyz, xyDz in plane
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  const int q = qx + (qy + qz * Q1D) * Q1D;
+                  const real_t O11 = op(q,0,e);
+                  const real_t O12 = op(q,1,e);
+                  const real_t O13 = op(q,2,e);
+                  const real_t O22 = op(q,3,e);
+                  const real_t O23 = op(q,4,e);
+                  const real_t O33 = op(q,5,e);
+                  const real_t gradX = grad[qz][qy][qx][0];
+                  const real_t gradY = grad[qz][qy][qx][1];
+                  const real_t gradZ = grad[qz][qy][qx][2];
+                  grad[qz][qy][qx][0] = (O11*gradX)+(O12*gradY)+(O13*gradZ);
+                  grad[qz][qy][qx][1] = (O12*gradX)+(O22*gradY)+(O23*gradZ);
+                  grad[qz][qy][qx][2] = (O13*gradX)+(O23*gradY)+(O33*gradZ);
+               }
+            }
+         }
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            real_t gradXY[max_D1D][max_D1D][3];
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  gradXY[dy][dx][0] = 0;
+                  gradXY[dy][dx][1] = 0;
+                  gradXY[dy][dx][2] = 0;
+               }
+            }
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               real_t gradX[max_D1D][3];
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  gradX[dx][0] = 0;
+                  gradX[dx][1] = 0;
+                  gradX[dx][2] = 0;
+               }
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  const real_t gX = grad[qz][qy][qx][0];
+                  const real_t gY = grad[qz][qy][qx][1];
+                  const real_t gZ = grad[qz][qy][qx][2];
+                  for (int dx = 0; dx < D1D; ++dx)
+                  {
+                     const real_t wx  = Bt(dx,qx);
+                     const real_t wDx = Gt(dx,qx);
+                     gradX[dx][0] += gX * wDx;
+                     gradX[dx][1] += gY * wx;
+                     gradX[dx][2] += gZ * wx;
+                  }
+               }
+               for (int dy = 0; dy < D1D; ++dy)
+               {
+                  const real_t wy  = Bt(dy,qy);
+                  const real_t wDy = Gt(dy,qy);
+                  for (int dx = 0; dx < D1D; ++dx)
+                  {
+                     gradXY[dy][dx][0] += gradX[dx][0] * wy;
+                     gradXY[dy][dx][1] += gradX[dx][1] * wDy;
+                     gradXY[dy][dx][2] += gradX[dx][2] * wy;
+                  }
+               }
+            }
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               const real_t wz  = Bt(dz,qz);
+               const real_t wDz = Gt(dz,qz);
+               for (int dy = 0; dy < D1D; ++dy)
+               {
+                  for (int dx = 0; dx < D1D; ++dx)
+                  {
+                     y(dx,dy,dz,c,e) +=
+                        ((gradXY[dy][dx][0] * wz) +
+                         (gradXY[dy][dx][1] * wz) +
+                         (gradXY[dy][dx][2] * wDz));
+                  }
+               }
+            }
+         }
+      }
+   });
+}
+
 // PA Diffusion Apply kernel
 void VectorDiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
@@ -490,29 +757,27 @@ void VectorDiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
       const Array<real_t> &Bt = maps->Bt;
       const Array<real_t> &Gt = maps->Gt;
       const Vector &D = pa_data;
-      ApplyPAKernels::Run(dim, sdim, D1D, Q1D, ne, B, G, Bt, Gt, D, x, y, D1D,
-                          Q1D, sdim);
+
+      if (dim == 2 && sdim == 3)
+      {
+         switch ((dofs1D << 4 ) | quad1D)
+         {
+            case 0x22: return PAVectorDiffusionApply2D<2,2,3>(ne,B,G,Bt,Gt,D,x,y);
+            case 0x33: return PAVectorDiffusionApply2D<3,3,3>(ne,B,G,Bt,Gt,D,x,y);
+            case 0x44: return PAVectorDiffusionApply2D<4,4,3>(ne,B,G,Bt,Gt,D,x,y);
+            case 0x55: return PAVectorDiffusionApply2D<5,5,3>(ne,B,G,Bt,Gt,D,x,y);
+            default:
+               return PAVectorDiffusionApply2D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D,sdim);
+         }
+      }
+      if (dim == 2 && sdim == 2)
+      { return PAVectorDiffusionApply2D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D,sdim); }
+
+      if (dim == 3 && sdim == 3)
+      { return PAVectorDiffusionApply3D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D); }
+
+      MFEM_ABORT("Unknown kernel.");
    }
 }
 
-/// \cond DO_NOT_DOCUMENT
-
-VectorDiffusionIntegrator::ApplyKernelType
-VectorDiffusionIntegrator::ApplyPAKernels::Fallback(int DIM, int, int, int)
-{
-   if (DIM == 2) { return internal::PAVectorDiffusionApply2D; }
-   else if (DIM == 3) { return internal::PAVectorDiffusionApply3D; }
-   else { MFEM_ABORT(""); }
-}
-
-VectorDiffusionIntegrator::Kernels::Kernels()
-{
-   VectorDiffusionIntegrator::AddSpecialization<2, 3, 2, 2>();
-   VectorDiffusionIntegrator::AddSpecialization<2, 3, 3, 3>();
-   VectorDiffusionIntegrator::AddSpecialization<2, 3, 4, 4>();
-   VectorDiffusionIntegrator::AddSpecialization<2, 3, 5, 5>();
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
 } // namespace mfem

fem/integ/lininteg_domain.cpp

@@ -9,19 +9,183 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "../../fem/kernels.hpp"
 #include "../../general/forall.hpp"
+#include "../../fem/kernels.hpp"
 #include "../fem.hpp"
 
-#include "lininteg_domain_kernels.hpp"
-
-/// \cond DO_NOT_DOCUMENT
-
 namespace mfem
 {
+
+template<int T_D1D = 0, int T_Q1D = 0>
+static void DLFEvalAssemble2D(const int vdim, const int ne, const int d,
+                              const int q,
+                              const int map_type, const int *markers, const real_t *b,
+                              const real_t *detj, const real_t *weights,
+                              const Vector &coeff, real_t *y)
+{
+   const auto F = coeff.Read();
+   const auto M = Reshape(markers, ne);
+   const auto B = Reshape(b, q, d);
+   const auto DETJ = Reshape(detj, q, q, ne);
+   const auto W = Reshape(weights, q, q);
+   const bool cst = coeff.Size() == vdim;
+   const auto C = cst ? Reshape(F,vdim,1,1,1) : Reshape(F,vdim,q,q,ne);
+   auto Y = Reshape(y, d,d, vdim, ne);
+
+   mfem::forall_2D(ne, q, q, [=] MFEM_HOST_DEVICE (int e)
+   {
+      if (M(e) == 0) { return; } // ignore
+
+      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+
+      MFEM_SHARED real_t sBt[Q*D];
+      MFEM_SHARED real_t sQQ[Q*Q];
+      MFEM_SHARED real_t sQD[Q*D];
+
+      const DeviceMatrix Bt(sBt, d, q);
+      kernels::internal::LoadB<D,Q>(d, q, B, sBt);
+
+      const DeviceMatrix QQ(sQQ, q, q);
+      const DeviceMatrix QD(sQD, q, d);
+
+      for (int c = 0; c < vdim; ++c)
+      {
+         const real_t cst_val = C(c,0,0,0);
+         MFEM_FOREACH_THREAD(x,x,q)
+         {
+            MFEM_FOREACH_THREAD(y,y,q)
+            {
+               const real_t detJ = (map_type == FiniteElement::VALUE) ? DETJ(x,y,e) : 1.0;
+               const real_t coeff_val = cst ? cst_val : C(c,x,y,e);
+               QQ(y,x) = W(x,y) * coeff_val * detJ;
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(qy,y,q)
+         {
+            MFEM_FOREACH_THREAD(dx,x,d)
+            {
+               real_t u = 0.0;
+               for (int qx = 0; qx < q; ++qx) { u += QQ(qy,qx) * Bt(dx,qx); }
+               QD(qy,dx) = u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(dy,y,d)
+         {
+            MFEM_FOREACH_THREAD(dx,x,d)
+            {
+               real_t u = 0.0;
+               for (int qy = 0; qy < q; ++qy) { u += QD(qy,dx) * Bt(dy,qy); }
+               Y(dx,dy,c,e) += u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0>
+static void DLFEvalAssemble3D(const int vdim, const int ne, const int d,
+                              const int q,
+                              const int map_type, const int *markers, const real_t *b,
+                              const real_t *detj, const real_t *weights,
+                              const Vector &coeff, real_t *y)
+{
+   const auto F = coeff.Read();
+   const auto M = Reshape(markers, ne);
+   const auto B = Reshape(b, q,d);
+   const auto DETJ = Reshape(detj, q, q, q, ne);
+   const auto W = Reshape(weights, q,q,q);
+   const bool cst_coeff = coeff.Size() == vdim;
+   const auto C = cst_coeff ? Reshape(F,vdim,1,1,1,1):Reshape(F,vdim,q,q,q,ne);
+
+   auto Y = Reshape(y, d,d,d, vdim, ne);
+
+   mfem::forall_2D(ne, q, q, [=] MFEM_HOST_DEVICE (int e)
+   {
+      if (M(e) == 0) { return; } // ignore
+
+      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MQD = (Q >= D) ? Q : D;
+
+      real_t u[D];
+
+      MFEM_SHARED real_t sBt[Q*D];
+      const DeviceMatrix Bt(sBt, d,q);
+      kernels::internal::LoadB<D,Q>(d,q,B,sBt);
+
+      MFEM_SHARED real_t sQQQ[MQD*MQD*MQD];
+      const DeviceCube QQQ(sQQQ, MQD, MQD, MQD);
+
+      for (int c = 0; c < vdim; ++c)
+      {
+         const real_t cst_val = C(c,0,0,0,0);
+         MFEM_FOREACH_THREAD(x,x,q)
+         {
+            MFEM_FOREACH_THREAD(y,y,q)
+            {
+               for (int z = 0; z < q; ++z)
+               {
+                  const real_t detJ = (map_type == FiniteElement::VALUE) ? DETJ(x,y,z,e) : 1.0;
+                  const real_t coeff_val = cst_coeff ? cst_val : C(c,x,y,z,e);
+                  QQQ(z,y,x) = W(x,y,z) * coeff_val * detJ;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(qx,x,q)
+         {
+            MFEM_FOREACH_THREAD(qy,y,q)
+            {
+               for (int dz = 0; dz < d; ++dz) { u[dz] = 0.0; }
+               for (int qz = 0; qz < q; ++qz)
+               {
+                  const real_t ZYX = QQQ(qz,qy,qx);
+                  for (int dz = 0; dz < d; ++dz) { u[dz] += ZYX * Bt(dz,qz); }
+               }
+               for (int dz = 0; dz < d; ++dz) { QQQ(dz,qy,qx) = u[dz]; }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(dz,y,d)
+         {
+            MFEM_FOREACH_THREAD(qx,x,q)
+            {
+               for (int dy = 0; dy < d; ++dy) { u[dy] = 0.0; }
+               for (int qy = 0; qy < q; ++qy)
+               {
+                  const real_t zYX = QQQ(dz,qy,qx);
+                  for (int dy = 0; dy < d; ++dy) { u[dy] += zYX * Bt(dy,qy); }
+               }
+               for (int dy = 0; dy < d; ++dy) { QQQ(dz,dy,qx) = u[dy]; }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(dz,y,d)
+         {
+            MFEM_FOREACH_THREAD(dy,x,d)
+            {
+               for (int dx = 0; dx < d; ++dx) { u[dx] = 0.0; }
+               for (int qx = 0; qx < q; ++qx)
+               {
+                  const real_t zyX = QQQ(dz,dy,qx);
+                  for (int dx = 0; dx < d; ++dx) { u[dx] += zyX * Bt(dx,qx); }
+               }
+               for (int dx = 0; dx < d; ++dx) { Y(dx,dy,dz,c,e) += u[dx]; }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
 static void DLFEvalAssemble(const FiniteElementSpace &fes,
                             const IntegrationRule *ir,
-                            const Array<int> &markers, const Vector &coeff,
+                            const Array<int> &markers,
+                            const Vector &coeff,
                             Vector &y)
 {
    Mesh *mesh = fes.GetMesh();
@@ -33,20 +197,50 @@ static void DLFEvalAssemble(const FiniteElementSpace &fes,
    constexpr int flags = GeometricFactors::DETERMINANTS;
    const GeometricFactors *geom = mesh->GetGeometricFactors(*ir, flags, mt);
    const int map_type = fes.GetTypicalFE()->GetMapType();
+   decltype(&DLFEvalAssemble2D<>) ker =
+      dim == 2 ? DLFEvalAssemble2D<> : DLFEvalAssemble3D<>;
+
+   if (dim==2)
+   {
+      if (d==1 && q==1) { ker=DLFEvalAssemble2D<1,1>; }
+      if (d==2 && q==2) { ker=DLFEvalAssemble2D<2,2>; }
+      if (d==3 && q==3) { ker=DLFEvalAssemble2D<3,3>; }
+      if (d==4 && q==4) { ker=DLFEvalAssemble2D<4,4>; }
+      if (d==5 && q==5) { ker=DLFEvalAssemble2D<5,5>; }
+      if (d==2 && q==3) { ker=DLFEvalAssemble2D<2,3>; }
+      if (d==3 && q==4) { ker=DLFEvalAssemble2D<3,4>; }
+      if (d==4 && q==5) { ker=DLFEvalAssemble2D<4,5>; }
+      if (d==5 && q==6) { ker=DLFEvalAssemble2D<5,6>; }
+   }
+
+   if (dim==3)
+   {
+      if (d==1 && q==1) { ker=DLFEvalAssemble3D<1,1>; }
+      if (d==2 && q==2) { ker=DLFEvalAssemble3D<2,2>; }
+      if (d==3 && q==3) { ker=DLFEvalAssemble3D<3,3>; }
+      if (d==4 && q==4) { ker=DLFEvalAssemble3D<4,4>; }
+      if (d==5 && q==5) { ker=DLFEvalAssemble3D<5,5>; }
+      if (d==2 && q==3) { ker=DLFEvalAssemble3D<2,3>; }
+      if (d==3 && q==4) { ker=DLFEvalAssemble3D<3,4>; }
+      if (d==4 && q==5) { ker=DLFEvalAssemble3D<4,5>; }
+      if (d==5 && q==6) { ker=DLFEvalAssemble3D<5,6>; }
+   }
+
+   MFEM_VERIFY(ker, "No kernel ndof " << d << " nqpt " << q);
 
    const int vdim = fes.GetVDim();
    const int ne = fes.GetMesh()->GetNE();
-   const real_t *B = maps.B.Read();
    const int *M = markers.Read();
+   const real_t *B = maps.B.Read();
    const real_t *detJ = geom->detJ.Read();
    const real_t *W = ir->GetWeights().Read();
    real_t *Y = y.ReadWrite();
-   DomainLFIntegrator::AssembleKernels::Run(dim, d, q, vdim, ne, d, q, map_type,
-                                            M, B, detJ, W, coeff, Y);
+   ker(vdim, ne, d, q, map_type, M, B, detJ, W, coeff, Y);
 }
 
 void DomainLFIntegrator::AssembleDevice(const FiniteElementSpace &fes,
-                                        const Array<int> &markers, Vector &b)
+                                        const Array<int> &markers,
+                                        Vector &b)
 {
    const FiniteElement &fe = *fes.GetTypicalFE();
    const int qorder = oa * fe.GetOrder() + ob;
@@ -72,49 +266,4 @@ void VectorDomainLFIntegrator::AssembleDevice(const FiniteElementSpace &fes,
    DLFEvalAssemble(fes, ir, markers, coeff, b);
 }
 
-DomainLFIntegrator::AssembleKernelType
-DomainLFIntegrator::AssembleKernels::Fallback(int DIM, int, int)
-{
-   switch (DIM)
-   {
-      case 1:
-         return DLFEvalAssemble1D<0, 0>;
-      case 2:
-         return DLFEvalAssemble2D<0, 0>;
-      case 3:
-         return DLFEvalAssemble3D<0, 0>;
-   }
-   MFEM_ABORT("");
-}
-
-DomainLFIntegrator::Kernels::Kernels()
-{
-   // 2D
-   // Q = P+1
-   DomainLFIntegrator::AddSpecialization<2, 1, 1>();
-   DomainLFIntegrator::AddSpecialization<2, 2, 2>();
-   DomainLFIntegrator::AddSpecialization<2, 3, 3>();
-   DomainLFIntegrator::AddSpecialization<2, 4, 4>();
-   DomainLFIntegrator::AddSpecialization<2, 5, 5>();
-   // Q = P+2
-   DomainLFIntegrator::AddSpecialization<2, 2, 3>();
-   DomainLFIntegrator::AddSpecialization<2, 3, 4>();
-   DomainLFIntegrator::AddSpecialization<2, 4, 5>();
-   DomainLFIntegrator::AddSpecialization<2, 5, 6>();
-   // 3D
-   // Q = P+1
-   DomainLFIntegrator::AddSpecialization<3, 1, 1>();
-   DomainLFIntegrator::AddSpecialization<3, 2, 2>();
-   DomainLFIntegrator::AddSpecialization<3, 3, 3>();
-   DomainLFIntegrator::AddSpecialization<3, 4, 4>();
-   DomainLFIntegrator::AddSpecialization<3, 5, 5>();
-   // Q = P+2
-   DomainLFIntegrator::AddSpecialization<3, 2, 3>();
-   DomainLFIntegrator::AddSpecialization<3, 3, 4>();
-   DomainLFIntegrator::AddSpecialization<3, 4, 5>();
-   DomainLFIntegrator::AddSpecialization<3, 5, 6>();
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
 } // namespace mfem

fem/integ/lininteg_domain_kernels.hpp

@@ -1,318 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_LININTEG_DOMAIN_KERNELS_HPP
-#define MFEM_LININTEG_DOMAIN_KERNELS_HPP
-
-#include "../../fem/kernels.hpp"
-#include "../../general/forall.hpp"
-#include "../fem.hpp"
-
-/// \cond DO_NOT_DOCUMENT
-
-namespace mfem
-{
-
-template <int T_D1D = 0, int T_Q1D = 0>
-static void DLFEvalAssemble1D(const int vdim, const int ne, const int d,
-                              const int q, const int map_type,
-                              const int *markers, const real_t *b,
-                              const real_t *detj, const real_t *weights,
-                              const Vector &coeff, real_t *y)
-{
-   {
-      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      MFEM_VERIFY(q <= Q, "");
-      MFEM_VERIFY(d <= D, "");
-   }
-
-   const auto F = coeff.Read();
-   const auto B = Reshape(b, q, d);
-   const auto DETJ = Reshape(detj, q, ne);
-   const bool cst = coeff.Size() == vdim;
-   const auto C = cst ? Reshape(F, vdim, 1, 1) : Reshape(F, vdim, q, ne);
-   auto Y = Reshape(y, d, vdim, ne);
-
-   mfem::forall_2D(ne, d, 1, [=] MFEM_HOST_DEVICE(int e)
-   {
-      if (markers[e] == 0)
-      {
-         return;
-      } // ignore
-
-      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-
-      MFEM_SHARED real_t sBt[Q * D];
-      const DeviceMatrix Bt(sBt, d, q);
-      kernels::internal::LoadB<D, Q>(d, q, B, sBt);
-
-      for (int c = 0; c < vdim; ++c)
-      {
-         const real_t cst_val = C(c, 0, 0);
-         MFEM_FOREACH_THREAD(dx, x, d)
-         {
-            real_t u = 0;
-            for (int qx = 0; qx < q; ++qx)
-            {
-               const real_t detJ =
-                  (map_type == FiniteElement::VALUE) ? DETJ(qx, e) : 1.0;
-               const real_t coeff_val = cst ? cst_val : C(c, qx, e);
-               u += weights[qx] * coeff_val * detJ * Bt(dx, qx);
-            }
-            Y(dx, c, e) += u;
-         }
-      }
-   });
-}
-
-template <int T_D1D = 0, int T_Q1D = 0>
-static void DLFEvalAssemble2D(const int vdim, const int ne, const int d,
-                              const int q, const int map_type,
-                              const int *markers, const real_t *b,
-                              const real_t *detj, const real_t *weights,
-                              const Vector &coeff, real_t *y)
-{
-   {
-      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      MFEM_VERIFY(q <= Q, "");
-      MFEM_VERIFY(d <= D, "");
-   }
-
-   const auto F = coeff.Read();
-   const auto B = Reshape(b, q, d);
-   const auto DETJ = Reshape(detj, q, q, ne);
-   const auto W = Reshape(weights, q, q);
-   const bool cst = coeff.Size() == vdim;
-   const auto C = cst ? Reshape(F, vdim, 1, 1, 1) : Reshape(F, vdim, q, q, ne);
-   auto Y = Reshape(y, d, d, vdim, ne);
-
-   mfem::forall_2D(ne, q, q, [=] MFEM_HOST_DEVICE(int e)
-   {
-      if (markers[e] == 0)
-      {
-         return;
-      } // ignore
-
-      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-
-      MFEM_SHARED real_t sBt[Q * D];
-      MFEM_SHARED real_t sQQ[Q * Q];
-      MFEM_SHARED real_t sQD[Q * D];
-
-      const DeviceMatrix Bt(sBt, d, q);
-      kernels::internal::LoadB<D, Q>(d, q, B, sBt);
-
-      const DeviceMatrix QQ(sQQ, q, q);
-      const DeviceMatrix QD(sQD, q, d);
-
-      for (int c = 0; c < vdim; ++c)
-      {
-         const real_t cst_val = C(c, 0, 0, 0);
-         MFEM_FOREACH_THREAD(x, x, q)
-         {
-            MFEM_FOREACH_THREAD(y, y, q)
-            {
-               const real_t detJ =
-                  (map_type == FiniteElement::VALUE) ? DETJ(x, y, e) : 1.0;
-               const real_t coeff_val = cst ? cst_val : C(c, x, y, e);
-               QQ(y, x) = W(x, y) * coeff_val * detJ;
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(qy, y, q)
-         {
-            MFEM_FOREACH_THREAD(dx, x, d)
-            {
-               real_t u = 0.0;
-               for (int qx = 0; qx < q; ++qx)
-               {
-                  u += QQ(qy, qx) * Bt(dx, qx);
-               }
-               QD(qy, dx) = u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(dy, y, d)
-         {
-            MFEM_FOREACH_THREAD(dx, x, d)
-            {
-               real_t u = 0.0;
-               for (int qy = 0; qy < q; ++qy)
-               {
-                  u += QD(qy, dx) * Bt(dy, qy);
-               }
-               Y(dx, dy, c, e) += u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
-}
-
-template <int T_D1D = 0, int T_Q1D = 0>
-static void DLFEvalAssemble3D(const int vdim, const int ne, const int d,
-                              const int q, const int map_type,
-                              const int* markers, const real_t *b,
-                              const real_t *detj, const real_t *weights,
-                              const Vector &coeff, real_t *y)
-{
-   {
-      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      MFEM_VERIFY(q <= Q, "");
-      MFEM_VERIFY(d <= D, "");
-   }
-
-   const auto F = coeff.Read();
-   const auto B = Reshape(b, q, d);
-   const auto DETJ = Reshape(detj, q, q, q, ne);
-   const auto W = Reshape(weights, q, q, q);
-   const bool cst_coeff = coeff.Size() == vdim;
-   const auto C =
-      cst_coeff ? Reshape(F, vdim, 1, 1, 1, 1) : Reshape(F, vdim, q, q, q, ne);
-
-   auto Y = Reshape(y, d, d, d, vdim, ne);
-
-   mfem::forall_2D(ne, q, q, [=] MFEM_HOST_DEVICE(int e)
-   {
-      if (markers[e] == 0)
-      {
-         return;
-      } // ignore
-
-      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQD = (Q >= D) ? Q : D;
-
-      real_t u[D];
-
-      MFEM_SHARED real_t sBt[Q * D];
-      const DeviceMatrix Bt(sBt, d, q);
-      kernels::internal::LoadB<D, Q>(d, q, B, sBt);
-
-      MFEM_SHARED real_t sQQQ[MQD * MQD * MQD];
-      const DeviceCube QQQ(sQQQ, MQD, MQD, MQD);
-
-      for (int c = 0; c < vdim; ++c)
-      {
-         const real_t cst_val = C(c, 0, 0, 0, 0);
-         MFEM_FOREACH_THREAD(x, x, q)
-         {
-            MFEM_FOREACH_THREAD(y, y, q)
-            {
-               for (int z = 0; z < q; ++z)
-               {
-                  const real_t detJ = (map_type == FiniteElement::VALUE)
-                                      ? DETJ(x, y, z, e)
-                                      : 1.0;
-                  const real_t coeff_val =
-                     cst_coeff ? cst_val : C(c, x, y, z, e);
-                  QQQ(z, y, x) = W(x, y, z) * coeff_val * detJ;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(qx, x, q)
-         {
-            MFEM_FOREACH_THREAD(qy, y, q)
-            {
-               for (int dz = 0; dz < d; ++dz)
-               {
-                  u[dz] = 0.0;
-               }
-               for (int qz = 0; qz < q; ++qz)
-               {
-                  const real_t ZYX = QQQ(qz, qy, qx);
-                  for (int dz = 0; dz < d; ++dz)
-                  {
-                     u[dz] += ZYX * Bt(dz, qz);
-                  }
-               }
-               for (int dz = 0; dz < d; ++dz)
-               {
-                  QQQ(dz, qy, qx) = u[dz];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(dz, y, d)
-         {
-            MFEM_FOREACH_THREAD(qx, x, q)
-            {
-               for (int dy = 0; dy < d; ++dy)
-               {
-                  u[dy] = 0.0;
-               }
-               for (int qy = 0; qy < q; ++qy)
-               {
-                  const real_t zYX = QQQ(dz, qy, qx);
-                  for (int dy = 0; dy < d; ++dy)
-                  {
-                     u[dy] += zYX * Bt(dy, qy);
-                  }
-               }
-               for (int dy = 0; dy < d; ++dy)
-               {
-                  QQQ(dz, dy, qx) = u[dy];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(dz, y, d)
-         {
-            MFEM_FOREACH_THREAD(dy, x, d)
-            {
-               for (int dx = 0; dx < d; ++dx)
-               {
-                  u[dx] = 0.0;
-               }
-               for (int qx = 0; qx < q; ++qx)
-               {
-                  const real_t zyX = QQQ(dz, dy, qx);
-                  for (int dx = 0; dx < d; ++dx)
-                  {
-                     u[dx] += zyX * Bt(dx, qx);
-                  }
-               }
-               for (int dx = 0; dx < d; ++dx)
-               {
-                  Y(dx, dy, dz, c, e) += u[dx];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
-}
-
-template <int DIM, int T_D1D, int T_Q1D>
-DomainLFIntegrator::AssembleKernelType
-DomainLFIntegrator::AssembleKernels::Kernel()
-{
-   switch (DIM)
-   {
-      case 1:
-         return DLFEvalAssemble1D<T_D1D, T_Q1D>;
-      case 2:
-         return DLFEvalAssemble2D<T_D1D, T_Q1D>;
-      case 3:
-         return DLFEvalAssemble3D<T_D1D, T_Q1D>;
-   }
-   MFEM_ABORT("");
-}
-/// \endcond DO_NOT_DOCUMENT
-
-} // namespace mfem
-#endif

fem/intrules.cpp

@@ -947,7 +947,6 @@ int Quadrature1D::CheckOpen(int type)
       case OpenUniform:
       case ClosedUniform:
       case OpenHalfUniform:
-      case ClosedGL:
          return type; // all types can work as open
       default:
          return Invalid;

fem/intrules_cut.hpp

@@ -346,13 +346,13 @@ private:
       template<typename T>
       T operator() (const blitz::TinyVector<T,3>& x) const
       {
-         const int el_order = el->GetOrder();
-         std::vector<T> u1(el_order+1);
-         std::vector<T> u2(el_order+1);
-         std::vector<T> u3(el_order+1);
-         TmplPoly_1D::CalcBernstein(el_order, x[0], u1.data());
-         TmplPoly_1D::CalcBernstein(el_order, x[1], u2.data());
-         TmplPoly_1D::CalcBernstein(el_order, x[2], u3.data());
+         int el_order=el->GetOrder();
+         T u1[el_order+1];
+         T u2[el_order+1];
+         T u3[el_order+1];
+         TmplPoly_1D::CalcBernstein(el_order, x[0], u1);
+         TmplPoly_1D::CalcBernstein(el_order, x[1], u2);
+         TmplPoly_1D::CalcBernstein(el_order, x[2], u3);
 
          const Array<int>& dof_map=el->GetDofMap();
 
@@ -370,17 +370,17 @@ private:
       template<typename T>
       blitz::TinyVector<T,3> grad(const blitz::TinyVector<T,3>& x) const
       {
-         const int el_order = el->GetOrder();
-         std::vector<T> u1(el_order+1);
-         std::vector<T> u2(el_order+1);
-         std::vector<T> u3(el_order+1);
-         std::vector<T> d1(el_order+1);
-         std::vector<T> d2(el_order+1);
-         std::vector<T> d3(el_order+1);
+         int el_order=el->GetOrder();
+         T u1[el_order+1];
+         T u2[el_order+1];
+         T u3[el_order+1];
+         T d1[el_order+1];
+         T d2[el_order+1];
+         T d3[el_order+1];
 
-         TmplPoly_1D::CalcBernstein(el_order,x[0], u1.data(), d1.data());
-         TmplPoly_1D::CalcBernstein(el_order,x[1], u2.data(), d2.data());
-         TmplPoly_1D::CalcBernstein(el_order,x[2], u3.data(), d3.data());
+         TmplPoly_1D::CalcBernstein(el_order,x[0], u1, d1);
+         TmplPoly_1D::CalcBernstein(el_order,x[1], u2, d2);
+         TmplPoly_1D::CalcBernstein(el_order,x[2], u3, d3);
 
          blitz::TinyVector<T,3> res(T(0.0),T(0.0),T(0.0));
 
@@ -415,11 +415,11 @@ private:
       template<typename T>
       T operator() (const blitz::TinyVector<T,2>& x) const
       {
-         const int el_order = el->GetOrder();
-         std::vector<T> u1(el_order+1);
-         std::vector<T> u2(el_order+1);
-         TmplPoly_1D::CalcBernstein(el_order, x[0], u1.data());
-         TmplPoly_1D::CalcBernstein(el_order, x[1], u2.data());
+         int el_order=el->GetOrder();
+         T u1[el_order+1];
+         T u2[el_order+1];
+         TmplPoly_1D::CalcBernstein(el_order, x[0], u1);
+         TmplPoly_1D::CalcBernstein(el_order, x[1], u2);
 
          const Array<int>& dof_map=el->GetDofMap();
 
@@ -437,14 +437,14 @@ private:
       template<typename T>
       blitz::TinyVector<T,2> grad(const blitz::TinyVector<T,2>& x) const
       {
-         const int el_order = el->GetOrder();
-         std::vector<T> u1(el_order+1);
-         std::vector<T> u2(el_order+1);
-         std::vector<T> d1(el_order+1);
-         std::vector<T> d2(el_order+1);
+         int el_order=el->GetOrder();
+         T u1[el_order+1];
+         T u2[el_order+1];
+         T d1[el_order+1];
+         T d2[el_order+1];
 
-         TmplPoly_1D::CalcBernstein(el_order,x[0], u1.data(), d1.data());
-         TmplPoly_1D::CalcBernstein(el_order,x[1], u2.data(), d2.data());
+         TmplPoly_1D::CalcBernstein(el_order,x[0], u1, d1);
+         TmplPoly_1D::CalcBernstein(el_order,x[1], u2, d2);
 
          blitz::TinyVector<T,2> res(T(0.0),T(0.0));
 

fem/kernel_dispatch.hpp

@@ -13,6 +13,7 @@
 #define MFEM_KERNEL_DISPATCH_HPP
 
 #include "../config/config.hpp"
+#include "../config/tconfig.hpp" // MFEM_ALWAYS_INLINE
 #include "kernel_reporter.hpp"
 #include <unordered_map>
 #include <tuple>
@@ -78,7 +79,7 @@ namespace mfem
   public:                                                                      \
     const char *kernel_name = MFEM_KERNEL_NAME(KernelName);                    \
     using KernelSignature = KernelType;                                        \
-    template <MFEM_PARAM_LIST P3> static KernelSignature Kernel();             \
+    template <MFEM_PARAM_LIST P3> static MFEM_EXPORT KernelSignature Kernel(); \
     static MFEM_EXPORT KernelSignature Fallback(MFEM_PARAM_LIST P1);           \
     static MFEM_EXPORT KernelName &Get() {                                     \
       static KernelName table;                                                 \
@@ -95,11 +96,13 @@ struct KernelDispatchKeyHash
 {
 private:
    template<int N>
+   inline MFEM_ALWAYS_INLINE
    size_t operator()(std::tuple<KernelParameters...> value) const { return 0; }
 
    // The hashing formula here is taken directly from the Boost library, with
    // the magic number 0x9e3779b9 chosen to minimize hashing collisions.
    template<std::size_t N, typename THead, typename... TTail>
+   inline MFEM_ALWAYS_INLINE
    size_t operator()(std::tuple<KernelParameters...> value) const
    {
       constexpr int Index = N - sizeof...(TTail) - 1;
@@ -109,6 +112,7 @@ private:
    }
 public:
    /// Returns the hash of the given @a value.
+   inline MFEM_ALWAYS_INLINE
    size_t operator()(std::tuple<KernelParameters...> value) const
    {
       return operator()<sizeof...(KernelParameters),KernelParameters...>(value);
@@ -137,7 +141,8 @@ class KernelDispatchTable<Kernels,
    /// Only valid when the function @a f is not a member function.
    template <typename F, typename... Args,
              typename std::enable_if<std::is_pointer<F>::value,bool>::type=true>
-   static void Invoke(F f, Args&&... args)
+   static inline MFEM_ALWAYS_INLINE
+   void Invoke(F f, Args&&... args)
    {
       f(std::forward<Args>(args)...);
    }
@@ -149,7 +154,8 @@ class KernelDispatchTable<Kernels,
    template <typename F, typename T, typename... Args,
              typename std::enable_if<
                 std::is_member_function_pointer<F>::value,bool>::type=true>
-   static void Invoke(F f, T&& t, Args&&... args)
+   static inline MFEM_ALWAYS_INLINE
+   void Invoke(F f, T&& t, Args&&... args)
    {
       (t.*f)(std::forward<Args>(args)...);
    }
@@ -164,7 +170,8 @@ public:
    /// If the kernel is a member function, then the first argument after @a
    /// params should be the object on which it is called.
    template<typename... Args>
-   static void Run(Params... params, Args&&... args)
+   static inline MFEM_ALWAYS_INLINE
+   void Run(Params... params, Args&&... args)
    {
       const auto &table = Kernels::Get().table;
       const std::tuple<Params...> key = std::make_tuple(params...);

fem/linearform_ext.cpp

@@ -51,7 +51,7 @@ void LinearFormExtension::Assemble()
       {
          // scan the attributes to set the markers to 0 or 1
          const int NE = fes.GetNE();
-         const auto attr = attributes->Read();
+         const auto attr = attributes.Read();
          const auto dimk = domain_integs_marker_k->Read();
          auto markers_w = markers.Write();
          mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
@@ -93,14 +93,13 @@ void LinearFormExtension::Assemble()
       else
       {
          // scan the attributes to set the markers to 0 or 1
-         const int NBE = bdr_face_attributes->Size();
-         const auto attr = bdr_face_attributes->Read();
+         const int NBE = bdr_attributes.Size();
+         const auto attr = bdr_attributes.Read();
          const auto attr_markers = boundary_integs_marker_k->Read();
          auto markers_w = bdr_markers.Write();
-         mfem::forall(NBE, [=] MFEM_HOST_DEVICE(int e)
+         mfem::forall(NBE, [=] MFEM_HOST_DEVICE (int e)
          {
-            markers_w[e] =
-               attr[e] > 0 ? (attr_markers[attr[e] - 1] == 1) : false;
+            markers_w[e] = attr_markers[attr[e]-1] == 1;
          });
       }
 
@@ -126,7 +125,8 @@ void LinearFormExtension::Update()
       //markers.UseDevice(true);
 
       // Gather the attributes on the host from all the elements
-      attributes = &mesh.GetElementAttributes();
+      attributes.SetSize(NE);
+      for (int i = 0; i < NE; ++i) { attributes[i] = mesh.GetAttribute(i); }
 
       elem_restrict_lex = fes.GetElementRestriction(ordering);
       MFEM_VERIFY(elem_restrict_lex, "Element restriction not available");
@@ -136,12 +136,34 @@ void LinearFormExtension::Update()
 
    if (lf->boundary_integs.Size() > 0)
    {
-      bdr_face_attributes = &mesh.GetBdrFaceAttributes();
-
-      const int nf_bdr = bdr_face_attributes->Size();
+      const int nf_bdr = fes.GetNFbyType(FaceType::Boundary);
       bdr_markers.SetSize(nf_bdr);
       // bdr_markers.UseDevice(true);
 
+      // The face restriction will give us "face E-vectors" on the boundary that
+      // are numbered in the order of the faces of mesh. This numbering will be
+      // different than the numbering of the boundary elements. We compute
+      // mappings so that the array `bdr_attributes[i]` gives the boundary
+      // attribute of the `i`th boundary face in the mesh face order.
+      std::unordered_map<int,int> f_to_be;
+      for (int i = 0; i < mesh.GetNBE(); ++i)
+      {
+         const int f = mesh.GetBdrElementFaceIndex(i);
+         f_to_be[f] = i;
+      }
+      MFEM_VERIFY(size_t(nf_bdr) == f_to_be.size(), "Incompatible sizes");
+      bdr_attributes.SetSize(nf_bdr);
+      int f_ind = 0;
+      for (int f = 0; f < mesh.GetNumFaces(); ++f)
+      {
+         if (f_to_be.find(f) != f_to_be.end())
+         {
+            const int be = f_to_be[f];
+            bdr_attributes[f_ind] = mesh.GetBdrAttribute(be);
+            ++f_ind;
+         }
+      }
+
       bdr_restrict_lex =
          dynamic_cast<const FaceRestriction*>(
             fes.GetFaceRestriction(ordering, FaceType::Boundary,

fem/linearform_ext.hpp

@@ -25,8 +25,7 @@ class LinearForm;
 class LinearFormExtension
 {
    /// Attributes of all mesh elements.
-   const Array<int> *attributes; // Not owned
-   const Array<int> *bdr_face_attributes; // Not owned
+   Array<int> attributes, bdr_attributes;
 
    /// Temporary markers for device kernels.
    Array<int> markers, bdr_markers;

fem/lininteg.cpp

@@ -35,19 +35,6 @@ void LinearFormIntegrator::AssembleRHSElementVect(
    mfem_error("LinearFormIntegrator::AssembleRHSElementVect(...)");
 }
 
-DomainLFIntegrator::DomainLFIntegrator(Coefficient &QF, int a, int b)
-   : DeltaLFIntegrator(QF), Q(QF), oa(a), ob(b)
-{
-   static Kernels kernels;
-}
-
-DomainLFIntegrator::DomainLFIntegrator(Coefficient &QF,
-                                       const IntegrationRule *ir)
-   : DeltaLFIntegrator(QF, ir), Q(QF), oa(1), ob(1)
-{
-   static Kernels kernels;
-}
-
 void DomainLFIntegrator::AssembleRHSElementVect(const FiniteElement &el,
                                                 ElementTransformation &Tr,
                                                 Vector &elvect)
@@ -279,13 +266,6 @@ void BoundaryTangentialLFIntegrator::AssembleRHSElementVect(
    }
 }
 
-VectorDomainLFIntegrator::VectorDomainLFIntegrator(VectorCoefficient &QF,
-                                                   const IntegrationRule *ir)
-   : DeltaLFIntegrator(QF, ir), Q(QF)
-{
-   static DomainLFIntegrator::Kernels kernels;
-}
-
 void VectorDomainLFIntegrator::AssembleRHSElementVect(
    const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
 {

fem/lininteg.hpp

@@ -18,8 +18,6 @@
 #include <random>
 #include "integrator.hpp"
 
-#include "kernel_dispatch.hpp"
-
 namespace mfem
 {
 
@@ -111,12 +109,14 @@ class DomainLFIntegrator : public DeltaLFIntegrator
    int oa, ob;
 public:
    /// Constructs a domain integrator with a given Coefficient
-   /// the old default was a = 1, b = 1
-   /// for simple elliptic problems a = 2, b = -2 is OK
-   DomainLFIntegrator(Coefficient &QF, int a = 2, int b = 0);
+   DomainLFIntegrator(Coefficient &QF, int a = 2, int b = 0)
+   // the old default was a = 1, b = 1
+   // for simple elliptic problems a = 2, b = -2 is OK
+      : DeltaLFIntegrator(QF), Q(QF), oa(a), ob(b) { }
 
    /// Constructs a domain integrator with a given Coefficient
-   DomainLFIntegrator(Coefficient &QF, const IntegrationRule *ir);
+   DomainLFIntegrator(Coefficient &QF, const IntegrationRule *ir)
+      : DeltaLFIntegrator(QF, ir), Q(QF), oa(1), ob(1) { }
 
    bool SupportsDevice() const override { return true; }
 
@@ -136,22 +136,6 @@ public:
                                  Vector &elvect) override;
 
    using LinearFormIntegrator::AssembleRHSElementVect;
-
-   /// args: vdim, ne, d1d, q1d, map_type, markers, B, detJ, W, coeff, y
-   using AssembleKernelType = void (*)(const int, const int, const int,
-                                       const int, const int, const int *,
-                                       const real_t *, const real_t *,
-                                       const real_t *, const Vector &coeff,
-                                       real_t *y);
-
-   /// parameters: use DIM, T_D1D, T_Q1D
-   MFEM_REGISTER_KERNELS(AssembleKernels, AssembleKernelType, (int, int, int));
-   struct Kernels { Kernels(); };
-
-   template <int DIM, int D1D, int Q1D> static void AddSpecialization()
-   {
-      AssembleKernels::Specialization<DIM, D1D, Q1D>::Add();
-   }
 };
 
 /// Class for domain integrator $ L(v) := (f, \nabla v) $
@@ -272,13 +256,14 @@ private:
 
 public:
    /// Constructs a domain integrator with a given VectorCoefficient
-   VectorDomainLFIntegrator(VectorCoefficient &QF,
-                            const IntegrationRule *ir = nullptr);
+   VectorDomainLFIntegrator(VectorCoefficient &QF)
+      : DeltaLFIntegrator(QF), Q(QF) { }
 
    bool SupportsDevice() const override { return true; }
 
    /// Method defining assembly on device
-   void AssembleDevice(const FiniteElementSpace &fes, const Array<int> &markers,
+   void AssembleDevice(const FiniteElementSpace &fes,
+                       const Array<int> &markers,
                        Vector &b) override;
 
    /** Given a particular Finite Element and a transformation (Tr)
@@ -292,12 +277,6 @@ public:
                                  Vector &elvect) override;
 
    using LinearFormIntegrator::AssembleRHSElementVect;
-
-   template <int DIM, int D1D, int Q1D> static void AddSpecialization()
-   {
-      // uses the same kernels for assembly
-      DomainLFIntegrator::AssembleKernels::Specialization<DIM, D1D, Q1D>::Add();
-   }
 };
 
 /** Class for domain integrator $ L(v) := (f, \nabla v) $, where
@@ -565,7 +544,7 @@ public:
     Specifically, given the Dirichlet data $u_D$, the linear form assembles the
     following integrals on the boundary:
    $$
-    \sigma \langle u_D, (Q \nabla v) \cdot n \rangle + \kappa \langle {h^{-1} Q} u_D, v \rangle,
+    \sigma \langle u_D, (Q \nabla v)) \cdot n \rangle + \kappa \langle {h^{-1} Q} u_D, v \rangle,
    $$
     where Q is a scalar or matrix diffusion coefficient and v is the test
     function. The parameters $\sigma$ and $\kappa$ should be the same as the ones

fem/lor/lor_batched.cpp

@@ -14,11 +14,9 @@
 #include "../../general/forall.hpp"
 #include <climits>
 #include "../pbilinearform.hpp"
-#include "../../fem/fe/face_map_utils.hpp"
 
 // Specializations
 #include "lor_h1.hpp"
-#include "lor_dg.hpp"
 #include "lor_nd.hpp"
 #include "lor_rt.hpp"
 
@@ -56,18 +54,17 @@ bool BatchedLORAssembly::FormIsSupported(BilinearForm &a)
    // Batched LOR requires all tensor elements
    if (!UsesTensorBasis(*a.FESpace())) { return false; }
 
-   if (dynamic_cast<const H1_FECollection*>(fec) ||
-       dynamic_cast<const DG_FECollection*>(fec))
+   if (dynamic_cast<const H1_FECollection*>(fec))
    {
-      return HasIntegrators<DiffusionIntegrator, MassIntegrator>(a);
+      if (HasIntegrators<DiffusionIntegrator, MassIntegrator>(a)) { return true; }
    }
    else if (dynamic_cast<const ND_FECollection*>(fec))
    {
-      return HasIntegrators<CurlCurlIntegrator, VectorFEMassIntegrator>(a);
+      if (HasIntegrators<CurlCurlIntegrator, VectorFEMassIntegrator>(a)) { return true; }
    }
    else if (dynamic_cast<const RT_FECollection*>(fec))
    {
-      return HasIntegrators<DivDivIntegrator, VectorFEMassIntegrator>(a);
+      if (HasIntegrators<DivDivIntegrator, VectorFEMassIntegrator>(a)) { return true; }
    }
    return false;
 }
@@ -78,14 +75,12 @@ void BatchedLORAssembly::FormLORVertexCoordinates(FiniteElementSpace &fes_ho,
    Mesh &mesh_ho = *fes_ho.GetMesh();
    mesh_ho.EnsureNodes();
 
-   const bool dg = fes_ho.IsDGSpace();
-
    // Get nodal points at the LOR vertices
    const int dim = mesh_ho.Dimension();
    const int sdim = mesh_ho.SpaceDimension();
    const int nel_ho = mesh_ho.GetNE();
    const int order = fes_ho.GetMaxElementOrder();
-   const int nd1d = dg ? order + 2 : order + 1;
+   const int nd1d = order + 1;
    const int ndof_per_el = static_cast<int>(pow(nd1d, dim));
 
    const GridFunction *nodal_gf = mesh_ho.GetNodes();
@@ -97,8 +92,7 @@ void BatchedLORAssembly::FormLORVertexCoordinates(FiniteElementSpace &fes_ho,
    Vector nodal_evec(nodal_restriction->Height());
    nodal_restriction->Mult(*nodal_gf, nodal_evec);
 
-   const IntegrationRule ir = GetLobattoIntRule(
-                                 mesh_ho.GetTypicalElementGeometry(), nd1d);
+   IntegrationRule ir = GetCollocatedIntRule(fes_ho);
 
    // Map from nodal E-vector to Q-vector at the LOR vertex points
    X_vert.SetSize(sdim*ndof_per_el*nel_ho);
@@ -165,7 +159,6 @@ int BatchedLORAssembly::FillI(SparseMatrix &A) const
    const auto K = dof_glob2loc_offsets_.Read();
    const auto map = Reshape(sparse_mapping.Read(), nnz_per_row, ndof_per_el);
 
-
    auto I = A.WriteI();
 
    mfem::forall(nvdof + 1, [=] MFEM_HOST_DEVICE (int ii) { I[ii] = 0; });
@@ -365,177 +358,6 @@ void BatchedLORAssembly::FillJAndData(SparseMatrix &A) const
    });
 }
 
-void BatchedLORAssembly::SparseIJToCSR_DG(OperatorHandle &A) const
-{
-   const int ndof_per_el = fes_ho.GetFE(0)->GetDof();
-   const int nel_ho = fes_ho.GetNE();
-   const int nnz_per_row = sparse_ij.Size()/ndof_per_el/nel_ho;
-   const int dim = fes_ho.GetMesh()->Dimension();
-   const int nrows = nel_ho*ndof_per_el;
-   const int p = fes_ho.GetMaxElementOrder();
-   const int pp1 = p + 1;
-   const int nnz = nrows*nnz_per_row;
-
-   const int face_nbr_vsize = [&]()
-   {
-#ifdef MFEM_USE_MPI
-      if (auto *par_fes = dynamic_cast<ParFiniteElementSpace*>(&fes_ho))
-      {
-         return par_fes->GetFaceNbrVSize();
-      }
-#endif
-      return 0;
-   }();
-
-   // If A contains an existing SparseMatrix, reuse it (and try to reuse its
-   // I, J, A arrays if they are big enough)
-   SparseMatrix *A_mat = A.Is<SparseMatrix>();
-   if (!A_mat)
-   {
-      A_mat = new SparseMatrix;
-      A.Reset(A_mat);
-   }
-
-   // The second argument (nrows + face_nbr_vsize) accounts for additional
-   // columns contributed by DG face neighbors in parallel finite element
-   // spaces. In serial, face_nbr_vsize is set to 0.
-   A_mat->OverrideSize(nrows, nrows + face_nbr_vsize);
-
-   EnsureCapacity(A_mat->GetMemoryI(), nrows + 1);
-   EnsureCapacity(A_mat->GetMemoryJ(), nnz);
-   EnsureCapacity(A_mat->GetMemoryData(), nnz);
-
-   Array<int> nbr_info(nel_ho*3*2*dim);
-   auto h_nbr_info = Reshape(nbr_info.HostWrite(), nel_ho, 2*dim, 3);
-   const int num_faces = fes_ho.GetMesh()->GetNumFaces();
-   for (int f = 0; f < num_faces; f++)
-   {
-      Mesh::FaceInformation finfo = fes_ho.GetMesh()->GetFaceInformation(f);
-      int e0 = finfo.element[0].index;
-      int f0 = finfo.element[0].local_face_id;
-      if (finfo.IsBoundary())
-      {
-         h_nbr_info(e0,f0,0) = -1;
-         h_nbr_info(e0,f0,1)= -1;
-         h_nbr_info(e0,f0,2)= -1;
-      }
-      else if (finfo.IsShared())
-      {
-         // Face neighbors elements are indexed after the last local element
-         h_nbr_info(e0,f0,0) = nel_ho + finfo.element[1].index;
-         h_nbr_info(e0,f0,1)= finfo.element[1].orientation;
-         h_nbr_info(e0,f0,2)= finfo.element[1].local_face_id;
-      }
-      else if (finfo.IsInterior())
-      {
-         int e1 = finfo.element[1].index;
-         int f1 = finfo.element[1].local_face_id;
-         h_nbr_info(e0,f0,0) = e1;
-         h_nbr_info(e0,f0,1)= finfo.element[1].orientation;
-         h_nbr_info(e0,f0,2)= f1;
-         h_nbr_info(e1,f1,0) = e0;
-         h_nbr_info(e1,f1,1) = finfo.element[1].orientation;
-         h_nbr_info(e1,f1,2) = f0;
-      }
-   };
-
-   auto h_I = A_mat->HostWriteI();
-   h_I[0] = 0;
-   for (int i = 0; i < nrows; ++i)
-   {
-      const int iel_ho = i / ndof_per_el;
-      const int iloc = i % ndof_per_el;
-      static const int lex_map_2[4] = {3, 1, 0, 2};
-      static const int lex_map_3[6] = {4, 2, 1, 3, 0, 5};
-      const int local_i[3] = {iloc % pp1, (iloc/pp1)%pp1, iloc/pp1/pp1};
-      int bdr_count = 0;
-      for (int n_idx = 0; n_idx < dim; ++n_idx)
-      {
-         for (int e_i = 0; e_i < 2; ++e_i)
-         {
-            const int j_lex = e_i + n_idx*2;
-            const int f = (dim == 3) ? lex_map_3[j_lex]:lex_map_2[j_lex];
-            const bool boundary = (local_i[n_idx] == e_i * p);
-            if (boundary)
-            {
-               int neighbor_idx = h_nbr_info(iel_ho, f, 0);
-               if (neighbor_idx == -1)
-               {
-                  ++bdr_count;
-               }
-            }
-         }
-      }
-      h_I[i+1] = h_I[i] + (nnz_per_row - bdr_count);
-   }
-
-   const auto V = Reshape(sparse_ij.Read(), nnz_per_row, ndof_per_el, nel_ho);
-   auto J = A_mat->WriteJ();
-   auto AV = A_mat->WriteData();
-   auto I = A_mat->ReadI();
-
-   auto d_nbr_info = Reshape(nbr_info.Read(), nel_ho, 2*dim, 3);
-   mfem::forall(nrows, [=] MFEM_HOST_DEVICE (int i)
-   {
-      const int e = i / ndof_per_el;
-      const int iloc = i % ndof_per_el;
-      const int local_x = iloc % pp1;
-      const int local_y = (iloc/pp1)%pp1;
-      const int local_z = iloc/pp1/pp1;
-      const int local_i[3] = {local_x, local_y, local_z};
-      int offset = I[i];
-      static const int lex_map_2[4] = {3, 1, 0, 2};
-      static const int lex_map_3[6] = {4,2,1,3,0,5};
-      const int *lex_map = (dim == 2) ? lex_map_2 : lex_map_3;
-      AV[offset] = V(0, iloc, e);
-      J[offset] = i;
-      ++offset;
-      for (int n_idx = 0; n_idx < dim; ++n_idx)
-      {
-         // qi is the face lexicographic index, obtained by taking the
-         // lexicographic index of the coordinates ommiting n_idx.
-         int qi = 0;
-         int stride = 1;
-         for (int d = 0; d < dim; ++d)
-         {
-            if (d != n_idx)
-            {
-               qi += local_i[d]*stride;
-               stride *= pp1;
-            }
-         }
-         for (int e_i = 0; e_i < 2; ++e_i)
-         {
-            const int j_lex = e_i + n_idx*2;
-            const int f = lex_map[j_lex];
-            const bool bdr = (local_i[n_idx] == e_i * p);
-            if (bdr)
-            {
-               const int nbr_e = d_nbr_info(e, f, 0);
-               const int nbr_ori = d_nbr_info(e, f, 1);
-               const int nbr_f = d_nbr_info(e, f, 2);
-               if (nbr_e != -1)
-               {
-                  const int nbr_loc_idx = internal::FaceIdxToVolIdx(
-                                             dim, qi, pp1, f, nbr_f, 1, nbr_ori);
-                  J[offset] = nbr_e*ndof_per_el + nbr_loc_idx;
-                  AV[offset] = V(f+1, iloc, e);
-                  ++offset;
-               }
-            }
-            else
-            {
-               int shift = (e_i == 0) ? -1 : 1;
-               for (int n = 0; n < n_idx; ++n) { shift *= pp1; }
-               J[offset] = i + shift;
-               AV[offset] = V(f+1, iloc, e);
-               ++offset;
-            }
-         }
-      }
-   });
-}
-
 void BatchedLORAssembly::SparseIJToCSR(OperatorHandle &A) const
 {
    const int nvdof = fes_ho.GetVSize();
@@ -550,11 +372,12 @@ void BatchedLORAssembly::SparseIJToCSR(OperatorHandle &A) const
    }
 
    A_mat->OverrideSize(nvdof, nvdof);
-   EnsureCapacity(A_mat->GetMemoryI(), nvdof + 1);
 
-   const int nnz = FillI(*A_mat);
-   EnsureCapacity(A_mat->GetMemoryJ(), nnz);
-   EnsureCapacity(A_mat->GetMemoryData(), nnz);
+   A_mat->GetMemoryI().New(nvdof+1, Device::GetDeviceMemoryType());
+   int nnz = FillI(*A_mat);
+
+   A_mat->GetMemoryJ().New(nnz, Device::GetDeviceMemoryType());
+   A_mat->GetMemoryData().New(nnz, Device::GetDeviceMemoryType());
    FillJAndData(*A_mat);
 }
 
@@ -608,19 +431,6 @@ void BatchedLORAssembly::AssembleWithoutBC(BilinearForm &a, OperatorHandle &A)
    // Assemble the matrix, depending on what the form is.
    // This fills in the arrays sparse_ij and sparse_mapping.
    const FiniteElementCollection *fec = fes_ho.FEColl();
-
-   // Handle DG case separately, because assembly of CSR matrix requires
-   // handling face terms.
-   if (dynamic_cast<const DG_FECollection*>(fec))
-   {
-      if (HasIntegrators<DiffusionIntegrator, MassIntegrator>(a))
-      {
-         AssemblyKernel<BatchedLOR_DG>(a);
-      }
-      SparseIJToCSR_DG(A);
-      return;
-   }
-
    if (dynamic_cast<const H1_FECollection*>(fec))
    {
       if (HasIntegrators<DiffusionIntegrator, MassIntegrator>(a))
@@ -643,47 +453,10 @@ void BatchedLORAssembly::AssembleWithoutBC(BilinearForm &a, OperatorHandle &A)
       }
    }
 
-   SparseIJToCSR(A);
+   return SparseIJToCSR(A);
 }
 
 #ifdef MFEM_USE_MPI
-void BatchedLORAssembly::ParAssemble_DG(SparseMatrix &A_local,
-                                        OperatorHandle &A)
-{
-   auto &par_fes = static_cast<ParFiniteElementSpace&>(fes_ho);
-
-   // handle the case when 'a' contains off-diagonal
-   const int lvsize = par_fes.GetVSize();
-   const Array<HYPRE_BigInt> &face_nbr_glob_ldof =
-      par_fes.GetFaceNbrGlobalDofMapArray();
-   const HYPRE_BigInt ldof_offset = par_fes.GetMyDofOffset();
-
-   const int nnz_local = A_local.NumNonZeroElems();
-   Array<HYPRE_BigInt> glob_J(nnz_local);
-
-   const HYPRE_BigInt *d_face_nbr_glob_ldof = face_nbr_glob_ldof.Read();
-   const int *d_J = A_local.ReadJ();
-   HYPRE_BigInt *d_glob_J = glob_J.Write();
-
-   mfem::forall(nnz_local, [=] MFEM_HOST_DEVICE (int i)
-   {
-      if (d_J[i] < lvsize)
-      {
-         d_glob_J[i] = d_J[i] + ldof_offset;
-      }
-      else
-      {
-         d_glob_J[i] = d_face_nbr_glob_ldof[d_J[i] - lvsize];
-      }
-   });
-
-   A.Reset(new HypreParMatrix(
-              par_fes.GetComm(), lvsize, par_fes.GlobalVSize(),
-              par_fes.GlobalVSize(), A_local.HostReadWriteI(),
-              glob_J.HostReadWrite(), A_local.HostReadWriteData(),
-              par_fes.GetDofOffsets(), par_fes.GetDofOffsets()));
-}
-
 void BatchedLORAssembly::ParAssemble(
    BilinearForm &a, const Array<int> &ess_dofs, OperatorHandle &A)
 {
@@ -691,18 +464,13 @@ void BatchedLORAssembly::ParAssemble(
    OperatorHandle A_local;
    AssembleWithoutBC(a, A_local);
 
-   if (dynamic_cast<const DG_FECollection*>(fes_ho.FEColl()))
-   {
-      ParAssemble_DG(*A_local.As<SparseMatrix>(), A);
-   }
-   else
-   {
-      ParBilinearForm *pa =
-         dynamic_cast<ParBilinearForm*>(&a);
-      pa->ParallelRAP(*A_local.As<SparseMatrix>(), A, true);
-      A.As<HypreParMatrix>()->EliminateBC(ess_dofs,
-                                          Operator::DiagonalPolicy::DIAG_ONE);
-   }
+   ParBilinearForm *pa =
+      dynamic_cast<ParBilinearForm*>(&a);
+
+   pa->ParallelRAP(*A_local.As<SparseMatrix>(), A, true);
+
+   A.As<HypreParMatrix>()->EliminateBC(ess_dofs,
+                                       Operator::DiagonalPolicy::DIAG_ONE);
 }
 #endif
 
@@ -736,22 +504,12 @@ BatchedLORAssembly::BatchedLORAssembly(FiniteElementSpace &fes_ho_)
    FormLORVertexCoordinates(fes_ho, X_vert);
 }
 
-IntegrationRule GetLobattoIntRule(Geometry::Type geom, int nd1d)
+IntegrationRule GetCollocatedIntRule(FiniteElementSpace &fes)
 {
    IntegrationRules irs(0, Quadrature1D::GaussLobatto);
+   const Geometry::Type geom = fes.GetMesh()->GetTypicalElementGeometry();
+   const int nd1d = fes.GetMaxElementOrder() + 1;
    return irs.Get(geom, 2*nd1d - 3);
 }
 
-IntegrationRule GetCollocatedIntRule(FiniteElementSpace &fes)
-{
-   const Geometry::Type geom = fes.GetMesh()->GetTypicalElementGeometry();
-   return GetLobattoIntRule(geom, fes.GetMaxElementOrder() + 1);
-}
-
-IntegrationRule GetCollocatedFaceIntRule(FiniteElementSpace &fes)
-{
-   const Geometry::Type geom = fes.GetMesh()->GetTypicalFaceGeometry();
-   return GetLobattoIntRule(geom, fes.GetMaxElementOrder() + 1);
-}
-
 } // namespace mfem