Formatting.

Grouping Mesh methods in Doxygen [mesh-group-doc-dev]

.github/workflows/builds-and-tests.yml

@@ -162,7 +162,7 @@ jobs:
 
     - name: get hypre
       if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
-      uses: mfem/github-actions/build-hypre@v2.2
+      uses: mfem/github-actions/build-hypre@v2.4
       with:
         archive: ${{ env.HYPRE_ARCHIVE }}
         dir: ${{ env.HYPRE_TOP_DIR }}
@@ -171,7 +171,7 @@ jobs:
 
     - name: get hypre (Windows)
       if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
-      uses: mfem/github-actions/build-hypre@v2.2
+      uses: mfem/github-actions/build-hypre@v2.4
       with:
         archive: ${{ env.HYPRE_ARCHIVE }}
         dir: ${{ env.HYPRE_TOP_DIR }}
@@ -190,7 +190,7 @@ jobs:
 
     - name: install metis
       if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@v2.2
+      uses: mfem/github-actions/build-metis@v2.4
       with:
         archive: ${{ env.METIS_ARCHIVE }}
         dir: ${{ env.METIS_TOP_DIR }}
@@ -217,7 +217,7 @@ jobs:
 
     # MFEM build and test
     - name: build
-      uses: mfem/github-actions/build-mfem@v2.3
+      uses: mfem/github-actions/build-mfem@v2.4
       env:
         VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
       with:
@@ -263,13 +263,15 @@ jobs:
       if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os != 'ubuntu-latest'
       run: |
         CTEST_CONFIG="Release"
-        cd ${{ env.MFEM_TOP_DIR }}/build && ctest --output-on-failure -C ${CTEST_CONFIG}
+        cd ${{ env.MFEM_TOP_DIR }}/build && \
+        ctest --output-on-failure -C ${CTEST_CONFIG} || \
+        ctest --rerun-failed --output-on-failure -C ${CTEST_CONFIG}
       shell: bash
 
     # Code coverage (process and upload reports)
     - name: codecov
       if: matrix.codecov == 'YES'
-      uses: mfem/github-actions/upload-coverage@v2.2
+      uses: mfem/github-actions/upload-coverage@v2.4
       with:
         name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
         project_dir: ${{ env.MFEM_TOP_DIR }}

.github/workflows/mfem-analysis.yml

@@ -57,7 +57,7 @@ jobs:
 
     - name: Get Hypre
       if: steps.hypre-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-hypre@v2.2
+      uses: mfem/github-actions/build-hypre@v2.4
       with:
         archive: ${{ env.HYPRE_ARCHIVE }}
         dir: ${{ env.HYPRE_TOP_DIR }}
@@ -72,14 +72,14 @@ jobs:
 
     - name: Install Metis
       if: steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@v2.2
+      uses: mfem/github-actions/build-metis@v2.4
       with:
         archive: ${{ env.METIS_ARCHIVE }}
         dir: ${{ env.METIS_TOP_DIR }}
 
     # MFEM build and test
     - name: build-mfem
-      uses: mfem/github-actions/build-mfem@v2.2
+      uses: mfem/github-actions/build-mfem@v2.4
       with:
         os: ${{ runner.os }}
         target: opt

.github/workflows/mfem-sanitizer.yml

@@ -0,0 +1,70 @@
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+name: "Sanitizer"
+
+permissions:
+  actions: write
+
+on:
+  push:
+    branches:
+      - master
+      - next
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  Serial:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Cancel Previous Runs
+      uses: styfle/cancel-workflow-action@0.11.0
+      with:
+        access_token: ${{ github.token }}
+
+    - name: MFEM Checkout
+      uses: actions/checkout@v3
+      with:
+        path: mfem
+
+    - name: MFEM Build
+      uses: mfem/github-actions/build-mfem@v2.4
+      with:
+        os: ${{ runner.os }}
+        target: opt
+        mpi: seq
+        hypre-dir: unused-hypre-dir
+        metis-dir: unused-metis-dir
+        mfem-dir: mfem
+        build-system: make
+        library-only: false
+        config-options:
+          CXX="clang++-14"
+          CXXFLAGS="-g -O1 -std=c++11
+                    -fsanitize=address
+                    -fno-omit-frame-pointer
+                    -fsanitize-address-use-after-scope"
+
+    - name: MFEM Info
+      working-directory: mfem
+      run: make info
+
+    - name: MFEM Sanitize
+      working-directory: mfem
+      run:
+        ASAN_OPTIONS="detect_leaks=1,
+                      strict_init_order=1,
+                      strict_string_checks=1,
+                      check_initialization_order=1,
+                      detect_stack_use_after_return=1"
+        make test

.gitignore

@@ -29,6 +29,7 @@ CMakeFiles/
 config/_config.hpp
 config/config.mk
 config/sample-runs-build.log
+config/user.cmake
 config/user.mk
 doc/CodeDocumentation.conf
 doc/CodeDocumentation.html
@@ -214,6 +215,7 @@ miniapps/meshing/pmesh-fitting
 miniapps/meshing/minimal-surface
 miniapps/meshing/pminimal-surface
 miniapps/meshing/polar-nc
+miniapps/meshing/mesh-quality
 miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
 miniapps/meshing/toroid-*.mesh
@@ -316,12 +318,28 @@ miniapps/solvers/ParaView
 miniapps/solvers/mesh.*
 miniapps/solvers/sol.*
 
+miniapps/hdiv-linear-solver/darcy
+miniapps/hdiv-linear-solver/grad_div
+
 miniapps/parelag/MultilevelHcurlHdivSolver
 miniapps/parelag/*.mesh
 
 miniapps/multidomain/multidomain
 miniapps/hooke/hooke
 
+miniapps/dpg/diffusion
+miniapps/dpg/pdiffusion
+miniapps/dpg/convection-diffusion
+miniapps/dpg/pconvection-diffusion
+miniapps/dpg/acoustics
+miniapps/dpg/pacoustics
+miniapps/dpg/maxwell
+miniapps/dpg/pmaxwell
+miniapps/dpg/ParaView
+
+miniapps/spde/generate_random_field
+miniapps/spde/ParaView
+
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
@@ -334,6 +352,8 @@ tests/unit/tmop_pa_tests_*
 tests/unit/ptmop_pa_tests_*
 tests/unit/ceed_tests
 tests/unit/debug_device_tests
+tests/unit/parallel_in_serial.mesh
+tests/unit/parallel_in_serial.gf
 
 # Benchmark binaries
 tests/benchmarks/bench_ceed

.gitlab/configs/report-build-and-test.yml

@@ -22,12 +22,10 @@
         date
         echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
         # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
+        # every 5 seconds; we may want to add a counter for the number of
         # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
+        while ! flock -n 9; do
+          sleep 5
         done
         echo "Acquired lock on '$PWD/autotest.lock'"
         date
@@ -57,12 +55,10 @@
         date
         echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
         # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
+        # every 5 seconds; we may want to add a counter for the number of
         # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
+        while ! flock -n 9; do
+          sleep 5
         done
         echo "Acquired lock on '$PWD/autotest.lock'"
         date

.gitlab/configs/setup-baseline.yml

@@ -47,12 +47,10 @@ setup_baseline:
         date
         echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
         # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
+        # every 5 seconds; we may want to add a counter for the number of
         # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
+        while ! flock -n 9; do
+          sleep 5
         done
         echo "Acquired lock on '$PWD/autotest.lock'"
         date

.gitlab/configs/setup-build-and-test.yml

@@ -35,13 +35,11 @@ setup:
       (
         date
         echo "Waiting to acquire lock on '$PWD/mfem-data.lock' ..."
-        # try to get an exclusive lock on fd 9 (mfem-data.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
+        # try to get an exclusive lock on fd 9 (mfem-data.lock) repeating the
+        # try every 5 seconds; we may want to add a counter for the number of
         # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
+        while ! flock -n 9; do
+          sleep 5
         done
         echo "Acquired lock on '$PWD/mfem-data.lock'"
         date
@@ -69,12 +67,10 @@ setup:
         date
         echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
         # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
+        # every 5 seconds; we may want to add a counter for the number of
         # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
+        while ! flock -n 9; do
+          sleep 5
         done
         echo "Acquired lock on '$PWD/autotest.lock'"
         date

.gitlab/lassen-build-and-test.yml

@@ -14,14 +14,14 @@ stages:
   - build_and_test
   - report
 
-opt_mpi_cuda_xl_16_1_1_8:
+opt_mpi_cuda_xl_16_1_1_12:
   variables:
-    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=70"
+    SPEC: "%xl@16.1.1.12 +mpi +cuda cuda_arch=70"
   extends: .build_and_test_on_lassen
 
 opt_mpi_cuda_hypre_cuda_xl:
   variables:
-    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
+    SPEC: "%xl@16.1.1.12 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
   extends: .build_and_test_on_lassen
 
 # Jobs report

.gitlab/quartz-baseline.yml

@@ -51,6 +51,8 @@ cleanup:
   script:
     - echo "BUILD_ROOT=${BUILD_ROOT}"
     - rm -rf "${BUILD_ROOT}" || true
+    - echo "CI_PROJECT_DIR=${CI_PROJECT_DIR}"
+    - make -C "${CI_PROJECT_DIR}" distclean
 
 report_baseline:
   extends: [.on_quartz]
@@ -66,12 +68,10 @@ report_baseline:
         date
         echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
         # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
+        # every 5 seconds; we may want to add a counter for the number of
         # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
+        while ! flock -n 9; do
+          sleep 5
         done
         echo "Acquired lock on '$PWD/autotest.lock'"
         date
@@ -82,12 +82,14 @@ report_baseline:
         rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-${BASELINE_TEST}-${CI_COMMIT_REF_SLUG}"
         rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir ${rundir})
         cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
-        # We create an autotest-email.html file, because that's how we signal that there was a diff (temporary).
-        if [[ -f ${rundir}/${BASELINE_TEST}.err ]]; then
-          cp ${rundir}/${BASELINE_TEST}.err ${rundir}/autotest-email.html
-        fi
         printf "%s\n" "" "Pipeline URL:" "$CI_PIPELINE_URL" \
           >> ${rundir}/pipeline.txt
+        # We create an autotest-email.html file, because that's how we signal
+        # that there was an error / diff (temporary).
+        if [[ -f ${rundir}/${BASELINE_TEST}.err ]] || \
+           [[ -f ${rundir}/${BASELINE_TEST}-${SYS_TYPE}.diff ]]; then
+          cp ${rundir}/pipeline.txt ${rundir}/autotest-email.html
+        fi
         msg="GitLab CI log for ${BASELINE_TEST} on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
         if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
           git pull && \

.gitlab/quartz-build-and-test.yml

@@ -27,39 +27,39 @@ allocate_resource:
   timeout: 6h
 
 # GitLab jobs for the Quartz machine at LLNL
-debug_ser_gcc_6_1_0:
+debug_ser_gcc_10:
   variables:
-    SPEC: "%gcc@6.1.0 +debug~mpi"
+    SPEC: "%gcc@10.3.1 +debug~mpi"
   extends: .build_and_test_on_quartz
 
-debug_par_gcc_6_1_0:
+debug_par_gcc_10:
   variables:
-    SPEC: "%gcc@6.1.0 +debug+mpi"
+    SPEC: "%gcc@10.3.1 +debug+mpi"
   extends: .build_and_test_on_quartz
 
-opt_ser_gcc_6_1_0:
+opt_ser_gcc_10:
   variables:
-    SPEC: "%gcc@6.1.0 ~mpi"
+    SPEC: "%gcc@10.3.1 ~mpi"
   extends: .build_and_test_on_quartz
 
-opt_par_gcc_6_1_0:
+opt_par_gcc_10:
   variables:
-    SPEC: "%gcc@6.1.0"
+    SPEC: "%gcc@10.3.1"
   extends: .build_and_test_on_quartz
 
-opt_par_gcc_6_1_0_sundials:
+opt_par_gcc_10_sundials:
   variables:
-    SPEC: "%gcc@6.1.0 +sundials"
+    SPEC: "%gcc@10.3.1 +sundials"
   extends: .build_and_test_on_quartz
 
-opt_par_gcc_6_1_0_petsc:
+opt_par_gcc_10_petsc:
   variables:
-    SPEC: "%gcc@6.1.0 +petsc ^petsc+mumps~superlu-dist"
+    SPEC: "%gcc@10.3.1 +petsc ^petsc+mumps~superlu-dist"
   extends: .build_and_test_on_quartz
 
-opt_par_gcc_6_1_0_pumi:
+opt_par_gcc_10_pumi:
   variables:
-    SPEC: "%gcc@6.1.0 +pumi"
+    SPEC: "%gcc@10.3.1 +pumi"
   extends: .build_and_test_on_quartz
 
 # Release

.gitlab/scripts/baseline

@@ -42,47 +42,34 @@ fi
 # post
 mkdir ${artifacts_path}
 
-if [[ -s ${glob_err} ]]
-then
-  echo "ERROR during ${BASELINE_TEST} execution";
-  echo "Here is the ${glob_err} file content";
-  cat ${glob_err}
-  cp ${glob_err} ${artifacts_path}/${glob_err}
-  exit 1;
-elif [[ ! -f ${base_patch} && ! -f ${base_out} ]]
-then
-  echo "Something went WRONG in ${BASELINE_TEST}:";
-  echo "Either ${base_patch} or ${base_out} should exists";
-  exit 1;
-elif [[ -f ${base_patch} ]]
-then
-  echo "${BASELINE_TEST}: Differences found, patch generated"
-  cp ${base_patch} ${artifacts_path}/${base_patch}
-elif [[ -f ${base_out} ]]
-then
-  echo "${BASELINE_TEST}: Differences found, replacement file generated"
-  cp ${base_out} ${artifacts_path}/${base_out}
-fi
-
+status=0
 if [[ -f ${BASELINE_TEST}.out ]]; then
   cp ${BASELINE_TEST}.out ${artifacts_path}
 fi
-
+if [[ -s ${glob_err} ]]; then
+  echo "ERROR during ${BASELINE_TEST} execution"
+  echo "Here is the ${glob_err} file content"
+  cat ${glob_err}
+  cp ${glob_err} ${artifacts_path}/${glob_err}
+  status=1
+fi
+if [[ -f ${base_patch} ]]; then
+  echo "${BASELINE_TEST}: Differences found, patch generated"
+  cp ${base_patch} ${artifacts_path}/${base_patch}
+elif [[ -f ${base_out} ]]; then
+  echo "${BASELINE_TEST}: Differences found, replacement file generated"
+  cp ${base_out} ${artifacts_path}/${base_out}
+fi
 # base_diff won't even exist if there is no difference.
-if [[ -f ${base_diff} ]]
-then
+if [[ -f ${base_diff} ]]; then
   echo "${BASELINE_TEST}: Relevant differences (filtered diff) ..."
   cat ${base_diff}
   cp ${base_diff} ${artifacts_path}/${base_diff}
-  # We create a .err file, because that's how we signal that there was a diff.
-  cp ${base_diff} ${artifacts_path}/gitlab-${BASELINE_TEST}-${MACHINE_NAME}.err
+  status=1
 fi
-
-if [[ ! -s ${base_diff} ]]
-then
+if [[ $status -eq 0 ]]; then
   echo "${BASELINE_TEST}: PASSED"
-  true
 else
   echo "${BASELINE_TEST}: FAILED"
-  false
 fi
+exit $status

CHANGELOG

@@ -11,14 +11,41 @@
 Version 4.5.3 (development)
 ===========================
 
+New and updated examples and miniapps
+-------------------------------------
+- Added a new miniapp, Mesh Quality, for evaluating mesh quality using size,
+  skewness, and aspect-ratio computed from the Jacobian of the transformation.
+
+- Added a new miniapp for interface and boundary fitting to implicit domains
+  defined using level-set functions. See miniapps/meshing/pmesh-fitting.cpp
+
+- Added new Discontinuous Petrov-Galerkin (DPG) miniapp which includes serial
+  and parallel examples for diffusion, convection-diffusion, acoustics and
+  Maxwell equations. The miniapp includes new classes such as (Par)DPGWeakForm,
+  (Par)ComplexDPGWeakForm and (Complex)BlockStaticCondensation. Three new
+  integrators are added in support of DPG systems: TraceIntegrator,
+  NormalTraceIntegrator and TangentTraceIntegrator.
+
+- Added a new H(div) solvers miniapp in miniapps/hdiv-linear-solver,
+  demonstrating the use of a matrix-free saddle-point solver methodology,
+  suitable for high-order discretizations and for GPU acceleration. Examples
+  illustrating the solution of Darcy and grad-div problems are included.
+
+- Added a random refinement option to the mesh-explorer miniapp to assist users
+  in experimenting with nonconforming meshes.
+
+- Moved the distance solver methods from miniapps/shifted to miniapps/common.
+
+Meshing improvements
+--------------------
 - Added new methods in the Mesh class to set and get attributes on NURBS patches
   and patch boundaries.
 
-New and updated examples and miniapps
--------------------------------------
-- Added a miniapp pmesh-fitting in miniapps/meshing for interface and boundary fitting to implicit domains defined using level-set functions.
+- Added HIP support to the SUNDIALS interface.
 
-- Moved the distance solver methods from miniapps/shifted to miniapps/common. 
+- TMOP improvement: added asymptotically-balanced compound metrics 90, 94, 328,
+  338. Added the tmop-metric-magnitude tool for tracking how metrics change
+  under geometric perturbations.
 
 Discretization improvements
 ---------------------------
@@ -27,6 +54,36 @@ Discretization improvements
 
 - VectorFEBoundaryFluxLFIntegrator is now supported on device/GPU.
 
+- Added support for p-refined meshes in FindPointsGSLIB.
+
+- Added radial basis functions and reproducing kernels as alternatives to 
+  polynomial functions. These are accessed through the LocalKernelFECollection.
+
+Linear and nonlinear solvers
+----------------------------
+- Updated interface to MUMPS direct solver to support multiple right-hand
+  sides, block low-rank compression, builds using 64-bit integers, and other
+  improvements.
+
+- Added an interface to the MKL Pardiso sparse direct solver developed by Intel.
+  This interface provides a serial (OpenMP shared memory) version of Pardiso for
+  use with SparseMatrix. This complements the existing parallel (MPI distributed
+  memory) version already available through the CPardiso MFEM integration.
+
+Integrations, testing and documentation
+---------------------------------------
+- Added an address sanitizer GitHub action for a serial build/test on Ubuntu,
+  based on Clang/LLVM (https://clang.llvm.org/docs/AddressSanitizer.html).
+
+Miscellaneous
+-------------
+- Improved lambda body debugging with the addition of mfem::forall functions.
+  These functions can take the place of the MFEM_FORALL macros, which have been
+  preserved for backwards compatibility.
+
+- Reorganized files for bilinear form, linear form, and nonlinear form integrators
+  in the fem/integ/ subdirectory.
+
 
 Version 4.5.2, released on March 23, 2023
 =========================================

CMakeLists.txt

@@ -82,7 +82,7 @@ if (MFEM_USE_CONDUIT OR
    #  * find_package(PETSc REQUIRED)
    set(XSDK_ENABLE_C ON)
 endif()
-if (MFEM_USE_STRUMPACK)
+if (MFEM_USE_STRUMPACK OR MFEM_USE_MUMPS)
   # Just needed to find the MPI_Fortran libraries to link with
   set(XSDK_ENABLE_Fortran ON)
 endif()
@@ -317,6 +317,9 @@ if (MFEM_USE_SUNDIALS)
   if (MFEM_USE_CUDA)
     list(APPEND SUNDIALS_COMPONENTS NVector_Cuda)
   endif()
+  if (MFEM_USE_HIP)
+    list(APPEND SUNDIALS_COMPONENTS NVector_Hip)
+  endif()
   find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS})
 endif()
 
@@ -333,6 +336,7 @@ endif()
 if (MFEM_USE_MUMPS)
   if (MFEM_USE_MPI)
     find_package(MUMPS REQUIRED mumps_common pord)
+    set(MFEM_MUMPS_VERSION ${MUMPS_VERSION})
   else()
     message(FATAL_ERROR " *** MUMPS requires that MPI be enabled.")
   endif()
@@ -466,12 +470,18 @@ if (MFEM_USE_ADIOS2)
    find_package(ADIOS2 REQUIRED)
 endif()
 
+# MKL CPardiso
 if (MFEM_USE_MKL_CPARDISO)
   if (MFEM_USE_MPI)
    find_package(MKL_CPARDISO REQUIRED MKL_SEQUENTIAL MKL_LP64 MKL_MPI_WRAPPER)
   endif()
 endif()
 
+# MKL Pardiso
+if (MFEM_USE_MKL_PARDISO)
+   find_package(MKL_PARDISO REQUIRED MKL_SEQUENTIAL MKL_LP64)
+endif()
+
 # PARELAG
 if (MFEM_USE_PARELAG)
    find_package(PARELAG REQUIRED)
@@ -521,8 +531,8 @@ find_package(Threads REQUIRED)
 set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
     SUNDIALS PETSC SLEPC MUMPS AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
     NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
-    ADIOS2 CUBLAS CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG
-    MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)
+    ADIOS2 CUBLAS CUSPARSE MKL_CPARDISO MKL_PARDISO AMGX CALIPER CODIPACK
+    BENCHMARK PARELAG MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)
 
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")

CONTRIBUTING.md

@@ -121,6 +121,7 @@ The MFEM source code has the following structure:
   ├── fem
   │   ├── ceed
   │   ├── fe
+  │   ├── integ
   │   ├── lor
   │   ├── moonolith
   │   ├── qinterp
@@ -136,6 +137,7 @@ The MFEM source code has the following structure:
   │   ├── common
   │   ├── electromagnetics
   │   ├── gslib
+  │   ├── hdiv-linear-solver
   │   ├── hooke
   │   ├── meshing
   │   ├── mtop
@@ -209,7 +211,7 @@ device/host memory manager.
 - The main device-relevant classes and sources are:
   + [`Device`](https://docs.mfem.org/html/device_8hpp.html)
   + [`MemoryManager`](https://docs.mfem.org/html/mem_manager_8hpp.html)
-  + the [`MFEM_FORALL`](https://docs.mfem.org/html/forall_8hpp.html) macro
+  + the [`mfem::forall`](https://docs.mfem.org/html/forall_8hpp.html) function
   + the [`cuda.hpp`](https://docs.mfem.org/html/cuda_8hpp.html) and [`occa.hpp`](https://docs.mfem.org/html/occa_8hpp.html) files
 
 #### Utilities, building and documentation

INSTALL

@@ -628,9 +628,13 @@ The specific libraries and their options are:
   both MPI and hypre.
   If MFEM_USE_CUDA is enabled, we expect that SUNDIALS is built with support
   for CUDA.
-  URL: http://computation.llnl.gov/projects/sundials/sundials-software
+  If MFEM_USE_HIP is enabled, we expect that SUNDIALS is built with support
+  for HIP.
+  URL: http://computing.llnl.gov/projects/sundials/sundials-software
   Options: SUNDIALS_OPT, SUNDIALS_LIB.
-  Versions: SUNDIALS >= 5.0.0, SUNDIALS >= 5.4.0 for CUDA support.
+  Versions: SUNDIALS >= 5.0.0,
+            SUNDIALS >= 5.4.0 for CUDA support, and
+            SUNDIALS >= 5.7.0 for HIP support.
 
 - SuiteSparse (optional), used when MFEM_USE_SUITESPARSE = YES.
   URL: http://faculty.cse.tamu.edu/davis/suitesparse.html

config/cmake/MFEMConfig.cmake.in

@@ -55,6 +55,8 @@ set(MFEM_USE_SIMD @MFEM_USE_SIMD@)
 set(MFEM_USE_ADIOS2 @MFEM_USE_ADIOS2@)
 set(MFEM_USE_MOONOLITH @MFEM_USE_MOONOLITH@)
 set(MFEM_USE_CODIPACK @MFEM_USE_CODIPACK@)
+set(MFEM_USE_MKL_CPARDISO @MFEM_USE_MKL_CPARDISO@)
+set(MFEM_USE_MKL_PARDISO @MFEM_USE_MKL_PARDISO@)
 set(MFEM_USE_ADFORWARD @MFEM_USE_ADFORWARD@)
 set(MFEM_USE_CALIPER @MFEM_USE_CALIPER@)
 set(MFEM_USE_ALGOIM @MFEM_USE_ALGOIM@)

config/cmake/config.hpp.in

@@ -80,96 +80,101 @@
 // Internal MFEM option: enable group/batch allocation for some small objects.
 #cmakedefine MFEM_USE_MEMALLOC
 
+// Which library functions to use in class StopWatch for measuring time.
+// For a list of the available options, see INSTALL.
+// If not defined, an option is selected automatically.
+#cmakedefine MFEM_TIMER_TYPE @MFEM_TIMER_TYPE@
+
+// Enable MFEM functionality based on the SUNDIALS libraries.
+#cmakedefine MFEM_USE_SUNDIALS
+
 // Enable MFEM functionality based on the SuiteSparse library.
 #cmakedefine MFEM_USE_SUITESPARSE
 
 // Enable MFEM functionality based on the SuperLU_DIST library.
 #cmakedefine MFEM_USE_SUPERLU
+#cmakedefine MFEM_USE_SUPERLU5
 
 // Enable MFEM functionality based on the MUMPS library.
 #cmakedefine MFEM_USE_MUMPS
+#cmakedefine MFEM_MUMPS_VERSION @MFEM_MUMPS_VERSION@
 
 // Enable MFEM functionality based on the STRUMPACK library.
 #cmakedefine MFEM_USE_STRUMPACK
 
-// Enable functionality based on the Ginkgo library
+// Enable functionality based on the Ginkgo library.
 #cmakedefine MFEM_USE_GINKGO
 
-// Enable MFEM functionality based on the AmgX library
+// Enable MFEM functionality based on the AmgX library.
 #cmakedefine MFEM_USE_AMGX
 
-// Enable MFEM functionality based on the GnuTLS library
+// Enable secure socket streams based on the GNUTLS library.
 #cmakedefine MFEM_USE_GNUTLS
 
-// Enable MFEM functionality based on the GSLIB library
-#cmakedefine MFEM_USE_GSLIB
-
-// Enable MFEM functionality based on the NetCDF library
-#cmakedefine MFEM_USE_NETCDF
-
-// Enable MFEM functionality based on the PETSc library
-#cmakedefine MFEM_USE_PETSC
-
-// Enable MFEM functionality based on the SLEPc library
-#cmakedefine MFEM_USE_SLEPC
-
-// Enable MFEM functionality based on the Sidre library
+// Enable Sidre support.
 #cmakedefine MFEM_USE_SIDRE
 
-// Enable the use of SIMD in the high performance templated classes
+// Enable the use of SIMD in the high performance templated classes.
 #cmakedefine MFEM_USE_SIMD
 
-// Enable MFEM functionality based on the FMS library
+// Enable FMS support.
 #cmakedefine MFEM_USE_FMS
 
-// Enable MFEM functionality based on Conduit
+// Enable Conduit support.
 #cmakedefine MFEM_USE_CONDUIT
 
-// Enable MFEM functionality based on the PUMI library
+// Enable functionality based on the NetCDF library (reading CUBIT files).
+#cmakedefine MFEM_USE_NETCDF
+
+// Enable functionality based on the PETSc library.
+#cmakedefine MFEM_USE_PETSC
+
+// Enable functionality based on the SLEPc library.
+#cmakedefine MFEM_USE_SLEPC
+
+// Enable functionality based on the MPFR library.
+#cmakedefine MFEM_USE_MPFR
+
+// Enable MFEM functionality based on the PUMI library.
 #cmakedefine MFEM_USE_PUMI
 
-// Enable MFEM functionality based on the Moonolith library
+// Enable Moonolith-based general interpolation between finite element spaces.
 #cmakedefine MFEM_USE_MOONOLITH
 
-// Enable MFEM functionality based on the HiOp library
+// Enable MFEM functionality based on the HIOP library.
 #cmakedefine MFEM_USE_HIOP
 
-// Build the GPU/CUDA-enabled version of the MFEM library.
+// Enable MFEM functionality based on the GSLIB library.
+#cmakedefine MFEM_USE_GSLIB
+
+// Build the NVIDIA GPU/CUDA-enabled version of the MFEM library.
 // Requires a CUDA compiler (nvcc).
 #cmakedefine MFEM_USE_CUDA
 
-// Build the HIP-enabled version of the MFEM library.
+// Build the AMD GPU/HIP-enabled version of the MFEM library.
 // Requires a HIP compiler (hipcc).
 #cmakedefine MFEM_USE_HIP
 
-// Enable MFEM functionality based on the RAJA library
+// Enable functionality based on the RAJA library.
 #cmakedefine MFEM_USE_RAJA
 
-// Enable MFEM functionality based on the OCCA library
+// Enable functionality based on the OCCA library.
 #cmakedefine MFEM_USE_OCCA
 
-// Enable MFEM functionality based on the libCEED library
+// Enable functionality based on the libCEED library.
 #cmakedefine MFEM_USE_CEED
 
-// Enable MFEM functionality based on the Umpire library
-#cmakedefine MFEM_USE_UMPIRE
-
-// Enable MFEM functionality based on the ADIOS2 library
-#cmakedefine MFEM_USE_ADIOS2
-
-// Enable MFEM functionality based on the Caliper library
+// Enable functionality based on the Caliper library.
 #cmakedefine MFEM_USE_CALIPER
 
-// Enable MFEM functionality based on the Algoim library
+// Enable functionality based on the Algoim library.
 #cmakedefine MFEM_USE_ALGOIM
 
-// Which library functions to use in class StopWatch for measuring time.
-// For a list of the available options, see INSTALL.
-// If not defined, an option is selected automatically.
-#define MFEM_TIMER_TYPE @MFEM_TIMER_TYPE@
+// Enable functionality based on the Umpire library.
+#cmakedefine MFEM_USE_UMPIRE
 
-// Enable MFEM functionality based on the SUNDIALS libraries.
-#cmakedefine MFEM_USE_SUNDIALS
+// Enable IO functionality based on the ADIOS2 library.
+#cmakedefine MFEM_USE_ADIOS2
 
 // Version of HYPRE used for building MFEM.
 #cmakedefine MFEM_HYPRE_VERSION @MFEM_HYPRE_VERSION@
@@ -181,13 +186,16 @@
 // Enable interface to the MKL CPardiso library.
 #cmakedefine MFEM_USE_MKL_CPARDISO
 
-// Use forward mode for automatic differentiation
+// Enable interface to the MKL Pardiso library.
+#cmakedefine MFEM_USE_MKL_PARDISO
+
+// Use forward mode for automatic differentiation.
 #cmakedefine MFEM_USE_ADFORWARD
 
-// Enable the use of the CoDiPack library for AD
+// Enable the use of the CoDiPack library for AD.
 #cmakedefine MFEM_USE_CODIPACK
 
-// Enable MFEM functionality based on the Google Benchmark library.
+// Enable functionality based on the Google Benchmark library.
 #cmakedefine MFEM_USE_BENCHMARK
 
 // Enable Enzyme for AD

config/cmake/modules/FindMKL_PARDISO.cmake

@@ -0,0 +1,27 @@
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - MKL_PARDISO_FOUND
+#   - MKL_PARDISO_LIBRARIES
+#   - MKL_PARDISO_INCLUDE_DIRS
+
+if(NOT MKL_LIBRARY_DIR)
+  message(WARNING "Using default MKL library path. Double check the variable MKL_LIBRARY_DIR")
+  set(MKL_LIBRARY_DIR "lib/intel64")
+endif()
+
+include(MfemCmakeUtilities)
+mfem_find_package(MKL_PARDISO MKL_PARDISO
+    MKL_PARDISO_DIR "include" mkl_pardiso.h ${MKL_LIBRARY_DIR} mkl_core
+  "Paths to headers required by MKL Pardiso." "Libraries required by MKL PARDISO."
+  ADD_COMPONENT MKL_LP64 "include" "" ${MKL_LIBRARY_DIR} mkl_intel_lp64
+  ADD_COMPONENT MKL_SEQUENTIAL "include" "" ${MKL_LIBRARY_DIR} mkl_sequential)

config/cmake/modules/FindMUMPS.cmake

@@ -11,8 +11,9 @@
 
 # Sets the following variables:
 #   - MUMPS_FOUND
-#   - MUMPS_INCLUDE_DIRS
 #   - MUMPS_LIBRARIES
+#   - MUMPS_INCLUDE_DIRS
+#   - MUMPS_VERSION
 
 include(MfemCmakeUtilities)
 mfem_find_package(MUMPS MUMPS MUMPS_DIR
@@ -21,3 +22,18 @@ mfem_find_package(MUMPS MUMPS MUMPS_DIR
   "Libraries required by MUMPS."
   ADD_COMPONENT mumps_common "include" dmumps_c.h "lib" mumps_common
   ADD_COMPONENT pord "include" dmumps_c.h "lib" pord)
+
+if (MUMPS_FOUND AND (NOT MUMPS_VERSION))
+  try_run(MUMPS_VERSION_RUN_RESULT MUMPS_VERSION_COMPILE_RESULT
+          ${CMAKE_CURRENT_BINARY_DIR}/config
+          ${CMAKE_CURRENT_SOURCE_DIR}/config/get_mumps_version.cpp
+          CMAKE_FLAGS -DINCLUDE_DIRECTORIES:STRING=${MUMPS_INCLUDE_DIRS}
+          RUN_OUTPUT_VARIABLE MUMPS_VERSION_OUTPUT)
+  if ((MUMPS_VERSION_RUN_RESULT EQUAL 0) AND MUMPS_VERSION_OUTPUT)
+    string(STRIP "${MUMPS_VERSION_OUTPUT}" MUMPS_VERSION)
+    set(MUMPS_VERSION ${MUMPS_VERSION} CACHE STRING "MUMPS version." FORCE)
+    message(STATUS "Found MUMPS version ${MUMPS_VERSION}")
+  else()
+    message(FATAL_ERROR "Unable to determine MUMPS version.")
+  endif()
+endif()

config/cmake/modules/FindSUNDIALS.cmake

@@ -22,8 +22,8 @@ mfem_find_package(SUNDIALS SUNDIALS SUNDIALS_DIR
     "include" nvector/nvector_serial.h "lib" sundials_nvecserial
   ADD_COMPONENT NVector_Cuda
     "include" nvector/nvector_cuda.h "lib" sundials_nveccuda
-  ADD_COMPONENT NVector_ParHyp
-    "include" nvector/nvector_parhyp.h "lib" sundials_nvecparhyp
+  ADD_COMPONENT NVector_Hip
+    "include" nvector/nvector_hip.h "lib" sundials_nvechip
   ADD_COMPONENT NVector_Parallel
     "include" nvector/nvector_parallel.h "lib" sundials_nvecparallel
   ADD_COMPONENT NVector_MPIPlusX

config/config.hpp.in

@@ -30,10 +30,10 @@
 #define MFEM_VERSION_MINOR (((MFEM_VERSION)/100)%100)
 #define MFEM_VERSION_PATCH ((MFEM_VERSION)%100)
 
-// The absolute path of the MFEM source prefix
+// The absolute path of the MFEM source prefix.
 // #define MFEM_SOURCE_DIR "@MFEM_SOURCE_DIR@"
 
-// The absolute path of the MFEM installation prefix
+// The absolute path of the MFEM installation prefix.
 // #define MFEM_INSTALL_DIR "@MFEM_INSTALL_DIR@"
 
 // Description of the git commit used to build MFEM.
@@ -91,7 +91,7 @@
 // Enable MFEM functionality based on the SuiteSparse library.
 // #define MFEM_USE_SUITESPARSE
 
-// Enable MFEM functionality based on the SuperLU library.
+// Enable MFEM functionality based on the SuperLU_DIST library.
 // #define MFEM_USE_SUPERLU
 // #define MFEM_USE_SUPERLU5
 
@@ -102,40 +102,40 @@
 // Enable MFEM functionality based on the STRUMPACK library.
 // #define MFEM_USE_STRUMPACK
 
-// Enable MFEM features based on the Ginkgo library
+// Enable MFEM features based on the Ginkgo library.
 // #define MFEM_USE_GINKGO
 
 // Enable MFEM functionality based on the AmgX library.
 // #define MFEM_USE_AMGX
 
-// Enable secure socket streams based on the GNUTLS library
+// Enable secure socket streams based on the GNUTLS library.
 // #define MFEM_USE_GNUTLS
 
-// Enable Sidre support
+// Enable Sidre support.
 // #define MFEM_USE_SIDRE
 
-// Enable the use of SIMD in the high performance templated classes
+// Enable the use of SIMD in the high performance templated classes.
 // #define MFEM_USE_SIMD
 
-// Enable FMS support
+// Enable FMS support.
 // #define MFEM_USE_FMS
 
-// Enable Conduit support
+// Enable Conduit support.
 // #define MFEM_USE_CONDUIT
 
-// Enable functionality based on the NetCDF library (reading CUBIT files)
+// Enable functionality based on the NetCDF library (reading CUBIT files).
 // #define MFEM_USE_NETCDF
 
-// Enable functionality based on the PETSc library
+// Enable functionality based on the PETSc library.
 // #define MFEM_USE_PETSC
 
-// Enable functionality based on the SLEPc library
+// Enable functionality based on the SLEPc library.
 // #define MFEM_USE_SLEPC
 
 // Enable functionality based on the MPFR library.
 // #define MFEM_USE_MPFR
 
-// Enable MFEM functionality based on the PUMI library
+// Enable MFEM functionality based on the PUMI library.
 // #define MFEM_USE_PUMI
 
 // Enable Moonolith-based general interpolation between finite element spaces.
@@ -144,7 +144,7 @@
 // Enable MFEM functionality based on the HIOP library.
 // #define MFEM_USE_HIOP
 
-// Enable MFEM functionality based on the GSLIB library
+// Enable MFEM functionality based on the GSLIB library.
 // #define MFEM_USE_GSLIB
 
 // Build the NVIDIA GPU/CUDA-enabled version of the MFEM library.
@@ -186,10 +186,13 @@
 // Enable interface to the MKL CPardiso library.
 // #define MFEM_USE_MKL_CPARDISO
 
-// Use forward mode for automatic differentiation
+// Enable interface to the MKL Pardiso library.
+// #define MFEM_USE_MKL_PARDISO
+
+// Use forward mode for automatic differentiation.
 // #define MFEM_USE_ADFORWARD
 
-// Enable the use of the CoDiPack library for AD
+// Enable the use of the CoDiPack library for AD.
 // #define MFEM_USE_CODIPACK
 
 // Enable functionality based on the Google Benchmark library.

config/config.mk.in

@@ -57,6 +57,7 @@ MFEM_USE_UMPIRE        = @MFEM_USE_UMPIRE@
 MFEM_USE_SIMD          = @MFEM_USE_SIMD@
 MFEM_USE_ADIOS2        = @MFEM_USE_ADIOS2@
 MFEM_USE_MKL_CPARDISO  = @MFEM_USE_MKL_CPARDISO@
+MFEM_USE_MKL_PARDISO   = @MFEM_USE_MKL_PARDISO@
 MFEM_USE_MOONOLITH     = @MFEM_USE_MOONOLITH@
 MFEM_USE_ADFORWARD     = @MFEM_USE_ADFORWARD@
 MFEM_USE_CODIPACK      = @MFEM_USE_CODIPACK@

config/defaults.cmake

@@ -60,6 +60,7 @@ option(MFEM_USE_ADIOS2 "Enable ADIOS2" OFF)
 option(MFEM_USE_CALIPER "Enable Caliper support" OFF)
 option(MFEM_USE_ALGOIM "Enable Algoim support" OFF)
 option(MFEM_USE_MKL_CPARDISO "Enable MKL CPardiso" OFF)
+option(MFEM_USE_MKL_PARDISO "Enable MKL Pardiso" OFF)
 option(MFEM_USE_ADFORWARD "Enable forward mode for AD" OFF)
 option(MFEM_USE_CODIPACK "Enable automatic differentiation (AD) using CoDiPack" OFF)
 option(MFEM_USE_BENCHMARK "Enable Google Benchmark" OFF)
@@ -134,16 +135,18 @@ set(ParMETIS_DIR "${MFEM_DIR}/../parmetis-4.0.3" CACHE PATH
 set(ParMETIS_REQUIRED_PACKAGES "METIS" CACHE STRING
     "Additional packages required by ParMETIS.")
 
-set(SuperLUDist_DIR "${MFEM_DIR}/../SuperLU_DIST_6.3.1" CACHE PATH
+set(SuperLUDist_DIR "${MFEM_DIR}/../SuperLU_DIST_8.1.2" CACHE PATH
     "Path to the SuperLU_DIST library.")
 # SuperLU_DIST may also depend on "OpenMP", depending on how it was compiled.
-set(SuperLUDist_REQUIRED_PACKAGES "MPI" "BLAS" "ParMETIS" CACHE STRING
+set(SuperLUDist_REQUIRED_PACKAGES "MPI" "ParMETIS" "METIS"
+    "LAPACK" "BLAS" CACHE STRING
     "Additional packages required by SuperLU_DIST.")
 
-set(MUMPS_DIR "${MFEM_DIR}/../MUMPS_5.2.0" CACHE PATH
+set(MUMPS_DIR "${MFEM_DIR}/../MUMPS_5.5.0" CACHE PATH
     "Path to the MUMPS library.")
-# Packages required by MUMPS, depending on how it was compiled.
-set(MUMPS_REQUIRED_PACKAGES "MPI" "BLAS" "METIS" "ScaLAPACK" CACHE STRING
+# MUMPS may also depend on "OpenMP", depending on how it was compiled.
+set(MUMPS_REQUIRED_PACKAGES "MPI" "MPI_Fortran" "ParMETIS" "METIS"
+    "ScaLAPACK" "LAPACK" "BLAS" CACHE STRING
     "Additional packages required by MUMPS.")
 # If the MPI package does not find all required Fortran libraries:
 # set(MUMPS_REQUIRED_LIBRARIES "gfortran" "mpi_mpifh" CACHE STRING
@@ -226,6 +229,8 @@ set(MKL_CPARDISO_DIR "" CACHE STRING "MKL installation path.")
 set(MKL_MPI_WRAPPER_LIB "mkl_blacs_mpich_lp64" CACHE STRING "MKL MPI wrapper library")
 set(MKL_LIBRARY_DIR "" CACHE STRING "Custom library subdirectory")
 
+set(MKL_PARDISO_DIR "" CACHE STRING "MKL installation path.")
+
 set(OCCA_DIR "${MFEM_DIR}/../occa" CACHE PATH "Path to OCCA")
 set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")

config/defaults.mk

@@ -160,6 +160,7 @@ MFEM_USE_UMPIRE        = NO
 MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
 MFEM_USE_MKL_CPARDISO  = NO
+MFEM_USE_MKL_PARDISO   = NO
 MFEM_USE_MOONOLITH     = NO
 MFEM_USE_ADFORWARD     = NO
 MFEM_USE_CODIPACK      = NO
@@ -266,6 +267,9 @@ endif
 ifeq ($(MFEM_USE_CUDA),YES)
    SUNDIALS_LIB += -lsundials_nveccuda
 endif
+ifeq ($(MFEM_USE_HIP),YES)
+   SUNDIALS_LIB += -lsundials_nvechip
+endif
 # If SUNDIALS was built with KLU:
 # MFEM_USE_SUITESPARSE = YES
 
@@ -284,10 +288,10 @@ ifeq ($(MFEM_USE_SUPERLU5),YES)
    SUPERLU_LIB = $(XLINKER)-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib\
       -lsuperlu_dist_5.1.0
 else
-   SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_6.3.1
+   SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_8.1.2
    SUPERLU_OPT = -I$(SUPERLU_DIR)/include
    SUPERLU_LIB = $(XLINKER)-rpath,$(SUPERLU_DIR)/lib64 -L$(SUPERLU_DIR)/lib64\
-      -lsuperlu_dist -lblas
+      -lsuperlu_dist $(LAPACK_LIB)
 endif
 
 # SCOTCH library configuration (required by STRUMPACK <= v2.1.0, optional in
@@ -311,7 +315,7 @@ MPI_FORTRAN_LIB = -lmpifort
 # MPI_FORTRAN_LIB += -lgfortran
 
 # MUMPS library configuration
-MUMPS_DIR = @MFEM_DIR@/../MUMPS_5.2.0
+MUMPS_DIR = @MFEM_DIR@/../MUMPS_5.5.0
 MUMPS_OPT = -I$(MUMPS_DIR)/include
 MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib -ldmumps\
  -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB)
@@ -484,7 +488,6 @@ ifdef GOTCHA_DIR
    CALIPER_LIB += $(XLINKER)-rpath,$(GOTCHA_DIR)/lib64 $(XLINKER)-rpath,$(GOTCHA_DIR)/lib -L$(GOTCHA_DIR)/lib64 -L$(GOTCHA_DIR)/lib -lgotcha
 endif
 
-
 # BLITZ library configuration
 BLITZ_DIR = @MFEM_DIR@/../blitz
 BLITZ_OPT = -I$(BLITZ_DIR)/include
@@ -539,6 +542,14 @@ MKL_CPARDISO_LIB = $(XLINKER)-rpath,$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
    -L$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR) -l$(MKL_MPI_WRAPPER)\
    -lmkl_intel_lp64 -lmkl_sequential -lmkl_core
 
+# MKL Pardiso library configuration
+MKL_PARDISO_DIR ?=
+MKL_LIBRARY_SUBDIR ?= lib
+MKL_PARDISO_OPT = -I$(MKL_PARDISO_DIR)/include
+MKL_PARDISO_LIB = $(XLINKER)-rpath,$(MKL_PARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
+   -L$(MKL_PARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
+   -lmkl_intel_lp64 -lmkl_sequential -lmkl_core
+
 # PARELAG library configuration
 PARELAG_DIR = @MFEM_DIR@/../parelag
 PARELAG_OPT = -I$(PARELAG_DIR)/src -I$(PARELAG_DIR)/build/src

doc/CodeDocumentation.conf.in

@@ -795,6 +795,7 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/miniapps/common \
                          @MFEM_SOURCE_DIR@/miniapps/electromagnetics \
                          @MFEM_SOURCE_DIR@/miniapps/gslib \
+                         @MFEM_SOURCE_DIR@/miniapps/hdiv-linear-solver \
                          @MFEM_SOURCE_DIR@/miniapps/hooke \
                          @MFEM_SOURCE_DIR@/miniapps/hooke/kernels \
                          @MFEM_SOURCE_DIR@/miniapps/hooke/materials \
@@ -810,7 +811,10 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/miniapps/shifted \
                          @MFEM_SOURCE_DIR@/miniapps/solvers \
                          @MFEM_SOURCE_DIR@/miniapps/tools \
-                         @MFEM_SOURCE_DIR@/miniapps/toys
+                         @MFEM_SOURCE_DIR@/miniapps/toys \
+                         @MFEM_SOURCE_DIR@/miniapps/spde \
+                         @MFEM_SOURCE_DIR@/miniapps/dpg \
+                         @MFEM_SOURCE_DIR@/miniapps/dpg/util
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses

doc/CodeDocumentation.dox

@@ -39,7 +39,7 @@ namespace mfem {
  * - Device
  * - Memory
  * - MemoryManager
- * - MFEM_FORALL macro in forall.hpp
+ * - mfem::forall functions in forall.hpp
  *
  * <H3>Example codes</H3>
  * - <a class="el" href="ex0_8cpp_source.html">Example 0</a>: simplest example, nodal H1 FEM for the Laplace problem
@@ -186,6 +186,7 @@ namespace mfem {
  * - <a class="el" href="extruder_8cpp_source.html">Extruder</a>: extrude a low-dimensional mesh into a higher dimension
  * - <a class="el" href="mesh-explorer_8cpp_source.html">Mesh Explorer</a>: visualize and manipulate meshes
  * - <a class="el" href="mesh-optimizer_8cpp_source.html">Mesh Optimizer</a>: optimize high-order meshes, <a class="el" href="mesh-optimizer_8cpp_source.html">serial</a> and <a class="el" href="pmesh-optimizer_8cpp_source.html">parallel</a> versions
+ * - <a class="el" href="mesh-quality_8cpp_source.html">Mesh Quality</a>: visualize and check mesh quality
  * - <a class="el" href="trimmer_8cpp_source.html">Trimmer</a>: trim elements from existing meshes
  * - <a class="el" href="display-basis_8cpp_source.html">Display Basis</a>: visualize finite element basis functions
  * - <a class="el" href="get-values_8cpp_source.html">Get Values</a>: extract field values via DataCollection classes
@@ -198,12 +199,15 @@ namespace mfem {
  * - <a class="el" href="distance_8cpp_source.html">Distance</a>: finite element distance function solver
  * - <a class="el" href="diffusion_8cpp_source.html">Shifted Diffusion</a>: shifted boundary diffusion solver
  * - <a class="el" href="extrapolate_8cpp_source.html">Extrapolation</a>: PDE-based extrapolation of finite element functions
- * - <a class="el" href="distance_8cpp_source.html">Block Solvers</a>: comparison of saddle point system solvers
+ * - <a class="el" href="block-solvers_8cpp_source.html">Block Solvers</a>: comparison of saddle point system solvers
  * - <a class="el" href="parheat_8cpp_source.html">Optimization gradients</a>: Gradients of PDE-constrained function
  * - <a class="el" href="par__example_8cpp_source.html">Parallel AD</a>: Parallel p-Laplacian example
  * - <a class="el" href="seq__example_8cpp_source.html">Serial AD</a>: Serial p-Laplacian example
  * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
  * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
+ * - <a class="el" href="generate__random__field_8cpp_source.html">SPDE Solvers</a>: SPDE solver random field generation
+ * - <a class="el" href="pdiffusion_8cpp_source.html">DPG Diffusion example</a>: DPG formulation for the diffusion problem
+ * - <a class="el" href="pmaxwell_8cpp_source.html">DPG Maxwell example</a>: DPG formulation for the indefinite Maxwell problem
  *
  * See also the <a class="el" href="https://mfem.org/examples/">examples documentation</a> online.
  */

examples/CMakeLists.txt

@@ -161,6 +161,15 @@ if (MFEM_ENABLE_TESTING)
       $<TARGET_FILE:ex11p> "-no-vis" "--superlu"
       ${MPIEXEC_POSTFLAGS})
   endif()
+
+  # If MUMPS is enabled, add a test run that uses it.
+  if (MFEM_USE_MUMPS)
+    add_test(NAME ex25p_mumps_np=${MFEM_MPI_NP}
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:ex25p> "-no-vis" "--mumps-solver"
+      ${MPIEXEC_POSTFLAGS})
+  endif()
 endif()
 
 # Include the examples/amgx directory if AmgX is enabled

examples/ex14.cpp

@@ -18,6 +18,8 @@
 //               ex14 -m ../data/amr-quad.mesh -r 3
 //               ex14 -m ../data/amr-hex.mesh
 //               ex14 -m ../data/fichera-amr.mesh
+//               ex14 -m ../data/inline-quad.mesh -r 2 -rk
+//               ex14 -m ../data/star.mesh -r 0 -o 3 -rk
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               discontinuous Galerkin (DG) finite element discretization of
@@ -43,10 +45,11 @@ int main(int argc, char *argv[])
    const char *mesh_file = "../data/star.mesh";
    int ref_levels = -1;
    int order = 1;
+   bool rk = false;
    double sigma = -1.0;
    double kappa = -1.0;
    double eta = 0.0;
-   bool visualization = 1;
+   bool visualization = true;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -54,7 +57,9 @@ int main(int argc, char *argv[])
    args.AddOption(&ref_levels, "-r", "--refine",
                   "Number of times to refine the mesh uniformly, -1 for auto.");
    args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) >= 0.");
+                  "Finite element order (polynomial degree) >= 0 OR reproducing kernel order.");
+   args.AddOption(&rk, "-rk", "--rk", "-no-rk", "--no-rk",
+                  "Use reproducing kernel functions");
    args.AddOption(&sigma, "-s", "--sigma",
                   "One of the three DG penalty parameters, typically +1/-1."
                   " See the documentation of class DGDiffusionIntegrator.");
@@ -75,6 +80,10 @@ int main(int argc, char *argv[])
    {
       kappa = (order+1)*(order+1);
    }
+   if (rk && sigma < 0.0)
+   {
+      sigma = 1.0;
+   }
    args.PrintOptions(cout);
 
    // 2. Read the mesh from the given mesh file. We can handle triangular,
@@ -104,7 +113,15 @@ int main(int argc, char *argv[])
 
    // 4. Define a finite element space on the mesh. Here we use discontinuous
    //    finite elements of the specified order >= 0.
-   FiniteElementCollection *fec = new DG_FECollection(order, dim);
+   FiniteElementCollection *fec;
+   if (rk)
+   {
+      fec = new LocalKernelFECollection(dim, 4, 6, order, 1.01 + order, 0.0);
+   }
+   else
+   {
+      fec = new DG_FECollection(order, dim);
+   }
    FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
    cout << "Number of unknowns: " << fespace->GetVSize() << endl;
 

examples/ex14p.cpp

@@ -17,6 +17,8 @@
 //               mpirun -np 4 ex14p -m ../data/inline-segment.mesh -rs 5
 //               mpirun -np 4 ex14p -m ../data/amr-quad.mesh -rs 3
 //               mpirun -np 4 ex14p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex14p -m ../data/star.mesh -rs 1 -o 2 -rk
+//               mpirun -np 4 ex14p -m ../data/amr-quad.mesh -rs 2 -rk
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               discontinuous Galerkin (DG) finite element discretization of
@@ -81,6 +83,7 @@ int main(int argc, char *argv[])
    int ser_ref_levels = -1;
    int par_ref_levels = 2;
    int order = 1;
+   bool rk = false;
    double sigma = -1.0;
    double kappa = -1.0;
    double eta = 0.0;
@@ -96,6 +99,8 @@ int main(int argc, char *argv[])
                   "Number of times to refine the mesh uniformly in parallel.");
    args.AddOption(&order, "-o", "--order",
                   "Finite element order (polynomial degree) >= 0.");
+   args.AddOption(&rk, "-rk", "--rk", "-no-rk", "--no-rk",
+                  "Use reproducing kernel functions");
    args.AddOption(&sigma, "-s", "--sigma",
                   "One of the three DG penalty parameters, typically +1/-1."
                   " See the documentation of class DGDiffusionIntegrator.");
@@ -119,6 +124,10 @@ int main(int argc, char *argv[])
    {
       kappa = (order+1)*(order+1);
    }
+   if (rk && sigma < 0.0)
+   {
+      sigma = 1.0;
+   }
    if (myid == 0)
    {
       args.PrintOptions(cout);
@@ -163,7 +172,15 @@ int main(int argc, char *argv[])
 
    // 6. Define a parallel finite element space on the parallel mesh. Here we
    //    use discontinuous finite elements of the specified order >= 0.
-   FiniteElementCollection *fec = new DG_FECollection(order, dim);
+   FiniteElementCollection *fec;
+   if (rk)
+   {
+      fec = new LocalKernelFECollection(dim, 4, 6, order, 1.01 + order, 0.0);
+   }
+   else
+   {
+      fec = new DG_FECollection(order, dim);
+   }
    ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
    HYPRE_BigInt size = fespace->GlobalTrueVSize();
    if (myid == 0)

examples/ex18.cpp

@@ -9,6 +9,8 @@
 //       ex18 -p 1 -r 0 -o 5 -s 6
 //       ex18 -p 2 -r 1 -o 1 -s 3
 //       ex18 -p 2 -r 0 -o 3 -s 3
+//       ex18 -p 1 -r 1 -o 3 -s 4 -rk
+//       ex18 -p 2 -r 0 -o 3 -s 3 -rk
 //
 // Description:  This example code solves the compressible Euler system of
 //               equations, a model nonlinear hyperbolic PDE, with a
@@ -65,6 +67,7 @@ int main(int argc, char *argv[])
    const char *mesh_file = "../data/periodic-square.mesh";
    int ref_levels = 1;
    int order = 3;
+   bool rk = false;
    int ode_solver_type = 4;
    double t_final = 2.0;
    double dt = -0.01;
@@ -84,6 +87,8 @@ int main(int argc, char *argv[])
                   "Number of times to refine the mesh uniformly.");
    args.AddOption(&order, "-o", "--order",
                   "Order (degree) of the finite elements.");
+   args.AddOption(&rk, "-rk", "--rk", "-no-rk", "--no-rk",
+                  "Use reproducing kernel functions");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
                   "ODE solver: 1 - Forward Euler,\n\t"
                   "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6.");
@@ -139,13 +144,21 @@ int main(int argc, char *argv[])
 
    // 5. Define the discontinuous DG finite element space of the given
    //    polynomial order on the refined mesh.
-   DG_FECollection fec(order, dim);
+   FiniteElementCollection *fec;
+   if (rk)
+   {
+      fec = new LocalKernelFECollection(dim, 5, 7, order, 2.01 + order, 1.0);
+   }
+   else
+   {
+      fec = new DG_FECollection(order, dim);
+   }
    // Finite element space for a scalar (thermodynamic quantity)
-   FiniteElementSpace fes(&mesh, &fec);
+   FiniteElementSpace fes(&mesh, fec);
    // Finite element space for a mesh-dim vector quantity (momentum)
-   FiniteElementSpace dfes(&mesh, &fec, dim, Ordering::byNODES);
+   FiniteElementSpace dfes(&mesh, fec, dim, Ordering::byNODES);
    // Finite element space for all variables together (total thermodynamic state)
-   FiniteElementSpace vfes(&mesh, &fec, num_equation, Ordering::byNODES);
+   FiniteElementSpace vfes(&mesh, fec, num_equation, Ordering::byNODES);
 
    // This example depends on this ordering of the space.
    MFEM_ASSERT(fes.GetOrdering() == Ordering::byNODES, "");
@@ -304,6 +317,7 @@ int main(int argc, char *argv[])
 
    // Free the used memory.
    delete ode_solver;
+   delete fec;
 
    return 0;
 }

examples/ex18p.cpp

@@ -9,6 +9,8 @@
 //       mpirun -np 4 ex18p -p 1 -rs 1 -rp 1 -o 5 -s 6
 //       mpirun -np 4 ex18p -p 2 -rs 1 -rp 1 -o 1 -s 3
 //       mpirun -np 4 ex18p -p 2 -rs 1 -rp 1 -o 3 -s 3
+//       mpirun -np 4 ex18p -p 1 -rs 1 -rp 0 -o 3 -s 4 -rk
+//       mpirun -np 4 ex18p -p 2 -rs 1 -rp 0 -o 3 -s 3 -rk
 //
 // Description:  This example code solves the compressible Euler system of
 //               equations, a model nonlinear hyperbolic PDE, with a
@@ -70,6 +72,7 @@ int main(int argc, char *argv[])
    int ser_ref_levels = 0;
    int par_ref_levels = 1;
    int order = 3;
+   bool rk = false;
    int ode_solver_type = 4;
    double t_final = 2.0;
    double dt = -0.01;
@@ -93,6 +96,8 @@ int main(int argc, char *argv[])
                   " partitioning.");
    args.AddOption(&order, "-o", "--order",
                   "Order (degree) of the finite elements.");
+   args.AddOption(&rk, "-rk", "--rk", "-no-rk", "--no-rk",
+                  "Use reproducing kernel functions");
    args.AddOption(&ode_solver_type, "-s", "--ode-solver",
                   "ODE solver: 1 - Forward Euler,\n\t"
                   "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6.");
@@ -161,13 +166,21 @@ int main(int argc, char *argv[])
 
    // 7. Define the discontinuous DG finite element space of the given
    //    polynomial order on the refined mesh.
-   DG_FECollection fec(order, dim);
+   FiniteElementCollection *fec;
+   if (rk)
+   {
+      fec = new LocalKernelFECollection(dim, 5, 7, order, 2.01 + order, 1.0);
+   }
+   else
+   {
+      fec = new DG_FECollection(order, dim);
+   }
    // Finite element space for a scalar (thermodynamic quantity)
-   ParFiniteElementSpace fes(&pmesh, &fec);
+   ParFiniteElementSpace fes(&pmesh, fec);
    // Finite element space for a mesh-dim vector quantity (momentum)
-   ParFiniteElementSpace dfes(&pmesh, &fec, dim, Ordering::byNODES);
+   ParFiniteElementSpace dfes(&pmesh, fec, dim, Ordering::byNODES);
    // Finite element space for all variables together (total thermodynamic state)
-   ParFiniteElementSpace vfes(&pmesh, &fec, num_equation, Ordering::byNODES);
+   ParFiniteElementSpace vfes(&pmesh, fec, num_equation, Ordering::byNODES);
 
    // This example depends on this ordering of the space.
    MFEM_ASSERT(fes.GetOrdering() == Ordering::byNODES, "");
@@ -375,6 +388,7 @@ int main(int argc, char *argv[])
 
    // Free the used memory.
    delete ode_solver;
+   delete fec;
 
    return 0;
 }

examples/ex25.cpp

@@ -32,6 +32,7 @@
 //               We recommend viewing Example 22 before viewing this example.
 
 #include "mfem.hpp"
+#include <memory>
 #include <fstream>
 #include <iostream>
 
@@ -44,7 +45,7 @@ using namespace std;
 using namespace mfem;
 
 // Class for setting up a simple Cartesian PML region
-class CartesianPML
+class PML
 {
 private:
    Mesh *mesh;
@@ -69,7 +70,7 @@ private:
 
 public:
    // Constructor
-   CartesianPML(Mesh *mesh_,Array2D<double> length_);
+   PML(Mesh *mesh_,Array2D<double> length_);
 
    // Return Computational Domain Boundary
    Array2D<double> GetCompDomainBdr() {return comp_dom_bdr;}
@@ -91,12 +92,12 @@ public:
 class PMLDiagMatrixCoefficient : public VectorCoefficient
 {
 private:
-   CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML *, Vector &);
+   PML * pml = nullptr;
+   void (*Function)(const Vector &, PML *, Vector &);
 public:
-   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, PML *,
                                               Vector &),
-                            CartesianPML * pml_)
+                            PML * pml_)
       : VectorCoefficient(dim), pml(pml_), Function(F)
    {}
 
@@ -125,13 +126,13 @@ void source(const Vector &x, Vector & f);
 
 // Functions for computing the necessary coefficients after PML stretching.
 // J is the Jacobian matrix of the stretching function
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_JT_J_inv_Re(const Vector &x, PML * pml, Vector &D);
+void detJ_JT_J_inv_Im(const Vector &x, PML * pml, Vector &D);
+void detJ_JT_J_inv_abs(const Vector &x, PML * pml, Vector &D);
 
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_inv_JT_J_Re(const Vector &x, PML * pml, Vector &D);
+void detJ_inv_JT_J_Im(const Vector &x, PML * pml, Vector &D);
+void detJ_inv_JT_J_abs(const Vector &x, PML * pml, Vector &D);
 
 Array2D<double> comp_domain_bdr;
 Array2D<double> domain_bdr;
@@ -267,7 +268,7 @@ int main(int argc, char *argv[])
          length = 0.25;
          break;
    }
-   CartesianPML * pml = new CartesianPML(mesh,length);
+   PML * pml = new PML(mesh,length);
    comp_domain_bdr = pml->GetCompDomainBdr();
    domain_bdr = pml->GetDomainBdr();
 
@@ -467,16 +468,14 @@ int main(int argc, char *argv[])
       offsets[2] = fespace->GetTrueVSize();
       offsets.PartialSum();
 
-      Operator *pc_r = nullptr;
-      Operator *pc_i = nullptr;
+      std::unique_ptr<Operator> pc_r;
+      std::unique_ptr<Operator> pc_i;
       double s = (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0;
       if (pa)
       {
          // Jacobi Smoother
-         OperatorJacobiSmoother *d00 = new OperatorJacobiSmoother(prec, ess_tdof_list);
-         ScaledOperator *d11 = new ScaledOperator(d00, s);
-         pc_r = d00;
-         pc_i = d11;
+         pc_r.reset(new OperatorJacobiSmoother(prec, ess_tdof_list));
+         pc_i.reset(new ScaledOperator(pc_r.get(), s));
       }
       else
       {
@@ -485,15 +484,13 @@ int main(int argc, char *argv[])
          prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
 
          // Gauss-Seidel Smoother
-         GSSmoother *gs00 = new GSSmoother(*PCOpAh.As<SparseMatrix>());
-         ScaledOperator *gs11 = new ScaledOperator(gs00, s);
-         pc_r = gs00;
-         pc_i = gs11;
+         pc_r.reset(new GSSmoother(*PCOpAh.As<SparseMatrix>()));
+         pc_i.reset(new ScaledOperator(pc_r.get(), s));
       }
 
       BlockDiagonalPreconditioner BlockDP(offsets);
-      BlockDP.SetDiagonalBlock(0, pc_r);
-      BlockDP.SetDiagonalBlock(1, pc_i);
+      BlockDP.SetDiagonalBlock(0, pc_r.get());
+      BlockDP.SetDiagonalBlock(1, pc_i.get());
 
       GMRESSolver gmres;
       gmres.SetPrintLevel(1);
@@ -807,7 +804,7 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
    }
 }
 
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_JT_J_inv_Re(const Vector &x, PML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det(1.0, 0.0);
@@ -824,7 +821,7 @@ void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D)
    }
 }
 
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_JT_J_inv_Im(const Vector &x, PML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -841,7 +838,7 @@ void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D)
    }
 }
 
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_JT_J_inv_abs(const Vector &x, PML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -858,7 +855,7 @@ void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D)
    }
 }
 
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_inv_JT_J_Re(const Vector &x, PML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det(1.0, 0.0);
@@ -883,7 +880,7 @@ void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D)
    }
 }
 
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_inv_JT_J_Im(const Vector &x, PML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -907,7 +904,7 @@ void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D)
    }
 }
 
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_inv_JT_J_abs(const Vector &x, PML * pml, Vector &D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -931,14 +928,14 @@ void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D)
    }
 }
 
-CartesianPML::CartesianPML(Mesh *mesh_, Array2D<double> length_)
+PML::PML(Mesh *mesh_, Array2D<double> length_)
    : mesh(mesh_), length(length_)
 {
    dim = mesh->Dimension();
    SetBoundaries();
 }
 
-void CartesianPML::SetBoundaries()
+void PML::SetBoundaries()
 {
    comp_dom_bdr.SetSize(dim, 2);
    dom_bdr.SetSize(dim, 2);
@@ -953,7 +950,7 @@ void CartesianPML::SetBoundaries()
    }
 }
 
-void CartesianPML::SetAttributes(Mesh *mesh_)
+void PML::SetAttributes(Mesh *mesh_)
 {
    // Initialize bdr attributes
    for (int i = 0; i < mesh_->GetNBE(); ++i)
@@ -1002,8 +999,8 @@ void CartesianPML::SetAttributes(Mesh *mesh_)
    mesh_->SetAttributes();
 }
 
-void CartesianPML::StretchFunction(const Vector &x,
-                                   vector<complex<double>> &dxs)
+void PML::StretchFunction(const Vector &x,
+                          vector<complex<double>> &dxs)
 {
    complex<double> zi = complex<double>(0., 1.);
 

examples/ex25p.cpp

@@ -44,7 +44,7 @@ using namespace std;
 using namespace mfem;
 
 // Class for setting up a simple Cartesian PML region
-class CartesianPML
+class PML
 {
 private:
    Mesh *mesh;
@@ -69,7 +69,7 @@ private:
 
 public:
    // Constructor
-   CartesianPML(Mesh *mesh_,Array2D<double> length_);
+   PML(Mesh *mesh_,Array2D<double> length_);
 
    // Return Computational Domain Boundary
    Array2D<double> GetCompDomainBdr() {return comp_dom_bdr;}
@@ -91,12 +91,12 @@ public:
 class PMLDiagMatrixCoefficient : public VectorCoefficient
 {
 private:
-   CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML *, Vector &);
+   PML * pml = nullptr;
+   void (*Function)(const Vector &, PML *, Vector &);
 public:
-   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, PML *,
                                               Vector &),
-                            CartesianPML * pml_)
+                            PML * pml_)
       : VectorCoefficient(dim), pml(pml_), Function(F)
    {}
 
@@ -125,13 +125,13 @@ void source(const Vector &x, Vector & f);
 
 // Functions for computing the necessary coefficients after PML stretching.
 // J is the Jacobian matrix of the stretching function
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_JT_J_inv_Re(const Vector &x, PML * pml, Vector & D);
+void detJ_JT_J_inv_Im(const Vector &x, PML * pml, Vector & D);
+void detJ_JT_J_inv_abs(const Vector &x, PML * pml, Vector & D);
 
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_inv_JT_J_Re(const Vector &x, PML * pml, Vector & D);
+void detJ_inv_JT_J_Im(const Vector &x, PML * pml, Vector & D);
+void detJ_inv_JT_J_abs(const Vector &x, PML * pml, Vector & D);
 
 Array2D<double> comp_domain_bdr;
 Array2D<double> domain_bdr;
@@ -295,7 +295,7 @@ int main(int argc, char *argv[])
          length = 0.25;
          break;
    }
-   CartesianPML * pml = new CartesianPML(mesh,length);
+   PML * pml = new PML(mesh,length);
    comp_domain_bdr = pml->GetCompDomainBdr();
    domain_bdr = pml->GetDomainBdr();
 
@@ -478,11 +478,11 @@ int main(int argc, char *argv[])
    if (!pa && mumps_solver)
    {
       HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
-      MUMPSSolver mumps;
+      MUMPSSolver mumps(A->GetComm());
       mumps.SetPrintLevel(0);
       mumps.SetMatrixSymType(MUMPSSolver::MatType::UNSYMMETRIC);
       mumps.SetOperator(*A);
-      mumps.Mult(B,X);
+      mumps.Mult(B, X);
       delete A;
    }
 #endif
@@ -524,16 +524,14 @@ int main(int argc, char *argv[])
       offsets[2] = fespace->GetTrueVSize();
       offsets.PartialSum();
 
-      Operator *pc_r = nullptr;
-      Operator *pc_i = nullptr;
+      std::unique_ptr<Operator> pc_r;
+      std::unique_ptr<Operator> pc_i;
       int s = (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0;
       if (pa)
       {
          // Jacobi Smoother
-         OperatorJacobiSmoother *d00 = new OperatorJacobiSmoother(prec, ess_tdof_list);
-         ScaledOperator *d11 = new ScaledOperator(d00, s);
-         pc_r = d00;
-         pc_i = d11;
+         pc_r.reset(new OperatorJacobiSmoother(prec, ess_tdof_list));
+         pc_i.reset(new ScaledOperator(pc_r.get(), s));
       }
       else
       {
@@ -541,15 +539,13 @@ int main(int argc, char *argv[])
          prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
 
          // Hypre AMS
-         HypreAMS *ams00 = new HypreAMS(*PCOpAh.As<HypreParMatrix>(), fespace);
-         ScaledOperator *ams11 = new ScaledOperator(ams00, s);
-         pc_r = ams00;
-         pc_i = ams11;
+         pc_r.reset(new HypreAMS(*PCOpAh.As<HypreParMatrix>(), fespace));
+         pc_i.reset(new ScaledOperator(pc_r.get(), s));
       }
 
       BlockDiagonalPreconditioner BlockDP(offsets);
-      BlockDP.SetDiagonalBlock(0, pc_r);
-      BlockDP.SetDiagonalBlock(1, pc_i);
+      BlockDP.SetDiagonalBlock(0, pc_r.get());
+      BlockDP.SetDiagonalBlock(1, pc_i.get());
 
       GMRESSolver gmres(MPI_COMM_WORLD);
       gmres.SetPrintLevel(1);
@@ -884,7 +880,7 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
    }
 }
 
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_JT_J_inv_Re(const Vector &x, PML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det(1.0, 0.0);
@@ -901,7 +897,7 @@ void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D)
    }
 }
 
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_JT_J_inv_Im(const Vector &x, PML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -918,7 +914,7 @@ void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D)
    }
 }
 
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_JT_J_inv_abs(const Vector &x, PML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -935,7 +931,7 @@ void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D)
    }
 }
 
-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_inv_JT_J_Re(const Vector &x, PML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det(1.0, 0.0);
@@ -960,7 +956,7 @@ void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D)
    }
 }
 
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_inv_JT_J_Im(const Vector &x, PML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -984,7 +980,7 @@ void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D)
    }
 }
 
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_inv_JT_J_abs(const Vector &x, PML * pml, Vector & D)
 {
    vector<complex<double>> dxs(dim);
    complex<double> det = 1.0;
@@ -1008,14 +1004,14 @@ void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D)
    }
 }
 
-CartesianPML::CartesianPML(Mesh *mesh_, Array2D<double> length_)
+PML::PML(Mesh *mesh_, Array2D<double> length_)
    : mesh(mesh_), length(length_)
 {
    dim = mesh->Dimension();
    SetBoundaries();
 }
 
-void CartesianPML::SetBoundaries()
+void PML::SetBoundaries()
 {
    comp_dom_bdr.SetSize(dim, 2);
    dom_bdr.SetSize(dim, 2);
@@ -1030,7 +1026,7 @@ void CartesianPML::SetBoundaries()
    }
 }
 
-void CartesianPML::SetAttributes(ParMesh *pmesh)
+void PML::SetAttributes(ParMesh *pmesh)
 {
    // Initialize bdr attributes
    for (int i = 0; i < pmesh->GetNBE(); ++i)
@@ -1080,8 +1076,8 @@ void CartesianPML::SetAttributes(ParMesh *pmesh)
    pmesh->SetAttributes();
 }
 
-void CartesianPML::StretchFunction(const Vector &x,
-                                   vector<complex<double>> &dxs)
+void PML::StretchFunction(const Vector &x,
+                          vector<complex<double>> &dxs)
 {
    complex<double> zi = complex<double>(0., 1.);
 

examples/ex9p.cpp

@@ -536,8 +536,10 @@ int main(int argc, char *argv[])
       if (!sout)
       {
          if (Mpi::Root())
+         {
             cout << "Unable to connect to GLVis server at "
                  << vishost << ':' << visport << endl;
+         }
          visualization = false;
          if (Mpi::Root())
          {
@@ -552,8 +554,10 @@ int main(int argc, char *argv[])
          sout << "pause\n";
          sout << flush;
          if (Mpi::Root())
+         {
             cout << "GLVis visualization paused."
                  << " Press space (in the GLVis window) to resume it.\n";
+         }
       }
    }
 

examples/sundials/CMakeLists.txt

@@ -68,11 +68,43 @@ if (MFEM_ENABLE_TESTING)
       add_test(NAME ${TEST_NAME}_ser
         COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
     else()
-      add_test(NAME ${TEST_NAME}_np=4
+      add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
         COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
         ${MPIEXEC_PREFLAGS}
         $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
         ${MPIEXEC_POSTFLAGS})
     endif()
   endforeach()
-endif()
+
+  # Add CUDA/HIP tests.
+  set(DEVICE_EXAMPLES
+      # serial examples with device support:
+      ex9
+      # parallel examples with device support:
+      ex9p)
+  set(MFEM_TEST_DEVICE)
+  if (MFEM_USE_CUDA)
+    set(MFEM_TEST_DEVICE "cuda")
+  elseif (MFEM_USE_HIP)
+    set(MFEM_TEST_DEVICE "hip")
+  endif()
+  if (MFEM_TEST_DEVICE)
+    foreach(TEST_NAME ${DEVICE_EXAMPLES})
+      string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+
+      set(THIS_TEST_OPTIONS "-no-vis" "-d" "${MFEM_TEST_DEVICE}")
+      list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+
+      if (NOT (${TEST_NAME} MATCHES ".*p$"))
+        add_test(NAME ${PFX}${TEST_NAME}_${MFEM_TEST_DEVICE}_ser
+          COMMAND ${PFX}${TEST_NAME} ${THIS_TEST_OPTIONS})
+      else()
+        add_test(NAME ${PFX}${TEST_NAME}_${MFEM_TEST_DEVICE}_np=${MFEM_MPI_NP}
+          COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+          ${MPIEXEC_PREFLAGS}
+          $<TARGET_FILE:${PFX}${TEST_NAME}> ${THIS_TEST_OPTIONS}
+          ${MPIEXEC_POSTFLAGS})
+      endif()
+    endforeach()
+  endif(MFEM_TEST_DEVICE)
+endif(MFEM_ENABLE_TESTING)

examples/sundials/README

@@ -12,8 +12,7 @@ use of MFEM features based on the SUNDIALS suite of time integration and
 non-linear solvers.
 
 To build these examples, make sure that MFEM is configured with the option
-"MFEM_USE_SUNDIALS = YES", see the top-level INSTALL file for details (version
-2.7 or higher of SUNDIALS is required).
+"MFEM_USE_SUNDIALS = YES", see the top-level INSTALL file for details.
 
 We recommend comparing the original example codes with the corresponding files
 in the current directory.

examples/sundials/ex9.cpp

@@ -280,15 +280,16 @@ int main(int argc, char *argv[])
       k.SetAssemblyLevel(AssemblyLevel::FULL);
    }
    m.AddDomainIntegrator(new MassIntegrator);
-   k.AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
+   constexpr double alpha = -1.0;
+   k.AddDomainIntegrator(new ConvectionIntegrator(velocity, alpha));
    k.AddInteriorFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
+      new NonconservativeDGTraceIntegrator(velocity, alpha));
    k.AddBdrFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
+      new NonconservativeDGTraceIntegrator(velocity, alpha));
 
    LinearForm b(&fes);
    b.AddBdrFaceIntegrator(
-      new BoundaryFlowIntegrator(inflow, velocity, -1.0, -0.5));
+      new BoundaryFlowIntegrator(inflow, velocity, alpha));
 
    m.Assemble();
    int skip_zeros = 0;

examples/sundials/ex9p.cpp

@@ -63,6 +63,66 @@ double inflow_function(const Vector &x);
 // Mesh bounding box
 Vector bb_min, bb_max;
 
+// Type of preconditioner for implicit time integrator
+enum class PrecType : int
+{
+   ILU = 0,
+   AIR = 1
+};
+
+#if MFEM_HYPRE_VERSION >= 21800
+// Algebraic multigrid preconditioner for advective problems based on
+// approximate ideal restriction (AIR). Most effective when matrix is
+// first scaled by DG block inverse, and AIR applied to scaled matrix.
+// See https://doi.org/10.1137/17M1144350.
+class AIR_prec : public Solver
+{
+private:
+   const HypreParMatrix *A;
+   // Copy of A scaled by block-diagonal inverse
+   HypreParMatrix A_s;
+
+   HypreBoomerAMG *AIR_solver;
+   int blocksize;
+
+public:
+   AIR_prec(int blocksize_) : AIR_solver(NULL), blocksize(blocksize_) { }
+
+   void SetOperator(const Operator &op)
+   {
+      width = op.Width();
+      height = op.Height();
+
+      A = dynamic_cast<const HypreParMatrix *>(&op);
+      MFEM_VERIFY(A != NULL, "AIR_prec requires a HypreParMatrix.")
+
+      // Scale A by block-diagonal inverse
+      BlockInverseScale(A, &A_s, NULL, NULL, blocksize,
+                        BlockInverseScaleJob::MATRIX_ONLY);
+      delete AIR_solver;
+      AIR_solver = new HypreBoomerAMG(A_s);
+      AIR_solver->SetAdvectiveOptions(1, "", "FA");
+      AIR_solver->SetPrintLevel(0);
+      AIR_solver->SetMaxLevels(50);
+   }
+
+   virtual void Mult(const Vector &x, Vector &y) const
+   {
+      // Scale the rhs by block inverse and solve system
+      HypreParVector z_s;
+      BlockInverseScale(A, NULL, &x, &z_s, blocksize,
+                        BlockInverseScaleJob::RHS_ONLY);
+      AIR_solver->Mult(z_s, y);
+   }
+
+   ~AIR_prec()
+   {
+      delete AIR_solver;
+   }
+};
+#endif
+
+
 class DG_Solver : public Solver
 {
 private:
@@ -70,24 +130,37 @@ private:
    SparseMatrix M_diag;
    HypreParMatrix *A;
    GMRESSolver linear_solver;
-   BlockILU prec;
+   Solver *prec;
    double dt;
 public:
-   DG_Solver(HypreParMatrix &M_, HypreParMatrix &K_, const FiniteElementSpace &fes)
+   DG_Solver(HypreParMatrix &M_, HypreParMatrix &K_, const FiniteElementSpace &fes,
+             PrecType prec_type)
       : M(M_),
         K(K_),
         A(NULL),
         linear_solver(M.GetComm()),
-        prec(fes.GetFE(0)->GetDof(),
-             BlockILU::Reordering::MINIMUM_DISCARDED_FILL),
         dt(-1.0)
    {
+      int block_size = fes.GetFE(0)->GetDof();
+      if (prec_type == PrecType::ILU)
+      {
+         prec = new BlockILU(block_size,
+                             BlockILU::Reordering::MINIMUM_DISCARDED_FILL);
+      }
+      else if (prec_type == PrecType::AIR)
+      {
+#if MFEM_HYPRE_VERSION >= 21800
+         prec = new AIR_prec(block_size);
+#else
+         MFEM_ABORT("Must have MFEM_HYPRE_VERSION >= 21800 to use AIR.\n");
+#endif
+      }
       linear_solver.iterative_mode = false;
       linear_solver.SetRelTol(1e-9);
       linear_solver.SetAbsTol(0.0);
       linear_solver.SetMaxIter(100);
       linear_solver.SetPrintLevel(0);
-      linear_solver.SetPreconditioner(prec);
+      linear_solver.SetPreconditioner(*prec);
 
       M.GetDiag(M_diag);
    }
@@ -120,10 +193,12 @@ public:
 
    ~DG_Solver()
    {
+      delete prec;
       delete A;
    }
 };
 
+
 /** A time-dependent operator for the right-hand side of the ODE. The DG weak
     form of du/dt = -v.grad(u) is M du/dt = K u + b, where M and K are the mass
     and advection matrices, and b describes the flow on the boundary. This can
@@ -141,7 +216,8 @@ private:
    mutable Vector z;
 
 public:
-   FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_, const Vector &b_);
+   FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_, const Vector &b_,
+                PrecType prec_type);
 
    virtual void Mult(const Vector &x, Vector &y) const;
    virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);
@@ -178,6 +254,11 @@ int main(int argc, char *argv[])
    bool adios2 = false;
    bool binary = false;
    int vis_steps = 5;
+#if MFEM_HYPRE_VERSION >= 21800
+   PrecType prec_type = PrecType::AIR;
+#else
+   PrecType prec_type = PrecType::ILU;
+#endif
 
    // Relative and absolute tolerances for CVODE and ARKODE.
    const double reltol = 1e-2, abstol = 1e-2;
@@ -218,6 +299,8 @@ int main(int argc, char *argv[])
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
                   "Time step.");
+   args.AddOption((int *)&prec_type, "-pt", "--prec-type", "Preconditioner for "
+                  "implicit solves. 0 for ILU, 1 for pAIR-AMG.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -238,13 +321,13 @@ int main(int argc, char *argv[])
    args.Parse();
    if (!args.Good())
    {
-      if (myid == 0)
+      if (Mpi::Root())
       {
          args.PrintUsage(cout);
       }
       return 1;
    }
-   if (myid == 0)
+   if (Mpi::Root())
    {
       args.PrintOptions(cout);
    }
@@ -252,7 +335,7 @@ int main(int argc, char *argv[])
    // check for valid ODE solver option
    if (ode_solver_type < 1 || ode_solver_type > 9)
    {
-      if (myid == 0)
+      if (Mpi::Root())
       {
          cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
       }
@@ -260,7 +343,7 @@ int main(int argc, char *argv[])
    }
 
    Device device(device_config);
-   if (myid == 0) { device.Print(); }
+   if (Mpi::Root()) { device.Print(); }
 
    // 3. Read the serial mesh from the given mesh file on all processors. We can
    //    handle geometrically periodic meshes in this code.
@@ -297,7 +380,7 @@ int main(int argc, char *argv[])
    ParFiniteElementSpace *fes = new ParFiniteElementSpace(pmesh, &fec);
 
    HYPRE_BigInt global_vSize = fes->GlobalTrueVSize();
-   if (myid == 0)
+   if (Mpi::Root())
    {
       cout << "Number of unknowns: " << global_vSize << endl;
    }
@@ -328,15 +411,16 @@ int main(int argc, char *argv[])
    }
 
    m->AddDomainIntegrator(new MassIntegrator);
-   k->AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
+   constexpr double alpha = -1.0;
+   k->AddDomainIntegrator(new ConvectionIntegrator(velocity, alpha));
    k->AddInteriorFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
+      new NonconservativeDGTraceIntegrator(velocity, alpha));
    k->AddBdrFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
+      new NonconservativeDGTraceIntegrator(velocity, alpha));
 
    ParLinearForm *b = new ParLinearForm(fes);
    b->AddBdrFaceIntegrator(
-      new BoundaryFlowIntegrator(inflow, velocity, -1.0, -0.5));
+      new BoundaryFlowIntegrator(inflow, velocity, alpha));
 
    int skip_zeros = 0;
    m->Assemble();
@@ -435,11 +519,13 @@ int main(int argc, char *argv[])
       sout.open(vishost, visport);
       if (!sout)
       {
-         if (myid == 0)
+         if (Mpi::Root())
+         {
             cout << "Unable to connect to GLVis server at "
                  << vishost << ':' << visport << endl;
+         }
          visualization = false;
-         if (myid == 0)
+         if (Mpi::Root())
          {
             cout << "GLVis visualization disabled.\n";
          }
@@ -451,15 +537,17 @@ int main(int argc, char *argv[])
          sout << "solution\n" << *pmesh << *u;
          sout << "pause\n";
          sout << flush;
-         if (myid == 0)
+         if (Mpi::Root())
+         {
             cout << "GLVis visualization paused."
                  << " Press space (in the GLVis window) to resume it.\n";
+         }
       }
    }
 
    // 9. Define the time-dependent evolution operator describing the ODE
    //    right-hand side, and define the ODE solver used for time integration.
-   FE_Evolution adv(*m, *k, *B);
+   FE_Evolution adv(*m, *k, *B, prec_type);
 
    double t = 0.0;
    adv.SetTime(t);
@@ -511,7 +599,7 @@ int main(int argc, char *argv[])
 
       if (done || ti % vis_steps == 0)
       {
-         if (myid == 0)
+         if (Mpi::Root())
          {
             cout << "time step: " << ti << ", time: " << t << endl;
             if (cvode) { cvode->PrintInfo(); }
@@ -590,7 +678,7 @@ int main(int argc, char *argv[])
 
 // Implementation of class FE_Evolution
 FE_Evolution::FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_,
-                           const Vector &b_)
+                           const Vector &b_, PrecType prec_type)
    : TimeDependentOperator(M_.Height()),
      b(b_),
      M_solver(M_.ParFESpace()->GetComm()),
@@ -617,7 +705,7 @@ FE_Evolution::FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_,
       HypreSmoother *hypre_prec = new HypreSmoother(M_mat, HypreSmoother::Jacobi);
       M_prec = hypre_prec;
 
-      dg_solver = new DG_Solver(M_mat, K_mat, *M_.FESpace());
+      dg_solver = new DG_Solver(M_mat, K_mat, *M_.FESpace(), prec_type);
    }
    else
    {
@@ -633,6 +721,10 @@ FE_Evolution::FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_,
    M_solver.SetPrintLevel(0);
 }
 
+// Solve the equation:
+//    u_t = M^{-1}(Ku + b),
+// by solving associated linear system
+//    (M - dt*K) d = K*u + b
 void FE_Evolution::ImplicitSolve(const double dt, const Vector &x, Vector &k)
 {
    K->Mult(x, z);

examples/sundials/makefile

@@ -23,6 +23,8 @@ MFEM_LIB_FILE = mfem_is_not_built
 
 SEQ_EXAMPLES = ex9 ex10 ex16
 PAR_EXAMPLES = ex9p ex10p ex16p
+SEQ_DEVICE_EXAMPLES = ex9
+PAR_DEVICE_EXAMPLES = ex9p
 ifeq ($(MFEM_USE_MPI),NO)
    EXAMPLES = $(SEQ_EXAMPLES)
 else
@@ -54,10 +56,22 @@ include $(MFEM_TEST_MK)
 RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
 SERIAL_NAME := Serial SUNDIALS example
 PARALLEL_NAME := Parallel SUNDIALS example
+SERIAL_CUDA_NAME := Serial SUNDIALS CUDA example
+PARALLEL_CUDA_NAME := Parallel SUNDIALS CUDA example
+SERIAL_HIP_NAME := Serial SUNDIALS HIP example
+PARALLEL_HIP_NAME := Parallel SUNDIALS HIP example
 %-test-par: %
 	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME))
 %-test-seq: %
 	@$(call mfem-test,$<,, $(SERIAL_NAME))
+%-test-par-cuda: %
+	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_CUDA_NAME),-d cuda)
+%-test-seq-cuda: %
+	@$(call mfem-test,$<,, $(SERIAL_CUDA_NAME),-d cuda)
+%-test-par-hip: %
+	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_HIP_NAME),-d hip)
+%-test-seq-hip: %
+	@$(call mfem-test,$<,, $(SERIAL_HIP_NAME),-d hip)
 
 # Testing: Specific execution options:
 # Example 9: test CVODE with CV_ADAMS (non-stiff implicit) time stepping
@@ -68,6 +82,16 @@ ex9-test-seq: ex9
 	@$(call mfem-test,$<,, $(SERIAL_NAME),$(EX9_ARGS))
 ex9p-test-par: ex9p
 	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME),$(EX9P_ARGS))
+ex9-test-seq-cuda: ex9
+	@$(call mfem-test,$<,, $(SERIAL_CUDA_NAME),-d cuda $(EX9_ARGS))
+ex9p-test-par-cuda: ex9p
+	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_CUDA_NAME),-d cuda \
+	   $(EX9P_ARGS))
+ex9-test-seq-hip: ex9
+	@$(call mfem-test,$<,, $(SERIAL_HIP_NAME),-d hip $(EX9_ARGS))
+ex9p-test-par-hip: ex9p
+	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_HIP_NAME),-d hip \
+	   $(EX9P_ARGS))
 # Example 10: test CVODE with CV_BDF (stiff implicit) time stepping
 EX10_COMMON_ARGS := -m ../../data/beam-quad.mesh -o 2 -s 5 -dt 0.15 -tf 6 -vs 10
 EX10_ARGS  := $(EX10_COMMON_ARGS) -r  2

examples/superlu/ex1p.cpp

@@ -67,6 +67,7 @@ int main(int argc, char *argv[])
    int slu_colperm = 4;
    int slu_rowperm = 1;
    int slu_iterref = 2;
+   int slu_npdep = 1;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -85,9 +86,11 @@ int main(int argc, char *argv[])
                   "6-ZOLTAN");
    args.AddOption(&slu_rowperm, "-rp", "--rowperm",
                   "SuperLU Row Permutation Method:  0-NOROWPERM, 1-LargeDiag");
-   args.AddOption(&slu_iterref, "-rp", "--rowperm",
+   args.AddOption(&slu_iterref, "-ir", "--iterref",
                   "SuperLU Iterative Refinement:  0-NOREFINE, 1-Single, "
                   "2-Double, 3-Extra");
+   args.AddOption(&slu_npdep, "-npdep", "--npdepth",
+                  "Depth of 3D parition for SuperLU (>= 7.2.0)");
 
    args.Parse();
    if (!args.Good())
@@ -214,7 +217,7 @@ int main(int argc, char *argv[])
    a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
 
    // 13. Solve the linear system A X = B utilizing SuperLU.
-   SuperLUSolver *superlu = new SuperLUSolver(MPI_COMM_WORLD);
+   SuperLUSolver *superlu = new SuperLUSolver(MPI_COMM_WORLD, slu_npdep);
    Operator *SLU_A = new SuperLURowLocMatrix(*A.As<HypreParMatrix>());
    superlu->SetPrintStatistics(true);
    superlu->SetSymmetricPattern(false);
@@ -281,10 +284,9 @@ int main(int argc, char *argv[])
    superlu->SetOperator(*SLU_A);
    superlu->SetPrintStatistics(true);
    superlu->Mult(B, X);
-   superlu->DismantleGrid();
 
-   delete SLU_A;
    delete superlu;
+   delete SLU_A;
 
    // 14. Recover the parallel grid function corresponding to X. This is the
    //     local finite element solution on each processor.

fem/CMakeLists.txt

@@ -13,28 +13,44 @@ set(SRCS
   bilinearform.cpp
   bilinearform_ext.cpp
   bilininteg.cpp
-  bilininteg_br2.cpp
-  bilininteg_convection_mf.cpp
-  bilininteg_convection_pa.cpp
-  bilininteg_convection_ea.cpp
-  bilininteg_dgtrace_pa.cpp
-  bilininteg_dgtrace_ea.cpp
-  bilininteg_diffusion_mf.cpp
-  bilininteg_diffusion_pa.cpp
-  bilininteg_diffusion_ea.cpp
-  bilininteg_divergence.cpp
-  bilininteg_hcurl.cpp
-  bilininteg_hdiv.cpp
-  bilininteg_vectorfe.cpp
-  bilininteg_gradient.cpp
-  bilininteg_mass_mf.cpp
-  bilininteg_mass_pa.cpp
-  bilininteg_mass_ea.cpp
-  bilininteg_transpose_ea.cpp
-  bilininteg_vecdiffusion.cpp
-  bilininteg_vecdiffusion_mf.cpp
-  bilininteg_vecmass.cpp
-  bilininteg_vecmass_mf.cpp
+  integ/bilininteg_br2.cpp
+  integ/bilininteg_convection_mf.cpp
+  integ/bilininteg_convection_pa.cpp
+  integ/bilininteg_convection_ea.cpp
+  integ/bilininteg_curlcurl_pa.cpp
+  integ/bilininteg_dgtrace_pa.cpp
+  integ/bilininteg_dgtrace_ea.cpp
+  integ/bilininteg_diffusion_mf.cpp
+  integ/bilininteg_diffusion_pa.cpp
+  integ/bilininteg_diffusion_ea.cpp
+  integ/bilininteg_divdiv_pa.cpp
+  integ/bilininteg_gradient_pa.cpp
+  integ/bilininteg_interp_pa.cpp
+  integ/bilininteg_mass_mf.cpp
+  integ/bilininteg_mass_pa.cpp
+  integ/bilininteg_mass_ea.cpp
+  integ/bilininteg_mixedcurl_pa.cpp
+  integ/bilininteg_mixedvecgrad_pa.cpp
+  integ/bilininteg_transpose_ea.cpp
+  integ/bilininteg_vecdiffusion_mf.cpp
+  integ/bilininteg_vecdiffusion_pa.cpp
+  integ/bilininteg_vecdiv_pa.cpp
+  integ/bilininteg_vecmass_mf.cpp
+  integ/bilininteg_vecmass_pa.cpp
+  integ/bilininteg_vectorfediv_pa.cpp
+  integ/bilininteg_vectorfemass_pa.cpp
+  integ/bilininteg_diffusion_kernels.cpp
+  integ/bilininteg_hcurl_kernels.cpp
+  integ/bilininteg_hdiv_kernels.cpp
+  integ/bilininteg_hcurlhdiv_kernels.cpp
+  integ/bilininteg_mass_kernels.cpp
+  integ/lininteg_boundary.cpp
+  integ/lininteg_boundary_flux.cpp
+  integ/lininteg_domain.cpp
+  integ/lininteg_domain_grad.cpp
+  integ/lininteg_domain_vectorfe.cpp
+  integ/nonlininteg_vecconvection_pa.cpp
+  integ/nonlininteg_vecconvection_mf.cpp
   coefficient.cpp
   complex_fem.cpp
   convergence.cpp
@@ -52,6 +68,7 @@ set(SRCS
   fe/fe_nd.cpp
   fe/fe_nurbs.cpp
   fe/fe_pos.cpp
+  fe/fe_rbf.cpp
   fe/fe_rt.cpp
   fe/fe_ser.cpp
   fe_coll.cpp
@@ -74,11 +91,6 @@ set(SRCS
   linearform.cpp
   linearform_ext.cpp
   lininteg.cpp
-  lininteg_boundary.cpp
-  lininteg_boundary_flux.cpp
-  lininteg_domain.cpp
-  lininteg_domain_grad.cpp
-  lininteg_vectorfe_domain.cpp
   lor/lor.cpp
   lor/lor_ads.cpp
   lor/lor_ams.cpp
@@ -91,8 +103,6 @@ set(SRCS
   nonlinearform_ext.cpp
   nonlininteg.cpp
   fespacehierarchy.cpp
-  nonlininteg_vectorconvection.cpp
-  nonlininteg_vectorconvection_mf.cpp
   qfunction.cpp
   qinterp/det.cpp
   qinterp/eval_by_nodes.cpp
@@ -143,7 +153,11 @@ set(HDRS
   bilinearform.hpp
   bilinearform_ext.hpp
   bilininteg.hpp
-  bilininteg_mass_pa.hpp
+  integ/bilininteg_diffusion_kernels.hpp
+  integ/bilininteg_hcurl_kernels.hpp
+  integ/bilininteg_hdiv_kernels.hpp
+  integ/bilininteg_hcurlhdiv_kernels.hpp
+  integ/bilininteg_mass_kernels.hpp
   coefficient.hpp
   complex_fem.hpp
   convergence.hpp
@@ -162,6 +176,7 @@ set(HDRS
   fe/fe_nd.hpp
   fe/fe_nurbs.hpp
   fe/fe_pos.hpp
+  fe/fe_rbf.hpp
   fe/fe_rt.hpp
   fe/fe_ser.hpp
   fe_coll.hpp

fem/bilinearform_ext.cpp

@@ -56,6 +56,9 @@ void MFBilinearFormExtension::Assemble()
    {
       integrators[i]->AssembleMF(*a->FESpace());
    }
+
+   MFEM_VERIFY(a->GetBBFI()->Size() == 0, "AddBoundaryIntegrator is not "
+               "currently supported in MFBilinearFormExtension");
 }
 
 void MFBilinearFormExtension::AssembleDiagonal(Vector &y) const
@@ -275,7 +278,9 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
       int_face_Y.UseDevice(true); // ensure 'int_face_Y = 0.0' is done on device
    }
 
-   if (bdr_face_restrict_lex == NULL && a->GetBFBFI()->Size() > 0)
+   const bool has_bdr_integs = (a->GetBFBFI()->Size() > 0 ||
+                                a->GetBBFI()->Size() > 0);
+   if (bdr_face_restrict_lex == NULL && has_bdr_integs)
    {
       bdr_face_restrict_lex = trial_fes->GetFaceRestriction(
                                  ElementDofOrdering::LEXICOGRAPHIC,
@@ -292,27 +297,27 @@ void PABilinearFormExtension::Assemble()
    SetupRestrictionOperators(L2FaceValues::DoubleValued);
 
    Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int integratorCount = integrators.Size();
-   for (int i = 0; i < integratorCount; ++i)
+   for (BilinearFormIntegrator *integ : integrators)
    {
-      integrators[i]->AssemblePA(*a->FESpace());
+      integ->AssemblePA(*a->FESpace());
    }
 
-   MFEM_VERIFY(a->GetBBFI()->Size() == 0,
-               "Partial assembly does not support AddBoundaryIntegrator yet.");
+   Array<BilinearFormIntegrator*> &bdr_integrators = *a->GetBBFI();
+   for (BilinearFormIntegrator *integ : bdr_integrators)
+   {
+      integ->AssemblePABoundary(*a->FESpace());
+   }
 
    Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
-   const int intFaceIntegratorCount = intFaceIntegrators.Size();
-   for (int i = 0; i < intFaceIntegratorCount; ++i)
+   for (BilinearFormIntegrator *integ : intFaceIntegrators)
    {
-      intFaceIntegrators[i]->AssemblePAInteriorFaces(*a->FESpace());
+      integ->AssemblePAInteriorFaces(*a->FESpace());
    }
 
    Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
-   const int boundFaceIntegratorCount = bdrFaceIntegrators.Size();
-   for (int i = 0; i < boundFaceIntegratorCount; ++i)
+   for (BilinearFormIntegrator *integ : bdrFaceIntegrators)
    {
-      bdrFaceIntegrators[i]->AssemblePABoundaryFaces(*a->FESpace());
+      integ->AssemblePABoundaryFaces(*a->FESpace());
    }
 }
 
@@ -323,20 +328,27 @@ void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
    const int iSz = integrators.Size();
    if (elem_restrict && !DeviceCanUseCeed())
    {
-      localY = 0.0;
-      for (int i = 0; i < iSz; ++i)
+      if (iSz > 0)
       {
-         integrators[i]->AssembleDiagonalPA(localY);
-      }
-      const ElementRestriction* H1elem_restrict =
-         dynamic_cast<const ElementRestriction*>(elem_restrict);
-      if (H1elem_restrict)
-      {
-         H1elem_restrict->MultTransposeUnsigned(localY, y);
+         localY = 0.0;
+         for (int i = 0; i < iSz; ++i)
+         {
+            integrators[i]->AssembleDiagonalPA(localY);
+         }
+         const ElementRestriction* H1elem_restrict =
+            dynamic_cast<const ElementRestriction*>(elem_restrict);
+         if (H1elem_restrict)
+         {
+            H1elem_restrict->MultTransposeUnsigned(localY, y);
+         }
+         else
+         {
+            elem_restrict->MultTranspose(localY, y);
+         }
       }
       else
       {
-         elem_restrict->MultTranspose(localY, y);
+         y = 0.0;
       }
    }
    else
@@ -348,6 +360,18 @@ void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
          integrators[i]->AssembleDiagonalPA(y);
       }
    }
+
+   Array<BilinearFormIntegrator*> &bdr_integs = *a->GetBBFI();
+   const int n_bdr_integs = bdr_integs.Size();
+   if (bdr_face_restrict_lex && n_bdr_integs > 0)
+   {
+      bdr_face_Y = 0.0;
+      for (int i = 0; i < n_bdr_integs; ++i)
+      {
+         bdr_integs[i]->AssembleDiagonalPA(bdr_face_Y);
+      }
+      bdr_face_restrict_lex->AddMultTransposeUnsigned(bdr_face_Y, y);
+   }
 }
 
 void PABilinearFormExtension::Update()
@@ -397,13 +421,20 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
    }
    else
    {
-      elem_restrict->Mult(x, localX);
-      localY = 0.0;
-      for (int i = 0; i < iSz; ++i)
+      if (iSz)
       {
-         integrators[i]->AddMultPA(localX, localY);
+         elem_restrict->Mult(x, localX);
+         localY = 0.0;
+         for (int i = 0; i < iSz; ++i)
+         {
+            integrators[i]->AddMultPA(localX, localY);
+         }
+         elem_restrict->MultTranspose(localY, y);
+      }
+      else
+      {
+         y = 0.0;
       }
-      elem_restrict->MultTranspose(localY, y);
    }
 
    Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
@@ -422,17 +453,24 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
       }
    }
 
-   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
-   const int bFISz = bdrFaceIntegrators.Size();
-   if (bdr_face_restrict_lex && bFISz>0)
+   Array<BilinearFormIntegrator*> &bdr_integs = *a->GetBBFI();
+   Array<BilinearFormIntegrator*> &bdr_face_integs = *a->GetBFBFI();
+   const int n_bdr_integs = bdr_integs.Size();
+   const int n_bdr_face_integs = bdr_face_integs.Size();
+   const bool has_bdr_integs = (n_bdr_face_integs > 0 || n_bdr_integs > 0);
+   if (bdr_face_restrict_lex && has_bdr_integs)
    {
       bdr_face_restrict_lex->Mult(x, bdr_face_X);
       if (bdr_face_X.Size()>0)
       {
          bdr_face_Y = 0.0;
-         for (int i = 0; i < bFISz; ++i)
+         for (int i = 0; i < n_bdr_integs; ++i)
          {
-            bdrFaceIntegrators[i]->AddMultPA(bdr_face_X, bdr_face_Y);
+            bdr_integs[i]->AddMultPA(bdr_face_X, bdr_face_Y);
+         }
+         for (int i = 0; i < n_bdr_face_integs; ++i)
+         {
+            bdr_face_integs[i]->AddMultPA(bdr_face_X, bdr_face_Y);
          }
          bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
       }
@@ -596,7 +634,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
       auto X = Reshape(useRestrict?localX.Read():x.Read(), NDOFS, ne);
       auto Y = Reshape(useRestrict?localY.ReadWrite():y.ReadWrite(), NDOFS, ne);
       auto A = Reshape(ea_data.Read(), NDOFS, NDOFS, ne);
-      MFEM_FORALL(glob_j, ne*NDOFS,
+      mfem::forall(ne*NDOFS, [=] MFEM_HOST_DEVICE (int glob_j)
       {
          const int e = glob_j/NDOFS;
          const int j = glob_j%NDOFS;
@@ -631,7 +669,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
          if (!factorize_face_terms)
          {
             auto A_int = Reshape(ea_data_int.Read(), NDOFS, NDOFS, 2, nf_int);
-            MFEM_FORALL(glob_j, nf_int*NDOFS,
+            mfem::forall(nf_int*NDOFS, [=] MFEM_HOST_DEVICE (int glob_j)
             {
                const int f = glob_j/NDOFS;
                const int j = glob_j%NDOFS;
@@ -650,7 +688,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
             });
          }
          auto A_ext = Reshape(ea_data_ext.Read(), NDOFS, NDOFS, 2, nf_int);
-         MFEM_FORALL(glob_j, nf_int*NDOFS,
+         mfem::forall(nf_int*NDOFS, [=] MFEM_HOST_DEVICE (int glob_j)
          {
             const int f = glob_j/NDOFS;
             const int j = glob_j%NDOFS;
@@ -687,7 +725,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
          auto X = Reshape(bdr_face_X.Read(), NDOFS, nf_bdr);
          auto Y = Reshape(bdr_face_Y.ReadWrite(), NDOFS, nf_bdr);
          auto A = Reshape(ea_data_bdr.Read(), NDOFS, NDOFS, nf_bdr);
-         MFEM_FORALL(glob_j, nf_bdr*NDOFS,
+         mfem::forall(nf_bdr*NDOFS, [=] MFEM_HOST_DEVICE (int glob_j)
          {
             const int f = glob_j/NDOFS;
             const int j = glob_j%NDOFS;
@@ -724,7 +762,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
       auto X = Reshape(useRestrict?localX.Read():x.Read(), NDOFS, ne);
       auto Y = Reshape(useRestrict?localY.ReadWrite():y.ReadWrite(), NDOFS, ne);
       auto A = Reshape(ea_data.Read(), NDOFS, NDOFS, ne);
-      MFEM_FORALL(glob_j, ne*NDOFS,
+      mfem::forall(ne*NDOFS, [=] MFEM_HOST_DEVICE (int glob_j)
       {
          const int e = glob_j/NDOFS;
          const int j = glob_j%NDOFS;
@@ -759,7 +797,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
          if (!factorize_face_terms)
          {
             auto A_int = Reshape(ea_data_int.Read(), NDOFS, NDOFS, 2, nf_int);
-            MFEM_FORALL(glob_j, nf_int*NDOFS,
+            mfem::forall(nf_int*NDOFS, [=] MFEM_HOST_DEVICE (int glob_j)
             {
                const int f = glob_j/NDOFS;
                const int j = glob_j%NDOFS;
@@ -778,7 +816,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
             });
          }
          auto A_ext = Reshape(ea_data_ext.Read(), NDOFS, NDOFS, 2, nf_int);
-         MFEM_FORALL(glob_j, nf_int*NDOFS,
+         mfem::forall(nf_int*NDOFS, [=] MFEM_HOST_DEVICE (int glob_j)
          {
             const int f = glob_j/NDOFS;
             const int j = glob_j%NDOFS;
@@ -815,7 +853,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
          auto X = Reshape(bdr_face_X.Read(), NDOFS, nf_bdr);
          auto Y = Reshape(bdr_face_Y.ReadWrite(), NDOFS, nf_bdr);
          auto A = Reshape(ea_data_bdr.Read(), NDOFS, NDOFS, nf_bdr);
-         MFEM_FORALL(glob_j, nf_bdr*NDOFS,
+         mfem::forall(nf_bdr*NDOFS, [=] MFEM_HOST_DEVICE (int glob_j)
          {
             const int f = glob_j/NDOFS;
             const int j = glob_j%NDOFS;
@@ -1030,13 +1068,13 @@ void FABilinearFormExtension::DGMult(const Vector &x, Vector &y) const
       const int local_size = a->FESpace()->GetVSize();
       auto dg_x_ptr = dg_x.Write();
       auto x_ptr = x.Read();
-      MFEM_FORALL(i,local_size,
+      mfem::forall(local_size, [=] MFEM_HOST_DEVICE (int i)
       {
          dg_x_ptr[i] = x_ptr[i];
       });
       const int shared_size = shared_x.Size();
       auto shared_x_ptr = shared_x.Read();
-      MFEM_FORALL(i,shared_size,
+      mfem::forall(shared_size, [=] MFEM_HOST_DEVICE (int i)
       {
          dg_x_ptr[local_size+i] = shared_x_ptr[i];
       });
@@ -1047,7 +1085,7 @@ void FABilinearFormExtension::DGMult(const Vector &x, Vector &y) const
          // DG Restriction
          auto dg_y_ptr = dg_y.Read();
          auto y_ptr = y.ReadWrite();
-         MFEM_FORALL(i,local_size,
+         mfem::forall(local_size, [=] MFEM_HOST_DEVICE (int i)
          {
             y_ptr[i] += dg_y_ptr[i];
          });
@@ -1091,13 +1129,13 @@ void FABilinearFormExtension::DGMultTranspose(const Vector &x, Vector &y) const
       const int local_size = a->FESpace()->GetVSize();
       auto dg_x_ptr = dg_x.Write();
       auto x_ptr = x.Read();
-      MFEM_FORALL(i,local_size,
+      mfem::forall(local_size, [=] MFEM_HOST_DEVICE (int i)
       {
          dg_x_ptr[i] = x_ptr[i];
       });
       const int shared_size = shared_x.Size();
       auto shared_x_ptr = shared_x.Read();
-      MFEM_FORALL(i,shared_size,
+      mfem::forall(shared_size, [=] MFEM_HOST_DEVICE (int i)
       {
          dg_x_ptr[local_size+i] = shared_x_ptr[i];
       });
@@ -1108,7 +1146,7 @@ void FABilinearFormExtension::DGMultTranspose(const Vector &x, Vector &y) const
          // DG Restriction
          auto dg_y_ptr = dg_y.Read();
          auto y_ptr = y.ReadWrite();
-         MFEM_FORALL(i,local_size,
+         mfem::forall(local_size, [=] MFEM_HOST_DEVICE (int i)
          {
             y_ptr[i] += dg_y_ptr[i];
          });
@@ -1446,7 +1484,7 @@ void PADiscreteLinearOperatorExtension::Assemble()
    }
 
    auto tm = test_multiplicity.ReadWrite();
-   MFEM_FORALL(i, test_multiplicity.Size(),
+   mfem::forall(test_multiplicity.Size(), [=] MFEM_HOST_DEVICE (int i)
    {
       tm[i] = 1.0 / tm[i];
    });
@@ -1498,7 +1536,7 @@ void PADiscreteLinearOperatorExtension::AddMultTranspose(
    MFEM_VERIFY(x.Size() == test_multiplicity.Size(), "Input vector of wrong size");
    auto xs = xscaled.ReadWrite();
    auto tm = test_multiplicity.Read();
-   MFEM_FORALL(i, x.Size(),
+   mfem::forall(x.Size(), [=] MFEM_HOST_DEVICE (int i)
    {
       xs[i] *= tm[i];
    });

fem/bilininteg.cpp

@@ -22,41 +22,47 @@ namespace mfem
 
 void BilinearFormIntegrator::AssemblePA(const FiniteElementSpace&)
 {
-   mfem_error ("BilinearFormIntegrator::AssemblePA(fes)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AssemblePA(fes)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssemblePA(const FiniteElementSpace&,
                                         const FiniteElementSpace&)
 {
-   mfem_error ("BilinearFormIntegrator::AssemblePA(fes, fes)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AssemblePA(fes, fes)\n"
+              "   is not implemented for this class.");
+}
+
+void BilinearFormIntegrator::AssemblePABoundary(const FiniteElementSpace&)
+{
+   MFEM_ABORT("BilinearFormIntegrator::AssemblePABoundary(fes)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssemblePAInteriorFaces(const FiniteElementSpace&)
 {
-   mfem_error ("BilinearFormIntegrator::AssemblePAInteriorFaces(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AssemblePAInteriorFaces(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssemblePABoundaryFaces(const FiniteElementSpace&)
 {
-   mfem_error ("BilinearFormIntegrator::AssemblePABoundaryFaces(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AssemblePABoundaryFaces(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssembleDiagonalPA(Vector &)
 {
-   mfem_error ("BilinearFormIntegrator::AssembleDiagonalPA(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AssembleDiagonalPA(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssembleEA(const FiniteElementSpace &fes,
                                         Vector &emat,
                                         const bool add)
 {
-   mfem_error ("BilinearFormIntegrator::AssembleEA(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AssembleEA(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace
@@ -65,8 +71,8 @@ void BilinearFormIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace
                                                      Vector &ea_data_ext,
                                                      const bool add)
 {
-   mfem_error ("BilinearFormIntegrator::AssembleEAInteriorFaces(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AssembleEAInteriorFaces(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace
@@ -74,8 +80,8 @@ void BilinearFormIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace
                                                      Vector &ea_data_bdr,
                                                      const bool add)
 {
-   mfem_error ("BilinearFormIntegrator::AssembleEABoundaryFaces(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AssembleEABoundaryFaces(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssembleDiagonalPA_ADAt(const Vector &, Vector &)
@@ -86,62 +92,62 @@ void BilinearFormIntegrator::AssembleDiagonalPA_ADAt(const Vector &, Vector &)
 
 void BilinearFormIntegrator::AddMultPA(const Vector &, Vector &) const
 {
-   mfem_error ("BilinearFormIntegrator::MultAssembled(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::MultAssembled(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AddMultTransposePA(const Vector &, Vector &) const
 {
-   mfem_error ("BilinearFormIntegrator::AddMultTransposePA(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AddMultTransposePA(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssembleMF(const FiniteElementSpace &fes)
 {
-   mfem_error ("BilinearFormIntegrator::AssembleMF(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AssembleMF(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AddMultMF(const Vector &, Vector &) const
 {
-   mfem_error ("BilinearFormIntegrator::AddMultMF(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AddMultMF(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AddMultTransposeMF(const Vector &, Vector &) const
 {
-   mfem_error ("BilinearFormIntegrator::AddMultTransposeMF(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AddMultTransposeMF(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssembleDiagonalMF(Vector &)
 {
-   mfem_error ("BilinearFormIntegrator::AssembleDiagonalMF(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AssembleDiagonalMF(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssembleElementMatrix (
    const FiniteElement &el, ElementTransformation &Trans,
    DenseMatrix &elmat )
 {
-   mfem_error ("BilinearFormIntegrator::AssembleElementMatrix(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AssembleElementMatrix(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssembleElementMatrix2 (
    const FiniteElement &el1, const FiniteElement &el2,
    ElementTransformation &Trans, DenseMatrix &elmat )
 {
-   mfem_error ("BilinearFormIntegrator::AssembleElementMatrix2(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AssembleElementMatrix2(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssembleFaceMatrix (
    const FiniteElement &el1, const FiniteElement &el2,
    FaceElementTransformations &Trans, DenseMatrix &elmat)
 {
-   mfem_error ("BilinearFormIntegrator::AssembleFaceMatrix(...)\n"
-               "   is not implemented for this class.");
+   MFEM_ABORT("BilinearFormIntegrator::AssembleFaceMatrix(...)\n"
+              "   is not implemented for this class.");
 }
 
 void BilinearFormIntegrator::AssembleFaceMatrix(
@@ -153,6 +159,16 @@ void BilinearFormIntegrator::AssembleFaceMatrix(
               " Integrator class.");
 }
 
+void BilinearFormIntegrator::AssembleTraceFaceMatrix (int elem,
+                                                      const FiniteElement &trial_face_fe,
+                                                      const FiniteElement &test_fe1,
+                                                      FaceElementTransformations &Trans,
+                                                      DenseMatrix &elmat)
+{
+   MFEM_ABORT("AssembleTraceFaceMatrix (DPG form) is not implemented for this"
+              " Integrator class.");
+}
+
 void BilinearFormIntegrator::AssembleElementVector(
    const FiniteElement &el, ElementTransformation &Tr, const Vector &elfun,
    Vector &elvect)
@@ -2633,7 +2649,7 @@ void VectorFEMassIntegrator::AssembleElementMatrix2(
    }
    else
    {
-      mfem_error("VectorFEMassIntegrator::AssembleElementMatrix2(...)\n"
+      MFEM_ABORT("VectorFEMassIntegrator::AssembleElementMatrix2(...)\n"
                  "   is not implemented for given trial and test bases.");
    }
 }
@@ -3997,6 +4013,216 @@ void NormalTraceJumpIntegrator::AssembleFaceMatrix(
    }
 }
 
+void TraceIntegrator::AssembleTraceFaceMatrix(int elem,
+                                              const FiniteElement &trial_face_fe,
+                                              const FiniteElement &test_fe,
+                                              FaceElementTransformations & Trans,
+                                              DenseMatrix &elmat)
+{
+   MFEM_VERIFY(test_fe.GetMapType() == FiniteElement::VALUE,
+               "TraceIntegrator::AssembleTraceFaceMatrix: Test space should be H1");
+   MFEM_VERIFY(trial_face_fe.GetMapType() == FiniteElement::INTEGRAL,
+               "TraceIntegrator::AssembleTraceFaceMatrix: Trial space should be RT trace");
+
+   int i, j, face_ndof, ndof;
+   int order;
+
+   face_ndof = trial_face_fe.GetDof();
+   ndof = test_fe.GetDof();
+
+   face_shape.SetSize(face_ndof);
+   shape.SetSize(ndof);
+
+   elmat.SetSize(ndof, face_ndof);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      order = test_fe.GetOrder();
+      order += trial_face_fe.GetOrder();
+      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+   }
+
+   int iel = Trans.Elem1->ElementNo;
+   if (iel != elem)
+   {
+      MFEM_VERIFY(elem == Trans.Elem2->ElementNo, "Elem != Trans.Elem2->ElementNo");
+   }
+
+   double scale = 1.0;
+   if (iel != elem) { scale = -1.; }
+   for (int p = 0; p < ir->GetNPoints(); p++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(p);
+
+      // Set the integration point in the face and the neighboring elements
+      Trans.SetAllIntPoints(&ip);
+      // Trace finite element shape function
+      trial_face_fe.CalcPhysShape(Trans,face_shape);
+
+      // Finite element shape function
+      ElementTransformation * eltrans = (iel == elem) ? Trans.Elem1 : Trans.Elem2;
+      test_fe.CalcPhysShape(*eltrans, shape);
+
+      face_shape *= Trans.Weight()*ip.weight*scale;
+      for (i = 0; i < ndof; i++)
+      {
+         for (j = 0; j < face_ndof; j++)
+         {
+            elmat(i, j) += shape(i) * face_shape(j);
+         }
+      }
+   }
+}
+
+void NormalTraceIntegrator::AssembleTraceFaceMatrix(int elem,
+                                                    const FiniteElement &trial_face_fe,
+                                                    const FiniteElement &test_fe,
+                                                    FaceElementTransformations &Trans,
+                                                    DenseMatrix &elmat)
+{
+   int i, j, face_ndof, ndof, dim;
+   int order;
+
+   MFEM_VERIFY(test_fe.GetMapType() == FiniteElement::H_DIV,
+               "NormalTraceIntegrator::AssembleTraceFaceMatrix: Test space should be RT");
+   MFEM_VERIFY(trial_face_fe.GetMapType() == FiniteElement::VALUE,
+               "NormalTraceIntegrator::AssembleTraceFaceMatrix: Trial space should be H1 (trace)");
+
+   face_ndof = trial_face_fe.GetDof();
+   ndof = test_fe.GetDof();
+   dim = test_fe.GetDim();
+
+   face_shape.SetSize(face_ndof);
+   normal.SetSize(dim);
+   shape.SetSize(ndof,dim);
+   shape_n.SetSize(ndof);
+
+   elmat.SetSize(ndof, face_ndof);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      order = test_fe.GetOrder();
+      order += trial_face_fe.GetOrder();
+      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+   }
+
+   int iel = Trans.Elem1->ElementNo;
+   if (iel != elem)
+   {
+      MFEM_VERIFY(elem == Trans.Elem2->ElementNo, "Elem != Trans.Elem2->ElementNo");
+   }
+
+   double scale = 1.0;
+   if (iel != elem) { scale = -1.; }
+
+   for (int p = 0; p < ir->GetNPoints(); p++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(p);
+      Trans.SetAllIntPoints(&ip);
+      trial_face_fe.CalcPhysShape(Trans, face_shape);
+      CalcOrtho(Trans.Jacobian(),normal);
+      ElementTransformation * etrans = (iel == elem) ? Trans.Elem1 : Trans.Elem2;
+      test_fe.CalcVShape(*etrans, shape);
+      shape.Mult(normal, shape_n);
+      face_shape *= ip.weight*scale;
+
+      for (i = 0; i < ndof; i++)
+      {
+         for (j = 0; j < face_ndof; j++)
+         {
+            elmat(i, j) += shape_n(i) * face_shape(j);
+         }
+      }
+   }
+}
+
+void TangentTraceIntegrator::AssembleTraceFaceMatrix(int elem,
+                                                     const FiniteElement &trial_face_fe,
+                                                     const FiniteElement &test_fe,
+                                                     FaceElementTransformations & Trans,
+                                                     DenseMatrix &elmat)
+{
+
+   MFEM_VERIFY(test_fe.GetMapType() == FiniteElement::H_CURL,
+               "TangentTraceIntegrator::AssembleTraceFaceMatrix: Test space should be ND");
+
+   int face_ndof, ndof, dim;
+   int order;
+   dim = test_fe.GetDim();
+   if (dim == 3)
+   {
+      std::string msg =
+         "Trial space should be ND face trace and test space should be a ND vector field in 3D ";
+      MFEM_VERIFY(trial_face_fe.GetMapType() == FiniteElement::H_CURL &&
+                  trial_face_fe.GetDim() == 2 && test_fe.GetDim() == 3, msg);
+   }
+   else
+   {
+      std::string msg =
+         "Trial space should be H1 edge trace and test space should be a ND vector field in 2D";
+      MFEM_VERIFY(trial_face_fe.GetMapType() == FiniteElement::VALUE &&
+                  trial_face_fe.GetDim() == 1 && test_fe.GetDim() == 2, msg);
+   }
+   face_ndof = trial_face_fe.GetDof();
+   ndof = test_fe.GetDof();
+
+   int dimc = (dim == 3) ? 3 : 1;
+
+   face_shape.SetSize(face_ndof,dimc);
+   shape_n.SetSize(ndof,dimc);
+   shape.SetSize(ndof,dim);
+   normal.SetSize(dim);
+   DenseMatrix face_shape_n(face_ndof,dimc);
+
+   elmat.SetSize(ndof, face_ndof);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      order = test_fe.GetOrder();
+      order += trial_face_fe.GetOrder();
+      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+   }
+
+   int iel = Trans.Elem1->ElementNo;
+   if (iel != elem)
+   {
+      MFEM_VERIFY(elem == Trans.Elem2->ElementNo, "Elem != Trans.Elem2->ElementNo");
+   }
+
+   double scale = 1.0;
+   if (iel != elem) { scale = -1.; }
+   for (int p = 0; p < ir->GetNPoints(); p++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(p);
+      // Set the integration point in the face and the neighboring elements
+      Trans.SetAllIntPoints(&ip);
+      // Trace finite element shape function
+      if (dim == 3)
+      {
+         trial_face_fe.CalcVShape(Trans,face_shape);
+      }
+      else
+      {
+         face_shape.GetColumnReference(0,temp);
+         trial_face_fe.CalcPhysShape(Trans,temp);
+      }
+      CalcOrtho(Trans.Jacobian(),normal);
+      ElementTransformation * eltrans = (iel == elem) ? Trans.Elem1 : Trans.Elem2;
+      test_fe.CalcVShape(*eltrans, shape);
+
+      // rotate
+      cross_product(normal, shape, shape_n);
+
+      const double w = scale*ip.weight;
+      AddMult_a_ABt(w,shape_n, face_shape, elmat);
+   }
+}
 
 void NormalInterpolator::AssembleElementMatrix2(
    const FiniteElement &dom_fe, const FiniteElement &ran_fe,

fem/bilininteg.hpp

@@ -61,6 +61,8 @@ public:
    virtual void AssemblePA(const FiniteElementSpace &trial_fes,
                            const FiniteElementSpace &test_fes);
 
+   virtual void AssemblePABoundary(const FiniteElementSpace &fes);
+
    virtual void AssemblePAInteriorFaces(const FiniteElementSpace &fes);
 
    virtual void AssemblePABoundaryFaces(const FiniteElementSpace &fes);
@@ -159,6 +161,15 @@ public:
                                    FaceElementTransformations &Trans,
                                    DenseMatrix &elmat);
 
+   /** Abstract method used for assembling TraceFaceIntegrators for
+       DPG weak formulations. */
+   virtual void AssembleTraceFaceMatrix(int elem,
+                                        const FiniteElement &trial_face_fe,
+                                        const FiniteElement &test_fe,
+                                        FaceElementTransformations &Trans,
+                                        DenseMatrix &elmat);
+
+
    /// @brief Perform the local action of the BilinearFormIntegrator.
    /// Note that the default implementation in the base class is general but not
    /// efficient.
@@ -292,6 +303,12 @@ public:
       bfi->AssemblePA(fes);
    }
 
+   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
+                           const FiniteElementSpace &test_fes)
+   {
+      bfi->AssemblePA(test_fes, trial_fes); // Reverse test and trial
+   }
+
    virtual void AssemblePAInteriorFaces(const FiniteElementSpace &fes)
    {
       bfi->AssemblePAInteriorFaces(fes);
@@ -2183,8 +2200,9 @@ protected:
    // PA extension
    const FiniteElementSpace *fespace;
    Vector pa_data;
-   const DofToQuad *maps;         ///< Not owned
-   const GeometricFactors *geom;  ///< Not owned
+   const DofToQuad *maps;                 ///< Not owned
+   const GeometricFactors *geom;          ///< Not owned
+   const FaceGeometricFactors *face_geom; ///< Not owned
    int dim, ne, nq, dofs1D, quad1D;
 
 public:
@@ -2211,6 +2229,8 @@ public:
 
    virtual void AssemblePA(const FiniteElementSpace &fes);
 
+   virtual void AssemblePABoundary(const FiniteElementSpace &fes);
+
    virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
                            const bool add);
 
@@ -3301,6 +3321,87 @@ public:
                                    DenseMatrix &elmat);
 };
 
+/** Integrator for the DPG form: < v, w > over a face (the interface) where
+    the trial variable v is defined on the interface
+    (H^-1/2 i.e., v:=u⋅n normal trace of H(div))
+    and the test variable w is in an H1-conforming space. */
+class TraceIntegrator : public BilinearFormIntegrator
+{
+private:
+   Vector face_shape, shape;
+public:
+   TraceIntegrator() { }
+   void AssembleTraceFaceMatrix(int elem,
+                                const FiniteElement &trial_face_fe,
+                                const FiniteElement &test_fe,
+                                FaceElementTransformations &Trans,
+                                DenseMatrix &elmat);
+};
+
+/** Integrator for the form: < v, w.n > over a face (the interface) where
+    the trial variable v is defined on the interface (H^1/2, i.e., trace of H1)
+    and the test variable w is in an H(div)-conforming space. */
+class NormalTraceIntegrator : public BilinearFormIntegrator
+{
+private:
+   Vector face_shape, normal, shape_n;
+   DenseMatrix shape;
+
+public:
+   NormalTraceIntegrator() { }
+   virtual void AssembleTraceFaceMatrix(int ielem,
+                                        const FiniteElement &trial_face_fe,
+                                        const FiniteElement &test_fe,
+                                        FaceElementTransformations &Trans,
+                                        DenseMatrix &elmat);
+};
+
+
+/** Integrator for the form: < v, w × n > over a face (the interface)
+ *  In 3D the trial variable v is defined on the interface (H^-1/2(curl), trace of H(curl))
+ *  In 2D it's defined on the interface (H^1/2, trace of H1)
+ *  The test variable w is in an H(curl)-conforming space. */
+class TangentTraceIntegrator : public BilinearFormIntegrator
+{
+private:
+   DenseMatrix face_shape, shape, shape_n;
+   Vector normal;
+   Vector temp;
+
+   void cross_product(const Vector & x, const DenseMatrix & Y, DenseMatrix & Z)
+   {
+      int dim = x.Size();
+      MFEM_VERIFY(Y.Width() == dim, "Size missmatch");
+      int dimc = dim == 3 ? dim : 1;
+      int h = Y.Height();
+      Z.SetSize(h,dimc);
+      if (dim == 3)
+      {
+         for (int i = 0; i<h; i++)
+         {
+            Z(i,0) = x(2) * Y(i,1) - x(1) * Y(i,2);
+            Z(i,1) = x(0) * Y(i,2) - x(2) * Y(i,0);
+            Z(i,2) = x(1) * Y(i,0) - x(0) * Y(i,1);
+         }
+      }
+      else
+      {
+         for (int i = 0; i<h; i++)
+         {
+            Z(i,0) = x(1) * Y(i,0) - x(0) * Y(i,1);
+         }
+      }
+   }
+
+public:
+   TangentTraceIntegrator() { }
+   void AssembleTraceFaceMatrix(int elem,
+                                const FiniteElement &trial_face_fe,
+                                const FiniteElement &test_fe,
+                                FaceElementTransformations &Trans,
+                                DenseMatrix &elmat);
+};
+
 /** Abstract class to serve as a base for local interpolators to be used in the
     DiscreteLinearOperator class. */
 class DiscreteInterpolator : public BilinearFormIntegrator { };
@@ -3336,7 +3437,7 @@ public:
 
 private:
    /// 1D finite element that generates and owns the 1D DofToQuad maps below
-   FiniteElement * dofquad_fe;
+   FiniteElement *dofquad_fe;
 
    bool B_id; // is the B basis operator (maps_C_C) the identity?
    const DofToQuad *maps_C_C; // one-d map with Lobatto rows, Lobatto columns
@@ -3351,6 +3452,8 @@ private:
 class IdentityInterpolator : public DiscreteInterpolator
 {
 public:
+   IdentityInterpolator(): dofquad_fe(NULL) { }
+
    virtual void AssembleElementMatrix2(const FiniteElement &dom_fe,
                                        const FiniteElement &ran_fe,
                                        ElementTransformation &Trans,
@@ -3365,9 +3468,11 @@ public:
    virtual void AddMultPA(const Vector &x, Vector &y) const;
    virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
 
+   virtual ~IdentityInterpolator() { delete dofquad_fe; }
+
 private:
    /// 1D finite element that generates and owns the 1D DofToQuad maps below
-   FiniteElement * dofquad_fe;
+   FiniteElement *dofquad_fe;
 
    const DofToQuad *maps_C_C; // one-d map with Lobatto rows, Lobatto columns
    const DofToQuad *maps_O_C; // one-d map with Legendre rows, Lobatto columns
@@ -3522,17 +3627,5 @@ protected:
    VectorCoefficient *VQ;
 };
 
-
-
-// PA Diffusion Assemble 2D kernel
-template<const int T_SDIM>
-void PADiffusionSetup2D(const int Q1D,
-                        const int coeffDim,
-                        const int NE,
-                        const Array<double> &w,
-                        const Vector &j,
-                        const Vector &c,
-                        Vector &d);
-
 }
 #endif

fem/bilininteg_mass_pa.cpp

@@ -1,736 +0,0 @@
-// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-#include "qfunction.hpp"
-#include "ceed/integrators/mass/mass.hpp"
-#include "bilininteg_mass_pa.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-// PA Mass Integrator
-
-// PA Mass Assemble kernel
-
-void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
-{
-   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
-                         Device::GetDeviceMemoryType() : pa_mt;
-
-   // Assuming the same element type
-   fespace = &fes;
-   Mesh *mesh = fes.GetMesh();
-   if (mesh->GetNE() == 0) { return; }
-   const FiniteElement &el = *fes.GetFE(0);
-   ElementTransformation *T0 = mesh->GetElementTransformation(0);
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, *T0);
-   if (DeviceCanUseCeed())
-   {
-      delete ceedOp;
-      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
-                         fes.IsVariableOrder();
-      if (mixed)
-      {
-         ceedOp = new ceed::MixedPAMassIntegrator(*this, fes, Q);
-      }
-      else
-      {
-         ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q);
-      }
-      return;
-   }
-   int map_type = el.GetMapType();
-   dim = mesh->Dimension();
-   ne = fes.GetMesh()->GetNE();
-   nq = ir->GetNPoints();
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::DETERMINANTS, mt);
-   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
-   dofs1D = maps->ndof;
-   quad1D = maps->nqpt;
-   pa_data.SetSize(ne*nq, mt);
-
-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
-
-   if (dim==1) { MFEM_ABORT("Not supported yet... stay tuned!"); }
-   if (dim==2)
-   {
-      const int NE = ne;
-      const int Q1D = quad1D;
-      const bool const_c = coeff.Size() == 1;
-      const bool by_val = map_type == FiniteElement::VALUE;
-      const auto W = Reshape(ir->GetWeights().Read(), Q1D,Q1D);
-      const auto J = Reshape(geom->detJ.Read(), Q1D,Q1D,NE);
-      const auto C = const_c ? Reshape(coeff.Read(), 1,1,1) :
-                     Reshape(coeff.Read(), Q1D,Q1D,NE);
-      auto v = Reshape(pa_data.Write(), Q1D,Q1D, NE);
-      MFEM_FORALL_2D(e, NE, Q1D,Q1D,1,
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               const double detJ = J(qx,qy,e);
-               const double coeff = const_c ? C(0,0,0) : C(qx,qy,e);
-               v(qx,qy,e) =  W(qx,qy) * coeff * (by_val ? detJ : 1.0/detJ);
-            }
-         }
-      });
-   }
-   if (dim==3)
-   {
-      const int NE = ne;
-      const int Q1D = quad1D;
-      const bool const_c = coeff.Size() == 1;
-      const bool by_val = map_type == FiniteElement::VALUE;
-      const auto W = Reshape(ir->GetWeights().Read(), Q1D,Q1D,Q1D);
-      const auto J = Reshape(geom->detJ.Read(), Q1D,Q1D,Q1D,NE);
-      const auto C = const_c ? Reshape(coeff.Read(), 1,1,1,1) :
-                     Reshape(coeff.Read(), Q1D,Q1D,Q1D,NE);
-      auto v = Reshape(pa_data.Write(), Q1D,Q1D,Q1D,NE);
-      MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(qz,z,Q1D)
-               {
-                  const double detJ = J(qx,qy,qz,e);
-                  const double coeff = const_c ? C(0,0,0,0) : C(qx,qy,qz,e);
-                  v(qx,qy,qz,e) = W(qx,qy,qz) * coeff * (by_val ? detJ : 1.0/detJ);
-               }
-            }
-         }
-      });
-   }
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void PAMassAssembleDiagonal2D(const int NE,
-                                     const Array<double> &b,
-                                     const Vector &d,
-                                     Vector &y,
-                                     const int d1d = 0,
-                                     const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto D = Reshape(d.Read(), Q1D, Q1D, NE);
-   auto Y = Reshape(y.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      double QD[MQ1][MD1];
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            QD[qx][dy] = 0.0;
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               QD[qx][dy] += B(qy, dy) * B(qy, dy) * D(qx, qy, e);
-            }
-         }
-      }
-      for (int dy = 0; dy < D1D; ++dy)
-      {
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               Y(dx,dy,e) += B(qx, dx) * B(qx, dx) * QD[qx][dy];
-            }
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
-static void SmemPAMassAssembleDiagonal2D(const int NE,
-                                         const Array<double> &b_,
-                                         const Vector &d_,
-                                         Vector &y_,
-                                         const int d1d = 0,
-                                         const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-   MFEM_VERIFY(D1D <= MD1, "");
-   MFEM_VERIFY(Q1D <= MQ1, "");
-   auto b = Reshape(b_.Read(), Q1D, D1D);
-   auto D = Reshape(d_.Read(), Q1D, Q1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      MFEM_SHARED double B[MQ1][MD1];
-      MFEM_SHARED double QDZ[NBZ][MQ1][MD1];
-      double (*QD)[MD1] = (double (*)[MD1])(QDZ + tidz);
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B[q][d] = b(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            QD[qx][dy] = 0.0;
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               QD[qx][dy] += B[qy][dy] * B[qy][dy] * D(qx, qy, e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               // might need absolute values on next line
-               Y(dx,dy,e) += B[qx][dx] * B[qx][dx] * QD[qx][dy];
-            }
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void PAMassAssembleDiagonal3D(const int NE,
-                                     const Array<double> &b,
-                                     const Vector &d,
-                                     Vector &y,
-                                     const int d1d = 0,
-                                     const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto D = Reshape(d.Read(), Q1D, Q1D, Q1D, NE);
-   auto Y = Reshape(y.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      double QQD[MQ1][MQ1][MD1];
-      double QDD[MQ1][MD1][MD1];
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               QQD[qx][qy][dz] = 0.0;
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  QQD[qx][qy][dz] += B(qz, dz) * B(qz, dz) * D(qx, qy, qz, e);
-               }
-            }
-         }
-      }
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               QDD[qx][dy][dz] = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  QDD[qx][dy][dz] += B(qy, dy) * B(qy, dy) * QQD[qx][qy][dz];
-               }
-            }
-         }
-      }
-      for (int dz = 0; dz < D1D; ++dz)
-      {
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               double t = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  t += B(qx, dx) * B(qx, dx) * QDD[qx][dy][dz];
-               }
-               Y(dx, dy, dz, e) += t;
-            }
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void SmemPAMassAssembleDiagonal3D(const int NE,
-                                         const Array<double> &b_,
-                                         const Vector &d_,
-                                         Vector &y_,
-                                         const int d1d = 0,
-                                         const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-   MFEM_VERIFY(D1D <= MD1, "");
-   MFEM_VERIFY(Q1D <= MQ1, "");
-   auto b = Reshape(b_.Read(), Q1D, D1D);
-   auto D = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      MFEM_SHARED double B[MQ1][MD1];
-      MFEM_SHARED double QQD[MQ1][MQ1][MD1];
-      MFEM_SHARED double QDD[MQ1][MD1][MD1];
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B[q][d] = b(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(dz,z,D1D)
-            {
-               QQD[qx][qy][dz] = 0.0;
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  QQD[qx][qy][dz] += B[qz][dz] * B[qz][dz] * D(qx, qy, qz, e);
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               QDD[qx][dy][dz] = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  QDD[qx][dy][dz] += B[qy][dy] * B[qy][dy] * QQD[qx][qy][dz];
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               double t = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  t += B[qx][dx] * B[qx][dx] * QDD[qx][dy][dz];
-               }
-               Y(dx, dy, dz, e) += t;
-            }
-         }
-      }
-   });
-}
-
-static void PAMassAssembleDiagonal(const int dim, const int D1D,
-                                   const int Q1D, const int NE,
-                                   const Array<double> &B,
-                                   const Vector &D,
-                                   Vector &Y)
-{
-   if (dim == 2)
-   {
-      switch ((D1D << 4 ) | Q1D)
-      {
-         case 0x22: return SmemPAMassAssembleDiagonal2D<2,2,16>(NE,B,D,Y);
-         case 0x33: return SmemPAMassAssembleDiagonal2D<3,3,16>(NE,B,D,Y);
-         case 0x44: return SmemPAMassAssembleDiagonal2D<4,4,8>(NE,B,D,Y);
-         case 0x55: return SmemPAMassAssembleDiagonal2D<5,5,8>(NE,B,D,Y);
-         case 0x66: return SmemPAMassAssembleDiagonal2D<6,6,4>(NE,B,D,Y);
-         case 0x77: return SmemPAMassAssembleDiagonal2D<7,7,4>(NE,B,D,Y);
-         case 0x88: return SmemPAMassAssembleDiagonal2D<8,8,2>(NE,B,D,Y);
-         case 0x99: return SmemPAMassAssembleDiagonal2D<9,9,2>(NE,B,D,Y);
-         default:   return PAMassAssembleDiagonal2D(NE,B,D,Y,D1D,Q1D);
-      }
-   }
-   else if (dim == 3)
-   {
-      switch ((D1D << 4 ) | Q1D)
-      {
-         case 0x23: return SmemPAMassAssembleDiagonal3D<2,3>(NE,B,D,Y);
-         case 0x24: return SmemPAMassAssembleDiagonal3D<2,4>(NE,B,D,Y);
-         case 0x26: return SmemPAMassAssembleDiagonal3D<2,6>(NE,B,D,Y);
-         case 0x34: return SmemPAMassAssembleDiagonal3D<3,4>(NE,B,D,Y);
-         case 0x35: return SmemPAMassAssembleDiagonal3D<3,5>(NE,B,D,Y);
-         case 0x45: return SmemPAMassAssembleDiagonal3D<4,5>(NE,B,D,Y);
-         case 0x48: return SmemPAMassAssembleDiagonal3D<4,8>(NE,B,D,Y);
-         case 0x56: return SmemPAMassAssembleDiagonal3D<5,6>(NE,B,D,Y);
-         case 0x67: return SmemPAMassAssembleDiagonal3D<6,7>(NE,B,D,Y);
-         case 0x78: return SmemPAMassAssembleDiagonal3D<7,8>(NE,B,D,Y);
-         case 0x89: return SmemPAMassAssembleDiagonal3D<8,9>(NE,B,D,Y);
-         default:   return PAMassAssembleDiagonal3D(NE,B,D,Y,D1D,Q1D);
-      }
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-
-void MassIntegrator::AssembleDiagonalPA(Vector &diag)
-{
-   if (DeviceCanUseCeed())
-   {
-      ceedOp->GetDiagonal(diag);
-   }
-   else
-   {
-      PAMassAssembleDiagonal(dim, dofs1D, quad1D, ne, maps->B, pa_data, diag);
-   }
-}
-
-
-#ifdef MFEM_USE_OCCA
-// OCCA PA Mass Apply 2D kernel
-static void OccaPAMassApply2D(const int D1D,
-                              const int Q1D,
-                              const int NE,
-                              const Array<double> &B,
-                              const Array<double> &Bt,
-                              const Vector &D,
-                              const Vector &X,
-                              Vector &Y)
-{
-   occa::properties props;
-   props["defines/D1D"] = D1D;
-   props["defines/Q1D"] = Q1D;
-   const occa::memory o_B = OccaMemoryRead(B.GetMemory(), B.Size());
-   const occa::memory o_Bt = OccaMemoryRead(Bt.GetMemory(), Bt.Size());
-   const occa::memory o_D = OccaMemoryRead(D.GetMemory(), D.Size());
-   const occa::memory o_X = OccaMemoryRead(X.GetMemory(), X.Size());
-   occa::memory o_Y = OccaMemoryReadWrite(Y.GetMemory(), Y.Size());
-   const occa_id_t id = std::make_pair(D1D,Q1D);
-   if (!Device::Allows(Backend::OCCA_CUDA))
-   {
-      static occa_kernel_t OccaMassApply2D_cpu;
-      if (OccaMassApply2D_cpu.find(id) == OccaMassApply2D_cpu.end())
-      {
-         const occa::kernel MassApply2D_CPU =
-            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
-                                        "MassApply2D_CPU", props);
-         OccaMassApply2D_cpu.emplace(id, MassApply2D_CPU);
-      }
-      OccaMassApply2D_cpu.at(id)(NE, o_B, o_Bt, o_D, o_X, o_Y);
-   }
-   else
-   {
-      static occa_kernel_t OccaMassApply2D_gpu;
-      if (OccaMassApply2D_gpu.find(id) == OccaMassApply2D_gpu.end())
-      {
-         const occa::kernel MassApply2D_GPU =
-            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
-                                        "MassApply2D_GPU", props);
-         OccaMassApply2D_gpu.emplace(id, MassApply2D_GPU);
-      }
-      OccaMassApply2D_gpu.at(id)(NE, o_B, o_Bt, o_D, o_X, o_Y);
-   }
-}
-
-// OCCA PA Mass Apply 3D kernel
-static void OccaPAMassApply3D(const int D1D,
-                              const int Q1D,
-                              const int NE,
-                              const Array<double> &B,
-                              const Array<double> &Bt,
-                              const Vector &D,
-                              const Vector &X,
-                              Vector &Y)
-{
-   occa::properties props;
-   props["defines/D1D"] = D1D;
-   props["defines/Q1D"] = Q1D;
-   const occa::memory o_B = OccaMemoryRead(B.GetMemory(), B.Size());
-   const occa::memory o_Bt = OccaMemoryRead(Bt.GetMemory(), Bt.Size());
-   const occa::memory o_D = OccaMemoryRead(D.GetMemory(), D.Size());
-   const occa::memory o_X = OccaMemoryRead(X.GetMemory(), X.Size());
-   occa::memory o_Y = OccaMemoryReadWrite(Y.GetMemory(), Y.Size());
-   const occa_id_t id = std::make_pair(D1D,Q1D);
-   if (!Device::Allows(Backend::OCCA_CUDA))
-   {
-      static occa_kernel_t OccaMassApply3D_cpu;
-      if (OccaMassApply3D_cpu.find(id) == OccaMassApply3D_cpu.end())
-      {
-         const occa::kernel MassApply3D_CPU =
-            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
-                                        "MassApply3D_CPU", props);
-         OccaMassApply3D_cpu.emplace(id, MassApply3D_CPU);
-      }
-      OccaMassApply3D_cpu.at(id)(NE, o_B, o_Bt, o_D, o_X, o_Y);
-   }
-   else
-   {
-      static occa_kernel_t OccaMassApply3D_gpu;
-      if (OccaMassApply3D_gpu.find(id) == OccaMassApply3D_gpu.end())
-      {
-         const occa::kernel MassApply3D_GPU =
-            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
-                                        "MassApply3D_GPU", props);
-         OccaMassApply3D_gpu.emplace(id, MassApply3D_GPU);
-      }
-      OccaMassApply3D_gpu.at(id)(NE, o_B, o_Bt, o_D, o_X, o_Y);
-   }
-}
-#endif // MFEM_USE_OCCA
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void PAMassApply2D(const int NE,
-                          const Array<double> &b_,
-                          const Array<double> &bt_,
-                          const Vector &d_,
-                          const Vector &x_,
-                          Vector &y_,
-                          const int d1d = 0,
-                          const int q1d = 0)
-{
-   MFEM_VERIFY(T_D1D ? T_D1D : d1d <= MAX_D1D, "");
-   MFEM_VERIFY(T_Q1D ? T_Q1D : q1d <= MAX_Q1D, "");
-
-   const auto B = b_.Read();
-   const auto Bt = bt_.Read();
-   const auto D = d_.Read();
-   const auto X = x_.Read();
-   auto Y = y_.ReadWrite();
-
-   MFEM_FORALL(e, NE,
-   {
-      internal::PAMassApply2D_Element(e, NE, B, Bt, D, X, Y, d1d, q1d);
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
-static void SmemPAMassApply2D(const int NE,
-                              const Array<double> &b_,
-                              const Array<double> &bt_,
-                              const Vector &d_,
-                              const Vector &x_,
-                              Vector &y_,
-                              const int d1d = 0,
-                              const int q1d = 0)
-{
-   MFEM_CONTRACT_VAR(bt_);
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-   MFEM_VERIFY(D1D <= MD1, "");
-   MFEM_VERIFY(Q1D <= MQ1, "");
-   const auto b = b_.Read();
-   const auto D = d_.Read();
-   const auto x = x_.Read();
-   auto Y = y_.ReadWrite();
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
-   {
-      internal::SmemPAMassApply2D_Element<T_D1D,T_Q1D,T_NBZ>(e, NE, b, D, x, Y, d1d, q1d);
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void PAMassApply3D(const int NE,
-                          const Array<double> &b_,
-                          const Array<double> &bt_,
-                          const Vector &d_,
-                          const Vector &x_,
-                          Vector &y_,
-                          const int d1d = 0,
-                          const int q1d = 0)
-{
-   MFEM_VERIFY(T_D1D ? T_D1D : d1d <= MAX_D1D, "");
-   MFEM_VERIFY(T_Q1D ? T_Q1D : q1d <= MAX_Q1D, "");
-
-   const auto B = b_.Read();
-   const auto Bt = bt_.Read();
-   const auto D = d_.Read();
-   const auto X = x_.Read();
-   auto Y = y_.ReadWrite();
-
-   MFEM_FORALL(e, NE,
-   {
-      internal::PAMassApply3D_Element(e, NE, B, Bt, D, X, Y, d1d, q1d);
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void SmemPAMassApply3D(const int NE,
-                              const Array<double> &b_,
-                              const Array<double> &bt_,
-                              const Vector &d_,
-                              const Vector &x_,
-                              Vector &y_,
-                              const int d1d = 0,
-                              const int q1d = 0)
-{
-   MFEM_CONTRACT_VAR(bt_);
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int M1Q = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int M1D = T_D1D ? T_D1D : MAX_D1D;
-   MFEM_VERIFY(D1D <= M1D, "");
-   MFEM_VERIFY(Q1D <= M1Q, "");
-   auto b = b_.Read();
-   auto d = d_.Read();
-   auto x = x_.Read();
-   auto y = y_.ReadWrite();
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
-   {
-      internal::SmemPAMassApply3D_Element<T_D1D,T_Q1D>(e, NE, b, d, x, y, d1d, q1d);
-   });
-}
-
-static void PAMassApply(const int dim,
-                        const int D1D,
-                        const int Q1D,
-                        const int NE,
-                        const Array<double> &B,
-                        const Array<double> &Bt,
-                        const Vector &D,
-                        const Vector &X,
-                        Vector &Y)
-{
-#ifdef MFEM_USE_OCCA
-   if (DeviceCanUseOcca())
-   {
-      if (dim == 2)
-      {
-         return OccaPAMassApply2D(D1D,Q1D,NE,B,Bt,D,X,Y);
-      }
-      if (dim == 3)
-      {
-         return OccaPAMassApply3D(D1D,Q1D,NE,B,Bt,D,X,Y);
-      }
-      MFEM_ABORT("OCCA PA Mass Apply unknown kernel!");
-   }
-#endif // MFEM_USE_OCCA
-   const int id = (D1D << 4) | Q1D;
-
-   if (dim == 2)
-   {
-      switch (id)
-      {
-         case 0x22: return SmemPAMassApply2D<2,2,16>(NE,B,Bt,D,X,Y);
-         case 0x24: return SmemPAMassApply2D<2,4,16>(NE,B,Bt,D,X,Y);
-         case 0x33: return SmemPAMassApply2D<3,3,16>(NE,B,Bt,D,X,Y);
-         case 0x34: return SmemPAMassApply2D<3,4,16>(NE,B,Bt,D,X,Y);
-         case 0x35: return SmemPAMassApply2D<3,5,16>(NE,B,Bt,D,X,Y);
-         case 0x36: return SmemPAMassApply2D<3,6,16>(NE,B,Bt,D,X,Y);
-         case 0x44: return SmemPAMassApply2D<4,4,8>(NE,B,Bt,D,X,Y);
-         case 0x46: return SmemPAMassApply2D<4,6,8>(NE,B,Bt,D,X,Y);
-         case 0x48: return SmemPAMassApply2D<4,8,4>(NE,B,Bt,D,X,Y);
-         case 0x55: return SmemPAMassApply2D<5,5,8>(NE,B,Bt,D,X,Y);
-         case 0x57: return SmemPAMassApply2D<5,7,8>(NE,B,Bt,D,X,Y);
-         case 0x58: return SmemPAMassApply2D<5,8,2>(NE,B,Bt,D,X,Y);
-         case 0x66: return SmemPAMassApply2D<6,6,4>(NE,B,Bt,D,X,Y);
-         case 0x77: return SmemPAMassApply2D<7,7,4>(NE,B,Bt,D,X,Y);
-         case 0x88: return SmemPAMassApply2D<8,8,2>(NE,B,Bt,D,X,Y);
-         case 0x99: return SmemPAMassApply2D<9,9,2>(NE,B,Bt,D,X,Y);
-         default:   return PAMassApply2D(NE,B,Bt,D,X,Y,D1D,Q1D);
-      }
-   }
-   else if (dim == 3)
-   {
-      switch (id)
-      {
-         case 0x22: return SmemPAMassApply3D<2,2>(NE,B,Bt,D,X,Y);
-         case 0x23: return SmemPAMassApply3D<2,3>(NE,B,Bt,D,X,Y);
-         case 0x24: return SmemPAMassApply3D<2,4>(NE,B,Bt,D,X,Y);
-         case 0x26: return SmemPAMassApply3D<2,6>(NE,B,Bt,D,X,Y);
-         case 0x34: return SmemPAMassApply3D<3,4>(NE,B,Bt,D,X,Y);
-         case 0x35: return SmemPAMassApply3D<3,5>(NE,B,Bt,D,X,Y);
-         case 0x36: return SmemPAMassApply3D<3,6>(NE,B,Bt,D,X,Y);
-         case 0x37: return SmemPAMassApply3D<3,7>(NE,B,Bt,D,X,Y);
-         case 0x45: return SmemPAMassApply3D<4,5>(NE,B,Bt,D,X,Y);
-         case 0x46: return SmemPAMassApply3D<4,6>(NE,B,Bt,D,X,Y);
-         case 0x48: return SmemPAMassApply3D<4,8>(NE,B,Bt,D,X,Y);
-         case 0x56: return SmemPAMassApply3D<5,6>(NE,B,Bt,D,X,Y);
-         case 0x58: return SmemPAMassApply3D<5,8>(NE,B,Bt,D,X,Y);
-         case 0x67: return SmemPAMassApply3D<6,7>(NE,B,Bt,D,X,Y);
-         case 0x78: return SmemPAMassApply3D<7,8>(NE,B,Bt,D,X,Y);
-         case 0x89: return SmemPAMassApply3D<8,9>(NE,B,Bt,D,X,Y);
-         case 0x9A: return SmemPAMassApply3D<9,10>(NE,B,Bt,D,X,Y);
-         default:   return PAMassApply3D(NE,B,Bt,D,X,Y,D1D,Q1D);
-      }
-   }
-   mfem::out << "Unknown kernel 0x" << std::hex << id << std::endl;
-   MFEM_ABORT("Unknown kernel.");
-}
-
-void MassIntegrator::AddMultPA(const Vector &x, Vector &y) const
-{
-   if (DeviceCanUseCeed())
-   {
-      ceedOp->AddMult(x, y);
-   }
-   else
-   {
-      PAMassApply(dim, dofs1D, quad1D, ne, maps->B, maps->Bt, pa_data, x, y);
-   }
-}
-
-void MassIntegrator::AddMultTransposePA(const Vector &x, Vector &y) const
-{
-   // Mass integrator is symmetric
-   AddMultPA(x, y);
-}
-
-} // namespace mfem

fem/ceed/interface/coefficient.hpp

@@ -288,7 +288,7 @@ void InitCoefficientWithIndices(mfem::Coefficient *Q, mfem::Mesh &mesh,
       auto in = Reshape(qFun.Read(), nq, ne);
       auto d_indices = Read(m_indices, nelem);
       auto out = Reshape(ceedCoeff->coeff.Write(), nq, nelem);
-      MFEM_FORALL(i, nelem * nq,
+      mfem::forall(nelem * nq, [=] MFEM_HOST_DEVICE (int i)
       {
          const int q = i%nq;
          const int sub_e = i/nq;
@@ -378,7 +378,7 @@ void InitCoefficientWithIndices(mfem::VectorCoefficient *VQ, mfem::Mesh &mesh,
       auto in = Reshape(qFun.Read(), dim, nq, ne);
       auto d_indices = Read(m_indices, nelem);
       auto out = Reshape(ceedCoeff->coeff.Write(), dim, nq, nelem);
-      MFEM_FORALL(i, nelem * nq,
+      mfem::forall(nelem * nq, [=] MFEM_HOST_DEVICE (int i)
       {
          const int q = i%nq;
          const int sub_e = i/nq;

fem/ceed/solvers/algebraic.cpp

@@ -519,7 +519,7 @@ int CeedVectorPointwiseMult(CeedVector a, const CeedVector b)
    ierr = CeedVectorGetArray(a, mem, &a_data); CeedChk(ierr);
    ierr = CeedVectorGetArrayRead(b, mem, &b_data); CeedChk(ierr);
    MFEM_VERIFY(int(length) == length, "length overflow");
-   MFEM_FORALL(i, length,
+   mfem::forall(length, [=] MFEM_HOST_DEVICE (int i)
    {a_data[i] *= b_data[i];});
 
    ierr = CeedVectorRestoreArray(a, &a_data); CeedChk(ierr);
@@ -593,7 +593,7 @@ void AlgebraicInterpolation::MultTranspose(const mfem::Vector& x,
                                  &multiplicitydata); PCeedChk(ierr);
    ierr = CeedVectorGetArrayWrite(fine_work, mem, &workdata); PCeedChk(ierr);
    MFEM_VERIFY((int)length == length, "length overflow");
-   MFEM_FORALL(i, length,
+   mfem::forall(length, [=] MFEM_HOST_DEVICE (int i)
    {workdata[i] = in_ptr[i] * multiplicitydata[i];});
    ierr = CeedVectorRestoreArrayRead(fine_multiplicity_r,
                                      &multiplicitydata);

fem/coefficient.cpp

@@ -144,11 +144,54 @@ double FunctionCoefficient::Eval(ElementTransformation & T,
    }
 }
 
+double CartesianCoefficient::Eval(ElementTransformation & T,
+                                  const IntegrationPoint & ip)
+{
+   T.Transform(ip, transip);
+   return transip[comp];
+}
+
+double CylindricalRadialCoefficient::Eval(ElementTransformation & T,
+                                          const IntegrationPoint & ip)
+{
+   T.Transform(ip, transip);
+   return sqrt(transip[0] * transip[0] + transip[1] * transip[1]);
+}
+
+double CylindricalAzimuthalCoefficient::Eval(ElementTransformation & T,
+                                             const IntegrationPoint & ip)
+{
+   T.Transform(ip, transip);
+   return atan2(transip[1], transip[0]);
+}
+
+double SphericalRadialCoefficient::Eval(ElementTransformation & T,
+                                        const IntegrationPoint & ip)
+{
+   T.Transform(ip, transip);
+   return sqrt(transip * transip);
+}
+
+double SphericalAzimuthalCoefficient::Eval(ElementTransformation & T,
+                                           const IntegrationPoint & ip)
+{
+   T.Transform(ip, transip);
+   return atan2(transip[1], transip[0]);
+}
+
+double SphericalPolarCoefficient::Eval(ElementTransformation & T,
+                                       const IntegrationPoint & ip)
+{
+   T.Transform(ip, transip);
+   return atan2(sqrt(transip[0] * transip[0] + transip[1] * transip[1]),
+                transip[2]);
+}
+
 double GridFunctionCoefficient::Eval (ElementTransformation &T,
                                       const IntegrationPoint &ip)
 {
    Mesh *gf_mesh = GridF->FESpace()->GetMesh();
-   if (T.mesh == gf_mesh)
+   if (T.mesh->GetNE() == gf_mesh->GetNE())
    {
       return GridF->GetValue(T, ip, Component);
    }
@@ -313,6 +356,13 @@ void PWVectorCoefficient::Eval(Vector &V, ElementTransformation &T,
    V = 0.0;
 }
 
+void PositionVectorCoefficient::Eval(Vector &V, ElementTransformation &T,
+                                     const IntegrationPoint &ip)
+{
+   V.SetSize(vdim);
+   T.Transform(ip, V);
+}
+
 void VectorFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
                                      const IntegrationPoint &ip)
 {
@@ -396,7 +446,7 @@ void VectorGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
                                          const IntegrationPoint &ip)
 {
    Mesh *gf_mesh = GridFunc->FESpace()->GetMesh();
-   if (T.mesh == gf_mesh)
+   if (T.mesh->GetNE() == gf_mesh->GetNE())
    {
       GridFunc->GetVectorValue(T, ip, V);
    }
@@ -444,7 +494,7 @@ void GradientGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
                                            const IntegrationPoint &ip)
 {
    Mesh *gf_mesh = GridFunc->FESpace()->GetMesh();
-   if (T.mesh == gf_mesh)
+   if (T.mesh->GetNE() == gf_mesh->GetNE())
    {
       GridFunc->GetGradient(T, V);
    }
@@ -485,7 +535,7 @@ void CurlGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
                                        const IntegrationPoint &ip)
 {
    Mesh *gf_mesh = GridFunc->FESpace()->GetMesh();
-   if (T.mesh == gf_mesh)
+   if (T.mesh->GetNE() == gf_mesh->GetNE())
    {
       GridFunc->GetCurl(T, V);
    }
@@ -507,7 +557,7 @@ double DivergenceGridFunctionCoefficient::Eval(ElementTransformation &T,
                                                const IntegrationPoint &ip)
 {
    Mesh *gf_mesh = GridFunc->FESpace()->GetMesh();
-   if (T.mesh == gf_mesh)
+   if (T.mesh->GetNE() == gf_mesh->GetNE())
    {
       return GridFunc->GetDivergence(T);
    }

fem/coefficient.hpp

@@ -258,6 +258,124 @@ public:
                        const IntegrationPoint &ip);
 };
 
+/// A common base class for returning individual components of the domain's
+/// Cartesian coordinates.
+class CartesianCoefficient : public Coefficient
+{
+protected:
+   int comp;
+   mutable Vector transip;
+
+   /// @a comp_ index of the desired component (0 -> x, 1 -> y, 2 -> z)
+   CartesianCoefficient(int comp_) : comp(comp_), transip(3) {}
+
+public:
+   /// Evaluate the coefficient at @a ip.
+   virtual double Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip);
+};
+
+/// Scalar coefficient which returns the x-component of the evaluation point
+class CartesianXCoefficient : public CartesianCoefficient
+{
+public:
+   CartesianXCoefficient() : CartesianCoefficient(0) {}
+};
+
+/// Scalar coefficient which returns the y-component of the evaluation point
+class CartesianYCoefficient : public CartesianCoefficient
+{
+public:
+   CartesianYCoefficient() : CartesianCoefficient(1) {}
+};
+
+/// Scalar coefficient which returns the z-component of the evaluation point
+class CartesianZCoefficient : public CartesianCoefficient
+{
+public:
+   CartesianZCoefficient() : CartesianCoefficient(2) {}
+};
+
+/// Scalar coefficient which returns the radial distance from the axis of
+/// the evaluation point in the cylindrical coordinate system
+class CylindricalRadialCoefficient : public Coefficient
+{
+private:
+   mutable Vector transip;
+
+public:
+   CylindricalRadialCoefficient() : transip(3) {}
+
+   /// Evaluate the coefficient at @a ip.
+   virtual double Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip);
+};
+
+/// Scalar coefficient which returns the angular position or azimuth (often
+/// denoted by theta) of the evaluation point in the cylindrical coordinate
+/// system
+class CylindricalAzimuthalCoefficient : public Coefficient
+{
+private:
+   mutable Vector transip;
+
+public:
+   CylindricalAzimuthalCoefficient() : transip(3) {}
+
+   /// Evaluate the coefficient at @a ip.
+   virtual double Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip);
+};
+
+/// Scalar coefficient which returns the height or altitude of
+/// the evaluation point in the cylindrical coordinate system
+typedef CartesianZCoefficient CylindricalZCoefficient;
+
+/// Scalar coefficient which returns the radial distance from the origin of
+/// the evaluation point in the spherical coordinate system
+class SphericalRadialCoefficient : public Coefficient
+{
+private:
+   mutable Vector transip;
+
+public:
+   SphericalRadialCoefficient() : transip(3) {}
+
+   /// Evaluate the coefficient at @a ip.
+   virtual double Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip);
+};
+
+/// Scalar coefficient which returns the azimuthal angle (often denoted by phi)
+/// of the evaluation point in the spherical coordinate system
+class SphericalAzimuthalCoefficient : public Coefficient
+{
+private:
+   mutable Vector transip;
+
+public:
+   SphericalAzimuthalCoefficient() : transip(3) {}
+
+   /// Evaluate the coefficient at @a ip.
+   virtual double Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip);
+};
+
+/// Scalar coefficient which returns the polar angle (often denoted by theta)
+/// of the evaluation point in the spherical coordinate system
+class SphericalPolarCoefficient : public Coefficient
+{
+private:
+   mutable Vector transip;
+
+public:
+   SphericalPolarCoefficient() : transip(3) {}
+
+   /// Evaluate the coefficient at @a ip.
+   virtual double Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip);
+};
+
 class GridFunction;
 
 /// Coefficient defined by a GridFunction. This coefficient is mesh dependent.
@@ -600,6 +718,22 @@ public:
    using VectorCoefficient::Eval;
 };
 
+/// A vector coefficient which returns the physical location of the
+/// evaluation point in the Cartesian coordinate system.
+class PositionVectorCoefficient : public VectorCoefficient
+{
+public:
+
+   PositionVectorCoefficient(int dim) : VectorCoefficient(dim) {}
+
+   using VectorCoefficient::Eval;
+   /// Evaluate the vector coefficient at @a ip.
+   virtual void Eval(Vector &V, ElementTransformation &T,
+                     const IntegrationPoint &ip);
+
+   virtual ~PositionVectorCoefficient() { }
+};
+
 /// A general vector function coefficient
 class VectorFunctionCoefficient : public VectorCoefficient
 {

fem/complex_fem.cpp

@@ -497,7 +497,7 @@ SesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
          auto d_X_r = X_r.Read();
          auto d_X_i = X_i.Read();
          auto d_idx = ess_tdof_list.Read();
-         MFEM_FORALL(i, n,
+         mfem::forall(n, [=] MFEM_HOST_DEVICE (int i)
          {
             const int j = d_idx[i];
             d_B_r[j] = d_X_r[j];
@@ -1230,7 +1230,7 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
       auto d_X_r = X_r.Read();
       auto d_X_i = X_i.Read();
       auto d_idx = ess_tdof_list.Read();
-      MFEM_FORALL(i, n,
+      mfem::forall(n, [=] MFEM_HOST_DEVICE (int i)
       {
          const int j = d_idx[i];
          d_B_r[j] = d_X_r[j];

fem/dgmassinv.cpp

@@ -107,7 +107,7 @@ void DGMassInverse::Update()
 {
    M->Assemble();
    M->AssembleDiagonal(diag_inv);
-   internal::MakeReciprocal(diag_inv.Size(), diag_inv.ReadWrite());
+   diag_inv.Reciprocal();
 }
 
 DGMassInverse::~DGMassInverse()
@@ -168,7 +168,7 @@ void DGMassInverse::DGMassCGIteration(const Vector &b_, Vector &u_) const
 
    constexpr int NB = Q1D ? Q1D : 1; // block size
 
-   MFEM_FORALL_2D(e, NE, NB, NB, 1,
+   mfem::forall_2D(NE, NB, NB, [=] MFEM_HOST_DEVICE (int e)
    {
       constexpr int NB = Q1D ? Q1D : 1; // redefine here for some compilers
 

fem/dgmassinv.hpp

@@ -87,6 +87,8 @@ public:
    ///
    /// If @ref iterative_mode is @a true, @a u is used as an initial guess.
    void Mult(const Vector &b, Vector &u) const;
+   /// Same as Mult() since the mass matrix is symmetric.
+   void MultTranspose(const Vector &b, Vector &u) const { Mult(b, u); }
    /// Not implemented. Aborts.
    void SetOperator(const Operator &op);
    /// Set the relative tolerance.
@@ -101,8 +103,8 @@ public:
    ~DGMassInverse();
 
    /// @brief Solve the system M b = u. <b>Not part of the public interface.</b>
-   /// @note This member function must be public because it contains an
-   /// MFEM_FORALL kernel (nvcc limitation)
+   /// @note This member function must be public because it defines an
+   /// extended lambda used in an mfem::forall kernel (nvcc limitation)
    template<int DIM, int D1D = 0, int Q1D = 0>
    void DGMassCGIteration(const Vector &b_, Vector &u_) const;
 };

fem/dgmassinv_kernels.hpp

@@ -12,9 +12,9 @@
 #ifndef MFEM_DGMASSINV_KERNELS_HPP
 #define MFEM_DGMASSINV_KERNELS_HPP
 
-#include "bilininteg_mass_pa.hpp"
 #include "../linalg/kernels.hpp"
 #include "kernels.hpp"
+#include "integ/bilininteg_mass_kernels.hpp"
 
 namespace mfem
 {
@@ -22,11 +22,6 @@ namespace mfem
 namespace internal
 {
 
-void MakeReciprocal(int n, double *x)
-{
-   MFEM_FORALL(i, n, x[i] = 1.0/x[i]; );
-}
-
 template <int DIM, int D1D, int Q1D>
 MFEM_HOST_DEVICE inline
 void DGMassApply(const int e,

fem/fe.hpp

@@ -26,5 +26,6 @@
 #include "fe/fe_nurbs.hpp"
 #include "fe/fe_pos.hpp"
 #include "fe/fe_ser.hpp"
+#include "fe/fe_rbf.hpp"
 
 #endif

fem/fe/fe_base.cpp

@@ -401,7 +401,7 @@ const DofToQuad &FiniteElement::GetDofToQuad(const IntegrationRule &ir,
          }
       }
    }
-   else
+   else if (range_type == VECTOR)
    {
       d2q->B.SetSize(nqpt*dim*dof);
       d2q->Bt.SetSize(dof*nqpt*dim);
@@ -419,6 +419,10 @@ const DofToQuad &FiniteElement::GetDofToQuad(const IntegrationRule &ir,
          }
       }
    }
+   else
+   {
+      // Skip B and Bt for unknown range type
+   }
    switch (deriv_type)
    {
       case GRAD:
@@ -472,7 +476,7 @@ const DofToQuad &FiniteElement::GetDofToQuad(const IntegrationRule &ir,
             {
                for (int j = 0; j < dof; j++)
                {
-                  d2q->G[i+nqpt*(d+dim*j)] = d2q->Gt[j+dof*(i+nqpt*d)] = curlshape(j, d);
+                  d2q->G[i+nqpt*(d+cdim*j)] = d2q->Gt[j+dof*(i+nqpt*d)] = curlshape(j, d);
                }
             }
          }
@@ -480,7 +484,8 @@ const DofToQuad &FiniteElement::GetDofToQuad(const IntegrationRule &ir,
       }
       case NONE:
       default:
-         MFEM_ABORT("invalid finite element derivative type");
+         // Skip G and Gt for unknown derivative type
+         break;
    }
    dof2quad_array.Append(d2q);
    return *d2q;

fem/fe/fe_base.hpp

@@ -1026,9 +1026,6 @@ private:
 
    static Array2D<int> binom;
 
-   static void CalcMono(const int p, const double x, double *u);
-   static void CalcMono(const int p, const double x, double *u, double *d);
-
    static void CalcChebyshev(const int p, const double x, double *u);
    static void CalcChebyshev(const int p, const double x, double *u, double *d);
    static void CalcChebyshev(const int p, const double x, double *u, double *d,
@@ -1165,6 +1162,9 @@ public:
    static void CalcLegendre(const int p, const double x, double *u);
    static void CalcLegendre(const int p, const double x, double *u, double *d);
 
+   static void CalcMono(const int p, const double x, double *u);
+   static void CalcMono(const int p, const double x, double *u, double *d);
+
    ~Poly_1D();
 };
 
@@ -1288,9 +1288,9 @@ public:
    const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
                                  DofToQuad::Mode mode) const override
    {
-      MFEM_VERIFY(mode != DofToQuad::FULL, "invalid mode requested");
-      return GetTensorDofToQuad(*this, ir, mode, basis1d, true,
-                                dof2quad_array);
+      return (mode == DofToQuad::FULL) ?
+             FiniteElement::GetDofToQuad(ir, mode) :
+             GetTensorDofToQuad(*this, ir, mode, basis1d, true, dof2quad_array);
    }
 
    const DofToQuad &GetDofToQuadOpen(const IntegrationRule &ir,

fem/fe/fe_rbf.hpp

@@ -0,0 +1,596 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_FE_RBF
+#define MFEM_FE_RBF
+
+#include "fe_base.hpp"
+
+namespace mfem
+{
+
+/** @brief Pure virtual class for dimensionless radial basis functions (RBFs).
+    Many RBFs are shaped like a Gaussian and are used here as an alternative
+    to polynomials in RBF and RK elements. The input for these
+    is a dimensionless radius r = ||x|| / h, where ||x|| is a distance and
+    h is the smoothing parameter, which controls the gradient of the RBF.
+    For reference: https://doi.org/10.1017/S0962492900000015 */
+class RBFKernel
+{
+public:
+   static const double GlobalRadius; // functions with r>=GR are considered global
+   RBFKernel() { };
+   virtual ~RBFKernel() { }
+
+   /// Evaluate the dimensionless RBF
+   virtual double BaseFunction(double r) const = 0;
+
+   /// Evaluate the derivative of the dimensionless RBF with respect to r
+   virtual double BaseDerivative(double r) const = 0;
+
+   /// Evaluate the second derivative of the dimensionless RBF with respect to r
+   virtual double BaseDerivative2(double r) const = 0;
+
+   /// The support radius, outside of which the function is zero if the function has compact support
+   virtual double Radius() const { return GlobalRadius; }
+
+   /// Does function have compact support?
+   virtual bool CompactSupport() const { return false; }
+
+   /** #brief This normalizes the smoothing parameter h such that h doesn't
+       need to be changed based on the choice of basis function */
+   virtual double HNorm() const = 0;
+};
+
+/// Gaussian RBF, exp(-r^2)
+class GaussianRBF : public RBFKernel
+{
+   // hNorm minimizes integral of Gaussian minus Wendland kernel over r=0,1
+   static const double hNorm;
+public:
+   GaussianRBF() { };
+   virtual ~GaussianRBF() { }
+
+   virtual double BaseFunction(double r) const;
+   virtual double BaseDerivative(double r) const;
+   virtual double BaseDerivative2(double r) const;
+
+   virtual double HNorm() const { return hNorm; }
+};
+
+/// Multiquadric RBF, sqrt(1+r^2)
+class MultiquadricRBF : public RBFKernel
+{
+   // Same as inverse multiquadric
+   static const double hNorm;
+public:
+   MultiquadricRBF() { };
+   virtual ~MultiquadricRBF() { }
+
+   virtual double BaseFunction(double r) const;
+   virtual double BaseDerivative(double r) const;
+   virtual double BaseDerivative2(double r) const;
+
+   virtual double HNorm() const { return hNorm; }
+};
+
+/// Inverse multiquadric RBF, 1/sqrt(1+r^2)
+class InvMultiquadricRBF : public RBFKernel
+{
+   // hNorm minimizes integral of Gaussian minus InvMQ kernel over r=0,0.5
+   static const double hNorm;
+public:
+   InvMultiquadricRBF() { };
+   virtual ~InvMultiquadricRBF() { }
+
+   virtual double BaseFunction(double r) const;
+   virtual double BaseDerivative(double r) const;
+   virtual double BaseDerivative2(double r) const;
+
+   virtual double HNorm() const { return hNorm; }
+};
+
+/** @brief Identitcal to the Gaussian RBF, but subtracted by a factor
+    such that the function is exactly zero at the chosen radius */
+class CompactGaussianRBF : public RBFKernel
+{
+   static const double hNorm;
+   const double radius;
+   double multK, shiftK;
+
+public:
+   CompactGaussianRBF(const double rad = 5.0);
+   virtual ~CompactGaussianRBF() { }
+
+   virtual double BaseFunction(double r) const;
+   virtual double BaseDerivative(double r) const;
+   virtual double BaseDerivative2(double r) const;
+
+   virtual double Radius() const { return radius; }
+
+   virtual double HNorm() const { return hNorm; }
+   virtual bool CompactSupport() const { return true; }
+};
+
+/// Identical to the Gaussian, but truncated (set to zero) at the chosen radius
+class TruncatedGaussianRBF : public RBFKernel
+{
+   static const double hNorm;
+   const double radius;
+public:
+   TruncatedGaussianRBF(const double rad = 5.0)
+      : radius(rad) { }
+   virtual ~TruncatedGaussianRBF() { }
+
+   virtual double BaseFunction(double r) const;
+   virtual double BaseDerivative(double r) const;
+   virtual double BaseDerivative2(double r) const;
+
+   virtual double Radius() const { return radius; }
+
+   virtual double HNorm() const { return hNorm; }
+   virtual bool CompactSupport() const { return true; }
+};
+
+/// Wendland 11 RBF, (1-r)^3 * (1+3r) if r < 1
+class Wendland11RBF : public RBFKernel
+{
+   static const double radius;
+
+public:
+   Wendland11RBF() { }
+   virtual ~Wendland11RBF() { }
+
+   virtual double BaseFunction(double r) const;
+   virtual double BaseDerivative(double r) const;
+   virtual double BaseDerivative2(double r) const;
+
+   virtual double Radius() const { return radius; }
+
+   virtual double HNorm() const { return 1.0 / radius; }
+   virtual bool CompactSupport() const { return true; }
+};
+
+/// Wendland 31 RBF, (1-r)^4 * (1+4r) if r < 1
+class Wendland31RBF : public RBFKernel
+{
+   static const double radius;
+
+public:
+   Wendland31RBF() { };
+   virtual ~Wendland31RBF() { }
+
+   virtual double BaseFunction(double r) const;
+   virtual double BaseDerivative(double r) const;
+   virtual double BaseDerivative2(double r) const;
+
+   virtual double Radius() const { return radius; }
+
+   virtual double HNorm() const { return 1.0 / radius; }
+   virtual bool CompactSupport() const { return true; }
+};
+
+/// Wendland 33 RBF, (1-r)^8 * (1+8r+25r^2+32r^3) if r < 1
+class Wendland33RBF : public RBFKernel
+{
+   static const double radius;
+
+public:
+   Wendland33RBF() { };
+   virtual ~Wendland33RBF() { }
+
+   virtual double BaseFunction(double r) const;
+   virtual double BaseDerivative(double r) const;
+   virtual double BaseDerivative2(double r) const;
+
+   virtual double Radius() const { return radius; }
+
+   virtual double HNorm() const { return 1.0 / radius; }
+   virtual bool CompactSupport() const { return true; }
+};
+
+/// Class for storing and creating the various RBFs
+class RBFType
+{
+public:
+   /// Represent each type of function for input/output
+   enum
+   {
+      Gaussian = 0,
+      Multiquadric = 1,
+      InvMultiquadric = 2,
+      TruncatedGaussian = 3,
+      CompactGaussian = 4,
+      Wendland11 = 5,
+      Wendland31 = 6,
+      Wendland33 = 7,
+      NumRBFTypes = 8
+   };
+
+   /// Return the requested RBF
+   static RBFKernel *GetRBF(const int rbfType)
+   {
+      switch (rbfType)
+      {
+         case RBFType::Gaussian:
+            return new GaussianRBF();
+         case RBFType::Multiquadric:
+            return new MultiquadricRBF();
+         case RBFType::InvMultiquadric:
+            return new InvMultiquadricRBF();
+         case RBFType::TruncatedGaussian:
+            return new TruncatedGaussianRBF();
+         case RBFType::CompactGaussian:
+            return new CompactGaussianRBF();
+         case RBFType::Wendland11:
+            return new Wendland11RBF();
+         case RBFType::Wendland31:
+            return new Wendland31RBF();
+         case RBFType::Wendland33:
+            return new Wendland33RBF();
+      }
+      MFEM_ABORT("unknown RBF type");
+      return NULL;
+   }
+
+   /// Abort if rbfType is invalid
+   static int Check(const int rbfType)
+   {
+      MFEM_VERIFY(0 <= rbfType && rbfType < NumRBFTypes,
+                  "unknown RBF type: " << rbfType);
+      return rbfType;
+   }
+
+   /// Convert rbf int to identifier for storage
+   static char GetChar(const int rbfType)
+   {
+      static const char ident[] = { 'G', 'M', 'I',
+                                    'T', 'C',
+                                    '1', '3', '6'
+                                  };
+      return ident[Check(rbfType)];
+   }
+
+   /// Convert identifier to rbf int
+   static int GetType(const char rbfIdent)
+   {
+      switch (rbfIdent)
+      {
+         case 'G': return Gaussian;
+         case 'M': return Multiquadric;
+         case 'I': return InvMultiquadric;
+         case 'T': return TruncatedGaussian;
+         case 'C': return CompactGaussian;
+         case '1': return Wendland11;
+         case '3': return Wendland31;
+         case '6': return Wendland33;
+      }
+      MFEM_ABORT("unknown RBF identifier: " << rbfIdent);
+      return -1;
+   }
+};
+
+/// Dimensionless distance metrics, whose output is the input for RBFs
+class DistanceMetric
+{
+protected:
+   int dim;
+public:
+   /** @brief Create a distance metric
+       @param D    Reference space dimension */
+   DistanceMetric(int D) { dim = D; }
+   virtual ~DistanceMetric() { }
+
+   /// Set the reference dimension
+   virtual void SetDim(int D) { dim = D; }
+
+   /// Given a vector of length D, return a physical distance
+   virtual void Distance(const Vector &x,
+                         double &r) const = 0;
+
+   /** @brief Given a vector of length D, return the gradient of the
+       distance with respect to the original coordinates */
+   virtual void DDistance(const Vector &x,
+                          Vector &dr) const = 0;
+
+   /** @brief Given a vector of length D, return the Hessian of the
+       distance with respect to the original coordinates */
+   virtual void DDDistance(const Vector &x,
+                           DenseMatrix &ddr) const = 0;
+
+   /// Create an Lp distance metric for the requested dimension and norm
+   static DistanceMetric *GetDistance(int dim, int pnorm);
+};
+
+/// Dimensionless distance with r = |x| + |y| + ...
+class L1Distance : public DistanceMetric
+{
+public:
+   L1Distance(int D) : DistanceMetric(D) { };
+   virtual ~L1Distance() { }
+
+   virtual void Distance(const Vector &x,
+                         double &r) const;
+   virtual void DDistance(const Vector &x,
+                          Vector &dr) const;
+   virtual void DDDistance(const Vector &x,
+                           DenseMatrix &ddr) const;
+};
+
+/// Dimensionless distance with r = (x^2 + y^2 + ...)^(1/2)
+class L2Distance : public DistanceMetric
+{
+public:
+   L2Distance(int D) : DistanceMetric(D) { };
+   virtual ~L2Distance() { }
+
+   virtual void Distance(const Vector &x,
+                         double &r) const;
+   virtual void DDistance(const Vector &x,
+                          Vector &dr) const;
+   virtual void DDDistance(const Vector &x,
+                           DenseMatrix &ddr) const;
+};
+
+/// Dimensionless distance with r = (x^p + y^p + ...)^(1/p)
+class LpDistance : public DistanceMetric
+{
+   const int p;
+   const double pinv;
+public:
+   LpDistance(int D, int pnorm)
+      : DistanceMetric(D),
+        p(pnorm),
+        pinv(1. / static_cast<double>(p))
+   { };
+   virtual ~LpDistance() { }
+
+   virtual void Distance(const Vector &x,
+                         double &r) const;
+   virtual void DDistance(const Vector &x,
+                          Vector &dr) const;
+   virtual void DDDistance(const Vector &x,
+                           DenseMatrix &ddr) const;
+};
+
+
+/** @brief Pure virtual class for a finite element with radial basis functions
+    instead of polynomials inside each element */
+class KernelFiniteElement : public ScalarFiniteElement
+{
+private:
+   // Choose whether to interpolate or project when Project is called
+   bool interpolate = false;
+public:
+   KernelFiniteElement(int D, Geometry::Type G, int Do, int O, int F)
+      : ScalarFiniteElement(D, G, Do, O, F) { }
+   virtual ~KernelFiniteElement() { }
+
+   /// Converts integration rule to vector
+   virtual void IntRuleToVec(const IntegrationPoint &ip,
+                             Vector &vec) const;
+
+   /// Is base RBF compact?
+   virtual bool IsCompact() const = 0;
+
+   /// Return base kernel
+   virtual const RBFKernel *Kernel() const = 0;
+
+   /** @brief Return whether shape function is a tensor product, used in providing indices for compact support */
+   virtual bool TensorIndexed() const { return false; }
+
+   /** @brief Get range of indices (start and end for each dimension) that
+       that are nonzero for compact support for the given integration point */
+   virtual void GetTensorIndices(const Vector &ip,
+                                 int (&indices)[3][2]) const
+   { MFEM_ABORT("GetTensorIndices(...)"); }
+
+   /** @brief Return total number of points in each dimension for tensor-
+       indexed points */
+   virtual void GetTensorNumPoints(int (&tNumPoints)[3]) const
+   { MFEM_ABORT("GetTensorNumPoints(...)"); }
+
+   using FiniteElement::Project;
+
+   virtual void Project(Coefficient &coeff, ElementTransformation &Trans,
+                        Vector &dofs) const;
+
+   virtual void Project(VectorCoefficient &vc, ElementTransformation &Trans,
+                        Vector &dofs) const;
+
+   virtual void Project(const FiniteElement &fe, ElementTransformation &Trans,
+                        DenseMatrix &I) const;
+
+   virtual void GetLocalInterpolation(ElementTransformation &Trans,
+                                      DenseMatrix &I) const
+   { ScalarLocalInterpolation(Trans, I, *this); }
+
+   virtual void GetTransferMatrix(const FiniteElement &fe,
+                                  ElementTransformation &Trans,
+                                  DenseMatrix &I) const
+   { CheckScalarFE(fe).ScalarLocalInterpolation(Trans, I, *this); }
+};
+
+/** @brief Finite element using base radial basis functions without
+    polynomial corrections. */
+class RBFFiniteElement : public KernelFiniteElement
+{
+private:
+#ifndef MFEM_THREAD_SAFE
+   mutable double r_scr, df_scr, ddf_scr;
+   mutable Vector x_scr, y_scr, dy_scr, dr_scr;
+   mutable DenseMatrix ddr_scr;
+   mutable int cInd[3][2];
+#endif
+   bool isCompact;
+   int dimPoints[3];
+   int numPointsD;
+   double delta; // Distance between points
+   double h;
+   double hPhys; // Shape parameter times distance between points times HNorm
+   double hPhysInv; // Inverse hPhys
+   double radPhys; // Radius adjusted by h
+   double faceFactor;
+   const RBFKernel *rbf;
+   const DistanceMetric *distance;
+   void InitializeGeometry();
+
+   // Get the dimensionless distance from x to the center of the RBF indexed i
+   virtual void DistanceVec(const int i,
+                            const Vector &x,
+                            Vector &y) const;
+
+public:
+   /** @brief Construct RBFFiniteElement
+       @param D           Reference space dimension
+       @param numPointsD  Number of points across the element in each dimension
+       @param rbfType     Type of radial basis function, from RBFType
+       @param distNorm    Norm used for distance, usually 2 = Euclidean distance
+       @param intOrder    Number of integration points per RBF point in each dimension
+       @param h           Shape parameter, approximately equal to the number of points in the support radius in one dimension
+       @param faceFactor  1.0 = points end on face, 0.0 = points end at dx/2 from face
+   */
+   RBFFiniteElement(const int D,
+                    const int numPointsD,
+                    const int rbfType,
+                    const int distNorm,
+                    const int intOrder,
+                    const double h,
+                    const double faceFactor);
+   virtual ~RBFFiniteElement() { delete rbf; delete distance; }
+
+   virtual bool TensorIndexed() const { return true; }
+   virtual void GetCompactIndices(const Vector &ip,
+                                  int (&indices)[3][2]) const;
+   virtual void GetGlobalIndices(const Vector &ip,
+                                 int (&indices)[3][2]) const;
+   virtual void GetTensorIndices(const Vector &ip,
+                                 int (&indices)[3][2]) const;
+   virtual void GetTensorNumPoints(int (&tNumPoints)[3]) const
+   {
+      tNumPoints[0] = dimPoints[0];
+      tNumPoints[1] = dimPoints[1];
+      tNumPoints[2] = dimPoints[2];
+   }
+
+   virtual bool IsCompact() const { return isCompact; }
+   virtual const RBFKernel *Kernel() const { return rbf; }
+
+   virtual void CalcShape(const IntegrationPoint &ip,
+                          Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void CalcHessian(const IntegrationPoint &ip,
+                            DenseMatrix &hess) const;
+};
+
+/** @brief Reproducing kernel finite element, which includes polynomial
+    corrections to the standard radial basis function finite element
+    to guarantee a chosen order of accuracy */
+class RKFiniteElement : public KernelFiniteElement
+{
+private:
+#ifndef MFEM_THREAD_SAFE
+   mutable double f_scr;
+   mutable Vector x_scr, y_scr, g_scr, c_scr, s_scr, p_scr, df_scr;
+   mutable DenseMatrix q_scr, dq_scr, M_scr;
+   mutable Vector dc_scr[3], dp_scr[3];
+   mutable DenseMatrix dM_scr[3];
+   mutable DenseMatrixInverse Minv_scr;
+   mutable int cInd[3][2];
+   mutable int dimPoints[3];
+#endif
+   int polyOrd, numPoly, numPoly1d;
+   KernelFiniteElement *baseFE;
+
+   // Get the vector of polynomials for the corrections, evaluated at x
+   virtual void GetPoly(const Vector &x,
+                        Vector &p) const;
+   virtual void GetDPoly(const Vector &x,
+                         Vector &p,
+                         Vector (&dp)[3]) const;
+
+   /* Helper functions that return pieces of the RK evaluation
+      W_{RK,i} = P_i^T C_i W_{RBF,i}, where
+      P_i = [1, x, y, z, ...] is the polynomial vector evaluated at the point i,
+      M_i = P_i P_i^T W_{RBF,i} is a matrix used in calculating the corrections,
+      C_i = M_i^{-1} G are the RK corrections,
+      G = [1, 0, 0, ...] is a convencience vector */
+   virtual void GetG(Vector &g) const;
+   virtual void GetM(const Vector &baseShape,
+                     const IntegrationPoint &ip,
+                     DenseMatrix &M) const;
+   virtual void GetDM(const Vector &baseShape,
+                      const DenseMatrix &baseDeriv,
+                      const IntegrationPoint &ip,
+                      DenseMatrix &M,
+                      DenseMatrix (&dM)[3]) const;
+   virtual void AddToM(const Vector &p,
+                       const double &f,
+                       DenseMatrix &M) const;
+   virtual void AddToDM(const Vector &p,
+                        const Vector (&dp)[3],
+                        const double &f,
+                        const Vector &df,
+                        DenseMatrix (&dM)[3]) const;
+
+   // Given the corrections and base values, calculate the RK value at the ip
+   virtual void CalculateValues(const Vector &c,
+                                const Vector &baseShape,
+                                const IntegrationPoint &ip,
+                                Vector &shape) const;
+   virtual void CalculateDValues(const Vector &c,
+                                 const Vector (&dc)[3],
+                                 const Vector &baseShape,
+                                 const DenseMatrix &baseDShape,
+                                 const IntegrationPoint &ip,
+                                 DenseMatrix &dshape) const;
+
+   // The corrections make the distance dimensionless, so no smoothing parameter
+   virtual void DistanceVec(const int i,
+                            const Vector &x,
+                            Vector &y) const;
+
+public:
+   /** @brief Construct RBFFiniteElement
+       @param D           Reference space dimension
+       @param numPointsD  Number of points across the element in each dimension
+       @param rbfType     Type of radial basis function, from RBFType
+       @param distNorm    Norm used for distance, usually 2 = Euclidean distance
+       @param order       Order of polynomial correction, >= 0
+       @param intOrder    Number of integration points per RBF point in each dimension
+       @param h           Shape parameter, approximately equal to the number of points in the support radius in one dimension
+       @param faceFactor  1.0 = points end on face, 0.0 = points end at dx/2 from face
+   */
+   RKFiniteElement(const int D,
+                   const int numPointsD,
+                   const int rbfType,
+                   const int distNorm,
+                   const int order,
+                   const int intOrder,
+                   const double h,
+                   const double faceFactor);
+   virtual ~RKFiniteElement() { delete baseFE; }
+
+   virtual bool IsCompact() const { return baseFE->IsCompact(); }
+   virtual const RBFKernel *Kernel() const { return baseFE->Kernel(); }
+
+   static int GetNumPoly(int polyOrd, int dim);
+
+   virtual void CalcShape(const IntegrationPoint &ip,
+                          Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+};
+
+} // end namespace mfem
+
+#endif

fem/fe_coll.cpp

@@ -347,6 +347,36 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name)
          fec = new NURBSFECollection();
       }
    }
+   else if (!strncmp(name, "RBF", 3) || !strncmp(name, "RK", 2))
+   {
+      // Example: RK4_G_2_V_2D_0020_1.00_4.01
+      // (RK order 4, Gaussian, L2 dist, Value map, 2 dimensions,
+      // 20 points across element, face factor 1.0, smoothing length of 4.01)
+      const int dim = atoi(name + 10);
+      const int numPoints = atoi(name + 13);
+      const double h = atof(name + 23);
+      const double faceFactor = atof(name + 18);
+      const int rbfType = RBFType::GetType(name[4]);
+      const int distNorm = atoi(name + 6);
+      const int mapType = (name[8] == 'V'
+                           ? FiniteElement::VALUE
+                           : FiniteElement::INTEGRAL);
+      const int intOrder = 2; // keep this fixed for now
+
+      if (!strncmp(name, "RK", 2))
+      {
+         int order = atoi(name + 2);
+         fec = new LocalKernelFECollection(dim, numPoints, rbfType,
+                                           order, h, faceFactor,
+                                           intOrder, distNorm, mapType);
+      }
+      else
+      {
+         fec = new LocalKernelFECollection(dim, numPoints, rbfType,
+                                           -1, h, faceFactor,
+                                           intOrder, distNorm, mapType);
+      }
+   }
    else
    {
       MFEM_ABORT("unknown FiniteElementCollection: " << name);
@@ -3510,4 +3540,184 @@ FiniteElementCollection *NURBSFECollection::GetTraceCollection() const
    return NULL;
 }
 
+LocalKernelFECollection::LocalKernelFECollection(const int dim,
+                                                 const int numPointsD,
+                                                 const int rbfType,
+                                                 const int order,
+                                                 const double h,
+                                                 const double faceFactor,
+                                                 const int intOrder,
+                                                 const int distNorm,
+                                                 const int mapType)
+{
+   const char *mapStr = NULL;
+   switch (mapType)
+   {
+      case FiniteElement::VALUE:    mapStr = "V"; break;
+      case FiniteElement::INTEGRAL: mapStr = "I"; break;
+      default:
+         MFEM_ABORT("invalid mapType: " << mapType);
+   }
+   if (order == -1)
+   {
+      snprintf(d_name, 32, "RBF_%c_%d_%s_%dD_%04d_%1.2f_%.2f",
+               (int)RBFType::GetChar(rbfType), distNorm,
+               mapStr, dim, numPointsD, faceFactor, h);
+   }
+   else if (order >= 0)
+   {
+      snprintf(d_name, 32, "RK%d_%c_%d_%s_%dD_%04d_%1.2f_%.2f", order,
+               (int)RBFType::GetChar(rbfType), distNorm,
+               mapStr, dim, numPointsD, faceFactor, h);
+   }
+   else
+   {
+      MFEM_ABORT("invalid order: " << order);
+   }
+
+   for (int g = 0; g < Geometry::NumGeom; ++g)
+   {
+      L2_Elements[g] = NULL;
+      Tr_Elements[g] = NULL;
+   }
+   for (int i = 0; i < 2; i++)
+   {
+      SegDofOrd[i] = NULL;
+   }
+   OtherDofOrd = NULL;
+
+   if (dim == 0)
+   {
+      L2_Elements[Geometry::POINT] = new PointFiniteElement;
+   }
+   else if (dim == 1)
+   {
+      if (order == -1)
+      {
+         L2_Elements[Geometry::SEGMENT]
+            = new RBFFiniteElement(1, numPointsD, rbfType, distNorm,
+                                   intOrder, h, faceFactor);
+      }
+      else
+      {
+         L2_Elements[Geometry::SEGMENT]
+            = new RKFiniteElement(1, numPointsD, rbfType, distNorm,
+                                  order, intOrder, h, faceFactor);
+      }
+      L2_Elements[Geometry::SEGMENT]->SetMapType(mapType);
+      Tr_Elements[Geometry::POINT] = new PointFiniteElement;
+   }
+   else if (dim == 2)
+   {
+      if (order == -1)
+      {
+         L2_Elements[Geometry::SQUARE]
+            = new RBFFiniteElement(2, numPointsD, rbfType, distNorm,
+                                   intOrder, h, faceFactor);
+         Tr_Elements[Geometry::SEGMENT]
+            = new RBFFiniteElement(1, numPointsD, rbfType, distNorm,
+                                   intOrder, h, faceFactor);
+      }
+      else
+      {
+         L2_Elements[Geometry::SQUARE]
+            = new RKFiniteElement(2, numPointsD, rbfType, distNorm,
+                                  order, intOrder, h, faceFactor);
+         Tr_Elements[Geometry::SEGMENT]
+            = new RKFiniteElement(1, numPointsD, rbfType, distNorm,
+                                  order, intOrder, h, faceFactor);
+      }
+      L2_Elements[Geometry::SQUARE]->SetMapType(mapType);
+   }
+   else if (dim == 3)
+   {
+      if (order == -1)
+      {
+         L2_Elements[Geometry::CUBE]
+            = new RBFFiniteElement(3, numPointsD, rbfType, distNorm,
+                                   intOrder, h, faceFactor);
+         Tr_Elements[Geometry::SQUARE]
+            = new RBFFiniteElement(2, numPointsD, rbfType, distNorm,
+                                   intOrder, h, faceFactor);
+      }
+      else
+      {
+         L2_Elements[Geometry::CUBE]
+            = new RKFiniteElement(3, numPointsD, rbfType, distNorm,
+                                  order, intOrder, h, faceFactor);
+         Tr_Elements[Geometry::SQUARE]
+            = new RKFiniteElement(2, numPointsD, rbfType, distNorm,
+                                  order, intOrder, h, faceFactor);
+      }
+      L2_Elements[Geometry::CUBE]->SetMapType(mapType);
+   }
+
+   if (dim == 1)
+   {
+      SegDofOrd[0] = new int[2*numPointsD];
+      SegDofOrd[1] = SegDofOrd[0] + numPointsD;
+      for (int i = 0; i < numPointsD; ++i)
+      {
+         SegDofOrd[0][i] = i;
+         SegDofOrd[1][i] = numPointsD - i - 1;
+      }
+   }
+   else
+   {
+      const int geomType = TensorBasisElement::GetTensorProductGeometry(dim);
+      const int dof = L2_Elements[geomType]->GetDof();
+      OtherDofOrd = new int[dof];
+      for (int i = 0; i < dof; ++i)
+      {
+         OtherDofOrd[i] = i;
+      }
+   }
 }
+
+LocalKernelFECollection::~LocalKernelFECollection()
+{
+   delete [] OtherDofOrd;
+   delete [] SegDofOrd[0];
+   for (int i = 0; i < Geometry::NumGeom; ++i)
+   {
+      delete L2_Elements[i];
+   }
+}
+
+const FiniteElement *
+LocalKernelFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
+{
+   return L2_Elements[GeomType];
+}
+
+const FiniteElement *
+LocalKernelFECollection::TraceFiniteElementForGeometry(Geometry::Type GeomType)
+const
+{
+   return Tr_Elements[GeomType];
+}
+
+int LocalKernelFECollection::DofForGeometry(Geometry::Type GeomType) const
+{
+   if (L2_Elements[GeomType])
+   {
+      return L2_Elements[GeomType]->GetDof();
+   }
+   return 0;
+}
+
+const int *LocalKernelFECollection::DofOrderForOrientation(
+   Geometry::Type GeomType,
+   int Or) const
+{
+   if (GeomType == Geometry::SEGMENT)
+   {
+      return (Or > 0) ? SegDofOrd[0] : SegDofOrd[1];
+   }
+   else
+   {
+      return (Or == 0) ? OtherDofOrd : NULL;
+   }
+}
+
+} // namespace mfem

fem/fe_coll.hpp

@@ -1305,6 +1305,64 @@ public:
    virtual int GetContType() const { return DISCONTINUOUS; }
 };
 
-}
+/** @brief Radial basis function collection.
+    Instead of using polynomials as a basis, these methods use
+    Gaussian-like functions centered at chosen points. Reproducing
+    kernels augment the radial basis functions so that the resultant
+    basis can exactly represent functions of a chosen polynomial degree. */
+class LocalKernelFECollection : public FiniteElementCollection
+{
+private:
+   int maxDim;
+   char d_name[32];
+   ScalarFiniteElement *Tr_Elements[Geometry::NumGeom];
+   ScalarFiniteElement *L2_Elements[Geometry::NumGeom];
+   int *SegDofOrd[2]; // for rotating segment dofs in 1D
+   int *OtherDofOrd;
+
+public:
+   /** @brief Create a radial basis function (RBF) or reproducing kernel (RK)
+       finite element collection.
+       @param D           Reference space dimension
+       @param numPointsD  Number of points across the element in each dimension
+       @param rbfType     Type of radial basis function, from RBFType
+       @param order       Order of polynomial correction, >= 0 for RK or -1 for
+                          RBF
+       @param h           Shape parameter, approximately equal to the number of
+                          points in the support radius in one dimension, should
+                          generally be > order
+       @param distNorm    Norm used for distance, usually 2 = Euclidean distance
+       @param intOrder    Number of integration points per RBF point in each
+                          dimension
+       @param faceFactor  1.0 = points end on face, 0.0 = points end at dx/2
+                          from face
+   */
+   LocalKernelFECollection(const int D,
+                           const int numPointsD,
+                           const int rbfType,
+                           const int order,
+                           const double h,
+                           const double faceFactor = 0.0,
+                           const int intOrder = 2,
+                           const int distNorm = 2,
+                           const int mapType = FiniteElement::VALUE);
+   virtual ~LocalKernelFECollection();
+
+   virtual const FiniteElement *
+   FiniteElementForGeometry(Geometry::Type GeomType) const;
+
+   virtual const FiniteElement *
+   TraceFiniteElementForGeometry(Geometry::Type GeomType) const;
+
+   virtual int DofForGeometry(Geometry::Type GeomType) const;
+
+   virtual const int * DofOrderForOrientation(Geometry::Type GeomType,
+                                              int Or) const;
+
+   virtual const char * Name() const { return d_name; }
+   virtual int GetContType() const { return DISCONTINUOUS; }
+};
+
+} // namespace mfem
 
 #endif

fem/fespace.cpp

@@ -385,21 +385,38 @@ void FiniteElementSpace::BuildBdrElementToDofTable() const
    if (bdr_elem_dof) { return; }
 
    Table *bel_dof = new Table;
+   Table *bel_fos = (mesh->Dimension() == 3) ? (new Table) : NULL;
    Array<int> dofs;
+   int F, Fo;
    bel_dof->MakeI(mesh->GetNBE());
+   if (bel_fos) { bel_fos->MakeI(mesh->GetNBE()); }
    for (int i = 0; i < mesh->GetNBE(); i++)
    {
       GetBdrElementDofs(i, dofs);
       bel_dof->AddColumnsInRow(i, dofs.Size());
+
+      if (bel_fos)
+      {
+         bel_fos->AddAColumnInRow(i);
+      }
    }
    bel_dof->MakeJ();
+   if (bel_fos) { bel_fos->MakeJ(); }
    for (int i = 0; i < mesh->GetNBE(); i++)
    {
       GetBdrElementDofs(i, dofs);
       bel_dof->AddConnections(i, (int *)dofs, dofs.Size());
+
+      if (bel_fos)
+      {
+         mesh->GetBdrElementFace(i, &F, &Fo);
+         bel_fos->AddConnection(i, Fo);
+      }
    }
    bel_dof->ShiftUpI();
+   if (bel_fos) { bel_fos->ShiftUpI(); }
    bdr_elem_dof = bel_dof;
+   bdr_elem_fos = bel_fos;
 }
 
 void FiniteElementSpace::BuildFaceToDofTable() const
@@ -1542,6 +1559,10 @@ FiniteElementSpace::RefinementOperator::~RefinementOperator()
 {
    delete old_elem_dof;
    delete old_elem_fos;
+   for (int i=0; i<old_DoFTrans.Size(); i++)
+   {
+      delete old_DoFTrans[i];
+   }
 }
 
 void FiniteElementSpace::RefinementOperator

fem/fespace.hpp

@@ -994,7 +994,9 @@ public:
 
    /// @brief Returns indices of degrees of freedom for the @a i'th element.
    /// The returned indices are offsets into an @ref ldof vector with @b vdim
-   /// not necessarily equal to 1. See also GetElementDofs().
+   /// not necessarily equal to 1. The returned indexes are always ordered
+   /// byNODES, irrespective of whether the space is byNODES or byVDIM.
+   /// See also GetElementDofs().
    ///
    /// @note In many cases the returned DofTransformation object will be NULL.
    /// In other cases see the documentation of the DofTransformation class for

fem/gridfunc.cpp

@@ -397,8 +397,6 @@ void GridFunction::GetNodalValues(int i, Array<double> &nval, int vdim) const
 {
    Array<int> vdofs;
 
-   int k;
-
    DofTransformation * doftrans = fes->GetElementVDofs(i, vdofs);
    const FiniteElement *FElem = fes->GetFE(i);
    const IntegrationRule *ElemVert =
@@ -419,7 +417,7 @@ void GridFunction::GetNodalValues(int i, Array<double> &nval, int vdim) const
       Vector shape(dof);
       if (FElem->GetMapType() == FiniteElement::VALUE)
       {
-         for (k = 0; k < n; k++)
+         for (int k = 0; k < n; k++)
          {
             FElem->CalcShape(ElemVert->IntPoint(k), shape);
             nval[k] = shape * (&loc_data[dof * vdim]);
@@ -428,7 +426,7 @@ void GridFunction::GetNodalValues(int i, Array<double> &nval, int vdim) const
       else
       {
          ElementTransformation *Tr = fes->GetElementTransformation(i);
-         for (k = 0; k < n; k++)
+         for (int k = 0; k < n; k++)
          {
             Tr->SetIntPoint(&ElemVert->IntPoint(k));
             FElem->CalcPhysShape(*Tr, shape);
@@ -440,7 +438,7 @@ void GridFunction::GetNodalValues(int i, Array<double> &nval, int vdim) const
    {
       ElementTransformation *Tr = fes->GetElementTransformation(i);
       DenseMatrix vshape(dof, FElem->GetDim());
-      for (k = 0; k < n; k++)
+      for (int k = 0; k < n; k++)
       {
          Tr->SetIntPoint(&ElemVert->IntPoint(k));
          FElem->CalcVShape(*Tr, vshape);
@@ -2401,7 +2399,11 @@ void GridFunction::ProjectDeltaCoefficient(DeltaCoefficient &delta_coeff,
                                      loc_mass);
             vals.SetSize(fe->GetDof());
             fe->ProjectDelta(j, vals);
-            fes->GetElementVDofs(i, vdofs);
+            const DofTransformation* const doftrans = fes->GetElementVDofs(i, vdofs);
+            if (doftrans)
+            {
+               doftrans->TransformPrimal(vals);
+            }
             SetSubVector(vdofs, vals);
             loc_mass_vals.SetSize(vals.Size());
             loc_mass.Mult(vals, loc_mass_vals);
@@ -2755,7 +2757,11 @@ void GridFunction::ProjectBdrCoefficientNormal(
          CalcOrtho(T->Jacobian(), nor);
          lvec(j) = (vc * nor);
       }
-      fes->GetBdrElementDofs(i, dofs);
+      const DofTransformation* const doftrans = fes->GetBdrElementDofs(i, dofs);
+      if (doftrans)
+      {
+         doftrans->TransformPrimal(lvec);
+      }
       SetSubVector(dofs, lvec);
    }
 #endif
@@ -4031,11 +4037,19 @@ double ZZErrorEstimator(BilinearFormIntegrator &blfi,
       {
          if (with_subdomains && ufes->GetAttribute(i) != s) { continue; }
 
-         ufes->GetElementVDofs(i, udofs);
-         ffes->GetElementVDofs(i, fdofs);
+         const DofTransformation* const utrans = ufes->GetElementVDofs(i, udofs);
+         const DofTransformation* const ftrans = ffes->GetElementVDofs(i, fdofs);
 
          u.GetSubVector(udofs, ul);
          flux.GetSubVector(fdofs, fla);
+         if (utrans)
+         {
+            utrans->InvTransformPrimal(ul);
+         }
+         if (ftrans)
+         {
+            ftrans->InvTransformPrimal(fla);
+         }
 
          Transf = ufes->GetElementTransformation(i);
          blfi.ComputeElementFlux(*ufes->GetFE(i), *Transf, ul,
@@ -4330,8 +4344,12 @@ double LSZZErrorEstimator(BilinearFormIntegrator &blfi,  // input
                                                     flux_order));
          int num_integration_pts = ir->GetNPoints();
 
-         ufes->GetElementVDofs(ielem, udofs);
+         const DofTransformation* const utrans = ufes->GetElementVDofs(ielem, udofs);
          u.GetSubVector(udofs, ul);
+         if (utrans)
+         {
+            utrans->InvTransformPrimal(ul);
+         }
          Transf = ufes->GetElementTransformation(ielem);
          FiniteElement *dummy = nullptr;
          blfi.ComputeElementFlux(*ufes->GetFE(ielem), *Transf, ul,

fem/gridfunc.hpp

@@ -684,6 +684,10 @@ public:
    /// Transform by the Space UpdateMatrix (e.g., on Mesh change).
    virtual void Update();
 
+   /** Return update counter, similar to Mesh::GetSequence(). Used to
+       check if it is up to date with the space. */
+   long GetSequence() const { return fes_sequence; }
+
    FiniteElementSpace *FESpace() { return fes; }
    const FiniteElementSpace *FESpace() const { return fes; }
 

fem/gslib.cpp

@@ -108,8 +108,7 @@ void FindPointsGSLIB::Setup(Mesh &m, const double bb_t, const double newt_tol,
                             const int npt_max)
 {
    MFEM_VERIFY(m.GetNodes() != NULL, "Mesh nodes are required.");
-   MFEM_VERIFY(!(m.GetNodes()->FESpace()->IsVariableOrder()),
-               "Variable order mesh is not currently supported.");
+   const int meshOrder = m.GetNodes()->FESpace()->GetMaxElementOrder();
 
    // call FreeData if FindPointsGSLIB::Setup has been called already
    if (setupflag) { FreeData(); }
@@ -117,30 +116,36 @@ void FindPointsGSLIB::Setup(Mesh &m, const double bb_t, const double newt_tol,
    crystal_init(cr, gsl_comm);
    mesh = &m;
    dim  = mesh->Dimension();
-   const FiniteElement *fe = mesh->GetNodalFESpace()->GetFE(0);
-   unsigned dof1D = fe->GetOrder() + 1;
+   unsigned dof1D = meshOrder + 1;
 
    SetupSplitMeshes();
    if (dim == 2)
    {
       if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
       ir_split[0] = new IntegrationRule(3*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], fe->GetOrder());
+      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder);
+
+      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
+      ir_split[1] = new IntegrationRule(pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder);
    }
    else if (dim == 3)
    {
+      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
+      ir_split[0] = new IntegrationRule(pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder);
 
       if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
       ir_split[1] = new IntegrationRule(4*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], fe->GetOrder());
+      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder);
 
       if (ir_split[2]) { delete ir_split[2]; ir_split[2] = NULL; }
       ir_split[2] = new IntegrationRule(3*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[2], ir_split[2], fe->GetOrder());
+      SetupIntegrationRuleForSplitMesh(mesh_split[2], ir_split[2], meshOrder);
 
       if (ir_split[3]) { delete ir_split[3]; ir_split[3] = NULL; }
       ir_split[3] = new IntegrationRule(8*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[3], ir_split[3], fe->GetOrder());
+      SetupIntegrationRuleForSplitMesh(mesh_split[3], ir_split[3], meshOrder);
    }
 
    GetNodalValues(mesh->GetNodes(), gsl_mesh);
@@ -179,7 +184,7 @@ void FindPointsGSLIB::FindPoints(const Vector &point_pos,
    gsl_ref.SetSize(points_cnt * dim);
    gsl_dist.SetSize(points_cnt);
 
-   auto xvFill = [&](const double *xv_base[], unsigned xv_stride[], int dim)
+   auto xvFill = [&](const double *xv_base[], unsigned xv_stride[])
    {
       for (int d = 0; d < dim; d++)
       {
@@ -199,7 +204,7 @@ void FindPointsGSLIB::FindPoints(const Vector &point_pos,
    {
       const double *xv_base[2];
       unsigned xv_stride[2];
-      xvFill(xv_base, xv_stride, dim);
+      xvFill(xv_base, xv_stride);
       findpts_2(gsl_code.GetData(), sizeof(unsigned int),
                 gsl_proc.GetData(), sizeof(unsigned int),
                 gsl_elem.GetData(), sizeof(unsigned int),
@@ -211,7 +216,7 @@ void FindPointsGSLIB::FindPoints(const Vector &point_pos,
    {
       const double *xv_base[3];
       unsigned xv_stride[3];
-      xvFill(xv_base, xv_stride, dim);
+      xvFill(xv_base, xv_stride);
       findpts_3(gsl_code.GetData(), sizeof(unsigned int),
                 gsl_proc.GetData(), sizeof(unsigned int),
                 gsl_elem.GetData(), sizeof(unsigned int),
@@ -333,9 +338,14 @@ void FindPointsGSLIB::SetupSplitMeshes()
             (*gf_rst_map[0])(j+k*npt) = quad_v[j][k];
          }
       }
+
+      mesh_split[1] = new Mesh(Mesh::MakeCartesian2D(1, 1,
+                                                     Element::QUADRILATERAL));
    }
    else if (mesh->Dimension() == 3)
    {
+      mesh_split[0] = new Mesh(Mesh::MakeCartesian3D(1, 1, 1,
+                                                     Element::HEXAHEDRON));
       // Tetrahedron
       {
          int Nvert = 15;
@@ -565,11 +575,12 @@ void FindPointsGSLIB::GetNodalValues(const GridFunction *gf_in,
    const GridFunction *nodes     = gf_in;
    const FiniteElementSpace *fes = nodes->FESpace();
    const int NE                  = mesh->GetNE();
-   const int vdim                = gf_in->FESpace()->GetVDim();
+   const int vdim                = fes->GetVDim();
 
    IntegrationRule *ir_split_temp = NULL;
 
-   const int dof_1D =  nodes->FESpace()->GetFE(0)->GetOrder()+1;
+   const int maxOrder = fes->GetMaxElementOrder();
+   const int dof_1D =  maxOrder+1;
    const int pts_el = std::pow(dof_1D, dim);
    const int pts_cnt = NE_split_total * pts_el;
    node_vals.SetSize(vdim * pts_cnt);
@@ -579,7 +590,7 @@ void FindPointsGSLIB::GetNodalValues(const GridFunction *gf_in,
 
    for (int e = 0; e < NE; e++)
    {
-      const FiniteElement *fe   = nodes->FESpace()->GetFE(e);
+      const FiniteElement *fe   = fes->GetFE(e);
       const Geometry::Type gt   = fe->GetGeomType();
       bool el_to_split = true;
       if (gt == Geometry::TRIANGLE)
@@ -598,16 +609,22 @@ void FindPointsGSLIB::GetNodalValues(const GridFunction *gf_in,
       {
          ir_split_temp = ir_split[3];
       }
-      else if (gt == Geometry::SQUARE || gt == Geometry::CUBE)
+      else if (gt == Geometry::SQUARE)
       {
-         el_to_split = false;
+         ir_split_temp = ir_split[1];
+         el_to_split = gf_in->FESpace()->IsVariableOrder();
+      }
+      else if (gt == Geometry::CUBE)
+      {
+         ir_split_temp = ir_split[0];
+         el_to_split = gf_in->FESpace()->IsVariableOrder();
       }
       else
       {
          MFEM_ABORT("Unsupported geometry type.");
       }
 
-      if (el_to_split) // Triangle/Tet/Prism
+      if (el_to_split) // Triangle/Tet/Prism or Quads/Hex but variable order
       {
          // Fill gsl_mesh with location of split points.
          Vector locval(vdim);
@@ -622,7 +639,7 @@ void FindPointsGSLIB::GetNodalValues(const GridFunction *gf_in,
             gsl_mesh_pt_index++;
          }
       }
-      else // Quad/Hex
+      else // Quad/Hex and constant polynomial order
       {
          const int dof_cnt_split = fe->GetDof();
 
@@ -803,8 +820,8 @@ void FindPointsGSLIB::MapRefPosAndElemIndices()
 void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
                                   Vector &field_out)
 {
-   const int  gf_order   = field_in.FESpace()->GetFE(0)->GetOrder(),
-              mesh_order = mesh->GetNodalFESpace()->GetFE(0)->GetOrder();
+   const int  gf_order   = field_in.FESpace()->GetMaxElementOrder(),
+              mesh_order = mesh->GetNodalFESpace()->GetMaxElementOrder();
 
    const FiniteElementCollection *fec_in =  field_in.FESpace()->FEColl();
    const H1_FECollection *fec_h1 = dynamic_cast<const H1_FECollection *>(fec_in);
@@ -812,7 +829,8 @@ void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
 
    if (fec_h1 && gf_order == mesh_order &&
        fec_h1->GetBasisType() == BasisType::GaussLobatto &&
-       !field_in.FESpace()->IsVariableOrder())
+       field_in.FESpace()->IsVariableOrder() ==
+       mesh->GetNodalFESpace()->IsVariableOrder())
    {
       InterpolateH1(field_in, field_out);
       return;
@@ -886,12 +904,21 @@ void FindPointsGSLIB::InterpolateH1(const GridFunction &field_in,
                                     Vector &field_out)
 {
    FiniteElementSpace ind_fes(mesh, field_in.FESpace()->FEColl());
+   if (field_in.FESpace()->IsVariableOrder())
+   {
+      for (int e = 0; e < ind_fes.GetMesh()->GetNE(); e++)
+      {
+         ind_fes.SetElementOrder(e, field_in.FESpace()->GetElementOrder(e));
+      }
+      ind_fes.Update(false);
+   }
    GridFunction field_in_scalar(&ind_fes);
    Vector node_vals;
 
    const int ncomp      = field_in.FESpace()->GetVDim(),
-             points_fld = field_in.Size() / ncomp,
-             points_cnt = gsl_code.Size();
+             points_fld = field_in.Size() / ncomp;
+   MFEM_VERIFY(points_cnt == gsl_code.Size(),
+               "FindPointsGSLIB::InterpolateH1: Inconsistent size of gsl_code");
 
    field_out.SetSize(points_cnt*ncomp);
    field_out = default_interp_value;
@@ -1111,8 +1138,7 @@ void OversetFindPointsGSLIB::Setup(Mesh &m, const int meshid,
                                    const int npt_max)
 {
    MFEM_VERIFY(m.GetNodes() != NULL, "Mesh nodes are required.");
-   MFEM_VERIFY(!(m.GetNodes()->FESpace()->IsVariableOrder()),
-               "Variable order mesh is not currently supported.");
+   const int meshOrder = m.GetNodes()->FESpace()->GetMaxElementOrder();
 
    // FreeData if OversetFindPointsGSLIB::Setup has been called already
    if (setupflag) { FreeData(); }
@@ -1128,21 +1154,29 @@ void OversetFindPointsGSLIB::Setup(Mesh &m, const int meshid,
    {
       if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
       ir_split[0] = new IntegrationRule(3*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], fe->GetOrder());
+      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder);
+
+      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
+      ir_split[1] = new IntegrationRule(pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder);
    }
    else if (dim == 3)
    {
+      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
+      ir_split[0] = new IntegrationRule(pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder);
+
       if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
       ir_split[1] = new IntegrationRule(4*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], fe->GetOrder());
+      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder);
 
       if (ir_split[2]) { delete ir_split[2]; ir_split[2] = NULL; }
       ir_split[2] = new IntegrationRule(3*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[2], ir_split[2], fe->GetOrder());
+      SetupIntegrationRuleForSplitMesh(mesh_split[2], ir_split[2], meshOrder);
 
       if (ir_split[3]) { delete ir_split[3]; ir_split[3] = NULL; }
       ir_split[3] = new IntegrationRule(8*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[3], ir_split[3], fe->GetOrder());
+      SetupIntegrationRuleForSplitMesh(mesh_split[3], ir_split[3], meshOrder);
    }
 
    GetNodalValues(mesh->GetNodes(), gsl_mesh);

fem/gslib.hpp

@@ -57,7 +57,9 @@ public:
 protected:
    Mesh *mesh;
    Array<Mesh *> mesh_split;  // Meshes used to split simplices.
-   Array<IntegrationRule *> ir_split; // IntegrationRules for simplex->Quad/Hex
+   // IntegrationRules for simplex->Quad/Hex and to project to highest polynomial
+   // order in-case of p-refinement.
+   Array<IntegrationRule *> ir_split;
    Array<FiniteElementSpace *>
    fes_rst_map; // FESpaces to map info Quad/Hex->Simplex
    Array<GridFunction *> gf_rst_map; // GridFunctions to map info Quad/Hex->Simplex

fem/integ/bilininteg_br2.cpp

@@ -9,8 +9,8 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "bilininteg.hpp"
-#include "pfespace.hpp"
+#include "../bilininteg.hpp"
+#include "../pfespace.hpp"
 #include <algorithm>
 
 namespace mfem

fem/integ/bilininteg_convection_ea.cpp

@@ -9,9 +9,9 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
+#include "../../general/forall.hpp"
+#include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
 
 namespace mfem
 {
@@ -34,7 +34,7 @@ static void EAConvectionAssemble1D(const int NE,
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, NE);
    auto A = Reshape(eadata.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
+   mfem::forall_2D(NE, D1D, D1D, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -86,7 +86,7 @@ static void EAConvectionAssemble2D(const int NE,
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, Q1D, 2, NE);
    auto A = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
+   mfem::forall_2D(NE, D1D, D1D, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -163,7 +163,7 @@ static void EAConvectionAssemble3D(const int NE,
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, Q1D, Q1D, 3, NE);
    auto A = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, D1D, D1D, D1D,
+   mfem::forall_3D(NE, D1D, D1D, D1D, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;

fem/integ/bilininteg_convection_mf.cpp

@@ -9,12 +9,9 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-#include "ceed/integrators/convection/convection.hpp"
-
-using namespace std;
+#include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
+#include "../ceed/integrators/convection/convection.hpp"
 
 namespace mfem
 {

fem/integ/bilininteg_convection_pa.cpp

@@ -9,18 +9,15 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-#include "qfunction.hpp"
-#include "ceed/integrators/convection/convection.hpp"
-#include "quadinterpolator.hpp"
+#include "../../general/forall.hpp"
+#include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
+#include "../qfunction.hpp"
+#include "../ceed/integrators/convection/convection.hpp"
 
 namespace mfem
 {
 
-// PA Convection Integrator
-
 // PA Convection Assemble 2D kernel
 static void PAConvectionSetup2D(const int NQ,
                                 const int NE,
@@ -41,7 +38,7 @@ static void PAConvectionSetup2D(const int NQ,
                   Reshape(vel.Read(), DIM,NQ,NE);
    auto y = Reshape(op.Write(), NQ,DIM,NE);
 
-   MFEM_FORALL(q_global, NE*NQ,
+   mfem::forall(NE*NQ, [=] MFEM_HOST_DEVICE (int q_global)
    {
       const int e = q_global / NQ;
       const int q = q_global % NQ;
@@ -78,7 +75,7 @@ static void PAConvectionSetup3D(const int NQ,
                   Reshape(vel.Read(), 3,1,1) :
                   Reshape(vel.Read(), 3,NQ,NE);
    auto y = Reshape(op.Write(), NQ,3,NE);
-   MFEM_FORALL(q_global, NE*NQ,
+   mfem::forall(NE*NQ, [=] MFEM_HOST_DEVICE (int q_global)
    {
       const int e = q_global / NQ;
       const int q = q_global % NQ;
@@ -135,6 +132,61 @@ static void PAConvectionSetup(const int dim,
    }
 }
 
+void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
+{
+   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
+                         Device::GetDeviceMemoryType() : pa_mt;
+   // Assumes tensor-product elements
+   Mesh *mesh = fes.GetMesh();
+   const FiniteElement &el = *fes.GetFE(0);
+   ElementTransformation &Trans = *fes.GetElementTransformation(0);
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, Trans);
+   if (DeviceCanUseCeed())
+   {
+      delete ceedOp;
+      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
+                         fes.IsVariableOrder();
+      if (mixed)
+      {
+         ceedOp = new ceed::MixedPAConvectionIntegrator(*this, fes, Q, alpha);
+      }
+      else
+      {
+         ceedOp = new ceed::PAConvectionIntegrator(fes, *ir, Q, alpha);
+      }
+      return;
+   }
+   const int dims = el.GetDim();
+   const int symmDims = dims;
+   nq = ir->GetNPoints();
+   dim = mesh->Dimension();
+   ne = fes.GetNE();
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mt);
+   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
+   dofs1D = maps->ndof;
+   quad1D = maps->nqpt;
+   pa_data.SetSize(symmDims * nq * ne, mt);
+
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector vel(*Q, qs, CoefficientStorage::COMPRESSED);
+
+   PAConvectionSetup(dim, nq, ne, ir->GetWeights(), geom->J,
+                     vel, alpha, pa_data);
+}
+
+void ConvectionIntegrator::AssembleDiagonalPA(Vector &diag)
+{
+   if (DeviceCanUseCeed())
+   {
+      ceedOp->GetDiagonal(diag);
+   }
+   else
+   {
+      MFEM_ABORT("AssembleDiagonalPA not yet implemented for"
+                 " ConvectionIntegrator.");
+   }
+}
+
 // PA Convection Apply 2D kernel
 template<int T_D1D = 0, int T_Q1D = 0> static
 void PAConvectionApply2D(const int ne,
@@ -159,7 +211,7 @@ void PAConvectionApply2D(const int ne,
    auto op = Reshape(op_.Read(), Q1D, Q1D, 2, NE);
    auto x = Reshape(x_.Read(), D1D, D1D, NE);
    auto y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL(e, NE,
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -279,7 +331,7 @@ void SmemPAConvectionApply2D(const int ne,
    auto op = Reshape(op_.Read(), Q1D, Q1D, 2, NE);
    auto x = Reshape(x_.Read(), D1D, D1D, NE);
    auto y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
+   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int e)
    {
       const int tidz = MFEM_THREAD_ID(z);
       const int D1D = T_D1D ? T_D1D : d1d;
@@ -406,7 +458,7 @@ void PAConvectionApply3D(const int ne,
    auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, 3, NE);
    auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
    auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL(e, NE,
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -587,7 +639,7 @@ void SmemPAConvectionApply3D(const int ne,
    auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, 3, NE);
    auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
    auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -791,7 +843,7 @@ void PAConvectionApplyT2D(const int ne,
    auto op = Reshape(op_.Read(), Q1D, Q1D, 2, NE);
    auto x = Reshape(x_.Read(), D1D, D1D, NE);
    auto y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL(e, NE,
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -907,7 +959,7 @@ void SmemPAConvectionApplyT2D(const int ne,
    auto op = Reshape(op_.Read(), Q1D, Q1D, 2, NE);
    auto x = Reshape(x_.Read(), D1D, D1D, NE);
    auto y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
+   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int e)
    {
       const int tidz = MFEM_THREAD_ID(z);
       const int D1D = T_D1D ? T_D1D : d1d;
@@ -1029,7 +1081,7 @@ void PAConvectionApplyT3D(const int ne,
    auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, 3, NE);
    auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
    auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL(e, NE,
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -1205,7 +1257,7 @@ void SmemPAConvectionApplyT3D(const int ne,
    auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, 3, NE);
    auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
    auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -1375,48 +1427,6 @@ void SmemPAConvectionApplyT3D(const int ne,
    });
 }
 
-void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
-{
-   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
-                         Device::GetDeviceMemoryType() : pa_mt;
-   // Assumes tensor-product elements
-   Mesh *mesh = fes.GetMesh();
-   const FiniteElement &el = *fes.GetFE(0);
-   ElementTransformation &Trans = *fes.GetElementTransformation(0);
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, Trans);
-   if (DeviceCanUseCeed())
-   {
-      delete ceedOp;
-      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
-                         fes.IsVariableOrder();
-      if (mixed)
-      {
-         ceedOp = new ceed::MixedPAConvectionIntegrator(*this, fes, Q, alpha);
-      }
-      else
-      {
-         ceedOp = new ceed::PAConvectionIntegrator(fes, *ir, Q, alpha);
-      }
-      return;
-   }
-   const int dims = el.GetDim();
-   const int symmDims = dims;
-   nq = ir->GetNPoints();
-   dim = mesh->Dimension();
-   ne = fes.GetNE();
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mt);
-   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
-   dofs1D = maps->ndof;
-   quad1D = maps->nqpt;
-   pa_data.SetSize(symmDims * nq * ne, mt);
-
-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector vel(*Q, qs, CoefficientStorage::COMPRESSED);
-
-   PAConvectionSetup(dim, nq, ne, ir->GetWeights(), geom->J,
-                     vel, alpha, pa_data);
-}
-
 static void PAConvectionApply(const int dim,
                               const int D1D,
                               const int Q1D,
@@ -1521,7 +1531,6 @@ static void PAConvectionApplyT(const int dim,
    MFEM_ABORT("Unknown kernel.");
 }
 
-// PA Convection Apply kernel
 void ConvectionIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
    if (DeviceCanUseCeed())
@@ -1536,7 +1545,6 @@ void ConvectionIntegrator::AddMultPA(const Vector &x, Vector &y) const
    }
 }
 
-// PA Convection Apply transpose kernel
 void ConvectionIntegrator::AddMultTransposePA(const Vector &x, Vector &y) const
 {
    if (DeviceCanUseCeed())
@@ -1552,17 +1560,4 @@ void ConvectionIntegrator::AddMultTransposePA(const Vector &x, Vector &y) const
    }
 }
 
-void ConvectionIntegrator::AssembleDiagonalPA(Vector &diag)
-{
-   if (DeviceCanUseCeed())
-   {
-      ceedOp->GetDiagonal(diag);
-   }
-   else
-   {
-      MFEM_ABORT("AssembleDiagonalPA not yet implemented for"
-                 " ConvectionIntegrator.");
-   }
-}
-
 } // namespace mfem

fem/integ/bilininteg_curlcurl_pa.cpp

@@ -0,0 +1,205 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../qfunction.hpp"
+#include "bilininteg_hcurl_kernels.hpp"
+
+namespace mfem
+{
+
+void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
+{
+   // Assumes tensor-product elements
+   Mesh *mesh = fes.GetMesh();
+   const FiniteElement *fel = fes.GetFE(0);
+
+   const VectorTensorFiniteElement *el =
+      dynamic_cast<const VectorTensorFiniteElement*>(fel);
+   MFEM_VERIFY(el != NULL, "Only VectorTensorFiniteElement is supported!");
+
+   const IntegrationRule *ir
+      = IntRule ? IntRule : &MassIntegrator::GetRule(*el, *el,
+                                                     *mesh->GetElementTransformation(0));
+
+   const int dims = el->GetDim();
+   MFEM_VERIFY(dims == 2 || dims == 3, "");
+
+   nq = ir->GetNPoints();
+   dim = mesh->Dimension();
+   MFEM_VERIFY(dim == 2 || dim == 3, "");
+
+   ne = fes.GetNE();
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
+   mapsC = &el->GetDofToQuad(*ir, DofToQuad::TENSOR);
+   mapsO = &el->GetDofToQuadOpen(*ir, DofToQuad::TENSOR);
+   dofs1D = mapsC->ndof;
+   quad1D = mapsC->nqpt;
+
+   MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");
+
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(qs, CoefficientStorage::SYMMETRIC);
+   if (Q) { coeff.Project(*Q); }
+   else if (MQ) { coeff.ProjectTranspose(*MQ); }
+   else if (DQ) { coeff.Project(*DQ); }
+   else { coeff.SetConstant(1.0); }
+
+   const int coeff_dim = coeff.GetVDim();
+   symmetric = (coeff_dim != dim*dim);
+   const int sym_dims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
+   const int ndata = (dim == 2) ? 1 : (symmetric ? sym_dims : dim*dim);
+   pa_data.SetSize(ndata * nq * ne, Device::GetMemoryType());
+
+   if (el->GetDerivType() != mfem::FiniteElement::CURL)
+   {
+      MFEM_ABORT("Unknown kernel.");
+   }
+
+   if (dim == 3)
+   {
+      internal::PACurlCurlSetup3D(quad1D, coeff_dim, ne, ir->GetWeights(), geom->J,
+                                  coeff, pa_data);
+   }
+   else
+   {
+      internal::PACurlCurlSetup2D(quad1D, ne, ir->GetWeights(), geom->J, coeff,
+                                  pa_data);
+   }
+}
+
+void CurlCurlIntegrator::AssembleDiagonalPA(Vector& diag)
+{
+   if (dim == 3)
+   {
+      if (Device::Allows(Backend::DEVICE_MASK))
+      {
+         const int ID = (dofs1D << 4) | quad1D;
+         switch (ID)
+         {
+            case 0x23:
+               return internal::SmemPACurlCurlAssembleDiagonal3D<2,3>(
+                         dofs1D,
+                         quad1D,
+                         symmetric, ne,
+                         mapsO->B, mapsC->B,
+                         mapsO->G, mapsC->G,
+                         pa_data, diag);
+            case 0x34:
+               return internal::SmemPACurlCurlAssembleDiagonal3D<3,4>(
+                         dofs1D,
+                         quad1D,
+                         symmetric, ne,
+                         mapsO->B, mapsC->B,
+                         mapsO->G, mapsC->G,
+                         pa_data, diag);
+            case 0x45:
+               return internal::SmemPACurlCurlAssembleDiagonal3D<4,5>(
+                         dofs1D,
+                         quad1D,
+                         symmetric, ne,
+                         mapsO->B, mapsC->B,
+                         mapsO->G, mapsC->G,
+                         pa_data, diag);
+            case 0x56:
+               return internal::SmemPACurlCurlAssembleDiagonal3D<5,6>(
+                         dofs1D,
+                         quad1D,
+                         symmetric, ne,
+                         mapsO->B, mapsC->B,
+                         mapsO->G, mapsC->G,
+                         pa_data, diag);
+            default:
+               return internal::SmemPACurlCurlAssembleDiagonal3D(
+                         dofs1D, quad1D,
+                         symmetric, ne,
+                         mapsO->B, mapsC->B,
+                         mapsO->G, mapsC->G,
+                         pa_data, diag);
+         }
+      }
+      else
+      {
+         internal::PACurlCurlAssembleDiagonal3D(dofs1D, quad1D, symmetric, ne,
+                                                mapsO->B, mapsC->B,
+                                                mapsO->G, mapsC->G,
+                                                pa_data, diag);
+      }
+   }
+   else if (dim == 2)
+   {
+      internal::PACurlCurlAssembleDiagonal2D(dofs1D, quad1D, ne,
+                                             mapsO->B, mapsC->G, pa_data, diag);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension!");
+   }
+}
+
+void CurlCurlIntegrator::AddMultPA(const Vector &x, Vector &y) const
+{
+   if (dim == 3)
+   {
+      if (Device::Allows(Backend::DEVICE_MASK))
+      {
+         const int ID = (dofs1D << 4) | quad1D;
+         switch (ID)
+         {
+            case 0x23:
+               return internal::SmemPACurlCurlApply3D<2,3>(
+                         dofs1D, quad1D,
+                         symmetric, ne,
+                         mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt,
+                         mapsC->G, mapsC->Gt, pa_data, x, y);
+            case 0x34:
+               return internal::SmemPACurlCurlApply3D<3,4>(
+                         dofs1D, quad1D,
+                         symmetric, ne,
+                         mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt,
+                         mapsC->G, mapsC->Gt, pa_data, x, y);
+            case 0x45:
+               return internal::SmemPACurlCurlApply3D<4,5>(
+                         dofs1D, quad1D,
+                         symmetric, ne,
+                         mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt,
+                         mapsC->G, mapsC->Gt, pa_data, x, y);
+            case 0x56:
+               return internal::SmemPACurlCurlApply3D<5,6>(
+                         dofs1D, quad1D,
+                         symmetric, ne,
+                         mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt,
+                         mapsC->G, mapsC->Gt, pa_data, x, y);
+            default:
+               return internal::SmemPACurlCurlApply3D(
+                         dofs1D, quad1D, symmetric, ne,
+                         mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt,
+                         mapsC->G, mapsC->Gt, pa_data, x, y);
+         }
+      }
+      else
+      {
+         internal::PACurlCurlApply3D(dofs1D, quad1D, symmetric, ne, mapsO->B, mapsC->B,
+                                     mapsO->Bt, mapsC->Bt, mapsC->G, mapsC->Gt,
+                                     pa_data, x, y);
+      }
+   }
+   else if (dim == 2)
+   {
+      internal::PACurlCurlApply2D(dofs1D, quad1D, ne, mapsO->B, mapsO->Bt,
+                                  mapsC->G, mapsC->Gt, pa_data, x, y);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension!");
+   }
+}
+
+} // namespace mfem

fem/integ/bilininteg_dgtrace_ea.cpp

@@ -9,9 +9,9 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
+#include "../../general/forall.hpp"
+#include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
 
 namespace mfem
 {
@@ -26,7 +26,7 @@ static void EADGTraceAssemble1DInt(const int NF,
    auto D = Reshape(padata.Read(), 2, 2, NF);
    auto A_int = Reshape(eadata_int.ReadWrite(), 2, NF);
    auto A_ext = Reshape(eadata_ext.ReadWrite(), 2, NF);
-   MFEM_FORALL(f, NF,
+   mfem::forall(NF, [=] MFEM_HOST_DEVICE (int f)
    {
       double val_int0, val_int1, val_ext01, val_ext10;
       val_int0  = D(0, 0, f);
@@ -58,7 +58,7 @@ static void EADGTraceAssemble1DBdr(const int NF,
 {
    auto D = Reshape(padata.Read(), 2, 2, NF);
    auto A_bdr = Reshape(eadata_bdr.ReadWrite(), NF);
-   MFEM_FORALL(f, NF,
+   mfem::forall(NF, [=] MFEM_HOST_DEVICE (int f)
    {
       if (add)
       {
@@ -89,7 +89,7 @@ static void EADGTraceAssemble2DInt(const int NF,
    auto D = Reshape(padata.Read(), Q1D, 2, 2, NF);
    auto A_int = Reshape(eadata_int.ReadWrite(), D1D, D1D, 2, NF);
    auto A_ext = Reshape(eadata_ext.ReadWrite(), D1D, D1D, 2, NF);
-   MFEM_FORALL_3D(f, NF, D1D, D1D, 1,
+   mfem::forall_2D(NF, D1D, D1D, [=] MFEM_HOST_DEVICE (int f)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -143,7 +143,7 @@ static void EADGTraceAssemble2DBdr(const int NF,
    auto B = Reshape(basis.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, 2, 2, NF);
    auto A_bdr = Reshape(eadata_bdr.ReadWrite(), D1D, D1D, NF);
-   MFEM_FORALL_3D(f, NF, D1D, D1D, 1,
+   mfem::forall_2D(NF, D1D, D1D, [=] MFEM_HOST_DEVICE (int f)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -187,7 +187,7 @@ static void EADGTraceAssemble3DInt(const int NF,
    auto D = Reshape(padata.Read(), Q1D, Q1D, 2, 2, NF);
    auto A_int = Reshape(eadata_int.ReadWrite(), D1D, D1D, D1D, D1D, 2, NF);
    auto A_ext = Reshape(eadata_ext.ReadWrite(), D1D, D1D, D1D, D1D, 2, NF);
-   MFEM_FORALL_3D(f, NF, D1D, D1D, 1,
+   mfem::forall_2D(NF, D1D, D1D, [=] MFEM_HOST_DEVICE (int f)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -283,7 +283,7 @@ static void EADGTraceAssemble3DBdr(const int NF,
    auto B = Reshape(basis.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, Q1D, 2, 2, NF);
    auto A_bdr = Reshape(eadata_bdr.ReadWrite(), D1D, D1D, D1D, D1D, NF);
-   MFEM_FORALL_3D(f, NF, D1D, D1D, 1,
+   mfem::forall_2D(NF, D1D, D1D, [=] MFEM_HOST_DEVICE (int f)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;

fem/integ/bilininteg_dgtrace_pa.cpp

@@ -9,16 +9,15 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-#include "qfunction.hpp"
-#include "restriction.hpp"
-
-using namespace std;
+#include "../../general/forall.hpp"
+#include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
+#include "../qfunction.hpp"
+#include "../restriction.hpp"
 
 namespace mfem
 {
+
 // PA DG Trace Integrator
 static void PADGTraceSetup2D(const int Q1D,
                              const int NF,
@@ -44,7 +43,7 @@ static void PADGTraceSetup2D(const int Q1D,
    auto W = w.Read();
    auto qd = Reshape(op.Write(), Q1D, 2, 2, NF);
 
-   MFEM_FORALL(tid, Q1D*NF,
+   mfem::forall(Q1D*NF, [=] MFEM_HOST_DEVICE (int tid)
    {
       const int f = tid / Q1D;
       const int q = tid % Q1D;
@@ -87,7 +86,7 @@ static void PADGTraceSetup3D(const int Q1D,
    auto W = w.Read();
    auto qd = Reshape(op.Write(), Q1D, Q1D, 2, 2, NF);
 
-   MFEM_FORALL(tid, Q1D*Q1D*NF,
+   mfem::forall(Q1D*Q1D*NF, [=] MFEM_HOST_DEVICE (int tid)
    {
       int f = tid / (Q1D * Q1D);
       int q2 = (tid / Q1D) % Q1D;
@@ -99,7 +98,7 @@ static void PADGTraceSetup3D(const int Q1D,
             const double v1 = const_v ? V(1,0,0,0) : V(1,q1,q2,f);
             const double v2 = const_v ? V(2,0,0,0) : V(2,q1,q2,f);
             const double dot = n(q1,q2,0,f) * v0 + n(q1,q2,1,f) * v1 +
-            /* */              n(q1,q2,2,f) * v2;
+                               n(q1,q2,2,f) * v2;
             const double abs = dot > 0.0 ? dot : -dot;
             const double w = W[q1+q2*Q1D]*r*d(q1,q2,f);
             qd(q1,q2,0,0,f) = w*( alpha/2 * dot + beta * abs );
@@ -267,7 +266,7 @@ void PADGTraceApply2D(const int NF,
    auto x = Reshape(x_.Read(), D1D, VDIM, 2, NF);
    auto y = Reshape(y_.ReadWrite(), D1D, VDIM, 2, NF);
 
-   MFEM_FORALL(f, NF,
+   mfem::forall(NF, [=] MFEM_HOST_DEVICE (int f)
    {
       const int VDIM = 1;
       const int D1D = T_D1D ? T_D1D : d1d;
@@ -358,7 +357,7 @@ void PADGTraceApply3D(const int NF,
    auto x = Reshape(x_.Read(), D1D, D1D, VDIM, 2, NF);
    auto y = Reshape(y_.ReadWrite(), D1D, D1D, VDIM, 2, NF);
 
-   MFEM_FORALL(f, NF,
+   mfem::forall(NF, [=] MFEM_HOST_DEVICE (int f)
    {
       const int VDIM = 1;
       const int D1D = T_D1D ? T_D1D : d1d;
@@ -503,7 +502,7 @@ void SmemPADGTraceApply3D(const int NF,
    auto x = Reshape(x_.Read(), D1D, D1D, 2, NF);
    auto y = Reshape(y_.ReadWrite(), D1D, D1D, 2, NF);
 
-   MFEM_FORALL_2D(f, NF, Q1D, Q1D, NBZ,
+   mfem::forall_2D_batch(NF, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int f)
    {
       const int tidz = MFEM_THREAD_ID(z);
       const int D1D = T_D1D ? T_D1D : d1d;
@@ -668,7 +667,7 @@ void PADGTraceApplyTranspose2D(const int NF,
    auto x = Reshape(x_.Read(), D1D, VDIM, 2, NF);
    auto y = Reshape(y_.ReadWrite(), D1D, VDIM, 2, NF);
 
-   MFEM_FORALL(f, NF,
+   mfem::forall(NF, [=] MFEM_HOST_DEVICE (int f)
    {
       const int VDIM = 1;
       const int D1D = T_D1D ? T_D1D : d1d;
@@ -764,7 +763,7 @@ void PADGTraceApplyTranspose3D(const int NF,
    auto x = Reshape(x_.Read(), D1D, D1D, VDIM, 2, NF);
    auto y = Reshape(y_.ReadWrite(), D1D, D1D, VDIM, 2, NF);
 
-   MFEM_FORALL(f, NF,
+   mfem::forall(NF, [=] MFEM_HOST_DEVICE (int f)
    {
       const int VDIM = 1;
       const int D1D = T_D1D ? T_D1D : d1d;
@@ -920,7 +919,7 @@ void SmemPADGTraceApplyTranspose3D(const int NF,
    auto x = Reshape(x_.Read(), D1D, D1D, 2, NF);
    auto y = Reshape(y_.ReadWrite(), D1D, D1D, 2, NF);
 
-   MFEM_FORALL_2D(f, NF, Q1D, Q1D, NBZ,
+   mfem::forall_2D_batch(NF, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int f)
    {
       const int tidz = MFEM_THREAD_ID(z);
       const int D1D = T_D1D ? T_D1D : d1d;

fem/integ/bilininteg_diffusion_ea.cpp

@@ -9,9 +9,9 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
+#include "../../general/forall.hpp"
+#include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
 
 namespace mfem
 {
@@ -33,7 +33,7 @@ static void EADiffusionAssemble1D(const int NE,
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, NE);
    auto A = Reshape(eadata.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
+   mfem::forall_2D(NE, D1D, D1D, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -85,7 +85,7 @@ static void EADiffusionAssemble2D(const int NE,
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, Q1D, 3, NE);
    auto A = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
+   mfem::forall_2D(NE, D1D, D1D, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -162,7 +162,7 @@ static void EADiffusionAssemble3D(const int NE,
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, Q1D, Q1D, 6, NE);
    auto A = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, D1D, D1D, D1D,
+   mfem::forall_3D(NE, D1D, D1D, D1D, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;

fem/integ/bilininteg_diffusion_kernels.cpp

@@ -0,0 +1,578 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "bilininteg_diffusion_kernels.hpp"
+
+namespace mfem
+{
+
+namespace internal
+{
+
+template<>
+void PADiffusionSetup2D<2>(const int Q1D,
+                           const int coeffDim,
+                           const int NE,
+                           const Array<double> &w,
+                           const Vector &j,
+                           const Vector &c,
+                           Vector &d);
+
+template<>
+void PADiffusionSetup2D<3>(const int Q1D,
+                           const int coeffDim,
+                           const int NE,
+                           const Array<double> &w,
+                           const Vector &j,
+                           const Vector &c,
+                           Vector &d);
+
+void PADiffusionSetup(const int dim,
+                      const int sdim,
+                      const int D1D,
+                      const int Q1D,
+                      const int coeffDim,
+                      const int NE,
+                      const Array<double> &W,
+                      const Vector &J,
+                      const Vector &C,
+                      Vector &D)
+{
+   if (dim == 1) { MFEM_ABORT("dim==1 not supported in PADiffusionSetup"); }
+   if (dim == 2)
+   {
+#ifdef MFEM_USE_OCCA
+      if (DeviceCanUseOcca())
+      {
+         OccaPADiffusionSetup2D(D1D, Q1D, NE, W, J, C, D);
+         return;
+      }
+#else
+      MFEM_CONTRACT_VAR(D1D);
+#endif // MFEM_USE_OCCA
+      if (sdim == 2) { PADiffusionSetup2D<2>(Q1D, coeffDim, NE, W, J, C, D); }
+      if (sdim == 3) { PADiffusionSetup2D<3>(Q1D, coeffDim, NE, W, J, C, D); }
+   }
+   if (dim == 3)
+   {
+#ifdef MFEM_USE_OCCA
+      if (DeviceCanUseOcca())
+      {
+         OccaPADiffusionSetup3D(D1D, Q1D, NE, W, J, C, D);
+         return;
+      }
+#endif // MFEM_USE_OCCA
+      PADiffusionSetup3D(Q1D, coeffDim, NE, W, J, C, D);
+   }
+}
+
+template<>
+void PADiffusionSetup2D<2>(const int Q1D,
+                           const int coeffDim,
+                           const int NE,
+                           const Array<double> &w,
+                           const Vector &j,
+                           const Vector &c,
+                           Vector &d)
+{
+   const bool symmetric = (coeffDim != 4);
+   const bool const_c = c.Size() == 1;
+   MFEM_VERIFY(coeffDim < 3 ||
+               !const_c, "Constant matrix coefficient not supported");
+   const auto W = Reshape(w.Read(), Q1D,Q1D);
+   const auto J = Reshape(j.Read(), Q1D,Q1D,2,2,NE);
+   const auto C = const_c ? Reshape(c.Read(), 1,1,1,1) :
+                  Reshape(c.Read(), coeffDim,Q1D,Q1D,NE);
+   auto D = Reshape(d.Write(), Q1D,Q1D, symmetric ? 3 : 4, NE);
+   mfem::forall_2D(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
+   {
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            const double J11 = J(qx,qy,0,0,e);
+            const double J21 = J(qx,qy,1,0,e);
+            const double J12 = J(qx,qy,0,1,e);
+            const double J22 = J(qx,qy,1,1,e);
+            const double w_detJ = W(qx,qy) / ((J11*J22)-(J21*J12));
+            if (coeffDim == 3 || coeffDim == 4) // Matrix coefficient
+            {
+               // First compute entries of R = MJ^{-T}, without det J factor.
+               const double M11 = C(0,qx,qy,e);
+               const double M12 = C(1,qx,qy,e);
+               const double M21 = symmetric ? M12 : C(2,qx,qy,e);
+               const double M22 = symmetric ? C(2,qx,qy,e) : C(3,qx,qy,e);
+               const double R11 = M11*J22 - M12*J12;
+               const double R21 = M21*J22 - M22*J12;
+               const double R12 = -M11*J21 + M12*J11;
+               const double R22 = -M21*J21 + M22*J11;
+
+               // Now set y to J^{-1}R.
+               D(qx,qy,0,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
+               D(qx,qy,1,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
+               D(qx,qy,2,e) = w_detJ * (symmetric ? (-J21*R12 + J11*R22) :
+                                        (J22*R12 - J12*R22)); // 2,2 or 1,2
+               if (!symmetric)
+               {
+                  D(qx,qy,3,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
+               }
+            }
+            else // Vector or scalar coefficient
+            {
+               const double C1 = const_c ? C(0,0,0,0) : C(0,qx,qy,e);
+               const double C2 = const_c ? C(0,0,0,0) :
+                                 (coeffDim == 2 ? C(1,qx,qy,e) : C(0,qx,qy,e));
+
+               D(qx,qy,0,e) =  w_detJ * (C2*J12*J12 + C1*J22*J22); // 1,1
+               D(qx,qy,1,e) = -w_detJ * (C2*J12*J11 + C1*J22*J21); // 1,2
+               D(qx,qy,2,e) =  w_detJ * (C2*J11*J11 + C1*J21*J21); // 2,2
+            }
+         }
+      }
+   });
+}
+
+template<>
+void PADiffusionSetup2D<3>(const int Q1D,
+                           const int coeffDim,
+                           const int NE,
+                           const Array<double> &w,
+                           const Vector &j,
+                           const Vector &c,
+                           Vector &d)
+{
+   MFEM_VERIFY(coeffDim == 1, "Matrix and vector coefficients not supported");
+   constexpr int DIM = 2;
+   constexpr int SDIM = 3;
+   const bool const_c = c.Size() == 1;
+   const auto W = Reshape(w.Read(), Q1D,Q1D);
+   const auto J = Reshape(j.Read(), Q1D,Q1D,SDIM,DIM,NE);
+   const auto C = const_c ? Reshape(c.Read(), 1,1,1) :
+                  Reshape(c.Read(), Q1D,Q1D,NE);
+   auto D = Reshape(d.Write(), Q1D,Q1D, 3, NE);
+   mfem::forall_2D(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
+   {
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            const double wq = W(qx,qy);
+            const double J11 = J(qx,qy,0,0,e);
+            const double J21 = J(qx,qy,1,0,e);
+            const double J31 = J(qx,qy,2,0,e);
+            const double J12 = J(qx,qy,0,1,e);
+            const double J22 = J(qx,qy,1,1,e);
+            const double J32 = J(qx,qy,2,1,e);
+            const double E = J11*J11 + J21*J21 + J31*J31;
+            const double G = J12*J12 + J22*J22 + J32*J32;
+            const double F = J11*J12 + J21*J22 + J31*J32;
+            const double iw = 1.0 / sqrt(E*G - F*F);
+            const double coeff = const_c ? C(0,0,0) : C(qx,qy,e);
+            const double alpha = wq * coeff * iw;
+            D(qx,qy,0,e) =  alpha * G; // 1,1
+            D(qx,qy,1,e) = -alpha * F; // 1,2
+            D(qx,qy,2,e) =  alpha * E; // 2,2
+         }
+      }
+   });
+}
+
+void PADiffusionSetup3D(const int Q1D,
+                        const int coeffDim,
+                        const int NE,
+                        const Array<double> &w,
+                        const Vector &j,
+                        const Vector &c,
+                        Vector &d)
+{
+   const bool symmetric = (coeffDim != 9);
+   const bool const_c = c.Size() == 1;
+   MFEM_VERIFY(coeffDim < 6 ||
+               !const_c, "Constant matrix coefficient not supported");
+   const auto W = Reshape(w.Read(), Q1D,Q1D,Q1D);
+   const auto J = Reshape(j.Read(), Q1D,Q1D,Q1D,3,3,NE);
+   const auto C = const_c ? Reshape(c.Read(), 1,1,1,1,1) :
+                  Reshape(c.Read(), coeffDim,Q1D,Q1D,Q1D,NE);
+   auto D = Reshape(d.Write(), Q1D,Q1D,Q1D, symmetric ? 6 : 9, NE);
+   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
+   {
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qz,z,Q1D)
+            {
+               const double J11 = J(qx,qy,qz,0,0,e);
+               const double J21 = J(qx,qy,qz,1,0,e);
+               const double J31 = J(qx,qy,qz,2,0,e);
+               const double J12 = J(qx,qy,qz,0,1,e);
+               const double J22 = J(qx,qy,qz,1,1,e);
+               const double J32 = J(qx,qy,qz,2,1,e);
+               const double J13 = J(qx,qy,qz,0,2,e);
+               const double J23 = J(qx,qy,qz,1,2,e);
+               const double J33 = J(qx,qy,qz,2,2,e);
+               const double detJ = J11 * (J22 * J33 - J32 * J23) -
+                                   J21 * (J12 * J33 - J32 * J13) +
+                                   J31 * (J12 * J23 - J22 * J13);
+               const double w_detJ = W(qx,qy,qz) / detJ;
+               // adj(J)
+               const double A11 = (J22 * J33) - (J23 * J32);
+               const double A12 = (J32 * J13) - (J12 * J33);
+               const double A13 = (J12 * J23) - (J22 * J13);
+               const double A21 = (J31 * J23) - (J21 * J33);
+               const double A22 = (J11 * J33) - (J13 * J31);
+               const double A23 = (J21 * J13) - (J11 * J23);
+               const double A31 = (J21 * J32) - (J31 * J22);
+               const double A32 = (J31 * J12) - (J11 * J32);
+               const double A33 = (J11 * J22) - (J12 * J21);
+
+               if (coeffDim == 6 || coeffDim == 9) // Matrix coefficient version
+               {
+                  // Compute entries of R = MJ^{-T} = M adj(J)^T, without det J.
+                  const double M11 = C(0, qx,qy,qz, e);
+                  const double M12 = C(1, qx,qy,qz, e);
+                  const double M13 = C(2, qx,qy,qz, e);
+                  const double M21 = (!symmetric) ? C(3, qx,qy,qz, e) : M12;
+                  const double M22 = (!symmetric) ? C(4, qx,qy,qz, e) : C(3, qx,qy,qz, e);
+                  const double M23 = (!symmetric) ? C(5, qx,qy,qz, e) : C(4, qx,qy,qz, e);
+                  const double M31 = (!symmetric) ? C(6, qx,qy,qz, e) : M13;
+                  const double M32 = (!symmetric) ? C(7, qx,qy,qz, e) : M23;
+                  const double M33 = (!symmetric) ? C(8, qx,qy,qz, e) : C(5, qx,qy,qz, e);
+
+                  const double R11 = M11*A11 + M12*A12 + M13*A13;
+                  const double R12 = M11*A21 + M12*A22 + M13*A23;
+                  const double R13 = M11*A31 + M12*A32 + M13*A33;
+                  const double R21 = M21*A11 + M22*A12 + M23*A13;
+                  const double R22 = M21*A21 + M22*A22 + M23*A23;
+                  const double R23 = M21*A31 + M22*A32 + M23*A33;
+                  const double R31 = M31*A11 + M32*A12 + M33*A13;
+                  const double R32 = M31*A21 + M32*A22 + M33*A23;
+                  const double R33 = M31*A31 + M32*A32 + M33*A33;
+
+                  // Now set D to J^{-1} R = adj(J) R
+                  D(qx,qy,qz,0,e) = w_detJ * (A11*R11 + A12*R21 + A13*R31); // 1,1
+                  const double D12 = w_detJ * (A11*R12 + A12*R22 + A13*R32);
+                  D(qx,qy,qz,1,e) = D12; // 1,2
+                  D(qx,qy,qz,2,e) = w_detJ * (A11*R13 + A12*R23 + A13*R33); // 1,3
+
+                  const double D22 = w_detJ * (A21*R12 + A22*R22 + A23*R32);
+                  const double D23 = w_detJ * (A21*R13 + A22*R23 + A23*R33);
+
+                  const double D33 = w_detJ * (A31*R13 + A32*R23 + A33*R33);
+
+                  D(qx,qy,qz,4,e) = symmetric ? D23 : D22; // 2,3 or 2,2
+                  D(qx,qy,qz,5,e) = symmetric ? D33 : D23; // 3,3 or 2,3
+
+                  if (symmetric)
+                  {
+                     D(qx,qy,qz,3,e) = D22; // 2,2
+                  }
+                  else
+                  {
+                     D(qx,qy,qz,3,e) = w_detJ * (A21*R11 + A22*R21 + A23*R31); // 2,1
+                     D(qx,qy,qz,6,e) = w_detJ * (A31*R11 + A32*R21 + A33*R31); // 3,1
+                     D(qx,qy,qz,7,e) = w_detJ * (A31*R12 + A32*R22 + A33*R32); // 3,2
+                     D(qx,qy,qz,8,e) = D33; // 3,3
+                  }
+               }
+               else  // Vector or scalar coefficient version
+               {
+                  const double C1 = const_c ? C(0,0,0,0,0) : C(0,qx,qy,qz,e);
+                  const double C2 = const_c ? C(0,0,0,0,0) :
+                                    (coeffDim == 3 ? C(1,qx,qy,qz,e) : C(0,qx,qy,qz,e));
+                  const double C3 = const_c ? C(0,0,0,0,0) :
+                                    (coeffDim == 3 ? C(2,qx,qy,qz,e) : C(0,qx,qy,qz,e));
+
+                  // detJ J^{-1} J^{-T} = (1/detJ) adj(J) adj(J)^T
+                  D(qx,qy,qz,0,e) = w_detJ * (C1*A11*A11 + C2*A12*A12 + C3*A13*A13); // 1,1
+                  D(qx,qy,qz,1,e) = w_detJ * (C1*A11*A21 + C2*A12*A22 + C3*A13*A23); // 2,1
+                  D(qx,qy,qz,2,e) = w_detJ * (C1*A11*A31 + C2*A12*A32 + C3*A13*A33); // 3,1
+                  D(qx,qy,qz,3,e) = w_detJ * (C1*A21*A21 + C2*A22*A22 + C3*A23*A23); // 2,2
+                  D(qx,qy,qz,4,e) = w_detJ * (C1*A21*A31 + C2*A22*A32 + C3*A23*A33); // 3,2
+                  D(qx,qy,qz,5,e) = w_detJ * (C1*A31*A31 + C2*A32*A32 + C3*A33*A33); // 3,3
+               }
+            }
+         }
+      }
+   });
+}
+
+#ifdef MFEM_USE_OCCA
+void OccaPADiffusionSetup2D(const int D1D,
+                            const int Q1D,
+                            const int NE,
+                            const Array<double> &W,
+                            const Vector &J,
+                            const Vector &C,
+                            Vector &op)
+{
+   occa::properties props;
+   props["defines/D1D"] = D1D;
+   props["defines/Q1D"] = Q1D;
+   const occa::memory o_W = OccaMemoryRead(W.GetMemory(), W.Size());
+   const occa::memory o_J = OccaMemoryRead(J.GetMemory(), J.Size());
+   const occa::memory o_C = OccaMemoryRead(C.GetMemory(), C.Size());
+   occa::memory o_op = OccaMemoryWrite(op.GetMemory(), op.Size());
+   const bool const_c = C.Size() == 1;
+   const occa_id_t id = std::make_pair(D1D,Q1D);
+   static occa_kernel_t OccaDiffSetup2D_ker;
+   if (OccaDiffSetup2D_ker.find(id) == OccaDiffSetup2D_ker.end())
+   {
+      const occa::kernel DiffusionSetup2D =
+         mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
+                                     "DiffusionSetup2D", props);
+      OccaDiffSetup2D_ker.emplace(id, DiffusionSetup2D);
+   }
+   OccaDiffSetup2D_ker.at(id)(NE, o_W, o_J, o_C, o_op, const_c);
+}
+
+void OccaPADiffusionSetup3D(const int D1D,
+                            const int Q1D,
+                            const int NE,
+                            const Array<double> &W,
+                            const Vector &J,
+                            const Vector &C,
+                            Vector &op)
+{
+   occa::properties props;
+   props["defines/D1D"] = D1D;
+   props["defines/Q1D"] = Q1D;
+   const occa::memory o_W = OccaMemoryRead(W.GetMemory(), W.Size());
+   const occa::memory o_J = OccaMemoryRead(J.GetMemory(), J.Size());
+   const occa::memory o_C = OccaMemoryRead(C.GetMemory(), C.Size());
+   occa::memory o_op = OccaMemoryWrite(op.GetMemory(), op.Size());
+   const bool const_c = C.Size() == 1;
+   const occa_id_t id = std::make_pair(D1D,Q1D);
+   static occa_kernel_t OccaDiffSetup3D_ker;
+   if (OccaDiffSetup3D_ker.find(id) == OccaDiffSetup3D_ker.end())
+   {
+      const occa::kernel DiffusionSetup3D =
+         mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
+                                     "DiffusionSetup3D", props);
+      OccaDiffSetup3D_ker.emplace(id, DiffusionSetup3D);
+   }
+   OccaDiffSetup3D_ker.at(id)(NE, o_W, o_J, o_C, o_op, const_c);
+}
+#endif // MFEM_USE_OCCA
+
+void PADiffusionAssembleDiagonal(const int dim,
+                                 const int D1D,
+                                 const int Q1D,
+                                 const int NE,
+                                 const bool symm,
+                                 const Array<double> &B,
+                                 const Array<double> &G,
+                                 const Vector &D,
+                                 Vector &Y)
+{
+   if (dim == 2)
+   {
+      switch ((D1D << 4 ) | Q1D)
+      {
+         case 0x22: return SmemPADiffusionDiagonal2D<2,2,8>(NE,symm,B,G,D,Y);
+         case 0x33: return SmemPADiffusionDiagonal2D<3,3,8>(NE,symm,B,G,D,Y);
+         case 0x44: return SmemPADiffusionDiagonal2D<4,4,4>(NE,symm,B,G,D,Y);
+         case 0x55: return SmemPADiffusionDiagonal2D<5,5,4>(NE,symm,B,G,D,Y);
+         case 0x66: return SmemPADiffusionDiagonal2D<6,6,2>(NE,symm,B,G,D,Y);
+         case 0x77: return SmemPADiffusionDiagonal2D<7,7,2>(NE,symm,B,G,D,Y);
+         case 0x88: return SmemPADiffusionDiagonal2D<8,8,1>(NE,symm,B,G,D,Y);
+         case 0x99: return SmemPADiffusionDiagonal2D<9,9,1>(NE,symm,B,G,D,Y);
+         default: return PADiffusionDiagonal2D(NE,symm,B,G,D,Y,D1D,Q1D);
+      }
+   }
+   else if (dim == 3)
+   {
+      switch ((D1D << 4 ) | Q1D)
+      {
+         case 0x22: return SmemPADiffusionDiagonal3D<2,2>(NE,symm,B,G,D,Y);
+         case 0x23: return SmemPADiffusionDiagonal3D<2,3>(NE,symm,B,G,D,Y);
+         case 0x34: return SmemPADiffusionDiagonal3D<3,4>(NE,symm,B,G,D,Y);
+         case 0x45: return SmemPADiffusionDiagonal3D<4,5>(NE,symm,B,G,D,Y);
+         case 0x46: return SmemPADiffusionDiagonal3D<4,6>(NE,symm,B,G,D,Y);
+         case 0x56: return SmemPADiffusionDiagonal3D<5,6>(NE,symm,B,G,D,Y);
+         case 0x67: return SmemPADiffusionDiagonal3D<6,7>(NE,symm,B,G,D,Y);
+         case 0x78: return SmemPADiffusionDiagonal3D<7,8>(NE,symm,B,G,D,Y);
+         case 0x89: return SmemPADiffusionDiagonal3D<8,9>(NE,symm,B,G,D,Y);
+         case 0x9A: return SmemPADiffusionDiagonal3D<9,10>(NE,symm,B,G,D,Y);
+         default: return PADiffusionDiagonal3D(NE,symm,B,G,D,Y,D1D,Q1D);
+      }
+   }
+   MFEM_ABORT("Unknown kernel.");
+}
+
+void PADiffusionApply(const int dim,
+                      const int D1D,
+                      const int Q1D,
+                      const int NE,
+                      const bool symm,
+                      const Array<double> &B,
+                      const Array<double> &G,
+                      const Array<double> &Bt,
+                      const Array<double> &Gt,
+                      const Vector &D,
+                      const Vector &X,
+                      Vector &Y)
+{
+#ifdef MFEM_USE_OCCA
+   if (DeviceCanUseOcca())
+   {
+      if (dim == 2)
+      {
+         OccaPADiffusionApply2D(D1D,Q1D,NE,B,G,Bt,Gt,D,X,Y);
+         return;
+      }
+      if (dim == 3)
+      {
+         OccaPADiffusionApply3D(D1D,Q1D,NE,B,G,Bt,Gt,D,X,Y);
+         return;
+      }
+      MFEM_ABORT("OCCA PADiffusionApply unknown kernel!");
+   }
+#endif // MFEM_USE_OCCA
+   const int id = (D1D << 4) | Q1D;
+
+   if (dim == 2)
+   {
+      switch (id)
+      {
+         case 0x22: return SmemPADiffusionApply2D<2,2,16>(NE,symm,B,G,D,X,Y);
+         case 0x33: return SmemPADiffusionApply2D<3,3,16>(NE,symm,B,G,D,X,Y);
+         case 0x44: return SmemPADiffusionApply2D<4,4,8>(NE,symm,B,G,D,X,Y);
+         case 0x55: return SmemPADiffusionApply2D<5,5,8>(NE,symm,B,G,D,X,Y);
+         case 0x66: return SmemPADiffusionApply2D<6,6,4>(NE,symm,B,G,D,X,Y);
+         case 0x77: return SmemPADiffusionApply2D<7,7,4>(NE,symm,B,G,D,X,Y);
+         case 0x88: return SmemPADiffusionApply2D<8,8,2>(NE,symm,B,G,D,X,Y);
+         case 0x99: return SmemPADiffusionApply2D<9,9,2>(NE,symm,B,G,D,X,Y);
+         default:   return PADiffusionApply2D(NE,symm,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
+      }
+   }
+
+   if (dim == 3)
+   {
+      switch (id)
+      {
+         case 0x22: return SmemPADiffusionApply3D<2,2>(NE,symm,B,G,D,X,Y);
+         case 0x23: return SmemPADiffusionApply3D<2,3>(NE,symm,B,G,D,X,Y);
+         case 0x34: return SmemPADiffusionApply3D<3,4>(NE,symm,B,G,D,X,Y);
+         case 0x45: return SmemPADiffusionApply3D<4,5>(NE,symm,B,G,D,X,Y);
+         case 0x46: return SmemPADiffusionApply3D<4,6>(NE,symm,B,G,D,X,Y);
+         case 0x56: return SmemPADiffusionApply3D<5,6>(NE,symm,B,G,D,X,Y);
+         case 0x58: return SmemPADiffusionApply3D<5,8>(NE,symm,B,G,D,X,Y);
+         case 0x67: return SmemPADiffusionApply3D<6,7>(NE,symm,B,G,D,X,Y);
+         case 0x78: return SmemPADiffusionApply3D<7,8>(NE,symm,B,G,D,X,Y);
+         case 0x89: return SmemPADiffusionApply3D<8,9>(NE,symm,B,G,D,X,Y);
+         default:   return PADiffusionApply3D(NE,symm,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
+      }
+   }
+   MFEM_ABORT("Unknown kernel: 0x"<<std::hex << id << std::dec);
+}
+
+#ifdef MFEM_USE_OCCA
+void OccaPADiffusionApply2D(const int D1D,
+                            const int Q1D,
+                            const int NE,
+                            const Array<double> &B,
+                            const Array<double> &G,
+                            const Array<double> &Bt,
+                            const Array<double> &Gt,
+                            const Vector &D,
+                            const Vector &X,
+                            Vector &Y)
+{
+   occa::properties props;
+   props["defines/D1D"] = D1D;
+   props["defines/Q1D"] = Q1D;
+   const occa::memory o_B = OccaMemoryRead(B.GetMemory(), B.Size());
+   const occa::memory o_G = OccaMemoryRead(G.GetMemory(), G.Size());
+   const occa::memory o_Bt = OccaMemoryRead(Bt.GetMemory(), Bt.Size());
+   const occa::memory o_Gt = OccaMemoryRead(Gt.GetMemory(), Gt.Size());
+   const occa::memory o_D = OccaMemoryRead(D.GetMemory(), D.Size());
+   const occa::memory o_X = OccaMemoryRead(X.GetMemory(), X.Size());
+   occa::memory o_Y = OccaMemoryReadWrite(Y.GetMemory(), Y.Size());
+   const occa_id_t id = std::make_pair(D1D,Q1D);
+   if (!Device::Allows(Backend::OCCA_CUDA))
+   {
+      static occa_kernel_t OccaDiffApply2D_cpu;
+      if (OccaDiffApply2D_cpu.find(id) == OccaDiffApply2D_cpu.end())
+      {
+         const occa::kernel DiffusionApply2D_CPU =
+            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
+                                        "DiffusionApply2D_CPU", props);
+         OccaDiffApply2D_cpu.emplace(id, DiffusionApply2D_CPU);
+      }
+      OccaDiffApply2D_cpu.at(id)(NE, o_B, o_G, o_Bt, o_Gt, o_D, o_X, o_Y);
+   }
+   else
+   {
+      static occa_kernel_t OccaDiffApply2D_gpu;
+      if (OccaDiffApply2D_gpu.find(id) == OccaDiffApply2D_gpu.end())
+      {
+         const occa::kernel DiffusionApply2D_GPU =
+            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
+                                        "DiffusionApply2D_GPU", props);
+         OccaDiffApply2D_gpu.emplace(id, DiffusionApply2D_GPU);
+      }
+      OccaDiffApply2D_gpu.at(id)(NE, o_B, o_G, o_Bt, o_Gt, o_D, o_X, o_Y);
+   }
+}
+
+void OccaPADiffusionApply3D(const int D1D,
+                            const int Q1D,
+                            const int NE,
+                            const Array<double> &B,
+                            const Array<double> &G,
+                            const Array<double> &Bt,
+                            const Array<double> &Gt,
+                            const Vector &D,
+                            const Vector &X,
+                            Vector &Y)
+{
+   occa::properties props;
+   props["defines/D1D"] = D1D;
+   props["defines/Q1D"] = Q1D;
+   const occa::memory o_B = OccaMemoryRead(B.GetMemory(), B.Size());
+   const occa::memory o_G = OccaMemoryRead(G.GetMemory(), G.Size());
+   const occa::memory o_Bt = OccaMemoryRead(Bt.GetMemory(), Bt.Size());
+   const occa::memory o_Gt = OccaMemoryRead(Gt.GetMemory(), Gt.Size());
+   const occa::memory o_D = OccaMemoryRead(D.GetMemory(), D.Size());
+   const occa::memory o_X = OccaMemoryRead(X.GetMemory(), X.Size());
+   occa::memory o_Y = OccaMemoryReadWrite(Y.GetMemory(), Y.Size());
+   const occa_id_t id = std::make_pair(D1D,Q1D);
+   if (!Device::Allows(Backend::OCCA_CUDA))
+   {
+      static occa_kernel_t OccaDiffApply3D_cpu;
+      if (OccaDiffApply3D_cpu.find(id) == OccaDiffApply3D_cpu.end())
+      {
+         const occa::kernel DiffusionApply3D_CPU =
+            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
+                                        "DiffusionApply3D_CPU", props);
+         OccaDiffApply3D_cpu.emplace(id, DiffusionApply3D_CPU);
+      }
+      OccaDiffApply3D_cpu.at(id)(NE, o_B, o_G, o_Bt, o_Gt, o_D, o_X, o_Y);
+   }
+   else
+   {
+      static occa_kernel_t OccaDiffApply3D_gpu;
+      if (OccaDiffApply3D_gpu.find(id) == OccaDiffApply3D_gpu.end())
+      {
+         const occa::kernel DiffusionApply3D_GPU =
+            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
+                                        "DiffusionApply3D_GPU", props);
+         OccaDiffApply3D_gpu.emplace(id, DiffusionApply3D_GPU);
+      }
+      OccaDiffApply3D_gpu.at(id)(NE, o_B, o_G, o_Bt, o_Gt, o_D, o_X, o_Y);
+   }
+}
+#endif // MFEM_USE_OCCA
+
+} // namespace internal
+
+} // namespace mfem

fem/integ/bilininteg_diffusion_kernels.hpp

@@ -9,189 +9,42 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-#include "qfunction.hpp"
-#include "ceed/integrators/diffusion/diffusion.hpp"
+#ifndef MFEM_BILININTEG_DIFFUSION_KERNELS_HPP
+#define MFEM_BILININTEG_DIFFUSION_KERNELS_HPP
 
-using namespace std;
+#include "../../config/config.hpp"
+#include "../../general/array.hpp"
+#include "../../general/forall.hpp"
+#include "../../linalg/dtensor.hpp"
+#include "../../linalg/vector.hpp"
+#include "../bilininteg.hpp"
 
 namespace mfem
 {
 
-// PA Diffusion Integrator
-
-// OCCA 2D Assemble kernel
-#ifdef MFEM_USE_OCCA
-static void OccaPADiffusionSetup2D(const int D1D,
-                                   const int Q1D,
-                                   const int NE,
-                                   const Array<double> &W,
-                                   const Vector &J,
-                                   const Vector &C,
-                                   Vector &op)
+namespace internal
 {
-   occa::properties props;
-   props["defines/D1D"] = D1D;
-   props["defines/Q1D"] = Q1D;
-   const occa::memory o_W = OccaMemoryRead(W.GetMemory(), W.Size());
-   const occa::memory o_J = OccaMemoryRead(J.GetMemory(), J.Size());
-   const occa::memory o_C = OccaMemoryRead(C.GetMemory(), C.Size());
-   occa::memory o_op = OccaMemoryWrite(op.GetMemory(), op.Size());
-   const bool const_c = C.Size() == 1;
-   const occa_id_t id = std::make_pair(D1D,Q1D);
-   static occa_kernel_t OccaDiffSetup2D_ker;
-   if (OccaDiffSetup2D_ker.find(id) == OccaDiffSetup2D_ker.end())
-   {
-      const occa::kernel DiffusionSetup2D =
-         mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
-                                     "DiffusionSetup2D", props);
-      OccaDiffSetup2D_ker.emplace(id, DiffusionSetup2D);
-   }
-   OccaDiffSetup2D_ker.at(id)(NE, o_W, o_J, o_C, o_op, const_c);
-}
 
-static void OccaPADiffusionSetup3D(const int D1D,
-                                   const int Q1D,
-                                   const int NE,
-                                   const Array<double> &W,
-                                   const Vector &J,
-                                   const Vector &C,
-                                   Vector &op)
-{
-   occa::properties props;
-   props["defines/D1D"] = D1D;
-   props["defines/Q1D"] = Q1D;
-   const occa::memory o_W = OccaMemoryRead(W.GetMemory(), W.Size());
-   const occa::memory o_J = OccaMemoryRead(J.GetMemory(), J.Size());
-   const occa::memory o_C = OccaMemoryRead(C.GetMemory(), C.Size());
-   occa::memory o_op = OccaMemoryWrite(op.GetMemory(), op.Size());
-   const bool const_c = C.Size() == 1;
-   const occa_id_t id = std::make_pair(D1D,Q1D);
-   static occa_kernel_t OccaDiffSetup3D_ker;
-   if (OccaDiffSetup3D_ker.find(id) == OccaDiffSetup3D_ker.end())
-   {
-      const occa::kernel DiffusionSetup3D =
-         mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
-                                     "DiffusionSetup3D", props);
-      OccaDiffSetup3D_ker.emplace(id, DiffusionSetup3D);
-   }
-   OccaDiffSetup3D_ker.at(id)(NE, o_W, o_J, o_C, o_op, const_c);
-}
-#endif // MFEM_USE_OCCA
+void PADiffusionSetup(const int dim,
+                      const int sdim,
+                      const int D1D,
+                      const int Q1D,
+                      const int coeffDim,
+                      const int NE,
+                      const Array<double> &W,
+                      const Vector &J,
+                      const Vector &C,
+                      Vector &D);
 
-template<>
-void PADiffusionSetup2D<2>(const int Q1D,
-                           const int coeffDim,
-                           const int NE,
-                           const Array<double> &w,
-                           const Vector &j,
-                           const Vector &c,
-                           Vector &d)
-{
-   const bool symmetric = (coeffDim != 4);
-   const bool const_c = c.Size() == 1;
-   MFEM_VERIFY(coeffDim < 3 ||
-               !const_c, "Constant matrix coefficient not supported");
-   const auto W = Reshape(w.Read(), Q1D,Q1D);
-   const auto J = Reshape(j.Read(), Q1D,Q1D,2,2,NE);
-   const auto C = const_c ? Reshape(c.Read(), 1,1,1,1) :
-                  Reshape(c.Read(), coeffDim,Q1D,Q1D,NE);
-   auto D = Reshape(d.Write(), Q1D,Q1D, symmetric ? 3 : 4, NE);
-   MFEM_FORALL_2D(e, NE, Q1D,Q1D,1,
-   {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            const double J11 = J(qx,qy,0,0,e);
-            const double J21 = J(qx,qy,1,0,e);
-            const double J12 = J(qx,qy,0,1,e);
-            const double J22 = J(qx,qy,1,1,e);
-            const double w_detJ = W(qx,qy) / ((J11*J22)-(J21*J12));
-            if (coeffDim == 3 || coeffDim == 4) // Matrix coefficient
-            {
-               // First compute entries of R = MJ^{-T}, without det J factor.
-               const double M11 = C(0,qx,qy,e);
-               const double M12 = C(1,qx,qy,e);
-               const double M21 = symmetric ? M12 : C(2,qx,qy,e);
-               const double M22 = symmetric ? C(2,qx,qy,e) : C(3,qx,qy,e);
-               const double R11 = M11*J22 - M12*J12;
-               const double R21 = M21*J22 - M22*J12;
-               const double R12 = -M11*J21 + M12*J11;
-               const double R22 = -M21*J21 + M22*J11;
-
-               // Now set y to J^{-1}R.
-               D(qx,qy,0,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
-               D(qx,qy,1,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
-               D(qx,qy,2,e) = w_detJ * (symmetric ? (-J21*R12 + J11*R22) :
-                                        (J22*R12 - J12*R22)); // 2,2 or 1,2
-               if (!symmetric)
-               {
-                  D(qx,qy,3,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
-               }
-            }
-            else // Vector or scalar coefficient
-            {
-               const double C1 = const_c ? C(0,0,0,0) : C(0,qx,qy,e);
-               const double C2 = const_c ? C(0,0,0,0) :
-                                 (coeffDim == 2 ? C(1,qx,qy,e) : C(0,qx,qy,e));
-
-               D(qx,qy,0,e) =  w_detJ * (C2*J12*J12 + C1*J22*J22); // 1,1
-               D(qx,qy,1,e) = -w_detJ * (C2*J12*J11 + C1*J22*J21); // 1,2
-               D(qx,qy,2,e) =  w_detJ * (C2*J11*J11 + C1*J21*J21); // 2,2
-            }
-         }
-      }
-   });
-}
-
-// PA Diffusion Assemble 2D kernel with 3D node coords
-template<>
-void PADiffusionSetup2D<3>(const int Q1D,
-                           const int coeffDim,
-                           const int NE,
-                           const Array<double> &w,
-                           const Vector &j,
-                           const Vector &c,
-                           Vector &d)
-{
-   MFEM_VERIFY(coeffDim == 1, "Matrix and vector coefficients not supported");
-   constexpr int DIM = 2;
-   constexpr int SDIM = 3;
-   const bool const_c = c.Size() == 1;
-   const auto W = Reshape(w.Read(), Q1D,Q1D);
-   const auto J = Reshape(j.Read(), Q1D,Q1D,SDIM,DIM,NE);
-   const auto C = const_c ? Reshape(c.Read(), 1,1,1) :
-                  Reshape(c.Read(), Q1D,Q1D,NE);
-   auto D = Reshape(d.Write(), Q1D,Q1D, 3, NE);
-   MFEM_FORALL_2D(e, NE, Q1D,Q1D,1,
-   {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            const double wq = W(qx,qy);
-            const double J11 = J(qx,qy,0,0,e);
-            const double J21 = J(qx,qy,1,0,e);
-            const double J31 = J(qx,qy,2,0,e);
-            const double J12 = J(qx,qy,0,1,e);
-            const double J22 = J(qx,qy,1,1,e);
-            const double J32 = J(qx,qy,2,1,e);
-            const double E = J11*J11 + J21*J21 + J31*J31;
-            const double G = J12*J12 + J22*J22 + J32*J32;
-            const double F = J11*J12 + J21*J22 + J31*J32;
-            const double iw = 1.0 / sqrt(E*G - F*F);
-            const double coeff = const_c ? C(0,0,0) : C(qx,qy,e);
-            const double alpha = wq * coeff * iw;
-            D(qx,qy,0,e) =  alpha * G; // 1,1
-            D(qx,qy,1,e) = -alpha * F; // 1,2
-            D(qx,qy,2,e) =  alpha * E; // 2,2
-         }
-      }
-   });
-}
+// PA Diffusion Assemble 2D kernel
+template<int T_SDIM>
+void PADiffusionSetup2D(const int Q1D,
+                        const int coeffDim,
+                        const int NE,
+                        const Array<double> &w,
+                        const Vector &j,
+                        const Vector &c,
+                        Vector &d);
 
 // PA Diffusion Assemble 3D kernel
 void PADiffusionSetup3D(const int Q1D,
@@ -200,217 +53,41 @@ void PADiffusionSetup3D(const int Q1D,
                         const Array<double> &w,
                         const Vector &j,
                         const Vector &c,
-                        Vector &d)
-{
-   const bool symmetric = (coeffDim != 9);
-   const bool const_c = c.Size() == 1;
-   MFEM_VERIFY(coeffDim < 6 ||
-               !const_c, "Constant matrix coefficient not supported");
-   const auto W = Reshape(w.Read(), Q1D,Q1D,Q1D);
-   const auto J = Reshape(j.Read(), Q1D,Q1D,Q1D,3,3,NE);
-   const auto C = const_c ? Reshape(c.Read(), 1,1,1,1,1) :
-                  Reshape(c.Read(), coeffDim,Q1D,Q1D,Q1D,NE);
-   auto D = Reshape(d.Write(), Q1D,Q1D,Q1D, symmetric ? 6 : 9, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qz,z,Q1D)
-            {
-               const double J11 = J(qx,qy,qz,0,0,e);
-               const double J21 = J(qx,qy,qz,1,0,e);
-               const double J31 = J(qx,qy,qz,2,0,e);
-               const double J12 = J(qx,qy,qz,0,1,e);
-               const double J22 = J(qx,qy,qz,1,1,e);
-               const double J32 = J(qx,qy,qz,2,1,e);
-               const double J13 = J(qx,qy,qz,0,2,e);
-               const double J23 = J(qx,qy,qz,1,2,e);
-               const double J33 = J(qx,qy,qz,2,2,e);
-               const double detJ = J11 * (J22 * J33 - J32 * J23) -
-               /* */               J21 * (J12 * J33 - J32 * J13) +
-               /* */               J31 * (J12 * J23 - J22 * J13);
-               const double w_detJ = W(qx,qy,qz) / detJ;
-               // adj(J)
-               const double A11 = (J22 * J33) - (J23 * J32);
-               const double A12 = (J32 * J13) - (J12 * J33);
-               const double A13 = (J12 * J23) - (J22 * J13);
-               const double A21 = (J31 * J23) - (J21 * J33);
-               const double A22 = (J11 * J33) - (J13 * J31);
-               const double A23 = (J21 * J13) - (J11 * J23);
-               const double A31 = (J21 * J32) - (J31 * J22);
-               const double A32 = (J31 * J12) - (J11 * J32);
-               const double A33 = (J11 * J22) - (J12 * J21);
+                        Vector &d);
 
-               if (coeffDim == 6 || coeffDim == 9) // Matrix coefficient version
-               {
-                  // Compute entries of R = MJ^{-T} = M adj(J)^T, without det J.
-                  const double M11 = C(0, qx,qy,qz, e);
-                  const double M12 = C(1, qx,qy,qz, e);
-                  const double M13 = C(2, qx,qy,qz, e);
-                  const double M21 = (!symmetric) ? C(3, qx,qy,qz, e) : M12;
-                  const double M22 = (!symmetric) ? C(4, qx,qy,qz, e) : C(3, qx,qy,qz, e);
-                  const double M23 = (!symmetric) ? C(5, qx,qy,qz, e) : C(4, qx,qy,qz, e);
-                  const double M31 = (!symmetric) ? C(6, qx,qy,qz, e) : M13;
-                  const double M32 = (!symmetric) ? C(7, qx,qy,qz, e) : M23;
-                  const double M33 = (!symmetric) ? C(8, qx,qy,qz, e) : C(5, qx,qy,qz, e);
-
-                  const double R11 = M11*A11 + M12*A12 + M13*A13;
-                  const double R12 = M11*A21 + M12*A22 + M13*A23;
-                  const double R13 = M11*A31 + M12*A32 + M13*A33;
-                  const double R21 = M21*A11 + M22*A12 + M23*A13;
-                  const double R22 = M21*A21 + M22*A22 + M23*A23;
-                  const double R23 = M21*A31 + M22*A32 + M23*A33;
-                  const double R31 = M31*A11 + M32*A12 + M33*A13;
-                  const double R32 = M31*A21 + M32*A22 + M33*A23;
-                  const double R33 = M31*A31 + M32*A32 + M33*A33;
-
-                  // Now set D to J^{-1} R = adj(J) R
-                  D(qx,qy,qz,0,e) = w_detJ * (A11*R11 + A12*R21 + A13*R31); // 1,1
-                  const double D12 = w_detJ * (A11*R12 + A12*R22 + A13*R32);
-                  D(qx,qy,qz,1,e) = D12; // 1,2
-                  D(qx,qy,qz,2,e) = w_detJ * (A11*R13 + A12*R23 + A13*R33); // 1,3
-
-                  const double D22 = w_detJ * (A21*R12 + A22*R22 + A23*R32);
-                  const double D23 = w_detJ * (A21*R13 + A22*R23 + A23*R33);
-
-                  const double D33 = w_detJ * (A31*R13 + A32*R23 + A33*R33);
-
-                  D(qx,qy,qz,4,e) = symmetric ? D23 : D22; // 2,3 or 2,2
-                  D(qx,qy,qz,5,e) = symmetric ? D33 : D23; // 3,3 or 2,3
-
-                  if (symmetric)
-                  {
-                     D(qx,qy,qz,3,e) = D22; // 2,2
-                  }
-                  else
-                  {
-                     D(qx,qy,qz,3,e) = w_detJ * (A21*R11 + A22*R21 + A23*R31); // 2,1
-                     D(qx,qy,qz,6,e) = w_detJ * (A31*R11 + A32*R21 + A33*R31); // 3,1
-                     D(qx,qy,qz,7,e) = w_detJ * (A31*R12 + A32*R22 + A33*R32); // 3,2
-                     D(qx,qy,qz,8,e) = D33; // 3,3
-                  }
-               }
-               else  // Vector or scalar coefficient version
-               {
-                  const double C1 = const_c ? C(0,0,0,0,0) : C(0,qx,qy,qz,e);
-                  const double C2 = const_c ? C(0,0,0,0,0) :
-                                    (coeffDim == 3 ? C(1,qx,qy,qz,e) : C(0,qx,qy,qz,e));
-                  const double C3 = const_c ? C(0,0,0,0,0) :
-                                    (coeffDim == 3 ? C(2,qx,qy,qz,e) : C(0,qx,qy,qz,e));
-
-                  // detJ J^{-1} J^{-T} = (1/detJ) adj(J) adj(J)^T
-                  D(qx,qy,qz,0,e) = w_detJ * (C1*A11*A11 + C2*A12*A12 + C3*A13*A13); // 1,1
-                  D(qx,qy,qz,1,e) = w_detJ * (C1*A11*A21 + C2*A12*A22 + C3*A13*A23); // 2,1
-                  D(qx,qy,qz,2,e) = w_detJ * (C1*A11*A31 + C2*A12*A32 + C3*A13*A33); // 3,1
-                  D(qx,qy,qz,3,e) = w_detJ * (C1*A21*A21 + C2*A22*A22 + C3*A23*A23); // 2,2
-                  D(qx,qy,qz,4,e) = w_detJ * (C1*A21*A31 + C2*A22*A32 + C3*A23*A33); // 3,2
-                  D(qx,qy,qz,5,e) = w_detJ * (C1*A31*A31 + C2*A32*A32 + C3*A33*A33); // 3,3
-               }
-            }
-         }
-      }
-   });
-}
-
-static void PADiffusionSetup(const int dim,
-                             const int sdim,
-                             const int D1D,
-                             const int Q1D,
-                             const int coeffDim,
-                             const int NE,
-                             const Array<double> &W,
-                             const Vector &J,
-                             const Vector &C,
-                             Vector &D)
-{
-   if (dim == 1) { MFEM_ABORT("dim==1 not supported in PADiffusionSetup"); }
-   if (dim == 2)
-   {
 #ifdef MFEM_USE_OCCA
-      if (DeviceCanUseOcca())
-      {
-         OccaPADiffusionSetup2D(D1D, Q1D, NE, W, J, C, D);
-         return;
-      }
-#else
-      MFEM_CONTRACT_VAR(D1D);
+// OCCA 2D Assemble kernel
+void OccaPADiffusionSetup2D(const int D1D,
+                            const int Q1D,
+                            const int NE,
+                            const Array<double> &W,
+                            const Vector &J,
+                            const Vector &C,
+                            Vector &op);
+
+// OCCA 3D Assemble kernel
+void OccaPADiffusionSetup3D(const int D1D,
+                            const int Q1D,
+                            const int NE,
+                            const Array<double> &W,
+                            const Vector &J,
+                            const Vector &C,
+                            Vector &op);
 #endif // MFEM_USE_OCCA
-      if (sdim == 2) { PADiffusionSetup2D<2>(Q1D, coeffDim, NE, W, J, C, D); }
-      if (sdim == 3) { PADiffusionSetup2D<3>(Q1D, coeffDim, NE, W, J, C, D); }
-   }
-   if (dim == 3)
-   {
-#ifdef MFEM_USE_OCCA
-      if (DeviceCanUseOcca())
-      {
-         OccaPADiffusionSetup3D(D1D, Q1D, NE, W, J, C, D);
-         return;
-      }
-#endif // MFEM_USE_OCCA
-      PADiffusionSetup3D(Q1D, coeffDim, NE, W, J, C, D);
-   }
-}
 
-void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
-{
-   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
-                         Device::GetDeviceMemoryType() : pa_mt;
-   // Assuming the same element type
-   fespace = &fes;
-   Mesh *mesh = fes.GetMesh();
-   if (mesh->GetNE() == 0) { return; }
-   const FiniteElement &el = *fes.GetFE(0);
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el);
-   if (DeviceCanUseCeed())
-   {
-      delete ceedOp;
-      MFEM_VERIFY(!VQ && !MQ,
-                  "Only scalar coefficient supported for DiffusionIntegrator"
-                  " with libCEED");
-      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
-                         fes.IsVariableOrder();
-      if (mixed)
-      {
-         ceedOp = new ceed::MixedPADiffusionIntegrator(*this, fes, Q);
-      }
-      else
-      {
-         ceedOp = new ceed::PADiffusionIntegrator(fes, *ir, Q);
-      }
-      return;
-   }
-   const int dims = el.GetDim();
-   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
-   const int nq = ir->GetNPoints();
-   dim = mesh->Dimension();
-   ne = fes.GetNE();
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mt);
-   const int sdim = mesh->SpaceDimension();
-   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
-   dofs1D = maps->ndof;
-   quad1D = maps->nqpt;
-
-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(qs, CoefficientStorage::COMPRESSED);
-
-   if (MQ) { coeff.ProjectTranspose(*MQ); }
-   else if (VQ) { coeff.Project(*VQ); }
-   else if (Q) { coeff.Project(*Q); }
-   else { coeff.SetConstant(1.0); }
-
-   const int coeff_dim = coeff.GetVDim();
-   symmetric = (coeff_dim != dims*dims);
-   const int pa_size = symmetric ? symmDims : dims*dims;
-
-   pa_data.SetSize(pa_size * nq * ne, mt);
-   PADiffusionSetup(dim, sdim, dofs1D, quad1D, coeff_dim, ne, ir->GetWeights(),
-                    geom->J, coeff, pa_data);
-}
+void PADiffusionAssembleDiagonal(const int dim,
+                                 const int D1D,
+                                 const int Q1D,
+                                 const int NE,
+                                 const bool symm,
+                                 const Array<double> &B,
+                                 const Array<double> &G,
+                                 const Vector &D,
+                                 Vector &Y);
 
+// PA Diffusion Diagonal 2D kernel
 template<int T_D1D = 0, int T_Q1D = 0>
-static void PADiffusionDiagonal2D(const int NE,
+inline void PADiffusionDiagonal2D(const int NE,
                                   const bool symmetric,
                                   const Array<double> &b,
                                   const Array<double> &g,
@@ -429,7 +106,7 @@ static void PADiffusionDiagonal2D(const int NE,
    // store necessary entries
    auto D = Reshape(d.Read(), Q1D*Q1D, symmetric ? 3 : 4, NE);
    auto Y = Reshape(y.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL(e, NE,
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -476,7 +153,7 @@ static void PADiffusionDiagonal2D(const int NE,
 
 // Shared memory PA Diffusion Diagonal 2D kernel
 template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
-static void SmemPADiffusionDiagonal2D(const int NE,
+inline void SmemPADiffusionDiagonal2D(const int NE,
                                       const bool symmetric,
                                       const Array<double> &b_,
                                       const Array<double> &g_,
@@ -496,7 +173,7 @@ static void SmemPADiffusionDiagonal2D(const int NE,
    auto g = Reshape(g_.Read(), Q1D, D1D);
    auto D = Reshape(d_.Read(), Q1D*Q1D, symmetric ? 3 : 4, NE);
    auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
+   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int e)
    {
       const int tidz = MFEM_THREAD_ID(z);
       const int D1D = T_D1D ? T_D1D : d1d;
@@ -569,8 +246,9 @@ static void SmemPADiffusionDiagonal2D(const int NE,
    });
 }
 
+// PA Diffusion Diagonal 3D kernel
 template<int T_D1D = 0, int T_Q1D = 0>
-static void PADiffusionDiagonal3D(const int NE,
+inline void PADiffusionDiagonal3D(const int NE,
                                   const bool symmetric,
                                   const Array<double> &b,
                                   const Array<double> &g,
@@ -590,7 +268,7 @@ static void PADiffusionDiagonal3D(const int NE,
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto Q = Reshape(d.Read(), Q1D*Q1D*Q1D, symmetric ? 6 : 9, NE);
    auto Y = Reshape(y.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL(e, NE,
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -614,8 +292,8 @@ static void PADiffusionDiagonal3D(const int NE,
                      {
                         const int q = qx + (qy + qz * Q1D) * Q1D;
                         const int ksym = j >= i ?
-                        3 - (3-i)*(2-i)/2 + j:
-                        3 - (3-j)*(2-j)/2 + i;
+                                         3 - (3-i)*(2-i)/2 + j:
+                                         3 - (3-j)*(2-j)/2 + i;
                         const int k = symmetric ? ksym : (i*DIM) + j;
                         const double O = Q(q,k,e);
                         const double Bz = B(qz,dz);
@@ -671,7 +349,7 @@ static void PADiffusionDiagonal3D(const int NE,
 
 // Shared memory PA Diffusion Diagonal 3D kernel
 template<int T_D1D = 0, int T_Q1D = 0>
-static void SmemPADiffusionDiagonal3D(const int NE,
+inline void SmemPADiffusionDiagonal3D(const int NE,
                                       const bool symmetric,
                                       const Array<double> &b_,
                                       const Array<double> &g_,
@@ -691,7 +369,7 @@ static void SmemPADiffusionDiagonal3D(const int NE,
    auto g = Reshape(g_.Read(), Q1D, D1D);
    auto D = Reshape(d_.Read(), Q1D*Q1D*Q1D, symmetric ? 6 : 9, NE);
    auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
    {
       const int tidz = MFEM_THREAD_ID(z);
       const int D1D = T_D1D ? T_D1D : d1d;
@@ -788,169 +466,48 @@ static void SmemPADiffusionDiagonal3D(const int NE,
    });
 }
 
-static void PADiffusionAssembleDiagonal(const int dim,
-                                        const int D1D,
-                                        const int Q1D,
-                                        const int NE,
-                                        const bool symm,
-                                        const Array<double> &B,
-                                        const Array<double> &G,
-                                        const Vector &D,
-                                        Vector &Y)
-{
-   if (dim == 2)
-   {
-      switch ((D1D << 4 ) | Q1D)
-      {
-         case 0x22: return SmemPADiffusionDiagonal2D<2,2,8>(NE,symm,B,G,D,Y);
-         case 0x33: return SmemPADiffusionDiagonal2D<3,3,8>(NE,symm,B,G,D,Y);
-         case 0x44: return SmemPADiffusionDiagonal2D<4,4,4>(NE,symm,B,G,D,Y);
-         case 0x55: return SmemPADiffusionDiagonal2D<5,5,4>(NE,symm,B,G,D,Y);
-         case 0x66: return SmemPADiffusionDiagonal2D<6,6,2>(NE,symm,B,G,D,Y);
-         case 0x77: return SmemPADiffusionDiagonal2D<7,7,2>(NE,symm,B,G,D,Y);
-         case 0x88: return SmemPADiffusionDiagonal2D<8,8,1>(NE,symm,B,G,D,Y);
-         case 0x99: return SmemPADiffusionDiagonal2D<9,9,1>(NE,symm,B,G,D,Y);
-         default: return PADiffusionDiagonal2D(NE,symm,B,G,D,Y,D1D,Q1D);
-      }
-   }
-   else if (dim == 3)
-   {
-      switch ((D1D << 4 ) | Q1D)
-      {
-         case 0x22: return SmemPADiffusionDiagonal3D<2,2>(NE,symm,B,G,D,Y);
-         case 0x23: return SmemPADiffusionDiagonal3D<2,3>(NE,symm,B,G,D,Y);
-         case 0x34: return SmemPADiffusionDiagonal3D<3,4>(NE,symm,B,G,D,Y);
-         case 0x45: return SmemPADiffusionDiagonal3D<4,5>(NE,symm,B,G,D,Y);
-         case 0x46: return SmemPADiffusionDiagonal3D<4,6>(NE,symm,B,G,D,Y);
-         case 0x56: return SmemPADiffusionDiagonal3D<5,6>(NE,symm,B,G,D,Y);
-         case 0x67: return SmemPADiffusionDiagonal3D<6,7>(NE,symm,B,G,D,Y);
-         case 0x78: return SmemPADiffusionDiagonal3D<7,8>(NE,symm,B,G,D,Y);
-         case 0x89: return SmemPADiffusionDiagonal3D<8,9>(NE,symm,B,G,D,Y);
-         case 0x9A: return SmemPADiffusionDiagonal3D<9,10>(NE,symm,B,G,D,Y);
-         default: return PADiffusionDiagonal3D(NE,symm,B,G,D,Y,D1D,Q1D);
-      }
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-
-void DiffusionIntegrator::AssembleDiagonalPA(Vector &diag)
-{
-   if (DeviceCanUseCeed())
-   {
-      ceedOp->GetDiagonal(diag);
-   }
-   else
-   {
-      if (pa_data.Size()==0) { AssemblePA(*fespace); }
-      PADiffusionAssembleDiagonal(dim, dofs1D, quad1D, ne, symmetric,
-                                  maps->B, maps->G, pa_data, diag);
-   }
-}
-
+void PADiffusionApply(const int dim,
+                      const int D1D,
+                      const int Q1D,
+                      const int NE,
+                      const bool symm,
+                      const Array<double> &B,
+                      const Array<double> &G,
+                      const Array<double> &Bt,
+                      const Array<double> &Gt,
+                      const Vector &D,
+                      const Vector &X,
+                      Vector &Y);
 
 #ifdef MFEM_USE_OCCA
 // OCCA PA Diffusion Apply 2D kernel
-static void OccaPADiffusionApply2D(const int D1D,
-                                   const int Q1D,
-                                   const int NE,
-                                   const Array<double> &B,
-                                   const Array<double> &G,
-                                   const Array<double> &Bt,
-                                   const Array<double> &Gt,
-                                   const Vector &D,
-                                   const Vector &X,
-                                   Vector &Y)
-{
-   occa::properties props;
-   props["defines/D1D"] = D1D;
-   props["defines/Q1D"] = Q1D;
-   const occa::memory o_B = OccaMemoryRead(B.GetMemory(), B.Size());
-   const occa::memory o_G = OccaMemoryRead(G.GetMemory(), G.Size());
-   const occa::memory o_Bt = OccaMemoryRead(Bt.GetMemory(), Bt.Size());
-   const occa::memory o_Gt = OccaMemoryRead(Gt.GetMemory(), Gt.Size());
-   const occa::memory o_D = OccaMemoryRead(D.GetMemory(), D.Size());
-   const occa::memory o_X = OccaMemoryRead(X.GetMemory(), X.Size());
-   occa::memory o_Y = OccaMemoryReadWrite(Y.GetMemory(), Y.Size());
-   const occa_id_t id = std::make_pair(D1D,Q1D);
-   if (!Device::Allows(Backend::OCCA_CUDA))
-   {
-      static occa_kernel_t OccaDiffApply2D_cpu;
-      if (OccaDiffApply2D_cpu.find(id) == OccaDiffApply2D_cpu.end())
-      {
-         const occa::kernel DiffusionApply2D_CPU =
-            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
-                                        "DiffusionApply2D_CPU", props);
-         OccaDiffApply2D_cpu.emplace(id, DiffusionApply2D_CPU);
-      }
-      OccaDiffApply2D_cpu.at(id)(NE, o_B, o_G, o_Bt, o_Gt, o_D, o_X, o_Y);
-   }
-   else
-   {
-      static occa_kernel_t OccaDiffApply2D_gpu;
-      if (OccaDiffApply2D_gpu.find(id) == OccaDiffApply2D_gpu.end())
-      {
-         const occa::kernel DiffusionApply2D_GPU =
-            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
-                                        "DiffusionApply2D_GPU", props);
-         OccaDiffApply2D_gpu.emplace(id, DiffusionApply2D_GPU);
-      }
-      OccaDiffApply2D_gpu.at(id)(NE, o_B, o_G, o_Bt, o_Gt, o_D, o_X, o_Y);
-   }
-}
+void OccaPADiffusionApply2D(const int D1D,
+                            const int Q1D,
+                            const int NE,
+                            const Array<double> &B,
+                            const Array<double> &G,
+                            const Array<double> &Bt,
+                            const Array<double> &Gt,
+                            const Vector &D,
+                            const Vector &X,
+                            Vector &Y);
 
 // OCCA PA Diffusion Apply 3D kernel
-static void OccaPADiffusionApply3D(const int D1D,
-                                   const int Q1D,
-                                   const int NE,
-                                   const Array<double> &B,
-                                   const Array<double> &G,
-                                   const Array<double> &Bt,
-                                   const Array<double> &Gt,
-                                   const Vector &D,
-                                   const Vector &X,
-                                   Vector &Y)
-{
-   occa::properties props;
-   props["defines/D1D"] = D1D;
-   props["defines/Q1D"] = Q1D;
-   const occa::memory o_B = OccaMemoryRead(B.GetMemory(), B.Size());
-   const occa::memory o_G = OccaMemoryRead(G.GetMemory(), G.Size());
-   const occa::memory o_Bt = OccaMemoryRead(Bt.GetMemory(), Bt.Size());
-   const occa::memory o_Gt = OccaMemoryRead(Gt.GetMemory(), Gt.Size());
-   const occa::memory o_D = OccaMemoryRead(D.GetMemory(), D.Size());
-   const occa::memory o_X = OccaMemoryRead(X.GetMemory(), X.Size());
-   occa::memory o_Y = OccaMemoryReadWrite(Y.GetMemory(), Y.Size());
-   const occa_id_t id = std::make_pair(D1D,Q1D);
-   if (!Device::Allows(Backend::OCCA_CUDA))
-   {
-      static occa_kernel_t OccaDiffApply3D_cpu;
-      if (OccaDiffApply3D_cpu.find(id) == OccaDiffApply3D_cpu.end())
-      {
-         const occa::kernel DiffusionApply3D_CPU =
-            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
-                                        "DiffusionApply3D_CPU", props);
-         OccaDiffApply3D_cpu.emplace(id, DiffusionApply3D_CPU);
-      }
-      OccaDiffApply3D_cpu.at(id)(NE, o_B, o_G, o_Bt, o_Gt, o_D, o_X, o_Y);
-   }
-   else
-   {
-      static occa_kernel_t OccaDiffApply3D_gpu;
-      if (OccaDiffApply3D_gpu.find(id) == OccaDiffApply3D_gpu.end())
-      {
-         const occa::kernel DiffusionApply3D_GPU =
-            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
-                                        "DiffusionApply3D_GPU", props);
-         OccaDiffApply3D_gpu.emplace(id, DiffusionApply3D_GPU);
-      }
-      OccaDiffApply3D_gpu.at(id)(NE, o_B, o_G, o_Bt, o_Gt, o_D, o_X, o_Y);
-   }
-}
+void OccaPADiffusionApply3D(const int D1D,
+                            const int Q1D,
+                            const int NE,
+                            const Array<double> &B,
+                            const Array<double> &G,
+                            const Array<double> &Bt,
+                            const Array<double> &Gt,
+                            const Vector &D,
+                            const Vector &X,
+                            Vector &Y);
 #endif // MFEM_USE_OCCA
 
 // PA Diffusion Apply 2D kernel
 template<int T_D1D = 0, int T_Q1D = 0>
-static void PADiffusionApply2D(const int NE,
+inline void PADiffusionApply2D(const int NE,
                                const bool symmetric,
                                const Array<double> &b_,
                                const Array<double> &g_,
@@ -973,7 +530,7 @@ static void PADiffusionApply2D(const int NE,
    auto D = Reshape(d_.Read(), Q1D*Q1D, symmetric ? 3 : 4, NE);
    auto X = Reshape(x_.Read(), D1D, D1D, NE);
    auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL(e, NE,
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -1072,7 +629,7 @@ static void PADiffusionApply2D(const int NE,
 
 // Shared memory PA Diffusion Apply 2D kernel
 template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
-static void SmemPADiffusionApply2D(const int NE,
+inline void SmemPADiffusionApply2D(const int NE,
                                    const bool symmetric,
                                    const Array<double> &b_,
                                    const Array<double> &g_,
@@ -1094,7 +651,7 @@ static void SmemPADiffusionApply2D(const int NE,
    auto D = Reshape(d_.Read(), Q1D*Q1D, symmetric ? 3 : 4, NE);
    auto x = Reshape(x_.Read(), D1D, D1D, NE);
    auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
+   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE(int e)
    {
       const int tidz = MFEM_THREAD_ID(z);
       const int D1D = T_D1D ? T_D1D : d1d;
@@ -1230,7 +787,7 @@ static void SmemPADiffusionApply2D(const int NE,
 
 // PA Diffusion Apply 3D kernel
 template<int T_D1D = 0, int T_Q1D = 0>
-static void PADiffusionApply3D(const int NE,
+inline void PADiffusionApply3D(const int NE,
                                const bool symmetric,
                                const Array<double> &b,
                                const Array<double> &g,
@@ -1252,7 +809,7 @@ static void PADiffusionApply3D(const int NE,
    auto D = Reshape(d_.Read(), Q1D*Q1D*Q1D, symmetric ? 6 : 9, NE);
    auto X = Reshape(x_.Read(), D1D, D1D, D1D, NE);
    auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL(e, NE,
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -1421,8 +978,9 @@ static void PADiffusionApply3D(const int NE,
    });
 }
 
+// Shared memory PA Diffusion Apply 3D kernel
 template<int T_D1D = 0, int T_Q1D = 0>
-static void SmemPADiffusionApply3D(const int NE,
+inline void SmemPADiffusionApply3D(const int NE,
                                    const bool symmetric,
                                    const Array<double> &b_,
                                    const Array<double> &g_,
@@ -1443,7 +1001,7 @@ static void SmemPADiffusionApply3D(const int NE,
    auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, symmetric ? 6 : 9, NE);
    auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
    auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -1643,99 +1201,8 @@ static void SmemPADiffusionApply3D(const int NE,
    });
 }
 
-static void PADiffusionApply(const int dim,
-                             const int D1D,
-                             const int Q1D,
-                             const int NE,
-                             const bool symm,
-                             const Array<double> &B,
-                             const Array<double> &G,
-                             const Array<double> &Bt,
-                             const Array<double> &Gt,
-                             const Vector &D,
-                             const Vector &X,
-                             Vector &Y)
-{
-#ifdef MFEM_USE_OCCA
-   if (DeviceCanUseOcca())
-   {
-      if (dim == 2)
-      {
-         OccaPADiffusionApply2D(D1D,Q1D,NE,B,G,Bt,Gt,D,X,Y);
-         return;
-      }
-      if (dim == 3)
-      {
-         OccaPADiffusionApply3D(D1D,Q1D,NE,B,G,Bt,Gt,D,X,Y);
-         return;
-      }
-      MFEM_ABORT("OCCA PADiffusionApply unknown kernel!");
-   }
-#endif // MFEM_USE_OCCA
-   const int id = (D1D << 4) | Q1D;
-
-   if (dim == 2)
-   {
-      switch (id)
-      {
-         case 0x22: return SmemPADiffusionApply2D<2,2,16>(NE,symm,B,G,D,X,Y);
-         case 0x33: return SmemPADiffusionApply2D<3,3,16>(NE,symm,B,G,D,X,Y);
-         case 0x44: return SmemPADiffusionApply2D<4,4,8>(NE,symm,B,G,D,X,Y);
-         case 0x55: return SmemPADiffusionApply2D<5,5,8>(NE,symm,B,G,D,X,Y);
-         case 0x66: return SmemPADiffusionApply2D<6,6,4>(NE,symm,B,G,D,X,Y);
-         case 0x77: return SmemPADiffusionApply2D<7,7,4>(NE,symm,B,G,D,X,Y);
-         case 0x88: return SmemPADiffusionApply2D<8,8,2>(NE,symm,B,G,D,X,Y);
-         case 0x99: return SmemPADiffusionApply2D<9,9,2>(NE,symm,B,G,D,X,Y);
-         default:   return PADiffusionApply2D(NE,symm,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
-      }
-   }
-
-   if (dim == 3)
-   {
-      switch (id)
-      {
-         case 0x22: return SmemPADiffusionApply3D<2,2>(NE,symm,B,G,D,X,Y);
-         case 0x23: return SmemPADiffusionApply3D<2,3>(NE,symm,B,G,D,X,Y);
-         case 0x34: return SmemPADiffusionApply3D<3,4>(NE,symm,B,G,D,X,Y);
-         case 0x45: return SmemPADiffusionApply3D<4,5>(NE,symm,B,G,D,X,Y);
-         case 0x46: return SmemPADiffusionApply3D<4,6>(NE,symm,B,G,D,X,Y);
-         case 0x56: return SmemPADiffusionApply3D<5,6>(NE,symm,B,G,D,X,Y);
-         case 0x58: return SmemPADiffusionApply3D<5,8>(NE,symm,B,G,D,X,Y);
-         case 0x67: return SmemPADiffusionApply3D<6,7>(NE,symm,B,G,D,X,Y);
-         case 0x78: return SmemPADiffusionApply3D<7,8>(NE,symm,B,G,D,X,Y);
-         case 0x89: return SmemPADiffusionApply3D<8,9>(NE,symm,B,G,D,X,Y);
-         default:   return PADiffusionApply3D(NE,symm,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
-      }
-   }
-   MFEM_ABORT("Unknown kernel: 0x"<<std::hex << id << std::dec);
-}
-
-// PA Diffusion Apply kernel
-void DiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
-{
-   if (DeviceCanUseCeed())
-   {
-      ceedOp->AddMult(x, y);
-   }
-   else
-   {
-      PADiffusionApply(dim, dofs1D, quad1D, ne, symmetric,
-                       maps->B, maps->G, maps->Bt, maps->Gt,
-                       pa_data, x, y);
-   }
-}
-
-void DiffusionIntegrator::AddMultTransposePA(const Vector &x, Vector &y) const
-{
-   if (symmetric)
-   {
-      AddMultPA(x, y);
-   }
-   else
-   {
-      MFEM_ABORT("DiffusionIntegrator::AddMultTransposePA only implemented in "
-                 "the symmetric case.")
-   }
-}
+} // namespace internal
 
 } // namespace mfem
+
+#endif

fem/integ/bilininteg_diffusion_mf.cpp

@@ -9,12 +9,9 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-#include "ceed/integrators/diffusion/diffusion.hpp"
-
-using namespace std;
+#include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
+#include "../ceed/integrators/diffusion/diffusion.hpp"
 
 namespace mfem
 {

fem/integ/bilininteg_diffusion_pa.cpp

@@ -0,0 +1,118 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
+#include "../qfunction.hpp"
+#include "../ceed/integrators/diffusion/diffusion.hpp"
+#include "bilininteg_diffusion_kernels.hpp"
+
+namespace mfem
+{
+
+void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
+{
+   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
+                         Device::GetDeviceMemoryType() : pa_mt;
+   // Assuming the same element type
+   fespace = &fes;
+   Mesh *mesh = fes.GetMesh();
+   if (mesh->GetNE() == 0) { return; }
+   const FiniteElement &el = *fes.GetFE(0);
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el);
+   if (DeviceCanUseCeed())
+   {
+      delete ceedOp;
+      MFEM_VERIFY(!VQ && !MQ,
+                  "Only scalar coefficient supported for DiffusionIntegrator"
+                  " with libCEED");
+      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
+                         fes.IsVariableOrder();
+      if (mixed)
+      {
+         ceedOp = new ceed::MixedPADiffusionIntegrator(*this, fes, Q);
+      }
+      else
+      {
+         ceedOp = new ceed::PADiffusionIntegrator(fes, *ir, Q);
+      }
+      return;
+   }
+   const int dims = el.GetDim();
+   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
+   const int nq = ir->GetNPoints();
+   dim = mesh->Dimension();
+   ne = fes.GetNE();
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mt);
+   const int sdim = mesh->SpaceDimension();
+   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
+   dofs1D = maps->ndof;
+   quad1D = maps->nqpt;
+
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(qs, CoefficientStorage::COMPRESSED);
+
+   if (MQ) { coeff.ProjectTranspose(*MQ); }
+   else if (VQ) { coeff.Project(*VQ); }
+   else if (Q) { coeff.Project(*Q); }
+   else { coeff.SetConstant(1.0); }
+
+   const int coeff_dim = coeff.GetVDim();
+   symmetric = (coeff_dim != dims*dims);
+   const int pa_size = symmetric ? symmDims : dims*dims;
+
+   pa_data.SetSize(pa_size * nq * ne, mt);
+   internal::PADiffusionSetup(dim, sdim, dofs1D, quad1D, coeff_dim, ne,
+                              ir->GetWeights(), geom->J, coeff, pa_data);
+}
+
+void DiffusionIntegrator::AssembleDiagonalPA(Vector &diag)
+{
+   if (DeviceCanUseCeed())
+   {
+      ceedOp->GetDiagonal(diag);
+   }
+   else
+   {
+      if (pa_data.Size()==0) { AssemblePA(*fespace); }
+      internal::PADiffusionAssembleDiagonal(dim, dofs1D, quad1D, ne, symmetric,
+                                            maps->B, maps->G, pa_data, diag);
+   }
+}
+
+void DiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
+{
+   if (DeviceCanUseCeed())
+   {
+      ceedOp->AddMult(x, y);
+   }
+   else
+   {
+      internal::PADiffusionApply(dim, dofs1D, quad1D, ne, symmetric,
+                                 maps->B, maps->G, maps->Bt, maps->Gt,
+                                 pa_data, x, y);
+   }
+}
+
+void DiffusionIntegrator::AddMultTransposePA(const Vector &x, Vector &y) const
+{
+   if (symmetric)
+   {
+      AddMultPA(x, y);
+   }
+   else
+   {
+      MFEM_ABORT("DiffusionIntegrator::AddMultTransposePA only implemented in "
+                 "the symmetric case.")
+   }
+}
+
+} // namespace mfem

fem/integ/bilininteg_divdiv_pa.cpp

@@ -0,0 +1,98 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
+#include "../qfunction.hpp"
+#include "bilininteg_hdiv_kernels.hpp"
+
+namespace mfem
+{
+
+void DivDivIntegrator::AssemblePA(const FiniteElementSpace &fes)
+{
+   // Assumes tensor-product elements
+   Mesh *mesh = fes.GetMesh();
+   const FiniteElement *fel = fes.GetFE(0);
+
+   const VectorTensorFiniteElement *el =
+      dynamic_cast<const VectorTensorFiniteElement*>(fel);
+   MFEM_VERIFY(el != NULL, "Only VectorTensorFiniteElement is supported!");
+
+   const IntegrationRule *ir = IntRule ? IntRule : &MassIntegrator::GetRule
+                               (*el, *el, *mesh->GetElementTransformation(0));
+
+   const int dims = el->GetDim();
+   MFEM_VERIFY(dims == 2 || dims == 3, "");
+
+   const int nq = ir->GetNPoints();
+   dim = mesh->Dimension();
+   MFEM_VERIFY(dim == 2 || dim == 3, "");
+
+   ne = fes.GetNE();
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
+   mapsC = &el->GetDofToQuad(*ir, DofToQuad::TENSOR);
+   mapsO = &el->GetDofToQuadOpen(*ir, DofToQuad::TENSOR);
+   dofs1D = mapsC->ndof;
+   quad1D = mapsC->nqpt;
+
+   MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");
+
+   pa_data.SetSize(nq * ne, Device::GetMemoryType());
+
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::FULL);
+
+   if (el->GetDerivType() == mfem::FiniteElement::DIV && dim == 3)
+   {
+      internal::PADivDivSetup3D(quad1D, ne, ir->GetWeights(), geom->J, coeff,
+                                pa_data);
+   }
+   else if (el->GetDerivType() == mfem::FiniteElement::DIV && dim == 2)
+   {
+      internal::PADivDivSetup2D(quad1D, ne, ir->GetWeights(), geom->J, coeff,
+                                pa_data);
+   }
+   else
+   {
+      MFEM_ABORT("Unknown kernel.");
+   }
+}
+
+void DivDivIntegrator::AssembleDiagonalPA(Vector& diag)
+{
+   if (dim == 3)
+   {
+      internal::PADivDivAssembleDiagonal3D(dofs1D, quad1D, ne,
+                                           mapsO->B, mapsC->G, pa_data, diag);
+   }
+   else
+   {
+      internal::PADivDivAssembleDiagonal2D(dofs1D, quad1D, ne,
+                                           mapsO->B, mapsC->G, pa_data, diag);
+   }
+}
+
+void DivDivIntegrator::AddMultPA(const Vector &x, Vector &y) const
+{
+   if (dim == 3)
+      internal::PADivDivApply3D(dofs1D, quad1D, ne, mapsO->B, mapsC->G,
+                                mapsO->Bt, mapsC->Gt, pa_data, x, y);
+   else if (dim == 2)
+      internal::PADivDivApply2D(dofs1D, quad1D, ne, mapsO->B, mapsC->G,
+                                mapsO->Bt, mapsC->Gt, pa_data, x, y);
+   else
+   {
+      MFEM_ABORT("Unsupported dimension!");
+   }
+}
+
+} // namespace mfem

fem/integ/bilininteg_gradient_pa.cpp

@@ -9,18 +9,14 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-#include "qfunction.hpp"
-
-using namespace std;
+#include "../../general/forall.hpp"
+#include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
+#include "../qfunction.hpp"
 
 namespace mfem
 {
 
-// PA Gradient Integrator
-
 /* Description of the *SetupND functions
    Inputs are as follows
    \b Q1D number of quadrature points in one dimension.
@@ -62,8 +58,8 @@ namespace mfem
    The shared memory (Smem) versions of the kernels differ from the regular
    versions in the following properties.
 
-   \b MFEM_FORALL is using only one level of parallelism.
-   \b MFEM_FORALL_ND uses an additional level of parallelism
+   \b mfem::forall is using only one level of parallelism.
+   \b mfem::forall_ND uses an additional level of parallelism
    \b MFEM_FOREACH_THREAD
 
    These macros allow automatic mapping of manually defined blocks to
@@ -87,7 +83,7 @@ static void PAGradientSetup2D(const int Q1D,
    const auto C = const_c ? Reshape(c.Read(), 1,1) :
                   Reshape(c.Read(), NQ, NE);
 
-   MFEM_FORALL(e, NE,
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
       for (int q = 0; q < NQ; ++q)
       {
@@ -122,7 +118,7 @@ static void PAGradientSetup3D(const int Q1D,
    const auto C = const_c ? Reshape(c.Read(), 1,1) :
                   Reshape(c.Read(), NQ,NE);
 
-   MFEM_FORALL(e, NE,
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
       for (int q = 0; q < NQ; ++q)
       {
@@ -242,7 +238,7 @@ static void PAGradientApply2D(const int NE,
    auto op = Reshape(op_.Read(), Q1D*Q1D, 2,2, NE);
    auto x = Reshape(x_.Read(), TR_D1D, TR_D1D, NE);
    auto y = Reshape(y_.ReadWrite(), TE_D1D, TE_D1D, 2, NE);
-   MFEM_FORALL(e, NE,
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
       const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
       const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -372,7 +368,7 @@ static void PAGradientApply3D(const int NE,
    auto op = Reshape(op_.Read(), Q1D*Q1D*Q1D, 3,3, NE);
    auto x = Reshape(x_.Read(), TR_D1D, TR_D1D, TR_D1D, NE);
    auto y = Reshape(y_.ReadWrite(), TE_D1D, TE_D1D, TE_D1D, 3, NE);
-   MFEM_FORALL(e, NE,
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
       const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
       const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -572,7 +568,8 @@ static void SmemPAGradientApply3D(const int NE,
    auto x = Reshape(x_.Read(), TR_D1D, TR_D1D, TR_D1D, NE);
    auto y = Reshape(y_.ReadWrite(), TE_D1D, TE_D1D, TE_D1D, 3, NE);
 
-   MFEM_FORALL_3D(e, NE, (Q1D>8)?8:Q1D, (Q1D>8)?8:Q1D, (Q1D>8)?8:Q1D,
+   mfem::forall_3D(NE, (Q1D>8)?8:Q1D, (Q1D>8)?8:Q1D, (Q1D>8)?8:Q1D,
+                   [=] MFEM_HOST_DEVICE (int e)
    {
       const int tidz = MFEM_THREAD_ID(z);
       const int D1DR = T_TR_D1D ? T_TR_D1D : tr_d1d;

fem/integ/bilininteg_hcurlhdiv_kernels.cpp

@@ -0,0 +1,578 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "bilininteg_hcurlhdiv_kernels.hpp"
+
+namespace mfem
+{
+
+namespace internal
+{
+
+// PA H(curl) x H(div) mass assemble 2D kernel, with factor
+// dF^{-1} C dF for a vector or matrix coefficient C.
+// If transpose, use dF^T C dF^{-T} for H(div) x H(curl).
+void PAHcurlHdivMassSetup2D(const int Q1D,
+                            const int coeffDim,
+                            const int NE,
+                            const bool transpose,
+                            const Array<double> &w_,
+                            const Vector &j,
+                            Vector &coeff_,
+                            Vector &op)
+{
+   const bool symmetric = (coeffDim != 4);
+   auto W = Reshape(w_.Read(), Q1D, Q1D);
+   auto J = Reshape(j.Read(), Q1D, Q1D, 2, 2, NE);
+   auto coeff = Reshape(coeff_.Read(), coeffDim, Q1D, Q1D, NE);
+   auto y = Reshape(op.Write(), 4, Q1D, Q1D, NE);
+
+   const int i11 = 0;
+   const int i12 = transpose ? 2 : 1;
+   const int i21 = transpose ? 1 : 2;
+   const int i22 = 3;
+
+   mfem::forall_2D(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
+   {
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            const double J11 = J(qx,qy,0,0,e);
+            const double J21 = J(qx,qy,1,0,e);
+            const double J12 = J(qx,qy,0,1,e);
+            const double J22 = J(qx,qy,1,1,e);
+            const double w_detJ = W(qx,qy) / ((J11*J22) - (J21*J12));
+
+            if (coeffDim == 3 || coeffDim == 4) // Matrix coefficient version
+            {
+               // First compute entries of R = MJ
+               const double M11 = coeff(i11,qx,qy,e);
+               const double M12 = (!symmetric) ? coeff(i12,qx,qy,e) : coeff(1,qx,qy,e);
+               const double M21 = (!symmetric) ? coeff(i21,qx,qy,e) : M12;
+               const double M22 = (!symmetric) ? coeff(i22,qx,qy,e) : coeff(2,qx,qy,e);
+
+               // J^{-1} M^T
+               const double R11 = ( J22*M11 - J12*M12); // 1,1
+               const double R12 = ( J22*M21 - J12*M22); // 1,2
+               const double R21 = (-J21*M11 + J11*M12); // 2,1
+               const double R22 = (-J21*M21 + J11*M22); // 2,2
+
+               // (RJ)^T
+               y(i11,qx,qy,e) = w_detJ * (R11*J11 + R12*J21); // 1,1
+               y(i21,qx,qy,e) = w_detJ * (R11*J12 + R12*J22); // 1,2 (transpose)
+               y(i12,qx,qy,e) = w_detJ * (R21*J11 + R22*J21); // 2,1 (transpose)
+               y(i22,qx,qy,e) = w_detJ * (R21*J12 + R22*J22); // 2,2
+            }
+            else if (coeffDim == 2) // Vector coefficient version
+            {
+               const double D1 = coeff(0,qx,qy,e);
+               const double D2 = coeff(1,qx,qy,e);
+               const double R11 = D1*J11;
+               const double R12 = D1*J12;
+               const double R21 = D2*J21;
+               const double R22 = D2*J22;
+               y(i11,qx,qy,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
+               y(i21,qx,qy,e) = w_detJ * ( J22*R12 - J12*R22); // 1,2 (transpose)
+               y(i12,qx,qy,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1 (transpose)
+               y(i22,qx,qy,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
+            }
+         }
+      }
+   });
+}
+
+// PA H(curl) x H(div) mass assemble 3D kernel, with factor
+// dF^{-1} C dF for a vector or matrix coefficient C.
+// If transpose, use dF^T C dF^{-T} for H(div) x H(curl).
+void PAHcurlHdivMassSetup3D(const int Q1D,
+                            const int coeffDim,
+                            const int NE,
+                            const bool transpose,
+                            const Array<double> &w_,
+                            const Vector &j,
+                            Vector &coeff_,
+                            Vector &op)
+{
+   const bool symmetric = (coeffDim != 9);
+   auto W = Reshape(w_.Read(), Q1D, Q1D, Q1D);
+   auto J = Reshape(j.Read(), Q1D, Q1D, Q1D, 3, 3, NE);
+   auto coeff = Reshape(coeff_.Read(), coeffDim, Q1D, Q1D, Q1D, NE);
+   auto y = Reshape(op.Write(), 9, Q1D, Q1D, Q1D, NE);
+
+   const int i11 = 0;
+   const int i12 = transpose ? 3 : 1;
+   const int i13 = transpose ? 6 : 2;
+   const int i21 = transpose ? 1 : 3;
+   const int i22 = 4;
+   const int i23 = transpose ? 7 : 5;
+   const int i31 = transpose ? 2 : 6;
+   const int i32 = transpose ? 5 : 7;
+   const int i33 = 8;
+
+   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
+   {
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qz,z,Q1D)
+            {
+               const double J11 = J(qx,qy,qz,0,0,e);
+               const double J21 = J(qx,qy,qz,1,0,e);
+               const double J31 = J(qx,qy,qz,2,0,e);
+               const double J12 = J(qx,qy,qz,0,1,e);
+               const double J22 = J(qx,qy,qz,1,1,e);
+               const double J32 = J(qx,qy,qz,2,1,e);
+               const double J13 = J(qx,qy,qz,0,2,e);
+               const double J23 = J(qx,qy,qz,1,2,e);
+               const double J33 = J(qx,qy,qz,2,2,e);
+               const double detJ = J11 * (J22 * J33 - J32 * J23) -
+                                   J21 * (J12 * J33 - J32 * J13) +
+                                   J31 * (J12 * J23 - J22 * J13);
+               const double w_detJ = W(qx,qy,qz) / detJ;
+               // adj(J)
+               const double A11 = (J22 * J33) - (J23 * J32);
+               const double A12 = (J32 * J13) - (J12 * J33);
+               const double A13 = (J12 * J23) - (J22 * J13);
+               const double A21 = (J31 * J23) - (J21 * J33);
+               const double A22 = (J11 * J33) - (J13 * J31);
+               const double A23 = (J21 * J13) - (J11 * J23);
+               const double A31 = (J21 * J32) - (J31 * J22);
+               const double A32 = (J31 * J12) - (J11 * J32);
+               const double A33 = (J11 * J22) - (J12 * J21);
+
+               if (coeffDim == 6 || coeffDim == 9) // Matrix coefficient version
+               {
+                  // First compute entries of R = M^T J
+                  const double M11 = (!symmetric) ? coeff(i11,qx,qy,qz,e) : coeff(0,qx,qy,qz,e);
+                  const double M12 = (!symmetric) ? coeff(i12,qx,qy,qz,e) : coeff(1,qx,qy,qz,e);
+                  const double M13 = (!symmetric) ? coeff(i13,qx,qy,qz,e) : coeff(2,qx,qy,qz,e);
+                  const double M21 = (!symmetric) ? coeff(i21,qx,qy,qz,e) : M12;
+                  const double M22 = (!symmetric) ? coeff(i22,qx,qy,qz,e) : coeff(3,qx,qy,qz,e);
+                  const double M23 = (!symmetric) ? coeff(i23,qx,qy,qz,e) : coeff(4,qx,qy,qz,e);
+                  const double M31 = (!symmetric) ? coeff(i31,qx,qy,qz,e) : M13;
+                  const double M32 = (!symmetric) ? coeff(i32,qx,qy,qz,e) : M23;
+                  const double M33 = (!symmetric) ? coeff(i33,qx,qy,qz,e) : coeff(5,qx,qy,qz,e);
+
+                  const double R11 = M11*J11 + M21*J21 + M31*J31;
+                  const double R12 = M11*J12 + M21*J22 + M31*J32;
+                  const double R13 = M11*J13 + M21*J23 + M31*J33;
+                  const double R21 = M12*J11 + M22*J21 + M32*J31;
+                  const double R22 = M12*J12 + M22*J22 + M32*J32;
+                  const double R23 = M12*J13 + M22*J23 + M32*J33;
+                  const double R31 = M13*J11 + M23*J21 + M33*J31;
+                  const double R32 = M13*J12 + M23*J22 + M33*J32;
+                  const double R33 = M13*J13 + M23*J23 + M33*J33;
+
+                  // y = (J^{-1} M^T J)^T
+                  y(i11,qx,qy,qz,e) = w_detJ * (A11*R11 + A12*R21 + A13*R31); // 1,1
+                  y(i21,qx,qy,qz,e) = w_detJ * (A11*R12 + A12*R22 + A13*R32); // 1,2
+                  y(i31,qx,qy,qz,e) = w_detJ * (A11*R13 + A12*R23 + A13*R33); // 1,3
+                  y(i12,qx,qy,qz,e) = w_detJ * (A21*R11 + A22*R21 + A23*R31); // 2,1
+                  y(i22,qx,qy,qz,e) = w_detJ * (A21*R12 + A22*R22 + A23*R32); // 2,2
+                  y(i32,qx,qy,qz,e) = w_detJ * (A21*R13 + A22*R23 + A23*R33); // 2,3
+                  y(i13,qx,qy,qz,e) = w_detJ * (A31*R11 + A32*R21 + A33*R31); // 3,1
+                  y(i23,qx,qy,qz,e) = w_detJ * (A31*R12 + A32*R22 + A33*R32); // 3,2
+                  y(i33,qx,qy,qz,e) = w_detJ * (A31*R13 + A32*R23 + A33*R33); // 3,3
+               }
+               else if (coeffDim == 3)  // Vector coefficient version
+               {
+                  const double D1 = coeff(0,qx,qy,qz,e);
+                  const double D2 = coeff(1,qx,qy,qz,e);
+                  const double D3 = coeff(2,qx,qy,qz,e);
+                  // detJ J^{-1} DJ = adj(J) DJ
+                  // transpose
+                  y(i11,qx,qy,qz,e) = w_detJ * (D1*A11*J11 + D2*A12*J21 + D3*A13*J31); // 1,1
+                  y(i21,qx,qy,qz,e) = w_detJ * (D1*A11*J12 + D2*A12*J22 + D3*A13*J32); // 1,2
+                  y(i31,qx,qy,qz,e) = w_detJ * (D1*A11*J13 + D2*A12*J23 + D3*A13*J33); // 1,3
+                  y(i12,qx,qy,qz,e) = w_detJ * (D1*A21*J11 + D2*A22*J21 + D3*A23*J31); // 2,1
+                  y(i22,qx,qy,qz,e) = w_detJ * (D1*A21*J12 + D2*A22*J22 + D3*A23*J32); // 2,2
+                  y(i32,qx,qy,qz,e) = w_detJ * (D1*A21*J13 + D2*A22*J23 + D3*A23*J33); // 2,3
+                  y(i13,qx,qy,qz,e) = w_detJ * (D1*A31*J11 + D2*A32*J21 + D3*A33*J31); // 3,1
+                  y(i23,qx,qy,qz,e) = w_detJ * (D1*A31*J12 + D2*A32*J22 + D3*A33*J32); // 3,2
+                  y(i33,qx,qy,qz,e) = w_detJ * (D1*A31*J13 + D2*A32*J23 + D3*A33*J33); // 3,3
+               }
+            }
+         }
+      }
+   });
+}
+
+// Mass operator for H(curl) and H(div) functions, using Piola transformations
+// u = dF^{-T} \hat{u} in H(curl), v = (1 / det dF) dF \hat{v} in H(div).
+void PAHcurlHdivMassApply2D(const int D1D,
+                            const int D1Dtest,
+                            const int Q1D,
+                            const int NE,
+                            const bool scalarCoeff,
+                            const bool trialHcurl,
+                            const bool transpose,
+                            const Array<double> &Bo_,
+                            const Array<double> &Bc_,
+                            const Array<double> &Bot_,
+                            const Array<double> &Bct_,
+                            const Vector &op_,
+                            const Vector &x_,
+                            Vector &y_)
+{
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
+
+   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
+   constexpr static int VDIM = 2;
+
+   auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
+   auto Bc = Reshape(Bc_.Read(), Q1D, D1D);
+   auto Bot = Reshape(Bot_.Read(), D1Dtest-1, Q1D);
+   auto Bct = Reshape(Bct_.Read(), D1Dtest, Q1D);
+   auto op = Reshape(op_.Read(), scalarCoeff ? 1 : 4, Q1D, Q1D, NE);
+   auto x = Reshape(x_.Read(), 2*(D1D-1)*D1D, NE);
+   auto y = Reshape(y_.ReadWrite(), 2*(D1Dtest-1)*D1Dtest, NE);
+
+   const int i12 = transpose ? 2 : 1;
+   const int i21 = transpose ? 1 : 2;
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      double mass[MAX_Q1D][MAX_Q1D][VDIM];
+
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            for (int c = 0; c < VDIM; ++c)
+            {
+               mass[qy][qx][c] = 0.0;
+            }
+         }
+      }
+
+      int osc = 0;
+      for (int c = 0; c < VDIM; ++c)  // loop over x, y trial components
+      {
+         const int D1Dy = trialHcurl ? ((c == 1) ? D1D - 1 : D1D) :
+                          ((c == 1) ? D1D : D1D - 1);
+         const int D1Dx = trialHcurl ? ((c == 0) ? D1D - 1 : D1D) :
+                          ((c == 0) ? D1D : D1D - 1);
+
+         for (int dy = 0; dy < D1Dy; ++dy)
+         {
+            double massX[MAX_Q1D];
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               massX[qx] = 0.0;
+            }
+
+            for (int dx = 0; dx < D1Dx; ++dx)
+            {
+               const double t = x(dx + (dy * D1Dx) + osc, e);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  massX[qx] += t * (trialHcurl ? ((c == 0) ? Bo(qx,dx) : Bc(qx,dx)) :
+                                    ((c == 0) ? Bc(qx,dx) : Bo(qx,dx)));
+               }
+            }
+
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const double wy = trialHcurl ? ((c == 1) ? Bo(qy,dy) : Bc(qy,dy)) :
+                                 ((c == 1) ? Bc(qy,dy) : Bo(qy,dy));
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  mass[qy][qx][c] += massX[qx] * wy;
+               }
+            }
+         }
+
+         osc += D1Dx * D1Dy;
+      }  // loop (c) over components
+
+      // Apply D operator.
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            const double O11 = op(0,qx,qy,e);
+            const double O12 = scalarCoeff ? 0.0 : op(i12,qx,qy,e);
+            const double O21 = scalarCoeff ? 0.0 : op(i21,qx,qy,e);
+            const double O22 = scalarCoeff ? O11 : op(3,qx,qy,e);
+            const double massX = mass[qy][qx][0];
+            const double massY = mass[qy][qx][1];
+            mass[qy][qx][0] = (O11*massX)+(O12*massY);
+            mass[qy][qx][1] = (O21*massX)+(O22*massY);
+         }
+      }
+
+      osc = 0;
+      for (int c = 0; c < VDIM; ++c)  // loop over x, y test components
+      {
+         const int D1Dy = trialHcurl ? ((c == 1) ? D1Dtest : D1Dtest - 1) :
+                          ((c == 1) ? D1Dtest - 1 : D1Dtest);
+         const int D1Dx = trialHcurl ? ((c == 0) ? D1Dtest : D1Dtest - 1) :
+                          ((c == 0) ? D1Dtest - 1 : D1Dtest);
+
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            double massX[HDIV_MAX_D1D];
+            for (int dx = 0; dx < D1Dx; ++dx)
+            {
+               massX[dx] = 0.0;
+            }
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  massX[dx] += mass[qy][qx][c] * (trialHcurl ?
+                                                  ((c == 0) ? Bct(dx,qx) : Bot(dx,qx)) :
+                                                  ((c == 0) ? Bot(dx,qx) : Bct(dx,qx)));
+               }
+            }
+            for (int dy = 0; dy < D1Dy; ++dy)
+            {
+               const double wy = trialHcurl ? ((c == 1) ? Bct(dy,qy) : Bot(dy,qy)) :
+                                 ((c == 1) ? Bot(dy,qy) : Bct(dy,qy));
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  y(dx + (dy * D1Dx) + osc, e) += massX[dx] * wy;
+               }
+            }
+         }
+
+         osc += D1Dx * D1Dy;
+      }  // loop c
+   }); // end of element loop
+}
+
+// Mass operator for H(curl) and H(div) functions, using Piola transformations
+// u = dF^{-T} \hat{u} in H(curl), v = (1 / det dF) dF \hat{v} in H(div).
+void PAHcurlHdivMassApply3D(const int D1D,
+                            const int D1Dtest,
+                            const int Q1D,
+                            const int NE,
+                            const bool scalarCoeff,
+                            const bool trialHcurl,
+                            const bool transpose,
+                            const Array<double> &Bo_,
+                            const Array<double> &Bc_,
+                            const Array<double> &Bot_,
+                            const Array<double> &Bct_,
+                            const Vector &op_,
+                            const Vector &x_,
+                            Vector &y_)
+{
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
+
+   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
+   constexpr static int VDIM = 3;
+
+   auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
+   auto Bc = Reshape(Bc_.Read(), Q1D, D1D);
+   auto Bot = Reshape(Bot_.Read(), D1Dtest-1, Q1D);
+   auto Bct = Reshape(Bct_.Read(), D1Dtest, Q1D);
+   auto op = Reshape(op_.Read(), scalarCoeff ? 1 : 9, Q1D, Q1D, Q1D, NE);
+   auto x = Reshape(x_.Read(), 3*(D1D-1)*D1D*(trialHcurl ? D1D : D1D-1), NE);
+   auto y = Reshape(y_.ReadWrite(), 3*(D1Dtest-1)*D1Dtest*
+                    (trialHcurl ? D1Dtest-1 : D1Dtest), NE);
+
+   const int i12 = transpose ? 3 : 1;
+   const int i13 = transpose ? 6 : 2;
+   const int i21 = transpose ? 1 : 3;
+   const int i23 = transpose ? 7 : 5;
+   const int i31 = transpose ? 2 : 6;
+   const int i32 = transpose ? 5 : 7;
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      double mass[MAX_Q1D][MAX_Q1D][MAX_Q1D][VDIM];
+
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               for (int c = 0; c < VDIM; ++c)
+               {
+                  mass[qz][qy][qx][c] = 0.0;
+               }
+            }
+         }
+      }
+
+      int osc = 0;
+      for (int c = 0; c < VDIM; ++c)  // loop over x, y, z trial components
+      {
+         const int D1Dz = trialHcurl ? ((c == 2) ? D1D - 1 : D1D) :
+                          ((c == 2) ? D1D : D1D - 1);
+         const int D1Dy = trialHcurl ? ((c == 1) ? D1D - 1 : D1D) :
+                          ((c == 1) ? D1D : D1D - 1);
+         const int D1Dx = trialHcurl ? ((c == 0) ? D1D - 1 : D1D) :
+                          ((c == 0) ? D1D : D1D - 1);
+
+         for (int dz = 0; dz < D1Dz; ++dz)
+         {
+            double massXY[MAX_Q1D][MAX_Q1D];
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  massXY[qy][qx] = 0.0;
+               }
+            }
+
+            for (int dy = 0; dy < D1Dy; ++dy)
+            {
+               double massX[MAX_Q1D];
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  massX[qx] = 0.0;
+               }
+
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  const double t = x(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e);
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     massX[qx] += t * (trialHcurl ? ((c == 0) ? Bo(qx,dx) : Bc(qx,dx)) :
+                                       ((c == 0) ? Bc(qx,dx) : Bo(qx,dx)));
+                  }
+               }
+
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  const double wy = trialHcurl ? ((c == 1) ? Bo(qy,dy) : Bc(qy,dy)) :
+                                    ((c == 1) ? Bc(qy,dy) : Bo(qy,dy));
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     const double wx = massX[qx];
+                     massXY[qy][qx] += wx * wy;
+                  }
+               }
+            }
+
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               const double wz = trialHcurl ? ((c == 2) ? Bo(qz,dz) : Bc(qz,dz)) :
+                                 ((c == 2) ? Bc(qz,dz) : Bo(qz,dz));
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     mass[qz][qy][qx][c] += massXY[qy][qx] * wz;
+                  }
+               }
+            }
+         }
+
+         osc += D1Dx * D1Dy * D1Dz;
+      }  // loop (c) over components
+
+      // Apply D operator.
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const double O11 = op(0,qx,qy,qz,e);
+               const double O12 = scalarCoeff ? 0.0 : op(i12,qx,qy,qz,e);
+               const double O13 = scalarCoeff ? 0.0 : op(i13,qx,qy,qz,e);
+               const double O21 = scalarCoeff ? 0.0 : op(i21,qx,qy,qz,e);
+               const double O22 = scalarCoeff ? O11 : op(4,qx,qy,qz,e);
+               const double O23 = scalarCoeff ? 0.0 : op(i23,qx,qy,qz,e);
+               const double O31 = scalarCoeff ? 0.0 : op(i31,qx,qy,qz,e);
+               const double O32 = scalarCoeff ? 0.0 : op(i32,qx,qy,qz,e);
+               const double O33 = scalarCoeff ? O11 : op(8,qx,qy,qz,e);
+               const double massX = mass[qz][qy][qx][0];
+               const double massY = mass[qz][qy][qx][1];
+               const double massZ = mass[qz][qy][qx][2];
+               mass[qz][qy][qx][0] = (O11*massX)+(O12*massY)+(O13*massZ);
+               mass[qz][qy][qx][1] = (O21*massX)+(O22*massY)+(O23*massZ);
+               mass[qz][qy][qx][2] = (O31*massX)+(O32*massY)+(O33*massZ);
+            }
+         }
+      }
+
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         double massXY[HDIV_MAX_D1D][HDIV_MAX_D1D];
+
+         osc = 0;
+         for (int c = 0; c < VDIM; ++c)  // loop over x, y, z test components
+         {
+            const int D1Dz = trialHcurl ? ((c == 2) ? D1Dtest : D1Dtest - 1) :
+                             ((c == 2) ? D1Dtest - 1 : D1Dtest);
+            const int D1Dy = trialHcurl ? ((c == 1) ? D1Dtest : D1Dtest - 1) :
+                             ((c == 1) ? D1Dtest - 1 : D1Dtest);
+            const int D1Dx = trialHcurl ? ((c == 0) ? D1Dtest : D1Dtest - 1) :
+                             ((c == 0) ? D1Dtest - 1 : D1Dtest);
+
+            for (int dy = 0; dy < D1Dy; ++dy)
+            {
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  massXY[dy][dx] = 0.0;
+               }
+            }
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               double massX[HDIV_MAX_D1D];
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  massX[dx] = 0.0;
+               }
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  for (int dx = 0; dx < D1Dx; ++dx)
+                  {
+                     massX[dx] += mass[qz][qy][qx][c] * (trialHcurl ?
+                                                         ((c == 0) ? Bct(dx,qx) : Bot(dx,qx)) :
+                                                         ((c == 0) ? Bot(dx,qx) : Bct(dx,qx)));
+                  }
+               }
+               for (int dy = 0; dy < D1Dy; ++dy)
+               {
+                  const double wy = trialHcurl ? ((c == 1) ? Bct(dy,qy) : Bot(dy,qy)) :
+                                    ((c == 1) ? Bot(dy,qy) : Bct(dy,qy));
+                  for (int dx = 0; dx < D1Dx; ++dx)
+                  {
+                     massXY[dy][dx] += massX[dx] * wy;
+                  }
+               }
+            }
+
+            for (int dz = 0; dz < D1Dz; ++dz)
+            {
+               const double wz = trialHcurl ? ((c == 2) ? Bct(dz,qz) : Bot(dz,qz)) :
+                                 ((c == 2) ? Bot(dz,qz) : Bct(dz,qz));
+               for (int dy = 0; dy < D1Dy; ++dy)
+               {
+                  for (int dx = 0; dx < D1Dx; ++dx)
+                  {
+                     y(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e) +=
+                        massXY[dy][dx] * wz;
+                  }
+               }
+            }
+
+            osc += D1Dx * D1Dy * D1Dz;
+         }  // loop c
+      }  // loop qz
+   }); // end of element loop
+}
+
+} // namespace internal
+
+} // namespace mfem

fem/integ/bilininteg_hcurlhdiv_kernels.hpp

@@ -0,0 +1,818 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_BILININTEG_HCURLHDIV_KERNELS_HPP
+#define MFEM_BILININTEG_HCURLHDIV_KERNELS_HPP
+
+#include "../../config/config.hpp"
+#include "../../general/array.hpp"
+#include "../../general/forall.hpp"
+#include "../../linalg/dtensor.hpp"
+#include "../../linalg/vector.hpp"
+#include "../bilininteg.hpp"
+
+namespace mfem
+{
+
+namespace internal
+{
+
+// PA H(curl)-H(div) Mass Apply 2D kernel
+void PAHcurlHdivMassSetup2D(const int Q1D,
+                            const int coeffDim,
+                            const int NE,
+                            const bool transpose,
+                            const Array<double> &w_,
+                            const Vector &j,
+                            Vector &coeff_,
+                            Vector &op);
+
+// PA H(curl)-H(div) Mass Assemble 3D kernel
+void PAHcurlHdivMassSetup3D(const int Q1D,
+                            const int coeffDim,
+                            const int NE,
+                            const bool transpose,
+                            const Array<double> &w_,
+                            const Vector &j,
+                            Vector &coeff_,
+                            Vector &op);
+
+// PA H(curl)-H(div) Mass Apply 2D kernel
+void PAHcurlHdivMassApply2D(const int D1D,
+                            const int D1Dtest,
+                            const int Q1D,
+                            const int NE,
+                            const bool scalarCoeff,
+                            const bool trialHcurl,
+                            const bool transpose,
+                            const Array<double> &Bo_,
+                            const Array<double> &Bc_,
+                            const Array<double> &Bot_,
+                            const Array<double> &Bct_,
+                            const Vector &op_,
+                            const Vector &x_,
+                            Vector &y_);
+
+// PA H(curl)-H(div) Mass Apply 3D kernel
+void PAHcurlHdivMassApply3D(const int D1D,
+                            const int D1Dtest,
+                            const int Q1D,
+                            const int NE,
+                            const bool scalarCoeff,
+                            const bool trialHcurl,
+                            const bool transpose,
+                            const Array<double> &Bo_,
+                            const Array<double> &Bc_,
+                            const Array<double> &Bot_,
+                            const Array<double> &Bct_,
+                            const Vector &op_,
+                            const Vector &x_,
+                            Vector &y_);
+
+// PA H(curl)-H(div) Curl Apply 3D kernel
+template<int T_D1D = 0, int T_D1D_TEST = 0, int T_Q1D = 0>
+inline void PAHcurlHdivApply3D(const int d1d,
+                               const int d1dtest,
+                               const int q1d,
+                               const int NE,
+                               const Array<double> &bo,
+                               const Array<double> &bc,
+                               const Array<double> &bot,
+                               const Array<double> &bct,
+                               const Array<double> &gc,
+                               const Vector &pa_data,
+                               const Vector &x,
+                               Vector &y)
+{
+   MFEM_VERIFY(T_D1D || d1d <= HCURL_MAX_D1D, "Error: d1d > HCURL_MAX_D1D");
+   MFEM_VERIFY(T_D1D_TEST ||
+               d1dtest <= HCURL_MAX_D1D, "Error: d1dtest > HCURL_MAX_D1D");
+   MFEM_VERIFY(T_Q1D || q1d <= HCURL_MAX_Q1D, "Error: q1d > HCURL_MAX_Q1D");
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int D1Dtest = T_D1D_TEST ? T_D1D_TEST : d1dtest;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
+   auto Bc = Reshape(bc.Read(), Q1D, D1D);
+   auto Bot = Reshape(bot.Read(), D1Dtest-1, Q1D);
+   auto Bct = Reshape(bct.Read(), D1Dtest, Q1D);
+   auto Gc = Reshape(gc.Read(), Q1D, D1D);
+   auto op = Reshape(pa_data.Read(), Q1D, Q1D, Q1D, 6, NE);
+   auto X = Reshape(x.Read(), 3*(D1D-1)*D1D*D1D, NE);
+   auto Y = Reshape(y.ReadWrite(), 3*(D1Dtest-1)*(D1Dtest-1)*D1Dtest, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      // Using Piola transformations (\nabla\times u) F = 1/det(dF) dF \hat{\nabla}\times\hat{u}
+      // for u in H(curl) and w = (1 / det (dF)) dF \hat{w} for w in H(div), we get
+      // (\nabla\times u) \cdot w = 1/det(dF)^2 \hat{\nabla}\times\hat{u}^T dF^T dF \hat{w}
+      // If c = 0, \hat{\nabla}\times\hat{u} reduces to [0, (u_0)_{x_2}, -(u_0)_{x_1}]
+      // If c = 1, \hat{\nabla}\times\hat{u} reduces to [-(u_1)_{x_2}, 0, (u_1)_{x_0}]
+      // If c = 2, \hat{\nabla}\times\hat{u} reduces to [(u_2)_{x_1}, -(u_2)_{x_0}, 0]
+
+      constexpr int VDIM = 3;
+      constexpr int MD1D = T_D1D ? T_D1D :
+                           HCURL_MAX_D1D;  // Assuming HDIV_MAX_D1D <= HCURL_MAX_D1D
+      constexpr int MQ1D = T_Q1D ? T_Q1D : HCURL_MAX_Q1D;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int D1Dtest = T_D1D_TEST ? T_D1D_TEST : d1dtest;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+      double curl[MQ1D][MQ1D][MQ1D][VDIM];
+      // curl[qz][qy][qx] will be computed as the vector curl at each quadrature point.
+
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               for (int c = 0; c < VDIM; ++c)
+               {
+                  curl[qz][qy][qx][c] = 0.0;
+               }
+            }
+         }
+      }
+
+      // We treat x, y, z components separately for optimization specific to each.
+
+      int osc = 0;
+
+      {
+         // x component
+         const int D1Dz = D1D;
+         const int D1Dy = D1D;
+         const int D1Dx = D1D - 1;
+
+         for (int dz = 0; dz < D1Dz; ++dz)
+         {
+            double gradXY[MQ1D][MQ1D][2];
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  for (int d = 0; d < 2; ++d)
+                  {
+                     gradXY[qy][qx][d] = 0.0;
+                  }
+               }
+            }
+
+            for (int dy = 0; dy < D1Dy; ++dy)
+            {
+               double massX[MQ1D];
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  massX[qx] = 0.0;
+               }
+
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  const double t = X(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e);
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     massX[qx] += t * Bo(qx,dx);
+                  }
+               }
+
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  const double wy = Bc(qy,dy);
+                  const double wDy = Gc(qy,dy);
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     const double wx = massX[qx];
+                     gradXY[qy][qx][0] += wx * wDy;
+                     gradXY[qy][qx][1] += wx * wy;
+                  }
+               }
+            }
+
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               const double wz = Bc(qz,dz);
+               const double wDz = Gc(qz,dz);
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     // \hat{\nabla}\times\hat{u} is [0, (u_0)_{x_2}, -(u_0)_{x_1}]
+                     curl[qz][qy][qx][1] += gradXY[qy][qx][1] * wDz; // (u_0)_{x_2}
+                     curl[qz][qy][qx][2] -= gradXY[qy][qx][0] * wz;  // -(u_0)_{x_1}
+                  }
+               }
+            }
+         }
+
+         osc += D1Dx * D1Dy * D1Dz;
+      }
+
+      {
+         // y component
+         const int D1Dz = D1D;
+         const int D1Dy = D1D - 1;
+         const int D1Dx = D1D;
+
+         for (int dz = 0; dz < D1Dz; ++dz)
+         {
+            double gradXY[MQ1D][MQ1D][2];
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  for (int d = 0; d < 2; ++d)
+                  {
+                     gradXY[qy][qx][d] = 0.0;
+                  }
+               }
+            }
+
+            for (int dx = 0; dx < D1Dx; ++dx)
+            {
+               double massY[MQ1D];
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  massY[qy] = 0.0;
+               }
+
+               for (int dy = 0; dy < D1Dy; ++dy)
+               {
+                  const double t = X(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e);
+                  for (int qy = 0; qy < Q1D; ++qy)
+                  {
+                     massY[qy] += t * Bo(qy,dy);
+                  }
+               }
+
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  const double wx = Bc(qx,dx);
+                  const double wDx = Gc(qx,dx);
+                  for (int qy = 0; qy < Q1D; ++qy)
+                  {
+                     const double wy = massY[qy];
+                     gradXY[qy][qx][0] += wDx * wy;
+                     gradXY[qy][qx][1] += wx * wy;
+                  }
+               }
+            }
+
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               const double wz = Bc(qz,dz);
+               const double wDz = Gc(qz,dz);
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     // \hat{\nabla}\times\hat{u} is [-(u_1)_{x_2}, 0, (u_1)_{x_0}]
+                     curl[qz][qy][qx][0] -= gradXY[qy][qx][1] * wDz; // -(u_1)_{x_2}
+                     curl[qz][qy][qx][2] += gradXY[qy][qx][0] * wz;  // (u_1)_{x_0}
+                  }
+               }
+            }
+         }
+
+         osc += D1Dx * D1Dy * D1Dz;
+      }
+
+      {
+         // z component
+         const int D1Dz = D1D - 1;
+         const int D1Dy = D1D;
+         const int D1Dx = D1D;
+
+         for (int dx = 0; dx < D1Dx; ++dx)
+         {
+            double gradYZ[MQ1D][MQ1D][2];
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  for (int d = 0; d < 2; ++d)
+                  {
+                     gradYZ[qz][qy][d] = 0.0;
+                  }
+               }
+            }
+
+            for (int dy = 0; dy < D1Dy; ++dy)
+            {
+               double massZ[MQ1D];
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  massZ[qz] = 0.0;
+               }
+
+               for (int dz = 0; dz < D1Dz; ++dz)
+               {
+                  const double t = X(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e);
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     massZ[qz] += t * Bo(qz,dz);
+                  }
+               }
+
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  const double wy = Bc(qy,dy);
+                  const double wDy = Gc(qy,dy);
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     const double wz = massZ[qz];
+                     gradYZ[qz][qy][0] += wz * wy;
+                     gradYZ[qz][qy][1] += wz * wDy;
+                  }
+               }
+            }
+
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const double wx = Bc(qx,dx);
+               const double wDx = Gc(qx,dx);
+
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     // \hat{\nabla}\times\hat{u} is [(u_2)_{x_1}, -(u_2)_{x_0}, 0]
+                     curl[qz][qy][qx][0] += gradYZ[qz][qy][1] * wx;  // (u_2)_{x_1}
+                     curl[qz][qy][qx][1] -= gradYZ[qz][qy][0] * wDx; // -(u_2)_{x_0}
+                  }
+               }
+            }
+         }
+      }
+
+      // Apply D operator.
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const double O11 = op(qx,qy,qz,0,e);
+               const double O12 = op(qx,qy,qz,1,e);
+               const double O13 = op(qx,qy,qz,2,e);
+               const double O22 = op(qx,qy,qz,3,e);
+               const double O23 = op(qx,qy,qz,4,e);
+               const double O33 = op(qx,qy,qz,5,e);
+
+               const double c1 = (O11 * curl[qz][qy][qx][0]) + (O12 * curl[qz][qy][qx][1]) +
+                                 (O13 * curl[qz][qy][qx][2]);
+               const double c2 = (O12 * curl[qz][qy][qx][0]) + (O22 * curl[qz][qy][qx][1]) +
+                                 (O23 * curl[qz][qy][qx][2]);
+               const double c3 = (O13 * curl[qz][qy][qx][0]) + (O23 * curl[qz][qy][qx][1]) +
+                                 (O33 * curl[qz][qy][qx][2]);
+
+               curl[qz][qy][qx][0] = c1;
+               curl[qz][qy][qx][1] = c2;
+               curl[qz][qy][qx][2] = c3;
+            }
+         }
+      }
+
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         double massXY[MD1D][MD1D];
+
+         osc = 0;
+
+         for (int c = 0; c < VDIM; ++c)  // loop over x, y, z components
+         {
+            const int D1Dz = (c == 2) ? D1Dtest : D1Dtest - 1;
+            const int D1Dy = (c == 1) ? D1Dtest : D1Dtest - 1;
+            const int D1Dx = (c == 0) ? D1Dtest : D1Dtest - 1;
+
+            for (int dy = 0; dy < D1Dy; ++dy)
+            {
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  massXY[dy][dx] = 0;
+               }
+            }
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               double massX[MD1D];
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  massX[dx] = 0;
+               }
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  for (int dx = 0; dx < D1Dx; ++dx)
+                  {
+                     massX[dx] += curl[qz][qy][qx][c] *
+                                  ((c == 0) ? Bct(dx,qx) : Bot(dx,qx));
+                  }
+               }
+               for (int dy = 0; dy < D1Dy; ++dy)
+               {
+                  const double wy = (c == 1) ? Bct(dy,qy) : Bot(dy,qy);
+                  for (int dx = 0; dx < D1Dx; ++dx)
+                  {
+                     massXY[dy][dx] += massX[dx] * wy;
+                  }
+               }
+            }
+
+            for (int dz = 0; dz < D1Dz; ++dz)
+            {
+               const double wz = (c == 2) ? Bct(dz,qz) : Bot(dz,qz);
+               for (int dy = 0; dy < D1Dy; ++dy)
+               {
+                  for (int dx = 0; dx < D1Dx; ++dx)
+                  {
+                     Y(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e) +=
+                        massXY[dy][dx] * wz;
+                  }
+               }
+            }
+
+            osc += D1Dx * D1Dy * D1Dz;
+         }  // loop c
+      }  // loop qz
+   }); // end of element loop
+}
+
+// PA H(curl)-H(div) Curl Apply Transpose 3D kernel
+template<int T_D1D = 0, int T_D1D_TEST = 0, int T_Q1D = 0>
+inline void PAHcurlHdivApplyTranspose3D(const int d1d,
+                                        const int d1dtest,
+                                        const int q1d,
+                                        const int NE,
+                                        const Array<double> &bo,
+                                        const Array<double> &bc,
+                                        const Array<double> &bot,
+                                        const Array<double> &bct,
+                                        const Array<double> &gct,
+                                        const Vector &pa_data,
+                                        const Vector &x,
+                                        Vector &y)
+{
+   MFEM_VERIFY(T_D1D || d1d <= HCURL_MAX_D1D, "Error: d1d > HCURL_MAX_D1D");
+   MFEM_VERIFY(T_D1D_TEST ||
+               d1dtest <= HCURL_MAX_D1D, "Error: d1dtest > HCURL_MAX_D1D");
+   MFEM_VERIFY(T_Q1D || q1d <= HCURL_MAX_Q1D, "Error: q1d > HCURL_MAX_Q1D");
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int D1Dtest = T_D1D_TEST ? T_D1D_TEST : d1dtest;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
+   auto Bc = Reshape(bc.Read(), Q1D, D1D);
+   auto Bot = Reshape(bot.Read(), D1Dtest-1, Q1D);
+   auto Bct = Reshape(bct.Read(), D1Dtest, Q1D);
+   auto Gct = Reshape(gct.Read(), D1D, Q1D);
+   auto op = Reshape(pa_data.Read(), Q1D, Q1D, Q1D, 6, NE);
+   auto X = Reshape(x.Read(), 3*(D1Dtest-1)*(D1Dtest-1)*D1Dtest, NE);
+   auto Y = Reshape(y.ReadWrite(), 3*(D1D-1)*D1D*D1D, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      // Using Piola transformations (\nabla\times u) F = 1/det(dF) dF \hat{\nabla}\times\hat{u}
+      // for u in H(curl) and w = (1 / det (dF)) dF \hat{w} for w in H(div), we get
+      // (\nabla\times u) \cdot w = 1/det(dF)^2 \hat{\nabla}\times\hat{u}^T dF^T dF \hat{w}
+      // If c = 0, \hat{\nabla}\times\hat{u} reduces to [0, (u_0)_{x_2}, -(u_0)_{x_1}]
+      // If c = 1, \hat{\nabla}\times\hat{u} reduces to [-(u_1)_{x_2}, 0, (u_1)_{x_0}]
+      // If c = 2, \hat{\nabla}\times\hat{u} reduces to [(u_2)_{x_1}, -(u_2)_{x_0}, 0]
+
+      constexpr int VDIM = 3;
+      constexpr int MD1D = T_D1D ? T_D1D :
+                           HCURL_MAX_D1D;  // Assuming HDIV_MAX_D1D <= HCURL_MAX_D1D
+      constexpr int MQ1D = T_Q1D ? T_Q1D : HCURL_MAX_Q1D;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int D1Dtest = T_D1D_TEST ? T_D1D_TEST : d1dtest;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+      double mass[MQ1D][MQ1D][MQ1D][VDIM];
+
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               for (int c = 0; c < VDIM; ++c)
+               {
+                  mass[qz][qy][qx][c] = 0.0;
+               }
+            }
+         }
+      }
+
+      int osc = 0;
+
+      for (int c = 0; c < VDIM; ++c)  // loop over x, y, z components
+      {
+         const int D1Dz = (c == 2) ? D1Dtest : D1Dtest - 1;
+         const int D1Dy = (c == 1) ? D1Dtest : D1Dtest - 1;
+         const int D1Dx = (c == 0) ? D1Dtest : D1Dtest - 1;
+
+         for (int dz = 0; dz < D1Dz; ++dz)
+         {
+            double massXY[MQ1D][MQ1D];
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  massXY[qy][qx] = 0.0;
+               }
+            }
+
+            for (int dy = 0; dy < D1Dy; ++dy)
+            {
+               double massX[MQ1D];
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  massX[qx] = 0.0;
+               }
+
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  const double t = X(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e);
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     massX[qx] += t * ((c == 0) ? Bc(qx,dx) : Bo(qx,dx));
+                  }
+               }
+
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  const double wy = (c == 1) ? Bc(qy,dy) : Bo(qy,dy);
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     const double wx = massX[qx];
+                     massXY[qy][qx] += wx * wy;
+                  }
+               }
+            }
+
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               const double wz = (c == 2) ? Bc(qz,dz) : Bo(qz,dz);
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     mass[qz][qy][qx][c] += massXY[qy][qx] * wz;
+                  }
+               }
+            }
+         }
+
+         osc += D1Dx * D1Dy * D1Dz;
+      }  // loop (c) over components
+
+      // Apply D operator.
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const double O11 = op(qx,qy,qz,0,e);
+               const double O12 = op(qx,qy,qz,1,e);
+               const double O13 = op(qx,qy,qz,2,e);
+               const double O22 = op(qx,qy,qz,3,e);
+               const double O23 = op(qx,qy,qz,4,e);
+               const double O33 = op(qx,qy,qz,5,e);
+               const double massX = mass[qz][qy][qx][0];
+               const double massY = mass[qz][qy][qx][1];
+               const double massZ = mass[qz][qy][qx][2];
+               mass[qz][qy][qx][0] = (O11*massX)+(O12*massY)+(O13*massZ);
+               mass[qz][qy][qx][1] = (O12*massX)+(O22*massY)+(O23*massZ);
+               mass[qz][qy][qx][2] = (O13*massX)+(O23*massY)+(O33*massZ);
+            }
+         }
+      }
+
+      // x component
+      osc = 0;
+      {
+         const int D1Dz = D1D;
+         const int D1Dy = D1D;
+         const int D1Dx = D1D - 1;
+
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            double gradXY12[MD1D][MD1D];
+            double gradXY21[MD1D][MD1D];
+
+            for (int dy = 0; dy < D1Dy; ++dy)
+            {
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  gradXY12[dy][dx] = 0.0;
+                  gradXY21[dy][dx] = 0.0;
+               }
+            }
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               double massX[MD1D][2];
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  for (int n = 0; n < 2; ++n)
+                  {
+                     massX[dx][n] = 0.0;
+                  }
+               }
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  for (int dx = 0; dx < D1Dx; ++dx)
+                  {
+                     const double wx = Bot(dx,qx);
+
+                     massX[dx][0] += wx * mass[qz][qy][qx][1];
+                     massX[dx][1] += wx * mass[qz][qy][qx][2];
+                  }
+               }
+               for (int dy = 0; dy < D1Dy; ++dy)
+               {
+                  const double wy = Bct(dy,qy);
+                  const double wDy = Gct(dy,qy);
+
+                  for (int dx = 0; dx < D1Dx; ++dx)
+                  {
+                     gradXY21[dy][dx] += massX[dx][0] * wy;
+                     gradXY12[dy][dx] += massX[dx][1] * wDy;
+                  }
+               }
+            }
+
+            for (int dz = 0; dz < D1Dz; ++dz)
+            {
+               const double wz = Bct(dz,qz);
+               const double wDz = Gct(dz,qz);
+               for (int dy = 0; dy < D1Dy; ++dy)
+               {
+                  for (int dx = 0; dx < D1Dx; ++dx)
+                  {
+                     // \hat{\nabla}\times\hat{u} is [0, (u_0)_{x_2}, -(u_0)_{x_1}]
+                     // (u_0)_{x_2} * (op * curl)_1 - (u_0)_{x_1} * (op * curl)_2
+                     Y(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc,
+                       e) += (gradXY21[dy][dx] * wDz) - (gradXY12[dy][dx] * wz);
+                  }
+               }
+            }
+         }  // loop qz
+
+         osc += D1Dx * D1Dy * D1Dz;
+      }
+
+      // y component
+      {
+         const int D1Dz = D1D;
+         const int D1Dy = D1D - 1;
+         const int D1Dx = D1D;
+
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            double gradXY02[MD1D][MD1D];
+            double gradXY20[MD1D][MD1D];
+
+            for (int dy = 0; dy < D1Dy; ++dy)
+            {
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  gradXY02[dy][dx] = 0.0;
+                  gradXY20[dy][dx] = 0.0;
+               }
+            }
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               double massY[MD1D][2];
+               for (int dy = 0; dy < D1Dy; ++dy)
+               {
+                  massY[dy][0] = 0.0;
+                  massY[dy][1] = 0.0;
+               }
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  for (int dy = 0; dy < D1Dy; ++dy)
+                  {
+                     const double wy = Bot(dy,qy);
+
+                     massY[dy][0] += wy * mass[qz][qy][qx][2];
+                     massY[dy][1] += wy * mass[qz][qy][qx][0];
+                  }
+               }
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  const double wx = Bct(dx,qx);
+                  const double wDx = Gct(dx,qx);
+
+                  for (int dy = 0; dy < D1Dy; ++dy)
+                  {
+                     gradXY02[dy][dx] += massY[dy][0] * wDx;
+                     gradXY20[dy][dx] += massY[dy][1] * wx;
+                  }
+               }
+            }
+
+            for (int dz = 0; dz < D1Dz; ++dz)
+            {
+               const double wz = Bct(dz,qz);
+               const double wDz = Gct(dz,qz);
+               for (int dy = 0; dy < D1Dy; ++dy)
+               {
+                  for (int dx = 0; dx < D1Dx; ++dx)
+                  {
+                     // \hat{\nabla}\times\hat{u} is [-(u_1)_{x_2}, 0, (u_1)_{x_0}]
+                     // -(u_1)_{x_2} * (op * curl)_0 + (u_1)_{x_0} * (op * curl)_2
+                     Y(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc,
+                       e) += (-gradXY20[dy][dx] * wDz) + (gradXY02[dy][dx] * wz);
+                  }
+               }
+            }
+         }  // loop qz
+
+         osc += D1Dx * D1Dy * D1Dz;
+      }
+
+      // z component
+      {
+         const int D1Dz = D1D - 1;
+         const int D1Dy = D1D;
+         const int D1Dx = D1D;
+
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            double gradYZ01[MD1D][MD1D];
+            double gradYZ10[MD1D][MD1D];
+
+            for (int dy = 0; dy < D1Dy; ++dy)
+            {
+               for (int dz = 0; dz < D1Dz; ++dz)
+               {
+                  gradYZ01[dz][dy] = 0.0;
+                  gradYZ10[dz][dy] = 0.0;
+               }
+            }
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               double massZ[MD1D][2];
+               for (int dz = 0; dz < D1Dz; ++dz)
+               {
+                  for (int n = 0; n < 2; ++n)
+                  {
+                     massZ[dz][n] = 0.0;
+                  }
+               }
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  for (int dz = 0; dz < D1Dz; ++dz)
+                  {
+                     const double wz = Bot(dz,qz);
+
+                     massZ[dz][0] += wz * mass[qz][qy][qx][0];
+                     massZ[dz][1] += wz * mass[qz][qy][qx][1];
+                  }
+               }
+               for (int dy = 0; dy < D1Dy; ++dy)
+               {
+                  const double wy = Bct(dy,qy);
+                  const double wDy = Gct(dy,qy);
+
+                  for (int dz = 0; dz < D1Dz; ++dz)
+                  {
+                     gradYZ01[dz][dy] += wy * massZ[dz][1];
+                     gradYZ10[dz][dy] += wDy * massZ[dz][0];
+                  }
+               }
+            }
+
+            for (int dx = 0; dx < D1Dx; ++dx)
+            {
+               const double wx = Bct(dx,qx);
+               const double wDx = Gct(dx,qx);
+
+               for (int dy = 0; dy < D1Dy; ++dy)
+               {
+                  for (int dz = 0; dz < D1Dz; ++dz)
+                  {
+                     // \hat{\nabla}\times\hat{u} is [(u_2)_{x_1}, -(u_2)_{x_0}, 0]
+                     // (u_2)_{x_1} * (op * curl)_0 - (u_2)_{x_0} * (op * curl)_1
+                     Y(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc,
+                       e) += (gradYZ10[dz][dy] * wx) - (gradYZ01[dz][dy] * wDx);
+                  }
+               }
+            }
+         }  // loop qx
+      }
+   }); // end of element loop
+}
+
+} // namespace internal
+
+} // namespace mfem
+
+#endif

fem/integ/bilininteg_hdiv_kernels.hpp

@@ -0,0 +1,715 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_BILININTEG_HDIV_KERNELS_HPP
+#define MFEM_BILININTEG_HDIV_KERNELS_HPP
+
+#include "../../config/config.hpp"
+#include "../../general/array.hpp"
+#include "../../general/forall.hpp"
+#include "../../linalg/dtensor.hpp"
+#include "../../linalg/vector.hpp"
+#include "../bilininteg.hpp"
+
+// Piola transformation in H(div): w = (1 / det (dF)) dF \hat{w}
+// div w = (1 / det (dF)) \hat{div} \hat{w}
+
+namespace mfem
+{
+
+namespace internal
+{
+
+// PA H(div) Mass Assemble 2D kernel
+void PAHdivMassSetup2D(const int Q1D,
+                       const int coeffDim,
+                       const int NE,
+                       const Array<double> &w,
+                       const Vector &j,
+                       Vector &coeff_,
+                       Vector &op);
+
+// PA H(div) Mass Assemble 3D kernel
+void PAHdivMassSetup3D(const int Q1D,
+                       const int coeffDim,
+                       const int NE,
+                       const Array<double> &w,
+                       const Vector &j,
+                       Vector &coeff_,
+                       Vector &op);
+
+// PA H(div) Mass Diagonal 2D kernel
+void PAHdivMassAssembleDiagonal2D(const int D1D,
+                                  const int Q1D,
+                                  const int NE,
+                                  const bool symmetric,
+                                  const Array<double> &Bo_,
+                                  const Array<double> &Bc_,
+                                  const Vector &op_,
+                                  Vector &diag_);
+
+// PA H(div) Mass Diagonal 3D kernel
+void PAHdivMassAssembleDiagonal3D(const int D1D,
+                                  const int Q1D,
+                                  const int NE,
+                                  const bool symmetric,
+                                  const Array<double> &Bo_,
+                                  const Array<double> &Bc_,
+                                  const Vector &op_,
+                                  Vector &diag_);
+
+void PAHdivMassApply(const int dim,
+                     const int D1D,
+                     const int Q1D,
+                     const int NE,
+                     const bool symmetric,
+                     const Array<double> &Bo,
+                     const Array<double> &Bc,
+                     const Array<double> &Bot,
+                     const Array<double> &Bct,
+                     const Vector &op,
+                     const Vector &x,
+                     Vector &y);
+
+// PA H(div) Mass Apply 2D kernel
+void PAHdivMassApply2D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const bool symmetric,
+                       const Array<double> &Bo_,
+                       const Array<double> &Bc_,
+                       const Array<double> &Bot_,
+                       const Array<double> &Bct_,
+                       const Vector &op_,
+                       const Vector &x_,
+                       Vector &y_);
+
+// PA H(div) Mass Apply 3D kernel
+void PAHdivMassApply3D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const bool symmetric,
+                       const Array<double> &Bo_,
+                       const Array<double> &Bc_,
+                       const Array<double> &Bot_,
+                       const Array<double> &Bct_,
+                       const Vector &op_,
+                       const Vector &x_,
+                       Vector &y_);
+
+// Shared memory PA H(div) Mass Apply 2D kernel
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAHdivMassApply2D(const int NE,
+                                  const bool symmetric,
+                                  const Array<double> &Bo_,
+                                  const Array<double> &Bc_,
+                                  const Array<double> &Bot_,
+                                  const Array<double> &Bct_,
+                                  const Vector &op_,
+                                  const Vector &x_,
+                                  Vector &y_,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
+{
+   MFEM_CONTRACT_VAR(Bot_);
+   MFEM_CONTRACT_VAR(Bct_);
+
+   static constexpr int VDIM = 2;
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto bo = Reshape(Bo_.Read(), Q1D, D1D-1);
+   const auto bc = Reshape(Bc_.Read(), Q1D, D1D);
+   const auto D = Reshape(op_.Read(), Q1D, Q1D, symmetric ? 3 : 4, NE);
+   const auto x = Reshape(x_.Read(), D1D*(D1D-1), VDIM, NE);
+   auto y = y_.ReadWrite();
+
+   mfem::forall_3D(NE, Q1D, Q1D, VDIM, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int tidz = MFEM_THREAD_ID(z);
+
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+      constexpr int MQ1 = T_Q1D ? T_Q1D : HDIV_MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : HDIV_MAX_D1D;
+      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
+
+      MFEM_SHARED double smo[MQ1*(MD1-1)];
+      DeviceMatrix Bo(smo, D1D-1, Q1D);
+
+      MFEM_SHARED double smc[MQ1*MD1];
+      DeviceMatrix Bc(smc, D1D, Q1D);
+
+      MFEM_SHARED double sm0[VDIM*MDQ*MDQ];
+      MFEM_SHARED double sm1[VDIM*MDQ*MDQ];
+      DeviceMatrix X(sm0, D1D*(D1D-1), VDIM);
+      DeviceCube QD(sm1, Q1D, D1D, VDIM);
+      DeviceCube QQ(sm0, Q1D, Q1D, VDIM);
+
+      // Load X, Bo and Bc into shared memory
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               if (qx < D1D && dy < (D1D-1)) { X(qx + dy*D1D,vd) = x(qx+dy*D1D,vd,e); }
+               if (tidz == 0)
+               {
+                  if (dy < (D1D-1)) { Bo(dy,qx) = bo(qx,dy); }
+                  Bc(dy,qx) = bc(qx,dy);
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // Apply B operator
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int nx = (vd == 0) ? D1D : D1D-1;
+         const int ny = (vd == 1) ? D1D : D1D-1;
+         DeviceCube Xxy(X, nx, ny, VDIM);
+         DeviceMatrix Bx = (vd == 0) ? Bc : Bo;
+         MFEM_FOREACH_THREAD(dy,y,ny)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double dq = 0.0;
+               for (int dx = 0; dx < nx; ++dx)
+               {
+                  dq += Xxy(dx,dy,vd) * Bx(dx,qx);
+               }
+               QD(qx,dy,vd) = dq;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int ny = (vd == 1) ? D1D : D1D-1;
+         DeviceMatrix By = (vd == 1) ? Bc : Bo;
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double qq = 0.0;
+               for (int dy = 0; dy < ny; ++dy)
+               {
+                  qq += QD(qx,dy,vd) * By(dy,qy);
+               }
+               QQ(qx,qy,vd) = qq;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // Apply D operator
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               const double Qx = QQ(qx,qy,0);
+               const double Qy = QQ(qx,qy,1);
+
+               const double D11 = D(qx,qy,0,e);
+               const double D12 = D(qx,qy,1,e);
+               const double D21 = symmetric ? D12 : D(qx,qy,2,e);
+               const double D22 = symmetric ? D(qx,qy,2,e) : D(qx,qy,3,e);
+
+               QQ(qx,qy,0) = D11*Qx + D12*Qy;
+               QQ(qx,qy,1) = D21*Qx + D22*Qy;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // Apply Bt operator
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int nx = (vd == 0) ? D1D : D1D-1;
+         DeviceMatrix Btx = (vd == 0) ? Bc : Bo;
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               double qd = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  qd += QQ(qx,qy,vd) * Btx(dx,qx);
+               }
+               QD(dx,qy,vd) = qd;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int nx = (vd == 0) ? D1D : D1D-1;
+         const int ny = (vd == 1) ? D1D : D1D-1;
+         DeviceMatrix Bty = (vd == 1) ? Bc : Bo;
+         DeviceTensor<4> Yxy(y, nx, ny, VDIM, NE);
+         MFEM_FOREACH_THREAD(dy,y,ny)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               double dd = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  dd += QD(dx,qy,vd) * Bty(dy,qy);
+               }
+               Yxy(dx,dy,vd,e) += dd;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+// Shared memory PA H(div) Mass Apply 3D kernel
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAHdivMassApply3D(const int NE,
+                                  const bool symmetric,
+                                  const Array<double> &Bo_,
+                                  const Array<double> &Bc_,
+                                  const Array<double> &Bot_,
+                                  const Array<double> &Bct_,
+                                  const Vector &op_,
+                                  const Vector &x_,
+                                  Vector &y_,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
+{
+   MFEM_CONTRACT_VAR(Bot_);
+   MFEM_CONTRACT_VAR(Bct_);
+
+   static constexpr int VDIM = 3;
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto bo = Reshape(Bo_.Read(), Q1D, D1D-1);
+   const auto bc = Reshape(Bc_.Read(), Q1D, D1D);
+   const auto D = Reshape(op_.Read(), Q1D, Q1D, Q1D, symmetric ? 6 : 9, NE);
+   const auto x = Reshape(x_.Read(), D1D*(D1D-1)*(D1D-1), VDIM, NE);
+   auto y = y_.ReadWrite();
+
+   mfem::forall_3D(NE, Q1D, Q1D, VDIM, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int tidz = MFEM_THREAD_ID(z);
+
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+      constexpr int MQ1 = T_Q1D ? T_Q1D : HDIV_MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : HDIV_MAX_D1D;
+      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
+
+      MFEM_SHARED double smo[MQ1*(MD1-1)];
+      DeviceMatrix Bo(smo, D1D-1, Q1D);
+
+      MFEM_SHARED double smc[MQ1*MD1];
+      DeviceMatrix Bc(smc, D1D, Q1D);
+
+      MFEM_SHARED double sm0[VDIM*MDQ*MDQ*MDQ];
+      MFEM_SHARED double sm1[VDIM*MDQ*MDQ*MDQ];
+      DeviceMatrix X(sm0, D1D*(D1D-1)*(D1D-1), VDIM);
+      DeviceTensor<4> QDD(sm1, Q1D, D1D, D1D, VDIM);
+      DeviceTensor<4> QQD(sm0, Q1D, Q1D, D1D, VDIM);
+      DeviceTensor<4> QQQ(sm1, Q1D, Q1D, Q1D, VDIM);
+      DeviceTensor<4> DQQ(sm0, D1D, Q1D, Q1D, VDIM);
+      DeviceTensor<4> DDQ(sm1, D1D, D1D, Q1D, VDIM);
+
+      // Load X into shared memory
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         MFEM_FOREACH_THREAD(dz,y,D1D-1)
+         {
+            MFEM_FOREACH_THREAD(dy,x,D1D-1)
+            {
+               MFEM_UNROLL(MD1)
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  X(dx+(dy+dz*(D1D-1))*D1D,vd) = x(dx+(dy+dz*(D1D-1))*D1D,vd,e);
+               }
+            }
+         }
+      }
+      // Load Bo and Bc into shared memory
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D-1)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               Bo(d,q) = bo(q,d);
+            }
+         }
+         MFEM_FOREACH_THREAD(d,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               Bc(d,q) = bc(q,d);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // Apply B operator
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int nx = (vd == 0) ? D1D : D1D-1;
+         const int ny = (vd == 1) ? D1D : D1D-1;
+         const int nz = (vd == 2) ? D1D : D1D-1;
+         DeviceTensor<4> Xxyz(X, nx, ny, nz, VDIM);
+         DeviceMatrix Bx = (vd == 0) ? Bc : Bo;
+         MFEM_FOREACH_THREAD(dy,y,ny)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double u[D1D];
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < nz; ++dz) { u[dz] = 0.0; }
+               MFEM_UNROLL(MD1)
+               for (int dx = 0; dx < nx; ++dx)
+               {
+                  MFEM_UNROLL(MD1)
+                  for (int dz = 0; dz < nz; ++dz)
+                  {
+                     u[dz] += Xxyz(dx,dy,dz,vd) * Bx(dx,qx);
+                  }
+               }
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < nz; ++dz) { QDD(qx,dy,dz,vd) = u[dz]; }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int ny = (vd == 1) ? D1D : D1D-1;
+         const int nz = (vd == 2) ? D1D : D1D-1;
+         DeviceMatrix By = (vd == 1) ? Bc : Bo;
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double u[D1D];
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < nz; ++dz) { u[dz] = 0.0; }
+               MFEM_UNROLL(MD1)
+               for (int dy = 0; dy < ny; ++dy)
+               {
+                  MFEM_UNROLL(MD1)
+                  for (int dz = 0; dz < nz; ++dz)
+                  {
+                     u[dz] += QDD(qx,dy,dz,vd) * By(dy,qy);
+                  }
+               }
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < nz; ++dz) { QQD(qx,qy,dz,vd) = u[dz]; }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int nz = (vd == 2) ? D1D : D1D-1;
+         DeviceMatrix Bz = (vd == 2) ? Bc : Bo;
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double u[Q1D];
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz) { u[qz] = 0.0; }
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < nz; ++dz)
+               {
+                  MFEM_UNROLL(MQ1)
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     u[qz] += QQD(qx,qy,dz,vd) * Bz(dz,qz);
+                  }
+               }
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz) { QQQ(qx,qy,qz,vd) = u[qz]; }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // Apply D operator
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  const double Qx = QQQ(qx,qy,qz,0);
+                  const double Qy = QQQ(qx,qy,qz,1);
+                  const double Qz = QQQ(qx,qy,qz,2);
+
+                  const double D11 = D(qx,qy,qz,0,e);
+                  const double D12 = D(qx,qy,qz,1,e);
+                  const double D13 = D(qx,qy,qz,2,e);
+                  const double D21 = symmetric ? D12 : D(qx,qy,qz,3,e);
+                  const double D22 = symmetric ? D(qx,qy,qz,3,e) : D(qx,qy,qz,4,e);
+                  const double D23 = symmetric ? D(qx,qy,qz,4,e) : D(qx,qy,qz,5,e);
+                  const double D31 = symmetric ? D13 : D(qx,qy,qz,6,e);
+                  const double D32 = symmetric ? D23 : D(qx,qy,qz,7,e);
+                  const double D33 = symmetric ? D(qx,qy,qz,5,e) : D(qx,qy,qz,8,e);
+
+                  QQQ(qx,qy,qz,0) = D11*Qx + D12*Qy + D13*Qz;
+                  QQQ(qx,qy,qz,1) = D21*Qx + D22*Qy + D23*Qz;
+                  QQQ(qx,qy,qz,2) = D31*Qx + D32*Qy + D33*Qz;
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // Apply Bt operator
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int nx = (vd == 0) ? D1D : D1D-1;
+         DeviceMatrix Btx = (vd == 0) ? Bc : Bo;
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               double u[Q1D];
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz) { u[qz] = 0.0; }
+               MFEM_UNROLL(MQ1)
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  MFEM_UNROLL(MQ1)
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     u[qz] += QQQ(qx,qy,qz,vd) * Btx(dx,qx);
+                  }
+               }
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz) { DQQ(dx,qy,qz,vd) = u[qz]; }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int nx = (vd == 0) ? D1D : D1D-1;
+         const int ny = (vd == 1) ? D1D : D1D-1;
+         DeviceMatrix Bty = (vd == 1) ? Bc : Bo;
+         MFEM_FOREACH_THREAD(dy,y,ny)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               double u[Q1D];
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz) { u[qz] = 0.0; }
+               MFEM_UNROLL(MQ1)
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  MFEM_UNROLL(MQ1)
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     u[qz] += DQQ(dx,qy,qz,vd) * Bty(dy,qy);
+                  }
+               }
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz) { DDQ(dx,dy,qz,vd) = u[qz]; }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int nx = (vd == 0) ? D1D : D1D-1;
+         const int ny = (vd == 1) ? D1D : D1D-1;
+         const int nz = (vd == 2) ? D1D : D1D-1;
+         DeviceTensor<5> Yxyz(y, nx, ny, nz, VDIM, NE);
+         DeviceMatrix Btz = (vd == 2) ? Bc : Bo;
+         MFEM_FOREACH_THREAD(dy,y,ny)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               double u[D1D];
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < nz; ++dz) { u[dz] = 0.0; }
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  MFEM_UNROLL(MD1)
+                  for (int dz = 0; dz < nz; ++dz)
+                  {
+                     u[dz] += DDQ(dx,dy,qz,vd) * Btz(dz,qz);
+                  }
+               }
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < nz; ++dz) { Yxyz(dx,dy,dz,vd,e) += u[dz]; }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+// PA H(div) div-div Assemble 2D kernel
+void PADivDivSetup2D(const int Q1D,
+                     const int NE,
+                     const Array<double> &w,
+                     const Vector &j,
+                     Vector &coeff_,
+                     Vector &op);
+
+// PA H(div) div-div Assemble 3D kernel
+void PADivDivSetup3D(const int Q1D,
+                     const int NE,
+                     const Array<double> &w,
+                     const Vector &j,
+                     Vector &coeff_,
+                     Vector &op);
+
+// PA H(div) div-div Diagonal 2D kernel
+void PADivDivAssembleDiagonal2D(const int D1D,
+                                const int Q1D,
+                                const int NE,
+                                const Array<double> &Bo_,
+                                const Array<double> &Gc_,
+                                const Vector &op_,
+                                Vector &diag_);
+
+// PA H(div) div-div Diagonal 3D kernel
+void PADivDivAssembleDiagonal3D(const int D1D,
+                                const int Q1D,
+                                const int NE,
+                                const Array<double> &Bo_,
+                                const Array<double> &Gc_,
+                                const Vector &op_,
+                                Vector &diag_);
+
+// PA H(div) div-div Apply 2D kernel
+void PADivDivApply2D(const int D1D,
+                     const int Q1D,
+                     const int NE,
+                     const Array<double> &Bo_,
+                     const Array<double> &Gc_,
+                     const Array<double> &Bot_,
+                     const Array<double> &Gct_,
+                     const Vector &op_,
+                     const Vector &x_,
+                     Vector &y_);
+
+// PA H(div) div-div Apply 3D kernel
+void PADivDivApply3D(const int D1D,
+                     const int Q1D,
+                     const int NE,
+                     const Array<double> &Bo_,
+                     const Array<double> &Gc_,
+                     const Array<double> &Bot_,
+                     const Array<double> &Gct_,
+                     const Vector &op_,
+                     const Vector &x_,
+                     Vector &y_);
+
+// PA H(div)-L2 Assemble 2D kernel
+void PAHdivL2Setup2D(const int Q1D,
+                     const int NE,
+                     const Array<double> &w,
+                     Vector &coeff_,
+                     Vector &op);
+
+// PA H(div)-L2 Assemble 3D kernel
+void PAHdivL2Setup3D(const int Q1D,
+                     const int NE,
+                     const Array<double> &w,
+                     Vector &coeff_,
+                     Vector &op);
+
+// PA H(div)-L2 Diagonal 2D kernel
+void PAHdivL2AssembleDiagonal_ADAt_2D(const int D1D,
+                                      const int Q1D,
+                                      const int L2D1D,
+                                      const int NE,
+                                      const Array<double> &L2Bo_,
+                                      const Array<double> &Gct_,
+                                      const Array<double> &Bot_,
+                                      const Vector &op_,
+                                      const Vector &D_,
+                                      Vector &diag_);
+
+// PA H(div)-L2 Diagonal 3D kernel
+void PAHdivL2AssembleDiagonal_ADAt_3D(const int D1D,
+                                      const int Q1D,
+                                      const int L2D1D,
+                                      const int NE,
+                                      const Array<double> &L2Bo_,
+                                      const Array<double> &Gct_,
+                                      const Array<double> &Bot_,
+                                      const Vector &op_,
+                                      const Vector &D_,
+                                      Vector &diag_);
+
+// PA H(div)-L2 Apply 2D kernel
+void PAHdivL2Apply2D(const int D1D,
+                     const int Q1D,
+                     const int L2D1D,
+                     const int NE,
+                     const Array<double> &Bo_,
+                     const Array<double> &Gc_,
+                     const Array<double> &L2Bot_,
+                     const Vector &op_,
+                     const Vector &x_,
+                     Vector &y_);
+
+// PA H(div)-L2 Apply Transpose 2D kernel
+void PAHdivL2ApplyTranspose2D(const int D1D,
+                              const int Q1D,
+                              const int L2D1D,
+                              const int NE,
+                              const Array<double> &L2Bo_,
+                              const Array<double> &Gct_,
+                              const Array<double> &Bot_,
+                              const Vector &op_,
+                              const Vector &x_,
+                              Vector &y_);
+
+// PA H(div)-L2 Apply 3D kernel
+void PAHdivL2Apply3D(const int D1D,
+                     const int Q1D,
+                     const int L2D1D,
+                     const int NE,
+                     const Array<double> &Bo_,
+                     const Array<double> &Gc_,
+                     const Array<double> &L2Bot_,
+                     const Vector &op_,
+                     const Vector &x_,
+                     Vector &y_);
+
+// PA H(div)-L2 Apply Transpose 3D kernel
+void PAHdivL2ApplyTranspose3D(const int D1D,
+                              const int Q1D,
+                              const int L2D1D,
+                              const int NE,
+                              const Array<double> &L2Bo_,
+                              const Array<double> &Gct_,
+                              const Array<double> &Bot_,
+                              const Vector &op_,
+                              const Vector &x_,
+                              Vector &y_);
+
+} // namespace internal
+
+} // namespace mfem
+
+#endif

fem/integ/bilininteg_mass_ea.cpp

@@ -9,9 +9,9 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
+#include "../../general/forall.hpp"
+#include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
 
 namespace mfem
 {
@@ -32,7 +32,7 @@ static void EAMassAssemble1D(const int NE,
    auto B = Reshape(basis.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, NE);
    auto M = Reshape(eadata.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
+   mfem::forall_2D(NE, D1D, D1D, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -82,7 +82,7 @@ static void EAMassAssemble2D(const int NE,
    auto B = Reshape(basis.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, Q1D, NE);
    auto M = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
+   mfem::forall_2D(NE, D1D, D1D, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -154,20 +154,51 @@ static void EAMassAssemble3D(const int NE,
    auto B = Reshape(basis.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, Q1D, Q1D, NE);
    auto M = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, D1D, D1D, D1D,
+   mfem::forall_3D(NE, D1D, D1D, D1D, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
       constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      double r_B[MQ1][MD1];
-      for (int d = 0; d < D1D; d++)
+      constexpr int DQ = T_D1D * T_Q1D;
+
+      // For quadratic and lower it's better to use registers but for higher-order you start to
+      // spill and it's better to use shared memory
+      constexpr bool USE_REG = DQ != 0 && DQ <= 12;
+      constexpr int MD1r = USE_REG ? MD1 : 1;
+      constexpr int MQ1r = USE_REG ? MQ1 : 1;
+      constexpr int MD1s = USE_REG ? 1 : MD1;
+      constexpr int MQ1s = USE_REG ? 1 : MQ1;
+
+      MFEM_SHARED double s_B[MQ1s][MD1s];
+      double r_B[MQ1r][MD1r];
+      double (*l_B)[MD1] = nullptr;
+      if (USE_REG)
       {
-         for (int q = 0; q < Q1D; q++)
+         for (int d = 0; d < D1D; d++)
          {
-            r_B[q][d] = B(q,d);
+            for (int q = 0; q < Q1D; q++)
+            {
+               r_B[q][d] = B(q,d);
+            }
          }
+         l_B = (double (*)[MD1])r_B;
       }
+      else
+      {
+         if (MFEM_THREAD_ID(z) == 0)
+         {
+            MFEM_FOREACH_THREAD(d,x,D1D)
+            {
+               MFEM_FOREACH_THREAD(q,y,Q1D)
+               {
+                  s_B[q][d] = B(q,d);
+               }
+            }
+         }
+         l_B = (double (*)[MD1])s_B;
+      }
+
       MFEM_SHARED double s_D[MQ1][MQ1][MQ1];
       MFEM_FOREACH_THREAD(k1,x,Q1D)
       {
@@ -199,9 +230,9 @@ static void EAMassAssemble3D(const int NE,
                            {
                               for (int k3 = 0; k3 < Q1D; ++k3)
                               {
-                                 val += r_B[k1][i1] * r_B[k1][j1]
-                                        * r_B[k2][i2] * r_B[k2][j2]
-                                        * r_B[k3][i3] * r_B[k3][j3]
+                                 val += l_B[k1][i1] * l_B[k1][j1]
+                                        * l_B[k2][i2] * l_B[k2][j2]
+                                        * l_B[k3][i3] * l_B[k3][j3]
                                         * s_D[k1][k2][k3];
                               }
                            }

fem/integ/bilininteg_mass_kernels.cpp

@@ -0,0 +1,333 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "bilininteg_mass_kernels.hpp"
+
+namespace mfem
+{
+
+namespace internal
+{
+
+// PA Mass Diagonal 1D kernel
+static void PAMassAssembleDiagonal1D(const int NE,
+                                     const Array<double> &b,
+                                     const Vector &d,
+                                     Vector &y,
+                                     const int D1D,
+                                     const int Q1D)
+{
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto D = Reshape(d.Read(), Q1D, NE);
+   auto Y = Reshape(y.ReadWrite(), D1D, NE);
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      for (int dx = 0; dx < D1D; ++dx)
+      {
+         Y(dx, e) = 0.0;
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            Y(dx, e) += B(qx, dx) * B(qx, dx) * D(qx, e);
+         }
+      }
+   });
+}
+
+void PAMassAssembleDiagonal(const int dim, const int D1D,
+                            const int Q1D, const int NE,
+                            const Array<double> &B,
+                            const Vector &D,
+                            Vector &Y)
+{
+   if (dim == 1)
+   {
+      return PAMassAssembleDiagonal1D(NE,B,D,Y,D1D,Q1D);
+   }
+   else if (dim == 2)
+   {
+      switch ((D1D << 4 ) | Q1D)
+      {
+         case 0x22: return SmemPAMassAssembleDiagonal2D<2,2,16>(NE,B,D,Y);
+         case 0x33: return SmemPAMassAssembleDiagonal2D<3,3,16>(NE,B,D,Y);
+         case 0x44: return SmemPAMassAssembleDiagonal2D<4,4,8>(NE,B,D,Y);
+         case 0x55: return SmemPAMassAssembleDiagonal2D<5,5,8>(NE,B,D,Y);
+         case 0x66: return SmemPAMassAssembleDiagonal2D<6,6,4>(NE,B,D,Y);
+         case 0x77: return SmemPAMassAssembleDiagonal2D<7,7,4>(NE,B,D,Y);
+         case 0x88: return SmemPAMassAssembleDiagonal2D<8,8,2>(NE,B,D,Y);
+         case 0x99: return SmemPAMassAssembleDiagonal2D<9,9,2>(NE,B,D,Y);
+         default:   return PAMassAssembleDiagonal2D(NE,B,D,Y,D1D,Q1D);
+      }
+   }
+   else if (dim == 3)
+   {
+      switch ((D1D << 4 ) | Q1D)
+      {
+         case 0x23: return SmemPAMassAssembleDiagonal3D<2,3>(NE,B,D,Y);
+         case 0x24: return SmemPAMassAssembleDiagonal3D<2,4>(NE,B,D,Y);
+         case 0x26: return SmemPAMassAssembleDiagonal3D<2,6>(NE,B,D,Y);
+         case 0x34: return SmemPAMassAssembleDiagonal3D<3,4>(NE,B,D,Y);
+         case 0x35: return SmemPAMassAssembleDiagonal3D<3,5>(NE,B,D,Y);
+         case 0x45: return SmemPAMassAssembleDiagonal3D<4,5>(NE,B,D,Y);
+         case 0x48: return SmemPAMassAssembleDiagonal3D<4,8>(NE,B,D,Y);
+         case 0x56: return SmemPAMassAssembleDiagonal3D<5,6>(NE,B,D,Y);
+         case 0x67: return SmemPAMassAssembleDiagonal3D<6,7>(NE,B,D,Y);
+         case 0x78: return SmemPAMassAssembleDiagonal3D<7,8>(NE,B,D,Y);
+         case 0x89: return SmemPAMassAssembleDiagonal3D<8,9>(NE,B,D,Y);
+         default:   return PAMassAssembleDiagonal3D(NE,B,D,Y,D1D,Q1D);
+      }
+   }
+   MFEM_ABORT("Unknown kernel.");
+}
+
+#ifdef MFEM_USE_OCCA
+void OccaPAMassApply2D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const Array<double> &B,
+                       const Array<double> &Bt,
+                       const Vector &D,
+                       const Vector &X,
+                       Vector &Y)
+{
+   occa::properties props;
+   props["defines/D1D"] = D1D;
+   props["defines/Q1D"] = Q1D;
+   const occa::memory o_B = OccaMemoryRead(B.GetMemory(), B.Size());
+   const occa::memory o_Bt = OccaMemoryRead(Bt.GetMemory(), Bt.Size());
+   const occa::memory o_D = OccaMemoryRead(D.GetMemory(), D.Size());
+   const occa::memory o_X = OccaMemoryRead(X.GetMemory(), X.Size());
+   occa::memory o_Y = OccaMemoryReadWrite(Y.GetMemory(), Y.Size());
+   const occa_id_t id = std::make_pair(D1D,Q1D);
+   if (!Device::Allows(Backend::OCCA_CUDA))
+   {
+      static occa_kernel_t OccaMassApply2D_cpu;
+      if (OccaMassApply2D_cpu.find(id) == OccaMassApply2D_cpu.end())
+      {
+         const occa::kernel MassApply2D_CPU =
+            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
+                                        "MassApply2D_CPU", props);
+         OccaMassApply2D_cpu.emplace(id, MassApply2D_CPU);
+      }
+      OccaMassApply2D_cpu.at(id)(NE, o_B, o_Bt, o_D, o_X, o_Y);
+   }
+   else
+   {
+      static occa_kernel_t OccaMassApply2D_gpu;
+      if (OccaMassApply2D_gpu.find(id) == OccaMassApply2D_gpu.end())
+      {
+         const occa::kernel MassApply2D_GPU =
+            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
+                                        "MassApply2D_GPU", props);
+         OccaMassApply2D_gpu.emplace(id, MassApply2D_GPU);
+      }
+      OccaMassApply2D_gpu.at(id)(NE, o_B, o_Bt, o_D, o_X, o_Y);
+   }
+}
+
+void OccaPAMassApply3D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const Array<double> &B,
+                       const Array<double> &Bt,
+                       const Vector &D,
+                       const Vector &X,
+                       Vector &Y)
+{
+   occa::properties props;
+   props["defines/D1D"] = D1D;
+   props["defines/Q1D"] = Q1D;
+   const occa::memory o_B = OccaMemoryRead(B.GetMemory(), B.Size());
+   const occa::memory o_Bt = OccaMemoryRead(Bt.GetMemory(), Bt.Size());
+   const occa::memory o_D = OccaMemoryRead(D.GetMemory(), D.Size());
+   const occa::memory o_X = OccaMemoryRead(X.GetMemory(), X.Size());
+   occa::memory o_Y = OccaMemoryReadWrite(Y.GetMemory(), Y.Size());
+   const occa_id_t id = std::make_pair(D1D,Q1D);
+   if (!Device::Allows(Backend::OCCA_CUDA))
+   {
+      static occa_kernel_t OccaMassApply3D_cpu;
+      if (OccaMassApply3D_cpu.find(id) == OccaMassApply3D_cpu.end())
+      {
+         const occa::kernel MassApply3D_CPU =
+            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
+                                        "MassApply3D_CPU", props);
+         OccaMassApply3D_cpu.emplace(id, MassApply3D_CPU);
+      }
+      OccaMassApply3D_cpu.at(id)(NE, o_B, o_Bt, o_D, o_X, o_Y);
+   }
+   else
+   {
+      static occa_kernel_t OccaMassApply3D_gpu;
+      if (OccaMassApply3D_gpu.find(id) == OccaMassApply3D_gpu.end())
+      {
+         const occa::kernel MassApply3D_GPU =
+            mfem::OccaDev().buildKernel("occa://mfem/fem/occa.okl",
+                                        "MassApply3D_GPU", props);
+         OccaMassApply3D_gpu.emplace(id, MassApply3D_GPU);
+      }
+      OccaMassApply3D_gpu.at(id)(NE, o_B, o_Bt, o_D, o_X, o_Y);
+   }
+}
+#endif // MFEM_USE_OCCA
+
+MFEM_HOST_DEVICE inline
+void PAMassApply1D_Element(const int e,
+                           const int NE,
+                           const double *b_,
+                           const double *bt_,
+                           const double *d_,
+                           const double *x_,
+                           double *y_,
+                           const int d1d = 0,
+                           const int q1d = 0)
+{
+   const int D1D = d1d;
+   const int Q1D = q1d;
+   auto B = ConstDeviceMatrix(b_, Q1D, D1D);
+   auto Bt = ConstDeviceMatrix(bt_, D1D, Q1D);
+   auto D = ConstDeviceMatrix(d_, Q1D, NE);
+   auto X = ConstDeviceMatrix(x_, D1D, NE);
+   auto Y = DeviceMatrix(y_, D1D, NE);
+
+   constexpr int max_Q1D = MAX_Q1D;
+   double XQ[max_Q1D];
+   for (int qx = 0; qx < Q1D; ++qx)
+   {
+      XQ[qx] = 0.0;
+   }
+   for (int dx = 0; dx < D1D; ++dx)
+   {
+      const double s = X(dx,e);
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         XQ[qx] += B(qx,dx)*s;
+      }
+   }
+   for (int qx = 0; qx < Q1D; ++qx)
+   {
+      const double q = XQ[qx]*D(qx,e);
+      for (int dx = 0; dx < D1D; ++dx)
+      {
+         Y(dx,e) += Bt(dx,qx) * q;
+      }
+   }
+}
+
+// PA Mass Apply 1D kernel
+static void PAMassApply1D(const int NE,
+                          const Array<double> &b_,
+                          const Array<double> &bt_,
+                          const Vector &d_,
+                          const Vector &x_,
+                          Vector &y_,
+                          const int d1d = 0,
+                          const int q1d = 0)
+{
+   MFEM_VERIFY(d1d <= MAX_D1D, "");
+   MFEM_VERIFY(q1d <= MAX_Q1D, "");
+
+   const auto B = b_.Read();
+   const auto Bt = bt_.Read();
+   const auto D = d_.Read();
+   const auto X = x_.Read();
+   auto Y = y_.ReadWrite();
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      internal::PAMassApply1D_Element(e, NE, B, Bt, D, X, Y, d1d, q1d);
+   });
+}
+
+void PAMassApply(const int dim,
+                 const int D1D,
+                 const int Q1D,
+                 const int NE,
+                 const Array<double> &B,
+                 const Array<double> &Bt,
+                 const Vector &D,
+                 const Vector &X,
+                 Vector &Y)
+{
+#ifdef MFEM_USE_OCCA
+   if (DeviceCanUseOcca())
+   {
+      if (dim == 2)
+      {
+         return OccaPAMassApply2D(D1D,Q1D,NE,B,Bt,D,X,Y);
+      }
+      if (dim == 3)
+      {
+         return OccaPAMassApply3D(D1D,Q1D,NE,B,Bt,D,X,Y);
+      }
+      MFEM_ABORT("OCCA PA Mass Apply unknown kernel!");
+   }
+#endif // MFEM_USE_OCCA
+   const int id = (D1D << 4) | Q1D;
+
+   if (dim == 1)
+   {
+      return PAMassApply1D(NE,B,Bt,D,X,Y,D1D,Q1D);
+   }
+   else if (dim == 2)
+   {
+      switch (id)
+      {
+         case 0x22: return SmemPAMassApply2D<2,2,16>(NE,B,Bt,D,X,Y);
+         case 0x24: return SmemPAMassApply2D<2,4,16>(NE,B,Bt,D,X,Y);
+         case 0x33: return SmemPAMassApply2D<3,3,16>(NE,B,Bt,D,X,Y);
+         case 0x34: return SmemPAMassApply2D<3,4,16>(NE,B,Bt,D,X,Y);
+         case 0x35: return SmemPAMassApply2D<3,5,16>(NE,B,Bt,D,X,Y);
+         case 0x36: return SmemPAMassApply2D<3,6,16>(NE,B,Bt,D,X,Y);
+         case 0x44: return SmemPAMassApply2D<4,4,8>(NE,B,Bt,D,X,Y);
+         case 0x46: return SmemPAMassApply2D<4,6,8>(NE,B,Bt,D,X,Y);
+         case 0x48: return SmemPAMassApply2D<4,8,4>(NE,B,Bt,D,X,Y);
+         case 0x55: return SmemPAMassApply2D<5,5,8>(NE,B,Bt,D,X,Y);
+         case 0x57: return SmemPAMassApply2D<5,7,8>(NE,B,Bt,D,X,Y);
+         case 0x58: return SmemPAMassApply2D<5,8,2>(NE,B,Bt,D,X,Y);
+         case 0x66: return SmemPAMassApply2D<6,6,4>(NE,B,Bt,D,X,Y);
+         case 0x77: return SmemPAMassApply2D<7,7,4>(NE,B,Bt,D,X,Y);
+         case 0x88: return SmemPAMassApply2D<8,8,2>(NE,B,Bt,D,X,Y);
+         case 0x99: return SmemPAMassApply2D<9,9,2>(NE,B,Bt,D,X,Y);
+         default:   return PAMassApply2D(NE,B,Bt,D,X,Y,D1D,Q1D);
+      }
+   }
+   else if (dim == 3)
+   {
+      switch (id)
+      {
+         case 0x22: return SmemPAMassApply3D<2,2>(NE,B,Bt,D,X,Y);
+         case 0x23: return SmemPAMassApply3D<2,3>(NE,B,Bt,D,X,Y);
+         case 0x24: return SmemPAMassApply3D<2,4>(NE,B,Bt,D,X,Y);
+         case 0x26: return SmemPAMassApply3D<2,6>(NE,B,Bt,D,X,Y);
+         case 0x34: return SmemPAMassApply3D<3,4>(NE,B,Bt,D,X,Y);
+         case 0x35: return SmemPAMassApply3D<3,5>(NE,B,Bt,D,X,Y);
+         case 0x36: return SmemPAMassApply3D<3,6>(NE,B,Bt,D,X,Y);
+         case 0x37: return SmemPAMassApply3D<3,7>(NE,B,Bt,D,X,Y);
+         case 0x45: return SmemPAMassApply3D<4,5>(NE,B,Bt,D,X,Y);
+         case 0x46: return SmemPAMassApply3D<4,6>(NE,B,Bt,D,X,Y);
+         case 0x48: return SmemPAMassApply3D<4,8>(NE,B,Bt,D,X,Y);
+         case 0x56: return SmemPAMassApply3D<5,6>(NE,B,Bt,D,X,Y);
+         case 0x58: return SmemPAMassApply3D<5,8>(NE,B,Bt,D,X,Y);
+         case 0x67: return SmemPAMassApply3D<6,7>(NE,B,Bt,D,X,Y);
+         case 0x78: return SmemPAMassApply3D<7,8>(NE,B,Bt,D,X,Y);
+         case 0x89: return SmemPAMassApply3D<8,9>(NE,B,Bt,D,X,Y);
+         case 0x9A: return SmemPAMassApply3D<9,10>(NE,B,Bt,D,X,Y);
+         default:   return PAMassApply3D(NE,B,Bt,D,X,Y,D1D,Q1D);
+      }
+   }
+   mfem::out << "Unknown kernel 0x" << std::hex << id << std::endl;
+   MFEM_ABORT("Unknown kernel.");
+}
+
+} // namespace internal
+
+} // namespace mfem

fem/integ/bilininteg_mass_kernels.hpp

@@ -9,12 +9,15 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#ifndef MFEM_BILININTEG_MASS_PA_HPP
-#define MFEM_BILININTEG_MASS_PA_HPP
+#ifndef MFEM_BILININTEG_MASS_KERNELS_HPP
+#define MFEM_BILININTEG_MASS_KERNELS_HPP
 
-#include "../config/config.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/dtensor.hpp"
+#include "../../config/config.hpp"
+#include "../../general/array.hpp"
+#include "../../general/forall.hpp"
+#include "../../linalg/dtensor.hpp"
+#include "../../linalg/vector.hpp"
+#include "../bilininteg.hpp"
 
 namespace mfem
 {
@@ -22,6 +25,315 @@ namespace mfem
 namespace internal
 {
 
+void PAMassAssembleDiagonal(const int dim, const int D1D,
+                            const int Q1D, const int NE,
+                            const Array<double> &B,
+                            const Vector &D,
+                            Vector &Y);
+
+// PA Mass Diagonal 2D kernel
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void PAMassAssembleDiagonal2D(const int NE,
+                                     const Array<double> &b,
+                                     const Vector &d,
+                                     Vector &y,
+                                     const int d1d = 0,
+                                     const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto D = Reshape(d.Read(), Q1D, Q1D, NE);
+   auto Y = Reshape(y.ReadWrite(), D1D, D1D, NE);
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      double QD[MQ1][MD1];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            QD[qx][dy] = 0.0;
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               QD[qx][dy] += B(qy, dy) * B(qy, dy) * D(qx, qy, e);
+            }
+         }
+      }
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               Y(dx,dy,e) += B(qx, dx) * B(qx, dx) * QD[qx][dy];
+            }
+         }
+      }
+   });
+}
+
+// Shared memory PA Mass Diagonal 2D kernel
+template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
+inline void SmemPAMassAssembleDiagonal2D(const int NE,
+                                         const Array<double> &b_,
+                                         const Vector &d_,
+                                         Vector &y_,
+                                         const int d1d = 0,
+                                         const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
+   auto b = Reshape(b_.Read(), Q1D, D1D);
+   auto D = Reshape(d_.Read(), Q1D, Q1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
+   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int tidz = MFEM_THREAD_ID(z);
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      MFEM_SHARED double B[MQ1][MD1];
+      MFEM_SHARED double QDZ[NBZ][MQ1][MD1];
+      double (*QD)[MD1] = (double (*)[MD1])(QDZ + tidz);
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B[q][d] = b(q,d);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            QD[qx][dy] = 0.0;
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               QD[qx][dy] += B[qy][dy] * B[qy][dy] * D(qx, qy, e);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               // might need absolute values on next line
+               Y(dx,dy,e) += B[qx][dx] * B[qx][dx] * QD[qx][dy];
+            }
+         }
+      }
+   });
+}
+
+// PA Mass Diagonal 3D kernel
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void PAMassAssembleDiagonal3D(const int NE,
+                                     const Array<double> &b,
+                                     const Vector &d,
+                                     Vector &y,
+                                     const int d1d = 0,
+                                     const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto D = Reshape(d.Read(), Q1D, Q1D, Q1D, NE);
+   auto Y = Reshape(y.ReadWrite(), D1D, D1D, D1D, NE);
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      double QQD[MQ1][MQ1][MD1];
+      double QDD[MQ1][MD1][MD1];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               QQD[qx][qy][dz] = 0.0;
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  QQD[qx][qy][dz] += B(qz, dz) * B(qz, dz) * D(qx, qy, qz, e);
+               }
+            }
+         }
+      }
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               QDD[qx][dy][dz] = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  QDD[qx][dy][dz] += B(qy, dy) * B(qy, dy) * QQD[qx][qy][dz];
+               }
+            }
+         }
+      }
+      for (int dz = 0; dz < D1D; ++dz)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               double t = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  t += B(qx, dx) * B(qx, dx) * QDD[qx][dy][dz];
+               }
+               Y(dx, dy, dz, e) += t;
+            }
+         }
+      }
+   });
+}
+
+// Shared memory PA Mass Diagonal 3D kernel
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAMassAssembleDiagonal3D(const int NE,
+                                         const Array<double> &b_,
+                                         const Vector &d_,
+                                         Vector &y_,
+                                         const int d1d = 0,
+                                         const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
+   auto b = Reshape(b_.Read(), Q1D, D1D);
+   auto D = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
+   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int tidz = MFEM_THREAD_ID(z);
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      MFEM_SHARED double B[MQ1][MD1];
+      MFEM_SHARED double QQD[MQ1][MQ1][MD1];
+      MFEM_SHARED double QDD[MQ1][MD1][MD1];
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B[q][d] = b(q,d);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(dz,z,D1D)
+            {
+               QQD[qx][qy][dz] = 0.0;
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  QQD[qx][qy][dz] += B[qz][dz] * B[qz][dz] * D(qx, qy, qz, e);
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               QDD[qx][dy][dz] = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  QDD[qx][dy][dz] += B[qy][dy] * B[qy][dy] * QQD[qx][qy][dz];
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dz,z,D1D)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               double t = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  t += B[qx][dx] * B[qx][dx] * QDD[qx][dy][dz];
+               }
+               Y(dx, dy, dz, e) += t;
+            }
+         }
+      }
+   });
+}
+
+void PAMassApply(const int dim,
+                 const int D1D,
+                 const int Q1D,
+                 const int NE,
+                 const Array<double> &B,
+                 const Array<double> &Bt,
+                 const Vector &D,
+                 const Vector &X,
+                 Vector &Y);
+
+#ifdef MFEM_USE_OCCA
+// OCCA PA Mass Apply 2D kernel
+void OccaPAMassApply2D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const Array<double> &B,
+                       const Array<double> &Bt,
+                       const Vector &D,
+                       const Vector &X,
+                       Vector &Y);
+
+// OCCA PA Mass Apply 3D kernel
+void OccaPAMassApply3D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const Array<double> &B,
+                       const Array<double> &Bt,
+                       const Vector &D,
+                       const Vector &X,
+                       Vector &Y);
+#endif // MFEM_USE_OCCA
+
 template <bool ACCUMULATE = true>
 MFEM_HOST_DEVICE inline
 void PAMassApply2D_Element(const int e,
@@ -625,6 +937,116 @@ void SmemPAMassApply3D_Element(const int e,
    MFEM_SYNC_THREAD;
 }
 
+// PA Mass Apply 2D kernel
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void PAMassApply2D(const int NE,
+                          const Array<double> &b_,
+                          const Array<double> &bt_,
+                          const Vector &d_,
+                          const Vector &x_,
+                          Vector &y_,
+                          const int d1d = 0,
+                          const int q1d = 0)
+{
+   MFEM_VERIFY(T_D1D ? T_D1D : d1d <= MAX_D1D, "");
+   MFEM_VERIFY(T_Q1D ? T_Q1D : q1d <= MAX_Q1D, "");
+
+   const auto B = b_.Read();
+   const auto Bt = bt_.Read();
+   const auto D = d_.Read();
+   const auto X = x_.Read();
+   auto Y = y_.ReadWrite();
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      internal::PAMassApply2D_Element(e, NE, B, Bt, D, X, Y, d1d, q1d);
+   });
+}
+
+// Shared memory PA Mass Apply 2D kernel
+template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
+inline void SmemPAMassApply2D(const int NE,
+                              const Array<double> &b_,
+                              const Array<double> &bt_,
+                              const Vector &d_,
+                              const Vector &x_,
+                              Vector &y_,
+                              const int d1d = 0,
+                              const int q1d = 0)
+{
+   MFEM_CONTRACT_VAR(bt_);
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
+   const auto b = b_.Read();
+   const auto D = d_.Read();
+   const auto x = x_.Read();
+   auto Y = y_.ReadWrite();
+   mfem::forall_2D_batch(NE, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE (int e)
+   {
+      internal::SmemPAMassApply2D_Element<T_D1D,T_Q1D,T_NBZ>(e, NE, b, D, x, Y, d1d,
+                                                             q1d);
+   });
+}
+
+// PA Mass Apply 3D kernel
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void PAMassApply3D(const int NE,
+                          const Array<double> &b_,
+                          const Array<double> &bt_,
+                          const Vector &d_,
+                          const Vector &x_,
+                          Vector &y_,
+                          const int d1d = 0,
+                          const int q1d = 0)
+{
+   MFEM_VERIFY(T_D1D ? T_D1D : d1d <= MAX_D1D, "");
+   MFEM_VERIFY(T_Q1D ? T_Q1D : q1d <= MAX_Q1D, "");
+
+   const auto B = b_.Read();
+   const auto Bt = bt_.Read();
+   const auto D = d_.Read();
+   const auto X = x_.Read();
+   auto Y = y_.ReadWrite();
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      internal::PAMassApply3D_Element(e, NE, B, Bt, D, X, Y, d1d, q1d);
+   });
+}
+
+// Shared memory PA Mass Apply 2D kernel
+template<int T_D1D = 0, int T_Q1D = 0>
+inline void SmemPAMassApply3D(const int NE,
+                              const Array<double> &b_,
+                              const Array<double> &bt_,
+                              const Vector &d_,
+                              const Vector &x_,
+                              Vector &y_,
+                              const int d1d = 0,
+                              const int q1d = 0)
+{
+   MFEM_CONTRACT_VAR(bt_);
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int M1Q = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int M1D = T_D1D ? T_D1D : MAX_D1D;
+   MFEM_VERIFY(D1D <= M1D, "");
+   MFEM_VERIFY(Q1D <= M1Q, "");
+   auto b = b_.Read();
+   auto d = d_.Read();
+   auto x = x_.Read();
+   auto y = y_.ReadWrite();
+   mfem::forall_2D(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
+   {
+      internal::SmemPAMassApply3D_Element<T_D1D,T_Q1D>(e, NE, b, d, x, y, d1d, q1d);
+   });
+}
+
 } // namespace internal
 
 } // namespace mfem

fem/integ/bilininteg_mass_mf.cpp

@@ -9,12 +9,9 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-#include "ceed/integrators/mass/mass.hpp"
-
-using namespace std;
+#include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
+#include "../ceed/integrators/mass/mass.hpp"
 
 namespace mfem
 {

fem/integ/bilininteg_mass_pa.cpp

@@ -0,0 +1,222 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../../general/forall.hpp"
+#include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
+#include "../qfunction.hpp"
+#include "../ceed/integrators/mass/mass.hpp"
+#include "bilininteg_mass_kernels.hpp"
+
+namespace mfem
+{
+
+void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
+{
+   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
+                         Device::GetDeviceMemoryType() : pa_mt;
+
+   // Assuming the same element type
+   fespace = &fes;
+   Mesh *mesh = fes.GetMesh();
+   if (mesh->GetNE() == 0) { return; }
+   const FiniteElement &el = *fes.GetFE(0);
+   ElementTransformation *T0 = mesh->GetElementTransformation(0);
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, *T0);
+   if (DeviceCanUseCeed())
+   {
+      delete ceedOp;
+      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
+                         fes.IsVariableOrder();
+      if (mixed)
+      {
+         ceedOp = new ceed::MixedPAMassIntegrator(*this, fes, Q);
+      }
+      else
+      {
+         ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q);
+      }
+      return;
+   }
+   int map_type = el.GetMapType();
+   dim = mesh->Dimension();
+   ne = fes.GetMesh()->GetNE();
+   nq = ir->GetNPoints();
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::DETERMINANTS, mt);
+   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
+   dofs1D = maps->ndof;
+   quad1D = maps->nqpt;
+   pa_data.SetSize(ne*nq, mt);
+
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
+
+   if (dim==1) { MFEM_ABORT("Not supported yet... stay tuned!"); }
+   if (dim==2)
+   {
+      const int NE = ne;
+      const int Q1D = quad1D;
+      const bool const_c = coeff.Size() == 1;
+      const bool by_val = map_type == FiniteElement::VALUE;
+      const auto W = Reshape(ir->GetWeights().Read(), Q1D,Q1D);
+      const auto J = Reshape(geom->detJ.Read(), Q1D,Q1D,NE);
+      const auto C = const_c ? Reshape(coeff.Read(), 1,1,1) :
+                     Reshape(coeff.Read(), Q1D,Q1D,NE);
+      auto v = Reshape(pa_data.Write(), Q1D,Q1D, NE);
+      mfem::forall_2D(NE,Q1D,Q1D, [=] MFEM_HOST_DEVICE (int e)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               const double detJ = J(qx,qy,e);
+               const double coeff = const_c ? C(0,0,0) : C(qx,qy,e);
+               v(qx,qy,e) =  W(qx,qy) * coeff * (by_val ? detJ : 1.0/detJ);
+            }
+         }
+      });
+   }
+   if (dim==3)
+   {
+      const int NE = ne;
+      const int Q1D = quad1D;
+      const bool const_c = coeff.Size() == 1;
+      const bool by_val = map_type == FiniteElement::VALUE;
+      const auto W = Reshape(ir->GetWeights().Read(), Q1D,Q1D,Q1D);
+      const auto J = Reshape(geom->detJ.Read(), Q1D,Q1D,Q1D,NE);
+      const auto C = const_c ? Reshape(coeff.Read(), 1,1,1,1) :
+                     Reshape(coeff.Read(), Q1D,Q1D,Q1D,NE);
+      auto v = Reshape(pa_data.Write(), Q1D,Q1D,Q1D,NE);
+      mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               MFEM_FOREACH_THREAD(qz,z,Q1D)
+               {
+                  const double detJ = J(qx,qy,qz,e);
+                  const double coeff = const_c ? C(0,0,0,0) : C(qx,qy,qz,e);
+                  v(qx,qy,qz,e) = W(qx,qy,qz) * coeff * (by_val ? detJ : 1.0/detJ);
+               }
+            }
+         }
+      });
+   }
+}
+
+void MassIntegrator::AssemblePABoundary(const FiniteElementSpace &fes)
+{
+   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
+                         Device::GetDeviceMemoryType() : pa_mt;
+
+   // Assuming the same element type
+   fespace = &fes;
+   Mesh *mesh = fes.GetMesh();
+   if (mesh->GetNBE() == 0) { return; }
+   const FiniteElement &el = *fes.GetBE(0);
+   ElementTransformation *T0 = mesh->GetBdrElementTransformation(0);
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, *T0);
+
+   int map_type = el.GetMapType();
+   dim = el.GetDim(); // Dimension of the boundary element, *not* the mesh
+   ne = fes.GetMesh()->GetNBE();
+   nq = ir->GetNPoints();
+   face_geom = mesh->GetFaceGeometricFactors(*ir, GeometricFactors::DETERMINANTS,
+                                             FaceType::Boundary, mt);
+   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
+   dofs1D = maps->ndof;
+   quad1D = maps->nqpt;
+   pa_data.SetSize(ne*nq, mt);
+
+   FaceQuadratureSpace qs(*mesh, *ir, FaceType::Boundary);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
+
+   const int NE = ne;
+   const int Q1D = quad1D;
+   const bool const_c = coeff.Size() == 1;
+   const bool by_val = map_type == FiniteElement::VALUE;
+   if (dim==1)
+   {
+      const auto W = Reshape(ir->GetWeights().Read(), Q1D);
+      const auto J = Reshape(face_geom->detJ.Read(), Q1D, NE);
+      const auto C = const_c ? Reshape(coeff.Read(), 1, 1) :
+                     Reshape(coeff.Read(), Q1D, NE);
+      auto v = Reshape(pa_data.Write(), Q1D, NE);
+      mfem::forall_2D(NE, Q1D, 1, [=] MFEM_HOST_DEVICE (int e)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            const double detJ = J(qx,e);
+            const double coeff = const_c ? C(0,0) : C(qx,e);
+            v(qx,e) =  W(qx) * coeff * (by_val ? detJ : 1.0/detJ);
+         }
+      });
+   }
+   else if (dim==2)
+   {
+      const auto W = Reshape(ir->GetWeights().Read(), Q1D,Q1D);
+      const auto J = Reshape(face_geom->detJ.Read(), Q1D,Q1D,NE);
+      const auto C = const_c ? Reshape(coeff.Read(), 1,1,1) :
+                     Reshape(coeff.Read(), Q1D,Q1D,NE);
+      auto v = Reshape(pa_data.Write(), Q1D,Q1D, NE);
+      mfem::forall_2D(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy,y,Q1D)
+            {
+               const double detJ = J(qx,qy,e);
+               const double coeff = const_c ? C(0,0,0) : C(qx,qy,e);
+               v(qx,qy,e) =  W(qx,qy) * coeff * (by_val ? detJ : 1.0/detJ);
+            }
+         }
+      });
+   }
+   else
+   {
+      MFEM_ABORT("Not supported.");
+   }
+}
+
+void MassIntegrator::AssembleDiagonalPA(Vector &diag)
+{
+   if (DeviceCanUseCeed())
+   {
+      ceedOp->GetDiagonal(diag);
+   }
+   else
+   {
+      internal::PAMassAssembleDiagonal(dim, dofs1D, quad1D, ne, maps->B, pa_data,
+                                       diag);
+   }
+}
+
+void MassIntegrator::AddMultPA(const Vector &x, Vector &y) const
+{
+   if (DeviceCanUseCeed())
+   {
+      ceedOp->AddMult(x, y);
+   }
+   else
+   {
+      internal::PAMassApply(dim, dofs1D, quad1D, ne, maps->B, maps->Bt, pa_data, x,
+                            y);
+   }
+}
+
+void MassIntegrator::AddMultTransposePA(const Vector &x, Vector &y) const
+{
+   // Mass integrator is symmetric
+   AddMultPA(x, y);
+}
+
+} // namespace mfem

fem/integ/bilininteg_mixedcurl_pa.cpp

@@ -0,0 +1,423 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
+#include "../qfunction.hpp"
+#include "bilininteg_hcurl_kernels.hpp"
+#include "bilininteg_hcurlhdiv_kernels.hpp"
+
+namespace mfem
+{
+
+void MixedScalarCurlIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
+                                           const FiniteElementSpace &test_fes)
+{
+   // Assumes tensor-product elements
+   Mesh *mesh = trial_fes.GetMesh();
+   const FiniteElement *fel = trial_fes.GetFE(0); // In H(curl)
+   const FiniteElement *eltest = test_fes.GetFE(0); // In scalar space
+
+   const VectorTensorFiniteElement *el =
+      dynamic_cast<const VectorTensorFiniteElement*>(fel);
+   MFEM_VERIFY(el != NULL, "Only VectorTensorFiniteElement is supported!");
+
+   if (el->GetDerivType() != mfem::FiniteElement::CURL)
+   {
+      MFEM_ABORT("Unknown kernel.");
+   }
+
+   const IntegrationRule *ir
+      = IntRule ? IntRule : &MassIntegrator::GetRule(*eltest, *eltest,
+                                                     *mesh->GetElementTransformation(0));
+
+   const int dims = el->GetDim();
+   MFEM_VERIFY(dims == 2, "");
+
+   const int nq = ir->GetNPoints();
+   dim = mesh->Dimension();
+   MFEM_VERIFY(dim == 2, "");
+
+   ne = test_fes.GetNE();
+   mapsC = &el->GetDofToQuad(*ir, DofToQuad::TENSOR);
+   mapsO = &el->GetDofToQuadOpen(*ir, DofToQuad::TENSOR);
+   dofs1D = mapsC->ndof;
+   quad1D = mapsC->nqpt;
+
+   MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");
+
+   if (el->GetOrder() == eltest->GetOrder())
+   {
+      dofs1Dtest = dofs1D;
+   }
+   else
+   {
+      dofs1Dtest = dofs1D - 1;
+   }
+
+   pa_data.SetSize(nq * ne, Device::GetMemoryType());
+
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::FULL);
+
+   if (dim == 2)
+   {
+      internal::PAHcurlL2Setup2D(quad1D, ne, ir->GetWeights(), coeff, pa_data);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension!");
+   }
+}
+
+void MixedScalarCurlIntegrator::AddMultPA(const Vector &x, Vector &y) const
+{
+   if (dim == 2)
+   {
+      internal::PAHcurlL2Apply2D(dofs1D, dofs1Dtest, quad1D, ne, mapsO->B,
+                                 mapsO->Bt, mapsC->Bt, mapsC->G, pa_data,
+                                 x, y);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension!");
+   }
+}
+
+void MixedScalarCurlIntegrator::AddMultTransposePA(const Vector &x,
+                                                   Vector &y) const
+{
+   if (dim == 2)
+   {
+      internal::PAHcurlL2ApplyTranspose2D(dofs1D, dofs1Dtest, quad1D, ne, mapsO->B,
+                                          mapsO->Bt, mapsC->B, mapsC->Gt, pa_data,
+                                          x, y);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension!");
+   }
+}
+
+void MixedVectorCurlIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
+                                           const FiniteElementSpace &test_fes)
+{
+   // Assumes tensor-product elements, with vector test and trial spaces.
+   Mesh *mesh = trial_fes.GetMesh();
+   const FiniteElement *trial_fel = trial_fes.GetFE(0);
+   const FiniteElement *test_fel = test_fes.GetFE(0);
+
+   const VectorTensorFiniteElement *trial_el =
+      dynamic_cast<const VectorTensorFiniteElement*>(trial_fel);
+   MFEM_VERIFY(trial_el != NULL, "Only VectorTensorFiniteElement is supported!");
+
+   const VectorTensorFiniteElement *test_el =
+      dynamic_cast<const VectorTensorFiniteElement*>(test_fel);
+   MFEM_VERIFY(test_el != NULL, "Only VectorTensorFiniteElement is supported!");
+
+   const IntegrationRule *ir
+      = IntRule ? IntRule : &MassIntegrator::GetRule(*trial_el, *trial_el,
+                                                     *mesh->GetElementTransformation(0));
+   const int dims = trial_el->GetDim();
+   MFEM_VERIFY(dims == 3, "");
+
+   const int nq = ir->GetNPoints();
+   dim = mesh->Dimension();
+   MFEM_VERIFY(dim == 3, "");
+
+   MFEM_VERIFY(trial_el->GetOrder() == test_el->GetOrder(), "");
+
+   ne = trial_fes.GetNE();
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
+   mapsC = &trial_el->GetDofToQuad(*ir, DofToQuad::TENSOR);
+   mapsO = &trial_el->GetDofToQuadOpen(*ir, DofToQuad::TENSOR);
+   mapsCtest = &test_el->GetDofToQuad(*ir, DofToQuad::TENSOR);
+   mapsOtest = &test_el->GetDofToQuadOpen(*ir, DofToQuad::TENSOR);
+   dofs1D = mapsC->ndof;
+   quad1D = mapsC->nqpt;
+   dofs1Dtest = mapsCtest->ndof;
+
+   MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");
+
+   testType = test_el->GetDerivType();
+   trialType = trial_el->GetDerivType();
+
+   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
+   coeffDim = (DQ ? 3 : 1);
+
+   const bool curlSpaces = (testType == mfem::FiniteElement::CURL &&
+                            trialType == mfem::FiniteElement::CURL);
+
+   const int ndata = curlSpaces ? (coeffDim == 1 ? 1 : 9) : symmDims;
+   pa_data.SetSize(ndata * nq * ne, Device::GetMemoryType());
+
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(qs, CoefficientStorage::FULL);
+   if (Q) { coeff.Project(*Q); }
+   else if (DQ) { coeff.Project(*DQ); }
+   else { coeff.SetConstant(1.0); }
+
+   if (testType == mfem::FiniteElement::CURL &&
+       trialType == mfem::FiniteElement::CURL && dim == 3)
+   {
+      if (coeffDim == 1)
+      {
+         internal::PAHcurlL2Setup3D(nq, coeffDim, ne, ir->GetWeights(), coeff, pa_data);
+      }
+      else
+      {
+         internal::PAHcurlHdivMassSetup3D(quad1D, coeffDim, ne, false, ir->GetWeights(),
+                                          geom->J, coeff, pa_data);
+      }
+   }
+   else if (testType == mfem::FiniteElement::DIV &&
+            trialType == mfem::FiniteElement::CURL && dim == 3 &&
+            test_fel->GetOrder() == trial_fel->GetOrder())
+   {
+      internal::PACurlCurlSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
+                                  coeff, pa_data);
+   }
+   else
+   {
+      MFEM_ABORT("Unknown kernel.");
+   }
+}
+
+void MixedVectorCurlIntegrator::AddMultPA(const Vector &x, Vector &y) const
+{
+   if (testType == mfem::FiniteElement::CURL &&
+       trialType == mfem::FiniteElement::CURL && dim == 3)
+   {
+      const int ndata = coeffDim == 1 ? 1 : 9;
+
+      if (Device::Allows(Backend::DEVICE_MASK))
+      {
+         const int ID = (dofs1D << 4) | quad1D;
+         switch (ID)
+         {
+            case 0x23:
+               return internal::SmemPAHcurlL2Apply3D<2,3>(
+                         dofs1D, quad1D, ndata, ne,
+                         mapsO->B, mapsC->B, mapsC->G,
+                         pa_data, x, y);
+            case 0x34:
+               return internal::SmemPAHcurlL2Apply3D<3,4>(
+                         dofs1D, quad1D, ndata, ne,
+                         mapsO->B, mapsC->B, mapsC->G,
+                         pa_data, x, y);
+            case 0x45:
+               return internal::SmemPAHcurlL2Apply3D<4,5>(
+                         dofs1D, quad1D, ndata, ne,
+                         mapsO->B, mapsC->B, mapsC->G,
+                         pa_data, x, y);
+            case 0x56:
+               return internal::SmemPAHcurlL2Apply3D<5,6>(
+                         dofs1D, quad1D, ndata, ne,
+                         mapsO->B, mapsC->B, mapsC->G,
+                         pa_data, x, y);
+            default:
+               return internal::SmemPAHcurlL2Apply3D(
+                         dofs1D, quad1D, ndata, ne,
+                         mapsO->B, mapsC->B, mapsC->G,
+                         pa_data, x, y);
+         }
+      }
+      else
+      {
+         internal::PAHcurlL2Apply3D(dofs1D, quad1D, ndata, ne, mapsO->B, mapsC->B,
+                                    mapsO->Bt, mapsC->Bt, mapsC->G, pa_data, x, y);
+      }
+   }
+   else if (testType == mfem::FiniteElement::DIV &&
+            trialType == mfem::FiniteElement::CURL && dim == 3)
+   {
+      internal::PAHcurlHdivApply3D(dofs1D, dofs1Dtest, quad1D, ne, mapsO->B,
+                                   mapsC->B, mapsOtest->Bt, mapsCtest->Bt, mapsC->G,
+                                   pa_data, x, y);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension or space!");
+   }
+}
+
+void MixedVectorCurlIntegrator::AddMultTransposePA(const Vector &x,
+                                                   Vector &y) const
+{
+   if (testType == mfem::FiniteElement::DIV &&
+       trialType == mfem::FiniteElement::CURL && dim == 3)
+   {
+      internal::PAHcurlHdivApplyTranspose3D(dofs1D, dofs1Dtest, quad1D, ne, mapsO->B,
+                                            mapsC->B, mapsOtest->Bt, mapsCtest->Bt,
+                                            mapsC->Gt, pa_data, x, y);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension or space!");
+   }
+}
+
+void MixedVectorWeakCurlIntegrator::AssemblePA(const FiniteElementSpace
+                                               &trial_fes,
+                                               const FiniteElementSpace &test_fes)
+{
+   // Assumes tensor-product elements, with vector test and trial spaces.
+   Mesh *mesh = trial_fes.GetMesh();
+   const FiniteElement *trial_fel = trial_fes.GetFE(0);
+   const FiniteElement *test_fel = test_fes.GetFE(0);
+
+   const VectorTensorFiniteElement *trial_el =
+      dynamic_cast<const VectorTensorFiniteElement*>(trial_fel);
+   MFEM_VERIFY(trial_el != NULL, "Only VectorTensorFiniteElement is supported!");
+
+   const VectorTensorFiniteElement *test_el =
+      dynamic_cast<const VectorTensorFiniteElement*>(test_fel);
+   MFEM_VERIFY(test_el != NULL, "Only VectorTensorFiniteElement is supported!");
+
+   const IntegrationRule *ir
+      = IntRule ? IntRule : &MassIntegrator::GetRule(*trial_el, *trial_el,
+                                                     *mesh->GetElementTransformation(0));
+   const int dims = trial_el->GetDim();
+   MFEM_VERIFY(dims == 3, "");
+
+   const int nq = ir->GetNPoints();
+   dim = mesh->Dimension();
+   MFEM_VERIFY(dim == 3, "");
+
+   MFEM_VERIFY(trial_el->GetOrder() == test_el->GetOrder(), "");
+
+   ne = trial_fes.GetNE();
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
+   mapsC = &test_el->GetDofToQuad(*ir, DofToQuad::TENSOR);
+   mapsO = &test_el->GetDofToQuadOpen(*ir, DofToQuad::TENSOR);
+   dofs1D = mapsC->ndof;
+   quad1D = mapsC->nqpt;
+
+   MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");
+
+   testType = test_el->GetDerivType();
+   trialType = trial_el->GetDerivType();
+
+   const bool curlSpaces = (testType == mfem::FiniteElement::CURL &&
+                            trialType == mfem::FiniteElement::CURL);
+
+   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
+
+   coeffDim = DQ ? 3 : 1;
+   const int ndata = curlSpaces ? (DQ ? 9 : 1) : symmDims;
+
+   pa_data.SetSize(ndata * nq * ne, Device::GetMemoryType());
+
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(qs, CoefficientStorage::FULL);
+   if (Q) { coeff.Project(*Q); }
+   else if (DQ) { coeff.Project(*DQ); }
+   else { coeff.SetConstant(1.0); }
+
+   if (trialType == mfem::FiniteElement::CURL && dim == 3)
+   {
+      if (coeffDim == 1)
+      {
+         internal::PAHcurlL2Setup3D(nq, coeffDim, ne, ir->GetWeights(), coeff, pa_data);
+      }
+      else
+      {
+         internal::PAHcurlHdivMassSetup3D(quad1D, coeffDim, ne, false, ir->GetWeights(),
+                                          geom->J, coeff, pa_data);
+      }
+   }
+   else if (trialType == mfem::FiniteElement::DIV && dim == 3 &&
+            test_el->GetOrder() == trial_el->GetOrder())
+   {
+      internal::PACurlCurlSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
+                                  coeff, pa_data);
+   }
+   else
+   {
+      MFEM_ABORT("Unknown kernel.");
+   }
+}
+
+void MixedVectorWeakCurlIntegrator::AddMultPA(const Vector &x, Vector &y) const
+{
+   if (testType == mfem::FiniteElement::CURL &&
+       trialType == mfem::FiniteElement::CURL && dim == 3)
+   {
+      const int ndata = coeffDim == 1 ? 1 : 9;
+      if (Device::Allows(Backend::DEVICE_MASK))
+      {
+         const int ID = (dofs1D << 4) | quad1D;
+         switch (ID)
+         {
+            case 0x23:
+               return internal::SmemPAHcurlL2ApplyTranspose3D<2,3>(
+                         dofs1D, quad1D, ndata,
+                         ne, mapsO->B, mapsC->B,
+                         mapsC->G, pa_data, x, y);
+            case 0x34:
+               return internal::SmemPAHcurlL2ApplyTranspose3D<3,4>(
+                         dofs1D, quad1D, ndata,
+                         ne, mapsO->B, mapsC->B,
+                         mapsC->G, pa_data, x, y);
+            case 0x45:
+               return internal::SmemPAHcurlL2ApplyTranspose3D<4,5>(
+                         dofs1D, quad1D, ndata,
+                         ne, mapsO->B, mapsC->B,
+                         mapsC->G, pa_data, x, y);
+            case 0x56:
+               return internal::SmemPAHcurlL2ApplyTranspose3D<5,6>(
+                         dofs1D, quad1D, ndata,
+                         ne, mapsO->B, mapsC->B,
+                         mapsC->G, pa_data, x, y);
+            default:
+               return internal::SmemPAHcurlL2ApplyTranspose3D(
+                         dofs1D, quad1D, ndata, ne,
+                         mapsO->B, mapsC->B,
+                         mapsC->G, pa_data, x, y);
+         }
+      }
+      else
+      {
+         internal::PAHcurlL2ApplyTranspose3D(dofs1D, quad1D, ndata, ne, mapsO->B,
+                                             mapsC->B, mapsO->Bt, mapsC->Bt, mapsC->Gt,
+                                             pa_data, x, y);
+      }
+   }
+   else if (testType == mfem::FiniteElement::CURL &&
+            trialType == mfem::FiniteElement::DIV && dim == 3)
+   {
+      internal::PAHcurlHdivApplyTranspose3D(dofs1D, dofs1D, quad1D, ne, mapsO->B,
+                                            mapsC->B, mapsO->Bt, mapsC->Bt,
+                                            mapsC->Gt, pa_data, x, y);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension or space!");
+   }
+}
+
+void MixedVectorWeakCurlIntegrator::AddMultTransposePA(const Vector &x,
+                                                       Vector &y) const
+{
+   if (testType == mfem::FiniteElement::CURL &&
+       trialType == mfem::FiniteElement::DIV && dim == 3)
+   {
+      internal::PAHcurlHdivApply3D(dofs1D, dofs1D, quad1D, ne, mapsO->B,
+                                   mapsC->B, mapsO->Bt, mapsC->Bt, mapsC->G,
+                                   pa_data, x, y);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension or space!");
+   }
+}
+
+} // namespace mfem

fem/integ/bilininteg_mixedvecgrad_pa.cpp

@@ -0,0 +1,757 @@
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../../general/forall.hpp"
+#include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
+#include "../qfunction.hpp"
+#include "bilininteg_diffusion_kernels.hpp"
+
+namespace mfem
+{
+
+// Apply to x corresponding to DOFs in H^1 (trial), whose gradients are
+// integrated against H(curl) test functions corresponding to y.
+static void PAHcurlH1Apply2D(const int D1D,
+                             const int Q1D,
+                             const int NE,
+                             const Array<double> &bc,
+                             const Array<double> &gc,
+                             const Array<double> &bot,
+                             const Array<double> &bct,
+                             const Vector &pa_data,
+                             const Vector &x,
+                             Vector &y)
+{
+   constexpr static int VDIM = 2;
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
+
+   auto Bc = Reshape(bc.Read(), Q1D, D1D);
+   auto Gc = Reshape(gc.Read(), Q1D, D1D);
+   auto Bot = Reshape(bot.Read(), D1D-1, Q1D);
+   auto Bct = Reshape(bct.Read(), D1D, Q1D);
+   auto op = Reshape(pa_data.Read(), Q1D, Q1D, 3, NE);
+   auto X = Reshape(x.Read(), D1D, D1D, NE);
+   auto Y = Reshape(y.ReadWrite(), 2*(D1D-1)*D1D, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      double mass[MAX_Q1D][MAX_Q1D][VDIM];
+
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            for (int c = 0; c < VDIM; ++c)
+            {
+               mass[qy][qx][c] = 0.0;
+            }
+         }
+      }
+
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         double gradX[MAX_Q1D][2];
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            gradX[qx][0] = 0.0;
+            gradX[qx][1] = 0.0;
+         }
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            const double s = X(dx,dy,e);
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               gradX[qx][0] += s * Bc(qx,dx);
+               gradX[qx][1] += s * Gc(qx,dx);
+            }
+         }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            const double wy  = Bc(qy,dy);
+            const double wDy = Gc(qy,dy);
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const double wx  = gradX[qx][0];
+               const double wDx = gradX[qx][1];
+               mass[qy][qx][0] += wDx * wy;
+               mass[qy][qx][1] += wx * wDy;
+            }
+         }
+      }
+
+      // Apply D operator.
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            const double O11 = op(qx,qy,0,e);
+            const double O12 = op(qx,qy,1,e);
+            const double O22 = op(qx,qy,2,e);
+            const double massX = mass[qy][qx][0];
+            const double massY = mass[qy][qx][1];
+            mass[qy][qx][0] = (O11*massX)+(O12*massY);
+            mass[qy][qx][1] = (O12*massX)+(O22*massY);
+         }
+      }
+
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         int osc = 0;
+
+         for (int c = 0; c < VDIM; ++c)  // loop over x, y components
+         {
+            const int D1Dy = (c == 1) ? D1D - 1 : D1D;
+            const int D1Dx = (c == 0) ? D1D - 1 : D1D;
+
+            double massX[MAX_D1D];
+            for (int dx = 0; dx < D1Dx; ++dx)
+            {
+               massX[dx] = 0;
+            }
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  massX[dx] += mass[qy][qx][c] * ((c == 0) ? Bot(dx,qx) : Bct(dx,qx));
+               }
+            }
+
+            for (int dy = 0; dy < D1Dy; ++dy)
+            {
+               const double wy = (c == 1) ? Bot(dy,qy) : Bct(dy,qy);
+
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  Y(dx + (dy * D1Dx) + osc, e) += massX[dx] * wy;
+               }
+            }
+
+            osc += D1Dx * D1Dy;
+         }  // loop c
+      }
+   }); // end of element loop
+}
+
+// Apply to x corresponding to DOFs in H(curl), integrated
+// against gradients of H^1 functions corresponding to y.
+static void PAHcurlH1ApplyTranspose2D(const int D1D,
+                                      const int Q1D,
+                                      const int NE,
+                                      const Array<double> &bc,
+                                      const Array<double> &bo,
+                                      const Array<double> &bct,
+                                      const Array<double> &gct,
+                                      const Vector &pa_data,
+                                      const Vector &x,
+                                      Vector &y)
+{
+   constexpr static int VDIM = 2;
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
+
+   auto Bc = Reshape(bc.Read(), Q1D, D1D);
+   auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
+   auto Bt = Reshape(bct.Read(), D1D, Q1D);
+   auto Gt = Reshape(gct.Read(), D1D, Q1D);
+   auto op = Reshape(pa_data.Read(), Q1D, Q1D, 3, NE);
+   auto X = Reshape(x.Read(), 2*(D1D-1)*D1D, NE);
+   auto Y = Reshape(y.ReadWrite(), D1D, D1D, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      double mass[MAX_Q1D][MAX_Q1D][VDIM];
+
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            for (int c = 0; c < VDIM; ++c)
+            {
+               mass[qy][qx][c] = 0.0;
+            }
+         }
+      }
+
+      int osc = 0;
+
+      for (int c = 0; c < VDIM; ++c)  // loop over x, y components
+      {
+         const int D1Dy = (c == 1) ? D1D - 1 : D1D;
+         const int D1Dx = (c == 0) ? D1D - 1 : D1D;
+
+         for (int dy = 0; dy < D1Dy; ++dy)
+         {
+            double massX[MAX_Q1D];
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               massX[qx] = 0.0;
+            }
+
+            for (int dx = 0; dx < D1Dx; ++dx)
+            {
+               const double t = X(dx + (dy * D1Dx) + osc, e);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  massX[qx] += t * ((c == 0) ? Bo(qx,dx) : Bc(qx,dx));
+               }
+            }
+
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const double wy = (c == 1) ? Bo(qy,dy) : Bc(qy,dy);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  mass[qy][qx][c] += massX[qx] * wy;
+               }
+            }
+         }
+
+         osc += D1Dx * D1Dy;
+      }  // loop (c) over components
+
+      // Apply D operator.
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            const double O11 = op(qx,qy,0,e);
+            const double O12 = op(qx,qy,1,e);
+            const double O22 = op(qx,qy,2,e);
+            const double massX = mass[qy][qx][0];
+            const double massY = mass[qy][qx][1];
+            mass[qy][qx][0] = (O11*massX)+(O12*massY);
+            mass[qy][qx][1] = (O12*massX)+(O22*massY);
+         }
+      }
+
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         double gradX[MAX_D1D][2];
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            gradX[dx][0] = 0;
+            gradX[dx][1] = 0;
+         }
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            const double gX = mass[qy][qx][0];
+            const double gY = mass[qy][qx][1];
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const double wx  = Bt(dx,qx);
+               const double wDx = Gt(dx,qx);
+               gradX[dx][0] += gX * wDx;
+               gradX[dx][1] += gY * wx;
+            }
+         }
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            const double wy  = Bt(dy,qy);
+            const double wDy = Gt(dy,qy);
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               Y(dx,dy,e) += ((gradX[dx][0] * wy) + (gradX[dx][1] * wDy));
+            }
+         }
+      }
+   }); // end of element loop
+}
+
+// Apply to x corresponding to DOFs in H^1 (trial), whose gradients are
+// integrated against H(curl) test functions corresponding to y.
+static void PAHcurlH1Apply3D(const int D1D,
+                             const int Q1D,
+                             const int NE,
+                             const Array<double> &bc,
+                             const Array<double> &gc,
+                             const Array<double> &bot,
+                             const Array<double> &bct,
+                             const Vector &pa_data,
+                             const Vector &x,
+                             Vector &y)
+{
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
+
+   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
+
+   constexpr static int VDIM = 3;
+
+   auto Bc = Reshape(bc.Read(), Q1D, D1D);
+   auto Gc = Reshape(gc.Read(), Q1D, D1D);
+   auto Bot = Reshape(bot.Read(), D1D-1, Q1D);
+   auto Bct = Reshape(bct.Read(), D1D, Q1D);
+   auto op = Reshape(pa_data.Read(), Q1D, Q1D, Q1D, 6, NE);
+   auto X = Reshape(x.Read(), D1D, D1D, D1D, NE);
+   auto Y = Reshape(y.ReadWrite(), 3*(D1D-1)*D1D*D1D, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      double mass[MAX_Q1D][MAX_Q1D][MAX_Q1D][VDIM];
+
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               for (int c = 0; c < VDIM; ++c)
+               {
+                  mass[qz][qy][qx][c] = 0.0;
+               }
+            }
+         }
+      }
+
+      for (int dz = 0; dz < D1D; ++dz)
+      {
+         double gradXY[MAX_Q1D][MAX_Q1D][3];
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               gradXY[qy][qx][0] = 0.0;
+               gradXY[qy][qx][1] = 0.0;
+               gradXY[qy][qx][2] = 0.0;
+            }
+         }
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            double gradX[MAX_Q1D][2];
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               gradX[qx][0] = 0.0;
+               gradX[qx][1] = 0.0;
+            }
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const double s = X(dx,dy,dz,e);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  gradX[qx][0] += s * Bc(qx,dx);
+                  gradX[qx][1] += s * Gc(qx,dx);
+               }
+            }
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const double wy  = Bc(qy,dy);
+               const double wDy = Gc(qy,dy);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  const double wx  = gradX[qx][0];
+                  const double wDx = gradX[qx][1];
+                  gradXY[qy][qx][0] += wDx * wy;
+                  gradXY[qy][qx][1] += wx * wDy;
+                  gradXY[qy][qx][2] += wx * wy;
+               }
+            }
+         }
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            const double wz  = Bc(qz,dz);
+            const double wDz = Gc(qz,dz);
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  mass[qz][qy][qx][0] += gradXY[qy][qx][0] * wz;
+                  mass[qz][qy][qx][1] += gradXY[qy][qx][1] * wz;
+                  mass[qz][qy][qx][2] += gradXY[qy][qx][2] * wDz;
+               }
+            }
+         }
+      }
+
+      // Apply D operator.
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const double O11 = op(qx,qy,qz,0,e);
+               const double O12 = op(qx,qy,qz,1,e);
+               const double O13 = op(qx,qy,qz,2,e);
+               const double O22 = op(qx,qy,qz,3,e);
+               const double O23 = op(qx,qy,qz,4,e);
+               const double O33 = op(qx,qy,qz,5,e);
+               const double massX = mass[qz][qy][qx][0];
+               const double massY = mass[qz][qy][qx][1];
+               const double massZ = mass[qz][qy][qx][2];
+               mass[qz][qy][qx][0] = (O11*massX)+(O12*massY)+(O13*massZ);
+               mass[qz][qy][qx][1] = (O12*massX)+(O22*massY)+(O23*massZ);
+               mass[qz][qy][qx][2] = (O13*massX)+(O23*massY)+(O33*massZ);
+            }
+         }
+      }
+
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         double massXY[MAX_D1D][MAX_D1D];
+
+         int osc = 0;
+
+         for (int c = 0; c < VDIM; ++c)  // loop over x, y, z components
+         {
+            const int D1Dz = (c == 2) ? D1D - 1 : D1D;
+            const int D1Dy = (c == 1) ? D1D - 1 : D1D;
+            const int D1Dx = (c == 0) ? D1D - 1 : D1D;
+
+            for (int dy = 0; dy < D1Dy; ++dy)
+            {
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  massXY[dy][dx] = 0.0;
+               }
+            }
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               double massX[MAX_D1D];
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  massX[dx] = 0;
+               }
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  for (int dx = 0; dx < D1Dx; ++dx)
+                  {
+                     massX[dx] += mass[qz][qy][qx][c] * ((c == 0) ? Bot(dx,qx) : Bct(dx,qx));
+                  }
+               }
+               for (int dy = 0; dy < D1Dy; ++dy)
+               {
+                  const double wy = (c == 1) ? Bot(dy,qy) : Bct(dy,qy);
+                  for (int dx = 0; dx < D1Dx; ++dx)
+                  {
+                     massXY[dy][dx] += massX[dx] * wy;
+                  }
+               }
+            }
+
+            for (int dz = 0; dz < D1Dz; ++dz)
+            {
+               const double wz = (c == 2) ? Bot(dz,qz) : Bct(dz,qz);
+               for (int dy = 0; dy < D1Dy; ++dy)
+               {
+                  for (int dx = 0; dx < D1Dx; ++dx)
+                  {
+                     Y(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e) += massXY[dy][dx] * wz;
+                  }
+               }
+            }
+
+            osc += D1Dx * D1Dy * D1Dz;
+         }  // loop c
+      }  // loop qz
+   }); // end of element loop
+}
+
+// Apply to x corresponding to DOFs in H(curl), integrated
+// against gradients of H^1 functions corresponding to y.
+static void PAHcurlH1ApplyTranspose3D(const int D1D,
+                                      const int Q1D,
+                                      const int NE,
+                                      const Array<double> &bc,
+                                      const Array<double> &bo,
+                                      const Array<double> &bct,
+                                      const Array<double> &gct,
+                                      const Vector &pa_data,
+                                      const Vector &x,
+                                      Vector &y)
+{
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
+
+   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
+
+   constexpr static int VDIM = 3;
+
+   auto Bc = Reshape(bc.Read(), Q1D, D1D);
+   auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
+   auto Bt = Reshape(bct.Read(), D1D, Q1D);
+   auto Gt = Reshape(gct.Read(), D1D, Q1D);
+   auto op = Reshape(pa_data.Read(), Q1D, Q1D, Q1D, 6, NE);
+   auto X = Reshape(x.Read(), 3*(D1D-1)*D1D*D1D, NE);
+   auto Y = Reshape(y.ReadWrite(), D1D, D1D, D1D, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      double mass[MAX_Q1D][MAX_Q1D][MAX_Q1D][VDIM];
+
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               for (int c = 0; c < VDIM; ++c)
+               {
+                  mass[qz][qy][qx][c] = 0.0;
+               }
+            }
+         }
+      }
+
+      int osc = 0;
+
+      for (int c = 0; c < VDIM; ++c)  // loop over x, y, z components
+      {
+         const int D1Dz = (c == 2) ? D1D - 1 : D1D;
+         const int D1Dy = (c == 1) ? D1D - 1 : D1D;
+         const int D1Dx = (c == 0) ? D1D - 1 : D1D;
+
+         for (int dz = 0; dz < D1Dz; ++dz)
+         {
+            double massXY[MAX_Q1D][MAX_Q1D];
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  massXY[qy][qx] = 0.0;
+               }
+            }
+
+            for (int dy = 0; dy < D1Dy; ++dy)
+            {
+               double massX[MAX_Q1D];
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  massX[qx] = 0.0;
+               }
+
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  const double t = X(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e);
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     massX[qx] += t * ((c == 0) ? Bo(qx,dx) : Bc(qx,dx));
+                  }
+               }
+
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  const double wy = (c == 1) ? Bo(qy,dy) : Bc(qy,dy);
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     const double wx = massX[qx];
+                     massXY[qy][qx] += wx * wy;
+                  }
+               }
+            }
+
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               const double wz = (c == 2) ? Bo(qz,dz) : Bc(qz,dz);
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     mass[qz][qy][qx][c] += massXY[qy][qx] * wz;
+                  }
+               }
+            }
+         }
+
+         osc += D1Dx * D1Dy * D1Dz;
+      }  // loop (c) over components
+
+      // Apply D operator.
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const double O11 = op(qx,qy,qz,0,e);
+               const double O12 = op(qx,qy,qz,1,e);
+               const double O13 = op(qx,qy,qz,2,e);
+               const double O22 = op(qx,qy,qz,3,e);
+               const double O23 = op(qx,qy,qz,4,e);
+               const double O33 = op(qx,qy,qz,5,e);
+               const double massX = mass[qz][qy][qx][0];
+               const double massY = mass[qz][qy][qx][1];
+               const double massZ = mass[qz][qy][qx][2];
+               mass[qz][qy][qx][0] = (O11*massX)+(O12*massY)+(O13*massZ);
+               mass[qz][qy][qx][1] = (O12*massX)+(O22*massY)+(O23*massZ);
+               mass[qz][qy][qx][2] = (O13*massX)+(O23*massY)+(O33*massZ);
+            }
+         }
+      }
+
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         double gradXY[MAX_D1D][MAX_D1D][3];
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               gradXY[dy][dx][0] = 0;
+               gradXY[dy][dx][1] = 0;
+               gradXY[dy][dx][2] = 0;
+            }
+         }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            double gradX[MAX_D1D][3];
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               gradX[dx][0] = 0;
+               gradX[dx][1] = 0;
+               gradX[dx][2] = 0;
+            }
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const double gX = mass[qz][qy][qx][0];
+               const double gY = mass[qz][qy][qx][1];
+               const double gZ = mass[qz][qy][qx][2];
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  const double wx  = Bt(dx,qx);
+                  const double wDx = Gt(dx,qx);
+                  gradX[dx][0] += gX * wDx;
+                  gradX[dx][1] += gY * wx;
+                  gradX[dx][2] += gZ * wx;
+               }
+            }
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               const double wy  = Bt(dy,qy);
+               const double wDy = Gt(dy,qy);
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  gradXY[dy][dx][0] += gradX[dx][0] * wy;
+                  gradXY[dy][dx][1] += gradX[dx][1] * wDy;
+                  gradXY[dy][dx][2] += gradX[dx][2] * wy;
+               }
+            }
+         }
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            const double wz  = Bt(dz,qz);
+            const double wDz = Gt(dz,qz);
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  Y(dx,dy,dz,e) +=
+                     ((gradXY[dy][dx][0] * wz) +
+                      (gradXY[dy][dx][1] * wz) +
+                      (gradXY[dy][dx][2] * wDz));
+               }
+            }
+         }
+      }  // loop qz
+   }); // end of element loop
+}
+
+void MixedVectorGradientIntegrator::AssemblePA(const FiniteElementSpace
+                                               &trial_fes,
+                                               const FiniteElementSpace &test_fes)
+{
+   // Assumes tensor-product elements, with a vector test space and H^1 trial space.
+   Mesh *mesh = trial_fes.GetMesh();
+   const FiniteElement *trial_fel = trial_fes.GetFE(0);
+   const FiniteElement *test_fel = test_fes.GetFE(0);
+
+   const NodalTensorFiniteElement *trial_el =
+      dynamic_cast<const NodalTensorFiniteElement*>(trial_fel);
+   MFEM_VERIFY(trial_el != NULL, "Only NodalTensorFiniteElement is supported!");
+
+   const VectorTensorFiniteElement *test_el =
+      dynamic_cast<const VectorTensorFiniteElement*>(test_fel);
+   MFEM_VERIFY(test_el != NULL, "Only VectorTensorFiniteElement is supported!");
+
+   const IntegrationRule *ir
+      = IntRule ? IntRule : &MassIntegrator::GetRule(*trial_el, *trial_el,
+                                                     *mesh->GetElementTransformation(0));
+   const int dims = trial_el->GetDim();
+   MFEM_VERIFY(dims == 2 || dims == 3, "");
+
+   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
+   const int nq = ir->GetNPoints();
+   dim = mesh->Dimension();
+   MFEM_VERIFY(dim == 2 || dim == 3, "");
+
+   MFEM_VERIFY(trial_el->GetOrder() == test_el->GetOrder(), "");
+
+   ne = trial_fes.GetNE();
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
+   mapsC = &test_el->GetDofToQuad(*ir, DofToQuad::TENSOR);
+   mapsO = &test_el->GetDofToQuadOpen(*ir, DofToQuad::TENSOR);
+   dofs1D = mapsC->ndof;
+   quad1D = mapsC->nqpt;
+
+   MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");
+
+   pa_data.SetSize(symmDims * nq * ne, Device::GetMemoryType());
+
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::FULL);
+
+   // Use the same setup functions as VectorFEMassIntegrator.
+   if (test_el->GetDerivType() == mfem::FiniteElement::CURL && dim == 3)
+   {
+      internal::PADiffusionSetup3D(quad1D, 1, ne, ir->GetWeights(), geom->J,
+                                   coeff, pa_data);
+   }
+   else if (test_el->GetDerivType() == mfem::FiniteElement::CURL && dim == 2)
+   {
+      internal::PADiffusionSetup2D<2>(quad1D, 1, ne, ir->GetWeights(), geom->J,
+                                      coeff, pa_data);
+   }
+   else
+   {
+      MFEM_ABORT("Unknown kernel.");
+   }
+}
+
+void MixedVectorGradientIntegrator::AddMultPA(const Vector &x, Vector &y) const
+{
+   if (dim == 3)
+   {
+      PAHcurlH1Apply3D(dofs1D, quad1D, ne, mapsC->B, mapsC->G,
+                       mapsO->Bt, mapsC->Bt, pa_data, x, y);
+   }
+   else if (dim == 2)
+   {
+      PAHcurlH1Apply2D(dofs1D, quad1D, ne, mapsC->B, mapsC->G,
+                       mapsO->Bt, mapsC->Bt, pa_data, x, y);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension!");
+   }
+}
+
+void MixedVectorGradientIntegrator::AddMultTransposePA(const Vector &x,
+                                                       Vector &y) const
+{
+   if (dim == 3)
+   {
+      PAHcurlH1ApplyTranspose3D(dofs1D, quad1D, ne, mapsC->B, mapsO->B,
+                                mapsC->Bt, mapsC->Gt, pa_data, x, y);
+   }
+   else if (dim == 2)
+   {
+      PAHcurlH1ApplyTranspose2D(dofs1D, quad1D, ne, mapsC->B, mapsO->B,
+                                mapsC->Bt, mapsC->Gt, pa_data, x, y);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension!");
+   }
+}
+
+} // namespace mfem

fem/integ/bilininteg_transpose_ea.cpp

@@ -9,8 +9,8 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
+#include "../../general/forall.hpp"
+#include "../bilininteg.hpp"
 
 namespace mfem
 {
@@ -27,7 +27,7 @@ void TransposeIntegrator::AssembleEA(const FiniteElementSpace &fes,
       const int dofs = fes.GetFE(0)->GetDof();
       auto A = Reshape(ea_data_tmp.Read(), dofs, dofs, ne);
       auto AT = Reshape(ea_data.ReadWrite(), dofs, dofs, ne);
-      MFEM_FORALL(e, ne,
+      mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e)
       {
          for (int i = 0; i < dofs; i++)
          {
@@ -46,7 +46,7 @@ void TransposeIntegrator::AssembleEA(const FiniteElementSpace &fes,
       if (ne == 0) { return; }
       const int dofs = fes.GetFE(0)->GetDof();
       auto A = Reshape(ea_data.ReadWrite(), dofs, dofs, ne);
-      MFEM_FORALL(e, ne,
+      mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e)
       {
          for (int i = 0; i < dofs; i++)
          {
@@ -80,7 +80,7 @@ void TransposeIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
       auto A_ext = Reshape(ea_data_ext_tmp.Read(), faceDofs, faceDofs, 2, nf);
       auto AT_int = Reshape(ea_data_int.ReadWrite(), faceDofs, faceDofs, 2, nf);
       auto AT_ext = Reshape(ea_data_ext.ReadWrite(), faceDofs, faceDofs, 2, nf);
-      MFEM_FORALL(f, nf,
+      mfem::forall(nf, [=] MFEM_HOST_DEVICE (int f)
       {
          for (int i = 0; i < faceDofs; i++)
          {
@@ -105,7 +105,7 @@ void TransposeIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
                                                fes.GetMesh()->GetFaceGeometry(0))->GetDof();
       auto A_int = Reshape(ea_data_int.ReadWrite(), faceDofs, faceDofs, 2, nf);
       auto A_ext = Reshape(ea_data_ext.ReadWrite(), faceDofs, faceDofs, 2, nf);
-      MFEM_FORALL(f, nf,
+      mfem::forall(nf, [=] MFEM_HOST_DEVICE (int f)
       {
          for (int i = 0; i < faceDofs; i++)
          {
@@ -149,7 +149,7 @@ void TransposeIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace& fes,
                                                fes.GetMesh()->GetFaceGeometry(0))->GetDof();
       auto A_bdr = Reshape(ea_data_bdr_tmp.Read(), faceDofs, faceDofs, nf);
       auto AT_bdr = Reshape(ea_data_bdr.ReadWrite(), faceDofs, faceDofs, nf);
-      MFEM_FORALL(f, nf,
+      mfem::forall(nf, [=] MFEM_HOST_DEVICE (int f)
       {
          for (int i = 0; i < faceDofs; i++)
          {
@@ -167,7 +167,7 @@ void TransposeIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace& fes,
       const int faceDofs = fes.GetTraceElement(0,
                                                fes.GetMesh()->GetFaceGeometry(0))->GetDof();
       auto A_bdr = Reshape(ea_data_bdr.ReadWrite(), faceDofs, faceDofs, nf);
-      MFEM_FORALL(f, nf,
+      mfem::forall(nf, [=] MFEM_HOST_DEVICE (int f)
       {
          for (int i = 0; i < faceDofs; i++)
          {

fem/integ/bilininteg_vecdiffusion_mf.cpp

@@ -9,12 +9,9 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-#include "ceed/integrators/diffusion/diffusion.hpp"
-
-using namespace std;
+#include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
+#include "../ceed/integrators/diffusion/diffusion.hpp"
 
 namespace mfem
 {

fem/integ/bilininteg_vecdiffusion_pa.cpp

@@ -9,19 +9,15 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-#include "qfunction.hpp"
-#include "ceed/integrators/diffusion/diffusion.hpp"
-
-using namespace std;
+#include "../../general/forall.hpp"
+#include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
+#include "../qfunction.hpp"
+#include "../ceed/integrators/diffusion/diffusion.hpp"
 
 namespace mfem
 {
 
-// PA Vector Diffusion Integrator
-
 // PA Diffusion Assemble 2D kernel
 static void PAVectorDiffusionSetup2D(const int Q1D,
                                      const int NE,
@@ -41,7 +37,7 @@ static void PAVectorDiffusionSetup2D(const int Q1D,
                   Reshape(c.Read(), NQ, NE);
 
 
-   MFEM_FORALL(e, NE,
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
       for (int q = 0; q < NQ; ++q)
       {
@@ -77,7 +73,7 @@ static void PAVectorDiffusionSetup3D(const int Q1D,
                   Reshape(c.Read(), NQ,NE);
 
 
-   MFEM_FORALL(e, NE,
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
       for (int q = 0; q < NQ; ++q)
       {
@@ -91,8 +87,8 @@ static void PAVectorDiffusionSetup3D(const int Q1D,
          const double J23 = J(q,1,2,e);
          const double J33 = J(q,2,2,e);
          const double detJ = J11 * (J22 * J33 - J32 * J23) -
-         /* */               J21 * (J12 * J33 - J32 * J13) +
-         /* */               J31 * (J12 * J23 - J22 * J13);
+                             J21 * (J12 * J33 - J32 * J13) +
+                             J31 * (J12 * J23 - J22 * J13);
 
          const double C1 = const_c ? C(0,0) : C(q,e);
 
@@ -197,7 +193,7 @@ void VectorDiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
       const auto C = const_c ? Reshape(coeff.Read(), 1,1) :
                      Reshape(coeff.Read(), NQ,ne);
 
-      MFEM_FORALL(e, ne,
+      mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e)
       {
          for (int q = 0; q < NQ; ++q)
          {
@@ -226,6 +222,209 @@ void VectorDiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
    }
 }
 
+template<int T_D1D = 0, int T_Q1D = 0>
+static void PAVectorDiffusionDiagonal2D(const int NE,
+                                        const Array<double> &b,
+                                        const Array<double> &g,
+                                        const Vector &d,
+                                        Vector &y,
+                                        const int d1d = 0,
+                                        const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto G = Reshape(g.Read(), Q1D, D1D);
+   // note the different shape for D, this is a (symmetric) matrix so we only
+   // store necessary entries
+   auto D = Reshape(d.Read(), Q1D*Q1D, 3, NE);
+   auto Y = Reshape(y.ReadWrite(), D1D, D1D, 2, NE);
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      // gradphi \cdot Q \gradphi has four terms
+      double QD0[MQ1][MD1];
+      double QD1[MQ1][MD1];
+      double QD2[MQ1][MD1];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            QD0[qx][dy] = 0.0;
+            QD1[qx][dy] = 0.0;
+            QD2[qx][dy] = 0.0;
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const int q = qx + qy * Q1D;
+               const double D0 = D(q,0,e);
+               const double D1 = D(q,1,e);
+               const double D2 = D(q,2,e);
+               QD0[qx][dy] += B(qy, dy) * B(qy, dy) * D0;
+               QD1[qx][dy] += B(qy, dy) * G(qy, dy) * D1;
+               QD2[qx][dy] += G(qy, dy) * G(qy, dy) * D2;
+            }
+         }
+      }
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            double temp = 0.0;
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               temp += G(qx, dx) * G(qx, dx) * QD0[qx][dy];
+               temp += G(qx, dx) * B(qx, dx) * QD1[qx][dy];
+               temp += B(qx, dx) * G(qx, dx) * QD1[qx][dy];
+               temp += B(qx, dx) * B(qx, dx) * QD2[qx][dy];
+            }
+            Y(dx,dy,0,e) += temp;
+            Y(dx,dy,1,e) += temp;
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0>
+static void PAVectorDiffusionDiagonal3D(const int NE,
+                                        const Array<double> &b,
+                                        const Array<double> &g,
+                                        const Vector &d,
+                                        Vector &y,
+                                        const int d1d = 0,
+                                        const int q1d = 0)
+{
+   constexpr int DIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto G = Reshape(g.Read(), Q1D, D1D);
+   auto Q = Reshape(d.Read(), Q1D*Q1D*Q1D, 6, NE);
+   auto Y = Reshape(y.ReadWrite(), D1D, D1D, D1D, 3, NE);
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      double QQD[MQ1][MQ1][MD1];
+      double QDD[MQ1][MD1][MD1];
+      for (int i = 0; i < DIM; ++i)
+      {
+         for (int j = 0; j < DIM; ++j)
+         {
+            // first tensor contraction, along z direction
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  for (int dz = 0; dz < D1D; ++dz)
+                  {
+                     QQD[qx][qy][dz] = 0.0;
+                     for (int qz = 0; qz < Q1D; ++qz)
+                     {
+                        const int q = qx + (qy + qz * Q1D) * Q1D;
+                        const int k = j >= i ?
+                                      3 - (3-i)*(2-i)/2 + j:
+                                      3 - (3-j)*(2-j)/2 + i;
+                        const double O = Q(q,k,e);
+                        const double Bz = B(qz,dz);
+                        const double Gz = G(qz,dz);
+                        const double L = i==2 ? Gz : Bz;
+                        const double R = j==2 ? Gz : Bz;
+                        QQD[qx][qy][dz] += L * O * R;
+                     }
+                  }
+               }
+            }
+            // second tensor contraction, along y direction
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               for (int dz = 0; dz < D1D; ++dz)
+               {
+                  for (int dy = 0; dy < D1D; ++dy)
+                  {
+                     QDD[qx][dy][dz] = 0.0;
+                     for (int qy = 0; qy < Q1D; ++qy)
+                     {
+                        const double By = B(qy,dy);
+                        const double Gy = G(qy,dy);
+                        const double L = i==1 ? Gy : By;
+                        const double R = j==1 ? Gy : By;
+                        QDD[qx][dy][dz] += L * QQD[qx][qy][dz] * R;
+                     }
+                  }
+               }
+            }
+            // third tensor contraction, along x direction
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               for (int dy = 0; dy < D1D; ++dy)
+               {
+                  for (int dx = 0; dx < D1D; ++dx)
+                  {
+                     double temp = 0.0;
+                     for (int qx = 0; qx < Q1D; ++qx)
+                     {
+                        const double Bx = B(qx,dx);
+                        const double Gx = G(qx,dx);
+                        const double L = i==0 ? Gx : Bx;
+                        const double R = j==0 ? Gx : Bx;
+                        temp += L * QDD[qx][dy][dz] * R;
+                     }
+                     Y(dx, dy, dz, 0, e) += temp;
+                     Y(dx, dy, dz, 1, e) += temp;
+                     Y(dx, dy, dz, 2, e) += temp;
+                  }
+               }
+            }
+         }
+      }
+   });
+}
+
+static void PAVectorDiffusionAssembleDiagonal(const int dim,
+                                              const int D1D,
+                                              const int Q1D,
+                                              const int NE,
+                                              const Array<double> &B,
+                                              const Array<double> &G,
+                                              const Vector &op,
+                                              Vector &y)
+{
+   if (dim == 2)
+   {
+      return PAVectorDiffusionDiagonal2D(NE, B, G, op, y, D1D, Q1D);
+   }
+   else if (dim == 3)
+   {
+      return PAVectorDiffusionDiagonal3D(NE, B, G, op, y, D1D, Q1D);
+   }
+   MFEM_ABORT("Dimension not implemented.");
+}
+
+void VectorDiffusionIntegrator::AssembleDiagonalPA(Vector &diag)
+{
+   if (DeviceCanUseCeed())
+   {
+      ceedOp->GetDiagonal(diag);
+   }
+   else
+   {
+      PAVectorDiffusionAssembleDiagonal(dim, dofs1D, quad1D, ne,
+                                        maps->B, maps->G,
+                                        pa_data, diag);
+   }
+}
+
 // PA Diffusion Apply 2D kernel
 template<int T_D1D = 0, int T_Q1D = 0, int T_VDIM = 0> static
 void PAVectorDiffusionApply2D(const int NE,
@@ -252,7 +451,7 @@ void PAVectorDiffusionApply2D(const int NE,
    auto D = Reshape(d_.Read(), Q1D*Q1D, 3, NE);
    auto x = Reshape(x_.Read(), D1D, D1D, VDIM, NE);
    auto y = Reshape(y_.ReadWrite(), D1D, D1D, VDIM, NE);
-   MFEM_FORALL(e, NE,
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -373,7 +572,7 @@ void PAVectorDiffusionApply3D(const int NE,
    auto op = Reshape(op_.Read(), Q1D*Q1D*Q1D, 6, NE);
    auto x = Reshape(x_.Read(), D1D, D1D, D1D, VDIM, NE);
    auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
-   MFEM_FORALL(e, NE,
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -581,212 +780,4 @@ void VectorDiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
    }
 }
 
-template<int T_D1D = 0, int T_Q1D = 0>
-static void PAVectorDiffusionDiagonal2D(const int NE,
-                                        const Array<double> &b,
-                                        const Array<double> &g,
-                                        const Vector &d,
-                                        Vector &y,
-                                        const int d1d = 0,
-                                        const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto G = Reshape(g.Read(), Q1D, D1D);
-   // note the different shape for D, this is a (symmetric) matrix so we only
-   // store necessary entries
-   auto D = Reshape(d.Read(), Q1D*Q1D, 3, NE);
-   auto Y = Reshape(y.ReadWrite(), D1D, D1D, 2, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      // gradphi \cdot Q \gradphi has four terms
-      double QD0[MQ1][MD1];
-      double QD1[MQ1][MD1];
-      double QD2[MQ1][MD1];
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            QD0[qx][dy] = 0.0;
-            QD1[qx][dy] = 0.0;
-            QD2[qx][dy] = 0.0;
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const int q = qx + qy * Q1D;
-               const double D0 = D(q,0,e);
-               const double D1 = D(q,1,e);
-               const double D2 = D(q,2,e);
-               QD0[qx][dy] += B(qy, dy) * B(qy, dy) * D0;
-               QD1[qx][dy] += B(qy, dy) * G(qy, dy) * D1;
-               QD2[qx][dy] += G(qy, dy) * G(qy, dy) * D2;
-            }
-         }
-      }
-      for (int dy = 0; dy < D1D; ++dy)
-      {
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            double temp = 0.0;
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               temp += G(qx, dx) * G(qx, dx) * QD0[qx][dy];
-               temp += G(qx, dx) * B(qx, dx) * QD1[qx][dy];
-               temp += B(qx, dx) * G(qx, dx) * QD1[qx][dy];
-               temp += B(qx, dx) * B(qx, dx) * QD2[qx][dy];
-            }
-            Y(dx,dy,0,e) += temp;
-            Y(dx,dy,1,e) += temp;
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void PAVectorDiffusionDiagonal3D(const int NE,
-                                        const Array<double> &b,
-                                        const Array<double> &g,
-                                        const Vector &d,
-                                        Vector &y,
-                                        const int d1d = 0,
-                                        const int q1d = 0)
-{
-   constexpr int DIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-   MFEM_VERIFY(D1D <= MD1, "");
-   MFEM_VERIFY(Q1D <= MQ1, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto G = Reshape(g.Read(), Q1D, D1D);
-   auto Q = Reshape(d.Read(), Q1D*Q1D*Q1D, 6, NE);
-   auto Y = Reshape(y.ReadWrite(), D1D, D1D, D1D, 3, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      double QQD[MQ1][MQ1][MD1];
-      double QDD[MQ1][MD1][MD1];
-      for (int i = 0; i < DIM; ++i)
-      {
-         for (int j = 0; j < DIM; ++j)
-         {
-            // first tensor contraction, along z direction
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  for (int dz = 0; dz < D1D; ++dz)
-                  {
-                     QQD[qx][qy][dz] = 0.0;
-                     for (int qz = 0; qz < Q1D; ++qz)
-                     {
-                        const int q = qx + (qy + qz * Q1D) * Q1D;
-                        const int k = j >= i ?
-                        3 - (3-i)*(2-i)/2 + j:
-                        3 - (3-j)*(2-j)/2 + i;
-                        const double O = Q(q,k,e);
-                        const double Bz = B(qz,dz);
-                        const double Gz = G(qz,dz);
-                        const double L = i==2 ? Gz : Bz;
-                        const double R = j==2 ? Gz : Bz;
-                        QQD[qx][qy][dz] += L * O * R;
-                     }
-                  }
-               }
-            }
-            // second tensor contraction, along y direction
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  for (int dy = 0; dy < D1D; ++dy)
-                  {
-                     QDD[qx][dy][dz] = 0.0;
-                     for (int qy = 0; qy < Q1D; ++qy)
-                     {
-                        const double By = B(qy,dy);
-                        const double Gy = G(qy,dy);
-                        const double L = i==1 ? Gy : By;
-                        const double R = j==1 ? Gy : By;
-                        QDD[qx][dy][dz] += L * QQD[qx][qy][dz] * R;
-                     }
-                  }
-               }
-            }
-            // third tensor contraction, along x direction
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               for (int dy = 0; dy < D1D; ++dy)
-               {
-                  for (int dx = 0; dx < D1D; ++dx)
-                  {
-                     double temp = 0.0;
-                     for (int qx = 0; qx < Q1D; ++qx)
-                     {
-                        const double Bx = B(qx,dx);
-                        const double Gx = G(qx,dx);
-                        const double L = i==0 ? Gx : Bx;
-                        const double R = j==0 ? Gx : Bx;
-                        temp += L * QDD[qx][dy][dz] * R;
-                     }
-                     Y(dx, dy, dz, 0, e) += temp;
-                     Y(dx, dy, dz, 1, e) += temp;
-                     Y(dx, dy, dz, 2, e) += temp;
-                  }
-               }
-            }
-         }
-      }
-   });
-}
-
-static void PAVectorDiffusionAssembleDiagonal(const int dim,
-                                              const int D1D,
-                                              const int Q1D,
-                                              const int NE,
-                                              const Array<double> &B,
-                                              const Array<double> &G,
-                                              const Vector &op,
-                                              Vector &y)
-{
-   if (dim == 2)
-   {
-      return PAVectorDiffusionDiagonal2D(NE, B, G, op, y, D1D, Q1D);
-   }
-   else if (dim == 3)
-   {
-      return PAVectorDiffusionDiagonal3D(NE, B, G, op, y, D1D, Q1D);
-   }
-   MFEM_ABORT("Dimension not implemented.");
-}
-
-void VectorDiffusionIntegrator::AssembleDiagonalPA(Vector &diag)
-{
-   if (DeviceCanUseCeed())
-   {
-      ceedOp->GetDiagonal(diag);
-   }
-   else
-   {
-      PAVectorDiffusionAssembleDiagonal(dim,
-                                        dofs1D,
-                                        quad1D,
-                                        ne,
-                                        maps->B,
-                                        maps->G,
-                                        pa_data,
-                                        diag);
-   }
-}
-
 } // namespace mfem

fem/integ/bilininteg_vecdiv_pa.cpp

@@ -9,17 +9,13 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-
-using namespace std;
+#include "../../general/forall.hpp"
+#include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
 
 namespace mfem
 {
 
-// PA Divergence Integrator
-
 // PA Divergence Assemble 2D kernel
 static void PADivergenceSetup2D(const int Q1D,
                                 const int NE,
@@ -33,7 +29,7 @@ static void PADivergenceSetup2D(const int Q1D,
    auto J = Reshape(j.Read(), NQ, 2, 2, NE);
    auto y = Reshape(op.Write(), NQ, 2, 2, NE);
 
-   MFEM_FORALL(e, NE,
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
       for (int q = 0; q < NQ; ++q)
       {
@@ -62,7 +58,7 @@ static void PADivergenceSetup3D(const int Q1D,
    auto W = w.Read();
    auto J = Reshape(j.Read(), NQ, 3, 3, NE);
    auto y = Reshape(op.Write(), NQ, 3, 3, NE);
-   MFEM_FORALL(e, NE,
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
       for (int q = 0; q < NQ; ++q)
       {
@@ -183,7 +179,7 @@ static void PADivergenceApply2D(const int NE,
    auto op = Reshape(op_.Read(), Q1D*Q1D, 2,2, NE);
    auto x = Reshape(x_.Read(), TR_D1D, TR_D1D, 2, NE);
    auto y = Reshape(y_.ReadWrite(), TE_D1D, TE_D1D, NE);
-   MFEM_FORALL(e, NE,
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
       const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
       const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -321,7 +317,7 @@ static void PADivergenceApplyTranspose2D(const int NE,
    auto op = Reshape(op_.Read(), Q1D*Q1D, 2,2, NE);
    auto x  = Reshape(x_.Read(), TE_D1D, TE_D1D, NE);
    auto y  = Reshape(y_.ReadWrite(), TR_D1D, TR_D1D, 2, NE);
-   MFEM_FORALL(e, NE,
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
       const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
       const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -437,7 +433,7 @@ static void PADivergenceApply3D(const int NE,
    auto op = Reshape(op_.Read(), Q1D*Q1D*Q1D, 3,3, NE);
    auto x = Reshape(x_.Read(), TR_D1D, TR_D1D, TR_D1D, 3, NE);
    auto y = Reshape(y_.ReadWrite(), TE_D1D, TE_D1D, TE_D1D, NE);
-   MFEM_FORALL(e, NE,
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
       const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
       const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -620,7 +616,7 @@ static void PADivergenceApplyTranspose3D(const int NE,
    auto op = Reshape(op_.Read(), Q1D*Q1D*Q1D, 3,3, NE);
    auto x  = Reshape(x_.Read(), TE_D1D, TE_D1D, TE_D1D, NE);
    auto y  = Reshape(y_.ReadWrite(), TR_D1D, TR_D1D, TR_D1D, 3, NE);
-   MFEM_FORALL(e, NE,
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
       const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
       const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -801,7 +797,7 @@ static void SmemPADivergenceApply3D(const int NE,
    auto x = Reshape(x_.Read(), TR_D1D, TR_D1D, TR_D1D, 3, NE);
    auto y = Reshape(y_.ReadWrite(), TE_D1D, TE_D1D, TE_D1D, NE);
 
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
    {
       constexpr int VDIM = 3;
       const int tidz = MFEM_THREAD_ID(z);
@@ -1040,11 +1036,25 @@ static void PADivergenceApply(const int dim,
 {
    if (dim == 2)
    {
-      return PADivergenceApply2D(NE,B,G,Bt,op,x,y,TR_D1D,TE_D1D,Q1D);
+      if (transpose)
+      {
+         return PADivergenceApplyTranspose2D(NE,B,G,Bt,op,x,y,TR_D1D,TE_D1D,Q1D);
+      }
+      else
+      {
+         return PADivergenceApply2D(NE,B,G,Bt,op,x,y,TR_D1D,TE_D1D,Q1D);
+      }
    }
    if (dim == 3)
    {
-      return PADivergenceApply3D(NE,B,G,Bt,op,x,y,TR_D1D,TE_D1D,Q1D);
+      if (transpose)
+      {
+         return PADivergenceApplyTranspose3D(NE,B,G,Bt,op,x,y,TR_D1D,TE_D1D,Q1D);
+      }
+      else
+      {
+         return PADivergenceApply3D(NE,B,G,Bt,op,x,y,TR_D1D,TE_D1D,Q1D);
+      }
    }
    MFEM_ABORT("Unknown kernel.");
 }