Merge branch 'gpu_em' into gpu-maxwell

Just use the regular Maxwell miniapp for full assembly.

.github/workflows/README.md

@@ -29,12 +29,16 @@ Runs a number of static repository-level sanity checks.
 
 - `branch-history` guards against accidental commits of large files using the `--history` option of the `config/githooks/pre-push` script.
 
+## `mfem-analysis.yml` (`build-analysis`)
+
+Checks if the code builds and satisfies minimal requirements.
+
+- `gitignore` builds hypre, METIS, and MFEM using `mfem/github-actions/build-hypre`, `mfem/github-actions/build-metis`, and `mfem/github-actions/build-mfem` and checks for correct `.gitignore` settings by running the `tests/scripts/gitignore` script.
+
 ## `builds-and-tests.yml`
 
 Runs a matrix of builds and tests runs with different compilers, OS, mfem/hypre settings, etc. Also processes and upload Codecov reports.
 
-One matrix job runs `tests/scripts/gitignore` after `make test-noclean` to check generated artifacts against `.gitignore`.
-
 Uses the following GitHub Actions from <https://github.com/mfem/github-actions>:
 
 - `mfem/github-actions/build-hypre`

.github/workflows/builds-and-tests.yml

@@ -111,7 +111,6 @@ jobs:
             build-system: make
             hypre-target: int64
             precision: fp64
-            gitignore-check: YES
           - os: ubuntu-latest
             target: opt
             codecov: NO
@@ -142,10 +141,6 @@ jobs:
 
     continue-on-error: ${{ matrix.enzyme && true || false }}
 
-    # Enable ccache for all jobs except Windows (would need sccache).
-    env:
-      USE_CCACHE: ${{ matrix.os != 'windows-latest' }}
-
     steps:
       # Fix 'No space left on device' errors for Ubuntu builds.
       - name: Run Actions Cleaner
@@ -294,52 +289,6 @@ jobs:
           echo "OMPI_CC=$LLVM_PREFIX/bin/clang" >> $GITHUB_ENV
           echo "OMPI_CXX=$LLVM_PREFIX/bin/clang++" >> $GITHUB_ENV
 
-      # Restore the compiler cache (ccache). The key embeds the run id, so new
-      # runs save a fresh snapshot; the restore-keys prefix warm-starts from the
-      # most recent prior run (incl. the base branch for PRs).
-      - name: cache ccache
-        if: ${{ env.USE_CCACHE == 'true' }}
-        uses: actions/cache@v5
-        with:
-          path: .ccache
-          key: ccache-${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}${{ matrix.enzyme && '-enzyme' || '' }}-${{ github.run_id }}
-          restore-keys: |
-            ccache-${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}${{ matrix.enzyme && '-enzyme' || '' }}-
-
-      # Configure ccache and select how it is injected into the MFEM build:
-      # - make:  set CXX="ccache g++"; for MPI, OMPI_CXX="ccache g++" so mpicxx
-      #          runs ccache around g++ (not ccache around the mpicxx wrapper).
-      # - cmake: set CMAKE_<LANG>_COMPILER_LAUNCHER=ccache.
-      # - enzyme: wrap the brew clang++ via OMPI_CXX.
-      # The chosen options are passed through build-mfem's 'config-options'
-      # input (see the build step below).
-      - name: configure ccache
-        if: ${{ env.USE_CCACHE == 'true' }}
-        run: |
-          command -v ccache >/dev/null 2>&1 || {
-            if [[ "${{ runner.os }}" == "Linux" ]]; then
-              sudo apt-get update && sudo apt-get install -y ccache
-            else
-              brew install ccache
-            fi
-          }
-          echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV
-          echo "CCACHE_MAXSIZE=1G" >> $GITHUB_ENV
-          echo "CCACHE_COMPILERCHECK=content" >> $GITHUB_ENV
-          # Ignore header timestamps (restamped by each checkout) so direct mode hits.
-          echo "CCACHE_SLOPPINESS=include_file_mtime,include_file_ctime,time_macros" >> $GITHUB_ENV
-          # Hash absolute paths relative to the workspace.
-          echo "CCACHE_BASEDIR=${{ github.workspace }}" >> $GITHUB_ENV
-          if [[ "${{ matrix.enzyme }}" == "true" ]]; then
-            echo "OMPI_CXX=ccache $LLVM_PREFIX/bin/clang++" >> $GITHUB_ENV
-          elif [[ "${{ matrix.build-system }}" == "cmake" ]]; then
-            echo 'CCACHE_CONFIG_OPTS=-DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache' >> $GITHUB_ENV
-          else
-            echo "OMPI_CXX=ccache g++" >> $GITHUB_ENV
-            echo 'CCACHE_CONFIG_OPTS=CXX="ccache g++" MPICXX="mpicxx"' >> $GITHUB_ENV
-          fi
-        shell: bash
-
       # MFEM build and test
       - name: build
         uses: mfem/github-actions/build-mfem@v2.7
@@ -355,14 +304,9 @@ jobs:
           metis-dir: ${{ env.METIS_TOP_DIR }}
           mfem-dir: ${{ env.MFEM_TOP_DIR }}
           precision: ${{ matrix.precision }}
-          config-options: ${{ matrix.config-opts }} ${{ env.CCACHE_CONFIG_OPTS }}
+          config-options: ${{ matrix.config-opts }}
           library-only: ${{ matrix.target == 'dbg' && matrix.os != 'ubuntu-latest' }}
 
-      - name: ccache stats
-        if: ${{ env.USE_CCACHE == 'true' }}
-        run: ccache -s
-        shell: bash
-
       # Run checks (and only checks) on debug targets
       - name: checks
         if: matrix.build-system == 'make' && matrix.target == 'dbg'
@@ -373,13 +317,7 @@ jobs:
       - name: tests
         if: matrix.build-system == 'make' && (matrix.target == 'opt' || matrix.os == 'ubuntu-latest')
         run: |
-          cd ${{ env.MFEM_TOP_DIR }}
-          if [[ "${{ matrix.gitignore-check }}" == "YES" ]]; then
-            make test-noclean
-          else
-            make test
-          fi
-        shell: bash
+          cd ${{ env.MFEM_TOP_DIR }} && make test
 
       - name: cmake checks
         if: matrix.build-system == 'cmake' && matrix.target == 'dbg'
@@ -431,9 +369,3 @@ jobs:
           directories: "fem general linalg mesh"
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-
-      - name: gitignore
-        if: matrix.gitignore-check == 'YES'
-        run: |
-          cd ${{ env.MFEM_TOP_DIR }}/tests/scripts
-          ./runtest gitignore

.github/workflows/cleanup-pr-caches.yml

@@ -1,42 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
----
-# A closed PR's caches can never be restored again, so delete them to free
-# space against the 10 GB per-repo cache limit.
-name: Cleanup PR caches
-
-on:
-  pull_request:
-    types: [closed]
-
-permissions:
-  actions: write
-
-jobs:
-  cleanup:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Delete caches for the closed PR
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          GH_REPO: ${{ github.repository }}
-          PR_REF: refs/pull/${{ github.event.pull_request.number }}/merge
-        run: |
-          echo "Deleting caches for $PR_REF"
-          while :; do
-            ids=$(gh cache list --ref "$PR_REF" --limit 100 --json id --jq '.[].id')
-            [ -n "$ids" ] || break
-            echo "$ids" | while read -r id; do
-              [ -n "$id" ] || continue
-              echo "Deleting cache $id"
-              gh cache delete "$id" || echo "  (already gone)"
-            done
-          done

.github/workflows/codeql-analysis.yml

@@ -14,19 +14,9 @@ name: "Static Analysis"
 on:
   push:
     branches: ["master", "next"]
-    paths-ignore: &docs-only-paths
-      - "**/*.md"
-      - "doc/**"
-      - ".binder/**"
-      - "CITATION.cff"
-      - "LICENSE"
-      - "NOTICE"
-      - "CHANGELOG"
-      - "INSTALL"
   pull_request:
     # The branches below must be a subset of the branches above
     branches: ["master"]
-    paths-ignore: *docs-only-paths
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

.github/workflows/mfem-analysis.yml

@@ -0,0 +1,102 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+name: "Build Analysis"
+
+permissions:
+  actions: write
+
+on:
+  push:
+    branches:
+      - master
+      - next
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  HYPRE_ARCHIVE: v2.19.0.tar.gz
+  HYPRE_TOP_DIR: hypre-2.19.0
+  METIS_ARCHIVE: metis-4.0.3.tar.gz
+  METIS_TOP_DIR: metis-4.0.3
+  COVERAGE_ENV: mfem-coverage
+  MFEM_ACTIONS_VERSION: v2.7
+
+jobs:
+  gitignore:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: checkout MFEM
+        uses: actions/checkout@v6
+        with:
+          path: mfem
+
+      - name: Get MPI (Linux)
+        run: |
+          sudo apt-get install openmpi-bin libopenmpi-dev
+          export OMPI_MCA_rmaps_base_oversubscribe=1
+
+      - name: Cache Hypre Install
+        id: hypre-cache
+        uses: actions/cache@v5
+        with:
+          path: ${{ env.HYPRE_TOP_DIR }}
+          key: ${{ runner.os }}-ompi-build-${{ env.HYPRE_TOP_DIR }}-int32-fp64-${{ env.MFEM_ACTIONS_VERSION }}
+
+      - name: Get Hypre
+        if: steps.hypre-cache.outputs.cache-hit != 'true'
+        uses: mfem/github-actions/build-hypre@v2.7
+        with:
+          archive: ${{ env.HYPRE_ARCHIVE }}
+          dir: ${{ env.HYPRE_TOP_DIR }}
+          target: int32
+          precision: fp64
+
+      - name: Cache Metis Install
+        id: metis-cache
+        uses: actions/cache@v5
+        with:
+          path: ${{ env.METIS_TOP_DIR }}
+          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-${{ env.MFEM_ACTIONS_VERSION }}
+
+      - name: Install Metis
+        if: steps.metis-cache.outputs.cache-hit != 'true'
+        uses: mfem/github-actions/build-metis@v2.7
+        with:
+          archive: ${{ env.METIS_ARCHIVE }}
+          dir: ${{ env.METIS_TOP_DIR }}
+
+      # MFEM build and test
+      - name: build-mfem
+        uses: mfem/github-actions/build-mfem@v2.7
+        with:
+          os: ${{ runner.os }}
+          target: opt
+          codecov: NO
+          mpi: par
+          build-system: make
+          hypre-dir: ${{ env.HYPRE_TOP_DIR }}
+          metis-dir: ${{ env.METIS_TOP_DIR }}
+          mfem-dir: mfem
+
+      - name: test (no clean)
+        run: |
+          cd mfem && make test-noclean
+
+      - name: gitignore
+        run: |
+          cd mfem/tests/scripts
+          ./runtest gitignore

.github/workflows/repo-check.yml

@@ -13,7 +13,6 @@ name: "Checks"
 
 permissions:
   actions: write
-  pull-requests: read
 
 on:
   push:
@@ -30,11 +29,6 @@ concurrency:
 # by checking if the workflow trigger is 'push' ("github.event_name == 'push'")
 # and if we are in a fork ("github.event.pull_request.head.repo.full_name !=
 # github.repository").
-#
-# The logic for the branch-history check is slightly different, since that check
-# also inspects the PR's labels to allow for overriding failures. In this case,
-# we run on all 'pull_request' triggers, but only run for 'push' triggers that
-# do not correspond to any open PRs.
 
 jobs:
   file-headers-check:
@@ -134,7 +128,10 @@ jobs:
 
   branch-history:
     if: |
-      github.ref != 'refs/heads/next' && github.ref != 'refs/heads/master'
+      github.ref != 'refs/heads/next' &&
+      github.ref != 'refs/heads/master' &&
+      (github.event_name == 'push' ||
+       github.event.pull_request.head.repo.full_name != github.repository)
     runs-on: ubuntu-latest
     steps:
       - name: checkout mfem
@@ -142,27 +139,7 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: check for pull request
-        id: check_pr
-        if: github.event_name == 'push'
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          pr_exists=$(gh pr list --repo "$GITHUB_REPOSITORY" \
-            --head "$GITHUB_REF_NAME" \
-            --state open \
-            --json number \
-            --jq 'length > 0')
-          echo "pr_exists=$pr_exists" >> "$GITHUB_OUTPUT"
-
       - name: branch-history
-        id: branch_history
-        if: |
-          (github.event_name == 'pull_request' ||
-           github.event_name == 'workflow_dispatch' ||
-           steps.check_pr.outputs.pr_exists == 'false')
-        continue-on-error: ${{ contains(github.event.pull_request.labels.*.name,
-                                        'branch-history-override') }}
         run: |
           # We override origin to make sure we point to the main repo.
           # This is to have consistent test results on PRs from forks.
@@ -170,20 +147,3 @@ jobs:
           git remote add origin https://github.com/mfem/mfem.git
           git checkout -b gh-actions-branch-history
           ./config/githooks/pre-push --history
-
-      - name: report branch-history override
-        if: steps.branch_history.outcome == 'failure'
-        run: |
-          echo "::warning::branch-history check failed, but the" \
-            "'branch-history-override' label is set."
-
-  codespell:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Codespell with annotations
-      uses: codespell-project/actions-codespell@master
-      with:
-        check_filenames: true
-        check_hidden: true
-        ignore_words_list: allright,ba,equil,esy,fo,hda,lod,nd,ned,numer,ot,pres,ro,seh,shat,solfes,strat,tbe,te,warmup
-

.github/workflows/sanitizers.yml

@@ -17,17 +17,7 @@ permissions:
 on:
   push:
     branches: ["master", "next"]
-    paths-ignore: &docs-only-paths
-      - "**/*.md"
-      - "doc/**"
-      - ".binder/**"
-      - "CITATION.cff"
-      - "LICENSE"
-      - "NOTICE"
-      - "CHANGELOG"
-      - "INSTALL"
   pull_request:
-    paths-ignore: *docs-only-paths
   workflow_dispatch:
 
 concurrency:

.gitignore

@@ -260,7 +260,6 @@ miniapps/meshing/polar-nc
 miniapps/meshing/mesh-quality
 miniapps/meshing/hpref
 miniapps/meshing/phpref
-miniapps/meshing/pref321
 miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
 miniapps/meshing/toroid-*.mesh

.gitlab/dane-baseline.yml

@@ -102,14 +102,12 @@ report_baseline:
         mkdir -p ${MACHINE_NAME}
         rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-${BASELINE_TEST}-${CI_COMMIT_REF_SLUG}"
         rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir ${rundir})
-        status=0
-        cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir} || { status=1; }
+        cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
         printf "%s\n" "" "Pipeline URL:" "$CI_PIPELINE_URL" \
           >> ${rundir}/pipeline.txt
         # We create an autotest-email.html file, because that's how we signal
         # that there was an error / diff (temporary).
-        if [[ $status -ne 0 ]] || \
-           [[ -f ${rundir}/${BASELINE_TEST}.err ]] || \
+        if [[ -f ${rundir}/${BASELINE_TEST}.err ]] || \
            [[ -f ${rundir}/${BASELINE_TEST}-${MACHINE_NAME}.diff ]]; then
           cp ${rundir}/pipeline.txt ${rundir}/autotest-email.html
         fi

CHANGELOG

@@ -15,10 +15,6 @@ Version 4.9.1 (development)
 
 Discretization improvements
 ---------------------------
-- Improved FindPointsGSLIB surface mesh capability with support for simplices
-  and an option to specify axis-aligned bounding box padding for near-surface
-  point queries.
-
 - Added GPU-enabled partial assembly for simplicial Bernstein H1 basis based on
   ragged tensor algorithms (see DOI: 10.1137/11082539X) for mass and diffusion
   integrators.
@@ -46,19 +42,8 @@ Discretization improvements
 
 - Extend FindPointsGSLIB to support surface meshes.
 
-- Added support for complex-valued mixed bilinear forms via the new classes
-  MixedSesquilinearForm and ParMixedSesquilinearForm, mirroring the existing
-  SesquilinearForm classes. Rectangular complex operators are now also
-  handled correctly by ComplexSparseMatrix::GetSystemMatrix and
-  ComplexHypreParMatrix::GetSystemMatrix, which previously assumed equal
-  trial and test spaces.
-
 Meshing improvements
 --------------------
-- Added support for nonuniform anisotropic mesh refinement on parallel quad/hex
-  meshes with arbitrary spacing in each direction. This enables in particular
-  3:1 refinement in parallel, as demonstrated in the new meshing miniapp pref321.
-
 - Added option to guarantee mesh validity during TMOP-based r-adaptivity, using
   bounds on the determinant of the mesh transformation Jacobian.
 
@@ -81,15 +66,6 @@ Linear and nonlinear solvers
 
 GPU computing
 -------------
-- Improved partial assembly for VectorDivergenceIntegrator with shared-memory
-  kernels, kernel registration, and transpose support.
-
-- Improved partial-assembly diagonal kernels for VectorMassIntegrator (shared-
-  memory specializations) and ElasticityIntegrator (no scratch Q-vector).
-
-- Added PA gradient and diagonal support for VectorConvectionNLFIntegrator
- (AssembleGradPA, AddMultGradPA, AssembleGradDiagonalPA).
-
 - Added device assembly support for 3D H(curl) VectorFEDomainLFIntegrator.
 
 - Added NVIDIA cuDSS library interface. Implementation examples have been
@@ -97,10 +73,6 @@ GPU computing
   details. Supported versions >= 0.6.0.
 
 - Allow specifying GPU kernel launch bounds for native and RAJA GPU backends.
-
-- Changed VectorFEMassIntegrator to use kernel specialization dispatch for
-  partial assembly.
-
 - Added support for FiniteElement::MapType::INTEGRAL spaces to
   QuadratureInterpolator.
 

CMakeLists.txt

@@ -88,9 +88,18 @@ if (MFEM_USE_STRUMPACK OR MFEM_USE_MUMPS)
   # Just needed to find the MPI_Fortran libraries to link with
   set(XSDK_ENABLE_Fortran ON)
 endif()
-# RAJA requires C++20:
-if (MFEM_USE_RAJA AND ("${CMAKE_CXX_STANDARD}" LESS "20"))
-  set(CMAKE_CXX_STANDARD 20 CACHE STRING "C++ standard to use." FORCE)
+# Ginkgo requires C++17:
+if ((MFEM_USE_GINKGO) AND ("${CMAKE_CXX_STANDARD}" LESS "17"))
+  set(CMAKE_CXX_STANDARD 17 CACHE STRING "C++ standard to use." FORCE)
+# Google Benchmark, SUNDIALS, STRUMPACK, Tribol, RAJA and Umpire require C++14:
+elseif ((MFEM_USE_BENCHMARK OR
+         MFEM_USE_SUNDIALS OR
+         MFEM_USE_STRUMPACK OR
+         MFEM_USE_TRIBOL OR
+         MFEM_USE_RAJA OR
+         MFEM_USE_UMPIRE) AND
+    ("${CMAKE_CXX_STANDARD}" LESS "14"))
+  set(CMAKE_CXX_STANDARD 14 CACHE STRING "C++ standard to use." FORCE)
 endif()
 
 # Include xSDK default CMake file.
@@ -230,13 +239,6 @@ else()
   set(MFEM_DEBUG OFF)
 endif()
 
-# Shadow warnings for clang only; GCC's -Wshadow flags more.
-if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
-  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -pedantic -Wall -Wshadow")
-elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -pedantic -Wall")
-endif()
-
 # Shared build on Windows
 if (WIN32 AND BUILD_SHARED_LIBS)
   # CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS works only with MSVC?

CONTRIBUTING.md

@@ -6,6 +6,7 @@
 <a href="https://github.com/mfem/mfem/blob/master/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-BSD-blue.svg"></a>
 <a href="https://github.com/mfem/mfem/releases/latest"><img alt="GitHub release" src="https://img.shields.io/github/v/release/mfem/mfem"></a>
 <a href="https://github.com/mfem/mfem/actions/workflows/repo-check.yml?query=branch%3Amaster"><img alt="Repo check" src="https://github.com/mfem/mfem/actions/workflows/repo-check.yml/badge.svg?branch=master"></a>
+<a href="https://github.com/mfem/mfem/actions/workflows/mfem-analysis.yml?query=branch%3Amaster"><img alt="Build Analysis" src="https://github.com/mfem/mfem/actions/workflows/mfem-analysis.yml/badge.svg?branch=master"></a>
 <a href="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml?query=branch%3Amaster"><img alt="Builds and Tests" src="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml/badge.svg?branch=master"></a>
 <a href="https://ci.appveyor.com/project/mfem/mfem"><img alt="Build Status" src="https://ci.appveyor.com/api/projects/status/19non9sqm6msi2wy?svg=true"></a>
 <a href="https://docs.mfem.org/html/index.html"><img alt="Documentation" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>

config/cmake/modules/FindSuiteSparse.cmake

@@ -22,15 +22,15 @@ include(MfemCmakeUtilities)
 mfem_find_package(SuiteSparse SuiteSparse SuiteSparse_DIR "" "" "" ""
   "Paths to headers required by SuiteSparse."
   "Libraries required by SuiteSparse."
-  ADD_COMPONENT "UMFPACK" "include;include/suitesparse;suitesparse" umfpack.h "lib" umfpack
-  ADD_COMPONENT "KLU" "include;include/suitesparse;suitesparse" klu.h "lib" klu
-  ADD_COMPONENT "AMD" "include;include/suitesparse;suitesparse" amd.h "lib" amd
-  ADD_COMPONENT "BTF" "include;include/suitesparse;suitesparse" btf.h "lib" btf
-  ADD_COMPONENT "CHOLMOD" "include;include/suitesparse;suitesparse" cholmod.h "lib" cholmod
-  ADD_COMPONENT "COLAMD" "include;include/suitesparse;suitesparse" colamd.h "lib" colamd
-  ADD_COMPONENT "CAMD" "include;include/suitesparse;suitesparse" camd.h "lib" camd
-  ADD_COMPONENT "CCOLAMD" "include;include/suitesparse;suitesparse" ccolamd.h "lib" ccolamd
-  ADD_COMPONENT "config" "include;include/suitesparse;suitesparse" SuiteSparse_config.h "lib"
+  ADD_COMPONENT "UMFPACK" "include;suitesparse" umfpack.h "lib" umfpack
+  ADD_COMPONENT "KLU" "include;suitesparse" klu.h "lib" klu
+  ADD_COMPONENT "AMD" "include;suitesparse" amd.h "lib" amd
+  ADD_COMPONENT "BTF" "include;suitesparse" btf.h "lib" btf
+  ADD_COMPONENT "CHOLMOD" "include;suitesparse" cholmod.h "lib" cholmod
+  ADD_COMPONENT "COLAMD" "include;suitesparse" colamd.h "lib" colamd
+  ADD_COMPONENT "CAMD" "include;suitesparse" camd.h "lib" camd
+  ADD_COMPONENT "CCOLAMD" "include;suitesparse" ccolamd.h "lib" ccolamd
+  ADD_COMPONENT "config" "include;suitesparse" SuiteSparse_config.h "lib"
     suitesparseconfig)
 
 if (SuiteSparse_FOUND AND METIS_VERSION_5)

config/defaults.mk

@@ -27,13 +27,7 @@ MPICXX = mpicxx
 
 BASE_FLAGS  = -std=c++17
 OPTIM_FLAGS = -O3 $(BASE_FLAGS)
-
-# Shadow warnings for clang only; GCC's -Wshadow flags more.
-SHADOW_WARNING_FLAG = $(if $(findstring clang,\
- $(shell $(MFEM_HOST_CXX) --version 2>/dev/null)),-Wshadow,)
-WARNING_FLAGS = -pedantic -Wall $(SHADOW_WARNING_FLAG)
-
-DEBUG_FLAGS = $(strip -g $(addprefix $(XCOMPILER),$(WARNING_FLAGS)) $(BASE_FLAGS))
+DEBUG_FLAGS = -g $(XCOMPILER)-Wall $(BASE_FLAGS)
 
 # Prefixes for passing flags to the compiler and linker when using CXX or MPICXX
 CXX_XCOMPILER =

config/githooks/README.md

@@ -39,8 +39,3 @@ when a picture was added for documentation.
 If that is the case, make sure the failure is indeed justified, and rerun the
 push command with the `--no-verify` option. This will skip the hooks, allowing
 you to push those changes.
-
-The `branch-history` check is run automatically through GitHub Actions. If a
-branch is known to have a large number of changes that are legitimate, the
-check can be overridden by setting  the label 'branch-history-override' on the
-pull request.

data/tinyzoo-2d.mesh

@@ -1,38 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see fem/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-# PYRAMID     = 7
-
-dimension
-2
-
-elements
-2
-1 3 0 1 4 3
-1 2 1 2 4
-
-boundary
-5
-1 1 0 1
-1 1 1 2
-1 1 2 4
-1 1 4 3
-1 1 3 0
-
-vertices
-5
-2
-0 0
-1 0
-2 0
-0 1
-1 1

doc/CodeDocumentation.dox

@@ -201,7 +201,6 @@ namespace mfem {
  * - <a class="el" href="nurbs__naca__cmesh_8cpp_source.html">NURBS NACA Mesher</a>: generate NURBS based mesh around a NACA foil
  * - <a class="el" href="nurbs__printfunc_8cpp_source.html">NURBS Printer</a>: print the NURBS-basis
  * - <a class="el" href="nurbs__mesh_info_8cpp_source.html">NURBS Mesh info</a>: print the info of a NURBS mesh
- * - <a class="el" href="nurbs__surface_8cpp_source.html">NURBS Surface</a>: interpolate a 3D Surface in a NURBS Patch
  *
  * <H3>Miniapps</H3>
  * - <a class="el" href="volta_8cpp_source.html">Volta</a>: simple electrostatics simulation code
@@ -246,9 +245,6 @@ namespace mfem {
  * - <a class="el" href="pdiffusion_8cpp_source.html">DPG Diffusion example</a>: DPG formulation for the diffusion problem
  * - <a class="el" href="pmaxwell_8cpp_source.html">DPG Maxwell example</a>: DPG formulation for the indefinite Maxwell problem
  * - <a class="el" href="lor__elast_8cpp_source.html">LOR Elasticity</a>: solve linear elasticity with LOR preconditioning on GPUs
- * - <a class="el" href="reflector_8cpp_source.html">Reflector Miniapp</a>: reflect a mesh about a plane
- * - <a class="el" href="ref321_8cpp_source.html">3:1 Refinement Miniapp</a>: perform 3:1 anisotropic mesh refinements
- * - <a class="el" href="pref321_8cpp_source.html">3:1 Refinement Miniapp</a>: parallel 3:1 anisotropic mesh refinements
  *
  * See also the <a class="el" href="https://mfem.org/examples/">examples documentation</a> online.
  */

examples/ex27.cpp

@@ -177,7 +177,7 @@ int main(int argc, char *argv[])
    Array<int> ess_tdof_list(0);
    if (h1 && mesh->bdr_attributes.Size())
    {
-      // For a continuous basis the linear system must be modifed to enforce an
+      // For a continuous basis the linear system must be modified to enforce an
       // essential (Dirichlet) boundary condition. In the DG case this is not
       // necessary as the boundary condition will only be enforced weakly.
       fespace.GetEssentialTrueDofs(dbc_bdr, ess_tdof_list);

fem/CMakeLists.txt

@@ -57,8 +57,6 @@ set(SRCS
   integ/lininteg_domain_grad.cpp
   integ/lininteg_domain_vectorfe.cpp
   integ/nonlininteg_vecconvection_pa.cpp
-  integ/nonlininteg_vecconvection_pa_diag.cpp
-  integ/nonlininteg_vecconvection_pa_grad.cpp
   integ/nonlininteg_vecconvection_mf.cpp
   coefficient.cpp
   complex_fem.cpp
@@ -135,7 +133,7 @@ set(SRCS
   tmop/assemble/diag2.cpp
   tmop/assemble/grad2_limit.cpp
   tmop/assemble/grad2.cpp
-  tmop/assemble/diag3_limit.cpp
+  tmop/assemble/diag3_limit.cpp  
   tmop/assemble/diag3.cpp
   tmop/assemble/grad3_limit.cpp
   tmop/assemble/grad3.cpp
@@ -206,11 +204,7 @@ set(HDRS
   integ/bilininteg_mass_kernels.hpp
   integ/bilininteg_mass_pa_simplices.hpp
   integ/bilininteg_vecdiffusion_pa.hpp
-  integ/bilininteg_vecdiv_pa.hpp
   integ/bilininteg_vecmass_pa.hpp
-  integ/nonlininteg_vecconvection_pa.hpp
-  integ/nonlininteg_vecconvection_pa_diag.hpp
-  integ/nonlininteg_vecconvection_pa_grad.hpp
   coefficient.hpp
   complex_fem.hpp
   convergence.hpp
@@ -317,7 +311,6 @@ set(HDRS
   tmop_tools.hpp
   tmop_amr.hpp
   gslib.hpp
-  gslib/gslib_kernel_helpers.hpp
   transfer.hpp
   hyperbolic.hpp
   integrator.hpp

fem/bilinearform.cpp

@@ -1255,6 +1255,31 @@ void BilinearForm::Mult(const Vector &x, Vector &y) const
    }
 }
 
+void BilinearForm::AddMult(const Vector &x, Vector &y, const real_t a) const
+{
+   if (ext)
+   {
+      ext->AddMult(x, y, a);
+   }
+   else
+   {
+      mat->AddMult(x, y, a);
+   }
+}
+
+void BilinearForm::AddMultTranspose(const Vector &x, Vector &y,
+                                    const real_t a) const
+{
+   if (ext)
+   {
+      ext->AddMultTranspose(x, y, a);
+   }
+   else
+   {
+      mat->AddMultTranspose(x, y, a);
+   }
+}
+
 void BilinearForm::MultTranspose(const Vector & x, Vector & y) const
 {
    if (ext)

fem/bilinearform.hpp

@@ -307,8 +307,8 @@ public:
    { mat->Mult(x, y); mat_e->AddMult(x, y); }
 
    /// Add the matrix vector multiple to a vector:  $ y += a M x $
-   void AddMult(const Vector &x, Vector &y, const real_t a = 1.0) const override
-   { mat -> AddMult (x, y, a); }
+   void AddMult(const Vector &x, Vector &y,
+                const real_t a = 1.0) const override;
 
    /** @brief Add the original uneliminated matrix vector multiple to a vector.
        The original matrix is $ M + Me $ so we have:
@@ -318,8 +318,7 @@ public:
 
    /// Add the matrix transpose vector multiplication:  $ y += a M^T x $
    void AddMultTranspose(const Vector & x, Vector & y,
-                         const real_t a = 1.0) const override
-   { mat->AddMultTranspose(x, y, a); }
+                         const real_t a = 1.0) const override;
 
    /** @brief Add the original uneliminated matrix transpose vector
        multiple to a vector. The original matrix is $ M + M_e $

fem/bilininteg.hpp

@@ -1110,20 +1110,36 @@ public:
    MixedDotProductIntegrator(VectorCoefficient &vq)
       : MixedScalarVectorIntegrator(vq, true) {}
 
-   inline virtual bool VerifyFiniteElementTypes(
+   inline bool VerifyFiniteElementTypes(
       const FiniteElement & trial_fe,
-      const FiniteElement & test_fe) const
+      const FiniteElement & test_fe) const override
    {
       return (trial_fe.GetRangeType() == mfem::FiniteElement::VECTOR &&
               test_fe.GetRangeType()  == mfem::FiniteElement::SCALAR );
    }
 
-   inline virtual const char * FiniteElementTypeFailureMessage() const
+   inline const char * FiniteElementTypeFailureMessage() const override
    {
       return "MixedDotProductIntegrator:  "
              "Trial space must be a vector field "
              "and the test space must be a scalar field";
    }
+
+   using BilinearFormIntegrator::AssemblePA;
+   void AssemblePA(const FiniteElementSpace &trial_fes,
+                   const FiniteElementSpace &test_fes) override;
+
+   void AddMultPA(const Vector&, Vector&) const override;
+   void AddMultTransposePA(const Vector&, Vector&) const override;
+
+private:
+   Vector pa_data;
+   const DofToQuad *mapsO = nullptr;      ///< Not owned. Trial open map.
+   const DofToQuad *mapsC = nullptr;      ///< Not owned. Trial closed map.
+   const DofToQuad *mapsTest = nullptr;   ///< Not owned. Scalar test map.
+   const GeometricFactors *geom = nullptr;///< Not owned.
+   int dim = 0, ne = 0, dofs1D = 0, dofs1Dtest = 0, quad1D = 0;
+   bool test_map_integral = false;
 };
 
 /** Class for integrating the bilinear form $a(u,v) := (-\vec{V} \cdot u, \nabla \cdot v)$ in 2D or
@@ -2689,22 +2705,14 @@ public:
    void AddMultMF(const Vector &x, Vector &y) const override;
    bool SupportsCeed() const override { return DeviceCanUseCeed(); }
 
-   // PA AddMultPA kernels
    using VectorMassAddMultPAType =
       void(*)(const int,  const int,
               const Array<real_t>&, const Vector&,
               const Vector&, Vector&, const int, const int);
+
    MFEM_REGISTER_KERNELS(VectorMassAddMultPA,
                          VectorMassAddMultPAType,
                          (int, int, int));
-
-   // PA DiagonalPA kernels
-   using VectorMassAssembleDiagonalPAType =
-      void(*)(const int, const int, const int,
-              const real_t*, const real_t*, real_t*);
-   MFEM_REGISTER_KERNELS(VectorMassAssembleDiagonalPA,
-                         VectorMassAssembleDiagonalPAType,
-                         (int /*dim*/, int /*q1d*/));
 };
 
 
@@ -3129,24 +3137,6 @@ public:
    void AddMultPA(const Vector &x, Vector &y) const override;
    void AddMultTransposePA(const Vector &x, Vector &y) const override;
 
-   using VectorDivergenceAddMultPAType =
-      void (*)(const int ne,
-               const Array<real_t> &b, const Array<real_t> &g, const Array<real_t> &bt,
-               const Vector &op, const Vector &x, Vector &y,
-               const int tr_d1d, const int te_d1d, const int q1d);
-   MFEM_REGISTER_KERNELS(VectorDivergenceAddMultPA,
-                         VectorDivergenceAddMultPAType,
-                         (int, int, int, int));
-
-   using VectorDivergenceAddMultTransposePAType =
-      void (*)(const int ne,
-               const Array<real_t> &bt, const Array<real_t> &gt, const Array<real_t> &b,
-               const Vector &q, const Vector &x, Vector &y,
-               const int tr_d1d, const int te_d1d, const int q1d);
-   MFEM_REGISTER_KERNELS(VectorDivergenceAddMultTransposePA,
-                         VectorDivergenceAddMultTransposePAType,
-                         (int, int, int, int));
-
    static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
                                          const FiniteElement &test_fe,
                                          const ElementTransformation &Trans);
@@ -3955,7 +3945,7 @@ class DiscreteInterpolator : public BilinearFormIntegrator { };
 
 
 /** Class for constructing the gradient as a DiscreteLinearOperator from an
-    $H^1$-conforming space to an $H(curl$-conforming space. The range space can be
+    $H^1$-conforming space to an $H(curl)$-conforming space. The range space can be
     vector $L_2$ space as well. */
 class GradientInterpolator : public DiscreteInterpolator
 {
@@ -4064,12 +4054,48 @@ public:
     discrete curl matrix. */
 class CurlInterpolator : public DiscreteInterpolator
 {
+   int dim, ne;
+   // "dof" are the domain fespace dof counts
+   int ndof_o;
+   // "quads" are the range fespace dof counts
+   int nquad_o;
+
+   Vector pa_data;
+
 public:
+   CurlInterpolator();
+
    void AssembleElementMatrix2(const FiniteElement &dom_fe,
                                const FiniteElement &ran_fe,
                                ElementTransformation &Trans,
                                DenseMatrix &elmat) override
    { ran_fe.ProjectCurl(dom_fe, Trans, elmat); }
+
+   void AssemblePA(const FiniteElementSpace &dom_fes,
+                   const FiniteElementSpace &ran_fes) override;
+   void AssemblePA(const FiniteElementSpace &fes) override
+   {
+      AssemblePA(fes, fes);
+   }
+   void AddMultPA(const Vector &x, Vector &y) const override;
+   void AddMultTransposePA(const Vector &x, Vector &y) const override;
+
+   using ApplyKernelType = void (*)(const int ne, const int ndof_o,
+                                    const int nquad_o, const Vector &pa,
+                                    const Vector &x, Vector &y);
+
+   /// arguments: DIM, ndof_o, nquad_o
+   MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int));
+   /// arguments: DIM, ndof_o, nquad_o
+   MFEM_REGISTER_KERNELS(ApplyTPAKernels, ApplyKernelType, (int, int, int));
+
+   template <int DIM, int NDOF_O, int NQUAD_O> static void AddSpecialization()
+   {
+      ApplyPAKernels::Specialization<DIM, NDOF_O, NQUAD_O>::Add();
+      ApplyTPAKernels::Specialization<DIM, NDOF_O, NQUAD_O>::Add();
+   }
+
+   struct Kernels { Kernels(); };
 };
 
 

fem/complex_fem.cpp

@@ -237,81 +237,6 @@ ComplexGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient
    gfi->SyncAliasMemory(*this);
 }
 
-real_t
-ComplexGridFunction::ComputeLpError(const real_t p,
-                                    Coefficient &exsolr,
-                                    Coefficient &exsoli,
-                                    Coefficient *weight,
-                                    const IntegrationRule *irs[],
-                                    const Array<int> *elems) const
-{
-   real_t error = 0.0;
-   const FiniteElement *fe;
-   ElementTransformation *T;
-   Vector valsr;
-   Vector valsi;
-
-   const GridFunction& gf_r = real();
-   const GridFunction& gf_i = imag();
-
-   for (int i = 0; i < fes->GetNE(); i++)
-   {
-      if (elems != NULL && (*elems)[i] == 0) { continue; }
-      fe = fes->GetFE(i);
-      const IntegrationRule *ir;
-      if (irs)
-      {
-         ir = irs[fe->GetGeomType()];
-      }
-      else
-      {
-         int intorder = 2*fe->GetOrder() + 3;
-         ir = &(IntRules.Get(fe->GetGeomType(), intorder));
-      }
-      real_t elem_error = 0.0;
-      gf_r.GetValues(i, *ir, valsr);
-      gf_i.GetValues(i, *ir, valsi);
-      T = fes->GetElementTransformation(i);
-      for (int j = 0; j < ir->GetNPoints(); j++)
-      {
-         const IntegrationPoint &ip = ir->IntPoint(j);
-         T->SetIntPoint(&ip);
-         real_t diffr = valsr(j) - exsolr.Eval(*T, ip);
-         real_t diffi = valsi(j) - exsoli.Eval(*T, ip);
-         real_t diff = hypot(diffr, diffi);
-         if (p < infinity())
-         {
-            diff = pow(diff, p);
-            if (weight)
-            {
-               diff *= weight->Eval(*T, ip);
-            }
-            elem_error += ip.weight * T->Weight() * diff;
-         }
-         else
-         {
-            if (weight)
-            {
-               diff *= weight->Eval(*T, ip);
-            }
-            error = std::max(error, diff);
-         }
-      }
-      if (p < infinity())
-      {
-         // negative quadrature weights may cause the error to be negative
-         error += fabs(elem_error);
-      }
-   }
-
-   if (p < infinity())
-   {
-      error = pow(error, 1./p);
-   }
-
-   return error;
-}
-
 void ComplexGridFunction::Save(std::ostream &os) const
 {
    os << "ComplexGridFunction\n";
@@ -718,8 +643,8 @@ SesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
 
    // A = A_r + i A_i
    A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::MFEM_SPARSEMAT) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::MFEM_SPARSEMAT))
+   if ( A_r.Type() == Operator::MFEM_SPARSEMAT ||
+        A_i.Type() == Operator::MFEM_SPARSEMAT )
    {
       ComplexSparseMatrix * A_sp =
          new ComplexSparseMatrix(A_r.As<SparseMatrix>(),
@@ -779,8 +704,8 @@ SesquilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
 
    // A = A_r + i A_i
    A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::MFEM_SPARSEMAT) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::MFEM_SPARSEMAT))
+   if ( A_r.Type() == Operator::MFEM_SPARSEMAT ||
+        A_i.Type() == Operator::MFEM_SPARSEMAT )
    {
       ComplexSparseMatrix * A_sp =
          new ComplexSparseMatrix(A_r.As<SparseMatrix>(),
@@ -843,426 +768,6 @@ SesquilinearForm::Update(FiniteElementSpace *nfes)
    if ( blfi ) { blfi->Update(nfes); }
 }
 
-bool
-MixedSesquilinearForm::RealInteg()
-{
-   int nint = mblfr->GetDBFI()->Size() + mblfr->GetBBFI()->Size() +
-              mblfr->GetFBFI()->Size() + mblfr->GetBFBFI()->Size() +
-              mblfr->GetTFBFI()->Size() + mblfr->GetBTFBFI()->Size();
-   return (nint != 0);
-}
-
-bool
-MixedSesquilinearForm::ImagInteg()
-{
-   int nint = mblfi->GetDBFI()->Size() + mblfi->GetBBFI()->Size() +
-              mblfi->GetFBFI()->Size() + mblfi->GetBFBFI()->Size() +
-              mblfi->GetTFBFI()->Size() + mblfi->GetBTFBFI()->Size();
-   return (nint != 0);
-}
-
-MixedSesquilinearForm::MixedSesquilinearForm(FiniteElementSpace * trial_fes,
-                                             FiniteElementSpace * test_fes,
-                                             ComplexOperator::Convention convention)
-   : conv(convention),
-     mblfr(new mfem::MixedBilinearForm(trial_fes, test_fes)),
-     mblfi(new mfem::MixedBilinearForm(trial_fes, test_fes))
-{
-}
-
-MixedSesquilinearForm::MixedSesquilinearForm(FiniteElementSpace * trial_fes,
-                                             FiniteElementSpace * test_fes,
-                                             MixedBilinearForm * bfr,
-                                             MixedBilinearForm * bfi,
-                                             ComplexOperator::Convention convention)
-   : conv(convention),
-     mblfr(new MixedBilinearForm(trial_fes, test_fes, bfr)),
-     mblfi(new MixedBilinearForm(trial_fes, test_fes, bfi))
-{
-}
-
-MixedSesquilinearForm::~MixedSesquilinearForm()
-{
-   delete mblfr;
-   delete mblfi;
-}
-
-void
-MixedSesquilinearForm::AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                                           BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      mblfr->AddDomainIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddDomainIntegrator(bfi_imag);
-   }
-}
-
-void
-MixedSesquilinearForm::AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                                           BilinearFormIntegrator * bfi_imag,
-                                           Array<int> & elem_marker)
-{
-   if (bfi_real)
-   {
-      mblfr->AddDomainIntegrator(bfi_real, elem_marker);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddDomainIntegrator(bfi_imag, elem_marker);
-   }
-}
-
-void
-MixedSesquilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
-                                             BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      mblfr->AddBoundaryIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddBoundaryIntegrator(bfi_imag);
-   }
-}
-
-void
-MixedSesquilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
-                                             BilinearFormIntegrator * bfi_imag,
-                                             Array<int> & bdr_marker)
-{
-   if (bfi_real)
-   {
-      mblfr->AddBoundaryIntegrator(bfi_real, bdr_marker);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddBoundaryIntegrator(bfi_imag, bdr_marker);
-   }
-}
-
-void
-MixedSesquilinearForm::AddInteriorFaceIntegrator(BilinearFormIntegrator *
-                                                 bfi_real,
-                                                 BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      mblfr->AddInteriorFaceIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddInteriorFaceIntegrator(bfi_imag);
-   }
-}
-
-void
-MixedSesquilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                            BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      mblfr->AddBdrFaceIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddBdrFaceIntegrator(bfi_imag);
-   }
-}
-
-void
-MixedSesquilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                            BilinearFormIntegrator * bfi_imag,
-                                            Array<int> & bdr_marker)
-{
-   if (bfi_real)
-   {
-      mblfr->AddBdrFaceIntegrator(bfi_real, bdr_marker);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddBdrFaceIntegrator(bfi_imag, bdr_marker);
-   }
-}
-
-void MixedSesquilinearForm::AddTraceFaceIntegrator(BilinearFormIntegrator *
-                                                   bfi_real,
-                                                   BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      mblfr->AddTraceFaceIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddTraceFaceIntegrator(bfi_imag);
-   }
-}
-
-void MixedSesquilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator
-                                                      *bfi_real,
-                                                      BilinearFormIntegrator *bfi_imag)
-{
-   if (bfi_real)
-   {
-      mblfr->AddBdrTraceFaceIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddBdrTraceFaceIntegrator(bfi_imag);
-   }
-}
-
-void MixedSesquilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator
-                                                      *bfi_real,
-                                                      BilinearFormIntegrator *bfi_imag,
-                                                      Array<int> &bdr_marker)
-{
-   if (bfi_real)
-   {
-      mblfr->AddBdrTraceFaceIntegrator(bfi_real, bdr_marker);
-   }
-   if (bfi_imag)
-   {
-      mblfi->AddBdrTraceFaceIntegrator(bfi_imag, bdr_marker);
-   }
-}
-
-void
-MixedSesquilinearForm::Assemble(int skip_zeros)
-{
-   mblfr->Assemble(skip_zeros);
-   mblfi->Assemble(skip_zeros);
-}
-
-void
-MixedSesquilinearForm::Finalize(int skip_zeros)
-{
-   mblfr->Finalize(skip_zeros);
-   mblfi->Finalize(skip_zeros);
-}
-
-ComplexSparseMatrix *
-MixedSesquilinearForm::AssembleComplexSparseMatrix()
-{
-   return new mfem::ComplexSparseMatrix(
-             &mblfr->SpMat(), &mblfi->SpMat(), false, false, conv);
-}
-
-void
-MixedSesquilinearForm::FormRectangularLinearSystem(const Array<int> &
-                                                   ess_trial_tdof_list,
-                                                   const Array<int> & ess_test_tdof_list,
-                                                   Vector & x,
-                                                   Vector & b,
-                                                   OperatorHandle & A,
-                                                   Vector & X,
-                                                   Vector & B)
-{
-   FiniteElementSpace * fes_trial = mblfr->TrialFESpace();
-   FiniteElementSpace * fes_test = mblfr->TestFESpace();
-   const int vsize_trial = fes_trial->GetVSize();
-   const int vsize_test = fes_test->GetVSize();
-
-   // Allocate temporary Vector
-   Vector b_0;
-   b_0.UseDevice(true);
-   b_0.SetSize(vsize_test);
-   b_0 = 0.0;
-
-   // Extract the real and imaginary parts of the input Vectors
-   MFEM_ASSERT(x.Size() == 2 * vsize_trial,
-               "Input GridFunction of incorrect size!");
-   x.Read();
-   Vector x_r;
-   x_r.MakeRef(x, 0, vsize_trial);
-   Vector x_i;
-   x_i.MakeRef(x, vsize_trial, vsize_trial);
-
-   MFEM_ASSERT(b.Size() == 2 * vsize_test, "Input LinearForm of incorrect size!");
-   b.Read();
-   Vector b_r;
-   b_r.MakeRef(b, 0, vsize_test);
-   Vector b_i;
-   b_i.MakeRef(b, vsize_test, vsize_test);
-
-   if (conv == ComplexOperator::BLOCK_SYMMETRIC)
-   {
-      b_i *= -1.0;
-   }
-
-   const int tvsize_trial = fes_trial->GetTrueVSize();
-   const int tvsize_test = fes_test->GetTrueVSize();
-   OperatorHandle A_r, A_i;
-
-   X.UseDevice(true);
-   X.SetSize(2 * tvsize_trial);
-   X = 0.0;
-
-   B.UseDevice(true);
-   B.SetSize(2 * tvsize_test);
-   B = 0.0;
-
-   Vector X_r;
-   X_r.MakeRef(X, 0, tvsize_trial);
-   Vector X_i;
-   X_i.MakeRef(X, tvsize_trial, tvsize_trial);
-   Vector B_r;
-   B_r.MakeRef(B, 0, tvsize_test);
-   Vector B_i;
-   B_i.MakeRef(B, tvsize_test, tvsize_test);
-
-   Vector X_0, B_0;
-
-   if (RealInteg())
-   {
-      b_0 = b_r;
-      mblfr->FormRectangularLinearSystem(
-         ess_trial_tdof_list, ess_test_tdof_list, x_r, b_0, A_r, X_0, B_0);
-      X_r = X_0;
-      B_r = B_0;
-
-      b_0 = b_i;
-      mblfr->FormRectangularLinearSystem(
-         ess_trial_tdof_list, ess_test_tdof_list, x_i, b_0, A_r, X_0, B_0);
-      X_i = X_0;
-      B_i = B_0;
-
-      if (ImagInteg())
-      {
-         b_0 = 0.0;
-         mblfi->FormRectangularLinearSystem(
-            ess_trial_tdof_list, ess_test_tdof_list, x_i, b_0, A_i, X_0, B_0);
-         B_r -= B_0;
-
-         b_0 = 0.0;
-         mblfi->FormRectangularLinearSystem(
-            ess_trial_tdof_list, ess_test_tdof_list, x_r, b_0, A_i, X_0, B_0);
-         B_i += B_0;
-      }
-   }
-   else if (ImagInteg())
-   {
-      b_0 = b_i;
-      mblfi->FormRectangularLinearSystem(
-         ess_trial_tdof_list, ess_test_tdof_list, x_r, b_0, A_i, X_0, B_0);
-      X_r = X_0;
-      B_i = B_0;
-
-      b_0 = b_r;
-      b_0 *= -1.0;
-      mblfi->FormRectangularLinearSystem(
-         ess_trial_tdof_list, ess_test_tdof_list, x_i, b_0, A_i, X_0, B_0);
-      X_i = X_0;
-      B_r = B_0;
-      B_r *= -1.0;
-   }
-   else
-   {
-      MFEM_ABORT("Real and Imaginary part of the Mixed Sesquilinear form are empty");
-   }
-
-   if (conv == ComplexOperator::BLOCK_SYMMETRIC)
-   {
-      B_i *= -1.0;
-      b_i *= -1.0;
-   }
-
-   x_r.SyncAliasMemory(x);
-   x_i.SyncAliasMemory(x);
-   b_r.SyncAliasMemory(b);
-   b_i.SyncAliasMemory(b);
-
-   X_r.SyncAliasMemory(X);
-   X_i.SyncAliasMemory(X);
-   B_r.SyncAliasMemory(B);
-   B_i.SyncAliasMemory(B);
-
-   // A = A_r + i A_i
-   A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::MFEM_SPARSEMAT) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::MFEM_SPARSEMAT))
-   {
-      ComplexSparseMatrix * A_hyp =
-         new ComplexSparseMatrix(A_r.As<SparseMatrix>(),
-                                 A_i.As<SparseMatrix>(),
-                                 A_r.OwnsOperator(),
-                                 A_i.OwnsOperator(),
-                                 conv);
-      A.Reset<ComplexSparseMatrix>(A_hyp, true);
-   }
-   else
-   {
-      ComplexOperator * A_op = new ComplexOperator(A_r.As<Operator>(),
-                                                   A_i.As<Operator>(),
-                                                   A_r.OwnsOperator(),
-                                                   A_i.OwnsOperator(),
-                                                   conv);
-      A.Reset<ComplexOperator>(A_op, true);
-   }
-   A_r.SetOperatorOwner(false);
-   A_i.SetOperatorOwner(false);
-}
-
-void
-MixedSesquilinearForm::FormRectangularSystemMatrix(const mfem::Array<int> &
-                                                   ess_trial_tdof_list,
-                                                   const mfem::Array<int> & ess_test_tdof_list,
-                                                   mfem::OperatorHandle & A)
-{
-   OperatorHandle A_r, A_i;
-   if (RealInteg())
-   {
-      mblfr->FormRectangularSystemMatrix(ess_trial_tdof_list, ess_test_tdof_list,
-                                         A_r);
-   }
-   if (ImagInteg())
-   {
-      mblfi->FormRectangularSystemMatrix(ess_trial_tdof_list, ess_test_tdof_list,
-                                         A_i);
-   }
-   if (!RealInteg() && !ImagInteg())
-   {
-      MFEM_ABORT("Both Real and Imaginary part of the Mixed Sesquilinear form are empty");
-   }
-
-   // A = A_r + i A_i
-   A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::MFEM_SPARSEMAT) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::MFEM_SPARSEMAT))
-   {
-      ComplexSparseMatrix * A_hyp =
-         new ComplexSparseMatrix(A_r.As<SparseMatrix>(),
-                                 A_i.As<SparseMatrix>(),
-                                 A_r.OwnsOperator(),
-                                 A_i.OwnsOperator(),
-                                 conv);
-      A.Reset<ComplexSparseMatrix>(A_hyp, true);
-   }
-   else
-   {
-      ComplexOperator * A_op = new ComplexOperator(A_r.As<Operator>(),
-                                                   A_i.As<Operator>(),
-                                                   A_r.OwnsOperator(),
-                                                   A_i.OwnsOperator(),
-                                                   conv);
-      A.Reset<ComplexOperator>(A_op, true);
-   }
-   A_r.SetOperatorOwner(false);
-   A_i.SetOperatorOwner(false);
-}
-
-void
-MixedSesquilinearForm::Update()
-{
-   mblfr->Update();
-   mblfi->Update();
-}
-
 
 #ifdef MFEM_USE_MPI
 
@@ -2034,8 +1539,8 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
 
    // A = A_r + i A_i
    A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::Hypre_ParCSR) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::Hypre_ParCSR))
+   if ( A_r.Type() == Operator::Hypre_ParCSR ||
+        A_i.Type() == Operator::Hypre_ParCSR )
    {
       ComplexHypreParMatrix * A_hyp =
          new ComplexHypreParMatrix(A_r.As<HypreParMatrix>(),
@@ -2102,8 +1607,8 @@ ParSesquilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
 
    // A = A_r + i A_i
    A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::Hypre_ParCSR) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::Hypre_ParCSR))
+   if ( A_r.Type() == Operator::Hypre_ParCSR ||
+        A_i.Type() == Operator::Hypre_ParCSR )
    {
       ComplexHypreParMatrix * A_hyp =
          new ComplexHypreParMatrix(A_r.As<HypreParMatrix>(),
@@ -2161,434 +1666,6 @@ ParSesquilinearForm::Update(FiniteElementSpace *nfes)
    if ( pblfi ) { pblfi->Update(nfes); }
 }
 
-bool
-ParMixedSesquilinearForm::RealInteg()
-{
-   int nint = pmblfr->GetDBFI()->Size() + pmblfr->GetBBFI()->Size() +
-              pmblfr->GetFBFI()->Size() + pmblfr->GetBFBFI()->Size() +
-              pmblfr->GetTFBFI()->Size() + pmblfr->GetBTFBFI()->Size();
-   return (nint != 0);
-}
-
-bool
-ParMixedSesquilinearForm::ImagInteg()
-{
-   int nint = pmblfi->GetDBFI()->Size() + pmblfi->GetBBFI()->Size() +
-              pmblfi->GetFBFI()->Size() + pmblfi->GetBFBFI()->Size() +
-              pmblfi->GetTFBFI()->Size() + pmblfi->GetBTFBFI()->Size();
-   return (nint != 0);
-}
-
-ParMixedSesquilinearForm::ParMixedSesquilinearForm(ParFiniteElementSpace *
-                                                   trial_fes,
-                                                   ParFiniteElementSpace * test_fes,
-                                                   ComplexOperator::Convention convention)
-   : conv(convention),
-     pmblfr(new ParMixedBilinearForm(trial_fes, test_fes)),
-     pmblfi(new ParMixedBilinearForm(trial_fes, test_fes))
-{
-}
-
-ParMixedSesquilinearForm::ParMixedSesquilinearForm(ParFiniteElementSpace *
-                                                   trial_fes,
-                                                   ParFiniteElementSpace * test_fes,
-                                                   ParMixedBilinearForm * pbfr,
-                                                   ParMixedBilinearForm * pbfi,
-                                                   ComplexOperator::Convention convention)
-   : conv(convention),
-     pmblfr(new ParMixedBilinearForm(trial_fes, test_fes, pbfr)),
-     pmblfi(new ParMixedBilinearForm(trial_fes, test_fes, pbfi))
-{
-}
-
-ParMixedSesquilinearForm::~ParMixedSesquilinearForm()
-{
-   delete pmblfr;
-   delete pmblfi;
-}
-
-void
-ParMixedSesquilinearForm::AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                                              BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddDomainIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddDomainIntegrator(bfi_imag);
-   }
-}
-
-void
-ParMixedSesquilinearForm::AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                                              BilinearFormIntegrator * bfi_imag,
-                                              Array<int> & elem_marker)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddDomainIntegrator(bfi_real, elem_marker);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddDomainIntegrator(bfi_imag, elem_marker);
-   }
-}
-
-void
-ParMixedSesquilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator *
-                                                bfi_real,
-                                                BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddBoundaryIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddBoundaryIntegrator(bfi_imag);
-   }
-}
-
-void
-ParMixedSesquilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator *
-                                                bfi_real,
-                                                BilinearFormIntegrator * bfi_imag,
-                                                Array<int> & bdr_marker)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddBoundaryIntegrator(bfi_real, bdr_marker);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddBoundaryIntegrator(bfi_imag, bdr_marker);
-   }
-}
-
-void
-ParMixedSesquilinearForm::AddInteriorFaceIntegrator(BilinearFormIntegrator *
-                                                    bfi_real,
-                                                    BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddInteriorFaceIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddInteriorFaceIntegrator(bfi_imag);
-   }
-}
-
-void
-ParMixedSesquilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator *
-                                               bfi_real,
-                                               BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddBdrFaceIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddBdrFaceIntegrator(bfi_imag);
-   }
-}
-
-void
-ParMixedSesquilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator *
-                                               bfi_real,
-                                               BilinearFormIntegrator * bfi_imag,
-                                               Array<int> & bdr_marker)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddBdrFaceIntegrator(bfi_real, bdr_marker);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddBdrFaceIntegrator(bfi_imag, bdr_marker);
-   }
-}
-
-void ParMixedSesquilinearForm::AddTraceFaceIntegrator(BilinearFormIntegrator *
-                                                      bfi_real,
-                                                      BilinearFormIntegrator * bfi_imag)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddTraceFaceIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddTraceFaceIntegrator(bfi_imag);
-   }
-}
-
-void ParMixedSesquilinearForm::AddBdrTraceFaceIntegrator(
-   BilinearFormIntegrator *bfi_real,
-   BilinearFormIntegrator *bfi_imag)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddBdrTraceFaceIntegrator(bfi_real);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddBdrTraceFaceIntegrator(bfi_imag);
-   }
-}
-
-void ParMixedSesquilinearForm::AddBdrTraceFaceIntegrator(
-   BilinearFormIntegrator *bfi_real,
-   BilinearFormIntegrator *bfi_imag,
-   Array<int> &bdr_marker)
-{
-   if (bfi_real)
-   {
-      pmblfr->AddBdrTraceFaceIntegrator(bfi_real, bdr_marker);
-   }
-   if (bfi_imag)
-   {
-      pmblfi->AddBdrTraceFaceIntegrator(bfi_imag, bdr_marker);
-   }
-}
-
-void
-ParMixedSesquilinearForm::Assemble(int skip_zeros)
-{
-   pmblfr->Assemble(skip_zeros);
-   pmblfi->Assemble(skip_zeros);
-}
-
-void
-ParMixedSesquilinearForm::Finalize(int skip_zeros)
-{
-   pmblfr->Finalize(skip_zeros);
-   pmblfi->Finalize(skip_zeros);
-}
-
-ComplexHypreParMatrix *
-ParMixedSesquilinearForm::ParallelAssemble()
-{
-   return new ComplexHypreParMatrix(
-             pmblfr->ParallelAssemble(), pmblfi->ParallelAssemble(), true, true, conv);
-}
-
-void
-ParMixedSesquilinearForm::FormRectangularLinearSystem(const Array<int> &
-                                                      ess_trial_tdof_list,
-                                                      const Array<int> & ess_test_tdof_list,
-                                                      Vector & x,
-                                                      Vector & b,
-                                                      OperatorHandle & A,
-                                                      Vector & X,
-                                                      Vector & B)
-{
-   FiniteElementSpace * pfes_trial = pmblfr->TrialFESpace();
-   FiniteElementSpace * pfes_test = pmblfr->TestFESpace();
-   const int vsize_trial = pfes_trial->GetVSize();
-   const int vsize_test = pfes_test->GetVSize();
-
-   // Allocate temporary Vector
-   Vector b_0;
-   b_0.UseDevice(true);
-   b_0.SetSize(vsize_test);
-   b_0 = 0.0;
-
-   // Extract the real and imaginary parts of the input Vectors
-   MFEM_ASSERT(x.Size() == 2 * vsize_trial,
-               "Input GridFunction of incorrect size!");
-   x.Read();
-   Vector x_r;
-   x_r.MakeRef(x, 0, vsize_trial);
-   Vector x_i;
-   x_i.MakeRef(x, vsize_trial, vsize_trial);
-
-   MFEM_ASSERT(b.Size() == 2 * vsize_test, "Input LinearForm of incorrect size!");
-   b.Read();
-   Vector b_r;
-   b_r.MakeRef(b, 0, vsize_test);
-   Vector b_i;
-   b_i.MakeRef(b, vsize_test, vsize_test);
-
-   if (conv == ComplexOperator::BLOCK_SYMMETRIC)
-   {
-      b_i *= -1.0;
-   }
-
-   const int tvsize_trial = pfes_trial->GetTrueVSize();
-   const int tvsize_test = pfes_test->GetTrueVSize();
-   OperatorHandle A_r, A_i;
-
-   X.UseDevice(true);
-   X.SetSize(2 * tvsize_trial);
-   X = 0.0;
-
-   B.UseDevice(true);
-   B.SetSize(2 * tvsize_test);
-   B = 0.0;
-
-   Vector X_r;
-   X_r.MakeRef(X, 0, tvsize_trial);
-   Vector X_i;
-   X_i.MakeRef(X, tvsize_trial, tvsize_trial);
-   Vector B_r;
-   B_r.MakeRef(B, 0, tvsize_test);
-   Vector B_i;
-   B_i.MakeRef(B, tvsize_test, tvsize_test);
-
-   Vector X_0, B_0;
-
-   if (RealInteg())
-   {
-      b_0 = b_r;
-      pmblfr->FormRectangularLinearSystem(
-         ess_trial_tdof_list, ess_test_tdof_list, x_r, b_0, A_r, X_0, B_0);
-      X_r = X_0;
-      B_r = B_0;
-
-      b_0 = b_i;
-      pmblfr->FormRectangularLinearSystem(
-         ess_trial_tdof_list, ess_test_tdof_list, x_i, b_0, A_r, X_0, B_0);
-      X_i = X_0;
-      B_i = B_0;
-
-      if (ImagInteg())
-      {
-         b_0 = 0.0;
-         pmblfi->FormRectangularLinearSystem(
-            ess_trial_tdof_list, ess_test_tdof_list, x_i, b_0, A_i, X_0, B_0);
-         B_r -= B_0;
-
-         b_0 = 0.0;
-         pmblfi->FormRectangularLinearSystem(
-            ess_trial_tdof_list, ess_test_tdof_list, x_r, b_0, A_i, X_0, B_0);
-         B_i += B_0;
-      }
-   }
-   else if (ImagInteg())
-   {
-      b_0 = b_i;
-      pmblfi->FormRectangularLinearSystem(
-         ess_trial_tdof_list, ess_test_tdof_list, x_r, b_0, A_i, X_0, B_0);
-      X_r = X_0;
-      B_i = B_0;
-
-      b_0 = b_r;
-      b_0 *= -1.0;
-      pmblfi->FormRectangularLinearSystem(
-         ess_trial_tdof_list, ess_test_tdof_list, x_i, b_0, A_i, X_0, B_0);
-      X_i = X_0;
-      B_r = B_0;
-      B_r *= -1.0;
-   }
-   else
-   {
-      MFEM_ABORT("Real and Imaginary part of the Mixed Sesquilinear form are empty");
-   }
-
-   if (conv == ComplexOperator::BLOCK_SYMMETRIC)
-   {
-      B_i *= -1.0;
-      b_i *= -1.0;
-   }
-
-   x_r.SyncAliasMemory(x);
-   x_i.SyncAliasMemory(x);
-   b_r.SyncAliasMemory(b);
-   b_i.SyncAliasMemory(b);
-
-   X_r.SyncAliasMemory(X);
-   X_i.SyncAliasMemory(X);
-   B_r.SyncAliasMemory(B);
-   B_i.SyncAliasMemory(B);
-
-   // A = A_r + i A_i
-   A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::Hypre_ParCSR) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::Hypre_ParCSR))
-   {
-      ComplexHypreParMatrix * A_hyp =
-         new ComplexHypreParMatrix(A_r.As<HypreParMatrix>(),
-                                   A_i.As<HypreParMatrix>(),
-                                   A_r.OwnsOperator(),
-                                   A_i.OwnsOperator(),
-                                   conv);
-      A.Reset<ComplexHypreParMatrix>(A_hyp, true);
-   }
-   else
-   {
-      ComplexOperator * A_op = new ComplexOperator(A_r.As<Operator>(),
-                                                   A_i.As<Operator>(),
-                                                   A_r.OwnsOperator(),
-                                                   A_i.OwnsOperator(),
-                                                   conv);
-      A.Reset<ComplexOperator>(A_op, true);
-   }
-   A_r.SetOperatorOwner(false);
-   A_i.SetOperatorOwner(false);
-}
-
-void
-ParMixedSesquilinearForm::FormRectangularSystemMatrix(const Array<int> &
-                                                      ess_trial_tdof_list,
-                                                      const Array<int> & ess_test_tdof_list,
-                                                      OperatorHandle & A)
-{
-   OperatorHandle A_r, A_i;
-   if (RealInteg())
-   {
-      pmblfr->FormRectangularSystemMatrix(ess_trial_tdof_list, ess_test_tdof_list,
-                                          A_r);
-   }
-   if (ImagInteg())
-   {
-      pmblfi->FormRectangularSystemMatrix(ess_trial_tdof_list, ess_test_tdof_list,
-                                          A_i);
-   }
-   if (!RealInteg() && !ImagInteg())
-   {
-      MFEM_ABORT("Both Real and Imaginary part of the Mixed Sesquilinear form are empty");
-   }
-
-   // A = A_r + i A_i
-   A.Clear();
-   if ((!A_r.Ptr() || A_r.Type() == Operator::Hypre_ParCSR) &&
-       (!A_i.Ptr() || A_i.Type() == Operator::Hypre_ParCSR))
-   {
-      ComplexHypreParMatrix * A_hyp =
-         new ComplexHypreParMatrix(A_r.As<HypreParMatrix>(),
-                                   A_i.As<HypreParMatrix>(),
-                                   A_r.OwnsOperator(),
-                                   A_i.OwnsOperator(),
-                                   conv);
-      A.Reset<ComplexHypreParMatrix>(A_hyp, true);
-   }
-   else
-   {
-      ComplexOperator * A_op = new ComplexOperator(A_r.As<Operator>(),
-                                                   A_i.As<Operator>(),
-                                                   A_r.OwnsOperator(),
-                                                   A_i.OwnsOperator(),
-                                                   conv);
-      A.Reset<ComplexOperator>(A_op, true);
-   }
-   A_r.SetOperatorOwner(false);
-   A_i.SetOperatorOwner(false);
-}
-
-void
-ParMixedSesquilinearForm::Update()
-{
-   pmblfr->Update();
-   pmblfi->Update();
-}
-
-
-
 #endif // MFEM_USE_MPI
 
 }

fem/complex_fem.hpp

@@ -166,75 +166,6 @@ public:
       return sqrt(err_r * err_r + err_i * err_i);
    }
 
-   /// @brief Returns Max|u_ex - u_h| error for complex-valued H1 or L2 elements
-   ///
-   /// Compute the $L_\infty$ error across the entire domain.
-   ///
-   /// @param[in] exsolr  Coefficient object reproducing the real part of the
-   ///                    anticipated values of the scalar field, Re(u_ex).
-   /// @param[in] exsoli  Coefficient object reproducing the imaginary part of
-   ///                    the anticipated values of the scalar field, Im(u_ex).
-   /// @param[in] irs        Optional pointer to an array of custom integration
-   ///                       rules e.g. higher order than the default rules. If
-   ///                       present the array will be indexed by
-   ///                       Geometry::Type.
-   ///
-   /// @note Uses ComputeLpError internally. See the ComputeLpError
-   ///       documentation for generalizations of this error computation.
-   ///
-   /// @note If an array of integration rules is provided through @a irs, be
-   ///       sure to include valid rules for each element type that may occur
-   ///       in the list of elements.
-   ///
-   virtual real_t ComputeMaxError(Coefficient &exsolr,
-                                  Coefficient &exsoli,
-                                  const IntegrationRule *irs[] = NULL) const
-   {
-      return ComputeLpError(infinity(), exsolr, exsoli, NULL, irs);
-   }
-
-   /// @brief Returns ||u_ex - u_h||_Lp for complex-valued H1 or L2 elements
-   ///
-   /// Computes:
-   ///    $$(\sum_{elems} \int_{elem} w \, |u_{ex} - u_h|^p)^{1/p}$$
-   /// Where:
-   ///    $$|u_{ex} - u_h| = \sqrt{Re(u_{ex} - u_h)^2 + Im(u_{ex} - u_h)^2}$$
-   ///
-   /// @param[in] p       Real value indicating the exponent of the $L^p$ norm.
-   ///                    To avoid domain errors p should have a positive value,
-   ///                    either finite or infinite.
-   /// @param[in] exsolr  Coefficient object reproducing the real part of the
-   ///                    anticipated values of the scalar field, Re(u_ex).
-   /// @param[in] exsoli  Coefficient object reproducing the imaginary part of
-   ///                    the anticipated values of the scalar field, Im(u_ex).
-   /// @param[in] weight  Optional pointer to a Coefficient object reproducing
-   ///                    a weighting function, w.
-   /// @param[in] irs     Optional pointer to an array of custom integration
-   ///                    rules e.g. higher order than the default rules. If
-   ///                    present the array will be indexed by Geometry::Type.
-   /// @param[in] elems   Optional pointer to a marker array, with a length
-   ///                    equal to the number of local elements, indicating
-   ///                    which elements to integrate over. Only those elements
-   ///                    corresponding to non-zero entries in @a elems will
-   ///                    contribute to the computed L2 error.
-   ///
-   /// @note If an array of integration rules is provided through @a irs, be
-   ///       sure to include valid rules for each element type that may occur
-   ///       in the list of elements.
-   ///
-   /// @note Quadratures with negative weights (as in some simplex integration
-   ///       rules in MFEM) can produce negative integrals even with
-   ///       non-negative integrands. To avoid returning negative errors this
-   ///       function uses the absolute values of the element-wise integrals.
-   ///       This may lead to results which are not entirely consistent with
-   ///       such integration rules.
-   virtual real_t ComputeLpError(const real_t p,
-                                 Coefficient &exsolr,
-                                 Coefficient &exsoli,
-                                 Coefficient *weight = NULL,
-                                 const IntegrationRule *irs[] = NULL,
-                                 const Array<int> *elems = NULL) const;
-
    /// Save the ComplexGridFunction to an output stream.
    virtual void Save(std::ostream &out) const;
 
@@ -505,186 +436,6 @@ public:
    virtual ~SesquilinearForm();
 };
 
-/** Class for a mixed sesquilinear form
-
-    A mixed sesquilinear form is a generalization of a mixed bilinear form to
-    complex-valued fields. Mixed sesquilinear forms are linear in the second
-    argument but the first argument involves a complex conjugate in the sense
-    that:
-
-                a(alpha u, beta v) = conj(alpha) beta a(u, v)
-
-    The @a convention argument in the class's constructor is documented in the
-    mfem::ComplexOperator class found in linalg/complex_operator.hpp.
-
-    When supplying integrators to the MixedSesquilinearForm either the real or
-    imaginary integrator can be NULL. This indicates that the corresponding
-    portion of the complex-valued material coefficient is equal to zero.
-*/
-class MixedSesquilinearForm
-{
-private:
-   ComplexOperator::Convention conv;
-
-   MixedBilinearForm * mblfr;
-   MixedBilinearForm * mblfi;
-
-   /* These methods check if the real/imag parts of the sesqulinear form are not
-      empty */
-   bool RealInteg();
-   bool ImagInteg();
-
-public:
-   MixedSesquilinearForm(
-      FiniteElementSpace * trial_fes,
-      FiniteElementSpace * test_fes,
-      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
-
-   /** @brief Create a MixedSesquilinearForm on the given trial and test
-       FiniteElementSpaces, using the same integrators as the
-       MixedBilinearForms @a bfr and @a bfi.
-
-       The FiniteElementSpace pointers are not owned by the newly constructed
-       object.
-
-       The integrators are copied as pointers and they are not owned by the
-       newly constructed MixedSesquilinearForm. */
-   MixedSesquilinearForm(
-      FiniteElementSpace * trial_fes,
-      FiniteElementSpace * test_fes,
-      MixedBilinearForm * bfr,
-      MixedBilinearForm * bfi,
-      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
-
-   ComplexOperator::Convention GetConvention() const { return conv; }
-   void SetConvention(const ComplexOperator::Convention & convention) { conv = convention; }
-
-   /// Set the desired assembly level.
-   /** Valid choices are:
-
-       - AssemblyLevel::LEGACY (default)
-       - AssemblyLevel::FULL
-       - AssemblyLevel::PARTIAL
-       - AssemblyLevel::ELEMENT
-       - AssemblyLevel::NONE
-
-       This method must be called before assembly. */
-   void SetAssemblyLevel(AssemblyLevel assembly_level)
-   {
-      mblfr->SetAssemblyLevel(assembly_level);
-      mblfi->SetAssemblyLevel(assembly_level);
-   }
-
-   MixedBilinearForm & real() { return *mblfr; }
-   MixedBilinearForm & imag() { return *mblfi; }
-   const MixedBilinearForm & real() const { return *mblfr; }
-   const MixedBilinearForm & imag() const { return *mblfi; }
-
-   /// Adds new Domain Integrator.
-   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                            BilinearFormIntegrator * bfi_imag);
-
-   /// Adds new Domain Integrator, restricted to specific attributes.
-   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                            BilinearFormIntegrator * bfi_imag,
-                            Array<int> & elem_marker);
-
-   /// Adds new Boundary Integrator.
-   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
-                              BilinearFormIntegrator * bfi_imag);
-
-   /** @brief Adds new boundary Integrator, restricted to specific boundary
-       attributes.
-
-       Assumes ownership of @a bfi.
-
-       The mfem::array @a bdr_marker is stored internally as a pointer to the given
-       mfem::Array<int> object. */
-   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
-                              BilinearFormIntegrator * bfi_imag,
-                              Array<int> & bdr_marker);
-
-   /// Adds new interior Face Integrator. Assumes ownership of @a bfi.
-   void AddInteriorFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag);
-
-   /// Adds new boundary Face Integrator. Assumes ownership of @a bfi.
-   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                             BilinearFormIntegrator * bfi_imag);
-
-   /** @brief Adds new boundary Face Integrator, restricted to specific boundary
-       attributes.
-
-       Assumes ownership of @a bfi.
-
-       The mfem::array @a bdr_marker is stored internally as a pointer to the given
-       mfem::Array<int> object. */
-   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                             BilinearFormIntegrator * bfi_imag,
-                             Array<int> & bdr_marker);
-
-   /** @brief Add a trace face integrator. Assumes ownership of @a bfi.
-
-       This type of integrator assembles terms over all faces of the mesh using
-       the face FE from the trial space and the two adjacent volume FEs from
-       the test space. */
-   void AddTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                               BilinearFormIntegrator * bfi_imag);
-
-   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag);
-
-   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag,
-                                  Array<int> &bdr_marker);
-
-   /// Assemble the local matrix
-   void Assemble(int skip_zeros = 1);
-
-   /// Finalizes the matrix initialization.
-   void Finalize(int skip_zeros = 1);
-
-   /// Updates the internal mixed forms with the new finite element space.
-   virtual void Update();
-
-   /** @brief Return a ComplexSparseMatrix wrapping the local (L-dof) real
-       and imaginary matrices of the form.
-
-       The returned wrapper has to be deleted by the caller, but it does not
-       own the wrapped real and imaginary matrices, which remain owned by
-       this form. */
-   ComplexSparseMatrix *AssembleComplexSparseMatrix();
-
-   /// Return the trial FE space associated with the MixedSesquilinearForm.
-   FiniteElementSpace *TrialFESpace() { return mblfr->TrialFESpace(); }
-
-   /// Read-only access to the associated trial FiniteElementSpace.
-   const FiniteElementSpace *TrialFESpace() const { return mblfr->TrialFESpace(); }
-
-   /// Return the test FE space associated with the MixedSesquilinearForm.
-   FiniteElementSpace *TestFESpace() { return mblfr->TestFESpace(); }
-
-   /// Read-only access to the associated test FiniteElementSpace.
-   const FiniteElementSpace *TestFESpace() const { return mblfr->TestFESpace(); }
-
-
-   void FormRectangularLinearSystem(const Array<int> & ess_trial_tdof_list,
-                                    const Array<int> & ess_test_tdof_list,
-                                    Vector & x,
-                                    Vector & b,
-                                    OperatorHandle & A,
-                                    Vector & X,
-                                    Vector & B);
-
-   void FormRectangularSystemMatrix(const Array<int> & ess_trial_tdof_list,
-                                    const Array<int> & ess_test_tdof_list,
-                                    OperatorHandle & A);
-
-   virtual ~MixedSesquilinearForm();
-};
-
 #ifdef MFEM_USE_MPI
 
 /// Class for parallel complex-valued grid function - real + imaginary part
@@ -1101,169 +852,6 @@ public:
    virtual ~ParSesquilinearForm();
 };
 
-/** Class for a parallel mixed sesquilinear form
-
-    A mixed sesquilinear form is a generalization of a mixed bilinear form to
-    complex-valued fields. Mixed sesquilinear forms are linear in the second
-    argument but the first argument involves a complex conjugate in the sense
-    that:
-
-                a(alpha u, beta v) = conj(alpha) beta a(u, v)
-
-    The @a convention argument in the class's constructor is documented in the
-    mfem::ComplexOperator class found in linalg/complex_operator.hpp.
-
-    When supplying integrators to the ParMixedSesquilinearForm either the real
-    or imaginary integrator can be NULL. This indicates that the corresponding
-    portion of the complex-valued material coefficient is equal to zero.
-*/
-class ParMixedSesquilinearForm
-{
-private:
-   ComplexOperator::Convention conv;
-
-   ParMixedBilinearForm * pmblfr;
-   ParMixedBilinearForm * pmblfi;
-
-   /* These methods check if the real/imag parts of the sesqulinear form are
-    not empty */
-   bool RealInteg();
-   bool ImagInteg();
-
-public:
-   ParMixedSesquilinearForm(
-      ParFiniteElementSpace * trial_fes,
-      ParFiniteElementSpace * test_fes,
-      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
-
-   /** @brief Create a ParMixedSesquilinearForm on the given trial and test
-       ParFiniteElementSpaces, using the same integrators as the
-       ParMixedBilinearForms @a pbfr and @a pbfi.
-
-       The ParFiniteElementSpace pointers are not owned by the newly
-       constructed object.
-
-       The integrators are copied as pointers and they are not owned by the
-       newly constructed ParMixedSesquilinearForm. */
-   ParMixedSesquilinearForm(
-      ParFiniteElementSpace * trial_fes,
-      ParFiniteElementSpace * test_fes,
-      ParMixedBilinearForm * pbfr,
-      ParMixedBilinearForm * pbfi,
-      ComplexOperator::Convention convention = ComplexOperator::HERMITIAN);
-
-   ComplexOperator::Convention GetConvention() const { return conv; }
-   void SetConvention(const ComplexOperator::Convention & convention) { conv = convention; }
-
-   /// Set the desired assembly level.
-   /** Valid choices are:
-
-       - AssemblyLevel::LEGACY (default)
-       - AssemblyLevel::FULL
-       - AssemblyLevel::PARTIAL
-       - AssemblyLevel::ELEMENT
-       - AssemblyLevel::NONE
-
-       This method must be called before assembly. */
-   void SetAssemblyLevel(AssemblyLevel assembly_level)
-   {
-      pmblfr->SetAssemblyLevel(assembly_level);
-      pmblfi->SetAssemblyLevel(assembly_level);
-   }
-
-   ParMixedBilinearForm & real() { return *pmblfr; }
-   ParMixedBilinearForm & imag() { return *pmblfi; }
-   const ParMixedBilinearForm & real() const { return *pmblfr; }
-   const ParMixedBilinearForm & imag() const { return *pmblfi; }
-
-   /// Adds new Domain Integrator.
-   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                            BilinearFormIntegrator * bfi_imag);
-
-   /// Adds new Domain Integrator, restricted to specific attributes.
-   void AddDomainIntegrator(BilinearFormIntegrator * bfi_real,
-                            BilinearFormIntegrator * bfi_imag,
-                            Array<int> & elem_marker);
-
-   /// Adds new Boundary Integrator.
-   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
-                              BilinearFormIntegrator * bfi_imag);
-
-   /** @brief Adds new boundary Integrator, restricted to specific boundary
-       attributes.
-
-       Assumes ownership of @a bfi.
-
-       The mfem::array @a bdr_marker is stored internally as a pointer to the given
-       mfem::Array<int> object. */
-   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi_real,
-                              BilinearFormIntegrator * bfi_imag,
-                              Array<int> & bdr_marker);
-
-   /// Adds new interior Face Integrator. Assumes ownership of @a bfi.
-   void AddInteriorFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag);
-
-   /// Adds new boundary Face Integrator. Assumes ownership of @a bfi.
-   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                             BilinearFormIntegrator * bfi_imag);
-
-   /** @brief Adds new boundary Face Integrator, restricted to specific boundary
-       attributes.
-
-       Assumes ownership of @a bfi.
-
-       The mfem::array @a bdr_marker is stored internally as a pointer to the given
-       mfem::Array<int> object. */
-   void AddBdrFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                             BilinearFormIntegrator * bfi_imag,
-                             Array<int> & bdr_marker);
-
-   /** @brief Add a trace face integrator. Assumes ownership of @a bfi.
-
-      This type of integrator assembles terms over all faces of the mesh using
-      the face FE from the trial space and the two adjacent volume FEs from
-      the test space. */
-   void AddTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                               BilinearFormIntegrator * bfi_imag);
-
-   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag);
-
-   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi_real,
-                                  BilinearFormIntegrator * bfi_imag,
-                                  Array<int> &bdr_marker);
-
-   /// Assemble the local matrix
-   void Assemble(int skip_zeros = 1);
-
-   /// Finalizes the matrix initialization.
-   void Finalize(int skip_zeros = 1);
-
-   /// Updates the internal mixed forms with the new finite element space.
-   virtual void Update();
-
-   /// Returns the matrix assembled on the true dofs, i.e. P^t A P.
-   /** The returned matrix has to be deleted by the caller. */
-   ComplexHypreParMatrix * ParallelAssemble();
-
-   void FormRectangularLinearSystem(const Array<int> & ess_trial_tdof_list,
-                                    const Array<int> & ess_test_tdof_list,
-                                    Vector & x,
-                                    Vector & b,
-                                    OperatorHandle & A,
-                                    Vector & X,
-                                    Vector & B);
-
-   void FormRectangularSystemMatrix(const Array<int> & ess_trial_tdof_list,
-                                    const Array<int> & ess_test_tdof_list,
-                                    OperatorHandle & A);
-
-   virtual ~ParMixedSesquilinearForm();
-};
-
 #endif // MFEM_USE_MPI
 
 }

fem/datacollection.cpp

@@ -809,7 +809,7 @@ ParaViewDataCollectionBase::ParaViewDataCollectionBase(
 
 void ParaViewDataCollectionBase::SetLevelsOfDetail(int levels_of_detail_)
 {
-   levels_of_detail = std::max(levels_of_detail_, 1);
+   levels_of_detail = levels_of_detail_;
 }
 
 void ParaViewDataCollectionBase::SetHighOrderOutput(bool high_order_output_)
@@ -1181,14 +1181,12 @@ void ParaViewDataCollection::SaveGFieldVTU(std::ostream &os, int ref_,
    DenseMatrix vval, pmat;
    std::vector<char> buf;
    int vec_dim = it->second->VectorDim();
-   int map_type = it->second->FESpace()->GetTypicalFE()->GetMapType();
    os << "<DataArray type=\"" << GetDataTypeString()
       << "\" Name=\"" << it->first
       << "\" NumberOfComponents=\"" << vec_dim << "\" "
       << VTKComponentLabels(vec_dim) << " "
       << "format=\"" << GetDataFormatString() << "\" >" << '\n';
-   if (vec_dim == 1 && (map_type == FiniteElement::VALUE ||
-                        map_type == FiniteElement::INTEGRAL))
+   if (vec_dim == 1)
    {
       for (int i = 0; i < mesh->GetNE(); i++)
       {

fem/fe/fe_base.hpp

@@ -1446,6 +1446,8 @@ public:
                                 dof2quad_array_open);
    }
 
+   const Poly_1D::Basis &GetOpenBasis1D() const { return obasis1d; }
+
    virtual ~VectorTensorFiniteElement();
 };
 

fem/fe/fe_fixed_order.hpp

@@ -307,12 +307,12 @@ public:
 
    /** @brief virtual function which evaluates the values of all
        shape functions at a given point ip and stores
-       them in the vector shape of dimension Dof (6) */
+       them in the vector shape of dimension Dof (4) */
    void CalcShape(const IntegrationPoint &ip, Vector &shape) const override;
 
    /** @brief virtual function which evaluates the values of all
        partial derivatives of all shape functions at a given
-       point ip and stores them in the matrix dshape (Dof x Dim) (6 x 3)
+       point ip and stores them in the matrix dshape (Dof x Dim) (4 x 3)
        so that each row contains the derivatives of one shape function */
    void CalcDShape(const IntegrationPoint &ip,
                    DenseMatrix &dshape) const override;
@@ -336,12 +336,12 @@ public:
 
    /** @brief virtual function which evaluates the values of all
        shape functions at a given point ip and stores
-       them in the vector shape of dimension Dof (5) */
+       them in the vector shape of dimension Dof (4) */
    void CalcShape(const IntegrationPoint &ip, Vector &shape) const override;
 
    /** @brief virtual function which evaluates the values of all
        partial derivatives of all shape functions at a given
-       point ip and stores them in the matrix dshape (Dof x Dim) (5 x 3)
+       point ip and stores them in the matrix dshape (Dof x Dim) (4 x 3)
        so that each row contains the derivatives of one shape function */
    void CalcDShape(const IntegrationPoint &ip,
                    DenseMatrix &dshape) const override;

fem/fe/fe_h1.cpp

@@ -1757,45 +1757,22 @@ H1_BergotPyramidElement::H1_BergotPyramidElement(const int p, const int btype)
       real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
       real_t z = ip.z;
 
-      if (std::abs(z - 1.0) < apex_tol)
-      {
-         // Compute the limit of the basis functions as z->1 with x and y on the
-         // line between the center of the base and the apex
-         o = 0;
-         for (int i = 0; i <= p; i++)
-            for (int j = 0; j <= p; j++)
-            {
-               int maxij = std::max(i, j);
-               for (int k = 0; k <= p - maxij; k++)
-                  if (i == 0 && j == 0)
-                  {
-                     T(o++, m) = ((k + 3.) * k + 2.) / 2.;
-                  }
-                  else
-                  {
-                     T(o++, m) = 0.;
-                  }
-            }
-      }
-      else
-      {
-         poly1d.CalcLegendre(p, x, shape_x.GetData());
-         poly1d.CalcLegendre(p, y, shape_y.GetData());
+      poly1d.CalcLegendre(p, x, shape_x.GetData());
+      poly1d.CalcLegendre(p, y, shape_y.GetData());
 
-         o = 0;
-         for (int i = 0; i <= p; i++)
+      o = 0;
+      for (int i = 0; i <= p; i++)
+      {
+         for (int j = 0; j <= p; j++)
          {
-            for (int j = 0; j <= p; j++)
-            {
-               int maxij = std::max(i, j);
-               FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                                z, 1.0, shape_z);
+            int maxij = std::max(i, j);
+            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
+                                             z, 1.0, shape_z);
 
-               for (int k = 0; k <= p - maxij; k++)
-               {
-                  T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
-                              pow(1.0 - ip.z, maxij);
-               }
+            for (int k = 0; k <= p - maxij; k++)
+            {
+               T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
+                           pow(1.0 - ip.z, maxij);
             }
          }
       }
@@ -1816,44 +1793,25 @@ void H1_BergotPyramidElement::CalcShape(const IntegrationPoint &ip,
    Vector u(dof);
 #endif
 
-   const real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
-   const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
-   const real_t z = ip.z;
+   real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
+   real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
+   real_t z = ip.z;
 
-   if (std::abs(z - 1.0) < apex_tol)
-   {
-      // Compute the limit of the basis functions as z->1 with x and y on the
-      // line between the center of the base and the apex
-      u = 0.;
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            for (int k = 0; k <= p - maxij; k++, o++)
-               if (i == 0 && j == 0)
-               {
-                  u(o) = ((k + 3.) * k + 2.) / 2.;
-               }
-         }
-   }
-   else
-   {
-      poly1d.CalcLegendre(p, x, shape_x.GetData());
-      poly1d.CalcLegendre(p, y, shape_y.GetData());
+   poly1d.CalcLegendre(p, x, shape_x.GetData());
+   poly1d.CalcLegendre(p, y, shape_y.GetData());
+
+   int o = 0;
+   for (int i = 0; i <= p; i++)
+      for (int j = 0; j <= p; j++)
+      {
+         int maxij = std::max(i, j);
+         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
+                                          shape_z);
+         for (int k = 0; k <= p - maxij; k++)
+            u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
+                     pow(1.0 - ip.z, maxij);
+      }
 
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                             z, 1.0, shape_z);
-            for (int k = 0; k <= p - maxij; k++)
-               u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
-                        pow(1.0 - ip.z, maxij);
-         }
-   }
    Ti.Mult(u, shape);
 }
 
@@ -1872,68 +1830,37 @@ void H1_BergotPyramidElement::CalcDShape(const IntegrationPoint &ip,
    Vector dshape_z(order+1);
    Vector dshape_z_dt(order+1);
 #endif
-   const real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
-   const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
-   const real_t z = ip.z;
+   real_t x = (ip.z < 1.0) ? (ip.x / (1.0 - ip.z)) : 0.0;
+   real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
+   real_t z = ip.z;
 
-   if (std::abs(z - 1.0) < apex_tol)
-   {
-      // Compute the limit of the gradients of the basis functions as
-      // z->1 with x and y on the line between the center of the base and the
-      // apex
-      du = 0.;
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
+   poly1d.CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
+   poly1d.CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
+
+   int o = 0;
+   for (int i = 0; i <= p; i++)
+      for (int j = 0; j <= p; j++)
+      {
+         int maxij = std::max(i, j);
+         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
+                                          shape_z, dshape_z, dshape_z_dt);
+
+         for (int k = 0; k <= p - maxij; k++, o++)
          {
-            int maxij = std::max(i, j);
-            for (int k = 0; k <= p - maxij; k++, o++)
-            {
-               if (i == 0 && j == 0)
-               {
-                  du(o,2) = (((k + 6.) * k + 11.) * k + 6.) * k / 6.;
-               }
-               else if (i == 1 && j == 0)
-               {
-                  du(o,0) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
-               }
-               else if (i == 0 && j == 1)
-               {
-                  du(o,1) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
-               }
-            }
+            du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
+                      pow(1.0 - ip.z, maxij - 1);
+            du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
+                      pow(1.0 - ip.z, maxij - 1);
+            du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
+                      pow(1.0 - ip.z, maxij) +
+                      (ip.x * dshape_x(i) * shape_y(j) +
+                       ip.y * shape_x(i) * dshape_y(j)) *
+                      shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
+                      maxij * shape_x(i) * shape_y(j) * shape_z(k) *
+                      pow(1.0 - ip.z, maxij - 1);
          }
-   }
-   else
-   {
-      poly1d.CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
-      poly1d.CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
+      }
 
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                             z, 1.0,
-                                             shape_z, dshape_z, dshape_z_dt);
-
-            for (int k = 0; k <= p - maxij; k++, o++)
-            {
-               du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
-                         pow(1.0 - ip.z, maxij - 1);
-               du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
-                         pow(1.0 - ip.z, maxij - 1);
-               du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
-                         pow(1.0 - ip.z, maxij) +
-                         (ip.x * dshape_x(i) * shape_y(j) +
-                          ip.y * shape_x(i) * dshape_y(j)) *
-                         shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
-                         maxij * shape_x(i) * shape_y(j) * shape_z(k) *
-                         (maxij > 0 ? pow(1.0 - ip.z, maxij - 1) : 0.0);
-            }
-         }
-   }
    Ti.Mult(du, dshape);
 }
 

fem/fe/fe_h1.hpp

@@ -208,8 +208,6 @@ private:
 #endif
    DenseMatrixInverse Ti;
 
-   static constexpr real_t apex_tol = 1e-8;
-
 public:
    H1_BergotPyramidElement(const int p,
                            const int btype = BasisType::GaussLobatto);

fem/fe/fe_l2.cpp

@@ -1106,16 +1106,9 @@ L2_BergotPyramidElement::L2_BergotPyramidElement(const int p, const int btype)
          {
             const real_t wik = op[i] + op[k] + op[p-i-k];
             const real_t w = wik * wjk * op[p-k];
-            if (std::abs(w) < apex_tol)
-            {
-               Nodes.IntPoint(o++).Set3(0.,0.,1.);
-            }
-            else
-            {
-               Nodes.IntPoint(o++).Set3(op[i] * (op[j] + op[p-j-k]) / w,
-                                        op[j] * (op[i] + op[p-i-k]) / w,
-                                        op[k] * op[p-k] / w);
-            }
+            Nodes.IntPoint(o++).Set3(op[i] * (op[j] + op[p-j-k]) / w,
+                                     op[j] * (op[j] + op[p-j-k]) / w,
+                                     op[k] * op[p-k] / w);
          }
       }
 
@@ -1132,45 +1125,22 @@ L2_BergotPyramidElement::L2_BergotPyramidElement(const int p, const int btype)
       const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
       const real_t z = ip.z;
 
-      if (std::abs(z - 1.0) < apex_tol)
-      {
-         // Compute the limit of the basis functions as z->1 with x and y on the
-         // line between the center of the base and the apex
-         o = 0;
-         for (int i = 0; i <= p; i++)
-            for (int j = 0; j <= p; j++)
-            {
-               int maxij = std::max(i, j);
-               for (int k = 0; k <= p - maxij; k++)
-                  if (i == 0 && j == 0)
-                  {
-                     T(o++, m) = ((k + 3.) * k + 2.) / 2.;
-                  }
-                  else
-                  {
-                     T(o++, m) = 0.;
-                  }
-            }
-      }
-      else
-      {
-         poly1d.CalcLegendre(p, x, shape_x.GetData());
-         poly1d.CalcLegendre(p, y, shape_y.GetData());
+      poly1d.CalcLegendre(p, x, shape_x.GetData());
+      poly1d.CalcLegendre(p, y, shape_y.GetData());
 
-         o = 0;
-         for (int i = 0; i <= p; i++)
+      o = 0;
+      for (int i = 0; i <= p; i++)
+      {
+         for (int j = 0; j <= p; j++)
          {
-            for (int j = 0; j <= p; j++)
-            {
-               int maxij = std::max(i, j);
-               FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                                z, 1.0, shape_z);
+            int maxij = std::max(i, j);
+            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
+                                             z, 1.0, shape_z);
 
-               for (int k = 0; k <= p - maxij; k++)
-               {
-                  T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
-                              pow(1.0 - ip.z, maxij);
-               }
+            for (int k = 0; k <= p - maxij; k++)
+            {
+               T(o++, m) = shape_x(i) * shape_y(j) * shape_z(k) *
+                           pow(1.0 - ip.z, maxij);
             }
          }
       }
@@ -1195,41 +1165,26 @@ void L2_BergotPyramidElement::CalcShape(const IntegrationPoint &ip,
    const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
    const real_t z = ip.z;
 
-   if (std::abs(z - 1.0) < apex_tol)
-   {
-      // Compute the limit of the basis functions as z->1 with x and y on the
-      // line between the center of the base and the apex
-      u = 0.;
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            for (int k = 0; k <= p - maxij; k++, o++)
-               if (i == 0 && j == 0)
-               {
-                  u(o) = ((k + 3.) * k + 2.) / 2.;
-               }
-         }
-   }
-   else
-   {
-      poly1d.CalcLegendre(p, x, shape_x.GetData());
-      poly1d.CalcLegendre(p, y, shape_y.GetData());
+   poly1d.CalcLegendre(p, x, shape_x.GetData());
+   poly1d.CalcLegendre(p, y, shape_y.GetData());
 
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                             z, 1.0, shape_z);
+   int o = 0;
+   for (int i = 0; i <= p; i++)
+   {
+      for (int j = 0; j <= p; j++)
+      {
+         int maxij = std::max(i, j);
+         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
+                                          shape_z);
 
-            for (int k = 0; k <= p - maxij; k++)
-               u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
-                        pow(1.0 - ip.z, maxij);
+         for (int k = 0; k <= p - maxij; k++)
+         {
+            u[o++] = shape_x(i) * shape_y(j) * shape_z(k) *
+                     pow(1.0 - ip.z, maxij);
          }
+      }
    }
+
    Ti.Mult(u, shape);
 }
 
@@ -1253,64 +1208,35 @@ void L2_BergotPyramidElement::CalcDShape(const IntegrationPoint &ip,
    const real_t y = (ip.z < 1.0) ? (ip.y / (1.0 - ip.z)) : 0.0;
    const real_t z = ip.z;
 
-   if (std::abs(z - 1.0) < apex_tol)
-   {
-      // Compute the limit of the gradients of the basis functions as
-      // z->1 with x and y on the line between the center of the base and the
-      // apex
-      du = 0.;
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            for (int k = 0; k <= p - maxij; k++, o++)
-            {
-               if (i == 0 && j == 0)
-               {
-                  du(o,2) = (((k + 6.) * k + 11.) * k + 6.) * k / 6.;
-               }
-               else if (i == 1 && j == 0)
-               {
-                  du(o,0) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
-               }
-               else if (i == 0 && j == 1)
-               {
-                  du(o,1) = ((((k + 10.) * k + 35.) * k + 50.) * k + 24.) / 24.;
-               }
-            }
-         }
-   }
-   else
-   {
-      Poly_1D::CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
-      Poly_1D::CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
+   Poly_1D::CalcLegendre(p, x, shape_x.GetData(), dshape_x.GetData());
+   Poly_1D::CalcLegendre(p, y, shape_y.GetData(), dshape_y.GetData());
 
-      int o = 0;
-      for (int i = 0; i <= p; i++)
-         for (int j = 0; j <= p; j++)
-         {
-            int maxij = std::max(i, j);
-            FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0),
-                                             z, 1.0,
-                                             shape_z, dshape_z, dshape_z_dt);
+   int o = 0;
+   for (int i = 0; i <= p; i++)
+   {
+      for (int j = 0; j <= p; j++)
+      {
+         int maxij = std::max(i, j);
+         FuentesPyramid::CalcScaledJacobi(p-maxij, 2.0 * (maxij + 1.0), z, 1.0,
+                                          shape_z, dshape_z, dshape_z_dt);
 
-            for (int k = 0; k <= p - maxij; k++, o++)
-            {
-               du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
-                         pow(1.0 - ip.z, maxij - 1);
-               du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
-                         pow(1.0 - ip.z, maxij - 1);
-               du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
-                         pow(1.0 - ip.z, maxij) +
-                         (ip.x * dshape_x(i) * shape_y(j) +
-                          ip.y * shape_x(i) * dshape_y(j)) *
-                         shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
-                         maxij * shape_x(i) * shape_y(j) * shape_z(k) *
-                         (maxij > 0 ? pow(1.0 - ip.z, maxij - 1) : 0.0);
-            }
+         for (int k = 0; k <= p - maxij; k++, o++)
+         {
+            du(o,0) = dshape_x(i) * shape_y(j) * shape_z(k) *
+                      pow(1.0 - ip.z, maxij - 1);
+            du(o,1) = shape_x(i) * dshape_y(j) * shape_z(k) *
+                      pow(1.0 - ip.z, maxij - 1);
+            du(o,2) = shape_x(i) * shape_y(j) * dshape_z(k) *
+                      pow(1.0 - ip.z, maxij) +
+                      (ip.x * dshape_x(i) * shape_y(j) +
+                       ip.y * shape_x(i) * dshape_y(j)) *
+                      shape_z(k) * pow(1.0 - ip.z, maxij - 2) -
+                      ((maxij > 0) ? (maxij * shape_x(i) * shape_y(j) * shape_z(k) *
+                                      pow(1.0 - ip.z, maxij - 1)) : 0.0);
          }
+      }
    }
+
    Ti.Mult(du, dshape);
 }
 

fem/fe/fe_l2.hpp

@@ -225,8 +225,6 @@ private:
 #endif
    DenseMatrixInverse Ti;
 
-   static constexpr real_t apex_tol = 1e-8;
-
 public:
    /// Construct the L2_PyramidElement of order @a p and BasisType @a btype
    L2_BergotPyramidElement(const int p,

fem/fe/fe_nd.cpp

@@ -1282,49 +1282,12 @@ ND_SegmentElement::ND_SegmentElement(const int p, const int ob_type)
    }
 }
 
-void ND_SegmentElement::CalcShape(const IntegrationPoint &ip,
-                                  Vector &shape) const
-{
-   if (obasis1d.IsIntegratedType()) { obasis1d.ScaleIntegrated(false); }
-   obasis1d.Eval(ip.x, shape);
-}
-
 void ND_SegmentElement::CalcVShape(const IntegrationPoint &ip,
                                    DenseMatrix &shape) const
 {
    Vector vshape(shape.Data(), dof);
 
-   CalcShape(ip, vshape);
-}
-
-void ND_SegmentElement::ProjectIntegrated(VectorCoefficient &vc,
-                                          ElementTransformation &Trans,
-                                          Vector &dofs) const
-{
-   MFEM_ASSERT(obasis1d.IsIntegratedType(), "Not integrated type");
-   real_t vk[Geometry::MaxDim];
-   Vector xk(vk, vc.GetVDim());
-
-   const real_t *cp = poly1d.ClosedPoints(dof, BasisType::GaussLobatto);
-   const IntegrationRule &ir = IntRules.Get(Geometry::SEGMENT, dof);
-   IntegrationPoint ip;
-
-   for (int i = 0; i < dof; i++)
-   {
-      const real_t h = cp[i+1] - cp[i];
-      real_t val = 0.0;
-
-      for (int q = 0; q < ir.GetNPoints(); q++)
-      {
-         const IntegrationPoint &ip1d = ir.IntPoint(q);
-         ip.x = cp[i] + h*ip1d.x;
-         Trans.SetIntPoint(&ip);
-         vc.Eval(xk, Trans, ip);
-         val += ip1d.weight*Trans.Jacobian().InnerProduct(tk, vk);
-      }
-
-      dofs(i) = val*h;
-   }
+   obasis1d.Eval(ip.x, vshape);
 }
 
 const real_t ND_WedgeElement::tk[15] =

fem/fe/fe_nd.hpp

@@ -303,7 +303,8 @@ public:
    /** @brief Construct the ND_SegmentElement of order @a p and open
        BasisType @a ob_type */
    ND_SegmentElement(const int p, const int ob_type = BasisType::GaussLegendre);
-   void CalcShape(const IntegrationPoint &ip, Vector &shape) const override;
+   void CalcShape(const IntegrationPoint &ip, Vector &shape) const override
+   { obasis1d.Eval(ip.x, shape); }
    void CalcVShape(const IntegrationPoint &ip,
                    DenseMatrix &shape) const override;
    void CalcVShape(ElementTransformation &Trans,
@@ -324,10 +325,7 @@ public:
    using FiniteElement::Project;
    void Project(VectorCoefficient &vc,
                 ElementTransformation &Trans, Vector &dofs) const override
-   {
-      if (obasis1d.IsIntegratedType()) { ProjectIntegrated(vc, Trans, dofs); }
-      else { Project_ND(tk, dof2tk, vc, Trans, dofs); }
-   }
+   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
    void ProjectMatrixCoefficient(MatrixCoefficient &mc,
                                  ElementTransformation &T,
                                  Vector &dofs) const override
@@ -340,11 +338,6 @@ public:
                     ElementTransformation &Trans,
                     DenseMatrix &grad) const override
    { ProjectGrad_ND(tk, dof2tk, fe, Trans, grad); }
-
-protected:
-   void ProjectIntegrated(VectorCoefficient &vc,
-                          ElementTransformation &Trans,
-                          Vector &dofs) const;
 };
 
 class ND_WedgeElement : public VectorFiniteElement

fem/fe/fe_pyramid.hpp

@@ -17,12 +17,6 @@
 namespace mfem
 {
 
-struct ScalarPyramid
-{
-   // Default basis type for H1 and L2 pyramids
-   static inline int DefaultType = 1; // Bergot(0) or Fuentes(1)
-};
-
 /** Base class for arbitrary order basis functions on pyramid-shaped elements
 
   This base class provides a common class to store temporary vectors,

fem/fe_coll.cpp

@@ -228,19 +228,7 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name)
    }
    else if (!strncmp(name, "H1_", 3))
    {
-      // Parse pyramid basis type if included in the name
-      const char *pyr = strstr(name, "Pyr");
-      if (pyr == NULL)
-      {
-         // Use default pyramid type elements
-         fec = new H1_FECollection(atoi(name + 7), atoi(name + 3));
-      }
-      else
-      {
-         // Use specific pyramid type elements
-         fec = new H1_FECollection(atoi(name + 7), atoi(name + 3),
-                                   BasisType::GaussLobatto, atoi(pyr + 3));
-      }
+      fec = new H1_FECollection(atoi(name + 7), atoi(name + 3));
    }
    else if (!strncmp(name, "H1Pos_Trace_", 12))
    {
@@ -257,44 +245,26 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name)
    }
    else if (!strncmp(name, "H1@", 3))
    {
-      // Parse pyramid basis type if included in the name
-      const char *pyr = strstr(name, "Pyr");
-      if (pyr == NULL)
-      {
-         // Use default pyramid type elements
-         fec = new H1_FECollection(atoi(name + 9), atoi(name + 5),
-                                   BasisType::GetType(name[3]));
-      }
-      else
-      {
-         // Use specific pyramid type elements
-         fec = new H1_FECollection(atoi(name + 9), atoi(name + 5),
-                                   BasisType::GetType(name[3]),
-                                   atoi(pyr + 3));
-      }
+      fec = new H1_FECollection(atoi(name + 9), atoi(name + 5),
+                                BasisType::GetType(name[3]));
    }
-   else if (!strncmp(name, "L2", 2))
+   else if (!strncmp(name, "L2_T", 4))
+      fec = new L2_FECollection(atoi(name + 10), atoi(name + 6),
+                                atoi(name + 4));
+   else if (!strncmp(name, "L2_", 3))
    {
-      // Parse Map Type
-      const int mtype = strstr(name, "Int") == NULL ?
-                        FiniteElement::VALUE : FiniteElement::INTEGRAL;
-
-      // Parse the base order
-      const int p = atoi(strstr(name, "_P") + 2);
-
-      // Parse the mesh dimension
-      const int dim = atoi(strstr(name, "D") - 1);
-
-      // Parse basis type if specified
-      const char *t = strstr(name, "_T");
-      const int btype = t == NULL ? BasisType::GaussLegendre : atoi(t + 2);
-
-      // Parse the pyramid type if specified
-      const char *pyr = strstr(name, "Pyr");
-      const int ptype = pyr == NULL ? 1 : atoi(pyr + 3);
-
-      // Create collection
-      fec = new L2_FECollection(p, dim, btype, mtype, ptype);
+      fec = new L2_FECollection(atoi(name + 7), atoi(name + 3));
+   }
+   else if (!strncmp(name, "L2Int_T", 7))
+   {
+      fec = new L2_FECollection(atoi(name + 13), atoi(name + 9),
+                                atoi(name + 7), FiniteElement::INTEGRAL);
+   }
+   else if (!strncmp(name, "L2Int_", 6))
+   {
+      fec = new L2_FECollection(atoi(name + 10), atoi(name + 6),
+                                BasisType::GaussLegendre,
+                                FiniteElement::INTEGRAL);
    }
    else if (!strncmp(name, "RT_Trace_", 9))
    {
@@ -1739,10 +1709,9 @@ const int *RT1_3DFECollection::DofOrderForOrientation(Geometry::Type GeomType,
 
 
 H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
-                                 const int pyr_type)
+                                 const int pyrtype)
    : FiniteElementCollection(p)
    , dim(dim)
-   , p_type(pyr_type)
 {
    MFEM_VERIFY(p >= 1, "H1_FECollection requires order >= 1.");
    MFEM_VERIFY(dim >= 0 && dim <= 3, "H1_FECollection requires 0 <= dim <= 3.");
@@ -1755,14 +1724,7 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
    {
       case BasisType::GaussLobatto:
       {
-         if (pyr_type == ScalarPyramid::DefaultType)
-         {
-            snprintf(h1_name, 32, "H1_%dD_P%d", dim, p);
-         }
-         else
-         {
-            snprintf(h1_name, 32, "H1_%dD_P%d_Pyr%d", dim, p, pyr_type);
-         }
+         snprintf(h1_name, 32, "H1_%dD_P%d", dim, p);
          break;
       }
       case BasisType::Positive:
@@ -1948,11 +1910,11 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
          H1_dof[Geometry::TETRAHEDRON] = (TriDof*pm3)/3;
          H1_dof[Geometry::CUBE] = QuadDof*pm1;
          H1_dof[Geometry::PRISM] = TriDof*pm1;
-         if (pyr_type == 0 || b_type == BasisType::Positive)
+         if (pyrtype == 0 || b_type == BasisType::Positive)
          {
             H1_dof[Geometry::PYRAMID] = pm2*pm1*(2*p-3)/6; // Bergot (JSC)
          }
-         else if (pyr_type == 1)
+         else if (pyrtype == 1)
          {
             H1_dof[Geometry::PYRAMID] = pm1*pm1*pm1; // Fuentes
          }
@@ -1973,15 +1935,13 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype,
                new H1_TetrahedronElement(p, btype);
             H1_Elements[Geometry::CUBE] = new H1_HexahedronElement(p, btype);
             H1_Elements[Geometry::PRISM] = new H1_WedgeElement(p, btype);
-            if (pyr_type == 0)
+            if (pyrtype == 0)
             {
-               H1_Elements[Geometry::PYRAMID] =
-                  new H1_BergotPyramidElement(p, btype);
+               H1_Elements[Geometry::PYRAMID] = new H1_BergotPyramidElement(p, btype);
             }
             else
             {
-               H1_Elements[Geometry::PYRAMID] =
-                  new H1_FuentesPyramidElement(p, btype);
+               H1_Elements[Geometry::PYRAMID] = new H1_FuentesPyramidElement(p, btype);
             }
          }
 
@@ -2188,7 +2148,6 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
    : FiniteElementCollection(p)
    , dim(dim)
    , m_type(map_type)
-   , p_type(pyr_type)
 {
    MFEM_VERIFY(p >= 0, "L2_FECollection requires order >= 0.");
 
@@ -2204,25 +2163,10 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
    switch (btype)
    {
       case BasisType::GaussLegendre:
-         if (pyr_type == ScalarPyramid::DefaultType)
-         {
-            snprintf(d_name, 32, "%s_%dD_P%d", prefix, dim, p);
-         }
-         else
-         {
-            snprintf(d_name, 32, "%s_%dD_P%d_Pyr%d", prefix, dim, p, pyr_type);
-         }
+         snprintf(d_name, 32, "%s_%dD_P%d", prefix, dim, p);
          break;
       default:
-         if (pyr_type == ScalarPyramid::DefaultType)
-         {
-            snprintf(d_name, 32, "%s_T%d_%dD_P%d", prefix, btype, dim, p);
-         }
-         else
-         {
-            snprintf(d_name, 32, "%s_T%d_%dD_P%d_Pyr%d",
-                     prefix, btype, dim, p, pyr_type);
-         }
+         snprintf(d_name, 32, "%s_T%d_%dD_P%d", prefix, btype, dim, p);
    }
 
    for (int g = 0; g < Geometry::NumGeom; g++)
@@ -2341,13 +2285,11 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
          L2_Elements[Geometry::PRISM] = new L2_WedgeElement(p, btype);
          if (pyr_type == 0)
          {
-            L2_Elements[Geometry::PYRAMID] =
-               new L2_BergotPyramidElement(p, btype);
+            L2_Elements[Geometry::PYRAMID] = new L2_BergotPyramidElement(p, btype);
          }
          else
          {
-            L2_Elements[Geometry::PYRAMID] =
-               new L2_FuentesPyramidElement(p, btype);
+            L2_Elements[Geometry::PYRAMID] = new L2_FuentesPyramidElement(p, btype);
          }
       }
 

fem/fe_coll.hpp

@@ -100,10 +100,6 @@ public:
       return FiniteElementForGeometry(GeomType);
    }
 
-   /** @brief Returns a collection of the trace elements.
-
-       @note The collection is owned by the caller and is NOT deleted in the
-       destructor. */
    virtual FiniteElementCollection *GetTraceCollection() const;
 
    virtual ~FiniteElementCollection();
@@ -290,7 +286,7 @@ protected:
 class H1_FECollection : public FiniteElementCollection
 {
 protected:
-   int dim, b_type, p_type;
+   int dim, b_type;
    char h1_name[32];
    FiniteElement *H1_Elements[Geometry::NumGeom];
    int H1_dof[Geometry::NumGeom];
@@ -299,7 +295,7 @@ protected:
 public:
    explicit H1_FECollection(const int p, const int dim = 3,
                             const int btype = BasisType::GaussLobatto,
-                            const int pyr_type = ScalarPyramid::DefaultType);
+                            const int pyrtype = 1);
 
    const FiniteElement *
    FiniteElementForGeometry(Geometry::Type GeomType) const override;
@@ -324,7 +320,7 @@ public:
    const int *GetDofMap(Geometry::Type GeomType, int p) const;
 
    FiniteElementCollection *Clone(int p) const override
-   { return new H1_FECollection(p, dim, b_type, p_type); }
+   { return new H1_FECollection(p, dim, b_type); }
 
    int GetConstructorOrder() const override
    { return base_p; }
@@ -371,7 +367,6 @@ private:
    int dim;
    int b_type; // BasisType
    int m_type; // map type
-   int p_type; // Pyramid type (0 -> Bergot, 1 -> Fuentes)
    char d_name[32];
    ScalarFiniteElement *L2_Elements[Geometry::NumGeom];
    ScalarFiniteElement *Tr_Elements[Geometry::NumGeom];
@@ -384,7 +379,7 @@ public:
    L2_FECollection(const int p, const int dim,
                    const int btype = BasisType::GaussLegendre,
                    const int map_type = FiniteElement::VALUE,
-                   const int pyr_type = ScalarPyramid::DefaultType);
+                   const int pyrtype = 1);
 
    const FiniteElement *
    FiniteElementForGeometry(Geometry::Type GeomType) const override;
@@ -414,7 +409,7 @@ public:
    int GetBasisType() const { return b_type; }
 
    FiniteElementCollection *Clone(int p) const override
-   { return new L2_FECollection(p, dim, b_type, m_type, p_type); }
+   { return new L2_FECollection(p, dim, b_type, m_type); }
 
    int GetConstructorOrder() const override
    { return base_p; }

fem/gslib.hpp

@@ -12,9 +12,6 @@
 #ifndef MFEM_GSLIB
 #define MFEM_GSLIB
 
-#include <map>
-#include <vector>
-
 #include "../config/config.hpp"
 #ifdef MFEM_USE_MPI
 #include "pgridfunc.hpp"
@@ -122,11 +119,6 @@ protected:
    // IntegrationRules for simplex->Quad/Hex and to project to p_max in-case of
    // p-refinement.
    Array<IntegrationRule *> ir_split;
-   /// Integration rules built at the field polynomial order (only for surface
-   /// meshes when mesh order is not the same as gridfunction order).
-   Array<IntegrationRule *> ir_split_sol;
-   /// Order at which #ir_split_sol was built; -1 means not built.
-   int ir_split_sol_order = -1;
    Array<FiniteElementSpace *> fes_rst_map; //FESpaces to map Quad/Hex->Simplex
    Array<GridFunction *> gf_rst_map; // GridFunctions to map Quad/Hex->Simplex
    FiniteElementCollection *fec_map_lin;
@@ -142,8 +134,6 @@ protected:
    AvgType avgtype;             // average type used for L2 functions
    Array<int> split_element_map;
    Array<int> split_element_index;
-   // Geometry::Type (as int) of the original element for each split quad.
-   Array<int> split_element_geom;
    int        NE_split_total;   // total number of elements after mesh splitting
    int        mesh_points_cnt;  // number of mesh nodes
    // Tolerance to ignore points found beyond the mesh boundary.
@@ -151,12 +141,6 @@ protected:
    double     bdr_tol;
    // Use CPU functions for Mesh/GridFunction on device for gslib1.0.7
    bool       gpu_to_cpu_fallback = false;
-   // Check if a point is inside the oriented bounding box of an
-   // element before the Newton iteration.
-   // Note: only used in MFEM implementation (not in gslib) which currently
-   // supports GPU kernels for area meshes in 2D, volume meshes in 3D,
-   // and surface meshes in 1D/2D/3D.
-   bool       obb_check = true;
 
    // Device specific data used for FindPoints
    struct DEV_STRUCT
@@ -178,16 +162,11 @@ protected:
       mutable double surf_dist_tol;
    } DEV;
 
-   // Helper function to setup and free gslib's crystal router.
-   void SetupCrystal(); // Called inside Setup and SetupSurf_base
-   void FreeCrystal();  // Called inside FreeData
-
-   /// Use GSLIB for communication and interpolation. Updates field_out on
-   /// host.
+   /// Use GSLIB for communication and interpolation
    virtual void InterpolateH1(const GridFunction &field_in, Vector &field_out,
                               const int field_out_ordering);
    /// Uses GSLIB Crystal Router for communication followed by MFEM's
-   /// interpolation functions. Updates field_out on host.
+   /// interpolation functions
    virtual void InterpolateGeneral(const GridFunction &field_in,
                                    Vector &field_out,
                                    const int field_out_ordering);
@@ -202,26 +181,12 @@ protected:
                                                  IntegrationRule *irule,
                                                  int order);
 
-   /** @brief Build integration rules at the given @a order for each split mesh
-    *  and store them in @a ir_out. Requires that \ref SetupSplitMeshes has
-    *  already been called. */
-   virtual void SetupIntegrationRules(const int order,
-                                      Array<IntegrationRule *> &ir_out);
-
    /** @brief Helper function that calls \ref SetupSplitMeshes and
-    * \ref SetupIntegrationRules. */
+    * \ref SetupIntegrationRuleForSplitMesh. */
    virtual void SetupSplitMeshesAndIntegrationRules(const int order);
 
-   /** @brief Get GridFunction value at the points expected by GSLIB.
-    *  @param[in]  gf_in       Grid function to evaluate.
-    *  @param[out] node_vals   Output values.
-    *  @param[in]  ir_in       If non-null, use these rules instead of #ir_split.
-    *  @param[in]  by_element  If true, output has element-major layout
-    *                          [nel][vdim][ndofs]; otherwise component-major
-    *                          layout [vdim][total_pts]. */
-   virtual void GetNodalValues(const GridFunction *gf_in, Vector &node_vals,
-                               const Array<IntegrationRule *> *ir_in = nullptr,
-                               bool by_element = false) const;
+   /// Get GridFunction value at the points expected by GSLIB.
+   virtual void GetNodalValues(const GridFunction *gf_in, Vector &node_vals) const;
 
    /** @brief Map {r,s,t} coordinates from [-1,1] to [0,1] for MFEM. For
     *  simplices, find the original element number (that was split into
@@ -326,60 +291,29 @@ protected:
    void findptsedge_setup_2(DEV_STRUCT &devs,
                             const double *const elx[2],
                             const unsigned n,
-                            const unsigned int nel,
+                            const uint nel,
                             const unsigned m,
-                            const double bbox_rel_size_inc,
-                            const unsigned int local_hash_size,
-                            const unsigned int global_hash_size,
-                            const Vector *aabb_sz_inc);
+                            const double bbox_tol,
+                            const uint local_hash_size,
+                            const uint global_hash_size);
 
    /// Preprocess 3D surface mesh needed for FindPoints.
    void findptssurf_setup_3(DEV_STRUCT &devs,
                             const double *const elx[3],
                             const unsigned n,
-                            const unsigned int nel,
+                            const uint nel,
                             const unsigned m,
-                            const double bbox_rel_size_inc,
-                            const unsigned int local_hash_size,
-                            const unsigned int global_hash_size,
-                            const int rD,
-                            const Vector *aabb_sz_inc);
+                            const double bbox_tol,
+                            const uint local_hash_size,
+                            const uint global_hash_size,
+                            const int rD);
 
-   /** @brief Shared implementation for the public surface-setup methods.
-    *
-    *  @details Initializes the surface-search data structures, builds the
-    *  split-element representation expected by gslib, and constructs the
-    *  element bounding boxes used by the MFEM surface kernels.
-    *
-    *  If @a aabb_sz_inc is null, the setup stores the default oriented
-    *  bounding boxes and uses @a bbox_rel_size_inc as their relative size
-    *  increase factor.
-    *
-    *  If @a aabb_sz_inc is non-null, the setup stores axis-aligned bounding
-    *  boxes only, applies the requested absolute AABB expansion in each
-    *  physical direction, and adjusts the tolerance @a bdr_tol so points
-    *  found in the expanded region are classified as border points.
-    *
-    *  @param[in] m                  Input surface mesh.
-    *  @param[in] bbox_rel_size_inc  Relative size increase applied when
-    *                                expanding each element bounding box during
-    *                                setup.
-    *  @param[in] aabb_sz_inc        Optional total absolute AABB expansion
-    *                                applied to the stored axis-aligned
-    *                                bounding boxes after construction.
-    *  @param[in] newt_tol           Newton tolerance for the point-search
-    *                                kernels.
-    */
-   void SetupSurf_Base(Mesh &m,
-                       const double bbox_rel_size_inc,
-                       const Vector *aabb_sz_inc,
-                       const double newt_tol);
 public:
    /// Serial constructor
    FindPointsGSLIB();
 
    /// Serial constructor + setup with given Mesh (see \ref Setup)
-   FindPointsGSLIB(Mesh &mesh_in, const double bbox_rel_size_inc = 0.1,
+   FindPointsGSLIB(Mesh &mesh_in, const double bb_t = 0.1,
                    const double newt_tol = 1.0e-12,
                    const int npt_max = 256);
 
@@ -388,7 +322,7 @@ public:
    FindPointsGSLIB(MPI_Comm comm_);
 
    /// Constructor + setup with given ParMesh (see \ref Setup)
-   FindPointsGSLIB(ParMesh &mesh_in, const double bbox_rel_size_inc = 0.1,
+   FindPointsGSLIB(ParMesh &mesh_in, const double bb_t = 0.1,
                    const double newt_tol = 1.0e-12,
                    const int npt_max = 256);
 #endif
@@ -404,59 +338,23 @@ public:
        Note: not tested with periodic (L2).
        Note: the input mesh \p m must have Nodes set.
 
-       @param[in] m                  Input mesh.
-       @param[in] bbox_rel_size_inc  (Optional) Relative size increase applied
-                                     when expanding each element bounding box.
-       @param[in] newt_tol           (Optional) Newton tolerance for the gslib
-                                     search methods.
-       @param[in] npt_max            (Optional) Number of points for
-                                     simultaneous iteration. This alters
-                                     performance and memory footprint.
+       @param[in] m         Input mesh.
+       @param[in] bb_t      (Optional) Relative size of bounding box around
+                            each element.
+       @param[in] newt_tol  (Optional) Newton tolerance for the gslib
+                            search methods.
+       @param[in] npt_max   (Optional) Number of points for simultaneous
+                            iteration. This alters performance and
+                            memory footprint.
    */
-   void Setup(Mesh &m, const double bbox_rel_size_inc = 0.1,
-              const double newt_tol = 1.0e-12,
+   void Setup(Mesh &m, const double bb_t = 0.1, const double newt_tol = 1.0e-12,
               const int npt_max = 256);
 
    /// Preprocess the surface mesh to compute data for FindPoints.
    void SetupSurf(Mesh &m,
-                  const double bbox_rel_size_inc = 0.1,
-                  const double newt_tol = 1.0e-12);
-
-   /** @brief Preprocess the surface mesh to compute data for FindPoints using
-    *  absolute AABB expansion.
-    *
-    *  @details This method computes only axis-aligned bounding boxes and
-    *  increases their total length by a user-specified amount in each
-    *  physical direction. The absolute AABB expansion is applied
-    *  symmetrically to the lower and upper bounds.
-    *
-    *  The size of @a aabb_sz_inc determines how the expansion values are
-    *  interpreted:
-    *  - `1`: one expansion value used in every direction for every element
-    *  - `NElements`: one expansion value per element, reused in x/y/z
-    *    directions
-    *  - `SpaceDim`: one expansion value per physical direction, reused for
-    *    every element
-    *  - `NElements*SpaceDim`: one expansion value per element and direction,
-    *    ordered as `(dx1,dy1,dz1, ... dxN,dyN,dzN)`
-    *
-    *  This method disables the oriented bounding-box precheck because the
-    *  stored boxes are modified only in their axis-aligned representation.
-    *
-    *  @param[in] m                  Input surface mesh.
-    *  @param[in] aabb_sz_inc        Total absolute AABB expansion applied in
-    *                                each physical direction to the stored
-    *                                axis-aligned bounding boxes.
-    *  @param[in] newt_tol           Newton tolerance for the point-search
-    *                                kernels.
-    *
-    *  @note We disable the oriented bounding box check with this setup.
-    *        @a bdr_tol is also adjusted so that all points in the AABBs can
-    *        be found.
-    */
-   void SetupSurfWithAABBExpansion(Mesh &m, const Vector &aabb_sz_inc,
-                                   const double newt_tol = 1.0e-12);
-
+                  const double bb_t = 0.1,
+                  const double newt_tol = 1.0e-12,
+                  const int npt_max = 256);
 
    /** @brief Searches positions given in physical space by \p point_pos.
 
@@ -503,8 +401,7 @@ public:
    /// Setup FindPoints and search positions
    void FindPoints(Mesh &m, const Vector &point_pos,
                    const int point_pos_ordering = Ordering::byNODES,
-                   const double bbox_rel_size_inc = 0.1,
-                   const double newt_tol = 1.0e-12,
+                   const double bb_t = 0.1, const double newt_tol = 1.0e-12,
                    const int npt_max = 256);
 
    /** @brief Interpolation of field values at prescribed reference space
@@ -516,11 +413,7 @@ public:
                               mesh that was given to Setup().
        @param[out] field_out  Interpolated values. For points that are not found
                               the value is set to #default_interp_value.
-                              The output ordering is determined from field_in.
-
-       @note: field_out is moved to device if field_in is on device. Otherwise,
-              field_out memory allocation is not changed.
-   */
+                              The output ordering is determined from field_in.*/
    virtual void Interpolate(const GridFunction &field_in, Vector &field_out);
 
    /// Interpolation of field values, with output ordering specification.
@@ -575,12 +468,7 @@ public:
     *  @details When using FindPoints, gslib may return points as found on the
     *  boundary even when they are slightly outside the domain. This tolerance
     *  is used to filter such points based on the distance^2 value and mark them
-    *  as not found.
-    *
-    *  @note When the SetupSurfWithAABBExpansion method is used for surface
-    *  meshes, this tolerance is automatically computed based on the size of
-    *  expanded AABBs. Using this method will override that computed tolerance.
-    *  */
+    *  as not found.*/
    virtual void SetDistanceToleranceForPointsFoundOnBoundary(double bdr_tol_)
    {
       bdr_tol = bdr_tol_;
@@ -715,28 +603,25 @@ public:
        Note: not tested with periodic meshes (L2).
        Note: the input mesh \p m must have Nodes set.
 
-       @param[in] m                  Input mesh.
-       @param[in] meshid             A unique # for each overlapping mesh.
-                                     This id is used to make sure that points
-                                     being searched are not looked for in the
-                                     mesh that they belong to.
-       @param[in] gfmax              (Optional) GridFunction in H1 that is used
-                                     as a discriminator when one point is
-                                     located in multiple meshes. The mesh that
-                                     maximizes gfmax is chosen. For example,
-                                     using the distance field based on the
-                                     overlapping boundaries is helpful for
-                                     convergence during Schwarz iterations.
-       @param[in] bbox_rel_size_inc  (Optional) Relative size increase applied
-                                     when expanding each element bounding box.
-       @param[in] newt_tol           (Optional) Newton tolerance for the gslib
-                                     search methods.
-       @param[in] npt_max            (Optional) Number of points for
-                                     simultaneous iteration. This alters
-                                     performance and memory footprint.*/
-   void Setup(Mesh &m, const int meshid, GridFunction *gfmax = nullptr,
-              const double bbox_rel_size_inc = 0.1,
-              const double newt_tol = 1.0e-12,
+       @param[in] m         Input mesh.
+       @param[in] meshid    A unique # for each overlapping mesh. This id is
+                            used to make sure that points being searched are not
+                            looked for in the mesh that they belong to.
+       @param[in] gfmax     (Optional) GridFunction in H1 that is used as a
+                            discriminator when one point is located in multiple
+                            meshes. The mesh that maximizes gfmax is chosen.
+                            For example, using the distance field based on the
+                            overlapping boundaries is helpful for convergence
+                            during Schwarz iterations.
+       @param[in] bb_t      (Optional) Relative size of bounding box around
+                            each element.
+       @param[in] newt_tol  (Optional) Newton tolerance for the gslib
+                            search methods.
+       @param[in] npt_max   (Optional) Number of points for simultaneous
+                            iteration. This alters performance and
+                            memory footprint.*/
+   void Setup(Mesh &m, const int meshid, GridFunction *gfmax = NULL,
+              const double bb_t = 0.1, const double newt_tol = 1.0e-12,
               const int npt_max = 256);
 
    /** Searches positions given in physical space by \p point_pos. All output
@@ -792,7 +677,7 @@ class GSOPGSLIB
 protected:
    struct gslib::crystal *cr;               // gslib's internal data
    struct gslib::comm *gsl_comm;            // gslib's internal data
-   struct gslib::gs_data *gsl_data = nullptr;
+   struct gslib::gs_data *gsl_data = NULL;
    int num_ids;
 
 public:

fem/gslib/findpts_local_2.cpp

@@ -11,7 +11,7 @@
 
 #include "../gslib.hpp"
 #include "../../general/forall.hpp"
-#include "gslib_kernel_helpers.hpp"
+#include "../../linalg/kernels.hpp"
 
 #ifdef MFEM_USE_GSLIB
 
@@ -27,6 +27,8 @@
 #pragma GCC diagnostic pop
 #endif
 
+#include <climits>
+
 namespace mfem
 {
 #if GSLIB_RELEASE_VERSION >= 10009
@@ -52,14 +54,127 @@ struct findptsElementGPT_t
    double x[DIM], jac[DIM * DIM], hes[4];
 };
 
-using dbl_range_t = gslib::dbl_range_t;
-using obbox_t = gslib::obbox_t<DIM>;
-using findptsLocalHashData_t = gslib::findptsLocalHashData_t<DIM>;
-using gslib::bbox_test;
-using gslib::hash_index;
-using gslib::l2norm2;
-using gslib::lag_eval_first_der;
-using gslib::lag_eval_second_der;
+struct dbl_range_t
+{
+   double min, max;
+};
+struct obbox_t
+{
+   double c0[DIM], A[DIM * DIM];
+   dbl_range_t x[DIM];
+};
+
+struct findptsLocalHashData_t
+{
+   int hash_n;
+   dbl_range_t bnd[DIM];
+   double fac[DIM];
+   unsigned int *offset;
+   int max;
+};
+
+// Eval the ith Lagrange interpolant and its first derivative at x.
+// Note: lCoeff stores pre-computed coefficients for fast evaluation.
+static MFEM_HOST_DEVICE inline void lag_eval_first_der(double *p0, double x,
+                                                       int i, const double *z,
+                                                       const double *lCoeff,
+                                                       int pN)
+{
+   double u0 = 1, u1 = 0;
+   for (int j = 0; j < pN; ++j)
+   {
+      if (i != j)
+      {
+         double d_j = 2 * (x - z[j]);
+         u1 = d_j * u1 + u0;
+         u0 = d_j * u0;
+      }
+   }
+   p0[i] = lCoeff[i] * u0;
+   p0[pN+i] = 2.0 * lCoeff[i] * u1;
+}
+
+// Eval the ith Lagrange interpolant and its first and second derivative at x.
+// Note: lCoeff stores pre-computed coefficients for fast evaluation.
+static MFEM_HOST_DEVICE inline void lag_eval_second_der(double *p0, double x,
+                                                        int i, const double *z,
+                                                        const double *lCoeff,
+                                                        int pN)
+{
+   double u0 = 1, u1 = 0, u2 = 0;
+   for (int j = 0; j < pN; ++j)
+   {
+      if (i != j)
+      {
+         double d_j = 2 * (x - z[j]);
+         u2 = d_j * u2 + u1;
+         u1 = d_j * u1 + u0;
+         u0 = d_j * u0;
+      }
+   }
+   p0[i] = lCoeff[i] * u0;
+   p0[pN+i] = 2.0 * lCoeff[i] * u1;
+   p0[2*pN+i] = 8.0 * lCoeff[i] * u2;
+}
+
+// Axis-aligned bounding box test.
+static MFEM_HOST_DEVICE inline double AABB_test(const obbox_t *const b,
+                                                const double x[2])
+{
+   double test = 1;
+   for (int d = 0; d < 2; ++d)
+   {
+      double b_d = (x[d] - b->x[d].min) * (b->x[d].max - x[d]);
+      test = test < 0 ? test : b_d;
+   }
+   return test;
+}
+
+// Axis-aligned bounding box test followed by oriented bounding-box test.
+static MFEM_HOST_DEVICE inline double bbox_test(const obbox_t *const b,
+                                                const double x[2])
+{
+   const double bxyz = AABB_test(b, x);
+   if (bxyz < 0)
+   {
+      return bxyz;
+   }
+   else
+   {
+      double dxyz[2];
+      for (int d = 0; d < 2; ++d)
+      {
+         dxyz[d] = x[d] - b->c0[d];
+      }
+      double test = 1;
+      for (int d = 0; d < 2; ++d)
+      {
+         double rst = 0;
+         for (int e = 0; e < 2; ++e)
+         {
+            rst += b->A[d * 2 + e] * dxyz[e];
+         }
+         double brst = (rst + 1) * (1 - rst);
+         test = test < 0 ? test : brst;
+      }
+      return test;
+   }
+}
+
+// Element index corresponding to hash mesh that the point is located in.
+static MFEM_HOST_DEVICE inline int hash_index(const findptsLocalHashData_t *p,
+                                              const double x[2])
+{
+   const int n = p->hash_n;
+   int sum = 0;
+   for (int d = 2 - 1; d >= 0; --d)
+   {
+      sum *= n;
+      int i = (int)floor((x[d] - p->bnd[d].min) * p->fac[d]);
+      sum += i < 0 ? 0 : (n - 1 < i ? n - 1 : i);
+   }
+   return sum;
+}
 
 /*Solve Ax=y. A is row-major */
 static MFEM_HOST_DEVICE inline void lin_solve_2(double x[2], const double A[4],
@@ -70,6 +185,12 @@ static MFEM_HOST_DEVICE inline void lin_solve_2(double x[2], const double A[4],
    x[1] = idet*(A[0]*y[1] - A[2]*y[0]);
 }
 
+/* L2 norm squared. */
+static MFEM_HOST_DEVICE inline double l2norm2(const double x[2])
+{
+   return x[0] * x[0] + x[1] * x[1];
+}
+
 /* the bit structure of flags is CSSRR
    the C bit --- 1<<4 --- is set when the point is converged
    RR is 0 = 00b if r is unconstrained,
@@ -231,7 +352,7 @@ static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsElementPoint_t *res,
                                                  const findptsElementPoint_t *p,
                                                  const double tol)
 {
-   const double dist2 = l2norm2<2>(resid);
+   const double dist2 = l2norm2(resid);
    const double decr = p->dist2 - dist2;
    const double pred = p->dist2p;
    for (int d = 0; d < 2; ++d)
@@ -574,25 +695,25 @@ static MFEM_HOST_DEVICE double tensor_ig2_j(double *g_partials,
 }
 
 template<int T_D1D = 0>
-static void FindPointsLocal2DKernel(const int npt,
-                                    const double tol,
-                                    const double *x,
-                                    const int point_pos_ordering,
-                                    const double *xElemCoord,
-                                    const int nel,
-                                    const double *wtend,
-                                    const double *boxinfo,
-                                    const int hash_n,
-                                    const double *hashMin,
-                                    const double *hashFac,
-                                    unsigned int *hashOffset,
-                                    unsigned int *const code_base,
-                                    unsigned int *const el_base,
-                                    double *const r_base,
-                                    double *const dist2_base,
-                                    const double *gll1D,
-                                    const double *lagcoeff,
-                                    const int pN = 0)
+static void FindPointsLocal2D_Kernel(const int npt,
+                                     const double tol,
+                                     const double *x,
+                                     const int point_pos_ordering,
+                                     const double *xElemCoord,
+                                     const int nel,
+                                     const double *wtend,
+                                     const double *boxinfo,
+                                     const int hash_n,
+                                     const double *hashMin,
+                                     const double *hashFac,
+                                     unsigned int *hashOffset,
+                                     unsigned int *const code_base,
+                                     unsigned int *const el_base,
+                                     double *const r_base,
+                                     double *const dist2_base,
+                                     const double *gll1D,
+                                     const double *lagcoeff,
+                                     const int pN = 0)
 {
    const int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
    const int D1D = T_D1D ? T_D1D : pN;
@@ -1054,45 +1175,30 @@ void FindPointsGSLIB::FindPointsLocal2(const Vector &point_pos,
    switch (DEV.dof1d)
    {
       case 2:
-         FindPointsLocal2DKernel<2>(npt, DEV.newt_tol, pp,
-                                    point_pos_ordering, pgslm,
-                                    NE_split_total, pwt, pbb,
-                                    DEV.lh_nx, plhm, plhf, plho,
-                                    pcode, pelem, pref, pdist,
-                                    pgll1d, plc);
-         break;
+         return FindPointsLocal2D_Kernel<2>(
+                   npt, DEV.newt_tol, pp, point_pos_ordering, pgslm, NE_split_total, pwt,
+                   pbb, DEV.lh_nx, plhm, plhf, plho, pcode, pelem, pref, pdist,
+                   pgll1d, plc);
       case 3:
-         FindPointsLocal2DKernel<3>(npt, DEV.newt_tol, pp,
-                                    point_pos_ordering, pgslm,
-                                    NE_split_total, pwt, pbb,
-                                    DEV.lh_nx, plhm, plhf, plho,
-                                    pcode, pelem, pref, pdist,
-                                    pgll1d, plc);
-         break;
+         return FindPointsLocal2D_Kernel<3>(
+                   npt, DEV.newt_tol, pp, point_pos_ordering, pgslm, NE_split_total, pwt,
+                   pbb, DEV.lh_nx, plhm, plhf, plho, pcode, pelem, pref, pdist,
+                   pgll1d, plc);
       case 4:
-         FindPointsLocal2DKernel<4>(npt, DEV.newt_tol, pp,
-                                    point_pos_ordering, pgslm,
-                                    NE_split_total, pwt, pbb,
-                                    DEV.lh_nx, plhm, plhf, plho,
-                                    pcode, pelem, pref, pdist,
-                                    pgll1d, plc);
-         break;
+         return FindPointsLocal2D_Kernel<4>(
+                   npt, DEV.newt_tol, pp, point_pos_ordering, pgslm, NE_split_total, pwt,
+                   pbb, DEV.lh_nx, plhm, plhf, plho, pcode, pelem, pref, pdist,
+                   pgll1d, plc);
       case 5:
-         FindPointsLocal2DKernel<5>(npt, DEV.newt_tol, pp,
-                                    point_pos_ordering, pgslm,
-                                    NE_split_total, pwt, pbb,
-                                    DEV.lh_nx, plhm, plhf, plho,
-                                    pcode, pelem, pref, pdist,
-                                    pgll1d, plc);
-         break;
+         return FindPointsLocal2D_Kernel<5>(
+                   npt, DEV.newt_tol, pp, point_pos_ordering, pgslm, NE_split_total, pwt,
+                   pbb, DEV.lh_nx, plhm, plhf, plho, pcode, pelem, pref, pdist,
+                   pgll1d, plc);
       default:
-         FindPointsLocal2DKernel(npt, DEV.newt_tol, pp,
-                                 point_pos_ordering, pgslm,
-                                 NE_split_total, pwt, pbb,
-                                 DEV.lh_nx, plhm, plhf, plho,
-                                 pcode, pelem, pref, pdist,
-                                 pgll1d, plc, DEV.dof1d);
-         break;
+         return FindPointsLocal2D_Kernel(npt, DEV.newt_tol, pp, point_pos_ordering,
+                                         pgslm, NE_split_total, pwt, pbb, DEV.lh_nx,
+                                         plhm, plhf, plho, pcode, pelem,
+                                         pref, pdist, pgll1d, plc, DEV.dof1d);
    }
 }
 #undef DIM2

fem/gslib/findpts_local_3.cpp

@@ -11,7 +11,9 @@
 
 #include "../gslib.hpp"
 #include "../../general/forall.hpp"
-#include "gslib_kernel_helpers.hpp"
+#include "../../linalg/kernels.hpp"
+
+#include <climits>
 
 #ifdef MFEM_USE_GSLIB
 
@@ -57,15 +59,128 @@ struct findptsElemPt
    double x[DIM], jac[DIM * DIM], hes[18];
 };
 
-using dbl_range_t = gslib::dbl_range_t;
-using obbox_t = gslib::obbox_t<DIM>;
-using findptsLocalHashData_t = gslib::findptsLocalHashData_t<DIM>;
-using gslib::bbox_test;
-using gslib::hash_index;
-using gslib::l2norm2;
-using gslib::lag_eval_first_der;
-using gslib::lag_eval_second_der;
-using gslib::lin_solve_sym_2;
+struct dbl_range_t
+{
+   double min, max;
+};
+
+struct obbox_t
+{
+   double c0[DIM], A[DIM * DIM];
+   dbl_range_t x[DIM];
+};
+
+struct findptsLocalHashData_t
+{
+   int hash_n;
+   dbl_range_t bnd[DIM];
+   double fac[DIM];
+   unsigned int *offset;
+   // int max;
+};
+
+// Eval the ith Lagrange interpolant and its first derivative at x.
+// Note: lCoeff stores pre-computed coefficients for fast evaluation.
+static MFEM_HOST_DEVICE inline void lag_eval_first_der(double *p0, double x,
+                                                       int i, const double *z,
+                                                       const double *lCoeff,
+                                                       int pN)
+{
+   double u0 = 1, u1 = 0;
+   for (int j = 0; j < pN; ++j)
+   {
+      if (i != j)
+      {
+         double d_j = 2*(x-z[j]);
+         u1 = d_j*u1+u0;
+         u0 = d_j*u0;
+      }
+   }
+   p0[i] = lCoeff[i]*u0;
+   p0[pN+i] = 2.0*lCoeff[i]*u1;
+}
+
+// Eval the ith Lagrange interpolant and its first and second derivative at x.
+// Note: lCoeff stores pre-computed coefficients for fast evaluation.
+static MFEM_HOST_DEVICE inline void lag_eval_second_der(double *p0, double x,
+                                                        int i, const double *z,
+                                                        const double *lCoeff,
+                                                        int pN)
+{
+   double u0 = 1, u1 = 0, u2 = 0;
+   for (int j = 0; j < pN; ++j)
+   {
+      if (i != j)
+      {
+         double d_j = 2*(x-z[j]);
+         u2 = d_j*u2+u1;
+         u1 = d_j*u1+u0;
+         u0 = d_j*u0;
+      }
+   }
+   p0[i] = lCoeff[i]*u0;
+   p0[pN+i] = 2.0*lCoeff[i]*u1;
+   p0[2*pN+i] = 8.0*lCoeff[i]*u2;
+}
+
+// Axis-aligned bounding box test.
+static MFEM_HOST_DEVICE inline double AABB_test(const obbox_t *const b,
+                                                const double x[3])
+{
+   double b_d;
+   for (int d = 0; d < 3; ++d)
+   {
+      b_d = (x[d]-b->x[d].min)*(b->x[d].max-x[d]);
+      if (b_d < 0) { return b_d; }
+   }
+   return b_d;
+}
+
+// Axis-aligned bounding box test followed by oriented bounding-box test.
+static MFEM_HOST_DEVICE inline double bbox_test(const obbox_t *const b,
+                                                const double x[3])
+{
+   const double bxyz = AABB_test(b, x);
+   if (bxyz < 0)
+   {
+      return bxyz;
+   }
+   else
+   {
+      double dxyz[3];
+      for (int d = 0; d < 3; ++d)
+      {
+         dxyz[d] = x[d]-b->c0[d];
+      }
+      double test = 1;
+      for (int d = 0; d < 3; ++d)
+      {
+         double rst = 0;
+         for (int e = 0; e < 3; ++e)
+         {
+            rst += b->A[d*3+e]*dxyz[e];
+         }
+         double brst = (rst+1)*(1-rst);
+         test = test < 0 ? test : brst;
+      }
+      return test;
+   }
+}
+
+// Element index corresponding to hash mesh that the point is located in.
+static MFEM_HOST_DEVICE inline int hash_index(const findptsLocalHashData_t *p,
+                                              const double x[3])
+{
+   const int n = p->hash_n;
+   int sum = 0;
+   for (int d = 3-1; d >= 0; --d)
+   {
+      sum *= n;
+      int i = (int)floor((x[d]-p->bnd[d].min)*p->fac[d]);
+      sum += i < 0 ? 0 : (n-1 < i ? n-1 : i);
+   }
+   return sum;
+}
 
 // Solve Ax=y. A is row-major.
 static MFEM_HOST_DEVICE inline void lin_solve_3(double x[3], const double A[9],
@@ -84,6 +199,22 @@ static MFEM_HOST_DEVICE inline void lin_solve_3(double x[3], const double A[9],
    x[2] = idet*(inv6*y[0]+inv7*y[1]+inv8*y[2]);
 }
 
+// Solve Ax=y. A is a symmetric 2x2 matrix.
+static MFEM_HOST_DEVICE inline void lin_solve_sym_2(double x[2],
+                                                    const double A[3],
+                                                    const double y[2])
+{
+   const double idet = 1 / (A[0]*A[2]-A[1]*A[1]);
+   x[0] = idet*(A[2]*y[0]-A[1]*y[1]);
+   x[1] = idet*(A[0]*y[1]-A[1]*y[0]);
+}
+
+// L2 norm.
+static MFEM_HOST_DEVICE inline double l2norm2(const double x[3])
+{
+   return x[0]*x[0]+x[1]*x[1]+x[2]*x[2];
+}
+
 /* the bit structure of flags is CTTSSRR
    the C bit --- 1<<6 --- is set when the point is converged
    RR is 0 = 00b if r is unconstrained,
@@ -328,7 +459,7 @@ static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsPt *res,
                                                  const findptsPt *p,
                                                  const double tol)
 {
-   const double dist2 = l2norm2<3>(resid);
+   const double dist2 = l2norm2(resid);
    const double decr = p->dist2-dist2;
    const double pred = p->dist2p;
    for (int d = 0; d < 3; ++d)
@@ -1678,36 +1809,33 @@ void FindPointsGSLIB::FindPointsLocal3(const Vector &point_pos,
    {
       case 2:
          FindPointsLocal3DKernel<2>(npt, DEV.newt_tol, pp, point_pos_ordering,
-                                    pgslm, NE_split_total, pwt, pbb,
-                                    DEV.lh_nx, plhm, plhf, plho,
-                                    pcode, pelem, pref, pdist, pgll1d, plc);
+                                    pgslm, NE_split_total, pwt, pbb, DEV.lh_nx, plhm,
+                                    plhf, plho, pcode, pelem, pref, pdist, pgll1d,
+                                    plc);
          break;
       case 3:
          FindPointsLocal3DKernel<3>(npt, DEV.newt_tol, pp, point_pos_ordering,
-                                    pgslm, NE_split_total, pwt, pbb,
-                                    DEV.lh_nx, plhm, plhf, plho,
-                                    pcode, pelem, pref, pdist, pgll1d, plc);
+                                    pgslm, NE_split_total, pwt, pbb, DEV.lh_nx, plhm,
+                                    plhf, plho, pcode, pelem, pref, pdist, pgll1d,
+                                    plc);
          break;
       case 4:
          FindPointsLocal3DKernel<4>(npt, DEV.newt_tol, pp, point_pos_ordering,
-                                    pgslm, NE_split_total, pwt, pbb,
-                                    DEV.lh_nx, plhm, plhf, plho,
-                                    pcode, pelem, pref, pdist, pgll1d, plc);
+                                    pgslm, NE_split_total, pwt, pbb, DEV.lh_nx, plhm,
+                                    plhf, plho, pcode, pelem, pref, pdist, pgll1d,
+                                    plc);
          break;
       case 5:
          FindPointsLocal3DKernel<5>(npt, DEV.newt_tol, pp, point_pos_ordering,
-                                    pgslm, NE_split_total, pwt, pbb,
-                                    DEV.lh_nx, plhm, plhf, plho,
-                                    pcode, pelem, pref, pdist, pgll1d, plc);
+                                    pgslm, NE_split_total, pwt, pbb, DEV.lh_nx, plhm,
+                                    plhf, plho, pcode, pelem, pref, pdist, pgll1d,
+                                    plc);
          break;
       default:
-         FindPointsLocal3DKernel(npt, DEV.newt_tol, pp,
-                                 point_pos_ordering, pgslm,
-                                 NE_split_total, pwt, pbb,
-                                 DEV.lh_nx, plhm, plhf, plho,
-                                 pcode, pelem, pref, pdist, pgll1d, plc,
+         FindPointsLocal3DKernel(npt, DEV.newt_tol, pp, point_pos_ordering, pgslm,
+                                 NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
+                                 plho, pcode, pelem, pref, pdist, pgll1d, plc,
                                  DEV.dof1d);
-         break;
    }
 }
 #undef pMax

fem/gslib/findptsedge_local_2.cpp

@@ -11,7 +11,6 @@
 
 #include "../gslib.hpp"
 #include "../../general/forall.hpp"
-#include "gslib_kernel_helpers.hpp"
 
 #ifdef MFEM_USE_GSLIB
 
@@ -53,14 +52,113 @@ struct findptsElementGPT_t
    double x[sDIM], jac[sDIM*rDIM], hes[sDIM*rDIM];
 };
 
-using dbl_range_t = gslib::dbl_range_t;
-using obbox_t = gslib::obbox_t<sDIM>;
-using findptsLocalHashData_t = gslib::findptsLocalHashData_t<sDIM>;
-using gslib::AABB_test;
-using gslib::bbox_test;
-using gslib::hash_index;
-using gslib::l2norm2;
-using gslib::lag_eval_second_der;
+struct dbl_range_t
+{
+   double min, max;
+};
+
+struct obbox_t
+{
+   double c0[sDIM], A[sDIM*sDIM];
+   dbl_range_t x[sDIM];
+};
+
+struct findptsLocalHashData_t
+{
+   int hash_n;
+   dbl_range_t bnd[sDIM];
+   double fac[sDIM];
+   unsigned int *offset;
+};
+
+static MFEM_HOST_DEVICE inline void lag_eval_second_der(double *p0, double x,
+                                                        int i, const double *z,
+                                                        const double *lCoeff,
+                                                        int pN)
+{
+   double u0 = 1, u1 = 0, u2 = 0;
+   for (int j = 0; j < pN; ++j)
+   {
+      if (i != j)
+      {
+         double d_j = 2 * (x-z[j]);
+         u2 = d_j * u2 + u1;
+         u1 = d_j * u1 + u0;
+         u0 = d_j * u0;
+      }
+   }
+   double *p1 = p0 + pN, *p2 = p0 + 2 * pN;
+   p0[i] = lCoeff[i] * u0;
+   p1[i] = 2.0 * lCoeff[i] * u1;
+   p2[i] = 8.0 * lCoeff[i] * u2;
+}
+
+/* positive when possibly inside */
+static MFEM_HOST_DEVICE inline double obbox_axis_test(const obbox_t *const b,
+                                                      const double x[sDIM])
+{
+   double b_d;
+   for (int d=0; d<sDIM; ++d)
+   {
+      b_d = (x[d] - b->x[d].min) * (b->x[d].max - x[d]);
+      if (b_d < 0)   // if outside in any dimension
+      {
+         return b_d;
+      }
+   }
+   return b_d;       // only positive if inside
+}
+
+/* positive when given point is possibly inside given obbox b */
+static MFEM_HOST_DEVICE inline double obbox_test(const obbox_t *const b,
+                                                 const double x[sDIM])
+{
+   const double bxyz = obbox_axis_test(b,x);
+   if (bxyz<0)   // test if point is in AABB
+   {
+      return bxyz;
+   }
+   else   // test OBB only if inside AABB
+   {
+      double dxyz[sDIM];
+      for (int d=0; d<sDIM; ++d)
+      {
+         dxyz[d] = x[d] - b->c0[d];
+      }
+      double test = 1;
+      for (int d=0; d<sDIM; ++d)
+      {
+         double rst = 0;
+         for (int e=0; e<sDIM; ++e)
+         {
+            rst += b->A[d*2 + e] * dxyz[e];
+         }
+         double brst = (rst+1)*(1-rst);
+         test = test<0 ? test : brst;
+      }
+      return test;
+   }
+}
+
+/* Hash index in the hash table to the elements that possibly contain the point x */
+static MFEM_HOST_DEVICE inline int hash_index(const findptsLocalHashData_t *p,
+                                              const double x[2])
+{
+   const int n = p->hash_n;
+   int sum = 0;
+   for (int d=sDIM-1; d>=0; --d)
+   {
+      sum *= n;
+      int i = (int)floor((x[d] - p->bnd[d].min) * p->fac[d]);
+      sum += i<0 ? 0 : (n-1 < i ? n-1 : i);
+   }
+   return sum;
+}
+
+static MFEM_HOST_DEVICE inline double l2norm2(const double x[2])
+{
+   return x[0] * x[0] + x[1] * x[1];
+}
 
 /* the bit structure of flags is CRR
    the C bit --- 1<<2 --- is set when the point is converged
@@ -89,29 +187,29 @@ static MFEM_HOST_DEVICE inline int point_index(const int x)
 /* check reduction in objective against prediction, and adjust
    trust region radius (p->tr) accordingly;
    may reject the prior step, returning 1; otherwise returns 0
-   sets out_pt->dist2, out_pt->index, out_pt->x, out_pt->oldr in any event,
-   leaving out_pt->r, out_pt->dr, out_pt->flags to be set when returning 0 */
-static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsElementPoint_t *out_pt,
+   sets out->dist2, out->index, out->x, out->oldr in any event,
+   leaving out->r, out->dr, out->flags to be set when returning 0 */
+static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsElementPoint_t *out,
                                                  const double resid[2],
                                                  const findptsElementPoint_t *p,
                                                  const double tol)
 {
-   const double dist2 = l2norm2<2>(resid);
+   const double dist2 = l2norm2(resid);
    const double decr = p->dist2 - dist2;
    const double pred = p->dist2p;
-   out_pt->x[0] = p->x[0];
-   out_pt->x[1] = p->x[1];
-   out_pt->oldr = p->r;
-   out_pt->dist2 = dist2;
+   out->x[0] = p->x[0];
+   out->x[1] = p->x[1];
+   out->oldr = p->r;
+   out->dist2 = dist2;
    if (decr >= 0.01*pred)
    {
       if (decr >= 0.9*pred)   // very good iteration
       {
-         out_pt->tr = p->tr*2;
+         out->tr = p->tr*2;
       }
       else                    // somewhat good iteration
       {
-         out_pt->tr = p->tr;
+         out->tr = p->tr;
       }
       return false;
    }
@@ -122,21 +220,21 @@ static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsElementPoint_t *out_pt,
          "very good iteration" --- this doubles the trust radius,
          which is why we divide by 4 below */
       double v0 = fabs(p->r - p->oldr);
-      out_pt->tr = v0/4.0;
-      out_pt->dist2 = p->dist2;
-      out_pt->r = p->oldr;
-      out_pt->flags = p->flags>>3;
-      out_pt->dist2p = -HUGE_VAL;
+      out->tr = v0/4.0;
+      out->dist2 = p->dist2;
+      out->r = p->oldr;
+      out->flags = p->flags>>3;
+      out->dist2p = -HUGE_VAL;
       if (pred < dist2*tol)
       {
-         out_pt->flags |= CONVERGED_FLAG;
+         out->flags |= CONVERGED_FLAG;
       }
       return true;
    }
 }
 
 static MFEM_HOST_DEVICE inline void newton_edge( findptsElementPoint_t *const
-                                                 out_pt,
+                                                 out,
                                                  const double jac[2],
                                                  const double rhess,
                                                  const double resid[2],
@@ -206,9 +304,9 @@ newton_edge_fin:
    {
       new_flags |= CONVERGED_FLAG;
    }
-   out_pt->r = newr;
-   out_pt->dist2p = -v;
-   out_pt->flags = flags | new_flags | ((p->flags & FLAG_MASK)<<3);
+   out->r = newr;
+   out->dist2p = -v;
+   out->flags = flags | new_flags | ((p->flags & FLAG_MASK)<<3);
 }
 
 static MFEM_HOST_DEVICE void seed_j( const double *elx[sDIM],
@@ -234,27 +332,26 @@ static MFEM_HOST_DEVICE void seed_j( const double *elx[sDIM],
 }
 
 template<int T_D1D = 0>
-static void FindPointsEdgeLocal2DKernel( const int     npt,
-                                         const double  tol,
-                                         const double  dist2tol,
-                                         const double  *x,
-                                         const int     point_pos_ordering,
-                                         const double  *xElemCoord,
-                                         const int     nel,
-                                         const double  *wtend,
-                                         const double  *boxinfo,
-                                         const bool    obb_check,
-                                         const int     hash_n,
-                                         const double  *hashMin,
-                                         const double  *hashFac,
-                                         unsigned int  *hashOffset,
-                                         unsigned int *const code_base,
-                                         unsigned int *const el_base,
-                                         double *const r_base,
-                                         double *const dist2_base,
-                                         const double  *gll1D,
-                                         const double  *lagcoeff,
-                                         const int     pN = 0 )
+static void FindPointsEdgeLocal2D_Kernel( const int     npt,
+                                          const double  tol,
+                                          const double  dist2tol,
+                                          const double  *x,
+                                          const int     point_pos_ordering,
+                                          const double  *xElemCoord,
+                                          const int     nel,
+                                          const double  *wtend,
+                                          const double  *boxinfo,
+                                          const int     hash_n,
+                                          const double  *hashMin,
+                                          const double  *hashFac,
+                                          unsigned int  *hashOffset,
+                                          unsigned int *const code_base,
+                                          unsigned int *const el_base,
+                                          double *const r_base,
+                                          double *const dist2_base,
+                                          const double  *gll1D,
+                                          const double  *lagcoeff,
+                                          const int     pN = 0 )
 {
    const int MD1   = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
    const int D1D   = T_D1D ? T_D1D : pN;
@@ -315,34 +412,22 @@ static void FindPointsEdgeLocal2DKernel( const int     npt,
       {
          const unsigned int el = *elp;
 
-         const int n_box_ents = obb_check ? (3*sDIM + sDIM2) : (2*sDIM);
-         bool pass_bb = true;
          obbox_t box;
-         if (obb_check)
+         int n_box_ents = 3*sDIM + sDIM2;
+
+         for (int idx = 0; idx < sDIM; ++idx)
          {
-            for (int idx = 0; idx < sDIM; ++idx)
-            {
-               box.c0[idx] = boxinfo[n_box_ents*el + idx];
-               box.x[idx].min = boxinfo[n_box_ents*el + sDIM + idx];
-               box.x[idx].max = boxinfo[n_box_ents*el + 2*sDIM + idx];
-            }
-            for (int idx = 0; idx < sDIM2; ++idx)
-            {
-               box.A[idx] = boxinfo[n_box_ents*el + 3*sDIM + idx];
-            }
-            pass_bb = (bbox_test(&box, x_i) >= 0);
-         }
-         else
-         {
-            for (int d = 0; d < sDIM; ++d)
-            {
-               box.x[d].min = boxinfo[n_box_ents*el + d];
-               box.x[d].max = boxinfo[n_box_ents*el + sDIM + d];
-            }
-            pass_bb = (AABB_test(&box, x_i) >= 0);
+            box.c0[idx] = boxinfo[n_box_ents*el + idx];
+            box.x[idx].min = boxinfo[n_box_ents*el + sDIM + idx];
+            box.x[idx].max = boxinfo[n_box_ents*el + 2*sDIM + idx];
          }
 
-         if (pass_bb)
+         for (int idx = 0; idx < sDIM2; ++idx)
+         {
+            box.A[idx] = boxinfo[n_box_ents*el + 3*sDIM + idx];
+         }
+
+         if (obbox_test(&box,x_i)>=0)
          {
             //------------ findpts_local ------------------
             {
@@ -431,14 +516,11 @@ static void FindPointsEdgeLocal2DKernel( const int     npt,
                            double *hess = jac + sDIM*rDIM;
 
                            findptsElementGEdge_t edge;
-                           for (int d=0; d<sDIM; ++d)
-                           {
-                              edge.x[d] = constraint_workspace + d*D1D;
-                           }
                            MFEM_FOREACH_THREAD(j,x,D1D)
                            {
                               for (int d=0; d<sDIM; ++d)
                               {
+                                 edge.x[d] = constraint_workspace + d*D1D;
                                  edge.x[d][j] = elx[d][j];
                               }
                            }
@@ -599,41 +681,28 @@ void FindPointsGSLIB::FindPointsEdgeLocal2( const Vector &point_pos,
    auto pgll1d = DEV.gll1d.ReadWrite(use_dev);
    auto plc = DEV.lagcoeff.Read(use_dev);
    double dist2tol = DEV.surf_dist_tol;
-   const bool obb_chk = obb_check;
    switch (DEV.dof1d)
    {
       case 2:
-         FindPointsEdgeLocal2DKernel<2>(npt, DEV.newt_tol, dist2tol,
-                                        pp, point_pos_ordering, pgslm,
-                                        NE_split_total, pwt, pbb, obb_chk,
-                                        DEV.lh_nx, plhm, plhf, plho,
-                                        pcode, pelem, pref, pdist,
-                                        pgll1d, plc);
-         break;
+         return FindPointsEdgeLocal2D_Kernel<2>(
+                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
+                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
+                   plho, pcode, pelem, pref, pdist, pgll1d, plc);
       case 3:
-         FindPointsEdgeLocal2DKernel<3>(npt, DEV.newt_tol, dist2tol,
-                                        pp, point_pos_ordering, pgslm,
-                                        NE_split_total, pwt, pbb, obb_chk,
-                                        DEV.lh_nx, plhm, plhf, plho,
-                                        pcode, pelem, pref, pdist,
-                                        pgll1d, plc);
-         break;
+         return FindPointsEdgeLocal2D_Kernel<3>(
+                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
+                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
+                   plho, pcode, pelem, pref, pdist, pgll1d, plc);
       case 4:
-         FindPointsEdgeLocal2DKernel<4>(npt, DEV.newt_tol, dist2tol,
-                                        pp, point_pos_ordering, pgslm,
-                                        NE_split_total, pwt, pbb, obb_chk,
-                                        DEV.lh_nx, plhm, plhf, plho,
-                                        pcode, pelem, pref, pdist,
-                                        pgll1d, plc);
-         break;
+         return FindPointsEdgeLocal2D_Kernel<4>(
+                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
+                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
+                   plho, pcode, pelem, pref, pdist, pgll1d, plc);
       default:
-         FindPointsEdgeLocal2DKernel(npt, DEV.newt_tol, dist2tol, pp,
-                                     point_pos_ordering, pgslm,
-                                     NE_split_total, pwt, pbb, obb_chk,
-                                     DEV.lh_nx, plhm, plhf, plho,
-                                     pcode, pelem, pref, pdist,
-                                     pgll1d, plc, DEV.dof1d);
-         break;
+         return FindPointsEdgeLocal2D_Kernel(
+                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
+                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
+                   plho, pcode, pelem, pref, pdist, pgll1d, plc, DEV.dof1d);
    }
 }
 #undef sDIM

fem/gslib/findptsedge_local_3.cpp

@@ -11,7 +11,6 @@
 
 #include "../gslib.hpp"
 #include "../../general/forall.hpp"
-#include "gslib_kernel_helpers.hpp"
 
 #ifdef MFEM_USE_GSLIB
 
@@ -55,14 +54,117 @@ struct findptsElementGPT_t
    double x[sDIM], jac[sDIM], hes[sDIM*(1+1)];
 };
 
-using dbl_range_t = gslib::dbl_range_t;
-using obbox_t = gslib::obbox_t<sDIM>;
-using findptsLocalHashData_t = gslib::findptsLocalHashData_t<sDIM>;
-using gslib::AABB_test;
-using gslib::bbox_test;
-using gslib::hash_index;
-using gslib::l2norm2;
-using gslib::lag_eval_second_der;
+struct dbl_range_t
+{
+   double min, max;
+};
+
+struct obbox_t
+{
+   double c0[sDIM], A[sDIM*sDIM];
+   dbl_range_t x[sDIM];
+};
+
+struct findptsLocalHashData_t
+{
+   int hash_n;
+   dbl_range_t bnd[sDIM];
+   double fac[sDIM];
+   unsigned int *offset;
+};
+
+static MFEM_HOST_DEVICE inline void lag_eval_second_der(double *p0, double x,
+                                                        int i, const double *z,
+                                                        const double *lCoeff,
+                                                        int pN)
+{
+   double u0 = 1, u1 = 0, u2 = 0;
+   for (int j=0; j<pN; ++j)
+   {
+      if (i!=j)
+      {
+         double d_j = 2 * (x-z[j]);
+         u2 = d_j * u2 + u1;
+         u1 = d_j * u1 + u0;
+         u0 = d_j * u0;
+      }
+   }
+   double *p1 = p0 + pN, *p2 = p0 + 2 * pN;
+   p0[i] = lCoeff[i] * u0;
+   p1[i] = 2.0 * lCoeff[i] * u1;
+   p2[i] = 8.0 * lCoeff[i] * u2;
+}
+
+/* positive when possibly inside */
+static MFEM_HOST_DEVICE inline double obbox_axis_test(const obbox_t *const b,
+                                                      const double x[sDIM])
+{
+   double b_d;
+   for (int d=0; d<sDIM; ++d)
+   {
+      b_d = (x[d] - b->x[d].min) * (b->x[d].max - x[d]);
+      if (b_d < 0)   // if outside in any dimension
+      {
+         return b_d;
+      }
+   }
+   return b_d;       // only positive if inside in all dimensions
+}
+
+/* positive when possibly inside */
+static MFEM_HOST_DEVICE inline double obbox_test(const obbox_t *const b,
+                                                 const double x[sDIM])
+{
+   const double bxyz = obbox_axis_test(b, x);
+   if (bxyz<0)
+   {
+      return bxyz;
+   }
+   else
+   {
+      double dxyz[3];
+      // dxyz: distance of the point from the center of the OBB
+      for (int d=0; d<sDIM; ++d)
+      {
+         dxyz[d] = x[d] - b->c0[d];
+      }
+      // transform dxyz to the local coordinate system of the OBB,
+      // and check if the point is inside the OBB [-1,1]^sDIM
+      double test = 1;
+      for (int d=0; d<sDIM; ++d)
+      {
+         double rst = 0;
+         for (int e=0; e<sDIM; ++e)
+         {
+            rst += b->A[d*sDIM + e] * dxyz[e];
+         }
+         double brst = (rst+1)*(1-rst);
+         test = test<0 ? test : brst;
+      }
+      return test;
+   }
+}
+
+/* Hash index in the hash table to the elements that possibly contain the point x */
+static MFEM_HOST_DEVICE inline int hash_index(const findptsLocalHashData_t *p,
+                                              const double x[sDIM])
+{
+   const int n = p->hash_n;
+   int sum = 0;
+   for (int d=sDIM-1; d>=0; --d)
+   {
+      sum *= n;
+      int i = (int)floor((x[d] - p->bnd[d].min) * p->fac[d]);
+      sum += i<0 ? 0 : (n-1 < i ? n-1 : i);
+   }
+   return sum;
+}
+
+
+static MFEM_HOST_DEVICE inline double norm2(const double x[sDIM])
+{
+   return ( x[0]*x[0] + x[1]*x[1] + x[2]*x[2] );
+}
 
 /* the bit structure of flags is CRR
    the C bit --- 1<<2 --- is set when the point is converged
@@ -73,46 +175,47 @@ using gslib::lag_eval_second_der;
 #define CONVERGED_FLAG (1u<<2)
 #define FLAG_MASK 0x07u
 
-/* returns the number of constrained reference coordinates, max 1
+/* returns the number of constrained reference coordinates, max 2
 */
 static MFEM_HOST_DEVICE inline int num_constrained(const int flags)
 {
-   return ((flags | flags>>1) & 1u);
+   const int y = (flags | flags>>1);
+   return (y & 1u) + (y>>2 & 1u);
 }
 
 static MFEM_HOST_DEVICE inline int point_index(const int x)
 {
-   return ((x>>1)&1u);
+   return ((x>>1)&1u) | ((x>>2)&2u);
 }
 
 /* check reduction in objective against prediction, and adjust
    trust region radius (p->tr) accordingly;
    may reject the prior step, returning 1; otherwise returns 0
-   sets out_pt->dist2, out_pt->index, out_pt->x, out_pt->oldr in any event,
-   leaving out_pt->r, out_pt->dr, out_pt->flags to be set when returning 0 */
-static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsElementPoint_t *out_pt,
+   sets out->dist2, out->index, out->x, out->oldr in any event,
+   leaving out->r, out->dr, out->flags to be set when returning 0 */
+static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsElementPoint_t *out,
                                                  const double resid[3],
                                                  const findptsElementPoint_t *p,
                                                  const double tol)
 {
-   const double dist2 = l2norm2<sDIM>(resid);
+   const double dist2 = norm2(resid);
    const double decr  = p->dist2 - dist2;
    const double pred  = p->dist2p;
    for (int d=0; d<sDIM; ++d)
    {
-      out_pt->x[d] = p->x[d];
+      out->x[d] = p->x[d];
    }
-   out_pt->oldr = p->r;
-   out_pt->dist2 = dist2;
+   out->oldr = p->r;
+   out->dist2 = dist2;
    if (decr>=0.01*pred)
    {
       if (decr>=0.9*pred)   // very good iteration
       {
-         out_pt->tr = 2*p->tr;
+         out->tr = 2*p->tr;
       }
       else   // good iteration
       {
-         out_pt->tr = p->tr;
+         out->tr = p->tr;
       }
       return false;
    }
@@ -123,21 +226,21 @@ static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsElementPoint_t *out_pt,
          "very good iteration" --- this doubles the trust radius,
          which is why we divide by 4 below */
       double v0 = fabs(p->r - p->oldr);
-      out_pt->tr = v0/4.0;
-      out_pt->dist2 = p->dist2;
-      out_pt->r = p->oldr;
-      out_pt->flags = p->flags>>3;
-      out_pt->dist2p = -HUGE_VAL;
+      out->tr = v0/4.0;
+      out->dist2 = p->dist2;
+      out->r = p->oldr;
+      out->flags = p->flags>>3;
+      out->dist2p = -HUGE_VAL;
       if (pred<dist2*tol)
       {
-         out_pt->flags |= CONVERGED_FLAG;
+         out->flags |= CONVERGED_FLAG;
       }
       return true;
    }
 }
 
 static MFEM_HOST_DEVICE inline void newton_edge(findptsElementPoint_t *const
-                                                out_pt,
+                                                out,
                                                 const double jac[sDIM*rDIM],
                                                 const double rhes,
                                                 const double resid[sDIM],
@@ -211,9 +314,9 @@ newton_edge_fin:
    {
       new_flags |= CONVERGED_FLAG;
    }
-   out_pt->r = nr;
-   out_pt->dist2p = -v;
-   out_pt->flags = flags | new_flags | ((p->flags & FLAG_MASK)<<3);
+   out->r = nr;
+   out->dist2p = -v;
+   out->flags = flags | new_flags | ((p->flags & FLAG_MASK)<<3);
 #undef EVAL
 }
 
@@ -235,32 +338,31 @@ static MFEM_HOST_DEVICE void seed_j(const double *elx[sDIM],
    {
       dx[d] = x[d] - elx[d][ir];
    }
-   dist2[ir] = l2norm2(dx);
+   dist2[ir] = norm2(dx);;
    r[ir] = z[ir];
 }
 
 template<int T_D1D = 0>
-static void FindPointsEdgeLocal3DKernel(const int npt,
-                                        const double tol,
-                                        const double dist2tol,
-                                        const double *x,
-                                        const int point_pos_ordering,
-                                        const double *xElemCoord,
-                                        const int nel,
-                                        const double *wtend,
-                                        const double *boxinfo,
-                                        const bool obb_check,
-                                        const int hash_n,
-                                        const double *hashMin,
-                                        const double *hashFac,
-                                        unsigned int *hashOffset,
-                                        unsigned int *const code_base,
-                                        unsigned int *const el_base,
-                                        double *const r_base,
-                                        double *const dist2_base,
-                                        const double *gll1D,
-                                        const double *lagcoeff,
-                                        const int pN = 0)
+static void FindPointsEdgeLocal3D_Kernel(const int npt,
+                                         const double tol,
+                                         const double dist2tol,
+                                         const double *x,
+                                         const int point_pos_ordering,
+                                         const double *xElemCoord,
+                                         const int nel,
+                                         const double *wtend,
+                                         const double *boxinfo,
+                                         const int hash_n,
+                                         const double *hashMin,
+                                         const double *hashFac,
+                                         unsigned int *hashOffset,
+                                         unsigned int *const code_base,
+                                         unsigned int *const el_base,
+                                         double *const r_base,
+                                         double *const dist2_base,
+                                         const double *gll1D,
+                                         const double *lagcoeff,
+                                         const int pN = 0)
 {
    const int MD1   = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
    const int D1D   = T_D1D ? T_D1D : pN;
@@ -317,35 +419,21 @@ static void FindPointsEdgeLocal3DKernel(const int npt,
       for (; elp!=ele; ++elp)
       {
          const unsigned int el = *elp;
-
-         const int n_box_ents = obb_check ? (3*sDIM + sDIM2) : (2*sDIM);
-         bool pass_bb = true;
          obbox_t box;
-         if (obb_check)
+         int n_box_ents = 3*sDIM + sDIM2;
+
+         for (int idx = 0; idx < sDIM; ++idx)
          {
-            for (int idx = 0; idx < sDIM; ++idx)
-            {
-               box.c0[idx] = boxinfo[n_box_ents*el + idx];
-               box.x[idx].min = boxinfo[n_box_ents*el + sDIM + idx];
-               box.x[idx].max = boxinfo[n_box_ents*el + 2*sDIM + idx];
-            }
-            for (int idx = 0; idx < sDIM2; ++idx)
-            {
-               box.A[idx] = boxinfo[n_box_ents*el + 3*sDIM + idx];
-            }
-            pass_bb = (bbox_test(&box, x_i) >= 0);
+            box.c0[idx] = boxinfo[n_box_ents*el + idx];
+            box.x[idx].min = boxinfo[n_box_ents*el + sDIM + idx];
+            box.x[idx].max = boxinfo[n_box_ents*el + 2*sDIM + idx];
          }
-         else
+         for (int idx = 0; idx < sDIM2; ++idx)
          {
-            for (int d = 0; d < sDIM; ++d)
-            {
-               box.x[d].min = boxinfo[n_box_ents*el + d];
-               box.x[d].max = boxinfo[n_box_ents*el + sDIM + d];
-            }
-            pass_bb = (AABB_test(&box, x_i) >= 0);
+            box.A[idx] = boxinfo[n_box_ents*el + 3*sDIM + idx];
          }
 
-         if (pass_bb)
+         if (obbox_test(&box, x_i)>=0)
          {
             //// findpts_local ////
             {
@@ -433,14 +521,11 @@ static void FindPointsEdgeLocal3DKernel(const int npt,
                            double *hess = jac + sDIM*rDIM;
 
                            findptsElementGEdge_t edge;
-                           for (int d=0; d<sDIM; ++d)
-                           {
-                              edge.x[d] = constraint_workspace + d*D1D;
-                           }
                            MFEM_FOREACH_THREAD(j,x,D1D)
                            {
                               for (int d=0; d<sDIM; ++d)
                               {
+                                 edge.x[d] = constraint_workspace + d*D1D;
                                  edge.x[d][j] = elx[d][j];
                               }
                            }
@@ -603,41 +688,28 @@ void FindPointsGSLIB::FindPointsEdgeLocal3(const Vector &point_pos,
    auto pgll1d = DEV.gll1d.ReadWrite(use_dev);
    auto plc = DEV.lagcoeff.Read(use_dev);
    double dist2tol = DEV.surf_dist_tol;
-   const bool obb_chk = obb_check;
    switch (DEV.dof1d)
    {
       case 2:
-         FindPointsEdgeLocal3DKernel<2>(npt, DEV.newt_tol, dist2tol,
-                                        pp, point_pos_ordering, pgslm,
-                                        NE_split_total, pwt, pbb, obb_chk,
-                                        DEV.lh_nx, plhm, plhf, plho,
-                                        pcode, pelem, pref, pdist,
-                                        pgll1d, plc);
-         break;
+         return FindPointsEdgeLocal3D_Kernel<2>(
+                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
+                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
+                   plho, pcode, pelem, pref, pdist, pgll1d, plc);
       case 3:
-         FindPointsEdgeLocal3DKernel<3>(npt, DEV.newt_tol, dist2tol,
-                                        pp, point_pos_ordering, pgslm,
-                                        NE_split_total, pwt, pbb, obb_chk,
-                                        DEV.lh_nx, plhm, plhf, plho,
-                                        pcode, pelem, pref, pdist,
-                                        pgll1d, plc);
-         break;
+         return FindPointsEdgeLocal3D_Kernel<3>(
+                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
+                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
+                   plho, pcode, pelem, pref, pdist, pgll1d, plc);
       case 4:
-         FindPointsEdgeLocal3DKernel<4>(npt, DEV.newt_tol, dist2tol,
-                                        pp, point_pos_ordering, pgslm,
-                                        NE_split_total, pwt, pbb, obb_chk,
-                                        DEV.lh_nx, plhm, plhf, plho,
-                                        pcode, pelem, pref, pdist,
-                                        pgll1d, plc);
-         break;
+         return FindPointsEdgeLocal3D_Kernel<4>(
+                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
+                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
+                   plho, pcode, pelem, pref, pdist, pgll1d, plc);
       default:
-         FindPointsEdgeLocal3DKernel(npt, DEV.newt_tol, dist2tol, pp,
-                                     point_pos_ordering, pgslm,
-                                     NE_split_total, pwt, pbb, obb_chk,
-                                     DEV.lh_nx, plhm, plhf, plho,
-                                     pcode, pelem, pref, pdist,
-                                     pgll1d, plc, DEV.dof1d);
-         break;
+         return FindPointsEdgeLocal3D_Kernel(
+                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
+                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
+                   plho, pcode, pelem, pref, pdist, pgll1d, plc, DEV.dof1d);
    }
 }
 #undef rDIM2

fem/gslib/findptssurf_local_3.cpp

@@ -11,7 +11,6 @@
 
 #include "../gslib.hpp"
 #include "../../general/forall.hpp"
-#include "gslib_kernel_helpers.hpp"
 #ifdef MFEM_USE_GSLIB
 
 #ifdef MFEM_HAVE_GCC_PRAGMA_DIAGNOSTIC
@@ -52,15 +51,124 @@ struct findptsElementGPT_t
    double x[sDIM], jac[sDIM*rDIM], hes[sDIM*(rDIM+1)];
 };
 
-using dbl_range_t = gslib::dbl_range_t;
-using obbox_t = gslib::obbox_t<sDIM>;
-using findptsLocalHashData_t = gslib::findptsLocalHashData_t<sDIM>;
-using gslib::AABB_test;
-using gslib::bbox_test;
-using gslib::hash_index;
-using gslib::l2norm2;
-using gslib::lag_eval_second_der;
-using gslib::lin_solve_sym_2;
+struct dbl_range_t
+{
+   double min, max;
+};
+
+struct obbox_t
+{
+   double c0[sDIM], A[sDIM*sDIM];
+   dbl_range_t x[sDIM];
+};
+
+struct findptsLocalHashData_t
+{
+   int hash_n;
+   dbl_range_t bnd[sDIM];
+   double fac[sDIM];
+   unsigned int *offset;
+};
+
+static MFEM_HOST_DEVICE inline void lag_eval_second_der(double *p0, double x,
+                                                        int i, const double *z,
+                                                        const double *lCoeff,
+                                                        int pN)
+{
+   double u0 = 1, u1 = 0, u2 = 0;
+   for (int j = 0; j < pN; ++j)
+   {
+      if (i != j)
+      {
+         double d_j = 2 * (x - z[j]);
+         u2 = d_j * u2 + u1;
+         u1 = d_j * u1 + u0;
+         u0 = d_j * u0;
+      }
+   }
+   p0[i] = lCoeff[i] * u0;
+   p0[pN+i] = 2.0 * lCoeff[i] * u1;
+   p0[2*pN+i] = 8.0 * lCoeff[i] * u2;
+}
+
+/* positive when possibly inside */
+static MFEM_HOST_DEVICE inline double AABB_test(const obbox_t *const b,
+                                                const double x[sDIM])
+{
+   double b_d;
+   for (int d=0; d<sDIM; ++d)
+   {
+      b_d = (x[d] - b->x[d].min) * (b->x[d].max - x[d]);
+      if (b_d < 0)   // if outside in any dimension
+      {
+         return b_d;
+      }
+   }
+   return b_d;       // only positive if inside in all dimensions
+}
+
+/* positive when possibly inside */
+static MFEM_HOST_DEVICE inline double bbox_test(const obbox_t *const b,
+                                                const double x[sDIM])
+{
+   const double bxyz = AABB_test(b, x);
+   if (bxyz<0)
+   {
+      return bxyz;
+   }
+   else
+   {
+      double dxyz[3];
+      // dxyz: distance of the point from the center of the OBB
+      for (int d=0; d<sDIM; ++d)
+      {
+         dxyz[d] = x[d] - b->c0[d];
+      }
+      // tranform dxyz to the local coordinate system of the OBB,
+      // and check if the point is inside the OBB [-1,1]^sDIM
+      double test = 1;
+      for (int d=0; d<sDIM; ++d)
+      {
+         double rst = 0;
+         for (int e=0; e<sDIM; ++e)
+         {
+            rst += b->A[d*sDIM + e] * dxyz[e];
+         }
+         double brst = (rst+1)*(1-rst);
+         test = test<0 ? test : brst;
+      }
+      return test;
+   }
+}
+
+/* Hash index in the hash table to the elements that possibly contain the point x */
+static MFEM_HOST_DEVICE inline int hash_index(const findptsLocalHashData_t *p,
+                                              const double x[sDIM])
+{
+   const int n = p->hash_n;
+   int sum = 0;
+   for (int d=sDIM-1; d>=0; --d)
+   {
+      sum *= n;
+      int i = (int)floor((x[d] - p->bnd[d].min) * p->fac[d]);
+      sum += i<0 ? 0 : (n-1 < i ? n-1 : i);
+   }
+   return sum;
+}
+
+static MFEM_HOST_DEVICE inline void lin_solve_sym_2(double x[2],
+                                                    const double A[3],
+                                                    const double y[2])
+{
+   const double idet = 1 / (A[0] * A[2] - A[1] * A[1]);
+   x[0] = idet * (A[2] * y[0] - A[1] * y[1]);
+   x[1] = idet * (A[0] * y[1] - A[1] * y[0]);
+}
+
+static MFEM_HOST_DEVICE inline double l2norm2(const double x[sDIM])
+{
+   return ( x[0]*x[0] + x[1]*x[1] + x[2]*x[2]);
+}
 
 /* the bit structure of flags is CSSRR
    the C bit --- 1<<4 --- is set when the point is converged
@@ -111,10 +219,18 @@ static MFEM_HOST_DEVICE inline int point_index(const int x)
    return ((x>>1)&1u) | ((x>>2)&2u);
 }
 
-static MFEM_HOST_DEVICE inline void
+static MFEM_HOST_DEVICE inline findptsElementGEdge_t
 get_edge(const double *elx[3], const double *wtend, int ei,
-         int &side_init, int jidx, int pN, findptsElementGEdge_t &edge)
+         double *workspace, int &side_init, int jidx, int pN)
 {
+   findptsElementGEdge_t edge;
+   for (int d=0; d<sDIM; ++d)
+   {
+      edge.x[d]     = workspace             + d*pN;
+      edge.dxdn[d]  = workspace +   sDIM*pN + d*pN;
+      edge.d2xdn[d] = workspace + 2*sDIM*pN + d*pN;
+   }
+
    // given edge index, compute normal and tangential directions
    const int dn = ei>>1, //0 for rmin/rmax, 1 for smin/smax
              de = plus_1_mod_2(dn); // 1 for rmin/rmax, 0 for smin/smax
@@ -140,6 +256,7 @@ get_edge(const double *elx[3], const double *wtend, int ei,
       edge.d2xdn[dd][jj] = sums_k[1];
 #undef ELX
    }
+   return edge;
 }
 
 static MFEM_HOST_DEVICE inline findptsElementGPT_t get_pt(const double *elx[3],
@@ -195,34 +312,34 @@ static MFEM_HOST_DEVICE inline findptsElementGPT_t get_pt(const double *elx[3],
 /* check reduction in objective against prediction, and adjust
    trust region radius (p->tr) accordingly;
    may reject the prior step, returning 1; otherwise returns 0
-   sets out_pt->dist2, out_pt->index, out_pt->x, out_pt->oldr in any event,
-   leaving out_pt->r, out_pt->dr, out_pt->flags to be set when returning 0 */
-static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsElementPoint_t *out_pt,
+   sets out->dist2, out->index, out->x, out->oldr in any event,
+   leaving out->r, out->dr, out->flags to be set when returning 0 */
+static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsElementPoint_t *out,
                                                  const double resid[3],
                                                  const findptsElementPoint_t *p,
                                                  const double tol)
 {
-   const double dist2 = l2norm2<sDIM>(resid);
+   const double dist2 = l2norm2(resid);
    const double decr  = p->dist2 - dist2;
    const double pred  = p->dist2p;
    for (int d=0; d<sDIM; ++d)
    {
-      out_pt->x[d] = p->x[d];
+      out->x[d] = p->x[d];
    }
    for (int d=0; d<rDIM; ++d)
    {
-      out_pt->oldr[d] = p->r[d];
+      out->oldr[d] = p->r[d];
    }
-   out_pt->dist2 = dist2;
+   out->dist2 = dist2;
    if (decr>=0.01*pred)
    {
       if (decr>=0.9*pred)   // very good iteration
       {
-         out_pt->tr = 2*p->tr;
+         out->tr = 2*p->tr;
       }
       else   // good iteration
       {
-         out_pt->tr = p->tr;
+         out->tr = p->tr;
       }
       return false;
    }
@@ -234,17 +351,17 @@ static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsElementPoint_t *out_pt,
          which is why we divide by 4 below */
       double v0 = fabs(p->r[0] - p->oldr[0]),
              v1 = fabs(p->r[1] - p->oldr[1]);
-      out_pt->tr     = ( v0>v1 ? v0 : v1 )/4;
-      out_pt->dist2  = p->dist2;
-      out_pt->flags   = p->flags >> 5;
-      out_pt->dist2p = -HUGE_VAL;
+      out->tr     = ( v0>v1 ? v0 : v1 )/4;
+      out->dist2  = p->dist2;
+      out->flags   = p->flags >> 5;
+      out->dist2p = -HUGE_VAL;
       for (int d=0; d<rDIM; ++d)
       {
-         out_pt->r[d] = p->oldr[d];
+         out->r[d] = p->oldr[d];
       }
       if (pred<dist2*tol)
       {
-         out_pt->flags |= CONVERGED_FLAG;
+         out->flags |= CONVERGED_FLAG;
       }
       return true;
    }
@@ -252,7 +369,7 @@ static MFEM_HOST_DEVICE bool reject_prior_step_q(findptsElementPoint_t *out_pt,
 
 /* minimize ||resid - jac * dr||_2, with |dr| <= tr, |r0+dr|<=1
    (exact solution of trust region problem) */
-static MFEM_HOST_DEVICE void newton_face( findptsElementPoint_t *const out_pt,
+static MFEM_HOST_DEVICE void newton_face( findptsElementPoint_t *const out,
                                           const double jac[sDIM*rDIM],
                                           const double rhes[3],
                                           const double resid[sDIM],
@@ -423,19 +540,19 @@ newton_face_constrained:
    }
 
 newton_face_fin:
-   out_pt->dist2p = -2*v;
+   out->dist2p = -2*v;
    dr[0] = r[0] - p->r[0];
    dr[1] = r[1] - p->r[1];
    if ( fabs(dr[0])+fabs(dr[1]) < tol)
    {
       new_flags |= CONVERGED_FLAG;
    }
-   out_pt->r[0] = r[0], out_pt->r[1] = r[1];
-   out_pt->flags = new_flags | ((p->flags & FLAG_MASK)<<5);
+   out->r[0] = r[0], out->r[1] = r[1];
+   out->flags = new_flags | ((p->flags & FLAG_MASK)<<5);
 }
 
 static MFEM_HOST_DEVICE inline void newton_edge(findptsElementPoint_t *const
-                                                out_pt,
+                                                out,
                                                 const double jac[sDIM*rDIM],
                                                 const double rhes,
                                                 const double resid[sDIM],
@@ -520,10 +637,10 @@ newton_edge_fin:
    {
       new_flags |= CONVERGED_FLAG;
    }
-   out_pt->r[de] = nr;
-   out_pt->r[dn] = p->r[dn];
-   out_pt->dist2p = -v;
-   out_pt->flags = flags | new_flags | ((p->flags & FLAG_MASK)<<5);
+   out->r[de] = nr;
+   out->r[dn] = p->r[dn];
+   out->dist2p = -v;
+   out->flags = flags | new_flags | ((p->flags & FLAG_MASK)<<5);
 #undef EVAL
 }
 
@@ -559,27 +676,26 @@ static MFEM_HOST_DEVICE void seed_j(const double *elx[sDIM],
 // global memory access of element coordinates.
 // Are the structs being stored in "local memory" or registers?
 template<int T_D1D = 0>
-static void FindPointsSurfLocal3DKernel(const int npt,
-                                        const double tol,
-                                        const double dist2tol,
-                                        const double *x,
-                                        const int point_pos_ordering,
-                                        const double *xElemCoord,
-                                        const int nel,
-                                        const double *wtend,
-                                        const double *boxinfo,
-                                        const bool obb_check,
-                                        const int hash_n,
-                                        const double *hashMin,
-                                        const double *hashFac,
-                                        unsigned int *hashOffset,
-                                        unsigned int *const code_base,
-                                        unsigned int *const el_base,
-                                        double *const r_base,
-                                        double *const dist2_base,
-                                        const double *gll1D,
-                                        const double *lagcoeff,
-                                        const int pN = 0)
+static void FindPointsSurfLocal3D_Kernel(const int npt,
+                                         const double tol,
+                                         const double dist2tol,
+                                         const double *x,
+                                         const int point_pos_ordering,
+                                         const double *xElemCoord,
+                                         const int nel,
+                                         const double *wtend,
+                                         const double *boxinfo,
+                                         const int hash_n,
+                                         const double *hashMin,
+                                         const double *hashFac,
+                                         unsigned int *hashOffset,
+                                         unsigned int *const code_base,
+                                         unsigned int *const el_base,
+                                         double *const r_base,
+                                         double *const dist2_base,
+                                         const double *gll1D,
+                                         const double *lagcoeff,
+                                         const int pN = 0)
 {
    const int MD1   = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
    const int D1D   = T_D1D ? T_D1D : pN;
@@ -637,36 +753,22 @@ static void FindPointsSurfLocal3DKernel(const int npt,
       {
          const unsigned int el = *elp;
 
-         const int n_box_ents = obb_check ? (3*sDIM + sDIM2) : (2*sDIM);
-         bool pass_bb = true;
+         // construct obbox on the fly
          obbox_t box;
-         if (obb_check)
+         int n_box_ents = 3*sDIM + sDIM2;
+         for (int idx = 0; idx < sDIM; ++idx)
          {
-            // construct obbox on the fly
-            for (int idx = 0; idx < sDIM; ++idx)
-            {
-               box.c0[idx] = boxinfo[n_box_ents*el + idx];
-               box.x[idx].min = boxinfo[n_box_ents*el + sDIM + idx];
-               box.x[idx].max = boxinfo[n_box_ents*el + 2*sDIM + idx];
-            }
-
-            for (int idx = 0; idx < sDIM2; ++idx)
-            {
-               box.A[idx] = boxinfo[n_box_ents*el + 3*sDIM + idx];
-            }
-            pass_bb = (bbox_test(&box, x_i) >= 0);
-         }
-         else
-         {
-            for (int d = 0; d < sDIM; ++d)
-            {
-               box.x[d].min = boxinfo[n_box_ents*el + d];
-               box.x[d].max = boxinfo[n_box_ents*el + sDIM + d];
-            }
-            pass_bb = (AABB_test(&box, x_i) >= 0);
+            box.c0[idx] = boxinfo[n_box_ents*el + idx];
+            box.x[idx].min = boxinfo[n_box_ents*el + sDIM + idx];
+            box.x[idx].max = boxinfo[n_box_ents*el + 2*sDIM + idx];
          }
 
-         if (!pass_bb) { continue; }
+         for (int idx = 0; idx < sDIM2; ++idx)
+         {
+            box.A[idx] = boxinfo[n_box_ents*el + 3*sDIM + idx];
+         }
+
+         if (bbox_test(&box, x_i) < 0) { continue; }
 
          //// findpts_local ////
          {
@@ -866,19 +968,13 @@ static void FindPointsSurfLocal3DKernel(const int npt,
                         double *hes_T = jac + sDIM*rDIM;
                         double *hes   = hes_T + hes_count*sDIM;
                         findptsElementGEdge_t edge;
-                        for (int d=0; d<sDIM; ++d)
-                        {
-                           edge.x[d]     = constraint_workspace + d*D1D;
-                           edge.dxdn[d]  = constraint_workspace + d*D1D
-                                           + sDIM*D1D;
-                           edge.d2xdn[d] = constraint_workspace + d*D1D
-                                           + 2*sDIM*D1D;
-                        }
 
                         MFEM_FOREACH_THREAD(j,x,D1D*sDIM)
                         {
-                           // One thread per physical component and edge DOF.
-                           get_edge(elx, wtend, ei, edge_init, j, D1D, edge);
+                           // utilized first D1D threads
+                           edge = get_edge(elx, wtend, ei,
+                                           constraint_workspace, edge_init, j,
+                                           D1D);
                         }
                         MFEM_SYNC_THREAD;
 
@@ -949,15 +1045,7 @@ static void FindPointsSurfLocal3DKernel(const int npt,
                               steep *= tmp->r[dn];
                               if (steep<0)
                               {
-                                 double face_hes[3] =
-                                 {
-                                    dn == 0 ? hes[2] : hes[0],
-                                    hes[1],
-                                    dn == 0 ? hes[0] : hes[2]
-                                 };
-                                 newton_face(fpt, jac, face_hes, resid,
-                                             tmp->flags & CONVERGED_FLAG,
-                                             tmp, tol);
+                                 newton_face( fpt,jac,hes,resid,tmp->flags&CONVERGED_FLAG,tmp,tol);
                               }
                               else
                               {
@@ -1123,42 +1211,29 @@ void FindPointsGSLIB::FindPointsSurfLocal3(const Vector &point_pos,
    auto pgll1d = DEV.gll1d.ReadWrite(use_dev);
    auto plc = DEV.lagcoeff.Read(use_dev);
    double dist2tol = DEV.surf_dist_tol;
-   const bool obb_chk = obb_check;
 
    switch (DEV.dof1d)
    {
       case 2:
-         FindPointsSurfLocal3DKernel<2>(npt, DEV.newt_tol, dist2tol,
-                                        pp, point_pos_ordering, pgslm,
-                                        NE_split_total, pwt, pbb, obb_chk,
-                                        DEV.lh_nx, plhm, plhf, plho,
-                                        pcode, pelem, pref, pdist,
-                                        pgll1d, plc);
-         break;
+         return FindPointsSurfLocal3D_Kernel<2>(
+                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
+                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
+                   plho, pcode, pelem, pref, pdist, pgll1d, plc);
       case 3:
-         FindPointsSurfLocal3DKernel<3>(npt, DEV.newt_tol, dist2tol,
-                                        pp, point_pos_ordering, pgslm,
-                                        NE_split_total, pwt, pbb, obb_chk,
-                                        DEV.lh_nx, plhm, plhf, plho,
-                                        pcode, pelem, pref, pdist,
-                                        pgll1d, plc);
-         break;
+         return FindPointsSurfLocal3D_Kernel<3>(
+                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
+                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
+                   plho, pcode, pelem, pref, pdist, pgll1d, plc);
       case 4:
-         FindPointsSurfLocal3DKernel<4>(npt, DEV.newt_tol, dist2tol,
-                                        pp, point_pos_ordering, pgslm,
-                                        NE_split_total, pwt, pbb, obb_chk,
-                                        DEV.lh_nx, plhm, plhf, plho,
-                                        pcode, pelem, pref, pdist,
-                                        pgll1d, plc);
-         break;
+         return FindPointsSurfLocal3D_Kernel<4>(
+                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
+                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
+                   plho, pcode, pelem, pref, pdist, pgll1d, plc);
       default:
-         FindPointsSurfLocal3DKernel(npt, DEV.newt_tol, dist2tol, pp,
-                                     point_pos_ordering, pgslm,
-                                     NE_split_total, pwt, pbb, obb_chk,
-                                     DEV.lh_nx, plhm, plhf, plho,
-                                     pcode, pelem, pref, pdist,
-                                     pgll1d, plc, DEV.dof1d);
-         break;
+         return FindPointsSurfLocal3D_Kernel(
+                   npt, DEV.newt_tol, dist2tol, pp, point_pos_ordering, pgslm,
+                   NE_split_total, pwt, pbb, DEV.lh_nx, plhm, plhf,
+                   plho, pcode, pelem, pref, pdist, pgll1d, plc, DEV.dof1d);
    }
 }
 

fem/gslib/gslib_kernel_helpers.hpp

@@ -1,190 +0,0 @@
-#ifndef MFEM_GSLIB_KERNEL_HELPERS_HPP
-#define MFEM_GSLIB_KERNEL_HELPERS_HPP
-
-#include "../../config/config.hpp"
-
-#include <cmath>
-
-namespace mfem
-{
-
-namespace gslib
-{
-
-struct dbl_range_t
-{
-   double min, max;
-};
-
-template <int SDIM>
-struct obbox_t
-{
-   double c0[SDIM], A[SDIM * SDIM];
-   dbl_range_t x[SDIM];
-};
-
-template <int SDIM>
-struct findptsLocalHashData_t
-{
-   int hash_n;
-   dbl_range_t bnd[SDIM];
-   double fac[SDIM];
-   unsigned int *offset;
-};
-
-// Eval the ith Lagrange interpolant at x.
-MFEM_HOST_DEVICE inline void lagrange_eval(double *p0, double x,
-                                           int i, int p_Nq,
-                                           double *z, double *lagrangeCoeff)
-{
-   double p_i = (1 << (p_Nq - 1));
-   for (int j = 0; j < p_Nq; ++j)
-   {
-      const double d_j = x - z[j];
-      p_i *= j == i ? 1 : d_j;
-   }
-   p0[i] = lagrangeCoeff[i] * p_i;
-}
-
-// Eval the ith Lagrange interpolant and its first derivative at x.
-MFEM_HOST_DEVICE inline void lag_eval_first_der(double *p0, double x,
-                                                int i, const double *z,
-                                                const double *lCoeff,
-                                                int pN)
-{
-   double u0 = 1, u1 = 0;
-   for (int j = 0; j < pN; ++j)
-   {
-      if (i != j)
-      {
-         const double d_j = 2 * (x - z[j]);
-         u1 = d_j * u1 + u0;
-         u0 = d_j * u0;
-      }
-   }
-   p0[i] = lCoeff[i] * u0;
-   p0[pN + i] = 2.0 * lCoeff[i] * u1;
-}
-
-// Eval the ith Lagrange interpolant and its first and second derivative at x.
-MFEM_HOST_DEVICE inline void lag_eval_second_der(double *p0, double x,
-                                                 int i, const double *z,
-                                                 const double *lCoeff,
-                                                 int pN)
-{
-   double u0 = 1, u1 = 0, u2 = 0;
-   for (int j = 0; j < pN; ++j)
-   {
-      if (i != j)
-      {
-         const double d_j = 2 * (x - z[j]);
-         u2 = d_j * u2 + u1;
-         u1 = d_j * u1 + u0;
-         u0 = d_j * u0;
-      }
-   }
-   p0[i] = lCoeff[i] * u0;
-   p0[pN + i] = 2.0 * lCoeff[i] * u1;
-   p0[2 * pN + i] = 8.0 * lCoeff[i] * u2;
-}
-
-// Solve Ax=y where A is a symmetric 2x2 matrix packed as {a00, a01, a11}.
-MFEM_HOST_DEVICE inline void lin_solve_sym_2(double x[2],
-                                             const double A[3],
-                                             const double y[2])
-{
-   const double idet = 1 / (A[0] * A[2] - A[1] * A[1]);
-   x[0] = idet * (A[2] * y[0] - A[1] * y[1]);
-   x[1] = idet * (A[0] * y[1] - A[1] * y[0]);
-}
-
-// Positive when the point is inside the axis-aligned bounding box.
-template <int SDIM>
-MFEM_HOST_DEVICE inline double AABB_test(const obbox_t<SDIM> *const b,
-                                         const double (&x)[SDIM])
-{
-   double test = 1.0;
-   for (int d = 0; d < SDIM; ++d)
-   {
-      const double b_d = (x[d] - b->x[d].min) * (b->x[d].max - x[d]);
-      test = test < 0.0 ? test : b_d;
-   }
-   return test;
-}
-
-// Positive when the point is inside the oriented bounding box.
-template <int SDIM>
-MFEM_HOST_DEVICE inline double bbox_test(const obbox_t<SDIM> *const b,
-                                         const double (&x)[SDIM])
-{
-   const double bxyz = AABB_test(b, x);
-   if (bxyz < 0.0)
-   {
-      return bxyz;
-   }
-
-   double dxyz[SDIM];
-   for (int d = 0; d < SDIM; ++d)
-   {
-      dxyz[d] = x[d] - b->c0[d];
-   }
-
-   double test = 1.0;
-   for (int d = 0; d < SDIM; ++d)
-   {
-      double rst = 0.0;
-      for (int e = 0; e < SDIM; ++e)
-      {
-         rst += b->A[d * SDIM + e] * dxyz[e];
-      }
-      const double brst = (rst + 1.0) * (1.0 - rst);
-      test = test < 0.0 ? test : brst;
-   }
-   return test;
-}
-
-// Hash index in the hash table for the point x.
-template <int SDIM>
-MFEM_HOST_DEVICE inline int hash_index(
-   const findptsLocalHashData_t<SDIM> *const p,
-   const double (&x)[SDIM])
-{
-   const int n = p->hash_n;
-   int sum = 0;
-   for (int d = SDIM - 1; d >= 0; --d)
-   {
-      sum *= n;
-      const int i = (int)floor((x[d] - p->bnd[d].min) * p->fac[d]);
-      sum += i < 0 ? 0 : (n - 1 < i ? n - 1 : i);
-   }
-   return sum;
-}
-
-// Squared Euclidean norm.
-template <int SDIM>
-MFEM_HOST_DEVICE inline double l2norm2(const double (&x)[SDIM])
-{
-   double sum = 0.0;
-   for (int d = 0; d < SDIM; ++d)
-   {
-      sum += x[d] * x[d];
-   }
-   return sum;
-}
-
-template <int SDIM>
-MFEM_HOST_DEVICE inline double l2norm2(const double *x)
-{
-   double sum = 0.0;
-   for (int d = 0; d < SDIM; ++d)
-   {
-      sum += x[d] * x[d];
-   }
-   return sum;
-}
-
-} // namespace gslib
-
-} // namespace mfem
-
-#endif

fem/gslib/interpolate_local_1.cpp

@@ -11,7 +11,7 @@
 
 #include "../gslib.hpp"
 #include "../../general/forall.hpp"
-#include "gslib_kernel_helpers.hpp"
+#include "../../linalg/kernels.hpp"
 
 #ifdef MFEM_USE_GSLIB
 
@@ -33,7 +33,17 @@ namespace mfem
 #define CODE_BORDER 1
 #define CODE_NOT_FOUND 2
 
-using gslib::lagrange_eval;
+static MFEM_HOST_DEVICE void lagrange_eval(double *p0, double x,
+                                           int i, int p_Nq,
+                                           double *z, double *lagrangeCoeff)
+{
+   double p_i = (1 << (p_Nq - 1));
+   for (int j=0; j<p_Nq; ++j)
+   {
+      p_i *= j==i ? 1 : x-z[j];
+   }
+   p0[i] = lagrangeCoeff[i] * p_i;
+}
 
 template<int T_D1D = 0>
 static void InterpolateLocal1DKernel(const double *const gf_in,
@@ -113,26 +123,21 @@ void FindPointsGSLIB::InterpolateLocal1( const Vector &field_in,
    auto plcf = DEV.lagcoeff_sol.ReadWrite(use_dev);
    switch (dof1Dsol)
    {
-      case 2:
-         InterpolateLocal1DKernel<2>(pfin, pgsl, pgslr, pfout,
-                                     npt, ncomp, pgll, plcf);
-         break;
-      case 3:
-         InterpolateLocal1DKernel<3>(pfin, pgsl, pgslr, pfout,
-                                     npt, ncomp, pgll, plcf);
-         break;
-      case 4:
-         InterpolateLocal1DKernel<4>(pfin, pgsl, pgslr, pfout,
-                                     npt, ncomp, pgll, plcf);
-         break;
-      case 5:
-         InterpolateLocal1DKernel<5>(pfin, pgsl, pgslr, pfout,
-                                     npt, ncomp, pgll, plcf);
-         break;
-      default:
-         InterpolateLocal1DKernel(pfin, pgsl, pgslr, pfout,
-                                  npt, ncomp, pgll, plcf, dof1Dsol);
-         break;
+      case 2: return InterpolateLocal1DKernel<2>(pfin, pgsl, pgslr, pfout,
+                                                    npt, ncomp,
+                                                    pgll, plcf);
+      case 3: return InterpolateLocal1DKernel<3>(pfin, pgsl, pgslr, pfout,
+                                                    npt, ncomp,
+                                                    pgll, plcf);
+      case 4: return InterpolateLocal1DKernel<4>(pfin, pgsl, pgslr, pfout,
+                                                    npt, ncomp,
+                                                    pgll, plcf);
+      case 5: return InterpolateLocal1DKernel<5>(pfin, pgsl, pgslr, pfout,
+                                                    npt, ncomp,
+                                                    pgll, plcf);
+      default: return InterpolateLocal1DKernel(pfin, pgsl, pgslr, pfout,
+                                                  npt, ncomp,
+                                                  pgll, plcf, dof1Dsol);
    }
 }
 #undef CODE_INTERNAL

fem/gslib/interpolate_local_2.cpp

@@ -11,7 +11,6 @@
 
 #include "../gslib.hpp"
 #include "../../general/forall.hpp"
-#include "gslib_kernel_helpers.hpp"
 
 #ifdef MFEM_USE_GSLIB
 
@@ -33,7 +32,18 @@ namespace mfem
 #define CODE_BORDER 1
 #define CODE_NOT_FOUND 2
 
-using gslib::lagrange_eval;
+static MFEM_HOST_DEVICE void lagrange_eval(double *p0, double x,
+                                           int i, int p_Nq,
+                                           double *z, double *lagrangeCoeff)
+{
+   double p_i = (1 << (p_Nq - 1));
+   for (int j = 0; j < p_Nq; ++j)
+   {
+      double d_j = x - z[j];
+      p_i *= j == i ? 1 : d_j;
+   }
+   p0[i] = lagrangeCoeff[i] * p_i;
+}
 
 template<int T_D1D = 0>
 static void InterpolateLocal2DKernel(const double *const gf_in,
@@ -122,26 +132,21 @@ void FindPointsGSLIB::InterpolateLocal2(const Vector &field_in,
    auto plcf = DEV.lagcoeff_sol.ReadWrite(use_dev);
    switch (dof1Dsol)
    {
-      case 2:
-         InterpolateLocal2DKernel<2>(pfin, pgsl, pgslr, pfout,
-                                     npt, ncomp, pgll, plcf);
-         break;
-      case 3:
-         InterpolateLocal2DKernel<3>(pfin, pgsl, pgslr, pfout,
-                                     npt, ncomp, pgll, plcf);
-         break;
-      case 4:
-         InterpolateLocal2DKernel<4>(pfin, pgsl, pgslr, pfout,
-                                     npt, ncomp, pgll, plcf);
-         break;
-      case 5:
-         InterpolateLocal2DKernel<5>(pfin, pgsl, pgslr, pfout,
-                                     npt, ncomp, pgll, plcf);
-         break;
-      default:
-         InterpolateLocal2DKernel(pfin, pgsl, pgslr, pfout,
-                                  npt, ncomp, pgll, plcf, dof1Dsol);
-         break;
+      case 2: return InterpolateLocal2DKernel<2>(pfin, pgsl, pgslr, pfout,
+                                                    npt, ncomp,
+                                                    pgll, plcf);
+      case 3: return InterpolateLocal2DKernel<3>(pfin, pgsl, pgslr, pfout,
+                                                    npt, ncomp,
+                                                    pgll, plcf);
+      case 4: return InterpolateLocal2DKernel<4>(pfin, pgsl, pgslr, pfout,
+                                                    npt, ncomp,
+                                                    pgll, plcf);
+      case 5: return InterpolateLocal2DKernel<5>(pfin, pgsl, pgslr, pfout,
+                                                    npt, ncomp,
+                                                    pgll, plcf);
+      default: return InterpolateLocal2DKernel(pfin, pgsl, pgslr, pfout,
+                                                  npt, ncomp,
+                                                  pgll, plcf, dof1Dsol);
    }
 }
 

fem/gslib/interpolate_local_3.cpp

@@ -11,7 +11,6 @@
 
 #include "../gslib.hpp"
 #include "../../general/forall.hpp"
-#include "gslib_kernel_helpers.hpp"
 
 #ifdef MFEM_USE_GSLIB
 
@@ -33,7 +32,18 @@ namespace mfem
 #define CODE_BORDER 1
 #define CODE_NOT_FOUND 2
 
-using gslib::lagrange_eval;
+static MFEM_HOST_DEVICE void lagrange_eval(double *p0, double x,
+                                           int i, int p_Nq,
+                                           double *z, double *lagrangeCoeff)
+{
+   double p_i = (1 << (p_Nq - 1));
+   for (int j = 0; j < p_Nq; ++j)
+   {
+      double d_j = x - z[j];
+      p_i *= j == i ? 1 : d_j;
+   }
+   p0[i] = lagrangeCoeff[i] * p_i;
+}
 
 template<int T_D1D = 0>
 static void InterpolateLocal3DKernel(const double *const gf_in,
@@ -125,26 +135,21 @@ void FindPointsGSLIB::InterpolateLocal3(const Vector &field_in,
    auto plcf = DEV.lagcoeff_sol.ReadWrite(use_dev);
    switch (dof1Dsol)
    {
-      case 2:
-         InterpolateLocal3DKernel<2>(pfin, pgsle, pgslr, pfout,
-                                     npt, ncomp, pgll, plcf);
-         break;
-      case 3:
-         InterpolateLocal3DKernel<3>(pfin, pgsle, pgslr, pfout,
-                                     npt, ncomp, pgll, plcf);
-         break;
-      case 4:
-         InterpolateLocal3DKernel<4>(pfin, pgsle, pgslr, pfout,
-                                     npt, ncomp, pgll, plcf);
-         break;
-      case 5:
-         InterpolateLocal3DKernel<5>(pfin, pgsle, pgslr, pfout,
-                                     npt, ncomp, pgll, plcf);
-         break;
-      default:
-         InterpolateLocal3DKernel(pfin, pgsle, pgslr, pfout,
-                                  npt, ncomp, pgll, plcf, dof1Dsol);
-         break;
+      case 2: return InterpolateLocal3DKernel<2>(pfin, pgsle, pgslr, pfout,
+                                                    npt, ncomp,
+                                                    pgll, plcf);
+      case 3: return InterpolateLocal3DKernel<3>(pfin, pgsle, pgslr, pfout,
+                                                    npt, ncomp,
+                                                    pgll, plcf);
+      case 4: return InterpolateLocal3DKernel<4>(pfin, pgsle, pgslr, pfout,
+                                                    npt, ncomp,
+                                                    pgll, plcf);
+      case 5: return InterpolateLocal3DKernel<5>(pfin, pgsle, pgslr, pfout,
+                                                    npt, ncomp,
+                                                    pgll, plcf);
+      default: return InterpolateLocal3DKernel(pfin, pgsle, pgslr, pfout,
+                                                  npt, ncomp,
+                                                  pgll, plcf, dof1Dsol);
    }
 }
 

fem/integ/bilininteg_elasticity_kernels.cpp

@@ -91,15 +91,15 @@ void ElasticityAddMultPA(const int dim, const int nDofs,
 void ElasticityAssembleDiagonalPA(const int dim, const int nDofs,
                                   const CoefficientVector &lambda,
                                   const CoefficientVector &mu, const GeometricFactors &geom,
-                                  const DofToQuad &maps, const IntegrationRule &ir, Vector &diag)
+                                  const DofToQuad &maps, QuadratureFunction &QVec, Vector &diag)
 {
    switch (dim)
    {
       case 2:
-         ElasticityAssembleDiagonalPA_<2>(nDofs, lambda, mu, geom, maps, ir, diag);
+         ElasticityAssembleDiagonalPA_<2>(nDofs, lambda, mu, geom, maps, QVec, diag);
          break;
       case 3:
-         ElasticityAssembleDiagonalPA_<3>(nDofs, lambda, mu, geom, maps, ir, diag);
+         ElasticityAssembleDiagonalPA_<3>(nDofs, lambda, mu, geom, maps, QVec, diag);
          break;
       default:
          MFEM_ABORT("Only dimensions 2 and 3 supported.");

fem/integ/bilininteg_elasticity_kernels.hpp

@@ -38,6 +38,7 @@
 #include "../../linalg/vector.hpp"
 #include "../../linalg/tensor.hpp"
 #include "../quadinterpolator.hpp"
+#include "../bilininteg.hpp"
 #include "../coefficient.hpp"
 #include "../qfunction.hpp"
 
@@ -132,12 +133,12 @@ void ElasticityAssembleEA(const int dim, const int i_block, const int j_block,
 /// @param[in] mu Quadrature function for second Lame param.
 /// @param[in] geom Geometric factors corresponding to fespace.
 /// @param[in] maps DofToQuad maps for one element (assume elements all same).
-/// @param[in] ir Integration rule.
+/// @param QVec Scratch Q-Vector. nQuad x dim x dim x dim x dim x numEls.
 /// @param[out] diag diagonal of A. nDofs x dim x numEls.
 void ElasticityAssembleDiagonalPA(const int dim, const int nDofs,
                                   const CoefficientVector &lambda,
                                   const CoefficientVector &mu, const GeometricFactors &geom,
-                                  const DofToQuad &maps, const IntegrationRule &ir, Vector &diag);
+                                  const DofToQuad &maps, QuadratureFunction &QVec, Vector &diag);
 
 /// Templated implementation of ElasticityAddMultPA.
 template<int dim, int i_block = -1, int j_block = -1>
@@ -279,67 +280,77 @@ void ElasticityAddMultPA_(const int nDofs, const FiniteElementSpace &fespace,
 template<int dim>
 void ElasticityAssembleDiagonalPA_(const int nDofs,
                                    const CoefficientVector &lambda,
-                                   const CoefficientVector &mu,
-                                   const GeometricFactors &geom,
-                                   const DofToQuad &maps,
-                                   const IntegrationRule &ir,
-                                   Vector &diag)
+                                   const CoefficientVector &mu, const GeometricFactors &geom,
+                                   const DofToQuad &maps, QuadratureFunction &QVec, Vector &diag)
 {
+   using future::tensor;
+   using future::make_tensor;
    using future::det;
    using future::inv;
-   using future::make_tensor;
-   using future::tensor;
 
    // Assuming all elements are the same
+   const auto &ir = QVec.GetIntRule(0);
    static constexpr int d = dim;
    const int numPoints = ir.GetNPoints();
-   const int numEls = lambda.Size() / numPoints;
-
+   const int numEls = lambda.Size()/numPoints;
    const auto lamDev = Reshape(lambda.Read(), numPoints, numEls);
    const auto muDev = Reshape(mu.Read(), numPoints, numEls);
    const auto J = Reshape(geom.J.Read(), numPoints, d, d, numEls);
+   auto Q = Reshape(QVec.ReadWrite(), numPoints, d,d, d, numEls);
    const real_t *ipWeights = ir.GetWeights().Read();
-   const auto G = Reshape(maps.G.Read(), numPoints, d, nDofs);
-   auto diagDev = Reshape(diag.Write(), nDofs, d, numEls);
+   mfem::forall_2D(numEls, numPoints,1, [=] MFEM_HOST_DEVICE (int e)
+   {
+      MFEM_FOREACH_THREAD(p, x,numPoints)
+      {
+         auto invJ = inv(make_tensor<d, d>(
+         [&](int i, int j) { return J(p, i, j, e); }));
+         const real_t w = ipWeights[p] /det(invJ);
+         for (int n = 0; n < d; n++)
+         {
+            for (int m = 0; m < d; m++)
+            {
+               for (int q = 0; q < d; q++)
+               {
+                  // compute contraction of 4*sym(grad(u))sym(grad(v)) term.
+                  // this contraction could be made slightly cheaper using Voigt
+                  // notation, but repeated entries are summed for simplicity.
+                  real_t contraction = 0.;
+                  for (int a = 0; a < d; a++)
+                  {
+                     for (int b = 0; b < d; b++)
+                     {
+                        contraction += ((a == q)*invJ(m,b) + (b==q)*invJ(m,a))*((a == q)
+                                                                                *invJ(n, b) + (b==q)*invJ(n,a));
+                     }
+                  }
+                  // lambda*div(u)*div(v) + 2*mu*sym(grad(u))*sym(grad(v))
+                  // contraction = 4*sym(grad(u))sym(grad(v))
+                  Q(p,m,n,q,e) = w*(lamDev(p, e)*invJ(m,q)*invJ(n,q)
+                                    + 0.5*muDev(p, e)*contraction);
+               }
+            }
+         }
+      }
+   });
 
+   // Reduce quadrature function to an E-Vector
+   const auto QRead = Reshape(QVec.Read(), numPoints, d, d, d, numEls);
+   auto diagDev = Reshape(diag.Write(), nDofs, d, numEls);
+   const auto G = Reshape(maps.G.Read(), numPoints, d, nDofs);
    mfem::forall_2D(numEls, d, nDofs, [=] MFEM_HOST_DEVICE (int e)
    {
-      MFEM_FOREACH_THREAD_DIRECT(i, y, nDofs)
+      MFEM_FOREACH_THREAD(i, y, nDofs)
       {
-         MFEM_FOREACH_THREAD_DIRECT(q, x, d)
+         MFEM_FOREACH_THREAD(q, x, d)
          {
-            real_t sum = 0.0;
-            for (int p = 0; p < numPoints; p++)
+            real_t sum = 0.;
+            for (int n = 0; n < d; n++)
             {
-               const auto invJ = inv(make_tensor<d, d>([&](int r, int c)
+               for (int m = 0; m < d; m++)
                {
-                  return J(p, r, c, e);
-               }));
-               const real_t w = ipWeights[p] / det(invJ);
-
-               for (int n = 0; n < d; n++)
-               {
-                  for (int m = 0; m < d; m++)
+                  for (int p = 0; p < numPoints; p++ )
                   {
-                     // compute contraction of 4*sym(grad(u))sym(grad(v)) term.
-                     // this contraction could be made slightly cheaper using Voigt
-                     // notation, but repeated entries are summed for simplicity.
-                     real_t contraction = 0.0;
-                     for (int a = 0; a < d; a++)
-                     {
-                        for (int b = 0; b < d; b++)
-                        {
-                           contraction +=
-                              ((a == q) * invJ(m, b) + (b == q) * invJ(m, a)) *
-                              ((a == q) * invJ(n, b) + (b == q) * invJ(n, a));
-                        }
-                     }
-                     // lambda*div(u)*div(v) + 2*mu*sym(grad(u))*sym(grad(v))
-                     // contraction = 4*sym(grad(u))sym(grad(v))
-                     const real_t Q =
-                        w * (lamDev(p, e) * invJ(m, q) * invJ(n, q)
-                             + 0.5 * muDev(p, e) * contraction);
-                     sum += Q * G(p, m, i) * G(p, n, i);
+                     sum += QRead(p,m,n,q,e)*G(p,m,i)*G(p,n,i);
                   }
                }
             }

fem/integ/bilininteg_elasticity_pa.cpp

@@ -10,6 +10,7 @@
 // CONTRIBUTING.md for details.
 
 #include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
 #include "../qfunction.hpp"
 #include "bilininteg_elasticity_kernels.hpp"
 
@@ -58,8 +59,9 @@ void ElasticityIntegrator::AssemblePA(const FiniteElementSpace &fes)
 
 void ElasticityIntegrator::AssembleDiagonalPA(Vector &diag)
 {
+   q_vec->SetVDim(vdim*vdim*vdim*vdim);
    internal::ElasticityAssembleDiagonalPA(vdim, ndofs, *lambda_quad, *mu_quad,
-                                          *geom, *maps, *IntRule, diag);
+                                          *geom, *maps, *q_vec, diag);
 }
 
 void ElasticityIntegrator::AddMultPA(const Vector &x, Vector &y) const

fem/integ/bilininteg_hcurl_kernels.cpp

@@ -148,12 +148,11 @@ void PAHcurlMassAssembleDiagonal3D(const int D1D,
 }
 
 void PAHcurlMassApply2D(const int NE, const bool symmetric,
-                        [[maybe_unused]] const bool scalar_coeff,
-                        const Array<real_t> &bo, const Array<real_t> &bc,
-                        const Array<real_t> &bot, const Array<real_t> &bct,
-                        const Vector &pa_data, const Vector &x, Vector &y,
-                        const int D1D, [[maybe_unused]] const int TestD1D,
-                        const int Q1D)
+                        const bool scalar_coeff, const Array<real_t> &bo,
+                        const Array<real_t> &bc, const Array<real_t> &bot,
+                        const Array<real_t> &bct, const Vector &pa_data,
+                        const Vector &x, Vector &y, const int D1D,
+                        const int TestD1D, const int Q1D)
 {
    MFEM_ASSERT(D1D == TestD1D,
                "Trial and Test space must have the same number of dofs");
@@ -276,12 +275,11 @@ void PAHcurlMassApply2D(const int NE, const bool symmetric,
 }
 
 void PAHcurlMassApply3D(const int NE, const bool symmetric,
-                        [[maybe_unused]] const bool scalar_coeff,
-                        const Array<real_t> &bo, const Array<real_t> &bc,
-                        const Array<real_t> &bot, const Array<real_t> &bct,
-                        const Vector &pa_data, const Vector &x, Vector &y,
-                        const int D1D, [[maybe_unused]] const int TestD1D,
-                        const int Q1D)
+                        const bool scalar_coeff, const Array<real_t> &bo,
+                        const Array<real_t> &bc, const Array<real_t> &bot,
+                        const Array<real_t> &bct, const Vector &pa_data,
+                        const Vector &x, Vector &y, const int D1D,
+                        const int TestD1D, const int Q1D)
 {
    MFEM_VERIFY(D1D == TestD1D,
                "Trial and test spaces must have same number of dofs");

fem/integ/bilininteg_hcurl_kernels.hpp

@@ -190,22 +190,19 @@ void PAHcurlMassApply2D(const int NE, const bool symmetric,
 
 // PA H(curl) Mass Apply 3D kernel
 void PAHcurlMassApply3D(const int NE, const bool symmetric,
-                        [[maybe_unused]] const bool scalar_coeff,
-                        const Array<real_t> &bo, const Array<real_t> &bc,
-                        const Array<real_t> &bot, const Array<real_t> &bct,
-                        const Vector &pa_data, const Vector &x, Vector &y,
-                        const int TrialD1D, [[maybe_unused]] const int TestD1D,
-                        const int Q1D);
+                        const bool scalar_coeff, const Array<real_t> &bo,
+                        const Array<real_t> &bc, const Array<real_t> &bot,
+                        const Array<real_t> &bct, const Vector &pa_data,
+                        const Vector &x, Vector &y, const int TrialD1D,
+                        const int TestD1D, const int Q1D);
 
 // Shared memory PA H(curl) Mass Apply 3D kernel
 template <int T_D1D = 0, int T_Q1D = 0, int TBATCH = 0, bool ACCUMULATE = true>
 inline void SmemPAHcurlMassApply3D(
-   const int NE, const bool symmetric, [[maybe_unused]] const bool scalar_coeff,
-   const Array<real_t> &bo, const Array<real_t> &bc,
-   [[maybe_unused]] const Array<real_t> &bot,
-   [[maybe_unused]] const Array<real_t> &bct, const Vector &pa_data,
-   const Vector &x, Vector &y, const int d1d = 0,
-   [[maybe_unused]] const int test_d1d = 0, const int q1d = 0)
+   const int NE, const bool symmetric, const bool scalar_coeff,
+   const Array<real_t> &bo, const Array<real_t> &bc, const Array<real_t> &bot,
+   const Array<real_t> &bct, const Vector &pa_data, const Vector &x, Vector &y,
+   const int d1d = 0, const int = 0, const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;

fem/integ/bilininteg_hcurlhdiv_kernels.hpp

@@ -864,8 +864,656 @@ inline void PAHcurlHdivApplyTranspose3D(const int d1d,
    }); // end of element loop
 }
 
+namespace curlinterp
+{
+constexpr int NBZ3D(int ndof_o, int nquad_o, int mdq)
+{
+   if (ndof_o <= 0 || nquad_o <= 0)
+   {
+      return 1;
+   }
+   int ndof_c = ndof_o + 1;
+   int nquad_c = nquad_o + 1;
+   // z dimension is capped at 64 on nvidia and amd gpus
+   int tmp =
+      std::min((128 + mdq * mdq * (mdq - 1) - 1) / (mdq * mdq * (mdq - 1)), 64);
+   int smem_req =
+      sizeof(mfem::real_t) *
+      ((3 * ndof_c * ndof_c * ndof_o + 2 * 2 * mdq * mdq * mdq) * tmp +
+       ndof_c * nquad_o + ndof_c * nquad_c + ndof_o * nquad_o);
+   // assume GPU has at least 48k shared memory
+   return std::max(std::min(tmp, (48 * 1024 + smem_req - 1) / smem_req), 1);
+}
+}
+
+template <int T_NDOF_O, int T_NQUAD_O>
+void CurlInterpolatorApply3DSmem(const int ne, const int ndof_o,
+                                 const int nquad_o, const Vector &pa,
+                                 const Vector &x_, Vector &y_)
+{
+   constexpr int mnd_o = T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
+   constexpr int mnq_o =
+      T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
+   constexpr int mndq = std::max(mnd_o + 1, mnq_o + 1);
+   constexpr int tbatch = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, mndq);
+   MFEM_VERIFY(ndof_o <= mnd_o, "Error: H(curl) order larger than supported");
+   MFEM_VERIFY(nquad_o <= mnq_o, "Error: H(div) order larger than supported");
+   int mnq = std::max(ndof_o + 1, nquad_o + 1);
+   auto pa_data = pa.Read();
+   auto x_d = x_.Read();
+   auto y_d = y_.ReadWrite();
+   mfem::forall_2D_batch<mndq * mndq * (mndq - 1) * tbatch>(
+      ne, mnq * mnq * (mnq - 1), 1, tbatch, [=] MFEM_HOST_DEVICE(int e)
+   {
+      constexpr int MND_O =
+         T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
+      constexpr int MNQ_O =
+         T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
+      constexpr int MNDQ = std::max(MND_O + 1, MNQ_O + 1);
+#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
+      constexpr int nbz = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, MNDQ);
+      int tidz = MFEM_THREAD_ID(z);
+      // Make mnq a local variable since capturing would result in different
+      // captures between host/device versions, and spuriously fails
+      int mnq = std::max(ndof_o + 1, nquad_o + 1);
+#else
+      constexpr int nbz = 1;
+      constexpr int tidz = 0;
+#endif
+      const int NDOF_O = T_NDOF_O ? T_NDOF_O : ndof_o;
+      const int NQUAD_O = T_NQUAD_O ? T_NQUAD_O : nquad_o;
+      const int NDOF_C = NDOF_O + 1;
+      const int NQUAD_C = NQUAD_O + 1;
+      MFEM_SHARED real_t
+      sBG[(MND_O + 1) * MNQ_O + (MND_O + 1) * (MNQ_O + 1) + MND_O * MNQ_O];
+      auto X_ = Reshape(x_d, 3 * NDOF_C * NDOF_C * NDOF_O, ne);
+      auto Y = Reshape(y_d, 3 * NQUAD_C * NQUAD_O * NQUAD_O, ne);
+      auto Gco = Reshape(sBG, NQUAD_O, NDOF_C);
+      auto Bcc = Reshape(sBG + NDOF_C * NQUAD_O, NQUAD_C, NDOF_C);
+      auto Boo =
+         Reshape(sBG + NDOF_C * NQUAD_O + NDOF_C * NQUAD_C, NQUAD_O, NDOF_O);
+      MFEM_SHARED real_t X[3][nbz][MND_O * (MND_O + 1) * (MND_O + 1)];
+      MFEM_SHARED real_t sm0[nbz * 2 * MNDQ * MNDQ * MNDQ];
+      MFEM_SHARED real_t sm1[nbz * 2 * MNDQ * MNDQ * MNDQ];
+
+      // shapes of buffers always use MNDQ to mitigate shared memory bank
+      // conflicts
+      real_t(*DDQ)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
+      real_t(*DQQ)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm1);
+      real_t(*QQQ)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
+      const int offset = NDOF_O * NDOF_C * NDOF_C;
+      const int offsetq = NQUAD_C * NQUAD_O * NQUAD_O;
+      MFEM_FOREACH_THREAD_DIRECT(ix, x, offset)
+      {
+         for (int dim = 0; dim < 3; ++dim)
+         {
+            X[dim][tidz][ix] = X_(ix + dim * offset, e);
+         }
+      }
+      // load basis functions data
+      if (tidz == 0)
+      {
+         auto npts = NDOF_C * NQUAD_O + NDOF_C * NQUAD_C + NDOF_O * NQUAD_O;
+         MFEM_FOREACH_THREAD(ix, x, npts) { sBG[ix] = pa_data[ix]; }
+      }
+      MFEM_SYNC_THREAD;
+
+      // x: Vz Bcc Gco Boo - Vy Bcc Boo Gco
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_C, NDOF_C,
+                                           NDOF_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_C; ++dx)
+         {
+            u += X[2][tidz][dx + (dy + dz * NDOF_C) * NDOF_C] * Bcc(qx, dx);
+         }
+         DDQ[0][tidz][dz][dy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_C, NDOF_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_C; ++dx)
+         {
+            u += X[1][tidz][dx + (dy + dz * NDOF_O) * NDOF_C] * Bcc(qx, dx);
+         }
+         DDQ[1][tidz][dz][dy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_C, NQUAD_O,
+                                           NDOF_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_C; ++dy)
+         {
+            u += DDQ[0][tidz][dz][dy][qx] * Gco(qy, dy);
+         }
+         DQQ[0][tidz][dz][qy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_C, NQUAD_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_O; ++dy)
+         {
+            u += DDQ[1][tidz][dz][dy][qx] * Boo(qy, dy);
+         }
+         DQQ[1][tidz][dz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_C, NQUAD_O,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_O; ++dz)
+         {
+            u += DQQ[0][tidz][dz][qy][qx] * Boo(qz, dz);
+         }
+         QQQ[0][tidz][qz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_C, NQUAD_O,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_C; ++dz)
+         {
+            u += DQQ[1][tidz][dz][qy][qx] * Gco(qz, dz);
+         }
+         Y(qx + (qy + qz * NQUAD_O) * NQUAD_C, e) =
+            QQQ[0][tidz][qz][qy][qx] - u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // y: Vx Boo Bcc Gco - Vz Gco Bcc Boo
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_C,
+                                           NDOF_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_O; ++dx)
+         {
+            u += X[0][tidz][dx + (dy + dz * NDOF_C) * NDOF_O] * Boo(qx, dx);
+         }
+         DDQ[0][tidz][dz][dy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_C,
+                                           NDOF_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_C; ++dx)
+         {
+            u += X[2][tidz][dx + (dy + dz * NDOF_C) * NDOF_C] * Gco(qx, dx);
+         }
+         DDQ[1][tidz][dz][dy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_C,
+                                           NDOF_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_C; ++dy)
+         {
+            u += DDQ[0][tidz][dz][dy][qx] * Bcc(qy, dy);
+         }
+         DQQ[0][tidz][dz][qy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_C,
+                                           NDOF_O, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_C; ++dy)
+         {
+            u += DDQ[1][tidz][dz][dy][qx] * Bcc(qy, dy);
+         }
+         DQQ[1][tidz][dz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_C; ++dz)
+         {
+            u += DQQ[0][tidz][dz][qy][qx] * Gco(qz, dz);
+         }
+         QQQ[0][tidz][qz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_O; ++dz)
+         {
+            u += DQQ[1][tidz][dz][qy][qx] * Boo(qz, dz);
+         }
+         Y(qx + (qy + qz * NQUAD_C) * NQUAD_O + offsetq, e) =
+            QQQ[0][tidz][qz][qy][qx] - u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // z: Vy Gco Boo Bcc - Vx Boo Gco Bcc
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_C; ++dx)
+         {
+            u += X[1][tidz][dx + (dy + dz * NDOF_O) * NDOF_C] * Gco(qx, dx);
+         }
+         DDQ[0][tidz][dz][dy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, dy, dz, x, NQUAD_O, NDOF_C,
+                                           NDOF_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int dx = 0; dx < NDOF_O; ++dx)
+         {
+            u += X[0][tidz][dx + (dy + dz * NDOF_C) * NDOF_O] * Boo(qx, dx);
+         }
+         DDQ[1][tidz][dz][dy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_O; ++dy)
+         {
+            u += DDQ[0][tidz][dz][dy][qx] * Boo(qy, dy);
+         }
+         DQQ[0][tidz][dz][qy][qx] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, dz, x, NQUAD_O, NQUAD_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dy = 0; dy < NDOF_C; ++dy)
+         {
+            u += DDQ[1][tidz][dz][dy][qx] * Gco(qy, dy);
+         }
+         DQQ[1][tidz][dz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_O,
+                                           NQUAD_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_C; ++dz)
+         {
+            u += DQQ[0][tidz][dz][qy][qx] * Bcc(qz, dz);
+         }
+         QQQ[0][tidz][qz][qy][qx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(qx, qy, qz, x, NQUAD_O, NQUAD_O,
+                                           NQUAD_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int dz = 0; dz < NDOF_C; ++dz)
+         {
+            u += DQQ[1][tidz][dz][qy][qx] * Bcc(qz, dz);
+         }
+         Y(qx + (qy + qz * NQUAD_O) * NQUAD_O + 2 * offsetq, e) =
+            QQQ[0][tidz][qz][qy][qx] - u;
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+template <int T_NDOF_O, int T_NQUAD_O>
+void CurlInterpolatorTApply3DSmem(const int ne, const int ndof_o,
+                                  const int nquad_o, const Vector &pa,
+                                  const Vector &x_, Vector &y_)
+{
+   constexpr int mnd_o = T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
+   constexpr int mnq_o =
+      T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
+   constexpr int mndq = std::max(mnd_o + 1, mnq_o + 1);
+   constexpr int tbatch = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, mndq);
+   MFEM_VERIFY(ndof_o <= mnd_o, "Error: H(curl) order larger than supported");
+   MFEM_VERIFY(nquad_o <= mnq_o, "Error: H(div) order larger than supported");
+   int mnq = std::max(ndof_o + 1, nquad_o + 1);
+   auto pa_data = pa.Read();
+   auto x_d = x_.Read();
+   auto y_d = y_.ReadWrite();
+   mfem::forall_2D_batch<mndq * mndq * (mndq - 1) * tbatch>(
+      ne, mnq * mnq * (mnq - 1), 1, tbatch, [=] MFEM_HOST_DEVICE(int e)
+   {
+      constexpr int MND_O =
+         T_NDOF_O ? T_NDOF_O : DofQuadLimits::HCURL_MAX_D1D - 1;
+      constexpr int MNQ_O =
+         T_NQUAD_O ? T_NQUAD_O : DofQuadLimits::HDIV_MAX_D1D - 1;
+      constexpr int MNDQ = std::max(MND_O + 1, MNQ_O + 1);
+#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
+      constexpr int nbz = curlinterp::NBZ3D(T_NDOF_O, T_NQUAD_O, MNDQ);
+      int tidz = MFEM_THREAD_ID(z);
+      // Make mnq a local variable since capturing would result in different
+      // captures between host/device versions, and spuriously fails
+      int mnq = std::max(ndof_o + 1, nquad_o + 1);
+#else
+      constexpr int nbz = 1;
+      constexpr int tidz = 0;
+#endif
+      const int NDOF_O = T_NDOF_O ? T_NDOF_O : ndof_o;
+      const int NQUAD_O = T_NQUAD_O ? T_NQUAD_O : nquad_o;
+      const int NDOF_C = NDOF_O + 1;
+      const int NQUAD_C = NQUAD_O + 1;
+      MFEM_SHARED real_t
+      sBG[(MND_O + 1) * MNQ_O + (MND_O + 1) * (MNQ_O + 1) + MND_O * MNQ_O];
+      auto X_ = Reshape(x_d, 3 * NQUAD_C * NQUAD_O * NQUAD_O, ne);
+      auto Y = Reshape(y_d, 3 * NDOF_C * NDOF_C * NDOF_O, ne);
+      auto Gco = Reshape(sBG, NQUAD_O, NDOF_C);
+      auto Bcc = Reshape(sBG + NDOF_C * NQUAD_O, NQUAD_C, NDOF_C);
+      auto Boo =
+         Reshape(sBG + NDOF_C * NQUAD_O + NDOF_C * NQUAD_C, NQUAD_O, NDOF_O);
+      MFEM_SHARED real_t X[3][nbz][MNQ_O * MNQ_O * (MNQ_O + 1)];
+      MFEM_SHARED real_t sm0[nbz * 2 * MNDQ * MNDQ * MNDQ];
+      MFEM_SHARED real_t sm1[nbz * 2 * MNDQ * MNDQ * MNDQ];
+
+      // shapes of buffers always use MNDQ to mitigate shared memory bank
+      // conflicts
+      real_t(*QQD)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
+      real_t(*QDD)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm1);
+      real_t(*DDD)[nbz][MNDQ][MNDQ][MNDQ] =
+         (real_t(*)[nbz][MNDQ][MNDQ][MNDQ])(sm0);
+      const int offset = NDOF_O * NDOF_C * NDOF_C;
+      const int offsetq = NQUAD_C * NQUAD_O * NQUAD_O;
+      MFEM_FOREACH_THREAD_DIRECT(ix, x, offsetq)
+      {
+         for (int dim = 0; dim < 3; ++dim)
+         {
+            X[dim][tidz][ix] = X_(ix + dim * offsetq, e);
+         }
+      }
+      // load basis functions data
+      if (tidz == 0)
+      {
+         auto npts = NDOF_C * NQUAD_O + NDOF_C * NQUAD_C + NDOF_O * NQUAD_O;
+         MFEM_FOREACH_THREAD(ix, x, npts) { sBG[ix] = pa_data[ix]; }
+      }
+      MFEM_SYNC_THREAD;
+
+      // x: Vy Boo Bcc Gco - Vz Boo Gco Bcc
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_O,
+                                           NQUAD_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_O; ++qz)
+         {
+            u += X[1][tidz][qx + (qy + qz * NQUAD_C) * NQUAD_O] * Gco(qz, dz);
+         }
+         QQD[0][tidz][qy][qx][dz] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_O,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_C; ++qz)
+         {
+            u += X[2][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_O] * Bcc(qz, dz);
+         }
+         QQD[1][tidz][qy][qx][dz] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_C; ++qy)
+         {
+            u += QQD[0][tidz][qy][qx][dz] * Bcc(qy, dy);
+         }
+         QDD[0][tidz][qx][dz][dy] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_O; ++qy)
+         {
+            u += QQD[1][tidz][qy][qx][dz] * Gco(qy, dy);
+         }
+         QDD[1][tidz][qx][dz][dy] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_O, NDOF_C,
+                                           NDOF_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_O; ++qx)
+         {
+            u += QDD[0][tidz][qx][dz][dy] * Boo(qx, dx);
+         }
+         DDD[0][tidz][dz][dy][dx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_O, NDOF_C,
+                                           NDOF_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_O; ++qx)
+         {
+            u += QDD[1][tidz][qx][dz][dy] * Boo(qx, dx);
+         }
+         Y(dx + (dy + dz * NDOF_C) * NDOF_O, e) =
+            DDD[0][tidz][dz][dy][dx] - u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // y: Vz Gco Boo Bcc - Vx Bcc Boo Gco
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_O,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_C; ++qz)
+         {
+            u += X[2][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_O] * Bcc(qz, dz);
+         }
+         QQD[0][tidz][qy][qx][dz] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_C, NQUAD_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_O; ++qz)
+         {
+            u += X[0][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_C] * Gco(qz, dz);
+         }
+         QQD[1][tidz][qy][qx][dz] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_O, NDOF_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_O; ++qy)
+         {
+            u += QQD[0][tidz][qy][qx][dz] * Boo(qy, dy);
+         }
+         QDD[0][tidz][qx][dz][dy] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_O, NDOF_C,
+                                           NQUAD_C, mnq - 1, mnq, mnq)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_O; ++qy)
+         {
+            u += QQD[1][tidz][qy][qx][dz] * Boo(qy, dy);
+         }
+         QDD[1][tidz][qx][dz][dy] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_O; ++qx)
+         {
+            u += QDD[0][tidz][qx][dz][dy] * Gco(qx, dx);
+         }
+         DDD[0][tidz][dz][dy][dx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_O,
+                                           NDOF_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_C; ++qx)
+         {
+            u += QDD[1][tidz][qx][dz][dy] * Bcc(qx, dx);
+         }
+         Y(dx + (dy + dz * NDOF_O) * NDOF_C + offset, e) =
+            DDD[0][tidz][dz][dy][dx] - u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // z: Vx Bcc Gco Boo - Vy Gco Bcc Boo
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_O, NQUAD_C,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_O; ++qz)
+         {
+            u += X[0][tidz][qx + (qy + qz * NQUAD_O) * NQUAD_C] * Boo(qz, dz);
+         }
+         QQD[0][tidz][qy][qx][dz] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dz, qx, qy, x, NDOF_O, NQUAD_O,
+                                           NQUAD_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int qz = 0; qz < NQUAD_O; ++qz)
+         {
+            u += X[1][tidz][qx + (qy + qz * NQUAD_C) * NQUAD_O] * Boo(qz, dz);
+         }
+         QQD[1][tidz][qy][qx][dz] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_O,
+                                           NQUAD_C, mnq, mnq - 1, mnq)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_O; ++qy)
+         {
+            u += QQD[0][tidz][qy][qx][dz] * Gco(qy, dy);
+         }
+         QDD[0][tidz][qx][dz][dy] = u;
+      }
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dy, dz, qx, x, NDOF_C, NDOF_O,
+                                           NQUAD_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qy = 0; qy < NQUAD_C; ++qy)
+         {
+            u += QQD[1][tidz][qy][qx][dz] * Bcc(qy, dy);
+         }
+         QDD[1][tidz][qx][dz][dy] = u;
+      }
+      MFEM_SYNC_THREAD;
+
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_C,
+                                           NDOF_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_C; ++qx)
+         {
+            u += QDD[0][tidz][qx][dz][dy] * Bcc(qx, dx);
+         }
+         DDD[0][tidz][dz][dy][dx] = u;
+      }
+      MFEM_SYNC_THREAD;
+      // threads assigned to mitigate bank conflicts
+      MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(dx, dy, dz, x, NDOF_C, NDOF_C,
+                                           NDOF_O, mnq, mnq, mnq - 1)
+      {
+         real_t u = 0;
+         for (int qx = 0; qx < NQUAD_O; ++qx)
+         {
+            u += QDD[1][tidz][qx][dz][dy] * Gco(qx, dx);
+         }
+         Y(dx + (dy + dz * NDOF_C) * NDOF_C + 2 * offset, e) =
+            DDD[0][tidz][dz][dy][dx] - u;
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
 } // namespace internal
 
+template <int DIM, int NDOF_O, int NQUAD_O>
+CurlInterpolator::ApplyKernelType
+CurlInterpolator::ApplyPAKernels::Kernel()
+{
+   if constexpr (DIM == 3)
+   {
+      return internal::CurlInterpolatorApply3DSmem<NDOF_O, NQUAD_O>;
+   }
+   MFEM_ABORT("Bad dimension!");
+}
+
+template <int DIM, int NDOF_O, int NQUAD_O>
+CurlInterpolator::ApplyKernelType
+CurlInterpolator::ApplyTPAKernels::Kernel()
+{
+   if constexpr (DIM == 3)
+   {
+      return internal::CurlInterpolatorTApply3DSmem<NDOF_O, NQUAD_O>;
+   }
+   MFEM_ABORT("Bad dimension!");
+}
+
 } // namespace mfem
 
 /// \endcond DO_NOT_DOCUMENT

fem/integ/bilininteg_interp_pa.cpp

@@ -14,6 +14,8 @@
 #include "../gridfunc.hpp"
 #include "../qfunction.hpp"
 
+#include "bilininteg_hcurlhdiv_kernels.hpp"
+
 namespace mfem
 {
 
@@ -1950,4 +1952,138 @@ void IdentityInterpolator::AddMultTransposePA(const Vector &x, Vector &y) const
    }
 }
 
+void CurlInterpolator::AssemblePA(const FiniteElementSpace &dom_fes,
+                                  const FiniteElementSpace &ran_fes)
+{
+   // TODO: 1D and 2D meshes
+   Mesh *mesh = dom_fes.GetMesh();
+   const VectorTensorFiniteElement *dom_el =
+      dynamic_cast<const VectorTensorFiniteElement *>(dom_fes.GetTypicalFE());
+   const VectorTensorFiniteElement *ran_el =
+      dynamic_cast<const VectorTensorFiniteElement *>(ran_fes.GetTypicalFE());
+   MFEM_VERIFY(dom_el != NULL, "Only VectorTensorFiniteElement is supported!");
+   MFEM_VERIFY(ran_el != NULL, "Only VectorTensorFiniteElement is supported!");
+   // only supports H(curl) -> H(div) because of discontinuity requirements
+   MFEM_VERIFY(dom_el->GetDerivType() == FiniteElement::CURL,
+               "Domain space must be H(curl)");
+   MFEM_VERIFY(ran_el->GetDerivType() == FiniteElement::DIV,
+               "Range space must be H(div)");
+
+   const int dims = dom_el->GetDim();
+   MFEM_VERIFY(dims == 3, "");
+   dim = mesh->Dimension();
+
+   ne = dom_fes.GetNE();
+   ndof_o = dom_el->GetOrder();
+   int ndof_c = ndof_o + 1;
+   nquad_o = ran_el->GetOrder();
+   int nquad_c = nquad_o + 1;
+
+   // extract the tensor product range dof locations
+   std::vector<real_t> qc(nquad_c);
+   std::vector<real_t> qo(nquad_o);
+   {
+      const IntegrationRule &ran_nodes = ran_el->GetNodes();
+      const Array<int> &quad_map = ran_el->GetDofMap();
+      for (int i = 0; i < nquad_c; ++i)
+      {
+         int idx = UnsignIndex(quad_map[i]);
+         qc[i] = ran_nodes.IntPoint(idx).x;
+      }
+      int offset = ndof_c * ndof_o * ndof_o;
+      for (int i = 0; i < nquad_o; ++i)
+      {
+         int idx = UnsignIndex(quad_map[i + offset]);
+         qo[i] = ran_nodes.IntPoint(idx).x;
+      }
+   }
+
+   // evaluate closed/open 1D basis (and their derivatives) at closed and
+   // open quads
+   // storage order: GCO, BCC, BOO
+   pa_data.SetSize(ndof_c * nquad_o + ndof_c * nquad_c + ndof_o * nquad_o);
+   auto ptr = pa_data.HostWrite();
+   auto &cbasis1d = dom_el->GetBasis1D();
+   auto &obasis1d = dom_el->GetOpenBasis1D();
+   Vector b, g;
+   b.SetSize(ndof_c);
+   g.SetSize(ndof_c);
+   for (int j = 0; j < nquad_o; ++j)
+   {
+      cbasis1d.Eval(qo[j], b, g);
+      for (int i = 0; i < ndof_c; ++i)
+      {
+         ptr[j + i * nquad_o] = g[i];
+      }
+   }
+   ptr += nquad_o * ndof_c;
+
+   for (int j = 0; j < nquad_c; ++j)
+   {
+      cbasis1d.Eval(qc[j], b);
+      for (int i = 0; i < ndof_c; ++i)
+      {
+         ptr[j + i * nquad_c] = b[i];
+      }
+   }
+   ptr += ndof_c * nquad_c;
+
+   b.SetSize(ndof_o);
+   for (int j = 0; j < nquad_o; ++j)
+   {
+      obasis1d.Eval(qo[j], b);
+      for (int i = 0; i < ndof_o; ++i)
+      {
+         ptr[j + i * nquad_o] = b[i];
+      }
+   }
+}
+
+CurlInterpolator::Kernels::Kernels()
+{
+   CurlInterpolator::AddSpecialization<3, 1, 1>();
+   CurlInterpolator::AddSpecialization<3, 2, 2>();
+   CurlInterpolator::AddSpecialization<3, 3, 3>();
+   CurlInterpolator::AddSpecialization<3, 4, 4>();
+   CurlInterpolator::AddSpecialization<3, 5, 5>();
+}
+
+CurlInterpolator::CurlInterpolator() { static Kernels kernels{}; }
+
+void CurlInterpolator::AddMultPA(const Vector &x, Vector &y) const
+{
+   ApplyPAKernels::Run(dim, ndof_o, nquad_o, ne, ndof_o, nquad_o, pa_data, x,
+                       y);
+}
+
+void CurlInterpolator::AddMultTransposePA(const Vector &x, Vector &y) const
+{
+   ApplyTPAKernels::Run(dim, ndof_o, nquad_o, ne, ndof_o, nquad_o, pa_data, x,
+                        y);
+}
+
+/// \cond DO_NOT_DOCUMENT
+
+CurlInterpolator::ApplyKernelType
+CurlInterpolator::ApplyPAKernels::Fallback(int DIM, int, int)
+{
+   if (DIM == 3)
+   {
+      return internal::CurlInterpolatorApply3DSmem<0, 0>;
+   }
+   MFEM_ABORT("Bad dimension!");
+}
+
+CurlInterpolator::ApplyKernelType
+CurlInterpolator::ApplyTPAKernels::Fallback(int DIM, int, int)
+{
+   if (DIM == 3)
+   {
+      return internal::CurlInterpolatorTApply3DSmem<0, 0>;
+   }
+   MFEM_ABORT("Bad dimension!");
+}
+
+/// \endcond DO_NOT_DOCUMENT
+
 } // namespace mfem

fem/integ/bilininteg_mixedcurl_pa.cpp

@@ -18,6 +18,779 @@
 namespace mfem
 {
 
+namespace
+{
+
+void PAHcurlDotSetup2D(const int q1d,
+                       const int ne,
+                       const bool test_map_integral,
+                       const Array<real_t> &w,
+                       const Vector &jacobians,
+                       const Vector &coeff,
+                       Vector &op)
+{
+   auto W = Reshape(w.Read(), q1d, q1d);
+   auto J = Reshape(jacobians.Read(), q1d, q1d, 2, 2, ne);
+   auto C = Reshape(coeff.Read(), 2, q1d, q1d, ne);
+   auto O = Reshape(op.Write(), 2, q1d, q1d, ne);
+
+   mfem::forall_2D(ne, q1d, q1d, [=] MFEM_HOST_DEVICE (int e)
+   {
+      MFEM_FOREACH_THREAD(qy, y, q1d)
+      {
+         MFEM_FOREACH_THREAD(qx, x, q1d)
+         {
+            const real_t J11 = J(qx, qy, 0, 0, e);
+            const real_t J12 = J(qx, qy, 1, 0, e);
+            const real_t J21 = J(qx, qy, 0, 1, e);
+            const real_t J22 = J(qx, qy, 1, 1, e);
+            const real_t detJ = (J11 * J22) - (J21 * J12);
+            const real_t scale = W(qx, qy) * (test_map_integral ? 1.0 / detJ : 1.0);
+            const real_t Vx = C(0, qx, qy, e);
+            const real_t Vy = C(1, qx, qy, e);
+
+            O(0, qx, qy, e) = scale * ( J22 * Vx - J12 * Vy);
+            O(1, qx, qy, e) = scale * (-J21 * Vx + J11 * Vy);
+         }
+      }
+   });
+}
+
+void PAHcurlDotSetup3D(const int q1d,
+                       const int ne,
+                       const bool test_map_integral,
+                       const Array<real_t> &w,
+                       const Vector &jacobians,
+                       const Vector &coeff,
+                       Vector &op)
+{
+   auto W = Reshape(w.Read(), q1d, q1d, q1d);
+   auto J = Reshape(jacobians.Read(), q1d, q1d, q1d, 3, 3, ne);
+   auto C = Reshape(coeff.Read(), 3, q1d, q1d, q1d, ne);
+   auto O = Reshape(op.Write(), 3, q1d, q1d, q1d, ne);
+
+   mfem::forall_3D(ne, q1d, q1d, q1d, [=] MFEM_HOST_DEVICE (int e)
+   {
+      MFEM_FOREACH_THREAD(qz, z, q1d)
+      {
+         MFEM_FOREACH_THREAD(qy, y, q1d)
+         {
+            MFEM_FOREACH_THREAD(qx, x, q1d)
+            {
+               const real_t J11 = J(qx, qy, qz, 0, 0, e);
+               const real_t J12 = J(qx, qy, qz, 0, 1, e);
+               const real_t J13 = J(qx, qy, qz, 0, 2, e);
+               const real_t J21 = J(qx, qy, qz, 1, 0, e);
+               const real_t J22 = J(qx, qy, qz, 1, 1, e);
+               const real_t J23 = J(qx, qy, qz, 1, 2, e);
+               const real_t J31 = J(qx, qy, qz, 2, 0, e);
+               const real_t J32 = J(qx, qy, qz, 2, 1, e);
+               const real_t J33 = J(qx, qy, qz, 2, 2, e);
+               const real_t detJ = J11 * (J22 * J33 - J32 * J23)
+                                   - J21 * (J12 * J33 - J32 * J13)
+                                   + J31 * (J12 * J23 - J22 * J13);
+               const real_t scale = W(qx, qy, qz) *
+                                    (test_map_integral ? 1.0 / detJ : 1.0);
+               const real_t Vx = C(0, qx, qy, qz, e);
+               const real_t Vy = C(1, qx, qy, qz, e);
+               const real_t Vz = C(2, qx, qy, qz, e);
+
+               O(0, qx, qy, qz, e) = scale *
+                                     ((J22 * J33 - J23 * J32) * Vx +
+                                      (J13 * J32 - J12 * J33) * Vy +
+                                      (J12 * J23 - J13 * J22) * Vz);
+               O(1, qx, qy, qz, e) = scale *
+                                     ((J23 * J31 - J21 * J33) * Vx +
+                                      (J11 * J33 - J13 * J31) * Vy +
+                                      (J13 * J21 - J11 * J23) * Vz);
+               O(2, qx, qy, qz, e) = scale *
+                                     ((J21 * J32 - J22 * J31) * Vx +
+                                      (J12 * J31 - J11 * J32) * Vy +
+                                      (J11 * J22 - J12 * J21) * Vz);
+            }
+         }
+      }
+   });
+}
+
+void PAHcurlDotApply2D(const int d1d,
+                       const int d1d_test,
+                       const int q1d,
+                       const int ne,
+                       const Array<real_t> &bo,
+                       const Array<real_t> &bc,
+                       const Array<real_t> &bt,
+                       const Vector &pa_data,
+                       const Vector &x,
+                       Vector &y)
+{
+   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D, "");
+   MFEM_VERIFY(d1d_test <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D, "");
+
+   auto Bo = Reshape(bo.Read(), q1d, d1d - 1);
+   auto Bc = Reshape(bc.Read(), q1d, d1d);
+   auto Bt = Reshape(bt.Read(), d1d_test, q1d);
+   auto O = Reshape(pa_data.Read(), 2, q1d, q1d, ne);
+   auto X = Reshape(x.Read(), 2 * (d1d - 1) * d1d, ne);
+   auto Y = Reshape(y.ReadWrite(), d1d_test, d1d_test, ne);
+
+   mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int MAX_D1D = DofQuadLimits::MAX_D1D;
+      constexpr int MAX_Q1D = DofQuadLimits::HCURL_MAX_Q1D;
+
+      real_t u0[MAX_Q1D][MAX_Q1D];
+      real_t u1[MAX_Q1D][MAX_Q1D];
+
+      for (int qy = 0; qy < q1d; ++qy)
+      {
+         for (int qx = 0; qx < q1d; ++qx)
+         {
+            u0[qy][qx] = 0.0;
+            u1[qy][qx] = 0.0;
+         }
+      }
+
+      int osc = 0;
+      for (int dy = 0; dy < d1d; ++dy)
+      {
+         real_t mass_x[MAX_Q1D];
+         for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] = 0.0; }
+         for (int dx = 0; dx < d1d - 1; ++dx)
+         {
+            const real_t t = X(dx + (dy * (d1d - 1)) + osc, e);
+            for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] += t * Bo(qx, dx); }
+         }
+         for (int qy = 0; qy < q1d; ++qy)
+         {
+            const real_t wy = Bc(qy, dy);
+            for (int qx = 0; qx < q1d; ++qx) { u0[qy][qx] += mass_x[qx] * wy; }
+         }
+      }
+
+      osc += (d1d - 1) * d1d;
+      for (int dy = 0; dy < d1d - 1; ++dy)
+      {
+         real_t mass_x[MAX_Q1D];
+         for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] = 0.0; }
+         for (int dx = 0; dx < d1d; ++dx)
+         {
+            const real_t t = X(dx + (dy * d1d) + osc, e);
+            for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] += t * Bc(qx, dx); }
+         }
+         for (int qy = 0; qy < q1d; ++qy)
+         {
+            const real_t wy = Bo(qy, dy);
+            for (int qx = 0; qx < q1d; ++qx) { u1[qy][qx] += mass_x[qx] * wy; }
+         }
+      }
+
+      for (int qy = 0; qy < q1d; ++qy)
+      {
+         real_t sol_x[MAX_D1D];
+         for (int dx = 0; dx < d1d_test; ++dx) { sol_x[dx] = 0.0; }
+         for (int qx = 0; qx < q1d; ++qx)
+         {
+            const real_t s = O(0, qx, qy, e) * u0[qy][qx]
+                             + O(1, qx, qy, e) * u1[qy][qx];
+            for (int dx = 0; dx < d1d_test; ++dx)
+            {
+               sol_x[dx] += s * Bt(dx, qx);
+            }
+         }
+         for (int dy = 0; dy < d1d_test; ++dy)
+         {
+            const real_t wy = Bt(dy, qy);
+            for (int dx = 0; dx < d1d_test; ++dx)
+            {
+               Y(dx, dy, e) += sol_x[dx] * wy;
+            }
+         }
+      }
+   });
+}
+
+void PAHcurlDotApplyTranspose2D(const int d1d,
+                                const int d1d_test,
+                                const int q1d,
+                                const int ne,
+                                const Array<real_t> &bo,
+                                const Array<real_t> &bc,
+                                const Array<real_t> &b,
+                                const Vector &pa_data,
+                                const Vector &x,
+                                Vector &y)
+{
+   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D, "");
+   MFEM_VERIFY(d1d_test <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D, "");
+
+   auto Bo = Reshape(bo.Read(), q1d, d1d - 1);
+   auto Bc = Reshape(bc.Read(), q1d, d1d);
+   auto B = Reshape(b.Read(), q1d, d1d_test);
+   auto O = Reshape(pa_data.Read(), 2, q1d, q1d, ne);
+   auto X = Reshape(x.Read(), d1d_test, d1d_test, ne);
+   auto Y = Reshape(y.ReadWrite(), 2 * (d1d - 1) * d1d, ne);
+
+   mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int MAX_D1D = DofQuadLimits::MAX_D1D;
+      constexpr int MAX_Q1D = DofQuadLimits::HCURL_MAX_Q1D;
+
+      real_t mass[MAX_Q1D][MAX_Q1D];
+      for (int qy = 0; qy < q1d; ++qy)
+      {
+         for (int qx = 0; qx < q1d; ++qx)
+         {
+            mass[qy][qx] = 0.0;
+         }
+      }
+
+      for (int dy = 0; dy < d1d_test; ++dy)
+      {
+         real_t sol_x[MAX_Q1D];
+         for (int qx = 0; qx < q1d; ++qx) { sol_x[qx] = 0.0; }
+         for (int dx = 0; dx < d1d_test; ++dx)
+         {
+            const real_t t = X(dx, dy, e);
+            for (int qx = 0; qx < q1d; ++qx) { sol_x[qx] += t * B(qx, dx); }
+         }
+         for (int qy = 0; qy < q1d; ++qy)
+         {
+            const real_t wy = B(qy, dy);
+            for (int qx = 0; qx < q1d; ++qx) { mass[qy][qx] += sol_x[qx] * wy; }
+         }
+      }
+
+      int osc = 0;
+      for (int qy = 0; qy < q1d; ++qy)
+      {
+         real_t mass_x[MAX_D1D];
+         for (int dx = 0; dx < d1d - 1; ++dx) { mass_x[dx] = 0.0; }
+         for (int qx = 0; qx < q1d; ++qx)
+         {
+            const real_t s = O(0, qx, qy, e) * mass[qy][qx];
+            for (int dx = 0; dx < d1d - 1; ++dx) { mass_x[dx] += s * Bo(qx, dx); }
+         }
+         for (int dy = 0; dy < d1d; ++dy)
+         {
+            const real_t wy = Bc(qy, dy);
+            for (int dx = 0; dx < d1d - 1; ++dx)
+            {
+               Y(dx + (dy * (d1d - 1)) + osc, e) += mass_x[dx] * wy;
+            }
+         }
+      }
+
+      osc += (d1d - 1) * d1d;
+      for (int qy = 0; qy < q1d; ++qy)
+      {
+         real_t mass_x[MAX_D1D];
+         for (int dx = 0; dx < d1d; ++dx) { mass_x[dx] = 0.0; }
+         for (int qx = 0; qx < q1d; ++qx)
+         {
+            const real_t s = O(1, qx, qy, e) * mass[qy][qx];
+            for (int dx = 0; dx < d1d; ++dx) { mass_x[dx] += s * Bc(qx, dx); }
+         }
+         for (int dy = 0; dy < d1d - 1; ++dy)
+         {
+            const real_t wy = Bo(qy, dy);
+            for (int dx = 0; dx < d1d; ++dx)
+            {
+               Y(dx + (dy * d1d) + osc, e) += mass_x[dx] * wy;
+            }
+         }
+      }
+   });
+}
+
+void PAHcurlDotApply3D(const int d1d,
+                       const int d1d_test,
+                       const int q1d,
+                       const int ne,
+                       const Array<real_t> &bo,
+                       const Array<real_t> &bc,
+                       const Array<real_t> &bt,
+                       const Vector &pa_data,
+                       const Vector &x,
+                       Vector &y)
+{
+   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D, "");
+   MFEM_VERIFY(d1d_test <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D, "");
+
+   auto Bo = Reshape(bo.Read(), q1d, d1d - 1);
+   auto Bc = Reshape(bc.Read(), q1d, d1d);
+   auto Bt = Reshape(bt.Read(), d1d_test, q1d);
+   auto O = Reshape(pa_data.Read(), 3, q1d, q1d, q1d, ne);
+   auto X = Reshape(x.Read(), 3 * (d1d - 1) * d1d * d1d, ne);
+   auto Y = Reshape(y.ReadWrite(), d1d_test, d1d_test, d1d_test, ne);
+
+   mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int MAX_D1D = DofQuadLimits::MAX_D1D;
+      constexpr int MAX_Q1D = DofQuadLimits::HCURL_MAX_Q1D;
+
+      real_t u[MAX_Q1D][MAX_Q1D][MAX_Q1D][3];
+      for (int qz = 0; qz < q1d; ++qz)
+      {
+         for (int qy = 0; qy < q1d; ++qy)
+         {
+            for (int qx = 0; qx < q1d; ++qx)
+            {
+               for (int c = 0; c < 3; ++c) { u[qz][qy][qx][c] = 0.0; }
+            }
+         }
+      }
+
+      int osc = 0;
+      for (int dz = 0; dz < d1d; ++dz)
+      {
+         real_t mass_xy[MAX_Q1D][MAX_Q1D];
+         for (int qy = 0; qy < q1d; ++qy)
+         {
+            for (int qx = 0; qx < q1d; ++qx) { mass_xy[qy][qx] = 0.0; }
+         }
+
+         for (int dy = 0; dy < d1d; ++dy)
+         {
+            real_t mass_x[MAX_Q1D];
+            for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] = 0.0; }
+            for (int dx = 0; dx < d1d - 1; ++dx)
+            {
+               const real_t t = X(dx + ((dy + (dz * d1d)) * (d1d - 1)) + osc, e);
+               for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] += t * Bo(qx, dx); }
+            }
+            for (int qy = 0; qy < q1d; ++qy)
+            {
+               const real_t wy = Bc(qy, dy);
+               for (int qx = 0; qx < q1d; ++qx) { mass_xy[qy][qx] += mass_x[qx] * wy; }
+            }
+         }
+
+         for (int qz = 0; qz < q1d; ++qz)
+         {
+            const real_t wz = Bc(qz, dz);
+            for (int qy = 0; qy < q1d; ++qy)
+            {
+               for (int qx = 0; qx < q1d; ++qx) { u[qz][qy][qx][0] += mass_xy[qy][qx] * wz; }
+            }
+         }
+      }
+
+      osc += (d1d - 1) * d1d * d1d;
+      for (int dz = 0; dz < d1d; ++dz)
+      {
+         real_t mass_xy[MAX_Q1D][MAX_Q1D];
+         for (int qy = 0; qy < q1d; ++qy)
+         {
+            for (int qx = 0; qx < q1d; ++qx) { mass_xy[qy][qx] = 0.0; }
+         }
+
+         for (int dy = 0; dy < d1d - 1; ++dy)
+         {
+            real_t mass_x[MAX_Q1D];
+            for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] = 0.0; }
+            for (int dx = 0; dx < d1d; ++dx)
+            {
+               const real_t t = X(dx + ((dy + (dz * (d1d - 1))) * d1d) + osc, e);
+               for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] += t * Bc(qx, dx); }
+            }
+            for (int qy = 0; qy < q1d; ++qy)
+            {
+               const real_t wy = Bo(qy, dy);
+               for (int qx = 0; qx < q1d; ++qx) { mass_xy[qy][qx] += mass_x[qx] * wy; }
+            }
+         }
+
+         for (int qz = 0; qz < q1d; ++qz)
+         {
+            const real_t wz = Bc(qz, dz);
+            for (int qy = 0; qy < q1d; ++qy)
+            {
+               for (int qx = 0; qx < q1d; ++qx) { u[qz][qy][qx][1] += mass_xy[qy][qx] * wz; }
+            }
+         }
+      }
+
+      osc += (d1d - 1) * d1d * d1d;
+      for (int dz = 0; dz < d1d - 1; ++dz)
+      {
+         real_t mass_xy[MAX_Q1D][MAX_Q1D];
+         for (int qy = 0; qy < q1d; ++qy)
+         {
+            for (int qx = 0; qx < q1d; ++qx) { mass_xy[qy][qx] = 0.0; }
+         }
+
+         for (int dy = 0; dy < d1d; ++dy)
+         {
+            real_t mass_x[MAX_Q1D];
+            for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] = 0.0; }
+            for (int dx = 0; dx < d1d; ++dx)
+            {
+               const real_t t = X(dx + ((dy + (dz * d1d)) * d1d) + osc, e);
+               for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] += t * Bc(qx, dx); }
+            }
+            for (int qy = 0; qy < q1d; ++qy)
+            {
+               const real_t wy = Bc(qy, dy);
+               for (int qx = 0; qx < q1d; ++qx) { mass_xy[qy][qx] += mass_x[qx] * wy; }
+            }
+         }
+
+         for (int qz = 0; qz < q1d; ++qz)
+         {
+            const real_t wz = Bo(qz, dz);
+            for (int qy = 0; qy < q1d; ++qy)
+            {
+               for (int qx = 0; qx < q1d; ++qx) { u[qz][qy][qx][2] += mass_xy[qy][qx] * wz; }
+            }
+         }
+      }
+
+      for (int qz = 0; qz < q1d; ++qz)
+      {
+         real_t mass_xy[MAX_D1D][MAX_D1D];
+         for (int dy = 0; dy < d1d_test; ++dy)
+         {
+            for (int dx = 0; dx < d1d_test; ++dx) { mass_xy[dy][dx] = 0.0; }
+         }
+
+         for (int qy = 0; qy < q1d; ++qy)
+         {
+            real_t mass_x[MAX_D1D];
+            for (int dx = 0; dx < d1d_test; ++dx) { mass_x[dx] = 0.0; }
+            for (int qx = 0; qx < q1d; ++qx)
+            {
+               const real_t s = O(0, qx, qy, qz, e) * u[qz][qy][qx][0]
+                                + O(1, qx, qy, qz, e) * u[qz][qy][qx][1]
+                                + O(2, qx, qy, qz, e) * u[qz][qy][qx][2];
+               for (int dx = 0; dx < d1d_test; ++dx) { mass_x[dx] += s * Bt(dx, qx); }
+            }
+            for (int dy = 0; dy < d1d_test; ++dy)
+            {
+               const real_t wy = Bt(dy, qy);
+               for (int dx = 0; dx < d1d_test; ++dx) { mass_xy[dy][dx] += mass_x[dx] * wy; }
+            }
+         }
+
+         for (int dz = 0; dz < d1d_test; ++dz)
+         {
+            const real_t wz = Bt(dz, qz);
+            for (int dy = 0; dy < d1d_test; ++dy)
+            {
+               for (int dx = 0; dx < d1d_test; ++dx)
+               {
+                  Y(dx, dy, dz, e) += mass_xy[dy][dx] * wz;
+               }
+            }
+         }
+      }
+   });
+}
+
+void PAHcurlDotApplyTranspose3D(const int d1d,
+                                const int d1d_test,
+                                const int q1d,
+                                const int ne,
+                                const Array<real_t> &bo,
+                                const Array<real_t> &bc,
+                                const Array<real_t> &b,
+                                const Vector &pa_data,
+                                const Vector &x,
+                                Vector &y)
+{
+   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D, "");
+   MFEM_VERIFY(d1d_test <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D, "");
+
+   auto Bo = Reshape(bo.Read(), q1d, d1d - 1);
+   auto Bc = Reshape(bc.Read(), q1d, d1d);
+   auto B = Reshape(b.Read(), q1d, d1d_test);
+   auto O = Reshape(pa_data.Read(), 3, q1d, q1d, q1d, ne);
+   auto X = Reshape(x.Read(), d1d_test, d1d_test, d1d_test, ne);
+   auto Y = Reshape(y.ReadWrite(), 3 * (d1d - 1) * d1d * d1d, ne);
+
+   mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int MAX_D1D = DofQuadLimits::MAX_D1D;
+      constexpr int MAX_Q1D = DofQuadLimits::HCURL_MAX_Q1D;
+
+      real_t mass[MAX_Q1D][MAX_Q1D][MAX_Q1D];
+      for (int qz = 0; qz < q1d; ++qz)
+      {
+         for (int qy = 0; qy < q1d; ++qy)
+         {
+            for (int qx = 0; qx < q1d; ++qx) { mass[qz][qy][qx] = 0.0; }
+         }
+      }
+
+      for (int dz = 0; dz < d1d_test; ++dz)
+      {
+         real_t mass_xy[MAX_Q1D][MAX_Q1D];
+         for (int qy = 0; qy < q1d; ++qy)
+         {
+            for (int qx = 0; qx < q1d; ++qx) { mass_xy[qy][qx] = 0.0; }
+         }
+
+         for (int dy = 0; dy < d1d_test; ++dy)
+         {
+            real_t mass_x[MAX_Q1D];
+            for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] = 0.0; }
+            for (int dx = 0; dx < d1d_test; ++dx)
+            {
+               const real_t t = X(dx, dy, dz, e);
+               for (int qx = 0; qx < q1d; ++qx) { mass_x[qx] += t * B(qx, dx); }
+            }
+            for (int qy = 0; qy < q1d; ++qy)
+            {
+               const real_t wy = B(qy, dy);
+               for (int qx = 0; qx < q1d; ++qx) { mass_xy[qy][qx] += mass_x[qx] * wy; }
+            }
+         }
+
+         for (int qz = 0; qz < q1d; ++qz)
+         {
+            const real_t wz = B(qz, dz);
+            for (int qy = 0; qy < q1d; ++qy)
+            {
+               for (int qx = 0; qx < q1d; ++qx) { mass[qz][qy][qx] += mass_xy[qy][qx] * wz; }
+            }
+         }
+      }
+
+      int osc = 0;
+      for (int qz = 0; qz < q1d; ++qz)
+      {
+         real_t mass_xy[MAX_D1D][MAX_D1D];
+         for (int dy = 0; dy < d1d; ++dy)
+         {
+            for (int dx = 0; dx < d1d - 1; ++dx) { mass_xy[dy][dx] = 0.0; }
+         }
+
+         for (int qy = 0; qy < q1d; ++qy)
+         {
+            real_t mass_x[MAX_D1D];
+            for (int dx = 0; dx < d1d - 1; ++dx) { mass_x[dx] = 0.0; }
+            for (int qx = 0; qx < q1d; ++qx)
+            {
+               const real_t s = O(0, qx, qy, qz, e) * mass[qz][qy][qx];
+               for (int dx = 0; dx < d1d - 1; ++dx) { mass_x[dx] += s * Bo(qx, dx); }
+            }
+            for (int dy = 0; dy < d1d; ++dy)
+            {
+               const real_t wy = Bc(qy, dy);
+               for (int dx = 0; dx < d1d - 1; ++dx) { mass_xy[dy][dx] += mass_x[dx] * wy; }
+            }
+         }
+
+         for (int dz = 0; dz < d1d; ++dz)
+         {
+            const real_t wz = Bc(qz, dz);
+            for (int dy = 0; dy < d1d; ++dy)
+            {
+               for (int dx = 0; dx < d1d - 1; ++dx)
+               {
+                  Y(dx + ((dy + (dz * d1d)) * (d1d - 1)) + osc, e) += mass_xy[dy][dx] * wz;
+               }
+            }
+         }
+      }
+
+      osc += (d1d - 1) * d1d * d1d;
+      for (int qz = 0; qz < q1d; ++qz)
+      {
+         real_t mass_xy[MAX_D1D][MAX_D1D];
+         for (int dy = 0; dy < d1d - 1; ++dy)
+         {
+            for (int dx = 0; dx < d1d; ++dx) { mass_xy[dy][dx] = 0.0; }
+         }
+
+         for (int qy = 0; qy < q1d; ++qy)
+         {
+            real_t mass_x[MAX_D1D];
+            for (int dx = 0; dx < d1d; ++dx) { mass_x[dx] = 0.0; }
+            for (int qx = 0; qx < q1d; ++qx)
+            {
+               const real_t s = O(1, qx, qy, qz, e) * mass[qz][qy][qx];
+               for (int dx = 0; dx < d1d; ++dx) { mass_x[dx] += s * Bc(qx, dx); }
+            }
+            for (int dy = 0; dy < d1d - 1; ++dy)
+            {
+               const real_t wy = Bo(qy, dy);
+               for (int dx = 0; dx < d1d; ++dx) { mass_xy[dy][dx] += mass_x[dx] * wy; }
+            }
+         }
+
+         for (int dz = 0; dz < d1d; ++dz)
+         {
+            const real_t wz = Bc(qz, dz);
+            for (int dy = 0; dy < d1d - 1; ++dy)
+            {
+               for (int dx = 0; dx < d1d; ++dx)
+               {
+                  Y(dx + ((dy + (dz * (d1d - 1))) * d1d) + osc, e) += mass_xy[dy][dx] * wz;
+               }
+            }
+         }
+      }
+
+      osc += (d1d - 1) * d1d * d1d;
+      for (int qz = 0; qz < q1d; ++qz)
+      {
+         real_t mass_xy[MAX_D1D][MAX_D1D];
+         for (int dy = 0; dy < d1d; ++dy)
+         {
+            for (int dx = 0; dx < d1d; ++dx) { mass_xy[dy][dx] = 0.0; }
+         }
+
+         for (int qy = 0; qy < q1d; ++qy)
+         {
+            real_t mass_x[MAX_D1D];
+            for (int dx = 0; dx < d1d; ++dx) { mass_x[dx] = 0.0; }
+            for (int qx = 0; qx < q1d; ++qx)
+            {
+               const real_t s = O(2, qx, qy, qz, e) * mass[qz][qy][qx];
+               for (int dx = 0; dx < d1d; ++dx) { mass_x[dx] += s * Bc(qx, dx); }
+            }
+            for (int dy = 0; dy < d1d; ++dy)
+            {
+               const real_t wy = Bc(qy, dy);
+               for (int dx = 0; dx < d1d; ++dx) { mass_xy[dy][dx] += mass_x[dx] * wy; }
+            }
+         }
+
+         for (int dz = 0; dz < d1d - 1; ++dz)
+         {
+            const real_t wz = Bo(qz, dz);
+            for (int dy = 0; dy < d1d; ++dy)
+            {
+               for (int dx = 0; dx < d1d; ++dx)
+               {
+                  Y(dx + ((dy + (dz * d1d)) * d1d) + osc, e) += mass_xy[dy][dx] * wz;
+               }
+            }
+         }
+      }
+   });
+}
+
+} // namespace
+
+void MixedDotProductIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
+                                           const FiniteElementSpace &test_fes)
+{
+   Mesh *mesh = trial_fes.GetMesh();
+   const FiniteElement *trial_fel = trial_fes.GetTypicalFE();
+   const FiniteElement *test_fel = test_fes.GetTypicalFE();
+
+   const VectorTensorFiniteElement *trial_el =
+      dynamic_cast<const VectorTensorFiniteElement*>(trial_fel);
+   MFEM_VERIFY(trial_el != NULL, "Only VectorTensorFiniteElement is supported!");
+
+   const TensorBasisElement *test_tensor_el =
+      dynamic_cast<const TensorBasisElement*>(test_fel);
+   MFEM_VERIFY(test_tensor_el != NULL,
+               "Only tensor-product scalar test elements are supported!");
+
+   MFEM_VERIFY(trial_el->GetDerivType() == mfem::FiniteElement::CURL,
+               "Only H(curl) trial spaces are supported!");
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == nullptr)
+   {
+      const int order = trial_fel->GetOrder() + test_fel->GetOrder()
+                        + mesh->GetTypicalElementTransformation()->OrderW();
+      ir = &IntRules.Get(trial_fel->GetGeomType(), order);
+   }
+
+   dim = mesh->Dimension();
+   MFEM_VERIFY(dim == 2 || dim == 3, "Unsupported dimension!");
+   MFEM_VERIFY(trial_el->GetDim() == dim && test_fel->GetDim() == dim,
+               "Trial/test dimension mismatch.");
+
+   ne = trial_fes.GetNE();
+   MFEM_VERIFY(ne == test_fes.GetNE(),
+               "Different meshes for test and trial spaces");
+
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
+   mapsC = &trial_el->GetDofToQuad(*ir, DofToQuad::TENSOR);
+   mapsO = &trial_el->GetDofToQuadOpen(*ir, DofToQuad::TENSOR);
+   mapsTest = &test_fel->GetDofToQuad(*ir, DofToQuad::TENSOR);
+
+   dofs1D = mapsC->ndof;
+   dofs1Dtest = mapsTest->ndof;
+   quad1D = mapsC->nqpt;
+   test_map_integral = (test_fel->GetMapType() == FiniteElement::INTEGRAL);
+
+   MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");
+   MFEM_VERIFY(quad1D == mapsTest->nqpt, "Trial/test quadrature mismatch");
+   MFEM_VERIFY(dofs1D <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D, "");
+   MFEM_VERIFY(dofs1Dtest <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(quad1D <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D, "");
+
+   const int nq = ir->GetNPoints();
+   if (dim == 2) { MFEM_VERIFY(nq == quad1D * quad1D, ""); }
+   else { MFEM_VERIFY(nq == quad1D * quad1D * quad1D, ""); }
+
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(*VQ, qs, CoefficientStorage::FULL);
+   MFEM_VERIFY(coeff.GetVDim() == dim, "Vector coefficient dimension mismatch.");
+
+   pa_data.SetSize(dim * nq * ne, Device::GetMemoryType());
+
+   if (dim == 2)
+   {
+      PAHcurlDotSetup2D(quad1D, ne, test_map_integral, ir->GetWeights(),
+                        geom->J, coeff, pa_data);
+   }
+   else
+   {
+      PAHcurlDotSetup3D(quad1D, ne, test_map_integral, ir->GetWeights(),
+                        geom->J, coeff, pa_data);
+   }
+}
+
+void MixedDotProductIntegrator::AddMultPA(const Vector &x, Vector &y) const
+{
+   if (dim == 2)
+   {
+      PAHcurlDotApply2D(dofs1D, dofs1Dtest, quad1D, ne,
+                        mapsO->B, mapsC->B, mapsTest->Bt, pa_data, x, y);
+   }
+   else if (dim == 3)
+   {
+      PAHcurlDotApply3D(dofs1D, dofs1Dtest, quad1D, ne,
+                        mapsO->B, mapsC->B, mapsTest->Bt, pa_data, x, y);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension!");
+   }
+}
+
+void MixedDotProductIntegrator::AddMultTransposePA(const Vector &x,
+                                                   Vector &y) const
+{
+   if (dim == 2)
+   {
+      PAHcurlDotApplyTranspose2D(dofs1D, dofs1Dtest, quad1D, ne,
+                                 mapsO->B, mapsC->B, mapsTest->B,
+                                 pa_data, x, y);
+   }
+   else if (dim == 3)
+   {
+      PAHcurlDotApplyTranspose3D(dofs1D, dofs1Dtest, quad1D, ne,
+                                 mapsO->B, mapsC->B, mapsTest->B,
+                                 pa_data, x, y);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension!");
+   }
+}
+
 void MixedScalarCurlIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
                                            const FiniteElementSpace &test_fes)
 {
@@ -320,6 +1093,7 @@ void MixedVectorWeakCurlIntegrator::AssemblePA(const FiniteElementSpace
    CoefficientVector coeff(qs, CoefficientStorage::FULL);
    if (Q) { coeff.Project(*Q); }
    else if (DQ) { coeff.Project(*DQ); }
+   else if (MQ) { MFEM_ABORT("Not implemented."); }
    else { coeff.SetConstant(1.0); }
 
    if (trialType == mfem::FiniteElement::CURL && dim == 3)

fem/integ/bilininteg_vecdiv_pa.hpp

@@ -1,365 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../config/config.hpp"
-#include "../../general/array.hpp"
-#include "../../general/forall.hpp"
-#include "../../linalg/dtensor.hpp"
-#include "../../linalg/vector.hpp"
-#include "../bilininteg.hpp"
-#include "../kernels.hpp"
-
-namespace mfem
-{
-
-/// \cond DO_NOT_DOCUMENT
-
-namespace internal
-{
-
-// Shared memory PA Divergence Apply 2D kernel
-template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
-inline void SmemPADivergenceApply2D(const int NE,
-                                    const Array<real_t> &b_,
-                                    const Array<real_t> &g_,
-                                    const Array<real_t> &bt_,
-                                    const Vector &q_,
-                                    const Vector &x_,
-                                    Vector &y_,
-                                    const int tr_d1d = 0,
-                                    const int te_d1d = 0,
-                                    const int q1d = 0)
-{
-   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
-   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   const auto B = b_.Read(), G = g_.Read(), Bt = bt_.Read();
-   const auto Q = Reshape(q_.Read(), Q1D, Q1D, 2, 2, NE);
-   const auto X = Reshape(x_.Read(), TR_D1D, TR_D1D, 2, NE);
-   auto Y = Reshape(y_.ReadWrite(), TE_D1D, TE_D1D, 1, NE);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
-
-      kernels::internal::vd_regs2d_t<2, 2, MQ1> g0, g1;
-      kernels::internal::v_regs2d_t<1, MQ1> r0, r1;
-
-      kernels::internal::LoadMatrix(TR_D1D, Q1D, B, sB);
-      kernels::internal::LoadMatrix(TR_D1D, Q1D, G, sG);
-
-      kernels::internal::LoadDofs2d(e, TR_D1D, X, g0);
-      kernels::internal::Grad2d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
-
-      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            r0[0][qy][qx] =
-               g1[0][0][qy][qx] * Q(qx, qy, 0, 0, e) +
-               g1[0][1][qy][qx] * Q(qx, qy, 1, 0, e) +
-               g1[1][0][qy][qx] * Q(qx, qy, 0, 1, e) +
-               g1[1][1][qy][qx] * Q(qx, qy, 1, 1, e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      kernels::internal::LoadMatrix<MQ1,true>(TE_D1D, Q1D, Bt, sB);
-      kernels::internal::EvalTranspose2d(TE_D1D, Q1D, smem, sB, r0, r1);
-      kernels::internal::WriteDofs2d(e, TE_D1D, r1, Y);
-   });
-}
-
-// Shared memory PA Divergence Apply 2D kernel transpose
-template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
-inline void SmemPADivergenceApplyTranspose2D(const int NE,
-                                             const Array<real_t> &bt,
-                                             const Array<real_t> &gt,
-                                             const Array<real_t> &b,
-                                             const Vector &q_,
-                                             const Vector &x_,
-                                             Vector &y_,
-                                             const int tr_d1d = 0,
-                                             const int te_d1d = 0,
-                                             const int q1d = 0)
-{
-   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
-   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   const auto Bt = bt.Read(), Gt = gt.Read(), B = b.Read();
-   const auto Q = Reshape(q_.Read(), Q1D, Q1D, 2, 2, NE);
-   const auto X  = Reshape(x_.Read(), TE_D1D, TE_D1D, 1, NE);
-   auto Y  = Reshape(y_.ReadWrite(), TR_D1D, TR_D1D, 2, NE);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
-
-      kernels::internal::v_regs2d_t<1, MQ1> r0, r1;
-      kernels::internal::vd_regs2d_t<2, 2, MQ1> g0, g1;
-
-      kernels::internal::LoadMatrix(TE_D1D, Q1D, B, sB);
-      kernels::internal::LoadDofs2d(e, TE_D1D, X, r0);
-      kernels::internal::Eval2d(TE_D1D, Q1D, smem, sB, r0, r1);
-
-      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            g0[0][0][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 0, 0, e);
-            g0[0][1][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 1, 0, e);
-            g0[1][0][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 0, 1, e);
-            g0[1][1][qy][qx] = r1[0][qy][qx] * Q(qx, qy, 1, 1, e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Bt, sB);
-      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Gt, sG);
-      kernels::internal::GradTranspose2d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
-      kernels::internal::WriteDofs2d(e, TR_D1D, g1, Y);
-   });
-}
-
-// Shared memory PA Divergence Apply 3D kernel transpose
-template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
-inline void SmemPADivergenceApplyTranspose3D(const int NE,
-                                             const Array<real_t> &bt,
-                                             const Array<real_t> &gt,
-                                             const Array<real_t> &b,
-                                             const Vector &q_,
-                                             const Vector &x_,
-                                             Vector &y_,
-                                             int tr_d1d = 0,
-                                             int te_d1d = 0,
-                                             int q1d = 0)
-{
-   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
-   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   const auto Bt = bt.Read(), Gt = gt.Read(), B = b.Read();
-   const auto Q = Reshape(q_.Read(), Q1D, Q1D, Q1D, 3, 3, NE);
-   const auto X = Reshape(x_.Read(), TE_D1D, TE_D1D, TE_D1D, 1, NE);
-   auto Y = Reshape(y_.ReadWrite(), TR_D1D, TR_D1D, TR_D1D, 3, NE);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
-
-      kernels::internal::v_regs3d_t<1, MQ1> r0, r1;
-      kernels::internal::vd_regs3d_t<3, 3, MQ1> g0, g1;
-
-      kernels::internal::LoadMatrix(TE_D1D, Q1D, B, sB);
-      kernels::internal::LoadDofs3d(e, TE_D1D, X, r0);
-      kernels::internal::Eval3d(TE_D1D, Q1D, smem, sB, r0, r1);
-
-      for (int qz = 0; qz < Q1D; qz++)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-            {
-               const auto r = r1[0][qz][qy][qx];
-               g0[0][0][qz][qy][qx] = r * Q(qx, qy, qz, 0, 0, e);
-               g0[0][1][qz][qy][qx] = r * Q(qx, qy, qz, 1, 0, e);
-               g0[0][2][qz][qy][qx] = r * Q(qx, qy, qz, 2, 0, e);
-
-               g0[1][0][qz][qy][qx] = r * Q(qx, qy, qz, 0, 1, e);
-               g0[1][1][qz][qy][qx] = r * Q(qx, qy, qz, 1, 1, e);
-               g0[1][2][qz][qy][qx] = r * Q(qx, qy, qz, 2, 1, e);
-
-               g0[2][0][qz][qy][qx] = r * Q(qx, qy, qz, 0, 2, e);
-               g0[2][1][qz][qy][qx] = r * Q(qx, qy, qz, 1, 2, e);
-               g0[2][2][qz][qy][qx] = r * Q(qx, qy, qz, 2, 2, e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Bt, sB);
-      kernels::internal::LoadMatrix<MQ1,true>(TR_D1D, Q1D, Gt, sG);
-      kernels::internal::GradTranspose3d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
-      kernels::internal::WriteDofs3d(e, TR_D1D, g1, Y);
-   });
-}
-
-// Shared memory PA Divergence Apply 3D kernel
-template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
-inline void SmemPADivergenceApply3D(const int NE,
-                                    const Array<real_t> &b_,
-                                    const Array<real_t> &g_,
-                                    const Array<real_t> &bt_,
-                                    const Vector &q_,
-                                    const Vector &x_,
-                                    Vector &y_,
-                                    const int tr_d1d = 0,
-                                    const int te_d1d = 0,
-                                    const int q1d = 0)
-{
-   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
-   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   const auto B = b_.Read(), G = g_.Read(), Bt = bt_.Read();
-   const auto Q = Reshape(q_.Read(), Q1D, Q1D, Q1D, 3,3, NE);
-   const auto X = Reshape(x_.Read(), TR_D1D, TR_D1D, TR_D1D, 3, NE);
-   auto Y = Reshape(y_.ReadWrite(), TE_D1D, TE_D1D, TE_D1D, 1, NE);
-
-   mfem::forall_2D<T_Q1D*T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MQ1][MQ1], sG[MQ1][MQ1];
-
-      kernels::internal::vd_regs3d_t<3, 3, MQ1> g0, g1;
-      kernels::internal::v_regs3d_t<1, MQ1> r0, r1;
-
-      kernels::internal::LoadMatrix(TR_D1D, Q1D, B, sB);
-      kernels::internal::LoadMatrix(TR_D1D, Q1D, G, sG);
-
-      kernels::internal::LoadDofs3d(e, TR_D1D, X, g0);
-      kernels::internal::Grad3d(TR_D1D, Q1D, smem, sB, sG, g0, g1);
-
-      for (int qz = 0; qz < Q1D; qz++)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-            {
-               r0[0][qz][qy][qx] =
-                  // c = 0
-                  g1[0][0][qz][qy][qx] * Q(qx, qy, qz, 0, 0, e) +
-                  g1[0][1][qz][qy][qx] * Q(qx, qy, qz, 1, 0, e) +
-                  g1[0][2][qz][qy][qx] * Q(qx, qy, qz, 2, 0, e) +
-                  // c = 1
-                  g1[1][0][qz][qy][qx] * Q(qx, qy, qz, 0, 1, e) +
-                  g1[1][1][qz][qy][qx] * Q(qx, qy, qz, 1, 1, e) +
-                  g1[1][2][qz][qy][qx] * Q(qx, qy, qz, 2, 1, e) +
-                  // c = 2
-                  g1[2][0][qz][qy][qx] * Q(qx, qy, qz, 0, 2, e) +
-                  g1[2][1][qz][qy][qx] * Q(qx, qy, qz, 1, 2, e) +
-                  g1[2][2][qz][qy][qx] * Q(qx, qy, qz, 2, 2, e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      kernels::internal::LoadMatrix<MQ1, true>(TE_D1D, Q1D, Bt, sB);
-      kernels::internal::EvalTranspose3d(TE_D1D, Q1D, smem, sB, r0, r1);
-      kernels::internal::WriteDofs3d(e, TE_D1D, r1, Y);
-   });
-}
-
-} // namespace internal
-
-template<int DIM, int T_TR_D1D, int T_TE_D1D, int T_Q1D>
-VectorDivergenceIntegrator::VectorDivergenceAddMultPAType
-VectorDivergenceIntegrator::VectorDivergenceAddMultPA::Kernel()
-{
-   static_assert(T_TR_D1D <= T_Q1D && T_TE_D1D <= T_Q1D);
-   if constexpr (DIM == 2)
-   {
-      return internal::SmemPADivergenceApply2D<T_TR_D1D, T_TE_D1D, T_Q1D>;
-   }
-   else if constexpr (DIM == 3)
-   {
-      return internal::SmemPADivergenceApply3D<T_TR_D1D, T_TE_D1D, T_Q1D>;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-inline VectorDivergenceIntegrator::VectorDivergenceAddMultPAType
-VectorDivergenceIntegrator::VectorDivergenceAddMultPA::Fallback
-(int dim, int tr_d1d, int te_d1d, int q1d)
-{
-   MFEM_VERIFY(tr_d1d <= q1d && te_d1d <= q1d, "");
-   MFEM_VERIFY(tr_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(te_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   if (dim == 2)
-   {
-      return internal::SmemPADivergenceApply2D;
-   }
-   else if (dim == 3)
-   {
-      return internal::SmemPADivergenceApply3D;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-template<int DIM, int T_TR_D1D, int T_TE_D1D, int T_Q1D>
-VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePAType
-VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePA::Kernel()
-{
-   static_assert(T_TR_D1D <= T_Q1D && T_TE_D1D <= T_Q1D);
-   if constexpr (DIM == 2)
-   {
-      return internal::SmemPADivergenceApplyTranspose2D<T_TR_D1D, T_TE_D1D, T_Q1D>;
-   }
-   else if constexpr (DIM == 3)
-   {
-      return internal::SmemPADivergenceApplyTranspose3D<T_TR_D1D, T_TE_D1D, T_Q1D>;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-inline VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePAType
-VectorDivergenceIntegrator::VectorDivergenceAddMultTransposePA::Fallback
-(int dim, int tr_d1d, int te_d1d, int q1d)
-{
-   MFEM_VERIFY(tr_d1d <= q1d && te_d1d <= q1d, "");
-   MFEM_VERIFY(tr_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(te_d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   if (dim == 2)
-   {
-      return internal::SmemPADivergenceApplyTranspose2D;
-   }
-   else if (dim == 3)
-   {
-      return internal::SmemPADivergenceApplyTranspose3D;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
-} // namespace mfem

fem/integ/bilininteg_vecmass_pa.cpp

@@ -205,40 +205,157 @@ void VectorMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 
 }
 
+template <const int T_D1D = 0, const int T_Q1D = 0>
+static void PAVectorMassAssembleDiagonal2D(const int NE,
+                                           const Array<real_t> &b,
+                                           const Vector &pa_data, Vector &diag,
+                                           const int d1d = 0, const int q1d = 0)
+{
+   constexpr int VDIM = 2;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   const auto B = Reshape(b.Read(), Q1D, D1D);
+   const auto D = Reshape(pa_data.Read(), Q1D, Q1D, NE);
+   auto Y = Reshape(diag.ReadWrite(), D1D, D1D, VDIM, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      real_t temp[max_Q1D][max_D1D];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            temp[qx][dy] = 0.0;
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               temp[qx][dy] += B(qy, dy) * B(qy, dy) * D(qx, qy, e);
+            }
+         }
+      }
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            real_t temp1 = 0.0;
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               temp1 += B(qx, dx) * B(qx, dx) * temp[qx][dy];
+            }
+            Y(dx, dy, 0, e) = temp1;
+            Y(dx, dy, 1, e) = temp1;
+         }
+      }
+   });
+}
+
+template <const int T_D1D = 0, const int T_Q1D = 0>
+static void PAVectorMassAssembleDiagonal3D(const int NE,
+                                           const Array<real_t> &B_,
+                                           const Vector &pa_data, Vector &diag,
+                                           const int d1d = 0, const int q1d = 0)
+{
+   constexpr int VDIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   const auto B = Reshape(B_.Read(), Q1D, D1D);
+   MFEM_VERIFY(pa_data.Size() == Q1D * Q1D * Q1D * NE, "pa_data size error");
+   const auto D = Reshape(pa_data.Read(), Q1D, Q1D, Q1D, NE);
+   auto Y = Reshape(diag.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      // the following variables are evaluated at compile time
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      real_t temp[max_Q1D][max_Q1D][max_D1D];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               temp[qx][qy][dz] = 0.0;
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  temp[qx][qy][dz] +=
+                     B(qz, dz) * B(qz, dz) * D(qx, qy, qz, e);
+               }
+            }
+         }
+      }
+      real_t temp2[max_Q1D][max_D1D][max_D1D];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               temp2[qx][dy][dz] = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  temp2[qx][dy][dz] +=
+                     B(qy, dy) * B(qy, dy) * temp[qx][qy][dz];
+               }
+            }
+         }
+      }
+      for (int dz = 0; dz < D1D; ++dz)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               real_t temp3 = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  temp3 += B(qx, dx) * B(qx, dx) * temp2[qx][dy][dz];
+               }
+               Y(dx, dy, dz, 0, e) = temp3;
+               Y(dx, dy, dz, 1, e) = temp3;
+               Y(dx, dy, dz, 2, e) = temp3;
+            }
+         }
+      }
+   });
+}
+
+static void PAVectorMassAssembleDiagonal(const int dim, const int D1D,
+                                         const int Q1D, const int NE,
+                                         const Array<real_t> &B,
+                                         const Vector &pa_data,
+                                         Vector &diag)
+{
+   if (dim == 2)
+   {
+      return PAVectorMassAssembleDiagonal2D(NE, B, pa_data, diag, D1D, Q1D);
+   }
+   else if (dim == 3)
+   {
+      return PAVectorMassAssembleDiagonal3D(NE, B, pa_data, diag, D1D, Q1D);
+   }
+   MFEM_ABORT("Dimension not implemented.");
+}
+
 void VectorMassIntegrator::AssembleDiagonalPA(Vector &diag)
 {
-   if (DeviceCanUseCeed()) { return ceedOp->GetDiagonal(diag); }
-
-   MFEM_VERIFY(coeff_vdim == 1, "coeff_vdim != 1");
-   MFEM_VERIFY(!VQ && !MQ, "VQ and MQ not supported");
-
-   // Add the VectorMassAssembleDiagonalPA specializations
-   static const auto vector_mass_assemble_diagonal_kernel_specializations =
-      ( // 2D
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 2>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 3>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 4>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 5>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 6>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 7>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<2, 8>::Add(),
-         // 3D
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 2>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 3>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 4>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 5>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 6>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 7>::Add(),
-         VectorMassIntegrator::VectorMassAssembleDiagonalPA::Specialization<3, 8>::Add(),
-         true);
-   MFEM_CONTRACT_VAR(vector_mass_assemble_diagonal_kernel_specializations);
-
-   VectorMassAssembleDiagonalPA::Run(dim, quad1D, // templated arguments
-                                     ne, dofs1D, quad1D,
-                                     maps->B.Read(),
-                                     pa_data.Read(),
-                                     diag.ReadWrite());
-
+   if (DeviceCanUseCeed()) { ceedOp->GetDiagonal(diag); }
+   else
+   {
+      MFEM_VERIFY(coeff_vdim == 1, "coeff_vdim != 1");
+      MFEM_VERIFY(!VQ && !MQ, "VQ and MQ not supported");
+      PAVectorMassAssembleDiagonal(dim, dofs1D, quad1D, ne, maps->B, pa_data, diag);
+   }
 }
 
 } // namespace mfem

fem/integ/bilininteg_vecmass_pa.hpp

@@ -176,146 +176,8 @@ void SmemPAVectorMassApply3D(const int NE,
    });
 }
 
-template <int T_Q1D = 0, int T_MDQ = 16>
-static void SmemPAVectorMassAssembleDiagonal2D(const int ne,
-                                               const int d1d,
-                                               const int q1d,
-                                               const real_t *b_r,
-                                               const real_t *d_r,
-                                               real_t *y_rw)
-{
-   constexpr int VDIM = 2;
-
-   const int D1D = d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(Q1D <= T_MDQ && D1D <= Q1D, "");
-
-   const auto B = Reshape(b_r, Q1D, D1D);
-   const auto D = Reshape(d_r, Q1D, Q1D, ne);
-   auto Y = Reshape(y_rw, D1D, D1D, VDIM, ne);
-
-   mfem::forall_2D<T_Q1D*T_Q1D>(
-      ne, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MDQ;
-
-      MFEM_SHARED real_t sm[MQ1][MQ1];
-
-      MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-         {
-            real_t u = 0.0;
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               u += B(qy, dy) * B(qy, dy) * D(qx, qy, e);
-            }
-            sm[qx][dy] = u;
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
-         {
-            real_t u = 0.0;
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               u += B(qx, dx) * B(qx, dx) * sm[qx][dy];
-            }
-            Y(dx, dy, 0, e) += u;
-            Y(dx, dy, 1, e) += u;
-         }
-      }
-   });
-}
-
-// T_MDQ <= 10 so the Q1D^3 thread block stays within the 1024/block GPU limit
-template <int T_Q1D = 0, int T_MDQ = 10>
-static void SmemPAVectorMassAssembleDiagonal3D(const int ne,
-                                               const int d1d,
-                                               const int q1d,
-                                               const real_t *b_r,
-                                               const real_t *d_r,
-                                               real_t *y_rw)
-{
-   constexpr int VDIM = 3;
-
-   const int D1D = d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(Q1D <= T_MDQ && D1D <= Q1D, "");
-
-   const auto B = Reshape(b_r, Q1D, D1D);
-   const auto D = Reshape(d_r, Q1D, Q1D, Q1D, ne);
-   auto Y = Reshape(y_rw, D1D, D1D, D1D, VDIM, ne);
-
-   mfem::forall_3D<T_Q1D*T_Q1D*T_Q1D>(
-      ne, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MDQ;
-
-      MFEM_SHARED real_t sm[2][MQ1][MQ1][MQ1];
-
-      MFEM_FOREACH_THREAD_DIRECT(dz, z, D1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-            {
-               real_t u = 0.0;
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u += B(qz, dz) * B(qz, dz) * D(qx, qy, qz, e);
-               }
-               sm[0][dz][qy][qx] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD_DIRECT(dz, z, D1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-            {
-               real_t u = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  u += B(qy, dy) * B(qy, dy) * sm[0][dz][qy][qx];
-               }
-               sm[1][dz][dy][qx] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD_DIRECT(dz, z, D1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
-            {
-               real_t u = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  u += B(qx, dx) * B(qx, dx) * sm[1][dz][dy][qx];
-               }
-               Y(dx, dy, dz, 0, e) += u;
-               Y(dx, dy, dz, 1, e) += u;
-               Y(dx, dy, dz, 2, e) += u;
-            }
-         }
-      }
-   });
-}
-
 } // namespace internal
 
-// AddMultPA kernels
 template<int DIM, int T_D1D, int T_Q1D>
 VectorMassIntegrator::VectorMassAddMultPAType
 VectorMassIntegrator::VectorMassAddMultPA::Kernel()
@@ -332,7 +194,7 @@ VectorMassIntegrator::VectorMassAddMultPA::Kernel()
 }
 
 inline VectorMassIntegrator::VectorMassAddMultPAType
-VectorMassIntegrator::VectorMassAddMultPA::Fallback(int dim, int, int)
+VectorMassIntegrator::VectorMassAddMultPA::Fallback(int dim, int d1d, int q1d)
 {
    if (dim == 2)
    {
@@ -342,37 +204,7 @@ VectorMassIntegrator::VectorMassAddMultPA::Fallback(int dim, int, int)
    {
       return internal::SmemPAVectorMassApply3D;
    }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-// DiagonalPA kernels
-template<int DIM, int T_Q1D>
-VectorMassIntegrator::VectorMassAssembleDiagonalPAType
-VectorMassIntegrator::VectorMassAssembleDiagonalPA::Kernel()
-{
-   if constexpr (DIM == 2)
-   {
-      return internal::SmemPAVectorMassAssembleDiagonal2D<T_Q1D>;
-   }
-   else if constexpr (DIM == 3)
-   {
-      return internal::SmemPAVectorMassAssembleDiagonal3D<T_Q1D>;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-inline VectorMassIntegrator::VectorMassAssembleDiagonalPAType
-VectorMassIntegrator::VectorMassAssembleDiagonalPA::Fallback(int dim, int)
-{
-   if (dim == 2)
-   {
-      return internal::SmemPAVectorMassAssembleDiagonal2D;
-   }
-   else if (dim == 3)
-   {
-      return internal::SmemPAVectorMassAssembleDiagonal3D;
-   }
-   MFEM_ABORT("Unsupported kernel");
+   else { MFEM_ABORT("Unsupported kernel"); }
 }
 
 /// \endcond DO_NOT_DOCUMENT

fem/integ/bilininteg_vectorfemass_kernels.hpp

@@ -13,6 +13,10 @@
 #define MFEM_BILININTEG_VECTORFEMASS_KERNELS_HPP
 
 #include "../../config/config.hpp"
+#include "../../general/array.hpp"
+#include "../../general/forall.hpp"
+#include "../../linalg/dtensor.hpp"
+#include "../../linalg/vector.hpp"
 #include "../bilininteg.hpp"
 
 #include "bilininteg_diffusion_kernels.hpp"
@@ -73,8 +77,15 @@ VectorFEMassIntegrator::ApplyPAKernels::Kernel()
       }
       else if constexpr (trial_div && test_div)
       {
-         // assumes TrialD1D == TestD1D
-         return internal::SmemPAHdivMassApply3D<TrialD1D, Q1D>;
+         if (Device::Allows(Backend::DEVICE_MASK))
+         {
+            // assumes TrialD1D == TestD1D
+            return internal::SmemPAHdivMassApply3D<TrialD1D, Q1D>;
+         }
+         else
+         {
+            return internal::PAHdivMassApply3D;
+         }
       }
       else if constexpr (trial_curl && test_div)
       {
@@ -93,8 +104,15 @@ VectorFEMassIntegrator::ApplyPAKernels::Kernel()
       }
       else if constexpr (trial_div && test_div)
       {
-         // assumes TrialD1D == TestD1D
-         return internal::SmemPAHdivMassApply2D<TrialD1D, Q1D>;
+         if (Device::Allows(Backend::DEVICE_MASK))
+         {
+            // assumes TrialD1D == TestD1D
+            return internal::SmemPAHdivMassApply2D<TrialD1D, Q1D>;
+         }
+         else
+         {
+            return internal::PAHdivMassApply2D;
+         }
       }
       else if constexpr (trial_curl && test_div)
       {

fem/integ/bilininteg_vectorfemass_pa.cpp

@@ -10,6 +10,8 @@
 // CONTRIBUTING.md for details.
 
 #include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
+#include "../qfunction.hpp"
 #include "bilininteg_vectorfemass_kernels.hpp"
 
 namespace mfem
@@ -70,7 +72,17 @@ VectorFEMassIntegrator::ApplyPAKernels::Fallback(
 VectorFEMassIntegrator::Kernels::Kernels()
 {
    // h(curl), h(curl)
-   // Q = P + 1 (3D)
+   // P = Q (3D)
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 2, 2, 2>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 3, 3, 3>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 4, 4, 4>();
+   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
+                          FiniteElement::CURL, 3, 5, 5, 5>();
+
+   // P = Q + 1 (3D)
    VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
                           FiniteElement::CURL, 3, 2, 2, 3>();
    VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
@@ -79,26 +91,8 @@ VectorFEMassIntegrator::Kernels::Kernels()
                           FiniteElement::CURL, 3, 4, 4, 5>();
    VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
                           FiniteElement::CURL, 3, 5, 5, 6>();
-   // Q = P + 2 (3D)
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 2, 2, 4>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 3, 3, 5>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 4, 4, 6>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 5, 5, 7>();
-   // Q = P + 4 (3D)
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 2, 2, 6>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 3, 3, 7>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 4, 4, 8>();
-   VectorFEMassIntegrator::AddSpecialization<FiniteElement::CURL,
-                          FiniteElement::CURL, 3, 5, 5, 9>();
    // h(div), h(div)
-   // Q = P (2D)
+   // P = Q (2D)
    VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
                           FiniteElement::DIV, 2, 2, 2, 2>();
    VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
@@ -108,7 +102,7 @@ VectorFEMassIntegrator::Kernels::Kernels()
    VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
                           FiniteElement::DIV, 2, 5, 5, 5>();
 
-   // Q = P + 1 (3D)
+   // P = Q + 1 (3D)
    VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
                           FiniteElement::DIV, 3, 2, 2, 3>();
    VectorFEMassIntegrator::AddSpecialization<FiniteElement::DIV,
@@ -339,6 +333,8 @@ void VectorFEMassIntegrator::AddAbsMultPA(const Vector &x, Vector &y) const
 
    Array<real_t> absBo(mapsO->B);
    Array<real_t> absBc(mapsC->B);
+   // Array<real_t> absBto(mapsO->Bt);
+   // Array<real_t> absBtc(mapsC->Bt);
    Array<real_t> absBto_t(mapsOtest->Bt);
    Array<real_t> absBtc_t(mapsCtest->Bt);
 

fem/integ/nonlininteg_vecconvection_pa.cpp

@@ -9,51 +9,21 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
+#include "../../general/forall.hpp"
+#include "../nonlininteg.hpp"
 #include "../ceed/integrators/nlconvection/nlconvection.hpp"
-#include "./nonlininteg_vecconvection_pa.hpp" // IWYU pragma: keep
-#include "./nonlininteg_vecconvection_pa_grad.hpp" // IWYU pragma: keep
-#include "./nonlininteg_vecconvection_pa_diag.hpp" // IWYU pragma: keep
 
 namespace mfem
 {
 
-VectorConvectionNLFIntegrator::Kernels::Kernels()
-{
-   // 2D
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 2, 2>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 2, 3>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 3, 4>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 3, 5>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 4, 5>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 4, 6>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 5, 7>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 5, 8>();
-   VectorConvectionNLFIntegrator::AddSpecialization<2, 6, 8>();
-   // 3D
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 2, 3>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 2, 4>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 2, 5>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 3, 4>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 3, 5>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 3, 6>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 4, 5>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 4, 6>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 4, 7>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 4, 8>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 5, 6>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 5, 7>();
-   VectorConvectionNLFIntegrator::AddSpecialization<3, 5, 8>();
-}
-
 void VectorConvectionNLFIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
    MFEM_ASSERT(fes.GetOrdering() == Ordering::byNODES,
                "PA Only supports Ordering::byNODES!");
    Mesh *mesh = fes.GetMesh();
    const FiniteElement &el = *fes.GetTypicalFE();
-   ElementTransformation &Tr = *mesh->GetTypicalElementTransformation();
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, Tr);
-
+   ElementTransformation &T = *mesh->GetTypicalElementTransformation();
+   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, T);
    if (DeviceCanUseCeed())
    {
       delete ceedOp;
@@ -69,124 +39,769 @@ void VectorConvectionNLFIntegrator::AssemblePA(const FiniteElementSpace &fes)
       }
       return;
    }
-
-   ne = mesh->GetNE();
-   nq = ir->GetNPoints();
    dim = mesh->Dimension();
-   MFEM_VERIFY(dim == 2 || dim == 3, "Dimension not supported");
-
-   const MemoryType mt = pa_mt == MemoryType::DEFAULT
-                         ? Device::GetDeviceMemoryType()
-                         : pa_mt;
-   pa_adj.SetSize(ne * nq * dim * dim, mt);
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mt);
+   ne = fes.GetMesh()->GetNE();
+   nq = ir->GetNPoints();
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
    maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
-   d1d = maps->ndof;
-   q1d = maps->nqpt;
-
-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
-
-   const int nq1d = q1d * q1d * (dim==3 ? q1d : 1);
-   MFEM_VERIFY(coeff.Size() == 1 || coeff.Size() == nq1d*ne, "Invalid coeff");
-   MFEM_VERIFY(ir->GetWeights().Size() == nq1d, "Invalid weights size");
-
-   const auto w_r = ir->GetWeights().Read();
-   const bool const_coeff = coeff.Size() == 1;
-
+   pa_data.SetSize(ne * nq * dim * dim, Device::GetMemoryType());
+   real_t COEFF = 1.0;
+   if (Q)
+   {
+      ConstantCoefficient *cQ = dynamic_cast<ConstantCoefficient *>(Q);
+      MFEM_VERIFY(cQ != NULL, "only ConstantCoefficient is supported!");
+      COEFF = cQ->constant;
+   }
+   const int NE = ne;
+   const int NQ = nq;
+   auto W = ir->GetWeights().Read();
+   if (dim == 1)
+   {
+      MFEM_ABORT("dim==1 not supported!");
+   }
    if (dim == 2)
    {
-      const int Q1D = q1d;
-      constexpr int VDIM = 2, DIM = 2;
-      const auto W = Reshape(w_r, Q1D, Q1D);
-      const auto C = const_coeff ?
-                     Reshape(coeff.Read(), 1, 1, 1) :
-                     Reshape(coeff.Read(), Q1D, Q1D, ne);
-      const auto J = Reshape(geom->J.Read(), Q1D, Q1D, VDIM, DIM, ne);
-      auto A = Reshape(pa_adj.Write(), VDIM, DIM, Q1D, Q1D, ne);
-
-      mfem::forall_2D(ne, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+      auto J = Reshape(geom->J.Read(), NQ, 2, 2, NE);
+      auto G = Reshape(pa_data.Write(), NQ, 2, 2, NE);
+      mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
       {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+         for (int q = 0; q < NQ; ++q)
          {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-            {
-               const real_t J11 = J(qx, qy, 0, 0, e), J12 = J(qx, qy, 0, 1, e);
-               const real_t J21 = J(qx, qy, 1, 0, e), J22 = J(qx, qy, 1, 1, e);
-               // adj(J)
-               const real_t A11 = +J22, A12 = -J12;
-               const real_t A21 = -J21, A22 = +J11;
-               // Store w * coeff * adj(J)
-               const real_t w = W(qx, qy);
-               const real_t c = const_coeff ? C(0, 0, 0) : C(qx, qy, e);
-               A(0, 0, qx, qy, e) = w * c * A11;
-               A(1, 0, qx, qy, e) = w * c * A12;
-               A(0, 1, qx, qy, e) = w * c * A21;
-               A(1, 1, qx, qy, e) = w * c * A22;
-            }
+            const real_t J11 = J(q, 0, 0, e);
+            const real_t J12 = J(q, 0, 1, e);
+            const real_t J21 = J(q, 1, 0, e);
+            const real_t J22 = J(q, 1, 1, e);
+            // Store wq * Q * adj(J)
+            G(q, 0, 0, e) = W[q] * COEFF * J22;  // 1,1
+            G(q, 0, 1, e) = W[q] * COEFF * -J12; // 1,2
+            G(q, 1, 0, e) = W[q] * COEFF * -J21; // 2,1
+            G(q, 1, 1, e) = W[q] * COEFF * J11;  // 2,2
          }
       });
    }
-   else if (dim == 3)
+   if (dim == 3)
    {
-      const int Q1D = q1d;
-      constexpr int VDIM = 3, DIM = 3;
-      const auto W = Reshape(w_r, Q1D, Q1D, Q1D);
-      const auto C = const_coeff ?
-                     Reshape(coeff.Read(), 1, 1, 1, 1) :
-                     Reshape(coeff.Read(), Q1D, Q1D, Q1D, ne);
-      const auto J = Reshape(geom->J.Read(), Q1D, Q1D, Q1D, VDIM, DIM, ne);
-      auto A = Reshape(pa_adj.Write(), VDIM, DIM, Q1D, Q1D, Q1D, ne);
-
-      mfem::forall_3D(ne, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
+      auto J = Reshape(geom->J.Read(), NQ, 3, 3, NE);
+      auto G = Reshape(pa_data.Write(), NQ, 3, 3, NE);
+      mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
       {
-         MFEM_FOREACH_THREAD_DIRECT(qz, z, Q1D)
+         for (int q = 0; q < NQ; ++q)
          {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
+            const real_t J11 = J(q, 0, 0, e);
+            const real_t J21 = J(q, 1, 0, e);
+            const real_t J31 = J(q, 2, 0, e);
+            const real_t J12 = J(q, 0, 1, e);
+            const real_t J22 = J(q, 1, 1, e);
+            const real_t J32 = J(q, 2, 1, e);
+            const real_t J13 = J(q, 0, 2, e);
+            const real_t J23 = J(q, 1, 2, e);
+            const real_t J33 = J(q, 2, 2, e);
+            const real_t cw = W[q] * COEFF;
+            // adj(J)
+            const real_t A11 = (J22 * J33) - (J23 * J32);
+            const real_t A12 = (J32 * J13) - (J12 * J33);
+            const real_t A13 = (J12 * J23) - (J22 * J13);
+            const real_t A21 = (J31 * J23) - (J21 * J33);
+            const real_t A22 = (J11 * J33) - (J13 * J31);
+            const real_t A23 = (J21 * J13) - (J11 * J23);
+            const real_t A31 = (J21 * J32) - (J31 * J22);
+            const real_t A32 = (J31 * J12) - (J11 * J32);
+            const real_t A33 = (J11 * J22) - (J12 * J21);
+            // Store wq * Q * adj(J)
+            G(q, 0, 0, e) = cw * A11; // 1,1
+            G(q, 0, 1, e) = cw * A12; // 1,2
+            G(q, 0, 2, e) = cw * A13; // 1,3
+            G(q, 1, 0, e) = cw * A21; // 2,1
+            G(q, 1, 1, e) = cw * A22; // 2,2
+            G(q, 1, 2, e) = cw * A23; // 2,3
+            G(q, 2, 0, e) = cw * A31; // 3,1
+            G(q, 2, 1, e) = cw * A32; // 3,2
+            G(q, 2, 2, e) = cw * A33; // 3,3
+         }
+      });
+   }
+}
+
+// PA Convection NL 2D kernel
+template<int T_D1D = 0, int T_Q1D = 0>
+static void PAConvectionNLApply2D(const int NE,
+                                  const Array<real_t> &b,
+                                  const Array<real_t> &g,
+                                  const Array<real_t> &bt,
+                                  const Vector &q_,
+                                  const Vector &x_,
+                                  Vector &y_,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto G = Reshape(g.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt.Read(), D1D, Q1D);
+   auto Q = Reshape(q_.Read(), Q1D * Q1D, 2, 2, NE);
+   auto x = Reshape(x_.Read(), D1D, D1D, 2, NE);
+   auto y = Reshape(y_.ReadWrite(), D1D, D1D, 2, NE);
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      real_t data[max_Q1D][max_Q1D][2];
+      real_t grad0[max_Q1D][max_Q1D][2];
+      real_t grad1[max_Q1D][max_Q1D][2];
+      real_t Z[max_Q1D][max_Q1D][2];
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            data[qy][qx][0] = 0.0;
+            data[qy][qx][1] = 0.0;
+            grad0[qy][qx][0] = 0.0;
+            grad0[qy][qx][1] = 0.0;
+            grad1[qy][qx][0] = 0.0;
+            grad1[qy][qx][1] = 0.0;
+         }
+      }
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         real_t dataX[max_Q1D][2];
+         real_t gradX0[max_Q1D][2];
+         real_t gradX1[max_Q1D][2];
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            dataX[qx][0] = 0.0;
+            dataX[qx][1] = 0.0;
+            gradX0[qx][0] = 0.0;
+            gradX0[qx][1] = 0.0;
+            gradX1[qx][0] = 0.0;
+            gradX1[qx][1] = 0.0;
+         }
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            const real_t s0 = x(dx, dy, 0, e);
+            const real_t s1 = x(dx, dy, 1, e);
+            for (int qx = 0; qx < Q1D; ++qx)
             {
-               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
+               const real_t Bx = B(qx, dx);
+               const real_t Gx = G(qx, dx);
+               dataX[qx][0] += s0 * Bx;
+               dataX[qx][1] += s1 * Bx;
+               gradX0[qx][0] += s0 * Gx;
+               gradX0[qx][1] += s0 * Bx;
+               gradX1[qx][0] += s1 * Gx;
+               gradX1[qx][1] += s1 * Bx;
+            }
+         }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            const real_t By = B(qy, dy);
+            const real_t Gy = G(qy, dy);
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               data[qy][qx][0] += dataX[qx][0] * By;
+               data[qy][qx][1] += dataX[qx][1] * By;
+               grad0[qy][qx][0] += gradX0[qx][0] * By;
+               grad0[qy][qx][1] += gradX0[qx][1] * Gy;
+               grad1[qy][qx][0] += gradX1[qx][0] * By;
+               grad1[qy][qx][1] += gradX1[qx][1] * Gy;
+            }
+         }
+      }
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            const int q = qx + qy * Q1D;
+            const real_t u1 = data[qy][qx][0];
+            const real_t u2 = data[qy][qx][1];
+            const real_t grad00 = grad0[qy][qx][0];
+            const real_t grad01 = grad0[qy][qx][1];
+            const real_t grad10 = grad1[qy][qx][0];
+            const real_t grad11 = grad1[qy][qx][1];
+            const real_t Dxu1 = grad00 * Q(q, 0, 0, e) + grad01 * Q(q, 1, 0, e);
+            const real_t Dyu1 = grad00 * Q(q, 0, 1, e) + grad01 * Q(q, 1, 1, e);
+            const real_t Dxu2 = grad10 * Q(q, 0, 0, e) + grad11 * Q(q, 1, 0, e);
+            const real_t Dyu2 = grad10 * Q(q, 0, 1, e) + grad11 * Q(q, 1, 1, e);
+            Z[qy][qx][0] = u1 * Dxu1 + u2 * Dyu1;
+            Z[qy][qx][1] = u1 * Dxu2 + u2 * Dyu2;
+         }
+      }
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         real_t Y[max_D1D][2];
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            Y[dx][0] = 0.0;
+            Y[dx][1] = 0.0;
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const real_t Btx = Bt(dx, qx);
+               Y[dx][0] += Btx * Z[qy][qx][0];
+               Y[dx][1] += Btx * Z[qy][qx][1];
+            }
+         }
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const real_t Bty = Bt(dy, qy);
+               y(dx, dy, 0, e) += Bty * Y[dx][0];
+               y(dx, dy, 1, e) += Bty * Y[dx][1];
+            }
+         }
+      }
+   });
+}
+
+// PA Convection NL 3D kernel
+template<int T_D1D = 0, int T_Q1D = 0>
+static void PAConvectionNLApply3D(const int NE,
+                                  const Array<real_t> &b,
+                                  const Array<real_t> &g,
+                                  const Array<real_t> &bt,
+                                  const Vector &q_,
+                                  const Vector &x_,
+                                  Vector &y_,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
+{
+   constexpr int VDIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto G = Reshape(g.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt.Read(), D1D, Q1D);
+   auto Q = Reshape(q_.Read(), Q1D * Q1D * Q1D, VDIM, VDIM, NE);
+   auto x = Reshape(x_.Read(), D1D, D1D, D1D, VDIM, NE);
+   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
+
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      constexpr int VDIM = 3;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      real_t data[max_Q1D][max_Q1D][max_Q1D][VDIM];
+      real_t grad0[max_Q1D][max_Q1D][max_Q1D][VDIM];
+      real_t grad1[max_Q1D][max_Q1D][max_Q1D][VDIM];
+      real_t grad2[max_Q1D][max_Q1D][max_Q1D][VDIM];
+      real_t Z[max_Q1D][max_Q1D][max_Q1D][VDIM];
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               data[qz][qy][qx][0] = 0.0;
+               data[qz][qy][qx][1] = 0.0;
+               data[qz][qy][qx][2] = 0.0;
+
+               grad0[qz][qy][qx][0] = 0.0;
+               grad0[qz][qy][qx][1] = 0.0;
+               grad0[qz][qy][qx][2] = 0.0;
+
+               grad1[qz][qy][qx][0] = 0.0;
+               grad1[qz][qy][qx][1] = 0.0;
+               grad1[qz][qy][qx][2] = 0.0;
+
+               grad2[qz][qy][qx][0] = 0.0;
+               grad2[qz][qy][qx][1] = 0.0;
+               grad2[qz][qy][qx][2] = 0.0;
+            }
+         }
+      }
+      for (int dz = 0; dz < D1D; ++dz)
+      {
+         real_t dataXY[max_Q1D][max_Q1D][VDIM];
+         real_t gradXY0[max_Q1D][max_Q1D][VDIM];
+         real_t gradXY1[max_Q1D][max_Q1D][VDIM];
+         real_t gradXY2[max_Q1D][max_Q1D][VDIM];
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               dataXY[qy][qx][0] = 0.0;
+               dataXY[qy][qx][1] = 0.0;
+               dataXY[qy][qx][2] = 0.0;
+
+               gradXY0[qy][qx][0] = 0.0;
+               gradXY0[qy][qx][1] = 0.0;
+               gradXY0[qy][qx][2] = 0.0;
+
+               gradXY1[qy][qx][0] = 0.0;
+               gradXY1[qy][qx][1] = 0.0;
+               gradXY1[qy][qx][2] = 0.0;
+
+               gradXY2[qy][qx][0] = 0.0;
+               gradXY2[qy][qx][1] = 0.0;
+               gradXY2[qy][qx][2] = 0.0;
+            }
+         }
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            real_t dataX[max_Q1D][VDIM];
+            real_t gradX0[max_Q1D][VDIM];
+            real_t gradX1[max_Q1D][VDIM];
+            real_t gradX2[max_Q1D][VDIM];
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               dataX[qx][0] = 0.0;
+               dataX[qx][1] = 0.0;
+               dataX[qx][2] = 0.0;
+
+               gradX0[qx][0] = 0.0;
+               gradX0[qx][1] = 0.0;
+               gradX0[qx][2] = 0.0;
+
+               gradX1[qx][0] = 0.0;
+               gradX1[qx][1] = 0.0;
+               gradX1[qx][2] = 0.0;
+
+               gradX2[qx][0] = 0.0;
+               gradX2[qx][1] = 0.0;
+               gradX2[qx][2] = 0.0;
+            }
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const real_t s0 = x(dx, dy, dz, 0, e);
+               const real_t s1 = x(dx, dy, dz, 1, e);
+               const real_t s2 = x(dx, dy, dz, 2, e);
+               for (int qx = 0; qx < Q1D; ++qx)
                {
-                  const real_t J11 = J(qx, qy, qz, 0, 0, e),
-                               J12 = J(qx, qy, qz, 0, 1, e),
-                               J13 = J(qx, qy, qz, 0, 2, e);
-                  const real_t J21 = J(qx, qy, qz, 1, 0, e),
-                               J22 = J(qx, qy, qz, 1, 1, e),
-                               J23 = J(qx, qy, qz, 1, 2, e);
-                  const real_t J31 = J(qx, qy, qz, 2, 0, e),
-                               J32 = J(qx, qy, qz, 2, 1, e),
-                               J33 = J(qx, qy, qz, 2, 2, e);
-                  const real_t c =
-                     const_coeff ? C(0, 0, 0, 0) : C(qx, qy, qz, e);
-                  const real_t cw = W(qx, qy, qz) * c;
-                  // adj(J)
-                  const real_t A11 = (J22 * J33) - (J23 * J32);
-                  const real_t A12 = (J32 * J13) - (J12 * J33);
-                  const real_t A13 = (J12 * J23) - (J22 * J13);
-                  const real_t A21 = (J31 * J23) - (J21 * J33);
-                  const real_t A22 = (J11 * J33) - (J13 * J31);
-                  const real_t A23 = (J21 * J13) - (J11 * J23);
-                  const real_t A31 = (J21 * J32) - (J31 * J22);
-                  const real_t A32 = (J31 * J12) - (J11 * J32);
-                  const real_t A33 = (J11 * J22) - (J12 * J21);
-                  // Store wq * coeff * adj(J)
-                  A(0, 0, qx, qy, qz, e) = cw * A11;
-                  A(1, 0, qx, qy, qz, e) = cw * A12;
-                  A(2, 0, qx, qy, qz, e) = cw * A13;
-                  A(0, 1, qx, qy, qz, e) = cw * A21;
-                  A(1, 1, qx, qy, qz, e) = cw * A22;
-                  A(2, 1, qx, qy, qz, e) = cw * A23;
-                  A(0, 2, qx, qy, qz, e) = cw * A31;
-                  A(1, 2, qx, qy, qz, e) = cw * A32;
-                  A(2, 2, qx, qy, qz, e) = cw * A33;
+                  const real_t Bx = B(qx, dx);
+                  const real_t Gx = G(qx, dx);
+
+                  dataX[qx][0] += s0 * Bx;
+                  dataX[qx][1] += s1 * Bx;
+                  dataX[qx][2] += s2 * Bx;
+
+                  gradX0[qx][0] += s0 * Gx;
+                  gradX0[qx][1] += s0 * Bx;
+                  gradX0[qx][2] += s0 * Bx;
+
+                  gradX1[qx][0] += s1 * Gx;
+                  gradX1[qx][1] += s1 * Bx;
+                  gradX1[qx][2] += s1 * Bx;
+
+                  gradX2[qx][0] += s2 * Gx;
+                  gradX2[qx][1] += s2 * Bx;
+                  gradX2[qx][2] += s2 * Bx;
+               }
+            }
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const real_t By = B(qy, dy);
+               const real_t Gy = G(qy, dy);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  dataXY[qy][qx][0] += dataX[qx][0] * By;
+                  dataXY[qy][qx][1] += dataX[qx][1] * By;
+                  dataXY[qy][qx][2] += dataX[qx][2] * By;
+
+                  gradXY0[qy][qx][0] += gradX0[qx][0] * By;
+                  gradXY0[qy][qx][1] += gradX0[qx][1] * Gy;
+                  gradXY0[qy][qx][2] += gradX0[qx][2] * By;
+
+                  gradXY1[qy][qx][0] += gradX1[qx][0] * By;
+                  gradXY1[qy][qx][1] += gradX1[qx][1] * Gy;
+                  gradXY1[qy][qx][2] += gradX1[qx][2] * By;
+
+                  gradXY2[qy][qx][0] += gradX2[qx][0] * By;
+                  gradXY2[qy][qx][1] += gradX2[qx][1] * Gy;
+                  gradXY2[qy][qx][2] += gradX2[qx][2] * By;
                }
             }
          }
-      });
-   }
-   else
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            const real_t Bz = B(qz, dz);
+            const real_t Gz = G(qz, dz);
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  data[qz][qy][qx][0] += dataXY[qy][qx][0] * Bz;
+                  data[qz][qy][qx][1] += dataXY[qy][qx][1] * Bz;
+                  data[qz][qy][qx][2] += dataXY[qy][qx][2] * Bz;
+
+                  grad0[qz][qy][qx][0] += gradXY0[qy][qx][0] * Bz;
+                  grad0[qz][qy][qx][1] += gradXY0[qy][qx][1] * Bz;
+                  grad0[qz][qy][qx][2] += gradXY0[qy][qx][2] * Gz;
+
+                  grad1[qz][qy][qx][0] += gradXY1[qy][qx][0] * Bz;
+                  grad1[qz][qy][qx][1] += gradXY1[qy][qx][1] * Bz;
+                  grad1[qz][qy][qx][2] += gradXY1[qy][qx][2] * Gz;
+
+                  grad2[qz][qy][qx][0] += gradXY2[qy][qx][0] * Bz;
+                  grad2[qz][qy][qx][1] += gradXY2[qy][qx][1] * Bz;
+                  grad2[qz][qy][qx][2] += gradXY2[qy][qx][2] * Gz;
+               }
+            }
+         }
+      }
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const int q = qx + Q1D * (qy + qz * Q1D);
+
+               const real_t u1 = data[qz][qy][qx][0];
+               const real_t u2 = data[qz][qy][qx][1];
+               const real_t u3 = data[qz][qy][qx][2];
+
+               const real_t grad00 = grad0[qz][qy][qx][0];
+               const real_t grad01 = grad0[qz][qy][qx][1];
+               const real_t grad02 = grad0[qz][qy][qx][2];
+
+               const real_t grad10 = grad1[qz][qy][qx][0];
+               const real_t grad11 = grad1[qz][qy][qx][1];
+               const real_t grad12 = grad1[qz][qy][qx][2];
+
+               const real_t grad20 = grad2[qz][qy][qx][0];
+               const real_t grad21 = grad2[qz][qy][qx][1];
+               const real_t grad22 = grad2[qz][qy][qx][2];
+
+               const real_t Dxu1 = grad00 * Q(q, 0, 0, e)
+                                   + grad01 * Q(q, 1, 0, e)
+                                   + grad02 * Q(q, 2, 0, e);
+               const real_t Dyu1 = grad00 * Q(q, 0, 1, e)
+                                   + grad01 * Q(q, 1, 1, e)
+                                   + grad02 * Q(q, 2, 1, e);
+               const real_t Dzu1 = grad00 * Q(q, 0, 2, e)
+                                   + grad01 * Q(q, 1, 2, e)
+                                   + grad02 * Q(q, 2, 2, e);
+
+               const real_t Dxu2 = grad10 * Q(q, 0, 0, e)
+                                   + grad11 * Q(q, 1, 0, e)
+                                   + grad12 * Q(q, 2, 0, e);
+               const real_t Dyu2 = grad10 * Q(q, 0, 1, e)
+                                   + grad11 * Q(q, 1, 1, e)
+                                   + grad12 * Q(q, 2, 1, e);
+               const real_t Dzu2 = grad10 * Q(q, 0, 2, e)
+                                   + grad11 * Q(q, 1, 2, e)
+                                   + grad12 * Q(q, 2, 2, e);
+
+               const real_t Dxu3 = grad20 * Q(q, 0, 0, e)
+                                   + grad21 * Q(q, 1, 0, e)
+                                   + grad22 * Q(q, 2, 0, e);
+               const real_t Dyu3 = grad20 * Q(q, 0, 1, e)
+                                   + grad21 * Q(q, 1, 1, e)
+                                   + grad22 * Q(q, 2, 1, e);
+               const real_t Dzu3 = grad20 * Q(q, 0, 2, e)
+                                   + grad21 * Q(q, 1, 2, e)
+                                   + grad22 * Q(q, 2, 2, e);
+
+               Z[qz][qy][qx][0] = u1 * Dxu1 + u2 * Dyu1 + u3 * Dzu1;
+               Z[qz][qy][qx][1] = u1 * Dxu2 + u2 * Dyu2 + u3 * Dzu2;
+               Z[qz][qy][qx][2] = u1 * Dxu3 + u2 * Dyu3 + u3 * Dzu3;
+            }
+         }
+      }
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         real_t opXY[max_D1D][max_D1D][VDIM];
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               opXY[dy][dx][0] = 0.0;
+               opXY[dy][dx][1] = 0.0;
+               opXY[dy][dx][2] = 0.0;
+            }
+         }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            real_t opX[max_D1D][VDIM];
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               opX[dx][0] = 0.0;
+               opX[dx][1] = 0.0;
+               opX[dx][2] = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  const real_t Btx = Bt(dx, qx);
+                  opX[dx][0] += Btx * Z[qz][qy][qx][0];
+                  opX[dx][1] += Btx * Z[qz][qy][qx][1];
+                  opX[dx][2] += Btx * Z[qz][qy][qx][2];
+               }
+            }
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  const real_t Bty = Bt(dy, qy);
+                  opXY[dy][dx][0] += Bty * opX[dx][0];
+                  opXY[dy][dx][1] += Bty * opX[dx][1];
+                  opXY[dy][dx][2] += Bty * opX[dx][2];
+               }
+            }
+         }
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  const real_t Btz = Bt(dz, qz);
+                  y(dx, dy, dz, 0, e) += Btz * opXY[dy][dx][0];
+                  y(dx, dy, dz, 1, e) += Btz * opXY[dy][dx][1];
+                  y(dx, dy, dz, 2, e) += Btz * opXY[dy][dx][2];
+               }
+            }
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0, int T_MAX_D1D = 0, int T_MAX_Q1D = 0>
+static void SmemPAConvectionNLApply3D(const int NE,
+                                      const Array<real_t> &b_,
+                                      const Array<real_t> &g_,
+                                      const Vector &d_,
+                                      const Vector &x_,
+                                      Vector &y_,
+                                      const int d1d = 0,
+                                      const int q1d = 0)
+{
+   constexpr int VDIM = 3;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int MD1 = T_D1D ? T_D1D : T_MAX_D1D;
+   constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX_Q1D;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
+
+   auto b = Reshape(b_.Read(), Q1D, D1D);
+   auto g = Reshape(g_.Read(), Q1D, D1D);
+   auto D = Reshape(d_.Read(), Q1D * Q1D * Q1D, VDIM, VDIM, NE);
+   auto x = Reshape(x_.Read(), D1D, D1D, D1D, VDIM, NE);
+   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
+
+   mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
    {
-      MFEM_ABORT("dim " << dim << " not supported!");
-   }
+      const int tidz = MFEM_THREAD_ID(z);
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : T_MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : T_MAX_Q1D;
+      MFEM_SHARED real_t BG[2][MQ1 * MD1];
+      real_t(*B)[MD1] = (real_t(*)[MD1])(BG + 0);
+      real_t(*G)[MD1] = (real_t(*)[MD1])(BG + 1);
+      real_t(*Bt)[MQ1] = (real_t(*)[MQ1])(BG + 0);
+      MFEM_SHARED real_t U[2][MQ1][MQ1][MQ1];
+      MFEM_SHARED real_t sm0[3][MQ1 * MQ1 * MQ1];
+      MFEM_SHARED real_t sm1[3][MQ1 * MQ1 * MQ1];
+      real_t(*DDQ0)[MD1][MQ1] = (real_t(*)[MD1][MQ1])(sm0 + 0);
+      real_t(*DDQ1)[MD1][MQ1] = (real_t(*)[MD1][MQ1])(sm0 + 1);
+      real_t(*X)[MD1][MD1] = (real_t(*)[MD1][MD1])(sm0 + 2);
+      real_t(*DQQ0)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm1 + 0);
+      real_t(*DQQ1)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm1 + 1);
+      real_t(*DQQ2)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm1 + 2);
+      real_t(*QQQ0)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm0 + 0);
+      real_t(*QQQ1)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm0 + 1);
+      real_t(*QQQ2)[MQ1][MQ1] = (real_t(*)[MQ1][MQ1])(sm0 + 2);
+      real_t(*QQD0)[MQ1][MD1] = (real_t(*)[MQ1][MD1])(sm1 + 0);
+      real_t(*QDD0)[MD1][MD1] = (real_t(*)[MD1][MD1])(sm0 + 0);
+      MFEM_SHARED real_t Z[MQ1][MQ1][MQ1];
+
+      for (int cy = 0; cy < VDIM; ++cy)
+      {
+         if (tidz == 0)
+         {
+            MFEM_FOREACH_THREAD(q, x, Q1D)
+            {
+               MFEM_FOREACH_THREAD(d, y, D1D)
+               {
+                  B[q][d] = b(q, d);
+                  G[q][d] = g(q, d);
+               }
+            }
+         }
+         MFEM_FOREACH_THREAD(qz, z, Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy, y, Q1D)
+            {
+               MFEM_FOREACH_THREAD(qx, x, Q1D) { Z[qz][qy][qx] = 0.0; }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         for (int c = 0; c < VDIM; ++c)
+         {
+            MFEM_FOREACH_THREAD(dz, z, D1D)
+            {
+               MFEM_FOREACH_THREAD(dy, y, D1D)
+               {
+                  MFEM_FOREACH_THREAD(dx, x, D1D)
+                  {
+                     X[dz][dy][dx] = x(dx, dy, dz, cy, e);
+                     U[0][dz][dy][dx] = x(dx, dy, dz, c, e);
+                  }
+               }
+            }
+            MFEM_SYNC_THREAD;
+            MFEM_FOREACH_THREAD(dz, z, D1D)
+            {
+               MFEM_FOREACH_THREAD(dy, y, D1D)
+               {
+                  MFEM_FOREACH_THREAD(qx, x, Q1D)
+                  {
+                     real_t u = 0.0;
+                     real_t v = 0.0;
+                     real_t z = 0.0;
+                     for (int dx = 0; dx < D1D; ++dx)
+                     {
+                        const real_t coord = X[dz][dy][dx];
+                        const real_t value = U[0][dz][dy][dx];
+                        u += coord * B[qx][dx];
+                        v += coord * G[qx][dx];
+                        z += value * B[qx][dx];
+                     }
+                     DDQ0[dz][dy][qx] = u;
+                     DDQ1[dz][dy][qx] = v;
+                     U[1][dz][dy][qx] = z;
+                  }
+               }
+            }
+            MFEM_SYNC_THREAD;
+            MFEM_FOREACH_THREAD(dz, z, D1D)
+            {
+               MFEM_FOREACH_THREAD(qy, y, Q1D)
+               {
+                  MFEM_FOREACH_THREAD(qx, x, Q1D)
+                  {
+                     real_t u = 0.0;
+                     real_t v = 0.0;
+                     real_t w = 0.0;
+                     real_t z = 0.0;
+                     for (int dy = 0; dy < D1D; ++dy)
+                     {
+                        u += DDQ1[dz][dy][qx] * B[qy][dy];
+                        v += DDQ0[dz][dy][qx] * G[qy][dy];
+                        w += DDQ0[dz][dy][qx] * B[qy][dy];
+                        z += U[1][dz][dy][qx] * B[qy][dy];
+                     }
+                     DQQ0[dz][qy][qx] = u;
+                     DQQ1[dz][qy][qx] = v;
+                     DQQ2[dz][qy][qx] = w;
+                     U[0][dz][qy][qx] = z;
+                  }
+               }
+            }
+            MFEM_SYNC_THREAD;
+            MFEM_FOREACH_THREAD(qz, z, Q1D)
+            {
+               MFEM_FOREACH_THREAD(qy, y, Q1D)
+               {
+                  MFEM_FOREACH_THREAD(qx, x, Q1D)
+                  {
+                     real_t u = 0.0;
+                     real_t v = 0.0;
+                     real_t w = 0.0;
+                     real_t z = 0.0;
+                     for (int dz = 0; dz < D1D; ++dz)
+                     {
+                        u += DQQ0[dz][qy][qx] * B[qz][dz];
+                        v += DQQ1[dz][qy][qx] * B[qz][dz];
+                        w += DQQ2[dz][qy][qx] * G[qz][dz];
+                        z += U[0][dz][qy][qx] * B[qz][dz];
+                     }
+                     QQQ0[qz][qy][qx] = u;
+                     QQQ1[qz][qy][qx] = v;
+                     QQQ2[qz][qy][qx] = w;
+                     U[1][qz][qy][qx] = z;
+                  }
+               }
+            }
+            MFEM_SYNC_THREAD;
+            MFEM_FOREACH_THREAD(qz, z, Q1D)
+            {
+               MFEM_FOREACH_THREAD(qy, y, Q1D)
+               {
+                  MFEM_FOREACH_THREAD(qx, x, Q1D)
+                  {
+                     const int q = qx + (qy + qz * Q1D) * Q1D;
+                     const real_t z = U[1][qz][qy][qx];
+                     const real_t gX = QQQ0[qz][qy][qx];
+                     const real_t gY = QQQ1[qz][qy][qx];
+                     const real_t gZ = QQQ2[qz][qy][qx];
+                     const real_t d = gX * D(q, 0, c, e) + gY * D(q, 1, c, e)
+                                      + gZ * D(q, 2, c, e);
+                     Z[qz][qy][qx] += z * d;
+                  }
+               }
+            }
+            MFEM_SYNC_THREAD;
+         } // for each conv component
+         if (tidz == 0)
+         {
+            MFEM_FOREACH_THREAD(d, y, D1D)
+            {
+               MFEM_FOREACH_THREAD(q, x, Q1D) { Bt[d][q] = b(q, d); }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(qz, z, Q1D)
+         {
+            MFEM_FOREACH_THREAD(qy, y, Q1D)
+            {
+               MFEM_FOREACH_THREAD(dx, x, D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     u += Z[qz][qy][qx] * Bt[dx][qx];
+                  }
+                  QQD0[qz][qy][dx] = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(qz, z, Q1D)
+         {
+            MFEM_FOREACH_THREAD(dy, y, D1D)
+            {
+               MFEM_FOREACH_THREAD(dx, x, D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qy = 0; qy < Q1D; ++qy)
+                  {
+                     u += QQD0[qz][qy][dx] * Bt[dy][qy];
+                  }
+                  QDD0[qz][dy][dx] = u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(dz, z, D1D)
+         {
+            MFEM_FOREACH_THREAD(dy, y, D1D)
+            {
+               MFEM_FOREACH_THREAD(dx, x, D1D)
+               {
+                  real_t u = 0.0;
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     u += QDD0[qz][dy][dx] * Bt[dz][qz];
+                  }
+                  Y(dx, dy, dz, cy, e) += u;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
 }
 
 void VectorConvectionNLFIntegrator::AddMultPA(const Vector &x, Vector &y) const
@@ -197,13 +812,26 @@ void VectorConvectionNLFIntegrator::AddMultPA(const Vector &x, Vector &y) const
    }
    else
    {
-      AddMultPAKernels::Run(dim, d1d, q1d, ne,
-                            maps->B.Read(),
-                            maps->G.Read(),
-                            pa_adj.Read(),
-                            x.Read(),
-                            y.ReadWrite(),
-                            d1d, q1d);
+      const int NE = ne;
+      const int D1D = maps->ndof;
+      const int Q1D = maps->nqpt;
+      const Vector &QV = pa_data;
+      const Array<real_t> &B = maps->B;
+      const Array<real_t> &G = maps->G;
+      const Array<real_t> &Bt = maps->Bt;
+      if (dim == 2)
+      {
+         return PAConvectionNLApply2D(NE, B, G, Bt, QV, x, y, D1D, Q1D);
+      }
+      if (dim == 3)
+      {
+         constexpr int T_MAX_D1D = 8;
+         constexpr int T_MAX_Q1D = 8;
+         MFEM_VERIFY(D1D <= T_MAX_D1D && Q1D <= T_MAX_Q1D, "Not yet implemented!");
+         return SmemPAConvectionNLApply3D<0, 0, T_MAX_D1D, T_MAX_Q1D>
+                (NE, B, G, QV, x, y, D1D, Q1D);
+      }
+      MFEM_ABORT("Not yet implemented!");
    }
 }
 

fem/integ/nonlininteg_vecconvection_pa.hpp

@@ -1,209 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../config/config.hpp"
-#include "../../general/forall.hpp"
-#include "../../linalg/dtensor.hpp"
-#include "../kernels.hpp"
-#include "../nonlininteg.hpp"
-
-namespace mfem
-{
-
-/// \cond DO_NOT_DOCUMENT
-
-namespace internal
-{
-
-// PA Convection NL 2D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAConvectionNLApply2D(const int NE,
-                                      const real_t *b,
-                                      const real_t *g,
-                                      const real_t *a,
-                                      const real_t *x,
-                                      real_t *y,
-                                      const int d1d = 0,
-                                      const int q1d = 0)
-{
-   static constexpr int VDIM = 2, DIM = 2;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto B = Reshape(b, Q1D, D1D);
-   const auto G = Reshape(g, Q1D, D1D);
-   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, NE);
-   const auto X = Reshape(x, D1D, D1D, VDIM, NE);
-   auto Y = Reshape(y, D1D, D1D, VDIM, NE);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1], sB[MD1][MQ1], sG[MD1][MQ1];
-
-      kernels::internal::vd_regs2d_t<VDIM, DIM, MQ1> g0, g1;
-      kernels::internal::v_regs2d_t<VDIM, MQ1> r0, r1;
-      kernels::internal::v_regs2d_t<VDIM, MQ1> s0, s1;
-
-      kernels::internal::LoadMatrix(D1D, Q1D, B, sB);
-      kernels::internal::LoadMatrix(D1D, Q1D, G, sG);
-
-      kernels::internal::LoadDofs2d(e, D1D, X, r0);
-      kernels::internal::Eval2d(D1D, Q1D, smem, sB, r0, r1); // u vector-value
-      kernels::internal::LoadDofs2d(e, D1D, X, g0);
-      kernels::internal::Grad2d(D1D, Q1D, smem, sB, sG, g0, g1); // u vector-gradient
-
-      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            const future::tensor<real_t, 2> U =
-            {
-               r1[0][qy][qx], r1[1][qy][qx]
-            };
-            const future::tensor<real_t, 2,2> gradU = {{
-                  {g1[0][0][qy][qx], g1[1][0][qy][qx]},
-                  {g1[0][1][qy][qx], g1[1][1][qy][qx]},
-               }
-            };
-            const future::tensor<real_t, 2,2> Q = {{
-                  {A(0,0,qx,qy,e), A(1,0,qx,qy,e)},
-                  {A(0,1,qx,qy,e), A(1,1,qx,qy,e)},
-               }
-            };
-            const future::tensor<real_t, 2> conv = transpose(gradU) * (Q * U);
-            s0[0][qy][qx] = conv[0];
-            s0[1][qy][qx] = conv[1];
-         }
-      }
-      MFEM_SYNC_THREAD;
-      kernels::internal::EvalTranspose2d(D1D, Q1D, smem, sB, s0, s1);
-      kernels::internal::WriteDofs2d(e, D1D, s1, Y);
-   });
-}
-
-// PA Convection NL 3D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAConvectionNLApply3D(const int NE,
-                                      const real_t *b,
-                                      const real_t *g,
-                                      const real_t *a,
-                                      const real_t *x,
-                                      real_t *y,
-                                      const int d1d = 0,
-                                      const int q1d = 0)
-{
-   static constexpr int VDIM = 3, DIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto B = Reshape(b, Q1D, D1D);
-   const auto G = Reshape(g, Q1D, D1D);
-   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, Q1D, NE);
-   const auto X = Reshape(x, D1D, D1D, D1D, VDIM, NE);
-   auto Y = Reshape(y, D1D, D1D, D1D, VDIM, NE);
-
-   mfem::forall_2D<T_Q1D*T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1], sB[MD1][MQ1], sG[MD1][MQ1];
-
-      kernels::internal::vd_regs3d_t<VDIM, DIM, MQ1> g0, g1;
-      kernels::internal::v_regs3d_t<VDIM, MQ1> r0, r1;
-      kernels::internal::v_regs3d_t<VDIM, MQ1> s0, s1;
-
-      kernels::internal::LoadMatrix(D1D, Q1D, B, sB);
-      kernels::internal::LoadMatrix(D1D, Q1D, G, sG);
-
-      kernels::internal::LoadDofs3d(e, D1D, X, r0);
-      kernels::internal::Eval3d(D1D, Q1D, smem, sB, r0, r1); // u vector-value
-      kernels::internal::LoadDofs3d(e, D1D, X, g0);
-      kernels::internal::Grad3d(D1D, Q1D, smem, sB, sG, g0, g1); // u vector-gradient
-
-      for (int qz = 0; qz < Q1D; qz++)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-            {
-               const future::tensor<real_t, 3> U =
-               {
-                  r1[0][qz][qy][qx], r1[1][qz][qy][qx], r1[2][qz][qy][qx]
-               };
-               const future::tensor<real_t, 3,3> gradU = {{
-                     {g1[0][0][qz][qy][qx], g1[1][0][qz][qy][qx], g1[2][0][qz][qy][qx]},
-                     {g1[0][1][qz][qy][qx], g1[1][1][qz][qy][qx], g1[2][1][qz][qy][qx]},
-                     {g1[0][2][qz][qy][qx], g1[1][2][qz][qy][qx], g1[2][2][qz][qy][qx]}
-                  }
-               };
-               const future::tensor<real_t, 3,3> Q = {{
-                     {A(0,0,qx,qy,qz,e), A(1,0,qx,qy,qz,e), A(2,0,qx,qy,qz,e)},
-                     {A(0,1,qx,qy,qz,e), A(1,1,qx,qy,qz,e), A(2,1,qx,qy,qz,e)},
-                     {A(0,2,qx,qy,qz,e), A(1,2,qx,qy,qz,e), A(2,2,qx,qy,qz,e)}
-                  }
-               };
-               const future::tensor<real_t, 3> conv = transpose(gradU) * (Q * U);
-               s0[0][qz][qy][qx] = conv[0];
-               s0[1][qz][qy][qx] = conv[1];
-               s0[2][qz][qy][qx] = conv[2];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      kernels::internal::EvalTranspose3d(D1D, Q1D, smem, sB, s0, s1);
-      kernels::internal::WriteDofs3d(e, D1D, s1, Y);
-   });
-}
-
-} // namespace internal
-
-template<int DIM, int T_D1D, int T_Q1D>
-VectorConvectionNLFIntegrator::AddMultPAType
-VectorConvectionNLFIntegrator::AddMultPAKernels::Kernel()
-{
-   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
-   if constexpr (DIM == 2)
-   {
-      return internal::SmemPAConvectionNLApply2D<T_D1D, T_Q1D>;
-   }
-   else if constexpr (DIM == 3)
-   {
-      return internal::SmemPAConvectionNLApply3D<T_D1D, T_Q1D>;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-inline VectorConvectionNLFIntegrator::AddMultPAType
-VectorConvectionNLFIntegrator::AddMultPAKernels::Fallback
-(int dim, int d1d, int q1d)
-{
-   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
-   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   if (dim == 2)
-   {
-      return internal::SmemPAConvectionNLApply2D<>;
-   }
-   else if (dim == 3)
-   {
-      return internal::SmemPAConvectionNLApply3D<>;
-   }
-   MFEM_ABORT("Unsupported kernel");
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
-} // namespace mfem

fem/integ/nonlininteg_vecconvection_pa_diag.cpp

@@ -1,50 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../ceed/interface/util.hpp"
-#include "./nonlininteg_vecconvection_pa_diag.hpp" // IWYU pragma: keep
-
-namespace mfem
-{
-
-void VectorConvectionNLFIntegrator::AssembleGradDiagonalPA(Vector &de) const
-{
-   MFEM_VERIFY(!DeviceCanUseCeed(),
-               "VectorConvectionNLFIntegrator PA gradients are not supported "
-               "with the libCEED backend");
-
-   if (dim == 2)
-   {
-      GradDiagPA2D::Run(d1d, q1d, ne,
-                        maps->B.Read(),
-                        maps->G.Read(),
-                        pa_adj.Read(),
-                        pa_u.Read(),
-                        de.ReadWrite(),
-                        d1d, q1d);
-   }
-   else if (dim == 3)
-   {
-      GradDiagPA3D::Run(d1d, q1d, ne,
-                        maps->B.Read(),
-                        maps->G.Read(),
-                        pa_adj.Read(),
-                        pa_u.Read(),
-                        de.ReadWrite(),
-                        d1d, q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-   }
-}
-
-} // namespace mfem

fem/integ/nonlininteg_vecconvection_pa_diag.hpp

@@ -1,302 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../config/config.hpp"
-#include "../../general/forall.hpp"
-#include "../../linalg/dtensor.hpp"
-#include "../kernels.hpp"
-#include "../nonlininteg.hpp"
-
-namespace mfem
-{
-
-/// \cond DO_NOT_DOCUMENT
-
-namespace internal
-{
-
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAConvectionNLGradDiagonal2D(const int NE,
-                                             const real_t *b,
-                                             const real_t *g,
-                                             const real_t *a,
-                                             const real_t *u,
-                                             real_t *de,
-                                             const int d1d,
-                                             const int q1d)
-{
-   static constexpr int VDIM = 2, DIM = 2;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, NE);
-   const auto U = Reshape(u, D1D, D1D, VDIM, NE);
-   auto D = Reshape(de, D1D, D1D, VDIM, NE);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t sM[3][MQ1][MQ1], sQ[3][MQ1][MQ1];
-      MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1];
-
-      kernels::internal::v_regs2d_t<VDIM, MQ1> r0, r1;
-      kernels::internal::vd_regs2d_t<VDIM, DIM, MQ1> g0, g1;
-
-      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
-      kernels::internal::LoadMatrix(D1D, Q1D, g, sG);
-
-      kernels::internal::LoadDofs2d(e, D1D, U, r0);
-      kernels::internal::Eval2d(D1D, Q1D, sM[0], sB, r0, r1);
-
-      kernels::internal::LoadDofs2d(e, D1D, U, g0);
-      kernels::internal::Grad2d(D1D, Q1D, sM[0], sB, sG, g0, g1);
-
-      for (int v = 0; v < VDIM; ++v)
-      {
-         future::tensor<real_t, VDIM> e_v = {};
-         e_v[v] = real_t(1);
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-            {
-               const future::tensor<real_t, VDIM> u_val =
-               {
-                  r1[0][qy][qx], r1[1][qy][qx]
-               };
-               const future::tensor<real_t, VDIM, DIM> Q_adj =
-               {
-                  {  { A(0, 0, qx, qy, e), A(1, 0, qx, qy, e) },
-                     { A(0, 1, qx, qy, e), A(1, 1, qx, qy, e) }
-                  }
-               };
-               const future::tensor<real_t, VDIM, DIM> grad_U =
-               {
-                  {  { g1[0][0][qy][qx], g1[1][0][qy][qx] },
-                     { g1[0][1][qy][qx], g1[1][1][qy][qx] }
-                  }
-               };
-               const auto one = Q_adj * u_val;
-               const auto two = transpose(grad_U) * (Q_adj * e_v);
-               sQ[0][qx][qy] = one[0];
-               sQ[1][qx][qy] = one[1];
-               sQ[2][qx][qy] = two[v];
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-            {
-               real_t s[3] = {};
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  const real_t By = sB[dy][qy], Gy = sG[dy][qy];
-                  s[0] += By * By * sQ[0][qx][qy];
-                  s[1] += Gy * By * sQ[1][qx][qy];
-                  s[2] += By * By * sQ[2][qx][qy];
-               }
-               sM[0][qx][dy] = s[0];
-               sM[1][qx][dy] = s[1];
-               sM[2][qx][dy] = s[2];
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
-            {
-               real_t d = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  const real_t Bx = sB[dx][qx], Gx = sG[dx][qx];
-                  d += Gx * Bx * sM[0][qx][dy] +
-                       Bx * Bx * sM[1][qx][dy] +
-                       Bx * Bx * sM[2][qx][dy];
-               }
-               D(dx, dy, v, e) += d;
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAConvectionNLGradDiagonal3D(const int NE,
-                                             const real_t *b,
-                                             const real_t *g,
-                                             const real_t *a,
-                                             const real_t *u,
-                                             real_t *de,
-                                             const int d1d,
-                                             const int q1d)
-{
-   static constexpr int VDIM = 3, DIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, Q1D, NE);
-   const auto U = Reshape(u, D1D, D1D, D1D, VDIM, NE);
-   auto D = Reshape(de, D1D, D1D, D1D, VDIM, NE);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(NE, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t sM[4][MQ1][MQ1], sQ[4][MQ1][MQ1];
-      MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1];
-
-      kernels::internal::v_regs3d_t<VDIM, MQ1> r0, r1;
-      kernels::internal::vd_regs3d_t<VDIM, DIM, MQ1> g0, g1;
-
-      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
-      kernels::internal::LoadMatrix(D1D, Q1D, g, sG);
-
-      kernels::internal::LoadDofs3d(e, D1D, U, r0);
-      kernels::internal::Eval3d(D1D, Q1D, sM[0], sB, r0, r1);
-
-      kernels::internal::LoadDofs3d(e, D1D, U, g0);
-      kernels::internal::Grad3d(D1D, Q1D, sM[0], sB, sG, g0, g1);
-
-      for (int v = 0; v < VDIM; ++v)
-      {
-         future::tensor<real_t, VDIM> e_v = {};
-         e_v[v] = real_t(1);
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-               {
-                  real_t s[4] = {};
-                  for (int qz = 0; qz < Q1D; ++qz)
-                  {
-                     const future::tensor<real_t, VDIM> u_val =
-                     {
-                        r1[0][qz][qy][qx], r1[1][qz][qy][qx], r1[2][qz][qy][qx]
-                     };
-                     const future::tensor<real_t, VDIM, DIM> Q_adj = {{
-                           {A(0,0,qx,qy,qz,e), A(1,0,qx,qy,qz,e), A(2,0,qx,qy,qz,e)},
-                           {A(0,1,qx,qy,qz,e), A(1,1,qx,qy,qz,e), A(2,1,qx,qy,qz,e)},
-                           {A(0,2,qx,qy,qz,e), A(1,2,qx,qy,qz,e), A(2,2,qx,qy,qz,e)}
-                        }
-                     };
-                     const future::tensor<real_t, VDIM, DIM> grad_U = {{
-                           {g1[0][0][qz][qy][qx], g1[1][0][qz][qy][qx], g1[2][0][qz][qy][qx]},
-                           {g1[0][1][qz][qy][qx], g1[1][1][qz][qy][qx], g1[2][1][qz][qy][qx]},
-                           {g1[0][2][qz][qy][qx], g1[1][2][qz][qy][qx], g1[2][2][qz][qy][qx]}
-                        }
-                     };
-                     const auto one = Q_adj * u_val;
-                     const auto two = transpose(grad_U) * (Q_adj * e_v);
-
-                     const real_t Bz = sB[dz][qz], Gz = sG[dz][qz];
-                     s[0] += one[0] * Bz * Bz;
-                     s[1] += one[1] * Bz * Bz;
-                     s[2] += one[2] * Bz * Gz;
-                     s[3] += two[v] * Bz * Bz;
-                  }
-                  sQ[0][qx][qy] = s[0];
-                  sQ[1][qx][qy] = s[1];
-                  sQ[2][qx][qy] = s[2];
-                  sQ[3][qx][qy] = s[3];
-               }
-            }
-            MFEM_SYNC_THREAD;
-
-            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-               {
-                  real_t s[4] = {};
-                  for (int qy = 0; qy < Q1D; ++qy)
-                  {
-                     const real_t By = sB[dy][qy], Gy = sG[dy][qy];
-                     s[0] += By * By * sQ[0][qx][qy];
-                     s[1] += Gy * By * sQ[1][qx][qy];
-                     s[2] += By * By * sQ[2][qx][qy];
-                     s[3] += By * By * sQ[3][qx][qy];
-                  }
-                  sM[0][dy][qx] = s[0];
-                  sM[1][dy][qx] = s[1];
-                  sM[2][dy][qx] = s[2];
-                  sM[3][dy][qx] = s[3];
-               }
-            }
-            MFEM_SYNC_THREAD;
-
-            MFEM_FOREACH_THREAD_DIRECT(dy, y, D1D)
-            {
-               MFEM_FOREACH_THREAD_DIRECT(dx, x, D1D)
-               {
-                  real_t d = 0.0;
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     const real_t Bx = sB[dx][qx], Gx = sG[dx][qx];
-                     d += Gx * Bx * sM[0][dy][qx];
-                     d += Bx * Bx * sM[1][dy][qx];
-                     d += Bx * Bx * sM[2][dy][qx];
-                     d += Bx * Bx * sM[3][dy][qx];
-                  }
-                  D(dx, dy, dz, v, e) += d;
-               }
-            }
-            MFEM_SYNC_THREAD;
-         }
-      }
-   });
-}
-
-} // namespace internal
-
-template<int T_D1D, int T_Q1D>
-VectorConvectionNLFIntegrator::GradDiagPAType
-VectorConvectionNLFIntegrator::GradDiagPA2D::Kernel()
-{
-   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
-   return internal::SmemPAConvectionNLGradDiagonal2D<T_D1D, T_Q1D>;
-}
-
-inline VectorConvectionNLFIntegrator::GradDiagPAType
-VectorConvectionNLFIntegrator::GradDiagPA2D::Fallback(int d1d, int q1d)
-{
-   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
-   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   return internal::SmemPAConvectionNLGradDiagonal2D<>;
-}
-
-template<int T_D1D, int T_Q1D>
-VectorConvectionNLFIntegrator::GradDiagPAType
-VectorConvectionNLFIntegrator::GradDiagPA3D::Kernel()
-{
-   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
-   return internal::SmemPAConvectionNLGradDiagonal3D<T_D1D, T_Q1D>;
-}
-
-inline VectorConvectionNLFIntegrator::GradDiagPAType
-VectorConvectionNLFIntegrator::GradDiagPA3D::Fallback(int d1d, int q1d)
-{
-   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
-   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   return internal::SmemPAConvectionNLGradDiagonal3D<>;
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
-} // namespace mfem

fem/integ/nonlininteg_vecconvection_pa_grad.cpp

@@ -1,64 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../ceed/interface/util.hpp"
-#include "./nonlininteg_vecconvection_pa_grad.hpp" // IWYU pragma: keep
-
-namespace mfem
-{
-
-void VectorConvectionNLFIntegrator::AssembleGradPA(
-   const Vector &u, const FiniteElementSpace &fes)
-{
-   MFEM_VERIFY(!DeviceCanUseCeed(),
-               "VectorConvectionNLFIntegrator PA gradients are not supported "
-               "with the libCEED backend");
-
-   this->pa_u = u;
-   AssemblePA(fes);
-}
-
-void VectorConvectionNLFIntegrator::AddMultGradPA(const Vector &x,
-                                                  Vector &y) const
-{
-   MFEM_VERIFY(!DeviceCanUseCeed(),
-               "VectorConvectionNLFIntegrator PA gradients are not supported "
-               "with the libCEED backend");
-
-   if (dim == 2)
-   {
-      AddMultGradPA2D::Run(d1d, q1d, ne,
-                           maps->B.Read(),
-                           maps->G.Read(),
-                           pa_adj.Read(),
-                           pa_u.Read(),
-                           x.Read(),
-                           y.ReadWrite(),
-                           d1d, q1d);
-   }
-   else if (dim == 3)
-   {
-      AddMultGradPA3D::Run(d1d, q1d, ne,
-                           maps->B.Read(),
-                           maps->G.Read(),
-                           pa_adj.Read(),
-                           pa_u.Read(),
-                           x.Read(),
-                           y.ReadWrite(),
-                           d1d, q1d);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension");
-   }
-}
-
-} // namespace mfem

fem/integ/nonlininteg_vecconvection_pa_grad.hpp

@@ -1,257 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "../../config/config.hpp"
-#include "../../general/forall.hpp"
-#include "../../linalg/dtensor.hpp"
-#include "../kernels.hpp"
-#include "../nonlininteg.hpp"
-
-namespace mfem
-{
-
-/// \cond DO_NOT_DOCUMENT
-
-namespace internal
-{
-
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAConvectionNLGradApply2D(const int ne,
-                                          const real_t *b,
-                                          const real_t *g,
-                                          const real_t *a,
-                                          const real_t *u,
-                                          const real_t *du,
-                                          real_t *y,
-                                          const int d1d,
-                                          const int q1d)
-{
-   static constexpr int VDIM = 2, DIM = 2;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, ne);
-   const auto U = Reshape(u, D1D, D1D, VDIM, ne);
-   const auto dU = Reshape(du, D1D, D1D, VDIM, ne);
-   auto Y = Reshape(y, D1D, D1D, VDIM, ne);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(ne, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1];
-
-      kernels::internal::vd_regs2d_t<VDIM, DIM, MQ1> g0, g1, g2;
-      kernels::internal::v_regs2d_t<DIM, MQ1> r0, r1, r2;
-
-      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
-      kernels::internal::LoadMatrix(D1D, Q1D, g, sG);
-
-      kernels::internal::LoadDofs2d(e, D1D, dU, g0);
-      kernels::internal::Grad2d(D1D, Q1D, smem, sB, sG, g0, g1); // δu gradient
-
-      kernels::internal::LoadDofs2d(e, D1D, U, r0);
-      kernels::internal::Eval2d(D1D, Q1D, smem, sB, r0, r2); // u value
-
-      kernels::internal::LoadDofs2d(e, D1D, dU, r0);
-      kernels::internal::Eval2d(D1D, Q1D, smem, sB, r0, r1); // δu value
-
-      kernels::internal::LoadDofs2d(e, D1D, U, g0);
-      kernels::internal::Grad2d(D1D, Q1D, smem, sB, sG, g0, g2); // u gradient
-
-      MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-         {
-            // First part of the Jacobian: u·∇δu
-            const future::tensor<real_t, DIM> u_val =
-            {
-               r2[0][qy][qx], r2[1][qy][qx]
-            };
-            const future::tensor<real_t, VDIM, DIM> Q_adj =
-            {
-               {  { A(0, 0, qx, qy, e), A(1, 0, qx, qy, e) },
-                  { A(0, 1, qx, qy, e), A(1, 1, qx, qy, e) }
-               }
-            };
-            const future::tensor<real_t, VDIM, DIM> grad_dU =
-            {
-               {  { g1[0][0][qy][qx], g1[1][0][qy][qx] },
-                  { g1[0][1][qy][qx], g1[1][1][qy][qx] }
-               }
-            };
-            const auto one = transpose(grad_dU) * (Q_adj * u_val);
-
-            // Second part of the Jacobian: δu·∇u
-            const future::tensor<real_t, DIM> du_val =
-            {
-               r1[0][qy][qx], r1[1][qy][qx]
-            };
-            const future::tensor<real_t, VDIM, DIM> grad_U =
-            {
-               {  { g2[0][0][qy][qx], g2[1][0][qy][qx] },
-                  { g2[0][1][qy][qx], g2[1][1][qy][qx] }
-               }
-            };
-            const auto two = transpose(grad_U) * (Q_adj * du_val);
-
-            // u⋅∇δu + δu⋅∇u
-            r0[0][qy][qx] = one[0] + two[0];
-            r0[1][qy][qx] = one[1] + two[1];
-         }
-      }
-      MFEM_SYNC_THREAD;
-      kernels::internal::EvalTranspose2d(D1D, Q1D, smem, sB, r0, r1);
-      kernels::internal::WriteDofs2d(e, D1D, r1, Y);
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-inline void SmemPAConvectionNLGradApply3D(const int ne,
-                                          const real_t *b,
-                                          const real_t *g,
-                                          const real_t *a,
-                                          const real_t *u,
-                                          const real_t *du,
-                                          real_t *y,
-                                          const int d1d,
-                                          const int q1d)
-{
-   static constexpr int VDIM = 3, DIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto A = Reshape(a, VDIM, DIM, Q1D, Q1D, Q1D, ne);
-   const auto U = Reshape(u, D1D, D1D, D1D, VDIM, ne);
-   const auto dU = Reshape(du, D1D, D1D, D1D, VDIM, ne);
-   auto Y = Reshape(y, D1D, D1D, D1D, VDIM, ne);
-
-   mfem::forall_2D<T_Q1D * T_Q1D>(ne, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int e)
-   {
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t smem[MQ1][MQ1];
-      MFEM_SHARED real_t sB[MD1][MQ1], sG[MD1][MQ1];
-
-      kernels::internal::v_regs3d_t<VDIM, MQ1> r0, r1, r2;
-      kernels::internal::vd_regs3d_t<VDIM, DIM, MQ1> g0, g1, g2;
-
-      kernels::internal::LoadMatrix(D1D, Q1D, b, sB);
-      kernels::internal::LoadMatrix(D1D, Q1D, g, sG);
-
-      kernels::internal::LoadDofs3d(e, D1D, dU, g0);
-      kernels::internal::Grad3d(D1D, Q1D, smem, sB, sG, g0, g1); // δu gradient
-
-      kernels::internal::LoadDofs3d(e, D1D, U, r0);
-      kernels::internal::Eval3d(D1D, Q1D, smem, sB, r0, r2); // u value
-
-      kernels::internal::LoadDofs3d(e, D1D, dU, r0);
-      kernels::internal::Eval3d(D1D, Q1D, smem, sB, r0, r1); // δu value
-
-      kernels::internal::LoadDofs3d(e, D1D, U, g0);
-      kernels::internal::Grad3d(D1D, Q1D, smem, sB, sG, g0, g2); // u gradient
-
-      for (int qz = 0; qz < Q1D; qz++)
-      {
-         MFEM_FOREACH_THREAD_DIRECT(qy, y, Q1D)
-         {
-            MFEM_FOREACH_THREAD_DIRECT(qx, x, Q1D)
-            {
-               // First part of the Jacobian: u·∇δu
-               const future::tensor<real_t, DIM> u_val =
-               {
-                  r2[0][qz][qy][qx],
-                  r2[1][qz][qy][qx],
-                  r2[2][qz][qy][qx]
-               };
-               const future::tensor<real_t, VDIM, DIM> Q_adj = {{
-                     {A(0,0,qx,qy,qz,e), A(1,0,qx,qy,qz,e), A(2,0,qx,qy,qz,e)},
-                     {A(0,1,qx,qy,qz,e), A(1,1,qx,qy,qz,e), A(2,1,qx,qy,qz,e)},
-                     {A(0,2,qx,qy,qz,e), A(1,2,qx,qy,qz,e), A(2,2,qx,qy,qz,e)}
-                  }
-               };
-               const future::tensor<real_t, DIM, DIM> grad_dU = {{
-                     {g1[0][0][qz][qy][qx], g1[1][0][qz][qy][qx], g1[2][0][qz][qy][qx]},
-                     {g1[0][1][qz][qy][qx], g1[1][1][qz][qy][qx], g1[2][1][qz][qy][qx]},
-                     {g1[0][2][qz][qy][qx], g1[1][2][qz][qy][qx], g1[2][2][qz][qy][qx]}
-                  }
-               };
-               const auto one = transpose(grad_dU) * (Q_adj * u_val);
-
-               // Second part of the Jacobian: δu·∇u
-               const future::tensor<real_t, DIM> du_val =
-               {
-                  r1[0][qz][qy][qx], r1[1][qz][qy][qx], r1[2][qz][qy][qx]
-               };
-               const future::tensor<real_t, VDIM, DIM> grad_U = {{
-                     {g2[0][0][qz][qy][qx], g2[1][0][qz][qy][qx], g2[2][0][qz][qy][qx]},
-                     {g2[0][1][qz][qy][qx], g2[1][1][qz][qy][qx], g2[2][1][qz][qy][qx]},
-                     {g2[0][2][qz][qy][qx], g2[1][2][qz][qy][qx], g2[2][2][qz][qy][qx]}
-                  }
-               };
-               const auto two = transpose(grad_U) * (Q_adj * du_val);
-
-               // u⋅∇δu + δu⋅∇u
-               r0[0][qz][qy][qx] = one[0] + two[0];
-               r0[1][qz][qy][qx] = one[1] + two[1];
-               r0[2][qz][qy][qx] = one[2] + two[2];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      kernels::internal::EvalTranspose3d(D1D, Q1D, smem, sB, r0, r1);
-      kernels::internal::WriteDofs3d(e, D1D, r1, Y);
-   });
-}
-
-} // namespace internal
-
-template<int T_D1D, int T_Q1D>
-VectorConvectionNLFIntegrator::AddMultGradPAType
-VectorConvectionNLFIntegrator::AddMultGradPA2D::Kernel()
-{
-   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
-   return internal::SmemPAConvectionNLGradApply2D<T_D1D, T_Q1D>;
-}
-
-inline VectorConvectionNLFIntegrator::AddMultGradPAType
-VectorConvectionNLFIntegrator::AddMultGradPA2D::Fallback(int d1d, int q1d)
-{
-   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
-   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   return internal::SmemPAConvectionNLGradApply2D<>;
-}
-
-template<int T_D1D, int T_Q1D>
-VectorConvectionNLFIntegrator::AddMultGradPAType
-VectorConvectionNLFIntegrator::AddMultGradPA3D::Kernel()
-{
-   static_assert(T_D1D <= T_Q1D, "d1d > q1d is not supported");
-   return internal::SmemPAConvectionNLGradApply3D<T_D1D, T_Q1D>;
-}
-
-inline VectorConvectionNLFIntegrator::AddMultGradPAType
-VectorConvectionNLFIntegrator::AddMultGradPA3D::Fallback(int d1d, int q1d)
-{
-   MFEM_VERIFY(d1d <= q1d, "d1d > q1d is not supported");
-   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   return internal::SmemPAConvectionNLGradApply3D<>;
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
-} // namespace mfem

fem/intrules.cpp

@@ -542,10 +542,7 @@ void QuadratureFunctions1D::GaussJacobi(const int np, const real_t alpha,
          return;
    }
 
-#ifdef MFEM_USE_MPFR
-   MFEM_WARNING("MPFR implementation of Gauss-Jacobi quadrature not implemented yet. Falling "
-                "back to double precision implementation...");
-#endif
+#ifndef MFEM_USE_MPFR
 
    const int n = np;
    // common constants for Jacobi polynomials
@@ -614,6 +611,13 @@ void QuadratureFunctions1D::GaussJacobi(const int np, const real_t alpha,
                                                 ab + 1) / ((1.0 - xi*xi)*pp*pp) / pow(2, ab);
       // map nodes and weights to the interval [0,1]
    }
+
+#else // MFEM_USE_MPFR is defined
+
+   MFEM_ABORT("MPFR implementation of Gauss-Jacobi quadrature not defined yet");
+
+#endif // MFEM_USE_MPFR
+
 }
 
 

fem/kernels.hpp

@@ -83,7 +83,7 @@ constexpr int SetMaxOf(int n) { return NextMultipleOf<4>(n); }
 #endif // CUDA/HIP && DEVICE_COMPILE
 
 /// Load 2D matrix into shared memory
-template <int MQ1, bool TRANSPOSE = false>
+template <int MQ1>
 inline MFEM_HOST_DEVICE void LoadMatrix(const int d1d, const int q1d,
                                         const real_t *M, real_t (*N)[MQ1])
 {
@@ -91,14 +91,7 @@ inline MFEM_HOST_DEVICE void LoadMatrix(const int d1d, const int q1d,
    {
       MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
       {
-         if constexpr (TRANSPOSE)
-         {
-            N[dy][qx] = M[qx * d1d + dy];
-         }
-         else
-         {
-            N[dy][qx] = M[dy * q1d + qx];
-         }
+         N[dy][qx] = M[dy * q1d + qx];
       }
    }
    MFEM_SYNC_THREAD;

fem/lor/lor_dg_impl.hpp

@@ -258,8 +258,8 @@ void BatchedLOR_DG::Assemble2D()
                   const int w_idx = (n_idx == 0) ? iy : ix;
                   const int x_idx = (n_idx == 0) ? i_0 : j_0;
 
-                  const real_t J1 = J(ix, iy, n_idx, !n_idx, iel_ho);
-                  const real_t J2 = J(ix, iy, !n_idx, !n_idx, iel_ho);
+                  const real_t J1 = J(ix, iy, n_idx, (int)!n_idx, iel_ho);
+                  const real_t J2 = J(ix, iy, (int)!n_idx, (int)!n_idx, iel_ho);
                   const real_t Jh = (J1*J1 + J2*J2) / detJ(ix, iy, iel_ho);
 
                   V(v_idx, ix, iy, iel_ho) =

fem/nonlinearform_ext.cpp

@@ -100,17 +100,6 @@ PANonlinearFormExtension::Gradient::Gradient(const PANonlinearFormExtension &e):
 
 void PANonlinearFormExtension::Gradient::AssembleGrad(const Vector &g)
 {
-   if (DeviceCanUseCeed())
-   {
-      for (int i = 0; i < ext.dnfi.Size(); ++i)
-      {
-         MFEM_VERIFY(dynamic_cast<VectorConvectionNLFIntegrator *>
-                     (ext.dnfi[i]) == nullptr,
-                     "VectorConvectionNLFIntegrator PA gradients are not supported "
-                     "with the libCEED backend");
-      }
-   }
-
    ext.elemR->Mult(g, ext.xe);
    for (int i = 0; i < ext.dnfi.Size(); ++i)
    {

fem/nonlininteg.cpp

@@ -954,74 +954,4 @@ void SkewSymmetricVectorConvectionNLFIntegrator::AssembleElementGrad(
    }
 }
 
-void ConvectiveVectorConvectionNLFIntegrator::AssemblePA(
-   const FiniteElementSpace &)
-{
-   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void ConvectiveVectorConvectionNLFIntegrator::AssembleGradPA(
-   const Vector &, const FiniteElementSpace &)
-{
-   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void ConvectiveVectorConvectionNLFIntegrator::AddMultPA(
-   const Vector &, Vector &) const
-{
-   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void ConvectiveVectorConvectionNLFIntegrator::AddMultGradPA(
-   const Vector &, Vector &) const
-{
-   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void ConvectiveVectorConvectionNLFIntegrator::AssembleGradDiagonalPA(
-   Vector &) const
-{
-   MFEM_ABORT("ConvectiveVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void SkewSymmetricVectorConvectionNLFIntegrator::AssemblePA(
-   const FiniteElementSpace &)
-{
-   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void SkewSymmetricVectorConvectionNLFIntegrator::AssembleGradPA(
-   const Vector &, const FiniteElementSpace &)
-{
-   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void SkewSymmetricVectorConvectionNLFIntegrator::AddMultPA(
-   const Vector &, Vector &) const
-{
-   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void SkewSymmetricVectorConvectionNLFIntegrator::AddMultGradPA(
-   const Vector &, Vector &) const
-{
-   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
-void SkewSymmetricVectorConvectionNLFIntegrator::AssembleGradDiagonalPA(
-   Vector &) const
-{
-   MFEM_ABORT("SkewSymmetricVectorConvectionNLFIntegrator does not support "
-              "partial assembly; use VectorConvectionNLFIntegrator");
-}
-
 }

fem/nonlininteg.hpp

@@ -18,7 +18,6 @@
 #include "fespace.hpp"
 #include "ceed/interface/operator.hpp"
 #include "integrator.hpp"
-#include "kernel_dispatch.hpp"
 
 namespace mfem
 {
@@ -385,17 +384,15 @@ private:
    DenseMatrix dshape, dshapex, EF, gradEF, ELV, elmat_comp;
    Vector shape;
    // PA extension
-   int dim, ne, nq, d1d, q1d;
-   Vector pa_adj, pa_u;
+   Vector pa_data;
    const DofToQuad *maps;         ///< Not owned
    const GeometricFactors *geom;  ///< Not owned
+   int dim, ne, nq;
 
 public:
-   struct Kernels { Kernels(); };
+   VectorConvectionNLFIntegrator(Coefficient &q): Q(&q) { }
 
-   VectorConvectionNLFIntegrator(Coefficient &q): Q(&q) { static Kernels kernels; }
-
-   VectorConvectionNLFIntegrator() { static Kernels kernels; }
+   VectorConvectionNLFIntegrator() = default;
 
    static const IntegrationRule &GetRule(const FiniteElement &fe,
                                          const ElementTransformation &T);
@@ -414,56 +411,13 @@ public:
 
    void AssemblePA(const FiniteElementSpace &fes) override;
 
-   void AssembleGradPA(const Vector &x, const FiniteElementSpace &fes) override;
+   void AssembleMF(const FiniteElementSpace &fes) override;
 
    void AddMultPA(const Vector &x, Vector &y) const override;
 
-   using AddMultPAType =
-      void(*)(const int ne, const real_t *B, const real_t *G, const real_t *A,
-              const real_t *x, real_t *y,
-              const int d1d, const int q1d);
-   MFEM_REGISTER_KERNELS(AddMultPAKernels, AddMultPAType, (int, int, int));
-
-   void AddMultGradPA(const Vector &x, Vector &y) const override;
-
-   using AddMultGradPAType =
-      void(*)(const int ne, const real_t *B, const real_t *G, const real_t *A,
-              const real_t *u, const real_t *x, real_t *y,
-              const int d1d, const int q1d);
-
-   MFEM_REGISTER_KERNELS(AddMultGradPA2D, AddMultGradPAType, (int, int));
-   MFEM_REGISTER_KERNELS(AddMultGradPA3D, AddMultGradPAType, (int, int));
-
-   void AssembleGradDiagonalPA(Vector &) const override;
-
-   using GradDiagPAType =
-      void (*)(const int ne, const real_t *B, const real_t *G, const real_t *A,
-               const real_t *u, real_t *y,
-               const int d1d, const int q1d);
-
-   MFEM_REGISTER_KERNELS(GradDiagPA2D, GradDiagPAType, (int, int));
-   MFEM_REGISTER_KERNELS(GradDiagPA3D, GradDiagPAType, (int, int));
-
-   template <int DIM, int D1D, int Q1D>
-   static void AddSpecialization()
-   {
-      AddMultPAKernels::Specialization<DIM, D1D, Q1D>::Add();
-      if constexpr (DIM == 2)
-      {
-         AddMultGradPA2D::Specialization<D1D, Q1D>::Add();
-         GradDiagPA2D::Specialization<D1D, Q1D>::Add();
-      }
-      else if constexpr (DIM == 3)
-      {
-         AddMultGradPA3D::Specialization<D1D, Q1D>::Add();
-         GradDiagPA3D::Specialization<D1D, Q1D>::Add();
-      }
-   }
-
-   void AssembleMF(const FiniteElementSpace &fes) override;
-
    void AddMultMF(const Vector &x, Vector &y) const override;
 
+
 protected:
    const IntegrationRule* GetDefaultIntegrationRule(
       const FiniteElement& trial_fe,
@@ -476,8 +430,7 @@ protected:
 
 
 /** This class is used to assemble the convective form of the nonlinear term
-    arising in the Navier-Stokes equations $(u \cdot \nabla v, w )$.
-    Partial assembly is not supported; use VectorConvectionNLFIntegrator. */
+    arising in the Navier-Stokes equations $(u \cdot \nabla v, w )$ */
 class ConvectiveVectorConvectionNLFIntegrator :
    public VectorConvectionNLFIntegrator
 {
@@ -495,20 +448,12 @@ public:
                             ElementTransformation &trans,
                             const Vector &elfun,
                             DenseMatrix &elmat) override;
-
-   using NonlinearFormIntegrator::AssemblePA;
-   void AssemblePA(const FiniteElementSpace &fes) override;
-   void AssembleGradPA(const Vector &x, const FiniteElementSpace &fes) override;
-   void AddMultPA(const Vector &x, Vector &y) const override;
-   void AddMultGradPA(const Vector &x, Vector &y) const override;
-   void AssembleGradDiagonalPA(Vector &diag) const override;
 };
 
 
 /** This class is used to assemble the skew-symmetric form of the nonlinear term
     arising in the Navier-Stokes equations
-    $.5*(u \cdot \nabla v, w ) - .5*(u \cdot \nabla w, v )$.
-    Partial assembly is not supported; use VectorConvectionNLFIntegrator. */
+    $.5*(u \cdot \nabla v, w ) - .5*(u \cdot \nabla w, v )$ */
 class SkewSymmetricVectorConvectionNLFIntegrator :
    public VectorConvectionNLFIntegrator
 {
@@ -526,13 +471,6 @@ public:
                             ElementTransformation &trans,
                             const Vector &elfun,
                             DenseMatrix &elmat) override;
-
-   using NonlinearFormIntegrator::AssemblePA;
-   void AssemblePA(const FiniteElementSpace &fes) override;
-   void AssembleGradPA(const Vector &x, const FiniteElementSpace &fes) override;
-   void AddMultPA(const Vector &x, Vector &y) const override;
-   void AddMultGradPA(const Vector &x, Vector &y) const override;
-   void AssembleGradDiagonalPA(Vector &diag) const override;
 };
 
 }

fem/particleset.cpp

@@ -10,7 +10,6 @@
 // CONTRIBUTING.md for details.
 
 #include "particleset.hpp"
-#include "../general/forall.hpp"
 
 #if defined(MFEM_USE_MPI) && defined(MFEM_USE_GSLIB)
 
@@ -226,7 +225,6 @@ void ParticleSet::AddParticles(const Array<IDType> &new_ids,
       }
    }
    // Add new ids
-   ids.HostReadWrite();
    ids.Append(new_ids);
 
    // Update data
@@ -246,102 +244,6 @@ void ParticleSet::AddParticles(const Array<IDType> &new_ids,
 #if defined(MFEM_USE_MPI) && defined(MFEM_USE_GSLIB)
 
 /// \cond DO_NOT_DOCUMENT
-// Static helper: gather selected particle-vector entries into a compact buffer.
-// nvcc does not allow extended host/device lambdas in non-public members.
-static void GatherParticleVectorDevice(const ParticleVector &pv,
-                                       const Array<int> &send_idxs,
-                                       Vector &send_data,
-                                       int nsend)
-{
-   const int vdim = pv.GetVDim();
-   const int ordering = pv.GetOrdering();
-   const int num_particles = pv.GetNumParticles();
-   const MemoryClass device_mc = Device::GetDeviceMemoryClass();
-   send_data.SetSize(nsend*vdim);
-   real_t *d_send_data =
-      send_data.GetMemory().Write(device_mc, send_data.Size());
-   const real_t *d_src = pv.GetMemory().Read(device_mc, pv.Size());
-   const int *d_send_idxs = send_idxs.GetMemory().Read(device_mc, nsend);
-
-   mfem::forall(nsend, [=] MFEM_HOST_DEVICE (int i)
-   {
-      const int p = d_send_idxs[i];
-      const int offset = (ordering == Ordering::byVDIM) ? p * vdim : p;
-      const int stride = (ordering == Ordering::byVDIM) ? 1 : num_particles;
-
-      for (int c = 0; c < vdim; c++)
-      {
-         d_send_data[i*vdim + c] = d_src[offset + c*stride];
-      }
-   });
-}
-
-// Static helper: gather selected tag values into a compact buffer.
-// nvcc does not allow extended host/device lambdas in non-public members.
-static void GatherParticleTagsDevice(const Array<int> &tag,
-                                     const Array<int> &send_idxs,
-                                     Array<int> &send_tag,
-                                     int nsend)
-{
-   const MemoryClass device_mc = Device::GetDeviceMemoryClass();
-   send_tag.SetSize(nsend);
-   int *d_send_tag = send_tag.GetMemory().Write(device_mc, nsend);
-   const int *d_tag = tag.GetMemory().Read(device_mc, tag.Size());
-   const int *d_send_idxs = send_idxs.GetMemory().Read(device_mc, nsend);
-
-   mfem::forall(nsend, [=] MFEM_HOST_DEVICE (int i)
-   {
-      d_send_tag[i] = d_tag[d_send_idxs[i]];
-   });
-}
-
-// Static helper: scatter compact particle-vector entries to particle storage.
-// nvcc does not allow extended host/device lambdas in non-public members.
-static void ScatterParticleVectorDevice(ParticleVector &pv,
-                                        const Vector &recv_data,
-                                        const Array<int> &recv_locs,
-                                        int nrecv)
-{
-   const int vdim = pv.GetVDim();
-   const int ordering = pv.GetOrdering();
-   const int num_particles = pv.GetNumParticles();
-   const MemoryClass device_mc = Device::GetDeviceMemoryClass();
-   const real_t *d_recv_data =
-      recv_data.GetMemory().Read(device_mc, recv_data.Size());
-   const int *d_recv_locs = recv_locs.GetMemory().Read(device_mc, nrecv);
-   real_t *d_dst = pv.GetMemory().ReadWrite(device_mc, pv.Size());
-
-   mfem::forall(nrecv, [=] MFEM_HOST_DEVICE (int i)
-   {
-      const int p = d_recv_locs[i];
-      const int offset = (ordering == Ordering::byVDIM) ? p * vdim : p;
-      const int stride = (ordering == Ordering::byVDIM) ? 1 : num_particles;
-
-      for (int c = 0; c < vdim; c++)
-      {
-         d_dst[offset + c*stride] = d_recv_data[i*vdim + c];
-      }
-   });
-}
-
-// Static helper: scatter compact tag values to particle storage.
-// nvcc does not allow extended host/device lambdas in non-public members.
-static void ScatterParticleTagsDevice(Array<int> &tag,
-                                      const Array<int> &recv_tag,
-                                      const Array<int> &recv_locs,
-                                      int nrecv)
-{
-   const MemoryClass device_mc = Device::GetDeviceMemoryClass();
-   const int *d_recv_tag = recv_tag.GetMemory().Read(device_mc, nrecv);
-   const int *d_recv_locs = recv_locs.GetMemory().Read(device_mc, nrecv);
-   int *d_tag = tag.GetMemory().ReadWrite(device_mc, tag.Size());
-
-   mfem::forall(nrecv, [=] MFEM_HOST_DEVICE (int i)
-   {
-      d_tag[d_recv_locs[i]] = d_recv_tag[i];
-   });
-}
-
 template<size_t NBytes>
 void ParticleSet::TransferParticlesImpl(ParticleSet &pset,
                                         const Array<int> &send_idxs,
@@ -364,129 +266,49 @@ void ParticleSet::TransferParticlesImpl(ParticleSet &pset,
    array_init(parr_t, &gsl_arr, send_idxs.Size());
    pdata_arr = (parr_t*) gsl_arr.ptr;
 
-   int nparticles = pset.GetNParticles();
-   int nsend      = send_idxs.Size();
    gsl_arr.n = send_idxs.Size();
-
-   const int *h_send_idxs_initial = send_idxs.HostRead();
-   const IDType *h_ids = pset.GetIDs().HostRead();
    for (int i = 0; i < send_idxs.Size(); i++)
    {
       parr_t &pdata = pdata_arr[i];
-      pdata.id = h_ids[h_send_idxs_initial[i]];
-   }
+      pdata.id = pset.GetIDs()[send_idxs[i]];
 
-   // Pack coords and fields into the GSLIB send buffer. Device-resident data
-   // is first gathered into a compact device buffer so that only selected
-   // particles are copied back to host. Host-resident data is packed directly.
-   int max_vdim = pset.Coords().GetVDim();
-   for (int f = 0; f < pset.GetNFields(); f++)
-   {
-      int f_vdim = pset.Field(f).GetVDim();
-      if (f_vdim > max_vdim) { max_vdim = f_vdim; }
-   }
-   Vector send_data;
-   Array<int> send_tag;
-   if (Device::IsEnabled())
-   {
-      send_data.SetSize(nsend * max_vdim); // allocate max size over all fields
-      send_tag.SetSize(nsend);
-   }
-
-   size_t counter = 0;
-   for (int f = -1; f < pset.GetNFields(); f++)
-   {
-      const ParticleVector &pv = f == -1 ? pset.Coords() : pset.Field(f);
-      const int vdim = pv.GetVDim();
-      const int ordering = pv.GetOrdering();
-      const int num_particles = pv.GetNumParticles();
-      const bool use_dev = Device::IsEnabled() && pv.UseDevice();
-
-      if (use_dev)
+      // Copy particle data directly into pdata
+      size_t counter = 0;
+      for (int f = -1; f < pset.GetNFields(); f++)
       {
-         GatherParticleVectorDevice(pv, send_idxs, send_data, nsend);
-
-         const real_t *h_send_data = send_data.HostRead();
-         for (int i = 0; i < nsend; i++)
+         ParticleVector &pv = (f == -1 ? pset.Coords() : pset.Field(f));
+         for (int c = 0; c < pv.GetVDim(); c++)
          {
-            std::memcpy(pdata_arr[i].data.data() + counter,
-                        h_send_data + i*vdim, vdim * sizeof(real_t));
-         }
-      }
-      else
-      {
-         const real_t *h_src = pv.HostRead();
-         const int *h_send_idxs = send_idxs.HostRead();
-         for (int i = 0; i < nsend; i++)
-         {
-            parr_t &pdata = pdata_arr[i];
-            const int p = h_send_idxs[i];
-            const int offset = (ordering == Ordering::byVDIM) ? p * vdim : p;
-            const int stride = (ordering == Ordering::byVDIM) ? 1 :
-                               num_particles;
-
-            for (int c = 0; c < vdim; c++)
-            {
-               std::memcpy(pdata.data.data() + counter + c*sizeof(real_t),
-                           h_src + offset + c*stride, sizeof(real_t));
-            }
+            std::memcpy(pdata.data.data() + counter, &pv(send_idxs[i], c),
+                        sizeof(real_t));
+            counter += sizeof(real_t);
          }
       }
 
-      counter += vdim*sizeof(real_t);
-   }
-
-   // Pack tags after all real_t data. Each tag uses the same selective
-   // device gather path when its Array is device-resident.
-   for (int t = 0; t < pset.GetNTags(); t++)
-   {
-      const Array<int> &tag = pset.Tag(t);
-      const size_t tag_counter = counter + t*sizeof(int);
-      const bool use_dev = Device::IsEnabled() && tag.UseDevice();
-
-      if (use_dev)
+      // Copy tags
+      for (int t = 0; t < pset.GetNTags(); t++)
       {
-         GatherParticleTagsDevice(tag, send_idxs, send_tag, nsend);
-
-         const int *h_send_tag = send_tag.HostRead();
-         for (int i = 0; i < nsend; i++)
-         {
-            std::memcpy(pdata_arr[i].data.data() + tag_counter,
-                        h_send_tag + i, sizeof(int));
-         }
-      }
-      else
-      {
-         const int *h_tag = tag.HostRead();
-         const int *h_send_idxs = send_idxs.HostRead();
-         for (int i = 0; i < nsend; i++)
-         {
-            std::memcpy(pdata_arr[i].data.data() + tag_counter,
-                        h_tag + h_send_idxs[i], sizeof(int));
-         }
+         Array<int> &tag_arr = pset.Tag(t);
+         std::memcpy(pdata.data.data() + counter, &tag_arr[send_idxs[i]],
+                     sizeof(int));
+         counter += sizeof(int);
       }
    }
 
+   int nparticles = pset.GetNParticles();
+   int nsend      = send_idxs.Size();
+
    // Transfer particles
    sarray_transfer_ext(parr_t, &gsl_arr, send_ranks.GetData(),
                        sizeof(unsigned int), pset.cr);
 
    // Make sure we have enough space for received particles
    int nrecv = (int) gsl_arr.n;
-
-   Vector recv_data;
-   Array<int> recv_tag;
-   if (Device::IsEnabled())
-   {
-      recv_data.SetSize(nrecv * max_vdim);
-      recv_tag.SetSize(nrecv);
-   }
-
    int ndelete = nsend - nrecv;
    if (ndelete > 0)
    {
       // Remove unneeded particles
-      auto datap = const_cast<int*>(send_idxs.HostRead());
+      auto datap = const_cast<int*>(send_idxs.GetData());
       Array<int> delete_idxs(datap + nrecv, ndelete);
       pset.RemoveParticles(delete_idxs);
    }
@@ -497,133 +319,47 @@ void ParticleSet::TransferParticlesImpl(ParticleSet &pset,
 
    pdata_arr = (parr_t*) gsl_arr.ptr;
 
-   // Make a list of new IDs to add
-   int num_new = nrecv > nsend ? nrecv - nsend : 0;
-   Array<IDType> new_ids(num_new);
-   for (int i = 0; i < num_new; i++)
-   {
-      new_ids[i] = pdata_arr[nsend + i].id;
-   }
-
-   // Add particles in batch
-   Array<int> new_indices;
-   if (num_new > 0)
-   {
-      pset.AddParticles(new_ids, &new_indices);
-   }
-
-   // Map each received packet to the local particle slot it updates.
-   Array<int> recv_locs(nrecv);
-   int *h_recv_locs = recv_locs.HostWrite();
-   const int *h_send_idxs_recv = send_idxs.HostRead();
+   // Add newly-recvd data directly to active state
    for (int i = 0; i < nrecv; i++)
    {
       parr_t &pdata = pdata_arr[i];
+      IDType id = pdata.id;
+
+      int new_loc_idx;
       if (i < nsend) // update existing particle
       {
-         h_recv_locs[i] = h_send_idxs_recv[i];
-         pset.UpdateID(h_recv_locs[i], pdata.id);
+         new_loc_idx = send_idxs[i];
+         pset.UpdateID(new_loc_idx, id);
       }
       else
       {
-         h_recv_locs[i] = new_indices[i - nsend];
+         // add new particle
+         Array<int> idx_temp;
+         pset.AddParticles(Array<IDType>({id}), &idx_temp);
+         new_loc_idx = idx_temp[0]; // Get index of newly-added particle
       }
-   }
 
-   // Unpack coords and fields from GSLIB host packets. Device-resident
-   // destinations use a compact host buffer followed by a device scatter.
-   size_t recv_counter = 0;
-   for (int f = -1; f < pset.GetNFields(); f++)
-   {
-      ParticleVector &pv = (f == -1 ? pset.Coords() : pset.Field(f));
-      const int vdim = pv.GetVDim();
-      const int ordering = pv.GetOrdering();
-      const int num_particles = pv.GetNumParticles();
-      const bool use_dev = Device::IsEnabled() && pv.UseDevice();
-
-      if (use_dev)
+      size_t counter = 0;
+      for (int f = -1; f < pset.GetNFields(); f++)
       {
-         recv_data.SetSize(nrecv*vdim);
-         real_t *h_recv_data = recv_data.HostWrite();
-
-         for (int i = 0; i < nrecv; i++)
+         ParticleVector &pv = (f == -1 ? pset.Coords() : pset.Field(f));
+         for (int c = 0; c < pv.GetVDim(); c++)
          {
-            std::memcpy(h_recv_data + i*vdim,
-                        pdata_arr[i].data.data() + recv_counter,
-                        vdim*sizeof(real_t));
-         }
-
-         ScatterParticleVectorDevice(pv, recv_data, recv_locs, nrecv);
-      }
-      else
-      {
-         real_t *h_dst = pv.HostReadWrite();
-         const int *h_recv_locs_read = recv_locs.HostRead();
-         for (int i = 0; i < nrecv; i++)
-         {
-            parr_t &pdata = pdata_arr[i];
-            const int p = h_recv_locs_read[i];
-            const int offset = (ordering == Ordering::byVDIM) ? p * vdim : p;
-            const int stride = (ordering == Ordering::byVDIM) ? 1 :
-                               num_particles;
-
-            for (int c = 0; c < vdim; c++)
-            {
-               std::memcpy(h_dst + offset + c*stride,
-                           pdata.data.data() + recv_counter + c*sizeof(real_t),
-                           sizeof(real_t));
-            }
+            real_t& val = pv(new_loc_idx, c);
+            std::memcpy(&val, pdata.data.data() + counter, sizeof(real_t));
+            counter += sizeof(real_t);
          }
       }
 
-      recv_counter += vdim*sizeof(real_t);
-   }
-
-   // Unpack tags after all real_t data, using the same compact scatter path
-   // for device-resident tag arrays.
-   for (int t = 0; t < pset.GetNTags(); t++)
-   {
-      Array<int> &tag = pset.Tag(t);
-      const size_t tag_counter = recv_counter + t*sizeof(int);
-      const bool use_dev = Device::IsEnabled() && tag.UseDevice();
-
-      if (use_dev)
+      for (int t = 0; t < pset.GetNTags(); t++)
       {
-         recv_tag.SetSize(nrecv);
-         int *h_recv_tag = recv_tag.HostWrite();
-
-         for (int i = 0; i < nrecv; i++)
-         {
-            std::memcpy(h_recv_tag + i,
-                        pdata_arr[i].data.data() + tag_counter, sizeof(int));
-         }
-
-         ScatterParticleTagsDevice(tag, recv_tag, recv_locs, nrecv);
-      }
-      else
-      {
-         int *h_tag = tag.HostReadWrite();
-         const int *h_recv_locs_read = recv_locs.HostRead();
-         for (int i = 0; i < nrecv; i++)
-         {
-            std::memcpy(h_tag + h_recv_locs_read[i],
-                        pdata_arr[i].data.data() + tag_counter, sizeof(int));
-         }
+         Array<int> &tag_arr = pset.Tag(t);
+         std::memcpy(&tag_arr[new_loc_idx],
+                     pdata.data.data() + counter, sizeof(int));
+         counter += sizeof(int);
       }
    }
    array_free(&gsl_arr);
-
-   // Restore Device validity if needed
-   for (int f = -1; f < pset.GetNFields(); f++)
-   {
-      ParticleVector &pv = (f == -1 ? pset.Coords() : pset.Field(f));
-      pv.ReadWrite(pv.UseDevice());
-   }
-   for (int t = 0; t < pset.GetNTags(); t++)
-   {
-      Array<int> &tag_arr = pset.Tag(t);
-      if (tag_arr.UseDevice()) { tag_arr.ReadWrite(true); }
-   }
 }
 
 template<size_t NBytes>
@@ -790,14 +526,11 @@ ParticleSet::ParticleSet(int id_stride_, IDType id_counter_, int num_particles,
                          int dim, Ordering::Type coords_ordering, const Array<int> &field_vdims,
                          const Array<Ordering::Type> &field_orderings,
                          const Array<const char*> &field_names_, int num_tags,
-                         const Array<const char*> &tag_names_,
-                         bool use_device)
+                         const Array<const char*> &tag_names_)
    : id_stride(id_stride_),
      id_counter(id_counter_),
      coords(dim, coords_ordering)
 {
-   if (use_device) { coords.UseDevice(true); }
-
    // Initialize fields
    for (int f = 0; f < field_vdims.Size(); f++)
    {
@@ -847,22 +580,21 @@ bool ParticleSet::IsValidParticle(const Particle &p) const
 }
 
 ParticleSet::ParticleSet(int num_particles, int dim,
-                         Ordering::Type coords_ordering,
-                         bool use_device)
+                         Ordering::Type coords_ordering)
    : ParticleSet(1, 0, num_particles, dim, coords_ordering, Array<int>(),
                  Array<Ordering::Type>(), Array<const char*>(), 0,
-                 Array<const char*>(), use_device)
+                 Array<const char*>())
 {
 
 }
 
 ParticleSet::ParticleSet(int num_particles, int dim,
                          const Array<int> &field_vdims, int num_tags,
-                         Ordering::Type all_ordering, bool use_device)
+                         Ordering::Type all_ordering)
    : ParticleSet(1, 0, num_particles, dim, all_ordering, field_vdims,
                  GetOrderingArray(all_ordering, field_vdims.Size()),
                  GetEmptyNameArray(field_vdims.Size()), num_tags,
-                 GetEmptyNameArray(num_tags), use_device)
+                 GetEmptyNameArray(num_tags))
 {
 }
 
@@ -870,11 +602,11 @@ ParticleSet::ParticleSet(int num_particles, int dim,
                          const Array<int> &field_vdims, const Array<const
                          char*> &field_names_, int num_tags,
                          const Array<const char*> &tag_names_,
-                         Ordering::Type all_ordering, bool use_device)
+                         Ordering::Type all_ordering)
    : ParticleSet(1, 0, num_particles, dim, all_ordering, field_vdims,
                  GetOrderingArray(all_ordering, field_vdims.Size()),
                  field_names_, num_tags,
-                 tag_names_, use_device)
+                 tag_names_)
 {
 
 }
@@ -884,9 +616,9 @@ ParticleSet::ParticleSet(int num_particles, int dim,
                          const Array<int> &field_vdims,
                          const Array<Ordering::Type> &field_orderings,
                          const Array<const char*> &field_names_, int num_tags,
-                         const Array<const char*> &tag_names_, bool use_device)
+                         const Array<const char*> &tag_names_)
    : ParticleSet(1, 0, num_particles, dim, coords_ordering, field_vdims,
-                 field_orderings, field_names_, num_tags, tag_names_, use_device)
+                 field_orderings, field_names_, num_tags, tag_names_)
 {
 
 }
@@ -895,21 +627,21 @@ ParticleSet::ParticleSet(int num_particles, int dim,
 
 #ifdef MFEM_USE_MPI
 ParticleSet::ParticleSet(MPI_Comm comm_, int rank_num_particles, int dim,
-                         Ordering::Type coords_ordering, bool use_device)
+                         Ordering::Type coords_ordering)
    : ParticleSet(comm_, rank_num_particles, dim, coords_ordering, Array<int>(),
                  Array<Ordering::Type>(), Array<const char*>(), 0,
-                 Array<const char*>(), use_device)
+                 Array<const char*>())
 {
 
 };
 
 ParticleSet::ParticleSet(MPI_Comm comm_, int rank_num_particles, int dim,
                          const Array<int> &field_vdims, int num_tags,
-                         Ordering::Type all_ordering, bool use_device)
+                         Ordering::Type all_ordering)
    : ParticleSet(comm_, rank_num_particles, dim, all_ordering, field_vdims,
                  GetOrderingArray(all_ordering, field_vdims.Size()),
                  GetEmptyNameArray(field_vdims.Size()), num_tags,
-                 GetEmptyNameArray(num_tags), use_device)
+                 GetEmptyNameArray(num_tags))
 {
 
 }
@@ -918,11 +650,11 @@ ParticleSet::ParticleSet(MPI_Comm comm_, int rank_num_particles, int dim,
                          const Array<int> &field_vdims, const Array<const
                          char*> &field_names_,
                          int num_tags, const Array<const char*> &tag_names_,
-                         Ordering::Type all_ordering, bool use_device)
+                         Ordering::Type all_ordering)
    : ParticleSet(comm_, rank_num_particles, dim, all_ordering, field_vdims,
                  GetOrderingArray(all_ordering, field_vdims.Size()),
                  field_names_, num_tags,
-                 tag_names_, use_device)
+                 tag_names_)
 {
 
 }
@@ -932,7 +664,7 @@ ParticleSet::ParticleSet(MPI_Comm comm_, int rank_num_particles, int dim,
                          const Array<int> &field_vdims,
                          const Array<Ordering::Type> &field_orderings,
                          const Array<const char*> &field_names_, int num_tags,
-                         const Array<const char*> &tag_names_, bool use_device)
+                         const Array<const char*> &tag_names_)
    : ParticleSet(GetSize(comm_), (IDType)GetRank(comm_),
                  rank_num_particles,
                  dim,
@@ -941,7 +673,7 @@ ParticleSet::ParticleSet(MPI_Comm comm_, int rank_num_particles, int dim,
                  field_orderings,
                  field_names_,
                  num_tags,
-                 tag_names_, use_device)
+                 tag_names_)
 {
    comm = comm_;
 #ifdef MFEM_USE_GSLIB
@@ -973,7 +705,6 @@ int ParticleSet::AddField(int vdim, Ordering::Type field_ordering,
    }
    fields.emplace_back(std::make_unique<ParticleVector>(vdim, field_ordering,
                                                         GetNParticles()));
-   if (coords.UseDevice()) { fields.back()->UseDevice(true); }
    field_names.emplace_back(field_name_str);
 
    return GetNFields() - 1;
@@ -987,7 +718,6 @@ int ParticleSet::AddTag(const char* tag_name)
       tag_name_str = GetDefaultTagName(tag_names.size());
    }
    tags.emplace_back(std::make_unique<Array<int>>(GetNParticles()));
-   if (coords.UseDevice()) { tags.back()->GetMemory().UseDevice(true); }
    tag_names.emplace_back(tag_name_str);
 
    return GetNTags() - 1;
@@ -1052,7 +782,7 @@ Particle ParticleSet::GetParticle(int i) const
 
    for (int t = 0; t < GetNTags(); t++)
    {
-      p.Tag(t) = Tag(t).HostRead()[i];
+      p.Tag(t) = Tag(t)[i];
    }
 
    return p;
@@ -1060,21 +790,13 @@ Particle ParticleSet::GetParticle(int i) const
 
 bool ParticleSet::IsParticleRefValid() const
 {
-   if (coords.GetOrdering() == Ordering::byNODES || coords.UseDevice())
+   if (coords.GetOrdering() == Ordering::byNODES)
    {
       return false;
    }
    for (int f = 0; f < GetNFields(); f++)
    {
-      if (fields[f]->GetOrdering() == Ordering::byNODES ||
-          fields[f]->UseDevice())
-      {
-         return false;
-      }
-   }
-   for (int t = 0; t < GetNTags(); t++)
-   {
-      if (tags[t]->UseDevice())
+      if (fields[f]->GetOrdering() == Ordering::byNODES)
       {
          return false;
       }
@@ -1084,10 +806,6 @@ bool ParticleSet::IsParticleRefValid() const
 
 Particle ParticleSet::GetParticleRef(int i)
 {
-   MFEM_ASSERT(IsParticleRefValid(),
-               "GetParticleRef is only valid when coordinates and fields are "
-               "ordered byVDIM and particle data is host-resident.");
-
    Particle p = CreateParticle();
 
    Coords().GetValuesRef(i, p.Coords());
@@ -1121,7 +839,7 @@ void ParticleSet::SetParticle(int i, const Particle &p)
 
    for (int t = 0; t < GetNTags(); t++)
    {
-      Tag(t).HostReadWrite()[i] = p.Tag(t);
+      Tag(t)[i] = p.Tag(t);
    }
 }
 
@@ -1182,15 +900,6 @@ void ParticleSet::PrintCSV(const char *fname, const Array<int> &field_idxs,
 #ifdef MFEM_USE_MPI
    int rank = GetRank(comm);
 #endif // MFEM_USE_MPI
-   // make sure we can read tag data on host. fields and coords will be read as
-   // needed in the loop below, so we don't need to pre-read them here.
-   for (int i = 0; i < GetNTags(); i++)
-   {
-      tags[i]->HostRead();
-   }
-   ids.HostRead();
-
-   // Write particle data
    for (int i = 0; i < GetNParticles(); i++)
    {
       ss_data << ids[i];

fem/particleset.hpp

@@ -211,12 +211,6 @@ public:
  *  byVDIM). The unique_ptrs to all the ParticleVectors are stored in the
  *  std::vector \ref fields.
  *
- *  @par Device Behavior:
- *  When a ParticleSet is constructed with \p use_device=true, \ref coords and
- *  all ParticleVector fields are marked to use device memory. Fields added
- *  later through \ref AddField inherit the current device mode (through
- *  \ref coords).
- *
  *  @par Tags:
  *  Tags represent integers associated with each particle. For a given tag,
  *  all particle data are stored in a single Array<int>. The unique_ptrs to all
@@ -375,10 +369,7 @@ protected:
     *  ID of a particle.
     */
    void UpdateID(int local_idx, IDType new_global_id)
-   {
-      ids.HostReadWrite();
-      ids[local_idx] = new_global_id;
-   }
+   { ids[local_idx] = new_global_id; }
 
    /** @brief Create a Particle object with the same spatial dimension,
     *  number of fields and field vdims, and number of tags as this ParticleSet.
@@ -408,14 +399,12 @@ protected:
     *  @param[in] field_names_        Array of field names.
     *  @param[in] num_tags            Number of tags to register.
     *  @param[in] tag_names_          Array of tag names.
-    *  @param[in] use_device          Use device memory for particle fields.
     */
    ParticleSet(int id_stride_, IDType id_counter_, int num_particles, int dim,
                Ordering::Type coords_ordering, const Array<int> &field_vdims,
                const Array<Ordering::Type> &field_orderings,
                const Array<const char*> &field_names_, int num_tags,
-               const Array<const char*> &tag_names_,
-               bool use_device);
+               const Array<const char*> &tag_names_);
 
 public:
 
@@ -424,12 +413,9 @@ public:
     *  @param[in] num_particles       Number of particles to initialize.
     *  @param[in] dim                 Particle spatial dimension.
     *  @param[in] coords_ordering     Ordering of coordinates.
-    *  @param[in] use_device          (Optional) Use device memory for particle
-    *                                 fields.
     */
    ParticleSet(int num_particles, int dim,
-               Ordering::Type coords_ordering=Ordering::byVDIM,
-               bool use_device=false);
+               Ordering::Type coords_ordering=Ordering::byVDIM);
 
    /** @brief Construct a serial ParticleSet with specified fields and tags at
     *  construction.
@@ -440,12 +426,9 @@ public:
     *  @param[in] num_tags            Number of tags to register.
     *  @param[in] all_ordering        (Optional) Ordering of coordinates and
     *                                 field ParticleVector.
-    *  @param[in] use_device          (Optional) Use device memory for particle
-    *                                 fields.
     */
    ParticleSet(int num_particles, int dim, const Array<int> &field_vdims,
-               int num_tags, Ordering::Type all_ordering=Ordering::byVDIM,
-               bool use_device=false);
+               int num_tags, Ordering::Type all_ordering=Ordering::byVDIM);
 
    /** @brief Construct a serial ParticleSet with specified fields and tags at
     *  construction, with names.
@@ -458,14 +441,11 @@ public:
     *  @param[in] tag_names_          Array of tag names.
     *  @param[in] all_ordering        (Optional) Ordering of coordinates and
     *                                 field ParticleVector.
-    *  @param[in] use_device          (Optional) Use device memory for particle
-    *                                 fields.
     */
    ParticleSet(int num_particles, int dim, const Array<int> &field_vdims,
                const Array<const char*> &field_names_, int num_tags,
                const Array<const char*> &tag_names_,
-               Ordering::Type all_ordering=Ordering::byVDIM,
-               bool use_device=false);
+               Ordering::Type all_ordering=Ordering::byVDIM);
 
    /** @brief Comprehensive serial constructor of ParticleSet.
     *
@@ -477,15 +457,12 @@ public:
     *  @param[in] field_names_        Array of field names.
     *  @param[in] num_tags            Number of tags to register.
     *  @param[in] tag_names_          Array of tag names.
-    *  @param[in] use_device          (Optional) Use device memory for particle
-    *                                 fields.
     */
    ParticleSet(int num_particles, int dim, Ordering::Type coords_ordering,
                const Array<int> &field_vdims,
                const Array<Ordering::Type> &field_orderings,
                const Array<const char*> &field_names_, int num_tags,
-               const Array<const char*> &tag_names_,
-               bool use_device=false);
+               const Array<const char*> &tag_names_);
 
 #ifdef MFEM_USE_MPI
    /** @brief Construct a parallel ParticleSet.
@@ -494,12 +471,9 @@ public:
     *  @param[in] rank_num_particles  Number of particles to initialize.
     *  @param[in] dim                 Particle spatial dimension.
     *  @param[in] coords_ordering     (Optional) Ordering of coordinates.
-    *  @param[in] use_device          (Optional) Use device memory for particle
-    *                                 fields.
     */
    ParticleSet(MPI_Comm comm_, int rank_num_particles, int dim,
-               Ordering::Type coords_ordering=Ordering::byVDIM,
-               bool use_device=false);
+               Ordering::Type coords_ordering=Ordering::byVDIM);
 
    /** @brief Construct a parallel ParticleSet with specified fields and tags
     *  at construction.
@@ -511,13 +485,10 @@ public:
     *  @param[in] num_tags            Number of tags to register.
     *  @param[in] all_ordering        (Optional) Ordering of coordinates and
     *                                 field ParticleVector.
-    *  @param[in] use_device          (Optional) Use device memory for particle
-    *                                 fields.
     */
    ParticleSet(MPI_Comm comm_, int rank_num_particles, int dim,
                const Array<int> &field_vdims, int num_tags,
-               Ordering::Type all_ordering=Ordering::byVDIM,
-               bool use_device=false);
+               Ordering::Type all_ordering=Ordering::byVDIM);
 
    /** @brief Construct a parallel ParticleSet with specified fields and tags
     *  at construction, with names (for PrintCSV()).
@@ -531,15 +502,12 @@ public:
     *  @param[in] tag_names_          Array of tag names.
     *  @param[in] all_ordering        (Optional) Ordering of coordinates and
     *                                 field ParticleVector.
-    *  @param[in] use_device          (Optional) Use device memory for particle
-    *                                 fields.
     */
    ParticleSet(MPI_Comm comm_, int rank_num_particles, int dim,
                const Array<int> &field_vdims,
                const Array<const char*> &field_names_,
                int num_tags, const Array<const char*> &tag_names_,
-               Ordering::Type all_ordering=Ordering::byVDIM,
-               bool use_device=false);
+               Ordering::Type all_ordering=Ordering::byVDIM);
 
    /** @brief Comprehensive parallel constructor of ParticleSet.
     *
@@ -552,15 +520,12 @@ public:
     *  @param[in] field_names_        Array of field names.
     *  @param[in] num_tags            Number of tags to register.
     *  @param[in] tag_names_          Array of tag names.
-    *  @param[in] use_device          (Optional) Use device memory for particle
-    *                                 fields.
     */
    ParticleSet(MPI_Comm comm_, int rank_num_particles, int dim,
                Ordering::Type coords_ordering, const Array<int> &field_vdims,
                const Array<Ordering::Type> &field_orderings,
                const Array<const char*> &field_names_, int num_tags,
-               const Array<const char*> &tag_names_,
-               bool use_device=false);
+               const Array<const char*> &tag_names_);
 
    /// Get the MPI communicator for this ParticleSet.
    MPI_Comm GetComm() const { return comm; };
@@ -580,8 +545,6 @@ public:
     *  @param[in] field_ordering   (Optional) Ordering::Type of the field.
     *  @param[in] field_name       (Optional) Name of the field.
     *
-    *  @note New fields inherit the current device mode of \ref coords.
-    *
     *  @return Index of the newly-added field.
     */
    int AddField(int vdim, Ordering::Type field_ordering=Ordering::byVDIM,
@@ -674,8 +637,8 @@ public:
 
    /** @brief Determine if GetParticleRef is valid.
     *
-    * Returns true when coordinates and all fields are ordered byVDIM and
-    * particle data is host-resident. Otherwise, false.
+    * If coordinates and all fields are ordered byVDIM, then returns true.
+    * Otherwise, false.
     */
    bool IsParticleRefValid() const;
 

fem/pgridfunc.cpp

@@ -22,20 +22,10 @@ using namespace std;
 namespace mfem
 {
 
-ParGridFunction::ParGridFunction(ParFiniteElementSpace *pf, GridFunction *gf,
-                                 bool preserve)
+ParGridFunction::ParGridFunction(ParFiniteElementSpace *pf, GridFunction *gf)
 {
    fes = pfes = pf;
    SetDataAndSize(gf->GetData(), gf->Size());
-
-   if (pfes->HaveDofSigns())
-   {
-      MFEM_VERIFY(!preserve, "Differing sign conventions for the serial and "
-                  "parallel grid functions will prevent preserving the serial "
-                  "GridFunctions in this context.");
-
-      pfes->ApplyDofSigns(HostReadWrite());
-   }
 }
 
 ParGridFunction::ParGridFunction(ParFiniteElementSpace *pf, HypreParVector *tv)

fem/pgridfunc.hpp

@@ -100,12 +100,8 @@ public:
    /// Construct a ParGridFunction using a GridFunction as external data.
    /** The parallel space @a *pf and the space used by @a *gf should match. The
        data from @a *gf is used as the local data of the ParGridFunction on each
-       processor. The ParGridFunction does not assume ownership of the data.
-       The boolean, @a preserve, indicates that the data stored in @a *gf should
-       remain unchanged. An error will occur if @a preserve is true and
-       construction of a valid ParGridFunction requires the data to change. */
-   ParGridFunction(ParFiniteElementSpace *pf, GridFunction *gf,
-                   bool preserve = true);
+       processor. The ParGridFunction does not assume ownership of the data. */
+   ParGridFunction(ParFiniteElementSpace *pf, GridFunction *gf);
 
    /** @brief Creates grid function on (all) dofs from a given vector on the
        true dofs, i.e. P tv. */

fem/qinterp/det.cpp

@@ -51,21 +51,48 @@ QuadratureInterpolator::DetKernels::Fallback(int DIM, int SDIM, int D1D,
 {
    if (DIM == 1)
    {
-      if (SDIM == 1) { return internal::quadrature_interpolator::Det1D; }
-      else if (SDIM == 2) { return internal::quadrature_interpolator::Det1DSurface<0,0,2>; }
-      else if (SDIM == 3) { return internal::quadrature_interpolator::Det1DSurface<0,0,3>; }
-      else { MFEM_ABORT(""); }
+      if (SDIM == 1)
+      {
+         return internal::quadrature_interpolator::Det1D;
+      }
+      else if (SDIM == 2)
+      {
+         return internal::quadrature_interpolator::Det1DSurface<0, 0, 2>;
+      }
+      else if (SDIM == 3)
+      {
+         return internal::quadrature_interpolator::Det1DSurface<0, 0, 3>;
+      }
+      else
+      {
+         MFEM_ABORT("");
+      }
+   }
+   else if (DIM == 2 && SDIM == 2)
+   {
+      return internal::quadrature_interpolator::Det2D<>;
+   }
+   else if (DIM == 2 && SDIM == 3)
+   {
+      return internal::quadrature_interpolator::Det2DSurface<>;
    }
-   else if (DIM == 2 && SDIM == 2) { return internal::quadrature_interpolator::Det2D; }
-   else if (DIM == 2 && SDIM == 3) { return internal::quadrature_interpolator::Det2DSurface; }
    else if (DIM == 3)
    {
       const int MD = DeviceDofQuadLimits::Get().MAX_DET_1D;
       const int MQ = DeviceDofQuadLimits::Get().MAX_DET_1D;
-      if (D1D <= MD && Q1D <= MQ) { return internal::quadrature_interpolator::Det3D<0,0,true>; }
-      else { return internal::quadrature_interpolator::Det3D<0,0,false>; }
+      if (D1D <= MD && Q1D <= MQ)
+      {
+         return internal::quadrature_interpolator::Det3D<0, 0, true>;
+      }
+      else
+      {
+         return internal::quadrature_interpolator::Det3D<0, 0, false>;
+      }
+   }
+   else
+   {
+      MFEM_ABORT("");
    }
-   else { MFEM_ABORT(""); }
 }
 
 /// @endcond

fem/qinterp/eval.hpp

@@ -31,9 +31,9 @@ namespace quadrature_interpolator
 {
 
 template <QVectorLayout Q_LAYOUT, bool Integral>
-static void ImplValues1D(const int NE, const real_t *b_, const real_t *detJ_,
-                         const real_t *x_, real_t *y_, const int vdim,
-                         const int d1d, const int q1d)
+static void IntValues1D(const int NE, const real_t *b_, const real_t *detJ_,
+                        const real_t *x_, real_t *y_, const int vdim,
+                        const int d1d, const int q1d)
 {
    mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
    {
@@ -72,15 +72,15 @@ template <QVectorLayout Q_LAYOUT>
 static void Values1D(const int NE, const real_t *b_, const real_t *x_,
                      real_t *y_, const int vdim, const int d1d, const int q1d)
 {
-   ImplValues1D<Q_LAYOUT, false>(NE, b_, nullptr, x_, y_, vdim, d1d, q1d);
+   IntValues1D<Q_LAYOUT, false>(NE, b_, nullptr, x_, y_, vdim, d1d, q1d);
 }
 
 // Template compute kernel for Values in 2D: tensor product version.
 template <QVectorLayout Q_LAYOUT, bool Integral, int T_VDIM = 0, int T_D1D = 0,
           int T_Q1D = 0, int T_NBZ = 1>
-static void ImplValues2D(const int NE, const real_t *b_, const real_t *detJ_,
-                         const real_t *x_, real_t *y_, const int vdim = 0,
-                         const int d1d = 0, const int q1d = 0)
+static void IntValues2D(const int NE, const real_t *b_, const real_t *detJ_,
+                        const real_t *x_, real_t *y_, const int vdim = 0,
+                        const int d1d = 0, const int q1d = 0)
 {
    static constexpr int NBZ = T_NBZ ? T_NBZ : 1;
 
@@ -159,16 +159,16 @@ static void Values2D(const int NE, const real_t *b_, const real_t *x_,
                      real_t *y_, const int vdim = 0, const int d1d = 0,
                      const int q1d = 0)
 {
-   return ImplValues2D<Q_LAYOUT, false, T_VDIM, T_D1D, T_Q1D, T_NBZ>(
+   return IntValues2D<Q_LAYOUT, false, T_VDIM, T_D1D, T_Q1D, T_NBZ>(
              NE, b_, nullptr, x_, y_, vdim, d1d, q1d);
 }
 
 // Template compute kernel for Values in 3D: tensor product version.
 template <QVectorLayout Q_LAYOUT, bool Integral, int T_VDIM = 0, int T_D1D = 0,
           int T_Q1D = 0>
-static void ImplValues3D(const int NE, const real_t *b_, const real_t *detJ_,
-                         const real_t *x_, real_t *y_, const int vdim = 0,
-                         const int d1d = 0, const int q1d = 0)
+static void IntValues3D(const int NE, const real_t *b_, const real_t *detJ_,
+                        const real_t *x_, real_t *y_, const int vdim = 0,
+                        const int d1d = 0, const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -251,23 +251,23 @@ static void Values3D(const int NE, const real_t *b_, const real_t *x_,
                      real_t *y_, const int vdim = 0, const int d1d = 0,
                      const int q1d = 0)
 {
-   return ImplValues3D<Q_LAYOUT, false, T_VDIM, T_D1D, T_Q1D>(
+   return IntValues3D<Q_LAYOUT, false, T_VDIM, T_D1D, T_Q1D>(
              NE, b_, nullptr, x_, y_, vdim, d1d, q1d);
 }
 
 template <bool Integral>
-void ImplEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
-                const real_t *detJ, const GeometricFactors *geom,
-                const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
-                Vector &q_der, Vector &q_det, const int eval_flags);
+void IntEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
+               const real_t *detJ, const GeometricFactors *geom,
+               const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
+               Vector &q_der, Vector &q_det, const int eval_flags);
 
 inline void Eval1D(const int NE, const int vdim, const QVectorLayout q_layout,
                    const GeometricFactors *geom, const DofToQuad &maps,
                    const Vector &e_vec, Vector &q_val, Vector &q_der, Vector &q_det,
                    const int eval_flags)
 {
-   ImplEval1D<false>(NE, vdim, q_layout, nullptr, geom, maps, e_vec, q_val,
-                     q_der, q_det, eval_flags);
+   IntEval1D<false>(NE, vdim, q_layout, nullptr, geom, maps, e_vec, q_val,
+                    q_der, q_det, eval_flags);
 }
 
 // Template compute kernel for 2D quadrature interpolation:
@@ -275,11 +275,11 @@ inline void Eval1D(const int NE, const int vdim, const QVectorLayout q_layout,
 // * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
 // * assumes 'maps.mode == FULL'.
 template <bool Integral, const int T_VDIM, const int T_ND, const int T_NQ>
-static void ImplEval2D(const int NE, const int vdim,
-                       const QVectorLayout q_layout, const real_t *detJ_,
-                       const GeometricFactors *geom, const DofToQuad &maps,
-                       const Vector &e_vec, Vector &q_val, Vector &q_der,
-                       Vector &q_det, const int eval_flags)
+static void
+IntEval2D(const int NE, const int vdim, const QVectorLayout q_layout,
+          const real_t *detJ_, const GeometricFactors *geom,
+          const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
+          Vector &q_der, Vector &q_det, const int eval_flags)
 {
    using QI = QuadratureInterpolator;
 
@@ -293,12 +293,6 @@ static void ImplEval2D(const int NE, const int vdim,
    MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 2, "");
    MFEM_VERIFY(ND <= QI::MAX_ND2D, "");
    MFEM_VERIFY(NQ <= QI::MAX_NQ2D, "");
-   if constexpr(Integral)
-   {
-      MFEM_VERIFY(!(eval_flags & (QI::DERIVATIVES | QI::PHYSICAL_DERIVATIVES |
-                                  QI::DETERMINANTS)),
-                  "Integral FE does not support computing derivatives");
-   }
    const auto B = Reshape(maps.B.Read(), NQ, ND);
    const auto G = Reshape(maps.G.Read(), NQ, 2, ND);
    const auto J = Reshape(geom ? geom->J.Read() : nullptr, NQ, 2, 2, NE);
@@ -455,9 +449,8 @@ static void Eval2D(const int NE, const int vdim, const QVectorLayout q_layout,
                    const Vector &e_vec, Vector &q_val, Vector &q_der,
                    Vector &q_det, const int eval_flags)
 {
-   ImplEval2D<false, T_VDIM, T_ND, T_NQ>(NE, vdim, q_layout, nullptr, geom,
-                                         maps, e_vec, q_val, q_der, q_det,
-                                         eval_flags);
+   IntEval2D<false, T_VDIM, T_ND, T_NQ>(NE, vdim, q_layout, nullptr, geom, maps,
+                                        e_vec, q_val, q_der, q_det, eval_flags);
 }
 
 // Template compute kernel for 3D quadrature interpolation:
@@ -465,11 +458,11 @@ static void Eval2D(const int NE, const int vdim, const QVectorLayout q_layout,
 // * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
 // * assumes 'maps.mode == FULL'.
 template <bool Integral, const int T_VDIM, const int T_ND, const int T_NQ>
-static void ImplEval3D(const int NE, const int vdim,
-                       const QVectorLayout q_layout, const real_t *detJ_,
-                       const GeometricFactors *geom, const DofToQuad &maps,
-                       const Vector &e_vec, Vector &q_val, Vector &q_der,
-                       Vector &q_det, const int eval_flags)
+static void
+IntEval3D(const int NE, const int vdim, const QVectorLayout q_layout,
+          const real_t *detJ_, const GeometricFactors *geom,
+          const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
+          Vector &q_der, Vector &q_det, const int eval_flags)
 {
    using QI = QuadratureInterpolator;
 
@@ -484,12 +477,6 @@ static void ImplEval3D(const int NE, const int vdim,
    MFEM_VERIFY(ND <= QI::MAX_ND3D, "");
    MFEM_VERIFY(NQ <= QI::MAX_NQ3D, "");
    MFEM_VERIFY(VDIM == 3 || !(eval_flags & QI::DETERMINANTS), "");
-   if constexpr(Integral)
-   {
-      MFEM_VERIFY(!(eval_flags & (QI::DERIVATIVES | QI::PHYSICAL_DERIVATIVES |
-                                  QI::DETERMINANTS)),
-                  "Integral FE does not support computing derivatives");
-   }
    const auto B = Reshape(maps.B.Read(), NQ, ND);
    const auto G = Reshape(maps.G.Read(), NQ, 3, ND);
    const auto J = Reshape(geom ? geom->J.Read() : nullptr, NQ, 3, 3, NE);
@@ -648,9 +635,8 @@ static void Eval3D(const int NE, const int vdim, const QVectorLayout q_layout,
                    const Vector &e_vec, Vector &q_val, Vector &q_der,
                    Vector &q_det, const int eval_flags)
 {
-   ImplEval3D<false, T_VDIM, T_ND, T_NQ>(NE, vdim, q_layout, nullptr, geom,
-                                         maps, e_vec, q_val, q_der, q_det,
-                                         eval_flags);
+   IntEval3D<false, T_VDIM, T_ND, T_NQ>(NE, vdim, q_layout, nullptr, geom, maps,
+                                        e_vec, q_val, q_der, q_det, eval_flags);
 }
 
 } // namespace quadrature_interpolator
@@ -663,9 +649,9 @@ template <int DIM, QVectorLayout Q_LAYOUT, int VDIM, int D1D, int Q1D, int NBZ>
 QuadratureInterpolator::IntTensorEvalKernelType
 QuadratureInterpolator::IntTensorEvalKernels::Kernel()
 {
-   if constexpr (DIM == 1) { return internal::quadrature_interpolator::ImplValues1D<Q_LAYOUT, true>; }
-   else if constexpr (DIM == 2) { return internal::quadrature_interpolator::ImplValues2D<Q_LAYOUT, true, VDIM, D1D, Q1D, NBZ>; }
-   else if constexpr (DIM == 3) { return internal::quadrature_interpolator::ImplValues3D<Q_LAYOUT, true, VDIM, D1D, Q1D>; }
+   if constexpr (DIM == 1) { return internal::quadrature_interpolator::IntValues1D<Q_LAYOUT, true>; }
+   else if constexpr (DIM == 2) { return internal::quadrature_interpolator::IntValues2D<Q_LAYOUT, true, VDIM, D1D, Q1D, NBZ>; }
+   else if constexpr (DIM == 3) { return internal::quadrature_interpolator::IntValues3D<Q_LAYOUT, true, VDIM, D1D, Q1D>; }
    MFEM_ABORT("");
 }
 
@@ -684,9 +670,9 @@ QuadratureInterpolator::IntEvalKernelType
 QuadratureInterpolator::IntEvalKernels::Kernel()
 {
    using namespace internal::quadrature_interpolator;
-   if constexpr (DIM == 1) { return ImplEval1D<true>; }
-   else if constexpr (DIM == 2) { return ImplEval2D<true,VDIM,ND,NQ>; }
-   else if constexpr (DIM == 3) { return ImplEval3D<true,VDIM,ND,NQ>; }
+   if constexpr (DIM == 1) { return IntEval1D<true>; }
+   else if constexpr (DIM == 2) { return IntEval2D<true,VDIM,ND,NQ>; }
+   else if constexpr (DIM == 3) { return IntEval3D<true,VDIM,ND,NQ>; }
    MFEM_ABORT("");
 }
 

fem/qinterp/eval_hdiv.hpp

@@ -136,8 +136,8 @@ inline void EvalHDiv2D(const int NE,
                {
                   qq += QD(qx,dy,vd) * By(dy,qy);
                }
-               if (FLAGS & (QuadratureInterpolator::PHYSICAL_VALUES |
-                            QuadratureInterpolator::PHYSICAL_MAGNITUDES))
+               if ((FLAGS & (QuadratureInterpolator::PHYSICAL_VALUES |
+                             QuadratureInterpolator::PHYSICAL_MAGNITUDES)) != 0)
                {
                   QQ(qx,qy,vd) = qq;
                }
@@ -153,8 +153,8 @@ inline void EvalHDiv2D(const int NE,
          }
       }
       MFEM_SYNC_THREAD;
-      if (FLAGS & (QuadratureInterpolator::PHYSICAL_VALUES |
-                   QuadratureInterpolator::PHYSICAL_MAGNITUDES))
+      if ((FLAGS & (QuadratureInterpolator::PHYSICAL_VALUES |
+                    QuadratureInterpolator::PHYSICAL_MAGNITUDES)) != 0)
       {
          if (tidz == 0)
          {
@@ -178,7 +178,7 @@ inline void EvalHDiv2D(const int NE,
                   const real_t detJ = kernels::Det<DIM>(J_loc);
                   kernels::Mult(DIM, DIM, J_loc, u_ref, u_phys);
                   kernels::Set(DIM, 1, 1_r/detJ, u_phys, u_phys);
-                  if (FLAGS & QuadratureInterpolator::PHYSICAL_VALUES)
+                  if ((FLAGS & QuadratureInterpolator::PHYSICAL_VALUES) != 0)
                   {
                      MFEM_UNROLL(DIM)
                      for (int sd = 0; sd < DIM; sd++)
@@ -193,7 +193,8 @@ inline void EvalHDiv2D(const int NE,
                         }
                      }
                   }
-                  else if (FLAGS & QuadratureInterpolator::PHYSICAL_MAGNITUDES)
+                  else if ((FLAGS &
+                            QuadratureInterpolator::PHYSICAL_MAGNITUDES) != 0)
                   {
                      y(qx,qy,0,e) = kernels::Norml2(DIM, u_phys);
                   }
@@ -372,8 +373,9 @@ inline void EvalHDiv3D(const int NE,
                MFEM_UNROLL(MQ1)
                for (int qz = 0; qz < Q1D; ++qz)
                {
-                  if (FLAGS & (QuadratureInterpolator::PHYSICAL_VALUES |
-                               QuadratureInterpolator::PHYSICAL_MAGNITUDES))
+                  if ((FLAGS & (QuadratureInterpolator::PHYSICAL_VALUES |
+                                QuadratureInterpolator::PHYSICAL_MAGNITUDES)) !=
+                      0)
                   {
                      QQQ(qx,qy,qz,vd) = u[qz];
                   }
@@ -390,8 +392,8 @@ inline void EvalHDiv3D(const int NE,
          }
       }
       MFEM_SYNC_THREAD;
-      if (FLAGS & (QuadratureInterpolator::PHYSICAL_VALUES |
-                   QuadratureInterpolator::PHYSICAL_MAGNITUDES))
+      if ((FLAGS & (QuadratureInterpolator::PHYSICAL_VALUES |
+                    QuadratureInterpolator::PHYSICAL_MAGNITUDES)) != 0)
       {
          MFEM_FOREACH_THREAD(qz,z,Q1D)
          {
@@ -415,7 +417,7 @@ inline void EvalHDiv3D(const int NE,
                   const real_t detJ = kernels::Det<DIM>(J_loc);
                   kernels::Mult(DIM, DIM, J_loc, u_ref, u_phys);
                   kernels::Set(DIM, 1, 1_r/detJ, u_phys, u_phys);
-                  if (FLAGS & QuadratureInterpolator::PHYSICAL_VALUES)
+                  if ((FLAGS & QuadratureInterpolator::PHYSICAL_VALUES) != 0)
                   {
                      MFEM_UNROLL(DIM)
                      for (int sd = 0; sd < DIM; sd++)
@@ -430,7 +432,8 @@ inline void EvalHDiv3D(const int NE,
                         }
                      }
                   }
-                  else if (FLAGS & QuadratureInterpolator::PHYSICAL_MAGNITUDES)
+                  else if ((FLAGS &
+                            QuadratureInterpolator::PHYSICAL_MAGNITUDES) != 0)
                   {
                      y(qx,qy,qz,0,e) = kernels::Norml2(DIM, u_phys);
                   }

fem/qinterp/grad.hpp

@@ -268,9 +268,8 @@ static void Derivatives3D(const int NE,
       DeviceMatrix B(BG[0], D1D, Q1D);
       DeviceMatrix G(BG[1], D1D, Q1D);
 
-      constexpr int MDQ = MD1 > MQ1 ? MD1 : MQ1;
-      MFEM_SHARED real_t sm0[3][MD1*MD1*MDQ];
-      MFEM_SHARED real_t sm1[3][MD1*MQ1*MQ1];
+      MFEM_SHARED real_t sm0[3][MQ1*MQ1*MQ1];
+      MFEM_SHARED real_t sm1[3][MQ1*MQ1*MQ1];
       DeviceTensor<3> X(sm0[2], D1D, D1D, D1D);
       DeviceTensor<3> DDQ0(sm0[0], D1D, D1D, Q1D);
       DeviceTensor<3> DDQ1(sm0[1], D1D, D1D, Q1D);

fem/quadinterpolator.cpp

@@ -111,10 +111,10 @@ namespace quadrature_interpolator
 // * assumes 'e_vec' is using ElementDofOrdering::NATIVE,
 // * assumes 'maps.mode == FULL'.
 template <bool Integral>
-void ImplEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
-                const real_t *detJ_, const GeometricFactors *geom,
-                const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
-                Vector &q_der, Vector &q_det, const int eval_flags)
+void IntEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
+               const real_t *detJ_, const GeometricFactors *geom,
+               const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
+               Vector &q_der, Vector &q_det, const int eval_flags)
 {
    using QI = QuadratureInterpolator;
 
@@ -123,12 +123,6 @@ void ImplEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
    MFEM_ASSERT(maps.mode == DofToQuad::FULL, "internal error");
    MFEM_ASSERT(!geom || geom->mesh->SpaceDimension() == 1, "");
    MFEM_VERIFY(vdim == 1 || !(eval_flags & QI::DETERMINANTS), "");
-   if constexpr(Integral)
-   {
-      MFEM_VERIFY(!(eval_flags & (QI::DERIVATIVES | QI::PHYSICAL_DERIVATIVES |
-                                  QI::DETERMINANTS)),
-                  "Integral FE does not support computing derivatives");
-   }
    const auto B_ = maps.B.Read();
    const auto G_ = maps.G.Read();
    const auto J = Reshape(geom ? geom->J.Read() : nullptr, nq, NE);
@@ -208,17 +202,17 @@ void ImplEval1D(const int NE, const int vdim, const QVectorLayout q_layout,
 }
 
 template void
-ImplEval1D<true>(const int NE, const int vdim, const QVectorLayout q_layout,
+IntEval1D<true>(const int NE, const int vdim, const QVectorLayout q_layout,
+                const real_t *detJ, const GeometricFactors *geom,
+                const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
+                Vector &q_der, Vector &q_det, const int eval_flags);
+
+template void
+IntEval1D<false>(const int NE, const int vdim, const QVectorLayout q_layout,
                  const real_t *detJ, const GeometricFactors *geom,
                  const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
                  Vector &q_der, Vector &q_det, const int eval_flags);
 
-template void
-ImplEval1D<false>(const int NE, const int vdim, const QVectorLayout q_layout,
-                  const real_t *detJ, const GeometricFactors *geom,
-                  const DofToQuad &maps, const Vector &e_vec, Vector &q_val,
-                  Vector &q_der, Vector &q_det, const int eval_flags);
-
 } // namespace quadrature_interpolator
 
 } // namespace internal
@@ -466,15 +460,15 @@ template <QVectorLayout Q_LAYOUT> auto IntFallbackTensorEvalKernel(int DIM)
 {
    if (DIM == 1)
    {
-      return ImplValues1D<Q_LAYOUT, true>;
+      return IntValues1D<Q_LAYOUT, true>;
    }
    else if (DIM == 2)
    {
-      return ImplValues2D<Q_LAYOUT, true>;
+      return IntValues2D<Q_LAYOUT, true>;
    }
    else if (DIM == 3)
    {
-      return ImplValues3D<Q_LAYOUT, true>;
+      return IntValues3D<Q_LAYOUT, true>;
    }
    MFEM_ABORT("");
 }

fem/quadinterpolator.hpp

@@ -117,10 +117,6 @@ public:
        FiniteElementSpace is a vector space) and their determinants are computed
        and stored in @a q_det.
 
-       For Integral spaces, the flags VALUES requests the computation of the
-       scalar field values. The result is stored in @a q_val. Derivative types
-       are not supported.
-
        For H(div)-conforming spaces, the flags VALUES / PHYSICAL_VALUES request
        the computation of the vector field values in reference or physical
        space, respectively. The flag PHYSICAL_MAGNITUDES requests the

general/backends.hpp

@@ -38,6 +38,15 @@
 #define CUB_IGNORE_DEPRECATED_CPP_DIALECT
 #define THRUST_IGNORE_DEPRECATED_CPP_DIALECT
 
+// MFEM only supports using RAJA/CAMP backends in default stream mode because
+// memory calls are performed outside of the RAJA ecosystem
+#ifndef CAMP_USE_PLATFORM_DEFAULT_STREAM
+#define CAMP_USE_PLATFORM_DEFAULT_STREAM 1
+#else
+#if !CAMP_USE_PLATFORM_DEFAULT_STREAM
+#error "MFEM only supports RAJA/CAMP with the default platform stream."
+#endif
+#endif
 #include "RAJA/RAJA.hpp"
 #if defined(RAJA_ENABLE_CUDA) && !defined(MFEM_USE_CUDA)
 #error When RAJA is built with CUDA, MFEM_USE_CUDA=YES is required
@@ -67,19 +76,12 @@ constexpr bool mfem_use_gpu = false;
 #define MFEM_FOREACH_THREAD(i,k,N) for(int i=0; i<N; i++)
 #define MFEM_FOREACH_THREAD_DIRECT(i,k,N) MFEM_FOREACH_THREAD(i,k,N)
 // Assigns a thread block shaped (SX,SY,SZ) contiguous in x.
-// Example (3,2,1) block:
-// 0 (0,0), 1 (1,0), 2 (2,0)
-// 3 (1,0), 4 (1,1), 5 (2,1)
 #define MFEM_FOREACH_THREAD_DIRECT_3D(ix, iy, iz, k, SX, SY, SZ)               \
    for (int iz = 0; iz < SZ; ++iz)                                             \
       for (int iy = 0; iy < SY; ++iy)                                          \
          for (int ix = 0; ix < SX; ++ix)
 // Assigns a thread block shaped (OX,OY,OZ) to work on items (SX,SY,SZ),
-// contiguous in x. This intentionally offsets threads within the block to avoid
-// shared memory bank conflicts.
-// Example (3,2,1) block assigned to work on (2,2,1) items:
-// 0 (0,0), 1 (1,0), 2 (N/A)
-// 3 (1,0), 4 (1,1), 5 (N/A)
+// contiguous in x. This intentionally offsets threads
 #define MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(ix, iy, iz, k, SX, SY, SZ, OX,    \
                                              OY, OZ)                           \
    MFEM_FOREACH_THREAD_DIRECT_3D(ix, iy, iz, k, SX, SY, SZ)

general/cuda.hpp

@@ -50,18 +50,11 @@ constexpr bool mfem_use_gpu = true;
 #define MFEM_FOREACH_THREAD(i,k,N) for(int i=threadIdx.k; i<N; i+=blockDim.k)
 #define MFEM_FOREACH_THREAD_DIRECT(i,k,N) if(const int i=threadIdx.k; i<N)
 // Assigns a thread block shaped (SX,SY,SZ) contiguous in x.
-// Example (3,2,1) block:
-// 0 (0,0), 1 (1,0), 2 (2,0)
-// 3 (1,0), 4 (1,1), 5 (2,1)
 #define MFEM_FOREACH_THREAD_DIRECT_3D(ix, iy, iz, k, SX, SY, SZ)               \
    if (int ix = threadIdx.k % (SX), iy = threadIdx.k / (SX), iz = iy / (SY);   \
        (iy %= (SY)), (threadIdx.k < (SX) * (SY) * (SZ)))
 // Assigns a thread block shaped (OX,OY,OZ) to work on items (SX,SY,SZ),
-// contiguous in x. This intentionally offsets threads within the block to avoid
-// shared memory bank conflicts.
-// Example (3,2,1) block assigned to work on (2,2,1) items:
-// 0 (0,0), 1 (1,0), 2 (N/A)
-// 3 (1,0), 4 (1,1), 5 (N/A)
+// contiguous in x. This intentionally offsets threads
 #define MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(ix, iy, iz, k, SX, SY, SZ, OX,    \
                                              OY, OZ)                           \
    if (int ix = threadIdx.k % (OX), iy = threadIdx.k / (OX), iz = iy / (OY);   \

general/device.cpp

@@ -581,9 +581,7 @@ void Device::Setup(const std::string &device_option, const int device_id)
    if (Allows(Backend::CUDA)) { CudaDeviceSetup(dev, ngpu); }
    if (Allows(Backend::HIP)) { HipDeviceSetup(dev, ngpu); }
    if (Allows(Backend::RAJA_CUDA) || Allows(Backend::RAJA_HIP))
-   {
-      RajaDeviceSetup(dev, ngpu);
-   }
+   { RajaDeviceSetup(dev, ngpu); }
    // The check for MFEM_USE_OCCA is in the function OccaDeviceSetup().
    if (Allows(Backend::OCCA_MASK)) { OccaDeviceSetup(dev); }
    if (Allows(Backend::CEED_MASK))

general/device.hpp

@@ -16,11 +16,6 @@
 #include "globals.hpp"
 #include "mem_manager.hpp"
 
-#ifdef MFEM_USE_RAJA
-#include "RAJA/RAJA.hpp"
-#endif
-
-#include <memory>
 #include <string>
 
 namespace mfem
@@ -271,18 +266,6 @@ public:
    static inline bool Allows(unsigned long b_mask)
    { return Get().backends & b_mask; }
 
-#if defined(MFEM_USE_RAJA) &&                                                  \
-   (defined(RAJA_ENABLE_CUDA) || defined(RAJA_ENABLE_HIP))
-   static inline auto GetRajaResource()
-   {
-#if defined(RAJA_ENABLE_CUDA)
-      return RAJA::resources::Cuda::CudaFromStream(0, Get().GetId());
-#elif defined(RAJA_ENABLE_HIP)
-      return RAJA::resources::Hip::HipFromStream(0, Get().GetId());
-#endif
-   }
-#endif
-
    /** @brief Get the current Host MemoryType. This is the MemoryType used by
        most MFEM classes when allocating memory used on the host.
    */

general/error.cpp

@@ -171,7 +171,14 @@ void mfem_error(const char *msg)
 #ifdef MFEM_USE_EXCEPTIONS
    if (mfem_error_action == MFEM_ERROR_THROW)
    {
-      throw ErrorException(msg);
+      if (msg)
+      {
+         throw ErrorException(msg);
+      }
+      else
+      {
+         throw ErrorException("");
+      }
    }
 #endif
 

general/forall.hpp

@@ -317,8 +317,8 @@ template <typename DBODY>
 void RajaCuWrap1D(const int N, DBODY &&d_body)
 {
    //true denotes asynchronous kernel
-   RAJA::forall<RAJA::cuda_exec<MFEM_CUDA_BLOCKS, true> >(
-      Device::GetRajaResource(), RAJA::RangeSegment(0, N), d_body);
+   RAJA::forall<RAJA::cuda_exec<MFEM_CUDA_BLOCKS,true>>(RAJA::RangeSegment(0,N),
+                                                        d_body);
 }
 
 template <typename DBODY>
@@ -331,9 +331,9 @@ void RajaCuWrap2D(const int N, DBODY &&d_body,
    using namespace RAJA;
    using RAJA::RangeSegment;
 
-   launch<cuda_launch_policy>(Device::GetRajaResource(),
-                              LaunchParams(Teams(G), Threads(X, Y, BZ)),
-                              [=] RAJA_DEVICE(LaunchContext ctx)
+   launch<cuda_launch_policy>
+   (LaunchParams(Teams(G), Threads(X, Y, BZ)),
+    [=] RAJA_DEVICE (LaunchContext ctx)
    {
 
       loop<cuda_teams_x>(ctx, RangeSegment(0, G), [&] (const int n)
@@ -349,6 +349,7 @@ void RajaCuWrap2D(const int N, DBODY &&d_body,
          });
 
       });
+
    });
 
    MFEM_GPU_CHECK(cudaGetLastError());
@@ -364,9 +365,9 @@ void RajaCuWrap2DLaunchBounds(const int N, DBODY &&d_body, const int X,
    using namespace RAJA;
    using RAJA::RangeSegment;
 
-   launch<cuda_launch_bounds_policy<LB> >(
-      Device::GetRajaResource(), LaunchParams(Teams(G), Threads(X, Y, BZ)),
-      [=] RAJA_DEVICE(LaunchContext ctx)
+   launch<cuda_launch_bounds_policy<LB> >
+   (LaunchParams(Teams(G), Threads(X, Y, BZ)),
+    [=] RAJA_DEVICE(LaunchContext ctx)
    {
       loop<cuda_teams_x>(ctx, RangeSegment(0, G), [&] (const int n)
       {
@@ -389,12 +390,13 @@ void RajaCuWrap3D(const int N, DBODY &&d_body,
    using namespace RAJA;
    using RAJA::RangeSegment;
 
-   launch<cuda_launch_policy>(Device::GetRajaResource(),
-                              LaunchParams(Teams(GRID), Threads(X, Y, Z)),
-                              [=] RAJA_DEVICE(LaunchContext ctx)
+   launch<cuda_launch_policy>
+   (LaunchParams(Teams(GRID), Threads(X, Y, Z)),
+    [=] RAJA_DEVICE (LaunchContext ctx)
    {
 
       loop<cuda_teams_x>(ctx, RangeSegment(0, N), d_body);
+
    });
 
    MFEM_GPU_CHECK(cudaGetLastError());
@@ -408,10 +410,12 @@ void RajaCuWrap3DLaunchBounds(const int N, DBODY &&d_body,
    using namespace RAJA;
    using RAJA::RangeSegment;
 
-   launch<cuda_launch_bounds_policy<LB> >(
-      Device::GetRajaResource(), LaunchParams(Teams(GRID), Threads(X, Y, Z)),
-      [=] RAJA_DEVICE(LaunchContext ctx)
-   { loop<cuda_teams_x>(ctx, RangeSegment(0, N), d_body); });
+   launch<cuda_launch_bounds_policy<LB> >
+   (LaunchParams(Teams(GRID), Threads(X, Y, Z)),
+    [=] RAJA_DEVICE(LaunchContext ctx)
+   {
+      loop<cuda_teams_x>(ctx, RangeSegment(0, N), d_body);
+   });
    MFEM_GPU_CHECK(cudaGetLastError());
 }
 
@@ -480,8 +484,8 @@ template <typename DBODY>
 void RajaHipWrap1D(const int N, DBODY &&d_body)
 {
    //true denotes asynchronous kernel
-   RAJA::forall<RAJA::hip_exec<MFEM_HIP_BLOCKS, true> >(
-      Device::GetRajaResource(), RAJA::RangeSegment(0, N), d_body);
+   RAJA::forall<RAJA::hip_exec<MFEM_HIP_BLOCKS,true>>(RAJA::RangeSegment(0,N),
+                                                      d_body);
 }
 
 template <typename DBODY>
@@ -494,9 +498,9 @@ void RajaHipWrap2D(const int N, DBODY &&d_body,
    using namespace RAJA;
    using RAJA::RangeSegment;
 
-   launch<hip_launch_policy>(Device::GetRajaResource(),
-                             LaunchParams(Teams(G), Threads(X, Y, BZ)),
-                             [=] RAJA_DEVICE(LaunchContext ctx)
+   launch<hip_launch_policy>
+   (LaunchParams(Teams(G), Threads(X, Y, BZ)),
+    [=] RAJA_DEVICE (LaunchContext ctx)
    {
 
       loop<hip_teams_x>(ctx, RangeSegment(0, G), [&] (const int n)
@@ -512,6 +516,7 @@ void RajaHipWrap2D(const int N, DBODY &&d_body,
          });
 
       });
+
    });
 
    MFEM_GPU_CHECK(hipGetLastError());
@@ -527,9 +532,9 @@ void RajaHipWrap2DLaunchBounds(const int N, DBODY &&d_body, const int X,
    using namespace RAJA;
    using RAJA::RangeSegment;
 
-   launch<hip_launch_bounds_policy<LB> >(
-      Device::GetRajaResource(), LaunchParams(Teams(G), Threads(X, Y, BZ)),
-      [=] RAJA_DEVICE(LaunchContext ctx)
+   launch<hip_launch_bounds_policy<LB> >
+   (LaunchParams(Teams(G), Threads(X, Y, BZ)),
+    [=] RAJA_DEVICE(LaunchContext ctx)
    {
       loop<hip_teams_x>(ctx, RangeSegment(0, G), [&] (const int n)
       {
@@ -552,12 +557,13 @@ void RajaHipWrap3D(const int N, DBODY &&d_body,
    using namespace RAJA;
    using RAJA::RangeSegment;
 
-   launch<hip_launch_policy>(Device::GetRajaResource(),
-                             LaunchParams(Teams(GRID), Threads(X, Y, Z)),
-                             [=] RAJA_DEVICE(LaunchContext ctx)
+   launch<hip_launch_policy>
+   (LaunchParams(Teams(GRID), Threads(X, Y, Z)),
+    [=] RAJA_DEVICE (LaunchContext ctx)
    {
 
       loop<hip_teams_x>(ctx, RangeSegment(0, N), d_body);
+
    });
 
    MFEM_GPU_CHECK(hipGetLastError());
@@ -571,10 +577,12 @@ void RajaHipWrap3DLaunchBounds(const int N, DBODY &&d_body, const int X,
    using namespace RAJA;
    using RAJA::RangeSegment;
 
-   launch<hip_launch_bounds_policy<LB> >(
-      Device::GetRajaResource(), LaunchParams(Teams(GRID), Threads(X, Y, Z)),
-      [=] RAJA_DEVICE(LaunchContext ctx)
-   { loop<hip_teams_x>(ctx, RangeSegment(0, N), d_body); });
+   launch<hip_launch_bounds_policy<LB> >
+   (LaunchParams(Teams(GRID), Threads(X, Y, Z)),
+    [=] RAJA_DEVICE(LaunchContext ctx)
+   {
+      loop<hip_teams_x>(ctx, RangeSegment(0, N), d_body);
+   });
    MFEM_GPU_CHECK(hipGetLastError());
 }
 

general/hip.hpp

@@ -52,19 +52,12 @@ constexpr bool mfem_use_gpu = true;
 #define MFEM_FOREACH_THREAD_DIRECT(i,k,N) \
    if(const int i=hipThreadIdx_ ##k; i<N)
 // Assigns a thread block shaped (SX,SY,SZ) contiguous in x.
-// Example (3,2,1) block:
-// 0 (0,0), 1 (1,0), 2 (2,0)
-// 3 (1,0), 4 (1,1), 5 (2,1)
 #define MFEM_FOREACH_THREAD_DIRECT_3D(ix, iy, iz, k, SX, SY, SZ)               \
    if (int ix = hipThreadIdx_##k % (SX), iy = hipThreadIdx_##k / (SX),         \
        iz = iy / (SY);                                                         \
        (iy %= (SY)), (hipThreadIdx_##k < (SX) * (SY) * (SZ)))
 // Assigns a thread block shaped (OX,OY,OZ) to work on items (SX,SY,SZ),
-// contiguous in x. This intentionally offsets threads within the block to avoid
-// shared memory bank conflicts.
-// Example (3,2,1) block assigned to work on (2,2,1) items:
-// 0 (0,0), 1 (1,0), 2 (N/A)
-// 3 (1,0), 4 (1,1), 5 (N/A)
+// contiguous in x. This intentionally offsets threads
 #define MFEM_FOREACH_THREAD_DIRECT_3D_OFFSET(ix, iy, iz, k, SX, SY, SZ, OX,    \
                                              OY, OZ)                           \
    if (int ix = hipThreadIdx_##k % (OX), iy = hipThreadIdx_##k / (OX),         \

general/optparser.cpp

@@ -148,9 +148,26 @@ void parseVector(char * str, Vector & var)
    }
 }
 
+bool parseEnumOption(const char *str, size_t &var,
+                     const std::vector<std::string> &options)
+{
+   for (size_t i = 0; i < options.size(); ++i)
+   {
+      auto &v = options[i];
+      if (v == str)
+      {
+         var = i;
+         // success
+         return true;
+      }
+   }
+   // failure
+   return false;
+}
+
 void OptionsParser::Parse()
 {
-   option_check.SetSize(options.Size());
+   option_check.SetSize(options.size());
    option_check = 0;
    for (int i = 1; i < argc; )
    {
@@ -161,9 +178,9 @@ void OptionsParser::Parse()
          return;
       }
 
-      for (int j = 0; true; j++)
+      for (size_t j = 0; true; j++)
       {
-         if (j >= options.Size())
+         if (j >= options.size())
          {
             // unrecognized option
             error_type = 2;
@@ -171,8 +188,8 @@ void OptionsParser::Parse()
             return;
          }
 
-         if (strcmp(argv[i], options[j].short_name) == 0 ||
-             strcmp(argv[i], options[j].long_name) == 0)
+         if (strcmp(argv[i], options[j].short_name.c_str()) == 0 ||
+             strcmp(argv[i], options[j].long_name.c_str()) == 0)
          {
             OptionType type = options[j].type;
 
@@ -224,6 +241,11 @@ void OptionsParser::Parse()
                case VECTOR:
                   parseVector(argv[i++], *(Vector*)(options[j].var_ptr) );
                   break;
+               case ENUM_OPTION:
+                  isValid =
+                     parseEnumOption(argv[i++], *(size_t *)(options[j].var_ptr),
+                                     options[j].options);
+                  break;
             }
 
             if (!isValid)
@@ -239,7 +261,7 @@ void OptionsParser::Parse()
    }
 
    // check for missing required options
-   for (int i = 0; i < options.Size(); i++)
+   for (size_t i = 0; i < options.size(); i++)
       if (options[i].required &&
           (option_check[i] == 0 ||
            (options[i].type == ENABLE && option_check[++i] == 0)))
@@ -323,6 +345,12 @@ void OptionsParser::WriteValue(const Option &opt, std::ostream &os)
          break;
       }
 
+      case ENUM_OPTION:
+      {
+         os << opt.options.at(*((size_t *)opt.var_ptr));
+         break;
+      }
+
       default: // provide a default to suppress warning
          break;
    }
@@ -333,7 +361,7 @@ void OptionsParser::PrintOptions(ostream &os) const
    static const char *indent = "   ";
 
    os << "Options used:\n";
-   for (int j = 0; j < options.Size(); j++)
+   for (size_t j = 0; j < options.size(); j++)
    {
       OptionType type = options[j].type;
 
@@ -355,7 +383,7 @@ void OptionsParser::PrintOptions(ostream &os) const
          os << options[j].long_name << " ";
          WriteValue(options[j], os);
       }
-      os << '\n';
+      os << std::endl;
    }
 }
 
@@ -410,12 +438,13 @@ void OptionsParser::PrintHelp(ostream &os) const
    static const char *line_sep = "";
    static const char *types[] = { " <int>", " <double>", " <string>",
                                   " <string>", "", "", " '<int>...'",
-                                  " '<double>...'"
+                                  " '<double>...'",
+                                  " <string>",
                                 };
 
    os << indent << "-h" << seprtr << "--help" << descr_sep
       << "Print this help message and exit.\n" << line_sep;
-   for (int j = 0; j < options.Size(); j++)
+   for (size_t j = 0; j < options.size(); j++)
    {
       OptionType type = options[j].type;
 
@@ -451,9 +480,22 @@ void OptionsParser::PrintHelp(ostream &os) const
       }
       os << descr_sep;
 
-      if (options[j].description)
+      if (options[j].description.size())
       {
-         os << options[j].description << '\n';
+         os << options[j].description << std::endl;
+      }
+      if (options[j].type == ENUM_OPTION)
+      {
+         os << "\tavailable choices: ";
+         for (size_t i = 0; i < options[j].options.size(); ++i)
+         {
+            os << '"' << options[j].options[i] << '"';
+            if (i + 1 < options[j].options.size())
+            {
+               os << ", ";
+            }
+         }
+         os << std::endl;
       }
       os << line_sep;
    }

general/optparser.hpp

@@ -15,6 +15,9 @@
 #include "../config/config.hpp"
 #include "array.hpp"
 
+#include <string>
+#include <vector>
+
 namespace mfem
 {
 
@@ -31,17 +34,18 @@ class Vector;
 class OptionsParser
 {
 public:
-   enum OptionType { INT, DOUBLE, STRING, STD_STRING, ENABLE, DISABLE, ARRAY, VECTOR };
+   enum OptionType { INT, DOUBLE, STRING, STD_STRING, ENABLE, DISABLE, ARRAY, VECTOR, ENUM_OPTION };
 
 private:
    struct Option
    {
       OptionType type;
       void *var_ptr;
-      const char *short_name;
-      const char *long_name;
-      const char *description;
+      std::string short_name;
+      std::string long_name;
+      std::string description;
       bool required;
+      std::vector<std::string> options;
 
       Option() = default;
 
@@ -49,11 +53,19 @@ private:
              const char *long_name_, const char *description_, bool req)
          : type(type_), var_ptr(var_ptr_), short_name(short_name_),
            long_name(long_name_), description(description_), required(req) { }
+
+      Option(size_t *var_ptr_, const char *short_name_, const char *long_name_,
+             const char *description_, std::vector<std::string> &&options_,
+             bool req)
+         : type(ENUM_OPTION), var_ptr(var_ptr_), short_name(short_name_),
+           long_name(long_name_), description(description_),
+           required(req), options(std::move(options_))
+      {}
    };
 
    int argc;
    char **argv;
-   Array<Option> options;
+   std::vector<Option> options;
    Array<int> option_check;
    // error_type can be:
    //  0 - no error
@@ -84,26 +96,26 @@ public:
                   const char *disable_long_name, const char *description,
                   bool required = false)
    {
-      options.Append(Option(ENABLE, var, enable_short_name, enable_long_name,
-                            description, required));
-      options.Append(Option(DISABLE, var, disable_short_name, disable_long_name,
-                            description, required));
+      options.emplace_back(ENABLE, var, enable_short_name, enable_long_name,
+                           description, required);
+      options.emplace_back(DISABLE, var, disable_short_name, disable_long_name,
+                           description, required);
    }
 
    /// Add an integer option and set 'var' to receive the value.
    void AddOption(int *var, const char *short_name, const char *long_name,
                   const char *description, bool required = false)
    {
-      options.Append(Option(INT, var, short_name, long_name, description,
-                            required));
+      options.emplace_back(INT, var, short_name, long_name, description,
+                           required);
    }
 
    /// Add a double option and set 'var' to receive the value.
    void AddOption(real_t *var, const char *short_name, const char *long_name,
                   const char *description, bool required = false)
    {
-      options.Append(Option(DOUBLE, var, short_name, long_name, description,
-                            required));
+      options.emplace_back(DOUBLE, var, short_name, long_name, description,
+                           required);
    }
 
    /// Add a string (char*) option and set 'var' to receive the value.
@@ -111,8 +123,8 @@ public:
                   const char *long_name, const char *description,
                   bool required = false)
    {
-      options.Append(Option(STRING, var, short_name, long_name, description,
-                            required));
+      options.emplace_back(STRING, var, short_name, long_name, description,
+                           required);
    }
 
    /// Add a string (std::string) option and set 'var' to receive the value.
@@ -120,8 +132,8 @@ public:
                   const char *long_name, const char *description,
                   bool required = false)
    {
-      options.Append(Option(STD_STRING, var, short_name, long_name, description,
-                            required));
+      options.emplace_back(STD_STRING, var, short_name, long_name, description,
+                           required);
    }
 
    /** Add an integer array (separated by spaces) option and set 'var' to
@@ -130,8 +142,8 @@ public:
                   const char *long_name, const char *description,
                   bool required = false)
    {
-      options.Append(Option(ARRAY, var, short_name, long_name, description,
-                            required));
+      options.emplace_back(ARRAY, var, short_name, long_name, description,
+                           required);
    }
 
    /** Add a vector (doubles separated by spaces) option and set 'var' to
@@ -140,8 +152,20 @@ public:
                   const char *long_name, const char *description,
                   bool required = false)
    {
-      options.Append(Option(VECTOR, var, short_name, long_name, description,
-                            required));
+      options.emplace_back(VECTOR, var, short_name, long_name, description,
+                           required);
+   }
+
+   /**
+    * Add an option which must be one of a given list of options
+    */
+   void AddOptionChoice(size_t *index, const char *short_name,
+                        const char *long_name, const char *description,
+                        std::vector<std::string> options_,
+                        bool required = false)
+   {
+      options.emplace_back(index, short_name, long_name, description,
+                           std::move(options_), required);
    }
 
    /** @brief Parse the command-line options.

linalg/batched/batched.cpp

@@ -13,7 +13,6 @@
 #include "native.hpp"
 #include "gpu_blas.hpp"
 #include "magma.hpp"
-#include "../../general/reducers.hpp"
 
 namespace mfem
 {
@@ -120,16 +119,4 @@ void BatchedLinAlgBase::MultTranspose(const DenseTensor &A, const Vector &x,
    AddMult(A, x, y, 1.0, 0.0, Op::T);
 }
 
-void VerifyBatchedLUInfo(const Array<int> &info_array, const char *message)
-{
-   static Array<int> workspace;
-   int status = 0;
-   const int *d_info = info_array.Read();
-   mfem::reduce(
-      info_array.Size(), status,
-   [=] MFEM_HOST_DEVICE (int i, int &r) { r |= d_info[i]; },
-   BOrReducer<int> {}, true, workspace);
-   MFEM_VERIFY(status == 0, message);
-}
-
 }

linalg/batched/batched.hpp

@@ -141,9 +141,6 @@ public:
    virtual ~BatchedLinAlgBase() { }
 };
 
-/// Check that all batched LU info values are zero.
-void VerifyBatchedLUInfo(const Array<int> &info_array, const char *message);
-
 } // namespace mfem
 
 #endif

linalg/batched/gpu_blas.cpp

@@ -126,8 +126,7 @@ void GPUBlasBatchedLinAlg::LUFactor(DenseTensor &A, Array<int> &P) const
    const blasStatus_t status = MFEM_GPUBLAS_PREFIX(getrfBatched)(
                                   GPUBlas::Handle(), n, d_A_ptrs, n, P.Write(),
                                   info_array.Write(), n_mat);
-   MFEM_VERIFY(status == MFEM_BLAS_SUCCESS, "GPU BLAS error.");
-   VerifyBatchedLUInfo(info_array, "Batch LU factorization failed");
+   MFEM_VERIFY(status == MFEM_BLAS_SUCCESS, "");
 }
 
 void GPUBlasBatchedLinAlg::LUSolve(
@@ -190,14 +189,12 @@ void GPUBlasBatchedLinAlg::Invert(DenseTensor &A) const
    status = MFEM_GPUBLAS_PREFIX(getrfBatched)(
                GPUBlas::Handle(), n, d_LU_ptrs, n, P.Write(),
                info_array.Write(), n_mat);
-   MFEM_VERIFY(status == MFEM_BLAS_SUCCESS, "GPU BLAS error.");
-   VerifyBatchedLUInfo(info_array, "Batch LU factorization failed");
+   MFEM_VERIFY(status == MFEM_BLAS_SUCCESS, "");
 
    status = MFEM_GPUBLAS_PREFIX(getriBatched)(
                GPUBlas::Handle(), n, d_LU_ptrs, n, P.ReadWrite(), d_A_ptrs, n,
                info_array.Write(), n_mat);
-   MFEM_VERIFY(status == MFEM_BLAS_SUCCESS, "GPU BLAS error.");
-   VerifyBatchedLUInfo(info_array, "Batch matrix inversion failed");
+   MFEM_VERIFY(status == MFEM_BLAS_SUCCESS, "");
 }
 
 #endif

linalg/batched/magma.cpp

@@ -99,8 +99,7 @@ void MagmaBatchedLinAlg::LUFactor(DenseTensor &A, Array<int> &P) const
    const magma_int_t status = MFEM_MAGMA_PREFIX(getrf_batched)(
                                  n, n, d_A_ptrs, n, d_P_ptrs,
                                  info_array.Write(), n_mat, Magma::Queue());
-   MFEM_VERIFY(status == MAGMA_SUCCESS, "MAGMA error.");
-   VerifyBatchedLUInfo(info_array, "Batch LU factorization failed");
+   MFEM_VERIFY(status == MAGMA_SUCCESS, "");
 }
 
 void MagmaBatchedLinAlg::LUSolve(
@@ -170,14 +169,12 @@ void MagmaBatchedLinAlg::Invert(DenseTensor &A) const
    status = MFEM_MAGMA_PREFIX(getrf_batched)(
                n, n, d_LU_ptrs, n, d_P_ptrs, info_array.Write(), n_mat,
                Magma::Queue());
-   MFEM_VERIFY(status == MAGMA_SUCCESS, "MAGMA error.");
-   VerifyBatchedLUInfo(info_array, "Batch LU factorization failed");
+   MFEM_VERIFY(status == MAGMA_SUCCESS, "");
 
    status = MFEM_MAGMA_PREFIX(getri_outofplace_batched)(
                n, d_LU_ptrs, n, d_P_ptrs, d_A_ptrs, n, info_array.Write(),
                n_mat, Magma::Queue());
-   MFEM_VERIFY(status == MAGMA_SUCCESS, "MAGMA error.");
-   VerifyBatchedLUInfo(info_array, "Batch matrix inversion failed");
+   MFEM_VERIFY(status == MAGMA_SUCCESS, "");
 }
 
 } // namespace mfem

linalg/complex_operator.cpp

@@ -246,10 +246,6 @@ SparseMatrix * ComplexSparseMatrix::GetSystemMatrix() const
    const int  nrows_i = (A_i)?A_i->Height():0;
    const int    nrows = std::max(nrows_r, nrows_i);
 
-   const int  ncols_r = (A_r)?A_r->Width():0;
-   const int  ncols_i = (A_i)?A_i->Width():0;
-   const int    ncols = std::max(ncols_r, ncols_i);
-
    const int     *I_r = (A_r)?A_r->GetI():NULL;
    const int     *I_i = (A_i)?A_i->GetI():NULL;
 
@@ -284,7 +280,7 @@ SparseMatrix * ComplexSparseMatrix::GetSystemMatrix() const
             J[I[i] + j] = J_r[I_r[i] + j];
             D[I[i] + j] = D_r[I_r[i] + j];
 
-            J[I[i+nrows] + off_i + j] = J_r[I_r[i] + j] + ncols;
+            J[I[i+nrows] + off_i + j] = J_r[I_r[i] + j] + nrows;
             D[I[i+nrows] + off_i + j] = factor*D_r[I_r[i] + j];
          }
       }
@@ -293,7 +289,7 @@ SparseMatrix * ComplexSparseMatrix::GetSystemMatrix() const
          const int off_r = (I_r)?(I_r[i+1] - I_r[i]):0;
          for (int j=0; j<I_i[i+1] - I_i[i]; j++)
          {
-            J[I[i] + off_r + j] =  J_i[I_i[i] + j] + ncols;
+            J[I[i] + off_r + j] =  J_i[I_i[i] + j] + nrows;
             D[I[i] + off_r + j] = -D_i[I_i[i] + j];
 
             J[I[i+nrows] + j] = J_i[I_i[i] + j];
@@ -896,12 +892,12 @@ ComplexHypreParMatrix::getColStartStop(const HypreParMatrix * A_r,
    HYPRE_BigInt loc_start_stop[2];
    offd_col_start_stop = new HYPRE_BigInt[2 * num_recv_procs];
 
-   const HYPRE_BigInt * col_part = (A_r) ? A_r->ColPart() :
-                                   ((A_i) ? A_i->ColPart() : NULL);
+   const HYPRE_BigInt * row_part = (A_r) ? A_r->RowPart() :
+                                   ((A_i) ? A_i->RowPart() : NULL);
 
-   int col_part_ind = (HYPRE_AssumedPartitionCheck()) ? 0 : myid_;
-   loc_start_stop[0] = col_part[col_part_ind];
-   loc_start_stop[1] = col_part[col_part_ind+1];
+   int row_part_ind = (HYPRE_AssumedPartitionCheck()) ? 0 : myid_;
+   loc_start_stop[0] = row_part[row_part_ind];
+   loc_start_stop[1] = row_part[row_part_ind+1];
 
    MPI_Request * req = new MPI_Request[send_procs.size()+recv_procs.size()];
    MPI_Status * stat = new MPI_Status[send_procs.size()+recv_procs.size()];

linalg/cudss.cpp

@@ -15,20 +15,10 @@
 
 #ifdef MFEM_USE_CUDSS
 
-#if CUDSS_VERSION >= 800
 #ifdef MFEM_USE_SINGLE
-#define CUDSS_REAL_T CUDSS_R_32F
+#define CUDA_REAL_T CUDA_R_32F
 #else
-#define CUDSS_REAL_T CUDSS_R_64F
-#endif
-#define CUDSS_INT_T CUDSS_R_32I
-#else
-#ifdef MFEM_USE_SINGLE
-#define CUDSS_REAL_T CUDA_R_32F
-#else
-#define CUDSS_REAL_T CUDA_R_64F
-#endif
-#define CUDSS_INT_T CUDA_R_32I
+#define CUDA_REAL_T CUDA_R_64F
 #endif
 
 // Define a cuDSS error check macro, MFEM_CUDSS_CHECK(x), where x returns/is of
@@ -75,13 +65,8 @@ CuDSSSolver::CuDSSSolver(MPI_Comm comm_) : mpi_comm(comm_)
 #endif
    MFEM_CUDSS_CHECK(cudssSetCommLayer(handle, comm_lib));
 
-#if CUDSS_VERSION >= 800
-   MFEM_CUDSS_CHECK(cudssDataSet(handle, solverData, CUDSS_DATA_COMM_HOST,
-                                 &mpi_comm, sizeof(MPI_Comm *)));
-#else
    MFEM_CUDSS_CHECK(cudssDataSet(handle, solverData, CUDSS_DATA_COMM,
                                  &mpi_comm, sizeof(MPI_Comm *)));
-#endif
 }
 #endif // MFEM_USE_MPI
 
@@ -272,19 +257,11 @@ void CuDSSSolver::SetMatrixCuDSS(int *csr_offsets, int *csr_columns,
          CuMemcpyDtoD(csr_offsets_d, csr_offsets, (n_loc + 1) * sizeof(int));
          CuMemcpyDtoD(csr_columns_d, csr_columns, nnz * sizeof(int));
 
-#if CUDSS_VERSION >= 800
          MFEM_CUDSS_CHECK(
             cudssMatrixCreateCsr(
                Ac.get(), n_global, n_global, nnz, csr_offsets_d, NULL,
-               csr_columns_d, csr_values_d, CUDSS_INT_T, CUDSS_INT_T, CUDSS_REAL_T,
-               mat_type, mview, CUDSS_BASE_ZERO));
-#else
-         MFEM_CUDSS_CHECK(
-            cudssMatrixCreateCsr(
-               Ac.get(), n_global, n_global, nnz, csr_offsets_d, NULL,
-               csr_columns_d, csr_values_d, CUDSS_INT_T, CUDSS_REAL_T,
-               mat_type, mview, CUDSS_BASE_ZERO));
-#endif
+               csr_columns_d, csr_values_d, CUDA_R_32I, CUDA_REAL_T, mat_type, mview,
+               CUDSS_BASE_ZERO));
       }
       else    // !reorder_reuse
       {
@@ -292,19 +269,11 @@ void CuDSSSolver::SetMatrixCuDSS(int *csr_offsets, int *csr_columns,
          {
             MFEM_CUDSS_CHECK(cudssMatrixDestroy(*Ac));
          }
-#if CUDSS_VERSION >= 800
          MFEM_CUDSS_CHECK(
             cudssMatrixCreateCsr(
-               Ac.get(), n_global, n_global, nnz, csr_offsets, NULL,
-               csr_columns, csr_values_d, CUDSS_INT_T, CUDSS_INT_T, CUDSS_REAL_T,
-               mat_type, mview, CUDSS_BASE_ZERO));
-#else
-         MFEM_CUDSS_CHECK(
-            cudssMatrixCreateCsr(
-               Ac.get(), n_global, n_global, nnz, csr_offsets, NULL,
-               csr_columns, csr_values_d, CUDSS_INT_T, CUDSS_REAL_T,
-               mat_type, mview, CUDSS_BASE_ZERO));
-#endif
+               Ac.get(), n_global, n_global, nnz, csr_offsets, NULL, csr_columns,
+               csr_values_d, CUDA_R_32I, CUDA_REAL_T, mat_type, mview,
+               CUDSS_BASE_ZERO));
       }
 #ifdef MFEM_USE_MPI
       if (Mpi::IsInitialized())
@@ -365,10 +334,10 @@ void CuDSSSolver::SetNumRHS(int nrhs_) const
       }
       // Create empty RHS and solution vectors
       MFEM_CUDSS_CHECK(cudssMatrixCreateDn(&xc, n_global, nrhs_, n_global, NULL,
-                                           CUDSS_REAL_T, CUDSS_LAYOUT_COL_MAJOR));
+                                           CUDA_REAL_T, CUDSS_LAYOUT_COL_MAJOR));
 
       MFEM_CUDSS_CHECK(cudssMatrixCreateDn(&yc, n_global, nrhs_, n_global, NULL,
-                                           CUDSS_REAL_T, CUDSS_LAYOUT_COL_MAJOR));
+                                           CUDA_REAL_T, CUDSS_LAYOUT_COL_MAJOR));
 
 #ifdef MFEM_USE_MPI
       MFEM_CUDSS_CHECK(cudssMatrixSetDistributionRow1d(xc, row_start, row_end));

linalg/ginkgo.hpp

@@ -39,7 +39,6 @@ namespace Ginkgo
 {
 
 template <typename T> using gko_array = gko::array<T>;
-#if defined(MFEM_USE_MPI) && GINKGO_BUILD_MPI
 // for inter-operability with hypre integer types
 using gko_hypre_int =
    std::conditional_t<sizeof(HYPRE_Int) == sizeof(std::int32_t), std::int32_t,
@@ -51,7 +50,6 @@ static_assert(!std::is_void_v<gko_hypre_int>,
               "HYPRE_Int type is incompatible with Ginkgo");
 static_assert(!std::is_void_v<gko_hypre_bigint>,
               "HYPRE_BigInt type is incompatible with Ginkgo");
-#endif
 
 /**
 * Helper class for a case where a wrapped MFEM Vector

linalg/hypre.cpp

@@ -2872,8 +2872,8 @@ void HypreParMatrix::Destroy()
    if (HypreUsingGPU() && ParCSROwner && (diagOwner < 0 || offdOwner < 0))
    {
       // Put the "host" or "hypre" pointers in {i,j,data} of A->{diag,offd}, so
-      // that they can be destroyed by mfem_hypre_TFree_host() or hypre when
-      // hypre_ParCSRMatrixDestroy(A) is called below, respectively.
+      // that they can be destroyed by hypre when hypre_ParCSRMatrixDestroy(A)
+      // is called below.
 
       // Check that if both diagOwner and offdOwner are negative then they have
       // the same value.
@@ -2882,33 +2882,7 @@ void HypreParMatrix::Destroy()
 
       MemoryClass mc = (diagOwner == -1 || offdOwner == -1) ?
                        Device::GetHostMemoryClass() : GetHypreMemoryClass();
-      Write(mc, diagOwner < 0, offdOwner < 0);
-      if (diagOwner == -1)
-      {
-         // Note: mfem_hypre_TFree_host() sets the pointer to NULL.
-         mfem_hypre_TFree_host(hypre_CSRMatrixI(A->diag));
-         if (hypre_CSRMatrixOwnsData(A->diag))
-         {
-            mfem_hypre_TFree_host(hypre_CSRMatrixJ(A->diag));
-            mfem_hypre_TFree_host(hypre_CSRMatrixData(A->diag));
-         }
-#if MFEM_HYPRE_VERSION >= 21800
-         hypre_CSRMatrixMemoryLocation(A->diag) = GetHypreMemoryLocation();
-#endif
-      }
-      if (offdOwner == -1)
-      {
-         // Note: mfem_hypre_TFree_host() sets the pointer to NULL.
-         mfem_hypre_TFree_host(hypre_CSRMatrixI(A->offd));
-         if (hypre_CSRMatrixOwnsData(A->offd))
-         {
-            mfem_hypre_TFree_host(hypre_CSRMatrixJ(A->offd));
-            mfem_hypre_TFree_host(hypre_CSRMatrixData(A->offd));
-         }
-#if MFEM_HYPRE_VERSION >= 21800
-         hypre_CSRMatrixMemoryLocation(A->offd) = GetHypreMemoryLocation();
-#endif
-      }
+      Write(mc, diagOwner < 0, offdOwner <0);
    }
 #endif