trying some things

AMGF Solver

.github/workflows/builds-and-tests.yml

@@ -132,12 +132,14 @@ jobs:
             hypre-target: int32
             precision: fp64
             enzyme: true
-            config-opts: MFEM_USE_ENZYME=YES ENZYME_DIR=$(brew --prefix enzyme)
+            config-opts: MFEM_USE_ENZYME=YES ENZYME_DIR=$(brew --prefix enzyme) LDFLAGS=-L$LLVM_PREFIX/lib/c++
 
     name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}${{ matrix.enzyme && '-enzyme' || '' }}
 
     runs-on: ${{ matrix.os }}
 
+    continue-on-error: ${{ matrix.enzyme && true || false }}
+
     steps:
       # Fix 'No space left on device' errors for Ubuntu builds.
       - name: Run Actions Cleaner
@@ -292,10 +294,12 @@ jobs:
         run: |
           export HOMEBREW_NO_INSTALL_CLEANUP=1
           brew update
-          brew install llvm@20 enzyme
-          echo "LLVM_PREFIX=$(brew --prefix llvm@20)" >> $GITHUB_ENV
-          echo "OMPI_CC=$(brew --prefix llvm@20)/bin/clang" >> $GITHUB_ENV
-          echo "OMPI_CXX=$(brew --prefix llvm@20)/bin/clang++" >> $GITHUB_ENV
+          brew install enzyme
+          ENZYME_LLVM=$(brew info enzyme | sed -n 's/^Required:.*\(llvm[^ ]*\).*/\1/p')
+          LLVM_PREFIX=$(brew --prefix $ENZYME_LLVM)
+          echo "LLVM_PREFIX=$LLVM_PREFIX" >> $GITHUB_ENV
+          echo "OMPI_CC=$LLVM_PREFIX/bin/clang" >> $GITHUB_ENV
+          echo "OMPI_CXX=$LLVM_PREFIX/bin/clang++" >> $GITHUB_ENV
 
       # MFEM build and test
       - name: build

.github/workflows/stale.yml

@@ -29,3 +29,47 @@ jobs:
         operations-per-run: 500
         exempt-issue-labels: "bug,WIP,ready-for-review,in-review,in-next"
         exempt-pr-labels: "bug,WIP,ready-for-review,in-review,in-next"
+
+  # Stale action for PRs with "in-review" label.
+  stale-in-review-pr:
+
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+      actions: write
+
+    steps:
+    - uses: actions/stale@v9
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-pr-message: ':warning: This PR has been automatically marked as stale because it has not had any activity in the last 150 days. *If no activity occurs in the next 30 days, it will be automatically closed.* Thank you for your contributions.'
+        only-pr-labels: "in-review"
+        days-before-pr-stale: 150
+        days-before-pr-close: 30
+        days-before-issue-stale: -1
+        days-before-issue-close: -1
+        stale-pr-label: 'stale'
+        operations-per-run: 500
+
+  # Stale action for PRs with "WIP" label.
+  stale-wip-pr:
+
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+      actions: write
+
+    steps:
+    - uses: actions/stale@v9
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-pr-message: ':warning: This PR has been automatically marked as stale because it has not had any activity in the last 300 days. *If no activity occurs in the next 30 days, it will be automatically closed.* Thank you for your contributions.'
+        only-pr-labels: "WIP"
+        days-before-pr-stale: 300
+        days-before-pr-close: 30
+        days-before-issue-stale: -1
+        days-before-issue-close: -1
+        stale-pr-label: 'stale'
+        operations-per-run: 500

.gitignore

@@ -19,9 +19,15 @@ CMakeFiles/
 # Clangd server cache
 *.cache*
 
+#vscode settings
+/.vscode/
+
 # Backup files
 *~
 
+# clangd index
+/.cache/
+
 # Default install location
 /mfem/
 
@@ -79,6 +85,7 @@ examples/sol_u.*
 examples/sol_p.*
 examples/sol_r.*
 examples/sol_i.*
+examples/sol_z.*
 examples/ex6p-checkpoint.*
 examples/order.*
 examples/ex9.mesh
@@ -208,10 +215,13 @@ miniapps/electromagnetics/volta
 miniapps/electromagnetics/tesla
 miniapps/electromagnetics/maxwell
 miniapps/electromagnetics/joule
+miniapps/electromagnetics/lorentz
 miniapps/electromagnetics/Volta-AMR*
 miniapps/electromagnetics/Tesla-AMR*
 miniapps/electromagnetics/Maxwell-Parallel*
 miniapps/electromagnetics/Joule_[0-9]*
+miniapps/electromagnetics/Lorentz_[0-9]*
+miniapps/electromagnetics/Lorentz.dat
 
 miniapps/gslib/field-diff
 miniapps/gslib/field-interp

.gitlab-ci.yml

@@ -9,91 +9,550 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
+# DESCRIPTION:
+###############################################################################
 # General GitLab pipelines configurations for supercomputers and Linux clusters
-# at Lawrence Livermore National Laboratory (LLNL). This entire pipeline is
-# LLNL-specific!
-
-include:
-  - project: 'lc-templates/id_tokens'
-    file: 'id_tokens.yml'
-
-# The pipeline is divided into stages. Usually, jobs in a given stage wait for
-# the preceding stages to complete before to start. However, we sometimes use
-# the "needs" keyword and express the DAG of jobs for more efficiency.
-# - We use setup and setup_baseline phases to download content outside of mfem
-#   directory.
-# - Allocate/Release is where Dane resource are allocated/released once for all.
-# - Build and Test is where we build and MFEM for multiple toolchains.
-# - Baseline_checks gathers baseline-type test suites execution
-# - Baseline_publish, only available on master, allows to update baseline
-#   results
-stages:
-  - sub-pipelines
+# at Lawrence Livermore National Laboratory (LLNL).
+# This entire pipeline is LLNL-specific
+#
+# Important note: This file is a template provided by llnl/radiuss-shared-ci.
+# Remains to set variable values, change the reference to the radiuss-shared-ci
+# repo, opt-in and out optional features. The project can then extend it with
+# additional stages.
+#
+# In addition, each project should copy over and complete:
+# - .gitlab/custom-jobs-and-variables.yml
+# - .gitlab/subscribed-pipelines.yml
+#
+# The jobs should be specified in a file local to the project,
+# - .gitlab/jobs/${CI_MACHINE}.yml
+# or generated (see LLNL/Umpire for an example).
+###############################################################################
+#   MAP OF GITLAB CI
+#######################
+#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+# File dependencies: direct, through jobs, through variables
+#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+# .gitlab-ci.yml
+#  ├── .build-and-test  [job]
+#  │    ├── .gitlab/custom-jobs-and-variables.yml
+#  │    │    ├── .custom_job          [job]
+#  │    │    ├── .reproducer_vars     [job]
+#  │    │    ├── .report_job_success  [job]
+#  │    │    │    └── .gitlab/scripts/report_build_and_test    [script]
+#  │    │    │         ├── .gitlab/scripts/safe_create_rundir  [script]
+#  │    │    │         └── .gitlab/scripts/git_try_to_push     [script]
+#  │    │    ├── .report_job_failure  [job]
+#  │    │    │    └── .gitlab/scripts/report_build_and_test    [script]
+#  │    │    │         ├── .gitlab/scripts/safe_create_rundir  [script]
+#  │    │    │         └── .gitlab/scripts/git_try_to_push     [script]
+#  │    │    └── JOB_CMD              [var]
+#  │    │         └── tests/gitlab/build_and_test            [script]
+#  │    │              └── tests/gitlab/get_mfem_uberenv     [script]
+#  │    ├── <radiuss-shared-ci>/pipelines/matrix.yml  [conditional]
+#  │    │    ├── .on_matrix               [job]
+#  │    │    ├── .matrix_reproducer_init  [job]
+#  │    │    ├── .matrix_reproducer_vars  [job]
+#  │    │    ├── .matrix_reproducer_job   [job]
+#  │    │    ├── .matrix_job_command      [job]
+#  │    │    └── .job_on_matrix           [job]
+#  │    ├── <radiuss-shared-ci>/pipelines/dane.yml  [conditional]
+#  │    │    ├── .on_dane               [job]
+#  │    │    ├── .dane_reproducer_init  [job]
+#  │    │    ├── .dane_reproducer_vars  [job]
+#  │    │    ├── .dane_reproducer_job   [job]
+#  │    │    ├── .dane_job_command      [job]
+#  │    │    ├── .job_on_dane           [job]
+#  │    │    ├── allocate_resources     [job]
+#  │    │    └── release_resources      [job]
+#  │    ├── <radiuss-shared-ci>/pipelines/tioga.yml  [conditional]
+#  │    │    ├── .on_tioga               [job]
+#  │    │    ├── .tioga_reproducer_init  [job]
+#  │    │    ├── .tioga_reproducer_vars  [job]
+#  │    │    ├── .tioga_reproducer_job   [job]
+#  │    │    ├── .tioga_job_command      [job]
+#  │    │    ├── .job_on_tioga           [job]
+#  │    │    ├── allocate_resources      [job]
+#  │    │    └── release_resources       [job]
+#  │    ├── <artifact>/matrix-jobs.yml  [conditional, from 'generate-job-lists']
+#  │    │    ├── .gitlab/jobs/matrix.yml
+#  │    │    │    ├── .matrix_reproducer_vars      [job]
+#  │    │    │    ├── setup                        [job]
+#  │    │    │    │    └── ./tests/gitlab/build_and_test_setup  [script]
+#  │    │    │    ├── opt_mpi_cuda_gcc             [job]
+#  │    │    │    └── opt_mpi_cuda_hypre_cuda_gcc  [job]
+#  │    │    └── .gitlab/jobs/matrix-reports.yml  [used conditionally]
+#  │    │         ├── report_job_success
+#  │    │         └── report_job_failure
+#  │    ├── <artifact>/dane-jobs.yml    [conditional, from 'generate-job-lists']
+#  │    │    ├── .gitlab/jobs/dane.yml
+#  │    │    │    ├── .dane_reproducer_vars    [job]
+#  │    │    │    ├── setup                    [job]
+#  │    │    │    │    └── ./tests/gitlab/build_and_test_setup  [script]
+#  │    │    │    ├── debug_ser_gcc_10         [job]
+#  │    │    │    ├── debug_par_gcc_10         [job]
+#  │    │    │    ├── opt_ser_gcc_10           [job]
+#  │    │    │    ├── opt_par_gcc_10           [job]
+#  │    │    │    ├── opt_par_gcc_10_sundials  [job]
+#  │    │    │    ├── opt_par_gcc_10_petsc     [job]
+#  │    │    │    └── opt_par_gcc_10_pumi      [job]
+#  │    │    └── .gitlab/jobs/dane-reports.yml    [used conditionally]
+#  │    │         ├── report_job_success
+#  │    │         └── report_job_failure
+#  │    └── <artifact>/tioga-jobs.yml   [conditional, from 'generate-job-lists']
+#  │         ├── .gitlab/jobs/tioga.yml
+#  │         │    ├── .tioga_reproducer_vars  [job]
+#  │         │    ├── setup                   [job]
+#  │         │    │    └── ./tests/gitlab/build_and_test_setup  [script]
+#  │         │    └── cce_16_0_1              [job]
+#  │         └── .gitlab/jobs/tioga-reports.yml   [used conditionally]
+#  │              ├── report_job_success
+#  │              └── report_job_failure
+#  └── .gitlab/subscribed-pipelines.yml
+#       ├── .machine-check         [job]
+#       ├── generate-job-lists     [job]
+#       ├── dane-up-check          [job]
+#       ├── dane-build-and-test    [job]
+#       ├── dane-baseline          [job]
+#       │    └── .gitlab/dane-baseline.yml
+#       │         ├── .on_dane                       [job]
+#       │         ├── baselinecheck_mfem_intel_dane  [job]
+#       │         │    └── .gitlab/scripts/baseline            [script]
+#       │         ├── cleanup                        [job]
+#       │         ├── report_baseline                [job]
+#       │         │    ├── .gitlab/scripts/safe_create_rundir  [script]
+#       │         │    └── .gitlab/scripts/git_try_to_push     [script]
+#       │         ├── baselinepublish_mfem_dane      [job]
+#       │         │    └── .gitlab/scripts/rebaseline          [script]
+#       │         ├── .gitlab/custom-jobs-and-variables.yml
+#       │         │    └── <same as above: see .gitlab-ci.yml/.build-and-test>
+#       │         └── .gitlab/configs/setup-baseline.yml
+#       │              └── setup_baseline            [job]
+#       ├── tioga-up-check         [job]
+#       ├── tioga-build-and-test   [job]
+#       ├── matrix-up-check        [job]
+#       └── matrix-build-and-test  [job]
+#
+#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+# File tree hierarchy with file contents highlights
+#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+# In addition to the files in the MFEM repo, the Gitlab CI uses files from the
+# radiuss/radiuss-shared-ci project, see below, after the <mfem-root> tree.
+#
+# <mfem root>
+#  ├── .gitlab-ci.yml  [this file]
+#  │    ├── <jobs>
+#  │    │    └── .build-and-test
+#  │    ├── <included files>
+#  │    │    ├── .gitlab/subscribed-pipelines.yml
+#  │    │    ├── .gitlab/custom-jobs-and-variables.yml  [by ".build-and-test"]
+#  │    │    ├── <artifact>  [by ".build-and-test"]
+#  │    │    │    ├── artifact: '${CI_MACHINE}-jobs.yml'
+#  │    │    │    └── job:      'generate-job-lists'
+#  │    │    └── <external>  [by ".build-and-test"]
+#  │    │         ├── project: 'radiuss/radiuss-shared-ci'
+#  │    │         ├── ref:     'v2025.09.1'
+#  │    │         └── file:    'pipelines/${CI_MACHINE}.yml'
+#  │    └── <defined variables>
+#  │         ├── CUSTOM_CI_BUILDS_DIR
+#  │         ├── USER_CI_TOP_DIR
+#  │         ├── SHARED_REPOS_DIR
+#  │         ├── AUTOTEST_ROOT
+#  │         ├── MFEM_DATA_DIR
+#  │         ├── AUTOTEST
+#  │         ├── AUTOTEST_COMMIT
+#  │         ├── REBASELINE
+#  │         ├── GITHUB_PROJECT_NAME
+#  │         └── GITHUB_PROJECT_ORG
+#  ├── .gitlab
+#  │    ├── configs
+#  │    │    └── setup-baseline.yml
+#  │    │         ├── <jobs>
+#  │    │         │    └── setup_baseline
+#  │    │         └── <used variables>
+#  │    │              ├── MACHINE_NAME
+#  │    │              ├── REBASELINE
+#  │    │              ├── AUTOTEST
+#  │    │              ├── AUTOTEST_COMMIT
+#  │    │              ├── BUILD_ROOT
+#  │    │              ├── TPLS_REPO
+#  │    │              ├── TESTS_REPO
+#  │    │              ├── AUTOTEST_ROOT
+#  │    │              └── AUTOTEST_REPO
+#  │    ├── jobs
+#  │    │    ├── matrix-reports.yml
+#  │    │    │    ├── <jobs>
+#  │    │    │    │    ├── report_job_success
+#  │    │    │    │    └── report_job_failure
+#  │    │    │    └── <used jobs>
+#  │    │    │         ├── .on_matrix
+#  │    │    │         ├── .report_job_success
+#  │    │    │         └── .report_job_failure
+#  │    │    ├── matrix.yml
+#  │    │    │    ├── <jobs>
+#  │    │    │    │    ├── .matrix_reproducer_vars
+#  │    │    │    │    ├── setup
+#  │    │    │    │    ├── opt_mpi_cuda_gcc
+#  │    │    │    │    └── opt_mpi_cuda_hypre_cuda_gcc
+#  │    │    │    ├── <used jobs>
+#  │    │    │    │    ├── .reproducer_vars
+#  │    │    │    │    ├── .on_matrix
+#  │    │    │    │    └── .job_on_matrix
+#  │    │    │    ├── <included and used files>
+#  │    │    │    │    └── tests/gitlab/build_and_test_setup  [by "setup"]
+#  │    │    │    └── <defined variables>
+#  │    │    │         └── SPEC
+#  │    │    ├── dane-reports.yml
+#  │    │    │    ├── <jobs>
+#  │    │    │    │    ├── report_job_success
+#  │    │    │    │    └── report_job_failure
+#  │    │    │    └── <used jobs>
+#  │    │    │         ├── .on_dane
+#  │    │    │         ├── .report_job_success
+#  │    │    │         └── .report_job_failure
+#  │    │    ├── dane.yml
+#  │    │    │    ├── <jobs>
+#  │    │    │    │    ├── .dane_reproducer_vars
+#  │    │    │    │    ├── setup
+#  │    │    │    │    ├── debug_ser_gcc_10
+#  │    │    │    │    ├── debug_par_gcc_10
+#  │    │    │    │    ├── opt_ser_gcc_10
+#  │    │    │    │    ├── opt_par_gcc_10
+#  │    │    │    │    ├── opt_par_gcc_10_sundials
+#  │    │    │    │    ├── opt_par_gcc_10_petsc
+#  │    │    │    │    └── opt_par_gcc_10_pumi
+#  │    │    │    ├── <used jobs>
+#  │    │    │    │    ├── .reproducer_vars
+#  │    │    │    │    ├── .on_dane
+#  │    │    │    │    └── .job_on_dane
+#  │    │    │    ├── <included and used files>
+#  │    │    │    │    └── tests/gitlab/build_and_test_setup  [by "setup"]
+#  │    │    │    └── <defined variables>
+#  │    │    │         ├── SPEC
+#  │    │    │         └── THREADS
+#  │    │    ├── tioga-reports.yml
+#  │    │    │    ├── <jobs>
+#  │    │    │    │    ├── report_job_success
+#  │    │    │    │    └── report_job_failure
+#  │    │    │    └── <used jobs>
+#  │    │    │         ├── .on_tioga
+#  │    │    │         ├── .report_job_success
+#  │    │    │         └── .report_job_failure
+#  │    │    └── tioga.yml
+#  │    │         ├── <jobs>
+#  │    │         │    ├── .tioga_reproducer_vars
+#  │    │         │    ├── setup
+#  │    │         │    └── opt_mpi_rocm_hypre_rocm
+#  │    │         ├── <used jobs>
+#  │    │         │    ├── .reproducer_vars
+#  │    │         │    ├── .on_tioga
+#  │    │         │    └── .job_on_tioga
+#  │    │         ├── <included and used files>
+#  │    │         │    └── tests/gitlab/build_and_test_setup  [by "setup"]
+#  │    │         └── <defined variables>
+#  │    │              ├── SPEC
+#  │    │              └── THREADS
+#  │    ├── scripts
+#  │    │    ├── baseline
+#  │    │    │    └── <used variables>
+#  │    │    │         ├── BASELINE_TEST
+#  │    │    │         ├── SYS_TYPE
+#  │    │    │         ├── MACHINE_NAME
+#  │    │    │         ├── CI_PROJECT_DIR
+#  │    │    │         ├── ARTIFACTS_DIR
+#  │    │    │         ├── BUILD_ROOT
+#  │    │    │         └── TPLS_DIR
+#  │    │    ├── git_try_to_push
+#  │    │    ├── rebaseline
+#  │    │    │    └── <used variables>
+#  │    │    │         ├── CI_PROJECT_DIR
+#  │    │    │         ├── ARTIFACTS_DIR
+#  │    │    │         ├── SYS_TYPE
+#  │    │    │         ├── BUILD_ROOT
+#  │    │    │         ├── MACHINE_NAME
+#  │    │    │         └── CI_PIPELINE_ID
+#  │    │    ├── report_build_and_test
+#  │    │    │    ├── <used files>
+#  │    │    │    │    ├── .gitlab/scripts/safe_create_rundir
+#  │    │    │    │    └── .gitlab/scripts/git_try_to_push
+#  │    │    │    └── <used variables>
+#  │    │    │         ├── AUTOTEST_ROOT
+#  │    │    │         ├── CI_COMMIT_REF_SLUG
+#  │    │    │         ├── CI_PROJECT_DIR
+#  │    │    │         ├── CI_PIPELINE_URL
+#  │    │    │         ├── AUTOTEST_COMMIT
+#  │    │    │         └── CI_MACHINE
+#  │    │    └── safe_create_rundir
+#  │    ├── custom-jobs-and-variables.yml
+#  │    │    ├── <jobs>
+#  │    │    │    ├── .custom_job
+#  │    │    │    ├── .reproducer_vars
+#  │    │    │    ├── .report_job_success
+#  │    │    │    └── .report_job_failure
+#  │    │    ├── <used files>
+#  │    │    │    ├── tests/gitlab/build_and_test  [in JOB_CMD]
+#  │    │    │    └── .gitlab/scripts/report_build_and_test [by .report_job_*]
+#  │    │    ├── <defined variables>
+#  │    │    │    ├── JOB_CMD
+#  │    │    │    ├── BUILD_ROOT
+#  │    │    │    ├── ALLOC_NAME
+#  │    │    │    ├── TPLS_REPO
+#  │    │    │    ├── TESTS_REPO
+#  │    │    │    ├── AUTOTEST_REPO
+#  │    │    │    ├── MFEM_DATA_REPO
+#  │    │    │    ├── ARTIFACTS_DIR: artifacts
+#  │    │    │    ├── SLURM_OVERLAP: 1
+#  │    │    │    ├── DANE_SHARED_ALLOC
+#  │    │    │    ├── DANE_JOB_ALLOC
+#  │    │    │    ├── TIOGA_SHARED_ALLOC
+#  │    │    │    ├── TIOGA_JOB_ALLOC
+#  │    │    │    └── MATRIX_JOB_ALLOC
+#  │    │    └── <used variables>
+#  │    │         ├── SPEC
+#  │    │         ├── BUILD_ROOT
+#  │    │         └── ...
+#  │    ├── dane-baseline.yml
+#  │    │    ├── <jobs>
+#  │    │    │    ├── .on_dane
+#  │    │    │    ├── baselinecheck_mfem_intel_dane
+#  │    │    │    ├── cleanup
+#  │    │    │    ├── report_baseline
+#  │    │    │    └── baselinepublish_mfem_dane
+#  │    │    ├── <included and used files>
+#  │    │    │    ├── .gitlab/custom-jobs-and-variables.yml
+#  │    │    │    ├── .gitlab/configs/setup-baseline.yml
+#  │    │    │    ├── .gitlab/scripts/rebaseline
+#  │    │    │    ├── .gitlab/scripts/baseline
+#  │    │    │    └── .gitlab/scripts/git_try_to_push
+#  │    │    ├── <defined variables>
+#  │    │    │    ├── BASELINE_TEST: baseline
+#  │    │    │    ├── MACHINE_NAME: dane
+#  │    │    │    ├── TPLS_DIR
+#  │    │    │    └── export MFEM_TEST_NP
+#  │    │    └── <used variables>
+#  │    │         ├── ON_DANE
+#  │    │         ├── AUTOTEST       [defined by .gitlab-ci.yml]
+#  │    │         ├── BUILD_ROOT     [defined by custom-jobs-and-variables.yml]
+#  │    │         ├── TPLS_DIR       [defined by this file]
+#  │    │         ├── ARTIFACTS_DIR  [defined by custom-jobs-and-variables.yml]
+#  │    │         ├── MACHINE_NAME   [defined by this file]
+#  │    │         ├── AUTOTEST_COMMIT  [defined by .gitlab-ci.yml]
+#  │    │         ├── AUTOTEST_ROOT  [defined by .gitlab-ci.yml]
+#  │    │         ├── BASELINE_TEST  [defined by this file]
+#  │    │         └── REBASELINE     [defined by .gitlab-ci.yml]
+#  │    └── subscribed-pipelines.yml
+#  │         ├── <jobs>
+#  │         │    ├── .machine-check
+#  │         │    ├── generate-job-lists
+#  │         │    ├── dane-up-check
+#  │         │    ├── dane-build-and-test
+#  │         │    ├── dane-baseline
+#  │         │    ├── tioga-up-check
+#  │         │    ├── tioga-build-and-test
+#  │         │    ├── matrix-up-check
+#  │         │    └── matrix-build-and-test
+#  │         ├── <used jobs>
+#  │         │    └── .build-and-test  [from ".gitlab-ci.yml"]
+#  │         ├── <included files>
+#  │         │    └── .gitlab/dane-baseline.yml  [by "dane-baseline"]
+#  │         └── <used variables>
+#  │              ├── GITHUB_PROJECT_ORG
+#  │              ├── GITHUB_PROJECT_NAME
+#  │              ├── AUTOTEST
+#  │              ├── AUTOTEST_COMMIT
+#  │              └── REBASELINE
+#  └── tests
+#       ├── gitlab
+#       │    ├── build_and_test
+#       │    │    ├── <builds and tests a given MFEM spec with uberenv>
+#       │    │    ├── <used files>
+#       │    │    │    ├── tests/uberenv/uberenv.py        [deps mode, cloned]
+#       │    │    │    └── tests/gitlab/get_mfem_uberenv   [deps mode]
+#       │    │    └── <used variables>
+#       │    │         ├── SYS_TYPE
+#       │    │         ├── THREADS      [num. parallel jobs to build MFEM]
+#       │    │         ├── MODULE_LIST  [modules to load]
+#       │    │         ├── CI_JOB_ID
+#       │    │         ├── USE_DEV_SHM
+#       │    │         ├── SPACK_DEBUG
+#       │    │         ├── DEBUG_MODE
+#       │    │         ├── REGISTRY_TOKEN
+#       │    │         ├── CI_REGISTRY_USER  (defined by Gitlab)
+#       │    │         ├── USER
+#       │    │         ├── CI_REGISTRY_IMAGE (defined by Gitlab)
+#       │    │         └── CI_JOB_TOKEN      (defined by Gitlab)
+#       │    ├── build_and_test_setup
+#       │    │    ├── <updates MFEM_DATA_REPO and AUTOTEST_REPO using locks>
+#       │    │    └── <used variables>
+#       │    │         ├── MFEM_DATA_REPO
+#       │    │         ├── SHARED_REPOS_DIR
+#       │    │         ├── AUTOTEST_REPO
+#       │    │         └── AUTOTEST_ROOT
+#       │    └── get_mfem_uberenv
+#       │         ├── <github.com/mfem/mfem-uberenv.git -> tests/uberenv>
+#       │         └── <defines the uberenv hash to use>
+#       └── uberenv  [cloned by tests/gitlab/get_mfem_uberenv]
+#            └── uberenv.py
+#
+# <root of radiuss/radiuss-shared-ci, ref: 'v2025.09.1'>
+#  └── pipelines
+#       ├── matrix.yml
+#       │    ├── <jobs>
+#       │    │    ├── .on_matrix
+#       │    │    ├── .matrix_reproducer_init
+#       │    │    ├── .matrix_reproducer_vars
+#       │    │    ├── .matrix_reproducer_job
+#       │    │    ├── .matrix_job_command
+#       │    │    └── .job_on_matrix
+#       │    ├── <used jobs>
+#       │    │    └── .custom_job  [from .gitlab/custom-jobs-and-variables.yml]
+#       │    └── <used variables>
+#       │         ├── ON_MATRIX
+#       │         ├── ADVANCED_JOB
+#       │         ├── ALL_TARGETS
+#       │         ├── SYS_TYPE
+#       │         ├── LLNL_SERVICE_USER
+#       │         ├── USER
+#       │         ├── GITHUB_PROJECT_NAME
+#       │         ├── GITHUB_PROJECT_ORG
+#       │         ├── MATRIX_JOB_ALLOC
+#       │         └── JOB_CMD
+#       ├── dane.yml
+#       │    ├── <jobs>
+#       │    │    ├── .on_dane
+#       │    │    ├── .dane_reproducer_init
+#       │    │    ├── .dane_reproducer_vars
+#       │    │    ├── .dane_reproducer_job
+#       │    │    ├── .dane_job_command
+#       │    │    ├── .job_on_dane
+#       │    │    ├── allocate_resources
+#       │    │    └── release_resources
+#       │    ├── <used jobs>
+#       │    │    └── .custom_job  [from .gitlab/custom-jobs-and-variables.yml]
+#       │    ├── <defined variables>
+#       │    │    └── export JOBID
+#       │    └── <used variables>
+#       │         ├── ON_DANE
+#       │         ├── ADVANCED_JOB
+#       │         ├── ALL_TARGETS
+#       │         ├── SYS_TYPE
+#       │         ├── LLNL_SERVICE_USER
+#       │         ├── USER
+#       │         ├── GITHUB_PROJECT_NAME
+#       │         ├── GITHUB_PROJECT_ORG
+#       │         ├── DANE_JOB_ALLOC
+#       │         ├── JOB_CMD
+#       │         ├── JOBID
+#       │         ├── ALLOC_NAME
+#       │         └── DANE_SHARED_ALLOC
+#       └── tioga.yml
+#            ├── <jobs>
+#            │    ├── .on_tioga
+#            │    ├── .tioga_reproducer_init
+#            │    ├── .tioga_reproducer_vars
+#            │    ├── .tioga_reproducer_job
+#            │    ├── .tioga_job_command
+#            │    ├── .job_on_tioga
+#            │    ├── allocate_resources
+#            │    └── release_resources
+#            ├── <used jobs>
+#            │    └── .custom_job  [from .gitlab/custom-jobs-and-variables.yml]
+#            ├── <defined variables>
+#            │    └── PROXY
+#            └── <used variables>
+#                 ├── ON_TIOGA
+#                 ├── ADVANCED_JOB
+#                 ├── ALL_TARGETS
+#                 ├── SYS_TYPE
+#                 ├── LLNL_SERVICE_USER
+#                 ├── USER
+#                 ├── GITHUB_PROJECT_NAME
+#                 ├── GITHUB_PROJECT_ORG
+#                 ├── TIOGA_JOB_ALLOC
+#                 ├── JOB_CMD
+#                 ├── PROXY
+#                 ├── ALLOC_NAME
+#                 └── TIOGA_SHARED_ALLOC
 
+###############################################################################
+# We define the following GitLab pipeline variables:
 variables:
+##### LC GITLAB CONFIGURATION
   CUSTOM_CI_BUILDS_DIR: "/usr/workspace/mfem/gitlab-runner"
+
+##### PROJECT VARIABLES
   USER_CI_TOP_DIR: "${CUSTOM_CI_BUILDS_DIR}/${GITLAB_USER_LOGIN}"
   SHARED_REPOS_DIR: "${USER_CI_TOP_DIR}/repos"
   AUTOTEST_ROOT: "${SHARED_REPOS_DIR}"
-  # MFEM_DATA_DIR is setup in '.gitlab/configs/setup-build-and-test.yml' and
-  # used in '.gitlab/configs/<machine>-config.yml':
   MFEM_DATA_DIR: "${SHARED_REPOS_DIR}/mfem-data"
-
+# AUTOTEST: enable (ON/YES) or disable (any other value) test reporting. See
+# also AUTOTEST_COMMIT.
+  AUTOTEST: "OFF"
+# AUTOTEST_COMMIT: used only when AUTOTEST is set to ON/YES.
+# * If AUTOTEST_COMMIT is set to ON/YES, reporting jobs will commit their
+#   files to the MFEM/autotest repo.
+# * If AUTOTEST_COMMIT is NOT set to ON/YES, reporting jobs will NOT commit
+#   their files to the MFEM/autotest repo. Instead they will just show the
+#   contents of the report files and remove them.
+  AUTOTEST_COMMIT: "ON"
+# REBASELINE:
 # Defines the default choice for updating the saved baseline results. By default
 # the baseline can only be updated from the master branch. This variable offers
 # the option to manually ask for rebaselining from another branch if necessary.
-  REBASELINE: "NO"
-  AUTOTEST: "NO"
-  # AUTOTEST_COMMIT: used only when AUTOTEST is set to YES.
-  # * If AUTOTEST_COMMIT is NOT set to NO, reporting jobs will commit their
-  #   files to the MFEM/autotest repo.
-  # * If AUTOTEST_COMMIT is set to NO, reporting jobs will NOT commit their
-  #   files to the MFEM/autotest repo. Instead they will just show the contents
-  #   of the report files and remove them.
-  AUTOTEST_COMMIT: "YES"
+  REBASELINE: "OFF"
 
-# Trigger subpipelines:
-dane-build-and-test:
-  stage: sub-pipelines
+##### SHARED_CI CONFIGURATION
+# Required information about GitHub repository
+  GITHUB_PROJECT_NAME: "mfem"
+  GITHUB_PROJECT_ORG: "MFEM"
+# Override the pattern describing branches that will skip the "draft PR filter
+# test".  Add protected branches here. See default value in
+# preliminary-ignore-draft-pr.yml.
+#  ALWAYS_RUN_PATTERN: ""
+
+###############################################################################
+##### High level stages
+# We organize the test-pipelines stage with sub-pipelines. Each sub-pipeline
+# corresponds to a test batch on a given machine.
+stages:
+  - prerequisites
+  - test-pipelines
+
+###############################################################################
+# Template for jobs triggering a build-and-test sub-pipeline:
+.build-and-test:
+  stage: test-pipelines
   variables:
-    # Explicitly pass down values that we want to be able to set when triggering
-    # pipelines manually or using scheduling
+    # Explicitly pass down values that are not always propagated to child
+    # pipelines, e.g. when a variable is set in the "Settings -> CI" web
+    # interface (project variables).
+    # Note: in some cases, this does not work as expected, e.g. when the
+    # variable is not re-defined in the web interface; in such cases, the child
+    # pipeline gets a definition like '${AUTOTEST}', i.e. it behaves as if
+    # AUTOTEST is undefined, even though there is a default value in
+    # .gitlab-ci.yml.
     AUTOTEST: "${AUTOTEST}"
     AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
   trigger:
-    include: .gitlab/dane-build-and-test.yml
+    include:
+      - local: '.gitlab/custom-jobs-and-variables.yml'
+      - project: 'radiuss/radiuss-shared-ci'
+        ref: 'v2025.09.1'
+        file: 'pipelines/${CI_MACHINE}.yml'
+      - artifact: '${CI_MACHINE}-jobs.yml'
+        job: 'generate-job-lists'
     strategy: depend
+    forward:
+      pipeline_variables: true
 
-dane-baseline:
-  stage: sub-pipelines
-  variables:
-    # Explicitly pass down values that we want to be able to set when triggering
-    # pipelines manually or using scheduling
-    REBASELINE: "${REBASELINE}"
-    AUTOTEST: "${AUTOTEST}"
-    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
-  trigger:
-    include: .gitlab/dane-baseline.yml
-    strategy: depend
-
-lassen-build-and-test:
-  stage: sub-pipelines
-  variables:
-    # Explicitly pass down values that we want to be able to set when triggering
-    # pipelines manually or using scheduling
-    AUTOTEST: "${AUTOTEST}"
-    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
-  trigger:
-    include: .gitlab/lassen-build-and-test.yml
-    strategy: depend
-
-corona-build-and-test:
-  stage: sub-pipelines
-  variables:
-    # Explicitly pass down values that we want to be able to set when triggering
-    # pipelines manually or using scheduling
-    AUTOTEST: "${AUTOTEST}"
-    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
-  trigger:
-    include: .gitlab/corona-build-and-test.yml
-    strategy: depend
+###############################################################################
+include:
+  # Sets ID tokens for every job using `default:`
+  - project: 'lc-templates/id_tokens'
+    file: 'id_tokens.yml'
+  # [Optional] checks preliminary to running the actual CI test
+  #- project: 'radiuss/radiuss-shared-ci'
+  #  ref: 'v2025.09.1'
+  #  file: 'preliminary-ignore-draft-pr.yml'
+  # pipelines subscribed by the project
+  - local: '.gitlab/subscribed-pipelines.yml'

.gitlab/README.md

@@ -8,6 +8,8 @@
                                https://mfem.org
 
 
+FIXME: this file needs to be updated
+
 This directory contains most of the GitLab CI configuration. MFEM runs both PR
 and nightly testing on GitLab.
 
@@ -15,9 +17,9 @@ and nightly testing on GitLab.
 
 ## Top level
 
-The root configuration file is `.gitlab-ci.yml` at the root of MFEM repo.
-This file only defines one stage, in which we trigger several
-sub-pipelines.
+The root configuration file is `.gitlab-ci.yml` at the root of MFEM repo. This
+file only defines three stages, a prerequisites one, and two main stages in
+which we trigger several sub-pipelines.
 
 We use sub-pipelines to isolate the test for one combination of `machine`
 and `test type`.
@@ -25,8 +27,8 @@ and `test type`.
 Machines typically include:
 
 * Dane: Intel Sapphire Rapids
-* Lassen: Power9 + Nvidia GPU
-* Corona: AMD GPU
+* Matrix: Intel Sapphire Rapids + Nvidia H100 GPU
+* Tioga: AMD MI250X GPU
 
 Test types include:
 
@@ -39,9 +41,31 @@ altering the scheduling, execution and displaying of the others.
 
 ## Sub-pipelines
 
-Each file is this directory is the root configuration file for one
-sub-pipeline. The naming reflects the corresponding couple (`machine`,
-`test_type`).
+### build-and-test
+
+The build-and-test sub-pipelines leverage RADIUSS Shared CI to share most of
+the CI implementation. RADIUSS Shared CI provides a shared CI infrastructure
+vetted on most LC systems of interest and efficiently leveraging each machine
+scheduler to increase CI throughput. The maintenance of RADIUSS Shared CI is
+shared among several RADIUSS projects.
+
+Jobs for the build-and-test sub-pipelines are defined in the jobs directory.
+Because build-and-test jobs leverage Uberenv and Spack to build the
+dependencies automatically, the jobs essentially consists in a `spack spec`
+defined in the jobs files, and some scheduling parameters defined in the
+`.gitlab/custom-jobs-and-variables.yml` file.
+
+Build-and-test jobs all run the `tests/gitlab/build_and_test` script.
+
+The build-and-test pipelines are controlled by the
+`.gitlab/subscribed-pipelines.yml` which defines which machines to run on and
+implements additional features like machine availability check, and job list
+generation.
+
+### baseline
+
+Baseline sub-pipelines are described by files with names reflecting the
+machine it runs on, e.g. `dane-baseline`.
 
 Those files define the *stages* and the *jobs* for the sub-pipeline. They
 also contain any configuration that cannot be shared. For the most part
@@ -63,11 +87,11 @@ usage function. This should be improved.
 
 # More testing
 
-## Adding a new target to a build_and_test pipeline
+## Adding a new target to a build-and-test pipeline
 
-`build_and_test` pipelines rely on Spack to install dependencies. Spack is
+`build-and-test` pipelines rely on Spack to install dependencies. Spack is
 driven by Uberenv which helps freezing Spack configuration: the goal being to
-point to specific commit in Spack and isolate its configuration so that it is
+point to a specific commit in Spack and isolate its configuration so that it is
 not influenced by the user environment. More documentation about this can be
 found in `tests/gitlab`.
 
@@ -82,7 +106,7 @@ spack spec to use. Adding a job on Dane for example resumes to:
 <job_name>:
   variables:
     SPEC: "<spack_spec>"
-  extends: .build_and_test_on_dane
+  extends: .job_on_dane
 ```
 
 The remaining and non trivial work is to make sure this spec is working. To

.gitlab/configs/common.yml

@@ -1,40 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-include:
-  - project: 'lc-templates/id_tokens'
-    file: 'id_tokens.yml'
-
-# We define the following GitLab pipeline variables:
-variables:
-
-# The path to the shared resource between all jobs. For example, external
-# repositories like 'tests' and 'tpls' are cloned here. Also, 'tpls' is built
-# once for all targets, so that build happen here. The BUILD_ROOT is unique to
-# the pipeline, preventing any form of concurrency with other pipelines. This
-# also means that the BUILD_ROOT directory will never be cleaned.
-# TODO: add a clean-up mechanism
-  BUILD_ROOT: ${USER_CI_TOP_DIR}/${CI_PROJECT_NAME}-${MACHINE_NAME}-pipeline-${CI_PIPELINE_ID}
-
-# On LLNL's Dane, there is only one allocation shared among jobs in order to
-# save time and resource. This allocation has to be uniquely named so that we
-# are sure to retrieve it.
-  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
-
-# Git repositories used in the pipeline
-  TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
-  TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
-  AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
-  MFEM_DATA_REPO: https://github.com/mfem/data.git
-
-# Directory used to place artifacts.
-  ARTIFACTS_DIR: artifacts
-  SLURM_OVERLAP: 1

.gitlab/configs/corona-config.yml

@@ -1,59 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# GitLab pipeline configuration for the Corona machine at LLNL
-variables:
-  MACHINE_NAME: corona
-
-.on_corona:
-  tags:
-    - shell
-    - corona
-  rules:
-    # Don't run corona jobs if...
-    # Note: This makes corona an "opt-in" machine. To activate builds on corona
-    # for a given GitLab clone of MFEM, go to Setting/CI-CD/variables, and set
-    # "ON_CORONA" to "ON". An LC account on for corona is required to trigger a
-    # pipeline there.
-    - if: '$CI_COMMIT_BRANCH =~ /_cnone/ || $ON_CORONA != "ON"'
-      when: never
-    # Don't run autotest update if...
-    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
-      when: never
-    # Report success on success status
-    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
-      when: on_success
-    # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
-      when: on_failure
-    # Always release resource
-    - if: '$CI_JOB_NAME =~ /release_resource/'
-      when: always
-    # Always cleanup
-    - if: '$CI_JOB_NAME =~ /cleanup/'
-      when: always
-    # Default is to run if previous stage succeeded
-    - when: on_success
-
-# Spack helped builds
-# Generic corona build job, extending build script
-.build_and_test_on_corona:
-  extends: [.on_corona]
-  stage: build_and_test
-  script:
-    # THREADS is used by 'tests/gitlab/build_and_test', run below
-    - export THREADS=12
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - echo ${MFEM_DATA_DIR}
-    - echo ${SPEC}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 15 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data

.gitlab/configs/dane-config.yml

@@ -1,56 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# GitLab pipelines configurations for the Dane machine at LLNL
-variables:
-  MACHINE_NAME: dane
-
-.on_dane:
-  tags:
-    - shell
-    - dane
-  rules:
-    # Don't run dane jobs if...
-    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_DANE == "OFF"'
-      when: never
-    # Don't run autotest update if...
-    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
-      when: never
-    # Report success on success status
-    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
-      when: on_success
-    # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
-      when: on_failure
-    # Always release resource
-    - if: '$CI_JOB_NAME =~ /release_resource/'
-      when: always
-    # Always cleanup
-    - if: '$CI_JOB_NAME =~ /cleanup/'
-      when: always
-    # Default is to run if previous stage succeeded
-    - when: on_success
-
-# Spack helped builds
-# Generic dane build job, extending build script
-.build_and_test_on_dane:
-  extends: [.on_dane]
-  stage: build_and_test
-  script:
-    # THREADS is used by 'tests/gitlab/build_and_test', run below
-    # Dane has 224 threads/node and we run 7 separate jobs: 224=7*32
-    - export THREADS=28
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - echo ${MFEM_DATA_DIR}
-    - echo ${SPEC}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) --reservation=ci -t 60 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data

.gitlab/configs/lassen-config.yml

@@ -1,48 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# GitLab pipelines configurations for the Lassen machine at LLNL
-variables:
-  MACHINE_NAME: lassen
-
-.on_lassen:
-  tags:
-    - shell
-    - lassen
-  rules:
-    - if: '$CI_COMMIT_BRANCH =~ /_lnone/ || $ON_LASSEN == "OFF"' #run except if ...
-      when: never
-    # Don't run autotest update if...
-    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
-      when: never
-    # Report success on success status
-    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
-      when: on_success
-    # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
-      when: on_failure
-    # Always cleanup
-    - if: '$CI_JOB_NAME =~ /cleanup/'
-      when: always
-    - when: on_success
-
-# Lassen uses a different job scheduler (spectrum lsf) that does not allow
-# pre-allocation the same way slurm does.  We use the pci queue on lassen
-# to speed-up the allocation.
-.build_and_test_on_lassen:
-  extends: [.on_lassen]
-  stage: build_and_test
-  script:
-    - echo ${MFEM_DATA_DIR}
-    - echo ${SPEC}
-    # Next script uses 'THREADS': leaving it empty --> it uses 'make all -j'
-    - lalloc 1 -W 45 -q pci --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
-  needs: [setup]

.gitlab/configs/report-build-and-test.yml

@@ -1,77 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Jobs report
-.report_job_success:
-  script:
-    - echo ${MACHINE_NAME}
-    - echo ${AUTOTEST}
-    - echo ${AUTOTEST_COMMIT}
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - cd ${AUTOTEST_ROOT}
-    - |
-      (
-        date
-        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
-        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -n 9; do
-          sleep 5
-        done
-        echo "Acquired lock on '$PWD/autotest.lock'"
-        date
-        # Report SUCCESS while holding the file lock on 'autotest.lock'.
-        # The next script uses the following environment variables:
-        # - MACHINE_NAME, AUTOTEST_ROOT, AUTOTEST_COMMIT
-        # - CI_COMMIT_REF_SLUG, CI_PROJECT_DIR, CI_PIPELINE_URL
-        # It also calls the script '.gitlab/scripts/safe_create_rundir'.
-        ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test_success
-        err=$?
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock
-
-.report_job_failure:
-  script:
-    - echo ${MACHINE_NAME}
-    - echo ${AUTOTEST}
-    - echo ${AUTOTEST_COMMIT}
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - cd ${AUTOTEST_ROOT}
-    - |
-      (
-        date
-        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
-        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -n 9; do
-          sleep 5
-        done
-        echo "Acquired lock on '$PWD/autotest.lock'"
-        date
-        # Report FAILURE while holding the file lock on 'autotest.lock'.
-        # The next script uses the following environment variables:
-        # - MACHINE_NAME, AUTOTEST_ROOT, AUTOTEST_COMMIT
-        # - CI_COMMIT_REF_SLUG, CI_PROJECT_DIR, CI_PIPELINE_URL
-        # It also calls the script '.gitlab/scripts/safe_create_rundir'.
-        ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test_failure
-        err=$?
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock

.gitlab/configs/setup-build-and-test.yml

@@ -1,90 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test
-# script then symlinks the repo to the parent directory of the MFEM source
-# directory. Unit tests that depend on the mfem/data repo will then detect that
-# this directory is present and be enabled.
-setup:
-  tags:
-    - shell
-    - dane
-  stage: setup
-  variables:
-    GIT_STRATEGY: none
-  script:
-    #
-    # Setup MFEM_DATA_DIR=${SHARED_REPOS_DIR}/mfem-data, see '.gitlab-ci.yml'
-    # and '.gitlab/configs/<machine>-config.yml'
-    #
-    - echo "MACHINE_NAME = ${MACHINE_NAME}"
-    - echo "AUTOTEST = ${AUTOTEST}"
-    - echo "AUTOTEST_COMMIT = ${AUTOTEST_COMMIT}"
-    - echo "SHARED_REPOS_DIR ${SHARED_REPOS_DIR}"
-    - mkdir -p ${SHARED_REPOS_DIR} && cd ${SHARED_REPOS_DIR}
-    - command -v flock || echo "Required command 'flock' not found"
-    - |
-      (
-        date
-        echo "Waiting to acquire lock on '$PWD/mfem-data.lock' ..."
-        # try to get an exclusive lock on fd 9 (mfem-data.lock) repeating the
-        # try every 5 seconds; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -n 9; do
-          sleep 5
-        done
-        echo "Acquired lock on '$PWD/mfem-data.lock'"
-        date
-        # clone/update the mfem/data repo while holding the file lock on
-        # 'mfem-data.lock'
-        err=0
-        if [[ ! -d "mfem-data" ]]; then
-          git clone ${MFEM_DATA_REPO} "mfem-data"
-        else
-          cd "mfem-data" && git pull && cd ..
-        fi || err=1
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> mfem-data.lock
-    #
-    # Setup ${AUTOTEST_ROOT}/autotest:
-    #
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - mkdir -p ${AUTOTEST_ROOT} && cd ${AUTOTEST_ROOT}
-    - |
-      (
-        date
-        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
-        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -n 9; do
-          sleep 5
-        done
-        echo "Acquired lock on '$PWD/autotest.lock'"
-        date
-        # clone/update the autotest repo while holding the file lock on
-        # 'autotest.lock'
-        err=0
-        if [[ ! -d "autotest" ]]; then
-          git clone ${AUTOTEST_REPO}
-        else
-          cd autotest && git pull && cd ..
-        fi || err=1
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock

.gitlab/corona-build-and-test.yml

@@ -1,67 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-stages:
-  - setup
-  - allocate_resource
-  - build_and_test
-  - release_resource_and_report
-
-# Slurm shared allocation
-allocate_resource:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_corona
-  stage: allocate_resource
-  script:
-    - echo ${ALLOC_NAME}
-    - salloc --exclusive --nodes=1 --partition=mi60 --time=45 --no-shell --job-name=${ALLOC_NAME}
-  timeout: 6h
-  needs: [setup]
-
-# Build and test jobs, simply provide a spec
-rocm_gcc_8.3.1:
-  variables:
-    SPEC: "@develop%gcc@8.3.1+rocm amdgpu_target=gfx906"
-  extends: .build_and_test_on_corona
-  needs: [allocate_resource]
-
-# Release slurm allocation
-release_resource:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_corona
-  stage: release_resource_and_report
-  script:
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
-  needs: [rocm_gcc_8.3.1]
-
-# Jobs report
-report_job_success:
-  stage: release_resource_and_report
-  extends:
-    - .on_corona
-    - .report_job_success
-
-report_job_failure:
-  stage: release_resource_and_report
-  extends:
-    - .on_corona
-    - .report_job_failure
-
-include:
-  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/corona-config.yml
-  - local: .gitlab/configs/setup-build-and-test.yml
-  - local: .gitlab/configs/report-build-and-test.yml

.gitlab/custom-jobs-and-variables.yml

@@ -0,0 +1,132 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+include:
+  - project: 'lc-templates/id_tokens'
+    file: 'id_tokens.yml'
+
+# We define the following GitLab pipeline variables:
+variables:
+# Set the build-and-test command.
+# Nested variables are allowed and useful to customize the job command. We
+# protect variables with quotes so that their value may remain a string even if
+# they contain whitespaces.
+  JOB_CMD:
+    value: tests/gitlab/build_and_test --spec \"${SPEC}\" --data-dir ${MFEM_DATA_DIR} --data
+# The path to the shared resource between all jobs in the 'dane-baseline'
+# pipeline. For example, external repositories like 'tests' and 'tpls' are
+# cloned here. Also, 'tpls' is built once for all targets, so that build happens
+# here. The BUILD_ROOT is unique to the pipeline, preventing any form of
+# concurrency with other pipelines. This directory is removed by the 'cleanup'
+# stage in the 'dane-baseline' pipeline.
+  BUILD_ROOT: ${USER_CI_TOP_DIR}/${CI_PROJECT_NAME}-${CI_MACHINE}-pipeline-${CI_PIPELINE_ID}
+
+# On LLNL's dane and tioga, the 'build-and-test' pipelines creates only one
+# allocation shared among jobs in the pipeline in order to save time and
+# resources. This allocation has to be uniquely named so that we are sure to
+# retrieve it and avoid collisions.
+  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
+
+# Git repositories used in the pipelines:
+# - TPLS_REPO and TESTS_REPO are used only by the 'dane-baseline' pipeline
+# - AUTOTEST_REPO is used by all pipelines
+# - MFEM_DATA_REPO is used only by the 'build-and-test' pipelines
+  TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
+  TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
+  AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
+  MFEM_DATA_REPO: https://github.com/mfem/data.git
+
+# Directory used to place artifacts:
+# - ARTIFACTS_DIR is only used by the 'dane-baseline' pipeline
+  ARTIFACTS_DIR: artifacts
+  SLURM_OVERLAP: 1
+
+# Dane
+# Arguments for top level allocation
+  DANE_SHARED_ALLOC: "--exclusive --reservation=ci --time=60 --nodes=1"
+# Arguments for job level allocation
+# Note: We repeat the reservation, necessary when jobs are manually re-triggered.
+  DANE_JOB_ALLOC: "--reservation=ci --overlap --nodes=1"
+
+# Tioga
+# Arguments for top level allocation
+  TIOGA_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=45m --nodes=1"
+# Arguments for job level allocation
+  TIOGA_JOB_ALLOC: "--nodes=1 --begin-time=+5s"
+
+# Matrix
+# Arguments for top level allocation
+  MATRIX_SHARED_ALLOC: "-p pdebug --exclusive --time=45 --nodes=1 -G 4"
+# Arguments for job level allocation
+# Note: We repeat the reservation, necessary when jobs are manually re-triggered.
+  MATRIX_JOB_ALLOC: "--overlap --nodes=1"
+
+# Configuration shared by build and test jobs specific to this project.
+# Not all configuration can be shared. Here projects can fine tune the
+# CI behavior.
+# See Umpire for an example (export junit test reports).
+.custom_job:
+  artifacts:
+    reports:
+
+# Note: this part is not used by the 'dane-baseline' pipeline.
+# FIXME: BUILD_ROOT, TPLS_REPO, TESTS_REPO are not needed here.
+#        Also, the definition of SHARED_REPOS_DIR is wrong.
+.reproducer_vars:
+  script:
+    - |
+      echo -e "
+      # Variables \n
+      export SPEC=\"${SPEC//\"/\\\"}\" \n
+      # Directories \n
+      export BUILD_ROOT=\"\${working_dir}\" \n
+      export SHARED_REPOS_DIR=\"\${BUILD_ROOT}/..\" \n
+      export MFEM_DATA_DIR=\"\${SHARED_REPOS_DIR}/mfem-data\" \n
+      # Repositories \n
+      export TPLS_REPO=\"${TPLS_REPO//\"/\\\"}\" \n
+      export TESTS_REPO=\"${TESTS_REPO//\"/\\\"}\" \n
+      export AUTOTEST_REPO=\"${AUTOTEST_REPO//\"/\\\"}\" \n
+      export MFEM_DATA_REPO=\"${MFEM_DATA_REPO//\"/\\\"}\" \n
+      # Setup directories \n
+      ./tests/gitlab/build_and_test_setup \n
+      # Using the CI build cache is optional and requires a token. Set it like so: \n
+      # export REGISTRY_TOKEN=\"<your token here>\" \n"
+      #
+
+# Jobs report
+.report_job_success:
+  script:
+    - ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test SUCCESS
+  rules:
+    - when: on_success
+
+.report_job_failure:
+  script:
+    - ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test FAILURE
+  rules:
+    - when: on_failure
+
+# Keep the following for debugging purposes: renaming this job from
+# '.show_variables' to 'show_variables' will insert this debug job at the
+# beginning of all child pipelines.
+.show_variables:
+  tags: [shell, oslic]
+  variables:
+    GIT_STRATEGY: none
+  stage: .pre
+  script:
+    - |
+      echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+      echo "AUTOTEST=${AUTOTEST}"
+      echo "AUTOTEST_COMMIT=${AUTOTEST_COMMIT}"
+      echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+      # Fail the job on purpose to prevent the rest of the pipeline from running
+      false

.gitlab/dane-baseline.yml

@@ -11,6 +11,7 @@
 
 variables:
   BASELINE_TEST: baseline
+  MACHINE_NAME: dane
 
 stages:
   - setup
@@ -19,6 +20,25 @@ stages:
   - cleanup
   - baseline_publish
 
+.on_dane:
+  tags:
+    - shell
+    - dane
+  rules:
+    # Don't run dane jobs if...
+    - if: '$ON_DANE == "OFF"'
+      when: never
+    # Don't run autotest update if...
+    # Note: in some cases, the content of AUTOTEST can be '${AUTOTEST}', so we
+    # need to treat that value as the default value of 'OFF'.
+    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "ON" && $AUTOTEST != "YES"'
+      when: never
+    # Always cleanup
+    - if: '$CI_JOB_NAME =~ /cleanup/'
+      when: always
+    # Default is to run if previous stage succeeded
+    - when: on_success
+
 baselinecheck_mfem_intel_dane:
   extends: [.on_dane]
   stage: baseline_check
@@ -29,6 +49,9 @@ baselinecheck_mfem_intel_dane:
     #       .gitlab/configs/setup-baseline.yml.
     TPLS_DIR: ${BUILD_ROOT}/tpls
   script:
+    - echo "AUTOTEST=$AUTOTEST"
+    - echo "AUTOTEST_COMMIT=$AUTOTEST_COMMIT"
+    - echo "AUTOTEST_ROOT=$AUTOTEST_ROOT"
     - echo ${BUILD_ROOT}
     - echo ${TPLS_DIR}
     # Used by the tests in MFEM/tests, dane has 224 threads/node:
@@ -89,7 +112,13 @@ report_baseline:
           cp ${rundir}/pipeline.txt ${rundir}/autotest-email.html
         fi
         msg="GitLab CI log for ${BASELINE_TEST} on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
-        if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
+        # Note: in some cases, the content of AUTOTEST_COMMIT can be
+        # '${AUTOTEST_COMMIT}', so we need to treat that value as the default
+        # value of 'ON'.
+        if [[ "$AUTOTEST_COMMIT" == '${AUTOTEST_COMMIT}' ]]; then
+          AUTOTEST_COMMIT="ON"
+        fi
+        if [[ "$AUTOTEST_COMMIT" == "ON" || "$AUTOTEST_COMMIT" == "YES" ]]; then
           git pull && \
           git add ${rundir} && \
           git commit -m "${msg}" && \
@@ -117,8 +146,8 @@ baselinepublish_mfem_dane:
   extends: [.on_dane]
   stage: baseline_publish
   rules:
-    # - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "YES"'
-    - if: '$REBASELINE == "YES"'
+    # - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "ON"'
+    - if: '$REBASELINE == "ON"'
       when: manual
   script:
     - echo ${BUILD_ROOT}
@@ -128,6 +157,5 @@ baselinepublish_mfem_dane:
     - .gitlab/scripts/rebaseline
 
 include:
-  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/dane-config.yml
+  - local: .gitlab/custom-jobs-and-variables.yml
   - local: .gitlab/configs/setup-baseline.yml

.gitlab/dane-build-and-test.yml

@@ -1,94 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-stages:
-  - setup
-  - allocate_resource
-  - build_and_test
-  - release_resource_and_report
-
-# Allocate
-allocate_resource:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_dane
-  stage: allocate_resource
-  script:
-    - echo ${ALLOC_NAME}
-    - salloc --exclusive --nodes=1 --reservation=ci --time=60 --no-shell --job-name=${ALLOC_NAME}
-  timeout: 6h
-
-# GitLab jobs for the Dane machine at LLNL
-debug_ser_gcc_10:
-  variables:
-    SPEC: "%gcc@10.3.1 +debug~mpi"
-  extends: .build_and_test_on_dane
-
-debug_par_gcc_10:
-  variables:
-    SPEC: "%gcc@10.3.1 +debug+mpi"
-  extends: .build_and_test_on_dane
-
-opt_ser_gcc_10:
-  variables:
-    SPEC: "%gcc@10.3.1 ~mpi"
-  extends: .build_and_test_on_dane
-
-opt_par_gcc_10:
-  variables:
-    SPEC: "%gcc@10.3.1"
-  extends: .build_and_test_on_dane
-
-opt_par_gcc_10_sundials:
-  variables:
-    SPEC: "%gcc@10.3.1 +sundials"
-  extends: .build_and_test_on_dane
-
-opt_par_gcc_10_petsc:
-  variables:
-    SPEC: "%gcc@10.3.1 +petsc ^petsc+mumps~superlu-dist"
-  extends: .build_and_test_on_dane
-
-opt_par_gcc_10_pumi:
-  variables:
-    SPEC: "%gcc@10.3.1 +pumi"
-  extends: .build_and_test_on_dane
-
-# Release
-release_resource:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_dane
-  stage: release_resource_and_report
-  script:
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
-
-# Jobs report
-report_job_success:
-  stage: release_resource_and_report
-  extends:
-    - .on_dane
-    - .report_job_success
-
-report_job_failure:
-  stage: release_resource_and_report
-  extends:
-    - .on_dane
-    - .report_job_failure
-
-include:
-  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/dane-config.yml
-  - local: .gitlab/configs/setup-build-and-test.yml
-  - local: .gitlab/configs/report-build-and-test.yml

.gitlab/jobs/dane-reports.yml

@@ -0,0 +1,19 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Jobs report
+report_job_success:
+  extends: [.on_dane, .report_job_success]
+  stage: jobs-stage-3
+
+report_job_failure:
+  extends: [.on_dane, .report_job_failure]
+  stage: jobs-stage-3

.gitlab/jobs/dane.yml

@@ -0,0 +1,87 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Override reproducer section to define MFEM specific variables.
+.dane_reproducer_vars:
+  script:
+    - !reference [.reproducer_vars, script]
+
+# TODO: Setup script should be defined as a bash script (but then GIT_STRATEGY
+#       cannot be "none" anymore).
+
+# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test
+# script then symlinks the repo to the parent directory of the MFEM source
+# directory. Unit tests that depend on the mfem/data repo will then detect that
+# this directory is present and be enabled.
+setup:
+  extends: .on_dane
+  stage: jobs-stage-1
+  script:
+    - ./tests/gitlab/build_and_test_setup
+
+
+########################
+# Overridden shared jobs
+########################
+# When using shared jobs, we can duplicate them here to override description and
+# add necessary changes.
+# We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS} So that
+# the comparison with the original job is easier.
+
+
+############
+# Extra jobs
+############
+# We do not recommend using ${PROJECT_<MACHINE>_VARIANTS} and
+# ${PROJECT_<MACHINE>_DEPS} in the extra jobs. There is not reason not to fully
+# describe the spec here.
+
+.mfem_job_on_dane:
+  extends: .job_on_dane
+  stage: jobs-stage-2
+  variables:
+    # Dane has 224 threads/node and we run 7 separate jobs: 224=7*32
+    THREADS: 28
+
+debug_ser_gcc_10:
+  extends: .mfem_job_on_dane
+  variables:
+    SPEC: "%gcc@10.3.1 +debug~mpi"
+
+debug_par_gcc_10:
+  extends: .mfem_job_on_dane
+  variables:
+    SPEC: "%gcc@10.3.1 +debug+mpi"
+
+opt_ser_gcc_10:
+  extends: .mfem_job_on_dane
+  variables:
+    SPEC: "%gcc@10.3.1 ~mpi"
+
+opt_par_gcc_10:
+  extends: .mfem_job_on_dane
+  variables:
+    SPEC: "%gcc@10.3.1"
+
+opt_par_gcc_10_sundials:
+  extends: .mfem_job_on_dane
+  variables:
+    SPEC: "%gcc@10.3.1 +sundials"
+
+opt_par_gcc_10_petsc:
+  extends: .mfem_job_on_dane
+  variables:
+    SPEC: "%gcc@10.3.1 +petsc ^petsc+mumps~superlu-dist"
+
+opt_par_gcc_10_pumi:
+  extends: .mfem_job_on_dane
+  variables:
+    SPEC: "%gcc@10.3.1 +pumi"

.gitlab/jobs/matrix-reports.yml

@@ -0,0 +1,19 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Jobs report
+report_job_success:
+  extends: [.on_matrix, .report_job_success]
+  stage: jobs-stage-3
+
+report_job_failure:
+  extends: [.on_matrix, .report_job_failure]
+  stage: jobs-stage-3

.gitlab/jobs/matrix.yml

@@ -0,0 +1,65 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Override reproducer section to define UMPIRE specific variables.
+.matrix_reproducer_vars:
+  script:
+    - !reference [.reproducer_vars, script]
+
+#TODO: Setup script should be defined as a bash script (but then GIT_STRATEGY cannot be "none" anymore).
+
+# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test
+# script then symlinks the repo to the parent directory of the MFEM source
+# directory. Unit tests that depend on the mfem/data repo will then detect that
+# this directory is present and be enabled.
+setup:
+  extends: .on_matrix
+  stage: jobs-stage-1
+  script:
+    - ./tests/gitlab/build_and_test_setup
+
+
+########################
+# Overridden shared jobs
+########################
+# When using shared jobs , we can duplicate them here to override description and add necessary changes.
+# We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS} So that
+# the comparison with the original job is easier.
+
+
+############
+# Extra jobs
+############
+# We do not recommend using ${PROJECT_<MACHINE>_VARIANTS} and
+# ${PROJECT_<MACHINE>_DEPS} in the extra jobs. There is not reason not to fully
+# describe the spec here.
+
+.mfem_job_on_matrix:
+  extends: .job_on_matrix
+  stage: jobs-stage-2
+  variables:
+    # We run 2 jobs on 1 node that has 112 threads
+    THREADS: 48
+    # These modules need to be consistent with the uberenv configurations:
+    MODULE_LIST: "gcc/10.3.1-magic cuda/12.9.1"
+
+allocate_resources:
+  timeout: 4h
+
+opt_mpi_cuda_gcc:
+  extends: .mfem_job_on_matrix
+  variables:
+    SPEC: "%gcc@10.3.1 +mpi +cuda cuda_arch=90"
+
+opt_mpi_cuda_hypre_cuda_gcc:
+  extends: .mfem_job_on_matrix
+  variables:
+    SPEC: "%gcc@10.3.1 +mpi +cuda cuda_arch=90 ^hypre+cuda"

.gitlab/jobs/tioga-reports.yml

@@ -0,0 +1,20 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Jobs report
+report_job_success:
+  extends: [.on_tioga, .report_job_success]
+  stage: jobs-stage-3
+
+report_job_failure:
+  extends: [.on_tioga, .report_job_failure]
+  stage: jobs-stage-3
+

.gitlab/jobs/tioga.yml

@@ -0,0 +1,70 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Override reproducer section to define UMPIRE specific variables.
+.tioga_reproducer_vars:
+  script:
+    - !reference [.reproducer_vars, script]
+
+#TODO: Setup script should be defined as a bash script (but then GIT_STRATEGY cannot be "none" anymore).
+
+# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test
+# script then symlinks the repo to the parent directory of the MFEM source
+# directory. Unit tests that depend on the mfem/data repo will then detect that
+# this directory is present and be enabled.
+setup:
+  extends: .on_tioga
+  stage: jobs-stage-1
+  script:
+    - ./tests/gitlab/build_and_test_setup
+
+########################
+# Overridden shared jobs
+########################
+# When using shared jobs , we can duplicate them here to override description and add necessary changes.
+# We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS} So that
+# the comparison with the original job is easier.
+
+
+############
+# Extra jobs
+############
+# We do not recommend using ${PROJECT_<MACHINE>_VARIANTS} and
+# ${PROJECT_<MACHINE>_DEPS} in the extra jobs. There is not reason not to fully
+# describe the spec here.
+
+# Build and test jobs, simply provide a spec
+
+#.tioga_job_command:
+#  script:
+#    - echo PROXY="${PROXY}"
+#    - echo TIOGA_JOB_ALLOC="${TIOGA_JOB_ALLOC}"
+#    - "printf '#!/bin/bash\n%s\n' \"${JOB_CMD}\" > flux_script.sh"
+#    - cat flux_script.sh
+#    - ${PROXY} flux watch $( ${PROXY} flux batch -o output.stdout.type=kvs ${TIOGA_JOB_ALLOC} flux_script.sh )
+#    - rm -f flux_script.sh
+
+.mfem_job_on_tioga:
+  extends: .job_on_tioga
+  stage: jobs-stage-2
+  variables:
+    # We run 1 job on 1 node that has 64 threads
+    THREADS: 64
+
+opt_mpi_rocm_hypre_rocm:
+  extends: .mfem_job_on_tioga
+  variables:
+    SPEC: "%rocmcc@=6.3.1 +rocm amdgpu_target=gfx90a ^hypre+rocm"
+
+# cce_16_0_1:
+#   extends: .mfem_job_on_tioga
+#   variables:
+#     SPEC: "%cce@=16.0.1"

.gitlab/lassen-build-and-test.yml

@@ -1,44 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-stages:
-  - setup
-  - build_and_test
-  - report
-
-opt_mpi_cuda_gcc:
-  variables:
-    SPEC: "%gcc@8.3.1 +mpi +cuda cuda_arch=70"
-  extends: .build_and_test_on_lassen
-
-opt_mpi_cuda_hypre_cuda_gcc:
-  variables:
-    SPEC: "%gcc@8.3.1 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
-  extends: .build_and_test_on_lassen
-
-# Jobs report
-report_job_success:
-  stage: report
-  extends:
-    - .on_lassen
-    - .report_job_success
-
-report_job_failure:
-  stage: report
-  extends:
-    - .on_lassen
-    - .report_job_failure
-
-include:
-  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/lassen-config.yml
-  - local: .gitlab/configs/setup-build-and-test.yml
-  - local: .gitlab/configs/report-build-and-test.yml

.gitlab/scripts/baseline

@@ -35,8 +35,6 @@ if [[ "${MACHINE_NAME}" == "dane" ]]; then
   salloc --nodes=1 --exclusive --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "corona" ]]; then
   salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
-elif [[ ${MACHINE_NAME} == "lassen" ]]; then
-  lalloc 1 -q pci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 else
   echo "Unknown machine: MACHINE_NAME=$MACHINE_NAME"
   exit 1

.gitlab/scripts/report_build_and_test

@@ -0,0 +1,118 @@
+#!/bin/bash
+
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+function info_msg ()
+{
+  echo "[Information:] ${1}"
+}
+
+function error_msg ()
+{
+  echo "[Error:] ${1}"
+}
+
+# Perform a report while holding a lock file to prevent concurrency on
+# the destination.
+# Usage:
+# locked_clone <report_function> <lock_name>
+function locked_report ()
+{
+  if ! command -v flock
+  then
+    error_msg "Required command 'flock' not found"
+    exit 1
+  fi
+
+  info_msg "Will report ${1} while holding a lock in ${2}"
+
+  ( date; info_msg "Waiting to acquire lock on '${PWD}/${2}.lock' ..."
+    # try to get an exclusive lock on fd 9 (mfem-data.lock) repeating the
+    # try every 5 seconds; we may want to add a counter for the number of
+    # retries to interrupt a potential infinite loop
+    while ! flock -n 9; do sleep 5; done
+    date; info_msg "Acquired lock on '${PWD}/${2}.lock'"
+
+    report ${1}
+    err=$?
+
+    # sleep for a period to allow NFS to propagate the above changes;
+    # clearly, there is no guarantee that other NFS clients will see the
+    # changes even after the timeout
+    sleep 10
+    exit $err
+  ) 9> ${2}.lock
+}
+
+function report ()
+{
+  if [[ "${1}" == "SUCCESS" ]]
+  then
+    info_msg "All the ${MACHINE_NAME} jobs passed"
+    status_msg="The 'build-and-test' jobs on ${MACHINE_NAME} were SUCCESSFUL."
+  elif [[ "${1}" == "FAILURE" ]]
+  then
+    info_msg "At least one failure on ${MACHINE_NAME}"
+    status_msg="Some 'build-and-test' jobs on ${MACHINE_NAME} FAILED."
+  else
+    error_msg "Unknown status: ${1} ... aborting"
+    exit 1
+  fi
+
+  cd ${AUTOTEST_ROOT}/autotest || \
+    { error_msg "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
+  mkdir -p ${MACHINE_NAME}
+
+  rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
+  rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir)
+
+  printf "%s\n" "${status_msg}" \
+    "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.err
+
+  msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
+
+  if [[ "${1}" == "FAILURE" ]]
+  then
+    # Create 'autotest-email.html' to indicate failure:
+    cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
+  fi
+
+  # Note: in some cases, the content of AUTOTEST_COMMIT can be
+  # '${AUTOTEST_COMMIT}', so we need to treat that value as the default
+  # value of 'ON'.
+  if [[ "$AUTOTEST_COMMIT" == '${AUTOTEST_COMMIT}' ]]; then
+    AUTOTEST_COMMIT="ON"
+  fi
+  if [[ "$AUTOTEST_COMMIT" == "ON" || "$AUTOTEST_COMMIT" == "YES" ]]; then
+    git pull && \
+    git add ${rundir} && \
+    git commit -m "${msg}" && \
+    ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
+  else
+    for file in ${rundir}/*; do
+      echo "------------------------------"
+      echo "Content of '$file'"
+      echo "******************************"
+      cat $file
+      echo "******************************"
+    done
+    rm -rf ${rundir} || true
+  fi
+}
+
+export MACHINE_NAME=${CI_MACHINE}
+info_msg "MACHINE_NAME is ${MACHINE_NAME}"
+info_msg "AUTOTEST_ROOT is ${AUTOTEST_ROOT}"
+info_msg "AUTOTEST=$AUTOTEST"
+info_msg "AUTOTEST_COMMIT=$AUTOTEST_COMMIT"
+
+cd ${AUTOTEST_ROOT} && locked_report ${1} autotest

.gitlab/scripts/report_build_and_test_failure

@@ -1,45 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-echo "Runs if there was at least one failure on ${MACHINE_NAME}"
-
-cd ${AUTOTEST_ROOT}/autotest || \
-  { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
-mkdir -p ${MACHINE_NAME}
-
-rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
-rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir)
-
-printf "%s\n" "Some 'build-and-test' jobs on ${MACHINE_NAME} FAILED." \
-  "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.err
-
-msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
-
-# Create 'autotest-email.html' to indicate failure:
-cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
-
-if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
-  git pull && \
-  git add ${rundir} && \
-  git commit -m "${msg}" && \
-  ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
-else
-  for file in ${rundir}/*; do
-    echo "------------------------------"
-    echo "Content of '$file'"
-    echo "******************************"
-    cat $file
-    echo "******************************"
-  done
-  rm -rf ${rundir} || true
-fi

.gitlab/scripts/report_build_and_test_success

@@ -1,42 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-echo "Can only run if all the ${MACHINE_NAME} jobs passed"
-
-cd ${AUTOTEST_ROOT}/autotest || \
-  { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
-mkdir -p ${MACHINE_NAME}
-
-rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
-rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir)
-
-printf "%s\n" "The 'build-and-test' jobs on ${MACHINE_NAME} were SUCCESSFUL." \
-  "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.out
-
-msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
-
-if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
-  git pull && \
-  git add ${rundir} && \
-  git commit -m "${msg}" && \
-  ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
-else
-  for file in ${rundir}/*; do
-    echo "------------------------------"
-    echo "Content of '$file'"
-    echo "******************************"
-    cat $file
-    echo "******************************"
-  done
-  rm -rf ${rundir} || true
-fi

.gitlab/subscribed-pipelines.yml

@@ -0,0 +1,130 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# The template job to test whether a machine is up.
+# Expects CI_MACHINE defined to machine name.
+.machine-check:
+  stage: prerequisites
+  tags: [shell, oslic]
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - |
+      if [[ $(jq '.[env.CI_MACHINE].total_nodes_up' /usr/global/tools/lorenz/data/loginnodeStatus) == 0 ]]
+      then
+        echo -e "\e[31mNo node available on ${CI_MACHINE}\e[0m"
+        false && \
+        curl --url "https://api.github.com/repos/${GITHUB_PROJECT_ORG}/${GITHUB_PROJECT_NAME}/statuses/${CI_COMMIT_SHA}" \
+             --header 'Content-Type: application/json' \
+             --header "authorization: Bearer ${GITHUB_TOKEN}" \
+             --data "{ \"state\": \"failure\", \"target_url\": \"${CI_PIPELINE_URL}\", \"description\": \"GitLab ${CI_MACHINE} down\", \"context\": \"ci/gitlab/${CI_MACHINE}\" }"
+        exit 1
+      fi
+
+###
+# Trigger a build-and-test pipeline for a machine.
+# Comment the jobs for machines you don’t need.
+###
+
+# One job to generate the job list for all the subpipelines
+generate-job-lists:
+  stage: prerequisites
+  tags: [shell, oslic]
+  variables:
+    LOCAL_JOBS_PATH: ".gitlab/jobs"
+  script:
+    - |
+      echo "AUTOTEST=$AUTOTEST"
+      echo "AUTOTEST_COMMIT=$AUTOTEST_COMMIT"
+      echo "AUTOTEST_ROOT=$AUTOTEST_ROOT"
+    - |
+      cat ${LOCAL_JOBS_PATH}/dane.yml > dane-jobs.yml
+      if [[ ${AUTOTEST} == "ON" || ${AUTOTEST} == "YES" ]]
+      then
+          cat ${LOCAL_JOBS_PATH}/dane-reports.yml >> dane-jobs.yml
+      fi
+    - |
+      cat ${LOCAL_JOBS_PATH}/matrix.yml > matrix-jobs.yml
+      if [[ ${AUTOTEST} == "ON" || ${AUTOTEST} == "YES" ]]
+      then
+          cat ${LOCAL_JOBS_PATH}/matrix-reports.yml >> matrix-jobs.yml
+      fi
+    - |
+      cat ${LOCAL_JOBS_PATH}/tioga.yml > tioga-jobs.yml
+      if [[ ${AUTOTEST} == "ON" || ${AUTOTEST} == "YES" ]]
+      then
+          cat ${LOCAL_JOBS_PATH}/tioga-reports.yml >> tioga-jobs.yml
+      fi
+  artifacts:
+    paths:
+      - dane-jobs.yml
+      - matrix-jobs.yml
+      - tioga-jobs.yml
+
+
+# DANE
+dane-up-check:
+  variables:
+    CI_MACHINE: "dane"
+  extends: [.machine-check]
+
+dane-build-and-test:
+  variables:
+    CI_MACHINE: "dane"
+  needs: [dane-up-check, generate-job-lists]
+  extends: [.build-and-test]
+
+# DANE, MFEM Specific
+dane-baseline:
+  stage: test-pipelines
+  variables:
+    # Explicitly pass down values that are not always propagated to child
+    # pipelines, e.g. when a variable is set in the "Settings -> CI" web
+    # interface (project variables).
+    # Note: in some cases, this does not work as expected, e.g. when the
+    # variable is not re-defined in the web interface; in such cases, the child
+    # pipeline gets a definition like '${AUTOTEST}', i.e. it behaves as if
+    # AUTOTEST is undefined, even though there is a default value in
+    # .gitlab-ci.yml.
+    AUTOTEST: "${AUTOTEST}"
+    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
+  trigger:
+    include: .gitlab/dane-baseline.yml
+    strategy: depend
+    forward:
+      pipeline_variables: true
+  needs: [dane-up-check]
+
+
+# TIOGA
+tioga-up-check:
+  variables:
+    CI_MACHINE: "tioga"
+  extends: [.machine-check]
+
+tioga-build-and-test:
+  variables:
+    CI_MACHINE: "tioga"
+  needs: [tioga-up-check, generate-job-lists]
+  extends: [.build-and-test]
+
+
+# Matrix
+matrix-up-check:
+  variables:
+    CI_MACHINE: "matrix"
+  extends: [.machine-check]
+
+matrix-build-and-test:
+  variables:
+    CI_MACHINE: "matrix"
+  needs: [matrix-up-check, generate-job-lists]
+  extends: [.build-and-test]

CHANGELOG

@@ -27,9 +27,34 @@ Discretization improvements
 - In the ParMoonolith integration, added support for variational resampling of
   H1 vector fields.
 
+- Added support for boundary integration to the hyperbolic framework. In this
+  regard, new classes `BdrHyperbolicDirichletIntegrator` and
+  `BoundaryHyperbolicFlowIntegrator` have been introduced for implementation
+  of weak Dirichlet boundary conditions with a general flux or for the linear
+  case respectively.
+
+- Added method to compute piecewise linear bounds on high-order functions on
+  tensor-product elements.
+
+- Parallel anisotropic refinement of hexahedral meshes is now supported,
+  provided that neighboring hexahedra are not refined in conflicting directions.
+  A new ParMesh method is added to check for such conflicts, before refinement.
+
 Meshing improvements
 --------------------
 
+- The TMOP kernel hierarchy has been restructured to reduce compilation time.
+  Most large kernels have been split into smaller, specific ones, with kernels
+  for each metric. The directory structure has been updated with assemble,
+  metrics, mult and tools subdirectories. The new kernel dispatch and
+  specialization system has also been integrated.
+  Unit tests have been revised to ensure --all tests pass.
+
+- Introduced NC-patch NURBS meshes, which are conforming element-wise but allow
+  for nonconforming patch topology. This new mesh format supports element
+  spacing formulas for refinement, as well as local refinement factors for a
+  subset of knot vectors.
+
 - Added support for higher order meshes in Mesh::MakeSimplicial and
   ParMesh::MakeSimplicial.
 
@@ -44,13 +69,28 @@ GPU computing
   set. This is most often used for setting constant essential boundary
   conditions. A new function Vector::SetSubVectorHost has been added in cases
   where host execution is always needed (e.g. when the DOFs array is small).
+
 - Introduced MFEM_FOREACH_THREAD_DIRECT, which directly maps loop tasks to GPU
   threads, assigning one task per thread.
+
 - Implemented a GPU-accelerated matrix-free AMR derefinement `GridFunction`
   update operator. This supports mixed geometry meshes and variable order
   spaces, and is the default derefinement operator constructed by
   `FiniteElementSpace::Update` and `ParFiniteElementSpace::Update`.
   The operator requires `FiniteElementSpace::Nonconforming() == true`.
+- Added new method: GridFunction::GetGradients, with GPU support, for computing
+  the gradients of a GridFunction on all elements.
+- Added GPU support in GradientGridFunctionCoefficient and
+  InnerProductCoefficient by implementing their Project methods.
+
+Linear and nonlinear solvers
+----------------------------
+- Added `FilteredSolver`: a base class for solvers with filtering. It handles cases 
+  where a solver performs well except in small subspaces, by adding a filtering step 
+  formulated as a subspace correction.
+- Added `AMGFSolver`: a derived class of `FilteredSolver`, specialized for 
+  AMG with Filtering (AMGF), providing robust preconditioning for linear systems 
+  arising in constrained optimization problems such as frictionless contact.
 
 New and updated examples and miniapps
 -------------------------------------
@@ -60,10 +100,22 @@ New and updated examples and miniapps
   operators as smoothers.
   These miniapps can be found in `miniapps/diag-smoothers`.
 
-API changes
+- Added a new miniapp (meshing/mesh-bounding-boxes) that computes the bounding
+  boxes for each element of a given mesh, and the bounds on the determinant of
+  the Jacobian of the transformation.
+
+- Added a new miniapp (tools/gridfunction-bounds) to compute piecewise linear
+  bounds on a given high-order grid function.
+
+- Added a new miniapp (electromagnetics/lorentz) which computes the trajectory
+  of a charged particle, subject to Lorentz forces, in electrostatic and/or
+  magnetostatic fields as computed by the volta or tesla miniapps.
+
+API changes:
 -----------
 - mfem::internal::tensor and mfem::internal::dual have been moved to
   mfem::future::tensor and mfem::future::dual.
+
 - API addition: in class `Operator`, added virtual functions: `AbsMult`, and
   `AbsMultTranspose`; in class `Vector`, added `Abs` and `Pow`.
 
@@ -71,16 +123,25 @@ Miscellaneous
 -------------
 - Added the "gpu", "raja-gpu", and "ceed-gpu" backend aliases/shortcuts which
   automatically select between CUDA or HIP.
+
 - The CUDA-specific names used by some of the unit tests like 'cunit_tests' and
   'pcunit_tests' were replaced by names using 'gpu' instead of 'c' (short for
   CUDA) or 'cuda'. These tests automatically run the CUDA/HIP tests based on the
   MFEM build configuration.
+
 - Added the option to enable GPU-aware MPI in MFEM using the environment
   variable 'MFEM_GPU_AWARE_MPI' set to any value. Setting this environment
   variable is an alternative to calling 'Device::SetGPUAwareMPI(true)'.
+
 - Added parallel Address Sanitizer, serial and parallel Undefined Behavior
   Sanitizer and serial Memory Sanitizer GitHub actions tests on Ubuntu.
 
+- FindPointsGSLIB has a new constructor that accepts the mesh object and
+  internally calls the Setup() method so that the user does not have to.
+  The FreeData() method has also been moved to the destructor so the user does
+  not need to manually free-up the memory if the destructor is called before
+  MPI_Finalize().
+
 Version 4.8, released on Apr 9, 2025
 ====================================
 

CMakeLists.txt

@@ -133,33 +133,49 @@ if (MFEM_USE_CUDA)
    if (NOT CMAKE_CUDA_HOST_COMPILER)
       set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
    endif()
-   if (CMAKE_VERSION VERSION_LESS 3.18.0)
-      set(CUDA_FLAGS "-arch=${CUDA_ARCH} ${CUDA_FLAGS}")
-   elseif (NOT CMAKE_CUDA_ARCHITECTURES)
-      string(REGEX REPLACE "^sm_" "" ARCH_NUMBER "${CUDA_ARCH}")
-      if ("${CUDA_ARCH}" STREQUAL "sm_${ARCH_NUMBER}")
-         set(CMAKE_CUDA_ARCHITECTURES "${ARCH_NUMBER}")
-      else()
-         message(FATAL_ERROR "Unknown CUDA_ARCH: ${CUDA_ARCH}")
-      endif()
+   if (NOT CMAKE_CUDA_ARCHITECTURES)
+      # make CUDA_ARCH resemble the same form as CMAKE_CUDA_ARCHITECTURES
+      string(REPLACE "sm_" "" CUDA_ARCH_TMP "${CUDA_ARCH}")
+      string(REPLACE "," ";" CUDA_ARCH "${CUDA_ARCH_TMP}")
+      set(CMAKE_CUDA_ARCHITECTURES "${CUDA_ARCH}")
    else()
       set(CUDA_ARCH "CMAKE_CUDA_ARCHITECTURES: ${CMAKE_CUDA_ARCHITECTURES}")
    endif()
    message(STATUS "Using CUDA architecture: ${CUDA_ARCH}")
    enable_language(CUDA)
    if (CMAKE_VERSION VERSION_LESS 3.18.0)
-     # backup try to detect if this is clang or nvcc
-     if(CMAKE_CUDA_COMPILER MATCHES "nvcc$")
-       # nvcc
-       set(MFEM_CUDA_COMPILER_IS_NVCC ON)
-       set(CUDA_FLAGS "${CUDA_FLAGS} --expt-extended-lambda --expt-relaxed-constexpr")
-     endif()
+      # backup try to detect if this is clang or nvcc
+      if(CMAKE_CUDA_COMPILER MATCHES "nvcc$")
+         # nvcc
+         set(MFEM_CUDA_COMPILER_IS_NVCC ON)
+         set(CUDA_FLAGS "${CUDA_FLAGS} --expt-extended-lambda --expt-relaxed-constexpr")
+         if ("all" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}"
+             OR "native" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}"
+             OR "all-major" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}")
+            set(CUDA_FLAGS "-arch=${CMAKE_CUDA_ARCHITECTURES} ${CUDA_FLAGS}")
+         else()
+            # build -gencode sequence for multiple architectures
+            foreach(ENTRY IN LISTS CMAKE_CUDA_ARCHITECTURES)
+               set(CUDA_FLAGS
+                   "-gencode arch=compute_${ENTRY},code=sm_${ENTRY} ${CUDA_FLAGS}")
+            endforeach()
+         endif()
+      else()
+         # build cuda-gpu-arch sequence for multiple architectures
+         # does not support all/all-major/native
+         foreach(ENTRY IN LISTS CMAKE_CUDA_ARCHITECTURES)
+            set(CUDA_FLAGS "-cuda-gpu-arch=sm_${ENTRY} ${CUDA_FLAGS}")
+         endforeach()
+      endif()
    else()
-     if (CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA")
-       # nvcc
-       set(MFEM_CUDA_COMPILER_IS_NVCC ON)
-       set(CUDA_FLAGS "${CUDA_FLAGS} --expt-extended-lambda --expt-relaxed-constexpr")
-     endif()
+      # TODO: all, native, all-major require CMake 3.24+
+      # backport support for CMake 3.18 to 3.24
+      if (CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA")
+         # nvcc
+         set(MFEM_CUDA_COMPILER_IS_NVCC ON)
+         set(CUDA_FLAGS
+             "${CUDA_FLAGS} --expt-extended-lambda --expt-relaxed-constexpr")
+      endif()
    endif()
    set(CMAKE_CUDA_STANDARD ${CMAKE_CXX_STANDARD} CACHE STRING
       "CUDA standard to use.")
@@ -242,10 +258,16 @@ endif()
 
 # AMD HIP
 if (MFEM_USE_HIP)
-  if (HIP_ARCH)
-    message(STATUS "Using HIP architecture: ${HIP_ARCH}")
-    set(GPU_TARGETS "${HIP_ARCH}" CACHE STRING "HIP targets to compile for")
+  if (NOT CMAKE_HIP_ARCHITECTURES)
+    if (HIP_ARCH)
+      set(CMAKE_HIP_ARCHITECTURES CACHE STRING "HIP targets to compile for" "${HIP_ARCH}")
+      set(GPU_TARGETS "${HIP_ARCH}" CACHE STRING "HIP targets to compile for" FORCE)
+    endif()
+  else()
+    set(HIP_ARCH CACHE STRING "HIP targets to compile for" "${CMAKE_HIP_ARCHITECTURES}")
+    set(GPU_TARGETS "${CMAKE_HIP_ARCHITECTURES}" CACHE STRING "HIP targets to compile for" FORCE)
   endif()
+  message(STATUS "Using HIP architecture: ${CMAKE_HIP_ARCHITECTURES}")
   if (ROCM_PATH)
     list(INSERT CMAKE_PREFIX_PATH 0 ${ROCM_PATH})
   endif()
@@ -278,6 +300,22 @@ if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
   endif()
 endif()
 
+# Warn user if deprecated FETCH_TPLS is provided
+if (DEFINED FETCH_TPLS)
+  message(STATUS "Setting MFEM_FETCH_TPLS to user-provided value of FETCH_TPLS (i.e., MFEM_FETCH_TPLS=${FETCH_TPLS})")
+  set (MFEM_FETCH_TPLS FETCH_TPLS)
+  message(DEPRECATION "The use of FETCH_TPLS is deprecated and will be removed in future verison. Please use MFEM_FETCH_TPLS instead.")
+endif()
+
+  # Umpire (must be included before hypre, so hypre can use it if needed)
+if (MFEM_USE_UMPIRE)
+   # umpire uses FindCUDA, which needs CMP0146=OLD in CMake >= 3.27
+   if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.27.0)
+      cmake_policy(SET CMP0146 OLD)
+   endif()
+   find_package(UMPIRE REQUIRED)
+endif()
+
 # MPI -> hypre; PETSc (optional)
 if (MFEM_USE_MPI)
   find_package(MPI REQUIRED)
@@ -495,14 +533,13 @@ endif()
 
 # RAJA
 if (MFEM_USE_RAJA)
+   # RAJA uses FindCUDA, which needs CMP0146=OLD in CMake >= 3.27
+   if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.27.0)
+      cmake_policy(SET CMP0146 OLD)
+   endif()
    find_package(RAJA REQUIRED)
 endif()
 
-# UMPIRE
-if (MFEM_USE_UMPIRE)
-   find_package(UMPIRE REQUIRED)
-endif()
-
 # GOOGLE-BENCHMARK
 if (MFEM_USE_BENCHMARK)
    find_package(Benchmark REQUIRED)
@@ -596,7 +633,7 @@ set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
     NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
     ADIOS2 MKL_CPARDISO MKL_PARDISO AMGX MAGMA CUSPARSE CUBLAS CALIPER CODIPACK
     BENCHMARK PARELAG TRIBOL MPI_CXX HIP HIPBLAS HIPSPARSE MOONOLITH BLITZ
-    ALGOIM ENZYME)
+    ALGOIM ENZYME CUDA::cudart)
 
 # Add all created targets and *_FOUND libraries in the variables TPL_TARGETS and
 # TPL_LIBRARIES, respectively.
@@ -614,6 +651,7 @@ foreach(TPL IN LISTS MFEM_TPLS)
     endif()
   endif()
 endforeach(TPL)
+
 list(REVERSE TPL_LIBRARIES)
 list(REMOVE_DUPLICATES TPL_LIBRARIES)
 list(REVERSE TPL_LIBRARIES)

INSTALL

@@ -123,7 +123,7 @@ Parallel build:
 
 Parallel build with fetching of hypre and METIS:
    mkdir <mfem-buil-dir> ; cd <mfem-build-dir>
-   cmake <mfem-source-dir> -DMFEM_USE_MPI=YES -DFETCH_TPLS=YES
+   cmake <mfem-source-dir> -DMFEM_USE_MPI=YES -DMFEM_FETCH_TPLS=YES
    make -j 4
 
 CUDA build:
@@ -859,7 +859,7 @@ The specific libraries and their options are:
   URL: https://github.com/CEED/libCEED
        https://ceed.exascaleproject.org/libceed
   Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED >= 0.12.
+  Versions: libCEED >= 0.12.0.
 
 - RAJA (optional), used when MFEM_USE_RAJA = YES.
   Beginning with MFEM v4.5.1, only RAJA v2022.10.3+ is supported.
@@ -1081,9 +1081,10 @@ The following options are CMake specific:
 MFEM_ENABLE_TESTING  - Enable the ctest framework for testing.
 MFEM_ENABLE_EXAMPLES - Build all of the examples by default.
 MFEM_ENABLE_MINIAPPS - Build all of the miniapps by default.
-FETCH_TPLS           - Enable fetching of all supported third-party libraries.
-HYPRE_FETCH          - Enable fetching of hypre.
-METIS_FETCH          - Enable fetching of metis.
+MFEM_FETCH_TPLS      - Enable fetching of all supported third-party libraries.
+MFEM_FETCH_GSLIB     - Enable fetching of gslib.
+MFEM_FETCH_HYPRE     - Enable fetching of hypre.
+MFEM_FETCH_METIS     - Enable fetching of metis.
 
 External libraries (CMake):
 ---------------------------
@@ -1149,6 +1150,7 @@ The MFEM CMake build system also provides fetching (automated building) for the
 packages/libraries listed below. Note that when fetching is enabled, any related
 auto-detection functionality is disabled.
 
+ - GSLIB
  - HYPRE
  - METIS
 

config/cmake/MFEMConfig.cmake.in

@@ -84,6 +84,31 @@ set_and_check(MFEM_LIBRARY_DIR "@PACKAGE_LIB_INSTALL_DIR@")
 
 check_required_components(MFEM)
 
+include(CMakeFindDependencyMacro)
+
+if (MFEM_USE_CUDA)
+  # required for projects linking to MFEM+CUDA, even if they don't use CUDA directly
+  find_dependency(CUDAToolkit)
+endif (MFEM_USE_CUDA)
+
+if (MFEM_USE_HIP)
+  # hip/rocm uses the modern MFEM way of linking to targets, need to find dependencies
+  find_dependency(HIP)
+  find_dependency(HIPBLAS)
+  find_dependency(HIPSPARSE)
+  if (MFEM_USE_MPI)
+    # assume HYPRE uses HIP
+    # alternatively could check HYPRE_USING_HIP
+    find_dependency(rocsparse)
+    find_dependency(rocrand)
+    find_dependency(rocsolver)
+  endif (MFEM_USE_MPI)
+endif (MFEM_USE_HIP)
+
+if (MFEM_USE_RAJA)
+   find_dependency(RAJA)
+endif()
+
 if (NOT TARGET mfem)
   include(${CMAKE_CURRENT_LIST_DIR}/MFEMTargets.cmake)
 endif (NOT TARGET mfem)

config/cmake/modules/FindGSLIB.cmake

@@ -9,10 +9,47 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-# Defines the following variables:
+# Defines the following variables if fetching of TPLs is disabled (default):
 #   - GSLIB_FOUND
 #   - GSLIB_LIBRARIES
 #   - GSLIB_INCLUDE_DIRS
+# otherwise, the following are defined:
+#   - GSLIB (imported library target)
+
+if (MFEM_FETCH_GSLIB OR MFEM_FETCH_TPLS)
+  enable_language(C)
+  string(TOUPPER "${CMAKE_BUILD_TYPE}" BUILD_TYPE)
+  set(GSLIB_FETCH_VERSION 1.0.9)
+  set(GSLIB_C_FLAGS ${CMAKE_C_FLAGS_${BUILD_TYPE}})
+  if (CMAKE_C_FLAGS)
+    set(GSLIB_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BUILD_TYPE}}")
+  endif()
+  if (BUILD_SHARED_LIBS)
+    set(GSLIB_C_FLAGS "${GSLIB_C_FLAGS} -fPIC")
+  endif()
+  add_library(GSLIB STATIC IMPORTED)
+  # define external project and create future include directory so it is present
+  # to pass CMake checks at end of MFEM configuration step
+  message(STATUS "Will fetch GSLIB ${GSLIB_FETCH_VERSION} to be built with ${GSLIB_C_FLAGS}")
+  set(PREFIX ${CMAKE_BINARY_DIR}/fetch/gslib)
+  include(ExternalProject)
+  ExternalProject_Add(gslib
+    GIT_REPOSITORY https://github.com/Nek5000/gslib
+    GIT_TAG v${GSLIB_FETCH_VERSION}
+    GIT_SHALLOW TRUE
+    UPDATE_DISCONNECTED TRUE
+    PREFIX ${PREFIX}
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND cd ${PREFIX}/src/gslib && $(MAKE) clean && $(MAKE) DESTDIR=${PREFIX} MPI=$<BOOL:${MFEM_USE_MPI}> "CFLAGS= ${GSLIB_C_FLAGS}"
+    INSTALL_COMMAND "")
+  file(MAKE_DIRECTORY ${PREFIX}/include)
+  # set imported library target properties
+  add_dependencies(GSLIB gslib)
+  set_target_properties(GSLIB PROPERTIES
+    IMPORTED_LOCATION ${PREFIX}/lib/libgs.a
+    INTERFACE_INCLUDE_DIRECTORIES ${PREFIX}/include)
+  return()
+endif()
 
 include(MfemCmakeUtilities)
 mfem_find_package(GSLIB GSLIB GSLIB_DIR "include" gslib.h "lib" gs

config/cmake/modules/FindHYPRE.cmake

@@ -27,6 +27,7 @@ if (HYPRE_FOUND OR TARGET HYPRE)
   if (HYPRE_USING_HIP)
     find_package(rocsparse REQUIRED)
     find_package(rocrand REQUIRED)
+    find_package(rocsolver REQUIRED)
   endif()
   if (HYPRE_LIBRARIES AND HYPRE_INCLUDE_DIRS AND HYPRE_VERSION)
     find_package_handle_standard_args(HYPRE
@@ -36,52 +37,92 @@ if (HYPRE_FOUND OR TARGET HYPRE)
   endif()
 endif()
 
-if (HYPRE_FETCH OR FETCH_TPLS)
+if (MFEM_FETCH_HYPRE OR MFEM_FETCH_TPLS)
   set(HYPRE_FETCH_VERSION 2.33.0)
+  set(HYPRE_FETCH_TAG "v${HYPRE_FETCH_VERSION}" CACHE STRING "Tag, branch, or commit for HYPRE")
   add_library(HYPRE STATIC IMPORTED)
   # set options and associated dependencies
-  set(CMAKE_OPTIONS)
-  list(APPEND CMAKE_OPTIONS -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE})
+  set(HYPRE_CMAKE_OPTIONS "")
+  list(APPEND HYPRE_CMAKE_OPTIONS -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE})
+  # collect all HYPRE_ENABLE variables and pass them to hypre, assuming they are BOOL.
+  get_cmake_property(all_vars VARIABLES)
+  foreach(var ${all_vars})
+    if(var MATCHES "^HYPRE_ENABLE")
+      list(APPEND HYPRE_CMAKE_OPTIONS "-D${var}:BOOL=${${var}}")
+    endif()
+  endforeach()
+  # process all MFEM_USE variables that impact hypre
   if (MFEM_USE_CUDA)
-    list(APPEND CMAKE_OPTIONS -DHYPRE_WITH_CUDA:BOOL=ON)
+    list(APPEND HYPRE_CMAKE_OPTIONS -DHYPRE_ENABLE_CUDA:BOOL=ON -DCMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES})
     find_package(CUDAToolkit REQUIRED)
     target_link_libraries(HYPRE INTERFACE CUDA::cusparse CUDA::curand CUDA::cublas)
   elseif (MFEM_USE_HIP)
-    list(APPEND CMAKE_OPTIONS -DHYPRE_WITH_HIP:BOOL=ON)
+    list(APPEND HYPRE_CMAKE_OPTIONS -DHYPRE_ENABLE_HIP:BOOL=ON)
     find_package(rocsparse REQUIRED)
     find_package(rocrand REQUIRED)
     target_link_libraries(HYPRE INTERFACE rocsparse rocrand)
   endif()
+  if (MFEM_USE_CUDA OR MFEM_USE_HIP)
+    if (MFEM_USE_UMPIRE)
+      if (EXISTS ${umpire_DIR})
+        list(APPEND HYPRE_CMAKE_OPTIONS -DHYPRE_ENABLE_UMPIRE:BOOL=ON -Dumpire_DIR:PATH=${umpire_DIR})
+      else()
+        message(FATAL_ERROR "MFEM_USE_UMPIRE=ON, however umpire_DIR isn't visible to HYPRE")
+      endif()
+    else()
+      list(APPEND HYPRE_CMAKE_OPTIONS -DHYPRE_ENABLE_UMPIRE:BOOL=OFF)
+      message(WARNING
+"================================================================================
+ Umpire is disabled while building HYPRE with GPU support.
+ This is not recommended for performance reasons!
+ Consider enabling Umpire with -DMFEM_USE_UMPIRE=ON and providing -DUMPIRE_DIR.
+================================================================================")
+    endif()
+  endif()
   if (MFEM_USE_SINGLE)
-    list(APPEND CMAKE_OPTIONS -DHYPRE_ENABLE_SINGLE:BOOL=ON)
+    list(APPEND HYPRE_CMAKE_OPTIONS -DHYPRE_ENABLE_SINGLE:BOOL=ON)
   endif()
   # define external project and create future include directory so it is present
   # to pass CMake checks at end of MFEM configuration step
-  message(STATUS "Will fetch HYPRE ${HYPRE_FETCH_VERSION} to be built with ${CMAKE_OPTIONS}")
-  set(PREFIX ${CMAKE_BINARY_DIR}/fetch/hypre)
+  message(STATUS "Will fetch HYPRE ${HYPRE_FETCH_TAG} to be built with ${HYPRE_CMAKE_OPTIONS}")
+  set(HYPRE_INSTALL ${CMAKE_BINARY_DIR}/fetch/hypre)
   include(ExternalProject)
   ExternalProject_Add(hypre
     GIT_REPOSITORY https://github.com/hypre-space/hypre.git
-    GIT_TAG v${HYPRE_FETCH_VERSION}
+    GIT_TAG ${HYPRE_FETCH_TAG}
     GIT_SHALLOW TRUE
+    GIT_PROGRESS TRUE
     UPDATE_DISCONNECTED TRUE
     SOURCE_SUBDIR src
-    PREFIX ${PREFIX}
-    CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${PREFIX} -DCMAKE_INSTALL_LIBDIR:PATH=lib ${CMAKE_OPTIONS})
-  file(MAKE_DIRECTORY ${PREFIX}/include)
+    PREFIX ${HYPRE_INSTALL}
+    BUILD_COMMAND ${CMAKE_COMMAND} --build . -- -j${CMAKE_BUILD_PARALLEL_LEVEL}
+    CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${HYPRE_INSTALL} -DCMAKE_INSTALL_LIBDIR:PATH=lib ${HYPRE_CMAKE_OPTIONS})
+  file(MAKE_DIRECTORY ${HYPRE_INSTALL}/include)
   # set imported library target properties
   add_dependencies(HYPRE hypre)
   set_target_properties(HYPRE PROPERTIES
-    IMPORTED_LOCATION ${PREFIX}/lib/libHYPRE.a
-    INTERFACE_INCLUDE_DIRECTORIES ${PREFIX}/include)
+    IMPORTED_LOCATION ${HYPRE_INSTALL}/lib/libHYPRE.a
+    INTERFACE_INCLUDE_DIRECTORIES ${HYPRE_INSTALL}/include)
   # convert HYPRE version to integer
-  string(REGEX MATCHALL "[0-9]+" HYPRE_SPLIT_VERSION ${HYPRE_FETCH_VERSION})
-  list(GET HYPRE_SPLIT_VERSION 0 HYPRE_MAJOR_VERSION)
-  list(GET HYPRE_SPLIT_VERSION 1 HYPRE_MINOR_VERSION)
-  list(GET HYPRE_SPLIT_VERSION 2 HYPRE_PATCH_VERSION)
-  math(EXPR HYPRE_VERSION "10000*${HYPRE_MAJOR_VERSION} + 100*${HYPRE_MINOR_VERSION} + ${HYPRE_PATCH_VERSION}")
-  # set cache variables that would otherwise be set after mfem_find_package call
-  set(HYPRE_VERSION ${HYPRE_VERSION} CACHE STRING "HYPRE version." FORCE)
+  if (HYPRE_FETCH_TAG MATCHES "^v?([0-9]+)\\.([0-9]+)\\.([0-9]+)$")
+    # Exact release tag X.Y.Z
+    string(REGEX MATCHALL "[0-9]+" HYPRE_SPLIT_VERSION "${HYPRE_FETCH_TAG}")
+  elseif (HYPRE_FETCH_VERSION MATCHES "([0-9]+)\\.([0-9]+)(\\.([0-9]+))?")
+    string(REGEX MATCHALL "[0-9]+" HYPRE_SPLIT_VERSION "${HYPRE_FETCH_VERSION}")
+  else (NOT DEFINED HYPRE_VERSION)
+    message(FATAL_ERROR "Unable to find HYPRE release version. Please provide it via -DHYPRE_VERSION")
+  endif()
+  if (HYPRE_SPLIT_VERSION AND NOT DEFINED HYPRE_VERSION)
+    list(GET HYPRE_SPLIT_VERSION 0 HYPRE_MAJOR_VERSION)
+    list(GET HYPRE_SPLIT_VERSION 1 HYPRE_MINOR_VERSION)
+    if (HYPRE_SPLIT_VERSION GREATER 2)
+      list(GET HYPRE_SPLIT_VERSION 2 HYPRE_PATCH_VERSION)
+    else()
+      set(HYPRE_PATCH_VERSION 0)
+    endif()
+    math(EXPR HYPRE_VERSION "10000*${HYPRE_MAJOR_VERSION} + 100*${HYPRE_MINOR_VERSION} + ${HYPRE_PATCH_VERSION}")
+    set(HYPRE_VERSION ${HYPRE_VERSION} CACHE STRING "HYPRE version." FORCE)
+  endif()
   return()
 endif()
 
@@ -149,7 +190,8 @@ endif()
 if (HYPRE_FOUND AND HYPRE_USING_HIP)
   find_package(rocsparse REQUIRED)
   find_package(rocrand REQUIRED)
-  list(APPEND HYPRE_LIBRARIES ${rocsparse_LIBRARIES} ${rocrand_LIBRARIES})
+  find_package(rocsolver REQUIRED)
+  list(APPEND HYPRE_LIBRARIES ${rocsparse_LIBRARIES} ${rocrand_LIBRARIES} roc::rocsolver roc::rocblas)
   set(HYPRE_LIBRARIES ${HYPRE_LIBRARIES} CACHE STRING
       "HYPRE libraries + dependencies." FORCE)
   message(STATUS "Updated HYPRE_LIBRARIES: ${HYPRE_LIBRARIES}")

config/cmake/modules/FindMETIS.cmake

@@ -18,7 +18,7 @@
 #   - METIS (imported library target)
 #   - METIS_VERSION_5 (cache variable)
 
-if (METIS_FETCH OR FETCH_TPLS)
+if (MFEM_FETCH_METIS OR MFEM_FETCH_TPLS)
   set(METIS_FETCH_VERSION 4.0.3)
   add_library(METIS STATIC IMPORTED)
   # define external project
@@ -32,6 +32,7 @@ if (METIS_FETCH OR FETCH_TPLS)
     UPDATE_DISCONNECTED TRUE
     PREFIX ${PREFIX}
     CONFIGURE_COMMAND tar -xzf ../metis/metis-${METIS_FETCH_VERSION}-mac.tgz --strip=1
+    BUILD_COMMAND $(MAKE) COPTIONS=-Wno-incompatible-pointer-types
     INSTALL_COMMAND mkdir -p ${PREFIX}/lib && cp libmetis.a ${PREFIX}/lib/)
   # set imported library target properties
   add_dependencies(METIS metis)

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -718,7 +718,7 @@ function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
   get_target_property(IsImported ${tgt} IMPORTED)
   # message(STATUS "${tgt}[IMPORTED]: ${IsImported}")
   # Generally, the possible target types are: STATIC_LIBRARY, MODULE_LIBRARY,
-  # SHARED_LIBRARY, INTERFACE_LIBRARY, EXECUTABLE.
+  # SHARED_LIBRARY, INTERFACE_LIBRARY, UNKNOWN_LIBRARY, EXECUTABLE.
   get_target_property(type ${tgt} TYPE)
   # message(STATUS "${tgt}[TYPE]: ${type}")
   unset(ImportConfig)
@@ -766,7 +766,7 @@ function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
     else()
       message(STATUS " *** Warning: [${tgt}] LOCATION not defined!")
     endif()
-  elseif ("${type}" STREQUAL "SHARED_LIBRARY")
+  elseif ("${type}" STREQUAL "SHARED_LIBRARY" OR "${type}" STREQUAL "UNKNOWN_LIBRARY")
     get_target_property(Location ${tgt} LOCATION)
     if (Location)
       get_filename_component(Dir ${Location} DIRECTORY)
@@ -932,12 +932,14 @@ function(mfem_export_mk_files)
   endif()
   set(MFEM_BUILD_TAG "${CMAKE_SYSTEM}")
   set(MFEM_PREFIX "${CMAKE_INSTALL_PREFIX}")
-  # For the next 4 variable, these are the values for the build-tree version of
+  # For the next 4 variables, these are the values for the build-tree version of
   # 'config.mk'
   set(MFEM_INC_DIR "${PROJECT_BINARY_DIR}")
   set(MFEM_LIB_DIR "${PROJECT_BINARY_DIR}")
   set(MFEM_TEST_MK "${PROJECT_SOURCE_DIR}/config/test.mk")
   set(MFEM_CONFIG_EXTRA "MFEM_BUILD_DIR ?= ${PROJECT_BINARY_DIR}")
+  # TODO: CUDA/HIP support:
+  set(MFEM_XLINKER "${CMAKE_CXX_LINKER_WRAPPER_FLAG}")
   set(MFEM_MPIEXEC ${MPIEXEC})
   if (NOT MFEM_MPIEXEC)
     set(MFEM_MPIEXEC "mpirun")

config/config.mk.in

@@ -88,6 +88,7 @@ MFEM_BUILD_TAG = @MFEM_BUILD_TAG@
 MFEM_PREFIX    = @MFEM_PREFIX@
 MFEM_INC_DIR   = @MFEM_INC_DIR@
 MFEM_LIB_DIR   = @MFEM_LIB_DIR@
+MFEM_XLINKER   = @MFEM_XLINKER@
 
 # Location of test.mk
 MFEM_TEST_MK = @MFEM_TEST_MK@

config/defaults.cmake

@@ -91,9 +91,10 @@ option(MFEM_ENABLE_BENCHMARKS "Build all of the benchmarks" OFF)
 
 # Allow a user to specify fetching of certain third-party libraries instead of
 # searching for existing installations.
-option(FETCH_TPLS "Enable fetching of all supported third-party libraries" OFF)
-option(HYPRE_FETCH "Enable fetching of hypre" OFF)
-option(METIS_FETCH "Enable fetching of METIS" OFF)
+option(MFEM_FETCH_TPLS "Enable fetching of all supported third-party libraries" OFF)
+option(MFEM_FETCH_GSLIB "Enable fetching of GSLIB" OFF)
+option(MFEM_FETCH_HYPRE "Enable fetching of hypre" OFF)
+option(MFEM_FETCH_METIS "Enable fetching of METIS" OFF)
 
 # Setting CXX/MPICXX on the command line or in user.cmake will overwrite the
 # autodetected C++ compiler.

config/defaults.mk

@@ -57,7 +57,7 @@ CUDA_DIR = $(or $(CUDA_HOME),$(patsubst %/,%,$(dir \
 CLANG_CUDA_FLAGS = -xcuda --cuda-path=$(CUDA_DIR) --cuda-gpu-arch=$(CUDA_ARCH)
 # flags for nvcc
 NVCC_FLAGS = -x=cu --expt-extended-lambda --expt-relaxed-constexpr \
- -arch=$(CUDA_ARCH)
+ -arch=$(CUDA_ARCH) -isystem "$(CUDA_DIR)/include"
 # Prefixes for passing flags to the host compiler and linker when using
 # CUDA_CXX=nvcc
 CUDA_XCOMPILER = -Xcompiler=

data/mfem.mesh

@@ -0,0 +1,593 @@
+MFEM mesh v1.0
+
+# Created by: Pointwise
+
+# MFEM Geometry Types:
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+
+dimension
+2
+
+elements
+160
+1 3 1 164 163 0
+1 3 164 165 162 163
+1 3 2 166 164 1
+1 3 166 132 165 164
+1 3 3 167 166 2
+1 3 167 131 132 166
+1 3 4 168 167 3
+1 3 168 130 131 167
+1 3 5 169 168 4
+1 3 169 129 130 168
+1 3 6 170 169 5
+1 3 170 128 129 169
+1 3 171 172 170 6
+1 3 172 127 128 170
+1 3 124 125 172 171
+1 3 125 126 127 172
+1 3 162 165 173 161
+1 3 165 132 133 173
+1 3 161 173 174 160
+1 3 173 133 134 174
+1 3 160 174 175 159
+1 3 174 134 135 175
+1 3 6 7 176 171
+1 3 7 8 177 176
+1 3 171 176 123 124
+1 3 176 177 122 123
+1 3 159 175 178 158
+1 3 175 135 136 178
+1 3 158 178 179 157
+1 3 178 136 137 179
+1 3 157 179 180 156
+1 3 179 137 138 180
+1 3 122 177 181 121
+1 3 177 8 182 181
+1 3 8 9 183 182
+1 3 9 10 184 183
+1 3 10 11 185 184
+1 3 11 12 186 185
+1 3 12 13 187 186
+1 3 13 14 15 187
+1 3 121 181 119 120
+1 3 181 182 118 119
+1 3 182 183 117 118
+1 3 183 184 188 117
+1 3 184 185 109 188
+1 3 185 186 108 109
+1 3 186 187 189 108
+1 3 187 15 16 189
+1 3 109 110 190 188
+1 3 110 111 191 190
+1 3 111 112 113 191
+1 3 188 190 116 117
+1 3 190 191 115 116
+1 3 191 113 114 115
+1 3 189 192 107 108
+1 3 192 193 106 107
+1 3 193 194 105 106
+1 3 194 195 104 105
+1 3 195 196 103 104
+1 3 16 17 192 189
+1 3 17 18 193 192
+1 3 18 19 194 193
+1 3 19 20 195 194
+1 3 20 21 196 195
+1 3 97 98 197 96
+1 3 98 99 198 197
+1 3 99 100 199 198
+1 3 100 101 200 199
+1 3 101 102 201 200
+1 3 102 103 202 201
+1 3 103 196 203 202
+1 3 196 21 22 203
+1 3 96 197 204 95
+1 3 197 198 39 204
+1 3 198 199 38 39
+1 3 199 200 205 38
+1 3 200 201 32 205
+1 3 201 202 31 32
+1 3 202 203 206 31
+1 3 203 22 23 206
+1 3 32 33 207 205
+1 3 33 34 35 207
+1 3 205 207 37 38
+1 3 207 35 36 37
+1 3 39 40 208 204
+1 3 40 41 209 208
+1 3 41 42 210 209
+1 3 42 43 211 210
+1 3 43 44 212 211
+1 3 204 208 94 95
+1 3 208 209 93 94
+1 3 209 210 92 93
+1 3 210 211 91 92
+1 3 211 212 90 91
+1 3 90 212 213 89
+1 3 212 44 214 213
+1 3 44 45 215 214
+1 3 45 46 216 215
+1 3 46 47 217 216
+1 3 47 48 218 217
+1 3 48 49 219 218
+1 3 49 50 51 219
+1 3 89 213 87 88
+1 3 213 214 86 87
+1 3 214 215 85 86
+1 3 215 216 84 85
+1 3 216 217 83 84
+1 3 217 218 82 83
+1 3 218 219 220 82
+1 3 219 51 52 220
+1 3 53 221 220 52
+1 3 221 81 82 220
+1 3 54 222 221 53
+1 3 222 80 81 221
+1 3 55 223 222 54
+1 3 223 79 80 222
+1 3 26 27 224 25
+1 3 27 28 29 224
+1 3 25 224 225 24
+1 3 224 29 30 225
+1 3 24 225 206 23
+1 3 225 30 31 206
+1 3 154 155 226 153
+1 3 155 156 180 226
+1 3 153 226 227 152
+1 3 226 180 138 227
+1 3 152 227 228 151
+1 3 227 138 139 228
+1 3 151 228 229 150
+1 3 228 139 140 229
+1 3 150 229 230 149
+1 3 229 140 141 230
+1 3 149 230 231 148
+1 3 230 141 142 231
+1 3 148 231 232 147
+1 3 231 142 143 232
+1 3 147 232 145 146
+1 3 232 143 144 145
+1 3 56 233 223 55
+1 3 233 78 79 223
+1 3 57 234 233 56
+1 3 234 77 78 233
+1 3 58 235 234 57
+1 3 235 76 77 234
+1 3 61 236 59 60
+1 3 236 235 58 59
+1 3 62 237 236 61
+1 3 237 76 235 236
+1 3 63 238 237 62
+1 3 238 75 76 237
+1 3 64 239 238 63
+1 3 239 74 75 238
+1 3 65 240 239 64
+1 3 240 73 74 239
+1 3 66 241 240 65
+1 3 241 72 73 240
+1 3 67 242 241 66
+1 3 242 71 72 241
+1 3 68 69 242 67
+1 3 69 70 71 242
+
+boundary
+164
+3 1 0 1
+3 1 1 2
+3 1 2 3
+3 1 3 4
+3 1 4 5
+3 1 5 6
+3 1 6 7
+3 1 7 8
+3 1 8 9
+3 1 9 10
+3 1 10 11
+3 1 11 12
+3 1 12 13
+3 1 13 14
+3 1 16 17
+3 1 17 18
+3 1 18 19
+3 1 19 20
+3 1 20 21
+3 1 21 22
+3 1 22 23
+3 1 23 24
+3 1 24 25
+3 1 25 26
+3 1 26 27
+3 1 27 28
+3 1 28 29
+3 1 29 30
+3 1 30 31
+3 1 31 32
+3 1 32 33
+3 1 33 34
+3 1 34 35
+3 1 35 36
+3 1 36 37
+3 1 37 38
+3 1 38 39
+3 1 39 40
+3 1 40 41
+3 1 41 42
+3 1 42 43
+3 1 43 44
+3 1 49 50
+3 1 48 49
+3 1 47 48
+3 1 46 47
+3 1 45 46
+3 1 44 45
+3 1 52 53
+3 1 53 54
+3 1 54 55
+3 1 57 58
+3 1 56 57
+3 1 55 56
+3 1 60 61
+3 1 61 62
+3 1 62 63
+3 1 63 64
+3 1 64 65
+3 1 65 66
+3 1 66 67
+3 1 67 68
+3 1 75 76
+3 1 74 75
+3 1 73 74
+3 1 72 73
+3 1 71 72
+3 1 70 71
+3 1 76 77
+3 1 77 78
+3 1 78 79
+3 1 81 82
+3 1 80 81
+3 1 79 80
+3 1 82 83
+3 1 83 84
+3 1 84 85
+3 1 85 86
+3 1 86 87
+3 1 87 88
+3 1 94 95
+3 1 93 94
+3 1 92 93
+3 1 91 92
+3 1 90 91
+3 1 96 97
+3 1 95 96
+3 1 97 98
+3 1 98 99
+3 1 99 100
+3 1 100 101
+3 1 101 102
+3 1 102 103
+3 1 107 108
+3 1 106 107
+3 1 105 106
+3 1 104 105
+3 1 103 104
+3 1 108 109
+3 1 109 110
+3 1 110 111
+3 1 111 112
+3 1 112 113
+3 1 113 114
+3 1 114 115
+3 1 115 116
+3 1 116 117
+3 1 119 120
+3 1 118 119
+3 1 117 118
+3 1 131 132
+3 1 130 131
+3 1 129 130
+3 1 128 129
+3 1 127 128
+3 1 126 127
+3 1 132 133
+3 1 133 134
+3 1 134 135
+3 1 137 138
+3 1 136 137
+3 1 135 136
+3 1 138 139
+3 1 139 140
+3 1 140 141
+3 1 141 142
+3 1 142 143
+3 1 143 144
+3 1 147 148
+3 1 146 147
+3 1 153 154
+3 1 152 153
+3 1 151 152
+3 1 150 151
+3 1 149 150
+3 1 148 149
+3 1 156 157
+3 1 157 158
+3 1 158 159
+3 1 161 162
+3 1 160 161
+3 1 159 160
+2 1 69 70
+2 1 68 69
+3 1 88 89
+3 1 89 90
+3 1 121 122
+3 1 120 121
+3 1 123 124
+3 1 122 123
+3 1 125 126
+3 1 124 125
+1 1 144 145
+1 1 145 146
+3 1 15 16
+3 1 14 15
+3 1 50 51
+3 1 51 52
+3 1 59 60
+3 1 58 59
+3 1 154 155
+3 1 155 156
+3 1 163 0
+3 1 162 163
+
+vertices
+243
+2
+4 4
+4 3.5
+4 3
+4 2.5
+4 2
+4 1.5
+4 1
+4.5 1
+5 1
+5 1.5
+5 2
+5 2.5
+5 3
+5 3.5
+5 4
+5.500 4
+6 4
+6.500 4
+7 4
+7.5 4
+8 4
+8.5 4
+9 4
+9.5 4
+10 4
+10.5 4
+11 4
+11 3.5
+11 3
+10.5 3
+10 3
+9.5 3
+9.5 2.5
+10 2.5
+10.5 2.5
+10.5 2
+10.5 1.5
+10 1.5
+9.5 1.5
+9.5 1
+10 1
+10.5 1
+11 1
+11.5 1
+12 1
+12 1.5
+12 2
+12 2.5
+12 3
+12 3.5
+12 4
+12.5 4
+13 4
+13.333 3.75
+13.666 3.5
+14.000 3.25
+14.333 3.5
+14.666 3.75
+15.000 4
+15.500 4
+16.000 4
+16.000 3.5
+16.000 3
+16.000 2.5
+16.000 2
+16.000 1.5
+16.000 1
+16.000 0.5
+16.000 0
+15.500 0
+15.000 0
+15.000 0.5000000000000002
+15.000 1
+15.000 1.5
+15.000 2
+15.000 2.5
+15.000 3
+14.666 2.75
+14.333 2.5
+14.000 2.25
+13.666 2.5
+13.333 2.75
+13 3
+13 2.5
+13 2
+13 1.5
+13 1
+13 0.500
+13 0
+12.5 0
+12 0
+11.5 0
+11 0
+10.5 0
+10 0
+9.5 0
+9 0
+8.5 0
+8.5 0.5
+8.5 1
+8.5 1.5
+8.5 2
+8.5 2.5
+8.5 3
+8 3
+7.5 3
+7 3
+6.500 3
+6 3
+6 2.5
+6.5 2.5
+7 2.5
+7.5 2.5
+7.5 2
+7.5 1.5
+7.000 1.5
+6.5 1.5
+6 1.5
+6 1
+6 0.5
+6 0
+5.5 0
+5 0
+4.5 0
+4 0
+3.5 0
+3 0
+3 0.500
+3 1
+3 1.5
+3 2
+3 2.5
+3 3
+2.666 2.75
+2.333 2.5
+2.000 2.25
+1.666 2.5
+1.333 2.75
+1.000 3
+1.000 2.5
+1.000 2
+1.000 1.5
+1.000 1
+1.000 0.5000
+1.000 0
+0.5000 0
+0.0000 0
+0.0000 0.5
+0.0000 1
+0.0000 1.5
+0.0000 2
+0.0000 2.5
+0.0000 3
+0.0000 3.5
+0.0000 4
+0.5000 4
+1.000 4
+1.333 3.75
+1.666 3.5
+2.000 3.25
+2.333 3.5
+2.666 3.75
+3 4
+3.5 4
+3.5 3.5
+3 3.5
+3.5 3
+3.5 2.5
+3.5 2
+3.5 1.5
+3.5 1
+4 0.5
+3.5 0.5
+2.666 3.25
+2.333 3
+2.000 2.75
+4.5 0.5
+5 0.5
+1.666 3
+1.333 3.25
+1.000 3.5
+5.5 0.5
+5.500 1
+5.500 1.5
+5.500 2
+5.500 2.5
+5.500 3
+5.500 3.5
+6 2
+6 3.5
+6.5 2
+7 2
+6.5 3.5
+7 3.5
+7.5 3.5
+8 3.5
+8.5 3.5
+9 0.5
+9 1
+9 1.5
+9 2
+9 2.5
+9 3
+9 3.5
+9.5 0.5
+9.5 2
+9.5 3.5
+10 2
+10 0.5
+10.5 0.5
+11 0.5
+11.5 0.5
+12 0.5
+12.5 0.500
+12.5 1
+12.5 1.5
+12.5 2
+12.5 2.5
+12.5 3
+12.5 3.5
+13 3.5
+13.333 3.250
+13.666 3
+14.000 2.75
+10.5 3.5
+10 3.5
+0.500 3.5
+0.500 3
+0.500 2.5
+0.500 2
+0.500 1.5
+0.500 1
+0.500 0.5
+14.333 3
+14.666 3.25
+15.000 3.5
+15.500 3.5
+15.500 3
+15.500 2.5
+15.500 2
+15.500 1.5
+15.500 1
+15.500 0.5

data/nc-nurbs3d.mesh

@@ -0,0 +1,342 @@
+MFEM NURBS NC-patch mesh v1.0
+dimension
+3
+
+elements
+13
+0 1 5 0 8 10 11 9 4 6 7 5
+0 1 5 0 18 8 24 32 30 23 36 38
+0 1 5 0 0 18 32 14 12 30 38 29
+0 1 5 0 32 24 10 20 38 36 26 35
+0 1 5 0 14 32 20 2 29 38 35 16
+0 1 5 0 30 23 36 38 31 22 37 39
+0 1 5 0 12 30 38 29 13 31 39 28
+0 1 5 0 38 36 26 35 39 37 27 34
+0 1 5 0 29 38 35 16 28 39 34 17
+0 1 5 0 31 22 37 39 19 9 25 33
+0 1 5 0 13 31 39 28 1 19 33 15
+0 1 5 0 39 37 27 34 33 25 11 21
+0 1 5 0 28 39 34 17 15 33 21 3
+
+boundary
+31
+9999 3 8 10 6 4
+9999 3 10 11 7 6
+9999 3 11 9 5 7
+9999 3 9 8 4 5
+9999 3 4 6 7 5
+9999 3 32 24 8 18
+9999 3 18 8 23 30
+9999 3 14 32 18 0
+9999 3 0 18 30 12
+9999 3 14 0 12 29
+9999 3 20 10 24 32
+9999 3 10 20 35 26
+9999 3 2 20 32 14
+9999 3 20 2 16 35
+9999 3 2 14 29 16
+9999 3 30 23 22 31
+9999 3 12 30 31 13
+9999 3 29 12 13 28
+9999 3 26 35 34 27
+9999 3 35 16 17 34
+9999 3 16 29 28 17
+9999 3 31 22 9 19
+9999 3 19 9 25 33
+9999 3 13 31 19 1
+9999 3 28 13 1 15
+9999 3 1 19 33 15
+9999 3 27 34 21 11
+9999 3 33 25 11 21
+9999 3 34 17 3 21
+9999 3 17 28 15 3
+9999 3 15 33 21 3
+
+vertex_to_knotspan
+8
+23 0 1 8 10 11 9
+22 0 2 8 10 11 9
+24 1 0 8 10 11 9
+36 1 1 8 10 11 9
+37 1 2 8 10 11 9
+25 1 3 8 10 11 9
+26 2 1 8 10 11 9
+27 2 2 8 10 11 9
+
+coordinates
+40
+3
+0 0 0
+0 1 0
+4 0 0
+4 1 0
+0 0 4
+0 1 4
+4 0 4
+4 1 4
+0 0 2
+0 1 2
+4 0 2
+4 1 2
+0 0.333333333333333 0
+0 0.666666666666667 0
+2 0 0
+2 1 0
+4 0.333333333333334 0
+4 0.666666666666667 0
+0 0 1
+0 1 1
+4 0 1
+4 1 1
+0 0.666666666666667 2
+0 0.333333333333333 2
+2 0 2
+2 1 2
+4 0.333333333333333 2
+4 0.666666666666667 2
+2 0.666666666666667 0
+2 0.333333333333333 0
+0 0.333333333333333 1
+0 0.666666666666667 1
+1.81325211007895 0 1
+1.81325211007895 1 1
+4 0.666666666666667 1
+4 0.333333333333333 1
+2 0.333333333333333 2
+2 0.666666666666667 2
+1.81325211007895 0.333333333333333 1
+1.81325211007895 0.666666666666667 1
+
+edges
+87
+0 8 10
+1 10 11
+0 9 11
+1 8 9
+0 4 6
+1 6 7
+0 5 7
+1 4 5
+2 4 8
+2 6 10
+2 7 11
+2 5 9
+9 18 8
+7 8 24
+9 32 24
+7 18 32
+9 30 23
+7 23 36
+9 38 36
+7 30 38
+3 18 30
+3 8 23
+3 24 36
+3 32 38
+8 0 18
+8 14 32
+7 0 14
+8 12 30
+8 29 38
+7 12 29
+3 0 12
+3 14 29
+6 24 10
+9 20 10
+6 32 20
+6 36 26
+9 35 26
+6 38 35
+3 10 26
+3 20 35
+8 2 20
+6 14 2
+8 16 35
+6 29 16
+3 2 16
+9 31 22
+7 22 37
+9 39 37
+7 31 39
+4 30 31
+4 23 22
+4 36 37
+4 38 39
+8 13 31
+8 28 39
+7 13 28
+4 12 13
+4 29 28
+6 37 27
+9 34 27
+6 39 34
+4 26 27
+4 35 34
+8 17 34
+6 28 17
+4 16 17
+9 19 9
+7 9 25
+9 33 25
+7 19 33
+5 31 19
+5 22 9
+5 37 25
+5 39 33
+8 1 19
+8 15 33
+7 1 15
+5 13 1
+5 28 15
+6 25 11
+9 21 11
+6 33 21
+5 27 11
+5 34 21
+8 3 21
+6 15 3
+5 17 3
+
+knotvectors
+10
+1 3 0 0 0.5 1 1
+1 4 0 0 0.333333333333333 0.666666666666667 1 1
+1 3 0 0 0.5 1 1
+1 2 0 0 1 1
+1 2 0 0 1 1
+1 2 0 0 1 1
+1 2 0 0 1 1
+1 2 0 0 1 1
+1 2 0 0 1 1
+1 2 0 0 1 1
+
+spacing
+0
+
+weights
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+
+FiniteElementSpace
+FiniteElementCollection: NURBS1
+VDim: 3
+Ordering: 1
+
+0 0 0
+0 1 0
+4 0 0
+4 1 0
+0 0 4
+0 1 4
+4 0 4
+4 1 4
+0 0 2
+0 1 2
+4 0 2
+4 1 2
+0 0.333333333333333 0
+0 0.666666666666667 0
+2 0 0
+2 1 0
+4 0.333333333333334 0
+4 0.666666666666667 0
+0 0 1
+0 1 1
+4 0 1
+4 1 1
+0 0.666666666666667 2
+0 0.333333333333333 2
+2 0 2
+2 1 2
+4 0.333333333333333 2
+4 0.666666666666667 2
+2 0.666666666666667 0
+2 0.333333333333333 0
+0 0.333333333333333 1
+0 0.666666666666667 1
+1.81325211007895 0 1
+1.81325211007895 1 1
+4 0.666666666666667 1
+4 0.333333333333333 1
+2 0.333333333333333 2
+2 0.666666666666667 2
+1.81325211007895 0.333333333333333 1
+1.81325211007895 0.666666666666667 1
+2 0 4
+4 0.333333333333333 4
+4 0.666666666666667 4
+2 1 4
+0 0.333333333333333 4
+0 0.666666666666667 4
+0 0 3
+4 0 3
+4 1 3
+0 1 3
+2 0 3
+4 0.333333333333333 3
+4 0.666666666666667 3
+2 1 3
+0 0.666666666666667 3
+0 0.333333333333333 3
+2 0.333333333333333 4
+2 0.666666666666667 4
+2 0.333333333333333 3
+2 0.666666666666667 3

data/nc3-nurbs.mesh

@@ -0,0 +1,96 @@
+MFEM NURBS NC-patch mesh v1.0
+dimension
+2
+
+# rank attr geom ref_type nodes/children
+elements
+3
+0 1 3 0 0 4 5 1
+0 1 3 0 6 7 4 2
+0 1 3 0 6 3 5 7
+
+# attr geom nodes
+boundary
+7
+1 1 0 4
+1 1 5 1
+1 1 1 0
+1 1 2 6
+1 1 6 3
+1 1 4 2
+1 1 5 3
+
+vertex_to_knotspan
+1
+7 1 4 5
+
+# top-level node coordinates
+coordinates
+8
+2
+0 0
+0 1
+2 0
+2 1
+1 0
+1 1
+2 0.5
+1 0.5
+
+edges
+11
+0 0 4
+1 4 5
+0 1 5
+1 0 1
+2 6 7
+4 7 4
+2 2 4
+4 6 2
+3 6 3
+2 3 5
+3 7 5
+
+knotvectors
+5
+1 3 0 0 0.5 1 1
+1 3 0 0 0.5 1 1
+1 2 0 0 1 1
+1 2 0 0 1 1
+1 2 0 0 1 1
+
+spacing
+0
+
+weights
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+
+FiniteElementSpace
+FiniteElementCollection: NURBS1
+VDim: 2
+Ordering: 1
+
+0 0
+0 1
+2 0
+2 1
+1 0
+1 1
+2 0.5
+1 0.5
+0.5 0
+0.5 1
+0 0.5
+0.5 0.5
+mfem_mesh_end

doc/CodeDocumentation.dox

@@ -202,6 +202,7 @@ namespace mfem {
  * - <a class="el" href="tesla_8cpp_source.html">Tesla</a>: simple magnetostatics simulation code
  * - <a class="el" href="maxwell_8cpp_source.html">Maxwell</a>: simple transient full-wave electromagnetics simulation code
  * - <a class="el" href="joule_8cpp_source.html">Joule</a>: transient magnetics and Joule heating miniapp
+ * - <a class="el" href="lorentz_8cpp_source.html">Lorentz</a>: simple particle tracking code based on the Lorentz force
  * - <a class="el" href="classmfem_1_1navier_1_1NavierSolver.html">Navier</a>: solve the transient incompressible Navier-Stokes equations
  * - <a class="el" href="mobius-strip_8cpp_source.html">Mobius Strip</a>: generate various Mobius strip-like meshes
  * - <a class="el" href="klein-bottle_8cpp_source.html">Klein Bottle</a>: generate three types of Klein bottle surfaces

examples/CMakeLists.txt

@@ -89,6 +89,7 @@ if (MFEM_USE_MPI)
     ex37p.cpp
     ex39p.cpp
     ex40p.cpp
+    ex999p.cpp
   )
 endif()
 
@@ -205,6 +206,15 @@ if (MFEM_ENABLE_TESTING)
       $<TARGET_FILE:ex25p> "-no-vis" "--mumps-solver"
       ${MPIEXEC_POSTFLAGS})
   endif()
+
+  # Parallel libCEED example
+  if (MFEM_USE_CEED AND MFEM_USE_MPI)
+    add_test(NAME ex1p_ceed_np=${MFEM_MPI_NP}
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:ex1p> "-no-vis" "-d ceed-cpu" "-pa" "-a"
+      ${MPIEXEC_POSTFLAGS})
+  endif()
 endif()
 
 # Include the examples/amgx directory if AmgX is enabled

examples/ex1.cpp

@@ -27,6 +27,7 @@
 //               ex1 -m ../data/fichera-amr.mesh
 //               ex1 -m ../data/mobius-strip.mesh
 //               ex1 -m ../data/mobius-strip.mesh -o -1 -sc
+//               ex1 -m ../data/nc3-nurbs.mesh -o -1
 //
 // Device sample runs:
 //               ex1 -pa -d cuda

examples/ex22.cpp

@@ -471,10 +471,13 @@ int main(int argc, char *argv[])
 
       ofstream sol_r_ofs("sol_r.gf");
       ofstream sol_i_ofs("sol_i.gf");
+      ofstream sol_z_ofs("sol_z.gf");
       sol_r_ofs.precision(8);
       sol_i_ofs.precision(8);
+      sol_z_ofs.precision(8);
       u.real().Save(sol_r_ofs);
       u.imag().Save(sol_i_ofs);
+      u.Save(sol_z_ofs);
    }
 
    // 14. Send the solution by socket to a GLVis server.

examples/ex22p.cpp

@@ -507,10 +507,11 @@ int main(int argc, char *argv[])
    // 15. Save the refined mesh and the solution in parallel. This output can be
    //     viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
    {
-      ostringstream mesh_name, sol_r_name, sol_i_name;
+      ostringstream mesh_name, sol_r_name, sol_i_name, sol_z_name;
       mesh_name << "mesh." << setfill('0') << setw(6) << myid;
       sol_r_name << "sol_r." << setfill('0') << setw(6) << myid;
       sol_i_name << "sol_i." << setfill('0') << setw(6) << myid;
+      sol_z_name << "sol_z." << setfill('0') << setw(6) << myid;
 
       ofstream mesh_ofs(mesh_name.str().c_str());
       mesh_ofs.precision(8);
@@ -518,10 +519,13 @@ int main(int argc, char *argv[])
 
       ofstream sol_r_ofs(sol_r_name.str().c_str());
       ofstream sol_i_ofs(sol_i_name.str().c_str());
+      ofstream sol_z_ofs(sol_z_name.str().c_str());
       sol_r_ofs.precision(8);
       sol_i_ofs.precision(8);
+      sol_z_ofs.precision(8);
       u.real().Save(sol_r_ofs);
       u.imag().Save(sol_i_ofs);
+      u.Save(sol_z_ofs);
    }
 
    // 16. Send the solution by socket to a GLVis server.

examples/ex999p.cpp

@@ -0,0 +1,159 @@
+#include <mfem.hpp>
+#include "nlohmann/json.hpp"
+#include "minja.hpp"
+#include "myqfunction.hpp"
+
+using namespace mfem;
+using namespace mfem::future;
+
+template<class T>
+struct remove_cvref
+{
+   using type = std::remove_cv_t<std::remove_reference_t<T>>;
+};
+
+template <typename qf_t>
+auto process(qf_t qf)
+{
+   using qfsig = typename create_function_signature<qf_t>::type;
+   using qfpar_t = typename qfsig::parameter_ts;
+   using qfout_t = typename qfsig::return_t;
+
+   auto qfparams = decay_tuple<qfpar_t> {};
+
+   auto in_str = apply([](auto&&... arg)
+   {
+      return std::vector<std::string>
+      {
+         std::string(get_type_name<typename remove_cvref<decltype(arg)>::type>())...
+      };
+   }, qfparams);
+
+   std::vector<std::string> out_str
+   {
+      std::string(get_type_name<typename remove_cvref<qfout_t>::type>())
+   };
+
+   return std::tuple{in_str, out_str};
+}
+
+int main()
+{
+   // load the kernel template
+   std::ifstream
+   kernel_istream("/Users/andrej1/repos/mfem/examples/kernel_skeleton.jinja");
+   if (!kernel_istream.is_open())
+   {
+      std::cerr << "error opening jinja template file" << std::endl;
+      return 1;
+   }
+   std::stringstream buffer;
+   buffer << kernel_istream.rdbuf();
+
+   std::string fileContent = buffer.str();
+   auto kernel_tmpl = minja::Parser::parse(buffer.str(), /* options= */ {});
+
+   auto [in_str, out_str] = process(myqfunction0);
+
+   for (auto &v : in_str)
+   {
+      std::cout << v << " ";
+   }
+   std::cout << std::endl;
+
+   const size_t DUMMY_STRIDE = 64*32*32;
+   const size_t basis_p_1d = 2;
+
+   json context_json{};
+   context_json["kernel_name"] = "demo";
+
+   context_json["spaces"].push_back(
+   {
+      {"P_1D", basis_p_1d},
+      {"dim", 3},
+      {"needs_value", true},
+      {"needs_grad", true},
+   });
+
+   context_json["spaces"].push_back(
+   {
+      {"P_1D", basis_p_1d},
+   });
+
+   context_json["inputs"].push_back(
+   {
+      {"name", "potential"},
+      {"space_idx", 0},
+      {"num_comp", 1},
+      {"comp_stride", DUMMY_STRIDE},
+      {"eval_grad", true},
+   });
+
+   context_json["inputs"].push_back(
+   {
+      {"name", "weights"},
+      {"space_idx", 0},
+      {"num_comp", 1},
+      {"comp_stride", DUMMY_STRIDE},
+      {"is_qdata", true},
+   });
+
+   context_json["outputs"].push_back(
+   {
+      {"name", "solution"},
+      {"space_idx", 0},
+      {"num_comp", 1},
+      {"comp_stride", DUMMY_STRIDE},
+      {"eval_grad", true},
+   });
+
+   const size_t nqf = 1;
+   const std::vector<std::string> qfunc_names = {"myqfunction0"};
+   const std::vector<std::vector<size_t>> qfunc_inputs = {{0, 1, 2}};
+
+   for (size_t i = 0; i < nqf; i++)
+   {
+      json inarr = json::array();
+      for (size_t j = 0; j < qfunc_inputs[i].size(); j++)
+      {
+         inarr.push_back(
+         {
+            {"index", j},
+            {"datatype", in_str[j]}
+         });
+      }
+
+      context_json["qfuncs"].push_back(
+      {
+         {"name", qfunc_names[i]},
+         {"inputs", inarr}
+      });
+   }
+
+   std::cout << context_json.dump(2) << std::endl;
+
+   auto context = minja::Context::make(context_json);
+   auto kernel_source = kernel_tmpl->render(context);
+
+   std::cout << ">>> generated kernel source\n"
+             << kernel_source
+             << "\n<<< generated kernel source\n"
+             << std::endl;
+
+   {
+      // test casting
+      std::vector<real_t> d(4);
+      int i = 0;
+      for (auto &v : d)
+      {
+         v = ++i;
+      }
+
+      mfem::future::tensor<real_t, 2, 2> *dudxi =
+         reinterpret_cast<mfem::future::tensor<real_t, 2, 2> *>(d.data());
+
+      std::cout << *dudxi << std::endl;
+   }
+
+   return 0;
+}

examples/kernel_skeleton.jinja

@@ -0,0 +1,21 @@
+#include "util.hpp"
+
+#define NUM_SPACES {{ spaces | count }}
+#define NUM_INPUTS {{ inputs | count }}
+#define NUM_OUTPUTS {{ outputs | count }}
+
+extern "C" __global__ void dfem_jit_{{kernel_name}}(int num_entities, const real_t *fields[NUM_INPUTS], real_t *outputs[NUM_OUTPUTS], const real_t *B[NUM_SPACES]) {
+  // transform fields
+  const real_t *inputs = ...;
+
+  // call qfunctions
+  {% for qf in qfuncs -%}
+  {
+    {%- for qfinput in qf.inputs %}
+    {{ qfinput.datatype }}* in{{ loop.index0 }} =
+      reinterpret_cast<{{ qfinput.datatype }}>(inputs[{{ qfinput.index }}]);
+    {% endfor %}
+    {{ qf.name }}({% for qfinput in qf.inputs %}*in{{ loop.index0 }}{{ "," if not loop.last else "" }}{% endfor %});
+  }
+  {% endfor %}
+}

examples/makefile

@@ -173,6 +173,12 @@ ex11p-test-cpardiso: ex11p
 	@$(call mfem-test,$<, $(RUN_MPI), MKL_CPARDISO example,--cpardiso)
 test-par-YES: ex11p-test-cpardiso
 endif
+ifeq ($(MFEM_USE_CEED),YES)
+ex1p-test-ceed: ex1p
+	@$(call mfem-test,$<, $(RUN_MPI),\
+	   Parallel libCEED example,-d ceed-cpu -pa -a)
+test-par-YES: ex1p-test-ceed
+endif
 
 # Testing: "test" target and mfem-test* variables are defined in config/test.mk
 
@@ -189,8 +195,8 @@ clean-build:
 clean-exec:
 	@rm -f refined.mesh displaced.mesh mesh.* ex5.mesh ex6p-checkpoint.*
 	@rm -rf Example5* Example9* Example15* Example16* Example23* ParaView
-	@rm -f sphere_refined.* sol.* sol_u.* sol_p.* sol_r.* sol_i.* order.*
-	@rm -f ex9.mesh ex9-mesh.* ex9-init.* ex9-final.*
+	@rm -f sphere_refined.* sol.* sol_u.* sol_p.* sol_r.* sol_i.* sol_z.*
+	@rm -f order.* ex9.mesh ex9-mesh.* ex9-init.* ex9-final.*
 	@rm -f deformed.* velocity.* elastic_energy.* mode_* mode_deriv_* flux.*
 	@rm -f ex5-p-*.bp ex9-p-*.bp ex12-p-*.bp ex16-p-*.bp
 	@rm -f ex16.mesh ex16-mesh.* ex16-init.* ex16-final.*

examples/myqfunction.hpp

@@ -0,0 +1,17 @@
+#include <mfem.hpp>
+
+using namespace mfem;
+using mfem::future::tensor;
+
+constexpr int dim = 2;
+
+tensor<real_t, dim, dim> myqfunction0(
+   const tensor<real_t, dim, dim> &dvdxi,
+   const tensor<real_t, dim, dim> &J,
+   const real_t &w)
+{
+   const auto invJ = inv(J);
+   const auto dvdx = dvdxi * invJ;
+   const auto test_function_terms = inv(J);
+   return dot(dvdx, J) * det(J) * w * test_function_terms;
+}

examples/nlohmann/json_fwd.hpp

@@ -0,0 +1,183 @@
+//     __ _____ _____ _____
+//  __|  |   __|     |   | |  JSON for Modern C++
+// |  |  |__   |  |  | | | |  version 3.12.0
+// |_____|_____|_____|_|___|  https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+#ifndef INCLUDE_NLOHMANN_JSON_FWD_HPP_
+#define INCLUDE_NLOHMANN_JSON_FWD_HPP_
+
+#include <cstdint> // int64_t, uint64_t
+#include <map>     // map
+#include <memory>  // allocator
+#include <string>  // string
+#include <vector>  // vector
+
+// #include <nlohmann/detail/abi_macros.hpp>
+//     __ _____ _____ _____
+//  __|  |   __|     |   | |  JSON for Modern C++
+// |  |  |__   |  |  | | | |  version 3.12.0
+// |_____|_____|_____|_|___|  https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013 - 2025 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+// This file contains all macro definitions affecting or depending on the ABI
+
+#ifndef JSON_SKIP_LIBRARY_VERSION_CHECK
+#if defined(NLOHMANN_JSON_VERSION_MAJOR) &&                                    \
+    defined(NLOHMANN_JSON_VERSION_MINOR) &&                                    \
+    defined(NLOHMANN_JSON_VERSION_PATCH)
+#if NLOHMANN_JSON_VERSION_MAJOR != 3 || NLOHMANN_JSON_VERSION_MINOR != 12 ||   \
+    NLOHMANN_JSON_VERSION_PATCH != 0
+#warning "Already included a different version of the library!"
+#endif
+#endif
+#endif
+
+#define NLOHMANN_JSON_VERSION_MAJOR 3  // NOLINT(modernize-macro-to-enum)
+#define NLOHMANN_JSON_VERSION_MINOR 12 // NOLINT(modernize-macro-to-enum)
+#define NLOHMANN_JSON_VERSION_PATCH 0  // NOLINT(modernize-macro-to-enum)
+
+#ifndef JSON_DIAGNOSTICS
+#define JSON_DIAGNOSTICS 0
+#endif
+
+#ifndef JSON_DIAGNOSTIC_POSITIONS
+#define JSON_DIAGNOSTIC_POSITIONS 0
+#endif
+
+#ifndef JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON
+#define JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON 0
+#endif
+
+#if JSON_DIAGNOSTICS
+#define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS _diag
+#else
+#define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS
+#endif
+
+#if JSON_DIAGNOSTIC_POSITIONS
+#define NLOHMANN_JSON_ABI_TAG_DIAGNOSTIC_POSITIONS _dp
+#else
+#define NLOHMANN_JSON_ABI_TAG_DIAGNOSTIC_POSITIONS
+#endif
+
+#if JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON
+#define NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON _ldvcmp
+#else
+#define NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON
+#endif
+
+#ifndef NLOHMANN_JSON_NAMESPACE_NO_VERSION
+#define NLOHMANN_JSON_NAMESPACE_NO_VERSION 0
+#endif
+
+// Construct the namespace ABI tags component
+#define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c) json_abi##a##b##c
+#define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b, c)                                 \
+  NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c)
+
+#define NLOHMANN_JSON_ABI_TAGS                                                 \
+  NLOHMANN_JSON_ABI_TAGS_CONCAT(                                               \
+      NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS,                                       \
+      NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON,                 \
+      NLOHMANN_JSON_ABI_TAG_DIAGNOSTIC_POSITIONS)
+
+// Construct the namespace version component
+#define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch)         \
+  _v##major##_##minor##_##patch
+#define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT(major, minor, patch)            \
+  NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch)
+
+#if NLOHMANN_JSON_NAMESPACE_NO_VERSION
+#define NLOHMANN_JSON_NAMESPACE_VERSION
+#else
+#define NLOHMANN_JSON_NAMESPACE_VERSION                                        \
+  NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT(NLOHMANN_JSON_VERSION_MAJOR,          \
+                                         NLOHMANN_JSON_VERSION_MINOR,          \
+                                         NLOHMANN_JSON_VERSION_PATCH)
+#endif
+
+// Combine namespace components
+#define NLOHMANN_JSON_NAMESPACE_CONCAT_EX(a, b) a##b
+#define NLOHMANN_JSON_NAMESPACE_CONCAT(a, b)                                   \
+  NLOHMANN_JSON_NAMESPACE_CONCAT_EX(a, b)
+
+#ifndef NLOHMANN_JSON_NAMESPACE
+#define NLOHMANN_JSON_NAMESPACE                                                \
+  nlohmann::NLOHMANN_JSON_NAMESPACE_CONCAT(NLOHMANN_JSON_ABI_TAGS,             \
+                                           NLOHMANN_JSON_NAMESPACE_VERSION)
+#endif
+
+#ifndef NLOHMANN_JSON_NAMESPACE_BEGIN
+#define NLOHMANN_JSON_NAMESPACE_BEGIN                                          \
+  namespace nlohmann {                                                         \
+  inline namespace NLOHMANN_JSON_NAMESPACE_CONCAT(                             \
+      NLOHMANN_JSON_ABI_TAGS, NLOHMANN_JSON_NAMESPACE_VERSION) {
+#endif
+
+#ifndef NLOHMANN_JSON_NAMESPACE_END
+#define NLOHMANN_JSON_NAMESPACE_END                                            \
+  } /* namespace (inline namespace) NOLINT(readability/namespace) */           \
+  } // namespace nlohmann
+#endif
+
+/*!
+@brief namespace for Niels Lohmann
+@see https://github.com/nlohmann
+@since version 1.0.0
+*/
+NLOHMANN_JSON_NAMESPACE_BEGIN
+
+/*!
+@brief default JSONSerializer template argument
+
+This serializer ignores the template arguments and uses ADL
+([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl))
+for serialization.
+*/
+template <typename T = void, typename SFINAE = void> struct adl_serializer;
+
+/// a class to store JSON values
+/// @sa https://json.nlohmann.me/api/basic_json/
+template <template <typename U, typename V, typename... Args> class ObjectType =
+          std::map,
+          template <typename U, typename... Args> class ArrayType = std::vector,
+          class StringType = std::string, class BooleanType = bool,
+          class NumberIntegerType = std::int64_t,
+          class NumberUnsignedType = std::uint64_t,
+          class NumberFloatType = double,
+          template <typename U> class AllocatorType = std::allocator,
+          template <typename T, typename SFINAE = void> class JSONSerializer =
+          adl_serializer,
+          class BinaryType =
+          std::vector<std::uint8_t>, // cppcheck-suppress syntaxError
+          class CustomBaseClass = void>
+class basic_json;
+
+/// @brief JSON Pointer defines a string syntax for identifying a specific value
+/// within a JSON document
+/// @sa https://json.nlohmann.me/api/json_pointer/
+template <typename RefStringType> class json_pointer;
+
+/*!
+@brief default specialization
+@sa https://json.nlohmann.me/api/json/
+*/
+using json = basic_json<>;
+
+/// @brief a minimal map-like container that preserves insertion order
+/// @sa https://json.nlohmann.me/api/ordered_map/
+template <class Key, class T, class IgnoredLess, class Allocator>
+struct ordered_map;
+
+/// @brief specialization that maintains the insertion order of object keys
+/// @sa https://json.nlohmann.me/api/ordered_json/
+using ordered_json = basic_json<nlohmann::ordered_map>;
+
+NLOHMANN_JSON_NAMESPACE_END
+
+#endif // INCLUDE_NLOHMANN_JSON_FWD_HPP_

fem/CMakeLists.txt

@@ -128,32 +128,46 @@ set(SRCS
   normal_deriv_restriction.cpp
   staticcond.cpp
   tmop.cpp
-  tmop/tmop_pa.cpp
-  tmop/tmop_pa_da3.cpp
-  tmop/tmop_pa_h2d.cpp
-  tmop/tmop_pa_h2d_c0.cpp
-  tmop/tmop_pa_h2m.cpp
-  tmop/tmop_pa_h2m_c0.cpp
-  tmop/tmop_pa_h2s.cpp
-  tmop/tmop_pa_h2s_c0.cpp
-  tmop/tmop_pa_h3d.cpp
-  tmop/tmop_pa_h3d_c0.cpp
-  tmop/tmop_pa_h3m.cpp
-  tmop/tmop_pa_h3m_c0.cpp
-  tmop/tmop_pa_h3s.cpp
-  tmop/tmop_pa_h3s_c0.cpp
-  tmop/tmop_pa_jp2.cpp
-  tmop/tmop_pa_jp3.cpp
-  tmop/tmop_pa_p2.cpp
-  tmop/tmop_pa_p2_c0.cpp
-  tmop/tmop_pa_p3.cpp
-  tmop/tmop_pa_p3_c0.cpp
-  tmop/tmop_pa_tc2.cpp
-  tmop/tmop_pa_tc3.cpp
-  tmop/tmop_pa_w2.cpp
-  tmop/tmop_pa_w2_c0.cpp
-  tmop/tmop_pa_w3.cpp
-  tmop/tmop_pa_w3_c0.cpp
+  tmop/pa.cpp
+  tmop/assemble/diag2_limit.cpp
+  tmop/assemble/diag2.cpp
+  tmop/assemble/grad2_limit.cpp
+  tmop/assemble/grad2.cpp
+  tmop/assemble/diag3_limit.cpp  
+  tmop/assemble/diag3.cpp
+  tmop/assemble/grad3_limit.cpp
+  tmop/assemble/grad3.cpp
+  tmop/metrics/001.cpp
+  tmop/metrics/002.cpp
+  tmop/metrics/007.cpp
+  tmop/metrics/056.cpp
+  tmop/metrics/077.cpp
+  tmop/metrics/080.cpp
+  tmop/metrics/094.cpp
+  tmop/metrics/302.cpp
+  tmop/metrics/303.cpp
+  tmop/metrics/315.cpp
+  tmop/metrics/318.cpp
+  tmop/metrics/321.cpp
+  tmop/metrics/332.cpp
+  tmop/metrics/338.cpp
+  tmop/mult/grad2_limit.cpp
+  tmop/mult/grad2.cpp
+  tmop/mult/mult2_limit.cpp
+  tmop/mult/mult2.cpp
+  tmop/mult/grad3_limit.cpp
+  tmop/mult/grad3.cpp
+  tmop/mult/mult3_limit.cpp
+  tmop/mult/mult3.cpp
+  tmop/tools/det2_jpr.cpp
+  tmop/tools/det3_jpr.cpp
+  tmop/tools/discrete.cpp
+  tmop/tools/energy2_limit.cpp
+  tmop/tools/energy2.cpp
+  tmop/tools/energy3_limit.cpp
+  tmop/tools/energy3.cpp
+  tmop/tools/target2.cpp
+  tmop/tools/target3.cpp
   tmop_tools.cpp
   tmop_amr.cpp
   gslib.cpp
@@ -171,12 +185,19 @@ set(HDRS
   bilinearform.hpp
   bilinearform_ext.hpp
   bilininteg.hpp
+  integ/lininteg_domain_kernels.hpp
+  integ/bilininteg_dgdiffusion_kernels.hpp
+  integ/bilininteg_dgtrace_kernels.hpp
+  integ/bilininteg_vecdiffusion_kernels.hpp
+  integ/bilininteg_convection_kernels.hpp
   integ/bilininteg_diffusion_kernels.hpp
   integ/bilininteg_elasticity_kernels.hpp
   integ/bilininteg_hcurl_kernels.hpp
   integ/bilininteg_hdiv_kernels.hpp
   integ/bilininteg_hcurlhdiv_kernels.hpp
   integ/bilininteg_mass_kernels.hpp
+  integ/bilininteg_vecdiffusion_pa.hpp
+  integ/bilininteg_vecmass_pa.hpp
   coefficient.hpp
   complex_fem.hpp
   convergence.hpp
@@ -241,8 +262,13 @@ set(HDRS
   lor/lor_ams.hpp
   lor/lor_batched.hpp
   lor/lor_h1.hpp
+  lor/lor_dg.hpp
   lor/lor_nd.hpp
   lor/lor_rt.hpp
+  lor/lor_h1_impl.hpp
+  lor/lor_dg_impl.hpp
+  lor/lor_nd_impl.hpp
+  lor/lor_rt_impl.hpp
   lor/lor_util.hpp
   multigrid.hpp
   nonlinearform.hpp
@@ -269,7 +295,12 @@ set(HDRS
   tfespace.hpp
   tintrules.hpp
   tmop.hpp
-  tmop/tmop_pa.hpp
+  tmop/pa.hpp
+  tmop/assemble/grad2.hpp
+  tmop/assemble/grad2.hpp
+  tmop/mult/mult2.hpp
+  tmop/mult/mult3.hpp
+  tmop/tools/energy2.hpp
   tmop_tools.hpp
   tmop_amr.hpp
   gslib.hpp

fem/bilininteg.cpp

@@ -3066,7 +3066,6 @@ void VectorDiffusionIntegrator::AssembleElementMatrix(
 
    for (int i = 0; i < ir -> GetNPoints(); i++)
    {
-
       const IntegrationPoint &ip = ir->IntPoint(i);
       el.CalcDShape(ip, dshape);
 

fem/bilininteg.hpp

@@ -23,6 +23,8 @@
 
 namespace mfem
 {
+class QuadratureSpace;
+class FaceQuadratureSpace;
 
 /// Abstract base class BilinearFormIntegrator
 class BilinearFormIntegrator : public NonlinearFormIntegrator
@@ -2494,8 +2496,7 @@ private:
 #endif
 
 public:
-   ConvectionIntegrator(VectorCoefficient &q, real_t a = 1.0)
-      : Q(&q) { alpha = a; }
+   ConvectionIntegrator(VectorCoefficient &q, real_t a = 1.0);
 
    void AssembleElementMatrix(const FiniteElement &,
                               ElementTransformation &,
@@ -2528,6 +2529,28 @@ public:
 
    bool SupportsCeed() const override { return DeviceCanUseCeed(); }
 
+   /// arguments: NE, B, G, Bt, Gt, pa_data, x, y, D1D, Q1D
+   using ApplyKernelType = void (*)(const int, const Array<real_t> &,
+                                    const Array<real_t> &,
+                                    const Array<real_t> &,
+                                    const Array<real_t> &, const Vector &,
+                                    const Vector &, Vector &, const int,
+                                    const int);
+
+   /// arguments: DIMS, D1D, Q1D
+   MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int));
+   /// arguments: DIMS, D1D, Q1D
+   MFEM_REGISTER_KERNELS(ApplyPATKernels, ApplyKernelType, (int, int, int));
+
+   template <int DIM, int D1D, int Q1D>
+   static void AddSpecialization()
+   {
+      ApplyPAKernels::Specialization<DIM, D1D, Q1D>::Add();
+      ApplyPATKernels::Specialization<DIM, D1D, Q1D>::Add();
+   }
+
+   struct Kernels { Kernels(); };
+
 protected:
    const IntegrationRule* GetDefaultIntegrationRule(
       const FiniteElement& trial_fe,
@@ -2573,41 +2596,40 @@ public:
     by scalar FE through standard transformation. */
 class VectorMassIntegrator: public BilinearFormIntegrator
 {
-private:
-   int vdim;
+   int vdim = -1, Q_order = 0;
    Vector shape, te_shape, vec;
    DenseMatrix partelmat;
    DenseMatrix mcoeff;
-   int Q_order;
 
 protected:
-   Coefficient *Q;
-   VectorCoefficient *VQ;
-   MatrixCoefficient *MQ;
+   Coefficient *Q = nullptr;
+   VectorCoefficient *VQ = nullptr;
+   MatrixCoefficient *MQ = nullptr;
    // PA extension
-   Vector pa_data;
    const DofToQuad *maps;         ///< Not owned
    const GeometricFactors *geom;  ///< Not owned
-   int dim, ne, nq, dofs1D, quad1D;
+   int ne, dim, dofs1D, quad1D, coeff_vdim;
+   Vector pa_data;
 
 public:
    /// Construct an integrator with coefficient 1.0
-   VectorMassIntegrator()
-      : vdim(-1), Q_order(0), Q(NULL), VQ(NULL), MQ(NULL) { }
+   VectorMassIntegrator() = default;
+
    /** Construct an integrator with scalar coefficient q.  If possible, save
        memory by using a scalar integrator since the resulting matrix is block
        diagonal with the same diagonal block repeated. */
-   VectorMassIntegrator(Coefficient &q, int qo = 0)
-      : vdim(-1), Q_order(qo), Q(&q), VQ(NULL), MQ(NULL) { }
-   VectorMassIntegrator(Coefficient &q, const IntegrationRule *ir)
-      : BilinearFormIntegrator(ir), vdim(-1), Q_order(0), Q(&q), VQ(NULL),
-        MQ(NULL) { }
+   VectorMassIntegrator(Coefficient &q, int qo = 0): Q_order(qo), Q(&q) { }
+
+   VectorMassIntegrator(Coefficient &q, const IntegrationRule *ir):
+      BilinearFormIntegrator(ir), Q(&q) { }
+
    /// Construct an integrator with diagonal coefficient q
-   VectorMassIntegrator(VectorCoefficient &q, int qo = 0)
-      : vdim(q.GetVDim()), Q_order(qo), Q(NULL), VQ(&q), MQ(NULL) { }
+   VectorMassIntegrator(VectorCoefficient &q, int qo = 0):
+      vdim(q.GetVDim()), Q_order(qo), VQ(&q) { }
+
    /// Construct an integrator with matrix coefficient q
-   VectorMassIntegrator(MatrixCoefficient &q, int qo = 0)
-      : vdim(q.GetVDim()), Q_order(qo), Q(NULL), VQ(NULL), MQ(&q) { }
+   VectorMassIntegrator(MatrixCoefficient &q, int qo = 0):
+      vdim(q.GetVDim()), Q_order(qo), MQ(&q) { }
 
    int GetVDim() const { return vdim; }
    void SetVDim(int vdim_) { vdim = vdim_; }
@@ -2619,6 +2641,7 @@ public:
                                const FiniteElement &test_fe,
                                ElementTransformation &Trans,
                                DenseMatrix &elmat) override;
+
    using BilinearFormIntegrator::AssemblePA;
    void AssemblePA(const FiniteElementSpace &fes) override;
    void AssembleMF(const FiniteElementSpace &fes) override;
@@ -2627,6 +2650,15 @@ public:
    void AddMultPA(const Vector &x, Vector &y) const override;
    void AddMultMF(const Vector &x, Vector &y) const override;
    bool SupportsCeed() const override { return DeviceCanUseCeed(); }
+
+   using VectorMassAddMultPAType =
+      void(*)(const int,  const int,
+              const Array<real_t>&, const Vector&,
+              const Vector&, Vector&, const int, const int);
+
+   MFEM_REGISTER_KERNELS(VectorMassAddMultPA,
+                         VectorMassAddMultPAType,
+                         (int, int, int));
 };
 
 
@@ -2798,15 +2830,13 @@ protected:
    bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient
 
 public:
-   CurlCurlIntegrator() { Q = NULL; DQ = NULL; MQ = NULL; }
+   CurlCurlIntegrator();
    /// Construct a bilinear form integrator for Nedelec elements
-   CurlCurlIntegrator(Coefficient &q, const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), Q(&q), DQ(NULL), MQ(NULL) { }
+   CurlCurlIntegrator(Coefficient &q, const IntegrationRule *ir = nullptr);
    CurlCurlIntegrator(DiagonalMatrixCoefficient &dq,
-                      const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), Q(NULL), DQ(&dq), MQ(NULL) { }
-   CurlCurlIntegrator(MatrixCoefficient &mq, const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), Q(NULL), DQ(NULL), MQ(&mq) { }
+                      const IntegrationRule *ir = nullptr);
+   CurlCurlIntegrator(MatrixCoefficient &mq,
+                      const IntegrationRule *ir = nullptr);
 
    /* Given a particular Finite Element, compute the
       element curl-curl matrix elmat */
@@ -2836,6 +2866,34 @@ public:
    void AssembleDiagonalPA(Vector& diag) override;
 
    const Coefficient *GetCoefficient() const { return Q; }
+
+   /// arguments: d1d, q1d, symmetric, NE, bo, bc, bot, bct, gc, gct, pa_data,
+   /// x, y, useAbs
+   using ApplyKernelType = void (*)(
+                              const int, const int, const bool, const int, const Array<real_t> &,
+                              const Array<real_t> &, const Array<real_t> &, const Array<real_t> &,
+                              const Array<real_t> &, const Array<real_t> &, const Vector &,
+                              const Vector &, Vector &, const bool);
+
+   /// arguments: d1d, q1d, symmetric, ne, Bo, Bc, Go, Gc, pa_data, diag
+   using DiagonalKernelType = void (*)(const int, const int, const bool,
+                                       const int, const Array<real_t> &,
+                                       const Array<real_t> &,
+                                       const Array<real_t> &,
+                                       const Array<real_t> &, const Vector &,
+                                       Vector &);
+
+   /// parameters: dim, d1d, q1d
+   MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int));
+   /// parameters: dim, d1d, q1d
+   MFEM_REGISTER_KERNELS(DiagonalPAKernels, DiagonalKernelType, (int, int, int));
+   struct Kernels { Kernels(); };
+
+   template <int DIM, int D1D, int Q1D> static void AddSpecialization()
+   {
+      ApplyPAKernels::Specialization<DIM, D1D, Q1D>::Add();
+      DiagonalPAKernels::Specialization<DIM, D1D, Q1D>::Add();
+   }
 };
 
 /** Integrator for $(\mathrm{curl}(u), \mathrm{curl}(v))$ for FE spaces defined by 'dim' copies of a
@@ -3071,39 +3129,34 @@ public:
     to be the spatial dimension (i.e. 2-dimension or 3-dimension). */
 class VectorDiffusionIntegrator : public BilinearFormIntegrator
 {
-protected:
-   Coefficient *Q = NULL;
-   VectorCoefficient *VQ = NULL;
-   MatrixCoefficient *MQ = NULL;
-
-   // PA extension
-   const DofToQuad *maps;         ///< Not owned
-   const GeometricFactors *geom;  ///< Not owned
-   int dim, sdim, ne, dofs1D, quad1D;
-   Vector pa_data;
-
-private:
-   DenseMatrix dshape, dshapedxt, pelmat;
    int vdim = -1;
+   DenseMatrix dshape, dshapedxt, pelmat;
    DenseMatrix mcoeff;
    Vector vcoeff;
 
+protected:
+   Coefficient *Q = nullptr;
+   VectorCoefficient *VQ = nullptr;
+   MatrixCoefficient *MQ = nullptr;
+   // PA extension
+   const DofToQuad *maps;         ///< Not owned
+   const GeometricFactors *geom;  ///< Not owned
+   int ne, dim, sdim, dofs1D, quad1D, coeff_vdim;
+   Vector pa_data;
+
 public:
-   VectorDiffusionIntegrator() { }
+   VectorDiffusionIntegrator(const IntegrationRule *ir = nullptr);
 
    /** \brief Integrator with unit coefficient for caller-specified vector
        dimension.
 
        If the vector dimension does not match the true dimension of the space,
        the resulting element matrix will be mathematically invalid. */
-   VectorDiffusionIntegrator(int vector_dimension)
-      : vdim(vector_dimension) { }
+   VectorDiffusionIntegrator(int vector_dimension);
 
-   VectorDiffusionIntegrator(Coefficient &q)
-      : Q(&q) { }
+   VectorDiffusionIntegrator(Coefficient &q);
 
-   VectorDiffusionIntegrator(Coefficient &q, const IntegrationRule *ir)
-      : BilinearFormIntegrator(ir), Q(&q) { }
+   VectorDiffusionIntegrator(Coefficient &q, const IntegrationRule *ir);
 
    /** \brief Integrator with scalar coefficient for caller-specified vector
        dimension.
@@ -3113,8 +3166,7 @@ public:
 
        If the vector dimension does not match the true dimension of the space,
        the resulting element matrix will be mathematically invalid. */
-   VectorDiffusionIntegrator(Coefficient &q, int vector_dimension)
-      : Q(&q), vdim(vector_dimension) { }
+   VectorDiffusionIntegrator(Coefficient &q, int vector_dimension);
 
    /** \brief Integrator with \c VectorCoefficient. The vector dimension of the
        \c FiniteElementSpace is assumed to be the same as the dimension of the
@@ -3125,8 +3177,7 @@ public:
 
        If the vector dimension does not match the true dimension of the space,
        the resulting element matrix will be mathematically invalid. */
-   VectorDiffusionIntegrator(VectorCoefficient &vq)
-      : VQ(&vq), vdim(vq.GetVDim()) { }
+   VectorDiffusionIntegrator(VectorCoefficient &vq);
 
    /** \brief Integrator with \c MatrixCoefficient. The vector dimension of the
        \c FiniteElementSpace is assumed to be the same as the dimension of the
@@ -3137,8 +3188,7 @@ public:
 
        If the vector dimension does not match the true dimension of the space,
        the resulting element matrix will be mathematically invalid. */
-   VectorDiffusionIntegrator(MatrixCoefficient& mq)
-      : MQ(&mq), vdim(mq.GetVDim()) { }
+   VectorDiffusionIntegrator(MatrixCoefficient& mq);
 
    void AssembleElementMatrix(const FiniteElement &el,
                               ElementTransformation &Trans,
@@ -3146,6 +3196,7 @@ public:
    void AssembleElementVector(const FiniteElement &el,
                               ElementTransformation &Tr,
                               const Vector &elfun, Vector &elvect) override;
+
    using BilinearFormIntegrator::AssemblePA;
    void AssemblePA(const FiniteElementSpace &fes) override;
    void AssembleMF(const FiniteElementSpace &fes) override;
@@ -3154,6 +3205,23 @@ public:
    void AddMultPA(const Vector &x, Vector &y) const override;
    void AddMultMF(const Vector &x, Vector &y) const override;
    bool SupportsCeed() const override { return DeviceCanUseCeed(); }
+
+   /// arguments: ne, coeff_vdim, B, G, pa_data, x, y, d1d, q1d, vdim
+   using ApplyKernelType = void (*)(const int, const int,
+                                    const Array<real_t> &, const Array<real_t> &,
+                                    const Vector &, const Vector &, Vector &,
+                                    const int, const int, const int);
+
+   /// arguments: dim, vdim, d1d, q1d
+   MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int, int));
+
+   template <int DIM, int VDIM, int D1D, int Q1D>
+   static void AddSpecialization()
+   {
+      ApplyPAKernels::Specialization<DIM, VDIM, D1D, Q1D>::Add();
+   }
+
+   // struct Kernels { Kernels(); };
 };
 
 /** Integrator for the linear elasticity form:
@@ -3307,8 +3375,8 @@ public:
 class DGTraceIntegrator : public BilinearFormIntegrator
 {
 protected:
-   Coefficient *rho;
-   VectorCoefficient *u;
+   Coefficient *rho = nullptr;
+   VectorCoefficient *u = nullptr;
    real_t alpha, beta;
    // PA extension
    Vector pa_data;
@@ -3321,17 +3389,16 @@ private:
    Vector tr_shape1, te_shape1, tr_shape2, te_shape2;
 
 public:
+   DGTraceIntegrator(real_t a, real_t b);
+
    /// Construct integrator with $\rho = 1$, $\beta = \alpha/2$.
-   DGTraceIntegrator(VectorCoefficient &u_, real_t a)
-   { rho = NULL; u = &u_; alpha = a; beta = 0.5*a; }
+   DGTraceIntegrator(VectorCoefficient &u_, real_t a);
 
    /// Construct integrator with $\rho = 1$.
-   DGTraceIntegrator(VectorCoefficient &u_, real_t a, real_t b)
-   { rho = NULL; u = &u_; alpha = a; beta = b; }
+   DGTraceIntegrator(VectorCoefficient &u_, real_t a, real_t b);
 
    DGTraceIntegrator(Coefficient &rho_, VectorCoefficient &u_,
-                     real_t a, real_t b)
-   { rho = &rho_; u = &u_; alpha = a; beta = b; }
+                     real_t a, real_t b);
 
    using BilinearFormIntegrator::AssembleFaceMatrix;
    void AssembleFaceMatrix(const FiniteElement &el1,
@@ -3370,6 +3437,26 @@ public:
    static const IntegrationRule &GetRule(Geometry::Type geom, int order,
                                          const ElementTransformation &T);
 
+   /// arguments: nf, B, Bt, pa_data, x, y, dofs1D, quad1D
+   using ApplyKernelType = void (*)(const int, const Array<real_t> &,
+                                    const Array<real_t> &, const Vector &,
+                                    const Vector &, Vector &, const int,
+                                    const int);
+
+   /// arguments: DIM, d1d, q1d
+   MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int));
+   /// arguments: DIM, d1d, q1d
+   MFEM_REGISTER_KERNELS(ApplyPATKernels, ApplyKernelType, (int, int, int));
+
+   template <int DIM, int D1D, int Q1D> static void AddSpecialization()
+   {
+      ApplyPAKernels::Specialization<DIM, D1D, Q1D>::Add();
+      ApplyPATKernels::Specialization<DIM, D1D, Q1D>::Add();
+   }
+
+   struct Kernels { Kernels(); };
+
+
 private:
    void SetupPA(const FiniteElementSpace &fes, FaceType type);
 };
@@ -3416,8 +3503,8 @@ public:
 class DGDiffusionIntegrator : public BilinearFormIntegrator
 {
 protected:
-   Coefficient *Q;
-   MatrixCoefficient *MQ;
+   Coefficient *Q = nullptr;
+   MatrixCoefficient *MQ = nullptr;
    real_t sigma, kappa;
 
    // these are not thread-safe!
@@ -3432,15 +3519,11 @@ protected:
    IntegrationRules irs{0, Quadrature1D::GaussLobatto};
 
 public:
-   DGDiffusionIntegrator(const real_t s, const real_t k)
-      : Q(NULL), MQ(NULL), sigma(s), kappa(k) { }
-   DGDiffusionIntegrator(Coefficient &q, const real_t s, const real_t k)
-      : Q(&q), MQ(NULL), sigma(s), kappa(k) { }
-   DGDiffusionIntegrator(MatrixCoefficient &q, const real_t s, const real_t k)
-      : Q(NULL), MQ(&q), sigma(s), kappa(k) { }
+   DGDiffusionIntegrator(const real_t s, const real_t k);
+   DGDiffusionIntegrator(Coefficient &q, const real_t s, const real_t k);
+   DGDiffusionIntegrator(MatrixCoefficient &q, const real_t s, const real_t k);
    using BilinearFormIntegrator::AssembleFaceMatrix;
-   void AssembleFaceMatrix(const FiniteElement &el1,
-                           const FiniteElement &el2,
+   void AssembleFaceMatrix(const FiniteElement &el1, const FiniteElement &el2,
                            FaceElementTransformations &Trans,
                            DenseMatrix &elmat) override;
 
@@ -3459,6 +3542,28 @@ public:
 
    const IntegrationRule &GetRule(int order, Geometry::Type geom);
 
+   real_t GetPenaltyParameter() const { return kappa; }
+
+   /// arguments: nf, B, Bt, G, Gt, sigma, pa_data, x, dxdn, y, dydn, dofs1D,
+   /// quad1D
+   using ApplyKernelType = void (*)(const int, const Array<real_t> &,
+                                    const Array<real_t> &,
+                                    const Array<real_t> &,
+                                    const Array<real_t> &, const real_t,
+                                    const Vector &, const Vector &_,
+                                    const Vector &, Vector &, Vector &,
+                                    const int, const int);
+
+   /// arguments: DIM, d1d, q1d
+   MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int));
+
+   template <int DIM, int D1D, int Q1D> static void AddSpecialization()
+   {
+      ApplyPAKernels::Specialization<DIM, D1D, Q1D>::Add();
+   }
+
+   struct Kernels { Kernels(); };
+
 private:
    void SetupPA(const FiniteElementSpace &fes, FaceType type);
 };

fem/ceed/integrators/convection/convection_qf.h

@@ -8,6 +8,7 @@
 // MFEM is free software; you can redistribute it and/or modify it under the
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
+#include <ceed/types.h>
 
 /// A structure used to pass additional data to f_build_conv and f_apply_conv
 struct ConvectionContext {
@@ -91,7 +92,7 @@ CEED_QFUNCTION(f_build_conv_const)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return 0;
+   return CEED_ERROR_SUCCESS;
 }
 
 /// libCEED Q-function for building quadrature data for a convection operator
@@ -167,7 +168,7 @@ CEED_QFUNCTION(f_build_conv_quad)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return 0;
+   return CEED_ERROR_SUCCESS;
 }
 
 /// libCEED Q-function for applying a conv operator
@@ -233,7 +234,7 @@ CEED_QFUNCTION(f_apply_conv)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return 0;
+   return CEED_ERROR_SUCCESS;
 }
 
 /// libCEED Q-function for applying a conv operator
@@ -381,7 +382,7 @@ CEED_QFUNCTION(f_apply_conv_mf_const)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return 0;
+   return CEED_ERROR_SUCCESS;
 }
 
 CEED_QFUNCTION(f_apply_conv_mf_quad)(void *ctx, CeedInt Q,
@@ -525,5 +526,5 @@ CEED_QFUNCTION(f_apply_conv_mf_quad)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return 0;
+   return CEED_ERROR_SUCCESS;
 }

fem/ceed/integrators/diffusion/diffusion_qf.h

@@ -8,7 +8,7 @@
 // MFEM is free software; you can redistribute it and/or modify it under the
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
-
+#include <ceed/types.h>
 
 /// A structure used to pass additional data to f_build_diff and f_apply_diff
 struct DiffusionContext { CeedInt dim, space_dim, vdim; CeedScalar coeff; };
@@ -85,7 +85,7 @@ CEED_QFUNCTION(f_build_diff_const)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return 0;
+   return CEED_ERROR_SUCCESS;
 }
 
 /// libCEED Q-function for building quadrature data for a diffusion operator
@@ -161,7 +161,7 @@ CEED_QFUNCTION(f_build_diff_quad)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return 0;
+   return CEED_ERROR_SUCCESS;
 }
 
 /// libCEED Q-function for applying a diff operator
@@ -241,7 +241,7 @@ CEED_QFUNCTION(f_apply_diff)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return 0;
+   return CEED_ERROR_SUCCESS;
 }
 
 /// libCEED Q-function for applying a diff operator
@@ -394,7 +394,7 @@ CEED_QFUNCTION(f_apply_diff_mf_const)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return 0;
+   return CEED_ERROR_SUCCESS;
 }
 
 CEED_QFUNCTION(f_apply_diff_mf_quad)(void *ctx, CeedInt Q,
@@ -549,5 +549,5 @@ CEED_QFUNCTION(f_apply_diff_mf_quad)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return 0;
+   return CEED_ERROR_SUCCESS;
 }

fem/ceed/integrators/mass/mass_qf.h

@@ -8,7 +8,7 @@
 // MFEM is free software; you can redistribute it and/or modify it under the
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
-
+#include <ceed/types.h>
 
 /// A structure used to pass additional data to f_build_diff and f_apply_diff
 struct MassContext { CeedInt dim, space_dim, vdim; CeedScalar coeff; };
@@ -53,7 +53,7 @@ CEED_QFUNCTION(f_build_mass_const)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return 0;
+   return CEED_ERROR_SUCCESS;
 }
 
 /// libCEED Q-function for building quadrature data for a mass operator with a
@@ -95,7 +95,7 @@ CEED_QFUNCTION(f_build_mass_quad)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return 0;
+   return CEED_ERROR_SUCCESS;
 }
 
 /// libCEED Q-function for applying a mass operator
@@ -135,7 +135,7 @@ CEED_QFUNCTION(f_apply_mass)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return 0;
+   return CEED_ERROR_SUCCESS;
 }
 
 /// libCEED Q-function for applying a diff operator
@@ -199,7 +199,7 @@ CEED_QFUNCTION(f_apply_mass_mf_const)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return 0;
+   return CEED_ERROR_SUCCESS;
 }
 
 CEED_QFUNCTION(f_apply_mass_mf_quad)(void *ctx, CeedInt Q,
@@ -266,5 +266,5 @@ CEED_QFUNCTION(f_apply_mass_mf_quad)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return 0;
+   return CEED_ERROR_SUCCESS;
 }

fem/ceed/integrators/nlconvection/nlconvection_qf.h

@@ -8,6 +8,7 @@
 // MFEM is free software; you can redistribute it and/or modify it under the
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
+#include <ceed/types.h>
 
 /// A structure used to pass additional data to f_build_conv and f_apply_conv
 struct NLConvectionContext { CeedInt dim, space_dim, vdim; CeedScalar coeff; };
@@ -87,7 +88,7 @@ CEED_QFUNCTION(f_build_conv_const)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return 0;
+   return CEED_ERROR_SUCCESS;
 }
 
 /// libCEED Q-function for building quadrature data for a convection operator
@@ -167,7 +168,7 @@ CEED_QFUNCTION(f_build_conv_quad)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return 0;
+   return CEED_ERROR_SUCCESS;
 }
 
 /// libCEED Q-function for applying a conv operator
@@ -247,7 +248,7 @@ CEED_QFUNCTION(f_apply_conv)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return 0;
+   return CEED_ERROR_SUCCESS;
 }
 
 /// libCEED Q-function for applying a conv operator
@@ -362,7 +363,7 @@ CEED_QFUNCTION(f_apply_conv_mf_const)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return 0;
+   return CEED_ERROR_SUCCESS;
 }
 
 CEED_QFUNCTION(f_apply_conv_mf_quad)(void *ctx, CeedInt Q,
@@ -475,5 +476,5 @@ CEED_QFUNCTION(f_apply_conv_mf_quad)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return 0;
+   return CEED_ERROR_SUCCESS;
 }

fem/ceed/interface/ceed.hpp

@@ -18,10 +18,21 @@
 
 #include <ceed.h>
 
-#if !CEED_VERSION_GE(0,12,0)
+#if !CEED_VERSION_GE(0, 12, 0)
 #error MFEM requires a libCEED version >= 0.12.0
 #endif
 
+#if !CEED_VERSION_GE(0, 13, 0)
+#define CeedOperatorCreateComposite(ceed, op)                                  \
+    CeedCompositeOperatorCreate((ceed), (op))
+#define CeedOperatorCompositeAddSub(op, sub)                                   \
+    CeedCompositeOperatorAddSub((op), (sub))
+#define CeedOperatorCompositeGetNumSub(op, num)                                \
+    CeedCompositeOperatorGetNumSub((op), (num))
+#define CeedOperatorCompositeGetSubList(op, list)                              \
+    CeedCompositeOperatorGetSubList((op), (list))
+#endif
+
 namespace mfem
 {
 

fem/ceed/interface/mixed_integrator.hpp

@@ -83,7 +83,7 @@ public:
       }
 
       // Create composite CeedOperator
-      CeedCompositeOperatorCreate(internal::ceed, &oper);
+      CeedOperatorCreateComposite(internal::ceed, &oper);
 
       // Create each sub-CeedOperator
       sub_ops.reserve(element_indices.size());
@@ -101,7 +101,7 @@ public:
          int nelem = *count[value.first];
          sub_op->Assemble(info, fes, ir, nelem, indices, Q);
          sub_ops.push_back(sub_op);
-         CeedCompositeOperatorAddSub(oper, sub_op->GetCeedOperator());
+         CeedOperatorCompositeAddSub(oper, sub_op->GetCeedOperator());
       }
 
       const int ndofs = fes.GetVDim() * fes.GetNDofs();

fem/ceed/interface/util.cpp

@@ -140,11 +140,7 @@ int CeedOperatorGetActiveField(CeedOperator oper, CeedOperatorField *field)
    CeedOperator *subops;
    if (isComposite)
    {
-#if CEED_VERSION_GE(0, 10, 2)
-      ierr = CeedCompositeOperatorGetSubList(oper, &subops); PCeedChk(ierr);
-#else
-      ierr = CeedOperatorGetSubList(oper, &subops); PCeedChk(ierr);
-#endif
+      ierr = CeedOperatorCompositeGetSubList(oper, &subops); PCeedChk(ierr);
       ierr = CeedOperatorGetQFunction(subops[0], &qf); PCeedChk(ierr);
    }
    else
@@ -171,7 +167,11 @@ int CeedOperatorGetActiveField(CeedOperator oper, CeedOperatorField *field)
    for (int i = 0; i < numinputfields; ++i)
    {
       ierr = CeedOperatorFieldGetVector(inputfields[i], &if_vector); PCeedChk(ierr);
-      if (if_vector == CEED_VECTOR_ACTIVE)
+      bool is_active = if_vector == CEED_VECTOR_ACTIVE;
+#if CEED_VERSION_GE(0, 13, 0)
+      ierr = CeedVectorDestroy(&if_vector); PCeedChk(ierr);
+#endif
+      if (is_active)
       {
          if (found)
          {

fem/ceed/solvers/algebraic.cpp

@@ -228,7 +228,7 @@ void AddToCompositeOperator(BilinearFormIntegrator *integ, CeedOperator op)
 {
    if (integ->SupportsCeed())
    {
-      CeedCompositeOperatorAddSub(op, integ->GetCeedOp().GetCeedOperator());
+      CeedOperatorCompositeAddSub(op, integ->GetCeedOp().GetCeedOperator());
    }
    else
    {
@@ -240,7 +240,7 @@ CeedOperator CreateCeedCompositeOperatorFromBilinearForm(BilinearForm &form)
 {
    int ierr;
    CeedOperator op;
-   ierr = CeedCompositeOperatorCreate(internal::ceed, &op); PCeedChk(ierr);
+   ierr = CeedOperatorCreateComposite(internal::ceed, &op); PCeedChk(ierr);
 
    MFEM_VERIFY(form.GetBBFI()->Size() == 0,
                "Not implemented for this integrator!");
@@ -271,18 +271,13 @@ CeedOperator CoarsenCeedCompositeOperator(
    MFEM_ASSERT(isComposite, "");
 
    CeedOperator op_coarse;
-   ierr = CeedCompositeOperatorCreate(internal::ceed,
+   ierr = CeedOperatorCreateComposite(internal::ceed,
                                       &op_coarse); PCeedChk(ierr);
 
    int nsub;
    CeedOperator *subops;
-#if CEED_VERSION_GE(0, 10, 2)
-   ierr = CeedCompositeOperatorGetNumSub(op, &nsub); PCeedChk(ierr);
-   ierr = CeedCompositeOperatorGetSubList(op, &subops); PCeedChk(ierr);
-#else
-   ierr = CeedOperatorGetNumSub(op, &nsub); PCeedChk(ierr);
-   ierr = CeedOperatorGetSubList(op, &subops); PCeedChk(ierr);
-#endif
+   ierr = CeedOperatorCompositeGetNumSub(op, &nsub); PCeedChk(ierr);
+   ierr = CeedOperatorCompositeGetSubList(op, &subops); PCeedChk(ierr);
    for (int isub=0; isub<nsub; ++isub)
    {
       CeedOperator subop = subops[isub];
@@ -294,7 +289,7 @@ CeedOperator CoarsenCeedCompositeOperator(
       // refcounted by existing objects
       ierr = CeedBasisDestroy(&basis_coarse); PCeedChk(ierr);
       ierr = CeedBasisDestroy(&basis_c2f); PCeedChk(ierr);
-      ierr = CeedCompositeOperatorAddSub(op_coarse, subop_coarse);
+      ierr = CeedOperatorCompositeAddSub(op_coarse, subop_coarse);
       PCeedChk(ierr);
       ierr = CeedOperatorDestroy(&subop_coarse); PCeedChk(ierr);
    }

fem/ceed/solvers/full-assembly.cpp

@@ -81,12 +81,27 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
       ierr = CeedOperatorFieldGetVector(input_fields[i], &vec); PCeedChk(ierr);
       if (vec == CEED_VECTOR_ACTIVE)
       {
-         ierr = CeedOperatorFieldGetBasis(input_fields[i], &basisin);
-         PCeedChk(ierr);
+         CeedBasis basis;
+         ierr = CeedOperatorFieldGetBasis(input_fields[i], &basis); PCeedChk(ierr);
+         if (!basisin)
+         {
+            ierr = CeedBasisReferenceCopy(basis, &basisin); PCeedChk(ierr);
+         }
+#if CEED_VERSION_GE(0, 13, 0)
+         ierr = CeedBasisDestroy(&basis); PCeedChk(ierr);
+#endif
          ierr = CeedBasisGetNumComponents(basisin, &ncomp); PCeedChk(ierr);
          ierr = CeedBasisGetDimension(basisin, &dim); PCeedChk(ierr);
-         ierr = CeedOperatorFieldGetElemRestriction(input_fields[i], &rstrin);
+         CeedElemRestriction rstr;
+         ierr = CeedOperatorFieldGetElemRestriction(input_fields[i], &rstr);
          PCeedChk(ierr);
+         if (!rstrin)
+         {
+            ierr = CeedElemRestrictionReferenceCopy(rstr, &rstrin); PCeedChk(ierr);
+         }
+#if CEED_VERSION_GE(0, 13, 0)
+         ierr = CeedElemRestrictionDestroy(&rstr); PCeedChk(ierr);
+#endif
          CeedEvalMode emode;
          ierr = CeedQFunctionFieldGetEvalMode(qffields[i], &emode);
          PCeedChk(ierr);
@@ -112,6 +127,9 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
                break; // Caught by QF Assembly
          }
       }
+#if CEED_VERSION_GE(0, 13, 0)
+      ierr = CeedVectorDestroy(&vec); PCeedChk(ierr);
+#endif
    }
 
    // Determine active output basis
@@ -127,11 +145,25 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
       ierr = CeedOperatorFieldGetVector(output_fields[i], &vec); PCeedChk(ierr);
       if (vec == CEED_VECTOR_ACTIVE)
       {
-         ierr = CeedOperatorFieldGetBasis(output_fields[i], &basisout);
-         PCeedChk(ierr);
-         ierr = CeedOperatorFieldGetElemRestriction(output_fields[i], &rstrout);
-         PCeedChk(ierr);
+         CeedBasis basis;
+         ierr = CeedOperatorFieldGetBasis(output_fields[i], &basis); PCeedChk(ierr);
+         if (!basisout)
+         {
+            ierr = CeedBasisReferenceCopy(basis, &basisout); PCeedChk(ierr);
+         }
+#if CEED_VERSION_GE(0, 13, 0)
+         ierr = CeedBasisDestroy(&basis); PCeedChk(ierr);
+#endif
+         CeedElemRestriction rstr;
+         ierr = CeedOperatorFieldGetElemRestriction(output_fields[i], &rstr);
          PCeedChk(ierr);
+         if (!rstrout)
+         {
+            ierr = CeedElemRestrictionReferenceCopy(rstr, &rstrout); PCeedChk(ierr);
+         }
+#if CEED_VERSION_GE(0, 13, 0)
+         ierr = CeedElemRestrictionDestroy(&rstr); PCeedChk(ierr);
+#endif
          CeedEvalMode emode;
          ierr = CeedQFunctionFieldGetEvalMode(qffields[i], &emode);
          PCeedChk(ierr);
@@ -157,6 +189,9 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
                break; // Caught by QF Assembly
          }
       }
+#if CEED_VERSION_GE(0, 13, 0)
+      ierr = CeedVectorDestroy(&vec); PCeedChk(ierr);
+#endif
    }
 
    CeedInt nelem, elemsize, nqpts;
@@ -200,7 +235,11 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
    PCeedChk(ierr);
 
    CeedInt layout[3];
+#if CEED_VERSION_GE(0, 13, 0)
+   ierr = CeedElemRestrictionGetELayout(rstr_q, layout); PCeedChk(ierr);
+#else
    ierr = CeedElemRestrictionGetELayout(rstr_q, &layout); PCeedChk(ierr);
+#endif
    ierr = CeedElemRestrictionDestroy(&rstr_q); PCeedChk(ierr);
 
    // enforce structurally symmetric for later elimination
@@ -285,6 +324,10 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
    ierr = CeedVectorRestoreArrayRead(assembledqf, &assembledqfarray);
    PCeedChk(ierr);
    ierr = CeedVectorDestroy(&assembledqf); PCeedChk(ierr);
+   ierr = CeedElemRestrictionDestroy(&rstrin); PCeedChk(ierr);
+   ierr = CeedElemRestrictionDestroy(&rstrout); PCeedChk(ierr);
+   ierr = CeedBasisDestroy(&basisin); PCeedChk(ierr);
+   ierr = CeedBasisDestroy(&basisout); PCeedChk(ierr);
    ierr = CeedHackFree(&emodein); PCeedChk(ierr);
    ierr = CeedHackFree(&emodeout); PCeedChk(ierr);
 
@@ -310,13 +353,8 @@ int CeedOperatorFullAssemble(CeedOperator op, SparseMatrix **mat)
    {
       CeedInt numsub;
       CeedOperator *subops;
-#if CEED_VERSION_GE(0, 10, 2)
-      CeedCompositeOperatorGetNumSub(op, &numsub);
-      ierr = CeedCompositeOperatorGetSubList(op, &subops); PCeedChk(ierr);
-#else
-      CeedOperatorGetNumSub(op, &numsub);
-      ierr = CeedOperatorGetSubList(op, &subops); PCeedChk(ierr);
-#endif
+      ierr = CeedOperatorCompositeGetNumSub(op, &numsub); PCeedChk(ierr);
+      ierr = CeedOperatorCompositeGetSubList(op, &subops); PCeedChk(ierr);
       for (int i = 0; i < numsub; ++i)
       {
          ierr = CeedSingleOperatorFullAssemble(subops[i], out); PCeedChk(ierr);

fem/ceed/solvers/solvers-atpmg.cpp

@@ -120,7 +120,11 @@ int CeedATPMGElemRestriction(int order,
    }
    ierr = CeedVectorRestoreArray(in_lvec, &lvec_data); PCeedChk(ierr);
    CeedInt in_layout[3];
+#if CEED_VERSION_GE(0, 13, 0)
+   ierr = CeedElemRestrictionGetELayout(er_in, in_layout); PCeedChk(ierr);
+#else
    ierr = CeedElemRestrictionGetELayout(er_in, &in_layout); PCeedChk(ierr);
+#endif
    if (in_layout[0] == 0 && in_layout[1] == 0 && in_layout[2] == 0)
    {
       return CeedError(ceed, 1, "Cannot interpret e-vector ordering of given"
@@ -664,7 +668,11 @@ int CeedATPMGOperator(CeedOperator oper, int order_reduction,
 
    for (int i = 0; i < numinputfields; ++i)
    {
+#if CEED_VERSION_GE(0, 13, 0)
+      const char * fieldname;
+#else
       char * fieldname;
+#endif
       ierr = CeedQFunctionFieldGetName(inputqfields[i], &fieldname); PCeedChk(ierr);
       if (if_vector[i] == CEED_VECTOR_ACTIVE)
       {
@@ -676,10 +684,19 @@ int CeedATPMGOperator(CeedOperator oper, int order_reduction,
          ierr = CeedOperatorSetField(coper, fieldname, er_input[i], basis_input[i],
                                      if_vector[i]); PCeedChk(ierr);
       }
+#if CEED_VERSION_GE(0, 13, 0)
+      ierr = CeedVectorDestroy(&if_vector[i]); PCeedChk(ierr);
+      ierr = CeedElemRestrictionDestroy(&er_input[i]); PCeedChk(ierr);
+      ierr = CeedBasisDestroy(&basis_input[i]); PCeedChk(ierr);
+#endif
    }
    for (int i = 0; i < numoutputfields; ++i)
    {
+#if CEED_VERSION_GE(0, 13, 0)
+      const char * fieldname;
+#else
       char * fieldname;
+#endif
       ierr = CeedQFunctionFieldGetName(outputqfields[i], &fieldname); PCeedChk(ierr);
       if (of_vector[i] == CEED_VECTOR_ACTIVE)
       {
@@ -691,6 +708,11 @@ int CeedATPMGOperator(CeedOperator oper, int order_reduction,
          ierr = CeedOperatorSetField(coper, fieldname, er_output[i], basis_output[i],
                                      of_vector[i]); PCeedChk(ierr);
       }
+#if CEED_VERSION_GE(0, 13, 0)
+      ierr = CeedVectorDestroy(&of_vector[i]); PCeedChk(ierr);
+      ierr = CeedElemRestrictionDestroy(&er_output[i]); PCeedChk(ierr);
+      ierr = CeedBasisDestroy(&basis_output[i]); PCeedChk(ierr);
+#endif
    }
    delete [] er_input;
    delete [] er_output;
@@ -741,7 +763,9 @@ int CeedOperatorGetOrder(CeedOperator oper, CeedInt * order)
    int P1d;
    ierr = CeedBasisGetNumNodes1D(basis, &P1d); PCeedChk(ierr);
    *order = P1d - 1;
-
+#if CEED_VERSION_GE(0, 13, 0)
+   ierr = CeedBasisDestroy(&basis); PCeedChk(ierr);
+#endif
    return 0;
 }
 

fem/coefficient.cpp

@@ -12,6 +12,7 @@
 // Implementation of Coefficient class
 
 #include "fem.hpp"
+#include "../general/forall.hpp"
 
 #include <cmath>
 #include <limits>
@@ -80,6 +81,49 @@ real_t PWConstCoefficient::Eval(ElementTransformation & T,
    return (constants(att-1));
 }
 
+void PWConstCoefficient::Project(QuadratureFunction &qf)
+{
+   auto &qs = *qf.GetSpace();
+
+   const bool compressed =
+      qs.Offsets(QSpaceOffsetStorage::COMPRESSED).Size() == 1;
+   const int *offsets = qs.Offsets(QSpaceOffsetStorage::COMPRESSED).Read();
+   const int ne = qs.GetNE();
+
+   const int *attributes = [&]()
+   {
+      if (dynamic_cast<QuadratureSpace*>(&qs) != nullptr)
+      {
+         return qs.GetMesh()->GetElementAttributes().Read();
+      }
+      else if (auto *qs_f = dynamic_cast<FaceQuadratureSpace*>(&qs))
+      {
+         MFEM_VERIFY(qs_f->GetFaceType() == FaceType::Boundary,
+                     "Interior faces do not have attributes.");
+         return qs.GetMesh()->GetBdrFaceAttributes().Read();
+      }
+      else
+      {
+         MFEM_ABORT("Unsupported case.");
+      }
+   }();
+
+   const real_t *d_c = constants.Read();
+   real_t *d_qf = qf.Write();
+
+   mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int a = attributes[e];
+      const real_t elementConstant = d_c[a - 1];
+      const int begin = compressed ? e*offsets[0] : offsets[e];
+      const int end = compressed ? (e+1)*offsets[0] : offsets[e+1];
+      for (int i = begin; i < end; ++i)
+      {
+         d_qf[i] = elementConstant;
+      }
+   });
+}
+
 void PWCoefficient::InitMap(const Array<int> & attr,
                             const Array<Coefficient*> & coefs)
 {
@@ -519,6 +563,26 @@ void GradientGridFunctionCoefficient::Eval(
    }
 }
 
+void GradientGridFunctionCoefficient::Project(QuadratureFunction &qf)
+{
+   const FiniteElementSpace &fes = *GridFunc->FESpace();
+   const Mesh &mesh = *fes.GetMesh();
+   const int sdim = mesh.SpaceDimension();
+   const int gf_vdim = fes.GetVDim(); // assumed to be 1 in this class
+   qf.SetVDim(sdim*gf_vdim);
+   if (mesh.GetNE() == 0) { return; }
+   // All mesh element must be the same type:
+   MFEM_VERIFY(mesh.GetNumGeometries(mesh.Dimension()) == 1,
+               "All mesh elements must be the same type!");
+   const IntegrationRule &ir = qf.GetIntRule(0);
+   // All elements must use the same quadrature rule:
+   MFEM_VERIFY(qf.Size() == sdim*gf_vdim*ir.GetNPoints()*mesh.GetNE(),
+               "All mesh elements must use the same quadrature rule!");
+   // QuadratureFunction uses the layout qf_vdim x nq x ne, i.e.
+   // gf_vdim x sdim x nq x nq, so we need to request QVectorLayout::byVDIM:
+   GridFunc->GetGradients(ir, qf, QVectorLayout::byVDIM);
+}
+
 CurlGridFunctionCoefficient::CurlGridFunctionCoefficient(
    const GridFunction *gf)
    : VectorCoefficient(0)
@@ -1021,6 +1085,29 @@ void SumCoefficient::SetTime(real_t t)
    this->Coefficient::SetTime(t);
 }
 
+void SumCoefficient::Project(QuadratureFunction &qf)
+{
+   if (a == nullptr)
+   {
+      // qf = alpha*aConst + beta * b
+      const real_t d_alpha_a = aConst*alpha;
+      const real_t d_beta = beta;
+      b->Project(qf);
+      auto d_qf = qf.ReadWrite();
+      mfem::forall(qf.Size(), [=] MFEM_HOST_DEVICE (int i)
+      {
+         d_qf[i] = d_alpha_a + d_beta*d_qf[i];
+      });
+   }
+   else
+   {
+      a->Project(qf);
+      QuadratureFunction qf_b(*qf.GetSpace());
+      b->Project(qf_b);
+      add(alpha, qf, beta, qf_b, qf);
+   }
+}
+
 void ProductCoefficient::SetTime(real_t t)
 {
    if (a) { a->SetTime(t); }
@@ -1028,6 +1115,23 @@ void ProductCoefficient::SetTime(real_t t)
    this->Coefficient::SetTime(t);
 }
 
+void ProductCoefficient::Project(QuadratureFunction &qf)
+{
+   if (a == nullptr)
+   {
+      // qf = aConst * b
+      b->Project(qf);
+      qf *= aConst;
+   }
+   else
+   {
+      a->Project(qf);
+      QuadratureFunction qf_b(qf.GetSpace());
+      b->Project(qf_b);
+      qf *= qf_b;
+   }
+}
+
 void RatioCoefficient::SetTime(real_t t)
 {
    if (a) { a->SetTime(t); }
@@ -1035,6 +1139,38 @@ void RatioCoefficient::SetTime(real_t t)
    this->Coefficient::SetTime(t);
 }
 
+void RatioCoefficient::Project(QuadratureFunction &qf)
+{
+   if (b == nullptr)
+   {
+      if (a == nullptr)
+      {
+         qf = aConst / bConst;
+      }
+      else
+      {
+         a->Project(qf);
+         qf *= 1.0/bConst;
+      }
+   }
+   else
+   {
+      if (a == nullptr)
+      {
+         b->Project(qf);
+         qf.Reciprocal();
+         qf *= aConst;
+      }
+      else
+      {
+         a->Project(qf);
+         QuadratureFunction qf_b(qf.GetSpace());
+         b->Project(qf_b);
+         qf /= qf_b;
+      }
+   }
+}
+
 void PowerCoefficient::SetTime(real_t t)
 {
    if (a) { a->SetTime(t); }
@@ -1065,6 +1201,41 @@ real_t InnerProductCoefficient::Eval(ElementTransformation &T,
    return va * vb;
 }
 
+void InnerProductCoefficient::Project(QuadratureFunction &qf)
+{
+   MFEM_VERIFY(a->GetVDim() == b->GetVDim(),
+               "Incompatible vector coefficients: a->GetVDim(): "
+               << a->GetVDim() << ", b->GetVDim(): " << b->GetVDim());
+
+   const int vdim = a->GetVDim();
+   MFEM_VERIFY(vdim >= 1, "invalid vdim: " << vdim);
+
+   // When running on device, make sure the output data is allocated before any
+   // local temporary data to reduce potential heap fragmentation:
+   auto dot_d = qf.Write();
+
+   QuadratureFunction qf_a(qf.GetSpace(), vdim);
+   QuadratureFunction qf_b(qf.GetSpace(), vdim);
+
+   a->Project(qf_a);
+   b->Project(qf_b);
+
+   auto a_d = qf_a.Read();
+   auto b_d = qf_b.Read();
+
+   mfem::forall(qf.GetSpace()->GetSize(), [=] MFEM_HOST_DEVICE (int i)
+   {
+      const real_t *ai = a_d + i*vdim;
+      const real_t *bi = b_d + i*vdim;
+      real_t dot = ai[0]*bi[0];
+      for (int d = 1; d < vdim; d++)
+      {
+         dot += ai[d]*bi[d];
+      }
+      dot_d[i] = dot;
+   });
+}
+
 VectorRotProductCoefficient::VectorRotProductCoefficient(VectorCoefficient &A,
                                                          VectorCoefficient &B)
    : a(&A), b(&B), va(A.GetVDim()), vb(B.GetVDim())

fem/coefficient.hpp

@@ -132,6 +132,9 @@ public:
    /// Evaluate the coefficient.
    real_t Eval(ElementTransformation &T,
                const IntegrationPoint &ip) override;
+
+   /// Fill the QuadratureFunction @a qf with the piecewise constant values.
+   void Project(QuadratureFunction &qf) override;
 };
 
 /** @brief A piecewise coefficient with the pieces keyed off the element
@@ -894,6 +897,9 @@ public:
    void Eval(DenseMatrix &M, ElementTransformation &T,
              const IntegrationRule &ir) override;
 
+   /// @copydoc VectorCoefficient::Project(QuadratureFunction &)
+   void Project(QuadratureFunction &qf) override;
+
    virtual ~GradientGridFunctionCoefficient() { }
 };
 
@@ -1450,6 +1456,9 @@ public:
    /// Set the time for internally stored coefficients
    void SetTime(real_t t) override;
 
+   /// @copydoc Coefficient::Project(QuadratureFunction &)
+   void Project(QuadratureFunction &qf) override;
+
    /// Reset the first term in the linear combination as a constant
    void SetAConst(real_t A) { a = NULL; aConst = A; }
    /// Return the first term in the linear combination
@@ -1631,6 +1640,9 @@ public:
    /// Set the time for internally stored coefficients
    void SetTime(real_t t) override;
 
+   /// @copydoc Coefficient::Project(QuadratureFunction &)
+   void Project(QuadratureFunction &qf) override;
+
    /// Reset the first term in the product as a constant
    void SetAConst(real_t A) { a = NULL; aConst = A; }
    /// Return the first term in the product
@@ -1679,6 +1691,9 @@ public:
    /// Set the time for internally stored coefficients
    void SetTime(real_t t) override;
 
+   /// @copydoc Coefficient::Project(QuadratureFunction &)
+   void Project(QuadratureFunction &qf) override;
+
    /// Reset the numerator in the ratio as a constant
    void SetAConst(real_t A) { a = NULL; aConst = A; }
    /// Return the numerator of the ratio
@@ -1771,6 +1786,9 @@ public:
    /// Evaluate the coefficient at @a ip.
    real_t Eval(ElementTransformation &T,
                const IntegrationPoint &ip) override;
+
+   /// @copydoc Coefficient::Project(QuadratureFunction &)
+   void Project(QuadratureFunction &qf) override;
 };
 
 /// Scalar coefficient defined as a cross product of two vectors in the xy-plane.

fem/complex_fem.cpp

@@ -11,14 +11,15 @@
 
 #include "complex_fem.hpp"
 #include "../general/forall.hpp"
+#include "../general/text.hpp"
 
 using namespace std;
 
 namespace mfem
 {
 
-ComplexGridFunction::ComplexGridFunction(FiniteElementSpace *fes)
-   : Vector(2*(fes->GetVSize()))
+ComplexGridFunction::ComplexGridFunction(FiniteElementSpace *f)
+   : Vector(2*(f->GetVSize())), fes(f), fec_owned(NULL)
 {
    UseDevice(true);
    this->Vector::operator=(0.0);
@@ -28,12 +29,88 @@ ComplexGridFunction::ComplexGridFunction(FiniteElementSpace *fes)
 
    gfi = new GridFunction();
    gfi->MakeRef(fes, *this, fes->GetVSize());
+
+   fes_sequence = fes->GetSequence();
+}
+
+ComplexGridFunction::ComplexGridFunction(Mesh *m, std::istream &input)
+   : Vector(), fes(NULL), fec_owned(NULL)
+{
+   string buff;
+
+   // Grid functions are stored on the device
+   UseDevice(true);
+
+   input >> std::ws;
+   getline(input, buff);  // 'ComplexGridFunction'
+   filter_dos(buff);
+   if (buff != "ComplexGridFunction")
+   {
+      MFEM_ABORT("unrecognized file header: " << buff);
+   }
+
+   fes = new FiniteElementSpace;
+   fec_owned = fes->Load(m, input);
+
+   skip_comment_lines(input, '#');
+   istream::int_type next_char = input.peek();
+   if (next_char == 'N') // First letter of "NURBS_patches"
+   {
+      getline(input, buff);
+      filter_dos(buff);
+      if (buff == "NURBS_patches")
+      {
+         MFEM_ABORT("NURBS not yet supported with ComplexGridFunction objects");
+      }
+      else
+      {
+         MFEM_ABORT("unknown section: " << buff);
+      }
+   }
+   else
+   {
+      Vector::Load(input, 2*fes->GetVSize());
+
+      // if the mesh is a legacy (v1.1) NC mesh, it has old vertex ordering
+      if (fes->Nonconforming() &&
+          fes->GetMesh()->ncmesh->IsLegacyLoaded())
+      {
+         // LegacyNCReorder();
+         MFEM_ABORT("LegacyNCReorder not supported for "
+                    "ComplexGridFunction objects");
+      }
+   }
+
+   gfr = new GridFunction();
+   gfr->MakeRef(fes, *this, 0);
+
+   gfi = new GridFunction();
+   gfi->MakeRef(fes, *this, fes->GetVSize());
+
+   fes_sequence = fes->GetSequence();
+}
+
+void ComplexGridFunction::Destroy()
+{
+   delete gfr; delete gfi;
+
+   if (fec_owned)
+   {
+      delete fes;
+      delete fec_owned;
+      fec_owned = NULL;
+   }
 }
 
 void
 ComplexGridFunction::Update()
 {
-   FiniteElementSpace *fes = gfr->FESpace();
+   if (fes->GetSequence() == fes_sequence)
+   {
+      return; // space and grid function are in sync, no-op
+   }
+   fes_sequence = fes->GetSequence();
+
    const int vsize = fes->GetVSize();
 
    const Operator *T = fes->GetUpdateOperator();
@@ -84,6 +161,17 @@ ComplexGridFunction::Update()
    }
 }
 
+int ComplexGridFunction::VectorDim() const
+{
+   const FiniteElement *fe = fes->GetTypicalFE();
+   if (!fe || fe->GetRangeType() == FiniteElement::SCALAR)
+   {
+      return fes->GetVDim();
+   }
+   return fes->GetVDim()*std::max(fes->GetMesh()->SpaceDimension(),
+                                  fe->GetRangeDim());
+}
+
 void
 ComplexGridFunction::ProjectCoefficient(Coefficient &real_coeff,
                                         Coefficient &imag_coeff)
@@ -149,6 +237,35 @@ ComplexGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient
    gfi->SyncAliasMemory(*this);
 }
 
+void ComplexGridFunction::Save(std::ostream &os) const
+{
+   os << "ComplexGridFunction\n";
+   fes->Save(os);
+   os << '\n';
+   if (fes->GetOrdering() == Ordering::byNODES)
+   {
+      Vector::Print(os, 1);
+   }
+   else
+   {
+      Vector::Print(os, fes->GetVDim());
+   }
+   os.flush();
+}
+
+void ComplexGridFunction::Save(const char *fname, int precision) const
+{
+   ofstream ofs(fname);
+   ofs.precision(precision);
+   Save(ofs);
+}
+
+std::ostream &operator<<(std::ostream &os, const ComplexGridFunction &sol)
+{
+   sol.Save(os);
+   return os;
+}
+
 
 ComplexLinearForm::ComplexLinearForm(FiniteElementSpace *fes,
                                      ComplexOperator::Convention convention)
@@ -654,8 +771,8 @@ SesquilinearForm::Update(FiniteElementSpace *nfes)
 
 #ifdef MFEM_USE_MPI
 
-ParComplexGridFunction::ParComplexGridFunction(ParFiniteElementSpace *pfes)
-   : Vector(2*(pfes->GetVSize()))
+ParComplexGridFunction::ParComplexGridFunction(ParFiniteElementSpace *pf)
+   : Vector(2*(pf->GetVSize())), pfes(pf), fec_owned(NULL)
 {
    UseDevice(true);
    this->Vector::operator=(0.0);
@@ -665,12 +782,105 @@ ParComplexGridFunction::ParComplexGridFunction(ParFiniteElementSpace *pfes)
 
    pgfi = new ParGridFunction();
    pgfi->MakeRef(pfes, *this, pfes->GetVSize());
+
+   fes_sequence = pfes->GetSequence();
+}
+
+ParComplexGridFunction::ParComplexGridFunction(ParMesh *m, std::istream &input)
+   : Vector(), pfes(NULL), fec_owned(NULL)
+{
+   string buff;
+
+   // Grid functions are stored on the device
+   UseDevice(true);
+
+   input >> std::ws;
+   getline(input, buff);  // 'ParComplexGridFunction'
+   filter_dos(buff);
+   if (buff != "ParComplexGridFunction")
+   {
+      MFEM_ABORT("unrecognized file header: " << buff);
+   }
+
+   FiniteElementSpace *fes = new FiniteElementSpace;
+   fec_owned = fes->Load(m, input);
+
+   pfes = new ParFiniteElementSpace(m, fec_owned, fes->GetVDim(),
+                                    fes->GetOrdering());
+
+   delete fes;
+
+   skip_comment_lines(input, '#');
+   istream::int_type next_char = input.peek();
+   if (next_char == 'N') // First letter of "NURBS_patches"
+   {
+      getline(input, buff);
+      filter_dos(buff);
+      if (buff == "NURBS_patches")
+      {
+         MFEM_ABORT("NURBS not yet supported with ComplexGridFunction objects");
+      }
+      else
+      {
+         MFEM_ABORT("unknown section: " << buff);
+      }
+   }
+   else
+   {
+      int vsize = pfes->GetVSize();
+      Vector::Load(input, 2*vsize);
+
+      real_t *data_  = const_cast<real_t*>(HostRead());
+      for (int i = 0; i < vsize; i++)
+      {
+         if (pfes->GetDofSign(i) < 0)
+         {
+            data_[i] = -data_[i];
+            data_[i+vsize] = -data_[i+vsize];
+         }
+      }
+
+
+      // if the mesh is a legacy (v1.1) NC mesh, it has old vertex ordering
+      if (pfes->Nonconforming() &&
+          pfes->GetMesh()->ncmesh->IsLegacyLoaded())
+      {
+         // LegacyNCReorder();
+         MFEM_ABORT("LegacyNCReorder not supported for "
+                    "ComplexGridFunction objects");
+      }
+   }
+
+   pgfr = new ParGridFunction();
+   pgfr->MakeRef(pfes, *this, 0);
+
+   pgfi = new ParGridFunction();
+   pgfi->MakeRef(pfes, *this, pfes->GetVSize());
+
+   fes_sequence = pfes->GetSequence();
+}
+
+void ParComplexGridFunction::Destroy()
+{
+   delete pgfr; delete pgfi;
+
+   if (fec_owned)
+   {
+      delete pfes;
+      delete fec_owned;
+      fec_owned = NULL;
+   }
 }
 
 void
 ParComplexGridFunction::Update()
 {
-   ParFiniteElementSpace *pfes = pgfr->ParFESpace();
+   if (pfes->GetSequence() == fes_sequence)
+   {
+      return; // space and grid function are in sync, no-op
+   }
+   fes_sequence = pfes->GetSequence();
+
    const int vsize = pfes->GetVSize();
 
    const Operator *T = pfes->GetUpdateOperator();
@@ -719,6 +929,17 @@ ParComplexGridFunction::Update()
    }
 }
 
+int ParComplexGridFunction::VectorDim() const
+{
+   const FiniteElement *fe = pfes->GetTypicalFE();
+   if (!fe || fe->GetRangeType() == FiniteElement::SCALAR)
+   {
+      return pfes->GetVDim();
+   }
+   return pfes->GetVDim()*std::max(pfes->GetMesh()->SpaceDimension(),
+                                   fe->GetRangeDim());
+}
+
 void
 ParComplexGridFunction::ProjectCoefficient(Coefficient &real_coeff,
                                            Coefficient &imag_coeff)
@@ -789,7 +1010,6 @@ ParComplexGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient
 void
 ParComplexGridFunction::Distribute(const Vector *tv)
 {
-   ParFiniteElementSpace *pfes = pgfr->ParFESpace();
    const int tvsize = pfes->GetTrueVSize();
 
    tv->Read();
@@ -807,7 +1027,6 @@ ParComplexGridFunction::Distribute(const Vector *tv)
 void
 ParComplexGridFunction::ParallelProject(Vector &tv) const
 {
-   ParFiniteElementSpace *pfes = pgfr->ParFESpace();
    const int tvsize = pfes->GetTrueVSize();
 
    tv.Write();
@@ -825,6 +1044,60 @@ ParComplexGridFunction::ParallelProject(Vector &tv) const
    tvi.SyncAliasMemory(tv);
 }
 
+void ParComplexGridFunction::Save(std::ostream &os) const
+{
+   os << "ParComplexGridFunction\n";
+   pfes->Save(os);
+   os << '\n';
+
+   int vsize = pfes->GetVSize();
+   real_t *data_  = const_cast<real_t*>(HostRead());
+   for (int i = 0; i < vsize; i++)
+   {
+      if (pfes->GetDofSign(i) < 0)
+      {
+         data_[i] = -data_[i];
+         data_[i+vsize] = -data_[i+vsize];
+      }
+   }
+
+   if (pfes->GetOrdering() == Ordering::byNODES)
+   {
+      Vector::Print(os, 1);
+   }
+   else
+   {
+      Vector::Print(os, pfes->GetVDim());
+   }
+
+   for (int i = 0; i < vsize; i++)
+   {
+      if (pfes->GetDofSign(i) < 0)
+      {
+         data_[i] = -data_[i];
+         data_[i+vsize] = -data_[i+vsize];
+      }
+   }
+
+   os.flush();
+}
+
+void ParComplexGridFunction::Save(const char *fname, int precision) const
+{
+   int rank = pfes->GetMyRank();
+   ostringstream fname_with_suffix;
+   fname_with_suffix << fname << "." << setfill('0') << setw(6) << rank;
+   ofstream ofs(fname_with_suffix.str().c_str());
+   ofs.precision(precision);
+   Save(ofs);
+}
+
+std::ostream &operator<<(std::ostream &os, const ParComplexGridFunction &sol)
+{
+   sol.Save(os);
+   return os;
+}
+
 
 ParComplexLinearForm::ParComplexLinearForm(ParFiniteElementSpace *pfes,
                                            ComplexOperator::Convention

fem/complex_fem.hpp

@@ -35,15 +35,53 @@ private:
    GridFunction * gfi;
 
 protected:
-   void Destroy() { delete gfr; delete gfi; }
+   /// FE space on which the grid function lives. Owned if #fec_owned
+   /// is not NULL.
+   FiniteElementSpace *fes;
+
+   /** @brief Used when the grid function is read from a file. It can also be
+       set explicitly, see MakeOwner().
+
+       If not NULL, this pointer is owned by the ComplexGridFunction. */
+   FiniteElementCollection *fec_owned;
+
+   long fes_sequence; // see FiniteElementSpace::sequence, Mesh::sequence
+
+   void Destroy();
 
 public:
    /** @brief Construct a ComplexGridFunction associated with the
        FiniteElementSpace @a *f. */
    ComplexGridFunction(FiniteElementSpace *f);
 
+   /** @brief Construct a ComplexGridFunction on the given Mesh, using the data
+       from @a input.
+
+       The content of @a input should be in the format created by the method
+       Save(). The reconstructed FiniteElementSpace and FiniteElementCollection
+       are owned by the ComplexGridFunction. */
+   ComplexGridFunction(Mesh *m, std::istream &input);
+
    void Update();
 
+   /** Return update counter, similar to Mesh::GetSequence(). Used to
+       check if it is up to date with the space. */
+   long GetSequence() const { return fes_sequence; }
+
+   /// Make the ComplexGridFunction the owner of #fec_owned and #fes.
+   /** If the new FiniteElementCollection, @a fec_, is NULL, ownership
+       of #fec_owned and #fes is taken away. */
+   void MakeOwner(FiniteElementCollection *fec_) { fec_owned = fec_; }
+
+   /// Returns a pointer to the FiniteElementCollection used to
+   /// construct this ComplexGridFunction if this class owns that
+   /// object. Otherwise this function will return NULL.
+   FiniteElementCollection *OwnFEC() { return fec_owned; }
+
+   /// Shortcut for calling FiniteElementSpace::GetVectorDim() on the
+   /// underlying #fes
+   int VectorDim() const;
+
    /// Assign constant values to the ComplexGridFunction data.
    ComplexGridFunction &operator=(const std::complex<real_t> & value)
    { *gfr = value.real(); *gfi = value.imag(); return *this; }
@@ -63,8 +101,8 @@ public:
                                              VectorCoefficient &imag_coeff,
                                              Array<int> &attr);
 
-   FiniteElementSpace *FESpace() { return gfr->FESpace(); }
-   const FiniteElementSpace *FESpace() const { return gfr->FESpace(); }
+   FiniteElementSpace *FESpace() { return fes; }
+   const FiniteElementSpace *FESpace() const { return fes; }
 
    GridFunction & real() { return *gfr; }
    GridFunction & imag() { return *gfi; }
@@ -79,11 +117,52 @@ public:
    /// @a gfr and @a gfi to match the ComplexGridFunction.
    void SyncAlias() { gfr->SyncAliasMemory(*this); gfi->SyncAliasMemory(*this); }
 
+   /// @brief Returns ||u_ex - u_h||_L2 for complex-valued scalar fields
+   ///
+   /// @see GridFunction::ComputeL2Error(Coefficient &exsol,
+   ///                                   const IntegrationRule *irs[],
+   ///                                   const Array<int> *elems) const
+   ///      for more detailed documentation.
+   virtual real_t ComputeL2Error(Coefficient &exsolr, Coefficient &exsoli,
+                                 const IntegrationRule *irs[] = NULL) const
+   {
+      real_t err_r = gfr->ComputeL2Error(exsolr, irs);
+      real_t err_i = gfi->ComputeL2Error(exsoli, irs);
+      return sqrt(err_r * err_r + err_i * err_i);
+   }
+
+   /// @brief Returns ||u_ex - u_h||_L2 for complex-valued vector fields
+   ///
+   /// @see GridFunction::ComputeL2Error(VectorCoefficient &exsol,
+   ///                                   const IntegrationRule *irs[],
+   ///                                   const Array<int> *elems) const
+   ///      for more detailed documentation.
+   virtual real_t ComputeL2Error(VectorCoefficient &exsolr,
+                                 VectorCoefficient &exsoli,
+                                 const IntegrationRule *irs[] = NULL,
+                                 Array<int> *elems = NULL) const
+   {
+      real_t err_r = gfr->ComputeL2Error(exsolr, irs, elems);
+      real_t err_i = gfi->ComputeL2Error(exsoli, irs, elems);
+      return sqrt(err_r * err_r + err_i * err_i);
+   }
+
+   /// Save the ComplexGridFunction to an output stream.
+   virtual void Save(std::ostream &out) const;
+
+   /// Save the ComplexGridFunction to a file
+   /** The given @a precision will be used for ASCII output. */
+   virtual void Save(const char *fname, int precision=16) const;
+
    /// Destroys the grid function.
    virtual ~ComplexGridFunction() { Destroy(); }
 
 };
 
+/** Overload operator<< for std::ostream and ComplexGridFunction; not valid
+    for the class ParComplexGridFunction */
+std::ostream &operator<<(std::ostream &out, const ComplexGridFunction &sol);
+
 /** Class for a complex-valued linear form
 
     The @a convention argument in the class's constructor is documented in the
@@ -345,12 +424,23 @@ public:
 class ParComplexGridFunction : public Vector
 {
 private:
-
    ParGridFunction * pgfr;
    ParGridFunction * pgfi;
 
 protected:
-   void Destroy() { delete pgfr; delete pgfi; }
+   /// FE space on which the grid function lives. Owned if #fec_owned
+   /// is not NULL.
+   ParFiniteElementSpace *pfes;
+
+   /** @brief Used when the grid function is read from a file. It can also be
+       set explicitly, see MakeOwner().
+
+       If not NULL, this pointer is owned by the ParComplexGridFunction. */
+   FiniteElementCollection *fec_owned;
+
+   long fes_sequence; // see FiniteElementSpace::sequence, Mesh::sequence
+
+   void Destroy();
 
 public:
 
@@ -358,8 +448,33 @@ public:
        ParFiniteElementSpace @a *pf. */
    ParComplexGridFunction(ParFiniteElementSpace *pf);
 
+   /** @brief Construct a ParComplexGridFunction on a given ParMesh,
+       @a pmesh, reading from an std::istream.
+
+       In the process, a ParFiniteElementSpace and a FiniteElementCollection are
+       constructed. The new ParComplexGridFunction assumes ownership of both. */
+   ParComplexGridFunction(ParMesh *pmesh, std::istream &input);
+
    void Update();
 
+   /** Return update counter, similar to Mesh::GetSequence(). Used to
+       check if it is up to date with the space. */
+   long GetSequence() const { return fes_sequence; }
+
+   /// Make the ParComplexGridFunction the owner of #fec_owned and #pfes.
+   /** If the new FiniteElementCollection, @a fec_, is NULL, ownership
+       of #fec_owned and #pfes is taken away. */
+   void MakeOwner(FiniteElementCollection *fec_) { fec_owned = fec_; }
+
+   /// Returns a pointer to the FiniteElementCollection used to
+   /// construct this ParComplexGridFunction if this class owns that
+   /// object. Otherwise this function will return NULL.
+   FiniteElementCollection *OwnFEC() { return fec_owned; }
+
+   /// Shortcut for calling FiniteElementSpace::GetVectorDim() on the
+   /// underlying #pfes
+   int VectorDim() const;
+
    /// Assign constant values to the ParComplexGridFunction data.
    ParComplexGridFunction &operator=(const std::complex<real_t> & value)
    { *pgfr = value.real(); *pgfi = value.imag(); return *this; }
@@ -385,11 +500,11 @@ public:
    /// Returns the vector restricted to the true dofs.
    void ParallelProject(Vector &tv) const;
 
-   FiniteElementSpace *FESpace() { return pgfr->FESpace(); }
-   const FiniteElementSpace *FESpace() const { return pgfr->FESpace(); }
+   FiniteElementSpace *FESpace() { return pfes; }
+   const FiniteElementSpace *FESpace() const { return pfes; }
 
-   ParFiniteElementSpace *ParFESpace() { return pgfr->ParFESpace(); }
-   const ParFiniteElementSpace *ParFESpace() const { return pgfr->ParFESpace(); }
+   ParFiniteElementSpace *ParFESpace() { return pfes; }
+   const ParFiniteElementSpace *ParFESpace() const { return pfes; }
 
    ParGridFunction & real() { return *pgfr; }
    ParGridFunction & imag() { return *pgfi; }
@@ -402,17 +517,32 @@ public:
 
    /// Update the alias memory location of the real and imaginary
    /// ParGridFunction @a pgfr and @a pgfi to match the ParComplexGridFunction.
-   void SyncAlias() { pgfr->SyncAliasMemory(*this); pgfi->SyncAliasMemory(*this); }
-
+   void SyncAlias()
+   { pgfr->SyncAliasMemory(*this); pgfi->SyncAliasMemory(*this); }
 
+   /// @brief Returns ||u_ex - u_h||_L2 in parallel for complex-valued
+   ///        scalar fields
+   ///
+   /// @see GridFunction::ComputeL2Error(Coefficient &exsol,
+   ///                                   const IntegrationRule *irs[],
+   ///                                   const Array<int> *elems) const
+   ///      for more detailed documentation.
    virtual real_t ComputeL2Error(Coefficient &exsolr, Coefficient &exsoli,
-                                 const IntegrationRule *irs[] = NULL) const
+                                 const IntegrationRule *irs[] = NULL,
+                                 Array<int> *elems = NULL) const
    {
-      real_t err_r = pgfr->ComputeL2Error(exsolr, irs);
-      real_t err_i = pgfi->ComputeL2Error(exsoli, irs);
-      return sqrt(err_r * err_r + err_i * err_i);
+      real_t err_r = pgfr->ComputeL2Error(exsolr, irs, elems);
+      real_t err_i = pgfi->ComputeL2Error(exsoli, irs, elems);
+      return hypot(err_r, err_i);
    }
 
+   /// @brief Returns ||u_ex - u_h||_L2 in parallel for complex-valued
+   ///        vector fields
+   ///
+   /// @see GridFunction::ComputeL2Error(VectorCoefficient &exsol,
+   ///                                   const IntegrationRule *irs[],
+   ///                                   const Array<int> *elems) const
+   ///      for more detailed documentation.
    virtual real_t ComputeL2Error(VectorCoefficient &exsolr,
                                  VectorCoefficient &exsoli,
                                  const IntegrationRule *irs[] = NULL,
@@ -420,15 +550,28 @@ public:
    {
       real_t err_r = pgfr->ComputeL2Error(exsolr, irs, elems);
       real_t err_i = pgfi->ComputeL2Error(exsoli, irs, elems);
-      return sqrt(err_r * err_r + err_i * err_i);
+      return hypot(err_r, err_i);
    }
 
+   /// Save the local portion of the ParComplexGridFunction
+   /** This differs from the serial ComplexGridFunction::Save in that it
+       takes into account the signs of the local dofs. */
+   void Save(std::ostream &out) const;
+
+   /// Save the ParComplexGridFunction to files
+   /** Saves one file for each MPI rank. The files will be given suffixes
+       according to the MPI rank. The given @a precision will be used for ASCII
+       output. */
+   void Save(const char *fname, int precision=16) const;
 
    /// Destroys grid function.
    virtual ~ParComplexGridFunction() { Destroy(); }
 
 };
 
+/** Overload operator<< for std::ostream and ParComplexGridFunction */
+std::ostream &operator<<(std::ostream &out, const ParComplexGridFunction &sol);
+
 /** Class for a complex-valued, parallel linear form
 
     The @a convention argument in the class's constructor is documented in the

fem/datacollection.cpp

@@ -310,9 +310,9 @@ void DataCollection::SaveField(const std::string &field_name)
    }
 }
 
-void DataCollection::SaveQField(const std::string &q_field_name)
+void DataCollection::SaveQField(const std::string &field_name)
 {
-   QFieldMapIterator it = q_field_map.find(q_field_name);
+   QFieldMapIterator it = q_field_map.find(field_name);
    if (it != q_field_map.end())
    {
       SaveOneQField(it);
@@ -780,6 +780,11 @@ void ParaViewDataCollectionBase::SetHighOrderOutput(bool high_order_output_)
    high_order_output = high_order_output_;
 }
 
+void ParaViewDataCollectionBase::SetBoundaryOutput(bool bdr_output_)
+{
+   bdr_output = bdr_output_;
+}
+
 void ParaViewDataCollectionBase::SetCompressionLevel(int compression_level_)
 {
    MFEM_ASSERT(compression_level_ >= -1 && compression_level_ <= 9,
@@ -935,16 +940,19 @@ void ParaViewDataCollection::Save()
    std::string vtu_prefix = col_path + "/" + GenerateVTUPath() + "/";
 
    // Save the local part of the mesh and grid functions fields to the local
-   // VTU file
+   // VTU file. Also save coefficient fields.
    {
       std::ofstream os(vtu_prefix + GenerateVTUFileName("proc", myid));
       os.precision(precision);
       SaveDataVTU(os, levels_of_detail);
    }
 
-   // Save the local part of the quadrature function fields
+   // Save the local part of the quadrature function fields.
    for (const auto &qfield : q_field_map)
    {
+      MFEM_VERIFY(!bdr_output,
+                  "QuadratureFunction output is not supported for "
+                  "ParaViewDataCollection on domain boundary!");
       const std::string &field_name = qfield.first;
       std::ofstream os(vtu_prefix + GenerateVTUFileName(field_name, myid));
       qfield.second->SaveVTU(os, pv_data_format, GetCompressionLevel(), field_name);
@@ -960,7 +968,7 @@ void ParaViewDataCollection::Save()
          std::ofstream pvtu_out(vtu_prefix + GeneratePVTUFileName("data"));
          WritePVTUHeader(pvtu_out);
 
-         // Grid function fields
+         // Grid function fields and coefficient fields
          pvtu_out << "<PPointData>\n";
          for (auto &field_it : field_map)
          {
@@ -971,7 +979,24 @@ void ParaViewDataCollection::Save()
                      << VTKComponentLabels(vec_dim) << " "
                      << "format=\"" << GetDataFormatString() << "\" />\n";
          }
+         for (auto &field_it : coeff_field_map)
+         {
+            int vec_dim = 1;
+            pvtu_out << "<PDataArray type=\"" << GetDataTypeString()
+                     << "\" Name=\"" << field_it.first
+                     << "\" NumberOfComponents=\"" << vec_dim << "\" "
+                     << "format=\"" << GetDataFormatString() << "\" />\n";
+         }
+         for (auto &field_it : vcoeff_field_map)
+         {
+            int vec_dim = field_it.second->GetVDim();
+            pvtu_out << "<PDataArray type=\"" << GetDataTypeString()
+                     << "\" Name=\"" << field_it.first
+                     << "\" NumberOfComponents=\"" << vec_dim << "\" "
+                     << "format=\"" << GetDataFormatString() << "\" />\n";
+         }
          pvtu_out << "</PPointData>\n";
+
          // Element attributes
          pvtu_out << "<PCellData>\n";
          pvtu_out << "\t<PDataArray type=\"Int32\" Name=\"" << "attribute"
@@ -1069,7 +1094,8 @@ void ParaViewDataCollection::SaveDataVTU(std::ostream &os, int ref)
    }
    os << " version=\"2.2\" byte_order=\"" << VTKByteOrder() << "\">\n";
    os << "<UnstructuredGrid>\n";
-   mesh->PrintVTU(os,ref,pv_data_format,high_order_output,GetCompressionLevel());
+   mesh->PrintVTU(os,ref,pv_data_format,high_order_output,GetCompressionLevel(),
+                  bdr_output);
 
    // dump out the grid functions as point data
    os << "<PointData >\n";
@@ -1077,8 +1103,21 @@ void ParaViewDataCollection::SaveDataVTU(std::ostream &os, int ref)
    // iterate over all grid functions
    for (FieldMapIterator it=field_map.begin(); it!=field_map.end(); ++it)
    {
+      MFEM_VERIFY(!bdr_output,
+                  "GridFunction output is not supported for "
+                  "ParaViewDataCollection on domain boundary!");
       SaveGFieldVTU(os,ref,it);
    }
+   // save the coefficient functions
+   // iterate over all Coefficient and VectorCoefficient functions
+   for (const auto &kv : coeff_field_map)
+   {
+      SaveCoeffFieldVTU(os, ref, kv.first, *kv.second);
+   }
+   for (const auto &kv : vcoeff_field_map)
+   {
+      SaveVCoeffFieldVTU(os, ref, kv.first, *kv.second);
+   }
    os << "</PointData>\n";
    // close the mesh
    os << "</Piece>\n"; // close the piece open in the PrintVTU method
@@ -1101,7 +1140,6 @@ void ParaViewDataCollection::SaveGFieldVTU(std::ostream &os, int ref_,
       << "format=\"" << GetDataFormatString() << "\" >" << '\n';
    if (vec_dim == 1)
    {
-      // scalar data
       for (int i = 0; i < mesh->GetNE(); i++)
       {
          RefG = GlobGeometryRefiner.Refine(
@@ -1131,11 +1169,131 @@ void ParaViewDataCollection::SaveGFieldVTU(std::ostream &os, int ref_,
          }
       }
    }
-
-   if (IsBinaryFormat())
+   if (pv_data_format != VTKFormat::ASCII)
    {
-      WriteVTKEncodedCompressed(os,buf.data(),buf.size(),GetCompressionLevel());
-      os << '\n';
+      WriteBase64WithSizeAndClear(os, buf, GetCompressionLevel());
+   }
+   os << "</DataArray>" << std::endl;
+}
+
+void ParaViewDataCollection::SaveCoeffFieldVTU(std::ostream &os, int ref_,
+                                               const std::string &name, Coefficient &coeff)
+{
+   RefinedGeometry *RefG;
+   real_t val;
+   std::vector<char> buf;
+   int vec_dim = 1;
+   os << "<DataArray type=\"" << GetDataTypeString()
+      << "\" Name=\"" << name
+      << "\" NumberOfComponents=\"" << vec_dim << "\""
+      << " format=\"" << GetDataFormatString() << "\" >" << '\n';
+   {
+      // scalar data
+      if (!bdr_output)
+      {
+         for (int i = 0; i < mesh->GetNE(); i++)
+         {
+            RefG = GlobGeometryRefiner.Refine(
+                      mesh->GetElementBaseGeometry(i), ref_, 1);
+
+            ElementTransformation *eltrans = mesh->GetElementTransformation(i);
+            const IntegrationRule *ir = &RefG->RefPts;
+            for (int j = 0; j < ir->GetNPoints(); j++)
+            {
+               const IntegrationPoint &ip = ir->IntPoint(j);
+               eltrans->SetIntPoint(&ip);
+               val = coeff.Eval(*eltrans, ip);
+               WriteBinaryOrASCII(os, buf, val, "\n", pv_data_format);
+            }
+         }
+      }
+      else
+      {
+         for (int i = 0; i < mesh->GetNBE(); i++)
+         {
+            RefG = GlobGeometryRefiner.Refine(
+                      mesh->GetBdrElementBaseGeometry(i), ref_, 1);
+
+            ElementTransformation *eltrans = mesh->GetBdrElementTransformation(i);
+            const IntegrationRule *ir = &RefG->RefPts;
+            for (int j = 0; j < ir->GetNPoints(); j++)
+            {
+               const IntegrationPoint &ip = ir->IntPoint(j);
+               eltrans->SetIntPoint(&ip);
+               val = coeff.Eval(*eltrans, ip);
+               WriteBinaryOrASCII(os, buf, val, "\n", pv_data_format);
+            }
+         }
+      }
+   }
+   if (pv_data_format != VTKFormat::ASCII)
+   {
+      WriteBase64WithSizeAndClear(os, buf, GetCompressionLevel());
+   }
+   os << "</DataArray>" << std::endl;
+}
+
+void ParaViewDataCollection::SaveVCoeffFieldVTU(std::ostream &os, int ref_,
+                                                const std::string &name, VectorCoefficient &coeff)
+{
+   RefinedGeometry *RefG;
+   Vector val;
+   std::vector<char> buf;
+   int vec_dim = coeff.GetVDim();
+   os << "<DataArray type=\"" << GetDataTypeString()
+      << "\" Name=\"" << name
+      << "\" NumberOfComponents=\"" << vec_dim << "\""
+      << " format=\"" << GetDataFormatString() << "\" >" << '\n';
+   {
+      // vector data
+      if (!bdr_output)
+      {
+         for (int i = 0; i < mesh->GetNE(); i++)
+         {
+            RefG = GlobGeometryRefiner.Refine(
+                      mesh->GetElementBaseGeometry(i), ref_, 1);
+
+            ElementTransformation *eltrans = mesh->GetElementTransformation(i);
+            const IntegrationRule *ir = &RefG->RefPts;
+            for (int j = 0; j < ir->GetNPoints(); j++)
+            {
+               const IntegrationPoint &ip = ir->IntPoint(j);
+               eltrans->SetIntPoint(&ip);
+               coeff.Eval(val, *eltrans, ip);
+               for (int jj = 0; jj < val.Size(); jj++)
+               {
+                  WriteBinaryOrASCII(os, buf, val(jj), " ", pv_data_format);
+               }
+               if (pv_data_format == VTKFormat::ASCII) { os << '\n'; }
+            }
+         }
+      }
+      else
+      {
+         for (int i = 0; i < mesh->GetNBE(); i++)
+         {
+            RefG = GlobGeometryRefiner.Refine(
+                      mesh->GetBdrElementBaseGeometry(i), ref_, 1);
+
+            ElementTransformation *eltrans = mesh->GetBdrElementTransformation(i);
+            const IntegrationRule *ir = &RefG->RefPts;
+            for (int j = 0; j < ir->GetNPoints(); j++)
+            {
+               const IntegrationPoint &ip = ir->IntPoint(j);
+               eltrans->SetIntPoint(&ip);
+               coeff.Eval(val, *eltrans, ip);
+               for (int jj = 0; jj < val.Size(); jj++)
+               {
+                  WriteBinaryOrASCII(os, buf, val(jj), " ", pv_data_format);
+               }
+               if (pv_data_format == VTKFormat::ASCII) { os << '\n'; }
+            }
+         }
+      }
+   }
+   if (pv_data_format != VTKFormat::ASCII)
+   {
+      WriteBase64WithSizeAndClear(os, buf, GetCompressionLevel());
    }
    os << "</DataArray>" << std::endl;
 }

fem/datacollection.hpp

@@ -133,6 +133,7 @@ private:
 
    /// A collection of named QuadratureFunctions
    typedef NamedFieldsMap<QuadratureFunction> QFieldMap;
+
 public:
    typedef GFieldMap::MapType FieldMapType;
    typedef GFieldMap::iterator FieldMapIterator;
@@ -249,10 +250,9 @@ public:
    { field_map.Deregister(field_name, own_data); }
 
    /// Add a QuadratureFunction to the collection.
-   virtual void RegisterQField(const std::string& q_field_name,
+   virtual void RegisterQField(const std::string& field_name,
                                QuadratureFunction *qf)
-   { q_field_map.Register(q_field_name, qf, own_data); }
-
+   { q_field_map.Register(field_name, qf, own_data); }
 
    /// Remove a QuadratureFunction from the collection
    virtual void DeregisterQField(const std::string& field_name)
@@ -280,13 +280,13 @@ public:
 #endif
 
    /// Check if a QuadratureFunction with the given name is in the collection.
-   bool HasQField(const std::string& q_field_name) const
-   { return q_field_map.Has(q_field_name); }
+   bool HasQField(const std::string& field_name) const
+   { return q_field_map.Has(field_name); }
 
    /// Get a pointer to a QuadratureFunction in the collection.
    /** Returns NULL if @a field_name is not in the collection. */
-   QuadratureFunction *GetQField(const std::string& q_field_name)
-   { return q_field_map.Get(q_field_name); }
+   QuadratureFunction *GetQField(const std::string& field_name)
+   { return q_field_map.Get(field_name); }
 
    /// Get a const reference to the internal field map.
    /** The keys in the map are the field names and the values are pointers to
@@ -302,11 +302,13 @@ public:
 
    /// Get a pointer to the mesh in the collection
    Mesh *GetMesh() { return mesh; }
+
    /// Set/change the mesh associated with the collection
    /** When passed a Mesh, assumes the serial case: MPI rank id is set to 0 and
        MPI num_procs is set to 1.  When passed a ParMesh, MPI info from the
        ParMesh is used to set the DataCollection's MPI rank and num_procs. */
    virtual void SetMesh(Mesh *new_mesh);
+
 #ifdef MFEM_USE_MPI
    /// Set/change the mesh associated with the collection.
    /** For this case, @a comm is used to set the DataCollection's MPI rank id
@@ -369,8 +371,7 @@ public:
    /// Save one field, assuming the collection directory already exists.
    virtual void SaveField(const std::string &field_name);
    /// Save one q-field, assuming the collection directory already exists.
-   virtual void SaveQField(const std::string &q_field_name);
-
+   virtual void SaveQField(const std::string &field_name);
    /// Load the collection. Not implemented in the base class DataCollection.
    virtual void Load(int cycle_ = 0);
 
@@ -510,7 +511,9 @@ protected:
    int compression_level = -1;
    bool high_order_output = false;
    bool restart_mode = false;
+   bool bdr_output = false;
    VTKFormat pv_data_format = VTKFormat::BINARY;
+
 public:
    ParaViewDataCollectionBase(const std::string &name, Mesh *mesh);
 
@@ -543,6 +546,10 @@ public:
    /// Reading high-order data requires ParaView 5.5 or later.
    void SetHighOrderOutput(bool high_order_output_);
 
+   /// @brief Configures collection to save only fields evaluated on boundaries of
+   /// the mesh.
+   void SetBoundaryOutput(bool bdr_output_);
+
    /// If compression is enabled, return the compression level, else return 0.
    int GetCompressionLevel() const;
 
@@ -564,8 +571,6 @@ public:
    ///
    /// If restart is enabled, new writes will preserve timestep metadata for any
    /// solutions prior to the currently defined time.
-   ///
-   /// Initially, restart mode is disabled.
    void UseRestartMode(bool restart_mode_);
 };
 
@@ -575,11 +580,23 @@ class ParaViewDataCollection : public ParaViewDataCollectionBase
 private:
    std::fstream pvd_stream;
 
+   /// A collection of named Coefficients and VectorCoefficients
+   using CoeffFieldMap = NamedFieldsMap<Coefficient>;
+   using VCoeffFieldMap = NamedFieldsMap<VectorCoefficient>;
+
+   /** A FieldMap mapping registered names to Coefficient and VectorCoefficient
+       pointers. */
+   CoeffFieldMap coeff_field_map;
+   VCoeffFieldMap vcoeff_field_map;
 protected:
    void WritePVTUHeader(std::ostream &out);
    void WritePVTUFooter(std::ostream &out, const std::string &vtu_prefix);
    void SaveDataVTU(std::ostream &out, int ref);
    void SaveGFieldVTU(std::ostream& out, int ref_, const FieldMapIterator& it);
+   void SaveCoeffFieldVTU(std::ostream& out, int ref_, const std::string &name,
+                          Coefficient &coeff);
+   void SaveVCoeffFieldVTU(std::ostream& out, int ref_, const std::string &name,
+                           VectorCoefficient& coeff);
    const char *GetDataFormatString() const;
    const char *GetDataTypeString() const;
 
@@ -598,6 +615,25 @@ public:
    ParaViewDataCollection(const std::string& collection_name,
                           Mesh *mesh_ = nullptr);
 
+   /// Get a const reference to the internal coefficient-field map.
+   const typename CoeffFieldMap::MapType &GetCoeffFieldMap() const
+   { return coeff_field_map.GetMap(); }
+   const typename VCoeffFieldMap::MapType &GetVCoeffFieldMap() const
+   { return vcoeff_field_map.GetMap(); }
+
+   /// Add a Coefficient or VectorCoefficient to the collection.
+   void RegisterCoeffField(const std::string& field_name, Coefficient *coeff)
+   { coeff_field_map.Register(field_name, coeff, own_data); }
+   void RegisterVCoeffField(const std::string& field_name,
+                            VectorCoefficient *vcoeff)
+   { vcoeff_field_map.Register(field_name, vcoeff, own_data); }
+
+   /// Remove a Coefficient or VectorCoefficient from the collection
+   void DeregisterCoeffField(const std::string& field_name)
+   { coeff_field_map.Deregister(field_name, own_data); }
+   void DeregisterVCoeffField(const std::string& field_name)
+   { vcoeff_field_map.Deregister(field_name, own_data); }
+
    /// Save the collection - the directory name is constructed based on the
    /// cycle value
    void Save() override;

fem/dfem/doperator.hpp

@@ -211,8 +211,8 @@ private:
 ///
 /// The operator is constructed with solution fields that it will act on and
 /// parameter fields that define coefficients. Quadrature functions are added by
-/// e.g. using AddDomainIntegrator() which specify how the operator evaluates f
-/// those functionas and parameters at quadrature points.
+/// e.g. using AddDomainIntegrator() which specify how the operator evaluates
+/// those functions and parameters at quadrature points.
 ///
 /// Derivatives can be computed by obtaining a DerivativeOperator using
 /// GetDerivative().
@@ -231,24 +231,71 @@ public:
       const std::vector<FieldDescriptor> &parameters,
       const ParMesh &mesh);
 
+   /// MultLevel enum to indicate if the T->L Operators are used in the
+   /// Mult method.
+   enum MultLevel
+   {
+      TVECTOR,
+      LVECTOR
+   };
+
+   /// @brief Set the MultLevel mode for the DifferentiableOperator.
+   /// The default is TVECTOR, which means that the Operator will use
+   /// T->L before Mult and L->T Operators after.
+   void SetMultLevel(MultLevel level)
+   {
+      mult_level = level;
+   }
+
    /// @brief Compute the action of the operator on a given vector.
    ///
-   /// @param solutions_t The solution vector in which to compute the action.
-   /// This has to be a T-dof vector.
-   /// @param result_t Result vector of the action of the operator on
-   /// solutions_t. The result is a T-dof vector.
-   void Mult(const Vector &solutions_t, Vector &result_t) const override
+   /// @param solutions_in The solution vector in which to compute the action.
+   /// This has to be a T-dof vector if MultLevel is set to TVECTOR, or L-dof
+   /// Vector if MultLevel is set to LVECTOR.
+   /// @param result_in Result vector of the action of the operator on
+   /// solutions. The result is a T-dof vector or L-dof vector depending on
+   /// the MultLevel.
+   void Mult(const Vector &solutions_in, Vector &result_in) const override
    {
       MFEM_ASSERT(!action_callbacks.empty(), "no integrators have been set");
-      prolongation(solutions, solutions_t, solutions_l);
-      residual_l = 0.0;
-      for (auto &action : action_callbacks)
+
+      if (mult_level == MultLevel::LVECTOR)
       {
-         action(solutions_l, parameters_l, residual_l);
+         get_lvectors(solutions, solutions_in, solutions_l);
+         result_in = 0.0;
+         for (auto &action : action_callbacks)
+         {
+            action(solutions_l, parameters_l, result_in);
+         }
+      }
+      else
+      {
+         prolongation(solutions, solutions_in, solutions_l);
+         residual_l = 0.0;
+         for (auto &action : action_callbacks)
+         {
+            action(solutions_l, parameters_l, residual_l);
+         }
+         prolongation_transpose(residual_l, result_in);
       }
-      prolongation_transpose(residual_l, result_t);
    }
 
+   /// @brief Add an integrator to the operator.
+   /// Called only from AddDomainIntegrator() and AddBoundaryIntegrator().
+   template <
+      typename entity_t,
+      typename qfunc_t,
+      typename input_t,
+      typename output_t,
+      typename derivative_ids_t>
+   void AddIntegrator(
+      qfunc_t &qfunc,
+      input_t inputs,
+      output_t outputs,
+      const IntegrationRule &integration_rule,
+      const Array<int> &attributes,
+      derivative_ids_t derivative_ids);
+
    /// @brief Add a domain integrator to the operator.
    ///
    /// @param qfunc The quadrature function to be added.
@@ -274,6 +321,31 @@ public:
       const Array<int> &domain_attributes,
       derivative_ids_t derivative_ids = std::make_index_sequence<0> {});
 
+   /// @brief Add a boundary integrator to the operator.
+   ///
+   /// @param qfunc The quadrature function to be added.
+   /// @param inputs Tuple of FieldOperators for the inputs of the quadrature
+   /// function.
+   /// @param outputs Tuple of FieldOperators for the outputs of the quadrature
+   /// function.
+   /// @param integration_rule IntegrationRule to use with this integrator.
+   /// @param boundary_attributes Boundary attributes marker array indicating over
+   /// which attributes this integrator will integrate over.
+   /// @param derivative_ids Derivatives to be made available for this
+   /// integrator.
+   template <
+      typename qfunc_t,
+      typename input_t,
+      typename output_t,
+      typename derivative_ids_t = decltype(std::make_index_sequence<0> {})>
+   void AddBoundaryIntegrator(
+      qfunc_t &qfunc,
+      input_t inputs,
+      output_t outputs,
+      const IntegrationRule &integration_rule,
+      const Array<int> &boundary_attributes,
+      derivative_ids_t derivative_ids = std::make_index_sequence<0> {});
+
    /// @brief Set the parameters for the operator.
    ///
    /// This has to be called before using Mult() or MultTranspose().
@@ -345,6 +417,8 @@ public:
 private:
    const ParMesh &mesh;
 
+   MultLevel mult_level = TVECTOR;
+
    std::vector<action_t> action_callbacks;
    std::map<size_t,
        std::vector<derivative_action_t>> derivative_action_callbacks;
@@ -354,7 +428,6 @@ private:
        std::vector<assemble_derivative_hypreparmatrix_callback_t>>
        assemble_derivative_hypreparmatrix_callbacks;
 
-
    std::vector<FieldDescriptor> solutions;
    std::vector<FieldDescriptor> parameters;
    // solutions and parameters
@@ -391,7 +464,52 @@ void DifferentiableOperator::AddDomainIntegrator(
    const Array<int> &domain_attributes,
    derivative_ids_t derivative_ids)
 {
-   using entity_t = Entity::Element;
+   AddIntegrator<Entity::Element>(
+      qfunc, inputs, outputs, integration_rule, domain_attributes, derivative_ids);
+}
+
+template <
+   typename qfunc_t,
+   typename input_t,
+   typename output_t,
+   typename derivative_ids_t>
+void DifferentiableOperator::AddBoundaryIntegrator(
+   qfunc_t &qfunc,
+   input_t inputs,
+   output_t outputs,
+   const IntegrationRule &integration_rule,
+   const Array<int> &boundary_attributes,
+   derivative_ids_t derivative_ids)
+{
+
+   if (mesh.GetNFbyType(FaceType::Boundary) != mesh.GetNBE())
+   {
+      MFEM_ABORT("AddBoundaryIntegrator on meshes with interior boundaries is not supported.");
+   }
+   AddIntegrator<Entity::BoundaryElement>(
+      qfunc, inputs, outputs, integration_rule, boundary_attributes, derivative_ids);
+}
+
+template <
+   typename entity_t,
+   typename qfunc_t,
+   typename input_t,
+   typename output_t,
+   typename derivative_ids_t>
+void DifferentiableOperator::AddIntegrator(
+   qfunc_t &qfunc,
+   input_t inputs,
+   output_t outputs,
+   const IntegrationRule &integration_rule,
+   const Array<int> &attributes,
+   derivative_ids_t derivative_ids)
+{
+   if constexpr (!(std::is_same_v<entity_t, Entity::Element> ||
+                   std::is_same_v<entity_t, Entity::BoundaryElement>))
+   {
+      static_assert(dfem::always_false<entity_t>,
+                    "entity type not supported in AddIntegrator");
+   }
 
    static constexpr size_t num_inputs =
       tuple_size<decltype(inputs)>::value;
@@ -445,32 +563,44 @@ void DifferentiableOperator::AddDomainIntegrator(
    auto output_to_field =
       create_descriptors_to_fields_map<entity_t>(fields, outputs);
 
-   // TODO: factor out
-   std::vector<int> inputs_vdim(num_inputs);
-   for_constexpr<num_inputs>([&](auto i)
+   const Array<int> *elem_attributes = nullptr;
+   if constexpr (std::is_same_v<entity_t, Entity::Element>)
    {
-      inputs_vdim[i] = get<i>(inputs).vdim;
-   });
-
-
-   Array<int> elem_attributes;
-   elem_attributes.SetSize(mesh.GetNE());
-   for (int i = 0; i < mesh.GetNE(); ++i)
+      elem_attributes = &mesh.GetElementAttributes();
+   }
+   else if constexpr (std::is_same_v<entity_t, Entity::BoundaryElement>)
    {
-      elem_attributes[i] = mesh.GetAttribute(i);
+      elem_attributes = &mesh.GetBdrFaceAttributes();
    }
 
    const auto output_fop = get<0>(outputs);
    test_space_field_idx = FindIdx(output_fop.GetFieldId(), fields);
 
    bool use_sum_factorization = false;
-   auto entity_element_type =
-      Element::TypeFromGeometry(mesh.GetTypicalElementGeometry());
-   if ((entity_element_type == Element::QUADRILATERAL ||
-        entity_element_type == Element::HEXAHEDRON) &&
-       use_tensor_product_structure == true)
+   Element::Type entity_element_type;
+   if constexpr (std::is_same_v<entity_t, Entity::Element>)
    {
-      use_sum_factorization = true;
+      entity_element_type =
+         Element::TypeFromGeometry(mesh.GetTypicalElementGeometry());
+
+      if ((entity_element_type == Element::QUADRILATERAL ||
+           entity_element_type == Element::HEXAHEDRON) &&
+          use_tensor_product_structure == true)
+      {
+         use_sum_factorization = true;
+      }
+   }
+   else if constexpr (std::is_same_v<entity_t, Entity::BoundaryElement>)
+   {
+      entity_element_type =
+         Element::TypeFromGeometry(mesh.GetTypicalFaceGeometry());
+
+      if ((entity_element_type == Element::SEGMENT ||
+           entity_element_type == Element::QUADRILATERAL) &&
+          use_tensor_product_structure == true)
+      {
+         use_sum_factorization = true;
+      }
    }
 
    ElementDofOrdering element_dof_ordering = ElementDofOrdering::NATIVE;
@@ -508,8 +638,17 @@ void DifferentiableOperator::AddDomainIntegrator(
    prolongation_transpose = get_prolongation_transpose(
                                fields[test_space_field_idx], output_fop, mesh.GetComm());
 
-   const int dimension = mesh.Dimension();
-   [[maybe_unused]] const int num_elements = GetNumEntities<Entity::Element>(mesh);
+   int dimension;
+   if constexpr (std::is_same_v<entity_t, Entity::Element>)
+   {
+      dimension = mesh.Dimension();
+   }
+   else if constexpr (std::is_same_v<entity_t, Entity::BoundaryElement>)
+   {
+      dimension = mesh.Dimension() - 1;
+   }
+
+   [[maybe_unused]] const int num_elements = GetNumEntities<entity_t>(mesh);
    const int num_entities = GetNumEntities<entity_t>(mesh);
    const int num_qp = integration_rule.GetNPoints();
 
@@ -583,6 +722,12 @@ void DifferentiableOperator::AddDomainIntegrator(
          thread_blocks.z = 1;
       }
    }
+   else if (dimension == 1)
+   {
+      thread_blocks.x = q1d;
+      thread_blocks.y = 1;
+      thread_blocks.z = 1;
+   }
 
    action_callbacks.push_back(
       // Explicitly capture everything we need, so we can make explicit choice
@@ -598,7 +743,7 @@ void DifferentiableOperator::AddDomainIntegrator(
          test_vdim,             // int (= output_fop.vdim)
          test_op_dim,           // int (derived from output_fop)
          inputs,                // mfem::future::tuple
-         domain_attributes,     // Array<int>
+         attributes,            // Array<int>
          ir_weights,            // DeviceTensor
          use_sum_factorization, // bool
          input_dtq_maps,        // std::array<DofToQuadMap, num_fields>
@@ -631,13 +776,13 @@ void DifferentiableOperator::AddDomainIntegrator(
                                           action_shmem_info.field_sizes,
                                           num_entities);
 
-      const bool has_attr = domain_attributes.Size() > 0;
-      const auto d_domain_attr = domain_attributes.Read();
-      const auto d_elem_attr = elem_attributes.Read();
+      const bool has_attr = attributes.Size() > 0;
+      const auto d_attr = attributes.Read();
+      const auto d_elem_attr = elem_attributes->Read();
 
       forall([=] MFEM_HOST_DEVICE (int e, void *shmem)
       {
-         if (has_attr && !d_domain_attr[d_elem_attr[e] - 1]) { return; }
+         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
 
          auto [input_dtq_shmem, output_dtq_shmem, fields_shmem, input_shmem,
                                 residual_shmem, scratch_shmem] =
@@ -707,7 +852,7 @@ void DifferentiableOperator::AddDomainIntegrator(
                test_vdim,             // int (= output_fop.vdim)
                test_op_dim,           // int (derived from output_fop)
                inputs,                // mfem::future::tuple
-               domain_attributes,     // Array<int>
+               attributes,            // Array<int>
                ir_weights,            // DeviceTensor
                use_sum_factorization, // bool
                input_dtq_maps,        // std::array<DofToQuadMap, num_fields>
@@ -745,14 +890,14 @@ void DifferentiableOperator::AddDomainIntegrator(
                                                shmem_info.direction_size,
                                                num_entities);
 
-            const auto d_elem_attr = elem_attributes.Read();
-            const bool has_attr = domain_attributes.Size() > 0;
-            const auto d_domain_attr = domain_attributes.Read();
+            const bool has_attr = attributes.Size() > 0;
+            const auto d_attr = attributes.Read();
+            const auto d_elem_attr = elem_attributes->Read();
 
             derivative_action_e = 0.0;
             forall([=] MFEM_HOST_DEVICE (int e, real_t *shmem)
             {
-               if (has_attr && !d_domain_attr[d_elem_attr[e] - 1]) { return; }
+               if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
 
                auto [input_dtq_shmem, output_dtq_shmem, fields_shmem,
                                       direction_shmem, input_shmem,

fem/dfem/integrate.hpp

@@ -95,6 +95,85 @@ void map_quadrature_data_to_fields_impl(
    }
 }
 
+template <typename output_t>
+MFEM_HOST_DEVICE
+void map_quadrature_data_to_fields_tensor_impl_1d(
+   DeviceTensor<2, real_t> &y,
+   const DeviceTensor<3, real_t> &f,
+   const output_t &output,
+   const DofToQuadMap &dtq,
+   std::array<DeviceTensor<1>, 6> &scratch_mem)
+{
+   [[maybe_unused]] auto B = dtq.B;
+   [[maybe_unused]] auto G = dtq.G;
+
+   if constexpr (is_value_fop<std::decay_t<output_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = B.GetShape();
+      const int vdim = output.vdim;
+      const int test_dim = output.size_on_qp / vdim;
+
+      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d);
+      auto yd = Reshape(&y(0, 0), d1d, vdim);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(dx, x, d1d)
+         {
+            real_t acc = 0.0;
+            for (int qx = 0; qx < q1d; qx++)
+            {
+               acc += fqp(vd, 0, qx) * B(qx, 0, dx);
+            }
+            yd(dx, vd) = acc;
+         }
+      }
+      MFEM_SYNC_THREAD;
+   }
+   else if constexpr (is_gradient_fop<std::decay_t<output_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = G.GetShape();
+      const int vdim = output.vdim;
+      const int test_dim = output.size_on_qp / vdim;
+      auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d);
+      auto yd = Reshape(&y(0, 0), d1d, vdim);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(dx, x, d1d)
+         {
+            real_t acc = 0.0;
+            for (int qx = 0; qx < q1d; qx++)
+            {
+               acc += fqp(vd, 0, qx) * G(qx, 0, dx);
+            }
+            yd(dx, vd) = acc;
+         }
+      }
+      MFEM_SYNC_THREAD;
+   }
+   else if constexpr (is_identity_fop<std::decay_t<output_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = B.GetShape();
+      auto fqp = Reshape(&f(0, 0, 0), output.size_on_qp, q1d);
+      auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d);
+
+      for (int sq = 0; sq < output.size_on_qp; sq++)
+      {
+         MFEM_FOREACH_THREAD(qx, x, q1d)
+         {
+            yqp(sq, qx) = fqp(sq, qx);
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+   else
+   {
+      MFEM_ABORT("quadrature data mapping to field is not implemented for"
+                 " this field descriptor with sum factorization on tensor product elements");
+   }
+}
+
 template <typename output_t>
 MFEM_HOST_DEVICE
 void map_quadrature_data_to_fields_tensor_impl_2d(
@@ -431,7 +510,11 @@ void map_quadrature_data_to_fields(
 {
    if (use_sum_factorization)
    {
-      if (dimension == 2)
+      if (dimension == 1)
+      {
+         map_quadrature_data_to_fields_tensor_impl_1d(y, f, output, dtq, scratch_mem);
+      }
+      else if (dimension == 2)
       {
          map_quadrature_data_to_fields_tensor_impl_2d(y, f, output, dtq, scratch_mem);
       }

fem/dfem/interpolate.hpp

@@ -338,6 +338,92 @@ void map_field_to_quadrature_data_tensor_product_2d(
    }
 }
 
+
+template <typename field_operator_t>
+MFEM_HOST_DEVICE inline
+void map_field_to_quadrature_data_tensor_product_1d(
+   DeviceTensor<2> &field_qp,
+   const DofToQuadMap &dtq,
+   const DeviceTensor<1> &field_e,
+   const field_operator_t &input,
+   const DeviceTensor<1, const real_t> &integration_weights,
+   const std::array<DeviceTensor<1>, 6> &scratch_mem)
+{
+   [[maybe_unused]] auto B = dtq.B;
+   [[maybe_unused]] auto G = dtq.G;
+
+   if constexpr (is_value_fop<std::decay_t<field_operator_t>>::value)
+   {
+      auto [q1d, unused, d1d] = B.GetShape();
+      const int vdim = input.vdim;
+      const auto field = Reshape(&field_e[0], d1d, vdim);
+      auto fqp = Reshape(&field_qp[0], vdim, q1d);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(qx, x, q1d)
+         {
+            real_t acc = 0.0;
+            for (int dx = 0; dx < d1d; dx++)
+            {
+               acc += B(qx, 0, dx) * field(dx, vd);
+            }
+            fqp(vd, qx) = acc;
+         }
+      }
+      MFEM_SYNC_THREAD;
+   }
+   else if constexpr (
+      is_gradient_fop<std::decay_t<field_operator_t>>::value)
+   {
+      const auto [q1d, unused, d1d] = B.GetShape();
+      const int vdim = input.vdim;
+      const int dim = input.dim;
+      const auto field = Reshape(&field_e[0], d1d, vdim);
+      auto fqp = Reshape(&field_qp[0], vdim, dim, q1d);
+
+      for (int vd = 0; vd < vdim; vd++)
+      {
+         MFEM_FOREACH_THREAD(qx, x, q1d)
+         {
+            real_t acc = 0.0;
+            for (int dx = 0; dx < d1d; dx++)
+            {
+               acc += G(qx, 0, dx) * field(dx, vd);
+            }
+            fqp(vd, 0, qx) = acc;
+         }
+         MFEM_SYNC_THREAD;
+      }
+   }
+   // TODO: Create separate function for clarity
+   else if constexpr (
+      std::is_same_v<std::decay_t<field_operator_t>, Weight>)
+   {
+      const int num_qp = integration_weights.GetShape()[0];
+      // TODO: eeek
+      const int q1d = (int)floor(std::pow(num_qp, 1.0/input.dim) + 0.5);
+      auto w = Reshape(&integration_weights[0], q1d);
+      auto f = Reshape(&field_qp[0], q1d);
+      MFEM_FOREACH_THREAD(qx, x, q1d)
+      {
+         f(qx) = w(qx);
+      }
+      MFEM_SYNC_THREAD;
+   }
+   else if constexpr (is_identity_fop<std::decay_t<field_operator_t>>::value)
+   {
+      const int q1d = B.GetShape()[0];
+      auto field = Reshape(&field_e[0], input.size_on_qp, q1d);
+      field_qp = field;
+   }
+   else
+   {
+      static_assert(dfem::always_false<std::decay_t<field_operator_t>>,
+                    "can't map field to quadrature data");
+   }
+}
+
 template <typename field_operator_t>
 MFEM_HOST_DEVICE
 void map_field_to_quadrature_data(
@@ -444,7 +530,13 @@ void map_fields_to_quadrature_data(
 
       if (use_sum_factorization)
       {
-         if (dimension == 2)
+         if (dimension == 1)
+         {
+            map_field_to_quadrature_data_tensor_product_1d(
+               fields_qp[i], dtqmaps[i], field_e, get<i>(fops),
+               integration_weights, scratch_mem);
+         }
+         else if (dimension == 2)
          {
             map_field_to_quadrature_data_tensor_product_2d(
                fields_qp[i], dtqmaps[i], field_e, get<i>(fops),
@@ -489,14 +581,20 @@ void map_field_to_quadrature_data_conditional(
    {
       if (use_sum_factorization)
       {
-         if (dimension == 2)
+         if (dimension == 1)
          {
-            map_field_to_quadrature_data_tensor_product_3d(
+            map_field_to_quadrature_data_tensor_product_1d(
+               field_qp, dtqmap, field_e, fop, integration_weights, scratch_mem);
+
+         }
+         else if (dimension == 2)
+         {
+            map_field_to_quadrature_data_tensor_product_2d(
                field_qp, dtqmap, field_e, fop, integration_weights, scratch_mem);
          }
          else if (dimension == 3)
          {
-            map_field_to_quadrature_data_tensor_product_2d(
+            map_field_to_quadrature_data_tensor_product_3d(
                field_qp, dtqmap, field_e, fop, integration_weights, scratch_mem);
          }
       }
@@ -547,7 +645,13 @@ void map_direction_to_quadrature_data_conditional(
       {
          if (use_sum_factorization)
          {
-            if (dimension == 2)
+            if (dimension == 1)
+            {
+               map_field_to_quadrature_data_tensor_product_1d(
+                  directions_qp[i], dtqmaps[i], direction_e, get<i>(fops),
+                  integration_weights, scratch_mem);
+            }
+            else if (dimension == 2)
             {
                map_field_to_quadrature_data_tensor_product_2d(
                   directions_qp[i], dtqmaps[i], direction_e, get<i>(fops),

fem/dfem/qfunction_apply.hpp

@@ -44,7 +44,16 @@ void call_qfunction(
 {
    if (use_sum_factorization)
    {
-      if (dimension == 2)
+      if (dimension == 1)
+      {
+         MFEM_FOREACH_THREAD(q, x, q1d)
+         {
+            auto qf_args = decay_tuple<qf_param_ts> {};
+            auto r = Reshape(&residual_shmem(0, q), rs_qp);
+            apply_kernel(r, qfunc, qf_args, input_shmem, q);
+         }
+      }
+      else if (dimension == 2)
       {
          MFEM_FOREACH_THREAD(qx, x, q1d)
          {
@@ -123,7 +132,22 @@ void call_qfunction_derivative_action(
 {
    if (use_sum_factorization)
    {
-      if (dimension == 2)
+      if (dimension == 1)
+      {
+         MFEM_FOREACH_THREAD(q, x, q1d)
+         {
+            auto r = Reshape(&residual_shmem(0, q), das_qp);
+            auto qf_args = decay_tuple<qf_param_ts> {};
+#ifdef MFEM_USE_ENZYME
+            auto qf_shadow_args = decay_tuple<qf_param_ts> {};
+            apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
+                                        shadow_shmem, q);
+#else
+            apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
+#endif
+         }
+      }
+      else if (dimension == 2)
       {
          MFEM_FOREACH_THREAD(qx, x, q1d)
          {
@@ -164,7 +188,10 @@ void call_qfunction_derivative_action(
             }
          }
       }
-      MFEM_SYNC_THREAD;
+      else
+      {
+         MFEM_ABORT_KERNEL("unsupported dimension");
+      }
    }
    else
    {
@@ -180,8 +207,8 @@ void call_qfunction_derivative_action(
          apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
 #endif
       }
-      MFEM_SYNC_THREAD;
    }
+   MFEM_SYNC_THREAD;
 }
 
 template <typename qfunc_t, typename args_ts, size_t num_args>

fem/dfem/qfunction_transform.hpp

@@ -44,7 +44,7 @@ void process_qf_arg(
    {
       for (int j = 0; j < n; j++)
       {
-         arg(j, i).value = u((i * m) + j);
+         arg(j, i).value = u((i * n) + j);
       }
    }
 }
@@ -94,8 +94,8 @@ void process_qf_arg(
    {
       for (int j = 0; j < n; j++)
       {
-         arg(j, i).value = u((i * m) + j);
-         arg(j, i).gradient = v((i * m) + j);
+         arg(j, i).value = u((i * n) + j);
+         arg(j, i).gradient = v((i * n) + j);
       }
    }
 }
@@ -181,6 +181,14 @@ void process_derivative_from_native_dual(
    }
 }
 
+template <typename T>
+MFEM_HOST_DEVICE inline
+void process_derivative_from_native_dual(
+   DeviceTensor<1, T> &r,
+   const dual<T, T> &x)
+{
+   r(0) = x.gradient;
+}
 
 template <typename T0, typename T1>
 MFEM_HOST_DEVICE inline
@@ -230,7 +238,7 @@ void process_qf_arg(
    {
       for (int j = 0; j < n; j++)
       {
-         arg(j, i) = u((i * m) + j);
+         arg(j, i) = u((i * n) + j);
       }
    }
 }
@@ -330,7 +338,7 @@ void process_qf_arg(
    {
       for (int j = 0; j < n; j++)
       {
-         arg(j, i) = u((i * m) + j);
+         arg(j, i) = u((i * n) + j);
       }
    }
 }

fem/dfem/tuple.hpp

@@ -454,7 +454,7 @@ MFEM_HOST_DEVICE constexpr auto operator+=(tuple<T...>& x,
  *
  * @tparam T the types stored in the tuples x and y
  * @tparam i integer sequence used to index the tuples
- * @param x tuple of values to be subracted from
+ * @param x tuple of values to be subtracted from
  * @param y tuple of values to subtract from x
  */
 template <typename... T, int... i>
@@ -596,7 +596,7 @@ MFEM_HOST_DEVICE constexpr auto div_helper(const real_t a,
  * @tparam T the types stored in the tuple y
  * @tparam i The integer sequence to i
  * @param x tuple of values
- * @param a the constant denomenator
+ * @param a the constant denominator
  * @return the returned tuple ratio
  */
 template <typename... T, int... i>
@@ -726,7 +726,7 @@ MFEM_HOST_DEVICE constexpr auto operator*(const tuple<T...>& x, const real_t a)
 
 /**
  * @tparam T the types stored in the tuple
- * @tparam i a list of indices used to acces each element of the tuple
+ * @tparam i a list of indices used to access each element of the tuple
  * @param out the ostream to write the output to
  * @param A the tuple of values
  * @brief helper used to implement printing a tuple of values

fem/dfem/util.hpp

@@ -944,7 +944,44 @@ const Operator *get_element_restriction(const FieldDescriptor &f,
       }
       else
       {
-         static_assert(dfem::always_false<T>, "can't use GetElementRestriction on type");
+         static_assert(dfem::always_false<T>,
+                       "can't use get_element_restriction on type");
+      }
+      return nullptr; // Unreachable, but avoids compiler warning
+   }, f.data);
+}
+
+/// @brief Get the face restriction operator for a field descriptor.
+///
+/// @param f the field descriptor.
+/// @param o the face dof ordering.
+/// @param ft the face type
+/// @param m indicator if single or double valued
+/// @returns the face restriction operator for the field descriptor in
+/// specified ordering.
+inline
+const Operator *get_face_restriction(const FieldDescriptor &f,
+                                     ElementDofOrdering o,
+                                     FaceType ft,
+                                     L2FaceValues m)
+{
+   return std::visit([&o, &ft, &m](auto&& arg) -> const Operator*
+   {
+      using T = std::decay_t<decltype(arg)>;
+      if constexpr (std::is_same_v<T, const FiniteElementSpace *> ||
+                    std::is_same_v<T, const ParFiniteElementSpace *>)
+      {
+         return arg->GetFaceRestriction(o, ft, m);
+      }
+      else if constexpr (std::is_same_v<T, const ParameterSpace *>)
+      {
+         // ParameterSpace does not support face restrictions
+         MFEM_ABORT("internal error");
+      }
+      else
+      {
+         static_assert(dfem::always_false<T>,
+                       "can't use get_face_restriction on type");
       }
       return nullptr; // Unreachable, but avoids compiler warning
    }, f.data);
@@ -965,6 +1002,11 @@ const Operator *get_restriction(const FieldDescriptor &f,
    {
       return get_element_restriction(f, o);
    }
+   else if constexpr (std::is_same_v<entity_t, Entity::BoundaryElement>)
+   {
+      return get_face_restriction(f, o, FaceType::Boundary,
+                                  L2FaceValues::SingleValued);
+   }
    MFEM_ABORT("restriction not implemented for Entity");
    return nullptr;
 }
@@ -974,7 +1016,7 @@ const Operator *get_restriction(const FieldDescriptor &f,
 /// @param f the field descriptor.
 /// @param o the element dof ordering.
 /// @param fop the field operator.
-/// @returns a tuple containting a std::function with the transpose
+/// @returns a tuple containing a std::function with the transpose
 /// restriction callback and it's height.
 template <typename entity_t, typename fop_t>
 inline std::tuple<std::function<void(const Vector&, Vector&)>, int>
@@ -1076,6 +1118,24 @@ void prolongation(const std::vector<FieldDescriptor> fields,
    }
 }
 
+inline
+void get_lvectors(const std::vector<FieldDescriptor> fields,
+                  const Vector &x,
+                  std::vector<Vector> &fields_l)
+{
+   int data_offset = 0;
+   for (std::size_t i = 0; i < fields.size(); i++)
+   {
+      const int sz = GetVSize(fields[i]);
+      fields_l[i].SetSize(sz);
+
+      const Vector x_i(const_cast<Vector&>(x), data_offset, sz);
+      fields_l[i] = x_i;
+
+      data_offset += sz;
+   }
+}
+
 /// @brief Get a transpose prolongation callback for a field descriptor.
 ///
 /// In the special case of a one field operator, the transpose prolongation
@@ -1371,7 +1431,7 @@ create_descriptors_to_fields_map(
       if constexpr (std::is_same_v<std::decay_t<decltype(fop)>, Weight>)
       {
          // TODO-bug: stealing dimension from the first field
-         fop.dim = GetDimension<Entity::Element>(fields[0]);
+         fop.dim = GetDimension<entity_t>(fields[0]);
          fop.vdim = 1;
          fop.size_on_qp = 1;
          map = -1;

fem/fe/face_map_utils.hpp

@@ -259,6 +259,30 @@ inline void FaceIdxToVolIdx3D(const int index, const int size1d,
    i = yz_plane ? level : _i;
 }
 
+MFEM_HOST_DEVICE
+inline int FaceIdxToVolIdx(int dim, int i, int size1d, int face0, int face1,
+                           int side, int orientation)
+{
+   if (dim == 2)
+   {
+      int ix, iy;
+      internal::FaceIdxToVolIdx2D(i, size1d, face0, face1, side, ix, iy);
+      return ix + iy*size1d;
+   }
+   else if (dim == 3)
+   {
+      int ix, iy, iz;
+      internal::FaceIdxToVolIdx3D(i, size1d, face0, face1, side, orientation,
+                                  ix, iy, iz);
+      return ix + size1d*iy + size1d*size1d*iz;
+   }
+   else
+   {
+      MFEM_ABORT_KERNEL("Invalid dimension");
+      return -1;
+   }
+};
+
 } // namespace internal
 
 } // namespace mfem

fem/fe/fe_base.cpp

@@ -661,65 +661,78 @@ void ScalarFiniteElement::ScalarLocalL2Restriction(
 void NodalFiniteElement::CreateLexicographicFullMap(const IntegrationRule &ir)
 const
 {
-   // Get the FULL version of the map.
-   auto &d2q = GetDofToQuad(ir, DofToQuad::FULL);
-   //Undo the native ordering which is what FiniteElement::GetDofToQuad returns.
-   auto *d2q_new = new DofToQuad(d2q);
-   d2q_new->mode = DofToQuad::LEXICOGRAPHIC_FULL;
-   const int nqpt = ir.GetNPoints();
 
-   const int b_dim = (range_type == VECTOR) ? dim : 1;
-
-   for (int i = 0; i < nqpt; i++)
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   #pragma omp critical (DofToQuad)
+#endif
    {
-      for (int d = 0; d < b_dim; d++)
+      // Get the FULL version of the map.
+      auto &d2q = GetDofToQuad(ir, DofToQuad::FULL);
+      //Undo the native ordering which is what FiniteElement::GetDofToQuad returns.
+      auto *d2q_new = new DofToQuad(d2q);
+      d2q_new->mode = DofToQuad::LEXICOGRAPHIC_FULL;
+      const int nqpt = ir.GetNPoints();
+
+      const int b_dim = (range_type == VECTOR) ? dim : 1;
+
+      for (int i = 0; i < nqpt; i++)
       {
-         for (int j = 0; j < dof; j++)
+         for (int d = 0; d < b_dim; d++)
          {
-            const double val = d2q.B[i + nqpt*(d+b_dim*lex_ordering[j])];
-            d2q_new->B[i+nqpt*(d+b_dim*j)] = val;
-            d2q_new->Bt[j+dof*(i+nqpt*d)] = val;
+            for (int j = 0; j < dof; j++)
+            {
+               const double val = d2q.B[i + nqpt*(d+b_dim*lex_ordering[j])];
+               d2q_new->B[i+nqpt*(d+b_dim*j)] = val;
+               d2q_new->Bt[j+dof*(i+nqpt*d)] = val;
+            }
          }
       }
-   }
 
-   const int g_dim = [this]()
-   {
-      switch (deriv_type)
+      const int g_dim = [this]()
       {
-         case GRAD: return dim;
-         case DIV: return 1;
-         case CURL: return cdim;
-         default: return 0;
-      }
-   }();
-
-   for (int i = 0; i < nqpt; i++)
-   {
-      for (int d = 0; d < g_dim; d++)
-      {
-         for (int j = 0; j < dof; j++)
+         switch (deriv_type)
          {
-            const double val = d2q.G[i + nqpt*(d+g_dim*lex_ordering[j])];
-            d2q_new->G[i+nqpt*(d+g_dim*j)] = val;
-            d2q_new->Gt[j+dof*(i+nqpt*d)] = val;
+            case GRAD: return dim;
+            case DIV: return 1;
+            case CURL: return cdim;
+            default: return 0;
+         }
+      }();
+
+      for (int i = 0; i < nqpt; i++)
+      {
+         for (int d = 0; d < g_dim; d++)
+         {
+            for (int j = 0; j < dof; j++)
+            {
+               const double val = d2q.G[i + nqpt*(d+g_dim*lex_ordering[j])];
+               d2q_new->G[i+nqpt*(d+g_dim*j)] = val;
+               d2q_new->Gt[j+dof*(i+nqpt*d)] = val;
+            }
          }
       }
-   }
 
-   dof2quad_array.Append(d2q_new);
+      dof2quad_array.Append(d2q_new);
+   }
 }
 
 const DofToQuad &NodalFiniteElement::GetDofToQuad(const IntegrationRule &ir,
                                                   DofToQuad::Mode mode) const
 {
-   //Should make this loop a function of FiniteElement
-   for (int i = 0; i < dof2quad_array.Size(); i++)
+   DofToQuad *d2q = nullptr;
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   #pragma omp critical (DofToQuad)
+#endif
    {
-      const DofToQuad &d2q = *dof2quad_array[i];
-      if (d2q.IntRule == &ir && d2q.mode == mode) { return d2q; }
+      //Should make this loop a function of FiniteElement
+      for (int i = 0; i < dof2quad_array.Size(); i++)
+      {
+         d2q = dof2quad_array[i];
+         if (d2q->IntRule == &ir && d2q->mode == mode) { break; }
+         d2q = nullptr;
+      }
    }
-
+   if (d2q) { return *d2q; }
    if (mode != DofToQuad::LEXICOGRAPHIC_FULL)
    {
       return FiniteElement::GetDofToQuad(ir, mode);
@@ -2620,8 +2633,12 @@ const DofToQuad &TensorBasisElement::GetTensorDofToQuad(
    {
       for (int i = 0; i < dof2quad_array.Size(); i++)
       {
-         d2q = dof2quad_array[i];
-         if (d2q->IntRule != &ir || d2q->mode != mode) { d2q = nullptr; }
+         auto* d2q_ = dof2quad_array[i];
+         if (d2q_->IntRule == &ir && d2q_->mode == mode)
+         {
+            d2q = d2q_;
+            break;
+         }
       }
       if (!d2q)
       {

fem/fe_coll.cpp

@@ -308,13 +308,25 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name)
                                           FiniteElement::INTEGRAL,
                                           BasisType::GetType(name[12]));
    }
-   else if (!strncmp(name, "RT_R1D",6))
+   else if (!strncmp(name, "RT_R1D_", 7))
    {
-      fec = new RT_R1D_FECollection(atoi(name+11),atoi(name + 7));
+      fec = new RT_R1D_FECollection(atoi(name + 11), atoi(name + 7));
    }
-   else if (!strncmp(name, "RT_R2D",6))
+   else if (!strncmp(name, "RT_R1D@", 7))
    {
-      fec = new RT_R2D_FECollection(atoi(name+11),atoi(name + 7));
+      fec = new RT_R1D_FECollection(atoi(name + 14), atoi(name + 10),
+                                    BasisType::GetType(name[7]),
+                                    BasisType::GetType(name[8]));
+   }
+   else if (!strncmp(name, "RT_R2D_", 7))
+   {
+      fec = new RT_R2D_FECollection(atoi(name + 11), atoi(name + 7));
+   }
+   else if (!strncmp(name, "RT_R2D@", 7))
+   {
+      fec = new RT_R2D_FECollection(atoi(name + 14), atoi(name + 10),
+                                    BasisType::GetType(name[7]),
+                                    BasisType::GetType(name[8]));
    }
    else if (!strncmp(name, "RT_", 3))
    {
@@ -336,13 +348,25 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name)
                                       BasisType::GetType(name[9]),
                                       BasisType::GetType(name[10]));
    }
-   else if (!strncmp(name, "ND_R1D",6))
+   else if (!strncmp(name, "ND_R1D_", 7))
    {
-      fec = new ND_R1D_FECollection(atoi(name+11),atoi(name + 7));
+      fec = new ND_R1D_FECollection(atoi(name + 11), atoi(name + 7));
    }
-   else if (!strncmp(name, "ND_R2D",6))
+   else if (!strncmp(name, "ND_R1D@", 7))
    {
-      fec = new ND_R2D_FECollection(atoi(name+11),atoi(name + 7));
+      fec = new ND_R1D_FECollection(atoi(name + 14), atoi(name + 10),
+                                    BasisType::GetType(name[7]),
+                                    BasisType::GetType(name[8]));
+   }
+   else if (!strncmp(name, "ND_R2D_", 7))
+   {
+      fec = new ND_R2D_FECollection(atoi(name + 11), atoi(name + 7));
+   }
+   else if (!strncmp(name, "ND_R2D@", 7))
+   {
+      fec = new ND_R2D_FECollection(atoi(name + 14), atoi(name + 10),
+                                    BasisType::GetType(name[7]),
+                                    BasisType::GetType(name[8]));
    }
    else if (!strncmp(name, "ND_", 3))
    {
@@ -401,9 +425,6 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name)
    {
       MFEM_ABORT("unknown FiniteElementCollection: " << name);
    }
-   MFEM_VERIFY(!strcmp(fec->Name(), name), "input name: \"" << name
-               << "\" does not match the created collection name: \""
-               << fec->Name() << '"');
 
    return fec;
 }
@@ -2459,8 +2480,7 @@ RT_FECollection::RT_FECollection(const int order, const int dim,
       const char *cb_name = BasisType::Name(cb_type); // this may abort
       MFEM_ABORT("unknown closed BasisType: " << cb_name);
    }
-   if (Quadrature1D::CheckOpen(op_type) == Quadrature1D::Invalid &&
-       ob_type != BasisType::IntegratedGLL)
+   if (Quadrature1D::CheckOpen(op_type) == Quadrature1D::Invalid)
    {
       const char *ob_name = BasisType::Name(ob_type); // this may abort
       MFEM_ABORT("unknown open BasisType: " << ob_name);
@@ -2518,6 +2538,7 @@ RT_FECollection::RT_FECollection(const int p, const int dim,
                                  const int map_type, const bool signs,
                                  const int ob_type)
    : FiniteElementCollection(p + 1)
+   , dim(dim)
    , ob_type(ob_type)
 {
    if (Quadrature1D::CheckOpen(BasisType::GetQuadrature1D(ob_type)) ==
@@ -2786,8 +2807,7 @@ ND_FECollection::ND_FECollection(const int p, const int dim,
    int cp_type = BasisType::GetQuadrature1D(cb_type);
 
    // Error checking
-   if (Quadrature1D::CheckOpen(op_type) == Quadrature1D::Invalid &&
-       ob_type != BasisType::IntegratedGLL)
+   if (Quadrature1D::CheckOpen(op_type) == Quadrature1D::Invalid)
    {
       const char *ob_name = BasisType::Name(ob_type);
       MFEM_ABORT("Invalid open basis point type: " << ob_name);

fem/fe_coll.hpp

@@ -120,7 +120,9 @@ public:
    | ND_Trace_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0  | H_CURL | H^{1/2}-conforming trace elements for H(curl) defined on the interface between mesh elements (faces) |
    | ND_Trace@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0 | H_CURL | H^{1/2}-conforming trace elements for H(curl) defined on the interface between mesh elements (faces) |
    | ND_R1D_[DIM]_[ORDER] | H(curl) | * | 1 / 0 | H_CURL | 3D H(curl)-conforming Nedelec vector elements in 1D. |
+   | ND_R1D@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H(curl) | * | * / * | H_CURL | 3D H(curl)-conforming Nedelec vector elements in 1D. |
    | ND_R2D_[DIM]_[ORDER] | H(curl) | * | 1 / 0 | H_CURL | 3D H(curl)-conforming Nedelec vector elements in 2D. |
+   | ND_R2D@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H(curl) | * | * / * | H_CURL | 3D H(curl)-conforming Nedelec vector elements in 2D. |
    | RT_[DIM]_[ORDER] | H(div) | * | 1 / 0 | H_DIV | Raviart-Thomas vector elements |
    | RT@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H(div) | * | * / * | H_DIV | Raviart-Thomas vector elements |
    | RT_Trace_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0 | INTEGRAL | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
@@ -128,7 +130,9 @@ public:
    | RT_Trace@[BTYPE]_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0 | INTEGRAL | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
    | RT_ValTrace@[BTYPE]_[DIM]_[ORDER] |  H^{1/2} | * | 1 / 0 | VALUE | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
    | RT_R1D_[DIM]_[ORDER] | H(div) | * | 1 / 0 | H_DIV | 3D H(div)-conforming Raviart-Thomas vector elements in 1D. |
+   | RT_R1D@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H(div) | * | * / * | H_DIV | 3D H(div)-conforming Raviart-Thomas vector elements in 1D. |
    | RT_R2D_[DIM]_[ORDER] | H(div) | * | 1 / 0 | H_DIV | 3D H(div)-conforming Raviart-Thomas vector elements in 2D. |
+   | RT_R2D@[CBTYPE][OBTYPE]_[DIM]_[ORDER] | H(div) | * | * / * | H_DIV | 3D H(div)-conforming Raviart-Thomas vector elements in 2D. |
    | L2_[DIM]_[ORDER] | L2 | * | 0 | VALUE | Discontinuous L2 elements |
    | L2_T[BTYPE]_[DIM]_[ORDER] | L2 | * | 0 | VALUE | Discontinuous L2 elements |
    | L2Int_[DIM]_[ORDER] | L2 | * | 0 | INTEGRAL | Discontinuous L2 elements |
@@ -464,6 +468,13 @@ public:
    RT_Trace_FECollection(const int p, const int dim,
                          const int map_type = FiniteElement::INTEGRAL,
                          const int ob_type = BasisType::GaussLegendre);
+
+   FiniteElementCollection *Clone(int p) const override
+   {
+      const int map_type = (strncmp(rt_name, "RT_Trace", 8) == 0)?
+                           (FiniteElement::INTEGRAL):(FiniteElement::VALUE);
+      return new RT_Trace_FECollection(p, dim, map_type, ob_type);
+   }
 };
 
 /** Arbitrary order discontinuous finite elements defined on the interface
@@ -475,6 +486,13 @@ public:
    DG_Interface_FECollection(const int p, const int dim,
                              const int map_type = FiniteElement::VALUE,
                              const int ob_type = BasisType::GaussLegendre);
+
+   FiniteElementCollection *Clone(int p) const override
+   {
+      const int map_type = (strncmp(rt_name, "DG_Iface", 8) == 0)?
+                           (FiniteElement::VALUE):(FiniteElement::INTEGRAL);
+      return new DG_Interface_FECollection(p, dim, map_type, ob_type);
+   }
 };
 
 /// Arbitrary order H(curl)-conforming Nedelec finite elements.

fem/fes_kernels.hpp

@@ -224,7 +224,7 @@ struct DerefineMatrixOpMultFunctor
                sum += sign * bsptr[boptr[k] + i + j * block_height] *
                       xptr[this->IndexX(col, vdim, k)];
             }
-#if defined(__CUDA_ARCH__) or defined(__HIP_DEVICE_COMPILE__)
+#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
             if (Atomic)
             {
                atomicAdd(yptr + this->IndexY(row, vdim), sum);

fem/fespace.hpp

@@ -683,8 +683,12 @@ public:
    NURBSExtension *GetNURBSext() { return NURBSext; }
    NURBSExtension *StealNURBSext();
 
-   bool Conforming() const { return mesh->Conforming() && cP == NULL; }
-   bool Nonconforming() const { return mesh->Nonconforming() || cP != NULL; }
+   bool Conforming() const
+   {
+      return NURBSext != NULL ||
+             (mesh->Conforming() && cP == NULL);
+   }
+   bool Nonconforming() const { return !Conforming(); }
 
    /** Set the prolongation operator of the space to an arbitrary sparse matrix,
        creating a copy of the argument. */
@@ -922,6 +926,9 @@ public:
    { return mesh->GetBdrElementType(i); }
 
    /// Returns ElementTransformation for the @a i-th element.
+   /// @note The returned pointer references an object owned by the associated
+   /// @a Mesh that will be modified by other calls to `GetElementTransformation`.
+   /// As such, this pointer should @b not be deleted by the caller.
    ElementTransformation *GetElementTransformation(int i) const
    { return mesh->GetElementTransformation(i); }
 

fem/gridfunc.cpp

@@ -68,7 +68,7 @@ GridFunction::GridFunction(Mesh *m, std::istream &input)
       Vector::Load(input, fes->GetVSize());
 
       // if the mesh is a legacy (v1.1) NC mesh, it has old vertex ordering
-      if (fes->Nonconforming() &&
+      if (fes->Nonconforming() && fes->GetMesh()->ncmesh &&
           fes->GetMesh()->ncmesh->IsLegacyLoaded())
       {
          LegacyNCReorder();
@@ -1374,6 +1374,50 @@ void GridFunction::GetVectorGradientHat(
    MultAtB(loc_data_mat, dshape, gh);
 }
 
+void GridFunction::GetGradients(const IntegrationRule &ir, Vector &grad,
+                                QVectorLayout ql, MemoryType d_mt) const
+{
+   const FiniteElement &fe = *fes->GetTypicalFE();
+   const int dim  = fe.GetDim();
+   const int vdim = fes->GetVDim();
+   const int NE   = fes->GetNE();
+   const int ND   = fe.GetDof();
+   const int NQ   = ir.GetNPoints();
+
+   MemoryType my_d_mt = (d_mt != MemoryType::DEFAULT) ? d_mt :
+                        Device::GetDeviceMemoryType();
+
+   // ql == QVectorLayout::byNODES :   NQ x VDIM x  DIM x NE
+   // ql == QVectorLayout::byVDIM  : VDIM x  DIM x NQPT x NE
+   grad.SetSize(dim*vdim*NQ*NE, my_d_mt);
+
+   const QuadratureInterpolator &qi = *fes->GetQuadratureInterpolator(ir);
+   qi.SetOutputLayout(ql);
+
+   const bool use_tensor_products = UsesTensorBasis(*fes);
+   qi.DisableTensorProducts(!use_tensor_products);
+   const ElementDofOrdering e_ordering = use_tensor_products ?
+                                         ElementDofOrdering::LEXICOGRAPHIC :
+                                         ElementDofOrdering::NATIVE;
+   const Operator *elem_restr = fes->GetElementRestriction(e_ordering);
+
+   // Pre-compute the geometric factors in order to set the desired MemoryType
+   // they use:
+   fes->GetMesh()->GetGeometricFactors(
+      ir, GeometricFactors::JACOBIANS, my_d_mt);
+
+   if (elem_restr) // currently, always true
+   {
+      Vector f_e(vdim*ND*NE, my_d_mt);
+      elem_restr->Mult(*this, f_e);
+      qi.PhysDerivatives(f_e, grad);
+   }
+   else
+   {
+      qi.PhysDerivatives(*this, grad);
+   }
+}
+
 real_t GridFunction::GetDivergence(ElementTransformation &T) const
 {
    DofTransformation doftrans;
@@ -2624,7 +2668,7 @@ void GridFunction::ProjectBdrCoefficient(Coefficient *coeff[],
    }
    for (int i = 0; i < values_counter.Size(); i++)
    {
-      MFEM_ASSERT(bool(values_counter[i]) == ess_vdofs_marker[i],
+      MFEM_ASSERT(bool(values_counter[i]) == bool(ess_vdofs_marker[i]),
                   "internal error");
    }
 #endif

fem/gridfunc.hpp

@@ -153,7 +153,8 @@ public:
    /// Shortcut for calling SetFromTrueDofs() with GetTrueVector() as argument.
    void SetFromTrueVector() { SetFromTrueDofs(GetTrueVector()); }
 
-   /// Returns the values in the vertices of i'th element for dimension vdim.
+   /** @brief Returns the values at the vertices of element @a i for the 1-based
+       dimension vdim. */
    void GetNodalValues(int i, Array<real_t> &nval, int vdim = 1) const;
 
    /** @name Element index Get Value Methods
@@ -308,7 +309,8 @@ public:
    /// For a vector grid function, makes sure that the ordering is byNODES.
    void ReorderByNodes();
 
-   /// Return the values as a vector on mesh vertices for dimension vdim.
+   /** @brief Returns the values as a vector at mesh vertices, for the 1-based
+       dimension vdim. */
    void GetNodalValues(Vector &nval, int vdim = 1) const;
 
    void GetVectorFieldNodalValues(Vector &val, int comp) const;
@@ -359,6 +361,33 @@ public:
        variable. */
    void GetVectorGradientHat(ElementTransformation &T, DenseMatrix &gh) const;
 
+   /** @brief Evaluate the gradients of the GridFunction at the given quadrature
+       points, @a ir, in all mesh elements. */
+   /** This method assumes that all mesh elements are the same type and that the
+       IntegrationRule @a ir is consistent with that type of element.
+
+       @param[in] ir     Quadrature points at which the gradients are to be
+                         evaluated.
+       @param[out] grad  Output vector of size `SDIM*VDIM*NQ*NE` where `SDIM` is
+                         the spatial dimention of the mesh, `VDIM` is the vector
+                         dimension of the GridFunction, `NQ` is the number of
+                         quadrature points in @a ir, and `NE` is the number of
+                         elements in the mesh. The layout of @a grad is
+                         determined by the parameter @a ql: when @a ql is
+                         QVectorLayout::byNODES, the layout is
+                         `NQ x VDIM x SDIM x NE`; when @a ql is
+                         QVectorLayout::byVDIM, the layout is
+                         `VDIM x SDIM x NQ x NE`.
+       @param[in] ql     Determines the layout of the output vector @a grad; see
+                         the description of @a grad for details.
+       @param[in] d_mt   MemoryType to use for allocating the output vector
+                         @a grad, as well the GeometricFactors and temporary
+                         vector used by the method. By default, the current
+                         device memory type is used. */
+   void GetGradients(const IntegrationRule &ir, Vector &grad,
+                     QVectorLayout ql = QVectorLayout::byNODES,
+                     MemoryType d_mt = MemoryType::DEFAULT) const;
+
    /** Compute $ (\int_{\Omega} (*this) \psi_i)/(\int_{\Omega} \psi_i) $,
        where $ \psi_i $ are the basis functions for the FE space of avgs.
        Both FE spaces should be scalar and on the same mesh. */

fem/gslib.cpp

@@ -85,9 +85,9 @@ namespace mfem
 {
 
 FindPointsGSLIB::FindPointsGSLIB()
-   : mesh(NULL),
-     fec_map_lin(NULL),
-     fdataD(NULL), cr(NULL), gsl_comm(NULL),
+   : mesh(nullptr),
+     fec_map_lin(nullptr),
+     fdataD(nullptr), cr(nullptr), gsl_comm(nullptr),
      dim(-1), points_cnt(-1), setupflag(false), default_interp_value(0),
      avgtype(AvgType::ARITHMETIC), bdr_tol(1e-8)
 {
@@ -97,10 +97,10 @@ FindPointsGSLIB::FindPointsGSLIB()
    gf_rst_map.SetSize(4);
    for (int i = 0; i < mesh_split.Size(); i++)
    {
-      mesh_split[i] = NULL;
-      ir_split[i] = NULL;
-      fes_rst_map[i] = NULL;
-      gf_rst_map[i] = NULL;
+      mesh_split[i] = nullptr;
+      ir_split[i] = nullptr;
+      fes_rst_map[i] = nullptr;
+      gf_rst_map[i] = nullptr;
    }
 
    gsl_comm = new gslib::comm;
@@ -117,27 +117,40 @@ FindPointsGSLIB::FindPointsGSLIB()
    crystal_init(cr, gsl_comm);
 }
 
+FindPointsGSLIB::FindPointsGSLIB(Mesh &mesh_in, const double bb_t,
+                                 const double newt_tol, const int npt_max)
+   : FindPointsGSLIB()
+{
+   Setup(mesh_in, bb_t, newt_tol, npt_max);
+}
+
 FindPointsGSLIB::~FindPointsGSLIB()
 {
-   crystal_free(cr);
-   comm_free(gsl_comm);
-   delete gsl_comm;
-   delete cr;
-   for (int i = 0; i < 4; i++)
+   FreeData();
+#ifdef MFEM_USE_MPI
+   if (!Mpi::IsFinalized())  // currently segfaults inside gslib otherwise
+#endif
    {
-      if (mesh_split[i]) { delete mesh_split[i]; mesh_split[i] = NULL; }
-      if (ir_split[i]) { delete ir_split[i]; ir_split[i] = NULL; }
-      if (fes_rst_map[i]) { delete fes_rst_map[i]; fes_rst_map[i] = NULL; }
-      if (gf_rst_map[i]) { delete gf_rst_map[i]; gf_rst_map[i] = NULL; }
+      crystal_free(cr);
+      comm_free(gsl_comm);
+      delete gsl_comm;
+      delete cr;
    }
-   if (fec_map_lin) { delete fec_map_lin; fec_map_lin = NULL; }
+   for (int i = 0; i < mesh_split.Size(); i++)
+   {
+      if (mesh_split[i]) { delete mesh_split[i]; mesh_split[i] = nullptr; }
+      if (ir_split[i]) { delete ir_split[i]; ir_split[i] = nullptr; }
+      if (fes_rst_map[i]) { delete fes_rst_map[i]; fes_rst_map[i] = nullptr; }
+      if (gf_rst_map[i]) { delete gf_rst_map[i]; gf_rst_map[i] = nullptr; }
+   }
+   if (fec_map_lin) { delete fec_map_lin; fec_map_lin = nullptr; }
 }
 
 #ifdef MFEM_USE_MPI
 FindPointsGSLIB::FindPointsGSLIB(MPI_Comm comm_)
-   : mesh(NULL),
-     fec_map_lin(NULL),
-     fdataD(NULL), cr(NULL), gsl_comm(NULL),
+   : mesh(nullptr),
+     fec_map_lin(nullptr),
+     fdataD(nullptr), cr(nullptr), gsl_comm(nullptr),
      dim(-1), points_cnt(-1), setupflag(false), default_interp_value(0),
      avgtype(AvgType::ARITHMETIC), bdr_tol(1e-8)
 {
@@ -147,10 +160,10 @@ FindPointsGSLIB::FindPointsGSLIB(MPI_Comm comm_)
    gf_rst_map.SetSize(4);
    for (int i = 0; i < mesh_split.Size(); i++)
    {
-      mesh_split[i] = NULL;
-      ir_split[i] = NULL;
-      fes_rst_map[i] = NULL;
-      gf_rst_map[i] = NULL;
+      mesh_split[i] = nullptr;
+      ir_split[i] = nullptr;
+      fes_rst_map[i] = nullptr;
+      gf_rst_map[i] = nullptr;
    }
 
    gsl_comm = new gslib::comm;
@@ -158,12 +171,21 @@ FindPointsGSLIB::FindPointsGSLIB(MPI_Comm comm_)
    comm_init(gsl_comm, comm_);
    crystal_init(cr, gsl_comm);
 }
+
+FindPointsGSLIB::FindPointsGSLIB(ParMesh &mesh_in, const double bb_t,
+                                 const double newt_tol, const int npt_max)
+   : FindPointsGSLIB(mesh_in.GetComm())
+{
+   Setup(mesh_in, bb_t, newt_tol, npt_max);
+}
 #endif
 
 void FindPointsGSLIB::Setup(Mesh &m, const double bb_t, const double newt_tol,
                             const int npt_max)
 {
    MFEM_VERIFY(m.GetNodes() != NULL, "Mesh nodes are required.");
+   MFEM_VERIFY(m.SpaceDimension() == m.Dimension(),
+               "Mesh spatial dimension and reference element dimension must be the same");
    const int meshOrder = m.GetNodes()->FESpace()->GetMaxElementOrder();
 
    // call FreeData if FindPointsGSLIB::Setup has been called already
@@ -171,37 +193,9 @@ void FindPointsGSLIB::Setup(Mesh &m, const double bb_t, const double newt_tol,
 
    mesh = &m;
    dim  = mesh->Dimension();
-   unsigned dof1D = meshOrder + 1;
+   const unsigned int dof1D = meshOrder+1;
 
-   SetupSplitMeshes();
-   if (dim == 2)
-   {
-      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
-      ir_split[0] = new IntegrationRule(3*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder);
-
-      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
-      ir_split[1] = new IntegrationRule(pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder);
-   }
-   else if (dim == 3)
-   {
-      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
-      ir_split[0] = new IntegrationRule(pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder);
-
-      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
-      ir_split[1] = new IntegrationRule(4*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder);
-
-      if (ir_split[2]) { delete ir_split[2]; ir_split[2] = NULL; }
-      ir_split[2] = new IntegrationRule(3*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[2], ir_split[2], meshOrder);
-
-      if (ir_split[3]) { delete ir_split[3]; ir_split[3] = NULL; }
-      ir_split[3] = new IntegrationRule(8*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[3], ir_split[3], meshOrder);
-   }
+   SetupSplitMeshesAndIntegrationRules(meshOrder);
 
    GetNodalValues(mesh->GetNodes(), gsl_mesh);
 
@@ -1128,13 +1122,18 @@ void FindPointsGSLIB::Interpolate(Mesh &m, const Vector &point_pos,
 void FindPointsGSLIB::FreeData()
 {
    if (!setupflag) { return; }
-   if (dim == 2)
+#ifdef MFEM_USE_MPI
+   if (!Mpi::IsFinalized())  // currently segfaults inside gslib otherwise
+#endif
    {
-      findpts_free_2((gslib::findpts_data_2 *)this->fdataD);
-   }
-   else
-   {
-      findpts_free_3((gslib::findpts_data_3 *)this->fdataD);
+      if (dim == 2)
+      {
+         findpts_free_2((gslib::findpts_data_2 *)this->fdataD);
+      }
+      else
+      {
+         findpts_free_3((gslib::findpts_data_3 *)this->fdataD);
+      }
    }
    gsl_code.DeleteAll();
    gsl_proc.DeleteAll();
@@ -1158,8 +1157,8 @@ void FindPointsGSLIB::FreeData()
 
 void FindPointsGSLIB::SetupSplitMeshes()
 {
-   fec_map_lin = new H1_FECollection(1, dim);
-   if (mesh->Dimension() == 2)
+   if (fec_map_lin == nullptr) { fec_map_lin = new H1_FECollection(1, dim); }
+   if (dim == 2)
    {
       int Nvert = 7;
       int NEsplit = 3;
@@ -1201,7 +1200,7 @@ void FindPointsGSLIB::SetupSplitMeshes()
       mesh_split[1] = new Mesh(Mesh::MakeCartesian2D(1, 1,
                                                      Element::QUADRILATERAL));
    }
-   else if (mesh->Dimension() == 3)
+   else if (dim == 3)
    {
       mesh_split[0] = new Mesh(Mesh::MakeCartesian3D(1, 1, 1,
                                                      Element::HEXAHEDRON));
@@ -1346,41 +1345,6 @@ void FindPointsGSLIB::SetupSplitMeshes()
          }
       }
    }
-
-   NE_split_total = 0;
-   split_element_map.SetSize(0);
-   split_element_index.SetSize(0);
-   int NEsplit = 0;
-   for (int e = 0; e < mesh->GetNE(); e++)
-   {
-      const Geometry::Type gt   = mesh->GetElement(e)->GetGeometryType();
-      if (gt == Geometry::TRIANGLE || gt == Geometry::PRISM)
-      {
-         NEsplit = 3;
-      }
-      else if (gt == Geometry::TETRAHEDRON)
-      {
-         NEsplit = 4;
-      }
-      else if (gt == Geometry::PYRAMID)
-      {
-         NEsplit = 8;
-      }
-      else if (gt == Geometry::SQUARE || gt == Geometry::CUBE)
-      {
-         NEsplit = 1;
-      }
-      else
-      {
-         MFEM_ABORT("Unsupported geometry type.");
-      }
-      NE_split_total += NEsplit;
-      for (int i = 0; i < NEsplit; i++)
-      {
-         split_element_map.Append(e);
-         split_element_index.Append(i);
-      }
-   }
 }
 
 void FindPointsGSLIB::SetupIntegrationRuleForSplitMesh(Mesh *meshin,
@@ -1431,6 +1395,79 @@ void FindPointsGSLIB::SetupIntegrationRuleForSplitMesh(Mesh *meshin,
    }
 }
 
+void FindPointsGSLIB::SetupSplitMeshesAndIntegrationRules(const int order)
+{
+   MFEM_VERIFY(mesh, "Setup FindPointsGSLIB with mesh first.");
+   const int dof1D = order+1;
+   const int dim = mesh->Dimension();
+
+   SetupSplitMeshes();
+   if (dim == 2)
+   {
+      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
+      ir_split[0] = new IntegrationRule(3*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], order);
+
+      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
+      ir_split[1] = new IntegrationRule(pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], order);
+   }
+   else if (dim == 3)
+   {
+      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
+      ir_split[0] = new IntegrationRule(pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], order);
+
+      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
+      ir_split[1] = new IntegrationRule(4*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], order);
+
+      if (ir_split[2]) { delete ir_split[2]; ir_split[2] = NULL; }
+      ir_split[2] = new IntegrationRule(3*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[2], ir_split[2], order);
+
+      if (ir_split[3]) { delete ir_split[3]; ir_split[3] = NULL; }
+      ir_split[3] = new IntegrationRule(8*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[3], ir_split[3], order);
+   }
+
+   // Setup map for non tensor-product elements
+   NE_split_total = 0;
+   split_element_map.SetSize(0);
+   split_element_index.SetSize(0);
+   int NEsplit = 0;
+   for (int e = 0; e < mesh->GetNE(); e++)
+   {
+      const Geometry::Type gt   = mesh->GetElement(e)->GetGeometryType();
+      if (gt == Geometry::TRIANGLE || gt == Geometry::PRISM)
+      {
+         NEsplit = 3;
+      }
+      else if (gt == Geometry::TETRAHEDRON)
+      {
+         NEsplit = 4;
+      }
+      else if (gt == Geometry::PYRAMID)
+      {
+         NEsplit = 8;
+      }
+      else if (gt == Geometry::SQUARE || gt == Geometry::CUBE)
+      {
+         NEsplit = 1;
+      }
+      else
+      {
+         MFEM_ABORT("Unsupported geometry type.");
+      }
+      NE_split_total += NEsplit;
+      for (int i = 0; i < NEsplit; i++)
+      {
+         split_element_map.Append(e);
+         split_element_index.Append(i);
+      }
+   }
+}
+
 void FindPointsGSLIB::GetNodalValues(const GridFunction *gf_in,
                                      Vector &node_vals)
 {
@@ -2081,6 +2118,19 @@ void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
    } // parallel
 }
 
+Array<unsigned int> FindPointsGSLIB::GetPointsNotFoundIndices() const
+{
+   Array<unsigned int> nf_idxs;
+   for (int i = 0; i < gsl_code.Size(); i++)
+   {
+      if (gsl_code[i] == 2)
+      {
+         nf_idxs.Append(i);
+      }
+   }
+   return nf_idxs;
+}
+
 void FindPointsGSLIB::DistributePointInfoToOwningMPIRanks(
    Array<unsigned int> &recv_elem, Vector &recv_ref,
    Array<unsigned int> &recv_code)
@@ -2386,6 +2436,10 @@ void OversetFindPointsGSLIB::Setup(Mesh &m, const int meshid,
 {
    MFEM_VERIFY(m.GetNodes() != NULL, "Mesh nodes are required.");
    const int meshOrder = m.GetNodes()->FESpace()->GetMaxElementOrder();
+   const int gfOrder = gfmax ? gfmax->FESpace()->GetMaxElementOrder() :
+                       meshOrder;
+   MFEM_VERIFY(meshOrder == gfOrder,
+               "Mesh order must match gfmax order in OversetFindPointsGSLIB.");
 
    // FreeData if OversetFindPointsGSLIB::Setup has been called already
    if (setupflag) { FreeData(); }
@@ -2395,35 +2449,7 @@ void OversetFindPointsGSLIB::Setup(Mesh &m, const int meshid,
    const FiniteElement *fe = mesh->GetNodalFESpace()->GetTypicalFE();
    unsigned dof1D = fe->GetOrder() + 1;
 
-   SetupSplitMeshes();
-   if (dim == 2)
-   {
-      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
-      ir_split[0] = new IntegrationRule(3*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder);
-
-      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
-      ir_split[1] = new IntegrationRule(pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder);
-   }
-   else if (dim == 3)
-   {
-      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
-      ir_split[0] = new IntegrationRule(pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder);
-
-      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
-      ir_split[1] = new IntegrationRule(4*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder);
-
-      if (ir_split[2]) { delete ir_split[2]; ir_split[2] = NULL; }
-      ir_split[2] = new IntegrationRule(3*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[2], ir_split[2], meshOrder);
-
-      if (ir_split[3]) { delete ir_split[3]; ir_split[3] = NULL; }
-      ir_split[3] = new IntegrationRule(8*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[3], ir_split[3], meshOrder);
-   }
+   SetupSplitMeshesAndIntegrationRules(meshOrder);
 
    GetNodalValues(mesh->GetNodes(), gsl_mesh);
 
@@ -2480,7 +2506,7 @@ void OversetFindPointsGSLIB::FindPoints(const Vector &point_pos,
 {
    MFEM_VERIFY(setupflag, "Use OversetFindPointsGSLIB::Setup before "
                "finding points.");
-   MFEM_VERIFY(overset, "Please setup FindPoints for overlapping grids.");
+   MFEM_VERIFY(overset, "Please use OversetFindPoints for overlapping grids.");
    points_cnt = point_pos.Size() / dim;
    unsigned int match = 0; // Don't find points in the mesh if point_id=mesh_id
 

fem/gslib.hpp

@@ -13,7 +13,11 @@
 #define MFEM_GSLIB
 
 #include "../config/config.hpp"
+#ifdef MFEM_USE_MPI
+#include "pgridfunc.hpp"
+#else
 #include "gridfunc.hpp"
+#endif
 
 #ifdef MFEM_USE_GSLIB
 
@@ -131,6 +135,10 @@ protected:
                                                  IntegrationRule *irule,
                                                  int order);
 
+   /// Helper function that calls \ref SetupSplitMeshes and
+   /// \ref SetupIntegrationRuleForSplitMesh.
+   virtual void SetupSplitMeshesAndIntegrationRules(const int order);
+
    /// Get GridFunction value at the points expected by GSLIB.
    virtual void GetNodalValues(const GridFunction *gf_in, Vector &node_vals);
 
@@ -190,14 +198,23 @@ protected:
    void InterpolateOnDevice(const Vector &field_in_evec, Vector &field_out,
                             const int nel, const int ncomp,
                             const int dof1dsol, const int ordering);
+
 public:
    FindPointsGSLIB();
+   FindPointsGSLIB(Mesh &mesh_in, const double bb_t = 0.1,
+                   const double newt_tol = 1.0e-12,
+                   const int npt_max = 256);
 
 #ifdef MFEM_USE_MPI
    FindPointsGSLIB(MPI_Comm comm_);
+   FindPointsGSLIB(ParMesh &mesh_in, const double bb_t = 0.1,
+                   const double newt_tol = 1.0e-12,
+                   const int npt_max = 256);
 #endif
 
    virtual ~FindPointsGSLIB();
+   FindPointsGSLIB(const FindPointsGSLIB&) = delete;
+   FindPointsGSLIB& operator=(const FindPointsGSLIB&) = delete;
 
    /** Initializes the internal mesh in gslib, by sending the positions of the
        Gauss-Lobatto nodes of the input Mesh object \p m.
@@ -212,8 +229,8 @@ public:
        @param[in] npt_max   (Optional) Number of points for simultaneous
                             iteration. This alters performance and
                             memory footprint.*/
-   void Setup(Mesh &m, const double bb_t = 0.1,
-              const double newt_tol = 1.0e-12,
+
+   void Setup(Mesh &m, const double bb_t = 0.1, const double newt_tol = 1.0e-12,
               const int npt_max = 256);
    /** Searches positions given in physical space by \p point_pos.
        These positions can be ordered byNodes: (XXX...,YYY...,ZZZ) or
@@ -289,7 +306,12 @@ public:
 
    /** Cleans up memory allocated internally by gslib.
        Note that in parallel, this must be called before MPI_Finalize(), as it
-       calls MPI_Comm_free() for internal gslib communicators. */
+       calls MPI_Comm_free() for internal gslib communicators. FreeData is
+       also called by the class destructor and there are no memory leaks if the
+       destructor is called before MPI_Finalize(). If the destructor is called
+       after MPI_Finalize(), there will be an error because gslib will try to
+       invoke some MPI functions.
+   */
    virtual void FreeData();
 
    /// Return code for each point searched by FindPoints: inside element (0), on
@@ -312,6 +334,9 @@ public:
    /// point found by FindPoints.
    virtual const Vector &GetGSLIBReferencePosition() const { return gsl_ref; }
 
+   /// Get array of indices of not-found points.
+   Array<unsigned int> GetPointsNotFoundIndices() const;
+
    /** @name Methods to support a custom interpolation procedure.
        \brief The physical-space point that the user seeks to interpolate at
        could be located inside an element on another mpi rank.

fem/hyperbolic.cpp

@@ -181,7 +181,7 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
    // current elements' the number of degrees of freedom
    // does not consider the number of equations
    const int dof1 = el1.GetDof();
-   const int dof2 = el2.GetDof();
+   const int dof2 = (Tr.Elem2No >= 0)?(el2.GetDof()):(0);
 
 #ifdef MFEM_THREAD_SAFE
    // Local storage for element integration
@@ -219,7 +219,9 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
    const IntegrationRule *ir = IntRule;
    if (!ir)
    {
-      const int order = 2*std::max(el1.GetOrder(), el2.GetOrder()) + IntOrderOffset;
+      const int max_el_order = dof2 ? std::max(el1.GetOrder(),
+                                               el2.GetOrder()) : el1.GetOrder();
+      const int order = 2*max_el_order + IntOrderOffset;
       ir = &IntRules.Get(Tr.GetGeometryType(), order);
    }
    // loop over integration points
@@ -231,18 +233,22 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
 
       // Calculate basis functions on both elements at the face
       el1.CalcShape(Tr.GetElement1IntPoint(), shape1);
-      el2.CalcShape(Tr.GetElement2IntPoint(), shape2);
 
       // Interpolate elfun at the point
       elfun1_mat.MultTranspose(shape1, state1);
-      elfun2_mat.MultTranspose(shape2, state2);
+
+      if (dof2)
+      {
+         // Calculate basis functions on both elements at the face
+         el2.CalcShape(Tr.GetElement2IntPoint(), shape2);
+         // Interpolate elfun at the point
+         elfun2_mat.MultTranspose(shape2, state2);
+      }
 
       // Get the normal vector and the flux on the face
       if (nor.Size() == 1)  // if 1D, use 1 or -1.
       {
-         // This assume the 1D integration point is in (0,1). This may not work
-         // if this changes.
-         nor(0) = (Tr.GetElement1IntPoint().x - 0.5) * 2.0;
+         nor(0) = 2*Tr.GetElement1IntPoint().x - 1.;
       }
       else
       {
@@ -250,14 +256,18 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
       }
       // Compute F(u+, x) and F(u-, x) with maximum characteristic speed
       // Compute hat(F) using evaluated quantities
-      const real_t speed = numFlux.Eval(state1, state2, nor, Tr, fluxN);
+      const real_t speed = (dof2) ? numFlux.Eval(state1, state2, nor, Tr, fluxN):
+                           fluxFunction.ComputeFluxDotN(state1, nor, Tr, fluxN);
 
       // Update the global max char speed
       max_char_speed = std::max(speed, max_char_speed);
 
       // pre-multiply integration weight to flux
       AddMult_a_VWt(-ip.weight*sign, shape1, fluxN, elvect1_mat);
-      AddMult_a_VWt(+ip.weight*sign, shape2, fluxN, elvect2_mat);
+      if (dof2)
+      {
+         AddMult_a_VWt(+ip.weight*sign, shape2, fluxN, elvect2_mat);
+      }
    }
 }
 
@@ -268,7 +278,7 @@ void HyperbolicFormIntegrator::AssembleFaceGrad(
    // current elements' the number of degrees of freedom
    // does not consider the number of equations
    const int dof1 = el1.GetDof();
-   const int dof2 = el2.GetDof();
+   const int dof2 = (Tr.Elem2No >= 0)?(el2.GetDof()):(0);
 
 #ifdef MFEM_THREAD_SAFE
    // Local storage for element integration
@@ -302,7 +312,9 @@ void HyperbolicFormIntegrator::AssembleFaceGrad(
    const IntegrationRule *ir = IntRule;
    if (!ir)
    {
-      const int order = 2*std::max(el1.GetOrder(), el2.GetOrder()) + IntOrderOffset;
+      const int max_el_order = dof2 ? std::max(el1.GetOrder(),
+                                               el2.GetOrder()) : el1.GetOrder();
+      const int order = 2*max_el_order + IntOrderOffset;
       ir = &IntRules.Get(Tr.GetGeometryType(), order);
    }
    // loop over integration points
@@ -312,20 +324,25 @@ void HyperbolicFormIntegrator::AssembleFaceGrad(
 
       Tr.SetAllIntPoints(&ip); // set face and element int. points
 
-      // Calculate basis functions on both elements at the face
+      // Calculate basis functions of the first element at the face
       el1.CalcShape(Tr.GetElement1IntPoint(), shape1);
-      el2.CalcShape(Tr.GetElement2IntPoint(), shape2);
 
       // Interpolate elfun at the point
       elfun1_mat.MultTranspose(shape1, state1);
-      elfun2_mat.MultTranspose(shape2, state2);
+
+      if (dof2)
+      {
+         // Calculate basis function of the second element at the face
+         el2.CalcShape(Tr.GetElement2IntPoint(), shape2);
+
+         // Interpolate elfun at the point
+         elfun2_mat.MultTranspose(shape2, state2);
+      }
 
       // Get the normal vector and the flux on the face
       if (nor.Size() == 1)  // if 1D, use 1 or -1.
       {
-         // This assume the 1D integration point is in (0,1). This may not work
-         // if this changes.
-         nor(0) = (Tr.GetElement1IntPoint().x - 0.5) * 2.0;
+         nor(0) = 2*Tr.GetElement1IntPoint().x - 1.;
       }
       else
       {
@@ -335,7 +352,14 @@ void HyperbolicFormIntegrator::AssembleFaceGrad(
       // Trial side 1
 
       // Compute hat(J) using evaluated quantities
-      numFlux.Grad(1, state1, state2, nor, Tr, JDotN);
+      if (dof2)
+      {
+         numFlux.Grad(1, state1, state2, nor, Tr, JDotN);
+      }
+      else
+      {
+         fluxFunction.ComputeFluxJacobianDotN(state1, nor, Tr, JDotN);
+      }
 
       const int ioff = fluxFunction.num_equations * dof1;
 
@@ -360,36 +384,325 @@ void HyperbolicFormIntegrator::AssembleFaceGrad(
             }
          }
 
-      // Trial side 2
+      if (dof2)
+      {
+         // Trial side 2
+
+         // Compute hat(J) using evaluated quantities
+         numFlux.Grad(2, state1, state2, nor, Tr, JDotN);
+
+         const int joff = ioff;
+
+         for (int di = 0; di < fluxFunction.num_equations; di++)
+            for (int dj = 0; dj < fluxFunction.num_equations; dj++)
+            {
+               // pre-multiply integration weight to Jacobian
+               const real_t w = +ip.weight * sign * JDotN(di,dj);
+               for (int j = 0; j < dof2; j++)
+               {
+                  // Test side 1
+                  for (int i = 0; i < dof1; i++)
+                  {
+                     elmat(i+dof1*di, joff+j+dof2*dj) += w * shape1(i) * shape2(j);
+                  }
+
+                  // Test side 2
+                  for (int i = 0; i < dof2; i++)
+                  {
+                     elmat(ioff+i+dof2*di, joff+j+dof2*dj) -= w * shape2(i) * shape2(j);
+                  }
+               }
+            }
+      }
+   }
+}
+
+BdrHyperbolicDirichletIntegrator::BdrHyperbolicDirichletIntegrator(
+   const NumericalFlux &numFlux,
+   VectorCoefficient &bdrState,
+   const int IntOrderOffset,
+   real_t sign)
+   : NonlinearFormIntegrator(),
+     numFlux(numFlux),
+     fluxFunction(numFlux.GetFluxFunction()),
+     u_vcoeff(bdrState),
+     IntOrderOffset(IntOrderOffset),
+     sign(sign),
+     num_equations(fluxFunction.num_equations)
+{
+   MFEM_VERIFY(fluxFunction.num_equations == bdrState.GetVDim(),
+               "Flux function does not match the vector dimension of the coefficient!");
+#ifndef MFEM_THREAD_SAFE
+   state_in.SetSize(num_equations);
+   state_out.SetSize(num_equations);
+   fluxN.SetSize(num_equations);
+   JDotN.SetSize(num_equations);
+   nor.SetSize(fluxFunction.dim);
+#endif
+   ResetMaxCharSpeed();
+}
+
+void BdrHyperbolicDirichletIntegrator::AssembleFaceVector(
+   const FiniteElement &el, const FiniteElement &,
+   FaceElementTransformations &Tr, const Vector &elfun, Vector &elvect)
+{
+   MFEM_ASSERT(Tr.Elem2No < 0, "Not a boundary face!");
+
+   // current elements' the number of degrees of freedom
+   // does not consider the number of equations
+   const int dof = el.GetDof();
+
+#ifdef MFEM_THREAD_SAFE
+   // Local storage for element integration
+
+   // shape function value at an integration point
+   Vector shape(dof);
+   // normal vector (usually not a unit vector)
+   Vector nor(Tr.GetSpaceDim());
+   // state value at an integration point - interior
+   Vector state_in(num_equations);
+   // state value at an integration point - boundary
+   Vector state_out(num_equations);
+   // hat(F)(u,x)
+   Vector fluxN(num_equations);
+#else
+   shape.SetSize(dof);
+#endif
+
+   elvect.SetSize(dof * num_equations);
+   elvect = 0.0;
+
+   const DenseMatrix elfun_mat(elfun.GetData(), dof, num_equations);
+
+   DenseMatrix elvect_mat(elvect.GetData(), dof, num_equations);
+
+   // Obtain integration rule. If integration is rule is given, then use it.
+   // Otherwise, get (2*p + IntOrderOffset) order integration rule
+   const IntegrationRule *ir = IntRule;
+   if (!ir)
+   {
+      const int order = 2*el.GetOrder() + IntOrderOffset;
+      ir = &IntRules.Get(Tr.GetGeometryType(), order);
+   }
+   // loop over integration points
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+
+      Tr.SetAllIntPoints(&ip); // set face and element int. points
+
+      // Calculate basis functions at the face
+      el.CalcShape(Tr.GetElement1IntPoint(), shape);
+
+      // Interpolate elfun at the point
+      elfun_mat.MultTranspose(shape, state_in);
+
+      // Evaluate boundary state at the point
+      u_vcoeff.Eval(state_out, Tr, ip);
+
+      // Get the normal vector and the flux on the face
+      if (nor.Size() == 1)  // if 1D, use 1 or -1.
+      {
+         nor(0) = 2*Tr.GetElement1IntPoint().x - 1.;
+      }
+      else
+      {
+         CalcOrtho(Tr.Jacobian(), nor);
+      }
+      // Compute F(u+, x) and F(u_b, x) with maximum characteristic speed
+      // Compute hat(F) using evaluated quantities
+      const real_t speed = numFlux.Eval(state_in, state_out, nor, Tr, fluxN);
+
+      // Update the global max char speed
+      max_char_speed = std::max(speed, max_char_speed);
+
+      // pre-multiply integration weight to flux
+      AddMult_a_VWt(-ip.weight*sign, shape, fluxN, elvect_mat);
+   }
+}
+
+void BdrHyperbolicDirichletIntegrator::AssembleFaceGrad(
+   const FiniteElement &el, const FiniteElement &,
+   FaceElementTransformations &Tr, const Vector &elfun, DenseMatrix &elmat)
+{
+   // current elements' the number of degrees of freedom
+   // does not consider the number of equations
+   const int dof = el.GetDof();
+
+#ifdef MFEM_THREAD_SAFE
+   // Local storage for element integration
+
+   // shape function value at an integration point
+   Vector shape(dof);
+   // normal vector (usually not a unit vector)
+   Vector nor(Tr.GetSpaceDim());
+   // state value at an integration point - interior
+   Vector state_in(num_equations);
+   // state value at an integration point - boundary
+   Vector state_out(num_equations);
+   // hat(J)(u,x)
+   DenseMatrix JDotN(num_equations);
+#else
+   shape.SetSize(dof);
+#endif
+
+   elmat.SetSize(dof * num_equations);
+   elmat = 0.0;
+
+   const DenseMatrix elfun_mat(elfun.GetData(), dof, num_equations);
+
+   // Obtain integration rule. If integration is rule is given, then use it.
+   // Otherwise, get (2*p + IntOrderOffset) order integration rule
+   const IntegrationRule *ir = IntRule;
+   if (!ir)
+   {
+      const int order = 2*el.GetOrder() + IntOrderOffset;
+      ir = &IntRules.Get(Tr.GetGeometryType(), order);
+   }
+   // loop over integration points
+   for (int q = 0; q < ir->GetNPoints(); q++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(q);
+
+      Tr.SetAllIntPoints(&ip); // set face and element int. points
+
+      // Calculate basis functions at the face
+      el.CalcShape(Tr.GetElement1IntPoint(), shape);
+
+      // Interpolate elfun at the point
+      elfun_mat.MultTranspose(shape, state_in);
+
+      // Evaluate boundary state at the point
+      u_vcoeff.Eval(state_out, Tr, ip);
+
+      // Get the normal vector and the flux on the face
+      if (nor.Size() == 1)  // if 1D, use 1 or -1.
+      {
+         nor(0) = 2*Tr.GetElement1IntPoint().x - 1.;
+      }
+      else
+      {
+         CalcOrtho(Tr.Jacobian(), nor);
+      }
 
       // Compute hat(J) using evaluated quantities
-      numFlux.Grad(2, state1, state2, nor, Tr, JDotN);
-
-      const int joff = ioff;
+      numFlux.Grad(1, state_in, state_out, nor, Tr, JDotN);
 
       for (int di = 0; di < fluxFunction.num_equations; di++)
          for (int dj = 0; dj < fluxFunction.num_equations; dj++)
          {
             // pre-multiply integration weight to Jacobian
-            const real_t w = +ip.weight * sign * JDotN(di,dj);
-            for (int j = 0; j < dof2; j++)
-            {
-               // Test side 1
-               for (int i = 0; i < dof1; i++)
+            const real_t w = -ip.weight * sign * JDotN(di,dj);
+            for (int j = 0; j < dof; j++)
+               for (int i = 0; i < dof; i++)
                {
-                  elmat(i+dof1*di, joff+j+dof2*dj) += w * shape1(i) * shape2(j);
+                  elmat(i+dof*di, j+dof*dj) += w * shape(i) * shape(j);
                }
-
-               // Test side 2
-               for (int i = 0; i < dof2; i++)
-               {
-                  elmat(ioff+i+dof2*di, joff+j+dof2*dj) -= w * shape2(i) * shape2(j);
-               }
-            }
          }
    }
 }
 
+BoundaryHyperbolicFlowIntegrator::BoundaryHyperbolicFlowIntegrator(
+   const FluxFunction &flux, VectorCoefficient &u, real_t alpha_, real_t beta_,
+   const int IntOrderOffset_)
+   : fluxFunction(flux), u_vcoeff(u), alpha(alpha_), beta(beta_),
+     IntOrderOffset(IntOrderOffset_)
+{
+   MFEM_VERIFY(fluxFunction.num_equations == u_vcoeff.GetVDim(),
+               "Flux function does not match the vector dimension of the coefficient!");
+#ifndef MFEM_THREAD_SAFE
+   state.SetSize(fluxFunction.num_equations);
+   nor.SetSize(fluxFunction.dim);
+   fluxN.SetSize(fluxFunction.num_equations);
+#endif
+   ResetMaxCharSpeed();
+}
+
+void BoundaryHyperbolicFlowIntegrator::AssembleRHSElementVect(
+   const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
+{
+   mfem_error("BoundaryHyperbolicFlowIntegrator::AssembleRHSElementVect\n"
+              "  is not implemented as boundary integrator!\n"
+              "  Use LinearForm::AddBdrFaceIntegrator instead of\n"
+              "  LinearForm::AddBoundaryIntegrator.");
+}
+
+void BoundaryHyperbolicFlowIntegrator::AssembleRHSElementVect(
+   const FiniteElement &el, FaceElementTransformations &Tr, Vector &elvect)
+{
+   // current elements' the number of degrees of freedom
+   // does not consider the number of equations
+   const int dof = el.GetDof();
+
+#ifdef MFEM_THREAD_SAFE
+   // Local storage for element integration
+
+   // shape function value at an integration point
+   Vector shape(dof);
+   // state value at an integration point
+   Vector state(fluxFunction.num_equations);
+   // normal vector (usually not a unit vector)
+   Vector nor(Tr.GetSpaceDim());
+   // hat(F)(u,x)
+   Vector fluxN(fluxFunction.num_equations);
+#else
+   shape.SetSize(dof);
+#endif
+
+   elvect.SetSize(dof * fluxFunction.num_equations);
+   elvect = 0.0;
+
+   DenseMatrix elvect_mat(elvect.GetData(), dof, fluxFunction.num_equations);
+
+   // Obtain integration rule. If integration is rule is given, then use it.
+   // Otherwise, get (2*p + IntOrderOffset) order integration rule
+   const IntegrationRule *ir = IntRule;
+   if (!ir)
+   {
+      const int order = 2*el.GetOrder() + IntOrderOffset;
+      ir = &IntRules.Get(Tr.GetGeometryType(), order);
+   }
+   // loop over integration points
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+
+      Tr.SetAllIntPoints(&ip); // set face and element int. points
+
+      // Calculate basis functions on both elements at the face
+      el.CalcShape(Tr.GetElement1IntPoint(), shape);
+
+      // Evaluate the coefficient at the point
+      u_vcoeff.Eval(state, Tr, ip);
+
+      // Get the normal vector and the flux on the face
+      if (nor.Size() == 1)  // if 1D, use 1 or -1.
+      {
+         nor(0) = 2*Tr.GetElement1IntPoint().x - 1.;
+      }
+      else
+      {
+         CalcOrtho(Tr.Jacobian(), nor);
+      }
+      // Compute F(u, x) with maximum characteristic speed
+      const real_t speed = fluxFunction.ComputeFluxDotN(state, nor, Tr, fluxN);
+
+      // Update the global max char speed
+      max_char_speed = std::max(speed, max_char_speed);
+
+      // pre-multiply integration weight to flux
+      const real_t a = 0.5 * alpha  * ip.weight;
+      const real_t b = beta * ip.weight;
+
+      for (int n = 0; n < fluxFunction.num_equations; n++)
+      {
+         fluxN(n) = a * fluxN(n) - b * fabs(fluxN(n));
+      }
+
+      AddMultVWt(shape, fluxN, elvect_mat);
+   }
+}
+
 real_t FluxFunction::ComputeFluxDotN(const Vector &U,
                                      const Vector &normal,
                                      FaceElementTransformations &Tr,

fem/hyperbolic.hpp

@@ -306,12 +306,14 @@ MFEM_DEPRECATED typedef NumericalFlux RiemannSolver;
 class HyperbolicFormIntegrator : public NonlinearFormIntegrator
 {
 private:
-   // The maximum characteristic speed, updated during element/face vector assembly
-   real_t max_char_speed;
    const NumericalFlux &numFlux;   // Numerical flux that maps F(u±,x) to F̂
    const FluxFunction &fluxFunction;
    const int IntOrderOffset; // integration order offset, 2*p + IntOrderOffset.
    const real_t sign;
+
+   // The maximum characteristic speed, updated during element/face vector assembly
+   real_t max_char_speed;
+
 #ifndef MFEM_THREAD_SAFE
    // Local storage for element integration
    Vector shape;              // shape function value at an integration point
@@ -331,8 +333,9 @@ private:
 
 public:
    const int num_equations;  // the number of equations
+
    /**
-    * @brief Construct a new Hyperbolic Form Integrator object
+    * @brief Construct a new HyperbolicFormIntegrator object
     *
     * @param[in] numFlux numerical flux
     * @param[in] IntOrderOffset integration order offset
@@ -343,21 +346,14 @@ public:
       const int IntOrderOffset = 0,
       const real_t sign = 1.);
 
-   /**
-    * @brief Reset the Max Char Speed 0
-    *
-    */
-   void ResetMaxCharSpeed()
-   {
-      max_char_speed = 0.0;
-   }
+   /// Reset the maximum characteristic speed to zero
+   void ResetMaxCharSpeed() { max_char_speed = 0.0; }
 
-   real_t GetMaxCharSpeed()
-   {
-      return max_char_speed;
-   }
+   /// Get the maximum characteristic speed
+   real_t GetMaxCharSpeed() const { return max_char_speed; }
 
-   const FluxFunction &GetFluxFunction() { return fluxFunction; }
+   /// Get the associated flux function
+   const FluxFunction &GetFluxFunction() const { return fluxFunction; }
 
    /**
     * @brief Implements (F(u), ∇v) with abstract F computed by
@@ -416,6 +412,182 @@ public:
                          const Vector &elfun, DenseMatrix &elmat) override;
 };
 
+/**
+ * @brief Abstract boundary hyperbolic form integrator, assembling
+ * <F̂(u⁻,u_b,x) n, [v]> term for scalar finite elements at the boundary.
+ *
+ * This form integrator is coupled with a NumericalFlux that implements the
+ * numerical flux F̂ at the boundary faces. The flux F is obtained from the
+ * FluxFunction assigned to the aforementioned NumericalFlux with the given
+ * boundary coefficient for the state u_b.
+ *
+ * Note the class can be used for imposing conditions on interior interfaces.
+ */
+class BdrHyperbolicDirichletIntegrator : public NonlinearFormIntegrator
+{
+private:
+   const NumericalFlux &numFlux;    // Numerical flux that maps F to F̂
+   const FluxFunction &fluxFunction;
+   VectorCoefficient &u_vcoeff;     // Boundary state vector coefficient
+   const int IntOrderOffset; // integration order offset, 2*p + IntOrderOffset.
+   const real_t sign;
+
+   // The maximum characteristic speed, updated during element/face vector assembly
+   real_t max_char_speed;
+
+#ifndef MFEM_THREAD_SAFE
+   // Local storage for element integration
+   Vector shape;  // shape function value at an integration point
+   Vector state_in;  // state value at an integration point - interior
+   Vector state_out;  // state value at an integration point - boundary
+   Vector nor;     // normal vector, see mfem::CalcOrtho()
+   Vector fluxN;   // F̂(u⁻,u_b,x) n
+   DenseMatrix JDotN;   // Ĵ(u⁻,u_b,x) n
+#endif
+
+public:
+   const int num_equations;  // the number of equations
+
+   /**
+    * @brief Construct a new BdrHyperbolicDirichletIntegrator object
+    *
+    * @param[in] numFlux numerical flux
+    * @param[in] bdrState boundary state coefficient
+    * @param[in] IntOrderOffset integration order offset
+    * @param[in] sign sign of the convection term
+    */
+   BdrHyperbolicDirichletIntegrator(
+      const NumericalFlux &numFlux,
+      VectorCoefficient &bdrState,
+      const int IntOrderOffset = 0,
+      const real_t sign = 1.);
+
+   /// Reset the maximum characteristic speed to zero
+   void ResetMaxCharSpeed() { max_char_speed = 0.0; }
+
+   /// Get the maximum characteristic speed
+   real_t GetMaxCharSpeed() const { return max_char_speed; }
+
+   /// Get the associated flux function
+   const FluxFunction &GetFluxFunction() const { return fluxFunction; }
+
+   /**
+    * @brief Implements <-F̂(u⁻,u_b,x) n, [v]> with abstract F̂ computed by
+    * NumericalFlux::Eval() of the numerical flux object
+    *
+    * @param[in] el1 finite element of the interior element
+    * @param[in] el2 not used
+    * @param[in] Tr face element transformations
+    * @param[in] elfun local coefficient of basis for the interior element
+    * @param[out] elvect evaluated dual vector <-F̂(u⁻,u_b,x) n, [v]>
+    */
+   void AssembleFaceVector(const FiniteElement &el1,
+                           const FiniteElement &el2,
+                           FaceElementTransformations &Tr,
+                           const Vector &elfun, Vector &elvect) override;
+
+   /**
+    * @brief Implements <-Ĵ(u⁻,u_b,x) n, [v]> with abstract Ĵ computed by
+    * NumericalFlux::Grad() of the numerical flux object
+    *
+    * @param[in] el1 finite element of the interior element
+    * @param[in] el2 not used
+    * @param[in] Tr face element transformations
+    * @param[in] elfun local coefficient of basis for the interior element
+    * @param[out] elmat evaluated Jacobian matrix <-Ĵ(u⁻,u_b,x) n, [v]>
+    */
+   void AssembleFaceGrad(const FiniteElement &el1,
+                         const FiniteElement &el2,
+                         FaceElementTransformations &Tr,
+                         const Vector &elfun, DenseMatrix &elmat) override;
+};
+
+/**
+ * @brief Abstract boundary hyperbolic linear form integrator, assembling
+ * <ɑ/2 F(u,x) n - β |F(u,x) n|, v> terms for scalar finite elements.
+ *
+ * This form integrator is coupled with a FluxFunction that evaluates the
+ * flux F at the boundary.
+ *
+ * Note the upwinding is performed component-wise. For general boundary
+ * integration with a numerical flux, see BdrHyperbolicDirichletIntegrator.
+ */
+class BoundaryHyperbolicFlowIntegrator : public LinearFormIntegrator
+{
+   const FluxFunction &fluxFunction;
+   VectorCoefficient &u_vcoeff;
+   const real_t alpha, beta;
+   const int IntOrderOffset; // integration order offset, 2*p + IntOrderOffset.
+
+   // The maximum characteristic speed, updated during face vector assembly
+   real_t max_char_speed;
+
+#ifndef MFEM_THREAD_SAFE
+   // Local storage for element integration
+   Vector shape;   // shape function value at an integration point
+   Vector state;   // state value at an integration point
+   Vector nor;     // normal vector, see mfem::CalcOrtho()
+   Vector fluxN;   // F(u,x) n
+#endif
+
+public:
+   /**
+    * @brief Construct a new BoundaryHyperbolicFlowIntegrator object
+    *
+    * @param[in] flux flux function
+    * @param[in] u vector state coefficient
+    * @param[in] alpha ɑ coefficient (β = ɑ/2)
+    * @param[in] IntOrderOffset integration order offset
+    */
+   BoundaryHyperbolicFlowIntegrator(const FluxFunction &flux, VectorCoefficient &u,
+                                    real_t alpha = -1., int IntOrderOffset = 0)
+      : BoundaryHyperbolicFlowIntegrator(flux, u, alpha, alpha/2., IntOrderOffset) { }
+
+   /**
+    * @brief Construct a new BoundaryHyperbolicFlowIntegrator object
+    *
+    * @param[in] flux flux function
+    * @param[in] u vector state coefficient
+    * @param[in] alpha ɑ coefficient
+    * @param[in] beta β coefficient
+    * @param[in] IntOrderOffset integration order offset
+    */
+   BoundaryHyperbolicFlowIntegrator(const FluxFunction &flux, VectorCoefficient &u,
+                                    real_t alpha, real_t beta, int IntOrderOffset = 0);
+
+   /// Reset the maximum characteristic speed to zero
+   void ResetMaxCharSpeed() { max_char_speed = 0.0; }
+
+   /// Get the maximum characteristic speed
+   real_t GetMaxCharSpeed() const { return max_char_speed; }
+
+   /// Get the associated flux function
+   const FluxFunction &GetFluxFunction() const { return fluxFunction; }
+
+   using LinearFormIntegrator::AssembleRHSElementVect;
+
+   /**
+    * @warning Boundary element integration not implemented, use
+    * AssembleRHSElementVect(const FiniteElement&,
+    * FaceElementTransformations &, Vector &) instead
+    */
+   void AssembleRHSElementVect(const FiniteElement &el,
+                               ElementTransformation &Tr,
+                               Vector &elvect) override;
+
+   /**
+    * @brief Implements <-F(u,x) n, v> with abstract F computed by
+    * FluxFunction::ComputeFluxDotN() of the flux function object
+    *
+    * @param[in] el finite element
+    * @param[in] Tr face element transformations
+    * @param[out] elvect evaluated dual vector <F(u,x) n, v>
+    */
+   void AssembleRHSElementVect(const FiniteElement &el,
+                               FaceElementTransformations &Tr,
+                               Vector &elvect) override;
+};
+
 
 /**
  * @brief Rusanov flux, also known as local Lax-Friedrichs,

fem/integ/bilininteg_curlcurl_pa.cpp

@@ -15,6 +15,86 @@
 namespace mfem
 {
 
+CurlCurlIntegrator::CurlCurlIntegrator() : Q(nullptr), DQ(nullptr), MQ(nullptr)
+{
+   static Kernels kernels;
+}
+
+CurlCurlIntegrator::CurlCurlIntegrator(Coefficient &q,
+                                       const IntegrationRule *ir)
+   : BilinearFormIntegrator(ir), Q(&q), DQ(nullptr), MQ(nullptr)
+{
+   static Kernels kernels;
+}
+
+CurlCurlIntegrator::CurlCurlIntegrator(DiagonalMatrixCoefficient &dq,
+                                       const IntegrationRule *ir)
+   : BilinearFormIntegrator(ir), Q(nullptr), DQ(&dq), MQ(nullptr)
+{
+   static Kernels kernels;
+}
+
+CurlCurlIntegrator::CurlCurlIntegrator(MatrixCoefficient &mq,
+                                       const IntegrationRule *ir)
+   : BilinearFormIntegrator(ir), Q(nullptr), DQ(nullptr), MQ(&mq)
+{
+   static Kernels kernels;
+}
+
+/// \cond DO_NOT_DOCUMENT
+
+CurlCurlIntegrator::Kernels::Kernels()
+{
+   CurlCurlIntegrator::AddSpecialization<3, 2, 3>();
+   CurlCurlIntegrator::AddSpecialization<3, 3, 4>();
+   CurlCurlIntegrator::AddSpecialization<3, 4, 5>();
+   CurlCurlIntegrator::AddSpecialization<3, 5, 6>();
+}
+
+CurlCurlIntegrator::ApplyKernelType
+CurlCurlIntegrator::ApplyPAKernels::Fallback(int DIM, int, int)
+{
+   if (DIM == 2) { return internal::PACurlCurlApply2D; }
+   else if (DIM == 3)
+   {
+      if (Device::Allows(Backend::DEVICE_MASK))
+      {
+         return internal::SmemPACurlCurlApply3D;
+      }
+      else
+      {
+         return internal::PACurlCurlApply3D;
+      }
+   }
+   else { MFEM_ABORT(""); }
+}
+
+CurlCurlIntegrator::DiagonalKernelType
+CurlCurlIntegrator::DiagonalPAKernels::Fallback(int DIM, int, int)
+{
+   if (DIM == 2)
+   {
+      return internal::PACurlCurlAssembleDiagonal2D;
+   }
+   else if (DIM == 3)
+   {
+      if (Device::Allows(Backend::DEVICE_MASK))
+      {
+         return internal::SmemPACurlCurlAssembleDiagonal3D;
+      }
+      else
+      {
+         return internal::PACurlCurlAssembleDiagonal3D;
+      }
+   }
+   else
+   {
+      MFEM_ABORT("");
+   }
+}
+
+/// \endcond DO_NOT_DOCUMENT
+
 void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
    // Assumes tensor-product elements
@@ -77,129 +157,16 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
 
 void CurlCurlIntegrator::AssembleDiagonalPA(Vector& diag)
 {
-   if (dim == 3)
-   {
-      if (Device::Allows(Backend::DEVICE_MASK))
-      {
-         const int ID = (dofs1D << 4) | quad1D;
-         switch (ID)
-         {
-            case 0x23:
-               return internal::SmemPACurlCurlAssembleDiagonal3D<2,3>(
-                         dofs1D,
-                         quad1D,
-                         symmetric, ne,
-                         mapsO->B, mapsC->B,
-                         mapsO->G, mapsC->G,
-                         pa_data, diag);
-            case 0x34:
-               return internal::SmemPACurlCurlAssembleDiagonal3D<3,4>(
-                         dofs1D,
-                         quad1D,
-                         symmetric, ne,
-                         mapsO->B, mapsC->B,
-                         mapsO->G, mapsC->G,
-                         pa_data, diag);
-            case 0x45:
-               return internal::SmemPACurlCurlAssembleDiagonal3D<4,5>(
-                         dofs1D,
-                         quad1D,
-                         symmetric, ne,
-                         mapsO->B, mapsC->B,
-                         mapsO->G, mapsC->G,
-                         pa_data, diag);
-            case 0x56:
-               return internal::SmemPACurlCurlAssembleDiagonal3D<5,6>(
-                         dofs1D,
-                         quad1D,
-                         symmetric, ne,
-                         mapsO->B, mapsC->B,
-                         mapsO->G, mapsC->G,
-                         pa_data, diag);
-            default:
-               return internal::SmemPACurlCurlAssembleDiagonal3D(
-                         dofs1D, quad1D,
-                         symmetric, ne,
-                         mapsO->B, mapsC->B,
-                         mapsO->G, mapsC->G,
-                         pa_data, diag);
-         }
-      }
-      else
-      {
-         internal::PACurlCurlAssembleDiagonal3D(dofs1D, quad1D, symmetric, ne,
-                                                mapsO->B, mapsC->B,
-                                                mapsO->G, mapsC->G,
-                                                pa_data, diag);
-      }
-   }
-   else if (dim == 2)
-   {
-      internal::PACurlCurlAssembleDiagonal2D(dofs1D, quad1D, ne,
-                                             mapsO->B, mapsC->G, pa_data, diag);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension!");
-   }
+   DiagonalPAKernels::Run(dim, dofs1D, quad1D, dofs1D, quad1D, symmetric, ne,
+                          mapsO->B, mapsC->B, mapsO->G, mapsC->G, pa_data,
+                          diag);
 }
 
 void CurlCurlIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
-   if (dim == 3)
-   {
-      if (Device::Allows(Backend::DEVICE_MASK))
-      {
-         const int ID = (dofs1D << 4) | quad1D;
-         switch (ID)
-         {
-            case 0x23:
-               return internal::SmemPACurlCurlApply3D<2,3>(
-                         dofs1D, quad1D,
-                         symmetric, ne,
-                         mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt,
-                         mapsC->G, mapsC->Gt, pa_data, x, y);
-            case 0x34:
-               return internal::SmemPACurlCurlApply3D<3,4>(
-                         dofs1D, quad1D,
-                         symmetric, ne,
-                         mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt,
-                         mapsC->G, mapsC->Gt, pa_data, x, y);
-            case 0x45:
-               return internal::SmemPACurlCurlApply3D<4,5>(
-                         dofs1D, quad1D,
-                         symmetric, ne,
-                         mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt,
-                         mapsC->G, mapsC->Gt, pa_data, x, y);
-            case 0x56:
-               return internal::SmemPACurlCurlApply3D<5,6>(
-                         dofs1D, quad1D,
-                         symmetric, ne,
-                         mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt,
-                         mapsC->G, mapsC->Gt, pa_data, x, y);
-            default:
-               return internal::SmemPACurlCurlApply3D(
-                         dofs1D, quad1D, symmetric, ne,
-                         mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt,
-                         mapsC->G, mapsC->Gt, pa_data, x, y);
-         }
-      }
-      else
-      {
-         internal::PACurlCurlApply3D(dofs1D, quad1D, symmetric, ne, mapsO->B, mapsC->B,
-                                     mapsO->Bt, mapsC->Bt, mapsC->G, mapsC->Gt,
-                                     pa_data, x, y);
-      }
-   }
-   else if (dim == 2)
-   {
-      internal::PACurlCurlApply2D(dofs1D, quad1D, ne, mapsO->B, mapsO->Bt,
-                                  mapsC->G, mapsC->Gt, pa_data, x, y);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension!");
-   }
+   ApplyPAKernels::Run(dim, dofs1D, quad1D, dofs1D, quad1D, symmetric, ne,
+                       mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt, mapsC->G,
+                       mapsC->Gt, pa_data, x, y, false);
 }
 
 void CurlCurlIntegrator::AddAbsMultPA(const Vector &x, Vector &y) const
@@ -209,61 +176,9 @@ void CurlCurlIntegrator::AddAbsMultPA(const Vector &x, Vector &y) const
    auto absO = mapsO->Abs();
    auto absC = mapsC->Abs();
 
-   if (dim == 3)
-   {
-      if (Device::Allows(Backend::DEVICE_MASK))
-      {
-         const int ID = (dofs1D << 4) | quad1D;
-         switch (ID)
-         {
-            case 0x23:
-               return internal::SmemPACurlCurlApply3D<2,3>(
-                         dofs1D, quad1D,
-                         symmetric, ne,
-                         absO.B, absC.B, absO.Bt, absC.Bt,
-                         absC.G, absC.Gt, abs_pa_data, x, y, true);
-            case 0x34:
-               return internal::SmemPACurlCurlApply3D<3,4>(
-                         dofs1D, quad1D,
-                         symmetric, ne,
-                         absO.B, absC.B, absO.Bt, absC.Bt,
-                         absC.G, absC.Gt, abs_pa_data, x, y, true);
-            case 0x45:
-               return internal::SmemPACurlCurlApply3D<4,5>(
-                         dofs1D, quad1D,
-                         symmetric, ne,
-                         absO.B, absC.B, absO.Bt, absC.Bt,
-                         absC.G, absC.Gt, abs_pa_data, x, y, true);
-            case 0x56:
-               return internal::SmemPACurlCurlApply3D<5,6>(
-                         dofs1D, quad1D,
-                         symmetric, ne,
-                         absO.B, absC.B, absO.Bt, absC.Bt,
-                         absC.G, absC.Gt, abs_pa_data, x, y, true);
-            default:
-               return internal::SmemPACurlCurlApply3D<0,0>(
-                         dofs1D, quad1D, symmetric, ne,
-                         absO.B, absC.B, absO.Bt, absC.Bt,
-                         absC.G, absC.Gt, abs_pa_data, x, y, true);
-         }
-      }
-      else
-      {
-         internal::PACurlCurlApply3D<0,0>(
-            dofs1D, quad1D, symmetric, ne,
-            absO.B, absC.B, absO.Bt, absC.Bt, absC.G, absC.Gt,
-            abs_pa_data, x, y, true);
-      }
-   }
-   else if (dim == 2)
-   {
-      internal::PACurlCurlApply2D(dofs1D, quad1D, ne, absO.B, absO.Bt,
-                                  absC.G, absC.Gt, abs_pa_data, x, y, true);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension!");
-   }
+   ApplyPAKernels::Run(dim, dofs1D, quad1D, dofs1D, quad1D, symmetric, ne,
+                       absO.B, absC.B, absO.Bt, absC.Bt, absC.G, absC.Gt,
+                       abs_pa_data, x, y, true);
 }
 
 } // namespace mfem

fem/integ/bilininteg_dgdiffusion_kernels.hpp

@@ -0,0 +1,500 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_BILININTEG_DGDIFFUSION_KERNELS_HPP
+#define MFEM_BILININTEG_DGDIFFUSION_KERNELS_HPP
+
+#include "../../general/forall.hpp"
+#include "../../mesh/face_nbr_geom.hpp"
+#include "../fe/face_map_utils.hpp"
+#include "../gridfunc.hpp"
+#include "../qfunction.hpp"
+
+/// \cond DO_NOT_DOCUMENT
+namespace mfem
+{
+
+namespace internal
+{
+
+template <int T_D1D = 0, int T_Q1D = 0>
+static void PADGDiffusionApply2D(const int NF, const Array<real_t> &b,
+                                 const Array<real_t> &bt,
+                                 const Array<real_t> &g,
+                                 const Array<real_t> &gt, const real_t sigma,
+                                 const Vector &pa_data, const Vector &x_,
+                                 const Vector &dxdn_, Vector &y_, Vector &dydn_,
+                                 const int d1d = 0, const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+
+   auto B_ = Reshape(b.Read(), Q1D, D1D);
+   auto G_ = Reshape(g.Read(), Q1D, D1D);
+
+   auto pa =
+      Reshape(pa_data.Read(), 6, Q1D, NF); // (q, 1/h, J00, J01, J10, J11)
+
+   auto x = Reshape(x_.Read(), D1D, 2, NF);
+   auto y = Reshape(y_.ReadWrite(), D1D, 2, NF);
+   auto dxdn = Reshape(dxdn_.Read(), D1D, 2, NF);
+   auto dydn = Reshape(dydn_.ReadWrite(), D1D, 2, NF);
+
+   const int NBX = std::max(D1D, Q1D);
+
+   mfem::forall_2D(NF, NBX, 2, [=] MFEM_HOST_DEVICE(int f) -> void
+   {
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      MFEM_SHARED real_t u0[max_D1D];
+      MFEM_SHARED real_t u1[max_D1D];
+      MFEM_SHARED real_t du0[max_D1D];
+      MFEM_SHARED real_t du1[max_D1D];
+
+      MFEM_SHARED real_t Bu0[max_Q1D];
+      MFEM_SHARED real_t Bu1[max_Q1D];
+      MFEM_SHARED real_t Bdu0[max_Q1D];
+      MFEM_SHARED real_t Bdu1[max_Q1D];
+
+      MFEM_SHARED real_t r[max_Q1D];
+
+      MFEM_SHARED real_t BG[2 * max_D1D * max_Q1D];
+      DeviceMatrix B(BG, Q1D, D1D);
+      DeviceMatrix G(BG + D1D * Q1D, Q1D, D1D);
+
+      if (MFEM_THREAD_ID(y) == 0)
+      {
+         MFEM_FOREACH_THREAD(p, x, Q1D)
+         {
+            for (int d = 0; d < D1D; ++d)
+            {
+               B(p, d) = B_(p, d);
+               G(p, d) = G_(p, d);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // copy edge values to u0, u1 and copy edge normals to du0, du1
+      MFEM_FOREACH_THREAD(side, y, 2)
+      {
+         real_t *u = (side == 0) ? u0 : u1;
+         real_t *du = (side == 0) ? du0 : du1;
+         MFEM_FOREACH_THREAD(d, x, D1D)
+         {
+            u[d] = x(d, side, f);
+            du[d] = dxdn(d, side, f);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // eval @ quad points
+      MFEM_FOREACH_THREAD(side, y, 2)
+      {
+         real_t *u = (side == 0) ? u0 : u1;
+         real_t *du = (side == 0) ? du0 : du1;
+         real_t *Bu = (side == 0) ? Bu0 : Bu1;
+         real_t *Bdu = (side == 0) ? Bdu0 : Bdu1;
+
+         MFEM_FOREACH_THREAD(p, x, Q1D)
+         {
+            const real_t Je_side[] = {pa(2 + 2 * side, p, f),
+                                      pa(2 + 2 * side + 1, p, f)
+                                     };
+
+            Bu[p] = 0.0;
+            Bdu[p] = 0.0;
+
+            for (int d = 0; d < D1D; ++d)
+            {
+               const real_t b = B(p, d);
+               const real_t g = G(p, d);
+
+               Bu[p] += b * u[d];
+               Bdu[p] += Je_side[0] * b * du[d] + Je_side[1] * g * u[d];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // term - < {Q du/dn}, [v] > +  kappa * < {Q/h} [u], [v] >:
+      if (MFEM_THREAD_ID(y) == 0)
+      {
+         MFEM_FOREACH_THREAD(p, x, Q1D)
+         {
+            const real_t q = pa(0, p, f);
+            const real_t hi = pa(1, p, f);
+            const real_t jump = Bu0[p] - Bu1[p];
+            const real_t avg = Bdu0[p] + Bdu1[p]; // = {Q du/dn} * w * det(J)
+            r[p] = -avg + hi * q * jump;
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(d, x, D1D)
+      {
+         real_t Br = 0.0;
+
+         for (int p = 0; p < Q1D; ++p)
+         {
+            Br += B(p, d) * r[p];
+         }
+
+         u0[d] = Br; // overwrite u0, u1
+         u1[d] = -Br;
+      } // for d
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(side, y, 2)
+      {
+         real_t *du = (side == 0) ? du0 : du1;
+         MFEM_FOREACH_THREAD(d, x, D1D) { du[d] = 0.0; }
+      }
+      MFEM_SYNC_THREAD;
+
+      // term sigma * < [u], {Q dv/dn} >
+      MFEM_FOREACH_THREAD(side, y, 2)
+      {
+         real_t *const du = (side == 0) ? du0 : du1;
+         real_t *const u = (side == 0) ? u0 : u1;
+
+         MFEM_FOREACH_THREAD(d, x, D1D)
+         {
+            for (int p = 0; p < Q1D; ++p)
+            {
+               const real_t Je[] = {pa(2 + 2 * side, p, f),
+                                    pa(2 + 2 * side + 1, p, f)
+                                   };
+               const real_t jump = Bu0[p] - Bu1[p];
+               const real_t r_p = Je[0] * jump; // normal
+               const real_t w_p = Je[1] * jump; // tangential
+               du[d] += sigma * B(p, d) * r_p;
+               u[d] += sigma * G(p, d) * w_p;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(side, y, 2)
+      {
+         real_t *u = (side == 0) ? u0 : u1;
+         real_t *du = (side == 0) ? du0 : du1;
+         MFEM_FOREACH_THREAD(d, x, D1D)
+         {
+            y(d, side, f) += u[d];
+            dydn(d, side, f) += du[d];
+         }
+      }
+   }); // mfem::forall
+}
+
+template <int T_D1D = 0, int T_Q1D = 0>
+static void PADGDiffusionApply3D(const int NF, const Array<real_t> &b,
+                                 const Array<real_t> &bt,
+                                 const Array<real_t> &g,
+                                 const Array<real_t> &gt, const real_t sigma,
+                                 const Vector &pa_data, const Vector &x_,
+                                 const Vector &dxdn_, Vector &y_, Vector &dydn_,
+                                 const int d1d = 0, const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+
+   auto B_ = Reshape(b.Read(), Q1D, D1D);
+   auto G_ = Reshape(g.Read(), Q1D, D1D);
+
+   // (J0[0], J0[1], J0[2], J1[0], J1[1], J1[2], q/h)
+   auto pa = Reshape(pa_data.Read(), 7, Q1D, Q1D, NF);
+
+   auto x = Reshape(x_.Read(), D1D, D1D, 2, NF);
+   auto y = Reshape(y_.ReadWrite(), D1D, D1D, 2, NF);
+   auto dxdn = Reshape(dxdn_.Read(), D1D, D1D, 2, NF);
+   auto dydn = Reshape(dydn_.ReadWrite(), D1D, D1D, 2, NF);
+
+   const int NBX = std::max(D1D, Q1D);
+
+   mfem::forall_3D(NF, NBX, NBX, 2, [=] MFEM_HOST_DEVICE(int f) -> void
+   {
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      MFEM_SHARED real_t u0[max_Q1D][max_Q1D];
+      MFEM_SHARED real_t u1[max_Q1D][max_Q1D];
+
+      MFEM_SHARED real_t du0[max_Q1D][max_Q1D];
+      MFEM_SHARED real_t du1[max_Q1D][max_Q1D];
+
+      MFEM_SHARED real_t Gu0[max_Q1D][max_Q1D];
+      MFEM_SHARED real_t Gu1[max_Q1D][max_Q1D];
+
+      MFEM_SHARED real_t Bu0[max_Q1D][max_Q1D];
+      MFEM_SHARED real_t Bu1[max_Q1D][max_Q1D];
+
+      MFEM_SHARED real_t Bdu0[max_Q1D][max_Q1D];
+      MFEM_SHARED real_t Bdu1[max_Q1D][max_Q1D];
+
+      MFEM_SHARED real_t kappa_Qh[max_Q1D][max_Q1D];
+
+      MFEM_SHARED real_t nJe[2][max_Q1D][max_Q1D][3];
+      MFEM_SHARED real_t BG[2 * max_D1D * max_Q1D];
+
+      // some buffers are reused multiple times, but for clarity have new names:
+      real_t(*Bj0)[max_Q1D] = Bu0;
+      real_t(*Bj1)[max_Q1D] = Bu1;
+      real_t(*Bjn0)[max_Q1D] = Bdu0;
+      real_t(*Bjn1)[max_Q1D] = Bdu1;
+      real_t(*Gj0)[max_Q1D] = Gu0;
+      real_t(*Gj1)[max_Q1D] = Gu1;
+
+      DeviceMatrix B(BG, Q1D, D1D);
+      DeviceMatrix G(BG + D1D * Q1D, Q1D, D1D);
+
+      // copy face values to u0, u1 and copy normals to du0, du1
+      MFEM_FOREACH_THREAD(side, z, 2)
+      {
+         real_t(*u)[max_Q1D] = (side == 0) ? u0 : u1;
+         real_t(*du)[max_Q1D] = (side == 0) ? du0 : du1;
+
+         MFEM_FOREACH_THREAD(d2, x, D1D)
+         {
+            MFEM_FOREACH_THREAD(d1, y, D1D)
+            {
+               u[d2][d1] = x(d1, d2, side,
+                             f); // copy transposed for better memory access
+               du[d2][d1] = dxdn(d1, d2, side, f);
+            }
+         }
+
+         MFEM_FOREACH_THREAD(p1, x, Q1D)
+         {
+            MFEM_FOREACH_THREAD(p2, y, Q1D)
+            {
+               for (int l = 0; l < 3; ++l)
+               {
+                  nJe[side][p2][p1][l] = pa(3 * side + l, p1, p2, f);
+               }
+
+               if (side == 0)
+               {
+                  kappa_Qh[p2][p1] = pa(6, p1, p2, f);
+               }
+            }
+         }
+
+         if (side == 0)
+         {
+            MFEM_FOREACH_THREAD(p, x, Q1D)
+            {
+               MFEM_FOREACH_THREAD(d, y, D1D)
+               {
+                  B(p, d) = B_(p, d);
+                  G(p, d) = G_(p, d);
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // eval u and normal derivative @ quad points
+      MFEM_FOREACH_THREAD(side, z, 2)
+      {
+         real_t(*u)[max_Q1D] = (side == 0) ? u0 : u1;
+         real_t(*du)[max_Q1D] = (side == 0) ? du0 : du1;
+         real_t(*Bu)[max_Q1D] = (side == 0) ? Bu0 : Bu1;
+         real_t(*Bdu)[max_Q1D] = (side == 0) ? Bdu0 : Bdu1;
+         real_t(*Gu)[max_Q1D] = (side == 0) ? Gu0 : Gu1;
+
+         MFEM_FOREACH_THREAD(p1, x, Q1D)
+         {
+            MFEM_FOREACH_THREAD(d2, y, D1D)
+            {
+               real_t bu = 0.0;
+               real_t bdu = 0.0;
+               real_t gu = 0.0;
+
+               for (int d1 = 0; d1 < D1D; ++d1)
+               {
+                  const real_t b = B(p1, d1);
+                  const real_t g = G(p1, d1);
+
+                  bu += b * u[d2][d1];
+                  bdu += b * du[d2][d1];
+                  gu += g * u[d2][d1];
+               }
+
+               Bu[p1][d2] = bu;
+               Bdu[p1][d2] = bdu;
+               Gu[p1][d2] = gu;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(side, z, 2)
+      {
+         real_t(*u)[max_Q1D] = (side == 0) ? u0 : u1;
+         real_t(*du)[max_Q1D] = (side == 0) ? du0 : du1;
+         real_t(*Bu)[max_Q1D] = (side == 0) ? Bu0 : Bu1;
+         real_t(*Gu)[max_Q1D] = (side == 0) ? Gu0 : Gu1;
+         real_t(*Bdu)[max_Q1D] = (side == 0) ? Bdu0 : Bdu1;
+
+         MFEM_FOREACH_THREAD(p2, x, Q1D)
+         {
+            MFEM_FOREACH_THREAD(p1, y, Q1D)
+            {
+               const real_t *Je = nJe[side][p2][p1];
+
+               real_t bbu = 0.0;
+               real_t bgu = 0.0;
+               real_t gbu = 0.0;
+               real_t bbdu = 0.0;
+
+               for (int d2 = 0; d2 < D1D; ++d2)
+               {
+                  const real_t b = B(p2, d2);
+                  const real_t g = G(p2, d2);
+                  bbu += b * Bu[p1][d2];
+                  gbu += g * Bu[p1][d2];
+                  bgu += b * Gu[p1][d2];
+                  bbdu += b * Bdu[p1][d2];
+               }
+
+               u[p2][p1] = bbu;
+               // du <- Q du/dn * w * det(J)
+               du[p2][p1] = Je[0] * bbdu + Je[1] * bgu + Je[2] * gbu;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(side, z, 2)
+      {
+         real_t(*Bj)[max_Q1D] = (side == 0) ? Bj0 : Bj1;
+         real_t(*Bjn)[max_Q1D] = (side == 0) ? Bjn0 : Bjn1;
+         real_t(*Gj)[max_Q1D] = (side == 0) ? Gj0 : Gj1;
+
+         MFEM_FOREACH_THREAD(d1, x, D1D)
+         {
+            MFEM_FOREACH_THREAD(p2, y, Q1D)
+            {
+               real_t bj = 0.0;
+               real_t bjn = 0.0;
+               real_t gj = 0.0;
+               real_t br = 0.0;
+
+               for (int p1 = 0; p1 < Q1D; ++p1)
+               {
+                  const real_t b = B(p1, d1);
+                  const real_t g = G(p1, d1);
+
+                  const real_t *Je = nJe[side][p2][p1];
+
+                  const real_t jump = u0[p2][p1] - u1[p2][p1];
+                  const real_t avg = du0[p2][p1] + du1[p2][p1];
+
+                  // r = - < {Q du/dn}, [v] > + kappa * < {Q/h} [u], [v] >
+                  const real_t r = -avg + kappa_Qh[p2][p1] * jump;
+
+                  // bj, gj, bjn contribute to sigma term
+                  bj += b * Je[0] * jump;
+                  gj += g * Je[1] * jump;
+                  bjn += b * Je[2] * jump;
+
+                  br += b * r;
+               }
+
+               Bj[d1][p2] = sigma * bj;
+               Bjn[d1][p2] = sigma * bjn;
+
+               // group br and gj together since we will multiply them both by B
+               // and then sum
+               const real_t sgn = (side == 0) ? 1.0 : -1.0;
+               Gj[d1][p2] = sgn * br + sigma * gj;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(side, z, 2)
+      {
+         real_t(*u)[max_Q1D] = (side == 0) ? u0 : u1;
+         real_t(*du)[max_Q1D] = (side == 0) ? du0 : du1;
+         real_t(*Bj)[max_Q1D] = (side == 0) ? Bj0 : Bj1;
+         real_t(*Bjn)[max_Q1D] = (side == 0) ? Bjn0 : Bjn1;
+         real_t(*Gj)[max_Q1D] = (side == 0) ? Gj0 : Gj1;
+
+         MFEM_FOREACH_THREAD(d2, x, D1D)
+         {
+            MFEM_FOREACH_THREAD(d1, y, D1D)
+            {
+               real_t bbj = 0.0;
+               real_t gbj = 0.0;
+               real_t bgj = 0.0;
+
+               for (int p2 = 0; p2 < Q1D; ++p2)
+               {
+                  const real_t b = B(p2, d2);
+                  const real_t g = G(p2, d2);
+
+                  bbj += b * Bj[d1][p2];
+                  bgj += b * Gj[d1][p2];
+                  gbj += g * Bjn[d1][p2];
+               }
+
+               du[d2][d1] = bbj;
+               u[d2][d1] = bgj + gbj;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // map back to y and dydn
+      MFEM_FOREACH_THREAD(side, z, 2)
+      {
+         const real_t(*u)[max_Q1D] = (side == 0) ? u0 : u1;
+         const real_t(*du)[max_Q1D] = (side == 0) ? du0 : du1;
+
+         MFEM_FOREACH_THREAD(d2, x, D1D)
+         {
+            MFEM_FOREACH_THREAD(d1, y, D1D)
+            {
+               y(d1, d2, side, f) += u[d2][d1];
+               dydn(d1, d2, side, f) += du[d2][d1];
+            }
+         }
+      }
+   });
+}
+
+} // namespace internal
+
+template <int DIM, int D1D, int Q1D>
+DGDiffusionIntegrator::ApplyKernelType
+DGDiffusionIntegrator::ApplyPAKernels::Kernel()
+{
+   if constexpr (DIM == 2)
+   {
+      return internal::PADGDiffusionApply2D<D1D, Q1D>;
+   }
+   else if constexpr (DIM == 3)
+   {
+      return internal::PADGDiffusionApply3D<D1D, Q1D>;
+   }
+   MFEM_ABORT("");
+}
+} // namespace mfem
+/// \endcond DO_NOT_DOCUMENT
+#endif

fem/integ/bilininteg_dgdiffusion_pa.cpp

@@ -11,42 +11,39 @@
 
 #include "../../general/forall.hpp"
 #include "../../mesh/face_nbr_geom.hpp"
+#include "../fe/face_map_utils.hpp"
 #include "../gridfunc.hpp"
 #include "../qfunction.hpp"
-#include "../fe/face_map_utils.hpp"
 
-using namespace std;
+#include "bilininteg_dgdiffusion_kernels.hpp"
 
 namespace mfem
 {
 
-static void PADGDiffusionSetup2D(const int Q1D,
-                                 const int NE,
-                                 const int NF,
+static void PADGDiffusionSetup2D(const int Q1D, const int NE, const int NF,
                                  const Array<real_t> &w,
                                  const GeometricFactors &el_geom,
                                  const FaceGeometricFactors &face_geom,
                                  const FaceNeighborGeometricFactors *nbr_geom,
-                                 const Vector &q,
-                                 const real_t sigma,
-                                 const real_t kappa,
-                                 Vector &pa_data,
+                                 const Vector &q, const real_t sigma,
+                                 const real_t kappa, Vector &pa_data,
                                  const Array<int> &face_info_)
 {
    const auto J_loc = Reshape(el_geom.J.Read(), Q1D, Q1D, 2, 2, NE);
    const auto detJe_loc = Reshape(el_geom.detJ.Read(), Q1D, Q1D, NE);
 
    const int n_nbr = nbr_geom ? nbr_geom->num_neighbor_elems : 0;
-   const auto J_shared = Reshape(nbr_geom ? nbr_geom->J.Read() : nullptr,
-                                 Q1D, Q1D, 2, 2, n_nbr);
-   const auto detJ_shared = Reshape(nbr_geom ? nbr_geom->detJ.Read() : nullptr,
-                                    Q1D, Q1D, n_nbr);
+   const auto J_shared =
+      Reshape(nbr_geom ? nbr_geom->J.Read() : nullptr, Q1D, Q1D, 2, 2, n_nbr);
+   const auto detJ_shared =
+      Reshape(nbr_geom ? nbr_geom->detJ.Read() : nullptr, Q1D, Q1D, n_nbr);
 
    const auto detJf = Reshape(face_geom.detJ.Read(), Q1D, NF);
    const auto n = Reshape(face_geom.normal.Read(), Q1D, 2, NF);
 
    const bool const_q = (q.Size() == 1);
-   const auto Q = const_q ? Reshape(q.Read(), 1,1) : Reshape(q.Read(), Q1D,NF);
+   const auto Q =
+      const_q ? Reshape(q.Read(), 1, 1) : Reshape(q.Read(), Q1D, NF);
 
    const auto W = w.Read();
 
@@ -56,7 +53,7 @@ static void PADGDiffusionSetup2D(const int Q1D,
    // (q, 1/h, J0_0, J0_1, J1_0, J1_1)
    auto pa = Reshape(pa_data.Write(), 6, Q1D, NF);
 
-   mfem::forall(NF, [=] MFEM_HOST_DEVICE (int f) -> void
+   mfem::forall(NF, [=] MFEM_HOST_DEVICE(int f) -> void
    {
       const int normal_dir[] = {face_info(0, f), face_info(1, f)};
       const int fid[] = {face_info(4, f), face_info(5, f)};
@@ -74,7 +71,7 @@ static void PADGDiffusionSetup2D(const int Q1D,
 
       for (int p = 0; p < Q1D; ++p)
       {
-         const real_t Qp = const_q ? Q(0,0) : Q(p, f);
+         const real_t Qp = const_q ? Q(0, 0) : Q(p, f);
          pa(0, p, f) = kappa * Qp * W[p] * detJf(p, f);
 
          real_t hi = 0.0;
@@ -85,17 +82,19 @@ static void PADGDiffusionSetup2D(const int Q1D,
 
             // Always opposite direction in "native" ordering
             // Need to multiply the native=>lex0 with native=>lex1 and negate
-            const int sgn = (side == 1) ? -1*sgn0*sgn1 : 1;
+            const int sgn = (side == 1) ? -1 * sgn0 * sgn1 : 1;
 
             const int e = el[side];
             const auto &J = (side == 1 && shared) ? J_shared : J_loc;
             const auto &detJ = (side == 1 && shared) ? detJ_shared : detJe_loc;
 
             real_t nJi[2];
-            nJi[0] = n(p,0,f)*J(i,j, 1,1, e) - n(p,1,f)*J(i,j,0,1,e);
-            nJi[1] = -n(p,0,f)*J(i,j,1,0, e) + n(p,1,f)*J(i,j,0,0,e);
+            nJi[0] =
+               n(p, 0, f) * J(i, j, 1, 1, e) - n(p, 1, f) * J(i, j, 0, 1, e);
+            nJi[1] =
+               -n(p, 0, f) * J(i, j, 1, 0, e) + n(p, 1, f) * J(i, j, 0, 0, e);
 
-            const real_t dJe = detJ(i,j,e);
+            const real_t dJe = detJ(i, j, e);
             const real_t dJf = detJf(p, f);
 
             const real_t w = factor * Qp * W[p] * dJf / dJe;
@@ -104,9 +103,9 @@ static void PADGDiffusionSetup2D(const int Q1D,
             const int ti = 1 - ni;
 
             // Normal
-            pa(2 + 2*side + 0, p, f) = w * nJi[ni];
+            pa(2 + 2 * side + 0, p, f) = w * nJi[ni];
             // Tangential
-            pa(2 + 2*side + 1, p, f) = sgn * w * nJi[ti];
+            pa(2 + 2 * side + 1, p, f) = sgn * w * nJi[ti];
 
             hi += factor * dJf / dJe;
          }
@@ -122,47 +121,43 @@ static void PADGDiffusionSetup2D(const int Q1D,
    });
 }
 
-static void PADGDiffusionSetup3D(const int Q1D,
-                                 const int NE,
-                                 const int NF,
+static void PADGDiffusionSetup3D(const int Q1D, const int NE, const int NF,
                                  const Array<real_t> &w,
                                  const GeometricFactors &el_geom,
                                  const FaceGeometricFactors &face_geom,
                                  const FaceNeighborGeometricFactors *nbr_geom,
-                                 const Vector &q,
-                                 const real_t sigma,
-                                 const real_t kappa,
-                                 Vector &pa_data,
+                                 const Vector &q, const real_t sigma,
+                                 const real_t kappa, Vector &pa_data,
                                  const Array<int> &face_info_)
 {
    const auto J_loc = Reshape(el_geom.J.Read(), Q1D, Q1D, Q1D, 3, 3, NE);
    const auto detJe_loc = Reshape(el_geom.detJ.Read(), Q1D, Q1D, Q1D, NE);
 
    const int n_nbr = nbr_geom ? nbr_geom->num_neighbor_elems : 0;
-   const auto J_shared = Reshape(nbr_geom ? nbr_geom->J.Read() : nullptr,
-                                 Q1D, Q1D, Q1D, 3, 3, n_nbr);
-   const auto detJ_shared = Reshape(nbr_geom ? nbr_geom->detJ.Read() : nullptr,
-                                    Q1D, Q1D, Q1D, n_nbr);
+   const auto J_shared = Reshape(nbr_geom ? nbr_geom->J.Read() : nullptr, Q1D,
+                                 Q1D, Q1D, 3, 3, n_nbr);
+   const auto detJ_shared =
+      Reshape(nbr_geom ? nbr_geom->detJ.Read() : nullptr, Q1D, Q1D, Q1D, n_nbr);
 
    const auto detJf = Reshape(face_geom.detJ.Read(), Q1D, Q1D, NF);
    const auto n = Reshape(face_geom.normal.Read(), Q1D, Q1D, 3, NF);
 
    const bool const_q = (q.Size() == 1);
-   const auto Q = const_q ? Reshape(q.Read(), 1, 1, 1)
-                  : Reshape(q.Read(), Q1D, Q1D, NF);
+   const auto Q =
+      const_q ? Reshape(q.Read(), 1, 1, 1) : Reshape(q.Read(), Q1D, Q1D, NF);
 
    const auto W = Reshape(w.Read(), Q1D, Q1D);
 
    // (perm[0], perm[1], perm[2], element_index, local_face_id, orientation)
    const auto face_info = Reshape(face_info_.Read(), 6, 2, NF);
-   constexpr int _el_ = 3; // offset in face_info for element index
+   constexpr int _el_ = 3;  // offset in face_info for element index
    constexpr int _fid_ = 4; // offset in face_info for local face id
-   constexpr int _or_ = 5; // offset in face_info for orientation
+   constexpr int _or_ = 5;  // offset in face_info for orientation
 
    // (J00, J01, J02, J10, J11, J12, q/h)
    const auto pa = Reshape(pa_data.Write(), 7, Q1D, Q1D, NF);
 
-   mfem::forall_2D(NF, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int f) -> void
+   mfem::forall_2D(NF, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int f) -> void
    {
       MFEM_SHARED int perm[2][3];
       MFEM_SHARED int el[2];
@@ -172,10 +167,7 @@ static void PADGDiffusionSetup3D(const int Q1D,
 
       MFEM_FOREACH_THREAD(side, x, 2)
       {
-         MFEM_FOREACH_THREAD(i, y, 3)
-         {
-            perm[side][i] = face_info(i, side, f);
-         }
+         MFEM_FOREACH_THREAD(i, y, 3) { perm[side][i] = face_info(i, side, f); }
 
          if (MFEM_THREAD_ID(y) == 0)
          {
@@ -200,16 +192,16 @@ static void PADGDiffusionSetup3D(const int Q1D,
       {
          MFEM_FOREACH_THREAD(p2, y, Q1D)
          {
-            const real_t Qp = const_q ? Q(0,0,0) : Q(p1, p2, f);
-            const real_t dJf = detJf(p1,p2,f);
+            const real_t Qp = const_q ? Q(0, 0, 0) : Q(p1, p2, f);
+            const real_t dJf = detJf(p1, p2, f);
 
             real_t hi = 0.0;
 
             for (int side = 0; side < nsides; ++side)
             {
                int i, j, k;
-               internal::FaceIdxToVolIdx3D(
-                  p1 + Q1D*p2, Q1D, fid[0], fid[1], side, ortn[1], i, j, k);
+               internal::FaceIdxToVolIdx3D(p1 + Q1D * p2, Q1D, fid[0], fid[1],
+                                           side, ortn[1], i, j, k);
 
                const int e = el[side];
                const auto &J = shared[side] ? J_shared : J_loc;
@@ -217,27 +209,45 @@ static void PADGDiffusionSetup3D(const int Q1D,
 
                // *INDENT-OFF*
                real_t nJi[3];
-               nJi[0] = (  -J(i,j,k, 1,2, e)*J(i,j,k, 2,1, e) + J(i,j,k, 1,1, e)*J(i,j,k, 2,2, e)) * n(p1,p2, 0, f)
-                        + ( J(i,j,k, 0,2, e)*J(i,j,k, 2,1, e) - J(i,j,k, 0,1, e)*J(i,j,k, 2,2, e)) * n(p1,p2, 1, f)
-                        + (-J(i,j,k, 0,2, e)*J(i,j,k, 1,1, e) + J(i,j,k, 0,1, e)*J(i,j,k, 1,2, e)) * n(p1,p2, 2, f);
+               nJi[0] = (-J(i, j, k, 1, 2, e) * J(i, j, k, 2, 1, e) +
+                         J(i, j, k, 1, 1, e) * J(i, j, k, 2, 2, e)) *
+                           n(p1, p2, 0, f) +
+                        (J(i, j, k, 0, 2, e) * J(i, j, k, 2, 1, e) -
+                         J(i, j, k, 0, 1, e) * J(i, j, k, 2, 2, e)) *
+                           n(p1, p2, 1, f) +
+                        (-J(i, j, k, 0, 2, e) * J(i, j, k, 1, 1, e) +
+                         J(i, j, k, 0, 1, e) * J(i, j, k, 1, 2, e)) *
+                           n(p1, p2, 2, f);
 
-               nJi[1] = (   J(i,j,k, 1,2, e)*J(i,j,k, 2,0, e) - J(i,j,k, 1,0, e)*J(i,j,k, 2,2, e)) * n(p1,p2, 0, f)
-                        + (-J(i,j,k, 0,2, e)*J(i,j,k, 2,0, e) + J(i,j,k, 0,0, e)*J(i,j,k, 2,2, e)) * n(p1,p2, 1, f)
-                        + ( J(i,j,k, 0,2, e)*J(i,j,k, 1,0, e) - J(i,j,k, 0,0, e)*J(i,j,k, 1,2, e)) * n(p1,p2, 2, f);
+               nJi[1] = (J(i, j, k, 1, 2, e) * J(i, j, k, 2, 0, e) -
+                         J(i, j, k, 1, 0, e) * J(i, j, k, 2, 2, e)) *
+                           n(p1, p2, 0, f) +
+                        (-J(i, j, k, 0, 2, e) * J(i, j, k, 2, 0, e) +
+                         J(i, j, k, 0, 0, e) * J(i, j, k, 2, 2, e)) *
+                           n(p1, p2, 1, f) +
+                        (J(i, j, k, 0, 2, e) * J(i, j, k, 1, 0, e) -
+                         J(i, j, k, 0, 0, e) * J(i, j, k, 1, 2, e)) *
+                           n(p1, p2, 2, f);
 
-               nJi[2] = (  -J(i,j,k, 1,1, e)*J(i,j,k, 2,0, e) + J(i,j,k, 1,0, e)*J(i,j,k, 2,1, e)) * n(p1,p2, 0, f)
-                        + ( J(i,j,k, 0,1, e)*J(i,j,k, 2,0, e) - J(i,j,k, 0,0, e)*J(i,j,k, 2,1, e)) * n(p1,p2, 1, f)
-                        + (-J(i,j,k, 0,1, e)*J(i,j,k, 1,0, e) + J(i,j,k, 0,0, e)*J(i,j,k, 1,1, e)) * n(p1,p2, 2, f);
+               nJi[2] = (-J(i, j, k, 1, 1, e) * J(i, j, k, 2, 0, e) +
+                         J(i, j, k, 1, 0, e) * J(i, j, k, 2, 1, e)) *
+                           n(p1, p2, 0, f) +
+                        (J(i, j, k, 0, 1, e) * J(i, j, k, 2, 0, e) -
+                         J(i, j, k, 0, 0, e) * J(i, j, k, 2, 1, e)) *
+                           n(p1, p2, 1, f) +
+                        (-J(i, j, k, 0, 1, e) * J(i, j, k, 1, 0, e) +
+                         J(i, j, k, 0, 0, e) * J(i, j, k, 1, 1, e)) *
+                           n(p1, p2, 2, f);
                // *INDENT-ON*
 
-               const real_t dJe = detJe(i,j,k,e);
+               const real_t dJe = detJe(i, j, k, e);
                const real_t val = factor * Qp * W(p1, p2) * dJf / dJe;
 
                for (int d = 0; d < 3; ++d)
                {
                   const int idx = std::abs(perm[side][d]) - 1;
                   const int sgn = (perm[side][d] < 0) ? -1 : 1;
-                  pa(3*side + d, p1, p2, f) = sgn * val * nJi[idx];
+                  pa(3 * side + d, p1, p2, f) = sgn * val * nJi[idx];
                }
 
                hi += factor * dJf / dJe;
@@ -257,7 +267,8 @@ static void PADGDiffusionSetup3D(const int Q1D,
 }
 
 static void PADGDiffusionSetupFaceInfo2D(const int nf, const Mesh &mesh,
-                                         const FaceType type, Array<int> &face_info_)
+                                         const FaceType type,
+                                         Array<int> &face_info_)
 {
    const int ne = mesh.GetNE();
 
@@ -326,8 +337,7 @@ inline void FaceNormalPermutation(int perm[3], const int face_id)
 
 // Assigns to perm the permutation as in FaceNormalPermutation for the second
 // element on the face but signed to indicate the sign of the normal derivative.
-inline void SignedFaceNormalPermutation(int perm[3],
-                                        const int face_id1,
+inline void SignedFaceNormalPermutation(int perm[3], const int face_id1,
                                         const int face_id2,
                                         const int orientation)
 {
@@ -386,17 +396,19 @@ inline void SignedFaceNormalPermutation(int perm[3],
 }
 
 static void PADGDiffusionSetupFaceInfo3D(const int nf, const Mesh &mesh,
-                                         const FaceType type, Array<int> &face_info_)
+                                         const FaceType type,
+                                         Array<int> &face_info_)
 {
    const int ne = mesh.GetNE();
 
    int fidx = 0;
-   // face_info array has 12 entries per face, 6 for each of the adjacent elements:
-   // (perm[0], perm[1], perm[2], element_index, local_face_id, orientation)
+   // face_info array has 12 entries per face, 6 for each of the adjacent
+   // elements: (perm[0], perm[1], perm[2], element_index, local_face_id,
+   // orientation)
    face_info_.SetSize(nf * 12);
-   constexpr int _e_ = 3; // offset for element index
+   constexpr int _e_ = 3;   // offset for element index
    constexpr int _fid_ = 4; // offset for local face id
-   constexpr int _or_ = 5; // offset for orientation
+   constexpr int _or_ = 5;  // offset for orientation
 
    auto face_info = Reshape(face_info_.HostWrite(), 6, 2, nf);
    for (int f = 0; f < mesh.GetNumFaces(); ++f)
@@ -408,9 +420,9 @@ static void PADGDiffusionSetupFaceInfo3D(const int nf, const Mesh &mesh,
          const int fid0 = f_info.element[0].local_face_id;
          const int or0 = f_info.element[0].orientation;
 
-         face_info(  _e_, 0, fidx) = f_info.element[0].index;
+         face_info(_e_, 0, fidx) = f_info.element[0].index;
          face_info(_fid_, 0, fidx) = fid0;
-         face_info( _or_, 0, fidx) = or0;
+         face_info(_or_, 0, fidx) = or0;
 
          FaceNormalPermutation(&face_info(0, 0, fidx), fid0);
 
@@ -421,16 +433,17 @@ static void PADGDiffusionSetupFaceInfo3D(const int nf, const Mesh &mesh,
 
             if (f_info.IsShared())
             {
-               face_info(  _e_, 1, fidx) = ne + f_info.element[1].index;
+               face_info(_e_, 1, fidx) = ne + f_info.element[1].index;
             }
             else
             {
-               face_info(  _e_, 1, fidx) = f_info.element[1].index;
+               face_info(_e_, 1, fidx) = f_info.element[1].index;
             }
             face_info(_fid_, 1, fidx) = fid1;
-            face_info( _or_, 1, fidx) = or1;
+            face_info(_or_, 1, fidx) = or1;
 
-            SignedFaceNormalPermutation(&face_info(0, 1, fidx), fid0, fid1, or1);
+            SignedFaceNormalPermutation(&face_info(0, 1, fidx), fid0, fid1,
+                                        or1);
          }
          else
          {
@@ -448,8 +461,8 @@ static void PADGDiffusionSetupFaceInfo3D(const int nf, const Mesh &mesh,
 void DGDiffusionIntegrator::SetupPA(const FiniteElementSpace &fes,
                                     FaceType type)
 {
-   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
-                         Device::GetDeviceMemoryType() : pa_mt;
+   const MemoryType mt =
+      (pa_mt == MemoryType::DEFAULT) ? Device::GetDeviceMemoryType() : pa_mt;
 
    const int ne = fes.GetNE();
    nf = fes.GetNFbyType(type);
@@ -458,16 +471,17 @@ void DGDiffusionIntegrator::SetupPA(const FiniteElementSpace &fes,
    Mesh &mesh = *fes.GetMesh();
    const Geometry::Type face_geom_type = mesh.GetTypicalFaceGeometry();
    const FiniteElement &el = *fes.GetTypicalTraceElement();
-   const int ir_order = IntRule ? IntRule->GetOrder()
+   const int ir_order = IntRule
+                        ? IntRule->GetOrder()
                         : GetRule(el.GetOrder(), face_geom_type).GetOrder();
    const IntegrationRule &ir = irs.Get(face_geom_type, ir_order);
    dim = mesh.Dimension();
-   const int q1d = (ir.GetOrder() + 3)/2;
-   MFEM_ASSERT(q1d == pow(real_t(ir.Size()), 1.0/(dim - 1)), "");
+   const int q1d = (ir.GetOrder() + 3) / 2;
+   MFEM_ASSERT(q1d == pow(real_t(ir.Size()), 1.0 / (dim - 1)), "");
 
    const auto vol_ir = irs.Get(mesh.GetTypicalElementGeometry(), ir_order);
-   const auto geom_flags = GeometricFactors::JACOBIANS |
-                           GeometricFactors::DETERMINANTS;
+   const auto geom_flags =
+      GeometricFactors::JACOBIANS | GeometricFactors::DETERMINANTS;
    const auto el_geom = mesh.GetGeometricFactors(vol_ir, geom_flags, mt);
 
    std::unique_ptr<FaceNeighborGeometricFactors> nbr_geom;
@@ -476,8 +490,8 @@ void DGDiffusionIntegrator::SetupPA(const FiniteElementSpace &fes,
       nbr_geom.reset(new FaceNeighborGeometricFactors(*el_geom));
    }
 
-   const auto face_geom_flags = FaceGeometricFactors::DETERMINANTS |
-                                FaceGeometricFactors::NORMALS;
+   const auto face_geom_flags =
+      FaceGeometricFactors::DETERMINANTS | FaceGeometricFactors::NORMALS;
    auto face_geom = mesh.GetFaceGeometricFactors(ir, face_geom_flags, type, mt);
    maps = &el.GetDofToQuad(ir, DofToQuad::TENSOR);
    dofs1D = maps->ndof;
@@ -489,9 +503,18 @@ void DGDiffusionIntegrator::SetupPA(const FiniteElementSpace &fes,
    // Evaluate the coefficient at the face quadrature points.
    FaceQuadratureSpace fqs(mesh, ir, type);
    CoefficientVector q(fqs, CoefficientStorage::COMPRESSED);
-   if (Q) { q.Project(*Q); }
-   else if (MQ) { MFEM_ABORT("Not yet implemented"); /* q.Project(*MQ); */ }
-   else { q.SetConstant(1.0); }
+   if (Q)
+   {
+      q.Project(*Q);
+   }
+   else if (MQ)
+   {
+      MFEM_ABORT("Not yet implemented"); /* q.Project(*MQ); */
+   }
+   else
+   {
+      q.SetConstant(1.0);
+   }
 
    Array<int> face_info;
    if (dim == 1)
@@ -501,14 +524,16 @@ void DGDiffusionIntegrator::SetupPA(const FiniteElementSpace &fes,
    else if (dim == 2)
    {
       PADGDiffusionSetupFaceInfo2D(nf, mesh, type, face_info);
-      PADGDiffusionSetup2D(quad1D, ne, nf, ir.GetWeights(), *el_geom, *face_geom,
-                           nbr_geom.get(), q, sigma, kappa, pa_data, face_info);
+      PADGDiffusionSetup2D(quad1D, ne, nf, ir.GetWeights(), *el_geom,
+                           *face_geom, nbr_geom.get(), q, sigma, kappa, pa_data,
+                           face_info);
    }
    else if (dim == 3)
    {
       PADGDiffusionSetupFaceInfo3D(nf, mesh, type, face_info);
-      PADGDiffusionSetup3D(quad1D, ne, nf, ir.GetWeights(), *el_geom, *face_geom,
-                           nbr_geom.get(), q, sigma, kappa, pa_data, face_info);
+      PADGDiffusionSetup3D(quad1D, ne, nf, ir.GetWeights(), *el_geom,
+                           *face_geom, nbr_geom.get(), q, sigma, kappa, pa_data,
+                           face_info);
    }
 }
 
@@ -524,529 +549,76 @@ void DGDiffusionIntegrator::AssemblePABoundaryFaces(
    SetupPA(fes, FaceType::Boundary);
 }
 
-template<int T_D1D = 0, int T_Q1D = 0> static
-void PADGDiffusionApply2D(const int NF,
-                          const Array<real_t> &b,
-                          const Array<real_t> &bt,
-                          const Array<real_t>& g,
-                          const Array<real_t>& gt,
-                          const real_t sigma,
-                          const Vector &pa_data,
-                          const Vector &x_,
-                          const Vector &dxdn_,
-                          Vector &y_,
-                          Vector &dydn_,
-                          const int d1d = 0,
-                          const int q1d = 0)
+void DGDiffusionIntegrator::AddMultPAFaceNormalDerivatives(const Vector &x,
+                                                           const Vector &dxdn,
+                                                           Vector &y,
+                                                           Vector &dydn) const
 {
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   auto B_ = Reshape(b.Read(), Q1D, D1D);
-   auto G_ = Reshape(g.Read(), Q1D, D1D);
-
-   auto pa = Reshape(pa_data.Read(), 6, Q1D, NF); // (q, 1/h, J00, J01, J10, J11)
-
-   auto x =    Reshape(x_.Read(),         D1D, 2, NF);
-   auto y =    Reshape(y_.ReadWrite(),    D1D, 2, NF);
-   auto dxdn = Reshape(dxdn_.Read(),      D1D, 2, NF);
-   auto dydn = Reshape(dydn_.ReadWrite(), D1D, 2, NF);
-
-   const int NBX = std::max(D1D, Q1D);
-
-   mfem::forall_2D(NF, NBX, 2, [=] MFEM_HOST_DEVICE (int f) -> void
-   {
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t u0[max_D1D];
-      MFEM_SHARED real_t u1[max_D1D];
-      MFEM_SHARED real_t du0[max_D1D];
-      MFEM_SHARED real_t du1[max_D1D];
-
-      MFEM_SHARED real_t Bu0[max_Q1D];
-      MFEM_SHARED real_t Bu1[max_Q1D];
-      MFEM_SHARED real_t Bdu0[max_Q1D];
-      MFEM_SHARED real_t Bdu1[max_Q1D];
-
-      MFEM_SHARED real_t r[max_Q1D];
-
-      MFEM_SHARED real_t BG[2*max_D1D*max_Q1D];
-      DeviceMatrix B(BG, Q1D, D1D);
-      DeviceMatrix G(BG + D1D*Q1D, Q1D, D1D);
-
-      if (MFEM_THREAD_ID(y) == 0)
-      {
-         MFEM_FOREACH_THREAD(p,x,Q1D)
-         {
-            for (int d = 0; d < D1D; ++d)
-            {
-               B(p,d) = B_(p,d);
-               G(p,d) = G_(p,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // copy edge values to u0, u1 and copy edge normals to du0, du1
-      MFEM_FOREACH_THREAD(side,y,2)
-      {
-         real_t *u = (side == 0) ? u0 : u1;
-         real_t *du = (side == 0) ? du0 : du1;
-         MFEM_FOREACH_THREAD(d,x,D1D)
-         {
-            u[d] = x(d, side, f);
-            du[d] = dxdn(d, side, f);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // eval @ quad points
-      MFEM_FOREACH_THREAD(side,y,2)
-      {
-         real_t *u = (side == 0) ? u0 : u1;
-         real_t *du = (side == 0) ? du0 : du1;
-         real_t *Bu = (side == 0) ? Bu0 : Bu1;
-         real_t *Bdu = (side == 0) ? Bdu0 : Bdu1;
-
-         MFEM_FOREACH_THREAD(p,x,Q1D)
-         {
-            const real_t Je_side[] = {pa(2 + 2*side, p, f), pa(2 + 2*side + 1, p, f)};
-
-            Bu[p] = 0.0;
-            Bdu[p] = 0.0;
-
-            for (int d = 0; d < D1D; ++d)
-            {
-               const real_t b = B(p,d);
-               const real_t g = G(p,d);
-
-               Bu[p] += b*u[d];
-               Bdu[p] += Je_side[0] * b * du[d] + Je_side[1] * g * u[d];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // term - < {Q du/dn}, [v] > +  kappa * < {Q/h} [u], [v] >:
-      if (MFEM_THREAD_ID(y) == 0)
-      {
-         MFEM_FOREACH_THREAD(p,x,Q1D)
-         {
-            const real_t q = pa(0, p, f);
-            const real_t hi = pa(1, p, f);
-            const real_t jump = Bu0[p] - Bu1[p];
-            const real_t avg = Bdu0[p] + Bdu1[p]; // = {Q du/dn} * w * det(J)
-            r[p] = -avg + hi * q * jump;
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(d,x,D1D)
-      {
-         real_t Br = 0.0;
-
-         for (int p = 0; p < Q1D; ++p)
-         {
-            Br += B(p, d) * r[p];
-         }
-
-         u0[d] =  Br; // overwrite u0, u1
-         u1[d] = -Br;
-      } // for d
-      MFEM_SYNC_THREAD;
-
-
-      MFEM_FOREACH_THREAD(side,y,2)
-      {
-         real_t *du = (side == 0) ? du0 : du1;
-         MFEM_FOREACH_THREAD(d,x,D1D)
-         {
-            du[d] = 0.0;
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // term sigma * < [u], {Q dv/dn} >
-      MFEM_FOREACH_THREAD(side,y,2)
-      {
-         real_t * const du = (side == 0) ? du0 : du1;
-         real_t * const u = (side == 0) ? u0 : u1;
-
-         MFEM_FOREACH_THREAD(d,x,D1D)
-         {
-            for (int p = 0; p < Q1D; ++p)
-            {
-               const real_t Je[] = {pa(2 + 2*side, p, f), pa(2 + 2*side + 1, p, f)};
-               const real_t jump = Bu0[p] - Bu1[p];
-               const real_t r_p = Je[0] * jump; // normal
-               const real_t w_p = Je[1] * jump; // tangential
-               du[d] += sigma * B(p, d) * r_p;
-               u[d] += sigma * G(p, d) * w_p;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(side,y,2)
-      {
-         real_t *u = (side == 0) ? u0 : u1;
-         real_t *du = (side == 0) ? du0 : du1;
-         MFEM_FOREACH_THREAD(d,x,D1D)
-         {
-            y(d, side, f) += u[d];
-            dydn(d, side, f) += du[d];
-         }
-      }
-   }); // mfem::forall
+   ApplyPAKernels::Run(dim, dofs1D, quad1D, nf, maps->B, maps->Bt, maps->G,
+                       maps->Gt, sigma, pa_data, x, dxdn, y, dydn, dofs1D,
+                       quad1D);
 }
 
-template <int T_D1D = 0, int T_Q1D = 0>
-static void PADGDiffusionApply3D(const int NF,
-                                 const Array<real_t>& b,
-                                 const Array<real_t>& bt,
-                                 const Array<real_t>& g,
-                                 const Array<real_t>& gt,
-                                 const real_t sigma,
-                                 const Vector& pa_data,
-                                 const Vector& x_,
-                                 const Vector& dxdn_,
-                                 Vector& y_,
-                                 Vector& dydn_,
-                                 const int d1d = 0,
-                                 const int q1d = 0)
+DGDiffusionIntegrator::DGDiffusionIntegrator(const real_t s, const real_t k)
+   : sigma(s), kappa(k)
 {
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   auto B_ = Reshape(b.Read(), Q1D, D1D);
-   auto G_ = Reshape(g.Read(), Q1D, D1D);
-
-   // (J0[0], J0[1], J0[2], J1[0], J1[1], J1[2], q/h)
-   auto pa = Reshape(pa_data.Read(), 7, Q1D, Q1D, NF);
-
-   auto x =    Reshape(x_.Read(),         D1D, D1D, 2, NF);
-   auto y =    Reshape(y_.ReadWrite(),    D1D, D1D, 2, NF);
-   auto dxdn = Reshape(dxdn_.Read(),      D1D, D1D, 2, NF);
-   auto dydn = Reshape(dydn_.ReadWrite(), D1D, D1D, 2, NF);
-
-   const int NBX = std::max(D1D, Q1D);
-
-   mfem::forall_3D(NF, NBX, NBX, 2, [=] MFEM_HOST_DEVICE (int f) -> void
-   {
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t u0[max_Q1D][max_Q1D];
-      MFEM_SHARED real_t u1[max_Q1D][max_Q1D];
-
-      MFEM_SHARED real_t du0[max_Q1D][max_Q1D];
-      MFEM_SHARED real_t du1[max_Q1D][max_Q1D];
-
-      MFEM_SHARED real_t Gu0[max_Q1D][max_Q1D];
-      MFEM_SHARED real_t Gu1[max_Q1D][max_Q1D];
-
-      MFEM_SHARED real_t Bu0[max_Q1D][max_Q1D];
-      MFEM_SHARED real_t Bu1[max_Q1D][max_Q1D];
-
-      MFEM_SHARED real_t Bdu0[max_Q1D][max_Q1D];
-      MFEM_SHARED real_t Bdu1[max_Q1D][max_Q1D];
-
-      MFEM_SHARED real_t kappa_Qh[max_Q1D][max_Q1D];
-
-      MFEM_SHARED real_t nJe[2][max_Q1D][max_Q1D][3];
-      MFEM_SHARED real_t BG[2*max_D1D*max_Q1D];
-
-      // some buffers are reused multiple times, but for clarity have new names:
-      real_t (*Bj0)[max_Q1D] = Bu0;
-      real_t (*Bj1)[max_Q1D] = Bu1;
-      real_t (*Bjn0)[max_Q1D] = Bdu0;
-      real_t (*Bjn1)[max_Q1D] = Bdu1;
-      real_t (*Gj0)[max_Q1D] = Gu0;
-      real_t (*Gj1)[max_Q1D] = Gu1;
-
-      DeviceMatrix B(BG, Q1D, D1D);
-      DeviceMatrix G(BG + D1D*Q1D, Q1D, D1D);
-
-      // copy face values to u0, u1 and copy normals to du0, du1
-      MFEM_FOREACH_THREAD(side, z, 2)
-      {
-         real_t (*u)[max_Q1D] = (side == 0) ? u0 : u1;
-         real_t (*du)[max_Q1D] = (side == 0) ? du0 : du1;
-
-         MFEM_FOREACH_THREAD(d2, x, D1D)
-         {
-            MFEM_FOREACH_THREAD(d1, y, D1D)
-            {
-               u[d2][d1] = x(d1, d2, side, f); // copy transposed for better memory access
-               du[d2][d1] = dxdn(d1, d2, side, f);
-            }
-         }
-
-         MFEM_FOREACH_THREAD(p1, x, Q1D)
-         {
-            MFEM_FOREACH_THREAD(p2, y, Q1D)
-            {
-               for (int l=0; l < 3; ++l)
-               {
-                  nJe[side][p2][p1][l] = pa(3*side + l, p1, p2, f);
-               }
-
-               if (side == 0)
-               {
-                  kappa_Qh[p2][p1] = pa(6, p1, p2, f);
-               }
-            }
-         }
-
-         if (side == 0)
-         {
-            MFEM_FOREACH_THREAD(p, x, Q1D)
-            {
-               MFEM_FOREACH_THREAD(d, y, D1D)
-               {
-                  B(p, d) = B_(p, d);
-                  G(p, d) = G_(p, d);
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // eval u and normal derivative @ quad points
-      MFEM_FOREACH_THREAD(side, z, 2)
-      {
-         real_t (*u)[max_Q1D] = (side == 0) ? u0 : u1;
-         real_t (*du)[max_Q1D] = (side == 0) ? du0 : du1;
-         real_t (*Bu)[max_Q1D] = (side == 0) ? Bu0 : Bu1;
-         real_t (*Bdu)[max_Q1D] = (side == 0) ? Bdu0 : Bdu1;
-         real_t (*Gu)[max_Q1D] = (side == 0) ? Gu0 : Gu1;
-
-         MFEM_FOREACH_THREAD(p1, x, Q1D)
-         {
-            MFEM_FOREACH_THREAD(d2, y, D1D)
-            {
-               real_t bu = 0.0;
-               real_t bdu = 0.0;
-               real_t gu = 0.0;
-
-               for (int d1=0; d1 < D1D; ++d1)
-               {
-                  const real_t b = B(p1, d1);
-                  const real_t g = G(p1, d1);
-
-                  bu += b * u[d2][d1];
-                  bdu += b * du[d2][d1];
-                  gu += g * u[d2][d1];
-               }
-
-               Bu[p1][d2] = bu;
-               Bdu[p1][d2] = bdu;
-               Gu[p1][d2] = gu;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(side, z, 2)
-      {
-         real_t (*u)[max_Q1D] = (side == 0) ? u0 : u1;
-         real_t (*du)[max_Q1D] = (side == 0) ? du0 : du1;
-         real_t (*Bu)[max_Q1D] = (side == 0) ? Bu0 : Bu1;
-         real_t (*Gu)[max_Q1D] = (side == 0) ? Gu0 : Gu1;
-         real_t (*Bdu)[max_Q1D] = (side == 0) ? Bdu0 : Bdu1;
-
-         MFEM_FOREACH_THREAD(p2, x, Q1D)
-         {
-            MFEM_FOREACH_THREAD(p1, y, Q1D)
-            {
-               const real_t * Je = nJe[side][p2][p1];
-
-               real_t bbu = 0.0;
-               real_t bgu = 0.0;
-               real_t gbu = 0.0;
-               real_t bbdu = 0.0;
-
-               for (int d2 = 0; d2 < D1D; ++d2)
-               {
-                  const real_t b = B(p2, d2);
-                  const real_t g = G(p2, d2);
-                  bbu += b * Bu[p1][d2];
-                  gbu += g * Bu[p1][d2];
-                  bgu += b * Gu[p1][d2];
-                  bbdu += b * Bdu[p1][d2];
-               }
-
-               u[p2][p1] = bbu;
-               // du <- Q du/dn * w * det(J)
-               du[p2][p1] = Je[0] * bbdu + Je[1] * bgu + Je[2] * gbu;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(side, z, 2)
-      {
-         real_t (*Bj)[max_Q1D] = (side == 0) ? Bj0 : Bj1;
-         real_t (*Bjn)[max_Q1D] = (side == 0) ? Bjn0 : Bjn1;
-         real_t (*Gj)[max_Q1D] = (side == 0) ? Gj0 : Gj1;
-
-         MFEM_FOREACH_THREAD(d1, x, D1D)
-         {
-            MFEM_FOREACH_THREAD(p2, y, Q1D)
-            {
-               real_t bj = 0.0;
-               real_t bjn = 0.0;
-               real_t gj = 0.0;
-               real_t br = 0.0;
-
-               for (int p1 = 0; p1 < Q1D; ++p1)
-               {
-                  const real_t b = B(p1, d1);
-                  const real_t g = G(p1, d1);
-
-                  const real_t * Je = nJe[side][p2][p1];
-
-                  const real_t jump = u0[p2][p1] - u1[p2][p1];
-                  const real_t avg = du0[p2][p1] + du1[p2][p1];
-
-                  // r = - < {Q du/dn}, [v] > + kappa * < {Q/h} [u], [v] >
-                  const real_t r = -avg + kappa_Qh[p2][p1] * jump;
-
-                  // bj, gj, bjn contribute to sigma term
-                  bj += b * Je[0] * jump;
-                  gj += g * Je[1] * jump;
-                  bjn += b * Je[2] * jump;
-
-                  br += b * r;
-               }
-
-               Bj[d1][p2] = sigma * bj;
-               Bjn[d1][p2] = sigma * bjn;
-
-               // group br and gj together since we will multiply them both by B
-               // and then sum
-               const real_t sgn = (side == 0) ? 1.0 : -1.0;
-               Gj[d1][p2] = sgn * br + sigma * gj;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(side, z, 2)
-      {
-         real_t (*u)[max_Q1D] = (side == 0) ? u0 : u1;
-         real_t (*du)[max_Q1D] = (side == 0) ? du0 : du1;
-         real_t (*Bj)[max_Q1D] = (side == 0) ? Bj0 : Bj1;
-         real_t (*Bjn)[max_Q1D] = (side == 0) ? Bjn0 : Bjn1;
-         real_t (*Gj)[max_Q1D] = (side == 0) ? Gj0 : Gj1;
-
-         MFEM_FOREACH_THREAD(d2, x, D1D)
-         {
-            MFEM_FOREACH_THREAD(d1, y, D1D)
-            {
-               real_t bbj = 0.0;
-               real_t gbj = 0.0;
-               real_t bgj = 0.0;
-
-               for (int p2 = 0; p2 < Q1D; ++p2)
-               {
-                  const real_t b = B(p2, d2);
-                  const real_t g = G(p2, d2);
-
-                  bbj += b * Bj[d1][p2];
-                  bgj += b * Gj[d1][p2];
-                  gbj += g * Bjn[d1][p2];
-               }
-
-               du[d2][d1] = bbj;
-               u[d2][d1] = bgj + gbj;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // map back to y and dydn
-      MFEM_FOREACH_THREAD(side, z, 2)
-      {
-         const real_t (*u)[max_Q1D] = (side == 0) ? u0 : u1;
-         const real_t (*du)[max_Q1D] = (side == 0) ? du0 : du1;
-
-         MFEM_FOREACH_THREAD(d2, x, D1D)
-         {
-            MFEM_FOREACH_THREAD(d1, y, D1D)
-            {
-               y(d1, d2, side, f) += u[d2][d1];
-               dydn(d1, d2, side, f) += du[d2][d1];
-            }
-         }
-      }
-   });
+   static Kernels kernels;
 }
 
-static void PADGDiffusionApply(const int dim,
-                               const int D1D,
-                               const int Q1D,
-                               const int NF,
-                               const Array<real_t> &B,
-                               const Array<real_t> &Bt,
-                               const Array<real_t> &G,
-                               const Array<real_t> &Gt,
-                               const real_t sigma,
-                               const Vector &pa_data,
-                               const Vector &x,
-                               const Vector &dxdn,
-                               Vector &y,
-                               Vector &dydn)
+DGDiffusionIntegrator::DGDiffusionIntegrator(Coefficient &q, const real_t s,
+                                             const real_t k)
+   : DGDiffusionIntegrator(s, k)
+{
+   Q = &q;
+}
+
+DGDiffusionIntegrator::DGDiffusionIntegrator(MatrixCoefficient &q,
+                                             const real_t s, const real_t k)
+   : DGDiffusionIntegrator(s, k)
+{
+   MQ = &q;
+}
+
+/// \cond DO_NOT_DOCUMENT
+
+DGDiffusionIntegrator::ApplyKernelType
+DGDiffusionIntegrator::ApplyPAKernels::Fallback(int dim, int, int)
 {
    if (dim == 2)
    {
-      auto kernel = PADGDiffusionApply2D<0,0>;
-      switch ((D1D << 4 ) | Q1D)
-      {
-         case 0x23: kernel = PADGDiffusionApply2D<2,3>; break;
-         case 0x34: kernel = PADGDiffusionApply2D<3,4>; break;
-         case 0x45: kernel = PADGDiffusionApply2D<4,5>; break;
-         case 0x56: kernel = PADGDiffusionApply2D<5,6>; break;
-         case 0x67: kernel = PADGDiffusionApply2D<6,7>; break;
-         case 0x78: kernel = PADGDiffusionApply2D<7,8>; break;
-         case 0x89: kernel = PADGDiffusionApply2D<8,9>; break;
-         case 0x9A: kernel = PADGDiffusionApply2D<9,10>; break;
-      }
-      kernel(NF, B, Bt, G, Gt, sigma, pa_data, x, dxdn, y, dydn, D1D, Q1D);
+      return internal::PADGDiffusionApply2D;
    }
    else if (dim == 3)
    {
-      auto kernel = PADGDiffusionApply3D<0,0>;
-      switch ((D1D << 4) | Q1D)
-      {
-         case 0x24: kernel = PADGDiffusionApply3D<2,4>; break;
-         case 0x35: kernel = PADGDiffusionApply3D<3,5>; break;
-         case 0x46: kernel = PADGDiffusionApply3D<4,6>; break;
-         case 0x57: kernel = PADGDiffusionApply3D<5,7>; break;
-         case 0x68: kernel = PADGDiffusionApply3D<6,8>; break;
-         case 0x79: kernel = PADGDiffusionApply3D<7,9>; break;
-         case 0x8A: kernel = PADGDiffusionApply3D<8,10>; break;
-         case 0x9B: kernel = PADGDiffusionApply3D<9,11>; break;
-      }
-      kernel(NF, B, Bt, G, Gt, sigma, pa_data, x, dxdn, y, dydn, D1D, Q1D);
+      return internal::PADGDiffusionApply3D;
    }
    else
    {
-      MFEM_ABORT("Unsupported dimension");
+      MFEM_ABORT("");
    }
 }
 
-void DGDiffusionIntegrator::AddMultPAFaceNormalDerivatives(
-   const Vector &x, const Vector &dxdn, Vector &y, Vector &dydn) const
+DGDiffusionIntegrator::Kernels::Kernels()
 {
-   PADGDiffusionApply(dim, dofs1D, quad1D, nf,
-                      maps->B, maps->Bt, maps->G, maps->Gt,
-                      sigma, pa_data, x, dxdn, y, dydn);
+   DGDiffusionIntegrator::AddSpecialization<2, 2, 3>();
+   DGDiffusionIntegrator::AddSpecialization<2, 3, 4>();
+   DGDiffusionIntegrator::AddSpecialization<2, 4, 5>();
+   DGDiffusionIntegrator::AddSpecialization<2, 5, 6>();
+   DGDiffusionIntegrator::AddSpecialization<2, 6, 7>();
+   DGDiffusionIntegrator::AddSpecialization<2, 7, 8>();
+   DGDiffusionIntegrator::AddSpecialization<2, 8, 9>();
+   DGDiffusionIntegrator::AddSpecialization<2, 9, 10>();
+
+   DGDiffusionIntegrator::AddSpecialization<3, 2, 4>();
+   DGDiffusionIntegrator::AddSpecialization<3, 3, 5>();
+   DGDiffusionIntegrator::AddSpecialization<3, 4, 6>();
+   DGDiffusionIntegrator::AddSpecialization<3, 5, 7>();
+   DGDiffusionIntegrator::AddSpecialization<3, 6, 8>();
+   DGDiffusionIntegrator::AddSpecialization<3, 7, 9>();
+   DGDiffusionIntegrator::AddSpecialization<3, 8, 10>();
+   DGDiffusionIntegrator::AddSpecialization<3, 9, 11>();
 }
 
+/// \endcond DO_NOT_DOCUMENT
+
 } // namespace mfem

fem/integ/bilininteg_dgtrace_kernels.hpp

@@ -0,0 +1,793 @@
+// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef BILININTEG_DGTRACE_KERNELS_HPP
+#define BILININTEG_DGTRACE_KERNELS_HPP
+
+#include "../../general/forall.hpp"
+#include "../bilininteg.hpp"
+#include "../gridfunc.hpp"
+#include "../qfunction.hpp"
+#include "../restriction.hpp"
+
+/// \cond DO_NOT_DOCUMENT
+namespace mfem
+{
+
+namespace internal
+{
+
+// PA DGTrace Apply 2D kernel for Gauss-Lobatto/Bernstein
+template <int T_D1D = 0, int T_Q1D = 0>
+static void PADGTraceApply2D(const int NF, const Array<real_t> &b,
+                             const Array<real_t> &bt, const Vector &op_,
+                             const Vector &x_, Vector &y_, const int d1d = 0,
+                             const int q1d = 0)
+{
+   const int VDIM = 1;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt.Read(), D1D, Q1D);
+   auto op = Reshape(op_.Read(), Q1D, 2, 2, NF);
+   auto x = Reshape(x_.Read(), D1D, VDIM, 2, NF);
+   auto y = Reshape(y_.ReadWrite(), D1D, VDIM, 2, NF);
+
+   mfem::forall(NF, [=] MFEM_HOST_DEVICE(int f)
+   {
+      const int VDIM = 1;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      // the following variables are evaluated at compile time
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      real_t u0[max_D1D][VDIM];
+      real_t u1[max_D1D][VDIM];
+      for (int d = 0; d < D1D; d++)
+      {
+         for (int c = 0; c < VDIM; c++)
+         {
+            u0[d][c] = x(d, c, 0, f);
+            u1[d][c] = x(d, c, 1, f);
+         }
+      }
+      real_t Bu0[max_Q1D][VDIM];
+      real_t Bu1[max_Q1D][VDIM];
+      for (int q = 0; q < Q1D; ++q)
+      {
+         for (int c = 0; c < VDIM; c++)
+         {
+            Bu0[q][c] = 0.0;
+            Bu1[q][c] = 0.0;
+         }
+         for (int d = 0; d < D1D; ++d)
+         {
+            const real_t b = B(q, d);
+            for (int c = 0; c < VDIM; c++)
+            {
+               Bu0[q][c] += b * u0[d][c];
+               Bu1[q][c] += b * u1[d][c];
+            }
+         }
+      }
+      real_t DBu[max_Q1D][VDIM];
+      for (int q = 0; q < Q1D; ++q)
+      {
+         for (int c = 0; c < VDIM; c++)
+         {
+            DBu[q][c] = op(q, 0, 0, f) * Bu0[q][c] + op(q, 1, 0, f) * Bu1[q][c];
+         }
+      }
+      real_t BDBu[max_D1D][VDIM];
+      for (int d = 0; d < D1D; ++d)
+      {
+         for (int c = 0; c < VDIM; c++)
+         {
+            BDBu[d][c] = 0.0;
+         }
+         for (int q = 0; q < Q1D; ++q)
+         {
+            const real_t b = Bt(d, q);
+            for (int c = 0; c < VDIM; c++)
+            {
+               BDBu[d][c] += b * DBu[q][c];
+            }
+         }
+         for (int c = 0; c < VDIM; c++)
+         {
+            y(d, c, 0, f) += BDBu[d][c];
+            y(d, c, 1, f) += -BDBu[d][c];
+         }
+      }
+   });
+}
+
+// PA DGTrace Apply 3D kernel for Gauss-Lobatto/Bernstein
+template <int T_D1D = 0, int T_Q1D = 0>
+static void PADGTraceApply3D(const int NF, const Array<real_t> &b,
+                             const Array<real_t> &bt, const Vector &op_,
+                             const Vector &x_, Vector &y_, const int d1d = 0,
+                             const int q1d = 0)
+{
+   const int VDIM = 1;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt.Read(), D1D, Q1D);
+   auto op = Reshape(op_.Read(), Q1D, Q1D, 2, 2, NF);
+   auto x = Reshape(x_.Read(), D1D, D1D, VDIM, 2, NF);
+   auto y = Reshape(y_.ReadWrite(), D1D, D1D, VDIM, 2, NF);
+
+   mfem::forall(NF, [=] MFEM_HOST_DEVICE(int f)
+   {
+      const int VDIM = 1;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      // the following variables are evaluated at compile time
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      real_t u0[max_D1D][max_D1D][VDIM];
+      real_t u1[max_D1D][max_D1D][VDIM];
+      for (int d1 = 0; d1 < D1D; d1++)
+      {
+         for (int d2 = 0; d2 < D1D; d2++)
+         {
+            for (int c = 0; c < VDIM; c++)
+            {
+               u0[d1][d2][c] = x(d1, d2, c, 0, f);
+               u1[d1][d2][c] = x(d1, d2, c, 1, f);
+            }
+         }
+      }
+      real_t Bu0[max_Q1D][max_D1D][VDIM];
+      real_t Bu1[max_Q1D][max_D1D][VDIM];
+      for (int q = 0; q < Q1D; ++q)
+      {
+         for (int d2 = 0; d2 < D1D; d2++)
+         {
+            for (int c = 0; c < VDIM; c++)
+            {
+               Bu0[q][d2][c] = 0.0;
+               Bu1[q][d2][c] = 0.0;
+            }
+            for (int d1 = 0; d1 < D1D; ++d1)
+            {
+               const real_t b = B(q, d1);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  Bu0[q][d2][c] += b * u0[d1][d2][c];
+                  Bu1[q][d2][c] += b * u1[d1][d2][c];
+               }
+            }
+         }
+      }
+      real_t BBu0[max_Q1D][max_Q1D][VDIM];
+      real_t BBu1[max_Q1D][max_Q1D][VDIM];
+      for (int q1 = 0; q1 < Q1D; ++q1)
+      {
+         for (int q2 = 0; q2 < Q1D; q2++)
+         {
+            for (int c = 0; c < VDIM; c++)
+            {
+               BBu0[q1][q2][c] = 0.0;
+               BBu1[q1][q2][c] = 0.0;
+            }
+            for (int d2 = 0; d2 < D1D; ++d2)
+            {
+               const real_t b = B(q2, d2);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  BBu0[q1][q2][c] += b * Bu0[q1][d2][c];
+                  BBu1[q1][q2][c] += b * Bu1[q1][d2][c];
+               }
+            }
+         }
+      }
+      real_t DBBu[max_Q1D][max_Q1D][VDIM];
+      for (int q1 = 0; q1 < Q1D; ++q1)
+      {
+         for (int q2 = 0; q2 < Q1D; q2++)
+         {
+            for (int c = 0; c < VDIM; c++)
+            {
+               DBBu[q1][q2][c] = op(q1, q2, 0, 0, f) * BBu0[q1][q2][c] +
+                                 op(q1, q2, 1, 0, f) * BBu1[q1][q2][c];
+            }
+         }
+      }
+      real_t BDBBu[max_Q1D][max_D1D][VDIM];
+      for (int q1 = 0; q1 < Q1D; ++q1)
+      {
+         for (int d2 = 0; d2 < D1D; d2++)
+         {
+            for (int c = 0; c < VDIM; c++)
+            {
+               BDBBu[q1][d2][c] = 0.0;
+            }
+            for (int q2 = 0; q2 < Q1D; ++q2)
+            {
+               const real_t b = Bt(d2, q2);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  BDBBu[q1][d2][c] += b * DBBu[q1][q2][c];
+               }
+            }
+         }
+      }
+      real_t BBDBBu[max_D1D][max_D1D][VDIM];
+      for (int d1 = 0; d1 < D1D; ++d1)
+      {
+         for (int d2 = 0; d2 < D1D; d2++)
+         {
+            for (int c = 0; c < VDIM; c++)
+            {
+               BBDBBu[d1][d2][c] = 0.0;
+            }
+            for (int q1 = 0; q1 < Q1D; ++q1)
+            {
+               const real_t b = Bt(d1, q1);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  BBDBBu[d1][d2][c] += b * BDBBu[q1][d2][c];
+               }
+            }
+            for (int c = 0; c < VDIM; c++)
+            {
+               y(d1, d2, c, 0, f) += BBDBBu[d1][d2][c];
+               y(d1, d2, c, 1, f) += -BBDBBu[d1][d2][c];
+            }
+         }
+      }
+   });
+}
+
+// Optimized PA DGTrace Apply 3D kernel for Gauss-Lobatto/Bernstein
+template <int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
+static void SmemPADGTraceApply3D(const int NF, const Array<real_t> &b,
+                                 const Array<real_t> &bt, const Vector &op_,
+                                 const Vector &x_, Vector &y_,
+                                 const int d1d = 0, const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt.Read(), D1D, Q1D);
+   auto op = Reshape(op_.Read(), Q1D, Q1D, 2, 2, NF);
+   auto x = Reshape(x_.Read(), D1D, D1D, 2, NF);
+   auto y = Reshape(y_.ReadWrite(), D1D, D1D, 2, NF);
+
+   mfem::forall_2D_batch(NF, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE(int f)
+   {
+      const int tidz = MFEM_THREAD_ID(z);
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      // the following variables are evaluated at compile time
+      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      MFEM_SHARED real_t u0[NBZ][max_D1D][max_D1D];
+      MFEM_SHARED real_t u1[NBZ][max_D1D][max_D1D];
+      MFEM_FOREACH_THREAD(d1, x, D1D)
+      {
+         MFEM_FOREACH_THREAD(d2, y, D1D)
+         {
+            u0[tidz][d1][d2] = x(d1, d2, 0, f);
+            u1[tidz][d1][d2] = x(d1, d2, 1, f);
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_SHARED real_t Bu0[NBZ][max_Q1D][max_D1D];
+      MFEM_SHARED real_t Bu1[NBZ][max_Q1D][max_D1D];
+      MFEM_FOREACH_THREAD(q1, x, Q1D)
+      {
+         MFEM_FOREACH_THREAD(d2, y, D1D)
+         {
+            real_t Bu0_ = 0.0;
+            real_t Bu1_ = 0.0;
+            for (int d1 = 0; d1 < D1D; ++d1)
+            {
+               const real_t b = B(q1, d1);
+               Bu0_ += b * u0[tidz][d1][d2];
+               Bu1_ += b * u1[tidz][d1][d2];
+            }
+            Bu0[tidz][q1][d2] = Bu0_;
+            Bu1[tidz][q1][d2] = Bu1_;
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_SHARED real_t BBu0[NBZ][max_Q1D][max_Q1D];
+      MFEM_SHARED real_t BBu1[NBZ][max_Q1D][max_Q1D];
+      MFEM_FOREACH_THREAD(q1, x, Q1D)
+      {
+         MFEM_FOREACH_THREAD(q2, y, Q1D)
+         {
+            real_t BBu0_ = 0.0;
+            real_t BBu1_ = 0.0;
+            for (int d2 = 0; d2 < D1D; ++d2)
+            {
+               const real_t b = B(q2, d2);
+               BBu0_ += b * Bu0[tidz][q1][d2];
+               BBu1_ += b * Bu1[tidz][q1][d2];
+            }
+            BBu0[tidz][q1][q2] = BBu0_;
+            BBu1[tidz][q1][q2] = BBu1_;
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_SHARED real_t DBBu[NBZ][max_Q1D][max_Q1D];
+      MFEM_FOREACH_THREAD(q1, x, Q1D)
+      {
+         MFEM_FOREACH_THREAD(q2, y, Q1D)
+         {
+            DBBu[tidz][q1][q2] = op(q1, q2, 0, 0, f) * BBu0[tidz][q1][q2] +
+                                 op(q1, q2, 1, 0, f) * BBu1[tidz][q1][q2];
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_SHARED real_t BDBBu[NBZ][max_Q1D][max_D1D];
+      MFEM_FOREACH_THREAD(q1, x, Q1D)
+      {
+         MFEM_FOREACH_THREAD(d2, y, D1D)
+         {
+            real_t BDBBu_ = 0.0;
+            for (int q2 = 0; q2 < Q1D; ++q2)
+            {
+               const real_t b = Bt(d2, q2);
+               BDBBu_ += b * DBBu[tidz][q1][q2];
+            }
+            BDBBu[tidz][q1][d2] = BDBBu_;
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(d1, x, D1D)
+      {
+         MFEM_FOREACH_THREAD(d2, y, D1D)
+         {
+            real_t BBDBBu_ = 0.0;
+            for (int q1 = 0; q1 < Q1D; ++q1)
+            {
+               const real_t b = Bt(d1, q1);
+               BBDBBu_ += b * BDBBu[tidz][q1][d2];
+            }
+            y(d1, d2, 0, f) += BBDBBu_;
+            y(d1, d2, 1, f) += -BBDBBu_;
+         }
+      }
+   });
+}
+
+// PA DGTrace Apply 2D kernel for Gauss-Lobatto/Bernstein
+template <int T_D1D = 0, int T_Q1D = 0>
+static void PADGTraceApplyTranspose2D(const int NF, const Array<real_t> &b,
+                                      const Array<real_t> &bt,
+                                      const Vector &op_, const Vector &x_,
+                                      Vector &y_, const int d1d = 0,
+                                      const int q1d = 0)
+{
+   const int VDIM = 1;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt.Read(), D1D, Q1D);
+   auto op = Reshape(op_.Read(), Q1D, 2, 2, NF);
+   auto x = Reshape(x_.Read(), D1D, VDIM, 2, NF);
+   auto y = Reshape(y_.ReadWrite(), D1D, VDIM, 2, NF);
+
+   mfem::forall(NF, [=] MFEM_HOST_DEVICE(int f)
+   {
+      const int VDIM = 1;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      // the following variables are evaluated at compile time
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      real_t u0[max_D1D][VDIM];
+      real_t u1[max_D1D][VDIM];
+      for (int d = 0; d < D1D; d++)
+      {
+         for (int c = 0; c < VDIM; c++)
+         {
+            u0[d][c] = x(d, c, 0, f);
+            u1[d][c] = x(d, c, 1, f);
+         }
+      }
+      real_t Bu0[max_Q1D][VDIM];
+      real_t Bu1[max_Q1D][VDIM];
+      for (int q = 0; q < Q1D; ++q)
+      {
+         for (int c = 0; c < VDIM; c++)
+         {
+            Bu0[q][c] = 0.0;
+            Bu1[q][c] = 0.0;
+         }
+         for (int d = 0; d < D1D; ++d)
+         {
+            const real_t b = B(q, d);
+            for (int c = 0; c < VDIM; c++)
+            {
+               Bu0[q][c] += b * u0[d][c];
+               Bu1[q][c] += b * u1[d][c];
+            }
+         }
+      }
+      real_t DBu0[max_Q1D][VDIM];
+      real_t DBu1[max_Q1D][VDIM];
+      for (int q = 0; q < Q1D; ++q)
+      {
+         for (int c = 0; c < VDIM; c++)
+         {
+            DBu0[q][c] =
+               op(q, 0, 0, f) * Bu0[q][c] + op(q, 0, 1, f) * Bu1[q][c];
+            DBu1[q][c] =
+               op(q, 1, 0, f) * Bu0[q][c] + op(q, 1, 1, f) * Bu1[q][c];
+         }
+      }
+      real_t BDBu0[max_D1D][VDIM];
+      real_t BDBu1[max_D1D][VDIM];
+      for (int d = 0; d < D1D; ++d)
+      {
+         for (int c = 0; c < VDIM; c++)
+         {
+            BDBu0[d][c] = 0.0;
+            BDBu1[d][c] = 0.0;
+         }
+         for (int q = 0; q < Q1D; ++q)
+         {
+            const real_t b = Bt(d, q);
+            for (int c = 0; c < VDIM; c++)
+            {
+               BDBu0[d][c] += b * DBu0[q][c];
+               BDBu1[d][c] += b * DBu1[q][c];
+            }
+         }
+         for (int c = 0; c < VDIM; c++)
+         {
+            y(d, c, 0, f) += BDBu0[d][c];
+            y(d, c, 1, f) += BDBu1[d][c];
+         }
+      }
+   });
+}
+
+// PA DGTrace Apply Transpose 3D kernel for Gauss-Lobatto/Bernstein
+template <int T_D1D = 0, int T_Q1D = 0>
+static void PADGTraceApplyTranspose3D(const int NF, const Array<real_t> &b,
+                                      const Array<real_t> &bt,
+                                      const Vector &op_, const Vector &x_,
+                                      Vector &y_, const int d1d = 0,
+                                      const int q1d = 0)
+{
+   const int VDIM = 1;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt.Read(), D1D, Q1D);
+   auto op = Reshape(op_.Read(), Q1D, Q1D, 2, 2, NF);
+   auto x = Reshape(x_.Read(), D1D, D1D, VDIM, 2, NF);
+   auto y = Reshape(y_.ReadWrite(), D1D, D1D, VDIM, 2, NF);
+
+   mfem::forall(NF, [=] MFEM_HOST_DEVICE(int f)
+   {
+      const int VDIM = 1;
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      // the following variables are evaluated at compile time
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      real_t u0[max_D1D][max_D1D][VDIM];
+      real_t u1[max_D1D][max_D1D][VDIM];
+      for (int d1 = 0; d1 < D1D; d1++)
+      {
+         for (int d2 = 0; d2 < D1D; d2++)
+         {
+            for (int c = 0; c < VDIM; c++)
+            {
+               u0[d1][d2][c] = x(d1, d2, c, 0, f);
+               u1[d1][d2][c] = x(d1, d2, c, 1, f);
+            }
+         }
+      }
+      real_t Bu0[max_Q1D][max_D1D][VDIM];
+      real_t Bu1[max_Q1D][max_D1D][VDIM];
+      for (int q1 = 0; q1 < Q1D; ++q1)
+      {
+         for (int d2 = 0; d2 < D1D; ++d2)
+         {
+            for (int c = 0; c < VDIM; c++)
+            {
+               Bu0[q1][d2][c] = 0.0;
+               Bu1[q1][d2][c] = 0.0;
+            }
+            for (int d1 = 0; d1 < D1D; ++d1)
+            {
+               const real_t b = B(q1, d1);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  Bu0[q1][d2][c] += b * u0[d1][d2][c];
+                  Bu1[q1][d2][c] += b * u1[d1][d2][c];
+               }
+            }
+         }
+      }
+      real_t BBu0[max_Q1D][max_Q1D][VDIM];
+      real_t BBu1[max_Q1D][max_Q1D][VDIM];
+      for (int q1 = 0; q1 < Q1D; ++q1)
+      {
+         for (int q2 = 0; q2 < Q1D; ++q2)
+         {
+            for (int c = 0; c < VDIM; c++)
+            {
+               BBu0[q1][q2][c] = 0.0;
+               BBu1[q1][q2][c] = 0.0;
+            }
+            for (int d2 = 0; d2 < D1D; ++d2)
+            {
+               const real_t b = B(q2, d2);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  BBu0[q1][q2][c] += b * Bu0[q1][d2][c];
+                  BBu1[q1][q2][c] += b * Bu1[q1][d2][c];
+               }
+            }
+         }
+      }
+      real_t DBu0[max_Q1D][max_Q1D][VDIM];
+      real_t DBu1[max_Q1D][max_Q1D][VDIM];
+      for (int q1 = 0; q1 < Q1D; ++q1)
+      {
+         for (int q2 = 0; q2 < Q1D; ++q2)
+         {
+            const real_t D00 = op(q1, q2, 0, 0, f);
+            const real_t D01 = op(q1, q2, 0, 1, f);
+            const real_t D10 = op(q1, q2, 1, 0, f);
+            const real_t D11 = op(q1, q2, 1, 1, f);
+            for (int c = 0; c < VDIM; c++)
+            {
+               DBu0[q1][q2][c] = D00 * BBu0[q1][q2][c] + D01 * BBu1[q1][q2][c];
+               DBu1[q1][q2][c] = D10 * BBu0[q1][q2][c] + D11 * BBu1[q1][q2][c];
+            }
+         }
+      }
+      real_t BDBu0[max_D1D][max_Q1D][VDIM];
+      real_t BDBu1[max_D1D][max_Q1D][VDIM];
+      for (int d1 = 0; d1 < D1D; ++d1)
+      {
+         for (int q2 = 0; q2 < Q1D; ++q2)
+         {
+            for (int c = 0; c < VDIM; c++)
+            {
+               BDBu0[d1][q2][c] = 0.0;
+               BDBu1[d1][q2][c] = 0.0;
+            }
+            for (int q1 = 0; q1 < Q1D; ++q1)
+            {
+               const real_t b = Bt(d1, q1);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  BDBu0[d1][q2][c] += b * DBu0[q1][q2][c];
+                  BDBu1[d1][q2][c] += b * DBu1[q1][q2][c];
+               }
+            }
+         }
+      }
+      real_t BBDBu0[max_D1D][max_D1D][VDIM];
+      real_t BBDBu1[max_D1D][max_D1D][VDIM];
+      for (int d1 = 0; d1 < D1D; ++d1)
+      {
+         for (int d2 = 0; d2 < D1D; ++d2)
+         {
+            for (int c = 0; c < VDIM; c++)
+            {
+               BBDBu0[d1][d2][c] = 0.0;
+               BBDBu1[d1][d2][c] = 0.0;
+            }
+            for (int q2 = 0; q2 < Q1D; ++q2)
+            {
+               const real_t b = Bt(d2, q2);
+               for (int c = 0; c < VDIM; c++)
+               {
+                  BBDBu0[d1][d2][c] += b * BDBu0[d1][q2][c];
+                  BBDBu1[d1][d2][c] += b * BDBu1[d1][q2][c];
+               }
+            }
+            for (int c = 0; c < VDIM; c++)
+            {
+               y(d1, d2, c, 0, f) += BBDBu0[d1][d2][c];
+               y(d1, d2, c, 1, f) += BBDBu1[d1][d2][c];
+            }
+         }
+      }
+   });
+}
+
+// Optimized PA DGTrace Apply Transpose 3D kernel for Gauss-Lobatto/Bernstein
+template <int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
+static void SmemPADGTraceApplyTranspose3D(const int NF, const Array<real_t> &b,
+                                          const Array<real_t> &bt,
+                                          const Vector &op_, const Vector &x_,
+                                          Vector &y_, const int d1d = 0,
+                                          const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt.Read(), D1D, Q1D);
+   auto op = Reshape(op_.Read(), Q1D, Q1D, 2, 2, NF);
+   auto x = Reshape(x_.Read(), D1D, D1D, 2, NF);
+   auto y = Reshape(y_.ReadWrite(), D1D, D1D, 2, NF);
+
+   mfem::forall_2D_batch(NF, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE(int f)
+   {
+      const int tidz = MFEM_THREAD_ID(z);
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      // the following variables are evaluated at compile time
+      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      MFEM_SHARED real_t u0[NBZ][max_D1D][max_D1D];
+      MFEM_SHARED real_t u1[NBZ][max_D1D][max_D1D];
+      MFEM_FOREACH_THREAD(d1, x, D1D)
+      {
+         MFEM_FOREACH_THREAD(d2, y, D1D)
+         {
+            u0[tidz][d1][d2] = x(d1, d2, 0, f);
+            u1[tidz][d1][d2] = x(d1, d2, 1, f);
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_SHARED real_t Bu0[NBZ][max_Q1D][max_D1D];
+      MFEM_SHARED real_t Bu1[NBZ][max_Q1D][max_D1D];
+      MFEM_FOREACH_THREAD(q1, x, Q1D)
+      {
+         MFEM_FOREACH_THREAD(d2, y, D1D)
+         {
+            real_t Bu0_ = 0.0;
+            real_t Bu1_ = 0.0;
+            for (int d1 = 0; d1 < D1D; ++d1)
+            {
+               const real_t b = B(q1, d1);
+               Bu0_ += b * u0[tidz][d1][d2];
+               Bu1_ += b * u1[tidz][d1][d2];
+            }
+            Bu0[tidz][q1][d2] = Bu0_;
+            Bu1[tidz][q1][d2] = Bu1_;
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_SHARED real_t BBu0[NBZ][max_Q1D][max_Q1D];
+      MFEM_SHARED real_t BBu1[NBZ][max_Q1D][max_Q1D];
+      MFEM_FOREACH_THREAD(q1, x, Q1D)
+      {
+         MFEM_FOREACH_THREAD(q2, y, Q1D)
+         {
+            real_t BBu0_ = 0.0;
+            real_t BBu1_ = 0.0;
+            for (int d2 = 0; d2 < D1D; ++d2)
+            {
+               const real_t b = B(q2, d2);
+               BBu0_ += b * Bu0[tidz][q1][d2];
+               BBu1_ += b * Bu1[tidz][q1][d2];
+            }
+            BBu0[tidz][q1][q2] = BBu0_;
+            BBu1[tidz][q1][q2] = BBu1_;
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_SHARED real_t DBBu0[NBZ][max_Q1D][max_Q1D];
+      MFEM_SHARED real_t DBBu1[NBZ][max_Q1D][max_Q1D];
+      MFEM_FOREACH_THREAD(q1, x, Q1D)
+      {
+         MFEM_FOREACH_THREAD(q2, y, Q1D)
+         {
+            const real_t D00 = op(q1, q2, 0, 0, f);
+            const real_t D01 = op(q1, q2, 0, 1, f);
+            const real_t D10 = op(q1, q2, 1, 0, f);
+            const real_t D11 = op(q1, q2, 1, 1, f);
+            const real_t u0q = BBu0[tidz][q1][q2];
+            const real_t u1q = BBu1[tidz][q1][q2];
+            DBBu0[tidz][q1][q2] = D00 * u0q + D01 * u1q;
+            DBBu1[tidz][q1][q2] = D10 * u0q + D11 * u1q;
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_SHARED real_t BDBBu0[NBZ][max_Q1D][max_D1D];
+      MFEM_SHARED real_t BDBBu1[NBZ][max_Q1D][max_D1D];
+      MFEM_FOREACH_THREAD(q1, x, Q1D)
+      {
+         MFEM_FOREACH_THREAD(d2, y, D1D)
+         {
+            real_t BDBBu0_ = 0.0;
+            real_t BDBBu1_ = 0.0;
+            for (int q2 = 0; q2 < Q1D; ++q2)
+            {
+               const real_t b = Bt(d2, q2);
+               BDBBu0_ += b * DBBu0[tidz][q1][q2];
+               BDBBu1_ += b * DBBu1[tidz][q1][q2];
+            }
+            BDBBu0[tidz][q1][d2] = BDBBu0_;
+            BDBBu1[tidz][q1][d2] = BDBBu1_;
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(d1, x, D1D)
+      {
+         MFEM_FOREACH_THREAD(d2, y, D1D)
+         {
+            real_t BBDBBu0_ = 0.0;
+            real_t BBDBBu1_ = 0.0;
+            for (int q1 = 0; q1 < Q1D; ++q1)
+            {
+               const real_t b = Bt(d1, q1);
+               BBDBBu0_ += b * BDBBu0[tidz][q1][d2];
+               BBDBBu1_ += b * BDBBu1[tidz][q1][d2];
+            }
+            y(d1, d2, 0, f) += BBDBBu0_;
+            y(d1, d2, 1, f) += BBDBBu1_;
+         }
+      }
+   });
+}
+
+} // namespace internal
+
+template <int DIM, int D1D, int Q1D>
+DGTraceIntegrator::ApplyKernelType DGTraceIntegrator::ApplyPAKernels::Kernel()
+{
+   if constexpr (DIM == 2)
+   {
+      return internal::PADGTraceApply2D<D1D, Q1D>;
+   }
+   else if constexpr (DIM == 3)
+   {
+      if constexpr (D1D == 3 || D1D == 4)
+      {
+         return internal::SmemPADGTraceApply3D<D1D, Q1D, 2>;
+      }
+      else
+      {
+         return internal::SmemPADGTraceApply3D<D1D, Q1D>;
+      }
+   }
+   MFEM_ABORT("");
+}
+
+template <int DIM, int D1D, int Q1D>
+DGTraceIntegrator::ApplyKernelType DGTraceIntegrator::ApplyPATKernels::Kernel()
+{
+   if constexpr (DIM == 2)
+   {
+      return internal::PADGTraceApplyTranspose2D<D1D, Q1D>;
+   }
+   else if constexpr (DIM == 3)
+   {
+      return internal::SmemPADGTraceApplyTranspose3D<D1D, Q1D>;
+   }
+   MFEM_ABORT("");
+}
+} // namespace mfem
+
+/// \endcond DO_NOT_DOCUMENT
+#endif

fem/integ/bilininteg_diffusion_kernels.cpp

@@ -19,6 +19,8 @@ namespace mfem
 DiffusionIntegrator::Kernels::Kernels()
 {
    // 2D
+   // Q = P+1
+   DiffusionIntegrator::AddSpecialization<2,1,1>();
    DiffusionIntegrator::AddSpecialization<2,2,2>();
    DiffusionIntegrator::AddSpecialization<2,3,3>();
    DiffusionIntegrator::AddSpecialization<2,4,4>();
@@ -27,17 +29,39 @@ DiffusionIntegrator::Kernels::Kernels()
    DiffusionIntegrator::AddSpecialization<2,7,7>();
    DiffusionIntegrator::AddSpecialization<2,8,8>();
    DiffusionIntegrator::AddSpecialization<2,9,9>();
+   // Q = P+2
+   DiffusionIntegrator::AddSpecialization<2,1,2>();
+   DiffusionIntegrator::AddSpecialization<2,2,3>();
+   DiffusionIntegrator::AddSpecialization<2,3,4>();
+   DiffusionIntegrator::AddSpecialization<2,4,5>();
+   DiffusionIntegrator::AddSpecialization<2,5,6>();
+   DiffusionIntegrator::AddSpecialization<2,6,7>();
+   DiffusionIntegrator::AddSpecialization<2,7,8>();
+   DiffusionIntegrator::AddSpecialization<2,8,9>();
+   DiffusionIntegrator::AddSpecialization<2,9,10>();
+   // others
    // 3D
+   // Q = P+1
+   DiffusionIntegrator::AddSpecialization<3,1,1>();
    DiffusionIntegrator::AddSpecialization<3,2,2>();
+   DiffusionIntegrator::AddSpecialization<3,3,3>();
+   DiffusionIntegrator::AddSpecialization<3,4,4>();
+   DiffusionIntegrator::AddSpecialization<3,5,5>();
+   DiffusionIntegrator::AddSpecialization<3,6,6>();
+   DiffusionIntegrator::AddSpecialization<3,7,7>();
+   DiffusionIntegrator::AddSpecialization<3,8,8>();
+   // Q = P+2
+   DiffusionIntegrator::AddSpecialization<3,1,2>();
    DiffusionIntegrator::AddSpecialization<3,2,3>();
    DiffusionIntegrator::AddSpecialization<3,3,4>();
    DiffusionIntegrator::AddSpecialization<3,4,5>();
-   DiffusionIntegrator::AddSpecialization<3,4,6>();
    DiffusionIntegrator::AddSpecialization<3,5,6>();
-   DiffusionIntegrator::AddSpecialization<3,5,8>();
    DiffusionIntegrator::AddSpecialization<3,6,7>();
    DiffusionIntegrator::AddSpecialization<3,7,8>();
    DiffusionIntegrator::AddSpecialization<3,8,9>();
+   // others
+   DiffusionIntegrator::AddSpecialization<3,4,6>();
+   DiffusionIntegrator::AddSpecialization<3,5,8>();
 }
 
 namespace internal

fem/integ/bilininteg_diffusion_kernels.hpp

@@ -672,12 +672,12 @@ inline void SmemPADiffusionApply2D(const int NE,
       real_t (*Gt)[MQ1] = (real_t (*)[MQ1]) (sBG+1);
       MFEM_SHARED real_t Xz[NBZ][MD1][MD1];
       MFEM_SHARED real_t GD[2][NBZ][MD1][MQ1];
-      MFEM_SHARED real_t GQ[2][NBZ][MD1][MQ1];
+      MFEM_SHARED real_t GQ[2][NBZ][MQ1][MQ1];
       real_t (*X)[MD1] = (real_t (*)[MD1])(Xz + tidz);
-      real_t (*DQ0)[MD1] = (real_t (*)[MD1])(GD[0] + tidz);
-      real_t (*DQ1)[MD1] = (real_t (*)[MD1])(GD[1] + tidz);
-      real_t (*QQ0)[MD1] = (real_t (*)[MD1])(GQ[0] + tidz);
-      real_t (*QQ1)[MD1] = (real_t (*)[MD1])(GQ[1] + tidz);
+      real_t (*DQ0)[MQ1] = (real_t (*)[MQ1])(GD[0] + tidz);
+      real_t (*DQ1)[MQ1] = (real_t (*)[MQ1])(GD[1] + tidz);
+      real_t (*QQ0)[MQ1] = (real_t (*)[MQ1])(GQ[0] + tidz);
+      real_t (*QQ1)[MQ1] = (real_t (*)[MQ1])(GQ[1] + tidz);
       MFEM_FOREACH_THREAD(dy,y,D1D)
       {
          MFEM_FOREACH_THREAD(dx,x,D1D)
@@ -769,8 +769,8 @@ inline void SmemPADiffusionApply2D(const int NE,
                u += Gt[dx][qx] * QQ0[qy][qx];
                v += Bt[dx][qx] * QQ1[qy][qx];
             }
-            DQ0[qy][dx] = u;
-            DQ1[qy][dx] = v;
+            DQ0[dx][qy] = u;
+            DQ1[dx][qy] = v;
          }
       }
       MFEM_SYNC_THREAD;
@@ -782,8 +782,8 @@ inline void SmemPADiffusionApply2D(const int NE,
             real_t v = 0.0;
             for (int qy = 0; qy < Q1D; ++qy)
             {
-               u += DQ0[qy][dx] * Bt[dy][qy];
-               v += DQ1[qy][dx] * Gt[dy][qy];
+               u += DQ0[dx][qy] * Bt[dy][qy];
+               v += DQ1[dx][qy] * Gt[dy][qy];
             }
             Y(dx,dy,e) += (u + v);
          }
@@ -1221,9 +1221,9 @@ using DiagonalKernelType = DiffusionIntegrator::DiagonalKernelType;
 template<int DIM, int T_D1D, int T_Q1D>
 ApplyKernelType DiffusionIntegrator::ApplyPAKernels::Kernel()
 {
-   if (DIM == 2) { return internal::SmemPADiffusionApply2D<T_D1D,T_Q1D>; }
-   else if (DIM == 3) { return internal::SmemPADiffusionApply3D<T_D1D, T_Q1D>; }
-   else { MFEM_ABORT(""); }
+   if constexpr (DIM == 2) { return internal::SmemPADiffusionApply2D<T_D1D,T_Q1D>; }
+   else if constexpr (DIM == 3) { return internal::SmemPADiffusionApply3D<T_D1D, T_Q1D>; }
+   MFEM_ABORT("");
 }
 
 inline
@@ -1237,9 +1237,9 @@ ApplyKernelType DiffusionIntegrator::ApplyPAKernels::Fallback(int DIM, int, int)
 template<int DIM, int D1D, int Q1D>
 DiagonalKernelType DiffusionIntegrator::DiagonalPAKernels::Kernel()
 {
-   if (DIM == 2) { return internal::SmemPADiffusionDiagonal2D<D1D,Q1D>; }
-   else if (DIM == 3) { return internal::SmemPADiffusionDiagonal3D<D1D, Q1D>; }
-   else { MFEM_ABORT(""); }
+   if constexpr (DIM == 2) { return internal::SmemPADiffusionDiagonal2D<D1D,Q1D>; }
+   else if constexpr (DIM == 3) { return internal::SmemPADiffusionDiagonal3D<D1D, Q1D>; }
+   MFEM_ABORT("");
 }
 
 inline DiagonalKernelType