Add some comments

Do BCs on device with device-aware MPI
gitignore update
2022-02-02 16:52:25 -08:00 · 2022-02-02 16:05:45 -08:00 · 2022-01-26 10:38:09 -08:00 · 2022-01-22 08:11:24 -08:00 · 2022-01-22 08:10:58 -08:00 · 2022-01-20 08:08:21 -08:00
252 changed files with 21079 additions and 2506 deletions
@@ -82,9 +82,9 @@ jobs:
      uses: mfem/github-actions/build-mfem@v2.0
      with:
        os: ${{ runner.os }}
-        target: optim
+        target: opt
        codecov: NO
-        mpi: parallel
+        mpi: par
        build-system: make
        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
        metis-dir: ${{ env.METIS_TOP_DIR }}
@@ -15,6 +15,19 @@
 CMakeCache.txt
 CMakeFiles/

+# NSYS files
+*.nsys-rep
+
+# Benchmark files
+*.tex
+*.org
+*.aux
+*.log
+*.gnuplot
+*.table
+*.pdf
+*.png
+
 # Backup files
 *~

@@ -225,6 +238,14 @@ miniapps/mtop/ParHeat*
 miniapps/mtop/seqheat
 miniapps/mtop/SeqHeat*

+miniapps/autodiff/paradiff
+miniapps/autodiff/seqadiff
+miniapps/autodiff/seqtest
+miniapps/autodiff/par_example
+miniapps/autodiff/seq_example
+miniapps/autodiff/seq_test
+miniapps/autodiff/Exampl*
+
 miniapps/navier/navier_mms
 miniapps/navier/navier_kovasznay
 miniapps/navier/navier_kovasznay_vs
@@ -308,6 +329,9 @@ tests/unit/debug_device_tests

 # Benchmark binaries
 tests/benchmarks/bench_ceed
+tests/benchmarks/bench_solvers
+tests/benchmarks/bench_linext
+tests/benchmarks/bench_lor_assembly
 tests/benchmarks/bench_tmop
 tests/benchmarks/bench_vector
 tests/benchmarks/bench_virtuals
@@ -29,12 +29,34 @@ stages:

 variables:
  CUSTOM_CI_BUILDS_DIR: "/usr/workspace/mfem/gitlab-runner"
+  USER_CI_TOP_DIR: "${CUSTOM_CI_BUILDS_DIR}/${GITLAB_USER_LOGIN}"
+  SHARED_REPOS_DIR: "${USER_CI_TOP_DIR}/repos"
+  AUTOTEST_ROOT: "${SHARED_REPOS_DIR}"
+  # MFEM_DATA_DIR is setup in '.gitlab/configs/setup-build-and-test.yml' and
+  # used in '.gitlab/configs/<machine>-config.yml':
+  MFEM_DATA_DIR: "${SHARED_REPOS_DIR}/mfem-data"
+
+# Defines the default choice for updating the saved baseline results. By default
+# the baseline can only be updated from the master branch. This variable offers
+# the option to manually ask for rebaselining from another branch if necessary.
+  REBASELINE: "NO"
+  AUTOTEST: "NO"
+  # AUTOTEST_COMMIT: used only when AUTOTEST is set to YES.
+  # * If AUTOTEST_COMMIT is NOT set to NO, reporting jobs will commit their
+  #   files to the MFEM/autotest repo.
+  # * If AUTOTEST_COMMIT is set to NO, reporting jobs will NOT commit their
+  #   files to the MFEM/autotest repo. Instead they will just show the contents
+  #   of the report files and remove them.
+  AUTOTEST_COMMIT: "YES"

 # Trigger subpipelines:
 quartz-build-and-test:
  stage: sub-pipelines
  variables:
-    _AUTOTEST: $AUTOTEST
+    # Explicitly pass down values that we want to be able to set when triggering
+    # pipelines manually or using scheduling
+    AUTOTEST: "${AUTOTEST}"
+    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
  trigger:
    include: .gitlab/quartz-build-and-test.yml
    strategy: depend
@@ -42,7 +64,11 @@ quartz-build-and-test:
 quartz-baseline:
  stage: sub-pipelines
  variables:
-    _AUTOTEST: $AUTOTEST
+    # Explicitly pass down values that we want to be able to set when triggering
+    # pipelines manually or using scheduling
+    REBASELINE: "${REBASELINE}"
+    AUTOTEST: "${AUTOTEST}"
+    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
  trigger:
    include: .gitlab/quartz-baseline.yml
    strategy: depend
@@ -50,7 +76,10 @@ quartz-baseline:
 lassen-build-and-test:
  stage: sub-pipelines
  variables:
-    _AUTOTEST: $AUTOTEST
+    # Explicitly pass down values that we want to be able to set when triggering
+    # pipelines manually or using scheduling
+    AUTOTEST: "${AUTOTEST}"
+    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
  trigger:
    include: .gitlab/lassen-build-and-test.yml
    strategy: depend
@@ -58,7 +87,10 @@ lassen-build-and-test:
 corona-build-and-test:
  stage: sub-pipelines
  variables:
-    _AUTOTEST: $AUTOTEST
+    # Explicitly pass down values that we want to be able to set when triggering
+    # pipelines manually or using scheduling
+    AUTOTEST: "${AUTOTEST}"
+    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
  trigger:
    include: .gitlab/corona-build-and-test.yml
    strategy: depend
@@ -18,19 +18,13 @@ variables:
 # the pipeline, preventing any form of concurrency with other pipelines. This
 # also means that the BUILD_ROOT directory will never be cleaned.
 # TODO: add a clean-up mechanism
-  BUILD_ROOT: ${CI_BUILDS_DIR}/MFEM_${MACHINE_NAME}/${CI_PROJECT_NAME}_${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID}
+  BUILD_ROOT: ${USER_CI_TOP_DIR}/${CI_PROJECT_NAME}-${MACHINE_NAME}-pipeline-${CI_PIPELINE_ID}

 # On LLNL's quartz, there is only one allocation shared among jobs in order to
 # save time and resource. This allocation has to be uniquely named so that we
 # are sure to retrieve it.
  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}

-# Defines the default choice for updating the saved baseline results. By default
-# the baseline can only be updated from the master branch. This variable offers
-# the option to manually ask for rebaselining from another branch if necessary.
-  _REBASELINE: "NO"
-  _AUTOTEST: "NO"
-
 # Git repositories used in the pipeline
  TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
  TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
@@ -40,5 +34,3 @@ variables:
 # Directory used to place artifacts.
  ARTIFACTS_DIR: artifacts
  SLURM_OVERLAP: 1
-
-
@@ -26,17 +26,20 @@ variables:
    - if: '$CI_COMMIT_BRANCH =~ /_cnone/ || $ON_CORONA != "ON"'
      when: never
    # Don’t run autotest update if...
-    - if: '$CI_JOB_NAME =~ /report/ && $_AUTOTEST != "YES"'
+    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
      when: never
    # Report success on success status
-    - if: '$CI_JOB_NAME =~ /report_job_success/ && $_AUTOTEST == "YES"'
+    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
      when: on_success
    # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $_AUTOTEST == "YES"'
+    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
      when: on_failure
    # Always release resource
    - if: '$CI_JOB_NAME =~ /release_resource/'
      when: always
+    # Always cleanup
+    - if: '$CI_JOB_NAME =~ /cleanup/'
+      when: always
    # Default is to run if previous stage succeeded
    - when: on_success

@@ -46,9 +49,11 @@ variables:
  extends: [.on_corona]
  stage: build_and_test
  script:
+    # THREADS is used by 'tests/gitlab/build_and_test', run below
    - export THREADS=12
    - echo ${ALLOC_NAME}
    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
    - echo ${JOBID}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 15 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --build-root "${BUILD_ROOT}" --data
-
+    - echo ${MFEM_DATA_DIR}
+    - echo ${SPEC}
+    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 15 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
@@ -21,14 +21,17 @@ variables:
    - if: '$CI_COMMIT_BRANCH =~ /_lnone/ || $ON_LASSEN == "OFF"' #run except if ...
      when: never
    # Don't run autotest update if...
-    - if: '$CI_JOB_NAME =~ /report/ && $_AUTOTEST != "YES"'
+    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
      when: never
    # Report success on success status
-    - if: '$CI_JOB_NAME =~ /report_job_success/ && $_AUTOTEST == "YES"'
+    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
      when: on_success
    # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $_AUTOTEST == "YES"'
+    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
      when: on_failure
+    # Always cleanup
+    - if: '$CI_JOB_NAME =~ /cleanup/'
+      when: always
    - when: on_success

 # Lassen uses a different job scheduler (spectrum lsf) that does not allow
@@ -39,5 +42,8 @@ variables:
  extends: [.on_lassen]
  stage: build_and_test
  script:
-    - lalloc 1 -W 30 -q pdebug tests/gitlab/build_and_test --spec "${SPEC}" --build-root "${BUILD_ROOT}" --data
+    - echo ${MFEM_DATA_DIR}
+    - echo ${SPEC}
+    # Next script uses 'THREADS': leaving it empty --> it uses 'make all -j'
+    - lalloc 1 -W 30 -q pdebug --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
  needs: [setup]
@@ -22,17 +22,20 @@ variables:
    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"'
      when: never
    # Don't run autotest update if...
-    - if: '$CI_JOB_NAME =~ /report/ && $_AUTOTEST != "YES"'
+    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
      when: never
    # Report success on success status
-    - if: '$CI_JOB_NAME =~ /report_job_success/ && $_AUTOTEST == "YES"'
+    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
      when: on_success
    # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $_AUTOTEST == "YES"'
+    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
      when: on_failure
    # Always release resource
    - if: '$CI_JOB_NAME =~ /release_resource/'
      when: always
+    # Always cleanup
+    - if: '$CI_JOB_NAME =~ /cleanup/'
+      when: always
    # Default is to run if previous stage succeeded
    - when: on_success

@@ -42,9 +45,11 @@ variables:
  extends: [.on_quartz]
  stage: build_and_test
  script:
+    # THREADS is used by 'tests/gitlab/build_and_test', run below
    - export THREADS=12
    - echo ${ALLOC_NAME}
    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
    - echo ${JOBID}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 30 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --build-root "${BUILD_ROOT}" --data
-
+    - echo ${MFEM_DATA_DIR}
+    - echo ${SPEC}
+    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 30 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
@@ -0,0 +1,81 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Jobs report
+.report_job_success:
+  script:
+    - echo ${MACHINE_NAME}
+    - echo ${AUTOTEST}
+    - echo ${AUTOTEST_COMMIT}
+    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
+    - cd ${AUTOTEST_ROOT}
+    - |
+      (
+        date
+        echo "Waiting to aquire lock on '$PWD/autotest.lock' ..."
+        # try to get an excusive lock on fd 9 (autotest.lock) repeating the try
+        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
+        # command to hang indefinitely sometimes, so we use the timeout & retry
+        # as a workaround; we may want to add a counter for the number of
+        # retries to interrupt a potential infinite loop
+        while ! flock -w 5 9; do
+          true
+        done
+        echo "Aquired lock on '$PWD/autotest.lock'"
+        date
+        # Report SUCCESS while holding the file lock on 'autotest.lock'.
+        # The next script uses the following environment variables:
+        # - MACHINE_NAME, AUTOTEST_ROOT, AUTOTEST_COMMIT
+        # - CI_COMMIT_REF_SLUG, CI_PROJECT_DIR, CI_PIPELINE_URL
+        # It also calls the script '.gitlab/scripts/safe_create_rundir'.
+        ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test_success
+        err=$?
+        # sleep for a period to allow NFS to propagate the above changes;
+        # clearly, there is no guarantee that other NFS clients will see the
+        # changes even after the timeout
+        sleep 10
+        exit $err
+      ) 9> autotest.lock
+
+.report_job_failure:
+  script:
+    - echo ${MACHINE_NAME}
+    - echo ${AUTOTEST}
+    - echo ${AUTOTEST_COMMIT}
+    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
+    - cd ${AUTOTEST_ROOT}
+    - |
+      (
+        date
+        echo "Waiting to aquire lock on '$PWD/autotest.lock' ..."
+        # try to get an excusive lock on fd 9 (autotest.lock) repeating the try
+        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
+        # command to hang indefinitely sometimes, so we use the timeout & retry
+        # as a workaround; we may want to add a counter for the number of
+        # retries to interrupt a potential infinite loop
+        while ! flock -w 5 9; do
+          true
+        done
+        echo "Aquired lock on '$PWD/autotest.lock'"
+        date
+        # Report FAILURE while holding the file lock on 'autotest.lock'.
+        # The next script uses the following environment variables:
+        # - MACHINE_NAME, AUTOTEST_ROOT, AUTOTEST_COMMIT
+        # - CI_COMMIT_REF_SLUG, CI_PROJECT_DIR, CI_PIPELINE_URL
+        # It also calls the script '.gitlab/scripts/safe_create_rundir'.
+        ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test_failure
+        err=$?
+        # sleep for a period to allow NFS to propagate the above changes;
+        # clearly, there is no guarantee that other NFS clients will see the
+        # changes even after the timeout
+        sleep 10
+        exit $err
+      ) 9> autotest.lock
@@ -9,13 +9,6 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

-# TPLS_DIR is used in .gitlab/scripts/baseline to provide the tpls location
-# when call the runtest script in MFEM test repo.
-# Note: the value must be consistent with what setup_baseline does.
-variables:
-  TPLS_DIR: ${BUILD_ROOT}/tpls
-  AUTOTEST_ROOT: ${CI_BUILDS_DIR}/MFEM_${MACHINE_NAME}_baseline
-
 # The setup_baseline job doesn't rely on MFEM git repo. It prepares a
 # pipeline-wide working directory downloading/updating external repos.
 # TODO:
@@ -30,13 +23,50 @@ setup_baseline:
  variables:
    GIT_STRATEGY: none
  script:
+    #
+    # Setup ${BUILD_ROOT}/tpls and ${BUILD_ROOT}/tests:
+    #
+    - echo "MACHINE_NAME = ${MACHINE_NAME}"
+    - echo "REBASELINE = ${REBASELINE}"
+    - echo "AUTOTEST = ${AUTOTEST}"
+    - echo "AUTOTEST_COMMIT = ${AUTOTEST_COMMIT}"
    - echo "BUILD_ROOT ${BUILD_ROOT}"
    - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
    - if [ ! -d "tpls" ]; then git clone ${TPLS_REPO}; fi
    - if [ ! -d "tests" ]; then git clone ${TESTS_REPO}; fi
    - cd tpls && git pull && cd ..
    - cd tests && git pull origin && cd ..
+    #
+    # Setup ${AUTOTEST_ROOT}/autotest:
+    #
    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
    - mkdir -p ${AUTOTEST_ROOT} && cd ${AUTOTEST_ROOT}
-    - if [ ! -d "autotest" ]; then git clone ${AUTOTEST_REPO}; fi
-    - cd autotest && git pull && cd ..
+    - command -v flock || echo "Required command 'flock' not found"
+    - |
+      (
+        date
+        echo "Waiting to aquire lock on '$PWD/autotest.lock' ..."
+        # try to get an excusive lock on fd 9 (autotest.lock) repeating the try
+        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
+        # command to hang indefinitely sometimes, so we use the timeout & retry
+        # as a workaround; we may want to add a counter for the number of
+        # retries to interrupt a potential infinite loop
+        while ! flock -w 5 9; do
+          true
+        done
+        echo "Aquired lock on '$PWD/autotest.lock'"
+        date
+        # clone/update the autotest repo while holding the file lock on
+        # 'autotest.lock'
+        err=0
+        if [[ ! -d "autotest" ]]; then
+          git clone ${AUTOTEST_REPO}
+        else
+          cd autotest && git pull && cd ..
+        fi || err=1
+        # sleep for a period to allow NFS to propagate the above changes;
+        # clearly, there is no guarantee that other NFS clients will see the
+        # changes even after the timeout
+        sleep 10
+        exit $err
+      ) 9> autotest.lock
@@ -9,13 +9,10 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

-variables:
-  AUTOTEST_ROOT: ${CI_BUILDS_DIR}/MFEM_${MACHINE_NAME}_build_and_test
-
-# setup clones the mfem/data repo in ${BUILD_ROOT}. The build_and_test script
-# then symlinks the repo to the parent directory of the MFEM source directory.
-# Unit tests that depend on the mfem/data repo will then detect that this
-# directory is present and be enabled.
+# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test
+# script then symlinks the repo to the parent directory of the MFEM source
+# directory. Unit tests that depend on the mfem/data repo will then detect that
+# this directory is present and be enabled.
 setup:
  tags:
    - shell
@@ -24,11 +21,74 @@ setup:
  variables:
    GIT_STRATEGY: none
  script:
-    - echo "BUILD_ROOT ${BUILD_ROOT}"
-    - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
-    - if [ ! -d data ]; then git clone ${MFEM_DATA_REPO}; fi
+    #
+    # Setup MFEM_DATA_DIR=${SHARED_REPOS_DIR}/mfem-data, see '.gitlab-ci.yml'
+    # and '.gitlab/configs/<machine>-config.yml'
+    #
+    - echo "MACHINE_NAME = ${MACHINE_NAME}"
+    - echo "AUTOTEST = ${AUTOTEST}"
+    - echo "AUTOTEST_COMMIT = ${AUTOTEST_COMMIT}"
+    - echo "SHARED_REPOS_DIR ${SHARED_REPOS_DIR}"
+    - mkdir -p ${SHARED_REPOS_DIR} && cd ${SHARED_REPOS_DIR}
+    - command -v flock || echo "Required command 'flock' not found"
+    - |
+      (
+        date
+        echo "Waiting to aquire lock on '$PWD/mfem-data.lock' ..."
+        # try to get an excusive lock on fd 9 (mfem-data.lock) repeating the try
+        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
+        # command to hang indefinitely sometimes, so we use the timeout & retry
+        # as a workaround; we may want to add a counter for the number of
+        # retries to interrupt a potential infinite loop
+        while ! flock -w 5 9; do
+          true
+        done
+        echo "Aquired lock on '$PWD/mfem-data.lock'"
+        date
+        # clone/update the mfem/data repo while holding the file lock on
+        # 'mfem-data.lock'
+        err=0
+        if [[ ! -d "mfem-data" ]]; then
+          git clone ${MFEM_DATA_REPO} "mfem-data"
+        else
+          cd "mfem-data" && git pull && cd ..
+        fi || err=1
+        # sleep for a period to allow NFS to propagate the above changes;
+        # clearly, there is no guarantee that other NFS clients will see the
+        # changes even after the timeout
+        sleep 10
+        exit $err
+      ) 9> mfem-data.lock
+    #
+    # Setup ${AUTOTEST_ROOT}/autotest:
+    #
    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
    - mkdir -p ${AUTOTEST_ROOT} && cd ${AUTOTEST_ROOT}
-    - if [ ! -d "autotest" ]; then git clone ${AUTOTEST_REPO}; fi
-    - cd autotest && git pull && cd ..
-
+    - |
+      (
+        date
+        echo "Waiting to aquire lock on '$PWD/autotest.lock' ..."
+        # try to get an excusive lock on fd 9 (autotest.lock) repeating the try
+        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
+        # command to hang indefinitely sometimes, so we use the timeout & retry
+        # as a workaround; we may want to add a counter for the number of
+        # retries to interrupt a potential infinite loop
+        while ! flock -w 5 9; do
+          true
+        done
+        echo "Aquired lock on '$PWD/autotest.lock'"
+        date
+        # clone/update the autotest repo while holding the file lock on
+        # 'autotest.lock'
+        err=0
+        if [[ ! -d "autotest" ]]; then
+          git clone ${AUTOTEST_REPO}
+        else
+          cd autotest && git pull && cd ..
+        fi || err=1
+        # sleep for a period to allow NFS to propagate the above changes;
+        # clearly, there is no guarantee that other NFS clients will see the
+        # changes even after the timeout
+        sleep 10
+        exit $err
+      ) 9> autotest.lock
@@ -22,6 +22,7 @@ allocate_resource:
  extends: .on_corona
  stage: allocate_resource
  script:
+    - echo ${ALLOC_NAME}
    - salloc --exclusive --nodes=1 --partition=mi60 --time=30 --no-shell --job-name=${ALLOC_NAME}
  timeout: 6h
  needs: [setup]
@@ -40,24 +41,27 @@ release_resource:
  extends: .on_corona
  stage: release_resource_and_report
  script:
+    - echo ${ALLOC_NAME}
    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
+    - echo ${JOBID}
    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
  needs: [rocm_gcc_8.3.1]

 # Jobs report
 report_job_success:
-  extends: .on_corona
  stage: release_resource_and_report
-  script:
-    - .gitlab/scripts/report_build_and_test_success
+  extends:
+    - .on_corona
+    - .report_job_success

 report_job_failure:
-  extends: .on_corona
  stage: release_resource_and_report
-  script:
-    - .gitlab/scripts/report_build_and_test_failure
+  extends:
+    - .on_corona
+    - .report_job_failure

 include:
  - local: .gitlab/configs/common.yml
  - local: .gitlab/configs/corona-config.yml
  - local: .gitlab/configs/setup-build-and-test.yml
+  - local: .gitlab/configs/report-build-and-test.yml
@@ -21,18 +21,19 @@ opt_mpi_cuda_xl_16_1_1_8:

 # Jobs report
 report_job_success:
-  extends: .on_lassen
  stage: report
-  script:
-    - .gitlab/scripts/report_build_and_test_success
+  extends:
+    - .on_lassen
+    - .report_job_success

 report_job_failure:
-  extends: .on_lassen
  stage: report
-  script:
-    - .gitlab/scripts/report_build_and_test_failure
+  extends:
+    - .on_lassen
+    - .report_job_failure

 include:
  - local: .gitlab/configs/common.yml
  - local: .gitlab/configs/lassen-config.yml
  - local: .gitlab/configs/setup-build-and-test.yml
+  - local: .gitlab/configs/report-build-and-test.yml
@@ -16,12 +16,26 @@ stages:
  - setup
  - baseline_check
  - baseline_report
+  - cleanup
  - baseline_publish

 baselinecheck_mfem_intel_quartz:
  extends: [.on_quartz]
  stage: baseline_check
+  variables:
+    # TPLS_DIR is used in .gitlab/scripts/baseline to provide the tpls location
+    # when call the runtest script in MFEM test repo.
+    # Note: the value must be consistent with the setup performed in
+    #       .gitlab/configs/setup-baseline.yml.
+    TPLS_DIR: ${BUILD_ROOT}/tpls
  script:
+    - echo ${BUILD_ROOT}
+    - echo ${TPLS_DIR}
+    # Used by the tests in MFEM/tests:
+    - export MFEM_TEST_NP=32
+    # The next script uses the following environment variables:
+    # * BASELINE_TEST, SYS_TYPE, CI_PROJECT_DIR, ARTIFACTS_DIR,
+    # * BUILD_ROOT, TPLS_DIR, MACHINE_NAME
    - .gitlab/scripts/baseline
  artifacts:
    when: always
@@ -29,33 +43,88 @@ baselinecheck_mfem_intel_quartz:
      - ${ARTIFACTS_DIR}
  allow_failure: true

+cleanup:
+  extends: .on_quartz
+  stage: cleanup
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - echo "BUILD_ROOT=${BUILD_ROOT}"
+    - rm -rf "${BUILD_ROOT}" || true
+
 report_baseline:
  extends: [.on_quartz]
  stage: baseline_report
  script:
-    - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - mkdir -p ${MACHINE_NAME}
-    - rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-${BASELINE_TEST}-${CI_COMMIT_REF_SLUG}"
-    - rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir ${rundir})
-    - cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
-    # We create an autotest-email.html file, because that's how we signal that there was a diff (temporary).
+    - echo ${MACHINE_NAME}
+    - echo ${AUTOTEST}
+    - echo ${AUTOTEST_COMMIT}
+    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
+    - cd ${AUTOTEST_ROOT}
    - |
-      if [[ -f ${rundir}/*.err ]]
-      then
-        echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/*.err
-        cp ${rundir}/*.err ${rundir}/autotest-email.html
-      fi
-    - git add ${rundir}
-    - git commit -am "GitLab CI log for ${BASELINE_TEST} on ${MACHINE_NAME} with intel ($(date +%Y-%m-%d))"
-    - git push origin master
+      (
+        date
+        echo "Waiting to aquire lock on '$PWD/autotest.lock' ..."
+        # try to get an excusive lock on fd 9 (autotest.lock) repeating the try
+        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
+        # command to hang indefinitely sometimes, so we use the timeout & retry
+        # as a workaround; we may want to add a counter for the number of
+        # retries to interrupt a potential infinite loop
+        while ! flock -w 5 9; do
+          true
+        done
+        echo "Aquired lock on '$PWD/autotest.lock'"
+        date
+        # ----------------------
+        cd ${AUTOTEST_ROOT}/autotest || \
+          { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
+        mkdir -p ${MACHINE_NAME}
+        rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-${BASELINE_TEST}-${CI_COMMIT_REF_SLUG}"
+        rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir ${rundir})
+        cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
+        # We create an autotest-email.html file, because that's how we signal that there was a diff (temporary).
+        if [[ -f ${rundir}/${BASELINE_TEST}.err ]]; then
+          cp ${rundir}/${BASELINE_TEST}.err ${rundir}/autotest-email.html
+        fi
+        printf "%s\n" "" "Pipeline URL:" "$CI_PIPELINE_URL" \
+          >> ${rundir}/pipeline.txt
+        msg="GitLab CI log for ${BASELINE_TEST} on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
+        if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
+          git pull && \
+          git add ${rundir} && \
+          git commit -m "${msg}" && \
+          git push origin master
+        else
+          for file in ${rundir}/*; do
+            echo "------------------------------"
+            echo "Content of '$file'"
+            echo "******************************"
+            cat $file
+            echo "******************************"
+          done
+          rm -rf ${rundir} || true
+        fi
+        err=$?
+        # ----------------------
+        # sleep for a period to allow NFS to propagate the above changes;
+        # clearly, there is no guarantee that other NFS clients will see the
+        # changes even after the timeout
+        sleep 10
+        exit $err
+      ) 9> autotest.lock

 baselinepublish_mfem_quartz:
  extends: [.on_quartz]
  stage: baseline_publish
  rules:
-    - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "YES"'
+    # - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "YES"'
+    - if: '$REBASELINE == "YES"'
      when: manual
  script:
+    - echo ${BUILD_ROOT}
+    - echo ${PWD}
+    - echo ${ARTIFACTS_DIR}
+    - ls -lA ${ARTIFACTS_DIR}
    - .gitlab/scripts/rebaseline

 include:
@@ -22,6 +22,7 @@ allocate_resource:
  extends: .on_quartz
  stage: allocate_resource
  script:
+    - echo ${ALLOC_NAME}
    - salloc --exclusive --nodes=1 --partition=pdebug --time=30 --no-shell --job-name=${ALLOC_NAME}
  timeout: 6h

@@ -73,23 +74,26 @@ release_resource:
  extends: .on_quartz
  stage: release_resource_and_report
  script:
+    - echo ${ALLOC_NAME}
    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
+    - echo ${JOBID}
    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})

 # Jobs report
 report_job_success:
-  extends: .on_quartz
  stage: release_resource_and_report
-  script:
-    - .gitlab/scripts/report_build_and_test_success
+  extends:
+    - .on_quartz
+    - .report_job_success

 report_job_failure:
-  extends: .on_quartz
  stage: release_resource_and_report
-  script:
-    - .gitlab/scripts/report_build_and_test_failure
+  extends:
+    - .on_quartz
+    - .report_job_failure

 include:
  - local: .gitlab/configs/common.yml
  - local: .gitlab/configs/quartz-config.yml
  - local: .gitlab/configs/setup-build-and-test.yml
+  - local: .gitlab/configs/report-build-and-test.yml
@@ -20,7 +20,8 @@ base_out=${base}.out
 artifacts_path=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}

 # prepare
-cd ${BUILD_ROOT}
+cd ${BUILD_ROOT} || \
+  { echo "Invalid BUILD_ROOT=$BUILD_ROOT"; exit 1; }
 ln -snf ${CI_PROJECT_DIR} mfem
 cd tests
 [[ -d _${BASELINE_TEST} ]] && rm -rf _${BASELINE_TEST}
@@ -33,6 +34,9 @@ elif [[ ${MACHINE_NAME} == "corona" ]]; then
  srun --nodes=1 -t 60 -p mi60 ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "lassen" ]]; then
  lalloc 1 -q pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+else
+  echo "Unknown machine: MACHINE_NAME=$MACHINE_NAME"
+  exit 1
 fi

 # post
@@ -60,6 +64,10 @@ then
  cp ${base_out} ${artifacts_path}/${base_out}
 fi

+if [[ -f ${BASELINE_TEST}.out ]]; then
+  cp ${BASELINE_TEST}.out ${artifacts_path}
+fi
+
 # base_diff won't even exist if there is no difference.
 if [[ -f ${base_diff} ]]
 then
@@ -13,20 +13,33 @@

 echo "Runs if there was at least one failure on ${MACHINE_NAME}"

-cd ${AUTOTEST_ROOT}/autotest && git pull
+cd ${AUTOTEST_ROOT}/autotest || \
+  { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
 mkdir -p ${MACHINE_NAME}

 rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
 rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir)

-echo "There was an error while running CI on ${MACHINE_NAME}" > ${rundir}/gitlab.err
-echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
+printf "%s\n" "Some 'build-and-test' jobs on ${MACHINE_NAME} FAILED." \
+  "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.err

 msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))"

+# Create 'autotest-email.html' to indicate failure:
 cp ${rundir}/gitlab.err ${rundir}/autotest-email.html

-git pull
-git add ${rundir}
-git commit -am "${msg}"
-git push origin master
+if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
+  git pull && \
+  git add ${rundir} && \
+  git commit -m "${msg}" && \
+  git push origin master
+else
+  for file in ${rundir}/*; do
+    echo "------------------------------"
+    echo "Content of '$file'"
+    echo "******************************"
+    cat $file
+    echo "******************************"
+  done
+  rm -rf ${rundir} || true
+fi
@@ -13,18 +13,30 @@

 echo "Can only run if all the ${MACHINE_NAME} jobs passed"

-cd ${AUTOTEST_ROOT}/autotest && git pull
+cd ${AUTOTEST_ROOT}/autotest || \
+  { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
 mkdir -p ${MACHINE_NAME}

 rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
 rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir)

-echo "The ${MACHINE_NAME} jobs were successful" > ${rundir}/gitlab.out
-echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
+printf "%s\n" "The 'build-and-test' jobs on ${MACHINE_NAME} were SUCCESSFUL." \
+  "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.out

 msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))"

-git pull
-git add ${rundir}
-git commit -am "${msg}"
-git push origin master
+if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
+  git pull && \
+  git add ${rundir} && \
+  git commit -m "${msg}" && \
+  git push origin master
+else
+  for file in ${rundir}/*; do
+    echo "------------------------------"
+    echo "Content of '$file'"
+    echo "******************************"
+    cat $file
+    echo "******************************"
+  done
+  rm -rf ${rundir} || true
+fi
@@ -10,6 +10,17 @@

 Version 4.3.1 (development)
 ===========================
+- More explicit and consistent formating of the output of iterative solvers
+  with the new IterativeSolver::PrintLevel options. See linalg/solvers.hpp.
+
+- Added support for automatic differentiation. Users can select between native
+  implementation and external library implementation during configuration. One
+  parallel and two serial examples are implemented in the miniapps/autodiff/
+  directory.
+
+- GridFunctionCoefficient (and the related vector, gradient, divergence, and
+  curl classes) now work properly with LORDiscretization and LORSolver.
+
 - Added support for mesh preprocessing to resolve fine scale problem data
  before simulation. This feature uses adaptive mesh refinement to control the
  associated data oscillation error. See the new Example 30/30p.
@@ -67,6 +78,22 @@ Version 4.3.1 (development)
 - Added initial TMOP-based capabilities for surface fitting and tangential
  relaxation in the mesh-optimizer and pmesh-optimizer miniapps.

+- Added ParMesh Adjaceny Set (adjset) creation support to the Conduit Mesh
+  Blueprint MFEM wrapper functions in ConduitDataCollection.
+
+- Added full assembly / device support for the LinearForm integrators:
+  * DomainLF: L(v) := (f, v)
+  * VectorDomainLF: L(v) := ((f1,...,fn), (v1,...,vn))
+  * DomainLFGrad: L(v) := (f,grad v)
+  * VectorDomainLFGrad:  L(v) := ((f1x,f1y,f1z,...,fnx,fny,fnz),grad(v1,...,vn))
+  It can be enabled by setting the linear assembly level to
+  'LinearAssemblyLevel::FULL' with the 'LinearForm::SetAssemblyLevel' method.
+
+- `HypreParVector` and `Vector` now support move semantics, and the copy
+  constructor for `HypreParVector` now copies the local vector data.
+
+- The HPC versions of ex1 and ex1p (in miniapps/performance) now support
+  runtime selection of either 2D or 3D meshes.

 Version 4.3, released on July 29, 2021
 ======================================
@@ -252,6 +252,11 @@ if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
  endif()
  find_package(OpenMP REQUIRED)
  set(OPENMP_LIBRARIES ${OpenMP_CXX_LIBRARIES})
+  if(APPLE)
+    # On macOS, the compiler needs additional help to find the <omp.h> header.
+    # See issue #2642 for more information.
+    include_directories(${OpenMP_CXX_INCLUDE_DIRS})
+  endif(APPLE)
 endif()

 # SuiteSparse (before SUNDIALS which may depend on KLU)
@@ -367,6 +372,12 @@ if (MFEM_USE_HIOP)
  # find_package updates HIOP_FOUND, HIOP_INCLUDE_DIRS, HIOP_LIBRARIES
 endif()

+# CoDiPack package
+if (MFEM_USE_CODIPACK)
+  find_package(CODIPACK REQUIRED)
+  # find_package updates CODIPACK_FOUND, CODIPACK_INCLUDE_DIRS, CODIPACK_LIBRARIES
+endif()
+
 # OCCA
 if (MFEM_USE_OCCA)
   find_package(OCCA REQUIRED)
@@ -446,7 +457,7 @@ endif()
 set(MFEM_TPLS OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS PETSC
    SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB NETCDF
    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
-    CUSPARSE MKL_CPARDISO AMGX CALIPER BENCHMARK PARELAG MPI_CXX)
+    CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG MPI_CXX)

 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
@@ -0,0 +1,133 @@
+# MFEM Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, caste, color, religion, or sexual
+identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the overall
+  community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or advances of
+  any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email address,
+  without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at mfem@llnl.gov.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident. Anyone involved in the reported behavior will recuse
+themselves from the investigation and decision making about the resolution of
+the complaint.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series of
+actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or permanent
+ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within the
+community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.1, available at
+[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
+
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
+[https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations
@@ -21,6 +21,9 @@ documentation; new examples and miniapps; HPC performance improvements; etc.
 MFEM is distributed under the terms of the BSD-3 license. All new contributions
 must be made under this license.

+Note also that MFEM has a [Code of Conduct](CODE_OF_CONDUCT.md). By participating
+in the MFEM community, you agree to abide by its rules.
+
 If you plan on contributing to MFEM, consider reviewing the
 [issue tracker](https://github.com/mfem/mfem/issues) first to check if a thread
 already exists for your desired feature or the bug you ran into. Use a pull
@@ -123,6 +126,7 @@ The MFEM source code has the following structure:
  ├── mesh
  ├── miniapps
  │   ├── adjoint
+  │   ├── autodiff
  │   ├── common
  │   ├── electromagnetics
  │   ├── gslib
@@ -137,6 +141,7 @@ The MFEM source code has the following structure:
  │   ├── tools
  │   └── toys
  └── tests
+      ├── benchmarks
      ├── convergence
      ├── gitlab
      ├── mem_manager
@@ -467,6 +467,14 @@ MFEM_USE_HIOP = YES/NO
   Enable the usage of HiOp (https://github.com/LLNL/hiop) in MFEM. HiOp is an
   HPC solver for nonlinear optimization problems.

+MFEM_USE_CODIPACK = YES/NO
+  Enable automatic differentiation using the CoDiPack library.
+  www.scicomp.uni-kl.de/codi/
+
+MFEM_USE_ADFORWARD = YES/NO
+  Enable forward mode for AD packages. This option is valid
+  only if the AD package supports two modes (backward/forward).
+
 MFEM_USE_CUDA = YES/NO
   Enables support for CUDA devices in MFEM. CUDA is a parallel computing
   platform and programming model for general computing on graphical processing
@@ -703,6 +711,11 @@ The specific libraries and their options are:
  Options: HIOP_OPT, HIOP_LIB.
  Versions: HIOP >= 0.4.6.

+- CoDiPack (optiobal), used with MFEM_USE_CODIPACK = YES
+  URL: https://www.scicomp.uni-kl.de/codi/
+  Options: CODIPACK_OPT
+  Versions: 1.9.3
+
 - GSLIB (optional), used when MFEM_USE_GSLIB = YES. The gslib library must be
  built prior to the MFEM build, as follows: download gslib-1.0.7, untar it at
  the same level as MFEM and create a symbolic link: "ln -s gslib-1.0.7 gslib".
@@ -908,6 +921,8 @@ MFEM_USE_MPFR
 MFEM_USE_ZLIB
 MFEM_USE_PUMI
 MFEM_USE_HIOP
+MFEM_USE_CODIPACK
+MFEM_USE_ADFORWARD
 MFEM_USE_CUDA
 MFEM_USE_OCCA
 MFEM_USE_CEED
@@ -967,6 +982,7 @@ The CMake build system adds auto-detection for the following packages/libraries:
 - POSIXCLOCKS
 - PUMI
 - HIOP
+ - CoDiPack
 - OCCA
 - RAJA
 - UMPIRE
@@ -12,6 +12,9 @@ to enable high-performance scalable finite element discretization research and
 application development on a wide variety of platforms, ranging from laptops to
 supercomputers.

+We welcome contributions and feedback from the community. Please see the file
+CONTRIBUTING.md for additional details about our development process.
+
 * For building instructions, see the file INSTALL, or type "make help".

 * Copyright and licensing information can be found in files LICENSE and NOTICE.
@@ -19,9 +22,6 @@ supercomputers.
 * The best starting point for new users interested in MFEM's features is to
  review the examples and miniapps at https://mfem.org/examples.

-* Developers interested in contributing to the library, should read the
-  instructions and documentation in the CONTRIBUTING.md file.
-
 Conceptually, MFEM can be viewed as a finite element toolbox that provides the
 building blocks for developing finite element algorithms in a manner similar to
 that of MATLAB for linear algebra methods. In particular, MFEM provides support
@@ -54,6 +54,8 @@ set(MFEM_USE_CEED @MFEM_USE_CEED@)
 set(MFEM_USE_UMPIRE @MFEM_USE_UMPIRE@)
 set(MFEM_USE_SIMD @MFEM_USE_SIMD@)
 set(MFEM_USE_ADIOS2 @MFEM_USE_ADIOS2@)
+set(MFEM_USE_CODIPACK @MFEM_USE_CODIPACK@)
+set(MFEM_USE_ADFORWARD @MFEM_USE_ADFORWARD@)
 set(MFEM_USE_CALIPER @MFEM_USE_CALIPER@)
 set(MFEM_USE_BENCHMARK @MFEM_USE_BENCHMARK@)
 set(MFEM_USE_PARELAG @MFEM_USE_PARELAG@)
@@ -175,6 +175,12 @@
 // Enable interface to the MKL CPardiso library.
 #cmakedefine MFEM_USE_MKL_CPARDISO

+// Use forward mode for automatic differentiation
+#cmakedefine MFEM_USE_ADFORWARD
+
+// Enable the use of the CoDiPack library for AD
+#cmakedefine MFEM_USE_CODIPACK
+
 // Enable MFEM functionality based on the Google Benchmark library.
 #cmakedefine MFEM_USE_BENCHMARK

@@ -0,0 +1,24 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Automatic differentiation using the CoDiPack library.
+#  www.scicomp.uni-kl.de/codi/
+# Sets the following variables:
+#   - CODIPACK_FOUND
+#   - CODIPACK_INCLUDE_DIRS
+#   - CODIPACK_LIBRARIES
+
+include(MfemCmakeUtilities)
+mfem_find_package(CODIPACK CODIPACK CODIPACK_DIR
+  "include" "codi.h"
+  "lib" ""
+  "Paths to headers required by CODIPACK."
+  "Libraries required by CODIPACK.")
@@ -180,6 +180,12 @@
 // Enable interface to the MKL CPardiso library.
 // #define MFEM_USE_MKL_CPARDISO

+// Use forward mode for automatic differentiation
+// #define MFEM_USE_ADFORWARD
+
+// Enable the use of the CoDiPack library for AD
+// #define MFEM_USE_CODIPACK
+
 // Enable functionality based on the Google Benchmark library.
 // #define MFEM_USE_BENCHMARK

@@ -58,6 +58,8 @@ MFEM_USE_UMPIRE        = @MFEM_USE_UMPIRE@
 MFEM_USE_SIMD          = @MFEM_USE_SIMD@
 MFEM_USE_ADIOS2        = @MFEM_USE_ADIOS2@
 MFEM_USE_MKL_CPARDISO  = @MFEM_USE_MKL_CPARDISO@
+MFEM_USE_ADFORWARD     = @MFEM_USE_ADFORWARD@
+MFEM_USE_CODIPACK      = @MFEM_USE_CODIPACK@
 MFEM_USE_BENCHMARK     = @MFEM_USE_BENCHMARK@
 MFEM_USE_PARELAG       = @MFEM_USE_PARELAG@

@@ -58,6 +58,8 @@ option(MFEM_USE_SIMD "Enable use of SIMD intrinsics" OFF)
 option(MFEM_USE_ADIOS2 "Enable ADIOS2" OFF)
 option(MFEM_USE_CALIPER "Enable Caliper support" OFF)
 option(MFEM_USE_MKL_CPARDISO "Enable MKL CPardiso" OFF)
+option(MFEM_USE_ADFORWARD "Enable forward mode for AD" OFF)
+option(MFEM_USE_CODIPACK "Enable automatic differentiation (AD) using CoDiPack" OFF)
 option(MFEM_USE_BENCHMARK "Enable Google Benchmark" OFF)
 option(MFEM_USE_PARELAG "Enable ParELAG" OFF)

@@ -243,6 +245,9 @@ set(BLAS_LIBRARIES "" CACHE STRING "The BLAS library.")
 set(LAPACK_INCLUDE_DIRS "" CACHE STRING "Path to LAPACK headers.")
 set(LAPACK_LIBRARIES "" CACHE STRING "The LAPACK library.")

+set(CODIPACK_INCLUDE_DIRS "${MFEM_DIR}/../CoDiPack/inlude" CACHE STRING "Path to CoDiPack headers.")
+set(CODIPACK_LIBRARIES "")
+
 # Some useful variables:
 set(CMAKE_SKIP_PREPROCESSED_SOURCE_RULES ON) # Skip *.i rules
 set(CMAKE_SKIP_ASSEMBLY_SOURCE_RULES  ON)    # Skip *.s rules
@@ -59,6 +59,9 @@ HIP_FLAGS = --amdgpu-target=$(HIP_ARCH)
 HIP_XCOMPILER =
 HIP_XLINKER   = -Wl,

+# Flags for generating dependencies.
+DEP_FLAGS = -MM -MT
+
 ifneq ($(NOTMAC),)
   AR      = ar
   ARFLAGS = crv
@@ -86,6 +89,9 @@ else
   BUILD_RPATH = $(XLINKER)-undefined,dynamic_lookup
   INSTALL_SOFLAGS = $(subst $1 ,,$(call MAKE_SOFLAGS,$(MFEM_LIB_DIR)))
   INSTALL_RPATH = $(XLINKER)-undefined,dynamic_lookup
+   # Silence unused command line argument warnings when generating dependencies
+   # with mpicxx and clang
+   DEP_FLAGS := -Wno-unused-command-line-argument $(DEP_FLAGS)
 endif

 # Set CXXFLAGS to overwrite the default selection of DEBUG_FLAGS/OPTIM_FLAGS
@@ -151,6 +157,8 @@ MFEM_USE_UMPIRE        = NO
 MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
 MFEM_USE_MKL_CPARDISO  = NO
+MFEM_USE_ADFORWARD     = NO
+MFEM_USE_CODIPACK      = NO
 MFEM_USE_BENCHMARK     = NO
 MFEM_USE_PARELAG       = NO

@@ -408,6 +416,11 @@ HIOP_DIR = @MFEM_DIR@/../hiop/install
 HIOP_OPT = -I$(HIOP_DIR)/include
 HIOP_LIB = -L$(HIOP_DIR)/lib -lhiop $(LAPACK_LIB)

+# CoDiPack
+CODIPACK_DIR = @MFEM_DIR@/../CoDiPack
+CODIPACK_OPT = -I$(CODIPACK_DIR)
+CODIPACK_LIB =
+
 # GSLIB library
 GSLIB_DIR = @MFEM_DIR@/../gslib/build
 GSLIB_OPT = -I$(GSLIB_DIR)/include
@@ -415,7 +428,7 @@ GSLIB_LIB = -L$(GSLIB_DIR)/lib -lgs

 # CUDA library configuration
 CUDA_OPT =
-CUDA_LIB = -lcusparse
+CUDA_LIB = -lcusparse -lnvToolsExt

 # HIP library configuration (currently not needed)
 HIP_OPT =
@@ -781,6 +781,7 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/miniapps/gslib \
                         @MFEM_SOURCE_DIR@/miniapps/meshing \
                         @MFEM_SOURCE_DIR@/miniapps/mtop \
+                         @MFEM_SOURCE_DIR@/miniapps/autodiff \
                         @MFEM_SOURCE_DIR@/miniapps/navier \
                         @MFEM_SOURCE_DIR@/miniapps/nurbs \
                         @MFEM_SOURCE_DIR@/miniapps/performance \
@@ -194,6 +194,8 @@ namespace mfem {
 * - <a class="el" href="parheat_8cpp_source.html">Optimization gradients</a>: Gradients of PDE-constrained function
 * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
 * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
+ * - <a class="el" href="par__example_8cpp_source.html">Parallel pLaplacian example using AD</a>: Parallel pLaplacian example
+ * - <a class="el" href="seq__example_8cpp_source.html">Serial pLaplacian example using AD</a>: Serial pLaplacian example
 *
 * See also the <a class="el" href="https://mfem.org/examples/">examples documentation</a> online.
 */
@@ -82,78 +82,80 @@ include_directories(BEFORE ${PROJECT_BINARY_DIR})
 add_mfem_examples(ALL_EXE_SRCS)

 # Add a test for each example
-foreach(SRC_FILE ${ALL_EXE_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+if (MFEM_ENABLE_TESTING)
+  foreach(SRC_FILE ${ALL_EXE_SRCS})
+    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})

-  set(THIS_TEST_OPTIONS "-no-vis")
-  if (${TEST_NAME} MATCHES "ex0p?")
-    set(THIS_TEST_OPTIONS)
-  endif()
-  if (${TEST_NAME} MATCHES "ex10p*")
-    list(APPEND THIS_TEST_OPTIONS "-tf" "5")
-  elseif(${TEST_NAME} MATCHES "ex15p*")
-    list(APPEND THIS_TEST_OPTIONS "-e" "1")
-  elseif(${TEST_NAME} MATCHES "ex27p*")
-    list(APPEND THIS_TEST_OPTIONS "-dg")
-  endif()
+    set(THIS_TEST_OPTIONS "-no-vis")
+    if (${TEST_NAME} MATCHES "ex0p?")
+      set(THIS_TEST_OPTIONS)
+    endif()
+    if (${TEST_NAME} MATCHES "ex10p*")
+      list(APPEND THIS_TEST_OPTIONS "-tf" "5")
+    elseif(${TEST_NAME} MATCHES "ex15p*")
+      list(APPEND THIS_TEST_OPTIONS "-e" "1")
+    elseif(${TEST_NAME} MATCHES "ex27p*")
+      list(APPEND THIS_TEST_OPTIONS "-dg")
+    endif()

-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
-
-# Add CUDA/HIP tests.
-set(DEVICE_EXAMPLES
-    # serial examples with device support:
-    ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
-    # parallel examples with device support:
-    ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p ex24p ex25p ex26p)
-set(MFEM_TEST_DEVICE)
-if (MFEM_USE_CUDA)
-  set(MFEM_TEST_DEVICE "cuda")
-elseif (MFEM_USE_HIP)
-  set(MFEM_TEST_DEVICE "hip")
-endif()
-if (MFEM_TEST_DEVICE)
-  foreach(TEST_NAME ${DEVICE_EXAMPLES})
-    set(THIS_TEST_OPTIONS "-no-vis" "-d" "${MFEM_TEST_DEVICE}")
    if (NOT (${TEST_NAME} MATCHES ".*p$"))
-      add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_ser
+      add_test(NAME ${TEST_NAME}_ser
        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-    elseif (MFEM_USE_MPI)
-      add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_np=${MFEM_MPI_NP}
+    else()
+      add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
        ${MPIEXEC_PREFLAGS}
        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
        ${MPIEXEC_POSTFLAGS})
    endif()
  endforeach()
-endif()

-# If STRUMPACK is enabled, add a test run that uses it.
-if (MFEM_USE_STRUMPACK)
-  add_test(NAME ex11p_strumpack_np=${MFEM_MPI_NP}
-    COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-    ${MPIEXEC_PREFLAGS}
-    $<TARGET_FILE:ex11p> "-no-vis" "--strumpack"
-    ${MPIEXEC_POSTFLAGS})
-endif()
+  # Add CUDA/HIP tests.
+  set(DEVICE_EXAMPLES
+      # serial examples with device support:
+      ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
+      # parallel examples with device support:
+      ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p ex24p ex25p ex26p)
+  set(MFEM_TEST_DEVICE)
+  if (MFEM_USE_CUDA)
+    set(MFEM_TEST_DEVICE "cuda")
+  elseif (MFEM_USE_HIP)
+    set(MFEM_TEST_DEVICE "hip")
+  endif()
+  if (MFEM_TEST_DEVICE)
+    foreach(TEST_NAME ${DEVICE_EXAMPLES})
+      set(THIS_TEST_OPTIONS "-no-vis" "-d" "${MFEM_TEST_DEVICE}")
+      if (NOT (${TEST_NAME} MATCHES ".*p$"))
+        add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_ser
+          COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+      elseif (MFEM_USE_MPI)
+        add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_np=${MFEM_MPI_NP}
+          COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+          ${MPIEXEC_PREFLAGS}
+          $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+          ${MPIEXEC_POSTFLAGS})
+      endif()
+    endforeach()
+  endif()

-# If SuperLU_DIST is enabled, add a test run that uses it.
-if (MFEM_USE_SUPERLU)
-  add_test(NAME ex11p_superlu_np=${MFEM_MPI_NP}
-    COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-    ${MPIEXEC_PREFLAGS}
-    $<TARGET_FILE:ex11p> "-no-vis" "--superlu"
-    ${MPIEXEC_POSTFLAGS})
+  # If STRUMPACK is enabled, add a test run that uses it.
+  if (MFEM_USE_STRUMPACK)
+    add_test(NAME ex11p_strumpack_np=${MFEM_MPI_NP}
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:ex11p> "-no-vis" "--strumpack"
+      ${MPIEXEC_POSTFLAGS})
+  endif()
+
+  # If SuperLU_DIST is enabled, add a test run that uses it.
+  if (MFEM_USE_SUPERLU)
+    add_test(NAME ex11p_superlu_np=${MFEM_MPI_NP}
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:ex11p> "-no-vis" "--superlu"
+      ${MPIEXEC_POSTFLAGS})
+  endif()
 endif()

 # Include the examples/amgx directory if AmgX is enabled
@@ -50,30 +50,32 @@ add_mfem_examples(AMGX_EXAMPLES_SRCS ${PFX} copy_amgx_json_files test_amgx)
 # which builds the examples and runs:
 #   ctest -R amgx

-# Command line options for the tests.
-# Example 1/1p:
-set(EX1_TEST_OPTS)
-set(EX1P_TEST_OPTS)
+if (MFEM_ENABLE_TESTING)
+  # Command line options for the tests.
+  # Example 1/1p:
+  set(EX1_TEST_OPTS)
+  set(EX1P_TEST_OPTS)

-# Add the tests: one test per source file.
-foreach(SRC_FILE ${AMGX_EXAMPLES_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-  set(TEST_NAME ${PFX}${TEST_NAME})
+  # Add the tests: one test per source file.
+  foreach(SRC_FILE ${AMGX_EXAMPLES_SRCS})
+    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+    string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+    set(TEST_NAME ${PFX}${TEST_NAME})

-  set(THIS_TEST_OPTIONS "-no-vis")
-  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+    set(THIS_TEST_OPTIONS "-no-vis")
+    list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+    # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")

-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
+    if (NOT (${TEST_NAME} MATCHES ".*p$"))
+      add_test(NAME ${TEST_NAME}_ser
+        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+    else()
+      add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
+        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+        ${MPIEXEC_PREFLAGS}
+        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+        ${MPIEXEC_POSTFLAGS})
+    endif()
+  endforeach()
+endif()
@@ -30,22 +30,22 @@ set(PREFIX caliper_)
 add_mfem_examples(CALIPER_EXE_SRCS ${PREFIX})

 # Add a test for each example
-foreach(SRC_FILE ${CALIPER_EXE_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-
-  set(THIS_TEST_OPTIONS "-no-vis")
-
-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
+if (MFEM_ENABLE_TESTING)
+  foreach(SRC_FILE ${CALIPER_EXE_SRCS})
+    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})

+    set(THIS_TEST_OPTIONS "-no-vis")

+    if (NOT (${TEST_NAME} MATCHES ".*p$"))
+      add_test(NAME ${TEST_NAME}_ser
+        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+    else()
+      add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
+        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+        ${MPIEXEC_PREFLAGS}
+        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+        ${MPIEXEC_POSTFLAGS})
+    endif()
+  endforeach()
+endif()
@@ -120,8 +120,8 @@ int main(int argc, char *argv[])
   //    largest number that gives a final mesh with no more than 50,000
   //    elements.
   {
-      int ref_levels =
-         (int)floor(log(50000./mesh.GetNE())/log(2.)/dim);
+      const int NE = Device::IsEnabled() ? 1e5 : 50000.;
+      const int ref_levels = (int)floor(log(NE/mesh.GetNE())/log(2.)/dim);
      for (int l = 0; l < ref_levels; l++)
      {
         mesh.UniformRefinement();
@@ -171,6 +171,7 @@ int main(int argc, char *argv[])
   LinearForm b(&fespace);
   ConstantCoefficient one(1.0);
   b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   if (Device::IsEnabled()) { b.SetAssemblyLevel(LinearAssemblyLevel::FULL); }
   b.Assemble();

   // 8. Define the solution vector x as a finite element grid function
@@ -184,7 +185,10 @@ int main(int argc, char *argv[])
   //    domain integrator.
   BilinearForm a(&fespace);
   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   // 'Default' E2E kernel
+   //a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   // Can be E2E, L2L, libCeed, etc.
+   a.AddDomainIntegrator(new MassIntegrator(one));

   // 10. Assemble the bilinear form and the corresponding linear system,
   //     applying any necessary transformations such as: eliminating boundary
@@ -197,7 +201,7 @@ int main(int argc, char *argv[])
   Vector B, X;
   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);

-   cout << "Size of linear system: " << A->Height() << endl;
+   //cout << "Size of linear system: " << A->Height() << endl;

   // 11. Solve the linear system A X = B.
   if (!pa)
@@ -223,10 +227,70 @@ int main(int argc, char *argv[])
            ceed::AlgebraicSolver M(a, ess_tdof_list);
            PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
         }
-         else
+         /*else
         {
            OperatorJacobiSmoother M(a, ess_tdof_list);
            PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
+         }*/
+         else
+         {
+            const int myid = 0;
+            const int max_it = 50;
+            const int print_lvl = -1;
+            const double rtol = 1e-12;
+
+            CGSolver cg;
+            cg.SetRelTol(rtol);
+            cg.SetOperator(*A);
+
+            // Warm-up CG solve (in case of JIT to avoid timing it)
+            {
+               Vector Y(X);
+               cg.SetMaxIter(2);
+               cg.SetPrintLevel(-1);
+               cg.Mult(B, Y);
+               MFEM_DEVICE_SYNC;
+            }
+
+            // benchmark this problem
+            {
+               tic_toc.Clear();
+               cg.SetMaxIter(max_it);
+               cg.SetPrintLevel(print_lvl);
+               {
+                  tic_toc.Start();
+                  cg.Mult(B, X);
+                  MFEM_DEVICE_SYNC;
+                  tic_toc.Stop();
+               }
+            }
+            // Final norm check
+            const double final_norm = cg.GetFinalNorm();
+            MFEM_VERIFY(final_norm < sqrt(rtol),
+                        "FinalNorm (" << final_norm << ") Error!");
+            // Number of iteration check
+            const int num_iter = cg.GetNumIterations();
+            MFEM_VERIFY(num_iter <= max_it,
+                        "NumIterations (" << num_iter << ") Error!");
+
+            const double rt = tic_toc.RealTime();
+            const double rt_min = rt, rt_max = rt;
+            const int dofs = fespace.GetVSize();
+            const int cg_iter = cg.GetNumIterations();
+            const double mdofs_max = ((1e-6 * dofs) * cg_iter) / rt_max;
+            const double mdofs_min = ((1e-6 * dofs) * cg_iter) / rt_min;
+            if (myid == 0)
+            {
+               mfem::out << "Total CG time:    " << rt_max << " (" << rt_min << ") sec."
+                         << std::endl;
+               mfem::out << "Time per CG step: "
+                         << rt_max / cg_iter << " ("
+                         << rt_min / cg_iter << ") sec." << std::endl;
+               mfem::out << "\033[32m";
+               mfem::out << "\"DOFs/sec\" in CG: " << mdofs_max << " ("
+                         << mdofs_min << ") million.";
+               mfem::out << "\033[m" << std::endl;
+            }
         }
      }
      else
@@ -149,7 +149,7 @@ void InitialDeformation(const Vector &x, Vector &y);

 void InitialVelocity(const Vector &x, Vector &v);

-void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
+void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
               GridFunction *field, const char *field_name = NULL,
               bool init_vis = false);

@@ -376,10 +376,10 @@ int main(int argc, char *argv[])
 }


-void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
+void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
               GridFunction *field, const char *field_name, bool init_vis)
 {
-   if (!out)
+   if (!os)
   {
      return;
   }
@@ -389,24 +389,25 @@ void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,

   mesh->SwapNodes(nodes, owns_nodes);

-   out << "solution\n" << *mesh << *field;
+   os << "solution\n" << *mesh << *field;

   mesh->SwapNodes(nodes, owns_nodes);

   if (init_vis)
   {
-      out << "window_size 800 800\n";
-      out << "window_title '" << field_name << "'\n";
+      os << "window_size 800 800\n";
+      os << "window_title '" << field_name << "'\n";
      if (mesh->SpaceDimension() == 2)
      {
-         out << "view 0 0\n"; // view from top
-         out << "keys jl\n";  // turn off perspective and light
+         os << "view 0 0\n"; // view from top
+         os << "keys jl\n";  // turn off perspective and light
      }
-      out << "keys cm\n";         // show colorbar and mesh
-      out << "autoscale value\n"; // update value-range; keep mesh-extents fixed
-      out << "pause\n";
+      os << "keys cm\n";         // show colorbar and mesh
+      // update value-range; keep mesh-extents fixed
+      os << "autoscale value\n";
+      os << "pause\n";
   }
-   out << flush;
+   os << flush;
 }


@@ -154,7 +154,8 @@ void InitialDeformation(const Vector &x, Vector &y);

 void InitialVelocity(const Vector &x, Vector &v);

-void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
+void visualize(ostream &os, ParMesh *mesh,
+               ParGridFunction *deformed_nodes,
               ParGridFunction *field, const char *field_name = NULL,
               bool init_vis = false);

@@ -438,10 +439,11 @@ int main(int argc, char *argv[])
   return 0;
 }

-void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
+void visualize(ostream &os, ParMesh *mesh,
+               ParGridFunction *deformed_nodes,
               ParGridFunction *field, const char *field_name, bool init_vis)
 {
-   if (!out)
+   if (!os)
   {
      return;
   }
@@ -451,25 +453,27 @@ void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,

   mesh->SwapNodes(nodes, owns_nodes);

-   out << "parallel " << mesh->GetNRanks() << " " << mesh->GetMyRank() << "\n";
-   out << "solution\n" << *mesh << *field;
+   os << "parallel " << mesh->GetNRanks()
+      << " " << mesh->GetMyRank() << "\n";
+   os << "solution\n" << *mesh << *field;

   mesh->SwapNodes(nodes, owns_nodes);

   if (init_vis)
   {
-      out << "window_size 800 800\n";
-      out << "window_title '" << field_name << "'\n";
+      os << "window_size 800 800\n";
+      os << "window_title '" << field_name << "'\n";
      if (mesh->SpaceDimension() == 2)
      {
-         out << "view 0 0\n"; // view from top
-         out << "keys jl\n";  // turn off perspective and light
+         os << "view 0 0\n"; // view from top
+         os << "keys jl\n";  // turn off perspective and light
      }
-      out << "keys cm\n";         // show colorbar and mesh
-      out << "autoscale value\n"; // update value-range; keep mesh-extents fixed
-      out << "pause\n";
+      os << "keys cm\n";         // show colorbar and mesh
+      // update value-range; keep mesh-extents fixed
+      os << "autoscale value\n";
+      os << "pause\n";
   }
-   out << flush;
+   os << flush;
 }


@@ -32,7 +32,7 @@ private:
   mutable DenseTensor flux;
   mutable Vector z;

-   void GetFlux(const DenseMatrix &state, DenseTensor &flux) const;
+   void GetFlux(const DenseMatrix &state_, DenseTensor &flux_) const;

 public:
   FE_Evolution(FiniteElementSpace &vfes_,
@@ -256,26 +256,26 @@ inline double ComputeMaxCharSpeed(const Vector &state, const int dim)
 }

 // Compute the flux at solution nodes.
-void FE_Evolution::GetFlux(const DenseMatrix &x, DenseTensor &flux) const
+void FE_Evolution::GetFlux(const DenseMatrix &x_, DenseTensor &flux_) const
 {
-   const int dof = flux.SizeI();
-   const int dim = flux.SizeJ();
+   const int flux_dof = flux_.SizeI();
+   const int flux_dim = flux_.SizeJ();

-   for (int i = 0; i < dof; i++)
+   for (int i = 0; i < flux_dof; i++)
   {
-      for (int k = 0; k < num_equation; k++) { state(k) = x(i, k); }
-      ComputeFlux(state, dim, f);
+      for (int k = 0; k < num_equation; k++) { state(k) = x_(i, k); }
+      ComputeFlux(state, flux_dim, f);

-      for (int d = 0; d < dim; d++)
+      for (int d = 0; d < flux_dim; d++)
      {
         for (int k = 0; k < num_equation; k++)
         {
-            flux(i, d, k) = f(k, d);
+            flux_(i, d, k) = f(k, d);
         }
      }

      // Update max char speed
-      const double mcs = ComputeMaxCharSpeed(state, dim);
+      const double mcs = ComputeMaxCharSpeed(state, flux_dim);
      if (mcs > max_char_speed) { max_char_speed = mcs; }
   }
 }
@@ -171,7 +171,7 @@ public:
 };

 // Visualization driver
-void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
+void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
               GridFunction *field, const char *field_name = NULL,
               bool init_vis = false);

@@ -542,10 +542,10 @@ RubberOperator::~RubberOperator()


 // Inline visualization
-void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
+void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
               GridFunction *field, const char *field_name, bool init_vis)
 {
-   if (!out)
+   if (!os)
   {
      return;
   }
@@ -555,23 +555,25 @@ void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,

   mesh->SwapNodes(nodes, owns_nodes);

-   out << "solution\n" << *mesh << *field;
+   os << "solution\n" << *mesh << *field;

   mesh->SwapNodes(nodes, owns_nodes);

   if (init_vis)
   {
-      out << "window_size 800 800\n";
-      out << "window_title '" << field_name << "'\n";
+      os << "window_size 800 800\n";
+      os << "window_title '" << field_name << "'\n";
      if (mesh->SpaceDimension() == 2)
      {
-         out << "view 0 0\n"; // view from top
-         out << "keys jlA\n"; // turn off perspective and light, +anti-aliasing
+         os << "view 0 0\n"; // view from top
+         // turn off perspective and light, +anti-aliasing
+         os << "keys jlA\n";
      }
-      out << "keys cmA\n";        // show colorbar and mesh, +anti-aliasing
-      out << "autoscale value\n"; // update value-range; keep mesh-extents fixed
+      os << "keys cmA\n"; // show colorbar and mesh, +anti-aliasing
+      // update value-range; keep mesh-extents fixed
+      os << "autoscale value\n";
   }
-   out << flush;
+   os << flush;
 }

 void ReferenceConfiguration(const Vector &x, Vector &y)
@@ -185,7 +185,8 @@ public:
 };

 // Visualization driver
-void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
+void visualize(ostream &os, ParMesh *mesh,
+               ParGridFunction *deformed_nodes,
               ParGridFunction *field, const char *field_name = NULL,
               bool init_vis = false);

@@ -617,10 +618,11 @@ RubberOperator::~RubberOperator()


 // Inline visualization
-void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
+void visualize(ostream &os, ParMesh *mesh,
+               ParGridFunction *deformed_nodes,
               ParGridFunction *field, const char *field_name, bool init_vis)
 {
-   if (!out)
+   if (!os)
   {
      return;
   }
@@ -630,24 +632,27 @@ void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,

   mesh->SwapNodes(nodes, owns_nodes);

-   out << "parallel " << mesh->GetNRanks() << " " << mesh->GetMyRank() << "\n";
-   out << "solution\n" << *mesh << *field;
+   os << "parallel " << mesh->GetNRanks() << " " << mesh->GetMyRank() <<
+      "\n";
+   os << "solution\n" << *mesh << *field;

   mesh->SwapNodes(nodes, owns_nodes);

   if (init_vis)
   {
-      out << "window_size 800 800\n";
-      out << "window_title '" << field_name << "'\n";
+      os << "window_size 800 800\n";
+      os << "window_title '" << field_name << "'\n";
      if (mesh->SpaceDimension() == 2)
      {
-         out << "view 0 0\n"; // view from top
-         out << "keys jlA\n"; // turn off perspective and light, +anti-aliasing
+         os << "view 0 0\n"; // view from top
+         // turn off perspective and light, +anti-aliasing
+         os << "keys jlA\n";
      }
-      out << "keys cmA\n";        // show colorbar and mesh, +anti-aliasing
-      out << "autoscale value\n"; // update value-range; keep mesh-extents fixed
+      os << "keys cmA\n"; // show colorbar and mesh, +anti-aliasing
+      // update value-range; keep mesh-extents fixed
+      os << "autoscale value\n";
   }
-   out << flush;
+   os << flush;
 }

 void ReferenceConfiguration(const Vector &x, Vector &y)
@@ -128,7 +128,7 @@ int main(int argc, char *argv[])
   //    more than 10,000 elements.
   {
      int ref_levels =
-         (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
+         (int)floor(log(1000./mesh.GetNE())/log(2.)/dim);
      for (int l = 0; l < ref_levels; l++)
      {
         mesh.UniformRefinement();
@@ -197,6 +197,7 @@ int main(int argc, char *argv[])
   ParLinearForm b(&fespace);
   ConstantCoefficient one(1.0);
   b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   if (Device::IsEnabled()) { b.SetAssemblyLevel(LinearAssemblyLevel::FULL); }
   b.Assemble();

   // 10. Define the solution vector x as a parallel finite element grid
@@ -210,7 +211,8 @@ int main(int argc, char *argv[])
   //     Diffusion domain integrator.
   ParBilinearForm a(&fespace);
   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   //a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a.AddDomainIntegrator(new MassIntegrator(one));

   // 12. Assemble the parallel bilinear form and the corresponding linear
   //     system, applying any necessary transformations such as: parallel
@@ -245,13 +247,67 @@ int main(int argc, char *argv[])
   {
      prec = new HypreBoomerAMG;
   }
+
+   const int max_it = 50;
+   const int print_lvl = -1;
+   const double rtol = 1e-12;
+
   CGSolver cg(MPI_COMM_WORLD);
-   cg.SetRelTol(1e-12);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(1);
+   cg.SetRelTol(rtol);
+   //cg.SetMaxIter(max_it);
+   //cg.SetPrintLevel(print_lvl);
   if (prec) { cg.SetPreconditioner(*prec); }
   cg.SetOperator(*A);
-   cg.Mult(B, X);
+
+   // Warm-up CG solve (in case of JIT to avoid timing it)
+   {
+      Vector Y(X);
+      cg.SetMaxIter(2);
+      cg.SetPrintLevel(-1);
+      cg.Mult(B, Y);
+      MFEM_DEVICE_SYNC;
+   }
+
+   // benchmark this problem
+   {
+      tic_toc.Clear();
+      cg.SetMaxIter(max_it);
+      cg.SetPrintLevel(print_lvl);
+      {
+         tic_toc.Start();
+         cg.Mult(B, X);
+         MFEM_DEVICE_SYNC;
+         tic_toc.Stop();
+      }
+   }
+   // Final norm check
+   const double final_norm = cg.GetFinalNorm();
+   MFEM_VERIFY(final_norm < sqrt(rtol),
+               "FinalNorm (" << final_norm << ") Error!");
+   // Number of iteration check
+   const int num_iter = cg.GetNumIterations();
+   MFEM_VERIFY(num_iter <= max_it,
+               "NumIterations (" << num_iter << ") Error!");
+
+   const double rt = tic_toc.RealTime();
+   const double rt_min = rt, rt_max = rt;
+   HYPRE_BigInt dofs = fespace.GlobalTrueVSize();
+   const int cg_iter = cg.GetNumIterations();
+   const double mdofs_max = ((1e-6 * dofs) * cg_iter) / rt_max;
+   const double mdofs_min = ((1e-6 * dofs) * cg_iter) / rt_min;
+   if (myid == 0)
+   {
+      mfem::out << "Total CG time:    " << rt_max << " (" << rt_min << ") sec."
+                << std::endl;
+      mfem::out << "Time per CG step: "
+                << rt_max / cg_iter << " ("
+                << rt_min / cg_iter << ") sec." << std::endl;
+      mfem::out << "\033[32m";
+      mfem::out << "\"DOFs/sec\" in CG: " << mdofs_max << " ("
+                << mdofs_min << ") million.";
+      mfem::out << "\033[m" << std::endl;
+   }
+   //cg.Mult(B, X);
   delete prec;

   // 14. Recover the parallel grid function corresponding to X. This is the
@@ -75,7 +75,7 @@ Mesh * GenerateSerialMesh(int ref);
 // alpha*n.Grad(sol) + beta*sol - gamma over the same boundary.
 double IntegrateBC(const GridFunction &sol, const Array<int> &bdr_marker,
                   double alpha, double beta, double gamma,
-                   double &err);
+                   double &error);

 int main(int argc, char *argv[])
 {
@@ -302,26 +302,26 @@ int main(int argc, char *argv[])
   {
      // Integrate the solution on the Dirichlet boundary and compare to the
      // expected value.
-      double err, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, err);
+      double error, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, error);

      bool hom_dbc = (dbc_val == 0.0);
-      err /=  hom_dbc ? 1.0 : fabs(dbc_val);
+      error /=  hom_dbc ? 1.0 : fabs(dbc_val);
      mfem::out << "Average of solution on Gamma_dbc:\t"
                << avg << ", \t"
                << (hom_dbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
   }
   {
      // Integrate n.Grad(u) on the inhomogeneous Neumann boundary and compare
      // to the expected value.
-      double err, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, err);
+      double error, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, error);

      bool hom_nbc = (nbc_val == 0.0);
-      err /=  hom_nbc ? 1.0 : fabs(nbc_val);
+      error /=  hom_nbc ? 1.0 : fabs(nbc_val);
      mfem::out << "Average of n.Grad(u) on Gamma_nbc:\t"
                << avg << ", \t"
                << (hom_nbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
   }
   {
      // Integrate n.Grad(u) on the homogeneous Neumann boundary and compare to
@@ -330,25 +330,26 @@ int main(int argc, char *argv[])
      nbc0_bdr = 0;
      nbc0_bdr[3] = 1;

-      double err, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, err);
+      double error, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, error);

      bool hom_nbc = true;
      mfem::out << "Average of n.Grad(u) on Gamma_nbc0:\t"
                << avg << ", \t"
                << (hom_nbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
   }
   {
      // Integrate n.Grad(u) + a * u on the Robin boundary and compare to the
      // expected value.
-      double err, avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val, err);
+      double error;
+      double avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val, error);

      bool hom_rbc = (rbc_b_val == 0.0);
-      err /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
+      error /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
      mfem::out << "Average of n.Grad(u)+a*u on Gamma_rbc:\t"
                << avg << ", \t"
                << (hom_rbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
   }

   // 14. Save the refined mesh and the solution. This output can be viewed
@@ -637,11 +638,11 @@ Mesh * GenerateSerialMesh(int ref)

 double IntegrateBC(const GridFunction &x, const Array<int> &bdr,
                   double alpha, double beta, double gamma,
-                   double &err)
+                   double &error)
 {
   double nrm = 0.0;
   double avg = 0.0;
-   err = 0.0;
+   error = 0.0;

   const bool a_is_zero = alpha == 0.0;
   const bool b_is_zero = beta == 0.0;
@@ -705,20 +706,20 @@ double IntegrateBC(const GridFunction &x, const Array<int> &bdr,

         // Integrate |alpha * n.Grad(x) + beta * x - gamma|^2
         val -= gamma;
-         err += (val*val) * ip.weight * face_weight;
+         error += (val*val) * ip.weight * face_weight;
      }
   }

   // Normalize by the length of the boundary
   if (std::abs(nrm) > 0.0)
   {
-      err /= nrm;
+      error /= nrm;
      avg /= nrm;
   }

   // Compute l2 norm of the error in the boundary condition (negative
-   // quadrature weights may produce negative 'err')
-   err = (err >= 0.0) ? sqrt(err) : -sqrt(-err);
+   // quadrature weights may produce negative 'error')
+   error = (error >= 0.0) ? sqrt(error) : -sqrt(-error);

   // Return the average value of alpha * n.Grad(x) + beta * x
   return avg;
@@ -75,7 +75,7 @@ Mesh * GenerateSerialMesh(int ref);
 // alpha*n.Grad(sol) + beta*sol - gamma over the same boundary.
 double IntegrateBC(const ParGridFunction &sol, const Array<int> &bdr_marker,
                   double alpha, double beta, double gamma,
-                   double &err);
+                   double &error);

 int main(int argc, char *argv[])
 {
@@ -321,26 +321,26 @@ int main(int argc, char *argv[])
   {
      // Integrate the solution on the Dirichlet boundary and compare to the
      // expected value.
-      double err, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, err);
+      double error, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, error);

      bool hom_dbc = (dbc_val == 0.0);
-      err /=  hom_dbc ? 1.0 : fabs(dbc_val);
+      error /=  hom_dbc ? 1.0 : fabs(dbc_val);
      mfem::out << "Average of solution on Gamma_dbc:\t"
                << avg << ", \t"
                << (hom_dbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
   }
   {
      // Integrate n.Grad(u) on the inhomogeneous Neumann boundary and compare
      // to the expected value.
-      double err, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, err);
+      double error, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, error);

      bool hom_nbc = (nbc_val == 0.0);
-      err /=  hom_nbc ? 1.0 : fabs(nbc_val);
+      error /=  hom_nbc ? 1.0 : fabs(nbc_val);
      mfem::out << "Average of n.Grad(u) on Gamma_nbc:\t"
                << avg << ", \t"
                << (hom_nbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
   }
   {
      // Integrate n.Grad(u) on the homogeneous Neumann boundary and compare to
@@ -349,25 +349,26 @@ int main(int argc, char *argv[])
      nbc0_bdr = 0;
      nbc0_bdr[3] = 1;

-      double err, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, err);
+      double error, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, error);

      bool hom_nbc = true;
      mfem::out << "Average of n.Grad(u) on Gamma_nbc0:\t"
                << avg << ", \t"
                << (hom_nbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
   }
   {
      // Integrate n.Grad(u) + a * u on the Robin boundary and compare to the
      // expected value.
-      double err, avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val, err);
+      double error, avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val,
+                                      error);

      bool hom_rbc = (rbc_b_val == 0.0);
-      err /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
+      error /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
      mfem::out << "Average of n.Grad(u)+a*u on Gamma_rbc:\t"
                << avg << ", \t"
                << (hom_rbc ? "absolute" : "relative")
-                << " error " << err << endl;
+                << " error " << error << endl;
   }

   // 15. Save the refined mesh and the solution in parallel. This output can be
@@ -667,11 +668,11 @@ double IntegrateBC(const ParGridFunction &x, const Array<int> &bdr,
   double loc_vals[3];
   double &nrm = loc_vals[0];
   double &avg = loc_vals[1];
-   double &err = loc_vals[2];
+   double &error = loc_vals[2];

   nrm = 0.0;
   avg = 0.0;
-   err = 0.0;
+   error = 0.0;

   const bool a_is_zero = alpha == 0.0;
   const bool b_is_zero = beta == 0.0;
@@ -735,7 +736,7 @@ double IntegrateBC(const ParGridFunction &x, const Array<int> &bdr,

         // Integrate |alpha * n.Grad(x) + beta * x - gamma|^2
         val -= gamma;
-         err += (val*val) * ip.weight * face_weight;
+         error += (val*val) * ip.weight * face_weight;
      }
   }

@@ -754,7 +755,7 @@ double IntegrateBC(const ParGridFunction &x, const Array<int> &bdr,
   }

   // Compute l2 norm of the error in the boundary condition (negative
-   // quadrature weights may produce negative 'err')
+   // quadrature weights may produce negative 'error')
   glb_err = (glb_err >= 0.0) ? sqrt(glb_err) : -sqrt(-glb_err);

   // Return the average value of alpha * n.Grad(x) + beta * x
@@ -167,9 +167,9 @@ int main(int argc, char *argv[])

   // 13. Compute error in the solution and its flux
   FunctionCoefficient uCoef(uExact);
-   double err = x.ComputeL2Error(uCoef);
+   double error = x.ComputeL2Error(uCoef);

-   cout << "|u - u_h|_2 = " << err << endl;
+   cout << "|u - u_h|_2 = " << error << endl;

   FiniteElementSpace flux_fespace(mesh, &fec, 3);
   GridFunction flux(&flux_fespace);
@@ -197,9 +197,9 @@ int main(int argc, char *argv[])

   // 15. Compute error in the solution and its flux
   FunctionCoefficient uCoef(uExact);
-   double err = x.ComputeL2Error(uCoef);
+   double error = x.ComputeL2Error(uCoef);

-   if (myid == 0) { cout << "|u - u_h|_2 = " << err << endl; }
+   if (myid == 0) { cout << "|u - u_h|_2 = " << error << endl; }

   ParFiniteElementSpace flux_fespace(&pmesh, &fec, 3);
   ParGridFunction flux(&flux_fespace);
@@ -255,10 +255,10 @@ int main(int argc, char *argv[])

   // 15. Compute and print the L^2 norm of the error.
   {
-      double err = x.ComputeL2Error(E);
+      double error = x.ComputeL2Error(E);
      if (myid == 0)
      {
-         cout << "\n|| E_h - E ||_{L^2} = " << err << '\n' << endl;
+         cout << "\n|| E_h - E ||_{L^2} = " << error << '\n' << endl;
      }
   }

@@ -256,10 +256,10 @@ int main(int argc, char *argv[])

   // 15. Compute and print the L^2 norm of the error.
   {
-      double err = x.ComputeL2Error(F);
+      double error = x.ComputeL2Error(F);
      if (myid == 0)
      {
-         cout << "\n|| F_h - F ||_{L^2} = " << err << '\n' << endl;
+         cout << "\n|| F_h - F ||_{L^2} = " << error << '\n' << endl;
      }
   }

@@ -125,6 +125,7 @@ int main(int argc, char *argv[])
   BilinearFormIntegrator *integ = new DiffusionIntegrator(one);
   a.AddDomainIntegrator(integ);
   b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   if (Device::IsEnabled()) { b.SetAssemblyLevel(LinearAssemblyLevel::FULL); }

   // 7. The solution vector x and the associated finite element grid function
   //    will be maintained over the AMR iterations. We initialize it to zero.
@@ -202,6 +202,7 @@ int main(int argc, char *argv[])
   BilinearFormIntegrator *integ = new DiffusionIntegrator(one);
   a.AddDomainIntegrator(integ);
   b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   if (Device::IsEnabled()) { b.SetAssemblyLevel(LinearAssemblyLevel::FULL); }

   // 12. The solution vector x and the associated finite element grid function
   //     will be maintained over the AMR iterations. We initialize it to zero.
@@ -282,10 +282,10 @@ int main(int argc, char *argv[])
   delete b;

   // 12. Compute and print the L^2 norm of the error.
-   double err = x.ComputeL2Error(sol_coef);
+   double error = x.ComputeL2Error(sol_coef);
   if (myid == 0)
   {
-      cout << "\nL2 norm of error: " << err << endl;
+      cout << "\nL2 norm of error: " << error << endl;
   }

   // 13. Save the refined mesh and the solution. This output can be viewed
@@ -31,29 +31,31 @@ add_mfem_examples(GINKGO_EXAMPLES_SRCS ${PFX} "" test_ginkgo)
 # which builds the examples and runs:
 #   ctest -R ginkgo

-# Command line options for the tests.
-set(EX1_COMMON_OPTS ex1 -m ../data/star.mesh --use_gko_solver)
-set(EX1_TEST_OPTS ${EX9_COMMON_OPTS})
+if (MFEM_ENABLE_TESTING)
+  # Command line options for the tests.
+  set(EX1_COMMON_OPTS ex1 -m ../data/star.mesh --use_gko_solver)
+  set(EX1_TEST_OPTS ${EX9_COMMON_OPTS})

-# Add the tests: one test per source file.
-foreach(SRC_FILE ${GINKGO_EXAMPLES_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-  set(TEST_NAME ${PFX}${TEST_NAME})
+  # Add the tests: one test per source file.
+  foreach(SRC_FILE ${GINKGO_EXAMPLES_SRCS})
+    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+    string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+    set(TEST_NAME ${PFX}${TEST_NAME})

-  set(THIS_TEST_OPTIONS "-no-vis")
-  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+    set(THIS_TEST_OPTIONS "-no-vis")
+    list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+    # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")

-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=4
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
+    if (NOT (${TEST_NAME} MATCHES ".*p$"))
+      add_test(NAME ${TEST_NAME}_ser
+        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+    else()
+      add_test(NAME ${TEST_NAME}_np=4
+        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+        ${MPIEXEC_PREFLAGS}
+        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+        ${MPIEXEC_POSTFLAGS})
+    endif()
+  endforeach()
+endif()
@@ -33,31 +33,33 @@ add_mfem_examples(HIOP_EXAMPLES_SRCS ${PFX} "" test_hiop)
 # which builds the examples and runs:
 #   ctest -R hiop

-# Command line options for the tests.
-# Example 9:
-set(EX9_COMMON_OPTS -m ../../data/periodic-segment.mesh -p 0 -dt 0.005)
-set(EX9_TEST_OPTS ${EX9_COMMON_OPTS}  -r  2 )
-set(EX9P_TEST_OPTS ${EX9_COMMON_OPTS})
+if (MFEM_ENABLE_TESTING)
+  # Command line options for the tests.
+  # Example 9:
+  set(EX9_COMMON_OPTS -m ../../data/periodic-segment.mesh -p 0 -dt 0.005)
+  set(EX9_TEST_OPTS ${EX9_COMMON_OPTS}  -r  2 )
+  set(EX9P_TEST_OPTS ${EX9_COMMON_OPTS})

-# Add the tests: one test per source file.
-foreach(SRC_FILE ${HIOP_EXAMPLES_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-  set(TEST_NAME ${PFX}${TEST_NAME})
+  # Add the tests: one test per source file.
+  foreach(SRC_FILE ${HIOP_EXAMPLES_SRCS})
+    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+    string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+    set(TEST_NAME ${PFX}${TEST_NAME})

-  set(THIS_TEST_OPTIONS "-no-vis")
-  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+    set(THIS_TEST_OPTIONS "-no-vis")
+    list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+    # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")

-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=4
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
+    if (NOT (${TEST_NAME} MATCHES ".*p$"))
+      add_test(NAME ${TEST_NAME}_ser
+        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+    else()
+      add_test(NAME ${TEST_NAME}_np=4
+        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4
+        ${MPIEXEC_PREFLAGS}
+        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+        ${MPIEXEC_POSTFLAGS})
+    endif()
+  endforeach()
+endif()
@@ -94,30 +94,32 @@ if (MFEM_USE_SLEPC)
 endif()

 # Add the tests: one test per command-line-variable.
-set(TEST_OPTIONS_VARS
-    EX1_ARGS_W EX1_ARGS_P EX2_ARGS EX3_ARGS EX4_ARGS EX4_HYB_ARGS
-    EX5_BDDC_LB_ARGS EX5_BDDC_GB_ARGS EX5_FSPL_ARGS EX6_ARGS EX6_NONOVL_ARGS
-    EX9_E_ARGS EX9_ES_ARGS EX9_IS_ARGS EX10_ARGS)
-if (MFEM_USE_SLEPC)
-    list(APPEND TEST_OPTIONS_VARS EX11_ARGS_SINV EX11_ARGS_LOBPCG EX11_ARGS_GD)
-endif()
-
-foreach(TEST_OPTIONS_VAR ${TEST_OPTIONS_VARS})
-  string(REGEX REPLACE "^(.+)_ARGS" "\\1" TEST_NAME_UC ${TEST_OPTIONS_VAR})
-  string(REGEX REPLACE "^([^_]+)" "\\1P" TEST_NAME_UC ${TEST_NAME_UC})
-  string(TOLOWER ${TEST_NAME_UC} TEST_NAME_FULL)
-  string(REGEX REPLACE "^([^_]+).*" "\\1" TEST_NAME ${TEST_NAME_FULL})
-  set(TEST_NAME_FULL ${PFX}${TEST_NAME_FULL})
-  set(TEST_NAME ${PFX}${TEST_NAME})
-  set(TEST_OPTIONS "-no-vis" ${${TEST_OPTIONS_VAR}})
-  # message(STATUS "${TEST_NAME_FULL} --> ${TEST_NAME} ${TEST_OPTIONS}")
-
-  # All PETSC tests are parallel.
-  if (MFEM_USE_MPI)
-    add_test(NAME ${TEST_NAME_FULL}_np=4
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
+if (MFEM_ENABLE_TESTING)
+  set(TEST_OPTIONS_VARS
+      EX1_ARGS_W EX1_ARGS_P EX2_ARGS EX3_ARGS EX4_ARGS EX4_HYB_ARGS
+      EX5_BDDC_LB_ARGS EX5_BDDC_GB_ARGS EX5_FSPL_ARGS EX6_ARGS EX6_NONOVL_ARGS
+      EX9_E_ARGS EX9_ES_ARGS EX9_IS_ARGS EX10_ARGS)
+  if (MFEM_USE_SLEPC)
+      list(APPEND TEST_OPTIONS_VARS EX11_ARGS_SINV EX11_ARGS_LOBPCG EX11_ARGS_GD)
  endif()
-endforeach()
+
+  foreach(TEST_OPTIONS_VAR ${TEST_OPTIONS_VARS})
+    string(REGEX REPLACE "^(.+)_ARGS" "\\1" TEST_NAME_UC ${TEST_OPTIONS_VAR})
+    string(REGEX REPLACE "^([^_]+)" "\\1P" TEST_NAME_UC ${TEST_NAME_UC})
+    string(TOLOWER ${TEST_NAME_UC} TEST_NAME_FULL)
+    string(REGEX REPLACE "^([^_]+).*" "\\1" TEST_NAME ${TEST_NAME_FULL})
+    set(TEST_NAME_FULL ${PFX}${TEST_NAME_FULL})
+    set(TEST_NAME ${PFX}${TEST_NAME})
+    set(TEST_OPTIONS "-no-vis" ${${TEST_OPTIONS_VAR}})
+    # message(STATUS "${TEST_NAME_FULL} --> ${TEST_NAME} ${TEST_OPTIONS}")
+
+    # All PETSC tests are parallel.
+    if (MFEM_USE_MPI)
+      add_test(NAME ${TEST_NAME_FULL}_np=4
+        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+        ${MPIEXEC_PREFLAGS}
+        $<TARGET_FILE:${TEST_NAME}> ${TEST_OPTIONS}
+        ${MPIEXEC_POSTFLAGS})
+    endif()
+  endforeach()
+endif()
@@ -37,37 +37,39 @@ add_mfem_examples(PUMI_EXAMPLES_SRCS ${PFX} "" test_pumi)
 # which builds the examples and runs:
 #   ctest -R pumi

-# Command line options for the tests.
-# TODO...
+if (MFEM_ENABLE_TESTING)
+  # Command line options for the tests.
+  # TODO...

-# Set the number of processors for the parallel examples. The value of
-# MFEM_MPI_NP is ignored.
-set(EX1_TEST_NP 1)
-set(EX1P_TEST_NP 8)
-set(EX2_TEST_NP 1)
-set(EX6P_TEST_NP 8)
+  # Set the number of processors for the parallel examples. The value of
+  # MFEM_MPI_NP is ignored.
+  set(EX1_TEST_NP 1)
+  set(EX1P_TEST_NP 8)
+  set(EX2_TEST_NP 1)
+  set(EX6P_TEST_NP 8)

-# Add the tests: one test per source file.
-foreach(SRC_FILE ${PUMI_EXAMPLES_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-  set(TEST_NAME ${PFX}${TEST_NAME})
+  # Add the tests: one test per source file.
+  foreach(SRC_FILE ${PUMI_EXAMPLES_SRCS})
+    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+    string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+    set(TEST_NAME ${PFX}${TEST_NAME})

-  set(THIS_TEST_OPTIONS "-no-vis")
-  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+    set(THIS_TEST_OPTIONS "-no-vis")
+    list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+    # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")

-  # All PUMI examples require MPI
-  if (FALSE)
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    set(TEST_NP ${${UP_TEST_NAME}_TEST_NP})
-    add_test(NAME ${TEST_NAME}_np=${TEST_NP}
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${TEST_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
+    # All PUMI examples require MPI
+    if (FALSE)
+      add_test(NAME ${TEST_NAME}_ser
+        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+    else()
+      set(TEST_NP ${${UP_TEST_NAME}_TEST_NP})
+      add_test(NAME ${TEST_NAME}_np=${TEST_NP}
+        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${TEST_NP}
+        ${MPIEXEC_PREFLAGS}
+        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+        ${MPIEXEC_POSTFLAGS})
+    endif()
+  endforeach()
+endif()
@@ -41,36 +41,38 @@ add_mfem_examples(SUNDIALS_EXAMPLES_SRCS ${PFX} "" test_sundials)
 # which builds the examples and runs:
 #   ctest -R sundials

-# Command line options for the tests.
-# Example 9: test CVODE with CV_ADAMS (non-stiff implicit) time stepping
-set(EX9_COMMON_OPTS -m ../../data/periodic-hexagon.mesh -p 0 -s 7)
-set(EX9_TEST_OPTS ${EX9_COMMON_OPTS}  -r  2 -dt 0.0018 -vs 25)
-set(EX9P_TEST_OPTS ${EX9_COMMON_OPTS} -rp 1 -dt 0.0009 -vs 50)
-# Example 10: test CVODE with CV_BDF (stiff implicit) time stepping
-set(EX10_COMMON_OPTS -m ../../data/beam-quad.mesh -o 2 -s 5 -dt 0.15 -tf 6 -vs 10)
-set(EX10_TEST_OPTS ${EX10_COMMON_OPTS}  -r  2)
-set(EX10P_TEST_OPTS ${EX10_COMMON_OPTS} -rp 1)
-# Example 16: use the default options
+if (MFEM_ENABLE_TESTING)
+  # Command line options for the tests.
+  # Example 9: test CVODE with CV_ADAMS (non-stiff implicit) time stepping
+  set(EX9_COMMON_OPTS -m ../../data/periodic-hexagon.mesh -p 0 -s 7)
+  set(EX9_TEST_OPTS ${EX9_COMMON_OPTS}  -r  2 -dt 0.0018 -vs 25)
+  set(EX9P_TEST_OPTS ${EX9_COMMON_OPTS} -rp 1 -dt 0.0009 -vs 50)
+  # Example 10: test CVODE with CV_BDF (stiff implicit) time stepping
+  set(EX10_COMMON_OPTS -m ../../data/beam-quad.mesh -o 2 -s 5 -dt 0.15 -tf 6 -vs 10)
+  set(EX10_TEST_OPTS ${EX10_COMMON_OPTS}  -r  2)
+  set(EX10P_TEST_OPTS ${EX10_COMMON_OPTS} -rp 1)
+  # Example 16: use the default options

-# Add the tests: one test per source file.
-foreach(SRC_FILE ${SUNDIALS_EXAMPLES_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-  set(TEST_NAME ${PFX}${TEST_NAME})
+  # Add the tests: one test per source file.
+  foreach(SRC_FILE ${SUNDIALS_EXAMPLES_SRCS})
+    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+    string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+    set(TEST_NAME ${PFX}${TEST_NAME})

-  set(THIS_TEST_OPTIONS "-no-vis")
-  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+    set(THIS_TEST_OPTIONS "-no-vis")
+    list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+    # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")

-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=4
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
+    if (NOT (${TEST_NAME} MATCHES ".*p$"))
+      add_test(NAME ${TEST_NAME}_ser
+        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+    else()
+      add_test(NAME ${TEST_NAME}_np=4
+        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+        ${MPIEXEC_PREFLAGS}
+        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+        ${MPIEXEC_POSTFLAGS})
+    endif()
+  endforeach()
+endif()
@@ -32,31 +32,32 @@ add_mfem_examples(SUPERLU_EXAMPLES_SRCS ${PFX} "" test_superlu)
 # The SuperLU tests can be run separately using the target "test_superlu"
 # which builds the examples and runs:
 #   ctest -R superlu
+if (MFEM_ENABLE_TESTING)
+  # Command line options for the tests.
+  # Example 1: Test SuperLU on the simple Poisson problem
+  set(EX1_COMMON_OPTS -m ../../data/star.mesh -p 2)
+  set(EX1P_TEST_OPTS ${EX1_COMMON_OPTS})

-# Command line options for the tests.
-# Example 1: Test SuperLU on the simple Poisson problem
-set(EX1_COMMON_OPTS -m ../../data/star.mesh -p 2)
-set(EX1P_TEST_OPTS ${EX1_COMMON_OPTS})
+  # Add the tests: one test per source file.
+  foreach(SRC_FILE ${SUPERLU_EXAMPLES_SRCS})
+    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+    string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+    set(TEST_NAME ${PFX}${TEST_NAME})

-# Add the tests: one test per source file.
-foreach(SRC_FILE ${SUPERLU_EXAMPLES_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-  set(TEST_NAME ${PFX}${TEST_NAME})
+    set(THIS_TEST_OPTIONS "-no-vis")
+    list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+    # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")

-  set(THIS_TEST_OPTIONS "-no-vis")
-  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
-
-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=4
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
+    if (NOT (${TEST_NAME} MATCHES ".*p$"))
+      add_test(NAME ${TEST_NAME}_ser
+        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+    else()
+      add_test(NAME ${TEST_NAME}_np=4
+        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+        ${MPIEXEC_PREFLAGS}
+        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+        ${MPIEXEC_POSTFLAGS})
+    endif()
+  endforeach()
+endif()
@@ -21,6 +21,8 @@ set(SRCS
  bilininteg_dgtrace_ea.cpp
  bilininteg_diffusion_mf.cpp
  bilininteg_diffusion_pa.cpp
+  bilininteg_diffusion_pa_diag_fast.cpp
+  bilininteg_diffusion_pa_fast.cpp
  bilininteg_diffusion_ea.cpp
  bilininteg_divergence.cpp
  bilininteg_hcurl.cpp
@@ -29,6 +31,11 @@ set(SRCS
  bilininteg_gradient.cpp
  bilininteg_mass_mf.cpp
  bilininteg_mass_pa.cpp
+  bilininteg_mass_pa_diag.cpp
+  bilininteg_mass_pa_fast.cpp
+  bilininteg_mass_pa_fast_amd.cpp
+  bilininteg_mass_pa_fast_hip.cpp
+  bilininteg_mass_pa_fast_diag.cpp
  bilininteg_mass_ea.cpp
  bilininteg_transpose_ea.cpp
  bilininteg_vecdiffusion.cpp
@@ -68,7 +75,16 @@ set(SRCS
  ceed/operator.cpp
  ceed/util.cpp
  linearform.cpp
+  linearform_ext.cpp
  lininteg.cpp
+  lininteg_domain.cpp
+  lininteg_domain_grad.cpp
+  lininteg_vector_domain.cpp
+  lininteg_vector_domain_grad.cpp
+  lor.cpp
+  lor_assembly.cpp
+  lor_assembly_ker.cpp
+  lor_assembly_ni3.cpp
  multigrid.cpp
  nonlinearform.cpp
  nonlinearform_ext.cpp
@@ -161,7 +177,12 @@ set(HDRS
  ceed/operator.hpp
  ceed/util.hpp
  linearform.hpp
+  linearform_ext.hpp
  lininteg.hpp
+  lininteg_domain.hpp
+  lininteg_domain_grad.hpp
+  lor.hpp
+  lor_assembly.hpp
  multigrid.hpp
  nonlinearform.hpp
  nonlinearform_ext.hpp
@@ -318,14 +318,31 @@ void PABilinearFormExtension::Assemble()
 void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
 {
   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
+   const int integrators_size = integrators.Size();

-   const int iSz = integrators.Size();
-   if (elem_restrict && !DeviceCanUseCeed())
+   // Scan the different action types we need to launch
+   bool E2E_actions = false,
+        L2L_actions = false;
+   for (int i = 0; i < integrators_size; ++i)
+   {
+      const ActionType action_type = integrators[i]->GetActionType();
+      E2E_actions |= action_type == ActionType::E2E;
+      L2L_actions |= action_type == ActionType::L2L;
+   }
+
+   // typically this is a large vector, so store on device
+   y.UseDevice(true);
+
+   // If E2E kernels are present, do their computation on local vectors
+   if (E2E_actions && elem_restrict)
   {
      localY = 0.0;
-      for (int i = 0; i < iSz; ++i)
+      for (int i = 0; i < integrators_size; ++i)
      {
-         integrators[i]->AssembleDiagonalPA(localY);
+         if (integrators[i]->GetActionType() == ActionType::E2E)
+         {
+            integrators[i]->AssembleDiagonalPA(localY);
+         }
      }
      const ElementRestriction* H1elem_restrict =
         dynamic_cast<const ElementRestriction*>(elem_restrict);
@@ -340,11 +357,20 @@ void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
   }
   else
   {
-      y.UseDevice(true); // typically this is a large vector, so store on device
+      // otherwise, initialize the y output
      y = 0.0;
-      for (int i = 0; i < iSz; ++i)
+   }
+
+   // Continue with the computation of the L2L integrators
+   if (L2L_actions || !elem_restrict)
+   {
+      for (int i = 0; i < integrators_size; ++i)
      {
-         integrators[i]->AssembleDiagonalPA(y);
+         const ActionType action = integrators[i]->GetActionType();
+         if (action == ActionType::L2L || !elem_restrict)
+         {
+            integrators[i]->AssembleDiagonalPA(y);
+         }
      }
   }
 }
@@ -383,27 +409,52 @@ void PABilinearFormExtension::FormLinearSystem(const Array<int> &ess_tdof_list,
 void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
 {
   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
+   const int integrators_size = integrators.Size();

-   const int iSz = integrators.Size();
-   if (DeviceCanUseCeed() || !elem_restrict)
+   // Scan the different action types we need to launch
+   bool E2E_actions = false,
+        L2L_actions = false;
+   for (int i = 0; i < integrators_size; ++i)
   {
-      y.UseDevice(true); // typically this is a large vector, so store on device
-      y = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultPA(x, y);
-      }
+      const ActionType action_type = integrators[i]->GetActionType();
+      E2E_actions |= action_type == ActionType::E2E;
+      L2L_actions |= action_type == ActionType::L2L;
   }
-   else
+
+   // typically this is a large vector, so store on device
+   y.UseDevice(true);
+
+   // If E2E kernels are present, do their computation on local vectors
+   if (E2E_actions && elem_restrict)
   {
      elem_restrict->Mult(x, localX);
      localY = 0.0;
-      for (int i = 0; i < iSz; ++i)
+      for (int i = 0; i < integrators_size; ++i)
      {
-         integrators[i]->AddMultPA(localX, localY);
+         if (integrators[i]->GetActionType() == ActionType::E2E)
+         {
+            integrators[i]->AddMultPA(localX, localY);
+         }
      }
      elem_restrict->MultTranspose(localY, y);
   }
+   else
+   {
+      // otherwise, initialize the y output
+      y = 0.0;
+   }
+
+   // Continue with the computation of the L2L integrators
+   if (L2L_actions || !elem_restrict)
+   {
+      for (int i = 0; i < integrators_size; ++i)
+      {
+         if (integrators[i]->GetActionType() == ActionType::L2L || !elem_restrict)
+         {
+            integrators[i]->AddMultPA(x, y);
+         }
+      }
+   }

   Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
   const int iFISz = intFaceIntegrators.Size();
@@ -244,6 +244,8 @@ public:
                                    Vector &flux, Vector *d_energy = NULL)
   { return 0.0; }

+   virtual bool SupportsBatchedLOR() const { return false; }
+
   virtual ~BilinearFormIntegrator() { }
 };

@@ -1977,7 +1979,7 @@ private:
   const DofToQuad *maps;         ///< Not owned
   const GeometricFactors *geom;  ///< Not owned
   int dim, ne, dofs1D, quad1D;
-   Vector pa_data;
+   Vector pa_data, CoG;
   bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient

 public:
@@ -2058,6 +2060,8 @@ public:
                                         const FiniteElement &test_fe);

   bool SupportsCeed() const { return DeviceCanUseCeed(); }
+
+   bool SupportsBatchedLOR() const { return true; }
 };

 /** Class for local mass matrix assembling a(u,v) := (Q u, v) */
@@ -768,8 +768,8 @@ void SmemPAConvectionApply3D(const int ne,

 void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
-   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
-                         Device::GetDeviceMemoryType() : pa_mt;
+   const MemoryType mt = (memory_type == MemoryType::DEFAULT) ?
+                         Device::GetDeviceMemoryType() : memory_type;
   // Assumes tensor-product elements
   Mesh *mesh = fes.GetMesh();
   const FiniteElement &el = *fes.GetFE(0);
@@ -9,18 +9,45 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.

+#define MFEM_DEBUG_COLOR 87
+#include "../general/debug.hpp"
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
 #include "ceed/diffusion.hpp"
+#include <string>
+
+#define MFEM_NVTX_COLOR Olive
+#include "../general/nvtx.hpp"

 using namespace std;

 namespace mfem
 {

+void NDK_PADiffusionAssembleDiagonal(const int dim,
+                                     const int D1D,
+                                     const int Q1D,
+                                     const int NE,
+                                     const bool symm,
+                                     const FiniteElementSpace *fes,
+                                     const DofToQuad *maps,
+                                     const Vector &D,
+                                     Vector &Y);
+
 // PA Diffusion Integrator

+void NDK_PADiffusionApply(const int dim,
+                          const int D1D,
+                          const int Q1D,
+                          const int NE,
+                          const Vector &CoG,
+                          const FiniteElementSpace *fes,
+                          const DofToQuad *maps,
+                          const Vector &D,
+                          const Vector &X,
+                          Vector &Y);
+
 // OCCA 2D Assemble kernel
 #ifdef MFEM_USE_OCCA
 static void OccaPADiffusionSetup2D(const int D1D,
@@ -349,10 +376,115 @@ static void PADiffusionSetup(const int dim,
   }
 }

+////////////////////////////////////////////////////////////////////////////////
+template<typename T> MFEM_HOST_DEVICE inline
+void HouseholderReflect(T *A, const T *v,
+                        const T b, const int m, const int n,
+                        const int row, const int col)
+{
+   for (int j = 0; j < n; j++)
+   {
+      T w = A[0*row + j*col];
+      for (int i = 1; i < m; i++) { w += v[i] * A[i*row + j*col]; }
+      A[0*row + j*col] -= b * w;
+      for (int i = 1; i < m; i++) { A[i*row + j*col] -= b * w * v[i]; }
+   }
+}
+
+template<int Q1D, typename T> MFEM_HOST_DEVICE inline
+void HouseholderApplyQ(T *A, const T *Q, const T *tau,
+                       const int k, const int row, const int col)
+{
+   T v[Q1D];
+   for (int ii=0; ii<k; ii++)
+   {
+      const int i = k-1-ii;
+      for (int j = i+1; j < Q1D; j++) { v[j] = Q[j*k+i]; }
+      // Apply Householder reflector (I - tau v v^T) coG^T
+      HouseholderReflect(&A[i*row], &v[i], tau[i], Q1D-i, Q1D, row, col);
+   }
+}
+
+template<int D1D, int Q1D, typename T> MFEM_HOST_DEVICE inline
+void QRFactorization(T *mat, T *tau)
+{
+   T v[Q1D];
+   DeviceMatrix B(mat, D1D, Q1D);
+   for (int i = 0; i < D1D; i++)
+   {
+      // Calculate Householder vector, magnitude
+      T sigma = 0.0;
+      v[i] = B(i,i);
+      for (int j = i + 1; j < Q1D; j++)
+      {
+         v[j] = B(i,j);
+         sigma += v[j] * v[j];
+      }
+      T norm = std::sqrt(v[i]*v[i] + sigma); // norm of v[i:m]
+      T Rii = -copysign(norm, v[i]);
+      v[i] -= Rii;
+      // norm of v[i:m] after modification above and scaling below
+      //   norm = sqrt(v[i]*v[i] + sigma) / v[i];
+      //   tau = 2 / (norm*norm)
+      tau[i] = 2 * v[i]*v[i] / (v[i]*v[i] + sigma);
+      for (int j=i+1; j<Q1D; j++) { v[j] /= v[i]; }
+      // Apply Householder reflector to lower right panel
+      HouseholderReflect(&mat[i*D1D+i+1], &v[i], tau[i],
+                         Q1D-i, D1D-i-1, D1D, 1);
+      // Save v
+      B(i,i) = Rii;
+      for (int j=i+1; j<Q1D; j++) { B(i,j) = v[j]; }
+   }
+}
+
+template<int D1D, int Q1D>
+void GetCollocatedGrad(const ConstDeviceMatrix &b,
+                       const ConstDeviceMatrix &g,
+                       const DeviceMatrix &CoG)
+{
+   double tau[Q1D];
+   double B1d[Q1D*D1D];
+   double G1d[Q1D*D1D];
+   DeviceMatrix B(B1d, D1D, Q1D);
+   DeviceMatrix G(G1d, D1D, Q1D);
+
+   for (int d = 0; d < D1D; d++)
+   {
+      for (int q = 0; q < Q1D; q++)
+      {
+         B(d,q) = b(q,d);
+         G(d,q) = g(q,d);
+      }
+   }
+   QRFactorization<D1D,Q1D>(B1d, tau);
+   // Apply Rinv, colograd1d = grad1d Rinv
+   for (int i = 0; i < Q1D; i++)
+   {
+      CoG(0,i) = G(0,i)/B(0,0);
+      for (int j = 1; j < D1D; j++)
+      {
+         CoG(j,i) = G(j,i);
+         for (int k = 0; k < j; k++) { CoG(j,i) -= B(j,k)*CoG(k,i); }
+         CoG(j,i) /= B(j,j);
+      }
+      for (int j = D1D; j < Q1D; j++) { CoG(j,i) = 0.0; }
+   }
+   // Apply Qtranspose, colograd = colograd Qtranspose
+   HouseholderApplyQ<Q1D>((double*)CoG, B1d, tau, D1D, 1, Q1D);
+}
+
+////////////////////////////////////////////////////////////////////////////////
 void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
-   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
-                         Device::GetDeviceMemoryType() : pa_mt;
+   const MemoryType mt = (memory_type == MemoryType::DEFAULT) ?
+                         Device::GetDeviceMemoryType() : memory_type;
+
+   // If device options allow fast kernels, set the action type to L2L
+   action_type =
+      (Device::FastKernelsEnabled())?
+      ActionType::L2L: // all fast kernel are L2L
+      ActionType::E2E; // default is E2E
+
   // Assuming the same element type
   fespace = &fes;
   Mesh *mesh = fes.GetMesh();
@@ -365,6 +497,7 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
      MFEM_VERIFY(!VQ && !MQ && !SMQ,
                  "Only scalar coefficient supported for DiffusionIntegrator"
                  " with libCEED");
+      action_type = ActionType::L2L;
      ceedOp = new ceed::PADiffusionIntegrator(fes, *ir, Q);
      return;
   }
@@ -490,6 +623,40 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
         }
      }
   }
+
+   if (Device::FastKernelsEnabled())
+   {
+      NVTX("CoG");
+      const int D1D = dofs1D;
+      const int Q1D = quad1D;
+      const int id = (D1D << 4) | Q1D;
+
+      CoG.SetSize(Q1D*Q1D);
+      CoG.UseDevice(true);
+      assert(CoG.UseDevice());
+
+      void (*KoG)(const ConstDeviceMatrix &b,
+                  const ConstDeviceMatrix &g,
+                  const DeviceMatrix &CoG) = nullptr;
+
+      switch (id) // orders 1~8
+      {
+         case 0x23: KoG=GetCollocatedGrad<2,3>; break; // 1
+         case 0x34: KoG=GetCollocatedGrad<3,4>; break; // 2
+         case 0x45: KoG=GetCollocatedGrad<4,5>; break; // 3
+         case 0x56: KoG=GetCollocatedGrad<5,6>; break; // 4
+         case 0x67: KoG=GetCollocatedGrad<6,7>; break; // 5
+         case 0x78: KoG=GetCollocatedGrad<7,8>; break; // 6
+         //case 0x89: KoG=GetCollocatedGrad<8,9>; break; // 7
+         //case 0x9A: KoG=GetCollocatedGrad<9,10>; break; // 8
+         default: MFEM_ABORT("Unknown kernel 0x" << std::hex << id << std::dec);
+      }
+
+      KoG(ConstDeviceMatrix(maps->B.HostRead(),Q1D,D1D),
+          ConstDeviceMatrix(maps->G.HostRead(),Q1D,D1D),
+          DeviceMatrix(CoG.HostReadWrite(),Q1D,Q1D));
+   }
+
   pa_data.SetSize((symmetric ? symmDims : MQfullDim) * nq * ne, mt);
   PADiffusionSetup(dim, sdim, dofs1D, quad1D, coeffDim, ne, ir->GetWeights(),
                    geom->J, coeff, pa_data);
@@ -869,6 +1036,7 @@ static void SmemPADiffusionDiagonal3D(const int NE,
                  }
               }
            }
+            MFEM_SYNC_THREAD;
         }
      }
   });
@@ -925,6 +1093,11 @@ void DiffusionIntegrator::AssembleDiagonalPA(Vector &diag)
   {
      ceedOp->GetDiagonal(diag);
   }
+   else if (Device::FastKernelsEnabled())
+   {
+      NDK_PADiffusionAssembleDiagonal(dim, dofs1D, quad1D, ne, symmetric,
+                                      fespace, maps, pa_data, diag);
+   }
   else
   {
      if (pa_data.Size()==0) { AssemblePA(*fespace); }
@@ -1180,6 +1353,7 @@ static void SmemPADiffusionApply2D(const int NE,
   auto D = Reshape(d_.Read(), Q1D*Q1D, symmetric ? 3 : 4, NE);
   auto x = Reshape(x_.Read(), D1D, D1D, NE);
   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
+
   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
   {
      const int tidz = MFEM_THREAD_ID(z);
@@ -1220,6 +1394,7 @@ static void SmemPADiffusionApply2D(const int NE,
         }
      }
      MFEM_SYNC_THREAD;
+
      MFEM_FOREACH_THREAD(dy,y,D1D)
      {
         MFEM_FOREACH_THREAD(qx,x,Q1D)
@@ -1545,6 +1720,7 @@ static void SmemPADiffusionApply3D(const int NE,
                                   const int d1d = 0,
                                   const int q1d = 0)
 {
+   MFEM_NVTX;
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
   constexpr int M1Q = T_Q1D ? T_Q1D : MAX_Q1D;
@@ -1556,7 +1732,7 @@ static void SmemPADiffusionApply3D(const int NE,
   auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, symmetric ? 6 : 9, NE);
   auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
   {
      const int D1D = T_D1D ? T_D1D : d1d;
      const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -1585,118 +1761,103 @@ static void SmemPADiffusionApply3D(const int NE,
      double (*QDD0)[MD1][MD1] = (double (*)[MD1][MD1]) (sm0+0);
      double (*QDD1)[MD1][MD1] = (double (*)[MD1][MD1]) (sm0+1);
      double (*QDD2)[MD1][MD1] = (double (*)[MD1][MD1]) (sm0+2);
-      MFEM_FOREACH_THREAD(dy,y,D1D)
+      MFEM_FOREACH_THREAD(dz,z,D1D)
      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
         {
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
+            MFEM_FOREACH_THREAD(dx,x,D1D)
            {
               X[dz][dy][dx] = x(dx,dy,dz,e);
            }
         }
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+      }
+      MFEM_SYNC_THREAD;
+      if (MFEM_THREAD_ID(z) == 0)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
         {
-            const int i = qi(qx,dy,Q1D);
-            const int j = dj(qx,dy,D1D);
-            const int k = qk(qx,dy,Q1D);
-            const int l = dl(qx,dy,D1D);
-            B[i][j] = b(qx,dy);
-            G[k][l] = g(qx,dy) * sign(qx,dy);
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               const int i = qi(qx,dy,Q1D);
+               const int j = dj(qx,dy,D1D);
+               const int k = qk(qx,dy,Q1D);
+               const int l = dl(qx,dy,D1D);
+               B[i][j] = b(qx,dy);
+               G[k][l] = g(qx,dy) * sign(qx,dy);
+            }
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
+      MFEM_FOREACH_THREAD(dz,z,D1D)
      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
         {
-            double u[D1D], v[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++) { u[dz] = v[dz] = 0.0; }
-            MFEM_UNROLL(MD1)
-            for (int dx = 0; dx < D1D; ++dx)
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
            {
-               const int i = qi(qx,dx,Q1D);
-               const int j = dj(qx,dx,D1D);
-               const int k = qk(qx,dx,Q1D);
-               const int l = dl(qx,dx,D1D);
-               const double s = sign(qx,dx);
+               double u = 0.0, v = 0.0;
+               MFEM_UNROLL(MD1)
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  const int i = qi(qx,dx,Q1D);
+                  const int j = dj(qx,dx,D1D);
+                  const int k = qk(qx,dx,Q1D);
+                  const int l = dl(qx,dx,D1D);
+                  const double s = sign(qx,dx);
+                  const double coords = X[dz][dy][dx];
+                  u += coords * B[i][j];
+                  v += coords * G[k][l] * s;
+               }
+               DDQ0[dz][dy][qx] = u;
+               DDQ1[dz][dy][qx] = v;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dz,z,D1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double u = 0.0, v = 0.0, w = 0.0;
+               MFEM_UNROLL(MD1)
+               for (int dy = 0; dy < D1D; ++dy)
+               {
+                  const int i = qi(qy,dy,Q1D);
+                  const int j = dj(qy,dy,D1D);
+                  const int k = qk(qy,dy,Q1D);
+                  const int l = dl(qy,dy,D1D);
+                  const double s = sign(qy,dy);
+                  u += DDQ1[dz][dy][qx] * B[i][j];
+                  v += DDQ0[dz][dy][qx] * G[k][l] * s;
+                  w += DDQ0[dz][dy][qx] * B[i][j];
+               }
+               DQQ0[dz][qy][qx] = u;
+               DQQ1[dz][qy][qx] = v;
+               DQQ2[dz][qy][qx] = w;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double u = 0.0, v = 0.0, w = 0.0;
               MFEM_UNROLL(MD1)
               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  const double coords = X[dz][dy][dx];
-                  u[dz] += coords * B[i][j];
-                  v[dz] += coords * G[k][l] * s;
-               }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               DDQ0[dz][dy][qx] = u[dz];
-               DDQ1[dz][dy][qx] = v[dz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double u[D1D], v[D1D], w[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++) { u[dz] = v[dz] = w[dz] = 0.0; }
-            MFEM_UNROLL(MD1)
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               const int i = qi(qy,dy,Q1D);
-               const int j = dj(qy,dy,D1D);
-               const int k = qk(qy,dy,Q1D);
-               const int l = dl(qy,dy,D1D);
-               const double s = sign(qy,dy);
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; dz++)
-               {
-                  u[dz] += DDQ1[dz][dy][qx] * B[i][j];
-                  v[dz] += DDQ0[dz][dy][qx] * G[k][l] * s;
-                  w[dz] += DDQ0[dz][dy][qx] * B[i][j];
-               }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++)
-            {
-               DQQ0[dz][qy][qx] = u[dz];
-               DQQ1[dz][qy][qx] = v[dz];
-               DQQ2[dz][qy][qx] = w[dz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double u[Q1D], v[Q1D], w[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; qz++) { u[qz] = v[qz] = w[qz] = 0.0; }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; qz++)
               {
                  const int i = qi(qz,dz,Q1D);
                  const int j = dj(qz,dz,D1D);
                  const int k = qk(qz,dz,Q1D);
                  const int l = dl(qz,dz,D1D);
                  const double s = sign(qz,dz);
-                  u[qz] += DQQ0[dz][qy][qx] * B[i][j];
-                  v[qz] += DQQ1[dz][qy][qx] * B[i][j];
-                  w[qz] += DQQ2[dz][qy][qx] * G[k][l] * s;
+                  u += DQQ0[dz][qy][qx] * B[i][j];
+                  v += DQQ1[dz][qy][qx] * B[i][j];
+                  w += DQQ2[dz][qy][qx] * G[k][l] * s;
               }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; qz++)
-            {
               const double O11 = d(qx,qy,qz,0,e);
               const double O12 = d(qx,qy,qz,1,e);
               const double O13 = d(qx,qy,qz,2,e);
@@ -1706,9 +1867,9 @@ static void SmemPADiffusionApply3D(const int NE,
               const double O31 = symmetric ? O13 : d(qx,qy,qz,6,e);
               const double O32 = symmetric ? O23 : d(qx,qy,qz,7,e);
               const double O33 = symmetric ? d(qx,qy,qz,5,e) : d(qx,qy,qz,8,e);
-               const double gX = u[qz];
-               const double gY = v[qz];
-               const double gZ = w[qz];
+               const double gX = u;
+               const double gY = v;
+               const double gZ = w;
               QQQ0[qz][qy][qx] = (O11*gX) + (O12*gY) + (O13*gZ);
               QQQ1[qz][qy][qx] = (O21*gX) + (O22*gY) + (O23*gZ);
               QQQ2[qz][qy][qx] = (O31*gX) + (O32*gY) + (O33*gZ);
@@ -1716,115 +1877,98 @@ static void SmemPADiffusionApply3D(const int NE,
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(d,y,D1D)
+      if (MFEM_THREAD_ID(z) == 0)
      {
-         MFEM_FOREACH_THREAD(q,x,Q1D)
+         MFEM_FOREACH_THREAD(d,y,D1D)
         {
-            const int i = qi(q,d,Q1D);
-            const int j = dj(q,d,D1D);
-            const int k = qk(q,d,Q1D);
-            const int l = dl(q,d,D1D);
-            Bt[j][i] = b(q,d);
-            Gt[l][k] = g(q,d) * sign(q,d);
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               const int i = qi(q,d,Q1D);
+               const int j = dj(q,d,D1D);
+               const int k = qk(q,d,Q1D);
+               const int l = dl(q,d,D1D);
+               Bt[j][i] = b(q,d);
+               Gt[l][k] = g(q,d) * sign(q,d);
+            }
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
         {
-            double u[Q1D], v[Q1D], w[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz) { u[qz] = v[qz] = w[qz] = 0.0; }
-            MFEM_UNROLL(MQ1)
-            for (int qx = 0; qx < Q1D; ++qx)
+            MFEM_FOREACH_THREAD(dx,x,D1D)
            {
-               const int i = qi(qx,dx,Q1D);
-               const int j = dj(qx,dx,D1D);
-               const int k = qk(qx,dx,Q1D);
-               const int l = dl(qx,dx,D1D);
-               const double s = sign(qx,dx);
+               double u = 0.0, v = 0.0, w = 0.0;
+               MFEM_UNROLL(MQ1)
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  const int i = qi(qx,dx,Q1D);
+                  const int j = dj(qx,dx,D1D);
+                  const int k = qk(qx,dx,Q1D);
+                  const int l = dl(qx,dx,D1D);
+                  const double s = sign(qx,dx);
+                  u += QQQ0[qz][qy][qx] * Gt[l][k] * s;
+                  v += QQQ1[qz][qy][qx] * Bt[j][i];
+                  w += QQQ2[qz][qy][qx] * Bt[j][i];
+               }
+               QQD0[qz][qy][dx] = u;
+               QQD1[qz][qy][dx] = v;
+               QQD2[qz][qy][dx] = w;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               double u = 0.0, v = 0.0, w = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  const int i = qi(qy,dy,Q1D);
+                  const int j = dj(qy,dy,D1D);
+                  const int k = qk(qy,dy,Q1D);
+                  const int l = dl(qy,dy,D1D);
+                  const double s = sign(qy,dy);
+                  u += QQD0[qz][qy][dx] * Bt[j][i];
+                  v += QQD1[qz][qy][dx] * Gt[l][k] * s;
+                  w += QQD2[qz][qy][dx] * Bt[j][i];
+               }
+               QDD0[qz][dy][dx] = u;
+               QDD1[qz][dy][dx] = v;
+               QDD2[qz][dy][dx] = w;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dz,z,D1D)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               double u = 0.0, v = 0.0, w = 0.0;
               MFEM_UNROLL(MQ1)
               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u[qz] += QQQ0[qz][qy][qx] * Gt[l][k] * s;
-                  v[qz] += QQQ1[qz][qy][qx] * Bt[j][i];
-                  w[qz] += QQQ2[qz][qy][qx] * Bt[j][i];
-               }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               QQD0[qz][qy][dx] = u[qz];
-               QQD1[qz][qy][dx] = v[qz];
-               QQD2[qz][qy][dx] = w[qz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[Q1D], v[Q1D], w[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz) { u[qz] = v[qz] = w[qz] = 0.0; }
-            MFEM_UNROLL(MQ1)
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const int i = qi(qy,dy,Q1D);
-               const int j = dj(qy,dy,D1D);
-               const int k = qk(qy,dy,Q1D);
-               const int l = dl(qy,dy,D1D);
-               const double s = sign(qy,dy);
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u[qz] += QQD0[qz][qy][dx] * Bt[j][i];
-                  v[qz] += QQD1[qz][qy][dx] * Gt[l][k] * s;
-                  w[qz] += QQD2[qz][qy][dx] * Bt[j][i];
-               }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               QDD0[qz][dy][dx] = u[qz];
-               QDD1[qz][dy][dx] = v[qz];
-               QDD2[qz][dy][dx] = w[qz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[D1D], v[D1D], w[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz) { u[dz] = v[dz] = w[dz] = 0.0; }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; ++dz)
               {
                  const int i = qi(qz,dz,Q1D);
                  const int j = dj(qz,dz,D1D);
                  const int k = qk(qz,dz,Q1D);
                  const int l = dl(qz,dz,D1D);
                  const double s = sign(qz,dz);
-                  u[dz] += QDD0[qz][dy][dx] * Bt[j][i];
-                  v[dz] += QDD1[qz][dy][dx] * Bt[j][i];
-                  w[dz] += QDD2[qz][dy][dx] * Gt[l][k] * s;
+                  u += QDD0[qz][dy][dx] * Bt[j][i];
+                  v += QDD1[qz][dy][dx] * Bt[j][i];
+                  w += QDD2[qz][dy][dx] * Gt[l][k] * s;
               }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               y(dx,dy,dz,e) += (u[dz] + v[dz] + w[dz]);
+               y(dx,dy,dz,e) += (u + v + w);
            }
         }
      }
+      MFEM_SYNC_THREAD;
   });
 }

@@ -1864,8 +2008,11 @@ static void PADiffusionApply(const int dim,
      switch (ID)
      {
         case 0x22: return SmemPADiffusionApply2D<2,2,16>(NE,symm,B,G,D,X,Y);
+         case 0x23: return SmemPADiffusionApply2D<2,3,16>(NE,symm,B,G,D,X,Y);
         case 0x33: return SmemPADiffusionApply2D<3,3,16>(NE,symm,B,G,D,X,Y);
+         case 0x34: return SmemPADiffusionApply2D<3,4,16>(NE,symm,B,G,D,X,Y);
         case 0x44: return SmemPADiffusionApply2D<4,4,8>(NE,symm,B,G,D,X,Y);
+         case 0x45: return SmemPADiffusionApply2D<4,5,8>(NE,symm,B,G,D,X,Y);
         case 0x55: return SmemPADiffusionApply2D<5,5,8>(NE,symm,B,G,D,X,Y);
         case 0x66: return SmemPADiffusionApply2D<6,6,4>(NE,symm,B,G,D,X,Y);
         case 0x77: return SmemPADiffusionApply2D<7,7,4>(NE,symm,B,G,D,X,Y);
@@ -1902,6 +2049,12 @@ void DiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
   {
      ceedOp->AddMult(x, y);
   }
+   else if (Device::FastKernelsEnabled())
+   {
+      NDK_PADiffusionApply(dim, dofs1D, quad1D, ne, CoG,
+                           fespace, maps,
+                           pa_data, x, y);
+   }
   else
   {
      PADiffusionApply(dim, dofs1D, quad1D, ne, symmetric,
@@ -0,0 +1,186 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#define MFEM_NVTX_COLOR Pink
+#include "../general/nvtx.hpp"
+
+#define MFEM_DEBUG_COLOR 206
+#include "../general/debug.hpp"
+#include "../general/forall.hpp"
+#include "bilininteg.hpp"
+#include "gridfunc.hpp"
+#include "ceed/diffusion.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+// Shared memory PA Diffusion Diagonal 3D kernel
+template<int D1D, int Q1D>
+static void NDK_SmemPADiffusionDiag3D(const int ndofs,
+                                      const int NE,
+                                      const bool symmetric,
+                                      const int *map_,
+                                      const double *b_,
+                                      const double *g_,
+                                      const double *d_,
+                                      double *y_)
+{
+   MFEM_NVTX;
+   constexpr int DIM = 3;
+   const auto MAP = Reshape(map_, D1D,D1D,D1D, NE);
+   auto b = Reshape(b_, Q1D, D1D);
+   auto g = Reshape(g_, Q1D, D1D);
+   auto D = Reshape(d_, Q1D*Q1D*Q1D, symmetric ? 6 : 9, NE);
+   auto Y = Reshape(y_, D1D, D1D, D1D, NE);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      const int tidz = MFEM_THREAD_ID(z);
+      MFEM_SHARED double BG[2][Q1D*D1D];
+      double (*B)[D1D] = (double (*)[D1D]) (BG+0);
+      double (*G)[D1D] = (double (*)[D1D]) (BG+1);
+      MFEM_SHARED double QQD[Q1D][Q1D][D1D];
+      MFEM_SHARED double QDD[Q1D][D1D][D1D];
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B[q][d] = b(q,d);
+               G[q][d] = g(q,d);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      for (int i = 0; i < DIM; ++i)
+      {
+         for (int j = 0; j < DIM; ++j)
+         {
+            // first tensor contraction, along z direction
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               MFEM_FOREACH_THREAD(qy,y,Q1D)
+               {
+                  MFEM_FOREACH_THREAD(dz,z,D1D)
+                  {
+                     QQD[qx][qy][dz] = 0.0;
+                     for (int qz = 0; qz < Q1D; ++qz)
+                     {
+                        const int q = qx + (qy + qz * Q1D) * Q1D;
+                        const int ksym = j >= i ?
+                                         3 - (3-i)*(2-i)/2 + j:
+                                         3 - (3-j)*(2-j)/2 + i;
+                        const int k = symmetric ? ksym : (i*DIM) + j;
+                        const double O = D(q,k,e);
+                        const double Bz = B[qz][dz];
+                        const double Gz = G[qz][dz];
+                        const double L = i==2 ? Gz : Bz;
+                        const double R = j==2 ? Gz : Bz;
+                        QQD[qx][qy][dz] += L * O * R;
+                     }
+                  }
+               }
+            }
+            MFEM_SYNC_THREAD;
+            // second tensor contraction, along y direction
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               MFEM_FOREACH_THREAD(dz,z,D1D)
+               {
+                  MFEM_FOREACH_THREAD(dy,y,D1D)
+                  {
+                     QDD[qx][dy][dz] = 0.0;
+                     for (int qy = 0; qy < Q1D; ++qy)
+                     {
+                        const double By = B[qy][dy];
+                        const double Gy = G[qy][dy];
+                        const double L = i==1 ? Gy : By;
+                        const double R = j==1 ? Gy : By;
+                        QDD[qx][dy][dz] += L * QQD[qx][qy][dz] * R;
+                     }
+                  }
+               }
+            }
+            MFEM_SYNC_THREAD;
+            // third tensor contraction, along x direction
+            MFEM_FOREACH_THREAD(dz,z,D1D)
+            {
+               MFEM_FOREACH_THREAD(dy,y,D1D)
+               {
+                  MFEM_FOREACH_THREAD(dx,x,D1D)
+                  {
+                     for (int qx = 0; qx < Q1D; ++qx)
+                     {
+                        const double Bx = B[qx][dx];
+                        const double Gx = G[qx][dx];
+                        const double L = i==0 ? Gx : Bx;
+                        const double R = j==0 ? Gx : Bx;
+                        const double lvr = L * QDD[qx][dy][dz] * R;
+                        const int gid = MAP(dx,dy,dz,e);
+                        const int idx = gid >= 0 ? gid : -1 - gid;
+                        AtomicAdd(Y[idx], lvr);
+                     }
+                  }
+               }
+            }
+            MFEM_SYNC_THREAD;
+         }
+      }
+   });
+}
+
+void NDK_PADiffusionAssembleDiagonal(const int dim,
+                                     const int D1D,
+                                     const int Q1D,
+                                     const int NE,
+                                     const bool symm,
+                                     const FiniteElementSpace *fes,
+                                     const DofToQuad *maps,
+                                     const Vector &D,
+                                     Vector &Y)
+{
+   MFEM_NVTX;
+   const int ND = fes->GetNDofs();
+   constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+   const Operator *ERop = fes->GetElementRestriction(ordering);
+   const ElementRestriction* ER = dynamic_cast<const ElementRestriction*>(ERop);
+   assert(ER);
+   const int *m = ER->GatherMap().Read();
+   const double *b = maps->B.Read();
+   const double *g = maps->G.Read();
+   const double *d = D.Read();
+   double *y = Y.ReadWrite();
+
+   if (dim == 2) {  assert(false); }
+   else if (dim == 3)
+   {
+      switch ((D1D << 4 ) | Q1D)
+      {
+         case 0x22: return NDK_SmemPADiffusionDiag3D<2,2>(ND,NE,symm,m,b,g,d,y);
+         case 0x23: return NDK_SmemPADiffusionDiag3D<2,3>(ND,NE,symm,m,b,g,d,y);
+         case 0x34: return NDK_SmemPADiffusionDiag3D<3,4>(ND,NE,symm,m,b,g,d,y);
+         case 0x45: return NDK_SmemPADiffusionDiag3D<4,5>(ND,NE,symm,m,b,g,d,y);
+         case 0x46: return NDK_SmemPADiffusionDiag3D<4,6>(ND,NE,symm,m,b,g,d,y);
+         case 0x56: return NDK_SmemPADiffusionDiag3D<5,6>(ND,NE,symm,m,b,g,d,y);
+         case 0x67: return NDK_SmemPADiffusionDiag3D<6,7>(ND,NE,symm,m,b,g,d,y);
+         case 0x78: return NDK_SmemPADiffusionDiag3D<7,8>(ND,NE,symm,m,b,g,d,y);
+         //case 0x89: return NDK_SmemPADiffusionDiag3D<8,9>(ND,NE,symm,m,b,g,d,y);
+         //case 0x9A: return NDK_SmemPADiffusionDiag3D<9,10>(ND,NE,symm,m,b,g,d,y);
+         default: MFEM_ABORT("Not implemented!");
+      }
+   }
+   MFEM_ABORT("Unknown kernel.");
+}
+
+} // namespace mfem
@@ -0,0 +1,332 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#define MFEM_NVTX_COLOR Pink
+#include "../general/nvtx.hpp"
+
+#define MFEM_DEBUG_COLOR 206
+#include "../general/debug.hpp"
+#include "../general/forall.hpp"
+#include "bilininteg.hpp"
+#include "gridfunc.hpp"
+#include "ceed/diffusion.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+////////////////////////////////////////////////////////////////////////////////
+template<int D1D, int Q1D, int NBZ, int NBK> static
+MFEM_GLOBAL MFEM_LAUNCH_BOUNDS(Q1D*Q1D*NBZ,NBK)
+void NDK_PADiffApply(const int NE,
+                     const int *MAP,
+                     const double *B,
+                     const double *G,
+                     const double *D,
+                     const double *X,
+                     double *Y)
+{
+   double r_qt, r_q[Q1D];
+   MFEM_SHARED double s_B[D1D][Q1D];
+   MFEM_SHARED double s_G[Q1D][Q1D];
+   MFEM_SHARED double s_Iq[NBZ][Q1D][Q1D][Q1D];
+   MFEM_SHARED double s_Gqr[NBZ][Q1D][Q1D];
+   MFEM_SHARED double s_Gqs[NBZ][Q1D][Q1D];
+
+   for (int be = MFEM_BLOCK_ID(x); be < (NE+NBZ-1)/NBZ; be += MFEM_GRID_DIM(x))
+   {
+      const int tz = MFEM_THREAD_ID(z);
+      const int e = be * NBZ + tz;
+      if (e>=NE) { return; }
+
+      // Scatter X
+      MFEM_FOREACH_THREAD(j,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(i,x,Q1D)
+         {
+            if (tz == 0) { s_G[j][i] = G[i+Q1D*j]; } // ok with init
+            if (tz == 0 && j<D1D) { s_B[j][i] = B[i+Q1D*j]; } // ok
+            if (j<D1D && i<D1D)
+            {
+               MFEM_UNROLL(D1D)
+               for (int k = 0; k < D1D; k++)
+               {
+                  const int gid = MAP[e*D1D*D1D*D1D + k*D1D*D1D + j*D1D + i];
+                  const int idx = gid >= 0 ? gid : -1 - gid;
+                  r_q[k] = X[idx];
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Grad1X
+      MFEM_FOREACH_THREAD(b,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(a,x,D1D)
+         {
+            MFEM_UNROLL(Q1D)
+            for (int k=0; k<Q1D; ++k)
+            {
+               double u = 0.0;
+               MFEM_UNROLL(D1D)
+               for (int c=0; c<D1D; ++c) { u += s_B[c][k] * r_q[c]; }
+               s_Iq[tz][k][b][a] = u;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Grad1Y
+      MFEM_FOREACH_THREAD(k,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(a,x,D1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int b=0; b<D1D; ++b) { r_q[b] = s_Iq[tz][k][b][a]; }
+            MFEM_UNROLL(Q1D)
+            for (int j=0; j<Q1D; ++j)
+            {
+               double u = 0.0;
+               MFEM_UNROLL(D1D)
+               for (int b=0; b<D1D; ++b) { u += s_B[b][j] * r_q[b]; }
+               s_Iq[tz][k][j][a] = u;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Grad1Z
+      MFEM_FOREACH_THREAD(k,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(j,x,Q1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int a=0; a<D1D; ++a) { r_q[a] = s_Iq[tz][k][j][a]; }
+            MFEM_UNROLL(Q1D)
+            for (int i=0; i<Q1D; ++i)
+            {
+               double u = 0.0;
+               MFEM_UNROLL(D1D)
+               for (int a=0; a<D1D; ++a) { u += s_B[a][i] * r_q[a]; }
+               s_Iq[tz][k][j][i] = u;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Flush
+      MFEM_FOREACH_THREAD(j,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(i,x,Q1D)
+         {
+            MFEM_UNROLL(Q1D)
+            for (int k = 0; k < Q1D; ++k) { r_q[k] = 0.0; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Q-Function
+      MFEM_UNROLL(Q1D)
+      for (int k = 0; k < Q1D; ++k)
+      {
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(j,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(i,x,Q1D)
+            {
+               double qr = 0.0, qs = 0.0, qt = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int m = 0; m < Q1D; ++m)
+               {
+                  const double Dim = s_G[i][m];
+                  const double Djm = s_G[j][m];
+                  const double Dkm = s_G[k][m];
+                  qr += Dim * s_Iq[tz][k][j][m];
+                  qs += Djm * s_Iq[tz][k][m][i];
+                  qt += Dkm * s_Iq[tz][m][j][i];
+               }
+               //(d, Q1D,Q1D,Q1D, 6, NE);
+               const int ebase = e * 6*Q1D*Q1D*Q1D;
+               const int gbase = i + j*Q1D + k*Q1D*Q1D;
+               const double D00 = D[0*Q1D*Q1D*Q1D + gbase + ebase];
+               const double D01 = D[1*Q1D*Q1D*Q1D + gbase + ebase];
+               const double D02 = D[2*Q1D*Q1D*Q1D + gbase + ebase];
+               const double D11 = D[3*Q1D*Q1D*Q1D + gbase + ebase];
+               const double D12 = D[4*Q1D*Q1D*Q1D + gbase + ebase];
+               const double D22 = D[5*Q1D*Q1D*Q1D + gbase + ebase];
+
+               s_Gqr[tz][j][i] = D00*qr + D01*qs + D02*qt;
+               s_Gqs[tz][j][i] = D01*qr + D11*qs + D12*qt;
+               r_qt = D02*qr + D12*qs + D22*qt;
+            }
+         }
+         MFEM_SYNC_THREAD;
+
+         MFEM_FOREACH_THREAD(j,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(i,x,Q1D)
+            {
+               double Aqtmp = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int m = 0; m < Q1D; ++m)
+               {
+                  const double Dmi = s_G[m][i];
+                  const double Dmj = s_G[m][j];
+                  const double Dkm = s_G[k][m];
+                  Aqtmp += Dmi * s_Gqr[tz][j][m];
+                  Aqtmp += Dmj * s_Gqs[tz][m][i];
+                  r_q[m] += Dkm * r_qt;
+               }
+               r_q[k] += Aqtmp;
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+
+      // GradZT
+      MFEM_FOREACH_THREAD(j,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(i,x,Q1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int c=0; c<D1D; ++c)
+            {
+               double u = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int k=0; k<Q1D; ++k) { u += s_B[c][k] * r_q[k]; }
+               s_Iq[tz][c][j][i] = u;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // GradYT
+      MFEM_FOREACH_THREAD(c,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(i,x,Q1D)
+         {
+            MFEM_UNROLL(Q1D)
+            for (int j=0; j<Q1D; ++j) { r_q[j] = s_Iq[tz][c][j][i]; }
+            MFEM_UNROLL(D1D)
+            for (int b=0; b<D1D; ++b)
+            {
+               double u = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int j=0; j<Q1D; ++j) { u += s_B[b][j] * r_q[j]; }
+               s_Iq[tz][c][b][i] = u;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // GradXT
+      MFEM_FOREACH_THREAD(c,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(b,x,D1D)
+         {
+            MFEM_UNROLL(Q1D)
+            for (int i=0; i<Q1D; ++i) { r_q[i] = s_Iq[tz][c][b][i]; }
+            MFEM_UNROLL(D1D)
+            for (int a=0; a<D1D; ++a)
+            {
+               double u = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int i=0; i<Q1D; ++i) { u += s_B[a][i] * r_q[i]; }
+               s_Iq[tz][c][b][a] = u;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Gather
+      MFEM_FOREACH_THREAD(j,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(i,x,D1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int k = 0; k < D1D; k++)
+            {
+               const int gid = MAP[e*D1D*D1D*D1D + k*D1D*D1D + j*D1D + i];
+               const int idx = gid >= 0 ? gid : -1 - gid;
+               const double output = s_Iq[tz][k][j][i];
+               AtomicAdd(Y[idx], output);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   }
+}
+
+void NDK_PADiffusionApply(const int dim,
+                          const int D1D,
+                          const int Q1D,
+                          const int NE,
+                          const Vector &CoG,
+                          const FiniteElementSpace *fes,
+                          const DofToQuad *maps,
+                          const Vector &D,
+                          const Vector &X,
+                          Vector &Y)
+{
+   //dbg();
+   MFEM_NVTX;
+   assert(dim == 3);
+
+   const double *b = maps->B.Read();
+   const double *d = D.Read();
+   const double *x = X.Read();
+   double *y = Y.ReadWrite();
+
+   const int id = (D1D << 4) | Q1D;
+
+   const int ND = fes->GetNDofs();
+   constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+   const Operator *ERop = fes->GetElementRestriction(ordering);
+   const ElementRestriction *ER = dynamic_cast<const ElementRestriction*>(ERop);
+   assert(ER);
+   const int *map = ER->GatherMap().Read();
+
+   const auto dM = Reshape(map, D1D,D1D,D1D, NE);
+   const auto dB = Reshape(b, Q1D,D1D);
+   const auto dG = Reshape(CoG.Read(), Q1D,Q1D);
+   const auto dD = Reshape(d, Q1D,Q1D,Q1D, 6, NE);
+   const auto dX = Reshape(x, ND);
+   auto dY = Reshape(y, ND);
+
+   void (*Ker)(const int NE,
+               const int *MAP,
+               const double *B,
+               const double *G,
+               const double *D,
+               const double *X,
+               double *Y) = nullptr;
+
+   int NBZ = 1;
+
+   switch (id) // orders 1~8
+   {
+      case 0x23: Ker=NDK_PADiffApply<2,3,16,5>; NBZ=16; break; // 1
+      case 0x34: Ker=NDK_PADiffApply<3,4,8,6>; NBZ=8; break; // 2
+      case 0x45: Ker=NDK_PADiffApply<4,5,4,4>; NBZ=4; break; // 3
+      case 0x56: Ker=NDK_PADiffApply<5,6,4,3>; NBZ=4; break; // 4
+      case 0x67: Ker=NDK_PADiffApply<6,7,1,3>; break; // 5
+      case 0x78: Ker=NDK_PADiffApply<7,8,1,0>; break; // 6
+      //case 0x89: Ker=NDK_PADiffApply<8,9,1,0>; break; // 7
+      //case 0x9A: Ker=NDK_PADiffApply<9,10,1,1>; break; // 8
+      default: MFEM_ABORT("Unknown kernel 0x" << std::hex << id << std::dec);
+   }
+   MFEM_CONTRACT_VAR(NBZ);
+   MFEM_LAUNCH_KERNEL(Ker,(NE+NBZ-1)/NBZ,dim3(Q1D,Q1D,NBZ),0,NE,dM,dB,dG,dD,dX,dY);
+}
+
+} // namespace mfem
@@ -19,14 +19,49 @@ using namespace std;
 namespace mfem
 {

+// Forward declaration of non-deterministic 'fast' kernels
+void NDK_PAMassApply(const int dim,
+                     const int D1D,
+                     const int Q1D,
+                     const int NE,
+                     const FiniteElementSpace *fes,
+                     const DofToQuad *maps,
+                     const Vector &D,
+                     const Vector &X,
+                     Vector &Y);
+
+void NDK_AMD_PAMassApply(const int dim,
+                         const int D1D,
+                         const int Q1D,
+                         const int NE,
+                         const FiniteElementSpace *fes,
+                         const DofToQuad *maps,
+                         const Vector &D,
+                         const Vector &X,
+                         Vector &Y);
+
+void NDK_HIP_PAMassApply(const int dim,
+                         const int D1D,
+                         const int Q1D,
+                         const int NE,
+                         const FiniteElementSpace *fes,
+                         const DofToQuad *maps,
+                         const Vector &D,
+                         const Vector &X,
+                         Vector &Y);
+
 // PA Mass Integrator

 // PA Mass Assemble kernel

 void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
-   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
-                         Device::GetDeviceMemoryType() : pa_mt;
+   const MemoryType mt = (memory_type == MemoryType::DEFAULT) ?
+                         Device::GetDeviceMemoryType() : memory_type;
+
+   // If device options allow fast kernels, set the action type to L2L
+   action_type =
+      Device::FastKernelsEnabled() ? ActionType::L2L : ActionType::E2E;

   // Assuming the same element type
   fespace = &fes;
@@ -38,7 +73,7 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
   if (DeviceCanUseCeed())
   {
      delete ceedOp;
-      ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q);
+      ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q, action_type);
      return;
   }
   dim = mesh->Dimension();
@@ -154,328 +189,6 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
   }
 }

-template<int T_D1D = 0, int T_Q1D = 0>
-static void PAMassAssembleDiagonal2D(const int NE,
-                                     const Array<double> &b,
-                                     const Vector &d,
-                                     Vector &y,
-                                     const int d1d = 0,
-                                     const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto D = Reshape(d.Read(), Q1D, Q1D, NE);
-   auto Y = Reshape(y.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      double QD[MQ1][MD1];
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            QD[qx][dy] = 0.0;
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               QD[qx][dy] += B(qy, dy) * B(qy, dy) * D(qx, qy, e);
-            }
-         }
-      }
-      for (int dy = 0; dy < D1D; ++dy)
-      {
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               Y(dx,dy,e) += B(qx, dx) * B(qx, dx) * QD[qx][dy];
-            }
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
-static void SmemPAMassAssembleDiagonal2D(const int NE,
-                                         const Array<double> &b_,
-                                         const Vector &d_,
-                                         Vector &y_,
-                                         const int d1d = 0,
-                                         const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-   MFEM_VERIFY(D1D <= MD1, "");
-   MFEM_VERIFY(Q1D <= MQ1, "");
-   auto b = Reshape(b_.Read(), Q1D, D1D);
-   auto D = Reshape(d_.Read(), Q1D, Q1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      MFEM_SHARED double B[MQ1][MD1];
-      MFEM_SHARED double QDZ[NBZ][MQ1][MD1];
-      double (*QD)[MD1] = (double (*)[MD1])(QDZ + tidz);
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B[q][d] = b(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            QD[qx][dy] = 0.0;
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               QD[qx][dy] += B[qy][dy] * B[qy][dy] * D(qx, qy, e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               // might need absolute values on next line
-               Y(dx,dy,e) += B[qx][dx] * B[qx][dx] * QD[qx][dy];
-            }
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void PAMassAssembleDiagonal3D(const int NE,
-                                     const Array<double> &b,
-                                     const Vector &d,
-                                     Vector &y,
-                                     const int d1d = 0,
-                                     const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto D = Reshape(d.Read(), Q1D, Q1D, Q1D, NE);
-   auto Y = Reshape(y.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      double QQD[MQ1][MQ1][MD1];
-      double QDD[MQ1][MD1][MD1];
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               QQD[qx][qy][dz] = 0.0;
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  QQD[qx][qy][dz] += B(qz, dz) * B(qz, dz) * D(qx, qy, qz, e);
-               }
-            }
-         }
-      }
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               QDD[qx][dy][dz] = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  QDD[qx][dy][dz] += B(qy, dy) * B(qy, dy) * QQD[qx][qy][dz];
-               }
-            }
-         }
-      }
-      for (int dz = 0; dz < D1D; ++dz)
-      {
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               double t = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  t += B(qx, dx) * B(qx, dx) * QDD[qx][dy][dz];
-               }
-               Y(dx, dy, dz, e) += t;
-            }
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void SmemPAMassAssembleDiagonal3D(const int NE,
-                                         const Array<double> &b_,
-                                         const Vector &d_,
-                                         Vector &y_,
-                                         const int d1d = 0,
-                                         const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-   MFEM_VERIFY(D1D <= MD1, "");
-   MFEM_VERIFY(Q1D <= MQ1, "");
-   auto b = Reshape(b_.Read(), Q1D, D1D);
-   auto D = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      MFEM_SHARED double B[MQ1][MD1];
-      MFEM_SHARED double QQD[MQ1][MQ1][MD1];
-      MFEM_SHARED double QDD[MQ1][MD1][MD1];
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B[q][d] = b(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(dz,z,D1D)
-            {
-               QQD[qx][qy][dz] = 0.0;
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  QQD[qx][qy][dz] += B[qz][dz] * B[qz][dz] * D(qx, qy, qz, e);
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               QDD[qx][dy][dz] = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  QDD[qx][dy][dz] += B[qy][dy] * B[qy][dy] * QQD[qx][qy][dz];
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               double t = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  t += B[qx][dx] * B[qx][dx] * QDD[qx][dy][dz];
-               }
-               Y(dx, dy, dz, e) += t;
-            }
-         }
-      }
-   });
-}
-
-static void PAMassAssembleDiagonal(const int dim, const int D1D,
-                                   const int Q1D, const int NE,
-                                   const Array<double> &B,
-                                   const Vector &D,
-                                   Vector &Y)
-{
-   if (dim == 2)
-   {
-      switch ((D1D << 4 ) | Q1D)
-      {
-         case 0x22: return SmemPAMassAssembleDiagonal2D<2,2,16>(NE,B,D,Y);
-         case 0x33: return SmemPAMassAssembleDiagonal2D<3,3,16>(NE,B,D,Y);
-         case 0x44: return SmemPAMassAssembleDiagonal2D<4,4,8>(NE,B,D,Y);
-         case 0x55: return SmemPAMassAssembleDiagonal2D<5,5,8>(NE,B,D,Y);
-         case 0x66: return SmemPAMassAssembleDiagonal2D<6,6,4>(NE,B,D,Y);
-         case 0x77: return SmemPAMassAssembleDiagonal2D<7,7,4>(NE,B,D,Y);
-         case 0x88: return SmemPAMassAssembleDiagonal2D<8,8,2>(NE,B,D,Y);
-         case 0x99: return SmemPAMassAssembleDiagonal2D<9,9,2>(NE,B,D,Y);
-         default:   return PAMassAssembleDiagonal2D(NE,B,D,Y,D1D,Q1D);
-      }
-   }
-   else if (dim == 3)
-   {
-      switch ((D1D << 4 ) | Q1D)
-      {
-         case 0x23: return SmemPAMassAssembleDiagonal3D<2,3>(NE,B,D,Y);
-         case 0x24: return SmemPAMassAssembleDiagonal3D<2,4>(NE,B,D,Y);
-         case 0x26: return SmemPAMassAssembleDiagonal3D<2,6>(NE,B,D,Y);
-         case 0x34: return SmemPAMassAssembleDiagonal3D<3,4>(NE,B,D,Y);
-         case 0x35: return SmemPAMassAssembleDiagonal3D<3,5>(NE,B,D,Y);
-         case 0x45: return SmemPAMassAssembleDiagonal3D<4,5>(NE,B,D,Y);
-         case 0x48: return SmemPAMassAssembleDiagonal3D<4,8>(NE,B,D,Y);
-         case 0x56: return SmemPAMassAssembleDiagonal3D<5,6>(NE,B,D,Y);
-         case 0x67: return SmemPAMassAssembleDiagonal3D<6,7>(NE,B,D,Y);
-         case 0x78: return SmemPAMassAssembleDiagonal3D<7,8>(NE,B,D,Y);
-         case 0x89: return SmemPAMassAssembleDiagonal3D<8,9>(NE,B,D,Y);
-         default:   return PAMassAssembleDiagonal3D(NE,B,D,Y,D1D,Q1D);
-      }
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-
-void MassIntegrator::AssembleDiagonalPA(Vector &diag)
-{
-   if (DeviceCanUseCeed())
-   {
-      ceedOp->GetDiagonal(diag);
-   }
-   else
-   {
-      PAMassAssembleDiagonal(dim, dofs1D, quad1D, ne, maps->B, pa_data, diag);
-   }
-}
-
-
 #ifdef MFEM_USE_OCCA
 // OCCA PA Mass Apply 2D kernel
 static void OccaPAMassApply2D(const int D1D,
@@ -1151,6 +864,159 @@ static void SmemPAMassApply3D(const int NE,
   });
 }

+template<int D1D, int Q1D>
+void SmemPAMassApply3D_v1(const int NE,
+                          const Array<double> &b_,
+                          const Array<double> &bt_,
+                          const Vector &d_,
+                          const Vector &x_,
+                          Vector &y_,
+                          const int d1d = 0,
+                          const int q1d = 0)
+{
+   MFEM_CONTRACT_VAR(bt_);
+   MFEM_CONTRACT_VAR(d1d);
+   MFEM_CONTRACT_VAR(q1d);
+   const auto B = Reshape(b_.Read(), Q1D,D1D);
+   const auto D = Reshape(d_.Read(), Q1D,Q1D,Q1D, NE);
+   const auto X = Reshape(x_.Read(), D1D,D1D,D1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), D1D,D1D,D1D, NE);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
+   {
+      double u[Q1D];
+      MFEM_SHARED double s_B[Q1D][D1D];
+      MFEM_SHARED double s_q[Q1D][Q1D][Q1D];
+
+      // Load input, B & X interpolation
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            s_B[qx][dy] = B(qx,dy);
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
+            MFEM_UNROLL(D1D)
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const double Bx = B(qx,dx);
+               MFEM_UNROLL(D1D)
+               for (int dz = 0; dz < D1D; ++dz)
+               {
+                  u[dz] += X(dx,dy,dz,e) * Bx;
+               }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { s_q[dz][dy][qx] = u[dz]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Y interpolation
+      MFEM_FOREACH_THREAD(dz,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            MFEM_UNROLL(Q1D)
+            for (int qy = 0; qy < Q1D; ++qy) { u[qy] = 0.0; }
+            MFEM_UNROLL(D1D)
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               const double zyX = s_q[dz][dy][qx];
+               MFEM_UNROLL(D1D)
+               for (int qy = 0; qy < Q1D; ++qy) { u[qy] += zyX * s_B[qy][dy]; }
+            }
+            MFEM_UNROLL(Q1D)
+            for (int qy = 0; qy < Q1D; ++qy) { s_q[dz][qy][qx] = u[qy]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Z interpolation, Q-function & Zt projection
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            // Z interpolation
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz) { u[qz] = 0.0; }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               const double zYX = s_q[dz][qy][qx];
+               MFEM_UNROLL(Q1D)
+               for (int qz = 0; qz < Q1D; ++qz) { u[qz] += zYX * s_B[qz][dz]; }
+            }
+
+            // Q-function
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               s_q[qz][qy][qx] = u[qz] * D(qx,qy,qz,e);
+            }
+
+            // Zt projection
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               const double ZYX = s_q[qz][qy][qx];
+               MFEM_UNROLL(D1D)
+               for (int dz = 0; dz < D1D; ++dz) { u[dz] += ZYX * s_B[qz][dz]; }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { s_q[dz][qy][qx] = u[dz]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Yt projection
+      MFEM_FOREACH_THREAD(dz,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int dy = 0; dy < D1D; ++dy) { u[dy] = 0.0; }
+            MFEM_UNROLL(Q1D)
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const double zYX = s_q[dz][qy][qx];
+               MFEM_UNROLL(D1D)
+               for (int dy = 0; dy < D1D; ++dy) { u[dy] += zYX * s_B[qy][dy]; }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dy = 0; dy < D1D; ++dy) { s_q[dz][dy][qx] = u[dy]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Xt projection & save output
+      MFEM_FOREACH_THREAD(dz,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dy,x,D1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int dx = 0; dx < D1D; ++dx) { u[dx] = 0.0; }
+            MFEM_UNROLL(Q1D)
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const double zyX = s_q[dz][dy][qx];
+               MFEM_UNROLL(D1D)
+               for (int dx = 0; dx < D1D; ++dx) { u[dx] += zyX * s_B[qx][dx]; }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const double output = u[dx];
+               Y(dx,dy,dz,e) += output;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
 static void PAMassApply(const int dim,
                        const int D1D,
                        const int Q1D,
@@ -1201,30 +1067,47 @@ static void PAMassApply(const int dim,
   }
   else if (dim == 3)
   {
+      const int ver = Device::KernelsVersion();
+      const int id = (ver << 8) | (D1D << 4) | Q1D;
+
+      static int ini = 0;
+      if (!ini++) { printf("\033[33mkernel #0x%x\033[m\n",id); }
+
      switch (id)
      {
-         case 0x22: return SmemPAMassApply3D<2,2>(NE,B,Bt,D,X,Y);
-         case 0x23: return SmemPAMassApply3D<2,3>(NE,B,Bt,D,X,Y);
-         case 0x24: return SmemPAMassApply3D<2,4>(NE,B,Bt,D,X,Y);
-         case 0x26: return SmemPAMassApply3D<2,6>(NE,B,Bt,D,X,Y);
-         case 0x34: return SmemPAMassApply3D<3,4>(NE,B,Bt,D,X,Y);
-         case 0x35: return SmemPAMassApply3D<3,5>(NE,B,Bt,D,X,Y);
-         case 0x36: return SmemPAMassApply3D<3,6>(NE,B,Bt,D,X,Y);
-         case 0x37: return SmemPAMassApply3D<3,7>(NE,B,Bt,D,X,Y);
-         case 0x45: return SmemPAMassApply3D<4,5>(NE,B,Bt,D,X,Y);
-         case 0x46: return SmemPAMassApply3D<4,6>(NE,B,Bt,D,X,Y);
-         case 0x48: return SmemPAMassApply3D<4,8>(NE,B,Bt,D,X,Y);
-         case 0x56: return SmemPAMassApply3D<5,6>(NE,B,Bt,D,X,Y);
-         case 0x58: return SmemPAMassApply3D<5,8>(NE,B,Bt,D,X,Y);
-         case 0x67: return SmemPAMassApply3D<6,7>(NE,B,Bt,D,X,Y);
-         case 0x78: return SmemPAMassApply3D<7,8>(NE,B,Bt,D,X,Y);
-         case 0x89: return SmemPAMassApply3D<8,9>(NE,B,Bt,D,X,Y);
-         case 0x9A: return SmemPAMassApply3D<9,10>(NE,B,Bt,D,X,Y);
-         default:   return PAMassApply3D(NE,B,Bt,D,X,Y,D1D,Q1D);
+         case 0x123: return SmemPAMassApply3D_v1<2,3>(NE,B,Bt,D,X,Y);
+         case 0x124: return SmemPAMassApply3D_v1<2,4>(NE,B,Bt,D,X,Y);
+         case 0x134: return SmemPAMassApply3D_v1<3,4>(NE,B,Bt,D,X,Y);
+         case 0x136: return SmemPAMassApply3D_v1<3,6>(NE,B,Bt,D,X,Y);
+         case 0x145: return SmemPAMassApply3D_v1<4,5>(NE,B,Bt,D,X,Y);
+         case 0x148: return SmemPAMassApply3D_v1<4,8>(NE,B,Bt,D,X,Y);
+         case 0x156: return SmemPAMassApply3D_v1<5,6>(NE,B,Bt,D,X,Y);
+         case 0x158: return SmemPAMassApply3D_v1<5,8>(NE,B,Bt,D,X,Y);
+         case 0x167: return SmemPAMassApply3D_v1<6,7>(NE,B,Bt,D,X,Y);
+         case 0x178: return SmemPAMassApply3D_v1<7,8>(NE,B,Bt,D,X,Y);
+
+         case 0x022: return SmemPAMassApply3D<2,2>(NE,B,Bt,D,X,Y);
+         case 0x023: return SmemPAMassApply3D<2,3>(NE,B,Bt,D,X,Y);
+         case 0x024: return SmemPAMassApply3D<2,4>(NE,B,Bt,D,X,Y);
+         case 0x026: return SmemPAMassApply3D<2,6>(NE,B,Bt,D,X,Y);
+         case 0x034: return SmemPAMassApply3D<3,4>(NE,B,Bt,D,X,Y);
+         case 0x035: return SmemPAMassApply3D<3,5>(NE,B,Bt,D,X,Y);
+         case 0x036: return SmemPAMassApply3D<3,6>(NE,B,Bt,D,X,Y);
+         case 0x037: return SmemPAMassApply3D<3,7>(NE,B,Bt,D,X,Y);
+         case 0x045: return SmemPAMassApply3D<4,5>(NE,B,Bt,D,X,Y);
+         case 0x046: return SmemPAMassApply3D<4,6>(NE,B,Bt,D,X,Y);
+         case 0x048: return SmemPAMassApply3D<4,8>(NE,B,Bt,D,X,Y);
+         case 0x056: return SmemPAMassApply3D<5,6>(NE,B,Bt,D,X,Y);
+         case 0x058: return SmemPAMassApply3D<5,8>(NE,B,Bt,D,X,Y);
+         case 0x067: return SmemPAMassApply3D<6,7>(NE,B,Bt,D,X,Y);
+         case 0x078: return SmemPAMassApply3D<7,8>(NE,B,Bt,D,X,Y);
+         case 0x089: return SmemPAMassApply3D<8,9>(NE,B,Bt,D,X,Y);
+         case 0x09A: return SmemPAMassApply3D<9,10>(NE,B,Bt,D,X,Y);
+
+         default:   break; //return PAMassApply3D(NE,B,Bt,D,X,Y,D1D,Q1D);
      }
   }
-   mfem::out << "Unknown kernel 0x" << std::hex << id << std::endl;
-   MFEM_ABORT("Unknown kernel.");
+   MFEM_ABORT("Unknown kernel 0x" << std::hex << id);
 }

 void MassIntegrator::AddMultPA(const Vector &x, Vector &y) const
@@ -1233,6 +1116,35 @@ void MassIntegrator::AddMultPA(const Vector &x, Vector &y) const
   {
      ceedOp->AddMult(x, y);
   }
+   else if (Device::FastKernelsEnabled())
+   {
+      const int version = Device::KernelsVersion();
+      MFEM_VERIFY(version < 4 || version==7, "Unsupported version!");
+      if (version == 3) // AMD
+      {
+         NDK_AMD_PAMassApply(dim, dofs1D, quad1D, ne,
+                             fespace, maps,
+                             pa_data, x, y);
+      }
+      // 4 E-vector
+      // 5 fused
+      // 6 MMA
+      else if (version == 7) // HIP
+      {
+         NDK_HIP_PAMassApply(dim, dofs1D, quad1D, ne,
+                             fespace, maps,
+                             pa_data, x, y);
+      }
+      // 0 legacy
+      // 1 fast
+      // 2 libP
+      else
+      {
+         NDK_PAMassApply(dim, dofs1D, quad1D, ne,
+                         fespace, maps,
+                         pa_data, x, y);
+      }
+   }
   else
   {
      PAMassApply(dim, dofs1D, quad1D, ne, maps->B, maps->Bt, pa_data, x, y);
@@ -0,0 +1,358 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../general/forall.hpp"
+#include "bilininteg.hpp"
+#include "gridfunc.hpp"
+#include "ceed/mass.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+void NDK_PAMassAssembleDiagonal(const int dim,
+                                const int D1D,
+                                const int Q1D,
+                                const int NE,
+                                const FiniteElementSpace *fes,
+                                const DofToQuad *maps,
+                                const Vector &D,
+                                Vector &Y);
+
+template<int T_D1D = 0, int T_Q1D = 0>
+static void PAMassAssembleDiagonal2D(const int NE,
+                                     const Array<double> &b,
+                                     const Vector &d,
+                                     Vector &y,
+                                     const int d1d = 0,
+                                     const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto D = Reshape(d.Read(), Q1D, Q1D, NE);
+   auto Y = Reshape(y.ReadWrite(), D1D, D1D, NE);
+   MFEM_FORALL(e, NE,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      double QD[MQ1][MD1];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            QD[qx][dy] = 0.0;
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               QD[qx][dy] += B(qy, dy) * B(qy, dy) * D(qx, qy, e);
+            }
+         }
+      }
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               Y(dx,dy,e) += B(qx, dx) * B(qx, dx) * QD[qx][dy];
+            }
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
+static void SmemPAMassAssembleDiagonal2D(const int NE,
+                                         const Array<double> &b_,
+                                         const Vector &d_,
+                                         Vector &y_,
+                                         const int d1d = 0,
+                                         const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
+   auto b = Reshape(b_.Read(), Q1D, D1D);
+   auto D = Reshape(d_.Read(), Q1D, Q1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
+   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
+   {
+      const int tidz = MFEM_THREAD_ID(z);
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      MFEM_SHARED double B[MQ1][MD1];
+      MFEM_SHARED double QDZ[NBZ][MQ1][MD1];
+      double (*QD)[MD1] = (double (*)[MD1])(QDZ + tidz);
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B[q][d] = b(q,d);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            QD[qx][dy] = 0.0;
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               QD[qx][dy] += B[qy][dy] * B[qy][dy] * D(qx, qy, e);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               // might need absolute values on next line
+               Y(dx,dy,e) += B[qx][dx] * B[qx][dx] * QD[qx][dy];
+            }
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0>
+static void PAMassAssembleDiagonal3D(const int NE,
+                                     const Array<double> &b,
+                                     const Vector &d,
+                                     Vector &y,
+                                     const int d1d = 0,
+                                     const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto D = Reshape(d.Read(), Q1D, Q1D, Q1D, NE);
+   auto Y = Reshape(y.ReadWrite(), D1D, D1D, D1D, NE);
+   MFEM_FORALL(e, NE,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      double QQD[MQ1][MQ1][MD1];
+      double QDD[MQ1][MD1][MD1];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               QQD[qx][qy][dz] = 0.0;
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  QQD[qx][qy][dz] += B(qz, dz) * B(qz, dz) * D(qx, qy, qz, e);
+               }
+            }
+         }
+      }
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               QDD[qx][dy][dz] = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  QDD[qx][dy][dz] += B(qy, dy) * B(qy, dy) * QQD[qx][qy][dz];
+               }
+            }
+         }
+      }
+      for (int dz = 0; dz < D1D; ++dz)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               double t = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  t += B(qx, dx) * B(qx, dx) * QDD[qx][dy][dz];
+               }
+               Y(dx, dy, dz, e) += t;
+            }
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0>
+static void SmemPAMassAssembleDiagonal3D(const int NE,
+                                         const Array<double> &b_,
+                                         const Vector &d_,
+                                         Vector &y_,
+                                         const int d1d = 0,
+                                         const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
+   const auto b = Reshape(b_.Read(), Q1D, D1D);
+   const auto D = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      const int tidz = MFEM_THREAD_ID(z);
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      MFEM_SHARED double B[MQ1][MD1];
+      MFEM_SHARED double QQD[MQ1][MQ1][MD1];
+      MFEM_SHARED double QDD[MQ1][MD1][MD1];
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B[q][d] = b(q,d);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(dz,z,D1D)
+            {
+               QQD[qx][qy][dz] = 0.0;
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  QQD[qx][qy][dz] += B[qz][dz] * B[qz][dz] * D(qx, qy, qz, e);
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               QDD[qx][dy][dz] = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  QDD[qx][dy][dz] += B[qy][dy] * B[qy][dy] * QQD[qx][qy][dz];
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dz,z,D1D)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               double t = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  t += B[qx][dx] * B[qx][dx] * QDD[qx][dy][dz];
+               }
+               Y(dx, dy, dz, e) += t;
+            }
+         }
+      }
+   });
+}
+
+static void PAMassAssembleDiagonal(const int dim, const int D1D,
+                                   const int Q1D, const int NE,
+                                   const Array<double> &B,
+                                   const Vector &D,
+                                   Vector &Y)
+{
+   if (dim == 2)
+   {
+      switch ((D1D << 4 ) | Q1D)
+      {
+         case 0x22: return SmemPAMassAssembleDiagonal2D<2,2,16>(NE,B,D,Y);
+         case 0x33: return SmemPAMassAssembleDiagonal2D<3,3,16>(NE,B,D,Y);
+         case 0x44: return SmemPAMassAssembleDiagonal2D<4,4,8>(NE,B,D,Y);
+         case 0x55: return SmemPAMassAssembleDiagonal2D<5,5,8>(NE,B,D,Y);
+         case 0x66: return SmemPAMassAssembleDiagonal2D<6,6,4>(NE,B,D,Y);
+         case 0x77: return SmemPAMassAssembleDiagonal2D<7,7,4>(NE,B,D,Y);
+         case 0x88: return SmemPAMassAssembleDiagonal2D<8,8,2>(NE,B,D,Y);
+         case 0x99: return SmemPAMassAssembleDiagonal2D<9,9,2>(NE,B,D,Y);
+         default:   return PAMassAssembleDiagonal2D(NE,B,D,Y,D1D,Q1D);
+      }
+   }
+   else if (dim == 3)
+   {
+      switch ((D1D << 4 ) | Q1D)
+      {
+         case 0x23: return SmemPAMassAssembleDiagonal3D<2,3>(NE,B,D,Y);
+         case 0x24: return SmemPAMassAssembleDiagonal3D<2,4>(NE,B,D,Y);
+         case 0x26: return SmemPAMassAssembleDiagonal3D<2,6>(NE,B,D,Y);
+         case 0x34: return SmemPAMassAssembleDiagonal3D<3,4>(NE,B,D,Y);
+         case 0x35: return SmemPAMassAssembleDiagonal3D<3,5>(NE,B,D,Y);
+         case 0x45: return SmemPAMassAssembleDiagonal3D<4,5>(NE,B,D,Y);
+         case 0x48: return SmemPAMassAssembleDiagonal3D<4,8>(NE,B,D,Y);
+         case 0x56: return SmemPAMassAssembleDiagonal3D<5,6>(NE,B,D,Y);
+         case 0x67: return SmemPAMassAssembleDiagonal3D<6,7>(NE,B,D,Y);
+         case 0x78: return SmemPAMassAssembleDiagonal3D<7,8>(NE,B,D,Y);
+         case 0x89: return SmemPAMassAssembleDiagonal3D<8,9>(NE,B,D,Y);
+         default:   return PAMassAssembleDiagonal3D(NE,B,D,Y,D1D,Q1D);
+      }
+   }
+   MFEM_ABORT("Unknown kernel.");
+}
+
+void MassIntegrator::AssembleDiagonalPA(Vector &diag)
+{
+   if (DeviceCanUseCeed())
+   {
+      ceedOp->GetDiagonal(diag);
+   }
+   else if (Device::FastKernelsEnabled())
+   {
+      NDK_PAMassAssembleDiagonal(dim, dofs1D, quad1D, ne,
+                                 fespace, maps,
+                                 pa_data, diag);
+   }
+   else
+   {
+      PAMassAssembleDiagonal(dim, dofs1D, quad1D, ne, maps->B, pa_data, diag);
+   }
+}
+
+} // namespace mfem
@@ -0,0 +1,684 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../general/forall.hpp"
+#include "gridfunc.hpp"
+#include "restriction.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+// Fast '0' non-deterministic 3D mass kernel
+template<int D1D, int Q1D>
+void NDK_SmemPAMassApply3D(const int ndofs,
+                           const int NE,
+                           const int *map,
+                           const double *b_,
+                           const double *d_,
+                           const double *x_,
+                           double *y_)
+{
+   const auto MAP = Reshape(map, D1D,D1D,D1D, NE);
+   const auto b = Reshape(b_, Q1D, D1D);
+   const auto D = Reshape(d_, Q1D, Q1D, Q1D, NE);
+   const auto X = Reshape(x_, ndofs);
+   const auto X1 = Reshape(x_, D1D,D1D,D1D, NE);
+   auto Y = Reshape(y_, ndofs);
+   auto Y1 = Reshape(y_, D1D,D1D,D1D, NE);
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
+   {
+      MFEM_SHARED double sDQ[Q1D*Q1D];
+      double (*B)[D1D] = (double (*)[D1D]) sDQ;
+      double (*Bt)[Q1D] = (double (*)[Q1D]) sDQ;
+      MFEM_SHARED double sm0[Q1D*Q1D*Q1D];
+      MFEM_SHARED double sm1[Q1D*Q1D*Q1D];
+      double (*DDQ)[D1D][Q1D] = (double (*)[D1D][Q1D]) sm1;
+      double (*DQQ)[Q1D][Q1D] = (double (*)[Q1D][Q1D]) sm0;
+      double (*QQQ)[Q1D][Q1D] = (double (*)[Q1D][Q1D]) sm1;
+      double (*QQD)[Q1D][D1D] = (double (*)[Q1D][D1D]) sm0;
+      double (*QDD)[D1D][D1D] = (double (*)[D1D][D1D]) sm1;
+
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            B[qx][dy] = b(qx,dy);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            double u[D1D];
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; dz++)
+            {
+               u[dz] = 0;
+            }
+            MFEM_UNROLL(D1D)
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               MFEM_UNROLL(D1D)
+               for (int dz = 0; dz < D1D; ++dz)
+               {
+                  if (map)
+                  {
+                     const int gid = MAP(dx, dy, dz, e);
+                     const int idx = gid >= 0 ? gid : -1 - gid;
+                     u[dz] += X(idx) * B[qx][dx];
+                  }
+                  else
+                  {
+                     u[dz] += X1(dx,dy,dz,e) * B[qx][dx];
+                  }
+               }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               DDQ[dz][dy][qx] = u[dz];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            double u[D1D];
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; dz++)
+            {
+               u[dz] = 0;
+            }
+            MFEM_UNROLL(D1D)
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               MFEM_UNROLL(D1D)
+               for (int dz = 0; dz < D1D; dz++)
+               {
+                  u[dz] += DDQ[dz][dy][qx] * B[qy][dy];
+               }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; dz++)
+            {
+               DQQ[dz][qy][qx] = u[dz];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            double u[Q1D];
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; qz++)
+            {
+               u[qz] = 0;
+            }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               MFEM_UNROLL(Q1D)
+               for (int qz = 0; qz < Q1D; qz++)
+               {
+                  u[qz] += DQQ[dz][qy][qx] * B[qz][dz];
+               }
+            }
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; qz++)
+            {
+               QQQ[qz][qy][qx] = u[qz] * D(qx,qy,qz,e);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(d,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(q,x,Q1D)
+         {
+            Bt[d][q] = b(q,d);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u[Q1D];
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               u[qz] = 0;
+            }
+            MFEM_UNROLL(Q1D)
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               MFEM_UNROLL(Q1D)
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  u[qz] += QQQ[qz][qy][qx] * Bt[dx][qx];
+               }
+            }
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               QQD[qz][qy][dx] = u[qz];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u[Q1D];
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               u[qz] = 0;
+            }
+            MFEM_UNROLL(Q1D)
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               MFEM_UNROLL(Q1D)
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  u[qz] += QQD[qz][qy][dx] * Bt[dy][qy];
+               }
+            }
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               QDD[qz][dy][dx] = u[qz];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u[D1D];
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               u[dz] = 0;
+            }
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               MFEM_UNROLL(D1D)
+               for (int dz = 0; dz < D1D; ++dz)
+               {
+                  u[dz] += QDD[qz][dy][dx] * Bt[dz][qz];
+               }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               if (map)
+               {
+                  const int gid = MAP(dx, dy, dz, e);
+                  const int idx = gid >= 0 ? gid : -1 - gid;
+                  AtomicAdd(Y(idx), u[dz]);
+               }
+               else
+               {
+                  Y1(dx,dy,dz,e) += u[dz];
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+// Fast '1' non-deterministic 3D mass kernel
+// Smem version melded toward registers + BZ-batch
+template<int D1D, int Q1D, int NBZ>
+void NDK_SmRgPAMassApply3D(const int ndofs,
+                           const int NE,
+                           const int *map,
+                           const double *b_,
+                           const double *d_,
+                           const double *x_,
+                           double *y_)
+{
+   const auto MAP = Reshape(map, D1D,D1D,D1D, NE);
+   const auto B = Reshape(b_, Q1D, D1D);
+   const auto D = Reshape(d_, Q1D, Q1D, Q1D, NE);
+   const auto X = Reshape(x_, ndofs);
+   const auto X1 = Reshape(x_, D1D,D1D,D1D, NE);
+   auto Y = Reshape(y_, ndofs);
+   auto Y1 = Reshape(y_, D1D,D1D,D1D, NE);
+
+   MFEM_FORALL_3D(be, (NE+NBZ-1)/NBZ, Q1D, Q1D, NBZ,
+   {
+      double u[Q1D];
+      const int tz = MFEM_THREAD_ID(z);
+      const int e = be * MFEM_THREAD_SIZE(z) + tz;
+
+      MFEM_SHARED double s_B[Q1D][D1D];
+      MFEM_SHARED double s_q[NBZ][Q1D][Q1D][Q1D];
+
+      // Load input, B & X interpolation
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            if (tz == 0) { s_B[qx][dy] = B(qx,dy); }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
+            MFEM_UNROLL(D1D)
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const double Bx = B(qx,dx);
+               MFEM_UNROLL(D1D)
+               for (int dz = 0; dz < D1D; ++dz)
+               {
+                  const int gid = map ? MAP(dx,dy,dz,e) : 0;
+                  const int idx = gid >= 0 ? gid : -1 - gid;
+                  u[dz] += (map ? X(idx) : X1(dx,dy,dz,e)) * Bx;
+               }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { s_q[tz][dz][dy][qx] = u[dz]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Y interpolation
+      MFEM_FOREACH_THREAD(dz,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            MFEM_UNROLL(Q1D)
+            for (int qy = 0; qy < Q1D; ++qy) { u[qy] = 0.0; }
+            MFEM_UNROLL(D1D)
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               const double zyX = s_q[tz][dz][dy][qx];
+               MFEM_UNROLL(D1D)
+               for (int qy = 0; qy < Q1D; ++qy) { u[qy] += zyX * s_B[qy][dy]; }
+            }
+            MFEM_UNROLL(Q1D)
+            for (int qy = 0; qy < Q1D; ++qy) { s_q[tz][dz][qy][qx] = u[qy]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Z interpolation, Q-function & Zt projection
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            // Z interpolation
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz) { u[qz] = 0.0; }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               const double zYX = s_q[tz][dz][qy][qx];
+               MFEM_UNROLL(Q1D)
+               for (int qz = 0; qz < Q1D; ++qz) { u[qz] += zYX * s_B[qz][dz]; }
+            }
+
+            // Q-function
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               s_q[tz][qz][qy][qx] = u[qz] * D(qx,qy,qz,e);
+            }
+
+            // Zt projection
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               const double ZYX = s_q[tz][qz][qy][qx];
+               MFEM_UNROLL(D1D)
+               for (int dz = 0; dz < D1D; ++dz) { u[dz] += ZYX * s_B[qz][dz]; }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { s_q[tz][dz][qy][qx] = u[dz]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Yt projection
+      MFEM_FOREACH_THREAD(dz,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int dy = 0; dy < D1D; ++dy) { u[dy] = 0.0; }
+            MFEM_UNROLL(Q1D)
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const double zYX = s_q[tz][dz][qy][qx];
+               MFEM_UNROLL(D1D)
+               for (int dy = 0; dy < D1D; ++dy) { u[dy] += zYX * s_B[qy][dy]; }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dy = 0; dy < D1D; ++dy) { s_q[tz][dz][dy][qx] = u[dy]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Xt projection & save output
+      MFEM_FOREACH_THREAD(dz,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dy,x,D1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int dx = 0; dx < D1D; ++dx) { u[dx] = 0.0; }
+            MFEM_UNROLL(Q1D)
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const double zyX = s_q[tz][dz][dy][qx];
+               MFEM_UNROLL(D1D)
+               for (int dx = 0; dx < D1D; ++dx) { u[dx] += zyX * s_B[qx][dx]; }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const double output = u[dx];
+               if (map)
+               {
+                  const int gid = MAP(dx,dy,dz,e);
+                  const int idx = gid >= 0 ? gid : -1 - gid;
+                  AtomicAdd(Y(idx), output);
+               }
+               else
+               {
+                  Y1(dx,dy,dz,e) += output;
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+// Fast '2' non-deterministic 3D mass kernel
+// Smem version with registers
+template<int D1D, int Q1D>
+void NDK_RegsPAMassApply3D(const int ndofs,
+                           const int NE,
+                           const int *map,
+                           const double *b_,
+                           const double *d_,
+                           const double *x_,
+                           double *y_)
+{
+   const auto MAP = Reshape(map, D1D,D1D,D1D, NE);
+   const auto B = Reshape(b_, Q1D,D1D);
+   const auto D = Reshape(d_, Q1D,Q1D,Q1D, NE);
+   const auto X = Reshape(x_, ndofs);
+   const auto X1 = Reshape(x_, D1D,D1D,D1D, NE);
+   auto Y = Reshape(y_, ndofs);
+   auto Y1 = Reshape(y_, D1D,D1D,D1D, NE);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
+   {
+      double r_wk[Q1D];
+      MFEM_SHARED double s_B[Q1D][D1D];
+      MFEM_SHARED double s_q[Q1D][Q1D][Q1D];
+
+      // Load s_B, load X in shared memory
+      MFEM_FOREACH_THREAD(b,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(a,x,Q1D)
+         {
+            if (a<D1D) { s_B[b][a] = B(b,a); }
+
+            MFEM_UNROLL(Q1D)
+            for (int i=0; i<Q1D; ++i) { r_wk[i] = 0.0; }
+
+            if (a<D1D && b<D1D)
+            {
+               MFEM_UNROLL(D1D)
+               for (int c=0; c<D1D; ++c)
+               {
+                  const int gid = map ? MAP(a,b,c,e) : 0;
+                  const int idx = gid >= 0 ? gid : -1 - gid;
+                  s_q[c][b][a] = map ? X(idx) : X1(a,b,c,e);
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Interpolate in X direction
+      MFEM_FOREACH_THREAD(c,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(b,x,Q1D)
+         {
+            if (b<D1D && c<D1D)
+            {
+               MFEM_UNROLL(D1D)
+               for (int a=0; a<D1D; ++a)
+               {
+                  const double q_cba = s_q[c][b][a];
+                  MFEM_UNROLL(Q1D)
+                  for (int i=0; i<Q1D; ++i) { r_wk[i] += s_B[i][a]*q_cba; }
+               }
+               // reg => s_mem
+               MFEM_UNROLL(Q1D)
+               for (int i=0; i<Q1D; ++i) { s_q[c][b][i] = r_wk[i]; }
+            }
+            MFEM_UNROLL(Q1D)
+            for (int j=0; j<Q1D; ++j) { r_wk[j] = 0.0; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Interpolate in Y direction
+      MFEM_FOREACH_THREAD(c,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(i,x,Q1D)
+         {
+            if (c<D1D)
+            {
+               MFEM_UNROLL(D1D)
+               for (int b=0; b<D1D; ++b)
+               {
+                  const double q_cbi = s_q[c][b][i];
+                  MFEM_UNROLL(Q1D)
+                  for (int j=0; j<Q1D; ++j) { r_wk[j] += s_B[j][b]*q_cbi; }
+               }
+               MFEM_UNROLL(Q1D)
+               for (int j=0; j<Q1D; ++j) { s_q[c][j][i] = r_wk[j]; }
+            }
+            MFEM_UNROLL(Q1D)
+            for (int k=0; k<Q1D; ++k) { r_wk[k] = 0.0; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(j,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(i,x,Q1D)
+         {
+            // Interpolate in Z direction
+            MFEM_UNROLL(D1D)
+            for (int c=0; c<D1D; ++c)
+            {
+               const double q_cji = s_q[c][j][i];
+               MFEM_UNROLL(Q1D)
+               for (int k=0; k<Q1D; ++k) { r_wk[k] += s_B[k][c]*q_cji; }
+            }
+
+            // Scale by Jacobian and integration weights
+            MFEM_UNROLL(Q1D)
+            for (int k=0; k<Q1D; ++k) { r_wk[k] *= D(i,j,k,e); }
+
+            // Project back in Z direction
+            MFEM_UNROLL(D1D)
+            for (int c=0; c<D1D; ++c)
+            {
+               double q_cji = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int k=0; k<Q1D; ++k) { q_cji += s_B[k][c] * r_wk[k]; }
+               s_q[c][j][i] = q_cji;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Project back in Y direction
+      MFEM_FOREACH_THREAD(c,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(i,x,Q1D)
+         {
+            MFEM_UNROLL(Q1D)
+            for (int j=0; j<Q1D; ++j) { r_wk[j] = s_q[c][j][i]; }
+            MFEM_UNROLL(D1D)
+            for (int b=0; b<D1D; ++b)
+            {
+               double q_cbi = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int j=0; j<Q1D; ++j) { q_cbi += s_B[j][b] * r_wk[j]; }
+               s_q[c][b][i] = q_cbi;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Project back in X direction
+      MFEM_FOREACH_THREAD(c,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(b,x,D1D)
+         {
+            MFEM_UNROLL(Q1D)
+            for (int i=0; i<Q1D; ++i) { r_wk[i] = s_q[c][b][i]; }
+            MFEM_UNROLL(D1D)
+            for (int a=0; a<D1D; ++a)
+            {
+               double q_cba = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int i=0; i<Q1D; ++i) { q_cba += s_B[i][a] * r_wk[i]; }
+               s_q[c][b][a] = q_cba;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Save back to memory
+      MFEM_FOREACH_THREAD(b,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(a,x,D1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int c=0; c<D1D; ++c)
+            {
+               const double q_cba = s_q[c][b][a];
+               const int gid = map ? MAP(a,b,c,e) : 0;
+               const int idx = gid >= 0 ? gid : -1 - gid;
+               AtomicAdd(map?Y(idx):Y1(a,b,c,e), q_cba);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+void NDK_PAMassApply(const int dim,
+                     const int D1D,
+                     const int Q1D,
+                     const int NE,
+                     const FiniteElementSpace *fes,
+                     const DofToQuad *maps,
+                     const Vector &D,
+                     const Vector &X,
+                     Vector &Y)
+{
+   const int ND = fes->GetNDofs();
+   constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+   const Operator *ERop = fes->GetElementRestriction(ordering);
+   const ElementRestriction* ER = dynamic_cast<const ElementRestriction*>(ERop);
+   const int *map = ER ? ER->GatherMap().Read() : nullptr;
+   const double *b = maps->B.Read();
+   const double *d = D.Read();
+   const double *x = X.Read();
+   double *y = Y.ReadWrite();
+
+   assert(dim == 3);
+   const int ver = Device::KernelsVersion();
+   const int id = (ver << 8) | (D1D << 4) | Q1D;
+
+   //printf("\033[32mkernel #0x%x\033[m\n",id); fflush(0);
+
+   switch (id) // orders 1~6
+   {
+      // Fast '0': Legacy non-deterministic 3D mass kernel
+      case 0x023: return NDK_SmemPAMassApply3D<2,3>(ND,NE,map,b,d,x,y);
+      case 0x024: return NDK_SmemPAMassApply3D<2,4>(ND,NE,map,b,d,x,y);
+      case 0x034: return NDK_SmemPAMassApply3D<3,4>(ND,NE,map,b,d,x,y);
+      case 0x036: return NDK_SmemPAMassApply3D<3,6>(ND,NE,map,b,d,x,y);
+      case 0x045: return NDK_SmemPAMassApply3D<4,5>(ND,NE,map,b,d,x,y);
+      case 0x046: return NDK_SmemPAMassApply3D<4,6>(ND,NE,map,b,d,x,y);
+      case 0x048: return NDK_SmemPAMassApply3D<4,8>(ND,NE,map,b,d,x,y);
+      case 0x056: return NDK_SmemPAMassApply3D<5,6>(ND,NE,map,b,d,x,y);
+      case 0x058: return NDK_SmemPAMassApply3D<5,8>(ND,NE,map,b,d,x,y);
+      case 0x067: return NDK_SmemPAMassApply3D<6,7>(ND,NE,map,b,d,x,y);
+      case 0x078: return NDK_SmemPAMassApply3D<7,8>(ND,NE,map,b,d,x,y);
+
+      // Fast '1': Legacy & half smem non-deterministic 3D mass kernel + Z-batch
+      case 0x123: return NDK_SmRgPAMassApply3D<2,3,32>(ND,NE,map,b,d,x,y);//1
+      case 0x124: return NDK_SmRgPAMassApply3D<2,4,16>(ND,NE,map,b,d,x,y);
+      case 0x134: return NDK_SmRgPAMassApply3D<3,4,16>(ND,NE,map,b,d,x,y);//2
+      case 0x136: return NDK_SmRgPAMassApply3D<3,6,8>(ND,NE,map,b,d,x,y);
+      case 0x145: return NDK_SmRgPAMassApply3D<4,5,4>(ND,NE,map,b,d,x,y);//3
+      case 0x146: return NDK_SmRgPAMassApply3D<4,6,4>(ND,NE,map,b,d,x,y);
+      case 0x148: return NDK_SmRgPAMassApply3D<4,8,4>(ND,NE,map,b,d,x,y);
+      case 0x156: return NDK_SmRgPAMassApply3D<5,6,4>(ND,NE,map,b,d,x,y);//4
+      case 0x158: return NDK_SmRgPAMassApply3D<5,8,1>(ND,NE,map,b,d,x,y);
+      case 0x167: return NDK_SmRgPAMassApply3D<6,7,1>(ND,NE,map,b,d,x,y);//5
+      case 0x178: return NDK_SmRgPAMassApply3D<7,8,1>(ND,NE,map,b,d,x,y);//6
+
+      // Fast '2': libP non-deterministic 3D mass kernel
+      case 0x223: return NDK_RegsPAMassApply3D<2,3>(ND,NE,map,b,d,x,y);
+      case 0x224: return NDK_RegsPAMassApply3D<2,4>(ND,NE,map,b,d,x,y);
+      case 0x234: return NDK_RegsPAMassApply3D<3,4>(ND,NE,map,b,d,x,y);
+      case 0x236: return NDK_RegsPAMassApply3D<3,6>(ND,NE,map,b,d,x,y);
+      case 0x245: return NDK_RegsPAMassApply3D<4,5>(ND,NE,map,b,d,x,y);
+      case 0x246: return NDK_RegsPAMassApply3D<4,6>(ND,NE,map,b,d,x,y);
+      case 0x248: return NDK_RegsPAMassApply3D<4,8>(ND,NE,map,b,d,x,y);
+      case 0x256: return NDK_RegsPAMassApply3D<5,6>(ND,NE,map,b,d,x,y);
+      case 0x258: return NDK_RegsPAMassApply3D<5,8>(ND,NE,map,b,d,x,y);
+      case 0x267: return NDK_RegsPAMassApply3D<6,7>(ND,NE,map,b,d,x,y);
+      case 0x278: return NDK_RegsPAMassApply3D<7,8>(ND,NE,map,b,d,x,y);
+
+      // Fast '3': libP + AMD specific non-deterministic 3D mass kernel
+      // in bilininteg_mass_pa_fast_amd.cpp
+
+      default: break;
+   }
+
+   MFEM_ABORT("Unknown kernel 0x" << std::hex << id);
+}
+
+} // namespace mfem
@@ -0,0 +1,250 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../general/forall.hpp"
+#include "gridfunc.hpp"
+#include "restriction.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+// Fast '3' non-deterministic 3D mass kernel
+template<int D1D, int Q1D>
+void NDK_AMD_PAMassApply3D(const int ndofs,
+                           const int NE,
+                           const int *map,
+                           const double *b_,
+                           const double *d_,
+                           const double *x_,
+                           double *y_)
+{
+   const auto MAP = Reshape(map, D1D,D1D,D1D, NE);
+   const auto B = Reshape(b_, Q1D,D1D);
+   const auto D = Reshape(d_, Q1D,Q1D,Q1D, NE);
+   const auto X = Reshape(x_, ndofs);
+   const auto X1 = Reshape(x_, D1D,D1D,D1D, NE);
+   auto Y = Reshape(y_, ndofs);
+   auto Y1 = Reshape(y_, D1D,D1D,D1D, NE);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
+   {
+      double r_wk[Q1D];
+      MFEM_SHARED double s_B[Q1D][D1D];
+      MFEM_SHARED double s_q[Q1D][Q1D][Q1D];
+
+      // Load s_B, load X in shared memory
+      MFEM_FOREACH_THREAD(b,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(a,x,Q1D)
+         {
+            if (a<D1D) { s_B[b][a] = B(b,a); }
+
+            MFEM_UNROLL(Q1D)
+            for (int i=0; i<Q1D; ++i) { r_wk[i] = 0.0; }
+
+            if (a<D1D && b<D1D)
+            {
+               MFEM_UNROLL(D1D)
+               for (int c=0; c<D1D; ++c)
+               {
+                  const int gid = map ? MAP(a,b,c,e) : 0;
+                  const int idx = gid >= 0 ? gid : -1 - gid;
+                  s_q[c][b][a] = map ? X(idx) : X1(a,b,c,e);
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Interpolate in X direction
+      MFEM_FOREACH_THREAD(c,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(b,x,Q1D)
+         {
+            if (b<D1D && c<D1D)
+            {
+               MFEM_UNROLL(D1D)
+               for (int a=0; a<D1D; ++a)
+               {
+                  const double q_cba = s_q[c][b][a];
+                  MFEM_UNROLL(Q1D)
+                  for (int i=0; i<Q1D; ++i) { r_wk[i] += s_B[i][a]*q_cba; }
+               }
+               // reg => s_mem
+               MFEM_UNROLL(Q1D)
+               for (int i=0; i<Q1D; ++i) { s_q[c][b][i] = r_wk[i]; }
+            }
+            MFEM_UNROLL(Q1D)
+            for (int j=0; j<Q1D; ++j) { r_wk[j] = 0.0; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Interpolate in Y direction
+      MFEM_FOREACH_THREAD(c,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(i,x,Q1D)
+         {
+            if (c<D1D)
+            {
+               MFEM_UNROLL(D1D)
+               for (int b=0; b<D1D; ++b)
+               {
+                  const double q_cbi = s_q[c][b][i];
+                  MFEM_UNROLL(Q1D)
+                  for (int j=0; j<Q1D; ++j) { r_wk[j] += s_B[j][b]*q_cbi; }
+               }
+               MFEM_UNROLL(Q1D)
+               for (int j=0; j<Q1D; ++j) { s_q[c][j][i] = r_wk[j]; }
+            }
+            MFEM_UNROLL(Q1D)
+            for (int k=0; k<Q1D; ++k) { r_wk[k] = 0.0; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(j,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(i,x,Q1D)
+         {
+            // Interpolate in Z direction
+            MFEM_UNROLL(D1D)
+            for (int c=0; c<D1D; ++c)
+            {
+               const double q_cji = s_q[c][j][i];
+               MFEM_UNROLL(Q1D)
+               for (int k=0; k<Q1D; ++k) { r_wk[k] += s_B[k][c]*q_cji; }
+            }
+
+            // Scale by Jacobian and integration weights
+            MFEM_UNROLL(Q1D)
+            for (int k=0; k<Q1D; ++k) { r_wk[k] *= D(i,j,k,e); }
+
+            // Project back in Z direction
+            MFEM_UNROLL(D1D)
+            for (int c=0; c<D1D; ++c)
+            {
+               double q_cji = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int k=0; k<Q1D; ++k) { q_cji += s_B[k][c] * r_wk[k]; }
+               s_q[c][j][i] = q_cji;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Project back in Y direction
+      MFEM_FOREACH_THREAD(c,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(i,x,Q1D)
+         {
+            MFEM_UNROLL(Q1D)
+            for (int j=0; j<Q1D; ++j) { r_wk[j] = s_q[c][j][i]; }
+            MFEM_UNROLL(D1D)
+            for (int b=0; b<D1D; ++b)
+            {
+               double q_cbi = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int j=0; j<Q1D; ++j) { q_cbi += s_B[j][b] * r_wk[j]; }
+               s_q[c][b][i] = q_cbi;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Project back in X direction
+      MFEM_FOREACH_THREAD(c,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(b,x,D1D)
+         {
+            MFEM_UNROLL(Q1D)
+            for (int i=0; i<Q1D; ++i) { r_wk[i] = s_q[c][b][i]; }
+            MFEM_UNROLL(D1D)
+            for (int a=0; a<D1D; ++a)
+            {
+               double q_cba = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int i=0; i<Q1D; ++i) { q_cba += s_B[i][a] * r_wk[i]; }
+               s_q[c][b][a] = q_cba;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Save back to memory
+      MFEM_FOREACH_THREAD(b,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(a,x,D1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int c=0; c<D1D; ++c)
+            {
+               const double q_cba = s_q[c][b][a];
+               const int gid = map ? MAP(a,b,c,e) : 0;
+               const int idx = gid >= 0 ? gid : -1 - gid;
+               AtomicAdd(map?Y(idx):Y1(a,b,c,e), q_cba);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+void NDK_AMD_PAMassApply(const int dim,
+                         const int D1D,
+                         const int Q1D,
+                         const int NE,
+                         const FiniteElementSpace *fes,
+                         const DofToQuad *maps,
+                         const Vector &D,
+                         const Vector &X,
+                         Vector &Y)
+{
+   const int ND = fes->GetNDofs();
+   constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+   const Operator *ERop = fes->GetElementRestriction(ordering);
+   const ElementRestriction* ER = dynamic_cast<const ElementRestriction*>(ERop);
+   const int *map = ER ? ER->GatherMap().Read() : nullptr;
+   const double *b = maps->B.Read();
+   const double *d = D.Read();
+   const double *x = X.Read();
+   double *y = Y.ReadWrite();
+
+   assert(dim == 3);
+   const int ver = Device::KernelsVersion();
+   const int id = (ver << 8) | (D1D << 4) | Q1D;
+
+   //printf("\033[32mkernel #0x%x\033[m\n",id); fflush(0);
+
+   switch (id) // orders 1~6
+   {
+      // Fast '3': libP + AMD specific non-deterministic 3D mass kernel
+      case 0x323: return NDK_AMD_PAMassApply3D<2,3>(ND,NE,map,b,d,x,y);
+      case 0x324: return NDK_AMD_PAMassApply3D<2,4>(ND,NE,map,b,d,x,y);
+      case 0x334: return NDK_AMD_PAMassApply3D<3,4>(ND,NE,map,b,d,x,y);
+      case 0x336: return NDK_AMD_PAMassApply3D<3,6>(ND,NE,map,b,d,x,y);
+      case 0x345: return NDK_AMD_PAMassApply3D<4,5>(ND,NE,map,b,d,x,y);
+      case 0x346: return NDK_AMD_PAMassApply3D<4,6>(ND,NE,map,b,d,x,y);
+      case 0x348: return NDK_AMD_PAMassApply3D<4,8>(ND,NE,map,b,d,x,y);
+      case 0x356: return NDK_AMD_PAMassApply3D<5,6>(ND,NE,map,b,d,x,y);
+      case 0x358: return NDK_AMD_PAMassApply3D<5,8>(ND,NE,map,b,d,x,y);
+      case 0x367: return NDK_AMD_PAMassApply3D<6,7>(ND,NE,map,b,d,x,y);
+      case 0x378: return NDK_AMD_PAMassApply3D<7,8>(ND,NE,map,b,d,x,y);
+
+      default: break;
+   }
+
+   MFEM_ABORT("Unknown kernel 0x" << std::hex << id);
+}
+
+} // namespace mfem
@@ -0,0 +1,269 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../general/forall.hpp"
+#include "gridfunc.hpp"
+#include "restriction.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+// Fast '0' non-deterministic 3D mass diag kernel
+template<int D1D, int Q1D>
+static void NDK_SmemPAMassDiag3D(const int ndofs,
+                                 const int NE,
+                                 const int *map_,
+                                 const double *b_,
+                                 const double *d_,
+                                 double *y_)
+{
+   const auto MAP = Reshape(map_, D1D,D1D,D1D, NE);
+   const auto b = Reshape(b_, Q1D, D1D);
+   const auto D = Reshape(d_, Q1D, Q1D, Q1D, NE);
+   auto Y = Reshape(y_, ndofs);
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      const int tidz = MFEM_THREAD_ID(z);
+      MFEM_SHARED double B[Q1D][D1D];
+      MFEM_SHARED double QQD[Q1D][Q1D][D1D];
+      MFEM_SHARED double QDD[Q1D][D1D][D1D];
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B[q][d] = b(q,d);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(dz,z,D1D)
+            {
+               QQD[qx][qy][dz] = 0.0;
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  QQD[qx][qy][dz] += B[qz][dz] * B[qz][dz] * D(qx, qy, qz, e);
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               QDD[qx][dy][dz] = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  QDD[qx][dy][dz] += B[qy][dy] * B[qy][dy] * QQD[qx][qy][dz];
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dz,z,D1D)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               double t = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  t += B[qx][dx] * B[qx][dx] * QDD[qx][dy][dz];
+               }
+               const int gid = MAP(dx, dy, dz, e);
+               const int idx = gid >= 0 ? gid : -1 - gid;
+               AtomicAdd(Y(idx), t);
+            }
+         }
+      }
+   });
+}
+
+// Fast non-deterministic 3D mass diag kernel
+// Less smem version with registers
+template<int D1D, int Q1D>
+static void NDK_RegsPAMassDiag3D(const int ndofs,
+                                 const int NE,
+                                 const int *map_,
+                                 const double *b_,
+                                 const double *d_,
+                                 double *y_)
+{
+   const auto MAP = Reshape(map_, D1D,D1D,D1D, NE);
+   const auto B = Reshape(b_, Q1D, D1D);
+   const auto D = Reshape(d_, Q1D, Q1D, Q1D, NE);
+   auto Y = Reshape(y_, ndofs);
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
+   {
+      double r_wk[Q1D];
+      MFEM_SHARED double s_B[Q1D][D1D];
+      MFEM_SHARED double s_q[Q1D][Q1D][Q1D];
+
+
+      MFEM_FOREACH_THREAD(d,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(q,x,Q1D)
+         {
+            if (d<D1D) { s_B[q][d] = B(q,d); }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(j,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(i,x,Q1D)
+         {
+            MFEM_UNROLL(Q1D)
+            for (int k=0; k<Q1D; ++k) { r_wk[k] = D(i,j,k,e); }
+            for (int c=0; c<D1D; ++c)
+            {
+               double q_cji = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int k=0; k<Q1D; ++k)
+               {
+                  const double Bkc = s_B[k][c];
+                  q_cji += Bkc * Bkc * r_wk[k];
+               }
+               s_q[c][j][i] = q_cji;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(c,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(i,x,Q1D)
+         {
+            for (int j=0; j<Q1D; ++j) { r_wk[j] = s_q[c][j][i]; }
+            MFEM_UNROLL(D1D)
+            for (int b=0; b<D1D; ++b)
+            {
+               double q_cbi = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int j=0; j<Q1D; ++j)
+               {
+                  const double Bjb = s_B[j][b];
+                  q_cbi += Bjb * Bjb * r_wk[j];
+               }
+               s_q[c][b][i] = q_cbi;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(c,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(b,x,D1D)
+         {
+            for (int i=0; i<Q1D; ++i) { r_wk[i] = s_q[c][b][i]; }
+            MFEM_UNROLL(D1D)
+            for (int a=0; a<D1D; ++a)
+            {
+               double q_cba = 0.0;
+               MFEM_UNROLL(Q1D)
+               for (int i=0; i<Q1D; ++i)
+               {
+                  const double Bia = s_B[i][a];
+                  q_cba += Bia * Bia * r_wk[i];
+               }
+               s_q[c][b][a] = q_cba;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(b,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(a,x,D1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int c=0; c<D1D; ++c)
+            {
+               const double q_cba = s_q[c][b][a];
+               const int gid = MAP(a,b,c,e);
+               const int idx = gid >= 0 ? gid : -1 - gid;
+               AtomicAdd(Y(idx), q_cba);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+void NDK_PAMassAssembleDiagonal(const int dim,
+                                const int D1D,
+                                const int Q1D,
+                                const int NE,
+                                const FiniteElementSpace *fes,
+                                const DofToQuad *maps,
+                                const Vector &D,
+                                Vector &Y)
+{
+   const int ND = fes->GetNDofs();
+   constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+   const Operator *ERop = fes->GetElementRestriction(ordering);
+   const ElementRestriction* ER = dynamic_cast<const ElementRestriction*>(ERop);
+   assert(ER);
+   const int *map = ER->GatherMap().Read();
+   const double *b = maps->B.Read();
+   const double *d = D.Read();
+   double *y = Y.ReadWrite();
+
+   assert(dim == 3);
+   const int ver = Device::KernelsVersion();
+   const int vid = (ver << 8) | (D1D << 4) | Q1D;
+
+   // Fast '0' non-deterministic 3D mass diag kernel
+   switch (vid) // orders 1~6
+   {
+      case 0x023: return NDK_SmemPAMassDiag3D<2,3>(ND,NE,map,b,d,y);
+      case 0x024: return NDK_SmemPAMassDiag3D<2,4>(ND,NE,map,b,d,y);
+      case 0x034: return NDK_SmemPAMassDiag3D<3,4>(ND,NE,map,b,d,y);
+      case 0x045: return NDK_SmemPAMassDiag3D<4,5>(ND,NE,map,b,d,y);
+      case 0x046: return NDK_SmemPAMassDiag3D<4,6>(ND,NE,map,b,d,y);
+      case 0x056: return NDK_SmemPAMassDiag3D<5,6>(ND,NE,map,b,d,y);
+      case 0x058: return NDK_SmemPAMassDiag3D<5,8>(ND,NE,map,b,d,y);
+      case 0x067: return NDK_SmemPAMassDiag3D<6,7>(ND,NE,map,b,d,y);
+      case 0x078: return NDK_SmemPAMassDiag3D<7,8>(ND,NE,map,b,d,y);
+      default: break;
+   }
+
+   // all other versions should fold back through these kernels
+   switch ((D1D << 4) | Q1D)
+   {
+      case 0x23: return NDK_RegsPAMassDiag3D<2,3>(ND,NE,map,b,d,y);
+      case 0x24: return NDK_RegsPAMassDiag3D<2,4>(ND,NE,map,b,d,y);
+      case 0x34: return NDK_RegsPAMassDiag3D<3,4>(ND,NE,map,b,d,y);
+      case 0x45: return NDK_RegsPAMassDiag3D<4,5>(ND,NE,map,b,d,y);
+      case 0x46: return NDK_RegsPAMassDiag3D<4,6>(ND,NE,map,b,d,y);
+      case 0x56: return NDK_RegsPAMassDiag3D<5,6>(ND,NE,map,b,d,y);
+      case 0x58: return NDK_RegsPAMassDiag3D<5,8>(ND,NE,map,b,d,y);
+      case 0x67: return NDK_RegsPAMassDiag3D<6,7>(ND,NE,map,b,d,y);
+      case 0x78: return NDK_RegsPAMassDiag3D<7,8>(ND,NE,map,b,d,y);
+      default: break;
+   }
+
+   MFEM_ABORT("Unknown kernel 0x" << std::hex << vid);
+}
+
+} // namespace mfem
@@ -0,0 +1,224 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../general/forall.hpp"
+#include "gridfunc.hpp"
+#include "restriction.hpp"
+
+using namespace std;
+
+namespace mfem
+{
+
+// Fast '7' HIP
+template<int D1D, int Q1D, int NBZ=1, int NBK=1> MFEM_GLOBAL static
+//MFEM_LAUNCH_BOUNDS(Q1D*Q1D*NBZ,NBK)
+void HIP_PAMassApply(const int NE,
+                     const int* MAP,
+                     const double* B,
+                     const double* D,
+                     const double* X,
+                     double* Y)
+{
+   double u[Q1D];
+   const int tz = MFEM_THREAD_ID(z);
+   MFEM_SHARED double s_B[Q1D][D1D];
+   MFEM_SHARED double s_q[NBZ][Q1D][Q1D][Q1D];
+
+   for (int be = MFEM_BLOCK_ID(x); be < (NE+NBZ-1)/NBZ; be += MFEM_GRID_DIM(x))
+   {
+      const int e = be * NBZ + tz;
+      if (e>=NE) { return; }
+
+      // Load input, B & X interpolation
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            if (tz == 0) { s_B[qx][dy] = B[qx+Q1D*dy]; }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
+            MFEM_UNROLL(D1D)
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const double Bx = B[qx+Q1D*dx];
+               MFEM_UNROLL(D1D)
+               for (int dz = 0; dz < D1D; ++dz)
+               {
+                  const int gid = MAP[e*D1D*D1D*D1D+dz*D1D*D1D+dy*D1D+dx];
+                  const int idx = gid >= 0 ? gid : -1 - gid;
+                  u[dz] = fma(X[idx], Bx, u[dz]);
+               }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { s_q[tz][dz][dy][qx] = u[dz]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Y interpolation
+      MFEM_FOREACH_THREAD(dz,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            MFEM_UNROLL(Q1D)
+            for (int qy = 0; qy < Q1D; ++qy) { u[qy] = 0.0; }
+            MFEM_UNROLL(D1D)
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               const double zyX = s_q[tz][dz][dy][qx];
+               MFEM_UNROLL(Q1D)
+               for (int qy = 0; qy < Q1D; ++qy) { u[qy] = fma(zyX,s_B[qy][dy],u[qy]); }
+            }
+            MFEM_UNROLL(Q1D)
+            for (int qy = 0; qy < Q1D; ++qy) { s_q[tz][dz][qy][qx] = u[qy]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Z interpolation, Q-function & Zt projection
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            // Z interpolation
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz) { u[qz] = 0.0; }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               const double zYX = s_q[tz][dz][qy][qx];
+               MFEM_UNROLL(Q1D)
+               for (int qz = 0; qz < Q1D; ++qz) { u[qz] = fma(zYX,s_B[qz][dz],u[qz]); }
+            }
+
+            // Q-function
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               const int idx = e*Q1D*Q1D*Q1D + qx + qy*Q1D + qz*Q1D*Q1D;
+               s_q[tz][qz][qy][qx] = u[qz] * D[idx];
+            }
+
+            // Zt projection
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
+            MFEM_UNROLL(Q1D)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               const double ZYX = s_q[tz][qz][qy][qx];
+               MFEM_UNROLL(D1D)
+               for (int dz = 0; dz < D1D; ++dz) { u[dz] = fma(ZYX,s_B[qz][dz],u[dz]); }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dz = 0; dz < D1D; ++dz) { s_q[tz][dz][qy][qx] = u[dz]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Yt projection
+      MFEM_FOREACH_THREAD(dz,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int dy = 0; dy < D1D; ++dy) { u[dy] = 0.0; }
+            MFEM_UNROLL(Q1D)
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const double zYX = s_q[tz][dz][qy][qx];
+               MFEM_UNROLL(D1D)
+               for (int dy = 0; dy < D1D; ++dy) { u[dy] = fma(zYX,s_B[qy][dy],u[dy]); }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dy = 0; dy < D1D; ++dy) { s_q[tz][dz][dy][qx] = u[dy]; }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // Xt projection & save output
+      MFEM_FOREACH_THREAD(dz,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dy,x,D1D)
+         {
+            MFEM_UNROLL(D1D)
+            for (int dx = 0; dx < D1D; ++dx) { u[dx] = 0.0; }
+            MFEM_UNROLL(Q1D)
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const double zyX = s_q[tz][dz][dy][qx];
+               MFEM_UNROLL(D1D)
+               for (int dx = 0; dx < D1D; ++dx) { u[dx] = fma(zyX,s_B[qx][dx],u[dx]); }
+            }
+            MFEM_UNROLL(D1D)
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const double output = u[dx];
+               const int gid = MAP[e*D1D*D1D*D1D+dz*D1D*D1D+dy*D1D+dx];
+               const int idx = gid >= 0 ? gid : -1 - gid;
+               AtomicAdd(Y[idx], output);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   }
+}
+
+
+void NDK_HIP_PAMassApply(const int dim,
+                         const int D1D,
+                         const int Q1D,
+                         const int NE,
+                         const FiniteElementSpace *fes,
+                         const DofToQuad *maps,
+                         const Vector &d,
+                         const Vector &x,
+                         Vector &y)
+{
+   assert(dim == 3);
+
+   constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+   const Operator *ERop = fes->GetElementRestriction(ordering);
+   const ElementRestriction *ER = dynamic_cast<const ElementRestriction*>(ERop);
+   assert(ER);
+
+   const auto M = ER->GatherMap().Read();
+   const double *B = maps->B.Read();
+   const double *D = d.Read();
+   const double *X = x.Read();
+   double *Y = y.ReadWrite();
+
+   void (*Ker)(const int NE,
+               const int *M,
+               const double *B,
+               const double *D,
+               const double *X,
+               double *Y) = nullptr;
+
+   const int ver = Device::KernelsVersion();
+   const int id = (ver << 8) | (D1D << 4) | Q1D;
+
+   switch (id) // orders 1~8
+   {
+      case 0x723: Ker=HIP_PAMassApply<2,3>; break; // 1
+      case 0x734: Ker=HIP_PAMassApply<3,4>; break; // 2
+      case 0x745: Ker=HIP_PAMassApply<4,5>; break; // 3
+      case 0x756: Ker=HIP_PAMassApply<5,6>; break; // 4
+      case 0x767: Ker=HIP_PAMassApply<6,7>; break; // 5
+      case 0x778: Ker=HIP_PAMassApply<7,8>; break; // 6
+      case 0x789: Ker=HIP_PAMassApply<8,9>; break; // 7
+      case 0x79A: Ker=HIP_PAMassApply<9,10>; break; // 8
+      default: MFEM_ABORT("Unknown kernel 0x" << std::hex << id << std::dec);
+   }
+   MFEM_LAUNCH_KERNEL(Ker,NE,dim3(Q1D,Q1D,1),0,NE,M,B,D,X,Y);
+}
+
+} // namespace mfem
@@ -34,7 +34,7 @@ void VectorMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
   if (DeviceCanUseCeed())
   {
      delete ceedOp;
-      ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q);
+      ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q, action_type);
      return;
   }
   dim = mesh->Dimension();
@@ -48,7 +48,8 @@ struct MassOperatorInfo : public OperatorInfo

 PAMassIntegrator::PAMassIntegrator(const mfem::FiniteElementSpace &fes,
                                   const mfem::IntegrationRule &irm,
-                                   mfem::Coefficient *Q)
+                                   mfem::Coefficient *Q,
+                                   mfem::ActionType &action_type)
   : PAIntegrator()
 {
 #ifdef MFEM_USE_CEED
@@ -57,6 +58,7 @@ PAMassIntegrator::PAMassIntegrator(const mfem::FiniteElementSpace &fes,
 #else
   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
 #endif
+   action_type = mfem::ActionType::L2L;
 }

 MFMassIntegrator::MFMassIntegrator(const mfem::FiniteElementSpace &fes,
@@ -27,7 +27,8 @@ class PAMassIntegrator : public PAIntegrator
 public:
   PAMassIntegrator(const mfem::FiniteElementSpace &fes,
                    const mfem::IntegrationRule &irm,
-                    mfem::Coefficient *Q);
+                    mfem::Coefficient *Q,
+                    mfem::ActionType &action_type);
 };

 /// Represent a MassIntegrator with AssemblyLevel::None using libCEED.
@@ -65,6 +65,8 @@ void RemoveBasisAndRestriction(const mfem::FiniteElementSpace *fes)
         itr++;
      }
   }
+#else
+   MFEM_CONTRACT_VAR(fes);
 #endif
 }

@@ -21,6 +21,33 @@ namespace mfem

 using namespace std;

+// Given an ElementTransformation and IntegrationPoint in a refined mesh,
+// return the ElementTransformation of the parent coarse element, and set
+// coarse_ip to the location of the original ip within the coarse element.
+ElementTransformation *RefinedToCoarse(
+   Mesh &coarse_mesh, const ElementTransformation &T,
+   const IntegrationPoint &ip, IntegrationPoint &coarse_ip)
+{
+   Mesh &fine_mesh = *T.mesh;
+   // Get the element transformation of the coarse element containing the
+   // fine element.
+   int fine_element = T.ElementNo;
+   const CoarseFineTransformations &cf = fine_mesh.GetRefinementTransforms();
+   int coarse_element = cf.embeddings[fine_element].parent;
+   ElementTransformation *coarse_T = coarse_mesh.GetElementTransformation(
+                                        coarse_element);
+   // Transform the integration point from fine element coordinates to coarse
+   // element coordinates.
+   Geometry::Type geom = T.GetGeometryType();
+   IntegrationPointTransformation fine_to_coarse;
+   IsoparametricTransformation &emb_tr = fine_to_coarse.Transf;
+   emb_tr.SetIdentityTransformation(geom);
+   emb_tr.SetPointMat(cf.point_matrices[geom](cf.embeddings[fine_element].matrix));
+   fine_to_coarse.Transform(ip, coarse_ip);
+   coarse_T->SetIntPoint(&coarse_ip);
+   return coarse_T;
+}
+
 double PWConstCoefficient::Eval(ElementTransformation & T,
                                const IntegrationPoint & ip)
 {
@@ -28,6 +55,52 @@ double PWConstCoefficient::Eval(ElementTransformation & T,
   return (constants(att-1));
 }

+void PWCoefficient::InitMap(const Array<int> & attr,
+                            const Array<Coefficient*> & coefs)
+{
+   MFEM_VERIFY(attr.Size() == coefs.Size(),
+               "PWCoefficient:  "
+               "Attribute and coefficient arrays have incompatible "
+               "dimensions.");
+
+   for (int i=0; i<attr.Size(); i++)
+   {
+      if (coefs[i] != NULL)
+      {
+         UpdateCoefficient(attr[i], *coefs[i]);
+      }
+   }
+}
+
+void PWCoefficient::SetTime(double t)
+{
+   Coefficient::SetTime(t);
+
+   std::map<int, Coefficient*>::iterator p = pieces.begin();
+   for (; p != pieces.end(); p++)
+   {
+      if (p->second != NULL)
+      {
+         p->second->SetTime(t);
+      }
+   }
+}
+
+double PWCoefficient::Eval(ElementTransformation &T,
+                           const IntegrationPoint &ip)
+{
+   const int att = T.Attribute;
+   std::map<int, Coefficient*>::const_iterator p = pieces.find(att);
+   if (p != pieces.end())
+   {
+      if ( p->second != NULL)
+      {
+         return p->second->Eval(T, ip);
+      }
+   }
+   return 0.0;
+}
+
 double FunctionCoefficient::Eval(ElementTransformation & T,
                                 const IntegrationPoint & ip)
 {
@@ -49,7 +122,17 @@ double FunctionCoefficient::Eval(ElementTransformation & T,
 double GridFunctionCoefficient::Eval (ElementTransformation &T,
                                      const IntegrationPoint &ip)
 {
-   return GridF -> GetValue (T, ip, Component);
+   Mesh *gf_mesh = GridF->FESpace()->GetMesh();
+   if (T.mesh == gf_mesh)
+   {
+      return GridF->GetValue(T, ip, Component);
+   }
+   else
+   {
+      IntegrationPoint coarse_ip;
+      ElementTransformation *coarse_T = RefinedToCoarse(*gf_mesh, T, ip, coarse_ip);
+      return GridF->GetValue(*coarse_T, coarse_ip, Component);
+   }
 }

 void TransformedCoefficient::SetTime(double t)
@@ -120,6 +203,63 @@ void VectorCoefficient::Eval(DenseMatrix &M, ElementTransformation &T,
   }
 }

+void PWVectorCoefficient::InitMap(const Array<int> & attr,
+                                  const Array<VectorCoefficient*> & coefs)
+{
+   MFEM_VERIFY(attr.Size() == coefs.Size(),
+               "PWVectorCoefficient:  "
+               "Attribute and coefficient arrays have incompatible "
+               "dimensions.");
+
+   for (int i=0; i<attr.Size(); i++)
+   {
+      if (coefs[i] != NULL)
+      {
+         UpdateCoefficient(attr[i], *coefs[i]);
+      }
+   }
+}
+
+void PWVectorCoefficient::UpdateCoefficient(int attr, VectorCoefficient & coef)
+{
+   MFEM_VERIFY(coef.GetVDim() == vdim,
+               "PWVectorCoefficient::UpdateCoefficient:  "
+               "VectorCoefficient has incompatible dimension.");
+   pieces[attr] = &coef;
+}
+
+void PWVectorCoefficient::SetTime(double t)
+{
+   VectorCoefficient::SetTime(t);
+
+   std::map<int, VectorCoefficient*>::iterator p = pieces.begin();
+   for (; p != pieces.end(); p++)
+   {
+      if (p->second != NULL)
+      {
+         p->second->SetTime(t);
+      }
+   }
+}
+
+void PWVectorCoefficient::Eval(Vector &V, ElementTransformation &T,
+                               const IntegrationPoint &ip)
+{
+   const int att = T.Attribute;
+   std::map<int, VectorCoefficient*>::const_iterator p = pieces.find(att);
+   if (p != pieces.end())
+   {
+      if ( p->second != NULL)
+      {
+         p->second->Eval(V, T, ip);
+         return;
+      }
+   }
+
+   V.SetSize(vdim);
+   V = 0.0;
+}
+
 void VectorFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
                                     const IntegrationPoint &ip)
 {
@@ -202,13 +342,30 @@ void VectorGridFunctionCoefficient::SetGridFunction(const GridFunction *gf)
 void VectorGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
                                         const IntegrationPoint &ip)
 {
-   GridFunc->GetVectorValue(T, ip, V);
+   Mesh *gf_mesh = GridFunc->FESpace()->GetMesh();
+   if (T.mesh == gf_mesh)
+   {
+      GridFunc->GetVectorValue(T, ip, V);
+   }
+   else
+   {
+      IntegrationPoint coarse_ip;
+      ElementTransformation *coarse_T = RefinedToCoarse(*gf_mesh, T, ip, coarse_ip);
+      GridFunc->GetVectorValue(*coarse_T, coarse_ip, V);
+   }
 }

 void VectorGridFunctionCoefficient::Eval(
   DenseMatrix &M, ElementTransformation &T, const IntegrationRule &ir)
 {
-   GridFunc->GetVectorValues(T, ir, M);
+   if (T.mesh == GridFunc->FESpace()->GetMesh())
+   {
+      GridFunc->GetVectorValues(T, ir, M);
+   }
+   else
+   {
+      VectorCoefficient::Eval(M, T, ir);
+   }
 }

 GradientGridFunctionCoefficient::GradientGridFunctionCoefficient (
@@ -228,13 +385,30 @@ void GradientGridFunctionCoefficient::SetGridFunction(const GridFunction *gf)
 void GradientGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
                                           const IntegrationPoint &ip)
 {
-   GridFunc->GetGradient(T, V);
+   Mesh *gf_mesh = GridFunc->FESpace()->GetMesh();
+   if (T.mesh == gf_mesh)
+   {
+      GridFunc->GetGradient(T, V);
+   }
+   else
+   {
+      IntegrationPoint coarse_ip;
+      ElementTransformation *coarse_T = RefinedToCoarse(*gf_mesh, T, ip, coarse_ip);
+      GridFunc->GetGradient(*coarse_T, V);
+   }
 }

 void GradientGridFunctionCoefficient::Eval(
   DenseMatrix &M, ElementTransformation &T, const IntegrationRule &ir)
 {
-   GridFunc->GetGradients(T, ir, M);
+   if (T.mesh == GridFunc->FESpace()->GetMesh())
+   {
+      GridFunc->GetGradients(T, ir, M);
+   }
+   else
+   {
+      VectorCoefficient::Eval(M, T, ir);
+   }
 }

 CurlGridFunctionCoefficient::CurlGridFunctionCoefficient(
@@ -260,7 +434,17 @@ void CurlGridFunctionCoefficient::SetGridFunction(const GridFunction *gf)
 void CurlGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
                                       const IntegrationPoint &ip)
 {
-   GridFunc->GetCurl(T, V);
+   Mesh *gf_mesh = GridFunc->FESpace()->GetMesh();
+   if (T.mesh == gf_mesh)
+   {
+      GridFunc->GetCurl(T, V);
+   }
+   else
+   {
+      IntegrationPoint coarse_ip;
+      ElementTransformation *coarse_T = RefinedToCoarse(*gf_mesh, T, ip, coarse_ip);
+      GridFunc->GetCurl(*coarse_T, V);
+   }
 }

 DivergenceGridFunctionCoefficient::DivergenceGridFunctionCoefficient (
@@ -272,7 +456,17 @@ DivergenceGridFunctionCoefficient::DivergenceGridFunctionCoefficient (
 double DivergenceGridFunctionCoefficient::Eval(ElementTransformation &T,
                                               const IntegrationPoint &ip)
 {
-   return GridFunc->GetDivergence(T);
+   Mesh *gf_mesh = GridFunc->FESpace()->GetMesh();
+   if (T.mesh == gf_mesh)
+   {
+      return GridFunc->GetDivergence(T);
+   }
+   else
+   {
+      IntegrationPoint coarse_ip;
+      ElementTransformation *coarse_T = RefinedToCoarse(*gf_mesh, T, ip, coarse_ip);
+      return GridFunc->GetDivergence(*coarse_T);
+   }
 }

 void VectorDeltaCoefficient::SetTime(double t)
@@ -331,6 +525,72 @@ void VectorRestrictedCoefficient::Eval(
   }
 }

+void PWMatrixCoefficient::InitMap(const Array<int> & attr,
+                                  const Array<MatrixCoefficient*> & coefs)
+{
+   MFEM_VERIFY(attr.Size() == coefs.Size(),
+               "PWMatrixCoefficient:  "
+               "Attribute and coefficient arrays have incompatible "
+               "dimensions.");
+
+   for (int i=0; i<attr.Size(); i++)
+   {
+      if (coefs[i] != NULL)
+      {
+         UpdateCoefficient(attr[i], *coefs[i]);
+      }
+   }
+}
+
+void PWMatrixCoefficient::UpdateCoefficient(int attr, MatrixCoefficient & coef)
+{
+   MFEM_VERIFY(coef.GetHeight() == height,
+               "PWMatrixCoefficient::UpdateCoefficient:  "
+               "MatrixCoefficient has incompatible height.");
+   MFEM_VERIFY(coef.GetWidth() == width,
+               "PWMatrixCoefficient::UpdateCoefficient:  "
+               "MatrixCoefficient has incompatible width.");
+   if (symmetric)
+   {
+      MFEM_VERIFY(coef.IsSymmetric(),
+                  "PWMatrixCoefficient::UpdateCoefficient:  "
+                  "MatrixCoefficient has incompatible symmetry.");
+   }
+   pieces[attr] = &coef;
+}
+
+void PWMatrixCoefficient::SetTime(double t)
+{
+   MatrixCoefficient::SetTime(t);
+
+   std::map<int, MatrixCoefficient*>::iterator p = pieces.begin();
+   for (; p != pieces.end(); p++)
+   {
+      if (p->second != NULL)
+      {
+         p->second->SetTime(t);
+      }
+   }
+}
+
+void PWMatrixCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
+                               const IntegrationPoint &ip)
+{
+   const int att = T.Attribute;
+   std::map<int, MatrixCoefficient*>::const_iterator p = pieces.find(att);
+   if (p != pieces.end())
+   {
+      if ( p->second != NULL)
+      {
+         p->second->Eval(K, T, ip);
+         return;
+      }
+   }
+
+   K.SetSize(height, width);
+   K = 0.0;
+}
+
 void MatrixFunctionCoefficient::SetTime(double t)
 {
   if (Q) { Q->SetTime(t); }
@@ -125,6 +125,87 @@ public:
                       const IntegrationPoint &ip);
 };

+/** @brief A piecewise coefficient with the pieces keyed off the element
+    attribute numbers.
+
+    A value of zero will be returned for any missing attribute numbers.
+
+    This object will not assume ownership of any Coefficient objects
+    passed to it. Consequently, the caller must ensure that the
+    individual Coefficient objects are not deleted while this
+    PWCoefficient is still in use.
+
+    \note The keys may either be domain attribute numbers or boundary
+    attribute numbers. If the PWCoefficient is used with a domain
+    integrator the keys are assumed to be domain attribute
+    numbers. Similarly, if the PWCoefficient is used with a boundary
+    integrator the keys are assumed to be boundary attribute numbers.
+*/
+class PWCoefficient : public Coefficient
+{
+private:
+   /** Internal data structure to store pointers to the appropriate
+       coefficients for different regions of the mesh. The keys used
+       in the map are the mesh attribute numbers (either element
+       attribute or boundary element attribute depending upon
+       context). The values returned for any missing attributes will
+       be zero. The coefficient pointers may be NULL in which case a
+       value of zero is returned.
+
+       The Coefficient objects contained in this map are NOT owned by
+       this PWCoefficient object. This means that they will not be
+       deleted when this object is deleted also the caller must ensure
+       that the various Coefficient objects are not deleted while this
+       PWCoefficient is still needed.
+   */
+   std::map<int, Coefficient*> pieces;
+
+   /** Convenience function to check for compatible array lengths,
+       loop over the arrays, and add their attribute/Coefficient pairs
+       to the internal data structure.
+   */
+   void InitMap(const Array<int> & attr,
+                const Array<Coefficient*> & coefs);
+
+public:
+
+   /// Constructs a piecewise coefficient
+   explicit PWCoefficient() {}
+
+   /// Construct the coefficient using arrays describing the pieces
+   /** \param attr - an array of attribute numbers for each piece
+       \param coefs - the corresponding array of Coefficient pointers
+       Any missing attributes or NULL coefficient pointers will result in a
+       value of zero being returned for that attribute.
+
+       \note Ownership of the Coefficient objects will NOT be
+       transferred to this object.
+   */
+   PWCoefficient(const Array<int> & attr,
+                 const Array<Coefficient*> & coefs)
+   { InitMap(attr, coefs); }
+
+   /// Set the time for time dependent coefficients
+   virtual void SetTime(double t);
+
+   /// Replace a set of coefficients
+   void UpdateCoefficients(const Array<int> & attr,
+                           const Array<Coefficient*> & coefs)
+   { InitMap(attr, coefs); }
+
+   /// Replace a single Coefficient for a particular attribute
+   void UpdateCoefficient(int attr, Coefficient & coef)
+   { pieces[attr] = &coef; }
+
+   /// Remove a single Coefficient for a particular attribute
+   void ZeroCoefficient(int attr)
+   { pieces.erase(attr); }
+
+   /// Evaluate the coefficient.
+   virtual double Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip);
+};
+
 /// A general function coefficient
 class FunctionCoefficient : public Coefficient
 {
@@ -413,6 +494,89 @@ public:
   const Vector& GetVec() { return vec; }
 };

+/** @brief A piecewise vector-valued coefficient with the pieces keyed off the
+    element attribute numbers.
+
+    A value of zero will be returned for any missing attribute numbers.
+
+    This object will not assume ownership of any VectorCoefficient
+    objects passed to it. Consequently, the caller must ensure that
+    the individual VectorCoefficient objects are not deleted while
+    this PWVectorCoefficient is still in use.
+
+    \note The keys may either be domain attribute numbers or boundary
+    attribute numbers. If the PWVectorCoefficient is used with a
+    domain integrator the keys are assumed to be domain attribute
+    numbers. Similarly, if the PWVectorCoefficient is used with a
+    boundary integrator the keys are assumed to be boundary attribute
+    numbers.
+*/
+class PWVectorCoefficient : public VectorCoefficient
+{
+private:
+   /** Internal data structure to store pointers to the appropriate
+       coefficients for different regions of the mesh. The keys used
+       in the map are the mesh attribute numbers (either element
+       attribute or boundary element attribute depending upon
+       context). The values returned for any missing attributes will
+       be zero. The coefficient pointers may be NULL in which case a
+       value of zero is returned.
+
+       The VectorCoefficient objects contained in this map are NOT
+       owned by this PWVectorCoefficient object. This means that they
+       will not be deleted when this object is deleted also the caller
+       must ensure that the various VectorCoefficient objects are not
+       deleted while this PWVectorCoefficient is still needed.
+   */
+   std::map<int, VectorCoefficient*> pieces;
+
+   /** Convenience function to check for compatible array lengths,
+       loop over the arrays, and add their attribute/VectorCoefficient
+       pairs to the internal data structure.
+   */
+   void InitMap(const Array<int> & attr,
+                const Array<VectorCoefficient*> & coefs);
+
+public:
+
+   /// Constructs a piecewise vector coefficient of dimension vd
+   explicit PWVectorCoefficient(int vd): VectorCoefficient(vd) {}
+
+   /// Construct the coefficient using arrays describing the pieces
+   /** \param vd - dimension of the vector-valued result
+       \param attr - an array of attribute numbers for each piece
+       \param coefs - the corresponding array of VectorCoefficient pointers
+       Any missing attributes or NULL coefficient pointers will result in a
+       zero vector being returned for that attribute.
+
+       \note Ownership of the VectorCoefficient objects will NOT be
+       transferred to this object.
+   */
+   PWVectorCoefficient(int vd, const Array<int> & attr,
+                       const Array<VectorCoefficient*> & coefs)
+      : VectorCoefficient(vd) { InitMap(attr, coefs); }
+
+   /// Set the time for time dependent coefficients
+   virtual void SetTime(double t);
+
+   /// Replace a set of coefficients
+   void UpdateCoefficients(const Array<int> & attr,
+                           const Array<VectorCoefficient*> & coefs)
+   { InitMap(attr, coefs); }
+
+   /// Replace a single Coefficient for a particular attribute
+   void UpdateCoefficient(int attr, VectorCoefficient & coef);
+
+   /// Remove a single VectorCoefficient for a particular attribute
+   void ZeroCoefficient(int attr)
+   { pieces.erase(attr); }
+
+   /// Evaluate the coefficient.
+   virtual void Eval(Vector &V, ElementTransformation &T,
+                     const IntegrationPoint &ip);
+   using VectorCoefficient::Eval;
+};
+
 /// A general vector function coefficient
 class VectorFunctionCoefficient : public VectorCoefficient
 {
@@ -782,6 +946,112 @@ public:
 };


+/** @brief A piecewise matrix-valued coefficient with the pieces keyed off the
+    element attribute numbers.
+
+    A value of zero will be returned for any missing attribute numbers.
+
+    This object will not assume ownership of any MatrixCoefficient
+    objects passed to it. Consequently, the caller must ensure that
+    the individual MatrixCoefficient objects are not deleted while
+    this PWMatrixCoefficient is still in use.
+
+    \note The keys may either be domain attribute numbers or boundary
+    attribute numbers. If the PWMatrixCoefficient is used with a
+    domain integrator the keys are assumed to be domain attribute
+    numbers. Similarly, if the PWMatrixCoefficient is used with a
+    boundary integrator the keys are assumed to be boundary attribute
+    numbers.
+*/
+class PWMatrixCoefficient : public MatrixCoefficient
+{
+private:
+   /** Internal data structure to store pointers to the appropriate
+       coefficients for different regions of the mesh. The keys used
+       in the map are the mesh attribute numbers (either element
+       attribute or boundary element attribute depending upon
+       context). The values returned for any missing attributes will
+       be zero. The coefficient pointers may be NULL in which case a
+       value of zero is returned.
+
+       The MatrixCoefficient objects contained in this map are NOT
+       owned by this PWMatrixCoefficient object. This means that they
+       will not be deleted when this object is deleted also the caller
+       must ensure that the various MatrixCoefficient objects are not
+       deleted while this PWMatrixCoefficient is still needed.
+   */
+   std::map<int, MatrixCoefficient*> pieces;
+
+   /** Convenience function to check for compatible array lengths,
+       loop over the arrays, and add their attribute/MatrixCoefficient
+       pairs to the internal data structure.
+   */
+   void InitMap(const Array<int> & attr,
+                const Array<MatrixCoefficient*> & coefs);
+
+public:
+
+   /// Constructs a piecewise matrix coefficient of dimension dim by dim
+   explicit PWMatrixCoefficient(int dim, bool symm = false)
+      : MatrixCoefficient(dim, symm) {}
+
+   /// Constructs a piecewise matrix coefficient of dimension h by w
+   explicit PWMatrixCoefficient(int h, int w, bool symm = false)
+      : MatrixCoefficient(h, w, symm) {}
+
+   /// Construct the coefficient using arrays describing the pieces
+   /** \param dim - size of the square matrix-valued result
+       \param attr - an array of attribute numbers for each piece
+       \param coefs - the corresponding array of MatrixCoefficient pointers
+       \param symm - true if the result will be symmetric, false otherwise
+       Any missing attributes or NULL coefficient pointers will result in a
+       zero matrix being returned.
+
+       \note Ownership of the MatrixCoefficient objects will NOT be
+       transferred to this object.
+   */
+   PWMatrixCoefficient(int dim, const Array<int> & attr,
+                       const Array<MatrixCoefficient*> & coefs,
+                       bool symm=false)
+      : MatrixCoefficient(dim, symm) { InitMap(attr, coefs); }
+
+   /// Construct the coefficient using arrays describing the pieces
+   /** \param h - height of the matrix-valued result
+       \param w - width of the matrix-valued result
+       \param attr - an array of attribute numbers for each piece
+       \param coefs - the corresponding array of MatrixCoefficient pointers
+       \param symm - true if the result will be symmetric, false otherwise
+       Any missing attributes or NULL coefficient pointers will result in a
+       zero matrix being returned for that attribute.
+
+       \note Ownership of the MatrixCoefficient objects will NOT be
+       transferred to this object.
+   */
+   PWMatrixCoefficient(int h, int w, const Array<int> & attr,
+                       const Array<MatrixCoefficient*> & coefs,
+                       bool symm=false)
+      : MatrixCoefficient(h, w, symm) { InitMap(attr, coefs); }
+
+   /// Set the time for time dependent coefficients
+   virtual void SetTime(double t);
+
+   /// Replace a set of coefficients
+   void UpdateCoefficients(const Array<int> & attr,
+                           const Array<MatrixCoefficient*> & coefs)
+   { InitMap(attr, coefs); }
+
+   /// Replace a single coefficient for a particular attribute
+   void UpdateCoefficient(int attr, MatrixCoefficient & coef);
+
+   /// Remove a single MatrixCoefficient for a particular attribute
+   void ZeroCoefficient(int attr)
+   { pieces.erase(attr); }
+
+   /// Evaluate the coefficient.
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
+                     const IntegrationPoint &ip);
+};
+
 /** @brief A matrix coefficient with an optional scalar coefficient multiplier
    \a q.  The matrix function can either be represented by a std function or
    a constant matrix provided when constructing this object.  */
@@ -195,6 +195,15 @@ ComplexLinearForm::AddDomainIntegrator(LinearFormIntegrator *lfi_real,
   if ( lfi_imag ) { lfi->AddDomainIntegrator(lfi_imag); }
 }

+void
+ComplexLinearForm::AddDomainIntegrator(LinearFormIntegrator *lfi_real,
+                                       LinearFormIntegrator *lfi_imag,
+                                       Array<int> &elem_attr_marker)
+{
+   if ( lfi_real ) { lfr->AddDomainIntegrator(lfi_real, elem_attr_marker); }
+   if ( lfi_imag ) { lfi->AddDomainIntegrator(lfi_imag, elem_attr_marker); }
+}
+
 void
 ComplexLinearForm::AddBoundaryIntegrator(LinearFormIntegrator *lfi_real,
                                         LinearFormIntegrator *lfi_imag)
@@ -317,6 +326,14 @@ void SesquilinearForm::AddDomainIntegrator(BilinearFormIntegrator *bfi_real,
   if (bfi_imag) { blfi->AddDomainIntegrator(bfi_imag); }
 }

+void SesquilinearForm::AddDomainIntegrator(BilinearFormIntegrator *bfi_real,
+                                           BilinearFormIntegrator *bfi_imag,
+                                           Array<int> & elem_marker)
+{
+   if (bfi_real) { blfr->AddDomainIntegrator(bfi_real, elem_marker); }
+   if (bfi_imag) { blfi->AddDomainIntegrator(bfi_imag, elem_marker); }
+}
+
 void
 SesquilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator *bfi_real,
                                        BilinearFormIntegrator *bfi_imag)
@@ -879,6 +896,15 @@ ParComplexLinearForm::AddDomainIntegrator(LinearFormIntegrator *lfi_real,
   if ( lfi_imag ) { plfi->AddDomainIntegrator(lfi_imag); }
 }

+void
+ParComplexLinearForm::AddDomainIntegrator(LinearFormIntegrator *lfi_real,
+                                          LinearFormIntegrator *lfi_imag,
+                                          Array<int> &elem_attr_marker)
+{
+   if ( lfi_real ) { plfr->AddDomainIntegrator(lfi_real, elem_attr_marker); }
+   if ( lfi_imag ) { plfi->AddDomainIntegrator(lfi_imag, elem_attr_marker); }
+}
+
 void
 ParComplexLinearForm::AddBoundaryIntegrator(LinearFormIntegrator *lfi_real,
                                            LinearFormIntegrator *lfi_imag)
@@ -1040,6 +1066,14 @@ void ParSesquilinearForm::AddDomainIntegrator(BilinearFormIntegrator *bfi_real,
   if (bfi_imag) { pblfi->AddDomainIntegrator(bfi_imag); }
 }

+void ParSesquilinearForm::AddDomainIntegrator(BilinearFormIntegrator *bfi_real,
+                                              BilinearFormIntegrator *bfi_imag,
+                                              Array<int> & elem_marker)
+{
+   if (bfi_real) { pblfr->AddDomainIntegrator(bfi_real, elem_marker); }
+   if (bfi_imag) { pblfi->AddDomainIntegrator(bfi_imag, elem_marker); }
+}
+
 void
 ParSesquilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator *bfi_real,
                                           BilinearFormIntegrator *bfi_imag)
@@ -128,6 +128,11 @@ public:
   void AddDomainIntegrator(LinearFormIntegrator *lfi_real,
                            LinearFormIntegrator *lfi_imag);

+   /// Adds new Domain Integrator, restricted to the given attributes.
+   void AddDomainIntegrator(LinearFormIntegrator *lfi_real,
+                            LinearFormIntegrator *lfi_imag,
+                            Array<int> &elem_attr_marker);
+
   /// Adds new Boundary Integrator.
   void AddBoundaryIntegrator(LinearFormIntegrator *lfi_real,
                              LinearFormIntegrator *lfi_imag);
@@ -260,6 +265,11 @@ public:
   void AddDomainIntegrator(BilinearFormIntegrator *bfi_real,
                            BilinearFormIntegrator *bfi_imag);

+   /// Adds new Domain Integrator, restricted to the given attributes.
+   void AddDomainIntegrator(BilinearFormIntegrator *bfi_real,
+                            BilinearFormIntegrator *bfi_imag,
+                            Array<int> &elem_marker);
+
   /// Adds new Boundary Integrator.
   void AddBoundaryIntegrator(BilinearFormIntegrator *bfi_real,
                              BilinearFormIntegrator *bfi_imag);
@@ -464,6 +474,11 @@ public:
   void AddDomainIntegrator(LinearFormIntegrator *lfi_real,
                            LinearFormIntegrator *lfi_imag);

+   /// Adds new Domain Integrator, restricted to specific attributes.
+   void AddDomainIntegrator(LinearFormIntegrator *lfi_real,
+                            LinearFormIntegrator *lfi_imag,
+                            Array<int> &elem_attr_marker);
+
   /// Adds new Boundary Integrator.
   void AddBoundaryIntegrator(LinearFormIntegrator *lfi_real,
                              LinearFormIntegrator *lfi_imag);
@@ -598,6 +613,11 @@ public:
   void AddDomainIntegrator(BilinearFormIntegrator *bfi_real,
                            BilinearFormIntegrator *bfi_imag);

+   /// Adds new Domain Integrator, restricted to specific attributes.
+   void AddDomainIntegrator(BilinearFormIntegrator *bfi_real,
+                            BilinearFormIntegrator *bfi_imag,
+                            Array<int> &elem_marker);
+
   /// Adds new Boundary Integrator.
   void AddBoundaryIntegrator(BilinearFormIntegrator *bfi_real,
                              BilinearFormIntegrator *bfi_imag);
@@ -330,7 +330,7 @@ ConduitDataCollection::BlueprintMeshToMesh(const Node &n_mesh,
         }
         else
         {
-            Node &(n_bndry_conn_conv) =
+            Node &n_bndry_conn_conv =
               n_conv["topologies"][bndry_topo_name]["elements/connectivity"];
            n_bndry_conn.to_int_array(n_bndry_conn_conv);
            bndry_indices = (n_bndry_conn_conv).value();
@@ -645,7 +645,8 @@ ConduitDataCollection::MeshToBlueprintMesh(Mesh *mesh,
                                           Node &n_mesh,
                                           const std::string &coordset_name,
                                           const std::string &main_topology_name,
-                                           const std::string &boundary_topology_name)
+                                           const std::string &boundary_topology_name,
+                                           const std::string &main_adjset_name)
 {
   int dim = mesh->SpaceDimension();

@@ -815,6 +816,83 @@ ConduitDataCollection::MeshToBlueprintMesh(Mesh *mesh,
         bndry_att_vals[i] = mesh->GetBdrAttribute(i);
      }
   }
+
+   ////////////////////////////////////////////
+   // Setup adjsets
+   ////////////////////////////////////////////
+
+#ifdef MFEM_USE_MPI
+   ParMesh *pmesh = dynamic_cast<ParMesh*>(mesh);
+   if (pmesh)
+   {
+      ////////////////////////////////////////////
+      // Setup main adjset
+      ////////////////////////////////////////////
+
+      Node &n_adjset = n_mesh["adjsets"][main_adjset_name];
+
+      n_adjset["association"] = "vertex";
+      n_adjset["topology"] = main_topology_name;
+      n_adjset["groups"].set(DataType::object());
+
+      const GroupTopology &pmesh_gtopo = pmesh->gtopo;
+      const int local_rank = pmesh->GetMyRank();
+      const int num_groups = pmesh_gtopo.NGroups();
+      // NOTE: skip the first group since its the local-only group
+      for (int i = 1; i < num_groups; i++)
+      {
+         const int num_group_nbrs = pmesh_gtopo.GetGroupSize(i);
+         const int *group_nbrs = pmesh_gtopo.GetGroup(i);
+         const int num_group_verts = pmesh->GroupNVertices(i);
+
+         // NOTE: 'neighbor' values are local to this processor, but Blueprint
+         // expects global domain identifiers, so we collapse this layer of
+         // indirection
+         Array<int> group_ranks(num_group_nbrs);
+         std::string group_name = "group";
+         {
+            for (int j = 0; j < num_group_nbrs; j++)
+            {
+               group_ranks[j] = pmesh_gtopo.GetNeighborRank(group_nbrs[j]);
+            }
+            group_ranks.Sort();
+            for (int j = 0; j < num_group_nbrs; j++)
+            {
+               group_name += "_" + std::to_string(group_ranks[j]);
+            }
+
+            // NOTE: Blueprint only wants remote ranks in its neighbor list,
+            // so we remove the local rank after the canonicalized Blueprint
+            // group name is formed
+            group_ranks.DeleteFirst(local_rank);
+         }
+         Node &n_group = n_adjset["groups"][group_name];
+
+         n_group["neighbors"].set(group_ranks.GetData(), group_ranks.Size());
+         n_group["values"].set(DataType::c_int(num_group_verts));
+
+         int_array group_vals = n_group["values"].value();
+         for (int j = 0; j < num_group_verts; j++)
+         {
+            group_vals[j] = pmesh->GroupVertex(i, j);
+         }
+      }
+
+      // NOTE: We don't create an adjset for face neighbor data because
+      // these faces aren't listed in the 'boundary_topology_name' topology
+      // (this topology only covers the faces between 'main_topology_name'
+      // elements and void). To include a face neighbor data adjset, this
+      // function would need to export a topology with either (1) all faces
+      // in the mesh topology or (2) all boundary faces, including neighbors.
+
+      ////////////////////////////////////////////
+      // Setup distributed state
+      ////////////////////////////////////////////
+
+      Node &n_domid = n_mesh["state/domain_id"];
+      n_domid.set(local_rank);
+   }
+#endif
 }

 //---------------------------------------------------------------------------//
@@ -166,7 +166,8 @@ public:
                                   conduit::Node &out,
                                   const std::string &coordset_name = "coords",
                                   const std::string &main_topology_name = "main",
-                                   const std::string &boundary_topology_name = "boundary");
+                                   const std::string &boundary_topology_name = "boundary",
+                                   const std::string &main_adjset_name = "main_adjset");

   /// Describes a MFEM grid function using the mesh blueprint
   /** Sets up passed conduit::Node out to describe the given grid function
@@ -85,6 +85,11 @@ void TransformPrimal(const DofTransformation *ran_dof_trans,
   }
 }

+void DofTransformation::InvTransformDual(Vector &v) const
+{
+   InvTransformDual(v.GetData());
+}
+
 void TransformDual(const DofTransformation *ran_dof_trans,
                   const DofTransformation *dom_dof_trans,
                   DenseMatrix &elmat)
@@ -195,6 +200,35 @@ void VDofTransformation::TransformDual(double *v) const
   }
 }

+void VDofTransformation::InvTransformDual(double *v) const
+{
+   int size = doftrans_->Size();
+
+   if ((Ordering::Type)ordering_ == Ordering::byNODES)
+   {
+      for (int i=0; i<vdim_; i++)
+      {
+         doftrans_->InvTransformDual(&v[i*size]);
+      }
+   }
+   else
+   {
+      Vector vec(size);
+      for (int i=0; i<vdim_; i++)
+      {
+         for (int j=0; j<size; j++)
+         {
+            vec(j) = v[j*vdim_+i];
+         }
+         doftrans_->InvTransformDual(vec);
+         for (int j=0; j<size; j++)
+         {
+            v[j*vdim_+i] = vec(j);
+         }
+      }
+   }
+}
+
 const double ND_DofTransformation::T_data[24] =
 {
   1.0,  0.0,  0.0,  1.0,
@@ -291,6 +325,26 @@ ND_TriDofTransformation::TransformDual(double *v) const
   }
 }

+void
+ND_TriDofTransformation::InvTransformDual(double *v) const
+{
+   int nedofs = order; // number of DoFs per edge
+   int nfdofs = order*(order-1); // number of DoFs per face
+
+   double data[2];
+   Vector v2(data, 2);
+
+   // Transform face DoFs
+   for (int f=0; f<1; f++)
+   {
+      for (int i=0; i<nfdofs/2; i++)
+      {
+         v2 = &v[3*nedofs + f*nfdofs + 2*i];
+         T(Fo[f]).MultTranspose(v2, &v[3*nedofs + f*nfdofs + 2*i]);
+      }
+   }
+}
+
 ND_TetDofTransformation::ND_TetDofTransformation(int p)
   : ND_DofTransformation(p*(p + 2)*(p + 3)/2, p)
 {
@@ -355,4 +409,24 @@ ND_TetDofTransformation::TransformDual(double *v) const
   }
 }

+void
+ND_TetDofTransformation::InvTransformDual(double *v) const
+{
+   int nedofs = order; // number of DoFs per edge
+   int nfdofs = order*(order-1); // number of DoFs per face
+
+   double data[2];
+   Vector v2(data, 2);
+
+   // Transform face DoFs
+   for (int f=0; f<4; f++)
+   {
+      for (int i=0; i<nfdofs/2; i++)
+      {
+         v2 = &v[6*nedofs + f*nfdofs + 2*i];
+         T(Fo[f]).MultTranspose(v2, &v[6*nedofs + f*nfdofs + 2*i]);
+      }
+   }
+}
+
 } // namespace mfem
@@ -102,6 +102,10 @@ public:
   virtual void TransformDual(double *v) const = 0;
   virtual void TransformDual(Vector &v) const;

+   /** Inverse Transform dual DoFs */
+   virtual void InvTransformDual(double *v) const = 0;
+   virtual void InvTransformDual(Vector &v) const;
+
   /** Transform a matrix of dual DoFs entries as computed by a
       BilinearFormIntegrator before summing into a BilinearForm object. */
   virtual void TransformDual(DenseMatrix &V) const;
@@ -183,10 +187,12 @@ public:
   using DofTransformation::TransformPrimal;
   using DofTransformation::InvTransformPrimal;
   using DofTransformation::TransformDual;
+   using DofTransformation::InvTransformDual;

   void TransformPrimal(double *v) const;
   void InvTransformPrimal(double *v) const;
   void TransformDual(double *v) const;
+   void InvTransformDual(double *v) const;
 };

 /** Abstract base class for high-order Nedelec spaces on elements with
@@ -235,6 +241,9 @@ public:
   void InvTransformPrimal(double *v) const;

   void TransformDual(double *v) const;
+
+   void InvTransformDual(double *v) const;
+   using DofTransformation::InvTransformDual;
 };

 /// DoF transformation implementation for the Nedelec basis on tetrahedra
@@ -246,12 +255,15 @@ public:
   using DofTransformation::TransformPrimal;
   using DofTransformation::InvTransformPrimal;
   using DofTransformation::TransformDual;
+   using DofTransformation::InvTransformDual;

   void TransformPrimal(double *v) const;

   void InvTransformPrimal(double *v) const;

   void TransformDual(double *v) const;
+
+   void InvTransformDual(double *v) const;
 };

 /// DoF transformation implementation for the Nedelec basis on wedge elements
@@ -264,12 +276,16 @@ public:
   using DofTransformation::TransformPrimal;
   using DofTransformation::InvTransformPrimal;
   using DofTransformation::TransformDual;
+   using DofTransformation::InvTransformDual;

   void TransformPrimal(double *v) const;

   void InvTransformPrimal(double *v) const;

   void TransformDual(double *v) const;
+
+   void InvTransformDual(double *v) const;
+
 };

 } // namespace mfem
@@ -21,7 +21,8 @@ ElementTransformation::ElementTransformation()
     EvalState(0),
     geom(Geometry::INVALID),
     Attribute(-1),
-     ElementNo(-1)
+     ElementNo(-1),
+     mesh(nullptr)
 { }

 double ElementTransformation::EvalWeight()
@@ -75,6 +75,12 @@ public:

   int Attribute, ElementNo, ElementType;

+   /// The Mesh object containing the element.
+   /** If the element transformation belongs to a mesh, this will point to the
+       containing Mesh object. ElementNo will be the number of the element in
+       this Mesh. This will be NULL if the element does not belong to a mesh. */
+   class Mesh *mesh;
+
   ElementTransformation();

   /** @brief Force the reevaluation of the Jacobian in the next call. */
@@ -749,11 +749,12 @@ public:
                           DenseMatrix &div) const;

   /** @brief Get an Array<int> that maps lexicographically ordered indices to
-       the indices of the respective nodes/dofs/basis functions. Lexicographic
-       ordering of nodes is defined in terms of reference-space coordinates
-       (x,y,z). Lexicographically ordered nodes are listed first in order of
-       increasing x-coordinate, and then in order of increasing y-coordinate,
-       and finally in order of increasing z-coordinate.
+       the indices of the respective nodes/dofs/basis functions.
+
+       Lexicographic ordering of nodes is defined in terms of reference-space
+       coordinates (x,y,z). Lexicographically ordered nodes are listed first in
+       order of increasing x-coordinate, and then in order of increasing
+       y-coordinate, and finally in order of increasing z-coordinate.

       For example, the six nodes of a quadratic triangle are lexicographically
       ordered as follows:
@@ -1709,8 +1709,7 @@ void FiniteElementSpace::RefinementOperator
            fespace->DofsToVDofs(vd, c_vdofs, old_ndofs);

            x.GetSubVector(f_vdofs, subX);
-            old_DoFTrans[geom]->InvTransformPrimal(subX);
-
+            doftrans->InvTransformDual(subX);
            for (int p = 0; p < f_dofs.Size(); ++p)
            {
               if (processed[DecodeDof(f_dofs[p])])
@@ -1719,9 +1718,9 @@ void FiniteElementSpace::RefinementOperator
               }
            }

-            lP.MultTranspose(subX, subY);
-            doftrans->TransformPrimal(subY);
-            y.AddElementVector(c_vdofs, subY);
+            lP.MultTranspose(subX, subYt);
+            old_DoFTrans[geom]->TransformDual(subYt);
+            y.AddElementVector(c_vdofs, subYt);
         }

         if (vdoftrans)
@@ -112,7 +112,7 @@ FmsFieldToGridFunction(FmsMesh fms_mesh, FmsField f, Mesh *mesh,
   // NOTE: transplanted from the FmsMeshToMesh function
   //       We should do this work once and save it.
   //--------------------------------------------------
-   FmsInt dim, n_vert, n_elem, space_dim;
+   FmsInt dim, n_elem, space_dim;

   // Find the first component that has coordinates - that will be the new mfem
   // mesh.
@@ -144,7 +144,6 @@ FmsFieldToGridFunction(FmsMesh fms_mesh, FmsField f, Mesh *mesh,
         n_ents[et] += num_ents;
      }
   }
-   n_vert = n_ents[FMS_VERTEX];
   //--------------------------------------------------

   // Interrogate the field.
@@ -48,6 +48,26 @@ MFEM_HOST_DEVICE inline void LoadB(const int D1D, const int Q1D,
   MFEM_SYNC_THREAD;
 }

+/// Load B1d matrice into shared memory
+MFEM_HOST_DEVICE inline void LoadB(const int D1D, const int Q1D,
+                                   const ConstDeviceMatrix &b,
+                                   const DeviceMatrix &B)
+{
+   const int tidz = MFEM_THREAD_ID(z);
+
+   if (tidz == 0)
+   {
+      MFEM_FOREACH_THREAD(d,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(q,x,Q1D)
+         {
+            B(q,d) = b(q,d);
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
 /// Load Bt1d matrices into shared memory
 template<int MD1, int MQ1>
 MFEM_HOST_DEVICE inline void LoadBt(const int D1D, const int Q1D,
@@ -120,6 +140,29 @@ MFEM_HOST_DEVICE inline void LoadBGt(const int D1D, const int Q1D,
   MFEM_SYNC_THREAD;
 }

+/// Load Bt1d & Gt1d matrices into shared memory w/o MAX
+MFEM_HOST_DEVICE inline void LoadBGt(const int D1D, const int Q1D,
+                                     const ConstDeviceMatrix &b,
+                                     const ConstDeviceMatrix &g,
+                                     const DeviceMatrix &Bt,
+                                     const DeviceMatrix &Gt)
+{
+   const int tidz = MFEM_THREAD_ID(z);
+
+   if (tidz == 0)
+   {
+      MFEM_FOREACH_THREAD(d,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(q,x,Q1D)
+         {
+            Bt(d,q) = b(q,d);
+            Gt(d,q) = g(q,d);
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
 /// Load 2D input scalar into shared memory
 template<int MD1, int NBZ>
 MFEM_HOST_DEVICE inline void LoadX(const int e, const int D1D,
@@ -628,6 +671,141 @@ MFEM_HOST_DEVICE inline void GradXt(const int D1D, const int Q1D,
   MFEM_SYNC_THREAD;
 }

+/// Atomic 2D Transposed Evaluation, 1/2
+MFEM_HOST_DEVICE inline void Atomic2DEvalYt(const int D1D, const int Q1D,
+                                            const DeviceMatrix &B,
+                                            const DeviceMatrix &QQ,
+                                            const DeviceMatrix &QD)
+{
+   MFEM_FOREACH_THREAD(qy,y,Q1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         double u = 0.0;
+         for (int qx = 0; qx < Q1D; ++qx) { u += QQ(qy,qx) * B(qx,dx); }
+         QD(qy,dx) = u;
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+/// Atomic 2D Transposed Evaluation, 2/2
+MFEM_HOST_DEVICE inline void Atomic2DEvalXt(const int D1D, const int Q1D,
+                                            const DeviceMatrix &B,
+                                            const DeviceMatrix &Q,
+                                            const DeviceTensor<3,const int> &I,
+                                            const DeviceMatrix &Y,
+                                            const int c,
+                                            const int e,
+                                            const bool byVDIM)
+{
+   MFEM_FOREACH_THREAD(dy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         double u = 0.0;
+         for (int qy = 0; qy < Q1D; ++qy) { u += Q(qy,dx) * B(qy,dy); }
+         const int gid = I(dx,dy,e);
+         const int idx = gid >= 0 ? gid : -1 - gid;
+         if (byVDIM) { AtomicAdd(Y(c,idx), u); }
+         else { AtomicAdd(Y(idx,c), u); }
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+MFEM_HOST_DEVICE inline
+void Atomic2DEvalTranspose(const int D1D,
+                           const int Q1D,
+                           const DeviceMatrix &B,
+                           const DeviceMatrix &QQ,
+                           const DeviceMatrix &QD,
+                           const DeviceTensor<3,const int> &I,
+                           const DeviceMatrix &Y,
+                           const int c,
+                           const int e,
+                           const bool byVDIM)
+{
+   kernels::internal::Atomic2DEvalYt(D1D,Q1D,B,QQ,QD);
+   kernels::internal::Atomic2DEvalXt(D1D,Q1D,B,QD,I,Y,c,e,byVDIM);
+}
+
+/// Atomic 2D Transposed Gradient, 1/2
+MFEM_HOST_DEVICE inline void AtomicGradYt(const int D1D, const int Q1D,
+                                          const DeviceMatrix &Bt,
+                                          const DeviceMatrix &Gt,
+                                          const DeviceMatrix &QQ0,
+                                          const DeviceMatrix &QQ1,
+                                          const DeviceMatrix &DQ0,
+                                          const DeviceMatrix &DQ1)
+{
+   MFEM_FOREACH_THREAD(qy,y,Q1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         double u = 0.0, v = 0.0;
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            u += Gt(dx,qx) * QQ0(qy,qx);
+            v += Bt(dx,qx) * QQ1(qy,qx);
+         }
+         DQ0(dx,qy) = u;
+         DQ1(dx,qy) = v;
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+/// Atomic 2D Transposed Gradient, 2/2
+MFEM_HOST_DEVICE inline void AtomicGradXt(const int D1D, const int Q1D,
+                                          const DeviceMatrix &Bt,
+                                          const DeviceMatrix &Gt,
+                                          const DeviceMatrix &DQ0,
+                                          const DeviceMatrix &DQ1,
+                                          const DeviceTensor<3,const int> &I,
+                                          const DeviceMatrix &Y,
+                                          const int c,
+                                          const int e,
+                                          const bool byVDIM)
+{
+   MFEM_FOREACH_THREAD(dy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         double u = 0.0, v = 0.0;
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            u += DQ0(dx,qy) * Bt(dy,qy);
+            v += DQ1(dx,qy) * Gt(dy,qy);
+         }
+         const double sum = u + v;
+         const int gid = I(dx,dy,e);
+         const int idx = gid >= 0 ? gid : -1-gid;
+         if (byVDIM) { AtomicAdd(Y(c,idx), sum); }
+         else { AtomicAdd(Y(idx,c), sum); }
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+MFEM_HOST_DEVICE inline
+void Atomic2DGradTranspose(const int D1D, const int Q1D,
+                           const DeviceMatrix &Bt,
+                           const DeviceMatrix &Gt,
+                           const DeviceMatrix &QQ0,
+                           const DeviceMatrix &QQ1,
+                           const DeviceMatrix &DQ0,
+                           const DeviceMatrix &DQ1,
+                           const DeviceTensor<3,const int> &I,
+                           const DeviceMatrix &Y,
+                           const int c,
+                           const int e,
+                           const bool byVDIM)
+{
+   kernels::internal::AtomicGradYt(D1D,Q1D,Bt,Gt,QQ0,QQ1,DQ0,DQ1);
+   kernels::internal::AtomicGradXt(D1D,Q1D,Bt,Gt,DQ0,DQ1,I,Y,c,e,byVDIM);
+}
+
 /// Load 3D scalar input vector into shared memory
 MFEM_HOST_DEVICE inline void LoadX(const int e, const int D1D,
                                   const DeviceTensor<4, const double> &x,
@@ -1543,6 +1721,269 @@ MFEM_HOST_DEVICE inline void GradXt(const int D1D, const int Q1D,
   }
 }

+/// Atomic 3D Transposed Evaluation, 1/3
+MFEM_HOST_DEVICE inline void Atomic3DEvalZt(const int D1D, const int Q1D,
+                                            double *u,
+                                            const DeviceMatrix &B,
+                                            const DeviceCube &Q)
+{
+   MFEM_FOREACH_THREAD(qx,x,Q1D)
+   {
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            const double ZYX = Q(qz,qy,qx);
+            for (int dz = 0; dz < D1D; ++dz) { u[dz] += ZYX * B(qz,dz); }
+         }
+         for (int dz = 0; dz < D1D; ++dz) { Q(dz,qy,qx) = u[dz]; }
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+/// Atomic 3D Transposed Evaluation, 2/3
+MFEM_HOST_DEVICE inline void Atomic3DEvalYt(const int D1D, const int Q1D,
+                                            double *u,
+                                            const DeviceMatrix &B,
+                                            const DeviceCube &Q)
+{
+   MFEM_FOREACH_THREAD(dz,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         for (int dy = 0; dy < D1D; ++dy) { u[dy] = 0.0; }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            const double zYX = Q(dz,qy,qx);
+            for (int dy = 0; dy < D1D; ++dy) { u[dy] += zYX * B(qy,dy); }
+         }
+         for (int dy = 0; dy < D1D; ++dy) { Q(dz,dy,qx) = u[dy]; }
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+/// Atomic 3D Transposed Evaluation, 3/3
+MFEM_HOST_DEVICE inline void Atomic3DEvalXt(const int D1D, const int Q1D,
+                                            double *u,
+                                            const DeviceMatrix &B,
+                                            const DeviceCube &Q,
+                                            const DeviceTensor<4,const int> &I,
+                                            const DeviceMatrix &Y,
+                                            const int c,
+                                            const int e,
+                                            const bool byVDIM)
+{
+   MFEM_FOREACH_THREAD(dz,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(dy,x,D1D)
+      {
+         for (int dx = 0; dx < D1D; ++dx) { u[dx] = 0.0; }
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            const double zyX = Q(dz,dy,qx);
+            for (int dx = 0; dx < D1D; ++dx) { u[dx] += zyX * B(qx,dx); }
+         }
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            const double val = u[dx];
+            const int gid = I(dx,dy,dz,e);
+            const int idx = gid >= 0 ? gid : -1 - gid;
+            if (byVDIM) { AtomicAdd(Y(c,idx), val); }
+            else { AtomicAdd(Y(idx,c), val); }
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+MFEM_HOST_DEVICE inline
+void Atomic3DEvalTranspose(const int D1D,
+                           const int Q1D,
+                           double *u,
+                           const DeviceMatrix &B,
+                           const DeviceCube &Q,
+                           const DeviceTensor<4,const int> &I,
+                           const DeviceMatrix &Y,
+                           const int c,
+                           const int e,
+                           const bool byVDIM)
+{
+   kernels::internal::Atomic3DEvalZt(D1D,Q1D,u,B,Q);
+   kernels::internal::Atomic3DEvalYt(D1D,Q1D,u,B,Q);
+   kernels::internal::Atomic3DEvalXt(D1D,Q1D,u,B,Q,I,Y,c,e,byVDIM);
+}
+
+// Half of B and G are stored in shared to get B, Bt, G and Gt.
+// Indices computation for SmemPADiffusionApply3D.
+static MFEM_HOST_DEVICE inline int qi(const int q, const int d, const int Q)
+{
+   return (q<=d) ? q : Q-1-q;
+}
+
+static MFEM_HOST_DEVICE inline int dj(const int q, const int d, const int D)
+{
+   return (q<=d) ? d : D-1-d;
+}
+
+static MFEM_HOST_DEVICE inline int qk(const int q, const int d, const int Q)
+{
+   return (q<=d) ? Q-1-q : q;
+}
+
+static MFEM_HOST_DEVICE inline int dl(const int q, const int d, const int D)
+{
+   return (q<=d) ? D-1-d : d;
+}
+
+static MFEM_HOST_DEVICE inline double sign(const int q, const int d)
+{
+   return (q<=d) ? -1.0 : 1.0;
+}
+
+/// Atomic 3D Transposed Gradient, 1/3
+MFEM_HOST_DEVICE inline void AtomicGradZt(const int D1D, const int Q1D,
+                                          const DeviceMatrix &Bt,
+                                          const DeviceMatrix &Gt,
+                                          const DeviceCube &QQQ0,
+                                          const DeviceCube &QQQ1,
+                                          const DeviceCube &QQQ2,
+                                          const DeviceCube &QQD0,
+                                          const DeviceCube &QQD1,
+                                          const DeviceCube &QQD2)
+{
+   MFEM_FOREACH_THREAD(qz,z,Q1D)
+   {
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u = 0.0, v = 0.0, w = 0.0;
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const int i = qi(qx,dx,Q1D);
+               const int j = dj(qx,dx,D1D);
+               const int k = qk(qx,dx,Q1D);
+               const int l = dl(qx,dx,D1D);
+               const double s = sign(qx,dx);
+               u += QQQ0(qz,qy,qx) * Gt(l,k) * s;
+               v += QQQ1(qz,qy,qx) * Bt(j,i);
+               w += QQQ2(qz,qy,qx) * Bt(j,i);
+            }
+            QQD0(qz,qy,dx) = u;
+            QQD1(qz,qy,dx) = v;
+            QQD2(qz,qy,dx) = w;
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+/// Atomic 3D Transposed Gradient, 2/3
+MFEM_HOST_DEVICE inline void AtomicGradYt(const int D1D, const int Q1D,
+                                          const DeviceMatrix &Bt,
+                                          const DeviceMatrix &Gt,
+                                          const DeviceCube &QQD0,
+                                          const DeviceCube &QQD1,
+                                          const DeviceCube &QQD2,
+                                          const DeviceCube &QDD0,
+                                          const DeviceCube &QDD1,
+                                          const DeviceCube &QDD2)
+{
+   MFEM_FOREACH_THREAD(qz,z,Q1D)
+   {
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u = 0.0, v = 0.0, w = 0.0;
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const int i = qi(qy,dy,Q1D);
+               const int j = dj(qy,dy,D1D);
+               const int k = qk(qy,dy,Q1D);
+               const int l = dl(qy,dy,D1D);
+               const double s = sign(qy,dy);
+               u += QQD0(qz,qy,dx) * Bt(j,i);
+               v += QQD1(qz,qy,dx) * Gt(l,k) * s;
+               w += QQD2(qz,qy,dx) * Bt(j,i);
+            }
+            QDD0(qz,dy,dx) = u;
+            QDD1(qz,dy,dx) = v;
+            QDD2(qz,dy,dx) = w;
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+/// Atomic 3D Transposed Gradient, 3/3
+MFEM_HOST_DEVICE inline void AtomicGradXt(const int D1D, const int Q1D,
+                                          const DeviceMatrix &Bt,
+                                          const DeviceMatrix &Gt,
+                                          const DeviceCube &QDD0,
+                                          const DeviceCube &QDD1,
+                                          const DeviceCube &QDD2,
+                                          const DeviceTensor<4,const int> &I,
+                                          const DeviceMatrix &Y,
+                                          const int c,
+                                          const int e,
+                                          const bool byVDIM)
+{
+   MFEM_FOREACH_THREAD(dz,z,D1D)
+   {
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u = 0.0, v = 0.0, w = 0.0;
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               const int i = qi(qz,dz,Q1D);
+               const int j = dj(qz,dz,D1D);
+               const int k = qk(qz,dz,Q1D);
+               const int l = dl(qz,dz,D1D);
+               const double s = sign(qz,dz);
+               u += QDD0(qz,dy,dx) * Bt(j,i);
+               v += QDD1(qz,dy,dx) * Bt(j,i);
+               w += QDD2(qz,dy,dx) * Gt(l,k) * s;
+            }
+            const double sum = u + v + w;
+            const int gid = I(dx,dy,dz,e);
+            const int idx = gid >= 0 ? gid : -1-gid;
+            if (byVDIM) { AtomicAdd(Y(c,idx), sum); }
+            else { AtomicAdd(Y(idx,c), sum); }
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+MFEM_HOST_DEVICE inline void Atomic3DGrad(const int D1D, const int Q1D,
+                                          const DeviceMatrix &Bt,
+                                          const DeviceMatrix &Gt,
+                                          const DeviceCube &QQ0,
+                                          const DeviceCube &QQ1,
+                                          const DeviceCube &QQ2,
+                                          const DeviceCube &QD0,
+                                          const DeviceCube &QD1,
+                                          const DeviceCube &QD2,
+                                          const DeviceCube &DD0,
+                                          const DeviceCube &DD1,
+                                          const DeviceCube &DD2,
+                                          const DeviceTensor<4,const int> &I,
+                                          const DeviceMatrix &Y,
+                                          const int c,
+                                          const int e,
+                                          const bool byVDIM)
+{
+   kernels::internal::AtomicGradZt(D1D,Q1D,Bt,Gt,QQ0,QQ1,QQ2,QD0,QD1,QD2);
+   kernels::internal::AtomicGradYt(D1D,Q1D,Bt,Gt,QD0,QD1,QD2,DD0,DD1,DD2);
+   kernels::internal::AtomicGradXt(D1D,Q1D,Bt,Gt,DD0,DD1,DD2,I,Y,c,e,byVDIM);
+}
+
 } // namespace kernels::internal

 } // namespace kernels
@@ -23,6 +23,7 @@ LinearForm::LinearForm(FiniteElementSpace *f, LinearForm *lf)
   UseDevice(true);

   fes = f;
+   ext = nullptr;
   extern_lfs = 1;

   // Copy the pointers to the integrators
@@ -69,14 +70,14 @@ void LinearForm::AddDomainIntegrator(LinearFormIntegrator *lfi,

 void LinearForm::AddBoundaryIntegrator (LinearFormIntegrator * lfi)
 {
-   boundary_integs.Append (lfi);
+   boundary_integs.Append(lfi);
   boundary_integs_marker.Append(NULL); // NULL -> all attributes are active
 }

 void LinearForm::AddBoundaryIntegrator (LinearFormIntegrator * lfi,
                                        Array<int> &bdr_attr_marker)
 {
-   boundary_integs.Append (lfi);
+   boundary_integs.Append(lfi);
   boundary_integs_marker.Append(&bdr_attr_marker);
 }

@@ -99,8 +100,35 @@ void LinearForm::AddInteriorFaceIntegrator(LinearFormIntegrator *lfi)
   interior_face_integs.Append(lfi);
 }

+void LinearForm::SetAssemblyLevel(LinearAssemblyLevel assembly_level)
+{
+   if (ext)
+   {
+      MFEM_ABORT("the assembly level has already been set!");
+   }
+   assembly = assembly_level;
+   switch (assembly)
+   {
+      case LinearAssemblyLevel::LEGACY:
+         break;
+      case LinearAssemblyLevel::FULL:
+         ext = new FullLinearFormExtension(this);
+         break;
+      default:
+         mfem_error("Unknown assembly level");
+   }
+}
+
 void LinearForm::Assemble()
 {
+   Vector::operator=(0.0);
+
+   // The above operation is executed on device because of UseDevice().
+   // The first use of AddElementVector() below will move it back to host
+   // because both 'vdofs' and 'elemvect' are on host.
+
+   if (ext) { return ext->Assemble(); }
+
   Array<int> vdofs;
   ElementTransformation *eltrans;
   DofTransformation *doftrans;
@@ -108,12 +136,6 @@ void LinearForm::Assemble()

   int i;

-   Vector::operator=(0.0);
-
-   // The above operation is executed on device because of UseDevice().
-   // The first use of AddElementVector() below will move it back to host
-   // because both 'vdofs' and 'elemvect' are on host.
-
   if (domain_integs.Size())
   {
      for (int k = 0; k < domain_integs.Size(); k++)
@@ -143,7 +165,7 @@ void LinearForm::Assemble()
               {
                  doftrans->TransformDual(elemvect);
               }
-               AddElementVector (vdofs, elemvect);
+               AddElementVector(vdofs, elemvect);
            }
         }
      }
@@ -274,6 +296,18 @@ void LinearForm::Assemble()
   }
 }

+void LinearForm::Update()
+{
+   SetSize(fes->GetVSize()); ResetDeltaLocations();
+   if (ext) { ext->Update(); }
+}
+
+void LinearForm::Update(FiniteElementSpace *f)
+{
+   fes = f;
+   Update();
+}
+
 void LinearForm::Update(FiniteElementSpace *f, Vector &v, int v_offset)
 {
   MFEM_ASSERT(v.Size() >= v_offset + f->GetVSize(), "");
@@ -281,6 +315,7 @@ void LinearForm::Update(FiniteElementSpace *f, Vector &v, int v_offset)
   v.UseDevice(true);
   this->Vector::MakeRef(v, v_offset, fes->GetVSize());
   ResetDeltaLocations();
+   if (ext) { ext->Update(); }
 }

 void LinearForm::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
@@ -292,6 +327,8 @@ void LinearForm::AssembleDelta()
 {
   if (domain_delta_integs.Size() == 0) { return; }

+   if (ext) { return ext->AssembleDelta(); }
+
   if (!HaveDeltaLocations())
   {
      int sdim = fes->GetMesh()->SpaceDimension();
@@ -356,6 +393,8 @@ LinearForm::~LinearForm()
      for (k=0; k < interior_face_integs.Size(); k++)
      { delete interior_face_integs[k]; }
   }
+
+   delete ext;
 }

 }
@@ -14,18 +14,39 @@

 #include "../config/config.hpp"
 #include "lininteg.hpp"
+#include "linearform_ext.hpp"
 #include "gridfunc.hpp"

 namespace mfem
 {

+/** @brief Enumeration defining the assembly level for linear form classes
+ *  derived from Vector. */
+enum class LinearAssemblyLevel
+{
+   /// Legacy assembled form, i.e. a global vector in MFEM format.
+   /// This assembly level is ALWAYS performed on the host.
+   LEGACY = 0,
+   /// Fully assembled form, i.e. a global vector in MFEM format.
+   /// This assembly is compatible with device execution.
+   FULL
+};
+
 /// Vector with associated FE space and LinearFormIntegrators.
 class LinearForm : public Vector
 {
+   friend LinearFormExtension;
+
 protected:
   /// FE space on which the LinearForm lives. Not owned.
   FiniteElementSpace *fes;

+   /// The assembly level of the form (legacy or full)
+   LinearAssemblyLevel assembly;
+
+   /** @brief Extension for supporting different assembly levels. */
+   LinearFormExtension *ext;
+
   /** @brief Indicates the LinearFormIntegrator%s stored in #domain_integs,
       #domain_delta_integs, #boundary_integs, and #boundary_face_integs are
       owned by another LinearForm. */
@@ -75,7 +96,7 @@ public:
   /// Creates linear form associated with FE space @a *f.
   /** The pointer @a f is not owned by the newly constructed object. */
   LinearForm(FiniteElementSpace *f) : Vector(f->GetVSize())
-   { fes = f; extern_lfs = 0; UseDevice(true); }
+   { fes = f; ext = nullptr; extern_lfs = 0; UseDevice(true); }

   /** @brief Create a LinearForm on the FiniteElementSpace @a f, using the
       same integrators as the LinearForm @a lf.
@@ -90,7 +111,7 @@ public:
   /** The associated FiniteElementSpace can be set later using one of the
       methods: Update(FiniteElementSpace *) or
       Update(FiniteElementSpace *, Vector &, int). */
-   LinearForm() { fes = NULL; extern_lfs = 0; UseDevice(true); }
+   LinearForm() { fes = NULL; ext = nullptr; extern_lfs = 0; UseDevice(true); }

   /// Construct a LinearForm using previously allocated array @a data.
   /** The LinearForm does not assume ownership of @a data which is assumed to
@@ -98,7 +119,7 @@ public:
       for externally allocated array, the pointer @a data can be NULL. The data
       array can be replaced later using the method SetData(). */
   LinearForm(FiniteElementSpace *f, double *data) : Vector(data, f->GetVSize())
-   { fes = f; extern_lfs = 0; }
+   { fes = f; ext = nullptr; extern_lfs = 0; }

   /// Copy assignment. Only the data of the base class Vector is copied.
   /** It is assumed that this object and @a rhs use FiniteElementSpace%s that
@@ -155,6 +176,9 @@ public:
       coefficients. */
   Array<LinearFormIntegrator*> *GetDLFI() { return &domain_integs; }

+   /// Access the element attribute marker (of length mesh->attributes)
+   Array<Array<int>*> *GetDLFIM() { return &domain_integs_marker; }
+
   /** @brief Access all integrators added with AddDomainIntegrator() which are
       DeltaLFIntegrator%s with delta coefficients. */
   Array<DeltaLFIntegrator*> *GetDLFI_Delta() { return &domain_delta_integs; }
@@ -173,6 +197,10 @@ public:
       corresponding pointer (to Array<int>) will be NULL. */
   Array<Array<int>*> *GetFLFI_Marker() { return &boundary_face_integs_marker; }

+   /// Set the desired assembly level, default is LinearAssemblyLevel::LEGACY.
+   /** This method must be called before assembly. */
+   void SetAssemblyLevel(LinearAssemblyLevel);
+
   /// Assembles the linear form i.e. sums over all domain/bdr integrators.
   void Assemble();

@@ -184,11 +212,10 @@ public:
       updated, e.g. after its associated Mesh object has been refined.

       @note This method does not perform assembly. */
-   void Update() { SetSize(fes->GetVSize()); ResetDeltaLocations(); }
+   void Update();

   /// Associate a new FE space, @a *f, with this object and Update() it. */
-   void Update(FiniteElementSpace *f)
-   { fes = f; SetSize(f->GetVSize()); ResetDeltaLocations(); }
+   void Update(FiniteElementSpace *f);

   /** @brief Associate a new FE space, @a *f, with this object and use the data
       of @a v, offset by @a v_offset, to initialize this object's Vector::data.
@@ -0,0 +1,104 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "linearform.hpp"
+#include "../general/forall.hpp"
+
+// Implementations of FullLinearFormExtension.
+
+namespace mfem
+{
+
+FullLinearFormExtension::FullLinearFormExtension(LinearForm *lf):
+   LinearFormExtension(lf)
+{
+   Update();
+}
+
+void FullLinearFormExtension::Assemble()
+{
+   MFEM_VERIFY(lf->Size() == lf->FESpace()->GetVSize(), "");
+
+   // Filter out the unsupported integrators
+   MFEM_VERIFY(lf->GetBLFI()->Size() == 0,
+               "Integrators added with AddBoundaryIntegrator() "
+               "are not supported!");
+
+   MFEM_VERIFY(lf->GetDLFI_Delta()->Size() == 0, ""
+               "Integrators added with AddDomainIntegrator() which are "
+               "DeltaLFIntegrators with delta coefficients "
+               "are not supported!");
+
+   MFEM_VERIFY(lf->GetIFLFI()->Size() == 0,
+               "Integrators added with AddInteriorFaceIntegrator() "
+               "are not supported!");
+
+   MFEM_VERIFY(lf->GetFLFI()->Size() == 0,
+               "Integrators added with AddBdrFaceIntegrator() "
+               " are not supported!");
+
+   const FiniteElementSpace &fes = *lf->FESpace();
+   const Array<Array<int>*> &domain_integs_marker = *lf->GetDLFIM();
+   const int mesh_attributes_size = fes.GetMesh()->attributes.Size();
+   const Array<LinearFormIntegrator*> &domain_integs = *lf->GetDLFI();
+
+   for (int k = 0; k < domain_integs.Size(); ++k)
+   {
+      // Get the markers for this integrator
+      const Array<int> *domain_integs_marker_k = domain_integs_marker[k];
+
+      // check if there are markers for this integrator
+      const bool has_markers_k = domain_integs_marker_k != nullptr;
+
+      if (has_markers_k)
+      {
+         // Element attribute marker should be of length mesh->attributes
+         MFEM_VERIFY(mesh_attributes_size == domain_integs_marker_k->Size(),
+                     "invalid element marker for domain linear form "
+                     "integrator #" << k << ", counting from zero");
+      }
+
+      const int NE = fes.GetNE();
+      auto markers_w = markers.Write();
+
+      // if there are no markers, just use the whole linear form (1)
+      if (!has_markers_k)
+      {
+         // done this way as operator= for array are still done on the host
+         MFEM_FORALL(e, NE, markers_w[e] = 1;);
+      }
+      else
+      {
+         // otherwise, scan the attributes to set the markers to 0 or 1
+         const auto attr = attributes.Read();
+         const auto dimk = domain_integs_marker_k->Read();
+         MFEM_FORALL(e, NE, markers_w[e] = dimk[attr[e]-1] == 1;);
+      }
+
+      domain_integs[k]->AssembleFull(fes, markers, *lf);
+   }
+}
+
+void FullLinearFormExtension::Update()
+{
+   MFEM_VERIFY(lf->Size() == lf->FESpace()->GetVSize(), "");
+
+   const int ne = lf->FESpace()->GetNE();
+   const Mesh &mesh = *lf->FESpace()->GetMesh();
+
+   markers.SetSize(ne);
+
+   // Gather the attributes on the host from all the elements
+   attributes.SetSize(ne);
+   for (int i = 0; i < ne; ++i) { attributes[i] = mesh.GetAttribute(i); }
+}
+
+} // namespace mfem
@@ -0,0 +1,71 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_LINEARFORM_EXT
+#define MFEM_LINEARFORM_EXT
+
+#include "../general/array.hpp"
+
+namespace mfem
+{
+
+class LinearForm;
+class FullLinearFormExtension;
+
+/// Class extending the LinearForm class to support different AssemblyLevels.
+class LinearFormExtension
+{
+protected:
+   /// Linear form from which this extension depends. Not owned.
+   LinearForm *lf;
+
+public:
+   LinearFormExtension(LinearForm *lf): lf(lf) { }
+
+   virtual ~LinearFormExtension() { }
+
+   /// Updates the linear form extension
+   virtual void Update() = 0;
+
+   /// Assemble at the level given for the linear form extension
+   virtual void Assemble() = 0;
+
+   /// Assembles delta functions of the linear form extension
+   virtual void AssembleDelta() = 0;
+};
+
+/// Data and methods for fully-assembled linear forms
+class FullLinearFormExtension : public LinearFormExtension
+{
+private:
+   /// Attributes of all mesh elements.
+   Array<int> attributes;
+
+   /// Temporary markers for device kernels.
+   Array<int> markers;
+
+public:
+   FullLinearFormExtension(LinearForm *lf);
+
+   /// Fully assembles the linear form, compatible with device execution.
+   /// Only integrators added with AddDomainIntegrator are supported.
+   void Assemble() override;
+
+   /// Fully assembles the delta functions of the linear form.
+   /// Not yet supported.
+   void AssembleDelta() override { MFEM_ABORT("Not yet supported!"); }
+
+   void Update() override;
+};
+
+} // namespace mfem
+
+#endif // MFEM_LINEARFORM_EXT
@@ -9,13 +9,21 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.

-
 #include "fem.hpp"
 #include <cmath>

 namespace mfem
 {

+void LinearFormIntegrator::AssembleFull(const FiniteElementSpace&,
+                                        const Array<int> &markers,
+                                        Vector &b)
+{
+   MFEM_CONTRACT_VAR(b);
+   MFEM_CONTRACT_VAR(markers);
+   mfem_error("LinearFormIntegrator::AssembleFull(...)");
+}
+
 void LinearFormIntegrator::AssembleRHSElementVect(
   const FiniteElement &el, FaceElementTransformations &Tr, Vector &elvect)
 {
@@ -280,12 +288,11 @@ void VectorDomainLFIntegrator::AssembleRHSElementVect(
      val = Tr.Weight();

      el.CalcShape(ip, shape);
-      Q.Eval (Qvec, Tr, ip);
+      Q.Eval(Qvec, Tr, ip);

      for (int k = 0; k < vdim; k++)
      {
         cf = val * Qvec(k);
-
         for (int s = 0; s < dof; s++)
         {
            elvect(dof*k+s) += ip.weight * cf * shape(s);
@@ -311,6 +318,54 @@ void VectorDomainLFIntegrator::AssembleDeltaElementVect(
   MultVWt(shape, Qvec, elvec_as_mat);
 }

+void VectorDomainLFGradIntegrator::AssembleRHSElementVect(
+   const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
+{
+   const int dim = el.GetDim();
+   const int dof = el.GetDof();
+   const int vdim = Q.GetVDim();
+   const int sdim = Tr.GetSpaceDim();
+
+   dshape.SetSize(dof,sdim);
+
+   elvect.SetSize(dof*(vdim/sdim));
+   elvect = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int intorder = 2 * el.GetOrder();
+      ir = &IntRules.Get(el.GetGeomType(), intorder);
+   }
+
+   Vector pelvect(dof);
+   Vector part_x(dim);
+
+   for (int q = 0; q < ir->GetNPoints(); q++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(q);
+
+      Tr.SetIntPoint(&ip);
+      el.CalcPhysDShape(Tr, dshape);
+
+      Q.Eval(Qvec, Tr, ip);
+      Qvec *= ip.weight * Tr.Weight();
+
+      for (int k = 0; k < vdim/sdim; k++)
+      {
+         for (int d=0; d < sdim; ++d) { part_x(d) = Qvec(k*sdim+d); }
+         dshape.Mult(part_x, pelvect);
+         for (int s = 0; s < dof; ++s) { elvect(s+k*dof) += pelvect(s); }
+      }
+   }
+}
+
+void VectorDomainLFGradIntegrator::AssembleDeltaElementVect(
+   const FiniteElement&, ElementTransformation&, Vector&)
+{
+   MFEM_ABORT("Not implemented!");
+}
+
 void VectorBoundaryLFIntegrator::AssembleRHSElementVect(
   const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
 {
@@ -27,6 +27,11 @@ protected:
   LinearFormIntegrator(const IntegrationRule *ir = NULL) { IntRule = ir; }

 public:
+   /// Method defining full assembly
+   virtual void AssembleFull(const FiniteElementSpace &fes,
+                             const Array<int> &markers,
+                             Vector &b);
+
   /** Given a particular Finite Element and a transformation (Tr)
       computes the element vector, elvect. */
   virtual void AssembleRHSElementVect(const FiniteElement &el,
@@ -110,6 +115,10 @@ public:
   DomainLFIntegrator(Coefficient &QF, const IntegrationRule *ir)
      : DeltaLFIntegrator(QF, ir), Q(QF), oa(1), ob(1) { }

+   virtual void AssembleFull(const FiniteElementSpace &fes,
+                             const Array<int> &markers,
+                             Vector &b);
+
   /** Given a particular Finite Element and a transformation (Tr)
       computes the element right hand side element vector, elvect. */
   virtual void AssembleRHSElementVect(const FiniteElement &el,
@@ -136,6 +145,10 @@ public:
   DomainLFGradIntegrator(VectorCoefficient &QF)
      : DeltaLFIntegrator(QF), Q(QF) { }

+   virtual void AssembleFull(const FiniteElementSpace &fes,
+                             const Array<int> &markers,
+                             Vector &b);
+
   /** Given a particular Finite Element and a transformation (Tr)
       computes the element right hand side element vector, elvect. */
   virtual void AssembleRHSElementVect(const FiniteElement &el,
@@ -223,6 +236,10 @@ public:
   VectorDomainLFIntegrator(VectorCoefficient &QF)
      : DeltaLFIntegrator(QF), Q(QF) { }

+   virtual void AssembleFull(const FiniteElementSpace &fes,
+                             const Array<int> &markers,
+                             Vector &b);
+
   /** Given a particular Finite Element and a transformation (Tr)
       computes the element right hand side element vector, elvect. */
   virtual void AssembleRHSElementVect(const FiniteElement &el,
@@ -236,6 +253,37 @@ public:
   using LinearFormIntegrator::AssembleRHSElementVect;
 };

+/** Class for domain integrator L(v) := (f, grad v), where
+    f=(f1x,f1y,f1z,...,fnx,fny,fnz) and v=(v1,...,vn). */
+class VectorDomainLFGradIntegrator : public DeltaLFIntegrator
+{
+private:
+   Vector shape, Qvec;
+   VectorCoefficient &Q;
+   DenseMatrix dshape;
+
+public:
+   /// Constructs the domain integrator (Q, grad v)
+   VectorDomainLFGradIntegrator(VectorCoefficient &QF)
+      : DeltaLFIntegrator(QF), Q(QF) { }
+
+   virtual void AssembleFull(const FiniteElementSpace &fes,
+                             const Array<int> &markers,
+                             Vector &b) override;
+
+   /** Given a particular Finite Element and a transformation (Tr)
+       computes the element right hand side element vector, elvect. */
+   virtual void AssembleRHSElementVect(const FiniteElement &el,
+                                       ElementTransformation &Tr,
+                                       Vector &elvect) override;
+
+   virtual void AssembleDeltaElementVect(const FiniteElement &fe,
+                                         ElementTransformation &Trans,
+                                         Vector &elvect) override;
+
+   using LinearFormIntegrator::AssembleRHSElementVect;
+};
+
 /** Class for boundary integration of L(v) := (g, v), where
    f=(f1,...,fn) and v=(v1,...,vn). */
 class VectorBoundaryLFIntegrator : public LinearFormIntegrator
@@ -0,0 +1,104 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "fem.hpp"
+#include "lininteg_domain.hpp"
+
+namespace mfem
+{
+
+using namespace internal::linearform_extension;
+
+void DomainLFIntegrator::AssembleFull(const FiniteElementSpace &fes,
+                                      const Array<int> &markers,
+                                      Vector &y)
+{
+   const int vdim = fes.GetVDim();
+   MFEM_VERIFY(vdim == 1, "vdim should be equal to 1!");
+   GetOrder_f qorder_fct = [&](int el_order) { return oa * el_order + ob; };
+   const IntegrationRule *ir = GetIntRuleFromOrder(fes, IntRule, qorder_fct);
+
+   Vector coeff;
+   const int NQ = ir->GetNPoints();
+   const int NE = fes.GetMesh()->GetNE();
+
+   if (ConstantCoefficient *cQ =
+          dynamic_cast<ConstantCoefficient*>(&Q))
+   {
+      coeff.SetSize(1);
+      coeff(0) = cQ->constant;
+   }
+   else if (QuadratureFunctionCoefficient *cQ =
+               dynamic_cast<QuadratureFunctionCoefficient*>(&Q))
+   {
+      const QuadratureFunction &qfun = cQ->GetQuadFunction();
+      MFEM_VERIFY(qfun.Size() == vdim*NE*NQ,
+                  "Incompatible QuadratureFunction dimension \n");
+      MFEM_VERIFY(ir == &qfun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different.\n");
+      qfun.Read();
+      coeff.MakeRef(const_cast<QuadratureFunction&>(qfun),0);
+   }
+   else
+   {
+      coeff.SetSize(NQ * NE);
+      auto C = Reshape(coeff.HostWrite(), NQ, NE);
+      for (int e = 0; e < NE; ++e)
+      {
+         ElementTransformation& T = *fes.GetElementTransformation(e);
+         for (int q = 0; q < NQ; ++q)
+         {
+            C(q,e) = Q.Eval(T, ir->IntPoint(q));
+         }
+      }
+   }
+
+   const int id = GetKernelId(fes,ir);
+   const int dim = fes.GetMesh()->Dimension();
+
+   LinearFormExtensionKernel_f ker = nullptr;
+   if (dim==2) { ker=VectorDomainLFIntegratorAssemble2D; }
+   if (dim==3) { ker=VectorDomainLFIntegratorAssemble3D; }
+
+   switch (id)
+   {
+      // 2D kernels, q=p+1
+      case 0x222: ker=VectorDomainLFIntegratorAssemble2D<2,2>; break;
+      case 0x233: ker=VectorDomainLFIntegratorAssemble2D<3,3>; break;
+      case 0x244: ker=VectorDomainLFIntegratorAssemble2D<4,4>; break;
+      case 0x255: ker=VectorDomainLFIntegratorAssemble2D<5,5>; break;
+
+      // 2D kernels, q=p+2
+      case 0x223: ker=VectorDomainLFIntegratorAssemble2D<2,3>; break;
+      case 0x234: ker=VectorDomainLFIntegratorAssemble2D<3,4>; break;
+      case 0x245: ker=VectorDomainLFIntegratorAssemble2D<4,5>; break;
+      case 0x256: ker=VectorDomainLFIntegratorAssemble2D<5,6>; break;
+
+      // 3D kernels, q=p+1, BENCH_FULL_DomainLF_3D_GLL
+      case 0x322: ker=VectorDomainLFIntegratorAssemble3D<2,2>; break;
+      case 0x333: ker=VectorDomainLFIntegratorAssemble3D<3,3>; break;
+      case 0x344: ker=VectorDomainLFIntegratorAssemble3D<4,4>; break;
+      case 0x355: ker=VectorDomainLFIntegratorAssemble3D<5,5>; break;
+      case 0x366: ker=VectorDomainLFIntegratorAssemble3D<6,6>; break;
+
+      // 3D kernels, q=p+2, BENCH_FULL_DomainLF_3D_GL
+      case 0x323: ker=VectorDomainLFIntegratorAssemble3D<2,3>; break;
+      case 0x334: ker=VectorDomainLFIntegratorAssemble3D<3,4>; break;
+      case 0x345: ker=VectorDomainLFIntegratorAssemble3D<4,5>; break;
+      case 0x356: ker=VectorDomainLFIntegratorAssemble3D<5,6>; break;
+      case 0x367: ker=VectorDomainLFIntegratorAssemble3D<6,7>; break;
+   }
+   MFEM_VERIFY(ker, "Unexpected kernel error!");
+   Launch(ker,fes,ir,coeff,markers,y);
+}
+
+} // namespace mfem
@@ -0,0 +1,263 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+// Internal header, included only by .cpp files
+
+#include "fem.hpp"
+
+#include "../fem/kernels.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+namespace internal
+{
+
+namespace linearform_extension
+{
+
+/// Signature of the kernels used for linear form extension
+using LinearFormExtensionKernel_f = void (*)(const int vdim,
+                                             const bool byVDIM,
+                                             const int ND,
+                                             const int NE,
+                                             const int d,
+                                             const int q,
+                                             const int *markers,
+                                             const double *b,
+                                             const double *g,
+                                             const int *idx,
+                                             const double *J,
+                                             const double *detJ,
+                                             const double *weights,
+                                             const Vector &coeff,
+                                             double *output);
+
+/// Signature of the function used to compute the quadrature order
+using GetOrder_f = std::function<int(int)>;
+
+/// Internal helper function to get the integration rule
+inline const IntegrationRule *GetIntRuleFromOrder(const FiniteElementSpace &fes,
+                                                  const IntegrationRule *IntRule,
+                                                  const GetOrder_f &qorder_fct)
+{
+   const FiniteElement &fe = *fes.GetFE(0);
+   const int qorder = qorder_fct(fe.GetOrder());
+   const Geometry::Type geom_type = fe.GetGeomType();
+   return IntRule ? IntRule : &IntRules.Get(geom_type, qorder);
+}
+
+/// Internal helper function to encode the ID of a LinearFormExtension kernel
+/// The ID is equal to: (dim << 8) |
+///                     (1D number of degrees of freedom << 4) |
+///                     (1D number of quadrature points)
+inline int GetKernelId(const FiniteElementSpace &fes,
+                       const IntegrationRule *ir)
+{
+   Mesh *mesh = fes.GetMesh();
+   const int dim = mesh->Dimension();
+   const FiniteElement &el = *fes.GetFE(0);
+   const DofToQuad &maps = el.GetDofToQuad(*ir, DofToQuad::TENSOR);
+   const int d = maps.ndof;
+   const int q = maps.nqpt;
+   return (dim << 8) | (d << 4) | q;
+}
+
+/// Internal helper function to launch the LinearFormExtension kernel
+inline void Launch(const LinearFormExtensionKernel_f &kernel,
+                   const FiniteElementSpace &fes,
+                   const IntegrationRule *ir,
+                   const Vector &coeff,
+                   const Array<int> &markers,
+                   Vector &y)
+{
+   Mesh *mesh = fes.GetMesh();
+   const int vdim = fes.GetVDim();
+   const bool byVDIM = fes.GetOrdering() == Ordering::byVDIM;
+
+   const FiniteElement &el = *fes.GetFE(0);
+   constexpr int flags = GeometricFactors::JACOBIANS |
+                         GeometricFactors::DETERMINANTS;
+   const MemoryType mt = Device::GetDeviceMemoryType();
+   const GeometricFactors *geom = mesh->GetGeometricFactors(*ir, flags, mt);
+   const DofToQuad &maps = el.GetDofToQuad(*ir, DofToQuad::TENSOR);
+   constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
+   const Operator *ERop = fes.GetElementRestriction(ordering);
+   const ElementRestriction* ER = dynamic_cast<const ElementRestriction*>(ERop);
+   MFEM_ASSERT(ER, "Not supported!");
+
+   const int *M = markers.Read();
+   const double *B = maps.B.Read();
+   const double *G = maps.G.Read();
+   const double *J = geom->J.Read();
+   const double *detJ = geom->detJ.Read();
+   const int *I = ER->GatherMap().Read();
+   const double *W = ir->GetWeights().Read();
+   double *Y = y.ReadWrite();
+
+   const int ND = fes.GetNDofs();
+   const int NE = fes.GetMesh()->GetNE();
+
+   const int d = maps.ndof;
+   const int q = maps.nqpt;
+
+   kernel(vdim, byVDIM, ND, NE, d, q, M, B, G, I, J, detJ, W, coeff, Y);
+}
+
+/// Internal assembly kernel for the 2D (Vector)DomainLFIntegrator
+template<int D=0, int Q=0> static
+void VectorDomainLFIntegratorAssemble2D(const int vdim,
+                                        const bool byVDIM,
+                                        const int ND,
+                                        const int NE,
+                                        const int d,
+                                        const int q,
+                                        const int *markers,
+                                        const double *b,
+                                        const double *, // g
+                                        const int *idx,
+                                        const double *, // jacobians
+                                        const double *detJ,
+                                        const double *weights,
+                                        const Vector &coeff,
+                                        double *y)
+{
+   constexpr bool USE_SMEM = D > 0 && Q > 0;
+
+   const bool cst_coeff = coeff.Size() == vdim;
+
+   const auto F = coeff.Read();
+   const auto M = Reshape(markers, NE);
+   const auto B = Reshape(b, q,d);
+   const auto DetJ = Reshape(detJ, q,q, NE);
+   const auto W = Reshape(weights, q,q);
+   const auto I = Reshape(idx, d,d, NE);
+   const auto C = cst_coeff ? Reshape(F,vdim,1,1,1) : Reshape(F,vdim,q,q,NE);
+
+   auto Y = Reshape(y, byVDIM ? vdim : ND, byVDIM ? ND : vdim);
+
+   const int sm_size = 2*q*(d+q);
+   constexpr int GRID = USE_SMEM ? 0 : 128;
+   double *gmem = ScratchMem<GRID>(sm_size);
+
+   MFEM_FORALL_3D_GRID(e, NE, q,q,1, GRID,
+   {
+      if (M(e) == 0) { /* ignore */ return; }
+
+      const int bid = MFEM_BLOCK_ID(x);
+      constexpr int SM_SIZE = 2*Q*(D+Q);
+      constexpr bool USE_SMEM = D > 0 && Q > 0;
+      MFEM_SHARED double SMEM[USE_SMEM ? SM_SIZE : 1];
+      double *sm = USE_SMEM ? SMEM : (gmem + sm_size*bid);
+      const DeviceMatrix Bt(DeviceMemAlloc(sm,q*d), q,d);
+      const DeviceMatrix QQ(DeviceMemAlloc(sm,q*q), q,q);
+      const DeviceMatrix QD(DeviceMemAlloc(sm,q*d), q,d);
+
+      kernels::internal::LoadB(d,q,B,Bt);
+
+      for (int c = 0; c < vdim; ++c)
+      {
+         const double cst_val = C(c,0,0,0);
+         MFEM_FOREACH_THREAD(x,x,q)
+         {
+            MFEM_FOREACH_THREAD(y,y,q)
+            {
+               const double detJ = DetJ(x,y,e);
+               const double coeff_val = cst_coeff ? cst_val : C(c,x,y,e);
+               QQ(y,x) = W(x,y) * coeff_val * detJ;
+            }
+         }
+         MFEM_SYNC_THREAD;
+         kernels::internal::Atomic2DEvalTranspose(d,q,Bt,QQ,QD,I,Y,c,e,byVDIM);
+      }
+   });
+}
+
+/// Internal assembly kernel for the 2D (Vector)DomainLFIntegrator
+template<int D=0, int Q=0> static
+void VectorDomainLFIntegratorAssemble3D(const int vdim,
+                                        const bool byVDIM,
+                                        const int ND,
+                                        const int NE,
+                                        const int d,
+                                        const int q,
+                                        const int *markers,
+                                        const double *b,
+                                        const double *, // g
+                                        const int *idx,
+                                        const double *, // jacobians
+                                        const double *detJ,
+                                        const double *weights,
+                                        const Vector &coeff,
+                                        double *y)
+{
+   constexpr bool USE_SMEM = D > 0 && Q > 0;
+
+   const bool cst_coeff = coeff.Size() == vdim;
+
+   const auto F = coeff.Read();
+   const auto M = Reshape(markers, NE);
+   const auto B = Reshape(b, q,d);
+   const auto DetJ = Reshape(detJ, q,q,q, NE);
+   const auto W = Reshape(weights, q,q,q);
+   const auto I = Reshape(idx, d,d,d, NE);
+   const auto C = cst_coeff ? Reshape(F,vdim,1,1,1,1):Reshape(F,vdim,q,q,q,NE);
+
+   auto Y = Reshape(y, byVDIM ? vdim : ND, byVDIM ? ND : vdim);
+
+   const int sm_size = q*d + q*q*q;
+   const int GRID = USE_SMEM ? 0 : 128;
+   double *gmem = ScratchMem<GRID>(sm_size);
+   MFEM_VERIFY(q < 32, "Unsupported quadrature order!");
+
+   MFEM_FORALL_3D_GRID(e, NE, q,q,1, GRID,
+   {
+      if (M(e) == 0) { /* ignore */ return; }
+
+      double u[Q>0?Q:32];
+
+      const int bid = MFEM_BLOCK_ID(x);
+      constexpr int SM_SIZE = Q*D + Q*Q*Q;
+      constexpr bool USE_SMEM = D > 0 && Q > 0;
+      MFEM_SHARED double SMEM[USE_SMEM ? SM_SIZE : 1];
+      double *sm = USE_SMEM ? SMEM : (gmem + sm_size*bid);
+      const DeviceCube QQQ(DeviceMemAlloc(sm,q*q*q), q,q,q);
+      const DeviceMatrix Bt(DeviceMemAlloc(sm,q*d), q,d);
+      kernels::internal::LoadB(d,q,B,Bt);
+
+      for (int c = 0; c < vdim; ++c)
+      {
+         const double cst_val = C(c,0,0,0,0);
+         MFEM_FOREACH_THREAD(x,x,q)
+         {
+            MFEM_FOREACH_THREAD(y,y,q)
+            {
+               for (int z = 0; z < q; ++z)
+               {
+                  const double detJ = DetJ(x,y,z,e);
+                  const double coeff_val = cst_coeff ? cst_val : C(c,x,y,z,e);
+                  QQQ(z,y,x) = W(x,y,z) * coeff_val * detJ;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         kernels::internal::Atomic3DEvalTranspose(d,q,u,Bt,QQQ,I,Y,c,e,byVDIM);
+      }
+   });
+}
+
+} // namespace linearform_extension
+
+} // namespace internal
+
+} // namespace mfem
@@ -0,0 +1,107 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "fem.hpp"
+#include "lininteg_domain.hpp"
+#include "lininteg_domain_grad.hpp"
+
+namespace mfem
+{
+
+using namespace internal::linearform_extension;
+
+void DomainLFGradIntegrator::AssembleFull(const FiniteElementSpace &fes,
+                                          const Array<int> &markers,
+                                          Vector &y)
+{
+   MFEM_VERIFY(fes.GetVDim()==1, "vdim != 1");
+   GetOrder_f gof = [](const int el_order) { return 2.0 * el_order; };
+   const IntegrationRule *ir = GetIntRuleFromOrder(fes, IntRule, gof);
+
+   Vector coeff;
+   const int NQ = ir->GetNPoints();
+   const int NE = fes.GetMesh()->GetNE();
+
+   if (VectorConstantCoefficient *vcQ =
+          dynamic_cast<VectorConstantCoefficient*>(&Q))
+   {
+      coeff = vcQ->GetVec();
+   }
+   else if (VectorQuadratureFunctionCoefficient *vqfQ =
+               dynamic_cast<VectorQuadratureFunctionCoefficient*>(&Q))
+   {
+      const QuadratureFunction &qfun = vqfQ->GetQuadFunction();
+      MFEM_VERIFY(qfun.Size() == NE*NQ,
+                  "Incompatible QuadratureFunction dimension \n");
+      MFEM_VERIFY(ir == &qfun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different.\n");
+      qfun.Read();
+      coeff.MakeRef(const_cast<QuadratureFunction&>(qfun),0);
+   }
+   else
+   {
+      const int qvdim = Q.GetVDim();
+      Vector Qvec(qvdim);
+      coeff.SetSize(qvdim * NQ * NE);
+      auto C = Reshape(coeff.HostWrite(), qvdim, NQ, NE);
+      for (int e = 0; e < NE; ++e)
+      {
+         ElementTransformation& T = *fes.GetElementTransformation(e);
+         for (int q = 0; q < NQ; ++q)
+         {
+            Q.Eval(Qvec, T, ir->IntPoint(q));
+            for (int c=0; c<qvdim; ++c)
+            {
+               C(c,q,e) = Qvec[c];
+            }
+         }
+      }
+   }
+
+   const int id = GetKernelId(fes,ir);
+   const int dim = fes.GetMesh()->Dimension();
+
+   LinearFormExtensionKernel_f ker = nullptr;
+   if (dim==2) { ker=VectorDomainLFGradIntegratorAssemble2D; }
+   if (dim==3) { ker=VectorDomainLFGradIntegratorAssemble3D; }
+
+   switch (id)
+   {
+      // 2D kernels, q=p+1
+      case 0x222: ker=VectorDomainLFGradIntegratorAssemble2D<2,2>; break;
+      case 0x233: ker=VectorDomainLFGradIntegratorAssemble2D<3,3>; break;
+      case 0x244: ker=VectorDomainLFGradIntegratorAssemble2D<4,4>; break;
+      case 0x255: ker=VectorDomainLFGradIntegratorAssemble2D<5,5>; break;
+
+      // 2D kernels, q=p+2
+      case 0x223: ker=VectorDomainLFGradIntegratorAssemble2D<2,3>; break;
+      case 0x234: ker=VectorDomainLFGradIntegratorAssemble2D<3,4>; break;
+      case 0x245: ker=VectorDomainLFGradIntegratorAssemble2D<4,5>; break;
+      case 0x256: ker=VectorDomainLFGradIntegratorAssemble2D<5,6>; break;
+
+      // 3D kernels, q=p+1
+      case 0x322: ker=VectorDomainLFGradIntegratorAssemble3D<2,2>; break;
+      case 0x333: ker=VectorDomainLFGradIntegratorAssemble3D<3,3>; break;
+      case 0x344: ker=VectorDomainLFGradIntegratorAssemble3D<4,4>; break;
+      case 0x355: ker=VectorDomainLFGradIntegratorAssemble3D<5,5>; break;
+
+      // 3D kernels, q=p+2
+      case 0x323: ker=VectorDomainLFGradIntegratorAssemble3D<2,3>; break;
+      case 0x334: ker=VectorDomainLFGradIntegratorAssemble3D<3,4>; break;
+      case 0x345: ker=VectorDomainLFGradIntegratorAssemble3D<4,5>; break;
+      case 0x356: ker=VectorDomainLFGradIntegratorAssemble3D<5,6>; break;
+   }
+   MFEM_VERIFY(ker, "Unexpected kernel error!");
+   Launch(ker,fes,ir,coeff,markers,y);
+}
+
+} // namespace mfem
@@ -0,0 +1,239 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+// Internal header, included only by .cpp files
+
+#include "fem.hpp"
+
+#include "../general/forall.hpp"
+#include "../fem/kernels.hpp"
+#include "../linalg/kernels.hpp"
+
+namespace mfem
+{
+
+namespace internal
+{
+
+namespace linearform_extension
+{
+
+////////////////////////////////////////////////////////////////////////////////
+template<int D=0, int Q=0> static
+void VectorDomainLFGradIntegratorAssemble2D(const int vdim,
+                                            const bool byVDIM,
+                                            const int ND,
+                                            const int NE,
+                                            const int d,
+                                            const int q,
+                                            const int *markers,
+                                            const double *b,
+                                            const double *g,
+                                            const int *idx,
+                                            const double *jacobians,
+                                            const double *detJ,
+                                            const double *weights,
+                                            const Vector &coeff,
+                                            double *y)
+{
+   constexpr int DIM = 2;
+   constexpr bool USE_SMEM = D > 0 && Q > 0;
+
+   const bool cst_coeff = coeff.Size() == vdim*DIM;
+
+   const auto F = coeff.Read();
+   const auto M = Reshape(markers, NE);
+   const auto B = Reshape(b, q,d);
+   const auto G = Reshape(g, q,d);
+   const auto J = Reshape(jacobians, q,q, DIM,DIM, NE);
+   const auto DetJ = Reshape(detJ, q,q, NE);
+   const auto W = Reshape(weights, q,q);
+   const auto I = Reshape(idx, d,d, NE);
+   const auto C = cst_coeff ?
+                  Reshape(F,DIM,vdim,1,1,1):
+                  Reshape(F,DIM,vdim,q,q,NE);
+
+   auto Y = Reshape(y, byVDIM ? vdim : ND, byVDIM ? ND : vdim);
+
+   const int sm_size = 2*d*q + 4*q*q;
+   const int GRID = USE_SMEM ? 0 : 128;
+   double *gmem = ScratchMem<GRID>(sm_size);
+
+   MFEM_FORALL_3D_GRID(e, NE, q,q,1, GRID,
+   {
+      if (M(e) == 0) { /* ignore */ return; }
+
+      const int bid = MFEM_BLOCK_ID(x);
+      constexpr int SM_SIZE = 2*D*Q + 4*Q*Q;
+      constexpr bool USE_SMEM = D > 0 && Q > 0;
+      MFEM_SHARED double SMEM[USE_SMEM ? SM_SIZE : 1];
+      double *sm = USE_SMEM ? SMEM : (gmem + sm_size*bid);
+
+      const DeviceMatrix Bt(DeviceMemAlloc(sm,q*d), d,q);
+      const DeviceMatrix Gt(DeviceMemAlloc(sm,q*d), d,q);
+      kernels::internal::LoadBGt(d,q,B,G,Bt,Gt);
+
+      const DeviceMatrix QQ0(DeviceMemAlloc(sm,q*q), q,q);
+      const DeviceMatrix QQ1(DeviceMemAlloc(sm,q*q), q,q);
+
+      const DeviceMatrix DQ0(DeviceMemAlloc(sm,d*q), d,q);
+      const DeviceMatrix DQ1(DeviceMemAlloc(sm,d*q), d,q);
+
+      for (int c = 0; c < vdim; ++c)
+      {
+         const double cst_val0 = C(0,c,0,0,0);
+         const double cst_val1 = C(1,c,0,0,0);
+
+         MFEM_FOREACH_THREAD(x,x,q)
+         {
+            MFEM_FOREACH_THREAD(y,y,q)
+            {
+               double Jloc[4], Jinv[4];
+               Jloc[0] = J(x,y,0,0,e);
+               Jloc[1] = J(x,y,1,0,e);
+               Jloc[2] = J(x,y,0,1,e);
+               Jloc[3] = J(x,y,1,1,e);
+               const double detJ = DetJ(x,y,e);
+               kernels::CalcInverse<2>(Jloc, Jinv);
+               const double weight = W(x,y);
+               const double u = cst_coeff ? cst_val0 : C(0,c,x,y,e);
+               const double v = cst_coeff ? cst_val1 : C(1,c,x,y,e);
+               QQ0(y,x) = Jinv[0]*u + Jinv[2]*v;
+               QQ1(y,x) = Jinv[1]*u + Jinv[3]*v;
+               QQ0(y,x) *= weight * detJ;
+               QQ1(y,x) *= weight * detJ;
+            }
+         }
+         MFEM_SYNC_THREAD;
+         kernels::internal::Atomic2DGradTranspose(d,q,Bt,Gt,
+                                                  QQ0,QQ1,DQ0,DQ1,
+                                                  I,Y,c,e,byVDIM);
+      }
+   });
+}
+
+template<int D=0, int Q=0> static
+void VectorDomainLFGradIntegratorAssemble3D(const int vdim,
+                                            const bool byVDIM,
+                                            const int ND,
+                                            const int NE,
+                                            const int d,
+                                            const int q,
+                                            const int *markers,
+                                            const double *b,
+                                            const double *g,
+                                            const int *idx,
+                                            const double *jacobians,
+                                            const double *detJ,
+                                            const double *weights,
+                                            const Vector &coeff,
+                                            double *y)
+{
+   constexpr int DIM = 3;
+   constexpr bool USE_SMEM = D > 0 && Q > 0;
+
+   const bool cst_coeff = coeff.Size() == vdim*DIM;
+
+   const auto F = coeff.Read();
+   const auto M = Reshape(markers, NE);
+   const auto B = Reshape(b, q,d);
+   const auto G = Reshape(g, q,d);
+   const auto J = Reshape(jacobians, q,q,q, DIM,DIM, NE);
+   const auto DetJ = Reshape(detJ, q,q,q, NE);
+   const auto W = Reshape(weights, q,q,q);
+   const auto I = Reshape(idx, d,d,d, NE);
+   const auto C = cst_coeff ?
+                  Reshape(F,DIM,vdim,1,1,1,1):
+                  Reshape(F,DIM,vdim,q,q,q,NE);
+
+   auto Y = Reshape(y, byVDIM ? vdim : ND, byVDIM ? ND : vdim);
+
+   const int sm_size = 2*q*d + 6*q*q*q;
+
+   const int GRID = USE_SMEM ? 0 : 128;
+   double *gmem = ScratchMem<GRID>(sm_size);
+
+   MFEM_FORALL_3D_GRID(e, NE, q,q,1, GRID,
+   {
+      if (M(e) == 0) { /* ignore */ return; }
+
+      const int bid = MFEM_BLOCK_ID(x);
+      constexpr int SM_SIZE = 2*Q*D + 6*Q*Q*Q;
+      constexpr bool USE_SMEM = D > 0 && Q > 0;
+      MFEM_SHARED double SMEM[USE_SMEM ? SM_SIZE : 1];
+      double *sm = USE_SMEM ? SMEM : (gmem + sm_size*bid);
+
+      const DeviceMatrix Bt(DeviceMemAlloc(sm,q*d), d,q);
+      const DeviceMatrix Gt(DeviceMemAlloc(sm,q*d), d,q);
+      kernels::internal::LoadBGt(d,q,B,G,Bt,Gt);
+
+      const DeviceCube QQ0(DeviceMemAlloc(sm,q*q*q), q,q,q);
+      const DeviceCube QQ1(DeviceMemAlloc(sm,q*q*q), q,q,q);
+      const DeviceCube QQ2(DeviceMemAlloc(sm,q*q*q), q,q,q);
+
+      const DeviceCube QD0(DeviceMemAlloc(sm,q*q*q), q,q,d);
+      const DeviceCube QD1(DeviceMemAlloc(sm,q*q*q), q,q,d);
+      const DeviceCube QD2(DeviceMemAlloc(sm,q*q*q), q,q,d);
+
+      const DeviceCube DD0(QQ0,q,d,d);
+      const DeviceCube DD1(QQ1,q,d,d);
+      const DeviceCube DD2(QQ2,q,d,d);
+
+      for (int c = 0; c < vdim; ++c)
+      {
+         const double cst_val_0 = C(0,c,0,0,0,0);
+         const double cst_val_1 = C(1,c,0,0,0,0);
+         const double cst_val_2 = C(2,c,0,0,0,0);
+
+         MFEM_FOREACH_THREAD(x,x,q)
+         {
+            MFEM_FOREACH_THREAD(y,y,q)
+            {
+               for (int z = 0; z < q; ++z)
+               {
+                  double Jloc[9], Jinv[9];
+                  for (int j = 0; j < 3; j++)
+                  {
+                     for (int i = 0; i < 3; i++)
+                     {
+                        Jloc[i+3*j] = J(x,y,z,i,j,e);
+                     }
+                  }
+                  const double detJ = DetJ(x,y,z,e);
+                  kernels::CalcInverse<3>(Jloc, Jinv);
+                  const double weight = W(x,y,z);
+                  const double u = cst_coeff ? cst_val_0 : C(0,c,x,y,z,e);
+                  const double v = cst_coeff ? cst_val_1 : C(1,c,x,y,z,e);
+                  const double w = cst_coeff ? cst_val_2 : C(2,c,x,y,z,e);
+                  QQ0(z,y,x) = Jinv[0]*u + Jinv[3]*v + Jinv[6]*w;
+                  QQ1(z,y,x) = Jinv[1]*u + Jinv[4]*v + Jinv[7]*w;
+                  QQ2(z,y,x) = Jinv[2]*u + Jinv[5]*v + Jinv[8]*w;
+                  QQ0(z,y,x) *= weight * detJ;
+                  QQ1(z,y,x) *= weight * detJ;
+                  QQ2(z,y,x) *= weight * detJ;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         kernels::internal::Atomic3DGrad(d,q,Bt,Gt,
+                                         QQ0,QQ1,QQ2,
+                                         QD0,QD1,QD2,
+                                         DD0,DD1,DD2,
+                                         I,Y,c,e,byVDIM);
+      }
+   });
+}
+
+} // namespace linearform_extension
+
+} // namespace internal
+
+} // namespace mfem
@@ -0,0 +1,102 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "fem.hpp"
+#include "lininteg_domain.hpp"
+
+namespace mfem
+{
+
+using namespace internal::linearform_extension;
+
+void VectorDomainLFIntegrator::AssembleFull(const FiniteElementSpace &fes,
+                                            const Array<int> &markers,
+                                            Vector &y)
+{
+   const int vdim = fes.GetVDim();
+   GetOrder_f gof = [](const int el_order) { return 2.0 * el_order; };
+   const IntegrationRule *ir = GetIntRuleFromOrder(fes, IntRule, gof);
+
+   Vector coeff;
+   const int NQ = ir->GetNPoints();
+   const int NE = fes.GetMesh()->GetNE();
+
+   if (VectorConstantCoefficient *vcQ =
+          dynamic_cast<VectorConstantCoefficient*>(&Q))
+   {
+      coeff = vcQ->GetVec();
+   }
+   else if (VectorQuadratureFunctionCoefficient *vQ =
+               dynamic_cast<VectorQuadratureFunctionCoefficient*>(&Q))
+   {
+      const QuadratureFunction &qfun = vQ->GetQuadFunction();
+      MFEM_VERIFY(qfun.Size() == vdim*NE*NQ,
+                  "Incompatible QuadratureFunction dimension \n");
+      MFEM_VERIFY(ir == &qfun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different.\n");
+      qfun.Read();
+      coeff.MakeRef(const_cast<QuadratureFunction&>(qfun),0);
+   }
+   else
+   {
+      Vector Qvec(vdim);
+      coeff.SetSize(vdim * NQ * NE);
+      auto C = Reshape(coeff.HostWrite(), vdim, NQ, NE);
+      for (int e = 0; e < NE; ++e)
+      {
+         ElementTransformation& T = *fes.GetElementTransformation(e);
+         for (int q = 0; q < NQ; ++q)
+         {
+            Q.Eval(Qvec, T, ir->IntPoint(q));
+            for (int c=0; c<vdim; ++c) { C(c,q,e) = Qvec[c]; }
+         }
+      }
+   }
+
+   LinearFormExtensionKernel_f ker = nullptr;
+   const int id = GetKernelId(fes,ir);
+   const int dim = fes.GetMesh()->Dimension();
+
+   if (dim==2) { ker = VectorDomainLFIntegratorAssemble2D<>; }
+   if (dim==3) { ker = VectorDomainLFIntegratorAssemble3D<>; }
+
+   switch (id)
+   {
+      // 2D kernels, q=p+1
+      case 0x222: ker=VectorDomainLFIntegratorAssemble2D<2,2>; break;
+      case 0x233: ker=VectorDomainLFIntegratorAssemble2D<3,3>; break;
+      case 0x244: ker=VectorDomainLFIntegratorAssemble2D<4,4>; break;
+      case 0x255: ker=VectorDomainLFIntegratorAssemble2D<5,5>; break;
+
+      // 2D kernels, q=p+2
+      case 0x223: ker=VectorDomainLFIntegratorAssemble2D<2,3>; break;
+      case 0x234: ker=VectorDomainLFIntegratorAssemble2D<3,4>; break;
+      case 0x245: ker=VectorDomainLFIntegratorAssemble2D<4,5>; break;
+      case 0x256: ker=VectorDomainLFIntegratorAssemble2D<5,6>; break;
+
+      // 3D kernels, q=p+1
+      case 0x322: ker=VectorDomainLFIntegratorAssemble3D<2,2>; break;
+      case 0x333: ker=VectorDomainLFIntegratorAssemble3D<3,3>; break;
+      case 0x344: ker=VectorDomainLFIntegratorAssemble3D<4,4>; break;
+      case 0x355: ker=VectorDomainLFIntegratorAssemble3D<5,5>; break;
+
+      // 3D kernels, q=p+2
+      case 0x323: ker=VectorDomainLFIntegratorAssemble3D<2,3>; break;
+      case 0x334: ker=VectorDomainLFIntegratorAssemble3D<3,4>; break;
+      case 0x345: ker=VectorDomainLFIntegratorAssemble3D<4,5>; break;
+      case 0x356: ker=VectorDomainLFIntegratorAssemble3D<5,6>; break;
+   }
+   MFEM_VERIFY(ker, "Unexpected kernel error!");
+   Launch(ker,fes,ir,coeff,markers,y);
+}
+
+} // namespace mfem
@@ -0,0 +1,116 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "fem.hpp"
+#include "lininteg_domain.hpp"
+#include "lininteg_domain_grad.hpp"
+
+namespace mfem
+{
+
+using namespace internal::linearform_extension;
+
+void VectorDomainLFGradIntegrator::AssembleFull(const FiniteElementSpace &fes,
+                                                const Array<int> &markers,
+                                                Vector &y)
+{
+   const int vdim = fes.GetVDim();
+   GetOrder_f gof = [](const int el_order) { return 2.0 * el_order; };
+   const IntegrationRule *ir = GetIntRuleFromOrder(fes, IntRule, gof);
+
+   Vector coeff;
+   const int NQ = ir->GetNPoints();
+   const int NE = fes.GetMesh()->GetNE();
+   const int NS = fes.GetMesh()->SpaceDimension();
+
+   if (VectorConstantCoefficient *vcQ =
+          dynamic_cast<VectorConstantCoefficient*>(&Q))
+   {
+      coeff = vcQ->GetVec();
+   }
+   else if (QuadratureFunctionCoefficient *qfQ =
+               dynamic_cast<QuadratureFunctionCoefficient*>(&Q))
+   {
+      const QuadratureFunction &qfun = qfQ->GetQuadFunction();
+      MFEM_VERIFY(qfun.Size() == NE*NQ,
+                  "Incompatible QuadratureFunction dimension \n");
+      MFEM_VERIFY(ir == &qfun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different.\n");
+      qfun.Read();
+      coeff.MakeRef(const_cast<QuadratureFunction&>(qfun),0);
+   }
+   else if (VectorQuadratureFunctionCoefficient* vqfQ =
+               dynamic_cast<VectorQuadratureFunctionCoefficient*>(&Q))
+   {
+      const QuadratureFunction &qFun = vqfQ->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == vdim * NS * NQ * NE,
+                  "Incompatible QuadratureFunction dimension \n");
+      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+      qFun.Read();
+      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+   }
+   else
+   {
+      Vector Qvec(vdim);
+      coeff.SetSize(vdim * NQ * NE);
+      auto C = Reshape(coeff.HostWrite(), vdim, NQ, NE);
+      for (int e = 0; e < NE; ++e)
+      {
+         ElementTransformation &Tr = *fes.GetElementTransformation(e);
+         for (int q = 0; q < NQ; ++q)
+         {
+            Q.Eval(Qvec, Tr, ir->IntPoint(q));
+            for (int c = 0; c<vdim; ++c) { C(c,q,e) = Qvec[c]; }
+         }
+      }
+   }
+
+   LinearFormExtensionKernel_f ker = nullptr;
+   const int id = GetKernelId(fes,ir);
+   const int dim = fes.GetMesh()->Dimension();
+
+   if (dim==2) { ker=VectorDomainLFGradIntegratorAssemble2D; }
+   if (dim==3) { ker=VectorDomainLFGradIntegratorAssemble3D; }
+
+   switch (id)
+   {
+      // 2D kernels, q=p+1
+      case 0x222: ker=VectorDomainLFGradIntegratorAssemble2D<2,2>; break;
+      case 0x233: ker=VectorDomainLFGradIntegratorAssemble2D<3,3>; break;
+      case 0x244: ker=VectorDomainLFGradIntegratorAssemble2D<4,4>; break;
+      case 0x255: ker=VectorDomainLFGradIntegratorAssemble2D<5,5>; break;
+
+      // 2D kernels, q=p+2
+      case 0x223: ker=VectorDomainLFGradIntegratorAssemble2D<2,3>; break;
+      case 0x234: ker=VectorDomainLFGradIntegratorAssemble2D<3,4>; break;
+      case 0x245: ker=VectorDomainLFGradIntegratorAssemble2D<4,5>; break;
+      case 0x256: ker=VectorDomainLFGradIntegratorAssemble2D<5,6>; break;
+
+      // 3D kernels, q=p+1
+      case 0x322: ker=VectorDomainLFGradIntegratorAssemble3D<2,2>; break;
+      case 0x333: ker=VectorDomainLFGradIntegratorAssemble3D<3,3>; break;
+      case 0x344: ker=VectorDomainLFGradIntegratorAssemble3D<4,4>; break;
+      case 0x355: ker=VectorDomainLFGradIntegratorAssemble3D<5,5>; break;
+
+      // 3D kernels, q=p+2
+      case 0x323: ker=VectorDomainLFGradIntegratorAssemble3D<2,3>; break;
+      case 0x334: ker=VectorDomainLFGradIntegratorAssemble3D<3,4>; break;
+      case 0x345: ker=VectorDomainLFGradIntegratorAssemble3D<4,5>; break;
+      case 0x356: ker=VectorDomainLFGradIntegratorAssemble3D<5,6>; break;
+   }
+   MFEM_VERIFY(ker, "Unexpected kernel error!");
+   Launch(ker,fes,ir,coeff,markers,y);
+}
+
+} // namespace mfem
--- a/Show More
+++ b/Show More