make style

revert make file
fix style
2021-09-13 10:51:52 -07:00 · 2021-09-13 18:42:10 +03:00 · 2021-09-13 18:41:43 +03:00 · 2021-09-13 11:39:22 +03:00 · 2021-09-13 11:34:44 +03:00 · 2021-09-03 11:59:51 +03:00
363 changed files with 26157 additions and 53032 deletions
@@ -1,12 +0,0 @@
-# extends https://github.com/jupyterhub/repo2docker/blob/main/repo2docker/buildpacks/conda/environment.yml
-# see https://mybinder.readthedocs.io/en/latest/using/config_files.html#environment-yml-install-a-conda-environment
-channels:
- conda-forge
-dependencies:
- xeus-cling=0.13.0
- xwidgets=0.26.0
-# NOTE: it's possible these aren't needed for the lab frontend
- widgetsnbextension=3.5.1
- pip
- pip:
-  - glvis==0.3.2
@@ -1,26 +0,0 @@
-#!/bin/bash
-
-set -e
-
-# cling is installed here (in bin) and will look in {dir}/include and {dir}/lib
-# without extra intervention (jk it doesn't look in {dir}/lib unless something
-# has been #included from {dir}/include first...)
-install_dir=/srv/conda/envs/notebook
-mkdir -p $install_dir
-
-# build and install mfem, which is the directory we start in
-make serial SHARED=YES -j8
-make install PREFIX=$install_dir
-
-# install xeus-glvis
-git clone https://github.com/GLVis/xeus-glvis.git
-pushd xeus-glvis
-make install prefix=$install_dir
-popd
-
-# install jupyter-lab extension
-jupyter labextension install @jupyter-widgets/jupyterlab-manager --no-build
-jupyter labextension install glvis-jupyter
-
-# fixup kernelspec, we could probably do this from sh but ¯\_(ツ)_/¯
-python .binder/update_kernel_env.py
@@ -1,14 +0,0 @@
-# Update the LD_LIBRARY_PATH of the C++14 kernel so it can find mfem without
-# extra pragma cling statements
-
-import json
-
-kernelspec = "/srv/conda/envs/notebook/share/jupyter/kernels/xcpp14/kernel.json"
-
-with open(kernelspec, "r") as f:
-    obj = json.load(f)
-
-obj["env"] = {"LD_LIBRARY_PATH": "/srv/conda/envs/notebook/lib"}
-
-with open(kernelspec, "w") as f:
-    json.dump(obj, f)
@@ -82,9 +82,9 @@ jobs:
      uses: mfem/github-actions/build-mfem@v2.0
      with:
        os: ${{ runner.os }}
-        target: opt
+        target: optim
        codecov: NO
-        mpi: par
+        mpi: parallel
        build-system: make
        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
        metis-dir: ${{ env.METIS_TOP_DIR }}
@@ -63,7 +63,7 @@ jobs:
        exit 1

  code-style:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-16.04 # needed for astyle 2.05.1

    steps:
    - name: checkout mfem
@@ -71,7 +71,7 @@ jobs:

    - name: get astyle
      run: |
-        sudo apt-get install astyle=3.1-1ubuntu2
+        sudo apt-get install astyle=2.05.1-0ubuntu1

    - name: style check
      run: |
@@ -105,9 +105,6 @@ jobs:

    - name: branch-history
      run: |
-        # We override origin to make sure we point to the main repo.
-        # This is to have consistent test results on PRs from forks.
-        git remote remove origin
-        git remote add origin https://github.com/mfem/mfem.git
+        git fetch origin master:master
        git checkout -b gh-actions-branch-history
        ./config/githooks/pre-push --history
@@ -15,19 +15,6 @@
 CMakeCache.txt
 CMakeFiles/

-# NSYS files
-*.nsys-rep
-
-# Benchmark files
-*.tex
-*.org
-*.aux
-*.log
-*.gnuplot
-*.table
-*.pdf
-*.png
-
 # Backup files
 *~

@@ -64,8 +51,6 @@ examples/ex1[04-9]
 examples/ex1[0-9]p
 examples/ex2[0-9]
 examples/ex2[0-9]p
-examples/ex30
-examples/ex30p

 examples/refined.mesh
 examples/displaced.mesh
@@ -238,14 +223,6 @@ miniapps/mtop/ParHeat*
 miniapps/mtop/seqheat
 miniapps/mtop/SeqHeat*

-miniapps/autodiff/paradiff
-miniapps/autodiff/seqadiff
-miniapps/autodiff/seqtest
-miniapps/autodiff/par_example
-miniapps/autodiff/seq_example
-miniapps/autodiff/seq_test
-miniapps/autodiff/Exampl*
-
 miniapps/navier/navier_mms
 miniapps/navier/navier_kovasznay
 miniapps/navier/navier_kovasznay_vs
@@ -311,15 +288,10 @@ miniapps/solvers/ParaView
 miniapps/solvers/mesh.*
 miniapps/solvers/sol.*

-miniapps/parelag/MultilevelHcurlHdivSolver
-miniapps/parelag/*.mesh
-
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
 tests/unit/punit_tests
-tests/unit/cunit_tests
-tests/unit/pcunit_tests
 tests/unit/sedov_tests_*
 tests/unit/psedov_tests_*
 tests/unit/tmop_pa_tests_*
@@ -327,15 +299,6 @@ tests/unit/ptmop_pa_tests_*
 tests/unit/ceed_tests
 tests/unit/debug_device_tests

-# Benchmark binaries
-tests/benchmarks/bench_ceed
-tests/benchmarks/bench_solvers
-tests/benchmarks/bench_linext
-tests/benchmarks/bench_lor_assembly
-tests/benchmarks/bench_tmop
-tests/benchmarks/bench_vector
-tests/benchmarks/bench_virtuals
-
 # Test script output
 tests/scripts/*.err
 tests/scripts/*.out
@@ -352,6 +315,3 @@ build-*/*
 # PETSc automated build
 petsc-build/*
 pkg.gitcommit
-
-# Jupyter Notebook Checkpoints
-.ipynb_checkpoints
@@ -13,84 +13,243 @@
 # at Lawrence Livermore National Laboratory (LLNL). This entire pipeline is
 # LLNL-specific!

+# We define the following GitLab pipeline variables:
+#
+# BUILD_ROOT:
+# The path to the shared resources between all jobs. For example, external
+# repositories like 'tests' and 'tpls' are cloned here. Also, 'tpls' is built
+# once for all targets, so that build happen here. The BUILD_ROOT is unique to
+# the pipeline, preventing any form of concurrency with other pipelines. This
+# also means that the BUILD_ROOT directory will never be cleaned.
+# TODO: add a clean-up mechanism
+#
+# REBASELINE:
+# Defines the default choice for updating the saved baseline results. By default
+# the baseline can only be updated from the master branch. This variable offers
+# the option to manually ask for rebaselining from another branch if necessary.
+#
+# MFEM_ALLOC_NAME:
+# On LLNL's quartz, there is only one allocation shared among jobs in order to
+# save time and resources. This allocation has to be uniquely named so that we
+# are sure to retrieve it.
+#
+# TPLS_REPO & TESTS_REPO:
+# Git repositories used in the pipeline
+#
+# ARTIFACTS_DIR:
+# Directory used to place artifacts.
+
+variables:
+  BUILD_ROOT: ${CI_BUILDS_DIR}/MFEM/${CI_PROJECT_NAME}_${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID}
+  AUTOTEST_ROOT: ${CI_BUILDS_DIR}/MFEM
+  REBASELINE: "NO"
+  AUTOTEST: "NO"
+  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
+  TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
+  TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
+  AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
+  MFEM_DATA_REPO: https://github.com/mfem/data.git
+  ARTIFACTS_DIR: artifacts
+  SLURM_OVERLAP: 1

 # The pipeline is divided into stages. Usually, jobs in a given stage wait for
 # the preceding stages to complete before to start. However, we sometimes use
 # the "needs" keyword and express the DAG of jobs for more efficiency.
 # - We use setup and setup_baseline phases to download content outside of mfem
 #   directory.
-# - Allocate/Release is where quartz resource are allocated/released once for all.
+# - Allocate/Release is where quartz resources are allocated/released once for all.
 # - Build and Test is where we build and MFEM for multiple toolchains.
 # - Baseline_checks gathers baseline-type test suites execution
 # - Baseline_publish, only available on master, allows to update baseline
 #   results
 stages:
-  - sub-pipelines
+  - setup
+  - q_allocate_resources
+  - q_build_and_test
+  - q_release_resources
+  - l_build_and_test
+  - c_build_and_test
+  - setup_baseline
+  - baseline_check
+  - baseline_to_autotest
+  - baseline_publish

-variables:
-  CUSTOM_CI_BUILDS_DIR: "/usr/workspace/mfem/gitlab-runner"
-  USER_CI_TOP_DIR: "${CUSTOM_CI_BUILDS_DIR}/${GITLAB_USER_LOGIN}"
-  SHARED_REPOS_DIR: "${USER_CI_TOP_DIR}/repos"
-  AUTOTEST_ROOT: "${SHARED_REPOS_DIR}"
-  # MFEM_DATA_DIR is setup in '.gitlab/configs/setup-build-and-test.yml' and
-  # used in '.gitlab/configs/<machine>-config.yml':
-  MFEM_DATA_DIR: "${SHARED_REPOS_DIR}/mfem-data"
-
-# Defines the default choice for updating the saved baseline results. By default
-# the baseline can only be updated from the master branch. This variable offers
-# the option to manually ask for rebaselining from another branch if necessary.
-  REBASELINE: "NO"
-  AUTOTEST: "NO"
-  # AUTOTEST_COMMIT: used only when AUTOTEST is set to YES.
-  # * If AUTOTEST_COMMIT is NOT set to NO, reporting jobs will commit their
-  #   files to the MFEM/autotest repo.
-  # * If AUTOTEST_COMMIT is set to NO, reporting jobs will NOT commit their
-  #   files to the MFEM/autotest repo. Instead they will just show the contents
-  #   of the report files and remove them.
-  AUTOTEST_COMMIT: "YES"
-
-# Trigger subpipelines:
-quartz-build-and-test:
-  stage: sub-pipelines
+# setup clones the mfem/data repo in ${BUILD_ROOT}. The build_and_test script
+# then symlinks the repo to the parent directory of the MFEM source directory.
+# Unit tests that depend on the mfem/data repo will then detect that this
+# directory is present and be enabled.
+setup:
+  tags:
+    - shell
+    - quartz
+  stage: setup
  variables:
-    # Explicitly pass down values that we want to be able to set when triggering
-    # pipelines manually or using scheduling
-    AUTOTEST: "${AUTOTEST}"
-    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
-  trigger:
-    include: .gitlab/quartz-build-and-test.yml
-    strategy: depend
+    GIT_STRATEGY: none
+  script:
+    - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
+    - if [ ! -d data ]; then git clone ${MFEM_DATA_REPO}; fi

-quartz-baseline:
-  stage: sub-pipelines
+# The setup_baseline job in setup stage_baseline doesn't rely on MFEM git repo.
+# It prepares a pipeline-wide working directory downloading/updating external
+# repos. TODO: updating tests and tpls is not necessary anymore since pipelines
+# are now using unique directories so repo are never shared with another
+# pipeline. This is not memory efficient (we keep a lot of data), hence this
+# reminder.
+# Note: This job can start immediately.
+setup_baseline:
+  tags:
+    - shell
+    - quartz
+  stage: setup_baseline
  variables:
-    # Explicitly pass down values that we want to be able to set when triggering
-    # pipelines manually or using scheduling
-    REBASELINE: "${REBASELINE}"
-    AUTOTEST: "${AUTOTEST}"
-    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
-  trigger:
-    include: .gitlab/quartz-baseline.yml
-    strategy: depend
+    GIT_STRATEGY: none
+  script:
+    - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
+    - if [ ! -d "tpls" ]; then git clone ${TPLS_REPO}; fi
+    - if [ ! -d "tests" ]; then git clone ${TESTS_REPO}; fi
+    - cd tpls && git pull && cd ..
+    - cd tests && git pull && cd ..
+    - cd ${AUTOTEST_ROOT}
+    - if [ ! -d "autotest" ]; then git clone ${AUTOTEST_REPO}; fi
+    - cd autotest && git pull && cd ..
+  needs: []

-lassen-build-and-test:
-  stage: sub-pipelines
-  variables:
-    # Explicitly pass down values that we want to be able to set when triggering
-    # pipelines manually or using scheduling
-    AUTOTEST: "${AUTOTEST}"
-    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
-  trigger:
-    include: .gitlab/lassen-build-and-test.yml
-    strategy: depend
+.build_toss_3_x86_64_ib_script:
+  script:
+    - export THREADS=12
+    - echo ${ALLOC_NAME}
+    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
+    - echo ${JOBID}
+    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 30 -N 1 tests/gitlab/build_and_test

-corona-build-and-test:
-  stage: sub-pipelines
+.build_toss_3_x86_64_ib_corona_script:
+  script:
+    - srun -p mi60 -t 15 -N 1 tests/gitlab/build_and_test
+
+# Lassen uses a different job scheduler (spectrum lsf) that does not allow
+# pre-allocation the same way slurm does.  We use pdebug queue on lassen to
+# speed-up the allocation.  However this would not be scalable to multiple
+# builds.
+.build_blueos_3_ppc64le_ib_script:
+  script:
+    - lalloc 1 -W 30 -q pdebug tests/gitlab/build_and_test
+
+# Shared script for baseline and sample-run-baseline, the value of BASELINE_TEST
+# differentiates between the two tests.
+.baseline_script: &baseline_script |
+  # locals
+  _glob_err=${BASELINE_TEST}.err
+  _base_diff=${BASELINE_TEST}-${SYS_TYPE}.diff
+  _base_patch=${BASELINE_TEST}-${SYS_TYPE}.patch
+  _base_out=${BASELINE_TEST}-${SYS_TYPE}.out
+  # prepare
+  cd ${BUILD_ROOT}
+  ln -snf ${CI_PROJECT_DIR} mfem
+  cd tests
+  mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
+  # run
+  srun --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${ADDITIONAL_DIR}"
+  # post
+  mkdir ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}
+  if [[ -s ${_glob_err} ]]
+  then
+    echo "ERROR during ${BASELINE_TEST} execution";
+    echo "Here is the ${_glob_err} file content";
+    cat ${_glob_err}
+    cp ${_glob_err} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_glob_err}
+    exit 1;
+  elif [[ ! -f ${_base_patch} && ! -f ${_base_out} ]]
+  then
+    echo "Something went WRONG in ${BASELINE_TEST}:";
+    echo "Either ${_base_patch} or ${_base_out} should exists";
+    exit 1;
+  elif [[ -f ${_base_patch} ]]
+  then
+    echo "${BASELINE_TEST}: Differences found, patch generated"
+    cp ${_base_patch} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_patch}
+  elif [[ -f ${_base_out} ]]
+  then
+    echo "${BASELINE_TEST}: Differences found, replacement file generated"
+    cp ${_base_out} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_out}
+  fi
+  # _base_diff won't even exist if there is no difference.
+  if [[ -f ${_base_diff} ]]
+  then
+    echo "${BASELINE_TEST}: Relevant differences (filtered diff) ..."
+    cat ${_base_diff}
+    cp ${_base_diff} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_diff}
+    # We create a .err file, because that's how we signal that there was a diff.
+    cp ${_base_diff} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/gitlab-${BASELINE_TEST}-${SYS_TYPE}.err
+  fi
+  if [[ ! -s ${_base_diff} ]]
+  then
+    echo "${BASELINE_TEST}: PASSED"
+    true
+  else
+    echo "${BASELINE_TEST}: FAILED"
+    false
+  fi
+
+# Actual templates for baseline checks
+.baselinecheck_mfem:
+  stage: baseline_check
  variables:
-    # Explicitly pass down values that we want to be able to set when triggering
-    # pipelines manually or using scheduling
-    AUTOTEST: "${AUTOTEST}"
-    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
-  trigger:
-    include: .gitlab/corona-build-and-test.yml
-    strategy: depend
+    BASELINE_TEST: baseline
+    ADDITIONAL_DIR: ${BUILD_ROOT}/tpls
+  script:
+    - *baseline_script
+  artifacts:
+    when: always
+    paths:
+      - ${ARTIFACTS_DIR}
+  allow_failure: true
+
+.samplebaselinecheck_mfem:
+  stage: baseline_check
+  variables:
+    BASELINE_TEST: sample-runs-baseline
+    ADDITIONAL_DIR: ""
+  script:
+    - *baseline_script
+  timeout: 4h
+  artifacts:
+    when: always
+    paths:
+      - ${ARTIFACTS_DIR}
+  allow_failure: true
+
+# This job can only be manually triggered on a pipeline for master branch, or if
+# the pipeline was triggered with REBASELINE="YES"
+.rebaseline_mfem:
+  stage: baseline_publish
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "YES"'
+      when: manual
+  script:
+    - export PATCH_FILE=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}.patch
+    - export FULL_FILE=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}.out
+    - export DIFF_FILE=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}.diff
+    - cd ${BUILD_ROOT}/tests
+    - |
+      if [[ ! -f "${DIFF_FILE}" ]]
+      then
+          echo "Nothing to be done: no relevant change in baseline"
+          exit 0
+      elif [[ -f "${PATCH_FILE}" ]]
+      then
+          patch "./baseline-${SYS_TYPE}.saved" < "${PATCH_FILE}"
+      elif [[ -f "${FULL_FILE}t" ]]
+      then
+          cp "${FULL_FILE}" "./baseline-${SYS_TYPE}.saved"
+      else
+          echo "File missing: expected ${PATCH_FILE} or ${FULL_FILE}"
+          exit 1
+      fi
+    - git add baseline-${SYS_TYPE}.saved
+    - git commit -m "${SYS_TYPE} rebaselined in GitLab pipeline ${CI_PIPELINE_ID}"
+    - git push origin master
+
+# The list on jobs is defined in machine-specific files.
+include:
+  - local: .gitlab/quartz.yml
+  - local: .gitlab/lassen.yml
@@ -1,94 +0,0 @@
-                    Finite Element Discretization Library
-                                   __
-                       _ __ ___   / _|  ___  _ __ ___
-                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
-                      | | | | | ||  _||  __/| | | | | |
-                      |_| |_| |_||_|   \___||_| |_| |_|
-
-                               https://mfem.org
-
-
-This directory contains most of the GitLab CI configuration. MFEM runs both PR
-and nightly testing on GitLab.
-
-# Structure
-
-## Top level
-
-The root configuration file is `.gitlab-ci.yml` at the root of MFEM repo.
-This file only defines one stage, in which we trigger several
-sub-pipelines.
-
-We use sub-pipelines to isolate the test for one combination of `machine`
-and `test type`.
-
-Machines typically include:
-
-* Quartz: Intel bi-socket x86
-* Lassen: Power9 + Nvidia GPU
-* Corona: AMD GPU
-
-Test types include:
-
-* Build and test: Spack driven build of dependencies, mfem build, mfem
-  test
-* Baseline: Script driven build of dependencies, thorough testing
-
-⚠️ The sub-pipeline design allows to add a new machine or a new test type without
-altering the scheduling, execution and displaying of the others.
-
-## Sub-pipelines
-
-Each file is this directory is the root configuration file for one
-sub-pipeline. The naming reflects the corresponding couple (`machine`,
-`test_type`).
-
-Those files define the *stages* and the *jobs* for the sub-pipeline. They
-also contain any configuration that cannot be shared. For the most part
-though, the configuration is shared and is placed in `.gitlab/configs`.
-
-We try to keep scripts out of the CI config and share them among similar
-jobs. They are gathered in `.gitlab/scripts`.
-
-## Scripts
-
-Scripts specific to the CI only are in `.gitlab/scripts`. It is best practice
-to keep scripts outside the CI configuration (no bash scripts embedded in a
-yaml file) because it helps with readability, maintenance and also with
-transition to another CI system.
-
-⚠️ Most of the scripts there are driven by environment variables and do not have a
-usage function. This should be improved.
-
-
-# More testing
-
-## Adding a new target to a build_and_test pipeline
-
-`build_and_test` pipelines rely on Spack to install dependencies. Spack is
-driven by Uberenv which helps freezing Spack configuration: the goal being to
-point to specific commit in Spack and isolate its configuration so that it is
-not influenced by the user environment. More documentation about this can be
-found in `tests/gitlab`.
-
-In the end, the MFEM target for which to build the dependencies is expressed
-with a spack spec of MFEM, within the limits permitted by the MFEM spack
-package.
-
-In any build-and-test sub-pipeline a job basically consists in defining the
-spack spec to use. Adding a job on quartz for example resumes to:
-
-```yaml
-<job_name>:
-  variables:
-    SPEC: "<spack_spec>"
-  extends: .build_and_test_on_quartz
-```
-
-The remaining and non trivial work is to make sure this spec is working. To
-test a spec before adding it, or reproduce a CI configuration, please refer to
-`tests/gitlab/reproduce-ci-jobs-interactively.md`.
-
-⚠️ It is assumed that the spack spec applies to `mfem@develop`. That's why in the
-CI all the specs start with the compiler or the variants to apply to mfem. The
-mechanism still works with a full spec.
@@ -1,36 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# We define the following GitLab pipeline variables:
-variables:
-
-# The path to the shared resource between all jobs. For example, external
-# repositories like 'tests' and 'tpls' are cloned here. Also, 'tpls' is built
-# once for all targets, so that build happen here. The BUILD_ROOT is unique to
-# the pipeline, preventing any form of concurrency with other pipelines. This
-# also means that the BUILD_ROOT directory will never be cleaned.
-# TODO: add a clean-up mechanism
-  BUILD_ROOT: ${USER_CI_TOP_DIR}/${CI_PROJECT_NAME}-${MACHINE_NAME}-pipeline-${CI_PIPELINE_ID}
-
-# On LLNL's quartz, there is only one allocation shared among jobs in order to
-# save time and resource. This allocation has to be uniquely named so that we
-# are sure to retrieve it.
-  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
-
-# Git repositories used in the pipeline
-  TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
-  TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
-  AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
-  MFEM_DATA_REPO: https://github.com/mfem/data.git
-
-# Directory used to place artifacts.
-  ARTIFACTS_DIR: artifacts
-  SLURM_OVERLAP: 1
@@ -1,59 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# GitLab pipeline configuration for the Corona machine at LLNL
-variables:
-  MACHINE_NAME: corona
-
-.on_corona:
-  tags:
-    - shell
-    - corona
-  rules:
-    # Don’t run corona jobs if...
-    # Note: This makes corona an "opt-in" machine. To activate builds on corona
-    # for a given GitLab clone of MFEM, go to Setting/CI-CD/variables, and set
-    # "ON_CORONA" to "ON". An LC account on for corona is required to trigger a
-    # pipeline there.
-    - if: '$CI_COMMIT_BRANCH =~ /_cnone/ || $ON_CORONA != "ON"'
-      when: never
-    # Don’t run autotest update if...
-    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
-      when: never
-    # Report success on success status
-    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
-      when: on_success
-    # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
-      when: on_failure
-    # Always release resource
-    - if: '$CI_JOB_NAME =~ /release_resource/'
-      when: always
-    # Always cleanup
-    - if: '$CI_JOB_NAME =~ /cleanup/'
-      when: always
-    # Default is to run if previous stage succeeded
-    - when: on_success
-
-# Spack helped builds
-# Generic corona build job, extending build script
-.build_and_test_on_corona:
-  extends: [.on_corona]
-  stage: build_and_test
-  script:
-    # THREADS is used by 'tests/gitlab/build_and_test', run below
-    - export THREADS=12
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - echo ${MFEM_DATA_DIR}
-    - echo ${SPEC}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 15 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
@@ -1,49 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# GitLab pipelines configurations for the Lassen machine at LLNL
-variables:
-  MACHINE_NAME: lassen
-
-.on_lassen:
-  tags:
-    - shell
-    - lassen
-  rules:
-    - if: '$CI_COMMIT_BRANCH =~ /_lnone/ || $ON_LASSEN == "OFF"' #run except if ...
-      when: never
-    # Don't run autotest update if...
-    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
-      when: never
-    # Report success on success status
-    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
-      when: on_success
-    # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
-      when: on_failure
-    # Always cleanup
-    - if: '$CI_JOB_NAME =~ /cleanup/'
-      when: always
-    - when: on_success
-
-# Lassen uses a different job scheduler (spectrum lsf) that does not allow
-# pre-allocation the same way slurm does.  We use pdebug queue on lassen
-# to speed-up the allocation.  However this would not be scalable to
-# multiple builds.
-.build_and_test_on_lassen:
-  extends: [.on_lassen]
-  stage: build_and_test
-  script:
-    - echo ${MFEM_DATA_DIR}
-    - echo ${SPEC}
-    # Next script uses 'THREADS': leaving it empty --> it uses 'make all -j'
-    - lalloc 1 -W 30 -q pdebug --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
-  needs: [setup]
@@ -1,55 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# GitLab pipelines configurations for the Quartz machine at LLNL
-variables:
-  MACHINE_NAME: quartz
-
-.on_quartz:
-  tags:
-    - shell
-    - quartz
-  rules:
-    # Don't run quartz jobs if...
-    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"'
-      when: never
-    # Don't run autotest update if...
-    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
-      when: never
-    # Report success on success status
-    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
-      when: on_success
-    # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
-      when: on_failure
-    # Always release resource
-    - if: '$CI_JOB_NAME =~ /release_resource/'
-      when: always
-    # Always cleanup
-    - if: '$CI_JOB_NAME =~ /cleanup/'
-      when: always
-    # Default is to run if previous stage succeeded
-    - when: on_success
-
-# Spack helped builds
-# Generic quartz build job, extending build script
-.build_and_test_on_quartz:
-  extends: [.on_quartz]
-  stage: build_and_test
-  script:
-    # THREADS is used by 'tests/gitlab/build_and_test', run below
-    - export THREADS=12
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - echo ${MFEM_DATA_DIR}
-    - echo ${SPEC}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 30 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
@@ -1,81 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Jobs report
-.report_job_success:
-  script:
-    - echo ${MACHINE_NAME}
-    - echo ${AUTOTEST}
-    - echo ${AUTOTEST_COMMIT}
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - cd ${AUTOTEST_ROOT}
-    - |
-      (
-        date
-        echo "Waiting to aquire lock on '$PWD/autotest.lock' ..."
-        # try to get an excusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
-        done
-        echo "Aquired lock on '$PWD/autotest.lock'"
-        date
-        # Report SUCCESS while holding the file lock on 'autotest.lock'.
-        # The next script uses the following environment variables:
-        # - MACHINE_NAME, AUTOTEST_ROOT, AUTOTEST_COMMIT
-        # - CI_COMMIT_REF_SLUG, CI_PROJECT_DIR, CI_PIPELINE_URL
-        # It also calls the script '.gitlab/scripts/safe_create_rundir'.
-        ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test_success
-        err=$?
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock
-
-.report_job_failure:
-  script:
-    - echo ${MACHINE_NAME}
-    - echo ${AUTOTEST}
-    - echo ${AUTOTEST_COMMIT}
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - cd ${AUTOTEST_ROOT}
-    - |
-      (
-        date
-        echo "Waiting to aquire lock on '$PWD/autotest.lock' ..."
-        # try to get an excusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
-        done
-        echo "Aquired lock on '$PWD/autotest.lock'"
-        date
-        # Report FAILURE while holding the file lock on 'autotest.lock'.
-        # The next script uses the following environment variables:
-        # - MACHINE_NAME, AUTOTEST_ROOT, AUTOTEST_COMMIT
-        # - CI_COMMIT_REF_SLUG, CI_PROJECT_DIR, CI_PIPELINE_URL
-        # It also calls the script '.gitlab/scripts/safe_create_rundir'.
-        ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test_failure
-        err=$?
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock
@@ -1,72 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# The setup_baseline job doesn't rely on MFEM git repo. It prepares a
-# pipeline-wide working directory downloading/updating external repos.
-# TODO:
-# updating tests and tpls is not necessary anymore since pipelines are now
-# using unique directories so repo are never shared with another pipeline. This
-# is not memory efficient (we keep a lot of data), hence this reminder.
-setup_baseline:
-  tags:
-    - shell
-    - quartz
-  stage: setup
-  variables:
-    GIT_STRATEGY: none
-  script:
-    #
-    # Setup ${BUILD_ROOT}/tpls and ${BUILD_ROOT}/tests:
-    #
-    - echo "MACHINE_NAME = ${MACHINE_NAME}"
-    - echo "REBASELINE = ${REBASELINE}"
-    - echo "AUTOTEST = ${AUTOTEST}"
-    - echo "AUTOTEST_COMMIT = ${AUTOTEST_COMMIT}"
-    - echo "BUILD_ROOT ${BUILD_ROOT}"
-    - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
-    - if [ ! -d "tpls" ]; then git clone ${TPLS_REPO}; fi
-    - if [ ! -d "tests" ]; then git clone ${TESTS_REPO}; fi
-    - cd tpls && git pull && cd ..
-    - cd tests && git pull origin && cd ..
-    #
-    # Setup ${AUTOTEST_ROOT}/autotest:
-    #
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - mkdir -p ${AUTOTEST_ROOT} && cd ${AUTOTEST_ROOT}
-    - command -v flock || echo "Required command 'flock' not found"
-    - |
-      (
-        date
-        echo "Waiting to aquire lock on '$PWD/autotest.lock' ..."
-        # try to get an excusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
-        done
-        echo "Aquired lock on '$PWD/autotest.lock'"
-        date
-        # clone/update the autotest repo while holding the file lock on
-        # 'autotest.lock'
-        err=0
-        if [[ ! -d "autotest" ]]; then
-          git clone ${AUTOTEST_REPO}
-        else
-          cd autotest && git pull && cd ..
-        fi || err=1
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock
@@ -1,94 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test
-# script then symlinks the repo to the parent directory of the MFEM source
-# directory. Unit tests that depend on the mfem/data repo will then detect that
-# this directory is present and be enabled.
-setup:
-  tags:
-    - shell
-    - quartz
-  stage: setup
-  variables:
-    GIT_STRATEGY: none
-  script:
-    #
-    # Setup MFEM_DATA_DIR=${SHARED_REPOS_DIR}/mfem-data, see '.gitlab-ci.yml'
-    # and '.gitlab/configs/<machine>-config.yml'
-    #
-    - echo "MACHINE_NAME = ${MACHINE_NAME}"
-    - echo "AUTOTEST = ${AUTOTEST}"
-    - echo "AUTOTEST_COMMIT = ${AUTOTEST_COMMIT}"
-    - echo "SHARED_REPOS_DIR ${SHARED_REPOS_DIR}"
-    - mkdir -p ${SHARED_REPOS_DIR} && cd ${SHARED_REPOS_DIR}
-    - command -v flock || echo "Required command 'flock' not found"
-    - |
-      (
-        date
-        echo "Waiting to aquire lock on '$PWD/mfem-data.lock' ..."
-        # try to get an excusive lock on fd 9 (mfem-data.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
-        done
-        echo "Aquired lock on '$PWD/mfem-data.lock'"
-        date
-        # clone/update the mfem/data repo while holding the file lock on
-        # 'mfem-data.lock'
-        err=0
-        if [[ ! -d "mfem-data" ]]; then
-          git clone ${MFEM_DATA_REPO} "mfem-data"
-        else
-          cd "mfem-data" && git pull && cd ..
-        fi || err=1
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> mfem-data.lock
-    #
-    # Setup ${AUTOTEST_ROOT}/autotest:
-    #
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - mkdir -p ${AUTOTEST_ROOT} && cd ${AUTOTEST_ROOT}
-    - |
-      (
-        date
-        echo "Waiting to aquire lock on '$PWD/autotest.lock' ..."
-        # try to get an excusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
-        done
-        echo "Aquired lock on '$PWD/autotest.lock'"
-        date
-        # clone/update the autotest repo while holding the file lock on
-        # 'autotest.lock'
-        err=0
-        if [[ ! -d "autotest" ]]; then
-          git clone ${AUTOTEST_REPO}
-        else
-          cd autotest && git pull && cd ..
-        fi || err=1
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock
@@ -1,67 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-stages:
-  - setup
-  - allocate_resource
-  - build_and_test
-  - release_resource_and_report
-
-# Slurm shared allocation
-allocate_resource:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_corona
-  stage: allocate_resource
-  script:
-    - echo ${ALLOC_NAME}
-    - salloc --exclusive --nodes=1 --partition=mi60 --time=30 --no-shell --job-name=${ALLOC_NAME}
-  timeout: 6h
-  needs: [setup]
-
-# Build and test jobs, simply provide a spec
-rocm_gcc_8.3.1:
-  variables:
-    SPEC: "@develop%gcc@8.3.1+rocm amdgpu_target=gfx906"
-  extends: .build_and_test_on_corona
-  needs: [allocate_resource]
-
-# Release slurm allocation
-release_resource:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_corona
-  stage: release_resource_and_report
-  script:
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
-  needs: [rocm_gcc_8.3.1]
-
-# Jobs report
-report_job_success:
-  stage: release_resource_and_report
-  extends:
-    - .on_corona
-    - .report_job_success
-
-report_job_failure:
-  stage: release_resource_and_report
-  extends:
-    - .on_corona
-    - .report_job_failure
-
-include:
-  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/corona-config.yml
-  - local: .gitlab/configs/setup-build-and-test.yml
-  - local: .gitlab/configs/report-build-and-test.yml
@@ -9,31 +9,26 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

-stages:
-  - setup
-  - build_and_test
-  - report
+# GitLab pipelines configurations for the Lassen machine at LLNL
+
+.on_lassen:
+  tags:
+    - shell
+    - lassen
+  rules:
+    - if: '$CI_COMMIT_BRANCH =~ /_lnone/ || $ON_LASSEN == "OFF"' #run except if ...
+      when: never
+    - when: on_success
+
+# Spack helped builds
+# Generic lassen build job, extending build script
+# Note: Lassen jobs can start as soon as the setup job is complete.
+.build_and_test_on_lassen:
+  extends: [.build_blueos_3_ppc64le_ib_script, .on_lassen]
+  stage: l_build_and_test
+  needs: [setup]

 opt_mpi_cuda_xl_16_1_1_8:
  variables:
    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=70"
  extends: .build_and_test_on_lassen
-
-# Jobs report
-report_job_success:
-  stage: report
-  extends:
-    - .on_lassen
-    - .report_job_success
-
-report_job_failure:
-  stage: report
-  extends:
-    - .on_lassen
-    - .report_job_failure
-
-include:
-  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/lassen-config.yml
-  - local: .gitlab/configs/setup-build-and-test.yml
-  - local: .gitlab/configs/report-build-and-test.yml
@@ -1,133 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-variables:
-  BASELINE_TEST: baseline
-
-stages:
-  - setup
-  - baseline_check
-  - baseline_report
-  - cleanup
-  - baseline_publish
-
-baselinecheck_mfem_intel_quartz:
-  extends: [.on_quartz]
-  stage: baseline_check
-  variables:
-    # TPLS_DIR is used in .gitlab/scripts/baseline to provide the tpls location
-    # when call the runtest script in MFEM test repo.
-    # Note: the value must be consistent with the setup performed in
-    #       .gitlab/configs/setup-baseline.yml.
-    TPLS_DIR: ${BUILD_ROOT}/tpls
-  script:
-    - echo ${BUILD_ROOT}
-    - echo ${TPLS_DIR}
-    # Used by the tests in MFEM/tests:
-    - export MFEM_TEST_NP=32
-    # The next script uses the following environment variables:
-    # * BASELINE_TEST, SYS_TYPE, CI_PROJECT_DIR, ARTIFACTS_DIR,
-    # * BUILD_ROOT, TPLS_DIR, MACHINE_NAME
-    - .gitlab/scripts/baseline
-  artifacts:
-    when: always
-    paths:
-      - ${ARTIFACTS_DIR}
-  allow_failure: true
-
-cleanup:
-  extends: .on_quartz
-  stage: cleanup
-  variables:
-    GIT_STRATEGY: none
-  script:
-    - echo "BUILD_ROOT=${BUILD_ROOT}"
-    - rm -rf "${BUILD_ROOT}" || true
-
-report_baseline:
-  extends: [.on_quartz]
-  stage: baseline_report
-  script:
-    - echo ${MACHINE_NAME}
-    - echo ${AUTOTEST}
-    - echo ${AUTOTEST_COMMIT}
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - cd ${AUTOTEST_ROOT}
-    - |
-      (
-        date
-        echo "Waiting to aquire lock on '$PWD/autotest.lock' ..."
-        # try to get an excusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
-        done
-        echo "Aquired lock on '$PWD/autotest.lock'"
-        date
-        # ----------------------
-        cd ${AUTOTEST_ROOT}/autotest || \
-          { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
-        mkdir -p ${MACHINE_NAME}
-        rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-${BASELINE_TEST}-${CI_COMMIT_REF_SLUG}"
-        rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir ${rundir})
-        cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
-        # We create an autotest-email.html file, because that's how we signal that there was a diff (temporary).
-        if [[ -f ${rundir}/${BASELINE_TEST}.err ]]; then
-          cp ${rundir}/${BASELINE_TEST}.err ${rundir}/autotest-email.html
-        fi
-        printf "%s\n" "" "Pipeline URL:" "$CI_PIPELINE_URL" \
-          >> ${rundir}/pipeline.txt
-        msg="GitLab CI log for ${BASELINE_TEST} on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
-        if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
-          git pull && \
-          git add ${rundir} && \
-          git commit -m "${msg}" && \
-          git push origin master
-        else
-          for file in ${rundir}/*; do
-            echo "------------------------------"
-            echo "Content of '$file'"
-            echo "******************************"
-            cat $file
-            echo "******************************"
-          done
-          rm -rf ${rundir} || true
-        fi
-        err=$?
-        # ----------------------
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock
-
-baselinepublish_mfem_quartz:
-  extends: [.on_quartz]
-  stage: baseline_publish
-  rules:
-    # - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "YES"'
-    - if: '$REBASELINE == "YES"'
-      when: manual
-  script:
-    - echo ${BUILD_ROOT}
-    - echo ${PWD}
-    - echo ${ARTIFACTS_DIR}
-    - ls -lA ${ARTIFACTS_DIR}
-    - .gitlab/scripts/rebaseline
-
-include:
-  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/quartz-config.yml
-  - local: .gitlab/configs/setup-baseline.yml
@@ -1,99 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-stages:
-  - setup
-  - allocate_resource
-  - build_and_test
-  - release_resource_and_report
-
-# Allocate
-allocate_resource:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_quartz
-  stage: allocate_resource
-  script:
-    - echo ${ALLOC_NAME}
-    - salloc --exclusive --nodes=1 --partition=pdebug --time=30 --no-shell --job-name=${ALLOC_NAME}
-  timeout: 6h
-
-# GitLab jobs for the Quartz machine at LLNL
-debug_ser_gcc_4_9_3:
-  variables:
-    SPEC: "%gcc@4.9.3 +debug~mpi"
-  extends: .build_and_test_on_quartz
-
-debug_ser_gcc_6_1_0:
-  variables:
-    SPEC: "%gcc@6.1.0 +debug~mpi"
-  extends: .build_and_test_on_quartz
-
-debug_par_gcc_6_1_0:
-  variables:
-    SPEC: "%gcc@6.1.0 +debug+mpi"
-  extends: .build_and_test_on_quartz
-
-opt_ser_gcc_6_1_0:
-  variables:
-    SPEC: "%gcc@6.1.0 ~mpi"
-  extends: .build_and_test_on_quartz
-
-opt_par_gcc_6_1_0:
-  variables:
-    SPEC: "%gcc@6.1.0"
-  extends: .build_and_test_on_quartz
-
-opt_par_gcc_6_1_0_sundials:
-  variables:
-    SPEC: "%gcc@6.1.0 +sundials"
-  extends: .build_and_test_on_quartz
-
-opt_par_gcc_6_1_0_petsc:
-  variables:
-    SPEC: "%gcc@6.1.0 +petsc ^petsc+mumps"
-  extends: .build_and_test_on_quartz
-
-opt_par_gcc_6_1_0_pumi:
-  variables:
-    SPEC: "%gcc@6.1.0 +pumi"
-  extends: .build_and_test_on_quartz
-
-# Release
-release_resource:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_quartz
-  stage: release_resource_and_report
-  script:
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
-
-# Jobs report
-report_job_success:
-  stage: release_resource_and_report
-  extends:
-    - .on_quartz
-    - .report_job_success
-
-report_job_failure:
-  stage: release_resource_and_report
-  extends:
-    - .on_quartz
-    - .report_job_failure
-
-include:
-  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/quartz-config.yml
-  - local: .gitlab/configs/setup-build-and-test.yml
-  - local: .gitlab/configs/report-build-and-test.yml
@@ -0,0 +1,184 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# GitLab pipelines configurations for the Quartz machine at LLNL
+
+.on_quartz:
+  tags:
+    - shell
+    - quartz
+  rules:
+    # Don't run quartz jobs if...
+    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"'
+      when: never
+    # Don't run autotest update if...
+    - if: '$CI_JOB_NAME =~ /update_autotest/ && $AUTOTEST != "YES"'
+      when: never
+    # Don't run autotest update if...
+    - if: '$CI_JOB_NAME =~ /q_report/ && $AUTOTEST != "YES"'
+      when: never
+    # Report success on success status
+    - if: '$CI_JOB_NAME =~ /q_report_success/ && $AUTOTEST == "YES"'
+      when: on_success
+    # Report failure on failure status
+    - if: '$CI_JOB_NAME =~ /q_report_failure/ && $AUTOTEST == "YES"'
+      when: on_failure
+    # Always release resources
+    - if: '$CI_JOB_NAME =~ /release_resources/'
+      when: always
+    # Default is to run if previous stage succeeded
+    - when: on_success
+
+# This is a yaml anchor, it can be used to avoid duplication like here.
+# The code below will simply be pasted wherever the anchor is placed.
+.safe_create_rundir: &safe_create_rundir |
+  if ! mkdir ${rundir}; then
+    n=1
+    while ! mkdir ${rundir}_${n}
+    do
+      n=$((n+1))
+    done
+    rundir=${rundir}_${n}
+  fi
+
+# Allocate
+q_allocate_resources:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_quartz
+  stage: q_allocate_resources
+  script:
+    - salloc --exclusive --nodes=1 --partition=pdebug --time=30 --no-shell --job-name=${ALLOC_NAME}
+  timeout: 6h
+
+# Release
+q_release_resources:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_quartz
+  stage: q_release_resources
+  script:
+    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
+    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
+
+# Release
+q_report_success:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_quartz
+  stage: q_release_resources
+  script:
+    - echo "Can only run if all the quartz jobs passed"
+    - cd ${AUTOTEST_ROOT}/autotest && git pull
+    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
+    - *safe_create_rundir
+    - echo "The Quartz jobs were successful" > ${rundir}/gitlab.out
+    - echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
+    - git add ${rundir}
+    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
+    - git push origin master
+
+q_report_failure:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_quartz
+  stage: q_release_resources
+  script:
+    - echo "Runs if there was at least one failure on quartz"
+    - cd ${AUTOTEST_ROOT}/autotest && git pull
+    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
+    - *safe_create_rundir
+    - echo "There was an error while running CI on Quartz" > ${rundir}/gitlab.err
+    - echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
+    - cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
+    - git add ${rundir}
+    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
+    - git push origin master
+
+# Spack helped builds
+# Generic quartz build job, extending build script
+.build_and_test_on_quartz:
+  extends: [.build_toss_3_x86_64_ib_script, .on_quartz]
+  stage: q_build_and_test
+
+# Build MFEM
+debug_ser_gcc_4_9_3:
+  variables:
+    SPEC: "%gcc@4.9.3 +debug~mpi"
+  extends: .build_and_test_on_quartz
+
+debug_ser_gcc_6_1_0:
+  variables:
+    SPEC: "%gcc@6.1.0 +debug~mpi"
+  extends: .build_and_test_on_quartz
+
+debug_par_gcc_6_1_0:
+  variables:
+    SPEC: "%gcc@6.1.0 +debug+mpi"
+  extends: .build_and_test_on_quartz
+
+opt_ser_gcc_6_1_0:
+  variables:
+    SPEC: "%gcc@6.1.0 ~mpi"
+  extends: .build_and_test_on_quartz
+
+opt_par_gcc_6_1_0:
+  variables:
+    SPEC: "%gcc@6.1.0"
+  extends: .build_and_test_on_quartz
+
+opt_par_gcc_6_1_0_sundials:
+  variables:
+    SPEC: "%gcc@6.1.0 +sundials"
+  extends: .build_and_test_on_quartz
+
+opt_par_gcc_6_1_0_petsc:
+  variables:
+    SPEC: "%gcc@6.1.0 +petsc ^petsc+mumps"
+  extends: .build_and_test_on_quartz
+
+opt_par_gcc_6_1_0_pumi:
+  variables:
+    SPEC: "%gcc@6.1.0 +pumi"
+  extends: .build_and_test_on_quartz
+
+# Baseline jobs form an independent set of jobs. We use `needs:[]` to specify
+# that "setup-baseline" can start immediately. Then, we have to use needs for
+# each one of the baseline jobs, otherwise they will wait for the rest of the
+# pipeline.
+
+baselinecheck_mfem_intel_quartz:
+  extends: [.baselinecheck_mfem, .on_quartz]
+  needs: [setup_baseline]
+
+update_autotest:
+  extends: [.on_quartz]
+  needs: [baselinecheck_mfem_intel_quartz]
+  stage: baseline_to_autotest
+  script:
+    - cd ${AUTOTEST_ROOT}/autotest && git pull
+    - rundir="quartz/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
+    - *safe_create_rundir
+    - cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
+    # We create an autotest-email.html file, because that's how we signal that there was a diff (temporary).
+    - |
+      if [[ -f ${rundir}/*.err ]]
+      then
+        echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/*.err
+        cp ${rundir}/*.err ${rundir}/autotest-email.html
+      fi
+    - git add ${rundir}
+    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
+    - git push origin master
+
+baselinepublish_mfem_quartz:
+  extends: [.on_quartz, .rebaseline_mfem]
+  needs: [baselinecheck_mfem_intel_quartz]
@@ -1,88 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# locals
-glob_err=${BASELINE_TEST}.err
-base=${BASELINE_TEST}-${SYS_TYPE}
-base_diff=${base}.diff
-base_patch=${base}.patch
-base_out=${base}.out
-artifacts_path=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}
-
-# prepare
-cd ${BUILD_ROOT} || \
-  { echo "Invalid BUILD_ROOT=$BUILD_ROOT"; exit 1; }
-ln -snf ${CI_PROJECT_DIR} mfem
-cd tests
-[[ -d _${BASELINE_TEST} ]] && rm -rf _${BASELINE_TEST}
-mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
-
-# run
-if [[ "${MACHINE_NAME}" == "quartz" || "${MACHINE_NAME}" == "ruby" ]]; then
-  srun --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
-elif [[ ${MACHINE_NAME} == "corona" ]]; then
-  srun --nodes=1 -t 60 -p mi60 ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
-elif [[ ${MACHINE_NAME} == "lassen" ]]; then
-  lalloc 1 -q pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
-else
-  echo "Unknown machine: MACHINE_NAME=$MACHINE_NAME"
-  exit 1
-fi
-
-# post
-mkdir ${artifacts_path}
-
-if [[ -s ${glob_err} ]]
-then
-  echo "ERROR during ${BASELINE_TEST} execution";
-  echo "Here is the ${glob_err} file content";
-  cat ${glob_err}
-  cp ${glob_err} ${artifacts_path}/${glob_err}
-  exit 1;
-elif [[ ! -f ${base_patch} && ! -f ${base_out} ]]
-then
-  echo "Something went WRONG in ${BASELINE_TEST}:";
-  echo "Either ${base_patch} or ${base_out} should exists";
-  exit 1;
-elif [[ -f ${base_patch} ]]
-then
-  echo "${BASELINE_TEST}: Differences found, patch generated"
-  cp ${base_patch} ${artifacts_path}/${base_patch}
-elif [[ -f ${base_out} ]]
-then
-  echo "${BASELINE_TEST}: Differences found, replacement file generated"
-  cp ${base_out} ${artifacts_path}/${base_out}
-fi
-
-if [[ -f ${BASELINE_TEST}.out ]]; then
-  cp ${BASELINE_TEST}.out ${artifacts_path}
-fi
-
-# base_diff won't even exist if there is no difference.
-if [[ -f ${base_diff} ]]
-then
-  echo "${BASELINE_TEST}: Relevant differences (filtered diff) ..."
-  cat ${base_diff}
-  cp ${base_diff} ${artifacts_path}/${base_diff}
-  # We create a .err file, because that's how we signal that there was a diff.
-  cp ${base_diff} ${artifacts_path}/gitlab-${BASELINE_TEST}-${MACHINE_NAME}.err
-fi
-
-if [[ ! -s ${base_diff} ]]
-then
-  echo "${BASELINE_TEST}: PASSED"
-  true
-else
-  echo "${BASELINE_TEST}: FAILED"
-  false
-fi
@@ -1,49 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# There will be collision between corona and quartz baselines.
-# Once the corresponding files have been generated, we can switch to machine
-# specific ref.
-ARTIFACT_PATH=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}
-#ARTIFACT_PATH=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}-${MACHINE_NAME}
-
-PATCH_FILE=${ARTIFACT_PATH}.patch
-FULL_FILE=${ARTIFACT_PATH}.out
-DIFF_FILE=${ARTIFACT_PATH}.diff
-
-# There will be collision between corona and quartz baselines.
-# Once the corresponding files have been generated, we can switch to machine
-# specific ref.
-SAVED_NAME=baseline-${SYS_TYPE}.saved
-#SAVED_NAME=baseline-${SYS_TYPE}-${MACHINE_NAME}.saved
-
-cd ${BUILD_ROOT}/tests
-
-if [[ ! -f "${DIFF_FILE}" ]]
-then
-    echo "Nothing to be done: no relevant change in baseline"
-    exit 0
-elif [[ -f "${PATCH_FILE}" ]]
-then
-    patch "${SAVED_NAME}" < "${PATCH_FILE}"
-elif [[ -f "${FULL_FILE}" ]]
-then
-    cp "${FULL_FILE}" "${SAVED_NAME}"
-else
-    echo "File missing: expected ${PATCH_FILE} or ${FULL_FILE}"
-    exit 1
-fi
-
-git add "${SAVED_NAME}"
-git commit -m "${SYS_TYPE} (${MACHINE_NAME}) rebaselined in GitLab pipeline ${CI_PIPELINE_ID}"
-git push origin master
@@ -1,45 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-echo "Runs if there was at least one failure on ${MACHINE_NAME}"
-
-cd ${AUTOTEST_ROOT}/autotest || \
-  { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
-mkdir -p ${MACHINE_NAME}
-
-rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
-rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir)
-
-printf "%s\n" "Some 'build-and-test' jobs on ${MACHINE_NAME} FAILED." \
-  "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.err
-
-msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
-
-# Create 'autotest-email.html' to indicate failure:
-cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
-
-if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
-  git pull && \
-  git add ${rundir} && \
-  git commit -m "${msg}" && \
-  git push origin master
-else
-  for file in ${rundir}/*; do
-    echo "------------------------------"
-    echo "Content of '$file'"
-    echo "******************************"
-    cat $file
-    echo "******************************"
-  done
-  rm -rf ${rundir} || true
-fi
@@ -1,42 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-echo "Can only run if all the ${MACHINE_NAME} jobs passed"
-
-cd ${AUTOTEST_ROOT}/autotest || \
-  { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
-mkdir -p ${MACHINE_NAME}
-
-rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
-rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir)
-
-printf "%s\n" "The 'build-and-test' jobs on ${MACHINE_NAME} were SUCCESSFUL." \
-  "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.out
-
-msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
-
-if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
-  git pull && \
-  git add ${rundir} && \
-  git commit -m "${msg}" && \
-  git push origin master
-else
-  for file in ${rundir}/*; do
-    echo "------------------------------"
-    echo "Content of '$file'"
-    echo "******************************"
-    cat $file
-    echo "******************************"
-  done
-  rm -rf ${rundir} || true
-fi
@@ -1,42 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# This script takes a seed for a directory name and appends it with a counter
-# incremented until it can create a new directory with it.
-
-# Usage:
-#
-# Expects 1 argument: a string that is use as a seed for the directory name.
-#
-# > rundir="desired_name"
-# > rundir=$(./safe_create_rundir $rundir)
-
-set -o errexit
-set -o nounset
-
-rundir=${1:-""}
-if [[ -z ${rundir} ]]; then
-  >&2 echo "The script expects a string as argument for directory creation."
-  exit 1
-fi
-
-if ! mkdir ${rundir}; then
-  n=1
-  while ! mkdir ${rundir}_${n}
-  do
-    n=$((n+1))
-  done
-  rundir=${rundir}_${n}
-fi
-
-echo $rundir
@@ -10,42 +10,8 @@

 Version 4.3.1 (development)
 ===========================
- More explicit and consistent formating of the output of iterative solvers
-  with the new IterativeSolver::PrintLevel options. See linalg/solvers.hpp.
-
- Added support for automatic differentiation. Users can select between native
-  implementation and external library implementation during configuration. One
-  parallel and two serial examples are implemented in the miniapps/autodiff/
-  directory.
-
- GridFunctionCoefficient (and the related vector, gradient, divergence, and
-  curl classes) now work properly with LORDiscretization and LORSolver.
-
- Added support for mesh preprocessing to resolve fine scale problem data
-  before simulation. This feature uses adaptive mesh refinement to control the
-  associated data oscillation error. See the new Example 30/30p.
-
- Switched from Artistic Style (astyle) version 2.05.1 to version 3.1 for code
-  formatting. See the "make style" target.
-
- Split the fem/fe.?pp files into separate files in the new fem/fe/ directory
-  to simplify and clarify the organization of FiniteElement classes.
-
 - Added support for hr-adaptivity using TMOP-based error estimator.

- Coefficient::SetTime now propagates the new time into internally stored
-  Coefficient objects.
-
- Added initial support for google-benchmarks in the tests/benchmarks directory.
-  It can be enabled with MFEM_USE_BENCHMARK=YES.
-
- Added Binder (mybinder.org) configuration files for C++ MFEM Jupyter Notebooks
-  with inline GLVis visualization as well as a new examples/jupyter/ directory
-  with a sample notebook based on Example 1. Implementation based on xeus-cling,
-  github.com/jupyter-xeus/xeus-cling + xeus-glvis, github.com/GLVis/xeus-glvis.
-
- Added 'double' atomicAdd implementation for previous versions of CUDA.
-
 - Adding lowest order Nedelec and Raviart-Thomas basis functions on wedge
  shaped elements.

@@ -61,39 +27,6 @@ Version 4.3.1 (development)
  functions on wedges and pyramids which are not amenable to reordering. The
  ReorientTetMesh method of the Mesh and ParMesh classes has been deprecated.

- Gmsh meshes where all elements have zero physical tag (the default Gmsh
-  output format if no physical groups are defined) are now successfully loaded,
-  and elements are reassigned attribute number 1.
-
- Added new miniapps that use the ParELAG library, its hybrid smoothers, and the
-  hierarchy of spaces created by the element-based AMG (AMGe) methodology in
-  ParELAG to build multigrid solvers for H(curl) and H(div) forms. See the
-  miniapps/parelag directory for more details.
-
- Fixed several MinGW build issues on Windows.
-
- Remove the 'u' flag in the ar command, to update all files in the archive,
-  avoiding file name collisions from different subdirectories.
-
- Added initial TMOP-based capabilities for surface fitting and tangential
-  relaxation in the mesh-optimizer and pmesh-optimizer miniapps.
-
- Added ParMesh Adjaceny Set (adjset) creation support to the Conduit Mesh
-  Blueprint MFEM wrapper functions in ConduitDataCollection.
-
- Added full assembly / device support for the LinearForm integrators:
-  * DomainLF: L(v) := (f, v)
-  * VectorDomainLF: L(v) := ((f1,...,fn), (v1,...,vn))
-  * DomainLFGrad: L(v) := (f,grad v)
-  * VectorDomainLFGrad:  L(v) := ((f1x,f1y,f1z,...,fnx,fny,fnz),grad(v1,...,vn))
-  It can be enabled by setting the linear assembly level to
-  'LinearAssemblyLevel::FULL' with the 'LinearForm::SetAssemblyLevel' method.
-
- `HypreParVector` and `Vector` now support move semantics, and the copy
-  constructor for `HypreParVector` now copies the local vector data.
-
- The HPC versions of ex1 and ex1p (in miniapps/performance) now support
-  runtime selection of either 2D or 3D meshes.

 Version 4.3, released on July 29, 2021
 ======================================
@@ -90,11 +90,6 @@ include("${CMAKE_CURRENT_SOURCE_DIR}/config/XSDKDefaults.cmake")

 # Enable languages.
 enable_language(CXX)
-if (MINGW)
-   # MinGW GCC does not expose the functions jn/_jn, yn/_yn (used in Example
-   # 25/25p) unless we use '-std=gnu++11':
-   set(CMAKE_CXX_EXTENSIONS ON)
-endif()
 if (MFEM_USE_CUDA)
   if (MFEM_USE_HIP)
      message(FATAL_ERROR " *** MFEM_USE_HIP cannot be combined with MFEM_USE_CUDA.")
@@ -180,6 +175,10 @@ else()
  set(MFEM_DEBUG OFF)
 endif()

+if (WIN32)
+	add_definitions(-D_USE_MATH_DEFINES)
+endif()
+
 # MPI -> hypre; PETSc (optional)
 if (MFEM_USE_MPI)
  find_package(MPI REQUIRED)
@@ -252,11 +251,6 @@ if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
  endif()
  find_package(OpenMP REQUIRED)
  set(OPENMP_LIBRARIES ${OpenMP_CXX_LIBRARIES})
-  if(APPLE)
-    # On macOS, the compiler needs additional help to find the <omp.h> header.
-    # See issue #2642 for more information.
-    include_directories(${OpenMP_CXX_INCLUDE_DIRS})
-  endif(APPLE)
 endif()

 # SuiteSparse (before SUNDIALS which may depend on KLU)
@@ -338,11 +332,11 @@ if (MFEM_USE_AMGX)
 endif()

 if (MFEM_USE_CONDUIT)
-    find_package(Conduit REQUIRED conduit relay blueprint)
+    find_package(Conduit REQUIRED conduit relay blueprint )
 endif()

 if (MFEM_USE_FMS)
-    find_package(FMS REQUIRED fms)
+    find_package(FMS REQUIRED fms )
 endif()

 # Axom/Sidre
@@ -372,12 +366,6 @@ if (MFEM_USE_HIOP)
  # find_package updates HIOP_FOUND, HIOP_INCLUDE_DIRS, HIOP_LIBRARIES
 endif()

-# CoDiPack package
-if (MFEM_USE_CODIPACK)
-  find_package(CODIPACK REQUIRED)
-  # find_package updates CODIPACK_FOUND, CODIPACK_INCLUDE_DIRS, CODIPACK_LIBRARIES
-endif()
-
 # OCCA
 if (MFEM_USE_OCCA)
   find_package(OCCA REQUIRED)
@@ -393,11 +381,6 @@ if (MFEM_USE_UMPIRE)
   find_package(UMPIRE REQUIRED)
 endif()

-# GOOGLE-BENCHMARK
-if (MFEM_USE_BENCHMARK)
-   find_package(Benchmark REQUIRED)
-endif()
-
 # Caliper
 if (MFEM_USE_CALIPER)
  find_package(Caliper REQUIRED)
@@ -426,11 +409,6 @@ if (MFEM_USE_MKL_CPARDISO)
  endif()
 endif()

-# PARELAG
-if (MFEM_USE_PARELAG)
-   find_package(PARELAG REQUIRED)
-endif()
-
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
  if (APPLE)
@@ -454,10 +432,10 @@ endif()
 #    With newer versions of SuiteSparse which include METIS header using 64-bit
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
-set(MFEM_TPLS OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS PETSC
+set(MFEM_TPLS MPI_CXX OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS PETSC
    SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB NETCDF
    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
-    CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG MPI_CXX)
+    CUSPARSE MKL_CPARDISO AMGX CALIPER)

 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
@@ -522,9 +500,13 @@ set(MFEM_INSTALL_DIR ${CMAKE_INSTALL_PREFIX} CACHE PATH
 # Declaring the library
 mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
 # message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
-target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES})
+if (CMAKE_VERSION VERSION_GREATER 2.8.11)
+  target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES})
+else()
+  target_link_libraries(mfem ${TPL_LIBRARIES})
+endif()
 if (MINGW)
-  target_link_libraries(mfem PRIVATE ws2_32)
+  target_link_libraries(mfem ws2_32)
 endif()
 set_target_properties(mfem PROPERTIES VERSION "${mfem_VERSION}")
 set_target_properties(mfem PROPERTIES SOVERSION "${mfem_VERSION}")
@@ -563,21 +545,15 @@ endif()
 set(MFEM_CUSTOM_TARGET_PREFIX CACHE STRING "")

 #-------------------------------------------------------------------------------
-# Examples, miniapps, benchmarks and testing
+# Examples, miniapps, and testing
 #-------------------------------------------------------------------------------

-# Enable testing and benchmarks if required
+# Enable testing if required
 if (MFEM_ENABLE_TESTING)
  enable_testing()
  set(MFEM_ALL_TESTS_TARGET_NAME tests)
  add_mfem_target(${MFEM_ALL_TESTS_TARGET_NAME} OFF)
  add_subdirectory(tests EXCLUDE_FROM_ALL)
-
-  # Create a target for all benchmarks and, optionally, enable it.
-  set(MFEM_ALL_BENCHMARKS_TARGET_NAME benchmarks)
-  add_mfem_target(${MFEM_ALL_BENCHMARKS_TARGET_NAME}
-                  ${MFEM_ENABLE_GOOGLE_BENCHMARKS})
-  add_subdirectory(tests/benchmarks EXCLUDE_FROM_ALL)
 endif()

 # Define a target that all examples and miniapps will depend on.
@@ -587,11 +563,7 @@ add_custom_target(${MFEM_EXEC_PREREQUISITES_TARGET_NAME})
 # Create a target for all examples and, optionally, enable it.
 set(MFEM_ALL_EXAMPLES_TARGET_NAME examples)
 add_mfem_target(${MFEM_ALL_EXAMPLES_TARGET_NAME} ${MFEM_ENABLE_EXAMPLES})
-if (MFEM_ENABLE_EXAMPLES)
-  add_subdirectory(examples) #install examples if enabled
-else()
-  add_subdirectory(examples EXCLUDE_FROM_ALL)
-endif()
+add_subdirectory(examples EXCLUDE_FROM_ALL)

 # Create a target for all miniapps and, optionally, enable it.
 set(MFEM_ALL_MINIAPPS_TARGET_NAME miniapps)
@@ -601,7 +573,6 @@ add_subdirectory(miniapps EXCLUDE_FROM_ALL)
 # Target to build all executables, i.e. everything.
 add_custom_target(exec)
 add_dependencies(exec
-  ${MFEM_ALL_BENCHMARKS_TARGET_NAME}
  ${MFEM_ALL_EXAMPLES_TARGET_NAME}
  ${MFEM_ALL_MINIAPPS_TARGET_NAME}
  ${MFEM_ALL_TESTS_TARGET_NAME})
@@ -1,133 +0,0 @@
-# MFEM Code of Conduct
-
-## Our Pledge
-
-We as members, contributors, and leaders pledge to make participation in our
-community a harassment-free experience for everyone, regardless of age, body
-size, visible or invisible disability, ethnicity, sex characteristics, gender
-identity and expression, level of experience, education, socio-economic status,
-nationality, personal appearance, race, caste, color, religion, or sexual
-identity and orientation.
-
-We pledge to act and interact in ways that contribute to an open, welcoming,
-diverse, inclusive, and healthy community.
-
-## Our Standards
-
-Examples of behavior that contributes to a positive environment for our
-community include:
-
-* Demonstrating empathy and kindness toward other people
-* Being respectful of differing opinions, viewpoints, and experiences
-* Giving and gracefully accepting constructive feedback
-* Accepting responsibility and apologizing to those affected by our mistakes,
-  and learning from the experience
-* Focusing on what is best not just for us as individuals, but for the overall
-  community
-
-Examples of unacceptable behavior include:
-
-* The use of sexualized language or imagery, and sexual attention or advances of
-  any kind
-* Trolling, insulting or derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or email address,
-  without their explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-## Enforcement Responsibilities
-
-Community leaders are responsible for clarifying and enforcing our standards of
-acceptable behavior and will take appropriate and fair corrective action in
-response to any behavior that they deem inappropriate, threatening, offensive,
-or harmful.
-
-Community leaders have the right and responsibility to remove, edit, or reject
-comments, commits, code, wiki edits, issues, and other contributions that are
-not aligned to this Code of Conduct, and will communicate reasons for moderation
-decisions when appropriate.
-
-## Scope
-
-This Code of Conduct applies within all community spaces, and also applies when
-an individual is officially representing the community in public spaces.
-Examples of representing our community include using an official e-mail address,
-posting via an official social media account, or acting as an appointed
-representative at an online or offline event.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported to the community leaders responsible for enforcement at mfem@llnl.gov.
-All complaints will be reviewed and investigated promptly and fairly.
-
-All community leaders are obligated to respect the privacy and security of the
-reporter of any incident. Anyone involved in the reported behavior will recuse
-themselves from the investigation and decision making about the resolution of
-the complaint.
-
-## Enforcement Guidelines
-
-Community leaders will follow these Community Impact Guidelines in determining
-the consequences for any action they deem in violation of this Code of Conduct:
-
-### 1. Correction
-
-**Community Impact**: Use of inappropriate language or other behavior deemed
-unprofessional or unwelcome in the community.
-
-**Consequence**: A private, written warning from community leaders, providing
-clarity around the nature of the violation and an explanation of why the
-behavior was inappropriate. A public apology may be requested.
-
-### 2. Warning
-
-**Community Impact**: A violation through a single incident or series of
-actions.
-
-**Consequence**: A warning with consequences for continued behavior. No
-interaction with the people involved, including unsolicited interaction with
-those enforcing the Code of Conduct, for a specified period of time. This
-includes avoiding interactions in community spaces as well as external channels
-like social media. Violating these terms may lead to a temporary or permanent
-ban.
-
-### 3. Temporary Ban
-
-**Community Impact**: A serious violation of community standards, including
-sustained inappropriate behavior.
-
-**Consequence**: A temporary ban from any sort of interaction or public
-communication with the community for a specified period of time. No public or
-private interaction with the people involved, including unsolicited interaction
-with those enforcing the Code of Conduct, is allowed during this period.
-Violating these terms may lead to a permanent ban.
-
-### 4. Permanent Ban
-
-**Community Impact**: Demonstrating a pattern of violation of community
-standards, including sustained inappropriate behavior, harassment of an
-individual, or aggression toward or disparagement of classes of individuals.
-
-**Consequence**: A permanent ban from any sort of public interaction within the
-community.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage],
-version 2.1, available at
-[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
-
-Community Impact Guidelines were inspired by
-[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
-
-For answers to common questions about this code of conduct, see the FAQ at
-[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
-[https://www.contributor-covenant.org/translations][translations].
-
-[homepage]: https://www.contributor-covenant.org
-[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
-[Mozilla CoC]: https://github.com/mozilla/diversity
-[FAQ]: https://www.contributor-covenant.org/faq
-[translations]: https://www.contributor-covenant.org/translations
@@ -21,9 +21,6 @@ documentation; new examples and miniapps; HPC performance improvements; etc.
 MFEM is distributed under the terms of the BSD-3 license. All new contributions
 must be made under this license.

-Note also that MFEM has a [Code of Conduct](CODE_OF_CONDUCT.md). By participating
-in the MFEM community, you agree to abide by its rules.
-
 If you plan on contributing to MFEM, consider reviewing the
 [issue tracker](https://github.com/mfem/mfem/issues) first to check if a thread
 already exists for your desired feature or the bug you ran into. Use a pull
@@ -45,7 +42,6 @@ back to them before issuing pull requests:
  - [New Feature Development](#new-feature-development)
  - [Developer Guidelines](#developer-guidelines)
  - [Pull Requests](#pull-requests)
-  - [MFEM PR Rules](#mfem-pr-rules)
  - [Pull Request Checklist](#pull-request-checklist)
  - [Master/Next Workflow](#masternext-workflow)
  - [Releases](#releases)
@@ -71,9 +67,8 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
  with regards to documentation and code styling.
 - Please do not commit large/binary files to the central repository (use a fork
  instead).
- Pull requests should be issued toward `mfem:master`. Make sure
-  to check the items off the [Pull Request Checklist](#pull-request-checklist) and
-  follow the [MFEM PR Rules](#mfem-pr-rules).
+- Pull requests  should be issued toward `mfem:master`. Make sure
+  to check the items off the [Pull Request Checklist](#pull-request-checklist).
 - When your contribution is fully working and ready to be reviewed, add
  the `ready-for-review` label.
 - PRs are treated similarly to journal submission with an "editor" assigning two
@@ -110,14 +105,12 @@ The MFEM source code has the following structure:
  │   ├── caliper
  │   ├── ginkgo
  │   ├── hiop
-  │   ├── jupyter
  │   ├── petsc
  │   ├── pumi
  │   ├── sundials
  |   └── superlu
  ├── fem
  │   ├── ceed
-  │   ├── fe
  │   ├── qinterp
  │   └── tmop
  ├── general
@@ -126,7 +119,6 @@ The MFEM source code has the following structure:
  ├── mesh
  ├── miniapps
  │   ├── adjoint
-  │   ├── autodiff
  │   ├── common
  │   ├── electromagnetics
  │   ├── gslib
@@ -134,14 +126,12 @@ The MFEM source code has the following structure:
  │   ├── mtop
  │   ├── navier
  │   ├── nurbs
-  │   ├── parelag
  │   ├── performance
  │   ├── shifted
  │   ├── solvers
  │   ├── tools
  │   └── toys
  └── tests
-      ├── benchmarks
      ├── convergence
      ├── gitlab
      ├── mem_manager
@@ -333,22 +323,15 @@ Before you can start, you need a GitHub account, here are a few suggestions:
    change the code by default.

 - Code specifics
-  - All new public, protected, and private classes, methods, data members, and
-    functions have Doxygen-style documentation in source comments.
-  - In addition to arguments and functionality, documentation should include the
-    current limitations of the code, any background information that is
-    implicitly assumed in the implementation, and the ownership and lifetime
-    of data.
+  - All significant new classes, methods and functions have Doxygen-style
+    documentation in source comments.
  - Consistent code styling is enforced with `make style` in the top-level
    directory. This requires [Artistic Style](http://astyle.sourceforge.net) (we
-    specifically use version 3.1). See also the file `config/mfem.astylerc`.
+    specifically use version 2.05.1). See also the file `config/mfem.astylerc`.
  - Use `mfem::out` and `mfem::err` instead of `std::cout` and `std::cerr` in
    internal library code. (You can use `std` in examples and miniapps.)
  - When manually resolving conflicts during a merge, make sure to mention the
    conflicted files in the commit message.
-  - All significant new features and changes should be documented in CHANGELOG.
-  - New examples and miniapps should have documentation on the MFEM webpage.
-

 ### Pull Requests

@@ -414,83 +397,6 @@ Before you can start, you need a GitHub account, here are a few suggestions:
 - If triggered, track the status of the LLNL GitLab tests. If failing, ask
  one of the _LLNL developers_ for details.

-
-### MFEM PR Rules
-
-The Pull Request (PR) approval process in MFEM is similar to the approval of papers in a peer-reviewed journal. In particular:
-
-1. There is an MFEM board of "editors" that evaluates new PRs and assigns "reviewers" for each PR.
-
-2. The assigned reviewers are responsible to carefully review and test the proposed PR.
-
-3. A PR can be (manually) merged in the *next* branch only if 2 of the assigned reviewers have approved it and it has passed internal testing. This merge can be performed by any of the assigned reviewers or by any of the editors.
-
-4. A PR can be merged in the *master* branch only if it has been tested successfully for a week in *next* and an editor has (optionally) taken a final look. This merge can be performed only by one of the editors.
-
-#### Responsibilities of Editors
-
-The current list of MFEM editors is:
-
- @v-dobrev (Veselin Dobrev)
- @tzanio (Tzanio Kolev)
- @pazner (Will Pazner)
- @mlstowell (Mark Stowell)
-
-**The responsibilities of the editors are:**
-
-1. To assign appropriate milestone and labels for new PRs, e.g. *bugfix*, *minor*, *api-change*, *high-impact*, etc.
-
-2. To assign at least 2 reviewers for new PRs. An editor can also be a reviewer. The editor, reviewers, and author should be listed as "Assignees" on the GitHub PR page. After assignment, the `in-review` label should be added.
-
-3. To complete the initial PR evaluation and assignments in a timely manner: 1 week from submission.
-
-4. To assist reviewers when they need help with their reviews (but also to stay out of the way when they don't).
-
-5. To remind the reviewers about timely completion of their review.
-
-6. To take a final look and complete the PR merge in *master*. The final look step is optional and shouldn't take more than 3 days.
-
-7. The assignment of bugfixes should be expedited proportional to their importance, e.g. in some cases the editor can assign much shorter review window.
-
-#### Responsibilities of Reviewers
-
-Everyone on the MFEM team can be asked to serve as a reviewer on a PR in their area of expertise.
-
-**The responsibilities of the reviewers are:**
-
-1. To let the editors know if the proposed assignment is not a good match for them.
-
-2. To communicate with the PR author, provide feedback and work with them to resolve issues.
-
-3. To ensure the quality of the PR by making sure that the code adheres to the [Developer Guidelines](#developer-guidelines), e.g. all methods, data members, and functions have documentation, including data ownership and lifetime, new examples/miniapps have a corresponding PR in mfem/web, major features have `CHANGELOG` entries, etc.
-
-3. To seek help from the editors in case of difficulties.
-
-4. To complete the review in a timely manner: 3 weeks from assignment.
-
-5. To test the PR thoroughly before merging in *next*. The PR author is also encouraged to perform testing and inform the reviewers about the results.
-
-6. To monitor the PR impact on the testing in the *next* branch and alert the editors that the PR is ready for merging in *master*.
-
-7. The review of bugfixes should be expedited proportional to their importance. The review window can be much less than three weeks in such cases.
-
-#### Responsibilities of Authors
-
-Authors should clearly indicate when a PR is ready for review (before that the PR should be marked as `Draft` or `[WIP]`).
-
-**The responsibilities of the authors are:**
-
-1. To follow the instructions and PR checklist in the `CONTRIBUTING.md` document in the MFEM repository.
-
-2. To respond to reviewer feedback in a timely manner.
-
-3. Authors are encouraged to perform testing and inform the reviewers about the results.
-
-4. Authors can use the "Reviewers" section of the GitHub PR page to suggest reviewers, but the "Assignees" section will show who the editor has assigned to do the reviews.
-
-5. To indicate when the PR is ready for review by adding the `ready-for-review` label.
-
-
 ### Pull Request Checklist

 Before a PR can be merged, it should satisfy the following:
@@ -544,9 +450,7 @@ Before a PR can be merged, it should satisfy the following:
     - [ ] The miniapps go at the end of the page, and are usually listed only under a specific "Application (PDE)" category.
     - [ ] Add a short description of the miniapp in the "Extensive Examples" section of `features.md`.
 - [ ] New capability:
-   - [ ] All new public, protected, and private classes, methods, data members, and functions have full Doxygen-style documentation in source comments. Documentation should include descriptions of member data, function arguments and return values, template parameters, and prerequisites for calling new functions.
-   - [ ] Pointer arguments and return values must specify whether ownership is being transferred or lent with the call.
-   - [ ] Any new functions should include descriptions of their intended use e.g. for internal use only, user-facing, etc., along with references to example code whenever possible/appropriate.
+   - [ ] All significant new classes, methods and functions have Doxygen-style documentation in source comments.
   - [ ] Consider adding new sample runs in existing examples to highlight the new capability.
   - [ ] Consider saving cool simulation pictures with the new capability in the Confluence gallery (LLNL only) or submitting them, via pull request, to the gallery section of the `mfem/web` repo.
   - [ ] If this is a major new feature, consider mentioning it in the short summary inside `README` *(rare)*.
@@ -557,7 +461,6 @@ Before a PR can be merged, it should satisfy the following:
 - [ ] (LLNL only) After merging:
   - [ ] Update internal tests to include the new features.

-
 ### Master/Next Workflow

 MFEM uses a `master`/`next`-branch workflow as described below:
@@ -649,10 +552,8 @@ MFEM uses a `master`/`next`-branch workflow as described below:
    - Update version and shortlinks in `src/index.md` and `src/download.md`.
    -  Use [cloc-1.62.pl](http://cloc.sourceforge.net/) and `ls -lh` to estimate the SLOC and the tarball size in `src/download.md`.

-
 ## LLNL Workflow

-
 ### Mirroring on Bitbucket

 - The GitHub `master` and `next` branches are mirrored to the LLNL institutional
@@ -672,17 +573,16 @@ MFEM uses a `master`/`next`-branch workflow as described below:
  - `mfem:gh-next` -- Bleeding-edge development version, may be broken, use at
     your own risk.

-
 ### Mirroring on GitLab

 - MFEM repository is also mirrored on the LLNL GitLab instance, in a
  semi-automated manner.

 - This instance is meant to complete CI testing with tests on Livermore
-  Computing systems. GitLab pipeline status is reported in the corresponding
+  Computing systems. Gitlab pipeline status is reported in the corresponding
  GitHub pull request.

- In GitLab pipelines, TPLs (dependencies) are built using Spack, driven by Uberenv.
+- In Gitlab pipelines, TPLs (dependencies) are built using Spack, driven by Uberenv.

 - No change to the MFEM repo can be made on this instance.

@@ -695,7 +595,6 @@ In addition, developers can set local git hooks to run some quick checks on
 commit or push, see the [README](config/githooks/README.md) in the `config/githooks`
 directory.

-
 ### Linux and Mac smoke tests
 We use GitHub Actions to drive the default tests on the `master` and `next`
 branches. See the `.github/workflows` files and the logs at
@@ -707,7 +606,6 @@ constraint on jobs. Two virtual machines are configured - Mac (OS X) and Linux.
 - Tests on the `master` branch are triggered whenever a PR is issued on this branch.
 - Tests on the `next` branch are currently scheduled to run each night.

-
 ### Windows smoke test
 We use Appveyor to test building with the MS Visual C++ compiler in a Windows
 environment, as well as to test the CMake build. See the `.appveyor` file and the
@@ -717,7 +615,6 @@ build logs at
 CMake is used to generate the MSVC Project files and drive the build.  A release
 and debug build is performed with a simple run of `ex1` to verify the executable.

-
 ### Tests at LLNL

 - We mirror the `master` and `next` branches internally (to `gh-master` and
@@ -459,22 +459,10 @@ MFEM_USE_UMPIRE = YES/NO
   discovery, provision, and management of memory on machines with multiple
   memory devices like NUMA and GPUs.

-MFEM_USE_BENCHMARK = YES/NO
-   Enables support for Google Benchmark, a library to support the benchmarking
-   of functions, in the tests/benchmarks directory.
-
 MFEM_USE_HIOP = YES/NO
   Enable the usage of HiOp (https://github.com/LLNL/hiop) in MFEM. HiOp is an
   HPC solver for nonlinear optimization problems.

-MFEM_USE_CODIPACK = YES/NO
-  Enable automatic differentiation using the CoDiPack library.
-  www.scicomp.uni-kl.de/codi/
-
-MFEM_USE_ADFORWARD = YES/NO
-  Enable forward mode for AD packages. This option is valid
-  only if the AD package supports two modes (backward/forward).
-
 MFEM_USE_CUDA = YES/NO
   Enables support for CUDA devices in MFEM. CUDA is a parallel computing
   platform and programming model for general computing on graphical processing
@@ -535,11 +523,6 @@ MFEM_USE_FMS = YES/NO
   convetion routines between FMS's FmsDataCollection structure and MFEM's
   DataCollection class, see the header file fem/fmsconvert.hpp.

-MFEM_USE_PARELAG = YES/NO
-   Enables the miniapps that use the ParELAG library. MFEM does not currently
-   use ParELAG. In fact, ParELAG is dependent on MFEM. Therefore, this option
-   currently only concerns the miniapps.
-
 MFEM_BUILD_TAG = (any value)
   An optional tag to characterize the build. Exported to config/config.mk.
   Can be used to identify the MFEM build from other makefiles.
@@ -709,12 +692,7 @@ The specific libraries and their options are:
 - HiOp (optional), used when MFEM_USE_HIOP = YES.
  URL: https://github.com/LLNL/hiop
  Options: HIOP_OPT, HIOP_LIB.
-  Versions: HIOP >= 0.4.6.
-
- CoDiPack (optiobal), used with MFEM_USE_CODIPACK = YES
-  URL: https://www.scicomp.uni-kl.de/codi/
-  Options: CODIPACK_OPT
-  Versions: 1.9.3
+  Versions: HIOP >= 0.4.

 - GSLIB (optional), used when MFEM_USE_GSLIB = YES. The gslib library must be
  built prior to the MFEM build, as follows: download gslib-1.0.7, untar it at
@@ -752,10 +730,10 @@ The specific libraries and their options are:
  Versions: libCEED >= 0.8.

 - RAJA (optional), used when MFEM_USE_RAJA = YES.
-  Beginning with MFEM v4.3, only RAJA v0.14.0+ is supported.
+  Beginning with MFEM v4.3, only RAJA v0.13.0+ is supported.
  URL: https://github.com/LLNL/RAJA
  Options: RAJA_DIR, RAJA_OPT, RAJA_LIB.
-  Versions: RAJA >= 0.14.0.
+  Versions: RAJA >= 0.13.0.

 - Caliper (optional), used when MFEM_USE_CALIPER = YES.
  URL: https://github.com/LLNL/Caliper
@@ -766,12 +744,7 @@ The specific libraries and their options are:
  Umpire requires camp when the Umpire version is >= 3.0.0.
  URL: https://github.com/LLNL/Umpire
  Options: UMPIRE_DIR, UMPIRE_OPT, UMPIRE_LIB.
-  Versions: Umpire >= 3.0.0.
-
- Benchmark, used when MFEM_USE_BENCHMARK = YES.
-  URL: https://github.com/google/benchmark
-  Options: BENCHMARK_DIR, BENCHMARK_LIB.
-  Versions: Benchmark >= 1.5.6.
+  Versions: Umpire >= 2.0.0.

 - MPFR (optional), used when MFEM_USE_MPFR = YES.
  URL: http://mpfr.org, it depends on the GMP library: https://gmplib.org
@@ -793,10 +766,6 @@ The specific libraries and their options are:
  Options: FMS_OPT, FMS_LIB.
  Versions: FMS >= 0.2.

- ParELAG, used when MFEM_USE_PARELAG = YES.
-  URL: https://github.com/LLNL/parelag
-  Options: PARELAG_DIR, PARELAG_OPT, PARELAG_LIB.
-
 Building with CMake
 ===================
 The MFEM build system consists of two steps: configuration and compilation.
@@ -921,8 +890,6 @@ MFEM_USE_MPFR
 MFEM_USE_ZLIB
 MFEM_USE_PUMI
 MFEM_USE_HIOP
-MFEM_USE_CODIPACK
-MFEM_USE_ADFORWARD
 MFEM_USE_CUDA
 MFEM_USE_OCCA
 MFEM_USE_CEED
@@ -931,8 +898,6 @@ MFEM_USE_UMPIRE
 MFEM_USE_SIDRE
 MFEM_USE_CALIPER
 MFEM_USE_FMS
-MFEM_USE_BENCHMARK
-MFEM_USE_PARELAG

 The following options are CMake specific:

@@ -982,15 +947,12 @@ The CMake build system adds auto-detection for the following packages/libraries:
 - POSIXCLOCKS
 - PUMI
 - HIOP
- - CoDiPack
 - OCCA
 - RAJA
 - UMPIRE
 - AXOM - Used when MFEM_USE_SIDRE is enabled
 - CALIPER
 - FMS
- - BENCHMARK
- - ParELAG

 The following built-in CMake packages are also used:

@@ -12,9 +12,6 @@ to enable high-performance scalable finite element discretization research and
 application development on a wide variety of platforms, ranging from laptops to
 supercomputers.

-We welcome contributions and feedback from the community. Please see the file
-CONTRIBUTING.md for additional details about our development process.
-
 * For building instructions, see the file INSTALL, or type "make help".

 * Copyright and licensing information can be found in files LICENSE and NOTICE.
@@ -22,6 +19,9 @@ CONTRIBUTING.md for additional details about our development process.
 * The best starting point for new users interested in MFEM's features is to
  review the examples and miniapps at https://mfem.org/examples.

+* Developers interested in contributing to the library, should read the
+  instructions and documentation in the CONTRIBUTING.md file.
+
 Conceptually, MFEM can be viewed as a finite element toolbox that provides the
 building blocks for developing finite element algorithms in a manner similar to
 that of MATLAB for linear algebra methods. In particular, MFEM provides support
@@ -283,11 +283,3 @@ ENDIF()
 IF (DEFINED TPL_ENABLE_UMPIRE)
  SET(MFEM_USE_UMPIRE ${TPL_ENABLE_UMPIRE} CACHE BOOL "Enable Umpire" FORCE)
 ENDIF()
-
-IF (DEFINED TPL_ENABLE_BENCHMARK)
-  SET(MFEM_USE_BENCHMARK ${TPL_ENABLE_BENCHMARK} CACHE BOOL "Enable Google-Benchmark" FORCE)
-ENDIF()
-
-IF (DEFINED TPL_ENABLE_PARELAG)
-  SET(MFEM_USE_PARELAG ${TPL_ENABLE_PARELAG} CACHE BOOL "Enable ParELAG" FORCE)
-ENDIF()
@@ -54,11 +54,7 @@ set(MFEM_USE_CEED @MFEM_USE_CEED@)
 set(MFEM_USE_UMPIRE @MFEM_USE_UMPIRE@)
 set(MFEM_USE_SIMD @MFEM_USE_SIMD@)
 set(MFEM_USE_ADIOS2 @MFEM_USE_ADIOS2@)
-set(MFEM_USE_CODIPACK @MFEM_USE_CODIPACK@)
-set(MFEM_USE_ADFORWARD @MFEM_USE_ADFORWARD@)
 set(MFEM_USE_CALIPER @MFEM_USE_CALIPER@)
-set(MFEM_USE_BENCHMARK @MFEM_USE_BENCHMARK@)
-set(MFEM_USE_PARELAG @MFEM_USE_PARELAG@)

 set(MFEM_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
 set(MFEM_CXX_FLAGS "@CMAKE_CXX_FLAGS@")
@@ -175,13 +175,4 @@
 // Enable interface to the MKL CPardiso library.
 #cmakedefine MFEM_USE_MKL_CPARDISO

-// Use forward mode for automatic differentiation
-#cmakedefine MFEM_USE_ADFORWARD
-
-// Enable the use of the CoDiPack library for AD
-#cmakedefine MFEM_USE_CODIPACK
-
-// Enable MFEM functionality based on the Google Benchmark library.
-#cmakedefine MFEM_USE_BENCHMARK
-
 #endif // MFEM_CONFIG_HEADER
@@ -1,22 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - BENCHMARK_FOUND
-#   - BENCHMARK_LIBRARIES
-#   - BENCHMARK_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(Benchmark BENCHMARK BENCHMARK_DIR
-        "include" "benchmark/benchmark.h"
-        "lib" "benchmark"
-        "Paths to headers required by Google Benchmark."
-        "Libraries required by Google Benchmark.")
@@ -1,24 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Automatic differentiation using the CoDiPack library.
-#  www.scicomp.uni-kl.de/codi/
-# Sets the following variables:
-#   - CODIPACK_FOUND
-#   - CODIPACK_INCLUDE_DIRS
-#   - CODIPACK_LIBRARIES
-
-include(MfemCmakeUtilities)
-mfem_find_package(CODIPACK CODIPACK CODIPACK_DIR
-  "include" "codi.h"
-  "lib" ""
-  "Paths to headers required by CODIPACK."
-  "Libraries required by CODIPACK.")
@@ -1,19 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - PARELAG_FOUND
-#   - PARELAG_LIBRARIES
-#   - PARELAG_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(PARELAG PARELAG PARELAG_DIR "" "" "" ""
-  "Paths to headers required by ParELAG." "Libraries required by ParELAG.")
@@ -100,8 +100,6 @@ macro(add_mfem_examples EXE_SRCS)

    string(REPLACE ".cpp" "" EXE_NAME "${EXE_PREFIX}${SRC_FILENAME}")
    mfem_add_executable(${EXE_NAME} ${SRC_FILE})
-    install(TARGETS ${EXE_NAME}
-            RUNTIME DESTINATION examples)
    add_dependencies(${MFEM_ALL_EXAMPLES_TARGET_NAME} ${EXE_NAME})
    if (EXE_NEEDED_BY)
      add_dependencies(${EXE_NEEDED_BY} ${EXE_NAME})
@@ -766,7 +764,7 @@ function(mfem_export_mk_files)
      MFEM_USE_GNUTLS MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC
      MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_CONDUIT MFEM_USE_PUMI
      MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA MFEM_USE_UMPIRE MFEM_USE_SIMD
-      MFEM_USE_ADIOS2 MFEM_USE_BENCHMARK MFEM_USE_PARELAG)
+      MFEM_USE_ADIOS2)
  foreach(var ${CONFIG_MK_BOOL_VARS})
    if (${var})
      set(${var} YES)
@@ -180,13 +180,4 @@
 // Enable interface to the MKL CPardiso library.
 // #define MFEM_USE_MKL_CPARDISO

-// Use forward mode for automatic differentiation
-// #define MFEM_USE_ADFORWARD
-
-// Enable the use of the CoDiPack library for AD
-// #define MFEM_USE_CODIPACK
-
-// Enable functionality based on the Google Benchmark library.
-// #define MFEM_USE_BENCHMARK
-
 #endif // MFEM_CONFIG_HEADER
@@ -58,10 +58,6 @@ MFEM_USE_UMPIRE        = @MFEM_USE_UMPIRE@
 MFEM_USE_SIMD          = @MFEM_USE_SIMD@
 MFEM_USE_ADIOS2        = @MFEM_USE_ADIOS2@
 MFEM_USE_MKL_CPARDISO  = @MFEM_USE_MKL_CPARDISO@
-MFEM_USE_ADFORWARD     = @MFEM_USE_ADFORWARD@
-MFEM_USE_CODIPACK      = @MFEM_USE_CODIPACK@
-MFEM_USE_BENCHMARK     = @MFEM_USE_BENCHMARK@
-MFEM_USE_PARELAG       = @MFEM_USE_PARELAG@

 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@
@@ -58,10 +58,6 @@ option(MFEM_USE_SIMD "Enable use of SIMD intrinsics" OFF)
 option(MFEM_USE_ADIOS2 "Enable ADIOS2" OFF)
 option(MFEM_USE_CALIPER "Enable Caliper support" OFF)
 option(MFEM_USE_MKL_CPARDISO "Enable MKL CPardiso" OFF)
-option(MFEM_USE_ADFORWARD "Enable forward mode for AD" OFF)
-option(MFEM_USE_CODIPACK "Enable automatic differentiation (AD) using CoDiPack" OFF)
-option(MFEM_USE_BENCHMARK "Enable Google Benchmark" OFF)
-option(MFEM_USE_PARELAG "Enable ParELAG" OFF)

 # Optional overrides for autodetected MPIEXEC and MPIEXEC_NUMPROC_FLAG
 # set(MFEM_MPIEXEC "mpirun" CACHE STRING "Command for running MPI tests")
@@ -78,7 +74,6 @@ set(MFEM_MPI_NP 4 CACHE STRING "Number of processes used for MPI tests")
 option(MFEM_ENABLE_TESTING "Enable the ctest framework for testing" ON)
 option(MFEM_ENABLE_EXAMPLES "Build all of the examples" OFF)
 option(MFEM_ENABLE_MINIAPPS "Build all of the miniapps" OFF)
-option(MFEM_ENABLE_GOOGLE_BENCHMARKS "Build all of the Google benchmarks" OFF)

 # Setting CXX/MPICXX on the command line or in user.cmake will overwrite the
 # autodetected C++ compiler.
@@ -229,25 +224,12 @@ set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")
 set(UMPIRE_DIR "${MFEM_DIR}/../umpire" CACHE PATH "Path to Umpire")
 set(CALIPER_DIR "${MFEM_DIR}/../caliper" CACHE PATH "Path to Caliper")
-set(BENCHMARK_DIR "${MFEM_DIR}/../google-benchmark" CACHE PATH
-    "Path to Google Benchmark")
-
-# Provide paths, since ParELAG is dependent on MFEM and MFEM needs to be
-# compiled (or at least cmake needs to succeed) before compiling ParELAG.
-set(PARELAG_DIR "${MFEM_DIR}/../parelag" CACHE PATH "Path to ParELAG")
-set(PARELAG_INCLUDE_DIRS "${PARELAG_DIR}/src;${PARELAG_DIR}/build/src" CACHE
-    STRING "Path to ParELAG headers.")
-set(PARELAG_LIBRARIES "${PARELAG_DIR}/build/src/libParELAG.a" CACHE STRING
-    "The ParELAG library.")

 set(BLAS_INCLUDE_DIRS "" CACHE STRING "Path to BLAS headers.")
 set(BLAS_LIBRARIES "" CACHE STRING "The BLAS library.")
 set(LAPACK_INCLUDE_DIRS "" CACHE STRING "Path to LAPACK headers.")
 set(LAPACK_LIBRARIES "" CACHE STRING "The LAPACK library.")

-set(CODIPACK_INCLUDE_DIRS "${MFEM_DIR}/../CoDiPack/inlude" CACHE STRING "Path to CoDiPack headers.")
-set(CODIPACK_LIBRARIES "")
-
 # Some useful variables:
 set(CMAKE_SKIP_PREPROCESSED_SOURCE_RULES ON) # Skip *.i rules
 set(CMAKE_SKIP_ASSEMBLY_SOURCE_RULES  ON)    # Skip *.s rules
@@ -59,12 +59,9 @@ HIP_FLAGS = --amdgpu-target=$(HIP_ARCH)
 HIP_XCOMPILER =
 HIP_XLINKER   = -Wl,

-# Flags for generating dependencies.
-DEP_FLAGS = -MM -MT
-
 ifneq ($(NOTMAC),)
   AR      = ar
-   ARFLAGS = crv
+   ARFLAGS = cruv
   RANLIB  = ranlib
   PICFLAG = $(XCOMPILER)-fPIC
   SO_EXT  = so
@@ -76,7 +73,7 @@ ifneq ($(NOTMAC),)
 else
   # Silence "has no symbols" warnings on Mac OS X
   AR      = ar
-   ARFLAGS = Scrv
+   ARFLAGS = Scruv
   RANLIB  = ranlib -no_warning_for_no_symbols
   PICFLAG = $(XCOMPILER)-fPIC
   SO_EXT  = dylib
@@ -89,9 +86,6 @@ else
   BUILD_RPATH = $(XLINKER)-undefined,dynamic_lookup
   INSTALL_SOFLAGS = $(subst $1 ,,$(call MAKE_SOFLAGS,$(MFEM_LIB_DIR)))
   INSTALL_RPATH = $(XLINKER)-undefined,dynamic_lookup
-   # Silence unused command line argument warnings when generating dependencies
-   # with mpicxx and clang
-   DEP_FLAGS := -Wno-unused-command-line-argument $(DEP_FLAGS)
 endif

 # Set CXXFLAGS to overwrite the default selection of DEBUG_FLAGS/OPTIM_FLAGS
@@ -157,10 +151,6 @@ MFEM_USE_UMPIRE        = NO
 MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
 MFEM_USE_MKL_CPARDISO  = NO
-MFEM_USE_ADFORWARD     = NO
-MFEM_USE_CODIPACK      = NO
-MFEM_USE_BENCHMARK     = NO
-MFEM_USE_PARELAG       = NO

 # MPI library compile and link flags
 # These settings are used only when building MFEM with MPI + HIP
@@ -416,11 +406,6 @@ HIOP_DIR = @MFEM_DIR@/../hiop/install
 HIOP_OPT = -I$(HIOP_DIR)/include
 HIOP_LIB = -L$(HIOP_DIR)/lib -lhiop $(LAPACK_LIB)

-# CoDiPack
-CODIPACK_DIR = @MFEM_DIR@/../CoDiPack
-CODIPACK_OPT = -I$(CODIPACK_DIR)
-CODIPACK_LIB =
-
 # GSLIB library
 GSLIB_DIR = @MFEM_DIR@/../gslib/build
 GSLIB_OPT = -I$(GSLIB_DIR)/include
@@ -428,7 +413,7 @@ GSLIB_LIB = -L$(GSLIB_DIR)/lib -lgs

 # CUDA library configuration
 CUDA_OPT =
-CUDA_LIB = -lcusparse -lnvToolsExt
+CUDA_LIB = -lcusparse

 # HIP library configuration (currently not needed)
 HIP_OPT =
@@ -444,11 +429,6 @@ CALIPER_DIR = @MFEM_DIR@/../caliper
 CALIPER_OPT = -I$(CALIPER_DIR)/include
 CALIPER_LIB = $(XLINKER)-rpath,$(CALIPER_DIR)/lib64 -L$(CALIPER_DIR)/lib64 -lcaliper

-# BENCHMARK library configuration
-BENCHMARK_DIR = @MFEM_DIR@/../google-benchmark
-BENCHMARK_OPT = -I$(BENCHMARK_DIR)/include
-BENCHMARK_LIB = -L$(BENCHMARK_DIR)/lib -lbenchmark -lpthread
-
 # libCEED library configuration
 CEED_DIR ?= @MFEM_DIR@/../libCEED
 CEED_OPT = -I$(CEED_DIR)/include
@@ -479,11 +459,6 @@ MKL_CPARDISO_LIB = $(XLINKER)-rpath,$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
   -L$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR) -l$(MKL_MPI_WRAPPER)\
   -lmkl_intel_lp64 -lmkl_sequential -lmkl_core

-# PARELAG library configuration
-PARELAG_DIR = @MFEM_DIR@/../parelag
-PARELAG_OPT = -I$(PARELAG_DIR)/src -I$(PARELAG_DIR)/build/src
-PARELAG_LIB = -L$(PARELAG_DIR)/build/src -lParELAG
-
 # If YES, enable some informational messages
 VERBOSE = NO

@@ -87,12 +87,12 @@ fi
 ## style check
 #if [[ "${option}" == "--style" || "${option}" == "" ]]; then
 if [[ "${option}" == "--style" ]]; then
-  if which astyle && [[ "$(astyle --version)" == "Artistic Style Version 3.1" ]]; then
+  if which astyle && [[ "$(astyle --version)" == "Artistic Style Version 2.05.1" ]]; then
    cd tests/scripts
    if ! ./runtest code-style; then code=1; fi
    cd -
  else
-    echo "Warning: astyle not found or version is not 3.1"
+    echo "Warning: astyle not found or version is not 2.05.1"
  fi
 fi

@@ -765,7 +765,6 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/linalg \
                         @MFEM_SOURCE_DIR@/mesh \
                         @MFEM_SOURCE_DIR@/fem \
-                         @MFEM_SOURCE_DIR@/fem/fe \
                         @MFEM_SOURCE_DIR@/examples \
                         @MFEM_SOURCE_DIR@/examples/caliper \
                         @MFEM_SOURCE_DIR@/examples/amgx \
@@ -781,15 +780,13 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/miniapps/gslib \
                         @MFEM_SOURCE_DIR@/miniapps/meshing \
                         @MFEM_SOURCE_DIR@/miniapps/mtop \
-                         @MFEM_SOURCE_DIR@/miniapps/autodiff \
                         @MFEM_SOURCE_DIR@/miniapps/navier \
                         @MFEM_SOURCE_DIR@/miniapps/nurbs \
                         @MFEM_SOURCE_DIR@/miniapps/performance \
                         @MFEM_SOURCE_DIR@/miniapps/shifted \
                         @MFEM_SOURCE_DIR@/miniapps/solvers \
                         @MFEM_SOURCE_DIR@/miniapps/tools \
-                         @MFEM_SOURCE_DIR@/miniapps/toys \
-                         @MFEM_SOURCE_DIR@/miniapps/parelag
+                         @MFEM_SOURCE_DIR@/miniapps/toys

 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
@@ -194,8 +194,6 @@ namespace mfem {
 * - <a class="el" href="parheat_8cpp_source.html">Optimization gradients</a>: Gradients of PDE-constrained function
 * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
 * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
- * - <a class="el" href="par__example_8cpp_source.html">Parallel pLaplacian example using AD</a>: Parallel pLaplacian example
- * - <a class="el" href="seq__example_8cpp_source.html">Serial pLaplacian example using AD</a>: Serial pLaplacian example
 *
 * See also the <a class="el" href="https://mfem.org/examples/">examples documentation</a> online.
 */
@@ -37,7 +37,6 @@ list(APPEND ALL_EXE_SRCS
  ex27.cpp
  ex28.cpp
  ex29.cpp
-  ex30.cpp
  )

 if (MFEM_USE_MPI)
@@ -71,7 +70,6 @@ if (MFEM_USE_MPI)
    ex27p.cpp
    ex28p.cpp
    ex29p.cpp
-    ex30p.cpp
    )
 endif()

@@ -82,80 +80,78 @@ include_directories(BEFORE ${PROJECT_BINARY_DIR})
 add_mfem_examples(ALL_EXE_SRCS)

 # Add a test for each example
-if (MFEM_ENABLE_TESTING)
-  foreach(SRC_FILE ${ALL_EXE_SRCS})
-    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+foreach(SRC_FILE ${ALL_EXE_SRCS})
+  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})

-    set(THIS_TEST_OPTIONS "-no-vis")
-    if (${TEST_NAME} MATCHES "ex0p?")
-      set(THIS_TEST_OPTIONS)
-    endif()
-    if (${TEST_NAME} MATCHES "ex10p*")
-      list(APPEND THIS_TEST_OPTIONS "-tf" "5")
-    elseif(${TEST_NAME} MATCHES "ex15p*")
-      list(APPEND THIS_TEST_OPTIONS "-e" "1")
-    elseif(${TEST_NAME} MATCHES "ex27p*")
-      list(APPEND THIS_TEST_OPTIONS "-dg")
-    endif()
+  set(THIS_TEST_OPTIONS "-no-vis")
+  if (${TEST_NAME} MATCHES "ex0p?")
+    set(THIS_TEST_OPTIONS)
+  endif()
+  if (${TEST_NAME} MATCHES "ex10p*")
+    list(APPEND THIS_TEST_OPTIONS "-tf" "5")
+  elseif(${TEST_NAME} MATCHES "ex15p*")
+    list(APPEND THIS_TEST_OPTIONS "-e" "1")
+  elseif(${TEST_NAME} MATCHES "ex27p*")
+    list(APPEND THIS_TEST_OPTIONS "-dg")
+  endif()

+  if (NOT (${TEST_NAME} MATCHES ".*p$"))
+    add_test(NAME ${TEST_NAME}_ser
+      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+  else()
+    add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+      ${MPIEXEC_POSTFLAGS})
+  endif()
+endforeach()
+
+# Add CUDA/HIP tests.
+set(DEVICE_EXAMPLES
+    # serial examples with device support:
+    ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
+    # parallel examples with device support:
+    ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p ex24p ex25p ex26p)
+set(MFEM_TEST_DEVICE)
+if (MFEM_USE_CUDA)
+  set(MFEM_TEST_DEVICE "cuda")
+elseif (MFEM_USE_HIP)
+  set(MFEM_TEST_DEVICE "hip")
+endif()
+if (MFEM_TEST_DEVICE)
+  foreach(TEST_NAME ${DEVICE_EXAMPLES})
+    set(THIS_TEST_OPTIONS "-no-vis" "-d" "${MFEM_TEST_DEVICE}")
    if (NOT (${TEST_NAME} MATCHES ".*p$"))
-      add_test(NAME ${TEST_NAME}_ser
+      add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_ser
        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-    else()
-      add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
+    elseif (MFEM_USE_MPI)
+      add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_np=${MFEM_MPI_NP}
        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
        ${MPIEXEC_PREFLAGS}
        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
        ${MPIEXEC_POSTFLAGS})
    endif()
  endforeach()
+endif()

-  # Add CUDA/HIP tests.
-  set(DEVICE_EXAMPLES
-      # serial examples with device support:
-      ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
-      # parallel examples with device support:
-      ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p ex24p ex25p ex26p)
-  set(MFEM_TEST_DEVICE)
-  if (MFEM_USE_CUDA)
-    set(MFEM_TEST_DEVICE "cuda")
-  elseif (MFEM_USE_HIP)
-    set(MFEM_TEST_DEVICE "hip")
-  endif()
-  if (MFEM_TEST_DEVICE)
-    foreach(TEST_NAME ${DEVICE_EXAMPLES})
-      set(THIS_TEST_OPTIONS "-no-vis" "-d" "${MFEM_TEST_DEVICE}")
-      if (NOT (${TEST_NAME} MATCHES ".*p$"))
-        add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_ser
-          COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-      elseif (MFEM_USE_MPI)
-        add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_np=${MFEM_MPI_NP}
-          COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-          ${MPIEXEC_PREFLAGS}
-          $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-          ${MPIEXEC_POSTFLAGS})
-      endif()
-    endforeach()
-  endif()
+# If STRUMPACK is enabled, add a test run that uses it.
+if (MFEM_USE_STRUMPACK)
+  add_test(NAME ex11p_strumpack_np=${MFEM_MPI_NP}
+    COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+    ${MPIEXEC_PREFLAGS}
+    $<TARGET_FILE:ex11p> "-no-vis" "--strumpack"
+    ${MPIEXEC_POSTFLAGS})
+endif()

-  # If STRUMPACK is enabled, add a test run that uses it.
-  if (MFEM_USE_STRUMPACK)
-    add_test(NAME ex11p_strumpack_np=${MFEM_MPI_NP}
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:ex11p> "-no-vis" "--strumpack"
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-
-  # If SuperLU_DIST is enabled, add a test run that uses it.
-  if (MFEM_USE_SUPERLU)
-    add_test(NAME ex11p_superlu_np=${MFEM_MPI_NP}
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:ex11p> "-no-vis" "--superlu"
-      ${MPIEXEC_POSTFLAGS})
-  endif()
+# If SuperLU_DIST is enabled, add a test run that uses it.
+if (MFEM_USE_SUPERLU)
+  add_test(NAME ex11p_superlu_np=${MFEM_MPI_NP}
+    COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+    ${MPIEXEC_PREFLAGS}
+    $<TARGET_FILE:ex11p> "-no-vis" "--superlu"
+    ${MPIEXEC_POSTFLAGS})
 endif()

 # Include the examples/amgx directory if AmgX is enabled
@@ -50,32 +50,30 @@ add_mfem_examples(AMGX_EXAMPLES_SRCS ${PFX} copy_amgx_json_files test_amgx)
 # which builds the examples and runs:
 #   ctest -R amgx

-if (MFEM_ENABLE_TESTING)
-  # Command line options for the tests.
-  # Example 1/1p:
-  set(EX1_TEST_OPTS)
-  set(EX1P_TEST_OPTS)
+# Command line options for the tests.
+# Example 1/1p:
+set(EX1_TEST_OPTS)
+set(EX1P_TEST_OPTS)

-  # Add the tests: one test per source file.
-  foreach(SRC_FILE ${AMGX_EXAMPLES_SRCS})
-    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-    string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-    set(TEST_NAME ${PFX}${TEST_NAME})
+# Add the tests: one test per source file.
+foreach(SRC_FILE ${AMGX_EXAMPLES_SRCS})
+  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+  set(TEST_NAME ${PFX}${TEST_NAME})

-    set(THIS_TEST_OPTIONS "-no-vis")
-    list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-    # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+  set(THIS_TEST_OPTIONS "-no-vis")
+  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")

-    if (NOT (${TEST_NAME} MATCHES ".*p$"))
-      add_test(NAME ${TEST_NAME}_ser
-        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-    else()
-      add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
-        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-        ${MPIEXEC_PREFLAGS}
-        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-        ${MPIEXEC_POSTFLAGS})
-    endif()
-  endforeach()
-endif()
+  if (NOT (${TEST_NAME} MATCHES ".*p$"))
+    add_test(NAME ${TEST_NAME}_ser
+      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+  else()
+    add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+      ${MPIEXEC_POSTFLAGS})
+  endif()
+endforeach()
@@ -30,22 +30,22 @@ set(PREFIX caliper_)
 add_mfem_examples(CALIPER_EXE_SRCS ${PREFIX})

 # Add a test for each example
-if (MFEM_ENABLE_TESTING)
-  foreach(SRC_FILE ${CALIPER_EXE_SRCS})
-    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+foreach(SRC_FILE ${CALIPER_EXE_SRCS})
+  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+
+  set(THIS_TEST_OPTIONS "-no-vis")
+
+  if (NOT (${TEST_NAME} MATCHES ".*p$"))
+    add_test(NAME ${TEST_NAME}_ser
+      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+  else()
+    add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+      ${MPIEXEC_POSTFLAGS})
+  endif()
+endforeach()

-    set(THIS_TEST_OPTIONS "-no-vis")

-    if (NOT (${TEST_NAME} MATCHES ".*p$"))
-      add_test(NAME ${TEST_NAME}_ser
-        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-    else()
-      add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
-        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-        ${MPIEXEC_PREFLAGS}
-        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-        ${MPIEXEC_POSTFLAGS})
-    endif()
-  endforeach()
-endif()
@@ -120,8 +120,8 @@ int main(int argc, char *argv[])
   //    largest number that gives a final mesh with no more than 50,000
   //    elements.
   {
-      const int NE = Device::IsEnabled() ? 1e5 : 50000.;
-      const int ref_levels = (int)floor(log(NE/mesh.GetNE())/log(2.)/dim);
+      int ref_levels =
+         (int)floor(log(50000./mesh.GetNE())/log(2.)/dim);
      for (int l = 0; l < ref_levels; l++)
      {
         mesh.UniformRefinement();
@@ -171,7 +171,6 @@ int main(int argc, char *argv[])
   LinearForm b(&fespace);
   ConstantCoefficient one(1.0);
   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   if (Device::IsEnabled()) { b.SetAssemblyLevel(LinearAssemblyLevel::FULL); }
   b.Assemble();

   // 8. Define the solution vector x as a finite element grid function
@@ -185,10 +184,7 @@ int main(int argc, char *argv[])
   //    domain integrator.
   BilinearForm a(&fespace);
   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   // 'Default' E2E kernel
-   //a.AddDomainIntegrator(new DiffusionIntegrator(one));
-   // Can be E2E, L2L, libCeed, etc.
-   a.AddDomainIntegrator(new MassIntegrator(one));
+   a.AddDomainIntegrator(new DiffusionIntegrator(one));

   // 10. Assemble the bilinear form and the corresponding linear system,
   //     applying any necessary transformations such as: eliminating boundary
@@ -201,7 +197,7 @@ int main(int argc, char *argv[])
   Vector B, X;
   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);

-   //cout << "Size of linear system: " << A->Height() << endl;
+   cout << "Size of linear system: " << A->Height() << endl;

   // 11. Solve the linear system A X = B.
   if (!pa)
@@ -227,70 +223,10 @@ int main(int argc, char *argv[])
            ceed::AlgebraicSolver M(a, ess_tdof_list);
            PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
         }
-         /*else
+         else
         {
            OperatorJacobiSmoother M(a, ess_tdof_list);
            PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
-         }*/
-         else
-         {
-            const int myid = 0;
-            const int max_it = 50;
-            const int print_lvl = -1;
-            const double rtol = 1e-12;
-
-            CGSolver cg;
-            cg.SetRelTol(rtol);
-            cg.SetOperator(*A);
-
-            // Warm-up CG solve (in case of JIT to avoid timing it)
-            {
-               Vector Y(X);
-               cg.SetMaxIter(2);
-               cg.SetPrintLevel(-1);
-               cg.Mult(B, Y);
-               MFEM_DEVICE_SYNC;
-            }
-
-            // benchmark this problem
-            {
-               tic_toc.Clear();
-               cg.SetMaxIter(max_it);
-               cg.SetPrintLevel(print_lvl);
-               {
-                  tic_toc.Start();
-                  cg.Mult(B, X);
-                  MFEM_DEVICE_SYNC;
-                  tic_toc.Stop();
-               }
-            }
-            // Final norm check
-            const double final_norm = cg.GetFinalNorm();
-            MFEM_VERIFY(final_norm < sqrt(rtol),
-                        "FinalNorm (" << final_norm << ") Error!");
-            // Number of iteration check
-            const int num_iter = cg.GetNumIterations();
-            MFEM_VERIFY(num_iter <= max_it,
-                        "NumIterations (" << num_iter << ") Error!");
-
-            const double rt = tic_toc.RealTime();
-            const double rt_min = rt, rt_max = rt;
-            const int dofs = fespace.GetVSize();
-            const int cg_iter = cg.GetNumIterations();
-            const double mdofs_max = ((1e-6 * dofs) * cg_iter) / rt_max;
-            const double mdofs_min = ((1e-6 * dofs) * cg_iter) / rt_min;
-            if (myid == 0)
-            {
-               mfem::out << "Total CG time:    " << rt_max << " (" << rt_min << ") sec."
-                         << std::endl;
-               mfem::out << "Time per CG step: "
-                         << rt_max / cg_iter << " ("
-                         << rt_min / cg_iter << ") sec." << std::endl;
-               mfem::out << "\033[32m";
-               mfem::out << "\"DOFs/sec\" in CG: " << mdofs_max << " ("
-                         << mdofs_min << ") million.";
-               mfem::out << "\033[m" << std::endl;
-            }
         }
      }
      else
@@ -149,7 +149,7 @@ void InitialDeformation(const Vector &x, Vector &y);

 void InitialVelocity(const Vector &x, Vector &v);

-void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
+void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
               GridFunction *field, const char *field_name = NULL,
               bool init_vis = false);

@@ -376,10 +376,10 @@ int main(int argc, char *argv[])
 }


-void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
+void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
               GridFunction *field, const char *field_name, bool init_vis)
 {
-   if (!os)
+   if (!out)
   {
      return;
   }
@@ -389,25 +389,24 @@ void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,

   mesh->SwapNodes(nodes, owns_nodes);

-   os << "solution\n" << *mesh << *field;
+   out << "solution\n" << *mesh << *field;

   mesh->SwapNodes(nodes, owns_nodes);

   if (init_vis)
   {
-      os << "window_size 800 800\n";
-      os << "window_title '" << field_name << "'\n";
+      out << "window_size 800 800\n";
+      out << "window_title '" << field_name << "'\n";
      if (mesh->SpaceDimension() == 2)
      {
-         os << "view 0 0\n"; // view from top
-         os << "keys jl\n";  // turn off perspective and light
+         out << "view 0 0\n"; // view from top
+         out << "keys jl\n";  // turn off perspective and light
      }
-      os << "keys cm\n";         // show colorbar and mesh
-      // update value-range; keep mesh-extents fixed
-      os << "autoscale value\n";
-      os << "pause\n";
+      out << "keys cm\n";         // show colorbar and mesh
+      out << "autoscale value\n"; // update value-range; keep mesh-extents fixed
+      out << "pause\n";
   }
-   os << flush;
+   out << flush;
 }


@@ -154,8 +154,7 @@ void InitialDeformation(const Vector &x, Vector &y);

 void InitialVelocity(const Vector &x, Vector &v);

-void visualize(ostream &os, ParMesh *mesh,
-               ParGridFunction *deformed_nodes,
+void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
               ParGridFunction *field, const char *field_name = NULL,
               bool init_vis = false);

@@ -439,11 +438,10 @@ int main(int argc, char *argv[])
   return 0;
 }

-void visualize(ostream &os, ParMesh *mesh,
-               ParGridFunction *deformed_nodes,
+void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
               ParGridFunction *field, const char *field_name, bool init_vis)
 {
-   if (!os)
+   if (!out)
   {
      return;
   }
@@ -453,27 +451,25 @@ void visualize(ostream &os, ParMesh *mesh,

   mesh->SwapNodes(nodes, owns_nodes);

-   os << "parallel " << mesh->GetNRanks()
-      << " " << mesh->GetMyRank() << "\n";
-   os << "solution\n" << *mesh << *field;
+   out << "parallel " << mesh->GetNRanks() << " " << mesh->GetMyRank() << "\n";
+   out << "solution\n" << *mesh << *field;

   mesh->SwapNodes(nodes, owns_nodes);

   if (init_vis)
   {
-      os << "window_size 800 800\n";
-      os << "window_title '" << field_name << "'\n";
+      out << "window_size 800 800\n";
+      out << "window_title '" << field_name << "'\n";
      if (mesh->SpaceDimension() == 2)
      {
-         os << "view 0 0\n"; // view from top
-         os << "keys jl\n";  // turn off perspective and light
+         out << "view 0 0\n"; // view from top
+         out << "keys jl\n";  // turn off perspective and light
      }
-      os << "keys cm\n";         // show colorbar and mesh
-      // update value-range; keep mesh-extents fixed
-      os << "autoscale value\n";
-      os << "pause\n";
+      out << "keys cm\n";         // show colorbar and mesh
+      out << "autoscale value\n"; // update value-range; keep mesh-extents fixed
+      out << "pause\n";
   }
-   os << flush;
+   out << flush;
 }


@@ -135,8 +135,8 @@ int main(int argc, char *argv[])
   a->AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
   if (eta > 0)
   {
-      a->AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
-      a->AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
+      a->AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
+      a->AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
   }
   a->Assemble();
   a->Finalize();
@@ -199,8 +199,8 @@ int main(int argc, char *argv[])
   a->AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
   if (eta > 0)
   {
-      a->AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
-      a->AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
+      a->AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
+      a->AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
   }
   a->Assemble();
   a->Finalize();
@@ -221,7 +221,7 @@ int main(int argc, char *argv[])
   {
      HyprePCG pcg(*A);
      pcg.SetTol(1e-12);
-      pcg.SetMaxIter(500);
+      pcg.SetMaxIter(200);
      pcg.SetPrintLevel(2);
      pcg.SetPreconditioner(*amg);
      pcg.Mult(*B, *X);
@@ -232,7 +232,7 @@ int main(int argc, char *argv[])
      GMRESSolver gmres(MPI_COMM_WORLD);
      gmres.SetAbsTol(0.0);
      gmres.SetRelTol(1e-12);
-      gmres.SetMaxIter(500);
+      gmres.SetMaxIter(200);
      gmres.SetKDim(10);
      gmres.SetPrintLevel(1);
      gmres.SetOperator(*A);
@@ -32,7 +32,7 @@ private:
   mutable DenseTensor flux;
   mutable Vector z;

-   void GetFlux(const DenseMatrix &state_, DenseTensor &flux_) const;
+   void GetFlux(const DenseMatrix &state, DenseTensor &flux) const;

 public:
   FE_Evolution(FiniteElementSpace &vfes_,
@@ -256,26 +256,26 @@ inline double ComputeMaxCharSpeed(const Vector &state, const int dim)
 }

 // Compute the flux at solution nodes.
-void FE_Evolution::GetFlux(const DenseMatrix &x_, DenseTensor &flux_) const
+void FE_Evolution::GetFlux(const DenseMatrix &x, DenseTensor &flux) const
 {
-   const int flux_dof = flux_.SizeI();
-   const int flux_dim = flux_.SizeJ();
+   const int dof = flux.SizeI();
+   const int dim = flux.SizeJ();

-   for (int i = 0; i < flux_dof; i++)
+   for (int i = 0; i < dof; i++)
   {
-      for (int k = 0; k < num_equation; k++) { state(k) = x_(i, k); }
-      ComputeFlux(state, flux_dim, f);
+      for (int k = 0; k < num_equation; k++) { state(k) = x(i, k); }
+      ComputeFlux(state, dim, f);

-      for (int d = 0; d < flux_dim; d++)
+      for (int d = 0; d < dim; d++)
      {
         for (int k = 0; k < num_equation; k++)
         {
-            flux_(i, d, k) = f(k, d);
+            flux(i, d, k) = f(k, d);
         }
      }

      // Update max char speed
-      const double mcs = ComputeMaxCharSpeed(state, flux_dim);
+      const double mcs = ComputeMaxCharSpeed(state, dim);
      if (mcs > max_char_speed) { max_char_speed = mcs; }
   }
 }
@@ -171,7 +171,7 @@ public:
 };

 // Visualization driver
-void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
+void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
               GridFunction *field, const char *field_name = NULL,
               bool init_vis = false);

@@ -542,10 +542,10 @@ RubberOperator::~RubberOperator()


 // Inline visualization
-void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
+void visualize(ostream &out, Mesh *mesh, GridFunction *deformed_nodes,
               GridFunction *field, const char *field_name, bool init_vis)
 {
-   if (!os)
+   if (!out)
   {
      return;
   }
@@ -555,25 +555,23 @@ void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,

   mesh->SwapNodes(nodes, owns_nodes);

-   os << "solution\n" << *mesh << *field;
+   out << "solution\n" << *mesh << *field;

   mesh->SwapNodes(nodes, owns_nodes);

   if (init_vis)
   {
-      os << "window_size 800 800\n";
-      os << "window_title '" << field_name << "'\n";
+      out << "window_size 800 800\n";
+      out << "window_title '" << field_name << "'\n";
      if (mesh->SpaceDimension() == 2)
      {
-         os << "view 0 0\n"; // view from top
-         // turn off perspective and light, +anti-aliasing
-         os << "keys jlA\n";
+         out << "view 0 0\n"; // view from top
+         out << "keys jlA\n"; // turn off perspective and light, +anti-aliasing
      }
-      os << "keys cmA\n"; // show colorbar and mesh, +anti-aliasing
-      // update value-range; keep mesh-extents fixed
-      os << "autoscale value\n";
+      out << "keys cmA\n";        // show colorbar and mesh, +anti-aliasing
+      out << "autoscale value\n"; // update value-range; keep mesh-extents fixed
   }
-   os << flush;
+   out << flush;
 }

 void ReferenceConfiguration(const Vector &x, Vector &y)
@@ -185,8 +185,7 @@ public:
 };

 // Visualization driver
-void visualize(ostream &os, ParMesh *mesh,
-               ParGridFunction *deformed_nodes,
+void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
               ParGridFunction *field, const char *field_name = NULL,
               bool init_vis = false);

@@ -618,11 +617,10 @@ RubberOperator::~RubberOperator()


 // Inline visualization
-void visualize(ostream &os, ParMesh *mesh,
-               ParGridFunction *deformed_nodes,
+void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes,
               ParGridFunction *field, const char *field_name, bool init_vis)
 {
-   if (!os)
+   if (!out)
   {
      return;
   }
@@ -632,27 +630,24 @@ void visualize(ostream &os, ParMesh *mesh,

   mesh->SwapNodes(nodes, owns_nodes);

-   os << "parallel " << mesh->GetNRanks() << " " << mesh->GetMyRank() <<
-      "\n";
-   os << "solution\n" << *mesh << *field;
+   out << "parallel " << mesh->GetNRanks() << " " << mesh->GetMyRank() << "\n";
+   out << "solution\n" << *mesh << *field;

   mesh->SwapNodes(nodes, owns_nodes);

   if (init_vis)
   {
-      os << "window_size 800 800\n";
-      os << "window_title '" << field_name << "'\n";
+      out << "window_size 800 800\n";
+      out << "window_title '" << field_name << "'\n";
      if (mesh->SpaceDimension() == 2)
      {
-         os << "view 0 0\n"; // view from top
-         // turn off perspective and light, +anti-aliasing
-         os << "keys jlA\n";
+         out << "view 0 0\n"; // view from top
+         out << "keys jlA\n"; // turn off perspective and light, +anti-aliasing
      }
-      os << "keys cmA\n"; // show colorbar and mesh, +anti-aliasing
-      // update value-range; keep mesh-extents fixed
-      os << "autoscale value\n";
+      out << "keys cmA\n";        // show colorbar and mesh, +anti-aliasing
+      out << "autoscale value\n"; // update value-range; keep mesh-extents fixed
   }
-   os << flush;
+   out << flush;
 }

 void ReferenceConfiguration(const Vector &x, Vector &y)
@@ -128,7 +128,7 @@ int main(int argc, char *argv[])
   //    more than 10,000 elements.
   {
      int ref_levels =
-         (int)floor(log(1000./mesh.GetNE())/log(2.)/dim);
+         (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
      for (int l = 0; l < ref_levels; l++)
      {
         mesh.UniformRefinement();
@@ -197,7 +197,6 @@ int main(int argc, char *argv[])
   ParLinearForm b(&fespace);
   ConstantCoefficient one(1.0);
   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   if (Device::IsEnabled()) { b.SetAssemblyLevel(LinearAssemblyLevel::FULL); }
   b.Assemble();

   // 10. Define the solution vector x as a parallel finite element grid
@@ -211,8 +210,7 @@ int main(int argc, char *argv[])
   //     Diffusion domain integrator.
   ParBilinearForm a(&fespace);
   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   //a.AddDomainIntegrator(new DiffusionIntegrator(one));
-   a.AddDomainIntegrator(new MassIntegrator(one));
+   a.AddDomainIntegrator(new DiffusionIntegrator(one));

   // 12. Assemble the parallel bilinear form and the corresponding linear
   //     system, applying any necessary transformations such as: parallel
@@ -247,67 +245,13 @@ int main(int argc, char *argv[])
   {
      prec = new HypreBoomerAMG;
   }
-
-   const int max_it = 50;
-   const int print_lvl = -1;
-   const double rtol = 1e-12;
-
   CGSolver cg(MPI_COMM_WORLD);
-   cg.SetRelTol(rtol);
-   //cg.SetMaxIter(max_it);
-   //cg.SetPrintLevel(print_lvl);
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(1);
   if (prec) { cg.SetPreconditioner(*prec); }
   cg.SetOperator(*A);
-
-   // Warm-up CG solve (in case of JIT to avoid timing it)
-   {
-      Vector Y(X);
-      cg.SetMaxIter(2);
-      cg.SetPrintLevel(-1);
-      cg.Mult(B, Y);
-      MFEM_DEVICE_SYNC;
-   }
-
-   // benchmark this problem
-   {
-      tic_toc.Clear();
-      cg.SetMaxIter(max_it);
-      cg.SetPrintLevel(print_lvl);
-      {
-         tic_toc.Start();
-         cg.Mult(B, X);
-         MFEM_DEVICE_SYNC;
-         tic_toc.Stop();
-      }
-   }
-   // Final norm check
-   const double final_norm = cg.GetFinalNorm();
-   MFEM_VERIFY(final_norm < sqrt(rtol),
-               "FinalNorm (" << final_norm << ") Error!");
-   // Number of iteration check
-   const int num_iter = cg.GetNumIterations();
-   MFEM_VERIFY(num_iter <= max_it,
-               "NumIterations (" << num_iter << ") Error!");
-
-   const double rt = tic_toc.RealTime();
-   const double rt_min = rt, rt_max = rt;
-   HYPRE_BigInt dofs = fespace.GlobalTrueVSize();
-   const int cg_iter = cg.GetNumIterations();
-   const double mdofs_max = ((1e-6 * dofs) * cg_iter) / rt_max;
-   const double mdofs_min = ((1e-6 * dofs) * cg_iter) / rt_min;
-   if (myid == 0)
-   {
-      mfem::out << "Total CG time:    " << rt_max << " (" << rt_min << ") sec."
-                << std::endl;
-      mfem::out << "Time per CG step: "
-                << rt_max / cg_iter << " ("
-                << rt_min / cg_iter << ") sec." << std::endl;
-      mfem::out << "\033[32m";
-      mfem::out << "\"DOFs/sec\" in CG: " << mdofs_max << " ("
-                << mdofs_min << ") million.";
-      mfem::out << "\033[m" << std::endl;
-   }
-   //cg.Mult(B, X);
+   cg.Mult(B, X);
   delete prec;

   // 14. Recover the parallel grid function corresponding to X. This is the
@@ -75,7 +75,7 @@ Mesh * GenerateSerialMesh(int ref);
 // alpha*n.Grad(sol) + beta*sol - gamma over the same boundary.
 double IntegrateBC(const GridFunction &sol, const Array<int> &bdr_marker,
                   double alpha, double beta, double gamma,
-                   double &error);
+                   double &err);

 int main(int argc, char *argv[])
 {
@@ -295,33 +295,43 @@ int main(int argc, char *argv[])
   //     element solution.
   a.RecoverFEMSolution(X, b, u);

-   // 13. Compute the various boundary integrals.
+   // 13. Build a mass matrix to help solve for n.Grad(u) where 'n' is a surface
+   //     normal.
+   BilinearForm m(&fespace);
+   m.AddDomainIntegrator(new MassIntegrator);
+   m.Assemble();
+
+   ess_tdof_list.SetSize(0);
+   OperatorPtr M;
+   m.FormSystemMatrix(ess_tdof_list, M);
+
+   // 14. Compute the various boundary integrals.
   mfem::out << endl
             << "Verifying boundary conditions" << endl
             << "=============================" << endl;
   {
      // Integrate the solution on the Dirichlet boundary and compare to the
      // expected value.
-      double error, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, error);
+      double err, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, err);

      bool hom_dbc = (dbc_val == 0.0);
-      error /=  hom_dbc ? 1.0 : fabs(dbc_val);
+      err /=  hom_dbc ? 1.0 : fabs(dbc_val);
      mfem::out << "Average of solution on Gamma_dbc:\t"
                << avg << ", \t"
                << (hom_dbc ? "absolute" : "relative")
-                << " error " << error << endl;
+                << " error " << err << endl;
   }
   {
      // Integrate n.Grad(u) on the inhomogeneous Neumann boundary and compare
      // to the expected value.
-      double error, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, error);
+      double err, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, err);

      bool hom_nbc = (nbc_val == 0.0);
-      error /=  hom_nbc ? 1.0 : fabs(nbc_val);
+      err /=  hom_nbc ? 1.0 : fabs(nbc_val);
      mfem::out << "Average of n.Grad(u) on Gamma_nbc:\t"
                << avg << ", \t"
                << (hom_nbc ? "absolute" : "relative")
-                << " error " << error << endl;
+                << " error " << err << endl;
   }
   {
      // Integrate n.Grad(u) on the homogeneous Neumann boundary and compare to
@@ -330,29 +340,28 @@ int main(int argc, char *argv[])
      nbc0_bdr = 0;
      nbc0_bdr[3] = 1;

-      double error, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, error);
+      double err, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, err);

      bool hom_nbc = true;
      mfem::out << "Average of n.Grad(u) on Gamma_nbc0:\t"
                << avg << ", \t"
                << (hom_nbc ? "absolute" : "relative")
-                << " error " << error << endl;
+                << " error " << err << endl;
   }
   {
      // Integrate n.Grad(u) + a * u on the Robin boundary and compare to the
      // expected value.
-      double error;
-      double avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val, error);
+      double err, avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val, err);

      bool hom_rbc = (rbc_b_val == 0.0);
-      error /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
+      err /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
      mfem::out << "Average of n.Grad(u)+a*u on Gamma_rbc:\t"
                << avg << ", \t"
                << (hom_rbc ? "absolute" : "relative")
-                << " error " << error << endl;
+                << " error " << err << endl;
   }

-   // 14. Save the refined mesh and the solution. This output can be viewed
+   // 15. Save the refined mesh and the solution. This output can be viewed
   //     later using GLVis: "glvis -m refined.mesh -g sol.gf".
   {
      ofstream mesh_ofs("refined.mesh");
@@ -363,7 +372,7 @@ int main(int argc, char *argv[])
      u.Save(sol_ofs);
   }

-   // 15. Send the solution by socket to a GLVis server.
+   // 16. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      string title_str = h1 ? "H1" : "DG";
@@ -376,7 +385,7 @@ int main(int argc, char *argv[])
               << " keys 'mmc'" << flush;
   }

-   // 16. Free the used memory.
+   // 17. Free the used memory.
   delete fec;
   delete mesh;

@@ -638,11 +647,11 @@ Mesh * GenerateSerialMesh(int ref)

 double IntegrateBC(const GridFunction &x, const Array<int> &bdr,
                   double alpha, double beta, double gamma,
-                   double &error)
+                   double &err)
 {
   double nrm = 0.0;
   double avg = 0.0;
-   error = 0.0;
+   err = 0.0;

   const bool a_is_zero = alpha == 0.0;
   const bool b_is_zero = beta == 0.0;
@@ -706,20 +715,20 @@ double IntegrateBC(const GridFunction &x, const Array<int> &bdr,

         // Integrate |alpha * n.Grad(x) + beta * x - gamma|^2
         val -= gamma;
-         error += (val*val) * ip.weight * face_weight;
+         err += (val*val) * ip.weight * face_weight;
      }
   }

   // Normalize by the length of the boundary
   if (std::abs(nrm) > 0.0)
   {
-      error /= nrm;
+      err /= nrm;
      avg /= nrm;
   }

   // Compute l2 norm of the error in the boundary condition (negative
-   // quadrature weights may produce negative 'error')
-   error = (error >= 0.0) ? sqrt(error) : -sqrt(-error);
+   // quadrature weights may produce negative 'err')
+   err = (err >= 0.0) ? sqrt(err) : -sqrt(-err);

   // Return the average value of alpha * n.Grad(x) + beta * x
   return avg;
@@ -75,7 +75,7 @@ Mesh * GenerateSerialMesh(int ref);
 // alpha*n.Grad(sol) + beta*sol - gamma over the same boundary.
 double IntegrateBC(const ParGridFunction &sol, const Array<int> &bdr_marker,
                   double alpha, double beta, double gamma,
-                   double &error);
+                   double &err);

 int main(int argc, char *argv[])
 {
@@ -314,33 +314,43 @@ int main(int argc, char *argv[])
   //     local finite element solution on each processor.
   a.RecoverFEMSolution(X, b, u);

-   // 14. Compute the various boundary integrals.
+   // 14. Build a mass matrix to help solve for n.Grad(u) where 'n' is a surface
+   //     normal.
+   ParBilinearForm m(&fespace);
+   m.AddDomainIntegrator(new MassIntegrator);
+   m.Assemble();
+
+   ess_tdof_list.SetSize(0);
+   OperatorPtr M;
+   m.FormSystemMatrix(ess_tdof_list, M);
+
+   // 15. Compute the various boundary integrals.
   mfem::out << endl
             << "Verifying boundary conditions" << endl
             << "=============================" << endl;
   {
      // Integrate the solution on the Dirichlet boundary and compare to the
      // expected value.
-      double error, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, error);
+      double err, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, err);

      bool hom_dbc = (dbc_val == 0.0);
-      error /=  hom_dbc ? 1.0 : fabs(dbc_val);
+      err /=  hom_dbc ? 1.0 : fabs(dbc_val);
      mfem::out << "Average of solution on Gamma_dbc:\t"
                << avg << ", \t"
                << (hom_dbc ? "absolute" : "relative")
-                << " error " << error << endl;
+                << " error " << err << endl;
   }
   {
      // Integrate n.Grad(u) on the inhomogeneous Neumann boundary and compare
      // to the expected value.
-      double error, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, error);
+      double err, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, err);

      bool hom_nbc = (nbc_val == 0.0);
-      error /=  hom_nbc ? 1.0 : fabs(nbc_val);
+      err /=  hom_nbc ? 1.0 : fabs(nbc_val);
      mfem::out << "Average of n.Grad(u) on Gamma_nbc:\t"
                << avg << ", \t"
                << (hom_nbc ? "absolute" : "relative")
-                << " error " << error << endl;
+                << " error " << err << endl;
   }
   {
      // Integrate n.Grad(u) on the homogeneous Neumann boundary and compare to
@@ -349,29 +359,28 @@ int main(int argc, char *argv[])
      nbc0_bdr = 0;
      nbc0_bdr[3] = 1;

-      double error, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, error);
+      double err, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, err);

      bool hom_nbc = true;
      mfem::out << "Average of n.Grad(u) on Gamma_nbc0:\t"
                << avg << ", \t"
                << (hom_nbc ? "absolute" : "relative")
-                << " error " << error << endl;
+                << " error " << err << endl;
   }
   {
      // Integrate n.Grad(u) + a * u on the Robin boundary and compare to the
      // expected value.
-      double error, avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val,
-                                      error);
+      double err, avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val, err);

      bool hom_rbc = (rbc_b_val == 0.0);
-      error /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
+      err /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
      mfem::out << "Average of n.Grad(u)+a*u on Gamma_rbc:\t"
                << avg << ", \t"
                << (hom_rbc ? "absolute" : "relative")
-                << " error " << error << endl;
+                << " error " << err << endl;
   }

-   // 15. Save the refined mesh and the solution in parallel. This output can be
+   // 16. Save the refined mesh and the solution in parallel. This output can be
   //     viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
   {
      ostringstream mesh_name, sol_name;
@@ -387,7 +396,7 @@ int main(int argc, char *argv[])
      u.Save(sol_ofs);
   }

-   // 16. Send the solution by socket to a GLVis server.
+   // 17. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      string title_str = h1 ? "H1" : "DG";
@@ -402,7 +411,7 @@ int main(int argc, char *argv[])
               << " keys 'mmc'" << flush;
   }

-   // 17. Free the used memory.
+   // 18. Free the used memory.
   delete fec;

   return 0;
@@ -668,11 +677,11 @@ double IntegrateBC(const ParGridFunction &x, const Array<int> &bdr,
   double loc_vals[3];
   double &nrm = loc_vals[0];
   double &avg = loc_vals[1];
-   double &error = loc_vals[2];
+   double &err = loc_vals[2];

   nrm = 0.0;
   avg = 0.0;
-   error = 0.0;
+   err = 0.0;

   const bool a_is_zero = alpha == 0.0;
   const bool b_is_zero = beta == 0.0;
@@ -736,7 +745,7 @@ double IntegrateBC(const ParGridFunction &x, const Array<int> &bdr,

         // Integrate |alpha * n.Grad(x) + beta * x - gamma|^2
         val -= gamma;
-         error += (val*val) * ip.weight * face_weight;
+         err += (val*val) * ip.weight * face_weight;
      }
   }

@@ -755,7 +764,7 @@ double IntegrateBC(const ParGridFunction &x, const Array<int> &bdr,
   }

   // Compute l2 norm of the error in the boundary condition (negative
-   // quadrature weights may produce negative 'error')
+   // quadrature weights may produce negative 'err')
   glb_err = (glb_err >= 0.0) ? sqrt(glb_err) : -sqrt(-glb_err);

   // Return the average value of alpha * n.Grad(x) + beta * x
@@ -167,9 +167,9 @@ int main(int argc, char *argv[])

   // 13. Compute error in the solution and its flux
   FunctionCoefficient uCoef(uExact);
-   double error = x.ComputeL2Error(uCoef);
+   double err = x.ComputeL2Error(uCoef);

-   cout << "|u - u_h|_2 = " << error << endl;
+   cout << "|u - u_h|_2 = " << err << endl;

   FiniteElementSpace flux_fespace(mesh, &fec, 3);
   GridFunction flux(&flux_fespace);
@@ -197,9 +197,9 @@ int main(int argc, char *argv[])

   // 15. Compute error in the solution and its flux
   FunctionCoefficient uCoef(uExact);
-   double error = x.ComputeL2Error(uCoef);
+   double err = x.ComputeL2Error(uCoef);

-   if (myid == 0) { cout << "|u - u_h|_2 = " << error << endl; }
+   if (myid == 0) { cout << "|u - u_h|_2 = " << err << endl; }

   ParFiniteElementSpace flux_fespace(&pmesh, &fec, 3);
   ParGridFunction flux(&flux_fespace);
@@ -1,195 +0,0 @@
-//                            MFEM Example 30
-//
-// Compile with: make ex30
-//
-// Sample runs:  ex30 -m ../data/square-disc.mesh -o 1
-//               ex30 -m ../data/square-disc.mesh -o 2
-//               ex30 -m ../data/square-disc.mesh -o 2 -me 1e3
-//               ex30 -m ../data/square-disc-nurbs.mesh -o 2
-//               ex30 -m ../data/star.mesh -o 2 -eo 4
-//               ex30 -m ../data/fichera.mesh -o 2 -me 1e4
-//               ex30 -m ../data/disc-nurbs.mesh -o 2
-//               ex30 -m ../data/ball-nurbs.mesh -o 2 -eo 3 -e 1e-2 -me 1e4
-//               ex30 -m ../data/star-surf.mesh -o 2
-//               ex30 -m ../data/square-disc-surf.mesh -o 2
-//               ex30 -m ../data/amr-quad.mesh -l 2
-//
-// Description:  This is an example of adaptive mesh refinement preprocessing
-//               which lowers the data oscillation [1] to a user-defined
-//               relative threshold. There is no PDE being solved.
-//
-//               MFEM's capability to work with both conforming and
-//               nonconforming meshes is demonstrated in example 6. In some
-//               problems, the material data or loading data is not sufficiently
-//               resolved on the initial mesh. This missing fine scale data
-//               reduces the accuracy of the solution as well as the accuracy
-//               of some local error estimators. By preprocessing the mesh
-//               before solving the PDE, many issues can be avoided.
-//
-//               [1] Morin, P., Nochetto, R. H., & Siebert, K. G. (2000).
-//                   Data oscillation and convergence of adaptive FEM. SIAM
-//                   Journal on Numerical Analysis, 38(2), 466-488.
-//
-//               [2] Mitchell, W. F. (2013). A collection of 2D elliptic
-//                   problems for testing adaptive grid refinement algorithms.
-//                   Applied mathematics and computation, 220, 350-364.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Piecewise-affine function which is sometimes mesh-conforming
-double affine_function(const Vector &p)
-{
-   double x = p(0), y = p(1);
-   if (x < 0.0)
-   {
-      return 1.0 + x + y;
-   }
-   else
-   {
-      return 1.0;
-   }
-}
-
-// Piecewise-constant function which is never mesh-conforming
-double jump_function(const Vector &p)
-{
-   if (p.Normlp(2.0) > 0.4 && p.Normlp(2.0) < 0.6) { return 1.0; }
-   return 5.0;
-}
-
-// Singular function derived from the Laplacian of the "steep wavefront"
-// problem in [2].
-double singular_function(const Vector &p)
-{
-   double x = p(0), y = p(1);
-   double alpha = 1000.0;
-   double xc = 0.75, yc = 0.5;
-   double r0 = 0.7;
-   double r = sqrt(pow(x - xc,2.0) + pow(y - yc,2.0));
-   double num = - ( alpha - pow(alpha,3) * (pow(r,2) - pow(r0,2)) );
-   double denom = pow(r * ( pow(alpha,2) * pow(r0,2) + pow(alpha,2) * pow(r,2) \
-                            - 2 * pow(alpha,2) * r0 * r + 1.0 ),2);
-   denom = max(denom,1e-8);
-   return num / denom;
-}
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../data/star.mesh";
-   int order = 1;
-   int nc_limit = 1;
-   int max_elems = 1e5;
-   double double_max_elems = double(max_elems);
-   bool visualization = true;
-   double osc_threshold = 1e-3;
-   int enriched_order = 5;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&nc_limit, "-l", "--nc-limit",
-                  "Maximum level of hanging nodes.");
-   args.AddOption(&double_max_elems, "-me", "--max-elems",
-                  "Stop after reaching this many elements.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&osc_threshold, "-e", "--error",
-                  "relative data oscillation threshold.");
-   args.AddOption(&enriched_order, "-eo", "--enriched_order",
-                  "Enriched quadrature order.");
-
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   max_elems = int(double_max_elems);
-   Mesh mesh(mesh_file, 1, 1);
-
-   // 2. Since a NURBS mesh can currently only be refined uniformly, we need to
-   //    convert it to a piecewise-polynomial curved mesh. First we refine the
-   //    NURBS mesh a bit more and then project the curvature to quadratic Nodes.
-   if (mesh.NURBSext)
-   {
-      for (int i = 0; i < 2; i++)
-      {
-         mesh.UniformRefinement();
-      }
-      mesh.SetCurvature(2);
-   }
-
-   // 3. Define functions and refiner.
-   FunctionCoefficient affine_coeff(affine_function);
-   FunctionCoefficient jump_coeff(jump_function);
-   FunctionCoefficient singular_coeff(singular_function);
-   CoefficientRefiner coeffrefiner(affine_coeff, order);
-
-   // 4. Connect to GLVis.
-   char vishost[] = "localhost";
-   int  visport   = 19916;
-   socketstream sol_sock;
-   if (visualization)
-   {
-      sol_sock.open(vishost, visport);
-   }
-
-   // 5. Define custom integration rule (optional).
-   const IntegrationRule *irs[Geometry::NumGeom];
-   int order_quad = 2*order + enriched_order;
-   for (int i = 0; i < Geometry::NumGeom; ++i)
-   {
-      irs[i] = &(IntRules.Get(i, order_quad));
-   }
-
-   // 6. Apply custom refiner settings.
-   coeffrefiner.SetIntRule(irs);
-   coeffrefiner.SetMaxElements(max_elems);
-   coeffrefiner.SetThreshold(osc_threshold);
-   coeffrefiner.SetNCLimit(nc_limit);
-   coeffrefiner.PrintWarnings();
-
-   // 7. Preprocess mesh to control osc (piecewise-affine function).
-   //    This is mostly just a verification check. The oscillation should
-   //    be zero if the function is mesh-conforming and order > 0.
-   coeffrefiner.PreprocessMesh(mesh);
-
-   mfem::out  << "\n";
-   mfem::out << "Function 0 (affine) \n";
-   mfem::out << "Number of Elements " << mesh.GetNE() << "\n";
-   mfem::out << "Osc error " << coeffrefiner.GetOsc() << "\n";
-
-   // 8. Preprocess mesh to control osc (jump function).
-   coeffrefiner.ResetCoefficient(jump_coeff);
-   coeffrefiner.PreprocessMesh(mesh);
-
-   mfem::out  << "\n";
-   mfem::out << "Function 1 (discontinuous) \n";
-   mfem::out << "Number of Elements " << mesh.GetNE() << "\n";
-   mfem::out << "Osc error " << coeffrefiner.GetOsc() << "\n";
-
-   // 9. Preprocess mesh to control osc (singular function).
-   coeffrefiner.ResetCoefficient(singular_coeff);
-   coeffrefiner.PreprocessMesh(mesh);
-
-   mfem::out  << "\n";
-   mfem::out << "Function 2 (singular) \n";
-   mfem::out << "Number of Elements " << mesh.GetNE() << "\n";
-   mfem::out << "Osc error " << coeffrefiner.GetOsc() << "\n";
-
-   sol_sock.precision(8);
-   sol_sock << "mesh\n" << mesh << flush;
-
-   return 0;
-}
@@ -1,241 +0,0 @@
-//                        MFEM Example 30 - Parallel Version
-//
-// Compile with: make ex30p
-//
-// Sample runs:  mpirun -np 4 ex30p -m ../data/square-disc.mesh -o 1
-//               mpirun -np 4 ex30p -m ../data/square-disc.mesh -o 2
-//               mpirun -np 4 ex30p -m ../data/square-disc.mesh -o 2 -me 1e3
-//               mpirun -np 4 ex30p -m ../data/square-disc-nurbs.mesh -o 2
-//               mpirun -np 4 ex30p -m ../data/star.mesh -o 2 -eo 4
-//               mpirun -np 4 oscp -m ../data/fichera.mesh -o 2 -me 1e4
-//               mpirun -np 4 ex30p -m ../data/disc-nurbs.mesh -o 2
-//               mpirun -np 4 ex30p -m ../data/ball-nurbs.mesh -o 2 -eo 3 -e 1e-2
-//               mpirun -np 4 ex30p -m ../data/star-surf.mesh -o 2
-//               mpirun -np 4 ex30p -m ../data/square-disc-surf.mesh -o 2
-//               mpirun -np 4 ex30p -m ../data/amr-quad.mesh -l 2
-//
-// Description:  This is an example of adaptive mesh refinement preprocessing
-//               which lowers the data oscillation [1] to a user-defined
-//               relative threshold. There is no PDE being solved.
-//
-//               MFEM's capability to work with both conforming and
-//               nonconforming meshes is demonstrated in example 6. In some
-//               problems, the material data or loading data is not sufficiently
-//               resolved on the initial mesh. This missing fine scale data
-//               reduces the accuracy of the solution as well as the accuracy
-//               of some local error estimators. By preprocessing the mesh
-//               before solving the PDE, many issues can be avoided.
-//
-//               [1] Morin, P., Nochetto, R. H., & Siebert, K. G. (2000).
-//                   Data oscillation and convergence of adaptive FEM. SIAM
-//                   Journal on Numerical Analysis, 38(2), 466-488.
-//
-//               [2] Mitchell, W. F. (2013). A collection of 2D elliptic
-//                   problems for testing adaptive grid refinement algorithms.
-//                   Applied mathematics and computation, 220, 350-364.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Piecewise-affine function which is sometimes mesh-conforming
-double affine_function(const Vector &p)
-{
-   double x = p(0), y = p(1);
-   if (x < 0.0)
-   {
-      return 1.0 + x + y;
-   }
-   else
-   {
-      return 1.0;
-   }
-}
-
-// Piecewise-constant function which is never mesh-conforming
-double jump_function(const Vector &p)
-{
-   if (p.Normlp(2.0) > 0.4 && p.Normlp(2.0) < 0.6) { return 1.0; }
-   return 5.0;
-}
-
-// Singular function derived from the Laplacian of the "steep wavefront"
-// problem in [2].
-double singular_function(const Vector &p)
-{
-   double x = p(0), y = p(1);
-   double alpha = 1000.0;
-   double xc = 0.75, yc = 0.5;
-   double r0 = 0.7;
-   double r = sqrt(pow(x - xc,2.0) + pow(y - yc,2.0));
-   double num = - ( alpha - pow(alpha,3) * (pow(r,2) - pow(r0,2)) );
-   double denom = pow(r * ( pow(alpha,2) * pow(r0,2) + pow(alpha,2) * pow(r,2) \
-                            - 2 * pow(alpha,2) * r0 * r + 1.0 ),2);
-   denom = max(denom,1e-8);
-   return num / denom;
-}
-
-int main(int argc, char *argv[])
-{
-   // 0. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 1. Parse command-line options.
-   const char *mesh_file = "../data/star.mesh";
-   int order = 1;
-   int nc_limit = 1;
-   int max_elems = 1e5;
-   double double_max_elems = double(max_elems);
-   bool visualization = true;
-   bool nc_simplices = true;
-   double osc_threshold = 1e-3;
-   int enriched_order = 5;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&nc_limit, "-l", "--nc-limit",
-                  "Maximum level of hanging nodes.");
-   args.AddOption(&double_max_elems, "-me", "--max-elems",
-                  "Stop after reaching this many elements.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&osc_threshold, "-e", "--error",
-                  "relative data oscillation threshold.");
-   args.AddOption(&enriched_order, "-eo", "--enriched_order",
-                  "Enriched quadrature order.");
-   args.AddOption(&nc_simplices, "-ns", "--nonconforming-simplices",
-                  "-cs", "--conforming-simplices",
-                  "For simplicial meshes, enable/disable nonconforming"
-                  " refinement");
-
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   max_elems = int(double_max_elems);
-   Mesh mesh(mesh_file, 1, 1);
-
-   // 2. Since a NURBS mesh can currently only be refined uniformly, we need to
-   //    convert it to a piecewise-polynomial curved mesh. First we refine the
-   //    NURBS mesh a bit more and then project the curvature to quadratic Nodes.
-   if (mesh.NURBSext)
-   {
-      for (int i = 0; i < 2; i++)
-      {
-         mesh.UniformRefinement();
-      }
-      mesh.SetCurvature(2);
-   }
-
-   // 3. Make sure the mesh is in the non-conforming mode to enable local
-   //    refinement of quadrilaterals/hexahedra. Simplices can be refined
-   //    either in conforming or in non-conforming mode. The conforming
-   //    mode however does not support dynamic partitioning.
-   mesh.EnsureNCMesh(nc_simplices);
-
-   // 4. Define a parallel mesh by partitioning the serial mesh.
-   //    Once the parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
-
-   // 5. Define functions and refiner.
-   FunctionCoefficient affine_coeff(affine_function);
-   FunctionCoefficient jump_coeff(jump_function);
-   FunctionCoefficient singular_coeff(singular_function);
-   CoefficientRefiner coeffrefiner(affine_coeff,order);
-
-   // 6. Connect to GLVis.
-   char vishost[] = "localhost";
-   int  visport   = 19916;
-   socketstream sol_sock;
-   if (visualization)
-   {
-      sol_sock.open(vishost, visport);
-   }
-
-   // 7. Define custom integration rule (optional).
-   const IntegrationRule *irs[Geometry::NumGeom];
-   int order_quad = 2*order + enriched_order;
-   for (int i=0; i < Geometry::NumGeom; ++i)
-   {
-      irs[i] = &(IntRules.Get(i, order_quad));
-   }
-
-   // 8. Apply custom refiner settings.
-   coeffrefiner.SetIntRule(irs);
-   coeffrefiner.SetMaxElements(max_elems);
-   coeffrefiner.SetThreshold(osc_threshold);
-   coeffrefiner.SetNCLimit(nc_limit);
-   coeffrefiner.PrintWarnings();
-
-   // 9. Preprocess mesh to control osc (piecewise-affine function).
-   //    This is mostly just a verification check. The oscillation should
-   //    be zero if the function is mesh-conforming and order > 0.
-   coeffrefiner.PreprocessMesh(pmesh);
-
-   int globalNE = pmesh.GetGlobalNE();
-   double osc = coeffrefiner.GetOsc();
-   if (myid == 0)
-   {
-      mfem::out  << "\n";
-      mfem::out << "Function 0 (affine) \n";
-      mfem::out << "Number of Elements " << globalNE << "\n";
-      mfem::out << "Osc error " << osc << "\n";
-   }
-
-   // 10. Preprocess mesh to control osc (jump function).
-   coeffrefiner.ResetCoefficient(jump_coeff);
-   coeffrefiner.PreprocessMesh(pmesh);
-
-   globalNE = pmesh.GetGlobalNE();
-   osc = coeffrefiner.GetOsc();
-   if (myid == 0)
-   {
-      mfem::out  << "\n";
-      mfem::out << "Function 1 (discontinuous) \n";
-      mfem::out << "Number of Elements " << globalNE << "\n";
-      mfem::out << "Osc error " << osc << "\n";
-   }
-
-   // 11. Preprocess mesh to control osc (singular function).
-   coeffrefiner.ResetCoefficient(singular_coeff);
-   coeffrefiner.PreprocessMesh(pmesh);
-
-   globalNE = pmesh.GetGlobalNE();
-   osc = coeffrefiner.GetOsc();
-   if (myid == 0)
-   {
-      mfem::out  << "\n";
-      mfem::out << "Function 2 (singular) \n";
-      mfem::out << "Number of Elements " << globalNE << "\n";
-      mfem::out << "Osc error " << osc << "\n";
-   }
-
-   sol_sock.precision(8);
-   sol_sock << "parallel " << num_procs << " " << myid << "\n";
-   sol_sock << "mesh\n" << pmesh << flush;
-
-   MPI_Finalize();
-   return 0;
-}
@@ -255,10 +255,10 @@ int main(int argc, char *argv[])

   // 15. Compute and print the L^2 norm of the error.
   {
-      double error = x.ComputeL2Error(E);
+      double err = x.ComputeL2Error(E);
      if (myid == 0)
      {
-         cout << "\n|| E_h - E ||_{L^2} = " << error << '\n' << endl;
+         cout << "\n|| E_h - E ||_{L^2} = " << err << '\n' << endl;
      }
   }

@@ -256,10 +256,10 @@ int main(int argc, char *argv[])

   // 15. Compute and print the L^2 norm of the error.
   {
-      double error = x.ComputeL2Error(F);
+      double err = x.ComputeL2Error(F);
      if (myid == 0)
      {
-         cout << "\n|| F_h - F ||_{L^2} = " << error << '\n' << endl;
+         cout << "\n|| F_h - F ||_{L^2} = " << err << '\n' << endl;
      }
   }

@@ -125,7 +125,6 @@ int main(int argc, char *argv[])
   BilinearFormIntegrator *integ = new DiffusionIntegrator(one);
   a.AddDomainIntegrator(integ);
   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   if (Device::IsEnabled()) { b.SetAssemblyLevel(LinearAssemblyLevel::FULL); }

   // 7. The solution vector x and the associated finite element grid function
   //    will be maintained over the AMR iterations. We initialize it to zero.
@@ -202,7 +202,6 @@ int main(int argc, char *argv[])
   BilinearFormIntegrator *integ = new DiffusionIntegrator(one);
   a.AddDomainIntegrator(integ);
   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   if (Device::IsEnabled()) { b.SetAssemblyLevel(LinearAssemblyLevel::FULL); }

   // 12. The solution vector x and the associated finite element grid function
   //     will be maintained over the AMR iterations. We initialize it to zero.
@@ -282,10 +282,10 @@ int main(int argc, char *argv[])
   delete b;

   // 12. Compute and print the L^2 norm of the error.
-   double error = x.ComputeL2Error(sol_coef);
+   double err = x.ComputeL2Error(sol_coef);
   if (myid == 0)
   {
-      cout << "\nL2 norm of error: " << error << endl;
+      cout << "\nL2 norm of error: " << err << endl;
   }

   // 13. Save the refined mesh and the solution. This output can be viewed
@@ -31,31 +31,29 @@ add_mfem_examples(GINKGO_EXAMPLES_SRCS ${PFX} "" test_ginkgo)
 # which builds the examples and runs:
 #   ctest -R ginkgo

-if (MFEM_ENABLE_TESTING)
-  # Command line options for the tests.
-  set(EX1_COMMON_OPTS ex1 -m ../data/star.mesh --use_gko_solver)
-  set(EX1_TEST_OPTS ${EX9_COMMON_OPTS})
+# Command line options for the tests.
+set(EX1_COMMON_OPTS ex1 -m ../data/star.mesh --use_gko_solver)
+set(EX1_TEST_OPTS ${EX9_COMMON_OPTS})

-  # Add the tests: one test per source file.
-  foreach(SRC_FILE ${GINKGO_EXAMPLES_SRCS})
-    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-    string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-    set(TEST_NAME ${PFX}${TEST_NAME})
+# Add the tests: one test per source file.
+foreach(SRC_FILE ${GINKGO_EXAMPLES_SRCS})
+  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+  set(TEST_NAME ${PFX}${TEST_NAME})

-    set(THIS_TEST_OPTIONS "-no-vis")
-    list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-    # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+  set(THIS_TEST_OPTIONS "-no-vis")
+  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")

-    if (NOT (${TEST_NAME} MATCHES ".*p$"))
-      add_test(NAME ${TEST_NAME}_ser
-        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-    else()
-      add_test(NAME ${TEST_NAME}_np=4
-        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-        ${MPIEXEC_PREFLAGS}
-        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-        ${MPIEXEC_POSTFLAGS})
-    endif()
-  endforeach()
-endif()
+  if (NOT (${TEST_NAME} MATCHES ".*p$"))
+    add_test(NAME ${TEST_NAME}_ser
+      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+  else()
+    add_test(NAME ${TEST_NAME}_np=4
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+      ${MPIEXEC_POSTFLAGS})
+  endif()
+endforeach()
@@ -33,33 +33,31 @@ add_mfem_examples(HIOP_EXAMPLES_SRCS ${PFX} "" test_hiop)
 # which builds the examples and runs:
 #   ctest -R hiop

-if (MFEM_ENABLE_TESTING)
-  # Command line options for the tests.
-  # Example 9:
-  set(EX9_COMMON_OPTS -m ../../data/periodic-segment.mesh -p 0 -dt 0.005)
-  set(EX9_TEST_OPTS ${EX9_COMMON_OPTS}  -r  2 )
-  set(EX9P_TEST_OPTS ${EX9_COMMON_OPTS})
+# Command line options for the tests.
+# Example 9:
+set(EX9_COMMON_OPTS -m ../../data/periodic-segment.mesh -p 0 -dt 0.005)
+set(EX9_TEST_OPTS ${EX9_COMMON_OPTS}  -r  2 )
+set(EX9P_TEST_OPTS ${EX9_COMMON_OPTS})

-  # Add the tests: one test per source file.
-  foreach(SRC_FILE ${HIOP_EXAMPLES_SRCS})
-    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-    string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-    set(TEST_NAME ${PFX}${TEST_NAME})
+# Add the tests: one test per source file.
+foreach(SRC_FILE ${HIOP_EXAMPLES_SRCS})
+  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+  set(TEST_NAME ${PFX}${TEST_NAME})

-    set(THIS_TEST_OPTIONS "-no-vis")
-    list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-    # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+  set(THIS_TEST_OPTIONS "-no-vis")
+  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")

-    if (NOT (${TEST_NAME} MATCHES ".*p$"))
-      add_test(NAME ${TEST_NAME}_ser
-        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-    else()
-      add_test(NAME ${TEST_NAME}_np=4
-        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4
-        ${MPIEXEC_PREFLAGS}
-        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-        ${MPIEXEC_POSTFLAGS})
-    endif()
-  endforeach()
-endif()
+  if (NOT (${TEST_NAME} MATCHES ".*p$"))
+    add_test(NAME ${TEST_NAME}_ser
+      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+  else()
+    add_test(NAME ${TEST_NAME}_np=4
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+      ${MPIEXEC_POSTFLAGS})
+  endif()
+endforeach()
@@ -1,31 +0,0 @@
-# Jupyter Notebooks using xeus-cling
-
-[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/mfem/mfem/master?filepath=examples%2Fjupyter%2Fex.ipynb)
-
-[xeus-cling](https://github.com/jupyter-xeus/xeus-cling) is a C++ Jupyter Kernel based on [cling](https://github.com/root-project/cling),
-which can be used to create interactive C++ MFEM and GLVis notebooks.
-
-Click on the `binder` button above for an interactive example.
-
-## Installing Locally
-
-In order to run notebooks locally you will need `xeus-cling` along with `mfem` and `xglvis`. We recommend you use
-[miniconda](https://docs.conda.io/en/latest/miniconda.html) or, if you already have it installed,
-[conda](https://docs.conda.io/projects/conda/en/latest/).
-
-1. Follow the install steps on https://github.com/jupyter-xeus/xeus-cling to install the C++ kernels
-2. Build and install a _shared_ version of mfem
-    * for example: `make serial SHARED=YES`
-3. Install [pyglvis](https://github.com/glvis/pyglvis)
-    * for the widget frontend
-4. Get [xeus-glvis](https://github.com/glvis/xeus-glvis) and `cp` the header to `{PREFIX}/glvis/xglvis.hpp`
-    * (this could be improved)
-
-## Running Locally
-
-Once you've installed Jupyter, the C++ Kernel, mfem, and glvis start the notebook server (`jupyter-notebook`)
-and open an existing example or a new `C++ 1x` kernel.
-
-You will _always_ need to `#pragma cling load("mfem")` and you may need to point the `cling` runtime at your
-mfem and/or glvis installs, do this with the
-`#pragma cling` [statements](https://xeus-cling.readthedocs.io/en/latest/build_options.html#using-third-party-libraries).
@@ -1,155 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "owned-extraction",
-   "metadata": {},
-   "source": [
-    "## Load the MFEM library\n",
-    "\n",
-    "Any non-default libraries must be loaded before you can `#include` files that use them. For more info see the [xeus-cling help](https://xeus-cling.readthedocs.io/en/latest/build_options.html)."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "waiting-portrait",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "#pragma cling load(\"mfem\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "foreign-recycling",
-   "metadata": {},
-   "source": [
-    "## MFEM Example 1"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "public-white",
-   "metadata": {},
-   "source": [
-    "This is the simplest MFEM example and a good starting point for new users. The example demonstrates the use of MFEM to define and solve an $H^1$ finite element discretization of the Laplace problem\n",
-    "\n",
-    "$$\n",
-    "-\\Delta u = 1\n",
-    "$$\n",
-    "\n",
-    "with homogeneous Dirichlet boundary conditions $u=0$.\n",
-    "\n",
-    "The example illustrates the use of the basic MFEM classes for defining the mesh, finite element space, as well as linear and bilinear forms corresponding to the left-hand side and right-hand side of the discrete linear system.\n",
-    "\n",
-    "Compare with MFEM's [ex1.cpp](https://github.com/mfem/mfem/blob/master/examples/ex1.cpp) and PyMFEM's [ex1.py](https://github.com/mfem/PyMFEM/blob/master/examples/ex1.py)."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "protective-darkness",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "#include <fstream>\n",
-    "#include <iostream>\n",
-    "#include <sstream>\n",
-    "\n",
-    "#include <mfem.hpp>\n",
-    "#include <glvis/xglvis.hpp>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "falling-monkey",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "using namespace std;\n",
-    "using namespace mfem;\n",
-    "\n",
-    "Mesh mesh = Mesh::MakeCartesian2D(5, 5, Element::TRIANGLE);\n",
-    "mesh.UniformRefinement();\n",
-    "\n",
-    "H1_FECollection fec(2, mesh.Dimension());\n",
-    "\n",
-    "FiniteElementSpace fespace(&mesh, &fec);\n",
-    "cout << \"Number of finite element unknowns: \" << fespace.GetTrueVSize() << endl;\n",
-    "\n",
-    "Array<int> ess_tdof_list;\n",
-    "if (mesh.bdr_attributes.Size())\n",
-    "{\n",
-    "  Array<int> ess_bdr(mesh.bdr_attributes.Max());\n",
-    "  ess_bdr = 1;\n",
-    "  fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);\n",
-    "}\n",
-    "\n",
-    "LinearForm b(&fespace);\n",
-    "ConstantCoefficient one(1.0);\n",
-    "b.AddDomainIntegrator(new DomainLFIntegrator(one));\n",
-    "b.Assemble();\n",
-    "\n",
-    "GridFunction x(&fespace);\n",
-    "x = 0.0;\n",
-    "\n",
-    "BilinearForm a(&fespace);\n",
-    "a.AddDomainIntegrator(new DiffusionIntegrator(one));\n",
-    "a.Assemble();\n",
-    "\n",
-    "OperatorPtr A;\n",
-    "Vector B, X;\n",
-    "a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);\n",
-    "\n",
-    "cout << \"Size of linear system: \" << A->Height() << endl;\n",
-    "\n",
-    "GSSmoother M((SparseMatrix&)(*A));\n",
-    "PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);\n",
-    "a.RecoverFEMSolution(X, b, x);"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "hawaiian-republican",
-   "metadata": {},
-   "source": [
-    "## GLVis Visualization\n",
-    "\n",
-    "For now we save the computational mesh and finite element solution in a string and pass that to the glvis widget, see https://github.com/glvis/xeus-glvis for the widget backend and https://github.com/GLVis/pyglvis/tree/master/js for the widget frontend."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ordinary-equation",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "std::stringstream ss;\n",
-    "ss << \"solution\\n\" << mesh << x << flush;\n",
-    "\n",
-    "auto glv = glvis::glvis();\n",
-    "glv.plot(ss.str() + \"keys Rjml\"); // the `+ \"keys ....\"' is optional\n",
-    "glv"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "C++14",
-   "language": "C++14",
-   "name": "xcpp14"
-  },
-  "language_info": {
-   "codemirror_mode": "text/x-c++src",
-   "file_extension": ".cpp",
-   "mimetype": "text/x-c++src",
-   "name": "c++",
-   "version": "14"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}
@@ -22,10 +22,10 @@ MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)

 SEQ_EXAMPLES = ex0 ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 \
-  ex17 ex18 ex19 ex20 ex21 ex22 ex23 ex24 ex25 ex26 ex27 ex28 ex29 ex30
+  ex17 ex18 ex19 ex20 ex21 ex22 ex23 ex24 ex25 ex26 ex27 ex28 ex29
 PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
  ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
-  ex25p ex26p ex27p ex28p ex29p ex30p
+  ex25p ex26p ex27p ex28p ex29p
 SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
 PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p \
  ex24p ex25p ex26p
@@ -94,32 +94,30 @@ if (MFEM_USE_SLEPC)
 endif()

 # Add the tests: one test per command-line-variable.
-if (MFEM_ENABLE_TESTING)
-  set(TEST_OPTIONS_VARS
-      EX1_ARGS_W EX1_ARGS_P EX2_ARGS EX3_ARGS EX4_ARGS EX4_HYB_ARGS
-      EX5_BDDC_LB_ARGS EX5_BDDC_GB_ARGS EX5_FSPL_ARGS EX6_ARGS EX6_NONOVL_ARGS
-      EX9_E_ARGS EX9_ES_ARGS EX9_IS_ARGS EX10_ARGS)
-  if (MFEM_USE_SLEPC)
-      list(APPEND TEST_OPTIONS_VARS EX11_ARGS_SINV EX11_ARGS_LOBPCG EX11_ARGS_GD)
-  endif()
-
-  foreach(TEST_OPTIONS_VAR ${TEST_OPTIONS_VARS})
-    string(REGEX REPLACE "^(.+)_ARGS" "\\1" TEST_NAME_UC ${TEST_OPTIONS_VAR})
-    string(REGEX REPLACE "^([^_]+)" "\\1P" TEST_NAME_UC ${TEST_NAME_UC})
-    string(TOLOWER ${TEST_NAME_UC} TEST_NAME_FULL)
-    string(REGEX REPLACE "^([^_]+).*" "\\1" TEST_NAME ${TEST_NAME_FULL})
-    set(TEST_NAME_FULL ${PFX}${TEST_NAME_FULL})
-    set(TEST_NAME ${PFX}${TEST_NAME})
-    set(TEST_OPTIONS "-no-vis" ${${TEST_OPTIONS_VAR}})
-    # message(STATUS "${TEST_NAME_FULL} --> ${TEST_NAME} ${TEST_OPTIONS}")
-
-    # All PETSC tests are parallel.
-    if (MFEM_USE_MPI)
-      add_test(NAME ${TEST_NAME_FULL}_np=4
-        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-        ${MPIEXEC_PREFLAGS}
-        $<TARGET_FILE:${TEST_NAME}> ${TEST_OPTIONS}
-        ${MPIEXEC_POSTFLAGS})
-    endif()
-  endforeach()
+set(TEST_OPTIONS_VARS
+    EX1_ARGS_W EX1_ARGS_P EX2_ARGS EX3_ARGS EX4_ARGS EX4_HYB_ARGS
+    EX5_BDDC_LB_ARGS EX5_BDDC_GB_ARGS EX5_FSPL_ARGS EX6_ARGS EX6_NONOVL_ARGS
+    EX9_E_ARGS EX9_ES_ARGS EX9_IS_ARGS EX10_ARGS)
+if (MFEM_USE_SLEPC)
+    list(APPEND TEST_OPTIONS_VARS EX11_ARGS_SINV EX11_ARGS_LOBPCG EX11_ARGS_GD)
 endif()
+
+foreach(TEST_OPTIONS_VAR ${TEST_OPTIONS_VARS})
+  string(REGEX REPLACE "^(.+)_ARGS" "\\1" TEST_NAME_UC ${TEST_OPTIONS_VAR})
+  string(REGEX REPLACE "^([^_]+)" "\\1P" TEST_NAME_UC ${TEST_NAME_UC})
+  string(TOLOWER ${TEST_NAME_UC} TEST_NAME_FULL)
+  string(REGEX REPLACE "^([^_]+).*" "\\1" TEST_NAME ${TEST_NAME_FULL})
+  set(TEST_NAME_FULL ${PFX}${TEST_NAME_FULL})
+  set(TEST_NAME ${PFX}${TEST_NAME})
+  set(TEST_OPTIONS "-no-vis" ${${TEST_OPTIONS_VAR}})
+  # message(STATUS "${TEST_NAME_FULL} --> ${TEST_NAME} ${TEST_OPTIONS}")
+
+  # All PETSC tests are parallel.
+  if (MFEM_USE_MPI)
+    add_test(NAME ${TEST_NAME_FULL}_np=4
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:${TEST_NAME}> ${TEST_OPTIONS}
+      ${MPIEXEC_POSTFLAGS})
+  endif()
+endforeach()
@@ -37,39 +37,37 @@ add_mfem_examples(PUMI_EXAMPLES_SRCS ${PFX} "" test_pumi)
 # which builds the examples and runs:
 #   ctest -R pumi

-if (MFEM_ENABLE_TESTING)
-  # Command line options for the tests.
-  # TODO...
+# Command line options for the tests.
+# TODO...

-  # Set the number of processors for the parallel examples. The value of
-  # MFEM_MPI_NP is ignored.
-  set(EX1_TEST_NP 1)
-  set(EX1P_TEST_NP 8)
-  set(EX2_TEST_NP 1)
-  set(EX6P_TEST_NP 8)
+# Set the number of processors for the parallel examples. The value of
+# MFEM_MPI_NP is ignored.
+set(EX1_TEST_NP 1)
+set(EX1P_TEST_NP 8)
+set(EX2_TEST_NP 1)
+set(EX6P_TEST_NP 8)

-  # Add the tests: one test per source file.
-  foreach(SRC_FILE ${PUMI_EXAMPLES_SRCS})
-    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-    string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-    set(TEST_NAME ${PFX}${TEST_NAME})
+# Add the tests: one test per source file.
+foreach(SRC_FILE ${PUMI_EXAMPLES_SRCS})
+  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+  set(TEST_NAME ${PFX}${TEST_NAME})

-    set(THIS_TEST_OPTIONS "-no-vis")
-    list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-    # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+  set(THIS_TEST_OPTIONS "-no-vis")
+  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")

-    # All PUMI examples require MPI
-    if (FALSE)
-      add_test(NAME ${TEST_NAME}_ser
-        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-    else()
-      set(TEST_NP ${${UP_TEST_NAME}_TEST_NP})
-      add_test(NAME ${TEST_NAME}_np=${TEST_NP}
-        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${TEST_NP}
-        ${MPIEXEC_PREFLAGS}
-        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-        ${MPIEXEC_POSTFLAGS})
-    endif()
-  endforeach()
-endif()
+  # All PUMI examples require MPI
+  if (FALSE)
+    add_test(NAME ${TEST_NAME}_ser
+      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+  else()
+    set(TEST_NP ${${UP_TEST_NAME}_TEST_NP})
+    add_test(NAME ${TEST_NAME}_np=${TEST_NP}
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${TEST_NP}
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+      ${MPIEXEC_POSTFLAGS})
+  endif()
+endforeach()
@@ -41,38 +41,36 @@ add_mfem_examples(SUNDIALS_EXAMPLES_SRCS ${PFX} "" test_sundials)
 # which builds the examples and runs:
 #   ctest -R sundials

-if (MFEM_ENABLE_TESTING)
-  # Command line options for the tests.
-  # Example 9: test CVODE with CV_ADAMS (non-stiff implicit) time stepping
-  set(EX9_COMMON_OPTS -m ../../data/periodic-hexagon.mesh -p 0 -s 7)
-  set(EX9_TEST_OPTS ${EX9_COMMON_OPTS}  -r  2 -dt 0.0018 -vs 25)
-  set(EX9P_TEST_OPTS ${EX9_COMMON_OPTS} -rp 1 -dt 0.0009 -vs 50)
-  # Example 10: test CVODE with CV_BDF (stiff implicit) time stepping
-  set(EX10_COMMON_OPTS -m ../../data/beam-quad.mesh -o 2 -s 5 -dt 0.15 -tf 6 -vs 10)
-  set(EX10_TEST_OPTS ${EX10_COMMON_OPTS}  -r  2)
-  set(EX10P_TEST_OPTS ${EX10_COMMON_OPTS} -rp 1)
-  # Example 16: use the default options
+# Command line options for the tests.
+# Example 9: test CVODE with CV_ADAMS (non-stiff implicit) time stepping
+set(EX9_COMMON_OPTS -m ../../data/periodic-hexagon.mesh -p 0 -s 7)
+set(EX9_TEST_OPTS ${EX9_COMMON_OPTS}  -r  2 -dt 0.0018 -vs 25)
+set(EX9P_TEST_OPTS ${EX9_COMMON_OPTS} -rp 1 -dt 0.0009 -vs 50)
+# Example 10: test CVODE with CV_BDF (stiff implicit) time stepping
+set(EX10_COMMON_OPTS -m ../../data/beam-quad.mesh -o 2 -s 5 -dt 0.15 -tf 6 -vs 10)
+set(EX10_TEST_OPTS ${EX10_COMMON_OPTS}  -r  2)
+set(EX10P_TEST_OPTS ${EX10_COMMON_OPTS} -rp 1)
+# Example 16: use the default options

-  # Add the tests: one test per source file.
-  foreach(SRC_FILE ${SUNDIALS_EXAMPLES_SRCS})
-    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-    string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-    set(TEST_NAME ${PFX}${TEST_NAME})
+# Add the tests: one test per source file.
+foreach(SRC_FILE ${SUNDIALS_EXAMPLES_SRCS})
+  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+  set(TEST_NAME ${PFX}${TEST_NAME})

-    set(THIS_TEST_OPTIONS "-no-vis")
-    list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-    # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+  set(THIS_TEST_OPTIONS "-no-vis")
+  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")

-    if (NOT (${TEST_NAME} MATCHES ".*p$"))
-      add_test(NAME ${TEST_NAME}_ser
-        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-    else()
-      add_test(NAME ${TEST_NAME}_np=4
-        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-        ${MPIEXEC_PREFLAGS}
-        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-        ${MPIEXEC_POSTFLAGS})
-    endif()
-  endforeach()
-endif()
+  if (NOT (${TEST_NAME} MATCHES ".*p$"))
+    add_test(NAME ${TEST_NAME}_ser
+      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+  else()
+    add_test(NAME ${TEST_NAME}_np=4
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+      ${MPIEXEC_POSTFLAGS})
+  endif()
+endforeach()
@@ -32,32 +32,31 @@ add_mfem_examples(SUPERLU_EXAMPLES_SRCS ${PFX} "" test_superlu)
 # The SuperLU tests can be run separately using the target "test_superlu"
 # which builds the examples and runs:
 #   ctest -R superlu
-if (MFEM_ENABLE_TESTING)
-  # Command line options for the tests.
-  # Example 1: Test SuperLU on the simple Poisson problem
-  set(EX1_COMMON_OPTS -m ../../data/star.mesh -p 2)
-  set(EX1P_TEST_OPTS ${EX1_COMMON_OPTS})

-  # Add the tests: one test per source file.
-  foreach(SRC_FILE ${SUPERLU_EXAMPLES_SRCS})
-    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-    string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-    set(TEST_NAME ${PFX}${TEST_NAME})
+# Command line options for the tests.
+# Example 1: Test SuperLU on the simple Poisson problem
+set(EX1_COMMON_OPTS -m ../../data/star.mesh -p 2)
+set(EX1P_TEST_OPTS ${EX1_COMMON_OPTS})

-    set(THIS_TEST_OPTIONS "-no-vis")
-    list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-    # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+# Add the tests: one test per source file.
+foreach(SRC_FILE ${SUPERLU_EXAMPLES_SRCS})
+  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+  set(TEST_NAME ${PFX}${TEST_NAME})

-    if (NOT (${TEST_NAME} MATCHES ".*p$"))
-      add_test(NAME ${TEST_NAME}_ser
-        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-    else()
-      add_test(NAME ${TEST_NAME}_np=4
-        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-        ${MPIEXEC_PREFLAGS}
-        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-        ${MPIEXEC_POSTFLAGS})
-    endif()
-  endforeach()
-endif()
+  set(THIS_TEST_OPTIONS "-no-vis")
+  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+
+  if (NOT (${TEST_NAME} MATCHES ".*p$"))
+    add_test(NAME ${TEST_NAME}_ser
+      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+  else()
+    add_test(NAME ${TEST_NAME}_np=4
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+      ${MPIEXEC_POSTFLAGS})
+  endif()
+endforeach()
@@ -21,8 +21,6 @@ set(SRCS
  bilininteg_dgtrace_ea.cpp
  bilininteg_diffusion_mf.cpp
  bilininteg_diffusion_pa.cpp
-  bilininteg_diffusion_pa_diag_fast.cpp
-  bilininteg_diffusion_pa_fast.cpp
  bilininteg_diffusion_ea.cpp
  bilininteg_divergence.cpp
  bilininteg_hcurl.cpp
@@ -31,11 +29,6 @@ set(SRCS
  bilininteg_gradient.cpp
  bilininteg_mass_mf.cpp
  bilininteg_mass_pa.cpp
-  bilininteg_mass_pa_diag.cpp
-  bilininteg_mass_pa_fast.cpp
-  bilininteg_mass_pa_fast_amd.cpp
-  bilininteg_mass_pa_fast_hip.cpp
-  bilininteg_mass_pa_fast_diag.cpp
  bilininteg_mass_ea.cpp
  bilininteg_transpose_ea.cpp
  bilininteg_vecdiffusion.cpp
@@ -50,15 +43,6 @@ set(SRCS
  eltrans.cpp
  estimators.cpp
  fe.cpp
-  fe/fe_base.cpp
-  fe/fe_fixed_order.cpp
-  fe/fe_h1.cpp
-  fe/fe_l2.cpp
-  fe/fe_nd.cpp
-  fe/fe_nurbs.cpp
-  fe/fe_pos.cpp
-  fe/fe_rt.cpp
-  fe/fe_ser.cpp
  fe_coll.cpp
  fespace.cpp
  geom.cpp
@@ -75,16 +59,7 @@ set(SRCS
  ceed/operator.cpp
  ceed/util.cpp
  linearform.cpp
-  linearform_ext.cpp
  lininteg.cpp
-  lininteg_domain.cpp
-  lininteg_domain_grad.cpp
-  lininteg_vector_domain.cpp
-  lininteg_vector_domain_grad.cpp
-  lor.cpp
-  lor_assembly.cpp
-  lor_assembly_ker.cpp
-  lor_assembly_ni3.cpp
  multigrid.cpp
  nonlinearform.cpp
  nonlinearform_ext.cpp
@@ -149,15 +124,6 @@ set(HDRS
  eltrans.hpp
  estimators.hpp
  fe.hpp
-  fe/fe_base.hpp
-  fe/fe_fixed_order.hpp
-  fe/fe_h1.hpp
-  fe/fe_l2.hpp
-  fe/fe_nd.hpp
-  fe/fe_nurbs.hpp
-  fe/fe_pos.hpp
-  fe/fe_rt.hpp
-  fe/fe_ser.hpp
  fe_coll.hpp
  fem.hpp
  fespace.hpp
@@ -177,12 +143,7 @@ set(HDRS
  ceed/operator.hpp
  ceed/util.hpp
  linearform.hpp
-  linearform_ext.hpp
  lininteg.hpp
-  lininteg_domain.hpp
-  lininteg_domain_grad.hpp
-  lor.hpp
-  lor_assembly.hpp
  multigrid.hpp
  nonlinearform.hpp
  nonlinearform_ext.hpp
@@ -969,7 +969,6 @@ void BilinearForm::EliminateVDofs(const Array<int> &vdofs,
                                  const Vector &sol, Vector &rhs,
                                  DiagonalPolicy dpolicy)
 {
-   vdofs.HostRead();
   for (int i = 0; i < vdofs.Size(); i++)
   {
      int vdof = vdofs[i];
@@ -318,31 +318,14 @@ void PABilinearFormExtension::Assemble()
 void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
 {
   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int integrators_size = integrators.Size();

-   // Scan the different action types we need to launch
-   bool E2E_actions = false,
-        L2L_actions = false;
-   for (int i = 0; i < integrators_size; ++i)
-   {
-      const ActionType action_type = integrators[i]->GetActionType();
-      E2E_actions |= action_type == ActionType::E2E;
-      L2L_actions |= action_type == ActionType::L2L;
-   }
-
-   // typically this is a large vector, so store on device
-   y.UseDevice(true);
-
-   // If E2E kernels are present, do their computation on local vectors
-   if (E2E_actions && elem_restrict)
+   const int iSz = integrators.Size();
+   if (elem_restrict && !DeviceCanUseCeed())
   {
      localY = 0.0;
-      for (int i = 0; i < integrators_size; ++i)
+      for (int i = 0; i < iSz; ++i)
      {
-         if (integrators[i]->GetActionType() == ActionType::E2E)
-         {
-            integrators[i]->AssembleDiagonalPA(localY);
-         }
+         integrators[i]->AssembleDiagonalPA(localY);
      }
      const ElementRestriction* H1elem_restrict =
         dynamic_cast<const ElementRestriction*>(elem_restrict);
@@ -357,20 +340,11 @@ void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
   }
   else
   {
-      // otherwise, initialize the y output
+      y.UseDevice(true); // typically this is a large vector, so store on device
      y = 0.0;
-   }
-
-   // Continue with the computation of the L2L integrators
-   if (L2L_actions || !elem_restrict)
-   {
-      for (int i = 0; i < integrators_size; ++i)
+      for (int i = 0; i < iSz; ++i)
      {
-         const ActionType action = integrators[i]->GetActionType();
-         if (action == ActionType::L2L || !elem_restrict)
-         {
-            integrators[i]->AssembleDiagonalPA(y);
-         }
+         integrators[i]->AssembleDiagonalPA(y);
      }
   }
 }
@@ -409,51 +383,26 @@ void PABilinearFormExtension::FormLinearSystem(const Array<int> &ess_tdof_list,
 void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
 {
   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int integrators_size = integrators.Size();

-   // Scan the different action types we need to launch
-   bool E2E_actions = false,
-        L2L_actions = false;
-   for (int i = 0; i < integrators_size; ++i)
+   const int iSz = integrators.Size();
+   if (DeviceCanUseCeed() || !elem_restrict)
   {
-      const ActionType action_type = integrators[i]->GetActionType();
-      E2E_actions |= action_type == ActionType::E2E;
-      L2L_actions |= action_type == ActionType::L2L;
-   }
-
-   // typically this is a large vector, so store on device
-   y.UseDevice(true);
-
-   // If E2E kernels are present, do their computation on local vectors
-   if (E2E_actions && elem_restrict)
-   {
-      elem_restrict->Mult(x, localX);
-      localY = 0.0;
-      for (int i = 0; i < integrators_size; ++i)
+      y.UseDevice(true); // typically this is a large vector, so store on device
+      y = 0.0;
+      for (int i = 0; i < iSz; ++i)
      {
-         if (integrators[i]->GetActionType() == ActionType::E2E)
-         {
-            integrators[i]->AddMultPA(localX, localY);
-         }
+         integrators[i]->AddMultPA(x, y);
      }
-      elem_restrict->MultTranspose(localY, y);
   }
   else
   {
-      // otherwise, initialize the y output
-      y = 0.0;
-   }
-
-   // Continue with the computation of the L2L integrators
-   if (L2L_actions || !elem_restrict)
-   {
-      for (int i = 0; i < integrators_size; ++i)
+      elem_restrict->Mult(x, localX);
+      localY = 0.0;
+      for (int i = 0; i < iSz; ++i)
      {
-         if (integrators[i]->GetActionType() == ActionType::L2L || !elem_restrict)
-         {
-            integrators[i]->AddMultPA(x, y);
-         }
+         integrators[i]->AddMultPA(localX, localY);
      }
+      elem_restrict->MultTranspose(localY, y);
   }

   Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
@@ -747,7 +747,6 @@ void DiffusionIntegrator::AssembleElementMatrix
 #ifdef MFEM_THREAD_SAFE
   DenseMatrix dshape(nd, dim), dshapedxt(nd, spaceDim);
   DenseMatrix dshapedxt_m(nd, MQ ? spaceDim : 0);
-   DenseMatrix M(MQ ? spaceDim : 0);
   Vector D(VQ ? VQ->GetVDim() : 0);
 #else
   dshape.SetSize(nd, dim);
@@ -985,8 +984,6 @@ void DiffusionIntegrator::ComputeElementFlux
                  "Unexpected height for MatrixCoefficient");
   }

-   MFEM_VERIFY(!SMQ, "SymmetricMatrixCoefficient not supported here");
-
 #ifdef MFEM_THREAD_SAFE
   DenseMatrix dshape(nd,dim), invdfdx(dim, spaceDim);
   DenseMatrix M(MQ ? spaceDim : 0);
@@ -999,7 +996,7 @@ void DiffusionIntegrator::ComputeElementFlux
 #endif
   vec.SetSize(dim);
   vecdxt.SetSize(spaceDim);
-   pointflux.SetSize(MQ || VQ ? spaceDim : 0);
+   pointflux.SetSize(MQ ? spaceDim : 0);

   const IntegrationRule &ir = fluxelem.GetNodes();
   fnd = ir.GetNPoints();
@@ -1015,45 +1012,36 @@ void DiffusionIntegrator::ComputeElementFlux
      CalcInverse(Trans.Jacobian(), invdfdx);
      invdfdx.MultTranspose(vec, vecdxt);

-      if (with_coef)
+      if (!MQ && !VQ)
      {
-         if (!MQ && !VQ)
+         if (Q && with_coef)
         {
-            if (Q)
-            {
-               vecdxt *= Q->Eval(Trans,ip);
-            }
-            for (j = 0; j < spaceDim; j++)
-            {
-               flux(fnd*j+i) = vecdxt(j);
-            }
+            vecdxt *= Q->Eval(Trans,ip);
         }
-         else
+         for (j = 0; j < spaceDim; j++)
         {
-            if (MQ)
-            {
-               MQ->Eval(M, Trans, ip);
-               M.Mult(vecdxt, pointflux);
-            }
-            else
-            {
-               VQ->Eval(D, Trans, ip);
-               for (int j=0; j<spaceDim; ++j)
-               {
-                  pointflux[j] = D[j] * vecdxt[j];
-               }
-            }
-            for (j = 0; j < spaceDim; j++)
-            {
-               flux(fnd*j+i) = pointflux(j);
-            }
+            flux(fnd*j+i) = vecdxt(j);
         }
      }
      else
      {
+         if (MQ)
+         {
+            MQ->Eval(M, Trans, ip);
+            M.Mult(vecdxt, pointflux);
+         }
+         else
+         {
+            VQ->Eval(D, Trans, ip);
+            for (int j=0; j<spaceDim; ++j)
+            {
+               pointflux[j] = D[j] * vecdxt[j];
+            }
+
+         }
         for (j = 0; j < spaceDim; j++)
         {
-            flux(fnd*j+i) = vecdxt(j);
+            flux(fnd*j+i) = pointflux(j);
         }
      }
   }
@@ -1069,13 +1057,8 @@ double DiffusionIntegrator::ComputeFluxEnergy

 #ifdef MFEM_THREAD_SAFE
   DenseMatrix M;
-   Vector D(VQ ? VQ->GetVDim() : 0);
-#else
-   D.SetSize(VQ ? VQ->GetVDim() : 0);
 #endif

-   MFEM_VERIFY(!SMQ, "SymmetricMatrixCoefficient not supported here");
-
   shape.SetSize(nd);
   pointflux.SetSize(spaceDim);
   if (d_energy) { vec.SetSize(spaceDim); }
@@ -1104,23 +1087,17 @@ double DiffusionIntegrator::ComputeFluxEnergy
      Trans.SetIntPoint(&ip);
      double w = Trans.Weight() * ip.weight;

-      if (MQ)
-      {
-         MQ->Eval(M, Trans, ip);
-         energy += w * M.InnerProduct(pointflux, pointflux);
-      }
-      else if (VQ)
-      {
-         VQ->Eval(D, Trans, ip);
-         D *= pointflux;
-         energy += w * (D * pointflux);
-      }
-      else
+      if (!MQ)
      {
         double e = (pointflux * pointflux);
         if (Q) { e *= Q->Eval(Trans, ip); }
         energy += w * e;
      }
+      else
+      {
+         MQ->Eval(M, Trans, ip);
+         energy += w * M.InnerProduct(pointflux, pointflux);
+      }

      if (d_energy)
      {
@@ -1130,7 +1107,7 @@ double DiffusionIntegrator::ComputeFluxEnergy
         {
            (*d_energy)[k] += w * vec[k] * vec[k];
         }
-         // TODO: Q, VQ, MQ
+         // TODO: Q, MQ
      }
   }

@@ -244,8 +244,6 @@ public:
                                    Vector &flux, Vector *d_energy = NULL)
   { return 0.0; }

-   virtual bool SupportsBatchedLOR() const { return false; }
-
   virtual ~BilinearFormIntegrator() { }
 };

@@ -1979,37 +1977,29 @@ private:
   const DofToQuad *maps;         ///< Not owned
   const GeometricFactors *geom;  ///< Not owned
   int dim, ne, dofs1D, quad1D;
-   Vector pa_data, CoG;
+   Vector pa_data;
   bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient

 public:
   /// Construct a diffusion integrator with coefficient Q = 1
-   DiffusionIntegrator(const IntegrationRule *ir = nullptr)
-      : BilinearFormIntegrator(ir),
-        Q(NULL), VQ(NULL), MQ(NULL), SMQ(NULL), maps(NULL), geom(NULL) { }
+   DiffusionIntegrator()
+      : Q(NULL), VQ(NULL), MQ(NULL), SMQ(NULL), maps(NULL), geom(NULL) { }

   /// Construct a diffusion integrator with a scalar coefficient q
-   DiffusionIntegrator(Coefficient &q, const IntegrationRule *ir = nullptr)
-      : BilinearFormIntegrator(ir),
-        Q(&q), VQ(NULL), MQ(NULL), SMQ(NULL), maps(NULL), geom(NULL) { }
+   DiffusionIntegrator(Coefficient &q)
+      : Q(&q), VQ(NULL), MQ(NULL), SMQ(NULL), maps(NULL), geom(NULL) { }

   /// Construct a diffusion integrator with a vector coefficient q
-   DiffusionIntegrator(VectorCoefficient &q,
-                       const IntegrationRule *ir = nullptr)
-      : BilinearFormIntegrator(ir),
-        Q(NULL), VQ(&q), MQ(NULL), SMQ(NULL), maps(NULL), geom(NULL) { }
+   DiffusionIntegrator(VectorCoefficient &q)
+      : Q(NULL), VQ(&q), MQ(NULL), SMQ(NULL), maps(NULL), geom(NULL) { }

   /// Construct a diffusion integrator with a matrix coefficient q
-   DiffusionIntegrator(MatrixCoefficient &q,
-                       const IntegrationRule *ir = nullptr)
-      : BilinearFormIntegrator(ir),
-        Q(NULL), VQ(NULL), MQ(&q), SMQ(NULL), maps(NULL), geom(NULL) { }
+   DiffusionIntegrator(MatrixCoefficient &q)
+      : Q(NULL), VQ(NULL), MQ(&q), SMQ(NULL), maps(NULL), geom(NULL) { }

   /// Construct a diffusion integrator with a symmetric matrix coefficient q
-   DiffusionIntegrator(SymmetricMatrixCoefficient &q,
-                       const IntegrationRule *ir = nullptr)
-      : BilinearFormIntegrator(ir),
-        Q(NULL), VQ(NULL), MQ(NULL), SMQ(&q), maps(NULL), geom(NULL) { }
+   DiffusionIntegrator(SymmetricMatrixCoefficient &q)
+      : Q(NULL), VQ(NULL), MQ(NULL), SMQ(&q), maps(NULL), geom(NULL) { }

   /** Given a particular Finite Element computes the element stiffness matrix
       elmat. */
@@ -2060,8 +2050,6 @@ public:
                                         const FiniteElement &test_fe);

   bool SupportsCeed() const { return DeviceCanUseCeed(); }
-
-   bool SupportsBatchedLOR() const { return true; }
 };

 /** Class for local mass matrix assembling a(u,v) := (Q u, v) */
@@ -2680,9 +2668,6 @@ public:
   VectorDiffusionIntegrator(Coefficient &q)
      : Q(&q) { }

-   VectorDiffusionIntegrator(Coefficient &q, const IntegrationRule *ir)
-      : BilinearFormIntegrator(ir), Q(&q) { }
-
   /** \brief Integrator with scalar coefficient for caller-specified vector
       dimension.

@@ -2942,11 +2927,10 @@ public:

    sum_e eta (r_e([u]), r_e([v]))

-    where r_e is the lifting operator defined on each edge e (potentially
-    weighted by a coefficient Q). The parameter eta can be chosen to be one to
-    obtain a stable discretization. The constructor for this integrator requires
-    the finite element space because the lifting operator depends on the
-    element-wise inverse mass matrix.
+    where r_e is the lifting operator defined on each edge e. The parameter eta
+    can be chosen to be one to obtain a stable discretization. The constructor
+    for this integrator requires the finite element space because the lifting
+    operator depends on the element-wise inverse mass matrix.

    BR2 stands for the second method of Bassi and Rebay:

@@ -2969,28 +2953,14 @@ protected:
   Array<int> ipiv;
   Array<int> ipiv_offsets, Minv_offsets;

-   Coefficient *Q;
-
   Vector shape1, shape2;

   DenseMatrix R11, R12, R21, R22;
   DenseMatrix MinvR11, MinvR12, MinvR21, MinvR22;
   DenseMatrix Re, MinvRe;

-   /// Precomputes the inverses (LU factorizations) of the local mass matrices.
-   /** @a fes must be a DG space, so the mass matrix is block diagonal, and its
-       inverse can be computed locally. This is required for the computation of
-       the lifting operators @a r_e.
-   */
-   void PrecomputeMassInverse(class FiniteElementSpace &fes);
-
 public:
-   DGDiffusionBR2Integrator(class FiniteElementSpace &fes, double e = 1.0);
-   DGDiffusionBR2Integrator(class FiniteElementSpace &fes, Coefficient &Q_,
-                            double e = 1.0);
-   MFEM_DEPRECATED DGDiffusionBR2Integrator(class FiniteElementSpace *fes,
-                                            double e = 1.0);
-
+   DGDiffusionBR2Integrator(class FiniteElementSpace *fes, double e = 1.0);
   using BilinearFormIntegrator::AssembleFaceMatrix;
   virtual void AssembleFaceMatrix(const FiniteElement &el1,
                                   const FiniteElement &el2,
@@ -16,39 +16,20 @@
 namespace mfem
 {

-DGDiffusionBR2Integrator::DGDiffusionBR2Integrator(
-   FiniteElementSpace &fes, double e) : eta(e), Q(NULL)
+DGDiffusionBR2Integrator::DGDiffusionBR2Integrator(FiniteElementSpace *fes,
+                                                   double e) : eta(e)
 {
-   PrecomputeMassInverse(fes);
-}
-
-DGDiffusionBR2Integrator::DGDiffusionBR2Integrator(
-   FiniteElementSpace &fes, Coefficient &Q_, double e) : eta(e), Q(&Q_)
-{
-   PrecomputeMassInverse(fes);
-}
-
-DGDiffusionBR2Integrator::DGDiffusionBR2Integrator(
-   FiniteElementSpace *fes, double e) : eta(e), Q(NULL)
-{
-   PrecomputeMassInverse(*fes);
-}
-
-void DGDiffusionBR2Integrator::PrecomputeMassInverse(FiniteElementSpace &fes)
-{
-   MFEM_VERIFY(fes.IsDGSpace(),
-               "The BR2 integrator is only defined for DG spaces.");
   // Precompute local mass matrix inverses needed for the lifting operators
   // First compute offsets and total size needed (e.g. for mixed meshes or
   // p-refinement)
-   int nel = fes.GetNE();
+   int nel = fes->GetNE();
   Minv_offsets.SetSize(nel+1);
   ipiv_offsets.SetSize(nel+1);
   ipiv_offsets[0] = 0;
   Minv_offsets[0] = 0;
   for (int i=0; i<nel; ++i)
   {
-      int dof = fes.GetFE(i)->GetDof();
+      int dof = fes->GetFE(i)->GetDof();
      ipiv_offsets[i+1] = ipiv_offsets[i] + dof;
      Minv_offsets[i+1] = Minv_offsets[i] + dof*dof;
   }
@@ -56,7 +37,7 @@ void DGDiffusionBR2Integrator::PrecomputeMassInverse(FiniteElementSpace &fes)
 #ifdef MFEM_USE_MPI
   // When running in parallel, we also need to compute the local mass matrices
   // of face neighbor elements
-   ParFiniteElementSpace *pfes = dynamic_cast<ParFiniteElementSpace *>(&fes);
+   ParFiniteElementSpace *pfes = dynamic_cast<ParFiniteElementSpace *>(fes);
   if (pfes != NULL)
   {
      ParMesh *pmesh = pfes->GetParMesh();
@@ -83,15 +64,15 @@ void DGDiffusionBR2Integrator::PrecomputeMassInverse(FiniteElementSpace &fes)
   {
      const FiniteElement *fe = NULL;
      ElementTransformation *tr = NULL;
-      if (i < fes.GetNE())
+      if (i < fes->GetNE())
      {
-         fe = fes.GetFE(i);
-         tr = fes.GetElementTransformation(i);
+         fe = fes->GetFE(i);
+         tr = fes->GetElementTransformation(i);
      }
      else
      {
 #ifdef MFEM_USE_MPI
-         int inbr = i - fes.GetNE();
+         int inbr = i - fes->GetNE();
         fe = pfes->GetFaceNbrFE(inbr);
         tr = pfes->GetParMesh()->GetFaceNbrElementTransformation(inbr);
 #endif
@@ -170,24 +151,21 @@ void DGDiffusionBR2Integrator::AssembleFaceMatrix(
   for (int p = 0; p < ir->GetNPoints(); p++)
   {
      const IntegrationPoint &ip = ir->IntPoint(p);
-      Trans.SetAllIntPoints(&ip);
+      IntegrationPoint eip1, eip2;

-      const IntegrationPoint &eip1 = Trans.Elem1->GetIntPoint();
+      Trans.Loc1.Transform(ip, eip1);
      el1.CalcShape(eip1, shape1);
-      double q = Q ? Q->Eval(*Trans.Elem1, eip1) : 1.0;
      if (ndof2)
      {
-         const IntegrationPoint &eip2 = Trans.Elem2->GetIntPoint();
+         Trans.Loc2.Transform(ip, eip2);
         el2.CalcShape(eip2, shape2);
-         // Set coefficient value q to the average of the values on either side
-         if (Q) { q = 0.5*(q + Q->Eval(*Trans.Elem2, eip2)); }
      }
-      // Take sqrt here because
-      //    eta (r_e([u]), r_e([v])) = (sqrt(eta) r_e([u]), sqrt(eta) r_e([v]))
-      double w = sqrt((factor + 1)*eta*q)*ip.weight*Trans.Face->Weight();
-      // r_e is defined by, (r_e([u]), tau) = <[u], {tau}>, so we pick up a
-      // factor of 0.5 on interior faces from the average term.
-      if (ndof2) { w *= 0.5; }
+
+      double w = factor*sqrt(eta)*ip.weight*Trans.Face->Weight();
+      if (ndof2)
+      {
+         w /= 2;
+      }

      for (int i = 0; i < ndof1; i++)
      {
@@ -17,14 +17,14 @@ namespace mfem
 {

 template<int T_D1D = 0, int T_Q1D = 0>
-static void EAConvectionAssemble1D(const int NE,
-                                   const Array<double> &b,
-                                   const Array<double> &g,
-                                   const Vector &padata,
-                                   Vector &eadata,
-                                   const bool add,
-                                   const int d1d = 0,
-                                   const int q1d = 0)
+void EAConvectionAssemble1D(const int NE,
+                            const Array<double> &b,
+                            const Array<double> &g,
+                            const Vector &padata,
+                            Vector &eadata,
+                            const bool add,
+                            const int d1d = 0,
+                            const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -69,14 +69,14 @@ static void EAConvectionAssemble1D(const int NE,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-static void EAConvectionAssemble2D(const int NE,
-                                   const Array<double> &b,
-                                   const Array<double> &g,
-                                   const Vector &padata,
-                                   Vector &eadata,
-                                   const bool add,
-                                   const int d1d = 0,
-                                   const int q1d = 0)
+void EAConvectionAssemble2D(const int NE,
+                            const Array<double> &b,
+                            const Array<double> &g,
+                            const Vector &padata,
+                            Vector &eadata,
+                            const bool add,
+                            const int d1d = 0,
+                            const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -146,14 +146,14 @@ static void EAConvectionAssemble2D(const int NE,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-static void EAConvectionAssemble3D(const int NE,
-                                   const Array<double> &b,
-                                   const Array<double> &g,
-                                   const Vector &padata,
-                                   Vector &eadata,
-                                   const bool add,
-                                   const int d1d = 0,
-                                   const int q1d = 0)
+void EAConvectionAssemble3D(const int NE,
+                            const Array<double> &b,
+                            const Array<double> &g,
+                            const Vector &padata,
+                            Vector &eadata,
+                            const bool add,
+                            const int d1d = 0,
+                            const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -21,13 +21,13 @@ namespace mfem
 // PA Convection Integrator

 // PA Convection Assemble 2D kernel
-static void PAConvectionSetup2D(const int NQ,
-                                const int NE,
-                                const Array<double> &w,
-                                const Vector &j,
-                                const Vector &vel,
-                                const double alpha,
-                                Vector &op)
+void PAConvectionSetup2D(const int NQ,
+                         const int NE,
+                         const Array<double> &w,
+                         const Vector &j,
+                         const Vector &vel,
+                         const double alpha,
+                         Vector &op)
 {
   constexpr int DIM = 2;

@@ -60,13 +60,13 @@ static void PAConvectionSetup2D(const int NQ,
 }

 // PA Convection Assemble 3D kernel
-static void PAConvectionSetup3D(const int NQ,
-                                const int NE,
-                                const Array<double> &w,
-                                const Vector &j,
-                                const Vector &vel,
-                                const double alpha,
-                                Vector &op)
+void PAConvectionSetup3D(const int NQ,
+                         const int NE,
+                         const Array<double> &w,
+                         const Vector &j,
+                         const Vector &vel,
+                         const double alpha,
+                         Vector &op)
 {
   constexpr int DIM = 3;
   constexpr int SDIM = DIM;
@@ -135,7 +135,7 @@ static void PAConvectionSetup(const int dim,
 }

 // PA Convection Apply 2D kernel
-template<int T_D1D = 0, int T_Q1D = 0> static
+template<int T_D1D = 0, int T_Q1D = 0>
 void PAConvectionApply2D(const int ne,
                         const Array<double> &b,
                         const Array<double> &g,
@@ -254,7 +254,7 @@ void PAConvectionApply2D(const int ne,
 }

 // Optimized PA Convection Apply 2D kernel
-template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0> static
+template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
 void SmemPAConvectionApply2D(const int ne,
                             const Array<double> &b,
                             const Array<double> &g,
@@ -382,7 +382,7 @@ void SmemPAConvectionApply2D(const int ne,
 }

 // PA Convection Apply 3D kernel
-template<int T_D1D = 0, int T_Q1D = 0> static
+template<int T_D1D = 0, int T_Q1D = 0>
 void PAConvectionApply3D(const int ne,
                         const Array<double> &b,
                         const Array<double> &g,
@@ -563,7 +563,7 @@ void PAConvectionApply3D(const int ne,
 }

 // Optimized PA Convection Apply 3D kernel
-template<int T_D1D = 0, int T_Q1D = 0> static
+template<int T_D1D = 0, int T_Q1D = 0>
 void SmemPAConvectionApply3D(const int ne,
                             const Array<double> &b,
                             const Array<double> &g,
@@ -768,8 +768,8 @@ void SmemPAConvectionApply3D(const int ne,

 void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
-   const MemoryType mt = (memory_type == MemoryType::DEFAULT) ?
-                         Device::GetDeviceMemoryType() : memory_type;
+   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
+                         Device::GetDeviceMemoryType() : pa_mt;
   // Assumes tensor-product elements
   Mesh *mesh = fes.GetMesh();
   const FiniteElement &el = *fes.GetFE(0);
@@ -16,12 +16,12 @@
 namespace mfem
 {

-static void EADGTraceAssemble1DInt(const int NF,
-                                   const Array<double> &basis,
-                                   const Vector &padata,
-                                   Vector &eadata_int,
-                                   Vector &eadata_ext,
-                                   const bool add)
+void EADGTraceAssemble1DInt(const int NF,
+                            const Array<double> &basis,
+                            const Vector &padata,
+                            Vector &eadata_int,
+                            Vector &eadata_ext,
+                            const bool add)
 {
   auto D = Reshape(padata.Read(), 2, 2, NF);
   auto A_int = Reshape(eadata_int.ReadWrite(), 2, NF);
@@ -50,11 +50,11 @@ static void EADGTraceAssemble1DInt(const int NF,
   });
 }

-static void EADGTraceAssemble1DBdr(const int NF,
-                                   const Array<double> &basis,
-                                   const Vector &padata,
-                                   Vector &eadata_bdr,
-                                   const bool add)
+void EADGTraceAssemble1DBdr(const int NF,
+                            const Array<double> &basis,
+                            const Vector &padata,
+                            Vector &eadata_bdr,
+                            const bool add)
 {
   auto D = Reshape(padata.Read(), 2, 2, NF);
   auto A_bdr = Reshape(eadata_bdr.ReadWrite(), NF);
@@ -72,14 +72,14 @@ static void EADGTraceAssemble1DBdr(const int NF,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-static void EADGTraceAssemble2DInt(const int NF,
-                                   const Array<double> &basis,
-                                   const Vector &padata,
-                                   Vector &eadata_int,
-                                   Vector &eadata_ext,
-                                   const bool add,
-                                   const int d1d = 0,
-                                   const int q1d = 0)
+void EADGTraceAssemble2DInt(const int NF,
+                            const Array<double> &basis,
+                            const Vector &padata,
+                            Vector &eadata_int,
+                            Vector &eadata_ext,
+                            const bool add,
+                            const int d1d = 0,
+                            const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -128,13 +128,13 @@ static void EADGTraceAssemble2DInt(const int NF,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-static void EADGTraceAssemble2DBdr(const int NF,
-                                   const Array<double> &basis,
-                                   const Vector &padata,
-                                   Vector &eadata_bdr,
-                                   const bool add,
-                                   const int d1d = 0,
-                                   const int q1d = 0)
+void EADGTraceAssemble2DBdr(const int NF,
+                            const Array<double> &basis,
+                            const Vector &padata,
+                            Vector &eadata_bdr,
+                            const bool add,
+                            const int d1d = 0,
+                            const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -170,14 +170,14 @@ static void EADGTraceAssemble2DBdr(const int NF,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-static void EADGTraceAssemble3DInt(const int NF,
-                                   const Array<double> &basis,
-                                   const Vector &padata,
-                                   Vector &eadata_int,
-                                   Vector &eadata_ext,
-                                   const bool add,
-                                   const int d1d = 0,
-                                   const int q1d = 0)
+void EADGTraceAssemble3DInt(const int NF,
+                            const Array<double> &basis,
+                            const Vector &padata,
+                            Vector &eadata_int,
+                            Vector &eadata_ext,
+                            const bool add,
+                            const int d1d = 0,
+                            const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -268,13 +268,13 @@ static void EADGTraceAssemble3DInt(const int NF,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-static void EADGTraceAssemble3DBdr(const int NF,
-                                   const Array<double> &basis,
-                                   const Vector &padata,
-                                   Vector &eadata_bdr,
-                                   const bool add,
-                                   const int d1d = 0,
-                                   const int q1d = 0)
+void EADGTraceAssemble3DBdr(const int NF,
+                            const Array<double> &basis,
+                            const Vector &padata,
+                            Vector &eadata_bdr,
+                            const bool add,
+                            const int d1d = 0,
+                            const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -19,16 +19,16 @@ using namespace std;
 namespace mfem
 {
 // PA DG Trace Integrator
-static void PADGTraceSetup2D(const int Q1D,
-                             const int NF,
-                             const Array<double> &w,
-                             const Vector &det,
-                             const Vector &nor,
-                             const Vector &rho,
-                             const Vector &vel,
-                             const double alpha,
-                             const double beta,
-                             Vector &op)
+void PADGTraceSetup2D(const int Q1D,
+                      const int NF,
+                      const Array<double> &w,
+                      const Vector &det,
+                      const Vector &nor,
+                      const Vector &rho,
+                      const Vector &vel,
+                      const double alpha,
+                      const double beta,
+                      Vector &op)
 {
   const int VDIM = 2;

@@ -61,16 +61,16 @@ static void PADGTraceSetup2D(const int Q1D,
   });
 }

-static void PADGTraceSetup3D(const int Q1D,
-                             const int NF,
-                             const Array<double> &w,
-                             const Vector &det,
-                             const Vector &nor,
-                             const Vector &rho,
-                             const Vector &vel,
-                             const double alpha,
-                             const double beta,
-                             Vector &op)
+void PADGTraceSetup3D(const int Q1D,
+                      const int NF,
+                      const Array<double> &w,
+                      const Vector &det,
+                      const Vector &nor,
+                      const Vector &rho,
+                      const Vector &vel,
+                      const double alpha,
+                      const double beta,
+                      Vector &op)
 {
   const int VDIM = 3;

@@ -301,7 +301,7 @@ void DGTraceIntegrator::AssemblePABoundaryFaces(const FiniteElementSpace& fes)
 }

 // PA DGTrace Apply 2D kernel for Gauss-Lobatto/Bernstein
-template<int T_D1D = 0, int T_Q1D = 0> static
+template<int T_D1D = 0, int T_Q1D = 0>
 void PADGTraceApply2D(const int NF,
                      const Array<double> &b,
                      const Array<double> &bt,
@@ -392,7 +392,7 @@ void PADGTraceApply2D(const int NF,
 }

 // PA DGTrace Apply 3D kernel for Gauss-Lobatto/Bernstein
-template<int T_D1D = 0, int T_Q1D = 0> static
+template<int T_D1D = 0, int T_Q1D = 0>
 void PADGTraceApply3D(const int NF,
                      const Array<double> &b,
                      const Array<double> &bt,
@@ -537,7 +537,7 @@ void PADGTraceApply3D(const int NF,
 }

 // Optimized PA DGTrace Apply 3D kernel for Gauss-Lobatto/Bernstein
-template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0> static
+template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
 void SmemPADGTraceApply3D(const int NF,
                          const Array<double> &b,
                          const Array<double> &bt,
@@ -701,7 +701,7 @@ static void PADGTraceApply(const int dim,
 }

 // PA DGTrace Apply 2D kernel for Gauss-Lobatto/Bernstein
-template<int T_D1D = 0, int T_Q1D = 0> static
+template<int T_D1D = 0, int T_Q1D = 0>
 void PADGTraceApplyTranspose2D(const int NF,
                               const Array<double> &b,
                               const Array<double> &bt,
@@ -797,7 +797,7 @@ void PADGTraceApplyTranspose2D(const int NF,
 }

 // PA DGTrace Apply Transpose 3D kernel for Gauss-Lobatto/Bernstein
-template<int T_D1D = 0, int T_Q1D = 0> static
+template<int T_D1D = 0, int T_Q1D = 0>
 void PADGTraceApplyTranspose3D(const int NF,
                               const Array<double> &b,
                               const Array<double> &bt,
@@ -953,7 +953,7 @@ void PADGTraceApplyTranspose3D(const int NF,
 }

 // Optimized PA DGTrace Apply Transpose 3D kernel for Gauss-Lobatto/Bernstein
-template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0> static
+template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
 void SmemPADGTraceApplyTranspose3D(const int NF,
                                   const Array<double> &b,
                                   const Array<double> &bt,
@@ -17,14 +17,14 @@ namespace mfem
 {

 template<int T_D1D = 0, int T_Q1D = 0>
-static void EADiffusionAssemble1D(const int NE,
-                                  const Array<double> &b,
-                                  const Array<double> &g,
-                                  const Vector &padata,
-                                  Vector &eadata,
-                                  const bool add,
-                                  const int d1d = 0,
-                                  const int q1d = 0)
+void EADiffusionAssemble1D(const int NE,
+                           const Array<double> &b,
+                           const Array<double> &g,
+                           const Vector &padata,
+                           Vector &eadata,
+                           const bool add,
+                           const int d1d = 0,
+                           const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -68,14 +68,14 @@ static void EADiffusionAssemble1D(const int NE,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-static void EADiffusionAssemble2D(const int NE,
-                                  const Array<double> &b,
-                                  const Array<double> &g,
-                                  const Vector &padata,
-                                  Vector &eadata,
-                                  const bool add,
-                                  const int d1d = 0,
-                                  const int q1d = 0)
+void EADiffusionAssemble2D(const int NE,
+                           const Array<double> &b,
+                           const Array<double> &g,
+                           const Vector &padata,
+                           Vector &eadata,
+                           const bool add,
+                           const int d1d = 0,
+                           const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -145,14 +145,14 @@ static void EADiffusionAssemble2D(const int NE,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-static void EADiffusionAssemble3D(const int NE,
-                                  const Array<double> &b,
-                                  const Array<double> &g,
-                                  const Vector &padata,
-                                  Vector &eadata,
-                                  const bool add,
-                                  const int d1d = 0,
-                                  const int q1d = 0)
+void EADiffusionAssemble3D(const int NE,
+                           const Array<double> &b,
+                           const Array<double> &g,
+                           const Vector &padata,
+                           Vector &eadata,
+                           const bool add,
+                           const int d1d = 0,
+                           const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -9,45 +9,18 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.

-#define MFEM_DEBUG_COLOR 87
-#include "../general/debug.hpp"
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
 #include "ceed/diffusion.hpp"
-#include <string>
-
-#define MFEM_NVTX_COLOR Olive
-#include "../general/nvtx.hpp"

 using namespace std;

 namespace mfem
 {

-void NDK_PADiffusionAssembleDiagonal(const int dim,
-                                     const int D1D,
-                                     const int Q1D,
-                                     const int NE,
-                                     const bool symm,
-                                     const FiniteElementSpace *fes,
-                                     const DofToQuad *maps,
-                                     const Vector &D,
-                                     Vector &Y);
-
 // PA Diffusion Integrator

-void NDK_PADiffusionApply(const int dim,
-                          const int D1D,
-                          const int Q1D,
-                          const int NE,
-                          const Vector &CoG,
-                          const FiniteElementSpace *fes,
-                          const DofToQuad *maps,
-                          const Vector &D,
-                          const Vector &X,
-                          Vector &Y);
-
 // OCCA 2D Assemble kernel
 #ifdef MFEM_USE_OCCA
 static void OccaPADiffusionSetup2D(const int D1D,
@@ -152,7 +125,7 @@ void PADiffusionSetup2D<2>(const int Q1D,
               D(qx,qy,0,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
               D(qx,qy,1,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
               D(qx,qy,2,e) = w_detJ * (symmetric ? (-J21*R12 + J11*R22) :
-                                        (J22*R12 - J12*R22)); // 2,2 or 1,2
+               (J22*R12 - J12*R22)); // 2,2 or 1,2
               if (!symmetric)
               {
                  D(qx,qy,3,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
@@ -376,115 +349,10 @@ static void PADiffusionSetup(const int dim,
   }
 }

-////////////////////////////////////////////////////////////////////////////////
-template<typename T> MFEM_HOST_DEVICE inline
-void HouseholderReflect(T *A, const T *v,
-                        const T b, const int m, const int n,
-                        const int row, const int col)
-{
-   for (int j = 0; j < n; j++)
-   {
-      T w = A[0*row + j*col];
-      for (int i = 1; i < m; i++) { w += v[i] * A[i*row + j*col]; }
-      A[0*row + j*col] -= b * w;
-      for (int i = 1; i < m; i++) { A[i*row + j*col] -= b * w * v[i]; }
-   }
-}
-
-template<int Q1D, typename T> MFEM_HOST_DEVICE inline
-void HouseholderApplyQ(T *A, const T *Q, const T *tau,
-                       const int k, const int row, const int col)
-{
-   T v[Q1D];
-   for (int ii=0; ii<k; ii++)
-   {
-      const int i = k-1-ii;
-      for (int j = i+1; j < Q1D; j++) { v[j] = Q[j*k+i]; }
-      // Apply Householder reflector (I - tau v v^T) coG^T
-      HouseholderReflect(&A[i*row], &v[i], tau[i], Q1D-i, Q1D, row, col);
-   }
-}
-
-template<int D1D, int Q1D, typename T> MFEM_HOST_DEVICE inline
-void QRFactorization(T *mat, T *tau)
-{
-   T v[Q1D];
-   DeviceMatrix B(mat, D1D, Q1D);
-   for (int i = 0; i < D1D; i++)
-   {
-      // Calculate Householder vector, magnitude
-      T sigma = 0.0;
-      v[i] = B(i,i);
-      for (int j = i + 1; j < Q1D; j++)
-      {
-         v[j] = B(i,j);
-         sigma += v[j] * v[j];
-      }
-      T norm = std::sqrt(v[i]*v[i] + sigma); // norm of v[i:m]
-      T Rii = -copysign(norm, v[i]);
-      v[i] -= Rii;
-      // norm of v[i:m] after modification above and scaling below
-      //   norm = sqrt(v[i]*v[i] + sigma) / v[i];
-      //   tau = 2 / (norm*norm)
-      tau[i] = 2 * v[i]*v[i] / (v[i]*v[i] + sigma);
-      for (int j=i+1; j<Q1D; j++) { v[j] /= v[i]; }
-      // Apply Householder reflector to lower right panel
-      HouseholderReflect(&mat[i*D1D+i+1], &v[i], tau[i],
-                         Q1D-i, D1D-i-1, D1D, 1);
-      // Save v
-      B(i,i) = Rii;
-      for (int j=i+1; j<Q1D; j++) { B(i,j) = v[j]; }
-   }
-}
-
-template<int D1D, int Q1D>
-void GetCollocatedGrad(const ConstDeviceMatrix &b,
-                       const ConstDeviceMatrix &g,
-                       const DeviceMatrix &CoG)
-{
-   double tau[Q1D];
-   double B1d[Q1D*D1D];
-   double G1d[Q1D*D1D];
-   DeviceMatrix B(B1d, D1D, Q1D);
-   DeviceMatrix G(G1d, D1D, Q1D);
-
-   for (int d = 0; d < D1D; d++)
-   {
-      for (int q = 0; q < Q1D; q++)
-      {
-         B(d,q) = b(q,d);
-         G(d,q) = g(q,d);
-      }
-   }
-   QRFactorization<D1D,Q1D>(B1d, tau);
-   // Apply Rinv, colograd1d = grad1d Rinv
-   for (int i = 0; i < Q1D; i++)
-   {
-      CoG(0,i) = G(0,i)/B(0,0);
-      for (int j = 1; j < D1D; j++)
-      {
-         CoG(j,i) = G(j,i);
-         for (int k = 0; k < j; k++) { CoG(j,i) -= B(j,k)*CoG(k,i); }
-         CoG(j,i) /= B(j,j);
-      }
-      for (int j = D1D; j < Q1D; j++) { CoG(j,i) = 0.0; }
-   }
-   // Apply Qtranspose, colograd = colograd Qtranspose
-   HouseholderApplyQ<Q1D>((double*)CoG, B1d, tau, D1D, 1, Q1D);
-}
-
-////////////////////////////////////////////////////////////////////////////////
 void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
-   const MemoryType mt = (memory_type == MemoryType::DEFAULT) ?
-                         Device::GetDeviceMemoryType() : memory_type;
-
-   // If device options allow fast kernels, set the action type to L2L
-   action_type =
-      (Device::FastKernelsEnabled())?
-      ActionType::L2L: // all fast kernel are L2L
-      ActionType::E2E; // default is E2E
-
+   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
+                         Device::GetDeviceMemoryType() : pa_mt;
   // Assuming the same element type
   fespace = &fes;
   Mesh *mesh = fes.GetMesh();
@@ -497,7 +365,6 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
      MFEM_VERIFY(!VQ && !MQ && !SMQ,
                  "Only scalar coefficient supported for DiffusionIntegrator"
                  " with libCEED");
-      action_type = ActionType::L2L;
      ceedOp = new ceed::PADiffusionIntegrator(fes, *ir, Q);
      return;
   }
@@ -623,54 +490,20 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
         }
      }
   }
-
-   if (Device::FastKernelsEnabled())
-   {
-      NVTX("CoG");
-      const int D1D = dofs1D;
-      const int Q1D = quad1D;
-      const int id = (D1D << 4) | Q1D;
-
-      CoG.SetSize(Q1D*Q1D);
-      CoG.UseDevice(true);
-      assert(CoG.UseDevice());
-
-      void (*KoG)(const ConstDeviceMatrix &b,
-                  const ConstDeviceMatrix &g,
-                  const DeviceMatrix &CoG) = nullptr;
-
-      switch (id) // orders 1~8
-      {
-         case 0x23: KoG=GetCollocatedGrad<2,3>; break; // 1
-         case 0x34: KoG=GetCollocatedGrad<3,4>; break; // 2
-         case 0x45: KoG=GetCollocatedGrad<4,5>; break; // 3
-         case 0x56: KoG=GetCollocatedGrad<5,6>; break; // 4
-         case 0x67: KoG=GetCollocatedGrad<6,7>; break; // 5
-         case 0x78: KoG=GetCollocatedGrad<7,8>; break; // 6
-         //case 0x89: KoG=GetCollocatedGrad<8,9>; break; // 7
-         //case 0x9A: KoG=GetCollocatedGrad<9,10>; break; // 8
-         default: MFEM_ABORT("Unknown kernel 0x" << std::hex << id << std::dec);
-      }
-
-      KoG(ConstDeviceMatrix(maps->B.HostRead(),Q1D,D1D),
-          ConstDeviceMatrix(maps->G.HostRead(),Q1D,D1D),
-          DeviceMatrix(CoG.HostReadWrite(),Q1D,Q1D));
-   }
-
   pa_data.SetSize((symmetric ? symmDims : MQfullDim) * nq * ne, mt);
   PADiffusionSetup(dim, sdim, dofs1D, quad1D, coeffDim, ne, ir->GetWeights(),
                    geom->J, coeff, pa_data);
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-static void PADiffusionDiagonal2D(const int NE,
-                                  const bool symmetric,
-                                  const Array<double> &b,
-                                  const Array<double> &g,
-                                  const Vector &d,
-                                  Vector &y,
-                                  const int d1d = 0,
-                                  const int q1d = 0)
+void PADiffusionDiagonal2D(const int NE,
+                           const bool symmetric,
+                           const Array<double> &b,
+                           const Array<double> &g,
+                           const Vector &d,
+                           Vector &y,
+                           const int d1d = 0,
+                           const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -729,14 +562,14 @@ static void PADiffusionDiagonal2D(const int NE,

 // Shared memory PA Diffusion Diagonal 2D kernel
 template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
-static void SmemPADiffusionDiagonal2D(const int NE,
-                                      const bool symmetric,
-                                      const Array<double> &b_,
-                                      const Array<double> &g_,
-                                      const Vector &d_,
-                                      Vector &y_,
-                                      const int d1d = 0,
-                                      const int q1d = 0)
+void SmemPADiffusionDiagonal2D(const int NE,
+                               const bool symmetric,
+                               const Array<double> &b_,
+                               const Array<double> &g_,
+                               const Vector &d_,
+                               Vector &y_,
+                               const int d1d = 0,
+                               const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -823,14 +656,14 @@ static void SmemPADiffusionDiagonal2D(const int NE,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-static void PADiffusionDiagonal3D(const int NE,
-                                  const bool symmetric,
-                                  const Array<double> &b,
-                                  const Array<double> &g,
-                                  const Vector &d,
-                                  Vector &y,
-                                  const int d1d = 0,
-                                  const int q1d = 0)
+void PADiffusionDiagonal3D(const int NE,
+                           const bool symmetric,
+                           const Array<double> &b,
+                           const Array<double> &g,
+                           const Vector &d,
+                           Vector &y,
+                           const int d1d = 0,
+                           const int q1d = 0)
 {
   constexpr int DIM = 3;
   const int D1D = T_D1D ? T_D1D : d1d;
@@ -924,14 +757,14 @@ static void PADiffusionDiagonal3D(const int NE,

 // Shared memory PA Diffusion Diagonal 3D kernel
 template<int T_D1D = 0, int T_Q1D = 0>
-static void SmemPADiffusionDiagonal3D(const int NE,
-                                      const bool symmetric,
-                                      const Array<double> &b_,
-                                      const Array<double> &g_,
-                                      const Vector &d_,
-                                      Vector &y_,
-                                      const int d1d = 0,
-                                      const int q1d = 0)
+void SmemPADiffusionDiagonal3D(const int NE,
+                               const bool symmetric,
+                               const Array<double> &b_,
+                               const Array<double> &g_,
+                               const Vector &d_,
+                               Vector &y_,
+                               const int d1d = 0,
+                               const int q1d = 0)
 {
   constexpr int DIM = 3;
   const int D1D = T_D1D ? T_D1D : d1d;
@@ -1036,7 +869,6 @@ static void SmemPADiffusionDiagonal3D(const int NE,
                  }
               }
            }
-            MFEM_SYNC_THREAD;
         }
      }
   });
@@ -1071,11 +903,9 @@ static void PADiffusionAssembleDiagonal(const int dim,
   {
      switch ((D1D << 4 ) | Q1D)
      {
-         case 0x22: return SmemPADiffusionDiagonal3D<2,2>(NE,symm,B,G,D,Y);
         case 0x23: return SmemPADiffusionDiagonal3D<2,3>(NE,symm,B,G,D,Y);
         case 0x34: return SmemPADiffusionDiagonal3D<3,4>(NE,symm,B,G,D,Y);
         case 0x45: return SmemPADiffusionDiagonal3D<4,5>(NE,symm,B,G,D,Y);
-         case 0x46: return SmemPADiffusionDiagonal3D<4,6>(NE,symm,B,G,D,Y);
         case 0x56: return SmemPADiffusionDiagonal3D<5,6>(NE,symm,B,G,D,Y);
         case 0x67: return SmemPADiffusionDiagonal3D<6,7>(NE,symm,B,G,D,Y);
         case 0x78: return SmemPADiffusionDiagonal3D<7,8>(NE,symm,B,G,D,Y);
@@ -1093,11 +923,6 @@ void DiffusionIntegrator::AssembleDiagonalPA(Vector &diag)
   {
      ceedOp->GetDiagonal(diag);
   }
-   else if (Device::FastKernelsEnabled())
-   {
-      NDK_PADiffusionAssembleDiagonal(dim, dofs1D, quad1D, ne, symmetric,
-                                      fespace, maps, pa_data, diag);
-   }
   else
   {
      if (pa_data.Size()==0) { AssemblePA(*fespace); }
@@ -1209,17 +1034,17 @@ static void OccaPADiffusionApply3D(const int D1D,

 // PA Diffusion Apply 2D kernel
 template<int T_D1D = 0, int T_Q1D = 0>
-static void PADiffusionApply2D(const int NE,
-                               const bool symmetric,
-                               const Array<double> &b_,
-                               const Array<double> &g_,
-                               const Array<double> &bt_,
-                               const Array<double> &gt_,
-                               const Vector &d_,
-                               const Vector &x_,
-                               Vector &y_,
-                               const int d1d = 0,
-                               const int q1d = 0)
+void PADiffusionApply2D(const int NE,
+                        const bool symmetric,
+                        const Array<double> &b_,
+                        const Array<double> &g_,
+                        const Array<double> &bt_,
+                        const Array<double> &gt_,
+                        const Vector &d_,
+                        const Vector &x_,
+                        Vector &y_,
+                        const int d1d = 0,
+                        const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -1331,15 +1156,15 @@ static void PADiffusionApply2D(const int NE,

 // Shared memory PA Diffusion Apply 2D kernel
 template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
-static void SmemPADiffusionApply2D(const int NE,
-                                   const bool symmetric,
-                                   const Array<double> &b_,
-                                   const Array<double> &g_,
-                                   const Vector &d_,
-                                   const Vector &x_,
-                                   Vector &y_,
-                                   const int d1d = 0,
-                                   const int q1d = 0)
+void SmemPADiffusionApply2D(const int NE,
+                            const bool symmetric,
+                            const Array<double> &b_,
+                            const Array<double> &g_,
+                            const Vector &d_,
+                            const Vector &x_,
+                            Vector &y_,
+                            const int d1d = 0,
+                            const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -1353,7 +1178,6 @@ static void SmemPADiffusionApply2D(const int NE,
   auto D = Reshape(d_.Read(), Q1D*Q1D, symmetric ? 3 : 4, NE);
   auto x = Reshape(x_.Read(), D1D, D1D, NE);
   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
-
   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
   {
      const int tidz = MFEM_THREAD_ID(z);
@@ -1394,7 +1218,6 @@ static void SmemPADiffusionApply2D(const int NE,
         }
      }
      MFEM_SYNC_THREAD;
-
      MFEM_FOREACH_THREAD(dy,y,D1D)
      {
         MFEM_FOREACH_THREAD(qx,x,Q1D)
@@ -1491,16 +1314,16 @@ static void SmemPADiffusionApply2D(const int NE,

 // PA Diffusion Apply 3D kernel
 template<int T_D1D = 0, int T_Q1D = 0>
-static void PADiffusionApply3D(const int NE,
-                               const bool symmetric,
-                               const Array<double> &b,
-                               const Array<double> &g,
-                               const Array<double> &bt,
-                               const Array<double> &gt,
-                               const Vector &d_,
-                               const Vector &x_,
-                               Vector &y_,
-                               int d1d = 0, int q1d = 0)
+void PADiffusionApply3D(const int NE,
+                        const bool symmetric,
+                        const Array<double> &b,
+                        const Array<double> &g,
+                        const Array<double> &bt,
+                        const Array<double> &gt,
+                        const Vector &d_,
+                        const Vector &x_,
+                        Vector &y_,
+                        int d1d = 0, int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -1710,17 +1533,16 @@ static MFEM_HOST_DEVICE inline double sign(const int q, const int d)
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-static void SmemPADiffusionApply3D(const int NE,
-                                   const bool symmetric,
-                                   const Array<double> &b_,
-                                   const Array<double> &g_,
-                                   const Vector &d_,
-                                   const Vector &x_,
-                                   Vector &y_,
-                                   const int d1d = 0,
-                                   const int q1d = 0)
+void SmemPADiffusionApply3D(const int NE,
+                            const bool symmetric,
+                            const Array<double> &b_,
+                            const Array<double> &g_,
+                            const Vector &d_,
+                            const Vector &x_,
+                            Vector &y_,
+                            const int d1d = 0,
+                            const int q1d = 0)
 {
-   MFEM_NVTX;
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
   constexpr int M1Q = T_Q1D ? T_Q1D : MAX_Q1D;
@@ -1732,7 +1554,7 @@ static void SmemPADiffusionApply3D(const int NE,
   auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, symmetric ? 6 : 9, NE);
   auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
   {
      const int D1D = T_D1D ? T_D1D : d1d;
      const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -1761,103 +1583,118 @@ static void SmemPADiffusionApply3D(const int NE,
      double (*QDD0)[MD1][MD1] = (double (*)[MD1][MD1]) (sm0+0);
      double (*QDD1)[MD1][MD1] = (double (*)[MD1][MD1]) (sm0+1);
      double (*QDD2)[MD1][MD1] = (double (*)[MD1][MD1]) (sm0+2);
-      MFEM_FOREACH_THREAD(dz,z,D1D)
+      MFEM_FOREACH_THREAD(dy,y,D1D)
      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
+         MFEM_FOREACH_THREAD(dx,x,D1D)
         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
            {
               X[dz][dy][dx] = x(dx,dy,dz,e);
            }
         }
-      }
-      MFEM_SYNC_THREAD;
-      if (MFEM_THREAD_ID(z) == 0)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               const int i = qi(qx,dy,Q1D);
-               const int j = dj(qx,dy,D1D);
-               const int k = qk(qx,dy,Q1D);
-               const int l = dl(qx,dy,D1D);
-               B[i][j] = b(qx,dy);
-               G[k][l] = g(qx,dy) * sign(qx,dy);
-            }
+            const int i = qi(qx,dy,Q1D);
+            const int j = dj(qx,dy,D1D);
+            const int k = qk(qx,dy,Q1D);
+            const int l = dl(qx,dy,D1D);
+            B[i][j] = b(qx,dy);
+            G[k][l] = g(qx,dy) * sign(qx,dy);
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
+      MFEM_FOREACH_THREAD(dy,y,D1D)
      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            double u[D1D], v[D1D];
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; dz++) { u[dz] = v[dz] = 0.0; }
+            MFEM_UNROLL(MD1)
+            for (int dx = 0; dx < D1D; ++dx)
            {
-               double u = 0.0, v = 0.0;
-               MFEM_UNROLL(MD1)
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  const int i = qi(qx,dx,Q1D);
-                  const int j = dj(qx,dx,D1D);
-                  const int k = qk(qx,dx,Q1D);
-                  const int l = dl(qx,dx,D1D);
-                  const double s = sign(qx,dx);
-                  const double coords = X[dz][dy][dx];
-                  u += coords * B[i][j];
-                  v += coords * G[k][l] * s;
-               }
-               DDQ0[dz][dy][qx] = u;
-               DDQ1[dz][dy][qx] = v;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double u = 0.0, v = 0.0, w = 0.0;
-               MFEM_UNROLL(MD1)
-               for (int dy = 0; dy < D1D; ++dy)
-               {
-                  const int i = qi(qy,dy,Q1D);
-                  const int j = dj(qy,dy,D1D);
-                  const int k = qk(qy,dy,Q1D);
-                  const int l = dl(qy,dy,D1D);
-                  const double s = sign(qy,dy);
-                  u += DDQ1[dz][dy][qx] * B[i][j];
-                  v += DDQ0[dz][dy][qx] * G[k][l] * s;
-                  w += DDQ0[dz][dy][qx] * B[i][j];
-               }
-               DQQ0[dz][qy][qx] = u;
-               DQQ1[dz][qy][qx] = v;
-               DQQ2[dz][qy][qx] = w;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double u = 0.0, v = 0.0, w = 0.0;
+               const int i = qi(qx,dx,Q1D);
+               const int j = dj(qx,dx,D1D);
+               const int k = qk(qx,dx,Q1D);
+               const int l = dl(qx,dx,D1D);
+               const double s = sign(qx,dx);
               MFEM_UNROLL(MD1)
               for (int dz = 0; dz < D1D; ++dz)
+               {
+                  const double coords = X[dz][dy][dx];
+                  u[dz] += coords * B[i][j];
+                  v[dz] += coords * G[k][l] * s;
+               }
+            }
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               DDQ0[dz][dy][qx] = u[dz];
+               DDQ1[dz][dy][qx] = v[dz];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            double u[D1D], v[D1D], w[D1D];
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; dz++) { u[dz] = v[dz] = w[dz] = 0.0; }
+            MFEM_UNROLL(MD1)
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               const int i = qi(qy,dy,Q1D);
+               const int j = dj(qy,dy,D1D);
+               const int k = qk(qy,dy,Q1D);
+               const int l = dl(qy,dy,D1D);
+               const double s = sign(qy,dy);
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < D1D; dz++)
+               {
+                  u[dz] += DDQ1[dz][dy][qx] * B[i][j];
+                  v[dz] += DDQ0[dz][dy][qx] * G[k][l] * s;
+                  w[dz] += DDQ0[dz][dy][qx] * B[i][j];
+               }
+            }
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; dz++)
+            {
+               DQQ0[dz][qy][qx] = u[dz];
+               DQQ1[dz][qy][qx] = v[dz];
+               DQQ2[dz][qy][qx] = w[dz];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            double u[Q1D], v[Q1D], w[Q1D];
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; qz++) { u[qz] = v[qz] = w[qz] = 0.0; }
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; qz++)
               {
                  const int i = qi(qz,dz,Q1D);
                  const int j = dj(qz,dz,D1D);
                  const int k = qk(qz,dz,Q1D);
                  const int l = dl(qz,dz,D1D);
                  const double s = sign(qz,dz);
-                  u += DQQ0[dz][qy][qx] * B[i][j];
-                  v += DQQ1[dz][qy][qx] * B[i][j];
-                  w += DQQ2[dz][qy][qx] * G[k][l] * s;
+                  u[qz] += DQQ0[dz][qy][qx] * B[i][j];
+                  v[qz] += DQQ1[dz][qy][qx] * B[i][j];
+                  w[qz] += DQQ2[dz][qy][qx] * G[k][l] * s;
               }
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; qz++)
+            {
               const double O11 = d(qx,qy,qz,0,e);
               const double O12 = d(qx,qy,qz,1,e);
               const double O13 = d(qx,qy,qz,2,e);
@@ -1867,9 +1704,9 @@ static void SmemPADiffusionApply3D(const int NE,
               const double O31 = symmetric ? O13 : d(qx,qy,qz,6,e);
               const double O32 = symmetric ? O23 : d(qx,qy,qz,7,e);
               const double O33 = symmetric ? d(qx,qy,qz,5,e) : d(qx,qy,qz,8,e);
-               const double gX = u;
-               const double gY = v;
-               const double gZ = w;
+               const double gX = u[qz];
+               const double gY = v[qz];
+               const double gZ = w[qz];
               QQQ0[qz][qy][qx] = (O11*gX) + (O12*gY) + (O13*gZ);
               QQQ1[qz][qy][qx] = (O21*gX) + (O22*gY) + (O23*gZ);
               QQQ2[qz][qy][qx] = (O31*gX) + (O32*gY) + (O33*gZ);
@@ -1877,98 +1714,115 @@ static void SmemPADiffusionApply3D(const int NE,
         }
      }
      MFEM_SYNC_THREAD;
-      if (MFEM_THREAD_ID(z) == 0)
+      MFEM_FOREACH_THREAD(d,y,D1D)
      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
+         MFEM_FOREACH_THREAD(q,x,Q1D)
         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               const int i = qi(q,d,Q1D);
-               const int j = dj(q,d,D1D);
-               const int k = qk(q,d,Q1D);
-               const int l = dl(q,d,D1D);
-               Bt[j][i] = b(q,d);
-               Gt[l][k] = g(q,d) * sign(q,d);
-            }
+            const int i = qi(q,d,Q1D);
+            const int j = dj(q,d,D1D);
+            const int k = qk(q,d,Q1D);
+            const int l = dl(q,d,D1D);
+            Bt[j][i] = b(q,d);
+            Gt[l][k] = g(q,d) * sign(q,d);
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         MFEM_FOREACH_THREAD(dx,x,D1D)
         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
+            double u[Q1D], v[Q1D], w[Q1D];
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz) { u[qz] = v[qz] = w[qz] = 0.0; }
+            MFEM_UNROLL(MQ1)
+            for (int qx = 0; qx < Q1D; ++qx)
            {
-               double u = 0.0, v = 0.0, w = 0.0;
-               MFEM_UNROLL(MQ1)
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  const int i = qi(qx,dx,Q1D);
-                  const int j = dj(qx,dx,D1D);
-                  const int k = qk(qx,dx,Q1D);
-                  const int l = dl(qx,dx,D1D);
-                  const double s = sign(qx,dx);
-                  u += QQQ0[qz][qy][qx] * Gt[l][k] * s;
-                  v += QQQ1[qz][qy][qx] * Bt[j][i];
-                  w += QQQ2[qz][qy][qx] * Bt[j][i];
-               }
-               QQD0[qz][qy][dx] = u;
-               QQD1[qz][qy][dx] = v;
-               QQD2[qz][qy][dx] = w;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               double u = 0.0, v = 0.0, w = 0.0;
-               MFEM_UNROLL(Q1D)
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  const int i = qi(qy,dy,Q1D);
-                  const int j = dj(qy,dy,D1D);
-                  const int k = qk(qy,dy,Q1D);
-                  const int l = dl(qy,dy,D1D);
-                  const double s = sign(qy,dy);
-                  u += QQD0[qz][qy][dx] * Bt[j][i];
-                  v += QQD1[qz][qy][dx] * Gt[l][k] * s;
-                  w += QQD2[qz][qy][dx] * Bt[j][i];
-               }
-               QDD0[qz][dy][dx] = u;
-               QDD1[qz][dy][dx] = v;
-               QDD2[qz][dy][dx] = w;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               double u = 0.0, v = 0.0, w = 0.0;
+               const int i = qi(qx,dx,Q1D);
+               const int j = dj(qx,dx,D1D);
+               const int k = qk(qx,dx,Q1D);
+               const int l = dl(qx,dx,D1D);
+               const double s = sign(qx,dx);
               MFEM_UNROLL(MQ1)
               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  u[qz] += QQQ0[qz][qy][qx] * Gt[l][k] * s;
+                  v[qz] += QQQ1[qz][qy][qx] * Bt[j][i];
+                  w[qz] += QQQ2[qz][qy][qx] * Bt[j][i];
+               }
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               QQD0[qz][qy][dx] = u[qz];
+               QQD1[qz][qy][dx] = v[qz];
+               QQD2[qz][qy][dx] = w[qz];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u[Q1D], v[Q1D], w[Q1D];
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz) { u[qz] = v[qz] = w[qz] = 0.0; }
+            MFEM_UNROLL(MQ1)
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const int i = qi(qy,dy,Q1D);
+               const int j = dj(qy,dy,D1D);
+               const int k = qk(qy,dy,Q1D);
+               const int l = dl(qy,dy,D1D);
+               const double s = sign(qy,dy);
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  u[qz] += QQD0[qz][qy][dx] * Bt[j][i];
+                  v[qz] += QQD1[qz][qy][dx] * Gt[l][k] * s;
+                  w[qz] += QQD2[qz][qy][dx] * Bt[j][i];
+               }
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               QDD0[qz][dy][dx] = u[qz];
+               QDD1[qz][dy][dx] = v[qz];
+               QDD2[qz][dy][dx] = w[qz];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u[D1D], v[D1D], w[D1D];
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz) { u[dz] = v[dz] = w[dz] = 0.0; }
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < D1D; ++dz)
               {
                  const int i = qi(qz,dz,Q1D);
                  const int j = dj(qz,dz,D1D);
                  const int k = qk(qz,dz,Q1D);
                  const int l = dl(qz,dz,D1D);
                  const double s = sign(qz,dz);
-                  u += QDD0[qz][dy][dx] * Bt[j][i];
-                  v += QDD1[qz][dy][dx] * Bt[j][i];
-                  w += QDD2[qz][dy][dx] * Gt[l][k] * s;
+                  u[dz] += QDD0[qz][dy][dx] * Bt[j][i];
+                  v[dz] += QDD1[qz][dy][dx] * Bt[j][i];
+                  w[dz] += QDD2[qz][dy][dx] * Gt[l][k] * s;
               }
-               y(dx,dy,dz,e) += (u + v + w);
+            }
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               y(dx,dy,dz,e) += (u[dz] + v[dz] + w[dz]);
            }
         }
      }
-      MFEM_SYNC_THREAD;
   });
 }

@@ -2008,11 +1862,8 @@ static void PADiffusionApply(const int dim,
      switch (ID)
      {
         case 0x22: return SmemPADiffusionApply2D<2,2,16>(NE,symm,B,G,D,X,Y);
-         case 0x23: return SmemPADiffusionApply2D<2,3,16>(NE,symm,B,G,D,X,Y);
         case 0x33: return SmemPADiffusionApply2D<3,3,16>(NE,symm,B,G,D,X,Y);
-         case 0x34: return SmemPADiffusionApply2D<3,4,16>(NE,symm,B,G,D,X,Y);
         case 0x44: return SmemPADiffusionApply2D<4,4,8>(NE,symm,B,G,D,X,Y);
-         case 0x45: return SmemPADiffusionApply2D<4,5,8>(NE,symm,B,G,D,X,Y);
         case 0x55: return SmemPADiffusionApply2D<5,5,8>(NE,symm,B,G,D,X,Y);
         case 0x66: return SmemPADiffusionApply2D<6,6,4>(NE,symm,B,G,D,X,Y);
         case 0x77: return SmemPADiffusionApply2D<7,7,4>(NE,symm,B,G,D,X,Y);
@@ -2026,7 +1877,6 @@ static void PADiffusionApply(const int dim,
   {
      switch (ID)
      {
-         case 0x22: return SmemPADiffusionApply3D<2,2>(NE,symm,B,G,D,X,Y);
         case 0x23: return SmemPADiffusionApply3D<2,3>(NE,symm,B,G,D,X,Y);
         case 0x34: return SmemPADiffusionApply3D<3,4>(NE,symm,B,G,D,X,Y);
         case 0x45: return SmemPADiffusionApply3D<4,5>(NE,symm,B,G,D,X,Y);
@@ -2049,12 +1899,6 @@ void DiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
   {
      ceedOp->AddMult(x, y);
   }
-   else if (Device::FastKernelsEnabled())
-   {
-      NDK_PADiffusionApply(dim, dofs1D, quad1D, ne, CoG,
-                           fespace, maps,
-                           pa_data, x, y);
-   }
   else
   {
      PADiffusionApply(dim, dofs1D, quad1D, ne, symmetric,
@@ -1,186 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#define MFEM_NVTX_COLOR Pink
-#include "../general/nvtx.hpp"
-
-#define MFEM_DEBUG_COLOR 206
-#include "../general/debug.hpp"
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-#include "ceed/diffusion.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-// Shared memory PA Diffusion Diagonal 3D kernel
-template<int D1D, int Q1D>
-static void NDK_SmemPADiffusionDiag3D(const int ndofs,
-                                      const int NE,
-                                      const bool symmetric,
-                                      const int *map_,
-                                      const double *b_,
-                                      const double *g_,
-                                      const double *d_,
-                                      double *y_)
-{
-   MFEM_NVTX;
-   constexpr int DIM = 3;
-   const auto MAP = Reshape(map_, D1D,D1D,D1D, NE);
-   auto b = Reshape(b_, Q1D, D1D);
-   auto g = Reshape(g_, Q1D, D1D);
-   auto D = Reshape(d_, Q1D*Q1D*Q1D, symmetric ? 6 : 9, NE);
-   auto Y = Reshape(y_, D1D, D1D, D1D, NE);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-      MFEM_SHARED double BG[2][Q1D*D1D];
-      double (*B)[D1D] = (double (*)[D1D]) (BG+0);
-      double (*G)[D1D] = (double (*)[D1D]) (BG+1);
-      MFEM_SHARED double QQD[Q1D][Q1D][D1D];
-      MFEM_SHARED double QDD[Q1D][D1D][D1D];
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B[q][d] = b(q,d);
-               G[q][d] = g(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      for (int i = 0; i < DIM; ++i)
-      {
-         for (int j = 0; j < DIM; ++j)
-         {
-            // first tensor contraction, along z direction
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               MFEM_FOREACH_THREAD(qy,y,Q1D)
-               {
-                  MFEM_FOREACH_THREAD(dz,z,D1D)
-                  {
-                     QQD[qx][qy][dz] = 0.0;
-                     for (int qz = 0; qz < Q1D; ++qz)
-                     {
-                        const int q = qx + (qy + qz * Q1D) * Q1D;
-                        const int ksym = j >= i ?
-                                         3 - (3-i)*(2-i)/2 + j:
-                                         3 - (3-j)*(2-j)/2 + i;
-                        const int k = symmetric ? ksym : (i*DIM) + j;
-                        const double O = D(q,k,e);
-                        const double Bz = B[qz][dz];
-                        const double Gz = G[qz][dz];
-                        const double L = i==2 ? Gz : Bz;
-                        const double R = j==2 ? Gz : Bz;
-                        QQD[qx][qy][dz] += L * O * R;
-                     }
-                  }
-               }
-            }
-            MFEM_SYNC_THREAD;
-            // second tensor contraction, along y direction
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               MFEM_FOREACH_THREAD(dz,z,D1D)
-               {
-                  MFEM_FOREACH_THREAD(dy,y,D1D)
-                  {
-                     QDD[qx][dy][dz] = 0.0;
-                     for (int qy = 0; qy < Q1D; ++qy)
-                     {
-                        const double By = B[qy][dy];
-                        const double Gy = G[qy][dy];
-                        const double L = i==1 ? Gy : By;
-                        const double R = j==1 ? Gy : By;
-                        QDD[qx][dy][dz] += L * QQD[qx][qy][dz] * R;
-                     }
-                  }
-               }
-            }
-            MFEM_SYNC_THREAD;
-            // third tensor contraction, along x direction
-            MFEM_FOREACH_THREAD(dz,z,D1D)
-            {
-               MFEM_FOREACH_THREAD(dy,y,D1D)
-               {
-                  MFEM_FOREACH_THREAD(dx,x,D1D)
-                  {
-                     for (int qx = 0; qx < Q1D; ++qx)
-                     {
-                        const double Bx = B[qx][dx];
-                        const double Gx = G[qx][dx];
-                        const double L = i==0 ? Gx : Bx;
-                        const double R = j==0 ? Gx : Bx;
-                        const double lvr = L * QDD[qx][dy][dz] * R;
-                        const int gid = MAP(dx,dy,dz,e);
-                        const int idx = gid >= 0 ? gid : -1 - gid;
-                        AtomicAdd(Y[idx], lvr);
-                     }
-                  }
-               }
-            }
-            MFEM_SYNC_THREAD;
-         }
-      }
-   });
-}
-
-void NDK_PADiffusionAssembleDiagonal(const int dim,
-                                     const int D1D,
-                                     const int Q1D,
-                                     const int NE,
-                                     const bool symm,
-                                     const FiniteElementSpace *fes,
-                                     const DofToQuad *maps,
-                                     const Vector &D,
-                                     Vector &Y)
-{
-   MFEM_NVTX;
-   const int ND = fes->GetNDofs();
-   constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
-   const Operator *ERop = fes->GetElementRestriction(ordering);
-   const ElementRestriction* ER = dynamic_cast<const ElementRestriction*>(ERop);
-   assert(ER);
-   const int *m = ER->GatherMap().Read();
-   const double *b = maps->B.Read();
-   const double *g = maps->G.Read();
-   const double *d = D.Read();
-   double *y = Y.ReadWrite();
-
-   if (dim == 2) {  assert(false); }
-   else if (dim == 3)
-   {
-      switch ((D1D << 4 ) | Q1D)
-      {
-         case 0x22: return NDK_SmemPADiffusionDiag3D<2,2>(ND,NE,symm,m,b,g,d,y);
-         case 0x23: return NDK_SmemPADiffusionDiag3D<2,3>(ND,NE,symm,m,b,g,d,y);
-         case 0x34: return NDK_SmemPADiffusionDiag3D<3,4>(ND,NE,symm,m,b,g,d,y);
-         case 0x45: return NDK_SmemPADiffusionDiag3D<4,5>(ND,NE,symm,m,b,g,d,y);
-         case 0x46: return NDK_SmemPADiffusionDiag3D<4,6>(ND,NE,symm,m,b,g,d,y);
-         case 0x56: return NDK_SmemPADiffusionDiag3D<5,6>(ND,NE,symm,m,b,g,d,y);
-         case 0x67: return NDK_SmemPADiffusionDiag3D<6,7>(ND,NE,symm,m,b,g,d,y);
-         case 0x78: return NDK_SmemPADiffusionDiag3D<7,8>(ND,NE,symm,m,b,g,d,y);
-         //case 0x89: return NDK_SmemPADiffusionDiag3D<8,9>(ND,NE,symm,m,b,g,d,y);
-         //case 0x9A: return NDK_SmemPADiffusionDiag3D<9,10>(ND,NE,symm,m,b,g,d,y);
-         default: MFEM_ABORT("Not implemented!");
-      }
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-
-} // namespace mfem
@@ -1,332 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#define MFEM_NVTX_COLOR Pink
-#include "../general/nvtx.hpp"
-
-#define MFEM_DEBUG_COLOR 206
-#include "../general/debug.hpp"
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-#include "ceed/diffusion.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-////////////////////////////////////////////////////////////////////////////////
-template<int D1D, int Q1D, int NBZ, int NBK> static
-MFEM_GLOBAL MFEM_LAUNCH_BOUNDS(Q1D*Q1D*NBZ,NBK)
-void NDK_PADiffApply(const int NE,
-                     const int *MAP,
-                     const double *B,
-                     const double *G,
-                     const double *D,
-                     const double *X,
-                     double *Y)
-{
-   double r_qt, r_q[Q1D];
-   MFEM_SHARED double s_B[D1D][Q1D];
-   MFEM_SHARED double s_G[Q1D][Q1D];
-   MFEM_SHARED double s_Iq[NBZ][Q1D][Q1D][Q1D];
-   MFEM_SHARED double s_Gqr[NBZ][Q1D][Q1D];
-   MFEM_SHARED double s_Gqs[NBZ][Q1D][Q1D];
-
-   for (int be = MFEM_BLOCK_ID(x); be < (NE+NBZ-1)/NBZ; be += MFEM_GRID_DIM(x))
-   {
-      const int tz = MFEM_THREAD_ID(z);
-      const int e = be * NBZ + tz;
-      if (e>=NE) { return; }
-
-      // Scatter X
-      MFEM_FOREACH_THREAD(j,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(i,x,Q1D)
-         {
-            if (tz == 0) { s_G[j][i] = G[i+Q1D*j]; } // ok with init
-            if (tz == 0 && j<D1D) { s_B[j][i] = B[i+Q1D*j]; } // ok
-            if (j<D1D && i<D1D)
-            {
-               MFEM_UNROLL(D1D)
-               for (int k = 0; k < D1D; k++)
-               {
-                  const int gid = MAP[e*D1D*D1D*D1D + k*D1D*D1D + j*D1D + i];
-                  const int idx = gid >= 0 ? gid : -1 - gid;
-                  r_q[k] = X[idx];
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Grad1X
-      MFEM_FOREACH_THREAD(b,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(a,x,D1D)
-         {
-            MFEM_UNROLL(Q1D)
-            for (int k=0; k<Q1D; ++k)
-            {
-               double u = 0.0;
-               MFEM_UNROLL(D1D)
-               for (int c=0; c<D1D; ++c) { u += s_B[c][k] * r_q[c]; }
-               s_Iq[tz][k][b][a] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Grad1Y
-      MFEM_FOREACH_THREAD(k,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(a,x,D1D)
-         {
-            MFEM_UNROLL(D1D)
-            for (int b=0; b<D1D; ++b) { r_q[b] = s_Iq[tz][k][b][a]; }
-            MFEM_UNROLL(Q1D)
-            for (int j=0; j<Q1D; ++j)
-            {
-               double u = 0.0;
-               MFEM_UNROLL(D1D)
-               for (int b=0; b<D1D; ++b) { u += s_B[b][j] * r_q[b]; }
-               s_Iq[tz][k][j][a] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Grad1Z
-      MFEM_FOREACH_THREAD(k,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(j,x,Q1D)
-         {
-            MFEM_UNROLL(D1D)
-            for (int a=0; a<D1D; ++a) { r_q[a] = s_Iq[tz][k][j][a]; }
-            MFEM_UNROLL(Q1D)
-            for (int i=0; i<Q1D; ++i)
-            {
-               double u = 0.0;
-               MFEM_UNROLL(D1D)
-               for (int a=0; a<D1D; ++a) { u += s_B[a][i] * r_q[a]; }
-               s_Iq[tz][k][j][i] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Flush
-      MFEM_FOREACH_THREAD(j,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(i,x,Q1D)
-         {
-            MFEM_UNROLL(Q1D)
-            for (int k = 0; k < Q1D; ++k) { r_q[k] = 0.0; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Q-Function
-      MFEM_UNROLL(Q1D)
-      for (int k = 0; k < Q1D; ++k)
-      {
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(j,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(i,x,Q1D)
-            {
-               double qr = 0.0, qs = 0.0, qt = 0.0;
-               MFEM_UNROLL(Q1D)
-               for (int m = 0; m < Q1D; ++m)
-               {
-                  const double Dim = s_G[i][m];
-                  const double Djm = s_G[j][m];
-                  const double Dkm = s_G[k][m];
-                  qr += Dim * s_Iq[tz][k][j][m];
-                  qs += Djm * s_Iq[tz][k][m][i];
-                  qt += Dkm * s_Iq[tz][m][j][i];
-               }
-               //(d, Q1D,Q1D,Q1D, 6, NE);
-               const int ebase = e * 6*Q1D*Q1D*Q1D;
-               const int gbase = i + j*Q1D + k*Q1D*Q1D;
-               const double D00 = D[0*Q1D*Q1D*Q1D + gbase + ebase];
-               const double D01 = D[1*Q1D*Q1D*Q1D + gbase + ebase];
-               const double D02 = D[2*Q1D*Q1D*Q1D + gbase + ebase];
-               const double D11 = D[3*Q1D*Q1D*Q1D + gbase + ebase];
-               const double D12 = D[4*Q1D*Q1D*Q1D + gbase + ebase];
-               const double D22 = D[5*Q1D*Q1D*Q1D + gbase + ebase];
-
-               s_Gqr[tz][j][i] = D00*qr + D01*qs + D02*qt;
-               s_Gqs[tz][j][i] = D01*qr + D11*qs + D12*qt;
-               r_qt = D02*qr + D12*qs + D22*qt;
-            }
-         }
-         MFEM_SYNC_THREAD;
-
-         MFEM_FOREACH_THREAD(j,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(i,x,Q1D)
-            {
-               double Aqtmp = 0.0;
-               MFEM_UNROLL(Q1D)
-               for (int m = 0; m < Q1D; ++m)
-               {
-                  const double Dmi = s_G[m][i];
-                  const double Dmj = s_G[m][j];
-                  const double Dkm = s_G[k][m];
-                  Aqtmp += Dmi * s_Gqr[tz][j][m];
-                  Aqtmp += Dmj * s_Gqs[tz][m][i];
-                  r_q[m] += Dkm * r_qt;
-               }
-               r_q[k] += Aqtmp;
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-
-      // GradZT
-      MFEM_FOREACH_THREAD(j,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(i,x,Q1D)
-         {
-            MFEM_UNROLL(D1D)
-            for (int c=0; c<D1D; ++c)
-            {
-               double u = 0.0;
-               MFEM_UNROLL(Q1D)
-               for (int k=0; k<Q1D; ++k) { u += s_B[c][k] * r_q[k]; }
-               s_Iq[tz][c][j][i] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // GradYT
-      MFEM_FOREACH_THREAD(c,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(i,x,Q1D)
-         {
-            MFEM_UNROLL(Q1D)
-            for (int j=0; j<Q1D; ++j) { r_q[j] = s_Iq[tz][c][j][i]; }
-            MFEM_UNROLL(D1D)
-            for (int b=0; b<D1D; ++b)
-            {
-               double u = 0.0;
-               MFEM_UNROLL(Q1D)
-               for (int j=0; j<Q1D; ++j) { u += s_B[b][j] * r_q[j]; }
-               s_Iq[tz][c][b][i] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // GradXT
-      MFEM_FOREACH_THREAD(c,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(b,x,D1D)
-         {
-            MFEM_UNROLL(Q1D)
-            for (int i=0; i<Q1D; ++i) { r_q[i] = s_Iq[tz][c][b][i]; }
-            MFEM_UNROLL(D1D)
-            for (int a=0; a<D1D; ++a)
-            {
-               double u = 0.0;
-               MFEM_UNROLL(Q1D)
-               for (int i=0; i<Q1D; ++i) { u += s_B[a][i] * r_q[i]; }
-               s_Iq[tz][c][b][a] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Gather
-      MFEM_FOREACH_THREAD(j,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(i,x,D1D)
-         {
-            MFEM_UNROLL(D1D)
-            for (int k = 0; k < D1D; k++)
-            {
-               const int gid = MAP[e*D1D*D1D*D1D + k*D1D*D1D + j*D1D + i];
-               const int idx = gid >= 0 ? gid : -1 - gid;
-               const double output = s_Iq[tz][k][j][i];
-               AtomicAdd(Y[idx], output);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-   }
-}
-
-void NDK_PADiffusionApply(const int dim,
-                          const int D1D,
-                          const int Q1D,
-                          const int NE,
-                          const Vector &CoG,
-                          const FiniteElementSpace *fes,
-                          const DofToQuad *maps,
-                          const Vector &D,
-                          const Vector &X,
-                          Vector &Y)
-{
-   //dbg();
-   MFEM_NVTX;
-   assert(dim == 3);
-
-   const double *b = maps->B.Read();
-   const double *d = D.Read();
-   const double *x = X.Read();
-   double *y = Y.ReadWrite();
-
-   const int id = (D1D << 4) | Q1D;
-
-   const int ND = fes->GetNDofs();
-   constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
-   const Operator *ERop = fes->GetElementRestriction(ordering);
-   const ElementRestriction *ER = dynamic_cast<const ElementRestriction*>(ERop);
-   assert(ER);
-   const int *map = ER->GatherMap().Read();
-
-   const auto dM = Reshape(map, D1D,D1D,D1D, NE);
-   const auto dB = Reshape(b, Q1D,D1D);
-   const auto dG = Reshape(CoG.Read(), Q1D,Q1D);
-   const auto dD = Reshape(d, Q1D,Q1D,Q1D, 6, NE);
-   const auto dX = Reshape(x, ND);
-   auto dY = Reshape(y, ND);
-
-   void (*Ker)(const int NE,
-               const int *MAP,
-               const double *B,
-               const double *G,
-               const double *D,
-               const double *X,
-               double *Y) = nullptr;
-
-   int NBZ = 1;
-
-   switch (id) // orders 1~8
-   {
-      case 0x23: Ker=NDK_PADiffApply<2,3,16,5>; NBZ=16; break; // 1
-      case 0x34: Ker=NDK_PADiffApply<3,4,8,6>; NBZ=8; break; // 2
-      case 0x45: Ker=NDK_PADiffApply<4,5,4,4>; NBZ=4; break; // 3
-      case 0x56: Ker=NDK_PADiffApply<5,6,4,3>; NBZ=4; break; // 4
-      case 0x67: Ker=NDK_PADiffApply<6,7,1,3>; break; // 5
-      case 0x78: Ker=NDK_PADiffApply<7,8,1,0>; break; // 6
-      //case 0x89: Ker=NDK_PADiffApply<8,9,1,0>; break; // 7
-      //case 0x9A: Ker=NDK_PADiffApply<9,10,1,1>; break; // 8
-      default: MFEM_ABORT("Unknown kernel 0x" << std::hex << id << std::dec);
-   }
-   MFEM_CONTRACT_VAR(NBZ);
-   MFEM_LAUNCH_KERNEL(Ker,(NE+NBZ-1)/NBZ,dim3(Q1D,Q1D,NBZ),0,NE,dM,dB,dG,dD,dX,dY);
-}
-
-} // namespace mfem
@@ -21,12 +21,12 @@ namespace mfem
 // PA Divergence Integrator

 // PA Divergence Assemble 2D kernel
-static void PADivergenceSetup2D(const int Q1D,
-                                const int NE,
-                                const Array<double> &w,
-                                const Vector &j,
-                                const double COEFF,
-                                Vector &op)
+void PADivergenceSetup2D(const int Q1D,
+                         const int NE,
+                         const Array<double> &w,
+                         const Vector &j,
+                         const double COEFF,
+                         Vector &op)
 {
   const int NQ = Q1D*Q1D;
   auto W = w.Read();
@@ -51,12 +51,12 @@ static void PADivergenceSetup2D(const int Q1D,
 }

 // PA Divergence Assemble 3D kernel
-static void PADivergenceSetup3D(const int Q1D,
-                                const int NE,
-                                const Array<double> &w,
-                                const Vector &j,
-                                const double COEFF,
-                                Vector &op)
+void PADivergenceSetup3D(const int Q1D,
+                         const int NE,
+                         const Array<double> &w,
+                         const Vector &j,
+                         const double COEFF,
+                         Vector &op)
 {
   const int NQ = Q1D*Q1D*Q1D;
   auto W = w.Read();
@@ -160,16 +160,16 @@ void VectorDivergenceIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,

 // PA Divergence Apply 2D kernel
 template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0>
-static void PADivergenceApply2D(const int NE,
-                                const Array<double> &b,
-                                const Array<double> &g,
-                                const Array<double> &bt,
-                                const Vector &op_,
-                                const Vector &x_,
-                                Vector &y_,
-                                const int tr_d1d = 0,
-                                const int te_d1d = 0,
-                                const int q1d = 0)
+void PADivergenceApply2D(const int NE,
+                         const Array<double> &b,
+                         const Array<double> &g,
+                         const Array<double> &bt,
+                         const Vector &op_,
+                         const Vector &x_,
+                         Vector &y_,
+                         const int tr_d1d = 0,
+                         const int te_d1d = 0,
+                         const int q1d = 0)
 {
   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -281,16 +281,16 @@ static void PADivergenceApply2D(const int NE,
 // Shared memory PA Divergence Apply 2D kernel
 template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0,
         const int T_NBZ = 0>
-static void SmemPADivergenceApply2D(const int NE,
-                                    const Array<double> &b_,
-                                    const Array<double> &g_,
-                                    const Array<double> &bt_,
-                                    const Vector &op_,
-                                    const Vector &x_,
-                                    Vector &y_,
-                                    const int tr_d1d = 0,
-                                    const int te_d1d = 0,
-                                    const int q1d = 0)
+void SmemPADivergenceApply2D(const int NE,
+                             const Array<double> &b_,
+                             const Array<double> &g_,
+                             const Array<double> &bt_,
+                             const Vector &op_,
+                             const Vector &x_,
+                             Vector &y_,
+                             const int tr_d1d = 0,
+                             const int te_d1d = 0,
+                             const int q1d = 0)
 {
   // TODO
   MFEM_ASSERT(false, "SHARED MEM NOT PROGRAMMED YET");
@@ -298,16 +298,16 @@ static void SmemPADivergenceApply2D(const int NE,

 // PA Divergence Apply 2D kernel transpose
 template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0>
-static void PADivergenceApplyTranspose2D(const int NE,
-                                         const Array<double> &bt,
-                                         const Array<double> &gt,
-                                         const Array<double> &b,
-                                         const Vector &op_,
-                                         const Vector &x_,
-                                         Vector &y_,
-                                         const int tr_d1d = 0,
-                                         const int te_d1d = 0,
-                                         const int q1d = 0)
+void PADivergenceApplyTranspose2D(const int NE,
+                                  const Array<double> &bt,
+                                  const Array<double> &gt,
+                                  const Array<double> &b,
+                                  const Vector &op_,
+                                  const Vector &x_,
+                                  Vector &y_,
+                                  const int tr_d1d = 0,
+                                  const int te_d1d = 0,
+                                  const int q1d = 0)
 {
   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -414,16 +414,16 @@ static void PADivergenceApplyTranspose2D(const int NE,

 // PA Vector Divergence Apply 3D kernel
 template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0>
-static void PADivergenceApply3D(const int NE,
-                                const Array<double> &b,
-                                const Array<double> &g,
-                                const Array<double> &bt,
-                                const Vector &op_,
-                                const Vector &x_,
-                                Vector &y_,
-                                int tr_d1d = 0,
-                                int te_d1d = 0,
-                                int q1d = 0)
+void PADivergenceApply3D(const int NE,
+                         const Array<double> &b,
+                         const Array<double> &g,
+                         const Array<double> &bt,
+                         const Vector &op_,
+                         const Vector &x_,
+                         Vector &y_,
+                         int tr_d1d = 0,
+                         int te_d1d = 0,
+                         int q1d = 0)
 {
   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -597,16 +597,16 @@ static void PADivergenceApply3D(const int NE,

 // PA Vector Divergence Apply 3D kernel
 template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0>
-static void PADivergenceApplyTranspose3D(const int NE,
-                                         const Array<double> &bt,
-                                         const Array<double> &gt,
-                                         const Array<double> &b,
-                                         const Vector &op_,
-                                         const Vector &x_,
-                                         Vector &y_,
-                                         int tr_d1d = 0,
-                                         int te_d1d = 0,
-                                         int q1d = 0)
+void PADivergenceApplyTranspose3D(const int NE,
+                                  const Array<double> &bt,
+                                  const Array<double> &gt,
+                                  const Array<double> &b,
+                                  const Vector &op_,
+                                  const Vector &x_,
+                                  Vector &y_,
+                                  int tr_d1d = 0,
+                                  int te_d1d = 0,
+                                  int q1d = 0)
 {
   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -775,16 +775,16 @@ static void PADivergenceApplyTranspose3D(const int NE,

 // Shared memory PA Vector Divergence Apply 3D kernel
 template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0>
-static void SmemPADivergenceApply3D(const int NE,
-                                    const Array<double> &b_,
-                                    const Array<double> &g_,
-                                    const Array<double> &bt_,
-                                    const Vector &q_,
-                                    const Vector &x_,
-                                    Vector &y_,
-                                    const int tr_d1d = 0,
-                                    const int te_d1d = 0,
-                                    const int q1d = 0)
+void SmemPADivergenceApply3D(const int NE,
+                             const Array<double> &b_,
+                             const Array<double> &g_,
+                             const Array<double> &bt_,
+                             const Vector &q_,
+                             const Vector &x_,
+                             Vector &y_,
+                             const int tr_d1d = 0,
+                             const int te_d1d = 0,
+                             const int q1d = 0)
 {
   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -70,12 +70,12 @@ namespace mfem
   the \b MFEM_SHARED keyword for local arrays. */

 // PA Gradient Assemble 2D kernel
-static void PAGradientSetup2D(const int Q1D,
-                              const int NE,
-                              const Array<double> &w,
-                              const Vector &j,
-                              const Vector &c,
-                              Vector &op)
+void PAGradientSetup2D(const int Q1D,
+                       const int NE,
+                       const Array<double> &w,
+                       const Vector &j,
+                       const Vector &c,
+                       Vector &op)
 {
   const int NQ = Q1D*Q1D;
   auto W = w.Read();
@@ -105,12 +105,12 @@ static void PAGradientSetup2D(const int Q1D,
 }

 // PA Gradient Assemble 3D kernel
-static void PAGradientSetup3D(const int Q1D,
-                              const int NE,
-                              const Array<double> &w,
-                              const Vector &j,
-                              const Vector &c,
-                              Vector &op)
+void PAGradientSetup3D(const int Q1D,
+                       const int NE,
+                       const Array<double> &w,
+                       const Vector &j,
+                       const Vector &c,
+                       Vector &op)
 {
   const int NQ = Q1D*Q1D*Q1D;
   auto W = w.Read();
@@ -254,16 +254,16 @@ void GradientIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,

 // PA Gradient Apply 2D kernel
 template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
-static void PAGradientApply2D(const int NE,
-                              const Array<double> &b,
-                              const Array<double> &g,
-                              const Array<double> &bt,
-                              const Vector &op_,
-                              const Vector &x_,
-                              Vector &y_,
-                              const int tr_d1d = 0,
-                              const int te_d1d = 0,
-                              const int q1d = 0)
+void PAGradientApply2D(const int NE,
+                       const Array<double> &b,
+                       const Array<double> &g,
+                       const Array<double> &bt,
+                       const Vector &op_,
+                       const Vector &x_,
+                       Vector &y_,
+                       const int tr_d1d = 0,
+                       const int te_d1d = 0,
+                       const int q1d = 0)
 {
   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -384,16 +384,16 @@ static void PAGradientApplyTranspose2D(const int NE,

 // PA Gradient Apply 3D kernel
 template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0>
-static void PAGradientApply3D(const int NE,
-                              const Array<double> &b,
-                              const Array<double> &g,
-                              const Array<double> &bt,
-                              const Vector &op_,
-                              const Vector &x_,
-                              Vector &y_,
-                              int tr_d1d = 0,
-                              int te_d1d = 0,
-                              int q1d = 0)
+void PAGradientApply3D(const int NE,
+                       const Array<double> &b,
+                       const Array<double> &g,
+                       const Array<double> &bt,
+                       const Vector &op_,
+                       const Vector &x_,
+                       Vector &y_,
+                       int tr_d1d = 0,
+                       int te_d1d = 0,
+                       int q1d = 0)
 {
   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -579,16 +579,16 @@ static void PAGradientApplyTranspose3D(const int NE,

 // Shared memory PA Gradient Apply 3D kernel
 template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0>
-static void SmemPAGradientApply3D(const int NE,
-                                  const Array<double> &b_,
-                                  const Array<double> &g_,
-                                  const Array<double> &bt_,
-                                  const Vector &d_,
-                                  const Vector &x_,
-                                  Vector &y_,
-                                  const int tr_d1d = 0,
-                                  const int te_d1d = 0,
-                                  const int q1d = 0)
+void SmemPAGradientApply3D(const int NE,
+                           const Array<double> &b_,
+                           const Array<double> &g_,
+                           const Array<double> &bt_,
+                           const Vector &d_,
+                           const Vector &x_,
+                           Vector &y_,
+                           const int tr_d1d = 0,
+                           const int te_d1d = 0,
+                           const int q1d = 0)
 {
   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -186,7 +186,7 @@ void PAHcurlMassAssembleDiagonal2D(const int D1D,
                  const double wy = (c == 1) ? Bo(qy,dy) : Bc(qy,dy);

                  mass[qx] += wy * wy * ((c == 0) ? op(qx,qy,0,e) :
-                                         op(qx,qy,symmetric ? 2 : 3, e));
+                  op(qx,qy,symmetric ? 2 : 3, e));
               }
            }

@@ -237,7 +237,7 @@ void PAHcurlMassAssembleDiagonal3D(const int D1D,
         const int D1Dx = (c == 0) ? D1D - 1 : D1D;

         const int opc = (c == 0) ? 0 : ((c == 1) ? (symmetric ? 3 : 4) :
-                                         (symmetric ? 5 : 8));
+         (symmetric ? 5 : 8));

         double mass[MAX_Q1D];

@@ -791,12 +791,12 @@ void SmemPAHcurlMassApply3D(const int D1D,
 }

 // PA H(curl) curl-curl assemble 2D kernel
-static void PACurlCurlSetup2D(const int Q1D,
-                              const int NE,
-                              const Array<double> &w,
-                              const Vector &j,
-                              Vector &coeff,
-                              Vector &op)
+void PACurlCurlSetup2D(const int Q1D,
+                       const int NE,
+                       const Array<double> &w,
+                       const Vector &j,
+                       Vector &coeff,
+                       Vector &op)
 {
   const int NQ = Q1D*Q1D;
   auto W = w.Read();
@@ -818,13 +818,13 @@ static void PACurlCurlSetup2D(const int Q1D,
 }

 // PA H(curl) curl-curl assemble 3D kernel
-static void PACurlCurlSetup3D(const int Q1D,
-                              const int coeffDim,
-                              const int NE,
-                              const Array<double> &w,
-                              const Vector &j,
-                              Vector &coeff,
-                              Vector &op)
+void PACurlCurlSetup3D(const int Q1D,
+                       const int coeffDim,
+                       const int NE,
+                       const Array<double> &w,
+                       const Vector &j,
+                       Vector &coeff,
+                       Vector &op)
 {
   const int NQ = Q1D*Q1D*Q1D;
   const bool symmetric = (coeffDim != 9);
@@ -1045,16 +1045,16 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
   }
 }

-static void PACurlCurlApply2D(const int D1D,
-                              const int Q1D,
-                              const int NE,
-                              const Array<double> &bo,
-                              const Array<double> &bot,
-                              const Array<double> &gc,
-                              const Array<double> &gct,
-                              const Vector &pa_data,
-                              const Vector &x,
-                              Vector &y)
+void PACurlCurlApply2D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const Array<double> &bo,
+                       const Array<double> &bot,
+                       const Array<double> &gc,
+                       const Array<double> &gct,
+                       const Vector &pa_data,
+                       const Vector &x,
+                       Vector &y)
 {
   constexpr static int VDIM = 2;
   constexpr static int MAX_D1D = HCURL_MAX_D1D;
@@ -1166,19 +1166,19 @@ static void PACurlCurlApply2D(const int D1D,
 }

 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-static void PACurlCurlApply3D(const int D1D,
-                              const int Q1D,
-                              const bool symmetric,
-                              const int NE,
-                              const Array<double> &bo,
-                              const Array<double> &bc,
-                              const Array<double> &bot,
-                              const Array<double> &bct,
-                              const Array<double> &gc,
-                              const Array<double> &gct,
-                              const Vector &pa_data,
-                              const Vector &x,
-                              Vector &y)
+void PACurlCurlApply3D(const int D1D,
+                       const int Q1D,
+                       const bool symmetric,
+                       const int NE,
+                       const Array<double> &bo,
+                       const Array<double> &bc,
+                       const Array<double> &bot,
+                       const Array<double> &bct,
+                       const Array<double> &gc,
+                       const Array<double> &gct,
+                       const Vector &pa_data,
+                       const Vector &x,
+                       Vector &y)
 {
   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
@@ -1677,19 +1677,19 @@ static void PACurlCurlApply3D(const int D1D,
 }

 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-static void SmemPACurlCurlApply3D(const int D1D,
-                                  const int Q1D,
-                                  const bool symmetric,
-                                  const int NE,
-                                  const Array<double> &bo,
-                                  const Array<double> &bc,
-                                  const Array<double> &bot,
-                                  const Array<double> &bct,
-                                  const Array<double> &gc,
-                                  const Array<double> &gct,
-                                  const Vector &pa_data,
-                                  const Vector &x,
-                                  Vector &y)
+void SmemPACurlCurlApply3D(const int D1D,
+                           const int Q1D,
+                           const bool symmetric,
+                           const int NE,
+                           const Array<double> &bo,
+                           const Array<double> &bc,
+                           const Array<double> &bot,
+                           const Array<double> &bct,
+                           const Array<double> &gc,
+                           const Array<double> &gct,
+                           const Vector &pa_data,
+                           const Vector &x,
+                           Vector &y)
 {
   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
@@ -2032,13 +2032,13 @@ void CurlCurlIntegrator::AddMultPA(const Vector &x, Vector &y) const
   }
 }

-static void PACurlCurlAssembleDiagonal2D(const int D1D,
-                                         const int Q1D,
-                                         const int NE,
-                                         const Array<double> &bo,
-                                         const Array<double> &gc,
-                                         const Vector &pa_data,
-                                         Vector &diag)
+void PACurlCurlAssembleDiagonal2D(const int D1D,
+                                  const int Q1D,
+                                  const int NE,
+                                  const Array<double> &bo,
+                                  const Array<double> &gc,
+                                  const Vector &pa_data,
+                                  Vector &diag)
 {
   constexpr static int VDIM = 2;
   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
@@ -2087,16 +2087,16 @@ static void PACurlCurlAssembleDiagonal2D(const int D1D,
 }

 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-static void PACurlCurlAssembleDiagonal3D(const int D1D,
-                                         const int Q1D,
-                                         const bool symmetric,
-                                         const int NE,
-                                         const Array<double> &bo,
-                                         const Array<double> &bc,
-                                         const Array<double> &go,
-                                         const Array<double> &gc,
-                                         const Vector &pa_data,
-                                         Vector &diag)
+void PACurlCurlAssembleDiagonal3D(const int D1D,
+                                  const int Q1D,
+                                  const bool symmetric,
+                                  const int NE,
+                                  const Array<double> &bo,
+                                  const Array<double> &bc,
+                                  const Array<double> &go,
+                                  const Array<double> &gc,
+                                  const Vector &pa_data,
+                                  Vector &diag)
 {
   constexpr static int VDIM = 3;
   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
@@ -2273,16 +2273,16 @@ static void PACurlCurlAssembleDiagonal3D(const int D1D,
 }

 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-static void SmemPACurlCurlAssembleDiagonal3D(const int D1D,
-                                             const int Q1D,
-                                             const bool symmetric,
-                                             const int NE,
-                                             const Array<double> &bo,
-                                             const Array<double> &bc,
-                                             const Array<double> &go,
-                                             const Array<double> &gc,
-                                             const Vector &pa_data,
-                                             Vector &diag)
+void SmemPACurlCurlAssembleDiagonal3D(const int D1D,
+                                      const int Q1D,
+                                      const bool symmetric,
+                                      const int NE,
+                                      const Array<double> &bo,
+                                      const Array<double> &bc,
+                                      const Array<double> &go,
+                                      const Array<double> &gc,
+                                      const Vector &pa_data,
+                                      Vector &diag)
 {
   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
@@ -2955,18 +2955,18 @@ void MixedVectorCurlIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
 // Apply to x corresponding to DOF's in H(curl) (trial), whose curl is
 // integrated against H(curl) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-static void PAHcurlL2Apply3D(const int D1D,
-                             const int Q1D,
-                             const int coeffDim,
-                             const int NE,
-                             const Array<double> &bo,
-                             const Array<double> &bc,
-                             const Array<double> &bot,
-                             const Array<double> &bct,
-                             const Array<double> &gc,
-                             const Vector &pa_data,
-                             const Vector &x,
-                             Vector &y)
+void PAHcurlL2Apply3D(const int D1D,
+                      const int Q1D,
+                      const int coeffDim,
+                      const int NE,
+                      const Array<double> &bo,
+                      const Array<double> &bc,
+                      const Array<double> &bot,
+                      const Array<double> &bct,
+                      const Array<double> &gc,
+                      const Vector &pa_data,
+                      const Vector &x,
+                      Vector &y)
 {
   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
@@ -3297,16 +3297,16 @@ static void PAHcurlL2Apply3D(const int D1D,
 // Apply to x corresponding to DOF's in H(curl) (trial), whose curl is
 // integrated against H(curl) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-static void SmemPAHcurlL2Apply3D(const int D1D,
-                                 const int Q1D,
-                                 const int coeffDim,
-                                 const int NE,
-                                 const Array<double> &bo,
-                                 const Array<double> &bc,
-                                 const Array<double> &gc,
-                                 const Vector &pa_data,
-                                 const Vector &x,
-                                 Vector &y)
+void SmemPAHcurlL2Apply3D(const int D1D,
+                          const int Q1D,
+                          const int coeffDim,
+                          const int NE,
+                          const Array<double> &bo,
+                          const Array<double> &bc,
+                          const Array<double> &gc,
+                          const Vector &pa_data,
+                          const Vector &x,
+                          Vector &y)
 {
   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
@@ -3585,18 +3585,18 @@ static void SmemPAHcurlL2Apply3D(const int D1D,
 // Apply to x corresponding to DOF's in H(curl) (trial), whose curl is
 // integrated against H(div) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-static void PAHcurlHdivApply3D(const int D1D,
-                               const int D1Dtest,
-                               const int Q1D,
-                               const int NE,
-                               const Array<double> &bo,
-                               const Array<double> &bc,
-                               const Array<double> &bot,
-                               const Array<double> &bct,
-                               const Array<double> &gc,
-                               const Vector &pa_data,
-                               const Vector &x,
-                               Vector &y)
+void PAHcurlHdivApply3D(const int D1D,
+                        const int D1Dtest,
+                        const int Q1D,
+                        const int NE,
+                        const Array<double> &bo,
+                        const Array<double> &bc,
+                        const Array<double> &bot,
+                        const Array<double> &bct,
+                        const Array<double> &gc,
+                        const Vector &pa_data,
+                        const Vector &x,
+                        Vector &y)
 {
   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
@@ -4071,18 +4071,18 @@ void MixedVectorWeakCurlIntegrator::AssemblePA(const FiniteElementSpace
 // Apply to x corresponding to DOF's in H(curl) (trial), integrated against curl
 // of H(curl) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-static void PAHcurlL2Apply3DTranspose(const int D1D,
-                                      const int Q1D,
-                                      const int coeffDim,
-                                      const int NE,
-                                      const Array<double> &bo,
-                                      const Array<double> &bc,
-                                      const Array<double> &bot,
-                                      const Array<double> &bct,
-                                      const Array<double> &gct,
-                                      const Vector &pa_data,
-                                      const Vector &x,
-                                      Vector &y)
+void PAHcurlL2Apply3DTranspose(const int D1D,
+                               const int Q1D,
+                               const int coeffDim,
+                               const int NE,
+                               const Array<double> &bo,
+                               const Array<double> &bc,
+                               const Array<double> &bot,
+                               const Array<double> &bct,
+                               const Array<double> &gct,
+                               const Vector &pa_data,
+                               const Vector &x,
+                               Vector &y)
 {
   // See PAHcurlL2Apply3D for comments.

@@ -4413,16 +4413,16 @@ static void PAHcurlL2Apply3DTranspose(const int D1D,
 }

 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-static void SmemPAHcurlL2Apply3DTranspose(const int D1D,
-                                          const int Q1D,
-                                          const int coeffDim,
-                                          const int NE,
-                                          const Array<double> &bo,
-                                          const Array<double> &bc,
-                                          const Array<double> &gc,
-                                          const Vector &pa_data,
-                                          const Vector &x,
-                                          Vector &y)
+void SmemPAHcurlL2Apply3DTranspose(const int D1D,
+                                   const int Q1D,
+                                   const int coeffDim,
+                                   const int NE,
+                                   const Array<double> &bo,
+                                   const Array<double> &bc,
+                                   const Array<double> &gc,
+                                   const Vector &pa_data,
+                                   const Vector &x,
+                                   Vector &y)
 {
   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
@@ -4675,13 +4675,13 @@ void MixedVectorWeakCurlIntegrator::AddMultPA(const Vector &x, Vector &y) const
 // Apply to x corresponding to DOFs in H^1 (domain) the (topological) gradient
 // to get a dof in H(curl) (range). You can think of the range as the "test" space
 // and the domain as the "trial" space, but there's no integration.
-static void PAHcurlApplyGradient2D(const int c_dofs1D,
-                                   const int o_dofs1D,
-                                   const int NE,
-                                   const Array<double> &B_,
-                                   const Array<double> &G_,
-                                   const Vector &x_,
-                                   Vector &y_)
+void PAHcurlApplyGradient2D(const int c_dofs1D,
+                            const int o_dofs1D,
+                            const int NE,
+                            const Array<double> &B_,
+                            const Array<double> &G_,
+                            const Vector &x_,
+                            Vector &y_)
 {
   auto B = Reshape(B_.Read(), c_dofs1D, c_dofs1D);
   auto G = Reshape(G_.Read(), o_dofs1D, c_dofs1D);
@@ -4753,12 +4753,12 @@ static void PAHcurlApplyGradient2D(const int c_dofs1D,
 }

 // Specialization of PAHcurlApplyGradient2D to the case where B is identity
-static void PAHcurlApplyGradient2DBId(const int c_dofs1D,
-                                      const int o_dofs1D,
-                                      const int NE,
-                                      const Array<double> &G_,
-                                      const Vector &x_,
-                                      Vector &y_)
+void PAHcurlApplyGradient2DBId(const int c_dofs1D,
+                               const int o_dofs1D,
+                               const int NE,
+                               const Array<double> &G_,
+                               const Vector &x_,
+                               Vector &y_)
 {
   auto G = Reshape(G_.Read(), o_dofs1D, c_dofs1D);

@@ -4822,7 +4822,7 @@ static void PAHcurlApplyGradient2DBId(const int c_dofs1D,
   });
 }

-static void PAHcurlApplyGradientTranspose2D(
+void PAHcurlApplyGradientTranspose2D(
   const int c_dofs1D, const int o_dofs1D, const int NE,
   const Array<double> &B_, const Array<double> &G_,
   const Vector &x_, Vector &y_)
@@ -4898,7 +4898,7 @@ static void PAHcurlApplyGradientTranspose2D(

 // Specialization of PAHcurlApplyGradientTranspose2D to the case where
 // B is identity
-static void PAHcurlApplyGradientTranspose2DBId(
+void PAHcurlApplyGradientTranspose2DBId(
   const int c_dofs1D, const int o_dofs1D, const int NE,
   const Array<double> &G_,
   const Vector &x_, Vector &y_)
@@ -4965,13 +4965,13 @@ static void PAHcurlApplyGradientTranspose2DBId(
   });
 }

-static void PAHcurlApplyGradient3D(const int c_dofs1D,
-                                   const int o_dofs1D,
-                                   const int NE,
-                                   const Array<double> &B_,
-                                   const Array<double> &G_,
-                                   const Vector &x_,
-                                   Vector &y_)
+void PAHcurlApplyGradient3D(const int c_dofs1D,
+                            const int o_dofs1D,
+                            const int NE,
+                            const Array<double> &B_,
+                            const Array<double> &G_,
+                            const Vector &x_,
+                            Vector &y_)
 {
   auto B = Reshape(B_.Read(), c_dofs1D, c_dofs1D);
   auto G = Reshape(G_.Read(), o_dofs1D, c_dofs1D);
@@ -5154,12 +5154,12 @@ static void PAHcurlApplyGradient3D(const int c_dofs1D,
 }

 // Specialization of PAHcurlApplyGradient3D to the case where
-static void PAHcurlApplyGradient3DBId(const int c_dofs1D,
-                                      const int o_dofs1D,
-                                      const int NE,
-                                      const Array<double> &G_,
-                                      const Vector &x_,
-                                      Vector &y_)
+void PAHcurlApplyGradient3DBId(const int c_dofs1D,
+                               const int o_dofs1D,
+                               const int NE,
+                               const Array<double> &G_,
+                               const Vector &x_,
+                               Vector &y_)
 {
   auto G = Reshape(G_.Read(), o_dofs1D, c_dofs1D);

@@ -5322,7 +5322,7 @@ static void PAHcurlApplyGradient3DBId(const int c_dofs1D,
   });
 }

-static void PAHcurlApplyGradientTranspose3D(
+void PAHcurlApplyGradientTranspose3D(
   const int c_dofs1D, const int o_dofs1D, const int NE,
   const Array<double> &B_, const Array<double> &G_,
   const Vector &x_, Vector &y_)
@@ -5507,7 +5507,7 @@ static void PAHcurlApplyGradientTranspose3D(
 }

 // Specialization of PAHcurlApplyGradientTranspose3D to the case where
-static void PAHcurlApplyGradientTranspose3DBId(
+void PAHcurlApplyGradientTranspose3DBId(
   const int c_dofs1D, const int o_dofs1D, const int NE,
   const Array<double> &G_,
   const Vector &x_, Vector &y_)
@@ -5789,14 +5789,14 @@ void GradientInterpolator::AddMultTransposePA(const Vector &x, Vector &y) const
   }
 }

-static void PAHcurlVecH1IdentityApply3D(const int c_dofs1D,
-                                        const int o_dofs1D,
-                                        const int NE,
-                                        const Array<double> &Bclosed,
-                                        const Array<double> &Bopen,
-                                        const Vector &pa_data,
-                                        const Vector &x_,
-                                        Vector &y_)
+void PAHcurlVecH1IdentityApply3D(const int c_dofs1D,
+                                 const int o_dofs1D,
+                                 const int NE,
+                                 const Array<double> &Bclosed,
+                                 const Array<double> &Bopen,
+                                 const Vector &pa_data,
+                                 const Vector &x_,
+                                 Vector &y_)
 {
   auto Bc = Reshape(Bclosed.Read(), c_dofs1D, c_dofs1D);
   auto Bo = Reshape(Bopen.Read(), o_dofs1D, c_dofs1D);
@@ -6002,14 +6002,14 @@ static void PAHcurlVecH1IdentityApply3D(const int c_dofs1D,
   });
 }

-static void PAHcurlVecH1IdentityApplyTranspose3D(const int c_dofs1D,
-                                                 const int o_dofs1D,
-                                                 const int NE,
-                                                 const Array<double> &Bclosed,
-                                                 const Array<double> &Bopen,
-                                                 const Vector &pa_data,
-                                                 const Vector &x_,
-                                                 Vector &y_)
+void PAHcurlVecH1IdentityApplyTranspose3D(const int c_dofs1D,
+                                          const int o_dofs1D,
+                                          const int NE,
+                                          const Array<double> &Bclosed,
+                                          const Array<double> &Bopen,
+                                          const Vector &pa_data,
+                                          const Vector &x_,
+                                          Vector &y_)
 {
   auto Bc = Reshape(Bclosed.Read(), c_dofs1D, c_dofs1D);
   auto Bo = Reshape(Bopen.Read(), o_dofs1D, c_dofs1D);
@@ -6228,14 +6228,14 @@ static void PAHcurlVecH1IdentityApplyTranspose3D(const int c_dofs1D,
   });
 }

-static void PAHcurlVecH1IdentityApply2D(const int c_dofs1D,
-                                        const int o_dofs1D,
-                                        const int NE,
-                                        const Array<double> &Bclosed,
-                                        const Array<double> &Bopen,
-                                        const Vector &pa_data,
-                                        const Vector &x_,
-                                        Vector &y_)
+void PAHcurlVecH1IdentityApply2D(const int c_dofs1D,
+                                 const int o_dofs1D,
+                                 const int NE,
+                                 const Array<double> &Bclosed,
+                                 const Array<double> &Bopen,
+                                 const Vector &pa_data,
+                                 const Vector &x_,
+                                 Vector &y_)
 {
   auto Bc = Reshape(Bclosed.Read(), c_dofs1D, c_dofs1D);
   auto Bo = Reshape(Bopen.Read(), o_dofs1D, c_dofs1D);
@@ -6327,14 +6327,14 @@ static void PAHcurlVecH1IdentityApply2D(const int c_dofs1D,
   });
 }

-static void PAHcurlVecH1IdentityApplyTranspose2D(const int c_dofs1D,
-                                                 const int o_dofs1D,
-                                                 const int NE,
-                                                 const Array<double> &Bclosed,
-                                                 const Array<double> &Bopen,
-                                                 const Vector &pa_data,
-                                                 const Vector &x_,
-                                                 Vector &y_)
+void PAHcurlVecH1IdentityApplyTranspose2D(const int c_dofs1D,
+                                          const int o_dofs1D,
+                                          const int NE,
+                                          const Array<double> &Bclosed,
+                                          const Array<double> &Bopen,
+                                          const Vector &pa_data,
+                                          const Vector &x_,
+                                          Vector &y_)
 {
   auto Bc = Reshape(Bclosed.Read(), c_dofs1D, c_dofs1D);
   auto Bo = Reshape(Bopen.Read(), o_dofs1D, c_dofs1D);
@@ -539,12 +539,12 @@ void PAHdivMassApply3D(const int D1D,

 // PA H(div) div-div assemble 2D kernel
 // NOTE: this is identical to PACurlCurlSetup3D
-static void PADivDivSetup2D(const int Q1D,
-                            const int NE,
-                            const Array<double> &w,
-                            const Vector &j,
-                            Vector &coeff_,
-                            Vector &op)
+void PADivDivSetup2D(const int Q1D,
+                     const int NE,
+                     const Array<double> &w,
+                     const Vector &j,
+                     Vector &coeff_,
+                     Vector &op)
 {
   const int NQ = Q1D*Q1D;
   auto W = w.Read();
@@ -565,12 +565,12 @@ static void PADivDivSetup2D(const int Q1D,
   });
 }

-static void PADivDivSetup3D(const int Q1D,
-                            const int NE,
-                            const Array<double> &w,
-                            const Vector &j,
-                            Vector &coeff_,
-                            Vector &op)
+void PADivDivSetup3D(const int Q1D,
+                     const int NE,
+                     const Array<double> &w,
+                     const Vector &j,
+                     Vector &coeff_,
+                     Vector &op)
 {
   const int NQ = Q1D*Q1D*Q1D;
   auto W = w.Read();
@@ -599,16 +599,16 @@ static void PADivDivSetup3D(const int Q1D,
   });
 }

-static void PADivDivApply2D(const int D1D,
-                            const int Q1D,
-                            const int NE,
-                            const Array<double> &Bo_,
-                            const Array<double> &Gc_,
-                            const Array<double> &Bot_,
-                            const Array<double> &Gct_,
-                            const Vector &op_,
-                            const Vector &x_,
-                            Vector &y_)
+void PADivDivApply2D(const int D1D,
+                     const int Q1D,
+                     const int NE,
+                     const Array<double> &Bo_,
+                     const Array<double> &Gc_,
+                     const Array<double> &Bot_,
+                     const Array<double> &Gct_,
+                     const Vector &op_,
+                     const Vector &x_,
+                     Vector &y_)
 {
   constexpr static int VDIM = 2;
   constexpr static int MAX_D1D = HDIV_MAX_D1D;
@@ -718,16 +718,16 @@ static void PADivDivApply2D(const int D1D,
   }); // end of element loop
 }

-static void PADivDivApply3D(const int D1D,
-                            const int Q1D,
-                            const int NE,
-                            const Array<double> &Bo_,
-                            const Array<double> &Gc_,
-                            const Array<double> &Bot_,
-                            const Array<double> &Gct_,
-                            const Vector &op_,
-                            const Vector &x_,
-                            Vector &y_)
+void PADivDivApply3D(const int D1D,
+                     const int Q1D,
+                     const int NE,
+                     const Array<double> &Bo_,
+                     const Array<double> &Gc_,
+                     const Array<double> &Bot_,
+                     const Array<double> &Gct_,
+                     const Vector &op_,
+                     const Vector &x_,
+                     Vector &y_)
 {
   MFEM_VERIFY(D1D <= HDIV_MAX_D1D, "Error: D1D > HDIV_MAX_D1D");
   MFEM_VERIFY(Q1D <= HDIV_MAX_Q1D, "Error: Q1D > HDIV_MAX_Q1D");
@@ -967,13 +967,13 @@ void DivDivIntegrator::AddMultPA(const Vector &x, Vector &y) const
   }
 }

-static void PADivDivAssembleDiagonal2D(const int D1D,
-                                       const int Q1D,
-                                       const int NE,
-                                       const Array<double> &Bo_,
-                                       const Array<double> &Gc_,
-                                       const Vector &op_,
-                                       Vector &diag_)
+void PADivDivAssembleDiagonal2D(const int D1D,
+                                const int Q1D,
+                                const int NE,
+                                const Array<double> &Bo_,
+                                const Array<double> &Gc_,
+                                const Vector &op_,
+                                Vector &diag_)
 {
   constexpr static int VDIM = 2;
   constexpr static int MAX_Q1D = HDIV_MAX_Q1D;
@@ -1023,13 +1023,13 @@ static void PADivDivAssembleDiagonal2D(const int D1D,
   });
 }

-static void PADivDivAssembleDiagonal3D(const int D1D,
-                                       const int Q1D,
-                                       const int NE,
-                                       const Array<double> &Bo_,
-                                       const Array<double> &Gc_,
-                                       const Vector &op_,
-                                       Vector &diag_)
+void PADivDivAssembleDiagonal3D(const int D1D,
+                                const int Q1D,
+                                const int NE,
+                                const Array<double> &Bo_,
+                                const Array<double> &Gc_,
+                                const Vector &op_,
+                                Vector &diag_)
 {
   MFEM_VERIFY(D1D <= HDIV_MAX_D1D, "Error: D1D > HDIV_MAX_D1D");
   MFEM_VERIFY(Q1D <= HDIV_MAX_Q1D, "Error: Q1D > HDIV_MAX_Q1D");
@@ -1104,11 +1104,11 @@ void DivDivIntegrator::AssembleDiagonalPA(Vector& diag)
 }

 // PA H(div)-L2 (div u, p) assemble 2D kernel
-static void PADivL2Setup2D(const int Q1D,
-                           const int NE,
-                           const Array<double> &w,
-                           Vector &coeff_,
-                           Vector &op)
+void PADivL2Setup2D(const int Q1D,
+                    const int NE,
+                    const Array<double> &w,
+                    Vector &coeff_,
+                    Vector &op)
 {
   const int NQ = Q1D*Q1D;
   auto W = w.Read();
@@ -1123,11 +1123,11 @@ static void PADivL2Setup2D(const int Q1D,
   });
 }

-static void PADivL2Setup3D(const int Q1D,
-                           const int NE,
-                           const Array<double> &w,
-                           Vector &coeff_,
-                           Vector &op)
+void PADivL2Setup3D(const int Q1D,
+                    const int NE,
+                    const Array<double> &w,
+                    Vector &coeff_,
+                    Vector &op)
 {
   const int NQ = Q1D*Q1D*Q1D;
   auto W = w.Read();
@@ -1225,16 +1225,16 @@ VectorFEDivergenceIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,

 // Apply to x corresponding to DOF's in H(div) (trial), whose divergence is
 // integrated against L_2 test functions corresponding to y.
-static void PAHdivL2Apply3D(const int D1D,
-                            const int Q1D,
-                            const int L2D1D,
-                            const int NE,
-                            const Array<double> &Bo_,
-                            const Array<double> &Gc_,
-                            const Array<double> &L2Bot_,
-                            const Vector &op_,
-                            const Vector &x_,
-                            Vector &y_)
+void PAHdivL2Apply3D(const int D1D,
+                     const int Q1D,
+                     const int L2D1D,
+                     const int NE,
+                     const Array<double> &Bo_,
+                     const Array<double> &Gc_,
+                     const Array<double> &L2Bot_,
+                     const Vector &op_,
+                     const Vector &x_,
+                     Vector &y_)
 {
   MFEM_VERIFY(D1D <= HDIV_MAX_D1D, "Error: D1D > HDIV_MAX_D1D");
   MFEM_VERIFY(Q1D <= HDIV_MAX_Q1D, "Error: Q1D > HDIV_MAX_Q1D");
@@ -1388,16 +1388,16 @@ static void PAHdivL2Apply3D(const int D1D,

 // Apply to x corresponding to DOF's in H(div) (trial), whose divergence is
 // integrated against L_2 test functions corresponding to y.
-static void PAHdivL2Apply2D(const int D1D,
-                            const int Q1D,
-                            const int L2D1D,
-                            const int NE,
-                            const Array<double> &Bo_,
-                            const Array<double> &Gc_,
-                            const Array<double> &L2Bot_,
-                            const Vector &op_,
-                            const Vector &x_,
-                            Vector &y_)
+void PAHdivL2Apply2D(const int D1D,
+                     const int Q1D,
+                     const int L2D1D,
+                     const int NE,
+                     const Array<double> &Bo_,
+                     const Array<double> &Gc_,
+                     const Array<double> &L2Bot_,
+                     const Vector &op_,
+                     const Vector &x_,
+                     Vector &y_)
 {
   constexpr static int VDIM = 2;
   constexpr static int MAX_D1D = HDIV_MAX_D1D;
@@ -1494,16 +1494,16 @@ static void PAHdivL2Apply2D(const int D1D,
   }); // end of element loop
 }

-static void PAHdivL2ApplyTranspose3D(const int D1D,
-                                     const int Q1D,
-                                     const int L2D1D,
-                                     const int NE,
-                                     const Array<double> &L2Bo_,
-                                     const Array<double> &Gct_,
-                                     const Array<double> &Bot_,
-                                     const Vector &op_,
-                                     const Vector &x_,
-                                     Vector &y_)
+void PAHdivL2ApplyTranspose3D(const int D1D,
+                              const int Q1D,
+                              const int L2D1D,
+                              const int NE,
+                              const Array<double> &L2Bo_,
+                              const Array<double> &Gct_,
+                              const Array<double> &Bot_,
+                              const Vector &op_,
+                              const Vector &x_,
+                              Vector &y_)
 {
   MFEM_VERIFY(D1D <= HDIV_MAX_D1D, "Error: D1D > HDIV_MAX_D1D");
   MFEM_VERIFY(Q1D <= HDIV_MAX_Q1D, "Error: Q1D > HDIV_MAX_Q1D");
@@ -1656,16 +1656,16 @@ static void PAHdivL2ApplyTranspose3D(const int D1D,
   }); // end of element loop
 }

-static void PAHdivL2ApplyTranspose2D(const int D1D,
-                                     const int Q1D,
-                                     const int L2D1D,
-                                     const int NE,
-                                     const Array<double> &L2Bo_,
-                                     const Array<double> &Gct_,
-                                     const Array<double> &Bot_,
-                                     const Vector &op_,
-                                     const Vector &x_,
-                                     Vector &y_)
+void PAHdivL2ApplyTranspose2D(const int D1D,
+                              const int Q1D,
+                              const int L2D1D,
+                              const int NE,
+                              const Array<double> &L2Bo_,
+                              const Array<double> &Gct_,
+                              const Array<double> &Bot_,
+                              const Vector &op_,
+                              const Vector &x_,
+                              Vector &y_)
 {
   constexpr static int VDIM = 2;
   constexpr static int MAX_D1D = HDIV_MAX_D1D;
@@ -1791,16 +1791,16 @@ void VectorFEDivergenceIntegrator::AddMultTransposePA(const Vector &x,
   }
 }

-static void PAHdivL2AssembleDiagonal_ADAt_3D(const int D1D,
-                                             const int Q1D,
-                                             const int L2D1D,
-                                             const int NE,
-                                             const Array<double> &L2Bo_,
-                                             const Array<double> &Gct_,
-                                             const Array<double> &Bot_,
-                                             const Vector &op_,
-                                             const Vector &D_,
-                                             Vector &diag_)
+void PAHdivL2AssembleDiagonal_ADAt_3D(const int D1D,
+                                      const int Q1D,
+                                      const int L2D1D,
+                                      const int NE,
+                                      const Array<double> &L2Bo_,
+                                      const Array<double> &Gct_,
+                                      const Array<double> &Bot_,
+                                      const Vector &op_,
+                                      const Vector &D_,
+                                      Vector &diag_)
 {
   MFEM_VERIFY(D1D <= HDIV_MAX_D1D, "Error: D1D > HDIV_MAX_D1D");
   MFEM_VERIFY(Q1D <= HDIV_MAX_Q1D, "Error: Q1D > HDIV_MAX_Q1D");
@@ -1916,16 +1916,16 @@ static void PAHdivL2AssembleDiagonal_ADAt_3D(const int D1D,
   }); // end of element loop
 }

-static void PAHdivL2AssembleDiagonal_ADAt_2D(const int D1D,
-                                             const int Q1D,
-                                             const int L2D1D,
-                                             const int NE,
-                                             const Array<double> &L2Bo_,
-                                             const Array<double> &Gct_,
-                                             const Array<double> &Bot_,
-                                             const Vector &op_,
-                                             const Vector &D_,
-                                             Vector &diag_)
+void PAHdivL2AssembleDiagonal_ADAt_2D(const int D1D,
+                                      const int Q1D,
+                                      const int L2D1D,
+                                      const int NE,
+                                      const Array<double> &L2Bo_,
+                                      const Array<double> &Gct_,
+                                      const Array<double> &Bot_,
+                                      const Vector &op_,
+                                      const Vector &D_,
+                                      Vector &diag_)
 {
   constexpr static int VDIM = 2;

@@ -17,13 +17,13 @@ namespace mfem
 {

 template<int T_D1D = 0, int T_Q1D = 0>
-static void EAMassAssemble1D(const int NE,
-                             const Array<double> &basis,
-                             const Vector &padata,
-                             Vector &eadata,
-                             const bool add,
-                             const int d1d = 0,
-                             const int q1d = 0)
+void EAMassAssemble1D(const int NE,
+                      const Array<double> &basis,
+                      const Vector &padata,
+                      Vector &eadata,
+                      const bool add,
+                      const int d1d = 0,
+                      const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -67,13 +67,13 @@ static void EAMassAssemble1D(const int NE,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-static void EAMassAssemble2D(const int NE,
-                             const Array<double> &basis,
-                             const Vector &padata,
-                             Vector &eadata,
-                             const bool add,
-                             const int d1d = 0,
-                             const int q1d = 0)
+void EAMassAssemble2D(const int NE,
+                      const Array<double> &basis,
+                      const Vector &padata,
+                      Vector &eadata,
+                      const bool add,
+                      const int d1d = 0,
+                      const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -139,13 +139,13 @@ static void EAMassAssemble2D(const int NE,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-static void EAMassAssemble3D(const int NE,
-                             const Array<double> &basis,
-                             const Vector &padata,
-                             Vector &eadata,
-                             const bool add,
-                             const int d1d = 0,
-                             const int q1d = 0)
+void EAMassAssemble3D(const int NE,
+                      const Array<double> &basis,
+                      const Vector &padata,
+                      Vector &eadata,
+                      const bool add,
+                      const int d1d = 0,
+                      const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -19,49 +19,14 @@ using namespace std;
 namespace mfem
 {

-// Forward declaration of non-deterministic 'fast' kernels
-void NDK_PAMassApply(const int dim,
-                     const int D1D,
-                     const int Q1D,
-                     const int NE,
-                     const FiniteElementSpace *fes,
-                     const DofToQuad *maps,
-                     const Vector &D,
-                     const Vector &X,
-                     Vector &Y);
-
-void NDK_AMD_PAMassApply(const int dim,
-                         const int D1D,
-                         const int Q1D,
-                         const int NE,
-                         const FiniteElementSpace *fes,
-                         const DofToQuad *maps,
-                         const Vector &D,
-                         const Vector &X,
-                         Vector &Y);
-
-void NDK_HIP_PAMassApply(const int dim,
-                         const int D1D,
-                         const int Q1D,
-                         const int NE,
-                         const FiniteElementSpace *fes,
-                         const DofToQuad *maps,
-                         const Vector &D,
-                         const Vector &X,
-                         Vector &Y);
-
 // PA Mass Integrator

 // PA Mass Assemble kernel

 void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
-   const MemoryType mt = (memory_type == MemoryType::DEFAULT) ?
-                         Device::GetDeviceMemoryType() : memory_type;
-
-   // If device options allow fast kernels, set the action type to L2L
-   action_type =
-      Device::FastKernelsEnabled() ? ActionType::L2L : ActionType::E2E;
+   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
+                         Device::GetDeviceMemoryType() : pa_mt;

   // Assuming the same element type
   fespace = &fes;
@@ -73,7 +38,7 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
   if (DeviceCanUseCeed())
   {
      delete ceedOp;
-      ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q, action_type);
+      ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q);
      return;
   }
   dim = mesh->Dimension();
@@ -189,6 +154,328 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
   }
 }

+template<int T_D1D = 0, int T_Q1D = 0>
+void PAMassAssembleDiagonal2D(const int NE,
+                              const Array<double> &b,
+                              const Vector &d,
+                              Vector &y,
+                              const int d1d = 0,
+                              const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto D = Reshape(d.Read(), Q1D, Q1D, NE);
+   auto Y = Reshape(y.ReadWrite(), D1D, D1D, NE);
+   MFEM_FORALL(e, NE,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      double QD[MQ1][MD1];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            QD[qx][dy] = 0.0;
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               QD[qx][dy] += B(qy, dy) * B(qy, dy) * D(qx, qy, e);
+            }
+         }
+      }
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               Y(dx,dy,e) += B(qx, dx) * B(qx, dx) * QD[qx][dy];
+            }
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
+void SmemPAMassAssembleDiagonal2D(const int NE,
+                                  const Array<double> &b_,
+                                  const Vector &d_,
+                                  Vector &y_,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
+   auto b = Reshape(b_.Read(), Q1D, D1D);
+   auto D = Reshape(d_.Read(), Q1D, Q1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
+   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
+   {
+      const int tidz = MFEM_THREAD_ID(z);
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      MFEM_SHARED double B[MQ1][MD1];
+      MFEM_SHARED double QDZ[NBZ][MQ1][MD1];
+      double (*QD)[MD1] = (double (*)[MD1])(QDZ + tidz);
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B[q][d] = b(q,d);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            QD[qx][dy] = 0.0;
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               QD[qx][dy] += B[qy][dy] * B[qy][dy] * D(qx, qy, e);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               // might need absolute values on next line
+               Y(dx,dy,e) += B[qx][dx] * B[qx][dx] * QD[qx][dy];
+            }
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0>
+void PAMassAssembleDiagonal3D(const int NE,
+                              const Array<double> &b,
+                              const Vector &d,
+                              Vector &y,
+                              const int d1d = 0,
+                              const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto D = Reshape(d.Read(), Q1D, Q1D, Q1D, NE);
+   auto Y = Reshape(y.ReadWrite(), D1D, D1D, D1D, NE);
+   MFEM_FORALL(e, NE,
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      double QQD[MQ1][MQ1][MD1];
+      double QDD[MQ1][MD1][MD1];
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               QQD[qx][qy][dz] = 0.0;
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  QQD[qx][qy][dz] += B(qz, dz) * B(qz, dz) * D(qx, qy, qz, e);
+               }
+            }
+         }
+      }
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               QDD[qx][dy][dz] = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  QDD[qx][dy][dz] += B(qy, dy) * B(qy, dy) * QQD[qx][qy][dz];
+               }
+            }
+         }
+      }
+      for (int dz = 0; dz < D1D; ++dz)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               double t = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  t += B(qx, dx) * B(qx, dx) * QDD[qx][dy][dz];
+               }
+               Y(dx, dy, dz, e) += t;
+            }
+         }
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0>
+void SmemPAMassAssembleDiagonal3D(const int NE,
+                                  const Array<double> &b_,
+                                  const Vector &d_,
+                                  Vector &y_,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
+   auto b = Reshape(b_.Read(), Q1D, D1D);
+   auto D = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   {
+      const int tidz = MFEM_THREAD_ID(z);
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      MFEM_SHARED double B[MQ1][MD1];
+      MFEM_SHARED double QQD[MQ1][MQ1][MD1];
+      MFEM_SHARED double QDD[MQ1][MD1][MD1];
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B[q][d] = b(q,d);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(dz,z,D1D)
+            {
+               QQD[qx][qy][dz] = 0.0;
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  QQD[qx][qy][dz] += B[qz][dz] * B[qz][dz] * D(qx, qy, qz, e);
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         MFEM_FOREACH_THREAD(dz,z,D1D)
+         {
+            MFEM_FOREACH_THREAD(dy,y,D1D)
+            {
+               QDD[qx][dy][dz] = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  QDD[qx][dy][dz] += B[qy][dy] * B[qy][dy] * QQD[qx][qy][dz];
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dz,z,D1D)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               double t = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  t += B[qx][dx] * B[qx][dx] * QDD[qx][dy][dz];
+               }
+               Y(dx, dy, dz, e) += t;
+            }
+         }
+      }
+   });
+}
+
+static void PAMassAssembleDiagonal(const int dim, const int D1D,
+                                   const int Q1D, const int NE,
+                                   const Array<double> &B,
+                                   const Vector &D,
+                                   Vector &Y)
+{
+   if (dim == 2)
+   {
+      switch ((D1D << 4 ) | Q1D)
+      {
+         case 0x22: return SmemPAMassAssembleDiagonal2D<2,2,16>(NE,B,D,Y);
+         case 0x33: return SmemPAMassAssembleDiagonal2D<3,3,16>(NE,B,D,Y);
+         case 0x44: return SmemPAMassAssembleDiagonal2D<4,4,8>(NE,B,D,Y);
+         case 0x55: return SmemPAMassAssembleDiagonal2D<5,5,8>(NE,B,D,Y);
+         case 0x66: return SmemPAMassAssembleDiagonal2D<6,6,4>(NE,B,D,Y);
+         case 0x77: return SmemPAMassAssembleDiagonal2D<7,7,4>(NE,B,D,Y);
+         case 0x88: return SmemPAMassAssembleDiagonal2D<8,8,2>(NE,B,D,Y);
+         case 0x99: return SmemPAMassAssembleDiagonal2D<9,9,2>(NE,B,D,Y);
+         default:   return PAMassAssembleDiagonal2D(NE,B,D,Y,D1D,Q1D);
+      }
+   }
+   else if (dim == 3)
+   {
+      switch ((D1D << 4 ) | Q1D)
+      {
+         case 0x23: return SmemPAMassAssembleDiagonal3D<2,3>(NE,B,D,Y);
+         case 0x24: return SmemPAMassAssembleDiagonal3D<2,4>(NE,B,D,Y);
+         case 0x26: return SmemPAMassAssembleDiagonal3D<2,6>(NE,B,D,Y);
+         case 0x34: return SmemPAMassAssembleDiagonal3D<3,4>(NE,B,D,Y);
+         case 0x35: return SmemPAMassAssembleDiagonal3D<3,5>(NE,B,D,Y);
+         case 0x45: return SmemPAMassAssembleDiagonal3D<4,5>(NE,B,D,Y);
+         case 0x48: return SmemPAMassAssembleDiagonal3D<4,8>(NE,B,D,Y);
+         case 0x56: return SmemPAMassAssembleDiagonal3D<5,6>(NE,B,D,Y);
+         case 0x67: return SmemPAMassAssembleDiagonal3D<6,7>(NE,B,D,Y);
+         case 0x78: return SmemPAMassAssembleDiagonal3D<7,8>(NE,B,D,Y);
+         case 0x89: return SmemPAMassAssembleDiagonal3D<8,9>(NE,B,D,Y);
+         default:   return PAMassAssembleDiagonal3D(NE,B,D,Y,D1D,Q1D);
+      }
+   }
+   MFEM_ABORT("Unknown kernel.");
+}
+
+void MassIntegrator::AssembleDiagonalPA(Vector &diag)
+{
+   if (DeviceCanUseCeed())
+   {
+      ceedOp->GetDiagonal(diag);
+   }
+   else
+   {
+      PAMassAssembleDiagonal(dim, dofs1D, quad1D, ne, maps->B, pa_data, diag);
+   }
+}
+
+
 #ifdef MFEM_USE_OCCA
 // OCCA PA Mass Apply 2D kernel
 static void OccaPAMassApply2D(const int D1D,
@@ -282,14 +569,14 @@ static void OccaPAMassApply3D(const int D1D,
 #endif // MFEM_USE_OCCA

 template<int T_D1D = 0, int T_Q1D = 0>
-static void PAMassApply2D(const int NE,
-                          const Array<double> &b_,
-                          const Array<double> &bt_,
-                          const Vector &d_,
-                          const Vector &x_,
-                          Vector &y_,
-                          const int d1d = 0,
-                          const int q1d = 0)
+void PAMassApply2D(const int NE,
+                   const Array<double> &b_,
+                   const Array<double> &bt_,
+                   const Vector &d_,
+                   const Vector &x_,
+                   Vector &y_,
+                   const int d1d = 0,
+                   const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -374,14 +661,14 @@ static void PAMassApply2D(const int NE,
 }

 template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
-static void SmemPAMassApply2D(const int NE,
-                              const Array<double> &b_,
-                              const Array<double> &bt_,
-                              const Vector &d_,
-                              const Vector &x_,
-                              Vector &y_,
-                              const int d1d = 0,
-                              const int q1d = 0)
+void SmemPAMassApply2D(const int NE,
+                       const Array<double> &b_,
+                       const Array<double> &bt_,
+                       const Vector &d_,
+                       const Vector &x_,
+                       Vector &y_,
+                       const int d1d = 0,
+                       const int q1d = 0)
 {
   MFEM_CONTRACT_VAR(bt_);
   const int D1D = T_D1D ? T_D1D : d1d;
@@ -497,14 +784,14 @@ static void SmemPAMassApply2D(const int NE,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-static void PAMassApply3D(const int NE,
-                          const Array<double> &b_,
-                          const Array<double> &bt_,
-                          const Vector &d_,
-                          const Vector &x_,
-                          Vector &y_,
-                          const int d1d = 0,
-                          const int q1d = 0)
+void PAMassApply3D(const int NE,
+                   const Array<double> &b_,
+                   const Array<double> &bt_,
+                   const Vector &d_,
+                   const Vector &x_,
+                   Vector &y_,
+                   const int d1d = 0,
+                   const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -638,14 +925,14 @@ static void PAMassApply3D(const int NE,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-static void SmemPAMassApply3D(const int NE,
-                              const Array<double> &b_,
-                              const Array<double> &bt_,
-                              const Vector &d_,
-                              const Vector &x_,
-                              Vector &y_,
-                              const int d1d = 0,
-                              const int q1d = 0)
+void SmemPAMassApply3D(const int NE,
+                       const Array<double> &b_,
+                       const Array<double> &bt_,
+                       const Vector &d_,
+                       const Vector &x_,
+                       Vector &y_,
+                       const int d1d = 0,
+                       const int q1d = 0)
 {
   MFEM_CONTRACT_VAR(bt_);
   const int D1D = T_D1D ? T_D1D : d1d;
@@ -864,159 +1151,6 @@ static void SmemPAMassApply3D(const int NE,
   });
 }

-template<int D1D, int Q1D>
-void SmemPAMassApply3D_v1(const int NE,
-                          const Array<double> &b_,
-                          const Array<double> &bt_,
-                          const Vector &d_,
-                          const Vector &x_,
-                          Vector &y_,
-                          const int d1d = 0,
-                          const int q1d = 0)
-{
-   MFEM_CONTRACT_VAR(bt_);
-   MFEM_CONTRACT_VAR(d1d);
-   MFEM_CONTRACT_VAR(q1d);
-   const auto B = Reshape(b_.Read(), Q1D,D1D);
-   const auto D = Reshape(d_.Read(), Q1D,Q1D,Q1D, NE);
-   const auto X = Reshape(x_.Read(), D1D,D1D,D1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D,D1D,D1D, NE);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
-   {
-      double u[Q1D];
-      MFEM_SHARED double s_B[Q1D][D1D];
-      MFEM_SHARED double s_q[Q1D][Q1D][Q1D];
-
-      // Load input, B & X interpolation
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            s_B[qx][dy] = B(qx,dy);
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
-            MFEM_UNROLL(D1D)
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               const double Bx = B(qx,dx);
-               MFEM_UNROLL(D1D)
-               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  u[dz] += X(dx,dy,dz,e) * Bx;
-               }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz) { s_q[dz][dy][qx] = u[dz]; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Y interpolation
-      MFEM_FOREACH_THREAD(dz,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            MFEM_UNROLL(Q1D)
-            for (int qy = 0; qy < Q1D; ++qy) { u[qy] = 0.0; }
-            MFEM_UNROLL(D1D)
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               const double zyX = s_q[dz][dy][qx];
-               MFEM_UNROLL(D1D)
-               for (int qy = 0; qy < Q1D; ++qy) { u[qy] += zyX * s_B[qy][dy]; }
-            }
-            MFEM_UNROLL(Q1D)
-            for (int qy = 0; qy < Q1D; ++qy) { s_q[dz][qy][qx] = u[qy]; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Z interpolation, Q-function & Zt projection
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            // Z interpolation
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz) { u[qz] = 0.0; }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               const double zYX = s_q[dz][qy][qx];
-               MFEM_UNROLL(Q1D)
-               for (int qz = 0; qz < Q1D; ++qz) { u[qz] += zYX * s_B[qz][dz]; }
-            }
-
-            // Q-function
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               s_q[qz][qy][qx] = u[qz] * D(qx,qy,qz,e);
-            }
-
-            // Zt projection
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               const double ZYX = s_q[qz][qy][qx];
-               MFEM_UNROLL(D1D)
-               for (int dz = 0; dz < D1D; ++dz) { u[dz] += ZYX * s_B[qz][dz]; }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz) { s_q[dz][qy][qx] = u[dz]; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Yt projection
-      MFEM_FOREACH_THREAD(dz,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            MFEM_UNROLL(D1D)
-            for (int dy = 0; dy < D1D; ++dy) { u[dy] = 0.0; }
-            MFEM_UNROLL(Q1D)
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const double zYX = s_q[dz][qy][qx];
-               MFEM_UNROLL(D1D)
-               for (int dy = 0; dy < D1D; ++dy) { u[dy] += zYX * s_B[qy][dy]; }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dy = 0; dy < D1D; ++dy) { s_q[dz][dy][qx] = u[dy]; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Xt projection & save output
-      MFEM_FOREACH_THREAD(dz,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dy,x,D1D)
-         {
-            MFEM_UNROLL(D1D)
-            for (int dx = 0; dx < D1D; ++dx) { u[dx] = 0.0; }
-            MFEM_UNROLL(Q1D)
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const double zyX = s_q[dz][dy][qx];
-               MFEM_UNROLL(D1D)
-               for (int dx = 0; dx < D1D; ++dx) { u[dx] += zyX * s_B[qx][dx]; }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               const double output = u[dx];
-               Y(dx,dy,dz,e) += output;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-   });
-}
-
 static void PAMassApply(const int dim,
                        const int D1D,
                        const int Q1D,
@@ -1067,47 +1201,28 @@ static void PAMassApply(const int dim,
   }
   else if (dim == 3)
   {
-      const int ver = Device::KernelsVersion();
-      const int id = (ver << 8) | (D1D << 4) | Q1D;
-
-      static int ini = 0;
-      if (!ini++) { printf("\033[33mkernel #0x%x\033[m\n",id); }
-
      switch (id)
      {
-         case 0x123: return SmemPAMassApply3D_v1<2,3>(NE,B,Bt,D,X,Y);
-         case 0x124: return SmemPAMassApply3D_v1<2,4>(NE,B,Bt,D,X,Y);
-         case 0x134: return SmemPAMassApply3D_v1<3,4>(NE,B,Bt,D,X,Y);
-         case 0x136: return SmemPAMassApply3D_v1<3,6>(NE,B,Bt,D,X,Y);
-         case 0x145: return SmemPAMassApply3D_v1<4,5>(NE,B,Bt,D,X,Y);
-         case 0x148: return SmemPAMassApply3D_v1<4,8>(NE,B,Bt,D,X,Y);
-         case 0x156: return SmemPAMassApply3D_v1<5,6>(NE,B,Bt,D,X,Y);
-         case 0x158: return SmemPAMassApply3D_v1<5,8>(NE,B,Bt,D,X,Y);
-         case 0x167: return SmemPAMassApply3D_v1<6,7>(NE,B,Bt,D,X,Y);
-         case 0x178: return SmemPAMassApply3D_v1<7,8>(NE,B,Bt,D,X,Y);
-
-         case 0x022: return SmemPAMassApply3D<2,2>(NE,B,Bt,D,X,Y);
-         case 0x023: return SmemPAMassApply3D<2,3>(NE,B,Bt,D,X,Y);
-         case 0x024: return SmemPAMassApply3D<2,4>(NE,B,Bt,D,X,Y);
-         case 0x026: return SmemPAMassApply3D<2,6>(NE,B,Bt,D,X,Y);
-         case 0x034: return SmemPAMassApply3D<3,4>(NE,B,Bt,D,X,Y);
-         case 0x035: return SmemPAMassApply3D<3,5>(NE,B,Bt,D,X,Y);
-         case 0x036: return SmemPAMassApply3D<3,6>(NE,B,Bt,D,X,Y);
-         case 0x037: return SmemPAMassApply3D<3,7>(NE,B,Bt,D,X,Y);
-         case 0x045: return SmemPAMassApply3D<4,5>(NE,B,Bt,D,X,Y);
-         case 0x046: return SmemPAMassApply3D<4,6>(NE,B,Bt,D,X,Y);
-         case 0x048: return SmemPAMassApply3D<4,8>(NE,B,Bt,D,X,Y);
-         case 0x056: return SmemPAMassApply3D<5,6>(NE,B,Bt,D,X,Y);
-         case 0x058: return SmemPAMassApply3D<5,8>(NE,B,Bt,D,X,Y);
-         case 0x067: return SmemPAMassApply3D<6,7>(NE,B,Bt,D,X,Y);
-         case 0x078: return SmemPAMassApply3D<7,8>(NE,B,Bt,D,X,Y);
-         case 0x089: return SmemPAMassApply3D<8,9>(NE,B,Bt,D,X,Y);
-         case 0x09A: return SmemPAMassApply3D<9,10>(NE,B,Bt,D,X,Y);
-
-         default:   break; //return PAMassApply3D(NE,B,Bt,D,X,Y,D1D,Q1D);
+         case 0x23: return SmemPAMassApply3D<2,3>(NE,B,Bt,D,X,Y);
+         case 0x24: return SmemPAMassApply3D<2,4>(NE,B,Bt,D,X,Y);
+         case 0x34: return SmemPAMassApply3D<3,4>(NE,B,Bt,D,X,Y);
+         case 0x35: return SmemPAMassApply3D<3,5>(NE,B,Bt,D,X,Y);
+         case 0x36: return SmemPAMassApply3D<3,6>(NE,B,Bt,D,X,Y);
+         case 0x37: return SmemPAMassApply3D<3,7>(NE,B,Bt,D,X,Y);
+         case 0x45: return SmemPAMassApply3D<4,5>(NE,B,Bt,D,X,Y);
+         case 0x46: return SmemPAMassApply3D<4,6>(NE,B,Bt,D,X,Y);
+         case 0x48: return SmemPAMassApply3D<4,8>(NE,B,Bt,D,X,Y);
+         case 0x56: return SmemPAMassApply3D<5,6>(NE,B,Bt,D,X,Y);
+         case 0x58: return SmemPAMassApply3D<5,8>(NE,B,Bt,D,X,Y);
+         case 0x67: return SmemPAMassApply3D<6,7>(NE,B,Bt,D,X,Y);
+         case 0x78: return SmemPAMassApply3D<7,8>(NE,B,Bt,D,X,Y);
+         case 0x89: return SmemPAMassApply3D<8,9>(NE,B,Bt,D,X,Y);
+         case 0x9A: return SmemPAMassApply3D<9,10>(NE,B,Bt,D,X,Y);
+         default:   return PAMassApply3D(NE,B,Bt,D,X,Y,D1D,Q1D);
      }
   }
-   MFEM_ABORT("Unknown kernel 0x" << std::hex << id);
+   mfem::out << "Unknown kernel 0x" << std::hex << id << std::endl;
+   MFEM_ABORT("Unknown kernel.");
 }

 void MassIntegrator::AddMultPA(const Vector &x, Vector &y) const
@@ -1116,35 +1231,6 @@ void MassIntegrator::AddMultPA(const Vector &x, Vector &y) const
   {
      ceedOp->AddMult(x, y);
   }
-   else if (Device::FastKernelsEnabled())
-   {
-      const int version = Device::KernelsVersion();
-      MFEM_VERIFY(version < 4 || version==7, "Unsupported version!");
-      if (version == 3) // AMD
-      {
-         NDK_AMD_PAMassApply(dim, dofs1D, quad1D, ne,
-                             fespace, maps,
-                             pa_data, x, y);
-      }
-      // 4 E-vector
-      // 5 fused
-      // 6 MMA
-      else if (version == 7) // HIP
-      {
-         NDK_HIP_PAMassApply(dim, dofs1D, quad1D, ne,
-                             fespace, maps,
-                             pa_data, x, y);
-      }
-      // 0 legacy
-      // 1 fast
-      // 2 libP
-      else
-      {
-         NDK_PAMassApply(dim, dofs1D, quad1D, ne,
-                         fespace, maps,
-                         pa_data, x, y);
-      }
-   }
   else
   {
      PAMassApply(dim, dofs1D, quad1D, ne, maps->B, maps->Bt, pa_data, x, y);
@@ -1,358 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-#include "ceed/mass.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-void NDK_PAMassAssembleDiagonal(const int dim,
-                                const int D1D,
-                                const int Q1D,
-                                const int NE,
-                                const FiniteElementSpace *fes,
-                                const DofToQuad *maps,
-                                const Vector &D,
-                                Vector &Y);
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void PAMassAssembleDiagonal2D(const int NE,
-                                     const Array<double> &b,
-                                     const Vector &d,
-                                     Vector &y,
-                                     const int d1d = 0,
-                                     const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto D = Reshape(d.Read(), Q1D, Q1D, NE);
-   auto Y = Reshape(y.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      double QD[MQ1][MD1];
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            QD[qx][dy] = 0.0;
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               QD[qx][dy] += B(qy, dy) * B(qy, dy) * D(qx, qy, e);
-            }
-         }
-      }
-      for (int dy = 0; dy < D1D; ++dy)
-      {
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               Y(dx,dy,e) += B(qx, dx) * B(qx, dx) * QD[qx][dy];
-            }
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
-static void SmemPAMassAssembleDiagonal2D(const int NE,
-                                         const Array<double> &b_,
-                                         const Vector &d_,
-                                         Vector &y_,
-                                         const int d1d = 0,
-                                         const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-   MFEM_VERIFY(D1D <= MD1, "");
-   MFEM_VERIFY(Q1D <= MQ1, "");
-   auto b = Reshape(b_.Read(), Q1D, D1D);
-   auto D = Reshape(d_.Read(), Q1D, Q1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      MFEM_SHARED double B[MQ1][MD1];
-      MFEM_SHARED double QDZ[NBZ][MQ1][MD1];
-      double (*QD)[MD1] = (double (*)[MD1])(QDZ + tidz);
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B[q][d] = b(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            QD[qx][dy] = 0.0;
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               QD[qx][dy] += B[qy][dy] * B[qy][dy] * D(qx, qy, e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               // might need absolute values on next line
-               Y(dx,dy,e) += B[qx][dx] * B[qx][dx] * QD[qx][dy];
-            }
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void PAMassAssembleDiagonal3D(const int NE,
-                                     const Array<double> &b,
-                                     const Vector &d,
-                                     Vector &y,
-                                     const int d1d = 0,
-                                     const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto D = Reshape(d.Read(), Q1D, Q1D, Q1D, NE);
-   auto Y = Reshape(y.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      double QQD[MQ1][MQ1][MD1];
-      double QDD[MQ1][MD1][MD1];
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               QQD[qx][qy][dz] = 0.0;
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  QQD[qx][qy][dz] += B(qz, dz) * B(qz, dz) * D(qx, qy, qz, e);
-               }
-            }
-         }
-      }
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               QDD[qx][dy][dz] = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  QDD[qx][dy][dz] += B(qy, dy) * B(qy, dy) * QQD[qx][qy][dz];
-               }
-            }
-         }
-      }
-      for (int dz = 0; dz < D1D; ++dz)
-      {
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               double t = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  t += B(qx, dx) * B(qx, dx) * QDD[qx][dy][dz];
-               }
-               Y(dx, dy, dz, e) += t;
-            }
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void SmemPAMassAssembleDiagonal3D(const int NE,
-                                         const Array<double> &b_,
-                                         const Vector &d_,
-                                         Vector &y_,
-                                         const int d1d = 0,
-                                         const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-   MFEM_VERIFY(D1D <= MD1, "");
-   MFEM_VERIFY(Q1D <= MQ1, "");
-   const auto b = Reshape(b_.Read(), Q1D, D1D);
-   const auto D = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      MFEM_SHARED double B[MQ1][MD1];
-      MFEM_SHARED double QQD[MQ1][MQ1][MD1];
-      MFEM_SHARED double QDD[MQ1][MD1][MD1];
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B[q][d] = b(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(dz,z,D1D)
-            {
-               QQD[qx][qy][dz] = 0.0;
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  QQD[qx][qy][dz] += B[qz][dz] * B[qz][dz] * D(qx, qy, qz, e);
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dz,z,D1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               QDD[qx][dy][dz] = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  QDD[qx][dy][dz] += B[qy][dy] * B[qy][dy] * QQD[qx][qy][dz];
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               double t = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  t += B[qx][dx] * B[qx][dx] * QDD[qx][dy][dz];
-               }
-               Y(dx, dy, dz, e) += t;
-            }
-         }
-      }
-   });
-}
-
-static void PAMassAssembleDiagonal(const int dim, const int D1D,
-                                   const int Q1D, const int NE,
-                                   const Array<double> &B,
-                                   const Vector &D,
-                                   Vector &Y)
-{
-   if (dim == 2)
-   {
-      switch ((D1D << 4 ) | Q1D)
-      {
-         case 0x22: return SmemPAMassAssembleDiagonal2D<2,2,16>(NE,B,D,Y);
-         case 0x33: return SmemPAMassAssembleDiagonal2D<3,3,16>(NE,B,D,Y);
-         case 0x44: return SmemPAMassAssembleDiagonal2D<4,4,8>(NE,B,D,Y);
-         case 0x55: return SmemPAMassAssembleDiagonal2D<5,5,8>(NE,B,D,Y);
-         case 0x66: return SmemPAMassAssembleDiagonal2D<6,6,4>(NE,B,D,Y);
-         case 0x77: return SmemPAMassAssembleDiagonal2D<7,7,4>(NE,B,D,Y);
-         case 0x88: return SmemPAMassAssembleDiagonal2D<8,8,2>(NE,B,D,Y);
-         case 0x99: return SmemPAMassAssembleDiagonal2D<9,9,2>(NE,B,D,Y);
-         default:   return PAMassAssembleDiagonal2D(NE,B,D,Y,D1D,Q1D);
-      }
-   }
-   else if (dim == 3)
-   {
-      switch ((D1D << 4 ) | Q1D)
-      {
-         case 0x23: return SmemPAMassAssembleDiagonal3D<2,3>(NE,B,D,Y);
-         case 0x24: return SmemPAMassAssembleDiagonal3D<2,4>(NE,B,D,Y);
-         case 0x26: return SmemPAMassAssembleDiagonal3D<2,6>(NE,B,D,Y);
-         case 0x34: return SmemPAMassAssembleDiagonal3D<3,4>(NE,B,D,Y);
-         case 0x35: return SmemPAMassAssembleDiagonal3D<3,5>(NE,B,D,Y);
-         case 0x45: return SmemPAMassAssembleDiagonal3D<4,5>(NE,B,D,Y);
-         case 0x48: return SmemPAMassAssembleDiagonal3D<4,8>(NE,B,D,Y);
-         case 0x56: return SmemPAMassAssembleDiagonal3D<5,6>(NE,B,D,Y);
-         case 0x67: return SmemPAMassAssembleDiagonal3D<6,7>(NE,B,D,Y);
-         case 0x78: return SmemPAMassAssembleDiagonal3D<7,8>(NE,B,D,Y);
-         case 0x89: return SmemPAMassAssembleDiagonal3D<8,9>(NE,B,D,Y);
-         default:   return PAMassAssembleDiagonal3D(NE,B,D,Y,D1D,Q1D);
-      }
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-
-void MassIntegrator::AssembleDiagonalPA(Vector &diag)
-{
-   if (DeviceCanUseCeed())
-   {
-      ceedOp->GetDiagonal(diag);
-   }
-   else if (Device::FastKernelsEnabled())
-   {
-      NDK_PAMassAssembleDiagonal(dim, dofs1D, quad1D, ne,
-                                 fespace, maps,
-                                 pa_data, diag);
-   }
-   else
-   {
-      PAMassAssembleDiagonal(dim, dofs1D, quad1D, ne, maps->B, pa_data, diag);
-   }
-}
-
-} // namespace mfem
@@ -1,684 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../general/forall.hpp"
-#include "gridfunc.hpp"
-#include "restriction.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-// Fast '0' non-deterministic 3D mass kernel
-template<int D1D, int Q1D>
-void NDK_SmemPAMassApply3D(const int ndofs,
-                           const int NE,
-                           const int *map,
-                           const double *b_,
-                           const double *d_,
-                           const double *x_,
-                           double *y_)
-{
-   const auto MAP = Reshape(map, D1D,D1D,D1D, NE);
-   const auto b = Reshape(b_, Q1D, D1D);
-   const auto D = Reshape(d_, Q1D, Q1D, Q1D, NE);
-   const auto X = Reshape(x_, ndofs);
-   const auto X1 = Reshape(x_, D1D,D1D,D1D, NE);
-   auto Y = Reshape(y_, ndofs);
-   auto Y1 = Reshape(y_, D1D,D1D,D1D, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
-   {
-      MFEM_SHARED double sDQ[Q1D*Q1D];
-      double (*B)[D1D] = (double (*)[D1D]) sDQ;
-      double (*Bt)[Q1D] = (double (*)[Q1D]) sDQ;
-      MFEM_SHARED double sm0[Q1D*Q1D*Q1D];
-      MFEM_SHARED double sm1[Q1D*Q1D*Q1D];
-      double (*DDQ)[D1D][Q1D] = (double (*)[D1D][Q1D]) sm1;
-      double (*DQQ)[Q1D][Q1D] = (double (*)[Q1D][Q1D]) sm0;
-      double (*QQQ)[Q1D][Q1D] = (double (*)[Q1D][Q1D]) sm1;
-      double (*QQD)[Q1D][D1D] = (double (*)[Q1D][D1D]) sm0;
-      double (*QDD)[D1D][D1D] = (double (*)[D1D][D1D]) sm1;
-
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            B[qx][dy] = b(qx,dy);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double u[D1D];
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; dz++)
-            {
-               u[dz] = 0;
-            }
-            MFEM_UNROLL(D1D)
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               MFEM_UNROLL(D1D)
-               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  if (map)
-                  {
-                     const int gid = MAP(dx, dy, dz, e);
-                     const int idx = gid >= 0 ? gid : -1 - gid;
-                     u[dz] += X(idx) * B[qx][dx];
-                  }
-                  else
-                  {
-                     u[dz] += X1(dx,dy,dz,e) * B[qx][dx];
-                  }
-               }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               DDQ[dz][dy][qx] = u[dz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double u[D1D];
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; dz++)
-            {
-               u[dz] = 0;
-            }
-            MFEM_UNROLL(D1D)
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               MFEM_UNROLL(D1D)
-               for (int dz = 0; dz < D1D; dz++)
-               {
-                  u[dz] += DDQ[dz][dy][qx] * B[qy][dy];
-               }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; dz++)
-            {
-               DQQ[dz][qy][qx] = u[dz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double u[Q1D];
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; qz++)
-            {
-               u[qz] = 0;
-            }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               MFEM_UNROLL(Q1D)
-               for (int qz = 0; qz < Q1D; qz++)
-               {
-                  u[qz] += DQQ[dz][qy][qx] * B[qz][dz];
-               }
-            }
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; qz++)
-            {
-               QQQ[qz][qy][qx] = u[qz] * D(qx,qy,qz,e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(d,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(q,x,Q1D)
-         {
-            Bt[d][q] = b(q,d);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[Q1D];
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               u[qz] = 0;
-            }
-            MFEM_UNROLL(Q1D)
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               MFEM_UNROLL(Q1D)
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u[qz] += QQQ[qz][qy][qx] * Bt[dx][qx];
-               }
-            }
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               QQD[qz][qy][dx] = u[qz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[Q1D];
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               u[qz] = 0;
-            }
-            MFEM_UNROLL(Q1D)
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               MFEM_UNROLL(Q1D)
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u[qz] += QQD[qz][qy][dx] * Bt[dy][qy];
-               }
-            }
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               QDD[qz][dy][dx] = u[qz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[D1D];
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               u[dz] = 0;
-            }
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               MFEM_UNROLL(D1D)
-               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  u[dz] += QDD[qz][dy][dx] * Bt[dz][qz];
-               }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               if (map)
-               {
-                  const int gid = MAP(dx, dy, dz, e);
-                  const int idx = gid >= 0 ? gid : -1 - gid;
-                  AtomicAdd(Y(idx), u[dz]);
-               }
-               else
-               {
-                  Y1(dx,dy,dz,e) += u[dz];
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-   });
-}
-
-// Fast '1' non-deterministic 3D mass kernel
-// Smem version melded toward registers + BZ-batch
-template<int D1D, int Q1D, int NBZ>
-void NDK_SmRgPAMassApply3D(const int ndofs,
-                           const int NE,
-                           const int *map,
-                           const double *b_,
-                           const double *d_,
-                           const double *x_,
-                           double *y_)
-{
-   const auto MAP = Reshape(map, D1D,D1D,D1D, NE);
-   const auto B = Reshape(b_, Q1D, D1D);
-   const auto D = Reshape(d_, Q1D, Q1D, Q1D, NE);
-   const auto X = Reshape(x_, ndofs);
-   const auto X1 = Reshape(x_, D1D,D1D,D1D, NE);
-   auto Y = Reshape(y_, ndofs);
-   auto Y1 = Reshape(y_, D1D,D1D,D1D, NE);
-
-   MFEM_FORALL_3D(be, (NE+NBZ-1)/NBZ, Q1D, Q1D, NBZ,
-   {
-      double u[Q1D];
-      const int tz = MFEM_THREAD_ID(z);
-      const int e = be * MFEM_THREAD_SIZE(z) + tz;
-
-      MFEM_SHARED double s_B[Q1D][D1D];
-      MFEM_SHARED double s_q[NBZ][Q1D][Q1D][Q1D];
-
-      // Load input, B & X interpolation
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            if (tz == 0) { s_B[qx][dy] = B(qx,dy); }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
-            MFEM_UNROLL(D1D)
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               const double Bx = B(qx,dx);
-               MFEM_UNROLL(D1D)
-               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  const int gid = map ? MAP(dx,dy,dz,e) : 0;
-                  const int idx = gid >= 0 ? gid : -1 - gid;
-                  u[dz] += (map ? X(idx) : X1(dx,dy,dz,e)) * Bx;
-               }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz) { s_q[tz][dz][dy][qx] = u[dz]; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Y interpolation
-      MFEM_FOREACH_THREAD(dz,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            MFEM_UNROLL(Q1D)
-            for (int qy = 0; qy < Q1D; ++qy) { u[qy] = 0.0; }
-            MFEM_UNROLL(D1D)
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               const double zyX = s_q[tz][dz][dy][qx];
-               MFEM_UNROLL(D1D)
-               for (int qy = 0; qy < Q1D; ++qy) { u[qy] += zyX * s_B[qy][dy]; }
-            }
-            MFEM_UNROLL(Q1D)
-            for (int qy = 0; qy < Q1D; ++qy) { s_q[tz][dz][qy][qx] = u[qy]; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Z interpolation, Q-function & Zt projection
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            // Z interpolation
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz) { u[qz] = 0.0; }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               const double zYX = s_q[tz][dz][qy][qx];
-               MFEM_UNROLL(Q1D)
-               for (int qz = 0; qz < Q1D; ++qz) { u[qz] += zYX * s_B[qz][dz]; }
-            }
-
-            // Q-function
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               s_q[tz][qz][qy][qx] = u[qz] * D(qx,qy,qz,e);
-            }
-
-            // Zt projection
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz) { u[dz] = 0.0; }
-            MFEM_UNROLL(Q1D)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               const double ZYX = s_q[tz][qz][qy][qx];
-               MFEM_UNROLL(D1D)
-               for (int dz = 0; dz < D1D; ++dz) { u[dz] += ZYX * s_B[qz][dz]; }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dz = 0; dz < D1D; ++dz) { s_q[tz][dz][qy][qx] = u[dz]; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Yt projection
-      MFEM_FOREACH_THREAD(dz,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            MFEM_UNROLL(D1D)
-            for (int dy = 0; dy < D1D; ++dy) { u[dy] = 0.0; }
-            MFEM_UNROLL(Q1D)
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const double zYX = s_q[tz][dz][qy][qx];
-               MFEM_UNROLL(D1D)
-               for (int dy = 0; dy < D1D; ++dy) { u[dy] += zYX * s_B[qy][dy]; }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dy = 0; dy < D1D; ++dy) { s_q[tz][dz][dy][qx] = u[dy]; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Xt projection & save output
-      MFEM_FOREACH_THREAD(dz,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dy,x,D1D)
-         {
-            MFEM_UNROLL(D1D)
-            for (int dx = 0; dx < D1D; ++dx) { u[dx] = 0.0; }
-            MFEM_UNROLL(Q1D)
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const double zyX = s_q[tz][dz][dy][qx];
-               MFEM_UNROLL(D1D)
-               for (int dx = 0; dx < D1D; ++dx) { u[dx] += zyX * s_B[qx][dx]; }
-            }
-            MFEM_UNROLL(D1D)
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               const double output = u[dx];
-               if (map)
-               {
-                  const int gid = MAP(dx,dy,dz,e);
-                  const int idx = gid >= 0 ? gid : -1 - gid;
-                  AtomicAdd(Y(idx), output);
-               }
-               else
-               {
-                  Y1(dx,dy,dz,e) += output;
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-   });
-}
-
-// Fast '2' non-deterministic 3D mass kernel
-// Smem version with registers
-template<int D1D, int Q1D>
-void NDK_RegsPAMassApply3D(const int ndofs,
-                           const int NE,
-                           const int *map,
-                           const double *b_,
-                           const double *d_,
-                           const double *x_,
-                           double *y_)
-{
-   const auto MAP = Reshape(map, D1D,D1D,D1D, NE);
-   const auto B = Reshape(b_, Q1D,D1D);
-   const auto D = Reshape(d_, Q1D,Q1D,Q1D, NE);
-   const auto X = Reshape(x_, ndofs);
-   const auto X1 = Reshape(x_, D1D,D1D,D1D, NE);
-   auto Y = Reshape(y_, ndofs);
-   auto Y1 = Reshape(y_, D1D,D1D,D1D, NE);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
-   {
-      double r_wk[Q1D];
-      MFEM_SHARED double s_B[Q1D][D1D];
-      MFEM_SHARED double s_q[Q1D][Q1D][Q1D];
-
-      // Load s_B, load X in shared memory
-      MFEM_FOREACH_THREAD(b,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(a,x,Q1D)
-         {
-            if (a<D1D) { s_B[b][a] = B(b,a); }
-
-            MFEM_UNROLL(Q1D)
-            for (int i=0; i<Q1D; ++i) { r_wk[i] = 0.0; }
-
-            if (a<D1D && b<D1D)
-            {
-               MFEM_UNROLL(D1D)
-               for (int c=0; c<D1D; ++c)
-               {
-                  const int gid = map ? MAP(a,b,c,e) : 0;
-                  const int idx = gid >= 0 ? gid : -1 - gid;
-                  s_q[c][b][a] = map ? X(idx) : X1(a,b,c,e);
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Interpolate in X direction
-      MFEM_FOREACH_THREAD(c,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(b,x,Q1D)
-         {
-            if (b<D1D && c<D1D)
-            {
-               MFEM_UNROLL(D1D)
-               for (int a=0; a<D1D; ++a)
-               {
-                  const double q_cba = s_q[c][b][a];
-                  MFEM_UNROLL(Q1D)
-                  for (int i=0; i<Q1D; ++i) { r_wk[i] += s_B[i][a]*q_cba; }
-               }
-               // reg => s_mem
-               MFEM_UNROLL(Q1D)
-               for (int i=0; i<Q1D; ++i) { s_q[c][b][i] = r_wk[i]; }
-            }
-            MFEM_UNROLL(Q1D)
-            for (int j=0; j<Q1D; ++j) { r_wk[j] = 0.0; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Interpolate in Y direction
-      MFEM_FOREACH_THREAD(c,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(i,x,Q1D)
-         {
-            if (c<D1D)
-            {
-               MFEM_UNROLL(D1D)
-               for (int b=0; b<D1D; ++b)
-               {
-                  const double q_cbi = s_q[c][b][i];
-                  MFEM_UNROLL(Q1D)
-                  for (int j=0; j<Q1D; ++j) { r_wk[j] += s_B[j][b]*q_cbi; }
-               }
-               MFEM_UNROLL(Q1D)
-               for (int j=0; j<Q1D; ++j) { s_q[c][j][i] = r_wk[j]; }
-            }
-            MFEM_UNROLL(Q1D)
-            for (int k=0; k<Q1D; ++k) { r_wk[k] = 0.0; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(j,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(i,x,Q1D)
-         {
-            // Interpolate in Z direction
-            MFEM_UNROLL(D1D)
-            for (int c=0; c<D1D; ++c)
-            {
-               const double q_cji = s_q[c][j][i];
-               MFEM_UNROLL(Q1D)
-               for (int k=0; k<Q1D; ++k) { r_wk[k] += s_B[k][c]*q_cji; }
-            }
-
-            // Scale by Jacobian and integration weights
-            MFEM_UNROLL(Q1D)
-            for (int k=0; k<Q1D; ++k) { r_wk[k] *= D(i,j,k,e); }
-
-            // Project back in Z direction
-            MFEM_UNROLL(D1D)
-            for (int c=0; c<D1D; ++c)
-            {
-               double q_cji = 0.0;
-               MFEM_UNROLL(Q1D)
-               for (int k=0; k<Q1D; ++k) { q_cji += s_B[k][c] * r_wk[k]; }
-               s_q[c][j][i] = q_cji;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Project back in Y direction
-      MFEM_FOREACH_THREAD(c,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(i,x,Q1D)
-         {
-            MFEM_UNROLL(Q1D)
-            for (int j=0; j<Q1D; ++j) { r_wk[j] = s_q[c][j][i]; }
-            MFEM_UNROLL(D1D)
-            for (int b=0; b<D1D; ++b)
-            {
-               double q_cbi = 0.0;
-               MFEM_UNROLL(Q1D)
-               for (int j=0; j<Q1D; ++j) { q_cbi += s_B[j][b] * r_wk[j]; }
-               s_q[c][b][i] = q_cbi;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Project back in X direction
-      MFEM_FOREACH_THREAD(c,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(b,x,D1D)
-         {
-            MFEM_UNROLL(Q1D)
-            for (int i=0; i<Q1D; ++i) { r_wk[i] = s_q[c][b][i]; }
-            MFEM_UNROLL(D1D)
-            for (int a=0; a<D1D; ++a)
-            {
-               double q_cba = 0.0;
-               MFEM_UNROLL(Q1D)
-               for (int i=0; i<Q1D; ++i) { q_cba += s_B[i][a] * r_wk[i]; }
-               s_q[c][b][a] = q_cba;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Save back to memory
-      MFEM_FOREACH_THREAD(b,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(a,x,D1D)
-         {
-            MFEM_UNROLL(D1D)
-            for (int c=0; c<D1D; ++c)
-            {
-               const double q_cba = s_q[c][b][a];
-               const int gid = map ? MAP(a,b,c,e) : 0;
-               const int idx = gid >= 0 ? gid : -1 - gid;
-               AtomicAdd(map?Y(idx):Y1(a,b,c,e), q_cba);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-   });
-}
-
-void NDK_PAMassApply(const int dim,
-                     const int D1D,
-                     const int Q1D,
-                     const int NE,
-                     const FiniteElementSpace *fes,
-                     const DofToQuad *maps,
-                     const Vector &D,
-                     const Vector &X,
-                     Vector &Y)
-{
-   const int ND = fes->GetNDofs();
-   constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
-   const Operator *ERop = fes->GetElementRestriction(ordering);
-   const ElementRestriction* ER = dynamic_cast<const ElementRestriction*>(ERop);
-   const int *map = ER ? ER->GatherMap().Read() : nullptr;
-   const double *b = maps->B.Read();
-   const double *d = D.Read();
-   const double *x = X.Read();
-   double *y = Y.ReadWrite();
-
-   assert(dim == 3);
-   const int ver = Device::KernelsVersion();
-   const int id = (ver << 8) | (D1D << 4) | Q1D;
-
-   //printf("\033[32mkernel #0x%x\033[m\n",id); fflush(0);
-
-   switch (id) // orders 1~6
-   {
-      // Fast '0': Legacy non-deterministic 3D mass kernel
-      case 0x023: return NDK_SmemPAMassApply3D<2,3>(ND,NE,map,b,d,x,y);
-      case 0x024: return NDK_SmemPAMassApply3D<2,4>(ND,NE,map,b,d,x,y);
-      case 0x034: return NDK_SmemPAMassApply3D<3,4>(ND,NE,map,b,d,x,y);
-      case 0x036: return NDK_SmemPAMassApply3D<3,6>(ND,NE,map,b,d,x,y);
-      case 0x045: return NDK_SmemPAMassApply3D<4,5>(ND,NE,map,b,d,x,y);
-      case 0x046: return NDK_SmemPAMassApply3D<4,6>(ND,NE,map,b,d,x,y);
-      case 0x048: return NDK_SmemPAMassApply3D<4,8>(ND,NE,map,b,d,x,y);
-      case 0x056: return NDK_SmemPAMassApply3D<5,6>(ND,NE,map,b,d,x,y);
-      case 0x058: return NDK_SmemPAMassApply3D<5,8>(ND,NE,map,b,d,x,y);
-      case 0x067: return NDK_SmemPAMassApply3D<6,7>(ND,NE,map,b,d,x,y);
-      case 0x078: return NDK_SmemPAMassApply3D<7,8>(ND,NE,map,b,d,x,y);
-
-      // Fast '1': Legacy & half smem non-deterministic 3D mass kernel + Z-batch
-      case 0x123: return NDK_SmRgPAMassApply3D<2,3,32>(ND,NE,map,b,d,x,y);//1
-      case 0x124: return NDK_SmRgPAMassApply3D<2,4,16>(ND,NE,map,b,d,x,y);
-      case 0x134: return NDK_SmRgPAMassApply3D<3,4,16>(ND,NE,map,b,d,x,y);//2
-      case 0x136: return NDK_SmRgPAMassApply3D<3,6,8>(ND,NE,map,b,d,x,y);
-      case 0x145: return NDK_SmRgPAMassApply3D<4,5,4>(ND,NE,map,b,d,x,y);//3
-      case 0x146: return NDK_SmRgPAMassApply3D<4,6,4>(ND,NE,map,b,d,x,y);
-      case 0x148: return NDK_SmRgPAMassApply3D<4,8,4>(ND,NE,map,b,d,x,y);
-      case 0x156: return NDK_SmRgPAMassApply3D<5,6,4>(ND,NE,map,b,d,x,y);//4
-      case 0x158: return NDK_SmRgPAMassApply3D<5,8,1>(ND,NE,map,b,d,x,y);
-      case 0x167: return NDK_SmRgPAMassApply3D<6,7,1>(ND,NE,map,b,d,x,y);//5
-      case 0x178: return NDK_SmRgPAMassApply3D<7,8,1>(ND,NE,map,b,d,x,y);//6
-
-      // Fast '2': libP non-deterministic 3D mass kernel
-      case 0x223: return NDK_RegsPAMassApply3D<2,3>(ND,NE,map,b,d,x,y);
-      case 0x224: return NDK_RegsPAMassApply3D<2,4>(ND,NE,map,b,d,x,y);
-      case 0x234: return NDK_RegsPAMassApply3D<3,4>(ND,NE,map,b,d,x,y);
-      case 0x236: return NDK_RegsPAMassApply3D<3,6>(ND,NE,map,b,d,x,y);
-      case 0x245: return NDK_RegsPAMassApply3D<4,5>(ND,NE,map,b,d,x,y);
-      case 0x246: return NDK_RegsPAMassApply3D<4,6>(ND,NE,map,b,d,x,y);
-      case 0x248: return NDK_RegsPAMassApply3D<4,8>(ND,NE,map,b,d,x,y);
-      case 0x256: return NDK_RegsPAMassApply3D<5,6>(ND,NE,map,b,d,x,y);
-      case 0x258: return NDK_RegsPAMassApply3D<5,8>(ND,NE,map,b,d,x,y);
-      case 0x267: return NDK_RegsPAMassApply3D<6,7>(ND,NE,map,b,d,x,y);
-      case 0x278: return NDK_RegsPAMassApply3D<7,8>(ND,NE,map,b,d,x,y);
-
-      // Fast '3': libP + AMD specific non-deterministic 3D mass kernel
-      // in bilininteg_mass_pa_fast_amd.cpp
-
-      default: break;
-   }
-
-   MFEM_ABORT("Unknown kernel 0x" << std::hex << id);
-}
-
-} // namespace mfem
@@ -1,250 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../general/forall.hpp"
-#include "gridfunc.hpp"
-#include "restriction.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-// Fast '3' non-deterministic 3D mass kernel
-template<int D1D, int Q1D>
-void NDK_AMD_PAMassApply3D(const int ndofs,
-                           const int NE,
-                           const int *map,
-                           const double *b_,
-                           const double *d_,
-                           const double *x_,
-                           double *y_)
-{
-   const auto MAP = Reshape(map, D1D,D1D,D1D, NE);
-   const auto B = Reshape(b_, Q1D,D1D);
-   const auto D = Reshape(d_, Q1D,Q1D,Q1D, NE);
-   const auto X = Reshape(x_, ndofs);
-   const auto X1 = Reshape(x_, D1D,D1D,D1D, NE);
-   auto Y = Reshape(y_, ndofs);
-   auto Y1 = Reshape(y_, D1D,D1D,D1D, NE);
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
-   {
-      double r_wk[Q1D];
-      MFEM_SHARED double s_B[Q1D][D1D];
-      MFEM_SHARED double s_q[Q1D][Q1D][Q1D];
-
-      // Load s_B, load X in shared memory
-      MFEM_FOREACH_THREAD(b,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(a,x,Q1D)
-         {
-            if (a<D1D) { s_B[b][a] = B(b,a); }
-
-            MFEM_UNROLL(Q1D)
-            for (int i=0; i<Q1D; ++i) { r_wk[i] = 0.0; }
-
-            if (a<D1D && b<D1D)
-            {
-               MFEM_UNROLL(D1D)
-               for (int c=0; c<D1D; ++c)
-               {
-                  const int gid = map ? MAP(a,b,c,e) : 0;
-                  const int idx = gid >= 0 ? gid : -1 - gid;
-                  s_q[c][b][a] = map ? X(idx) : X1(a,b,c,e);
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Interpolate in X direction
-      MFEM_FOREACH_THREAD(c,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(b,x,Q1D)
-         {
-            if (b<D1D && c<D1D)
-            {
-               MFEM_UNROLL(D1D)
-               for (int a=0; a<D1D; ++a)
-               {
-                  const double q_cba = s_q[c][b][a];
-                  MFEM_UNROLL(Q1D)
-                  for (int i=0; i<Q1D; ++i) { r_wk[i] += s_B[i][a]*q_cba; }
-               }
-               // reg => s_mem
-               MFEM_UNROLL(Q1D)
-               for (int i=0; i<Q1D; ++i) { s_q[c][b][i] = r_wk[i]; }
-            }
-            MFEM_UNROLL(Q1D)
-            for (int j=0; j<Q1D; ++j) { r_wk[j] = 0.0; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Interpolate in Y direction
-      MFEM_FOREACH_THREAD(c,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(i,x,Q1D)
-         {
-            if (c<D1D)
-            {
-               MFEM_UNROLL(D1D)
-               for (int b=0; b<D1D; ++b)
-               {
-                  const double q_cbi = s_q[c][b][i];
-                  MFEM_UNROLL(Q1D)
-                  for (int j=0; j<Q1D; ++j) { r_wk[j] += s_B[j][b]*q_cbi; }
-               }
-               MFEM_UNROLL(Q1D)
-               for (int j=0; j<Q1D; ++j) { s_q[c][j][i] = r_wk[j]; }
-            }
-            MFEM_UNROLL(Q1D)
-            for (int k=0; k<Q1D; ++k) { r_wk[k] = 0.0; }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(j,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(i,x,Q1D)
-         {
-            // Interpolate in Z direction
-            MFEM_UNROLL(D1D)
-            for (int c=0; c<D1D; ++c)
-            {
-               const double q_cji = s_q[c][j][i];
-               MFEM_UNROLL(Q1D)
-               for (int k=0; k<Q1D; ++k) { r_wk[k] += s_B[k][c]*q_cji; }
-            }
-
-            // Scale by Jacobian and integration weights
-            MFEM_UNROLL(Q1D)
-            for (int k=0; k<Q1D; ++k) { r_wk[k] *= D(i,j,k,e); }
-
-            // Project back in Z direction
-            MFEM_UNROLL(D1D)
-            for (int c=0; c<D1D; ++c)
-            {
-               double q_cji = 0.0;
-               MFEM_UNROLL(Q1D)
-               for (int k=0; k<Q1D; ++k) { q_cji += s_B[k][c] * r_wk[k]; }
-               s_q[c][j][i] = q_cji;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Project back in Y direction
-      MFEM_FOREACH_THREAD(c,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(i,x,Q1D)
-         {
-            MFEM_UNROLL(Q1D)
-            for (int j=0; j<Q1D; ++j) { r_wk[j] = s_q[c][j][i]; }
-            MFEM_UNROLL(D1D)
-            for (int b=0; b<D1D; ++b)
-            {
-               double q_cbi = 0.0;
-               MFEM_UNROLL(Q1D)
-               for (int j=0; j<Q1D; ++j) { q_cbi += s_B[j][b] * r_wk[j]; }
-               s_q[c][b][i] = q_cbi;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Project back in X direction
-      MFEM_FOREACH_THREAD(c,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(b,x,D1D)
-         {
-            MFEM_UNROLL(Q1D)
-            for (int i=0; i<Q1D; ++i) { r_wk[i] = s_q[c][b][i]; }
-            MFEM_UNROLL(D1D)
-            for (int a=0; a<D1D; ++a)
-            {
-               double q_cba = 0.0;
-               MFEM_UNROLL(Q1D)
-               for (int i=0; i<Q1D; ++i) { q_cba += s_B[i][a] * r_wk[i]; }
-               s_q[c][b][a] = q_cba;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Save back to memory
-      MFEM_FOREACH_THREAD(b,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(a,x,D1D)
-         {
-            MFEM_UNROLL(D1D)
-            for (int c=0; c<D1D; ++c)
-            {
-               const double q_cba = s_q[c][b][a];
-               const int gid = map ? MAP(a,b,c,e) : 0;
-               const int idx = gid >= 0 ? gid : -1 - gid;
-               AtomicAdd(map?Y(idx):Y1(a,b,c,e), q_cba);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-   });
-}
-
-void NDK_AMD_PAMassApply(const int dim,
-                         const int D1D,
-                         const int Q1D,
-                         const int NE,
-                         const FiniteElementSpace *fes,
-                         const DofToQuad *maps,
-                         const Vector &D,
-                         const Vector &X,
-                         Vector &Y)
-{
-   const int ND = fes->GetNDofs();
-   constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;
-   const Operator *ERop = fes->GetElementRestriction(ordering);
-   const ElementRestriction* ER = dynamic_cast<const ElementRestriction*>(ERop);
-   const int *map = ER ? ER->GatherMap().Read() : nullptr;
-   const double *b = maps->B.Read();
-   const double *d = D.Read();
-   const double *x = X.Read();
-   double *y = Y.ReadWrite();
-
-   assert(dim == 3);
-   const int ver = Device::KernelsVersion();
-   const int id = (ver << 8) | (D1D << 4) | Q1D;
-
-   //printf("\033[32mkernel #0x%x\033[m\n",id); fflush(0);
-
-   switch (id) // orders 1~6
-   {
-      // Fast '3': libP + AMD specific non-deterministic 3D mass kernel
-      case 0x323: return NDK_AMD_PAMassApply3D<2,3>(ND,NE,map,b,d,x,y);
-      case 0x324: return NDK_AMD_PAMassApply3D<2,4>(ND,NE,map,b,d,x,y);
-      case 0x334: return NDK_AMD_PAMassApply3D<3,4>(ND,NE,map,b,d,x,y);
-      case 0x336: return NDK_AMD_PAMassApply3D<3,6>(ND,NE,map,b,d,x,y);
-      case 0x345: return NDK_AMD_PAMassApply3D<4,5>(ND,NE,map,b,d,x,y);
-      case 0x346: return NDK_AMD_PAMassApply3D<4,6>(ND,NE,map,b,d,x,y);
-      case 0x348: return NDK_AMD_PAMassApply3D<4,8>(ND,NE,map,b,d,x,y);
-      case 0x356: return NDK_AMD_PAMassApply3D<5,6>(ND,NE,map,b,d,x,y);
-      case 0x358: return NDK_AMD_PAMassApply3D<5,8>(ND,NE,map,b,d,x,y);
-      case 0x367: return NDK_AMD_PAMassApply3D<6,7>(ND,NE,map,b,d,x,y);
-      case 0x378: return NDK_AMD_PAMassApply3D<7,8>(ND,NE,map,b,d,x,y);
-
-      default: break;
-   }
-
-   MFEM_ABORT("Unknown kernel 0x" << std::hex << id);
-}
-
-} // namespace mfem
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Tzanio	967908029f	make style	2021-09-13 10:51:52 -07:00
$AMCBRIDGE\ahudym$ AMCBRIDGE\ahudym	8560248ea4	revert make file	2021-09-13 18:42:10 +03:00
$AMCBRIDGE\ahudym$ AMCBRIDGE\ahudym	fa85cf2471	fix style	2021-09-13 18:41:43 +03:00
$AMCBRIDGE\ahudym$ AMCBRIDGE\ahudym	8b34881d46	change #elif with #else	2021-09-13 11:39:22 +03:00
$AMCBRIDGE\ahudym$ AMCBRIDGE\ahudym	de0cad550d	cast to SOCKET only for WIN32 platform	2021-09-13 11:34:44 +03:00
$AMCBRIDGE\ahudym$ AMCBRIDGE\ahudym	1f7f84d7a2	1. Remove "static" keyword from functions used as lambdas on "device" 2. Add _USE_MATH_DEFINES to CmakeLists.txt (without this change VS doesn't "see" this preprocessor in config.hpp) 3. Remove asm definition - asm is not allowed in VS anymore 4. Convert "const double Epsilon" to constexpr (fixes CUDA compilation error)	2021-09-03 11:59:51 +03:00