wip

modification and examples
Another Lambda example
2021-07-10 10:33:09 -07:00 · 2021-07-02 11:21:48 -07:00 · 2021-05-26 15:53:39 -07:00 · 2021-05-26 15:11:05 -07:00 · 2021-05-25 21:00:57 -07:00 · 2021-05-25 17:53:33 -07:00
319 changed files with 15396 additions and 24919 deletions
@@ -12,15 +12,6 @@
 # In this CI section, we build different variants of mfem and run test on them.
 name: builds-and-tests

-# Github actions can use the default "GITHUB_TOKEN". By default, this token
-# is set to have permissive access. However, this is not a good practice
-# security-wise. Here we use an external action, so we restrict the
-# permission to the minimum required.
-# When the 'permissions' is set, all the scopes not mentioned are set to the
-# most restrictive setting. So the following is enough.
-permissions:
-  actions: write
-
 on:
  push:
    branches:
@@ -47,47 +38,30 @@ jobs:
    strategy:
      matrix:
        os: [ubuntu-18.04, macos-10.15]
-        target: [dbg, opt]
-        mpi: [seq, par]
+        target: [debug, optim]
+        mpi: [sequential, parallel]
        build-system: [make]
-        hypre-target: [int32]
-        # 'include' allows us to:
-        # - Add a variable to all jobs without creating a new matrix dimension.
-        #   Codecov is defined that way.
-        # - Add a new combination.
-        #   'build-system: cmake' and 'hypre-target: int64'
+        # 'include' allows us to
+        # - add a variable without creating a new matrix dimension.
+        # - add a new combination ('build-system: cmake' case here)
        #
        # note: we will gather coverage info for any non-debug run except the
        # CMake build.
        include:
-          - target: dbg
+          - target: debug
            codecov: NO
-          - target: opt
+          - target: optim
            codecov: YES
          - os: ubuntu-18.04
-            target: opt
+            target: optim
            codecov: NO
-            mpi: par
+            mpi: parallel
            build-system: cmake
-            hypre-target: int32
-          - os: ubuntu-18.04
-            target: opt
-            codecov: NO
-            mpi: par
-            build-system: make
-            hypre-target: int64
-    name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
+    name: ${{ matrix.os }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.build-system }}

    runs-on: ${{ matrix.os }}

    steps:
-    # This external action allows to interrupt a workflow already running on
-    # the same branch to save resource
-    - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.9.0
-      with:
-        access_token: ${{ github.token }}
-
    # Checkout MFEM in "mfem" subdirectory. Final path:
    # /home/runner/work/mfem/mfem/mfem
    # Note: Done now to access "install-hypre" and "install-metis" actions.
@@ -102,7 +76,7 @@ jobs:
    # TODO: It would be nice to have only one step, e.g. with a dedicated
    # action, but I (@adrienbernede) don't see how at the moment.
    - name: get MPI (Linux)
-      if: matrix.mpi == 'par' && matrix.os == 'ubuntu-18.04'
+      if: matrix.mpi == 'parallel' && matrix.os == 'ubuntu-18.04'
      run: |
        sudo apt-get install mpich libmpich-dev
        export MAKE_CXX_FLAG="MPICXX=mpic++"
@@ -113,11 +87,11 @@ jobs:
        sudo apt-get install lcov

    - name: Set up Homebrew
-      if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-10.15'
+      if: ( matrix.mpi == 'parallel' || matrix.codecov == 'YES' ) && matrix.os == 'macos-10.15'
      uses: Homebrew/actions/setup-homebrew@c4aafe8c4620bf08883dd4679c374f11e73329d3

    - name: get MPI (MacOS)
-      if: matrix.mpi == 'par' && matrix.os == 'macos-10.15'
+      if: matrix.mpi == 'parallel' && matrix.os == 'macos-10.15'
      run: |
        export HOMEBREW_NO_INSTALL_CLEANUP=1
        brew install openmpi
@@ -133,40 +107,39 @@ jobs:
    # Install will only run on cache miss.
    - name: cache hypre
      id: hypre-cache
-      if: matrix.mpi == 'par'
+      if: matrix.mpi == 'parallel'
      uses: actions/cache@v2
      with:
        path: ${{ env.HYPRE_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.0
+        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2

    - name: get hypre
-      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-hypre@v2.0
+      if: matrix.mpi == 'parallel' && steps.hypre-cache.outputs.cache-hit != 'true'
+      uses: mfem/github-actions/build-hypre@v1.0
      with:
-        archive: ${{ env.HYPRE_ARCHIVE }}
-        dir: ${{ env.HYPRE_TOP_DIR }}
-        target: ${{ matrix.hypre-target }}
+        hypre-archive: ${{ env.HYPRE_ARCHIVE }}
+        hypre-dir: ${{ env.HYPRE_TOP_DIR }}

    # Get Metis through cache, or build it.
    # Install will only run on cache miss.
    - name: cache metis
      id: metis-cache
-      if: matrix.mpi == 'par'
+      if: matrix.mpi == 'parallel'
      uses: actions/cache@v2
      with:
        path: ${{ env.METIS_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.0
+        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2

    - name: install metis
-      if: matrix.mpi == 'par' && steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@v2.0
+      if: matrix.mpi == 'parallel' && steps.metis-cache.outputs.cache-hit != 'true'
+      uses: mfem/github-actions/build-metis@v1.0
      with:
-        archive: ${{ env.METIS_ARCHIVE }}
-        dir: ${{ env.METIS_TOP_DIR }}
+        metis-archive: ${{ env.METIS_ARCHIVE }}
+        metis-dir: ${{ env.METIS_TOP_DIR }}

    # MFEM build and test
    - name: build
-      uses: mfem/github-actions/build-mfem@v2.0
+      uses: mfem/github-actions/build-mfem@v1.0
      with:
        os: ${{ matrix.os }}
        target: ${{ matrix.target }}
@@ -179,17 +152,17 @@ jobs:

    # Run checks (and only checks) on debug targets
    - name: checks
-      if: matrix.build-system == 'make' && matrix.target == 'dbg'
+      if: matrix.build-system == 'make' && matrix.target == 'debug'
      run: |
        cd ${{ env.MFEM_TOP_DIR }}  && make check

    - name: unit tests
-      if: matrix.build-system == 'make' && matrix.target == 'opt'
+      if: matrix.build-system == 'make' && matrix.target == 'optim'
      run: |
        cd ${{ env.MFEM_TOP_DIR }} && make unittest

    - name: tests
-      if: matrix.build-system == 'make' && matrix.target == 'opt'
+      if: matrix.build-system == 'make' && matrix.target == 'optim'
      run: |
        cd ${{ env.MFEM_TOP_DIR }} && make test

@@ -201,8 +174,8 @@ jobs:
    # Code coverage (process and upload reports)
    - name: codecov
      if: matrix.codecov == 'YES'
-      uses: mfem/github-actions/upload-coverage@v2.0
+      uses: mfem/github-actions/upload-coverage@v1.0
      with:
-        name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
+        name: ${{ matrix.os }}-${{ matrix.mpi }}
        project_dir: ${{ env.MFEM_TOP_DIR }}
        directories: "fem general linalg mesh"
@@ -11,9 +11,6 @@

 name: build-analysis

-permissions:
-  actions: write
-
 on:
  push:
    branches:
@@ -33,11 +30,6 @@ jobs:
    runs-on: ubuntu-18.04

    steps:
-    - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.9.0
-      with:
-        access_token: ${{ github.token }}
-
    - name: checkout MFEM
      uses: actions/checkout@v2
      with:
@@ -53,33 +45,32 @@ jobs:
      uses: actions/cache@v2
      with:
        path: ${{ env.HYPRE_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.0
+        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2

    - name: Get Hypre
      if: steps.hypre-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-hypre@v2.0
+      uses: mfem/github-actions/build-hypre@master
      with:
-        archive: ${{ env.HYPRE_ARCHIVE }}
-        dir: ${{ env.HYPRE_TOP_DIR }}
-        target: int32
+        hypre-archive: ${{ env.HYPRE_ARCHIVE }}
+        hypre-dir: ${{ env.HYPRE_TOP_DIR }}

    - name: Cache Metis Install
      id: metis-cache
      uses: actions/cache@v2
      with:
        path: ${{ env.METIS_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.0
+        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2

    - name: Install Metis
      if: steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@v2.0
+      uses: mfem/github-actions/build-metis@master
      with:
-        archive: ${{ env.METIS_ARCHIVE }}
-        dir: ${{ env.METIS_TOP_DIR }}
+        metis-archive: ${{ env.METIS_ARCHIVE }}
+        metis-dir: ${{ env.METIS_TOP_DIR }}

    # MFEM build and test
    - name: build-mfem
-      uses: mfem/github-actions/build-mfem@v2.0
+      uses: mfem/github-actions/build-mfem@master
      with:
        os: ${{ runner.os }}
        target: optim
@@ -11,56 +11,34 @@

 name: repo-check

-permissions:
-  actions: write
-
 on:
  push:
+    branches:
+      - master
+      - next
+  pull_request:

 jobs:
-  file-headers-check:
-    runs-on: ubuntu-18.04
+  copyright-check:
+    runs-on: ubuntu-18.04 # needed for astyle 2.05.1

    steps:
-    - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.9.0
-      with:
-        access_token: ${{ github.token }}
    - name: checkout mfem
      uses: actions/checkout@v2
+      with:
+        path: mfem

    - name: copyright check
-      id: copyright
      run: |
-        ./config/githooks/pre-push --copyright
-
-      continue-on-error: true
-
-    - name: license check
-      id: license
-      run: |
-        ./config/githooks/pre-push --license
-      continue-on-error: true
-
-    - name: release check
-      id: release
-      run: |
-        ./config/githooks/pre-push --release
-      continue-on-error: true
-
-    - name: wrap-up
-      if: steps.copyright.outcome != 'success' || steps.license.outcome != 'success' || steps.release.outcome != 'success'
-      run: |
-        if [[ "${{ steps.copyright.outcome }}" != "success" ]]; then
-          echo "copyright check failed, unroll log for details"
+        cd mfem
+        if git grep -l "^#.*\-2020" > matches.txt
+        then
+          echo "Please update the following files to Copyright (c) 2010-2021:"
+          cat matches.txt
+          exit 1
+        else
+          echo "No outdated copyright found."
        fi
-        if [[ "${{ steps.license.outcome }}" != "success" ]]; then
-          echo "license check failed, unroll log for details"
-        fi
-        if [[ "${{ steps.release.outcome }}" != "success" ]]; then
-          echo "release check failed, unroll log for details"
-        fi
-        exit 1

  code-style:
    runs-on: ubuntu-16.04 # needed for astyle 2.05.1
@@ -75,7 +53,8 @@ jobs:

    - name: style check
      run: |
-        ./config/githooks/pre-push --style
+        cd tests/scripts
+        ./runtest code-style

  documentation:
    runs-on: ubuntu-18.04
@@ -107,4 +86,5 @@ jobs:
      run: |
        git fetch origin master:master
        git checkout -b gh-actions-branch-history
-        ./config/githooks/pre-push --history
+        cd tests/scripts
+        ./runtest branch-history
@@ -26,7 +26,6 @@ CMakeFiles/
 config/_config.hpp
 config/config.mk
 config/sample-runs-build.log
-config/user.mk
 doc/CodeDocumentation.conf
 doc/CodeDocumentation.html
 doc/CodeDocumentation
@@ -249,10 +248,6 @@ miniapps/performance/sol.*

 miniapps/shifted/distance
 miniapps/shifted/ParaViewDistance
-miniapps/shifted/diffusion
-miniapps/shifted/diffusion.mesh
-miniapps/shifted/diffusion.gf
-miniapps/shifted/ParaViewDiffusion

 miniapps/tools/display-basis
 miniapps/tools/load-dc
@@ -297,7 +292,6 @@ tests/unit/psedov_tests_*
 tests/unit/tmop_pa_tests_*
 tests/unit/ptmop_pa_tests_*
 tests/unit/ceed_tests
-tests/unit/debug_device_tests

 # Test script output
 tests/scripts/*.err
@@ -40,67 +40,43 @@
 # Directory used to place artifacts.

 variables:
-  BUILD_ROOT: ${CI_BUILDS_DIR}/MFEM/${CI_PROJECT_NAME}_${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID}
-  AUTOTEST_ROOT: ${CI_BUILDS_DIR}/MFEM
+  BUILD_ROOT: ${CI_BUILDS_DIR}/${CI_PROJECT_NAME}_${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID}
  REBASELINE: "NO"
-  AUTOTEST: "NO"
  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
  TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
  TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
-  AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
-  MFEM_DATA_REPO: https://github.com/mfem/data.git
  ARTIFACTS_DIR: artifacts
-  SLURM_OVERLAP: 1

-# The pipeline is divided into stages. Usually, jobs in a given stage wait for
-# the preceding stages to complete before to start. However, we sometimes use
-# the "needs" keyword and express the DAG of jobs for more efficiency.
-# - We use setup and setup_baseline phases to download content outside of mfem
-#   directory.
+# The pipeline is divided into stages. Usually, these are also synchronization
+# points, however, we use "needs" keyword to express the DAG of jobs for more
+# efficiency.
+# - We use setup phase to download content outside of mfem directory.
 # - Allocate/Release is where quartz resources are allocated/released once for all.
 # - Build and Test is where we build and MFEM for multiple toolchains.
 # - Baseline_checks gathers baseline-type test suites execution
 # - Baseline_publish, only available on master, allows to update baseline
 #   results
 stages:
-  - setup
  - q_allocate_resources
  - q_build_and_test
  - q_release_resources
  - l_build_and_test
  - c_build_and_test
-  - setup_baseline
+  - setup
  - baseline_check
-  - baseline_to_autotest
  - baseline_publish

-# setup clones the mfem/data repo in ${BUILD_ROOT}. The build_and_test script
-# then symlinks the repo to the parent directory of the MFEM source directory.
-# Unit tests that depend on the mfem/data repo will then detect that this
-# directory is present and be enabled.
+# The setup job in setup stage don't rely on MFEM git repo. It prepares a
+# pipeline-wide working directory downloading/updating external repos.
+# TODO: updating tests and tpls is not necessary anymore since pipelines are
+# now using unique directories so repo are never shared with another pipeline.
+# This is not memory efficient (we keep a lot of data), hence this reminder.
+# Setup
 setup:
  tags:
    - shell
    - quartz
  stage: setup
-  variables:
-    GIT_STRATEGY: none
-  script:
-    - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
-    - if [ ! -d data ]; then git clone ${MFEM_DATA_REPO}; fi
-
-# The setup_baseline job in setup stage_baseline doesn't rely on MFEM git repo.
-# It prepares a pipeline-wide working directory downloading/updating external
-# repos. TODO: updating tests and tpls is not necessary anymore since pipelines
-# are now using unique directories so repo are never shared with another
-# pipeline. This is not memory efficient (we keep a lot of data), hence this
-# reminder.
-# Note: This job can start immediately.
-setup_baseline:
-  tags:
-    - shell
-    - quartz
-  stage: setup_baseline
  variables:
    GIT_STRATEGY: none
  script:
@@ -109,9 +85,6 @@ setup_baseline:
    - if [ ! -d "tests" ]; then git clone ${TESTS_REPO}; fi
    - cd tpls && git pull && cd ..
    - cd tests && git pull && cd ..
-    - cd ${AUTOTEST_ROOT}
-    - if [ ! -d "autotest" ]; then git clone ${AUTOTEST_REPO}; fi
-    - cd autotest && git pull && cd ..
  needs: []

 .build_toss_3_x86_64_ib_script:
@@ -126,10 +99,10 @@ setup_baseline:
  script:
    - srun -p mi60 -t 15 -N 1 tests/gitlab/build_and_test

-# Lassen uses a different job scheduler (spectrum lsf) that does not allow
-# pre-allocation the same way slurm does.  We use pdebug queue on lassen to
-# speed-up the allocation.  However this would not be scalable to multiple
-# builds.
+# Lassen uses a different job scheduler (spectrum lsf) that does not
+# allow pre-allocation the same way slurm does.
+# We use pdebug queue on lassen to speed-up the allocation.
+# However this would not be scalable to multiple builds.
 .build_blueos_3_ppc64le_ib_script:
  script:
    - lalloc 1 -W 30 -q pdebug tests/gitlab/build_and_test
@@ -138,10 +111,16 @@ setup_baseline:
 # differentiates between the two tests.
 .baseline_script: &baseline_script |
  # locals
+  _glob_out=${BASELINE_TEST}.out
  _glob_err=${BASELINE_TEST}.err
  _base_diff=${BASELINE_TEST}-${SYS_TYPE}.diff
  _base_patch=${BASELINE_TEST}-${SYS_TYPE}.patch
  _base_out=${BASELINE_TEST}-${SYS_TYPE}.out
+
+  _out=${BASELINE_TEST}-${SYS_TYPE}.out
+  _ref=../${BASELINE_TEST}-${SYS_TYPE}.saved
+  _out_txt=${BASELINE_TEST}.txt
+  _diff=${BASELINE_TEST}-diff.txt
  # prepare
  cd ${BUILD_ROOT}
  ln -snf ${CI_PROJECT_DIR} mfem
@@ -156,7 +135,7 @@ setup_baseline:
    echo "ERROR during ${BASELINE_TEST} execution";
    echo "Here is the ${_glob_err} file content";
    cat ${_glob_err}
-    cp ${_glob_err} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_glob_err}
+    cp ${_glob_err} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_glob_err}.txt
    exit 1;
  elif [[ ! -f ${_base_patch} && ! -f ${_base_out} ]]
  then
@@ -166,20 +145,18 @@ setup_baseline:
  elif [[ -f ${_base_patch} ]]
  then
    echo "${BASELINE_TEST}: Differences found, patch generated"
-    cp ${_base_patch} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_patch}
+    cp ${_base_patch} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_patch}.txt
  elif [[ -f ${_base_out} ]]
  then
    echo "${BASELINE_TEST}: Differences found, replacement file generated"
-    cp ${_base_out} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_out}
+    cp ${_base_out} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_out}.txt
  fi
  # _base_diff won't even exist if there is no difference.
  if [[ -f ${_base_diff} ]]
  then
    echo "${BASELINE_TEST}: Relevant differences (filtered diff) ..."
    cat ${_base_diff}
-    cp ${_base_diff} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_diff}
-    # We create a .err file, because that's how we signal that there was a diff.
-    cp ${_base_diff} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/gitlab-${BASELINE_TEST}-${SYS_TYPE}.err
+    cp ${_base_diff} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_diff}.txt
  fi
  if [[ ! -s ${_base_diff} ]]
  then
@@ -218,7 +195,7 @@ setup_baseline:
      - ${ARTIFACTS_DIR}
  allow_failure: true

-# This job can only be manually triggered on a pipeline for master branch, or if
+# This job can only be manually triggers on a pipeline for master branch, or if
 # the pipeline was triggered with REBASELINE="YES"
 .rebaseline_mfem:
  stage: baseline_publish
@@ -231,18 +208,19 @@ setup_baseline:
    - export DIFF_FILE=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}.diff
    - cd ${BUILD_ROOT}/tests
    - |
-      if [[ ! -f "${DIFF_FILE}" ]]
+      if [[ ! -f "${DIFF_FILE}.txt" ]]
      then
          echo "Nothing to be done: no relevant change in baseline"
          exit 0
-      elif [[ -f "${PATCH_FILE}" ]]
+      elif [[ -f "${PATCH_FILE}.txt" ]]
      then
+          mv ${PATCH_FILE}.txt ${PATCH_FILE}
          patch "./baseline-${SYS_TYPE}.saved" < "${PATCH_FILE}"
-      elif [[ -f "${FULL_FILE}t" ]]
+      elif [[ -f "${FULL_FILE}.txt" ]]
      then
-          cp "${FULL_FILE}" "./baseline-${SYS_TYPE}.saved"
+          cp "${FULL_FILE}.txt" "./baseline-${SYS_TYPE}.saved"
      else
-          echo "File missing: expected ${PATCH_FILE} or ${FULL_FILE}"
+          echo "File missing: expected ${PATCH_FILE}.txt or ${FULL_FILE}.txt"
          exit 1
      fi
    - git add baseline-${SYS_TYPE}.saved
@@ -22,13 +22,12 @@

 # Spack helped builds
 # Generic lassen build job, extending build script
-# Note: Lassen jobs can start as soon as the setup job is complete.
 .build_and_test_on_lassen:
  extends: [.build_blueos_3_ppc64le_ib_script, .on_lassen]
  stage: l_build_and_test
-  needs: [setup]
+  needs: []

 opt_mpi_cuda_xl_16_1_1_8:
  variables:
-    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=70"
+    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=sm_70"
  extends: .build_and_test_on_lassen
@@ -16,39 +16,12 @@
    - shell
    - quartz
  rules:
-    # Don't run quartz jobs if...
-    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"'
+    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"' #run except if ...
      when: never
-    # Don't run autotest update if...
-    - if: '$CI_JOB_NAME =~ /update_autotest/ && $AUTOTEST != "YES"'
-      when: never
-    # Don't run autotest update if...
-    - if: '$CI_JOB_NAME =~ /q_report/ && $AUTOTEST != "YES"'
-      when: never
-    # Report success on success status
-    - if: '$CI_JOB_NAME =~ /q_report_success/ && $AUTOTEST == "YES"'
-      when: on_success
-    # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /q_report_failure/ && $AUTOTEST == "YES"'
-      when: on_failure
-    # Always release resources
    - if: '$CI_JOB_NAME =~ /release_resources/'
      when: always
-    # Default is to run if previous stage succeeded
    - when: on_success

-# This is a yaml anchor, it can be used to avoid duplication like here.
-# The code below will simply be pasted wherever the anchor is placed.
-.safe_create_rundir: &safe_create_rundir |
-  if ! mkdir ${rundir}; then
-    n=1
-    while ! mkdir ${rundir}_${n}
-    do
-      n=$((n+1))
-    done
-    rundir=${rundir}_${n}
-  fi
-
 # Allocate
 q_allocate_resources:
  variables:
@@ -69,40 +42,6 @@ q_release_resources:
    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})

-# Release
-q_report_success:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_quartz
-  stage: q_release_resources
-  script:
-    - echo "Can only run if all the quartz jobs passed"
-    - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
-    - *safe_create_rundir
-    - echo "The Quartz jobs were successful" > ${rundir}/gitlab.out
-    - echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
-    - git add ${rundir}
-    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
-    - git push origin master
-
-q_report_failure:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_quartz
-  stage: q_release_resources
-  script:
-    - echo "Runs if there was at least one failure on quartz"
-    - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
-    - *safe_create_rundir
-    - echo "There was an error while running CI on Quartz" > ${rundir}/gitlab.err
-    - echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
-    - cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
-    - git add ${rundir}
-    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
-    - git push origin master
-
 # Spack helped builds
 # Generic quartz build job, extending build script
 .build_and_test_on_quartz:
@@ -150,34 +89,10 @@ opt_par_gcc_6_1_0_pumi:
    SPEC: "%gcc@6.1.0 +pumi"
  extends: .build_and_test_on_quartz

-# Baseline jobs form an independent set of jobs. We use `needs:[]` to specify
-# that "setup-baseline" can start immediately. Then, we have to use needs for
-# each one of the baseline jobs, otherwise they will wait for the rest of the
-# pipeline.
-
+# Baseline
 baselinecheck_mfem_intel_quartz:
  extends: [.baselinecheck_mfem, .on_quartz]
-  needs: [setup_baseline]
-
-update_autotest:
-  extends: [.on_quartz]
-  needs: [baselinecheck_mfem_intel_quartz]
-  stage: baseline_to_autotest
-  script:
-    - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - rundir="quartz/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
-    - *safe_create_rundir
-    - cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
-    # We create an autotest-email.html file, because that's how we signal that there was a diff (temporary).
-    - |
-      if [[ -f ${rundir}/*.err ]]
-      then
-        echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/*.err
-        cp ${rundir}/*.err ${rundir}/autotest-email.html
-      fi
-    - git add ${rundir}
-    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
-    - git push origin master
+  needs: [setup]

 baselinepublish_mfem_quartz:
  extends: [.on_quartz, .rebaseline_mfem]
@@ -0,0 +1,469 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+language: cpp
+
+os: linux
+dist: bionic
+
+stages:
+  - checks
+  - tests
+  - optional
+
+env:
+  global:
+    - HYPRE_ARCHIVE=v2.19.0.tar.gz
+      HYPRE_URL=https://github.com/hypre-space/hypre/archive/$HYPRE_ARCHIVE
+      HYPRE_TOP_DIR=hypre-2.19.0
+
+jobs:
+  include:
+
+    # ========================
+    #         Checks
+    # ========================
+    # - code-style
+    # - documentation
+    # - gitignore
+
+    - stage: checks
+      os: linux
+      dist: xenial
+      name: "code-style"
+      addons:
+        apt:
+          packages:
+            - astyle=2.05.1-0ubuntu1
+      script:
+        - cd ${TRAVIS_BUILD_DIR}
+        - cd tests/scripts
+        - ./runtest code-style
+
+    - stage: checks
+      os: linux
+      name: "documentation"
+      addons:
+        apt:
+          packages:
+            - doxygen
+            - graphviz
+      script:
+        - cd ${TRAVIS_BUILD_DIR}
+        - cd tests/scripts
+        - ./runtest documentation
+
+    - stage: checks
+      os: linux
+      name: "gitignore"
+      addons:
+        apt:
+          packages:
+            - mpich
+            - libmpich-dev
+      env: MPI=YES
+      before_script:
+        - cd ${TRAVIS_BUILD_DIR}
+        - mpicxx -v
+        - make config MFEM_USE_MPI=YES MFEM_MPI_NP=2
+        - make all -j3
+        - make test-noclean
+      script:
+        - cd tests/scripts
+        - ./runtest gitignore
+      cache:
+        ccache: true
+        directories:
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
+          rm -f Lib/*.{c,o}
+
+    # ========================
+    #  Optional Checks/Tests
+    # ========================
+    # - branch-history
+
+    - stage: optional
+      name: "branch-history"
+      if: branch != next
+      # need full git history for the binary/big files check
+      git:
+        depth: false
+      script:
+        - cd ${TRAVIS_BUILD_DIR}
+        # update master
+        - git fetch origin master:master
+        # checkout a branch (otherwise Travis works in detached head)
+        - git checkout -b travis_tests
+        - cd tests/scripts
+        - ./runtest branch-history
+
+    # ========================
+    #       Linux tests
+    # ========================
+    # - serial + debug
+    # - serial
+    # - parallel + debug
+    # - parallel
+
+    - stage: tests
+      os: linux
+      compiler: gcc
+      name: "Linux: Serial + Debug"
+      env: DEBUG=YES
+           MPI=NO
+           CODECOV=NO
+           MFEM_TEST_TARGET=check
+      cache:
+        ccache: true
+
+    - os: linux
+      compiler: gcc
+      name: "Linux: Serial"
+      env: DEBUG=NO
+           MPI=NO
+           CODECOV=NO
+           MFEM_TEST_TARGET=test
+      cache:
+        ccache: true
+
+    - os: linux
+      compiler: gcc
+      name: "Linux: Parallel + Debug"
+      addons:
+        apt:
+          # sources:
+            # - ubuntu-toolchain-r-test
+          packages:
+            # GCC 4.9
+            # - g++-4.9
+            # MPICH
+            - mpich
+            - libmpich-dev
+            # OpenMPI
+            # - openmpi-bin
+            # - libopenmpi-dev
+      env: DEBUG=YES
+           MPI=YES
+           CODECOV=NO
+           MFEM_TEST_TARGET=check
+           NPROCS=2
+      cache:
+        ccache: true
+        directories:
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
+          rm -f Lib/*.{c,o}
+
+    - os: linux
+      compiler: gcc
+      name: "Linux: Parallel"
+      addons:
+        apt:
+          # sources:
+            # - ubuntu-toolchain-r-test
+          packages:
+            # GCC 4.9
+            # - g++-4.9
+            # MPICH
+            - mpich
+            - libmpich-dev
+            # OpenMPI
+            # - openmpi-bin
+            # - libopenmpi-dev
+      env: DEBUG=NO
+           MPI=YES
+           CODECOV=YES
+           MFEM_TEST_TARGET=test
+           NPROCS=2
+      cache:
+        ccache: true
+        directories:
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
+          rm -f Lib/*.{c,o}
+
+    - os: linux
+      compiler: gcc
+      name: "Linux: Parallel (cmake)"
+      addons:
+        apt:
+          packages:
+            - mpich
+            - libmpich-dev
+      env: MPI=YES
+           NPROCS=2
+      script:
+        - cd ${TRAVIS_BUILD_DIR}
+        - mkdir ${TRAVIS_BUILD_DIR}/build
+        - cd ${TRAVIS_BUILD_DIR}/build
+        - cmake ..
+             -DMFEM_USE_MPI=ON
+             -DHYPRE_DIR=${TRAVIS_BUILD_DIR}/../$HYPRE_TOP_DIR/src/hypre
+             -DMFEM_MPI_NP=$NPROCS
+        - make -j3 mfem examples
+        - cd ${TRAVIS_BUILD_DIR}/build/tests/unit
+        - make -j3
+        - ctest --output-on-failure
+      cache:
+        ccache: true
+        directories:
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
+          rm -f Lib/*.{c,o}
+
+    # ========================
+    #      Mac OS X tests
+    # ========================
+    # - serial + debug
+    # - serial
+    # - parallel + debug
+    # - parallel
+
+    - os: osx
+      osx_image: xcode11.2
+      compiler: clang
+      name: "Mac: Serial + Debug"
+      addons:
+        homebrew:
+          packages:
+            - ccache
+      env: DEBUG=YES
+           MPI=NO
+           CODECOV=NO
+           MFEM_TEST_TARGET=check
+      cache:
+        ccache: true
+
+    - os: osx
+      osx_image: xcode11.2
+      compiler: clang
+      name: "Mac: Serial"
+      addons:
+        homebrew:
+          packages:
+            - ccache
+      env: DEBUG=NO
+           MPI=NO
+           CODECOV=NO
+           MFEM_TEST_TARGET=test
+      cache:
+        ccache: true
+
+    - os: osx
+      osx_image: xcode11.2
+      compiler: clang
+      name: "Mac: Parallel + Debug"
+      addons:
+        homebrew:
+          packages:
+            - ccache
+      env: DEBUG=YES
+           MPI=YES
+           CODECOV=NO
+           MFEM_TEST_TARGET=check
+           NPROCS=4
+           TMPDIR=/tmp
+      cache:
+        ccache: true
+        directories:
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+          - $HOME/local-cached
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
+          rm -f Lib/*.{c,o}
+
+    - os: osx
+      osx_image: xcode11.2
+      compiler: clang
+      name: "Mac: Parallel"
+      addons:
+        homebrew:
+          packages:
+            - ccache
+      env: DEBUG=NO
+           MPI=YES
+           CODECOV=YES
+           MFEM_TEST_TARGET=test
+           NPROCS=4
+           TMPDIR=/tmp
+      cache:
+        ccache: true
+        directories:
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+          - $HOME/local-cached
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
+          rm -f Lib/*.{c,o}
+
+before_install:
+   # No addon for brew yet, have to install OSX packages this way.
+   # - if [ $TRAVIS_OS_NAME == "osx" ] && [ $MPI == "YES" ]; then
+   #      brew install open-mpi;
+   #   fi
+
+   # Disable ccache while building dependencies that are cached:
+   - echo "before \$PATH = $PATH";
+     export PATH=${PATH//\/usr\/lib\/ccache:/};
+     echo "after  \$PATH = $PATH"
+
+   # On Mac OS X, build and cache OpenMPI 2.1.6:
+   - if [ $TRAVIS_OS_NAME == "osx" ] && [ $MPI == "YES" ]; then
+        if [ ! -e $HOME/local-cached/bin/mpicc ]; then
+           mkdir -p $HOME/builds && cd $HOME/builds &&
+           wget https://download.open-mpi.org/release/open-mpi/v2.1/openmpi-2.1.6.tar.bz2 &&
+           tar jxf openmpi-2.1.6.tar.bz2 &&
+           mkdir openmpi-build && cd openmpi-build &&
+           ../openmpi-2.1.6/configure --prefix=$HOME/local-cached &&
+           make -j3 all && make install;
+        fi;
+        PATH=$HOME/local-cached/bin:$PATH;
+        cd $TRAVIS_BUILD_DIR;
+     fi
+
+   # Update environment to find g++ 4.9 installation first.
+   # - if [ $TRAVIS_OS_NAME == "linux" ]; then
+   #      mkdir -p latest-gcc-symlinks;
+   #      ln -s /usr/bin/g++-4.9 latest-gcc-symlinks/g++;
+   #      ln -s /usr/bin/gcc-4.9 latest-gcc-symlinks/gcc;
+   #      ln -s /usr/bin/gcov-4.9 latest-gcc-symlinks/gcov;
+   #      export PATH=$PWD/latest-gcc-symlinks:$PATH;
+   #   fi
+
+   # Install tool to upload code coverage reports to coveralls.io
+   - if [ "$CODECOV" == "YES" ]; then
+        export PYTHONUSERBASE=$HOME/local;
+        pip install --user cpp-coveralls;
+        pip install --user pyyaml;
+        PATH=$HOME/local/bin:$PATH;
+     fi
+
+install:
+   # Set MPI compilers, print compiler version
+   - if [ $MPI == "YES" ]; then
+        if [ "$TRAVIS_OS_NAME" == "linux" ]; then
+           export MPICH_CC="$CC";
+           export MPICH_CXX="$CXX";
+        else
+           export OMPI_CC="$CC";
+           export OMPI_CXX="$CXX";
+           mpic++ --showme:version;
+        fi;
+        mpic++ -v;
+     else
+        $CXX -v;
+     fi
+
+   # Back out of the mfem directory to install the libraries
+   - cd ..
+
+   # hypre
+   - if [ $MPI == "YES" ]; then
+        if [ ! -e $HYPRE_TOP_DIR/src/hypre/lib/libHYPRE.a ]; then
+           wget $HYPRE_URL;
+           rm -rf $HYPRE_TOP_DIR;
+           tar xvzf $HYPRE_ARCHIVE;
+           cd $HYPRE_TOP_DIR/src;
+           ./configure --disable-fortran CC=mpicc CXX=mpic++;
+           make -j3;
+           cd ../..;
+        else
+            echo "Reusing cached $HYPRE_TOP_DIR/";
+        fi;
+        ln -s $HYPRE_TOP_DIR hypre;
+     else
+        echo "Serial build, not using hypre";
+     fi
+
+   # METIS, use a mirror because the original source server is not always up.
+   # Original url:
+   #    http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/OLD/metis-4.0.3.tar.gz
+   - if [ $MPI == "YES" ]; then
+        if [ ! -e metis-4.0/libmetis.a ]; then
+           wget https://mfem.github.io/tpls/metis-4.0.3.tar.gz;
+           tar xvzf metis-4.0.3.tar.gz;
+           make -j3 -C metis-4.0.3/Lib CC="$CC" OPTFLAGS="-O2";
+           rm -rf metis-4.0;
+           mv metis-4.0.3 metis-4.0;
+        else
+           echo "Reusing cached metis-4.0/";
+        fi;
+     fi
+
+   # Re-enable ccache on linux; enable ccache on mac os:
+   - if [ $TRAVIS_OS_NAME == "linux" ]; then
+        export PATH="/usr/lib/ccache:$PATH";
+     else
+        if [ $TRAVIS_OS_NAME == "osx" ]; then
+           export PATH="/usr/local/opt/ccache/libexec:$PATH";
+        fi;
+     fi
+
+   - printf "which \$CC  = "; which $CC;
+     printf "which \$CXX = "; which $CXX
+
+script:
+   # Compiler
+   - if [ $MPI == "YES" ]; then
+        export MYCXX=mpic++;
+        export MAKE_CXX_FLAG=MPICXX=$MYCXX;
+     else
+        export MYCXX="$CXX";
+        export MAKE_CXX_FLAG=CXX=$MYCXX;
+     fi
+
+   # Print the compiler version
+   - $MYCXX -v
+
+   # Set some variables
+   - cd $TRAVIS_BUILD_DIR;
+     CPPFLAGS="";
+     SKIP_TEST_DIRS="";
+     if [ "$CODECOV" == "YES" ]; then
+        CPPFLAGS="--coverage -g";
+     fi;
+     if [ "$TRAVIS_OS_NAME" != "linux" ] || [ "$DEBUG" == "YES" ]; then
+        CPPFLAGS+=" -pedantic -Wall -Werror";
+     fi
+
+   # Configure the library
+   - make config MFEM_USE_MPI=$MPI MFEM_DEBUG=$DEBUG $MAKE_CXX_FLAG
+        MFEM_MPI_NP=$NPROCS CPPFLAGS="$CPPFLAGS"
+   # Show the configuration
+   - make info
+   # Build the library
+   - make -j3
+   # Build the examples and the miniapps
+   - make -j3 all
+   # Run tests
+   - make $MFEM_TEST_TARGET SKIP_TEST_DIRS="$SKIP_TEST_DIRS"
+
+after_success:
+   - if [ "$CODECOV" == "YES" ]; then
+        coveralls --include fem --include general --include linalg --include
+           mesh --exclude /usr --gcov-options '\-lp' --root $TRAVIS_BUILD_DIR;
+     fi
@@ -8,87 +8,40 @@
                               https://mfem.org


-Version 4.3.1 (development)
+Version 4.2.1 (development)
 ===========================
- Added support for hr-adaptivity using TMOP-based error estimator.
+- Added initial support for GPU-accelerated versions of PETSc that works with
+  MFEM_USE_CUDA if PETSc has been configured with CUDA support. Examples 1 and 9
+  in the examples/petsc directory have been modified to work with --device cuda.
+  Examples with GAMG (ex1p) and SLEPc (ex11p) are also provided.

- Adding lowest order Nedelec and Raviart-Thomas basis functions on wedge
-  shaped elements.
+- Memory management:
+  * Added method Device::SetMemoryTypes that can be used to change the default
+    host and device MemoryTypes before Device setup.
+  * In class MemoryManager, added methods GetDualMemoryType and
+    SetDualMemoryType; dual MemoryTypes are used to determine the second
+    MemoryType (host or device) when only one MemoryType is specified in methods
+    of class Memory.
+  * Added Memory constructor for setting both the host and device MemoryTypes.
+  * Switched the default behavior of device memory allocations so that they
+    are deferred until the device pointer is needed.
+  * Added a second Umpire device MemoryType, DEVICE_UMPIRE_2, with
+    corresponding allocator that can be set with the method
+    MemoryManager::SetUmpireDevice2AllocatorName.
+  * Added HOST_PINNED MemoryType and a pinned host allocator for CUDA and HIP.

- Added initial support for meshes with pyramidal elements, including several
-  pyramidal meshes in the data/ directory and support for the lowest order H1,
-  Nedelec, Raviart-Thomas, and L2 basis functions on pyramids.
+- Added support for Caliper: a library to integrate performance profiling
+  capabilities into applications. See examples/caliper for more details.

- Updated the hypre interface according to changes in hypre-2.22.1. The ADS
-  solver is now fully working on GPUs.
+- Added support for explicit vectorization in the high-performance templated
+  code for Fujitsu's A64FX ARM microprocessor architecture.

- Tetrahedral meshes no longer need to be reordered to support high order
-  Nedelec basis functions. This will allow future support for Nedelec basis
-  functions on wedges and pyramids which are not amenable to reordering. The
-  ReorientTetMesh method of the Mesh and ParMesh classes has been deprecated.
+- Added AlgebraicCeedSolver that does matrix-free algebraic p-multigrid for
+  diffusion problems with the Ceed backend.

-
-Version 4.3, released on July 29, 2021
-======================================
-
-Discretization improvements
---------------------------
- Variable order spaces, p- and hp-refinement. This is the initial (serial)
-  support for variable-order FiniteElementCollection and FiniteElementSpace.
-  The new method FiniteElementSpace::SetElementOrder can be called to set an
-  arbitrary order for each mesh element. The conforming interpolation matrix
-  will now automatically constrain p- and hp- interfaces, enabling general
-  hp-refinement in both 2D and 3D, on uniform or mixed NC meshes. Support for
-  parallel variable-order spaces will follow shortly.
-
- Extended the support for field transfer between high-order and low-order
-  refined finite element spaces to include: dual fields and H1 fields (both
-  primary and dual). These are illustrated in the lor-transfer miniapp.
-
- Improved libCEED integration, including support for VectorCoefficient,
-  ConvectionIntegrator, and VectorConvectionNLFIntegrator with libCEED backends.
-
- Extending support for L2 basis functions using MapTypes VALUE and INTEGRAL in
-  linear interpolators and GridFunction "GetValue" methods.
-
- Changed the interface for the error estimator and implemented the Kelly error
-  indicator for scalar-valued problems, supported in serial and parallel builds.
-
- Added support for the "BR2" discontinuous Galerkin discretization for
-  diffusion via DGDiffusionBR2Integrator (see Example 14/14p).
-
- Added convective and skew-symmetric integrators for the nonlinear term in the
-  Navier-Stokes equations.
-
- Added new classes DenseSymmetricMatrix and SymmetricMatrixCoefficient for
-  efficient evaluation of symmetric matrix coefficients. This replaces the now
-  deprecated EvalSymmetric in MatrixCoefficient. Added DiagonalMatrixCoefficient
-  for clarity, which is a typedef of VectorCoefficient.
-
- Added support for nonscalar coefficient with VectorDiffusionIntegrator.
-
-Linear and nonlinear solvers
----------------------------
- Added support for AMG preconditioners on GPUs based on the hypre library
-  (version 2.22.0 or later). These include BoomerAMG, AMS and ADS and most
-  MFEM examples that use hypre have been ported to support this functionality.
-  The GPU preconditioners require that both hypre and MFEM are built with CUDA
-  support. Hypre builds with CUDA and unified memory are also supported and
-  can be used with `-d cuda:uvm` as a command-line option.
-
- Added support for AMG preconditioners for non-symmetric systems (e.g.
-  advection-dominated problems) using hypre's approximate ideal restriction
-  (AIR) AMG. Requires hypre version 2.14.0 or newer. Usage is illustrated in
-  example 9/9p.
-
- Added new functionality for constructing low-order refined discretizations and
-  solvers, see the LORDiscretization and LORSolver classes. A new basis type for
-  H(curl) and H(div) spaces is introduced to give spectral equivalence. This
-  functionality is illustrated in the LOR solvers miniapp in miniapps/solvers.
-
- Generalized the Multigrid class to support non-geometric multigrid. Previous
-  functionality, based on FiniteElementSpaceHierarchy, is now available in the
-  derived class GeometricMultigrid.
+- Introduced new options for the mesh-explorer miniapp to visualize the actual
+  element attributes in parallel meshes while retaining the visualization of
+  the domain decomposition.

 - Introduced solver interface for linear problems with constraints, a few
  concrete solvers that implement the interface, and a demonstration of their
@@ -99,18 +52,19 @@ Linear and nonlinear solvers
  as described in Barker and Kolev 2020 (https://doi.org/10.1002/nla.2348). See
  Example 3p and linalg/auxiliary.?pp.

- Improved interface for using the Ginkgo library, including: support for matrix-
-  free operators in Ginkgo solvers, new wrappers for Ginkgo preconditioners, HIP
-  support, and reduction of unnecessary data copies.
+- Added a new miniapp block-solvers that compares the performance of various
+  solvers for mixed finite element discretization of the second order scalar
+  elliptic equations. Currently available solvers in the miniapp include a
+  block-diagonal preconditioner that is based on approximate Schur complement
+  (implemented in ex5p), and a newly implemented solver DivFreeSolver, which
+  exploits a multilevel decomposition of the Raviart-Thomas space and its
+  divergence-free subspace. See the miniapps/solvers directory for more details.

- Added initial support for hypre's mixed integer (mixedint) capability, which
-  uses different data types for local and global indices in order to save memory
-  in large problems. This capability requires that hypre was configured with the
-  --enable-mixedint option. Note that this option is currently tested only in
-  ex1p, ex3p, and ex4p, and may not work in more general settings.
+- Added a new miniapp for computing (signed) distance functions to a point
+  source or zero level set. See miniapps/shifted/distance.cpp.

- Added AlgebraicCeedSolver that does matrix-free algebraic p-multigrid for
-  diffusion problems with the Ceed backend.
+- Added matrix-free GPU-enabled implementations of GradientInterpolator and
+  IdentityInterpolator.

 - Added interface to MUMPS direct solver. Its usage is demonstrated in ex25p.
  See http://mumps.enseeiht.fr/ for more details. Supported versions >= 5.1.1.
@@ -118,17 +72,6 @@ Linear and nonlinear solvers
 - Added three ESDIRK time integrators: implicit trapezoid rule, L-stable
  ESDIRK-32, and A-stable ESDIRK-33.

- Implemented a variable step-size IMEX (VSSIMEX) method for the Navier miniapp.
-
- Implemented an adaptive linear solver tolerance option for NewtonSolver based
-  on the algorithm of Eisenstat and Walker.
-
-Meshing improvements
--------------------
- Added support for reading high-order Lagrange meshes in VTK format. Arbitrary-
-  orders and all element types are supported. See the VTK blog for more info:
-  https://blog.kitware.com/wp-content/uploads/2018/09/Source_Issue_43.pdf.
-
 - Introduced a new non-conforming mesh format that fixes known inconsistencies
  of legacy "MFEM mesh v1.1" NC format and works consistently in both serial and
  parallel. ParMesh::ParPrint can now print non-conforming AMR meshes that can
@@ -137,26 +80,108 @@ Meshing improvements
  NC data files are compatible with serial code, e.g., can be viewed with serial
  GLVis. Loading of legacy NC mesh files is still supported.

- Added FMS support (https://github.com/CEED/FMS) to mfem. FMS can represent
-  unstructured high-order meshes with general high-order finite element fields
-  on them. When enabled, mfem can convert data collections to/from FMS data
-  collections in memory. In addition, an FMS data collection class was added so
-  the convert-dc miniapp can read and generate data files in FMS format.
+- Added support for 1D non-conforming meshes (which can be useful for parallel
+  load balancing and derefinement).
+
+- Added a "scaled Jacobian" visualization option in the Mesh Explorer miniapp to
+  help identify elements with poor mesh quality.
+
+- Added support for the "BR2" discontinuous Galerkin discretization for
+  diffusion via DGDiffusionBR2Integrator (see Example 14/14p).
+
+- Generalized the Multigrid class to support non-geometric multigrid. The
+  previous functionality, based on FiniteElementSpaceHierarchy, is now available
+  in the derived class GeometricMultigrid.
+
+- Upgraded the Catch unit test framework from version 2.13.0 to version 2.13.2.
+
+- The TMOP mesh optimization algorithms were extended to GPU:
+   - QualityMetric #1, #2, #7 and #77 are available in 2D, #302, #303, #315
+     and #321 in 3D
+   - Both AnalyticAdaptTC and DiscreteAdaptTC TargetConstructor are available
+   - Kernels for normalization and limiting have been added
+   - The AdvectorCG now also supports AssemblyLevel::PARTIAL
+
+- Added a new command line boolean option (`--all`) to the unit tests to launch
+  *all* non-regression tests.
+
+- Added support for different modes of QuadratureInterpolator on GPU.
+  The layout (QVectorLayout::byNODES|byVDIM) and the tensor products modes can
+  be enabled before calling the Mult, Values, Derivatives, PhysDerivatives and
+  Determinants methods.
+
+- Implemented a filter method for the Navier miniapp to stabilize highly
+  turbulent flows in direct numerical simulation.
+
+- Added HIP support to the CMake build system.
+
+- Added support for reading high-order Lagrange meshes in VTK format. Arbitrary-
+  orders and all element types are supported. See the VTK blog for more info:
+  https://blog.kitware.com/wp-content/uploads/2018/09/Source_Issue_43.pdf.
+
+- Added support for reading VTK meshes in XML format.
+
+- Added partial assembly and device support to Example 25/25p, with diagonal
+  preconditioning.
+
+- Implemented a variable step-size IMEX (VSSIMEX) method for the Navier miniapp.

 - Added new mesh quality metrics and improved the untangling capabilities of the
  TMOP-based mesh optimization algorithms.

- The TMOP mesh optimization algorithms were extended to GPU:
-  * QualityMetric 1, 2, 7, 77 are available in 2D, 302, 303, 315, 321 in 3D
-  * Both AnalyticAdaptTC and DiscreteAdaptTC TargetConstructor are available
-  * Kernels for normalization and limiting have been added
-  * The AdvectorCG now also supports AssemblyLevel::PARTIAL
+- Added convective and skew-symmetric integrators for the nonlinear term in the
+  Navier-Stokes equations.
+
+- Added new miniapp directory mtop/ with optimization-oriented block parametric
+  non-linear form and abstract integrators. Two new miniapps, ParHeat and
+  SeqHeat, demonstrate parallel and sequential implementation of gradients
+  evaluation for linear diffusion with discrete density.
+
+- Changed the interface for the error estimator.
+
+- Implemented the parallel Kelly error indicator for scalar-valued problems.
+
+- Added new classes DenseSymmetricMatrix and SymmetricMatrixCoefficient for
+  efficient evaluation of symmetric matrix coefficients. This replaces the now
+  deprecated EvalSymmetric in MatrixCoefficient. Added DiagonalMatrixCoefficient
+  for clarity, which is a typedef of VectorCoefficient.
+
+- Added support for AMG preconditioners for non-symmetric systems (e.g.
+  advection-dominated problems) using hypre's approximate ideal restriction
+  (AIR) AMG. Requires hypre version 2.14.0 or newer. Usage is illustrated in
+  example 9/9p.
+
+- Implemented an adaptive linear solver tolerance option for NewtonSolver based
+  on the algorithm of Eisenstat and Walker.
+
+- Added support for nonscalar coefficient with VectorDiffusionIntegrator.
+
+- Extending support for L2 basis functions using MapTypes VALUE and INTEGRAL in
+  linear interpolators and GridFunction "GetValue" methods.
+
+- Variable order spaces, p- and hp-refinement. This is the initial (serial)
+  support for variable-order FiniteElementCollection and FiniteElementSpace.
+  The new method FiniteElementSpace::SetElementOrder can be called to set an
+  arbitrary order for each mesh element. The conforming interpolation matrix
+  will now automatically constrain p- and hp- interfaces, enabling general
+  hp-refinement in both 2D and 3D, on uniform or mixed NC meshes. Support for
+  parallel variable-order spaces will follow shortly.

 - Added support for creating refined meshes for all element types (e.g. by
  splitting high-order elements into low-order refined elements), including
  mixed meshes. The LOR Transfer miniapp (miniapps/tools/lor-transfer.cpp) now
  supports meshes with any element geometry.

+- Gitlab CI: use Spack (and Uberenv) to automate the build of TPLs.
+
+- Added new miniapps demonstrating: 1) the use of GSLIB for overlapping grids,
+  see gslib/schwarz_ex1, and 2) coupling different physics in different domains,
+  see navier/cht. Note that gslib v1.0.7 is require (see INSTALL for details).
+
+- Added a new, very simple example (ex0 and parallel version ex0p). This
+  example solves a simple Poisson problem using H1 elements (the same problem as
+  ex1), but is intended to be extremely simple and approachable for new users.
+
 - Meshes consisting of any type of elements (including mixed meshes) can be
  converted to all-simplex meshes using Mesh::MakeSimplicial.

@@ -169,147 +194,36 @@ Meshing improvements
  requisite periodic vertex mappings can be created with
  Mesh::CreatePeriodicVertexMapping.

- Added support for 1D non-conforming meshes (which can be useful for parallel
-  load balancing and derefinement).
+- Added support for transferring dual fields between high-order and low-order
+  refined finite element spaces using the transposed versions of the
+  L2ProjectionGridTransfer operators. This functionality is illustrated in the
+  lor-transfer miniapp.

- Added sample meshes in the `data` subdirectory showing the reference elements
-  of the six currently supported element types; ref-segment.mesh,
-  ref-triangle.mesh, ref-square.mesh, ref-tetrahedron.mesh, ref-cube.mesh, and
-  ref-prism.mesh.
+- Added initial support for hypre's mixed integer (mixedint) capability, which
+  uses different data types for local and global indices in order to save memory
+  in large problems. This capability requires that hypre was configured with the
+  --enable-mixedint option. Note that this option is currently tested only in
+  ex1p and may not work in more general settings.

-High-performance computing
--------------------------
- Added initial support for GPU-accelerated versions of PETSc that works with
-  MFEM_USE_CUDA if PETSc has been configured with CUDA support. Examples 1 and 9
-  in the examples/petsc directory have been modified to work with --device cuda.
-  Examples with GAMG (ex1p) and SLEPc (ex11p) are also provided.
+- Added new functionality for constructing low-order refined discretizations and
+  solvers, see the LORDiscretization and LORSolver classes. A new basis type for
+  H(curl) and H(div) spaces is introduced to give spectral equivalence. This
+  functionality is illustrated in the LOR solvers miniapp in miniapps/solvers.

- Added support for explicit vectorization in the high-performance templated
-  code for Fujitsu's A64FX ARM microprocessor architecture.
+libCEED integration improvements
+--------------------------------
+- Refactor the libCEED integration

- Added support for different modes of QuadratureInterpolator on GPU.
-  The layout (QVectorLayout::byNODES|byVDIM) and the tensor products modes can
-  be enabled before calling the Mult, Values, Derivatives, PhysDerivatives and
-  Determinants methods.
+- Add support for VectorCoefficient with libCEED backends.

- Added method Device::SetMemoryTypes that can be used to change the default
-  host and device MemoryTypes before Device setup.
-
- In class MemoryManager, added methods GetDualMemoryType and SetDualMemoryType;
-  dual MemoryTypes are used to determine the second MemoryType (host or device)
-  when only one MemoryType is specified in methods of class Memory.
-
- Added Memory constructor for setting both the host and device MemoryTypes.
-
- Switched the default behavior of device memory allocations so that they are
-  deferred until the device pointer is needed.
-
- Added a second Umpire device MemoryType, DEVICE_UMPIRE_2, with corresponding
-  allocator that can be set with the method SetUmpireDevice2AllocatorName.
-
- Added HOST_PINNED MemoryType and a pinned host allocator for CUDA and HIP.
-
- Added matrix-free GPU-enabled implementations of GradientInterpolator and
-  IdentityInterpolator.
-
-New and updated examples and miniapps
-------------------------------------
- Added a new, very simple example (ex0 and parallel version ex0p). This example
-  solves a simple Poisson problem using H1 elements (the same problem as ex1),
-  but is intended to be extremely simple and approachable for new users.
-
- Added new miniapps demonstrating: 1) the use of GSLIB for overlapping grids,
-  see gslib/schwarz_ex1, and 2) coupling different physics in different domains,
-  see navier/cht. Note that gslib v1.0.7 is require (see INSTALL for details).
-
- Added a new miniapp for computing (signed) distance functions to a point
-  source or zero level set. See miniapps/shifted/distance.cpp.
-
- Added a high-order extension of the shifted boundary method to solve PDEs on
-  non body-fitted meshes. This is illustrated in the new Shifted Diffusion
-  miniapp, see miniapps/shifted/diffusion.cpp.
-
- Added new miniapp directory mtop/ with optimization-oriented block parametric
-  non-linear form and abstract integrators. Two new miniapps, ParHeat and
-  SeqHeat, demonstrate parallel and sequential implementation of gradients
-  evaluation for linear diffusion with discrete density.
-
- Added a new miniapp block-solvers that compares the performance of various
-  solvers for mixed finite element discretization of the second order scalar
-  elliptic equations. Currently available solvers in the miniapp include a
-  block-diagonal preconditioner that is based on approximate Schur complement
-  (implemented in ex5p), and a newly implemented solver DivFreeSolver, which
-  exploits a multilevel decomposition of the Raviart-Thomas space and its
-  divergence-free subspace. See the miniapps/solvers directory for more details.
-
- Introduced new options for the mesh-explorer miniapp to visualize the actual
-  element attributes in parallel meshes while retaining the visualization of the
-  domain decomposition.
-
- Added partial assembly and device support to Example 25/25p, with diagonal
-  preconditioning.
-
- Implemented a filter method for the Navier miniapp to stabilize highly
-  turbulent flows in direct numerical simulation.
-
-Improved testing
----------------
- Transitioned from Travis to GitHub Action for testing/CI on GitHub.
-
- Use Spack (and Uberenv) to automate TPL building in LLNL GitLab tests.
-
- Extended `make test` to include GPU tests when MFEM is built with CUDA or HIP
-  support.
-
- Added a set of suggested git hooks for developers in config/githooks.
-
- Added support for Caliper: a library to integrate performance profiling
-  capabilities into applications. See examples/caliper for more details.
-
- Added a new command line boolean option (`--all`) to the unit tests to launch
-  *all* non-regression tests.
-
- Upgraded the Catch unit test framework from version 2.13.0 to version 2.13.2.
-
-Miscellaneous
-------------
- The following integrations have updated minimum version requirements:
-  * CUDA >= 10.1.168
-  * Ginkgo >= 1.4.0
-  * GSLIB >= 1.0.7
-  * HIOP >= 0.4
-  * HYPRE >= 2.20.0 for mixedint support
-  * HYPRE >= 2.22.0 for CUDA support
-  * libCEED >= 0.8
-  * PETSc >= 3.15.0 for CUDA support
-  * RAJA >= 0.13.0
-  see INSTALL for more details.
-
- Added a "scaled Jacobian" visualization option in the Mesh Explorer miniapp to
-  help identify elements with poor mesh quality.
-
- Added support for reading VTK meshes in XML format.
-
- Added makefile rule to generate TAGS table for vi or Emacs users.
-
- Added HIP support to the CMake build system.
-
- Various other simplifications, extensions, and bugfixes in the code.
-
-API changes
-----------
- Added an abstract interface `mfem::FaceRestriction` for `H1FaceRestriction`
-  and `L2FaceRestriction`.
-  In order to conform with the semantic of `MultTranspose` in `mfem::Operator`,
-  `mfem::FaceRestriction::MultTranspose` now sets instead of adding values, and
-  `mfem::FaceRestriction::AddMultTranspose` should replace previous calls to
-  `mfem::FaceRestriction::MultTranspose`.
+- Add support for ConvectionIntegrator, and VectorConvectionNLFIntegrator with
+  libCEED backends.


 Version 4.2, released on October 30, 2020
 =========================================

-High-performance computing
+High-Performance Computing
 --------------------------
 - Added support for explicit vectorization in the high-performance templated
  code, which can now take advantage of specific classes on the following
@@ -391,6 +305,9 @@ Linear and nonlinear solvers
  matrix with the function HypreParMatrixFromBlocks. This could be useful for
  solving block systems with parallel direct solvers such as STRUMPACK.

+- Added CUDA support for SUNDIALS ODE integrators. See the updated SUNDIALS
+  modification of Example 9/9p.
+
 - Added wrappers for hypre's flexible GMRES solver and the new parallel ILU
  preconditioner. The latter requires hypre version 2.19.0 or later.

@@ -501,7 +418,7 @@ New and updated examples and miniapps
    L2, with partial assembly support in Example 24/24p.
  * Weak Dirichlet boundary conditions (Nitsche) to the NURBS miniapp.

-Data management and visualization
+Data management and Visualization
 ---------------------------------
 - Added support for ADIOS2 for parallel I/O with ParaView visualization. See
  Examples 5, 9, 12, 16. The classes adios2stream and ADIOS2DataCollection
@@ -1,66 +0,0 @@
-cff-version: 1.2.0
-message: "If you use MFEM, please cite it as follows."
-authors:
- family-names: "MFEM Team"
-title: "MFEM: Modular Finite Element Methods [Software]"
-doi: 10.11578/dc.20171025.1248
-url: "https://mfem.org"
-preferred-citation:
-  type: article
-  authors:
-  - family-names: "Anderson"
-    given-names: "Robert"
-    orcid: "https://orcid.org/0000-0002-3508-9944"
-  - family-names: "Andrej"
-    given-names: "Julian"
-    orcid: "https://orcid.org/0000-0001-7661-4840"
-  - family-names: "Barker"
-    given-names: "Andrew"
-    orcid: "https://orcid.org/0000-0003-3572-911X"
-  - family-names: "Bramwell"
-    given-names: "Jamie"
-  - family-names: "Camier"
-    given-names: "Jean-Sylvain"
-    orcid: "https://orcid.org/0000-0003-2421-1999"
-  - family-names: "Cerveny"
-    given-names: "Jakub"
-    orcid: "https://orcid.org/0000-0003-4231-2531"
-  - family-names: "Dobrev"
-    given-names: "Veselin"
-    orcid: "https://orcid.org/0000-0003-1793-5622"
-  - family-names: "Dudouit"
-    given-names: "Yohann"
-    orcid: "https://orcid.org/0000-0001-5831-561X"
-  - family-names: "Fisher"
-    given-names: "Aaron"
-  - family-names: "Kolev"
-    given-names: "Tzanio"
-    orcid: "https://orcid.org/0000-0002-2810-3090"
-  - family-names: "Pazner"
-    given-names: "Will"
-    orcid: "https://orcid.org/0000-0003-4885-2934"
-  - family-names: "Stowell"
-    given-names: "Mark"
-    orcid: "https://orcid.org/0000-0002-5389-7435"
-  - family-names: "Tomov"
-    given-names: "Vladimir"
-    orcid: "https://orcid.org/0000-0002-1846-6816"
-  - family-names: "Akkerman"
-    given-names: "Ido"
-    orcid: "https://orcid.org/0000-0002-5937-0300"
-  - family-names: "Dahm"
-    given-names: "Johann"
-    orcid: "https://orcid.org/0000-0001-9657-3564"
-  - family-names: "Medina"
-    given-names: "David"
-  - family-names: "Zampini"
-    given-names: "Stefano"
-    orcid: "https://orcid.org/0000-0002-0435-0433"
-  doi: "10.1016/j.camwa.2020.06.009"
-  journal: "Computers \\& Mathematics with Applications"
-  month: 1
-  start: 42 # First page number
-  end: 74 # Last page number
-  title: "MFEM: A Modular Finite Element Methods Library"
-  volume: 81
-  year: 2021
@@ -16,9 +16,6 @@ set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH

 # Require C++11 and disable compiler-specific extensions
 set(CMAKE_CXX_STANDARD 11)
-if (MFEM_USE_GINKGO)
-  set(CMAKE_CXX_STANDARD 14)
-endif()
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)

@@ -54,7 +51,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.3.1)
+set(${PROJECT_NAME}_VERSION 4.2.1)

 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -102,9 +99,6 @@ if (MFEM_USE_CUDA)
   endif()
   enable_language(CUDA)
   set(CMAKE_CUDA_STANDARD 11)
-   if (MFEM_USE_GINKGO)
-     set(CMAKE_CUDA_STANDARD 14)
-   endif()
   set(CMAKE_CUDA_STANDARD_REQUIRED ON)
   set(CMAKE_CUDA_EXTENSIONS OFF)
   set(CUDA_FLAGS "--expt-extended-lambda")
@@ -175,10 +169,6 @@ else()
  set(MFEM_DEBUG OFF)
 endif()

-if (WIN32)
-	add_definitions(-D_USE_MATH_DEFINES)
-endif()
-
 # MPI -> hypre; PETSc (optional)
 if (MFEM_USE_MPI)
  find_package(MPI REQUIRED)
@@ -250,7 +240,6 @@ if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
    message(FATAL_ERROR " *** MFEM_USE_LEGACY_OPENMP requires MFEM_THREAD_SAFE=ON.")
  endif()
  find_package(OpenMP REQUIRED)
-  set(OPENMP_LIBRARIES ${OpenMP_CXX_LIBRARIES})
 endif()

 # SuiteSparse (before SUNDIALS which may depend on KLU)
@@ -335,10 +324,6 @@ if (MFEM_USE_CONDUIT)
    find_package(Conduit REQUIRED conduit relay blueprint )
 endif()

-if (MFEM_USE_FMS)
-    find_package(FMS REQUIRED fms )
-endif()
-
 # Axom/Sidre
 if (MFEM_USE_SIDRE)
  find_package(Axom REQUIRED Axom)
@@ -366,6 +351,30 @@ if (MFEM_USE_HIOP)
  # find_package updates HIOP_FOUND, HIOP_INCLUDE_DIRS, HIOP_LIBRARIES
 endif()

+# ADEPT package
+if (MFEM_USE_ADEPT)
+        find_package(ADEPT REQUIRED)
+        # find_package updates ADEPT_FOUND, ADEPT_INCLUDE_DIRS, ADEPT_LIBRARIES
+endif()
+
+# FADBAD++ package
+if (MFEM_USE_FADBADPP)
+        find_package(FADBADPP REQUIRED)
+        # find_package updates FADBADPP_FOUND, FADBADPP_INCLUDE_DIRS, FADBADPP_LIBRARIES
+endif()
+
+# CoDiPack package
+if (MFEM_USE_CODIPACK)
+	find_package(CODIPACK REQUIRED)
+	# find_package updates CODIPACK_FOUND, CODIPACK_INCLUDE_DIRS, CODIPACK_LIBRARIES
+endif()
+
+# Eigen package
+if (MFEM_USE_EIGEN)
+	find_package(EIGEN REQUIRED)
+	# find_package updates EIGEN_FOUND, EIGEN_INCLUDE_DIRS, EIGEN_LIBRARIES
+endif()
+
 # OCCA
 if (MFEM_USE_OCCA)
   find_package(OCCA REQUIRED)
@@ -392,9 +401,6 @@ if (MFEM_USE_HIP)
  if (HIP_ARCH)
    message(STATUS "Using HIP architecture: ${HIP_ARCH}")
    list(APPEND HIP_HIPCC_FLAGS "--amdgpu-target=${HIP_ARCH}")
-    if (MFEM_USE_GINKGO)
-      list(APPEND HIP_HIPCC_FLAGS "-std=c++14")
-    endif()
  endif()
 endif()

@@ -433,10 +439,9 @@ endif()
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
 set(MFEM_TPLS MPI_CXX OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS PETSC
-    SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB NETCDF
+    SLEPC MESQUITE MUMPS STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
-    CUSPARSE MKL_CPARDISO AMGX CALIPER)
-
+    CUSPARSE MKL_CPARDISO AMGX CALIPER FADBADPP ADEPT CODIPACK EIGEN)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")
@@ -455,9 +460,6 @@ include_directories(${TPL_INCLUDE_DIRS})
 if (OPENMP_FOUND)
  message(STATUS "MFEM: using package OpenMP")
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
-  if (MFEM_USE_CUDA)
-    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=${OpenMP_CXX_FLAGS}")
-  endif()
 endif()

 message(STATUS "MFEM build type: CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}")
@@ -4,9 +4,7 @@

 <p align="center">
 <a href="https://github.com/mfem/mfem/blob/master/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-BSD-brightgreen.svg"></a>
-<a href="https://github.com/mfem/mfem/actions?query=workflow%3Arepo-check+branch%3Amaster"><img alt="Repo check" src="https://github.com/mfem/mfem/actions/workflows/repo-check.yml/badge.svg?branch=master"></a>
-<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuild-analysis+branch%3Amaster"><img alt="Build Analysis" src="https://github.com/mfem/mfem/actions/workflows/mfem-analysis.yml/badge.svg?branch=master"></a>
-<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuilds-and-tests+branch%3Amaster"><img alt="Builds and Tests" src="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml/badge.svg?branch=master"></a>
+<a href="https://travis-ci.org/mfem/mfem"><img alt="Build Status" src="https://travis-ci.org/mfem/mfem.svg?branch=master"></a>
 <a href="https://ci.appveyor.com/project/mfem/mfem"><img alt="Build Status" src="https://ci.appveyor.com/api/projects/status/19non9sqm6msi2wy?svg=true"></a>
 <a href="https://mfem.github.io/doxygen/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
 </p>
@@ -65,8 +63,6 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
  development branches off `mfem:master`.
 - Please follow the [developer guidelines](#developer-guidelines), in particular
  with regards to documentation and code styling.
- Please do not commit large/binary files to the central repository (use a fork
-  instead).
 - Pull requests  should be issued toward `mfem:master`. Make sure
  to check the items off the [Pull Request Checklist](#pull-request-checklist).
 - When your contribution is fully working and ready to be reviewed, add
@@ -75,7 +71,6 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
  reviewers to evaluate the changes.
 - The reviewers have 3 weeks to evaluate the PR and work with the author to
  fix issues and implement improvements.
- During review there should be no force pushes/rewriting history in the branch.
 - After approval, MFEM developers merge the PR manually in the [mfem:next branch](#masternext-workflow).
 - After a week of testing in `mfem:next`, the original PR is merged in `mfem:master`.
 - We use [milestones](https://github.com/mfem/mfem/milestones) to coordinate the
@@ -96,8 +91,8 @@ The MFEM source code has the following structure:
 ```
  .
  ├── config
-  │   ├── cmake
-  │   └── githooks
+  │   └── cmake
+  │       └── ...
  ├── data
  ├── doc
  ├── examples
@@ -134,10 +129,10 @@ The MFEM source code has the following structure:
  └── tests
      ├── convergence
      ├── gitlab
-      ├── mem_manager
      ├── par-mesh-format
      ├── scripts
      └── unit
+          └── ...
 ```

 #### Main directories and classes
@@ -368,10 +363,6 @@ Before you can start, you need a GitHub account, here are a few suggestions:
  two reviewers to evaluate the changes. The reviewers have 3 weeks to evaluate
  the PR and work with the author to implement improvements and fix issues.

- Once the `ready-for-review` label has been applied and reviewers have been
-  assigned, the PR is considered under review. To help with the review process
-  there should be no force pushes/rewriting history in the branch.
-
 - After approval, the PR is [tested](#masternext-workflow) for a week with
  other approved PRs in the `mfem:next` branch.

@@ -379,20 +370,16 @@ Before you can start, you need a GitHub account, here are a few suggestions:
  `mfem:next`, see the [README](tests/scripts/README) file in that directory
  for more details.

- Track the GitHub Actions and Appveyor [continuous integration](#automated-testing)
+- Track the Travis CI, Github Actions and Appveyor [continuous integration](#automated-testing)
  builds at the end of the PR. These should generally run clean, so address any
  errors as soon as possible. Please ask if you are unsure how to do that.

- Note that some tests, such as the `branch-history` check in GitHub Actions
-  are safeguards that are allowed to fail in certain cases.
+- Note that some tests, such as the `branch-history` check in Travis and Github
+  Actions are safeguards that are allowed to fail in certain cases.

 - Other tests, such as the `code-style`, `documentation` and `gitignore`
-  checks in GitHub Actions enforce MFEM-specific rules which are explained in
-  the error messages and the `tests/scripts` directory.
-
- Also note that the tests `branch-history` and `repos-checks` found in GitHub
-  Actions can be triggered automatically before each push using git hooks. See
-  the [git hooks README](config/githooks/README.md) for a detailed explanation.
+  checks in Travis and Github Actions enforce MFEM-specific rules which are
+  explained in the error messages and the `tests/scripts` directory.

 - If triggered, track the status of the LLNL GitLab tests. If failing, ask
  one of the _LLNL developers_ for details.
@@ -412,7 +399,7 @@ Before a PR can be merged, it should satisfy the following:
    - [ ] Does `make` or `cmake` have a new target?
    - [ ] Did the requirements or the installation process change? *(rare)*
 - [ ] Update continuous integration server configurations if necessary (e.g. with new version requirements for each of MFEM's dependencies)
-    - [ ] `.github`
+    - [ ] `.travis.yml`
    - [ ] `.appveyor.yml`
 - [ ] Update `.gitignore`:
    - [ ] Check if `make distclean; git status` shows any files that were generated from the source by the project (not an IDE) but we don't want to track in the repository.
@@ -529,7 +516,7 @@ MFEM uses a `master`/`next`-branch workflow as described below:
    - [ ] `doc/CodeDocumentation.conf.in`
 - [ ] Check that version requirements for each of MFEM's dependencies are documented in `INSTALL` and up-to-date
 - [ ] Check that continuous integration server configurations reflect the dependency version requirements of the new release
-    - [ ] `.github`
+    - [ ] `.travis.yml`
    - [ ] `.appveyor.yml`
 - [ ] Update the `CHANGELOG` to organize all release contributions
 - [ ] Review the whole source code once over
@@ -591,17 +578,14 @@ MFEM uses a `master`/`next`-branch workflow as described below:
 MFEM has several levels of automated testing running on GitHub, as well as on
 local Mac and Linux workstations, and Livermore Computing clusters at LLNL.

-In addition, developers can set local git hooks to run some quick checks on
-commit or push, see the [README](config/githooks/README.md) in the `config/githooks`
-directory.
-
 ### Linux and Mac smoke tests
-We use GitHub Actions to drive the default tests on the `master` and `next`
-branches. See the `.github/workflows` files and the logs at
-[https://github.com/mfem/mfem/actions](https://github.com/mfem/mfem/actions).
+We use Travis CI and Github Actions to drive the default tests on the `master`
+and `next` branches. See the `.travis` file and the logs at
+[https://travis-ci.org/mfem/mfem](https://travis-ci.org/mfem/mfem).

-Testing using GitHub Actions should be kept lightweight, as there is a time
-constraint on jobs. Two virtual machines are configured - Mac (OS X) and Linux.
+Testing using Travis CI and Github Actions should be kept lightweight, as there
+is a time constraint on jobs. Two virtual machines are configured - Mac (OS X)
+and Linux.

 - Tests on the `master` branch are triggered whenever a PR is issued on this branch.
 - Tests on the `next` branch are currently scheduled to run each night.
@@ -58,7 +58,6 @@ following package managers:

 - Spack, https://github.com/spack/spack
 - OpenHPC, http://openhpc.community
- Conda-forge, https://conda-forge.org (pre-built binaries linked with OpenMPI/MPICH, hypre, and METIS)
 - Homebrew/Science, https://github.com/Homebrew/homebrew-science (deprecated)

 We also recommend downloading and building the MFEM-based GLVis visualization
@@ -463,6 +462,27 @@ MFEM_USE_HIOP = YES/NO
   Enable the usage of HiOp (https://github.com/LLNL/hiop) in MFEM. HiOp is an
   HPC solver for nonlinear optimization problems.

+MFEM_USE_ADEPT = YES/NO
+  Enable automatic differentiation using the ADEPT library.
+  (http://www.met.reading.ac.uk/clouds/adept)
+  Please, compile the library with flag --disable-openmp.
+
+MFEM_USE_FADBADPP = YES/NO
+  Enable automatic differentiation using the FADBAD++ library.
+  www.fadbad.com/fadbad.html
+
+MFEM_USE_CODIPACK = YES/NO
+  Enable automatic differentiation using the CoDiPack library.
+  www.scicomp.uni-kl.de/codi/
+
+MFEM_USE_EIGEN = YES/NO
+  Enable the Eigen library.
+  https://gitlab.com/libeigen/eigen
+
+MFEM_USE_ADFORWARD = YES/NO
+  Enable forward mode for AD packages. This option is valid
+  only if the AD package supports two modes (backward/forward).
+
 MFEM_USE_CUDA = YES/NO
   Enables support for CUDA devices in MFEM. CUDA is a parallel computing
   platform and programming model for general computing on graphical processing
@@ -474,7 +494,7 @@ MFEM_USE_HIP = YES/NO
   Enables support for AMD devices in MFEM. HIP is a heterogeneous-compute
   interface for portability developed by AMD that can target both AMD and
   NVIDIA GPUs. The variable HIP_ARCH is used to specify the AMD GPU processor
-   used during compilation (by default, HIP_ARCH=gfx900). When enabled, this
+   used during compilation (by default, HIP_ARCH=gfx900).  When enabled, this
   option uses the HIP_* build options, see below.

 MFEM_USE_RAJA = YES/NO
@@ -516,13 +536,6 @@ MFEM_USE_CALIPER = YES/NO
   profiling at runtime with Caliper's configuration API. Alternatively, one
   can configure Caliper through environment variables or config files.

-MFEM_USE_FMS = YES/NO
-   Enables support for the FMS library which consists of the DataCollection
-   sub-class mfem::FMSDataCollection for I/O in FMS formats, see the header file
-   fem/fmsdatacollection.hpp. In addition, this option enables in-memory
-   convetion routines between FMS's FmsDataCollection structure and MFEM's
-   DataCollection class, see the header file fem/fmsconvert.hpp.
-
 MFEM_BUILD_TAG = (any value)
   An optional tag to characterize the build. Exported to config/config.mk.
   Can be used to identify the MFEM build from other makefiles.
@@ -547,9 +560,8 @@ The specific libraries and their options are:
  See also the "Specific options for hypre" section at the end of this file.
  URL: https://github.com/hypre-space/hypre and https://www.llnl.gov/casc/hypre
  Options: HYPRE_OPT, HYPRE_LIB.
-  Versions: HYPRE >= 2.10.0b  (HYPRE built without CUDA)
-            HYPRE >= 2.20.0   (HYPRE built with '--enable-mixedint')
-            HYPRE >= 2.22.1   (HYPRE built with CUDA)
+  Versions: HYPRE >= 2.10.0b,
+            HYPRE >= 2.20.0 for '--enable-mixedint' support.

 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
  MFEM_USE_METIS_5 = YES (default is to use METIS 4).
@@ -619,11 +631,10 @@ The specific libraries and their options are:
  Versions: STRUMPACK >= 3.0.0.

 - Ginkgo (optional), used when MFEM_USE_GINKGO = YES. Note that Ginkgo needs a
-  C++ compiler that supports the C++-14 standard. For additional requirements
-  and dependencies of specific modules, see the Ginkgo webpage below.
+  C++ compiler that supports the C++-11 standard. For additional requirements
+  and dependencies of specific modules see the Ginkgo webpage below.
  URL: https://ginkgo-project.github.io
-  Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or Debug).
-  Versions: Ginkgo >= 1.4.0.
+  Options: GINKGO_OPT (Not used), GINKGO_LIB.

 - AmgX (optional), used when MFEM_USE_AMGX = YES.
  URL: https://github.com/NVIDIA/AMGX
@@ -694,6 +705,26 @@ The specific libraries and their options are:
  Options: HIOP_OPT, HIOP_LIB.
  Versions: HIOP >= 0.4.

+- ADEPT (optional), used with MFEM_USE_ADEPT = YES
+  URL: www.met.reading.ac.uk/clouds/adept/
+  Options: ADEPT_OPT, ADEPT_LIB
+  Versions: 1.1 and 2.0.5
+
+- FADBAD++ (optiobal), used with MFEM_USE_FADBADPP = YES
+  URL: www.fadbad.com/fadbad.html
+  Options: FADBADPP_OPT
+  Versions: 2.1
+
+- CoDiPack (optiobal), used with MFEM_USE_CODIPACK = YES
+  URL: https://www.scicomp.uni-kl.de/codi/
+  Options: CODIPACK_OPT
+  Versions: 1.9.3
+
+- Eigen (optiobal), used with MFEM_USE_EIGEN = YES
+  URL: https://gitlab.com/libeigen/eigen
+  Options: EIGEN_OPT
+  Versions: 3.3.9
+
 - GSLIB (optional), used when MFEM_USE_GSLIB = YES. The gslib library must be
  built prior to the MFEM build, as follows: download gslib-1.0.7, untar it at
  the same level as MFEM and create a symbolic link: "ln -s gslib-1.0.7 gslib".
@@ -761,11 +792,6 @@ The specific libraries and their options are:
  URL: https://zlib.net
  Options: ZLIB_OPT, ZLIB_LIB.

- FMS (optional), used when MFEM_USE_FMS = YES.
-  URL: https://github.com/CEED/FMS
-  Options: FMS_OPT, FMS_LIB.
-  Versions: FMS >= 0.2.
-
 Building with CMake
 ===================
 The MFEM build system consists of two steps: configuration and compilation.
@@ -890,6 +916,11 @@ MFEM_USE_MPFR
 MFEM_USE_ZLIB
 MFEM_USE_PUMI
 MFEM_USE_HIOP
+MFEM_USE_ADEPT
+MFEM_USE_FADBADPP
+MFEM_USE_CODIPACK
+MFEM_USE_EIGEN
+MFEM_USE_ADFORWARD
 MFEM_USE_CUDA
 MFEM_USE_OCCA
 MFEM_USE_CEED
@@ -897,7 +928,6 @@ MFEM_USE_RAJA
 MFEM_USE_UMPIRE
 MFEM_USE_SIDRE
 MFEM_USE_CALIPER
-MFEM_USE_FMS

 The following options are CMake specific:

@@ -947,12 +977,15 @@ The CMake build system adds auto-detection for the following packages/libraries:
 - POSIXCLOCKS
 - PUMI
 - HIOP
+ - ADEPT
+ - FADBAD++
+ - CoDiPack
+ - Eigen
 - OCCA
 - RAJA
 - UMPIRE
 - AXOM - Used when MFEM_USE_SIDRE is enabled
 - CALIPER
- - FMS

 The following built-in CMake packages are also used:

@@ -970,7 +1003,7 @@ config/config.hpp.in:

   cp config/config.hpp.in config/_config.hpp

-The file config/_config.hpp can then be edited to enable desired options. The
+The file config/_config.hpp can then be edited to enable desired options.  The
 MFEM library is simply a combination of all object files obtained by compiling
 the .cpp source files in the source directories: general, linalg, mesh, and fem.

@@ -978,7 +1011,7 @@ the .cpp source files in the source directories: general, linalg, mesh, and fem.
 Specifying an MPI job launcher
 ==============================
 By default, MFEM will use 'mpirun -np #' to launch any of its parallel tests or
-miniapps, where # is the number of MPI tasks. An alternate MPI launcher can be
+miniapps, where # is the number of MPI tasks.  An alternate MPI launcher can be
 provided by setting the MFEM_MPIEXEC and MFEM_MPIEXEC_NP config variables.

 MFEM will expect the launcher command, plus the command line option to allow it
@@ -256,10 +256,6 @@ IF (DEFINED TPL_ENABLE_SIDRE)
  SET(MFEM_USE_SIDRE ${TPL_ENABLE_SIDRE} CACHE BOOL "Enable Axom/Sidre usage" FORCE)
 ENDIF()

-IF (DEFINED TPL_ENABLE_FMS)
-  SET(MFEM_USE_FMS ${TPL_ENABLE_FMS} CACHE BOOL "Enable FMS usage" FORCE)
-ENDIF()
-
 IF (DEFINED TPL_ENABLE_CONDUIT)
  SET(MFEM_USE_CONDUIT ${TPL_ENABLE_CONDUIT} CACHE BOOL "Enable Conduit usage" FORCE)
 ENDIF()
@@ -44,7 +44,6 @@ set(MFEM_USE_PETSC @MFEM_USE_PETSC@)
 set(MFEM_USE_SLEPC @MFEM_USE_SLEPC@)
 set(MFEM_USE_MPFR @MFEM_USE_MPFR@)
 set(MFEM_USE_SIDRE @MFEM_USE_SIDRE@)
-set(MFEM_USE_FMS @MFEM_USE_FMS@)
 set(MFEM_USE_CONDUIT @MFEM_USE_CONDUIT@)
 set(MFEM_USE_PUMI @MFEM_USE_PUMI@)
 set(MFEM_USE_CUDA @MFEM_USE_CUDA@)
@@ -54,6 +53,11 @@ set(MFEM_USE_CEED @MFEM_USE_CEED@)
 set(MFEM_USE_UMPIRE @MFEM_USE_UMPIRE@)
 set(MFEM_USE_SIMD @MFEM_USE_SIMD@)
 set(MFEM_USE_ADIOS2 @MFEM_USE_ADIOS2@)
+set(MFEM_USE_ADEPT @MFEM_USE_ADEPT@)
+set(MFEM_USE_FADBADPP @MFEM_USE_FADBADPP@)
+set(MFEM_USE_CODIPACK @MFEM_USE_CODIPACK@)
+set(MFEM_USE_EIGEN @MFEM_USE_EIGEN@)
+set(MFEM_USE_ADFORWARD @MFEM_USE_ADFORWARD@)
 set(MFEM_USE_CALIPER @MFEM_USE_CALIPER@)

 set(MFEM_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
@@ -119,9 +119,6 @@
 // Enable the use of SIMD in the high performance templated classes
 #cmakedefine MFEM_USE_SIMD

-// Enable MFEM functionality based on the FMS library
-#cmakedefine MFEM_USE_FMS
-
 // Enable MFEM functionality based on Conduit
 #cmakedefine MFEM_USE_CONDUIT

@@ -172,6 +169,21 @@
 // library.
 #cmakedefine MFEM_USE_SIMMETRIX

+// use ADEPT library for AD
+#cmakedefine MFEM_USE_ADEPT
+
+// use FADBAD++ library for AD
+#cmakedefine MFEM_USE_FADBADPP
+
+// use CoDiPack library for AD
+#cmakedefine MFEM_USE_CODIPACK
+
+// use Eigen library
+#cmakedefine MFEM_USE_EIGEN
+
+// use forward mode for automatic differentiation
+#cmakedefine MFEM_USE_ADFORWARD
+
 // Enable interface to the MKL CPardiso library.
 #cmakedefine MFEM_USE_MKL_CPARDISO

@@ -0,0 +1,23 @@
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability see http://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.
+
+# Sets the following variables:
+#   - ADEPT_FOUND
+#   - ADEPT_INCLUDE_DIRS
+#   - ADEPT_LIBRARIES
+
+include(MfemCmakeUtilities)
+mfem_find_package(ADEPT ADEPT ADEPT_DIR
+  "include" "adept.hpp"
+  "lib" "libadept.so"
+  "Paths to headers required by ADEPT."
+  "Libraries required by ADEPT.")
+
@@ -0,0 +1,23 @@
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability see http://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.
+
+# Sets the following variables:
+#   - CODIPACK_FOUND
+#   - CODIPACK_INCLUDE_DIRS
+#   - CODIPACK_LIBRARIES
+
+include(MfemCmakeUtilities)
+mfem_find_package(CODIPACK CODIPACK CODIPACK_DIR
+  "include" "codi.h"
+  "lib" ""
+  "Paths to headers required by CODIPACK."
+  "Libraries required by CODIPACK.")
+
@@ -0,0 +1,23 @@
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability see http://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.
+
+# Sets the following variables:
+#   - EIGEN_FOUND
+#   - EIGEN_INCLUDE_DIRS
+#   - EIGEN_LIBRARIES
+
+include(MfemCmakeUtilities)
+mfem_find_package(EIGEN EIGEN EIGEN_DIR
+  "include" "codi.h"
+  "lib" ""
+  "Paths to headers required by EIGEN."
+  "Libraries required by EIGEN.")
+
@@ -0,0 +1,23 @@
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability see http://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.
+
+# Sets the following variables:
+#   - FADBADPP_FOUND
+#   - FADBADPP_INCLUDE_DIRS
+#   - FADBADPP_LIBRARIES
+
+include(MfemCmakeUtilities)
+mfem_find_package(FADBADPP FADBADPP FADBADPP_DIR
+  "include" "fadiff.h"
+  "lib" ""
+  "Paths to headers required by FADBADPP."
+  "Libraries required by FADBADPP.")
+
@@ -1,20 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - FMS_FOUND
-#   - FMS_LIBRARIES
-#   - FMS_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(FMS FMS FMS_DIR
-  "include" fms.h "lib" fms
-  "Paths to headers required by FMS." "Libraries required by FMS.")
@@ -47,7 +47,6 @@ endfunction()
 macro(mfem_add_executable NAME)
  if (MFEM_USE_HIP)
    hip_add_executable(${NAME} ${ARGN})
-    set_target_properties(${NAME} PROPERTIES LINKER_LANGUAGE CXX)
  else()
    add_executable(${NAME} ${ARGN})
  endif()
@@ -117,9 +117,6 @@
 // Enable the use of SIMD in the high performance templated classes
 // #define MFEM_USE_SIMD

-// Enable FMS support
-// #define MFEM_USE_FMS
-
 // Enable Conduit support
 // #define MFEM_USE_CONDUIT

@@ -177,6 +174,21 @@
 // library.
 // #define MFEM_USE_SIMMETRIX

+// use ADEPT library for AD
+// #define MFEM_USE_ADEPT
+
+// use FADBAD++ library for AD
+// #define MFEM_USE_FADBADPP
+
+// use CoDiPack library for AD
+// #define MFEM_USE_CODIPACK
+
+// use Eigen library 
+// #define MFEM_USE_EIGEN
+
+// use forward mode for automatic differentiation
+// #define MFEM_USE_ADFORWARD
+
 // Enable interface to the MKL CPardiso library.
 // #define MFEM_USE_MKL_CPARDISO

@@ -43,10 +43,14 @@ MFEM_USE_PETSC         = @MFEM_USE_PETSC@
 MFEM_USE_SLEPC         = @MFEM_USE_SLEPC@
 MFEM_USE_MPFR          = @MFEM_USE_MPFR@
 MFEM_USE_SIDRE         = @MFEM_USE_SIDRE@
-MFEM_USE_FMS           = @MFEM_USE_FMS@
 MFEM_USE_CONDUIT       = @MFEM_USE_CONDUIT@
 MFEM_USE_PUMI          = @MFEM_USE_PUMI@
 MFEM_USE_HIOP          = @MFEM_USE_HIOP@
+MFEM_USE_ADEPT         = @MFEM_USE_ADEPT@
+MFEM_USE_FADBADPP      = @MFEM_USE_FADBADPP@
+MFEM_USE_CODIPACK      = @MFEM_USE_CODIPACK@
+MFEM_USE_EIGEN         = @MFEM_USE_EIGEN@
+MFEM_USE_ADFORWARD     = @MFEM_USE_ADFORWARD@
 MFEM_USE_GSLIB         = @MFEM_USE_GSLIB@
 MFEM_USE_CUDA          = @MFEM_USE_CUDA@
 MFEM_USE_HIP           = @MFEM_USE_HIP@
@@ -45,7 +45,6 @@ option(MFEM_USE_PETSC "Enable PETSc support." OFF)
 option(MFEM_USE_SLEPC "Enable SLEPc support." OFF)
 option(MFEM_USE_MPFR "Enable MPFR usage." OFF)
 option(MFEM_USE_SIDRE "Enable Axom/Sidre usage" OFF)
-option(MFEM_USE_FMS "Enable FMS usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
 option(MFEM_USE_PUMI "Enable PUMI" OFF)
 option(MFEM_USE_HIOP "Enable HiOp" OFF)
@@ -56,6 +55,11 @@ option(MFEM_USE_CEED "Enable CEED" OFF)
 option(MFEM_USE_UMPIRE "Enable Umpire" OFF)
 option(MFEM_USE_SIMD "Enable use of SIMD intrinsics" OFF)
 option(MFEM_USE_ADIOS2 "Enable ADIOS2" OFF)
+option(MFEM_USE_ADEPT "Enable AD using ADEPT" OFF)
+option(MFEM_USE_FADBADPP "Enable AD using FADBAD++" OFF)
+option(MFEM_USE_CODIPACK "Enable AD using CoDiPack" OFF)
+option(MFEM_USE_EIGEN "Enable Eigen" OFF)
+option(MFEM_USE_ADFORWARD "Enable forward mode for AD" OFF)
 option(MFEM_USE_CALIPER "Enable Caliper support" OFF)
 option(MFEM_USE_MKL_CPARDISO "Enable MKL CPardiso" OFF)

@@ -97,11 +101,6 @@ set(HYPRE_DIR "${MFEM_DIR}/../hypre/src/hypre" CACHE PATH
 # If hypre was compiled to depend on BLAS and LAPACK:
 # set(HYPRE_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
 #     "Packages that HYPRE depends on.")
-if (MFEM_USE_CUDA)
-   # This is only necessary when hypre is built with cuda:
-   set(HYPRE_REQUIRED_LIBRARIES "-lcusparse" "-lcurand" CACHE STRING
-       "Libraries that HYPRE depends on.")
-endif()

 set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library.")

@@ -138,10 +137,10 @@ set(MUMPS_DIR "${MFEM_DIR}/../MUMPS_5.2.0" CACHE PATH
    "Path to the MUMPS library.")
 # Packages required by MUMPS, depending on how it was compiled.
 set(MUMPS_REQUIRED_PACKAGES "MPI" "BLAS" "METIS" "ScaLAPACK" CACHE STRING
-    "Additional packages required by MUMPS.")
+    "Additional packages required by MUMPS.")    
 # If the MPI package does not find all required Fortran libraries:
 # set(MUMPS_REQUIRED_LIBRARIES "gfortran" "mpi_mpifh" CACHE STRING
-#     "Additional libraries required by MUMPS.")
+#     "Additional libraries required by MUMPS.")    

 set(STRUMPACK_DIR "${MFEM_DIR}/../STRUMPACK-build" CACHE PATH
    "Path to the STRUMPACK library.")
@@ -193,12 +192,6 @@ set(SLEPC_ARCH "arch-linux2-c-debug" CACHE STRING "SLEPC build architecture.")

 set(MPFR_DIR "" CACHE PATH "Path to the MPFR library.")

-set(FMS_DIR "${MFEM_DIR}/../fms" CACHE PATH
-    "Path to the FMS library.")
-# If FMS is built with Conduit:
-# set(FMS_REQUIRED_PACKAGES "Conduit/relay" CACHE STRING
-#     "Additional packages required by FMS.")
-
 set(CONDUIT_DIR "${MFEM_DIR}/../conduit" CACHE PATH
    "Path to the Conduit library.")

@@ -230,6 +223,18 @@ set(BLAS_LIBRARIES "" CACHE STRING "The BLAS library.")
 set(LAPACK_INCLUDE_DIRS "" CACHE STRING "Path to LAPACK headers.")
 set(LAPACK_LIBRARIES "" CACHE STRING "The LAPACK library.")

+set(ADEPT_INCLUDE_DIRS "${MFEM_DIR}/../adept-1.1/include" CACHE STRING "Path to ADEPT headers.")
+set(ADEPT_LIBRARIES "-L${MFEM_DIR}/../adept-1.1/lib -ladept" CACHE STRING "The ADEPT library.")
+
+set(FADBADPP_INCLUDE_DIRS "${MFEM_DIR}/../FADBAD++" CACHE STRING "Path to FADBAD++ headers.")
+set(FADBADPP_LIBRARIES "")
+
+set(CODIPACK_INCLUDE_DIRS "${MFEM_DIR}/../CoDiPack/inlude" CACHE STRING "Path to CoDoPack headers.")
+set(CODIPACK_LIBRARIES "")
+
+set(EIGEN_INCLUDE_DIRS "${MFEM_DIR}/../Eigen/" CACHE STRING "Path to Eigen headers.")
+set(EIGEN_LIBRARIES "")
+
 # Some useful variables:
 set(CMAKE_SKIP_PREPROCESSED_SOURCE_RULES ON) # Skip *.i rules
 set(CMAKE_SKIP_ASSEMBLY_SOURCE_RULES  ON)    # Skip *.s rules
@@ -18,9 +18,6 @@
 # Some choices below are based on the OS type:
 NOTMAC := $(subst Darwin,,$(shell uname -s))

-ETAGS_BIN = $(shell command -v etags 2> /dev/null)
-EGREP_BIN = $(shell command -v egrep 2> /dev/null)
-
 CXX = g++
 MPICXX = mpicxx

@@ -136,7 +133,6 @@ MFEM_USE_PETSC         = NO
 MFEM_USE_SLEPC         = NO
 MFEM_USE_MPFR          = NO
 MFEM_USE_SIDRE         = NO
-MFEM_USE_FMS           = NO
 MFEM_USE_CONDUIT       = NO
 MFEM_USE_PUMI          = NO
 MFEM_USE_HIOP          = NO
@@ -150,6 +146,11 @@ MFEM_USE_CALIPER       = NO
 MFEM_USE_UMPIRE        = NO
 MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
+MFEM_USE_ADEPT         = NO
+MFEM_USE_FADBADPP      = NO
+MFEM_USE_CODIAPCK      = NO
+MFEM_USE_EIGEN         = NO
+MFEM_USE_ADFORWARD     = NO
 MFEM_USE_MKL_CPARDISO  = NO

 # MPI library compile and link flags
@@ -175,10 +176,6 @@ LIBUNWIND_LIB = $(if $(NOTMAC),-lunwind -ldl,)
 HYPRE_DIR = @MFEM_DIR@/../hypre/src/hypre
 HYPRE_OPT = -I$(HYPRE_DIR)/include
 HYPRE_LIB = -L$(HYPRE_DIR)/lib -lHYPRE
-ifeq (YES,$(MFEM_USE_CUDA))
-   # This is only necessary when hypre is built with cuda:
-   HYPRE_LIB += -lcusparse -lcurand
-endif

 # METIS library configuration
 ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK)$(MFEM_USE_MUMPS),NONONO)
@@ -293,23 +290,9 @@ STRUMPACK_LIB = -L$(STRUMPACK_DIR)/lib -lstrumpack $(MPI_FORTRAN_LIB)\

 # Ginkgo library configuration (currently not needed)
 GINKGO_DIR = @MFEM_DIR@/../ginkgo/install
-GINKGO_BUILD_TYPE=Release
-ifeq ($(MFEM_USE_GINKGO),YES)
-   BASE_FLAGS = -std=c++14
-endif
 GINKGO_OPT = -isystem $(GINKGO_DIR)/include
-GINKGO_LIB_DIR = $(sort $(dir $(wildcard $(GINKGO_DIR)/lib*/libginkgo*.a  $(GINKGO_DIR)/lib*/libginkgo*.so $(GINKGO_DIR)/lib*/libginkgo*.dylib $(GINKGO_DIR)/lib*/libginkgo*.dll)))
-ALL_GINKGO_LIBS_DEBUG = $(notdir $(basename $(wildcard $(GINKGO_DIR)/lib*/libginkgo*d.a  $(GINKGO_DIR)/lib*/libginkgo*d.so $(GINKGO_DIR)/lib*/libginkgo*d.dylib $(GINKGO_DIR)/lib*/libginkgo*d.dll)))
-ALL_GINKGO_LIBS = $(notdir $(basename $(wildcard $(GINKGO_DIR)/lib*/libginkgo*.a  $(GINKGO_DIR)/lib*/libginkgo*.so $(GINKGO_DIR)/lib*/libginkgo*.dylib $(GINKGO_DIR)/lib*/libginkgo*.dll)))
-ALL_GINKGO_LIBS_RELEASE = $(filter-out $(ALL_GINKGO_LIBS_DEBUG),$(ALL_GINKGO_LIBS))
-GINKGO_LINK = $(subst libginkgo,-lginkgo,$(ALL_GINKGO_LIBS_RELEASE))
-ifeq ($(GINKGO_BUILD_TYPE),Debug)
-  ifneq (,$(ALL_GINKGO_LIBS_DEBUG))
-    GINKGO_LINK = $(subst libginkgo,-lginkgo,$(ALL_GINKGO_LIBS_DEBUG))
-  endif
-else
-endif
-GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_LIB_DIR) -L$(GINKGO_LIB_DIR) $(GINKGO_LINK)
+GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_DIR)/lib -L$(GINKGO_DIR)/lib -lginkgo\
+ -lginkgo_omp -lginkgo_cuda -lginkgo_reference

 # AmgX library configuration
 AMGX_DIR = @MFEM_DIR@/../amgx
@@ -362,11 +345,6 @@ endif
 MPFR_OPT =
 MPFR_LIB = -lmpfr

-# FMS and required libraries configuration
-FMS_DIR = $(MFEM_DIR)/../fms
-FMS_OPT = -I$(FMS_DIR)/include
-FMS_LIB = -Wl,-rpath,$(FMS_DIR)/lib -L$(FMS_DIR)/lib -lfms
-
 # Conduit and required libraries configuration
 CONDUIT_DIR = @MFEM_DIR@/../conduit
 CONDUIT_OPT = -I$(CONDUIT_DIR)/include/conduit
@@ -406,6 +384,26 @@ HIOP_DIR = @MFEM_DIR@/../hiop/install
 HIOP_OPT = -I$(HIOP_DIR)/include
 HIOP_LIB = -L$(HIOP_DIR)/lib -lhiop $(LAPACK_LIB)

+# ADEPT
+ADEPT_DIR = @MFEM_DIR@/../adept-1.1
+ADEPT_OPT = -I$(ADEPT_DIR)/include
+ADEPT_LIB = -L$(ADEPT_DIR)/lib -ladept
+
+# FADBAD++
+FADBADPP_DIR = @MFEM_DIR@/../FADBAD++
+FADBADPP_OPT = -I$(FADBADPP_DIR)
+FADBADPP_LIB = -L.
+
+# CoDiPack
+CODIPACK_DIR = @MFEM_DIR@/../CoDiPack
+CODIPACK_OPT = -I$(CODIPACK_DIR)
+CODIPACK_LIB = -L.
+
+# Eigen
+EIGEN_DIR = @MFEM_DIR@/../Eigen
+EIGEN_OPT = -I$(EIGEN_DIR)
+EIGEN_LIB = -L.
+
 # GSLIB library
 GSLIB_DIR = @MFEM_DIR@/../gslib/build
 GSLIB_OPT = -I$(GSLIB_DIR)/include
@@ -1,41 +0,0 @@
-                    Finite Element Discretization Library
-                                   __
-                       _ __ ___   / _|  ___  _ __ ___
-                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
-                      | | | | | ||  _||  __/| | | | | |
-                      |_| |_| |_||_|   \___||_| |_| |_|
-
-                               https://mfem.org
-
-
-This directory contains recommended git hooks, which are scripts that can be
-used to improve your development experience with MFEM:
-
-### The hooks
-
-* `pre-commit` is a hook that will be applied before each commit and run
-`astyle` on the code. This will ensure that your changes comply with the MFEM
-code styling guidelines.
-
-* `pre-push` is a hook that will be applied before each push to run a quick set
-of tests that verify that your files headers are in compliance, and that you did
-not add any large files to the repo.
-
-### Setup
-
-To setup the git hooks, run `make hooks`, which creates symlinks to the hooks in
-the `.git/hooks` directory. Individual hooks can be enabled by manually creating
-symlinks.
-
-(You may also copy the scripts directly and customize them further, but this way
-you may miss additional updates in the future.)
-
-### Failures
-
-The `branch-history` check can fail in some cases when the history is OK. For
-example, when a large number of files were modified for a legitimate reason, or
-when a picture was added for documentation.
-
-If that is the case, make sure the failure is indeed justified, and rerun the
-push command with the `--no-verify` option. This will skip the hooks, allowing
-you to push those changes.
@@ -1,4 +0,0 @@
-#!/bin/sh
-
-# Apply automated code formatting
-make -C $(git rev-parse --show-toplevel) style
@@ -1,107 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-option=${1:-""}
-
-if [[ "${option}" == "--help" ]]; then
-  echo "This script runs checks on the repository."
-  echo "It has 2 modes: with and without an option."
-  echo ""
-  echo "Options are used in GitHub Actions and can be:"
-  echo "  --copyright"
-  echo "  --license"
-  echo "  --release"
-  echo "  --style"
-  echo "  --history"
-  echo ""
-  echo "As a githook, the script is used without options."
-  echo "In that case, it will run all the checks except style."
-  echo ""
-  echo "Use --help to print this help message."
-fi
-
-cd $(git rev-parse --show-toplevel)
-
-# copyright check
-copyright=true
-if [[ "${option}" == "--copyright" || "${option}" == "" ]]; then
-  if git grep -l "^\(#\|//\).*\(\-2020\|\ 2010,\)" > matches.txt; then
-    echo "Please update the following files to Copyright (c) 2010-2021:"
-    cat matches.txt
-    copyright=false
-  fi
-fi
-
-# license check
-license=true
-if [[ "${option}" == "--license" || "${option}" == "" ]]; then
-  if git grep -li "^\(#\|//\).*GNU\ Lesser\ General\ Public\ License" > matches.txt; then
-    echo "Please update the following files to the BSD-3 license:"
-    cat matches.txt
-    license=false
-  fi
-fi
-
-# release check
-release=true
-if [[ "${option}" == "--release" || "${option}" == "" ]]; then
-  if git grep -l "^\(#\|//\).*LLNL\-CODE\-443211" > matches.txt
-  then
-    echo "Please update the following files to LLNL-CODE-806117:"
-    cat matches.txt
-    release=false
-  fi
-fi
-
-# wrap-up
-code=0
-if ! $copyright ; then
-  echo "copyright check failed, unroll log for details"
-  code=1
-fi
-if ! $license ; then
-  echo "license check failed, unroll log for details"
-  code=1
-fi
-if ! $release ; then
-  echo "release check failed, unroll log for details"
-  code=1
-fi
-
-# `code-style` is not just a check, it will actually reformat the code if
-# necessary. This means that if one pushes while the repo is in dirty state
-# (changes not staged), those changes may be mixed with format changes.
-# To activate this, you will need to hard-copy this hook script in the hook
-# directory and uncomment only then. (See README.md)
-#
-## style check
-#if [[ "${option}" == "--style" || "${option}" == "" ]]; then
-if [[ "${option}" == "--style" ]]; then
-  if which astyle && [[ "$(astyle --version)" == "Artistic Style Version 2.05.1" ]]; then
-    cd tests/scripts
-    if ! ./runtest code-style; then code=1; fi
-    cd -
-  else
-    echo "Warning: astyle not found or version is not 2.05.1"
-  fi
-fi
-
-# branch-history
-if [[ "${option}" == "--history" || "${option}" == "" ]]; then
-  git fetch origin master:master
-  cd tests/scripts
-  if ! ./runtest branch-history; then code=1; fi
-  cd -
-fi
-
-exit $code
@@ -39,7 +39,7 @@ set -- $$($(1) $(SHELL) -c "$(2)" 2>&1); while [ "$$#" -gt 3 ]; do shift; done
 endef
 define TIMECMD.NOTGNU
 set -- $$($(1) -l $(SHELL) -c "{ $(2); } > /dev/null 2>&1" 2>&1; echo $$?); \
-set -- "$$1"s "$$(($$7/1024))"kB "$${!#}"
+set -- "$$1"s "$$(($$7/1024))"kB "$${60}"
 endef
 define TIMECMD.BASH
 TIMEFORMAT=$$'%3Rs'; \
@@ -57,27 +57,22 @@ TIMECMD := $(word 1,$(TIMECMD))
 ifneq (,$(filter test%,$(MAKECMDGOALS)))
   MAKEFLAGS += -k
 endif
-# Test runs of the examples/miniapps with parameters - check exit code:
-# 0 means success, 255 means the test was skipped, anything else means error
+# Test runs of the examples/miniapps with parameters - check exit code
 mfem-test = \
   printf "   $(3) [$(2) $(1) ... ]: "; \
   $(call $(TIMEFUN),$(TIMECMD),$(2) ./$(1) $(if $(5),,-no-vis )$(4) \
     > $(1).stderr 2>&1); \
-   err="$$3"; \
-   if [ "$$3" = 0 ]; then $(PRINT_OK); \
-   else if [ "$$3" = 255 ]; then $(PRINT_SKIP); err=0; \
-   else $(PRINT_FAILED); cat $(1).stderr; fi; fi; \
-   rm -f $(1).stderr; exit $$err
+   if [ "$$3" = 0 ]; \
+   then $(PRINT_OK); else $(PRINT_FAILED); cat $(1).stderr; fi; \
+   rm -f $(1).stderr; exit $$3

 # Test runs of the examples/miniapps - check exit code and if a file exists
-# See mfem-test for the interpretation of the error code
 mfem-test-file = \
   printf "   $(3) [$(2) $(1) ... ]: "; \
   $(call $(TIMEFUN),$(TIMECMD),$(2) ./$(1) -no-vis > $(1).stderr 2>&1); \
   err="$$3"; \
-   if [ "$$3" = 0 ] && [ -e $(4) ]; then $(PRINT_OK); \
-   else if [ "$$3" = 255 ] && [ -e $(4) ]; then $(PRINT_SKIP); err=0; \
-   else $(PRINT_FAILED); cat $(1).stderr; err=64; fi; fi; \
+   if [ "$$3" = 0 ] && [ -e $(4) ]; \
+   then $(PRINT_OK); else $(PRINT_FAILED); cat $(1).stderr; err=64; fi; \
   rm -f $(1).stderr; exit $$err

 .PHONY: test test-par-YES test-par-NO test-ser test-par test-clean test-print
@@ -85,26 +80,6 @@ mfem-test-file = \
 # What sets of tests to run in serial and parallel
 test-par-YES: $(PAR_$(MFEM_TESTS):=-test-par) $(SEQ_$(MFEM_TESTS):=-test-seq)
 test-par-NO:  $(SEQ_$(MFEM_TESTS):=-test-seq)
-ifeq ($(MFEM_USE_CUDA),YES)
-.PHONY: test-par-YES-cuda test-par-NO-cuda test-ser-cuda test-par-cuda test-cuda
-test-par-YES: test-par-YES-cuda
-test-par-NO:  test-par-NO-cuda
-test-par-YES-cuda: test-par-cuda test-ser-cuda
-test-par-NO-cuda:  test-ser-cuda
-test-ser-cuda: $(SEQ_DEVICE_$(MFEM_TESTS):=-test-seq-cuda)
-test-par-cuda: $(PAR_DEVICE_$(MFEM_TESTS):=-test-par-cuda)
-test-cuda: test-par-$(MFEM_USE_MPI)-cuda clean-exec
-endif
-ifeq ($(MFEM_USE_HIP),YES)
-.PHONY: test-par-YES-hip test-par-NO-hip test-ser-hip test-par-hip test-hip
-test-par-YES: test-par-YES-hip
-test-par-NO:  test-par-NO-hip
-test-par-YES-hip: test-par-hip test-ser-hip
-test-par-NO-hip:  test-ser-hip
-test-ser-hip: $(SEQ_DEVICE_$(MFEM_TESTS):=-test-seq-hip)
-test-par-hip: $(PAR_DEVICE_$(MFEM_TESTS):=-test-par-hip)
-test-hip: test-par-$(MFEM_USE_MPI)-hip clean-exec
-endif
 test-ser:     test-par-NO
 test-par:     test-par-YES
 test:         all test-par-$(MFEM_USE_MPI) clean-exec
@@ -1,9 +0,0 @@
-MFEM INLINE mesh v1.0
-
-type = pyramid
-nx = 4
-ny = 4
-nz = 4
-sx = 1.0
-sy = 1.0
-sz = 1.0
@@ -1,43 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-# PYRAMID     = 7
-#
-
-dimension
-3
-
-elements
-2
-1 7 4 3 2 1 0
-1 7 1 2 3 4 5
-
-boundary
-8
-1 2 0 2 1
-2 2 0 3 2
-3 2 0 4 3
-4 2 0 1 4
-5 2 1 2 5
-6 2 2 3 5
-7 2 3 4 5
-8 2 4 1 5
-
-vertices
-6
-3
-0 0 -1
-1 0 0
-0 1 0
-1 0 0
-0 -1 0
-0 0 1
@@ -1,41 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-3
-
-elements
-1
-1 5 0 1 2 3 4 5 6 7
-
-boundary
-6
-1 3 3 2 1 0
-2 3 0 1 5 4
-3 3 1 2 6 5
-4 3 2 3 7 6
-5 3 3 0 4 7
-6 3 4 5 6 7
-
-vertices
-8
-3
-0 0 0
-1 0 0
-1 1 0
-0 1 0
-0 0 1
-1 0 1
-1 1 1
-0 1 1
@@ -1,38 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-3
-
-elements
-1
-1 6 0 1 2 3 4 5
-
-boundary
-5
-1 2 0 2 1
-2 2 3 4 5
-3 3 0 1 4 3
-4 3 1 2 5 4
-5 3 2 0 3 5
-
-vertices
-6
-3
-0 0 0
-1 0 0
-0 1 0
-0 0 1
-1 0 1
-0 1 1
@@ -1,38 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-# PYRAMID     = 7
-#
-
-dimension
-3
-
-elements
-1
-1 7 0 1 2 3 4
-
-boundary
-5
-1 3 3 2 1 0
-2 2 0 1 4
-3 2 1 2 4
-4 2 2 3 4
-5 2 3 0 4
-
-vertices
-5
-3
-0 0 0
-1 0 0
-1 1 0
-0 1 0
-0 0 1
@@ -1,31 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-1
-
-elements
-1
-1 1 0 1
-
-boundary
-2
-1 0 0
-2 0 1
-
-vertices
-2
-1
-0
-1
@@ -1,35 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-2
-
-elements
-1
-1 3 0 1 2 3
-
-boundary
-4
-1 1 0 1
-2 1 1 2
-3 1 2 3
-4 1 3 0
-
-vertices
-4
-2
-0 0
-1 0
-1 1
-0 1
@@ -1,35 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-3
-
-elements
-1
-1 4 0 1 2 3
-
-boundary
-4
-1 2 1 2 3
-2 2 0 3 2
-3 2 0 1 3
-4 2 0 2 1
-
-vertices
-4
-3
-0 0 0
-1 0 0
-0 1 0
-0 0 1
@@ -1,33 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-2
-
-elements
-1
-1 2 0 1 2
-
-boundary
-3
-1 1 0 1
-2 1 1 2
-3 1 2 0
-
-vertices
-3
-2
-0 0
-1 0
-0 1
@@ -1,246 +0,0 @@
-FMS: 100
-DataCollection/Name: star
-DataCollection/NumberOfFieldDescriptors: 1
-DataCollection/FieldDescriptors/0/Name: CoordsDescriptor
-DataCollection/FieldDescriptors/0/ComponentName: volume
-DataCollection/FieldDescriptors/0/Type: 0
-DataCollection/FieldDescriptors/0/FixedOrder/Size: 3
-DataCollection/FieldDescriptors/0/FixedOrder/Type: FMS_UINT64
-DataCollection/FieldDescriptors/0/FixedOrder/Values: [0, 1, 3]
-DataCollection/FieldDescriptors/0/NumDofs: 211
-DataCollection/NumberOfFields: 1
-DataCollection/Fields/0/Name: Coords
-DataCollection/Fields/0/LayoutType: 0
-DataCollection/Fields/0/NumberOfVectorComponents: 2
-DataCollection/Fields/0/FieldDescriptorName: CoordsDescriptor
-DataCollection/Fields/0/Data/Size: 422
-DataCollection/Fields/0/Data/Type: FMS_DOUBLE
-DataCollection/Fields/0/Data/Values: [-0.016886, 1.000000, 0.309017, 
-1.309020, -0.809017, -0.500000, 
-0.809017, -1.618030, 0.309017, 
-0.500000, 1.309020, 0.519420, 
-1.154510, 0.809019, 0.147680, 
-0.095492, -0.654508, -0.415586, 
-1.213520, -1.213520, -0.392210, 
-0.654508, -0.095492, 0.139949, 
-0.809019, 1.154510, 0.660184, 
-0.264063, -0.800064, -0.231060, 
-0.663691, 0.183114, 0.317639, 
-0.543082, 0.598483, 0.345112, 
-0.478298, 0.027703, 0.095229, 
-0.012368, -0.092534, -0.334412, 
-0.313767, -0.140526, -0.293881, 
-0.534056, -0.660290, -0.537646, 
-0.655590, -0.121396, -0.274504, 
-0.346497, -0.296570, 0.004737, 
-0.098835, 0.069287, 0.082675, 
-0.318799, 0.467183, 0.564505, 
-0.595190, 0.846237, 0.671735, 
-1.051500, 1.103010, 0.964008, 
-0.821603, 1.257520, 1.206010, 
-1.142350, 0.975686, 0.781273, 
-0.717257, 0.475684, 0.642352, 
-0.268930, 0.211049, 0.174181, 
-0.039345, -0.147746, -0.177481, 
-0.365164, -0.230328, -0.551503, 
-0.603005, -0.497587, -0.389864, 
-0.757514, -0.706011, -0.675487, 
-0.528946, -0.943851, -1.078690, 
-1.087600, -0.955467, -1.483190, 
-1.348360, -1.483190, -1.348360, 
-1.085930, -0.938010, -0.943851, 
-1.078690, -0.681476, -0.540944, 
-0.757514, -0.706011, -0.540614, 
-0.367058, -0.551503, -0.603005, 
-0.365164, -0.230328, -0.138552, 
-0.206896, 0.174181, 0.039345, 
-0.268468, 0.222269, 0.475684, 
-0.642352, 0.759791, 0.719381, 
-1.142350, 0.975686, 1.257520, 
-1.206010, 0.972837, 0.836119, 
-1.051500, 1.103010, 0.214572, 
-0.407449, 0.288323, 0.449827, 
-0.086700, -0.027358, -0.200560, 
-0.166595, -0.271802, -0.418426, 
-0.426131, -0.551441, -0.096117, 
-0.206969, -0.027946, -0.184969, 
-0.211136, 0.260131, 0.407172, 
-0.430781, 0.718277, 0.885068, 
-0.753103, 0.957692, 0.866273, 
-1.024530, 0.934099, 1.093820, 
-0.348422, 0.524463, 0.404903, 
-0.587376, 0.054525, 0.146431, 
-0.078026, -0.007795, -0.329488, 
-0.302967, -0.488115, -0.439332, 
-0.498515, -0.453527, -0.633059, 
-0.570251, -0.655787, -0.791132, 
-0.802013, -0.956872, -1.094720, 
-1.208560, -1.207720, -1.339910, 
-0.693371, -0.795404, -0.803594, 
-0.945068, -0.464668, -0.631721, 
-0.458968, -0.547876, -0.332878, 
-0.485696, -0.286385, -0.408481, 
-0.053021, -0.058200, 0.110846, 
-0.002086, 0.381384, 0.416784, 
-0.551133, 0.613261, 0.872474, 
-0.901208, 1.038300, 1.084660, 
-0.737459, 0.751250, 0.890002, 
-0.915210, 0.010915, 0.000000, 
-0.951057, 0.951057, 0.587785, 
-1.538840, -0.587785, 0.000000, 
-0.951057, -1.538840, -0.951057, 
-0.015847, 0.475529, 0.951057, 
-0.492248, 1.244950, 1.063310, 
-0.274399, 0.293893, -0.293892, 
-0.296404, -1.063310, -1.244950, 
-0.453865, -0.951057, -0.475529, 
-0.466620, 0.792932, -0.013913, 
-0.748783, -0.497528, 0.021382, 
-0.017158, 0.172591, 0.330125, 
-0.458568, 0.457971, 0.137740, 
-0.299049, 0.588394, 0.667324, 
-0.432341, 0.634346, 0.117322, 
-0.193603, 0.211702, 0.098278, 
-0.199438, -0.077304, -0.082243, 
-0.216296, -0.458634, -0.592374, 
-0.563926, -0.680404, -0.135751, 
-0.302942, -0.469005, -0.453640, 
-0.182727, -0.314240, 0.024270, 
-0.021546, 0.158510, 0.317019, 
-0.485799, 0.492951, 0.792548, 
-0.634038, 0.951057, 0.951057, 
-0.777915, 0.613430, 0.951057, 
-0.951057, 0.793994, 0.635800, 
-1.049020, 1.146990, 1.084480, 
-0.924310, 1.440880, 1.342910, 
-1.380330, 1.221820, 0.948209, 
-0.856297, 0.746293, 0.904802, 
-0.476242, 0.393234, 0.489821, 
-0.391857, 0.194471, 0.075751, 
-0.097964, 0.195929, -0.097964, 
-0.195928, -0.173234, -0.078922, 
-0.489821, -0.391856, -0.467007, 
-0.397859, -0.746293, -0.904802, 
-0.945206, -0.849559, -1.380330, 
-1.221820, -1.440880, -1.342910, 
-1.100830, -0.923191, -1.049020, 
-1.146990, -0.774515, -0.621542, 
-0.951057, -0.951057, -0.803055, 
-0.635255, -0.951057, -0.951057, 
-0.792548, -0.634038, -0.454301, 
-0.479369, -0.158510, -0.317019, 
-0.149331, 0.178643, 0.295860, 
-0.306275, 0.246225, 0.406610, 
-0.367954, 0.498458, -0.014929, 
-0.084917, -0.094272, -0.000726, 
-0.245374, -0.340755, -0.435351, 
-0.490564, -0.176355, -0.294974, 
-0.148010, -0.328246, 0.136248, 
-0.182883, 0.328957, 0.309903, 
-0.646446, 0.622546, 0.800859, 
-0.801803, 0.643087, 0.654473, 
-0.796963, 0.816799, 0.748189, 
-0.898148, 0.819092, 0.970033, 
-1.049540, 1.162660, 1.115310, 
-1.266700, 0.539960, 0.728396, 
-0.656318, 0.783152, 0.302716, 
-0.403858, 0.219006, 0.309252, 
-0.009667, 0.107161, -0.118211, 
-0.011786, -0.284704, -0.179858, 
-0.411567, -0.296676, -0.570168, 
-0.635348, -0.726784, -0.793211, 
-1.016860, -1.112970, -1.170910, 
-1.281530, -0.737860, -0.851723, 
-0.883732, -0.995070, -0.653153, 
-0.769939, -0.631918, -0.797688, 
-0.616289, -0.806819, -0.638485, 
-0.790356, -0.136399, -0.322769, 
-0.165339, -0.309622]
-DataCollection/Mesh/PartitionInfo/Size: 2
-DataCollection/Mesh/PartitionInfo/Type: FMS_UINT64
-DataCollection/Mesh/PartitionInfo/Values: [0, 1]
-DataCollection/Mesh/NumDomainNames: 1
-DataCollection/Mesh/NumComponents: 1
-DataCollection/Mesh/NumTags: 0
-DataCollection/Mesh/DomainNames/0/Name: Domain
-DataCollection/Mesh/DomainNames/0/NumDomains: 1
-DataCollection/Mesh/DomainNames/0/Domains/0/Dimension: 2
-DataCollection/Mesh/DomainNames/0/Domains/0/NumVertices: 31
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/EntityType: FMS_EDGE
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/NumEntities: 50
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Size: 100
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Type: FMS_INT32
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Values: [11, 0, 26, 
-11, 26, 14, 
-14, 0, 27, 
-14, 27, 17, 
-17, 0, 28, 
-17, 28, 20, 
-20, 0, 29, 
-20, 29, 23, 
-23, 0, 30, 
-23, 30, 11, 
-11, 1, 12, 
-1, 26, 12, 
-12, 3, 13, 
-3, 26, 13, 
-13, 2, 14, 
-2, 15, 2, 
-27, 15, 15, 
-5, 16, 5, 
-27, 16, 16, 
-4, 17, 4, 
-18, 4, 28, 
-18, 18, 7, 
-19, 7, 28, 
-19, 19, 6, 
-20, 6, 21, 
-6, 29, 21, 
-21, 9, 22, 
-9, 29, 22, 
-22, 8, 23, 
-8, 24, 8, 
-30, 24, 24, 
-10, 25, 10, 
-30, 25, 25, 1]
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/EntityType: FMS_QUADRILATERAL
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/NumEntities: 20
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Size: 80
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Type: FMS_INT32
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Values: [0, 1, 2, 
-3, 3, 4, 
-5, 6, 6, 
-7, 8, 9, 
-9, 10, 11, 
-12, 12, 13, 
-14, 0, 15, 
-16, 17, 1, 
-17, 18, 19, 
-20, 2, 20, 
-21, 22, 22, 
-23, 24, 4, 
-24, 25, 26, 
-27, 5, 27, 
-28, 29, 29, 
-30, 31, 7, 
-31, 32, 33, 
-34, 8, 34, 
-35, 36, 36, 
-37, 38, 10, 
-38, 39, 40, 
-41, 11, 41, 
-42, 43, 43, 
-44, 45, 13, 
-45, 46, 47, 
-48, 14, 48, 
-49, 15]
-DataCollection/Mesh/Components/0/Name: volume
-DataCollection/Mesh/Components/0/Dimension: 2
-DataCollection/Mesh/Components/0/NumEntities: 20
-DataCollection/Mesh/Components/0/Coordinates: Coords
-DataCollection/Mesh/Components/0/NumParts: 1
-DataCollection/Mesh/Components/0/Parts/0/DomainName: Domain
-DataCollection/Mesh/Components/0/Parts/0/DomainID: 0
-DataCollection/Mesh/Components/0/Parts/0/FullDomain: Yes
-DataCollection/Mesh/Components/0/Relations/Size: 0
-DataCollection/Mesh/Components/0/Relations/Type: FMS_UINT64
@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.

-PROJECT_NUMBER         = v4.3.1
+PROJECT_NUMBER         = v4.2.1

 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -189,7 +189,6 @@ namespace mfem {
 * - <a class="el" href="field-diff_8cpp_source.html">Field Diff</a>: compare grid functions on different meshes
 * - <a class="el" href="field-interp_8cpp_source.html">Field Interp</a>: transfer a grid functions between meshes
 * - <a class="el" href="distance_8cpp_source.html">Distance</a>: finite element distance function solver
- * - <a class="el" href="diffusion_8cpp_source.html">Shifted Diffusion</a>: shifted boundary diffusion solver
 * - <a class="el" href="distance_8cpp_source.html">Block Solvers</a>: comparison of saddle point system solvers
 * - <a class="el" href="parheat_8cpp_source.html">Optimization gradients</a>: Gradients of PDE-constrained function
 * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
@@ -84,9 +84,8 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})

-  set(THIS_TEST_OPTIONS "-no-vis")
-  if (${TEST_NAME} MATCHES "ex0p?")
-    set(THIS_TEST_OPTIONS)
+  if (NOT (${TEST_NAME} MATCHES "ex0p?"))
+    set(THIS_TEST_OPTIONS "-no-vis")
  endif()
  if (${TEST_NAME} MATCHES "ex10p*")
    list(APPEND THIS_TEST_OPTIONS "-tf" "5")
@@ -108,34 +107,6 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
  endif()
 endforeach()

-# Add CUDA/HIP tests.
-set(DEVICE_EXAMPLES
-    # serial examples with device support:
-    ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
-    # parallel examples with device support:
-    ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p ex24p ex25p ex26p)
-set(MFEM_TEST_DEVICE)
-if (MFEM_USE_CUDA)
-  set(MFEM_TEST_DEVICE "cuda")
-elseif (MFEM_USE_HIP)
-  set(MFEM_TEST_DEVICE "hip")
-endif()
-if (MFEM_TEST_DEVICE)
-  foreach(TEST_NAME ${DEVICE_EXAMPLES})
-    set(THIS_TEST_OPTIONS "-no-vis" "-d" "${MFEM_TEST_DEVICE}")
-    if (NOT (${TEST_NAME} MATCHES ".*p$"))
-      add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_ser
-        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-    elseif (MFEM_USE_MPI)
-      add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_np=${MFEM_MPI_NP}
-        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-        ${MPIEXEC_PREFLAGS}
-        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-        ${MPIEXEC_POSTFLAGS})
-    endif()
-  endforeach()
-endif()
-
 # If STRUMPACK is enabled, add a test run that uses it.
 if (MFEM_USE_STRUMPACK)
  add_test(NAME ex11p_strumpack_np=${MFEM_MPI_NP}
@@ -1,18 +0,0 @@
-                    Finite Element Discretization Library
-                                   __
-                       _ __ ___   / _|  ___  _ __ ___
-                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
-                      | | | | | ||  _||  __/| | | | | |
-                      |_| |_| |_||_|   \___||_| |_| |_|
-
-                               https://mfem.org
-
-This directory contains modifications of the example codes that illustrate the
-use of MFEM features based on the Caliper performance profiling library.
-
-To build these examples, make sure that MFEM is configured with the option
-"MFEM_USE_CALIPER = YES", see the top-level INSTALL file for details (version
-2.5.0 of Caliper is recommended, though older versions may work too).
-
-We recommend comparing the original example codes with the corresponding files
-in the current directory.
@@ -206,9 +206,9 @@ int main(int argc, char *argv[])
   cout << "Size of linear system: " << A->Height() << endl;

   // 11. Solve the linear system A X = B.
+   MFEM_PERF_BEGIN("Solve A X=B");
   if (!pa)
   {
-      MFEM_PERF_SCOPE("Solve A X=B (FA)");
 #ifndef MFEM_USE_SUITESPARSE
      // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
      GSSmoother M((SparseMatrix&)(*A));
@@ -223,7 +223,6 @@ int main(int argc, char *argv[])
   }
   else // Jacobi preconditioning in partial assembly mode
   {
-      MFEM_PERF_SCOPE("Solve A X=B (PA)");
      if (UsesTensorBasis(fespace))
      {
         OperatorJacobiSmoother M(a, ess_tdof_list);
@@ -234,6 +233,7 @@ int main(int argc, char *argv[])
         CG(*A, B, X, 1, 400, 1e-12, 0.0);
      }
   }
+   MFEM_PERF_END("Solve A X=B");
   // 12. Recover the solution as a finite element grid function.
   a.RecoverFEMSolution(X, b, x);

@@ -231,29 +231,28 @@ int main(int argc, char *argv[])
   // 13. Solve the linear system A X = B.
   //     * With full assembly, use the BoomerAMG preconditioner from hypre.
   //     * With partial assembly, use Jacobi smoothing, for now.
+   MFEM_PERF_BEGIN("Solve A X = B");
+   Solver *prec = NULL;
+   if (pa)
   {
-      MFEM_PERF_SCOPE("Solve A X=B");
-      Solver *prec = NULL;
-      if (pa)
+      if (UsesTensorBasis(fespace))
      {
-         if (UsesTensorBasis(fespace))
-         {
-            prec = new OperatorJacobiSmoother(a, ess_tdof_list);
-         }
+         prec = new OperatorJacobiSmoother(a, ess_tdof_list);
      }
-      else
-      {
-         prec = new HypreBoomerAMG;
-      }
-      CGSolver cg(MPI_COMM_WORLD);
-      cg.SetRelTol(1e-12);
-      cg.SetMaxIter(2000);
-      cg.SetPrintLevel(1);
-      if (prec) { cg.SetPreconditioner(*prec); }
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete prec;
   }
+   else
+   {
+      prec = new HypreBoomerAMG;
+   }
+   CGSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(1);
+   if (prec) { cg.SetPreconditioner(*prec); }
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+   delete prec;
+   MFEM_PERF_END("Solve A X = B");
   // 14. Recover the parallel grid function corresponding to X. This is the
   //     local finite element solution on each processor.
   a.RecoverFEMSolution(X, b, x);
@@ -9,7 +9,6 @@
 //               ex1 -m ../data/fichera.mesh
 //               ex1 -m ../data/fichera-mixed.mesh
 //               ex1 -m ../data/toroid-wedge.mesh
-//               ex1 -m ../data/octahedron.mesh -o 1
 //               ex1 -m ../data/periodic-annulus-sector.msh
 //               ex1 -m ../data/periodic-torus-sector.msh
 //               ex1 -m ../data/square-disc-p2.vtk -o 2
@@ -55,7 +55,6 @@ int main(int argc, char *argv[])
   int order = 1;
   int nev = 5;
   bool visualization = 1;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -72,8 +71,6 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -89,18 +86,13 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors. We
+   // 3. Read the (serial) mesh from the given mesh file on all processors. We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 5. Refine the serial mesh on all processors to increase the resolution. In
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ref_levels' of uniform refinement (2 by default, or
   //    specified on the command line with -rs).
   for (int lev = 0; lev < ser_ref_levels; lev++)
@@ -108,7 +100,7 @@ int main(int argc, char *argv[])
      mesh->UniformRefinement();
   }

-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution (1 time by
   //    default, or specified on the command line with -rp). Once the parallel
   //    mesh is defined, the serial mesh can be deleted.
@@ -118,8 +110,9 @@ int main(int argc, char *argv[])
   {
      pmesh->UniformRefinement();
   }
+   pmesh->ReorientTetMesh();

-   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Nedelec finite elements of the specified order.
   FiniteElementCollection *fec = new ND_FECollection(order, dim);
   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
@@ -129,7 +122,7 @@ int main(int argc, char *argv[])
      cout << "Number of unknowns: " << size << endl;
   }

-   // 8. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
+   // 7. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
   //    element space. The first corresponds to the curl curl, while the second
   //    is a simple mass matrix needed on the right hand side of the
   //    generalized eigenvalue problem below. The boundary conditions are
@@ -171,7 +164,7 @@ int main(int argc, char *argv[])
   delete a;
   delete m;

-   // 9. Define and configure the AME eigensolver and the AMS preconditioner for
+   // 8. Define and configure the AME eigensolver and the AMS preconditioner for
   //    A to be used within the solver. Set the matrices which define the
   //    generalized eigenproblem A x = lambda M x.
   HypreAMS *ams = new HypreAMS(*A,fespace);
@@ -187,15 +180,15 @@ int main(int argc, char *argv[])
   ame->SetMassMatrix(*M);
   ame->SetOperator(*A);

-   // 10. Compute the eigenmodes and extract the array of eigenvalues. Define a
-   //     parallel grid function to represent each of the eigenmodes returned by
-   //     the solver.
+   // 9. Compute the eigenmodes and extract the array of eigenvalues. Define a
+   //    parallel grid function to represent each of the eigenmodes returned by
+   //    the solver.
   Array<double> eigenvalues;
   ame->Solve();
   ame->GetEigenvalues(eigenvalues);
   ParGridFunction x(fespace);

-   // 11. Save the refined mesh and the modes in parallel. This output can be
+   // 10. Save the refined mesh and the modes in parallel. This output can be
   //     viewed later using GLVis: "glvis -np <np> -m mesh -g mode".
   {
      ostringstream mesh_name, mode_name;
@@ -220,7 +213,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 12. Send the solution by socket to a GLVis server.
+   // 11. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -260,7 +253,7 @@ int main(int argc, char *argv[])
      mode_sock.close();
   }

-   // 13. Free the used memory.
+   // 12. Free the used memory.
   delete ame;
   delete ams;
   delete M;
@@ -19,14 +19,6 @@
 //               ex15 -m ../data/square-disc.mesh
 //               ex15 -m ../data/escher.mesh -r 2 -tf 0.3
 //
-//               Kelly estimator:
-//
-//               ex15 -est 1 -e 0.0001
-//               ex15 -est 1 -o 1 -y 0.4
-//               ex15 -est 1 -o 4 -y 0.1
-//               ex15 -est 1 -n 5
-//               ex15 -est 1 -p 1 -n 3
-//
 // Description:  Building on Example 6, this example demonstrates dynamic AMR.
 //               The mesh is adapted to a time-dependent solution by refinement
 //               as well as by derefinement. For simplicity, the solution is
@@ -36,10 +28,10 @@
 //               At each outer iteration the right hand side function is changed
 //               to mimic a time dependent problem.  Within each inner iteration
 //               the problem is solved on a sequence of meshes which are locally
-//               refined according to a simple ZZ or Kelly error estimator.  At
-//               the end of the inner iteration the error estimates are also
-//               used to identify any elements which may be over-refined and a
-//               single derefinement step is performed.
+//               refined according to a simple ZZ error estimator.  At the end
+//               of the inner iteration the error estimates are also used to
+//               identify any elements which may be over-refined and a single
+//               derefinement step is performed.
 //
 //               The example demonstrates MFEM's capability to refine and
 //               derefine nonconforming meshes, in 2D and 3D, and on linear,
@@ -86,7 +78,6 @@ int main(int argc, char *argv[])
   int nc_limit = 3;         // maximum level of hanging nodes
   bool visualization = true;
   bool visit = false;
-   int which_estimator = 0;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -107,9 +98,6 @@ int main(int argc, char *argv[])
                  "Maximum level of hanging nodes.");
   args.AddOption(&t_final, "-tf", "--t-final",
                  "Final time; start time is 0.");
-   args.AddOption(&which_estimator, "-est", "--estimator",
-                  "Which estimator to use: "
-                  "0 = ZZ, 1 = Kelly. Defaults to ZZ.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -198,38 +186,19 @@ int main(int argc, char *argv[])
   visit_dc.RegisterField("solution", &x);
   int vis_cycle = 0;

-   // 9. As in Example 6, we set up an estimator that will be used to obtain
-   //    element error indicators. The integrator needs to provide the method
-   //    ComputeElementFlux. The smoothed flux space is a vector valued H1 (ZZ)
-   //    or L2 (Kelly) space here.
-   L2_FECollection flux_fec(order, dim);
-   ErrorEstimator* estimator{nullptr};
-
-   switch (which_estimator)
-   {
-      case 1:
-      {
-         auto flux_fes = new FiniteElementSpace(&mesh, &flux_fec, sdim);
-         estimator = new KellyErrorEstimator(*integ, x, flux_fes);
-         break;
-      }
-
-      default:
-         std::cout << "Unknown estimator. Falling back to ZZ." << std::endl;
-      case 0:
-      {
-         auto flux_fes = new FiniteElementSpace(&mesh, &fec, sdim);
-         estimator = new ZienkiewiczZhuEstimator(*integ, x, flux_fes);
-         break;
-      }
-   }
+   // 9. As in Example 6, we set up a Zienkiewicz-Zhu estimator that will be
+   //    used to obtain element error indicators. The integrator needs to
+   //    provide the method ComputeElementFlux. The smoothed flux space is a
+   //    vector valued H1 space here.
+   FiniteElementSpace flux_fespace(&mesh, &fec, sdim);
+   ZienkiewiczZhuEstimator estimator(*integ, x, flux_fespace);

   // 10. As in Example 6, we also need a refiner. This time the refinement
   //     strategy is based on a fixed threshold that is applied locally to each
   //     element. The global threshold is turned off by setting the total error
   //     fraction to zero. We also enforce a maximum refinement ratio between
   //     adjacent elements.
-   ThresholdRefiner refiner(*estimator);
+   ThresholdRefiner refiner(estimator);
   refiner.SetTotalErrorFraction(0.0); // use purely local threshold
   refiner.SetLocalErrorGoal(max_elem_error);
   refiner.PreferConformingRefinement();
@@ -238,7 +207,7 @@ int main(int argc, char *argv[])
   // 11. A derefiner selects groups of elements that can be coarsened to form
   //     a larger element. A conservative enough threshold needs to be set to
   //     prevent derefining elements that would immediately be refined again.
-   ThresholdDerefiner derefiner(*estimator);
+   ThresholdDerefiner derefiner(estimator);
   derefiner.SetThreshold(hysteresis * max_elem_error);
   derefiner.SetNCLimit(nc_limit);

@@ -339,8 +308,6 @@ int main(int argc, char *argv[])
      b.Update();
   }

-   delete estimator;
-
   return 0;
 }

@@ -223,14 +223,13 @@ int main(int argc, char *argv[])
   visit_dc.RegisterField("solution", &x);
   int vis_cycle = 0;

-   // 10. As in Example 6p, we set up an estimator that will be used to obtain
-   //     element error indicators. The integrator needs to provide the method
-   //     ComputeElementFlux. We supply an L2 space for the discontinuous flux
-   //     and an H(div) space for the smoothed flux.
+   // 10. As in Example 6p, we set up a Zienkiewicz-Zhu estimator that will be
+   //     used to obtain element error indicators. The integrator needs to
+   //     provide the method ComputeElementFlux. We supply an L2 space for the
+   //     discontinuous flux and an H(div) space for the smoothed flux.
   L2_FECollection flux_fec(order, dim);
   RT_FECollection smooth_flux_fec(order-1, dim);
-   ErrorEstimator* estimator{nullptr};
-
+   ErrorEstimator* estimator;
   switch (which_estimator)
   {
      case 1:
@@ -249,7 +248,7 @@ int main(int argc, char *argv[])
      default:
         if (myid == 0)
         {
-            std::cout << "Unknown estimator. Falling back to L2ZZ." << std::endl;
+            std::cout << "Unkown estimator. Falling back to L2ZZ." << std::endl;
         }
      case 0:
      {
@@ -24,10 +24,7 @@
 //               class ConductionOperator defining C(u)), as well as their
 //               implicit time integration. Note that implementing the method
 //               ConductionOperator::ImplicitSolve is the only requirement for
-//               high-order implicit (SDIRK) time integration. In this example,
-//               the diffusion operator is linearized by evaluating with the
-//               lagged solution from the previous timestep, so there is only
-//               a linear solve.
+//               high-order implicit (SDIRK) time integration.
 //
 //               We recommend viewing examples 2, 9 and 10 before viewing this
 //               example.
@@ -329,8 +326,8 @@ ConductionOperator::ConductionOperator(FiniteElementSpace &f, double al,
 void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
 {
   // Compute:
-   //    du_dt = M^{-1}*-Ku
-   // for du_dt, where K is linearized by using u from the previous timestep
+   //    du_dt = M^{-1}*-K(u)
+   // for du_dt
   Kmat.Mult(u, z);
   z.Neg(); // z = -z
   M_solver.Mult(z, du_dt);
@@ -341,7 +338,7 @@ void ConductionOperator::ImplicitSolve(const double dt,
 {
   // Solve the equation:
   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
-   // for du_dt, where K is linearized by using u from the previous timestep
+   // for du_dt
   if (!T)
   {
      T = Add(1.0, Mmat, dt, Kmat);
@@ -24,11 +24,8 @@
 //               class ConductionOperator defining C(u)), as well as their
 //               implicit time integration. Note that implementing the method
 //               ConductionOperator::ImplicitSolve is the only requirement for
-//               high-order implicit (SDIRK) time integration. In this example,
-//               the diffusion operator is linearized by evaluating with the
-//               lagged solution from the previous timestep, so there is only
-//               a linear solve. Optional saving with ADIOS2
-//               (adios2.readthedocs.io) is also illustrated.
+//               high-order implicit (SDIRK) time integration. Optional saving
+//               with ADIOS2 (adios2.readthedocs.io) is also illustrated.
 //
 //               We recommend viewing examples 2, 9 and 10 before viewing this
 //               example.
@@ -423,8 +420,8 @@ ConductionOperator::ConductionOperator(ParFiniteElementSpace &f, double al,
 void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
 {
   // Compute:
-   //    du_dt = M^{-1}*-Ku
-   // for du_dt, where K is linearized by using u from the previous timestep
+   //    du_dt = M^{-1}*-K(u)
+   // for du_dt
   Kmat.Mult(u, z);
   z.Neg(); // z = -z
   M_solver.Mult(z, du_dt);
@@ -435,7 +432,7 @@ void ConductionOperator::ImplicitSolve(const double dt,
 {
   // Solve the equation:
   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
-   // for du_dt, where K is linearized by using u from the previous timestep
+   // for du_dt
   if (!T)
   {
      T = Add(1.0, Mmat, dt, Kmat);
@@ -196,12 +196,6 @@ void InitialDeformation(const Vector &x, Vector &y);

 int main(int argc, char *argv[])
 {
-#ifdef HYPRE_USING_CUDA
-   cout << "\nAs of mfem-4.3 and hypre-2.22.0 (July 2021) this example\n"
-        << "is NOT supported with the CUDA version of hypre.\n\n";
-   return 255;
-#endif
-
   // 1. Initialize MPI
   MPI_Session mpi;
   const int myid = mpi.WorldRank();
@@ -444,19 +438,15 @@ JacobianPreconditioner::JacobianPreconditioner(Array<ParFiniteElementSpace *>
 void JacobianPreconditioner::Mult(const Vector &k, Vector &y) const
 {
   // Extract the blocks from the input and output vectors
-   Vector disp_in;
-   disp_in.MakeRef(const_cast<Vector&>(k), block_trueOffsets[0],
-                   block_trueOffsets[1]-block_trueOffsets[0]);
-   Vector pres_in;
-   pres_in.MakeRef(const_cast<Vector&>(k), block_trueOffsets[1],
-                   block_trueOffsets[2]-block_trueOffsets[1]);
+   Vector disp_in(k.GetData() + block_trueOffsets[0],
+                  block_trueOffsets[1]-block_trueOffsets[0]);
+   Vector pres_in(k.GetData() + block_trueOffsets[1],
+                  block_trueOffsets[2]-block_trueOffsets[1]);

-   Vector disp_out;
-   disp_out.MakeRef(y, block_trueOffsets[0],
-                    block_trueOffsets[1]-block_trueOffsets[0]);
-   Vector pres_out;
-   pres_out.MakeRef(y, block_trueOffsets[1],
-                    block_trueOffsets[2]-block_trueOffsets[1]);
+   Vector disp_out(y.GetData() + block_trueOffsets[0],
+                   block_trueOffsets[1]-block_trueOffsets[0]);
+   Vector pres_out(y.GetData() + block_trueOffsets[1],
+                   block_trueOffsets[2]-block_trueOffsets[1]);

   Vector temp(block_trueOffsets[1]-block_trueOffsets[0]);
   Vector temp2(block_trueOffsets[1]-block_trueOffsets[0]);
@@ -469,9 +459,6 @@ void JacobianPreconditioner::Mult(const Vector &k, Vector &y) const
   subtract(disp_in, temp, temp2);

   stiff_pcg->Mult(temp2, disp_out);
-
-   disp_out.SyncAliasMemory(y);
-   pres_out.SyncAliasMemory(y);
 }

 void JacobianPreconditioner::SetOperator(const Operator &op)
@@ -486,10 +473,7 @@ void JacobianPreconditioner::SetOperator(const Operator &op)

      if (!spaces[0]->GetParMesh()->Nonconforming())
      {
-#ifndef HYPRE_USING_CUDA
-         // Not available yet when hypre is built with CUDA
         stiff_prec_amg->SetElasticityOptions(spaces[0]);
-#endif
      }

      stiff_prec = stiff_prec_amg;
@@ -9,7 +9,6 @@
 //               mpirun -np 4 ex1p -m ../data/fichera.mesh
 //               mpirun -np 4 ex1p -m ../data/fichera-mixed.mesh
 //               mpirun -np 4 ex1p -m ../data/toroid-wedge.mesh
-//               mpirun -np 4 ex1p -m ../data/octahedron.mesh -o 1
 //               mpirun -np 4 ex1p -m ../data/periodic-annulus-sector.msh
 //               mpirun -np 4 ex1p -m ../data/periodic-torus-sector.msh
 //               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
@@ -90,8 +89,7 @@ int main(int argc, char *argv[])
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
 #ifdef MFEM_USE_CEED
-   args.AddOption(&algebraic_ceed, "-a", "--algebraic",
-                  "-no-a", "--no-algebraic",
+   args.AddOption(&algebraic_ceed, "-a", "--algebraic", "-no-a", "--no-algebraic",
                  "Use algebraic Ceed solver");
 #endif
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -199,15 +197,15 @@ int main(int argc, char *argv[])
   b.AddDomainIntegrator(new DomainLFIntegrator(one));
   b.Assemble();

-   // 10. Define the solution vector x as a parallel finite element grid
-   //     function corresponding to fespace. Initialize x with initial guess of
-   //     zero, which satisfies the boundary conditions.
+   // 10. Define the solution vector x as a parallel finite element grid function
+   //     corresponding to fespace. Initialize x with initial guess of zero,
+   //     which satisfies the boundary conditions.
   ParGridFunction x(&fespace);
   x = 0.0;

   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the
-   //     Diffusion domain integrator.
+   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
+   //     domain integrator.
   ParBilinearForm a(&fespace);
   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   a.AddDomainIntegrator(new DiffusionIntegrator(one));
@@ -3,10 +3,6 @@
 // Compile with: make ex20
 //
 // Sample runs:  ex20
-//               ex20 -p 1 -o 1 -n 120 -dt 0.1
-//               ex20 -p 1 -o 2 -n 60 -dt 0.2
-//               ex20 -p 1 -o 3 -n 40 -dt 0.3
-//               ex20 -p 1 -o 4 -n 30 -dt 0.4
 //
 // Description: This example demonstrates the use of the variable order,
 //              symplectic ODE integration algorithm.  Symplectic integration
@@ -235,7 +231,6 @@ int main(int argc, char *argv[])
   // 9. Finalize the GLVis output
   if (visualization)
   {
-      mesh.FinalizeQuadMesh(1);
      H1_FECollection fec(order = 1, 2);
      FiniteElementSpace fespace(&mesh, &fec);
      GridFunction energy(&fespace);
@@ -3,10 +3,6 @@
 // Compile with: make ex20p
 //
 // Sample runs:  mpirun -np 4 ex20p
-//               mpirun -np 4 ex20p -p 1 -o 1 -n 120 -dt 0.1
-//               mpirun -np 4 ex20p -p 1 -o 2 -n 60 -dt 0.2
-//               mpirun -np 4 ex20p -p 1 -o 3 -n 40 -dt 0.3
-//               mpirun -np 4 ex20p -p 1 -o 4 -n 30 -dt 0.4
 //
 // Description: This example demonstrates the use of the variable order,
 //              symplectic ODE integration algorithm.  Symplectic integration
@@ -172,7 +168,7 @@ int main(int argc, char *argv[])
   }

   // 6. Create a Mesh for visualization in phase space
-   int nverts = (visualization) ? 2*num_procs*(nsteps+1) : 0;
+   int nverts = (visualization) ? (num_procs+1)*(nsteps+1) : 0;
   int nelems = (visualization) ? (nsteps * num_procs) : 0;
   Mesh mesh(2, nverts, nelems, 0, 3);

@@ -194,9 +190,9 @@ int main(int argc, char *argv[])

         if (visualization)
         {
+            mesh.AddVertex(x0);
            for (int j = 0; j < num_procs; j++)
            {
-               mesh.AddVertex(x0);
               x1[0] = q(0);
               x1[1] = p(0);
               x1[2] = 0.0;
@@ -220,17 +216,17 @@ int main(int argc, char *argv[])
      if (visualization)
      {
         x0[2] = t;
+         mesh.AddVertex(x0);
         for (int j = 0; j < num_procs; j++)
         {
-            mesh.AddVertex(x0);
            x1[0] = q(0);
            x1[1] = p(0);
            x1[2] = t;
            mesh.AddVertex(x1);
-            v[0] = 2 * num_procs * i + 2 * j;
-            v[1] = 2 * num_procs * (i + 1) + 2 * j;
-            v[2] = 2 * num_procs * (i + 1) + 2 * j + 1;
-            v[3] = 2 * num_procs * i + 2 * j + 1;
+            v[0] = (num_procs + 1) * i;
+            v[1] = (num_procs + 1) * (i + 1);
+            v[2] = (num_procs + 1) * (i + 1) + j + 1;
+            v[3] = (num_procs + 1) * i + j + 1;
            mesh.AddQuad(v);
            part[num_procs * i + j] = j;
         }
@@ -13,8 +13,6 @@
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 1
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 2
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 2 -pa
-//               ex22 -m ../data/inline-wedge.mesh -o 1
-//               ex22 -m ../data/inline-pyramid.mesh -o 1
 //               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0
 //
 // Device sample runs:
@@ -13,8 +13,6 @@
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 1
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 2
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 1 -p 2 -pa
-//               mpirun -np 4 ex22p -m ../data/inline-wedge.mesh -o 1
-//               mpirun -np 4 ex22p -m ../data/inline-pyramid.mesh -o 1
 //               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0
 //
 // Device sample runs:
@@ -113,6 +113,7 @@ int main(int argc, char *argv[])
         mesh->UniformRefinement();
      }
   }
+   mesh->ReorientTetMesh();

   // 5. Define a finite element space on the mesh. Here we use Nedelec or
   //    Raviart-Thomas finite elements of the specified order.
@@ -141,6 +141,7 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
+   pmesh->ReorientTetMesh();

   // 7. Define a parallel finite element space on the parallel mesh. Here we
   //    use Nedelec or Raviart-Thomas finite elements of the specified order.
@@ -92,7 +92,7 @@ class PMLDiagMatrixCoefficient : public VectorCoefficient
 {
 private:
   CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML *, Vector &);
+   void (*Function)(const Vector &, CartesianPML * , Vector &);
 public:
   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
                                              Vector &),
@@ -277,8 +277,10 @@ int main(int argc, char *argv[])
      mesh->UniformRefinement();
   }

-   // 6. Set element attributes in order to distinguish elements in the
-   //    PML region
+   // 6. Reorient mesh in case of a tet mesh
+   mesh->ReorientTetMesh();
+
+   // Set element attributes in order to distinguish elements in the PML region
   pml->SetAttributes(mesh);

   // 7. Define a finite element space on the mesh. Here we use the Nedelec
@@ -92,7 +92,7 @@ class PMLDiagMatrixCoefficient : public VectorCoefficient
 {
 private:
   CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML *, Vector &);
+   void (*Function)(const Vector &, CartesianPML * , Vector &);
 public:
   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
                                              Vector &),
@@ -316,6 +316,9 @@ int main(int argc, char *argv[])
      }
   }

+   // 7a. Reorient mesh in case of a tet mesh
+   pmesh->ReorientTetMesh();
+
   // 8. Set element attributes in order to distinguish elements in the PML
   pml->SetAttributes(pmesh);

@@ -105,7 +105,7 @@ private:
      Vector diag(fespace.GetTrueVSize());
      bfs.Last()->AssembleDiagonal(diag);

-      Solver* smoother = new OperatorChebyshevSmoother(*opr, diag,
+      Solver* smoother = new OperatorChebyshevSmoother(opr.Ptr(), diag,
                                                       *essentialTrueDofs.Last(), 2);
      AddLevel(opr.Ptr(), smoother, true, true);
   }
@@ -115,7 +115,7 @@ private:
      Vector diag(fespace.GetTrueVSize());
      bfs.Last()->AssembleDiagonal(diag);

-      Solver* smoother = new OperatorChebyshevSmoother(*opr, diag,
+      Solver* smoother = new OperatorChebyshevSmoother(opr.Ptr(), diag,
                                                       *essentialTrueDofs.Last(), 2, fespace.GetParMesh()->GetComm());

      AddLevel(opr.Ptr(), smoother, true, true);
@@ -81,12 +81,6 @@ Mesh * build_trapezoid_mesh(double offset)

 int main(int argc, char *argv[])
 {
-#ifdef HYPRE_USING_CUDA
-   cout << "\nAs of mfem-4.3 and hypre-2.22.0 (July 2021) this example\n"
-        << "is NOT supported with the CUDA version of hypre.\n\n";
-   return 255;
-#endif
-
   // 1. Initialize MPI.
   int num_procs, myid;
   MPI_Init(&argc, &argv);
@@ -366,7 +360,6 @@ int main(int argc, char *argv[])
   }
   delete pmesh;

-   // HYPRE_Finalize();
   MPI_Finalize();

   return 0;
@@ -61,7 +61,6 @@ int main(int argc, char *argv[])
   bool visualization = 1;
   bool amg_elast = 0;
   bool reorder_space = false;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -79,8 +78,6 @@ int main(int argc, char *argv[])
                  "Enable or disable GLVis visualization.");
   args.AddOption(&reorder_space, "-nodes", "--by-nodes", "-vdim", "--by-vdim",
                  "Use byNODES ordering of vector space instead of byVDIM");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -96,12 +93,7 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
@@ -117,14 +109,14 @@ int main(int argc, char *argv[])
      return 3;
   }

-   // 5. Select the order of the finite element discretization space. For NURBS
+   // 4. Select the order of the finite element discretization space. For NURBS
   //    meshes, we increase the order by degree elevation.
   if (mesh->NURBSext)
   {
      mesh->DegreeElevate(order, order);
   }

-   // 6. Refine the serial mesh on all processors to increase the resolution. In
+   // 5. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ref_levels' of uniform refinement. We choose
   //    'ref_levels' to be the largest number that gives a final mesh with no
   //    more than 1,000 elements.
@@ -137,7 +129,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 7. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -150,7 +142,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 8. Define a parallel finite element space on the parallel mesh. Here we
+   // 7. Define a parallel finite element space on the parallel mesh. Here we
   //    use vector finite elements, i.e. dim copies of a scalar finite element
   //    space. We use the ordering by vector dimension (the last argument of
   //    the FiniteElementSpace constructor) which is expected in the systems
@@ -183,7 +175,7 @@ int main(int argc, char *argv[])
           << "Assembling: " << flush;
   }

-   // 9. Determine the list of true (i.e. parallel conforming) essential
+   // 8. Determine the list of true (i.e. parallel conforming) essential
   //    boundary dofs. In this example, the boundary conditions are defined by
   //    marking only boundary attribute 1 from the mesh as essential and
   //    converting it to a list of true dofs.
@@ -192,14 +184,14 @@ int main(int argc, char *argv[])
   ess_bdr[0] = 1;
   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);

-   // 10. Set up the parallel linear form b(.) which corresponds to the
-   //     right-hand side of the FEM linear system. In this case, b_i equals the
-   //     boundary integral of f*phi_i where f represents a "pull down" force on
-   //     the Neumann part of the boundary and phi_i are the basis functions in
-   //     the finite element fespace. The force is defined by the object f, which
-   //     is a vector of Coefficient objects. The fact that f is non-zero on
-   //     boundary attribute 2 is indicated by the use of piece-wise constants
-   //     coefficient for its last component.
+   // 9. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system. In this case, b_i equals the
+   //    boundary integral of f*phi_i where f represents a "pull down" force on
+   //    the Neumann part of the boundary and phi_i are the basis functions in
+   //    the finite element fespace. The force is defined by the object f, which
+   //    is a vector of Coefficient objects. The fact that f is non-zero on
+   //    boundary attribute 2 is indicated by the use of piece-wise constants
+   //    coefficient for its last component.
   VectorArrayCoefficient f(dim);
   for (int i = 0; i < dim-1; i++)
   {
@@ -220,13 +212,13 @@ int main(int argc, char *argv[])
   }
   b->Assemble();

-   // 11. Define the solution vector x as a parallel finite element grid
+   // 10. Define the solution vector x as a parallel finite element grid
   //     function corresponding to fespace. Initialize x with initial guess of
   //     zero, which satisfies the boundary conditions.
   ParGridFunction x(fespace);
   x = 0.0;

-   // 12. Set up the parallel bilinear form a(.,.) on the finite element space
+   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
   //     corresponding to the linear elasticity integrator with piece-wise
   //     constants coefficient lambda and mu.
   Vector lambda(pmesh->attributes.Max());
@@ -241,7 +233,7 @@ int main(int argc, char *argv[])
   ParBilinearForm *a = new ParBilinearForm(fespace);
   a->AddDomainIntegrator(new ElasticityIntegrator(lambda_func, mu_func));

-   // 13. Assemble the parallel bilinear form and the corresponding linear
+   // 12. Assemble the parallel bilinear form and the corresponding linear
   //     system, applying any necessary transformations such as: parallel
   //     assembly, eliminating boundary conditions, applying conforming
   //     constraints for non-conforming AMR, static condensation, etc.
@@ -258,7 +250,7 @@ int main(int argc, char *argv[])
      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
   }

-   // 14. Define and apply a parallel PCG solver for A X = B with the BoomerAMG
+   // 13. Define and apply a parallel PCG solver for A X = B with the BoomerAMG
   //     preconditioner from hypre.
   HypreBoomerAMG *amg = new HypreBoomerAMG(A);
   if (amg_elast && !a->StaticCondensationIsEnabled())
@@ -276,11 +268,11 @@ int main(int argc, char *argv[])
   pcg->SetPreconditioner(*amg);
   pcg->Mult(B, X);

-   // 15. Recover the parallel grid function corresponding to X. This is the
+   // 14. Recover the parallel grid function corresponding to X. This is the
   //     local finite element solution on each processor.
   a->RecoverFEMSolution(X, *b, x);

-   // 16. For non-NURBS meshes, make the mesh curved based on the finite element
+   // 15. For non-NURBS meshes, make the mesh curved based on the finite element
   //     space. This means that we define the mesh elements through a fespace
   //     based transformation of the reference element.  This allows us to save
   //     the displaced mesh as a curved mesh when using high-order finite
@@ -292,7 +284,7 @@ int main(int argc, char *argv[])
      pmesh->SetNodalFESpace(fespace);
   }

-   // 17. Save in parallel the displaced mesh and the inverted solution (which
+   // 16. Save in parallel the displaced mesh and the inverted solution (which
   //     gives the backward displacements to the original grid). This output
   //     can be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
   {
@@ -313,7 +305,7 @@ int main(int argc, char *argv[])
      x.Save(sol_ofs);
   }

-   // 18. Send the above data by socket to a GLVis server.  Use the "n" and "b"
+   // 17. Send the above data by socket to a GLVis server.  Use the "n" and "b"
   //     keys in GLVis to visualize the displacements.
   if (visualization)
   {
@@ -325,7 +317,7 @@ int main(int argc, char *argv[])
      sol_sock << "solution\n" << *pmesh << x << flush;
   }

-   // 19. Free the used memory.
+   // 18. Free the used memory.
   delete pcg;
   delete amg;
   delete a;
@@ -16,8 +16,6 @@
 //               ex3 -m ../data/beam-hex-nurbs.mesh
 //               ex3 -m ../data/amr-hex.mesh
 //               ex3 -m ../data/fichera-amr.mesh
-//               ex3 -m ../data/ref-prism.mesh -o 1
-//               ex3 -m ../data/octahedron.mesh -o 1
 //               ex3 -m ../data/star-surf.mesh -o 1
 //               ex3 -m ../data/mobius-strip.mesh -f 0.1
 //               ex3 -m ../data/klein-bottle.mesh -f 0.1
@@ -115,6 +113,7 @@ int main(int argc, char *argv[])
         mesh->UniformRefinement();
      }
   }
+   mesh->ReorientTetMesh();

   // 5. Define a finite element space on the mesh. Here we use the Nedelec
   //    finite elements of the specified order.
@@ -16,8 +16,6 @@
 //               mpirun -np 4 ex3p -m ../data/beam-hex-nurbs.mesh
 //               mpirun -np 4 ex3p -m ../data/amr-quad.mesh -o 2
 //               mpirun -np 4 ex3p -m ../data/amr-hex.mesh
-//               mpirun -np 4 ex3p -m ../data/ref-prism.mesh -o 1
-//               mpirun -np 4 ex3p -m ../data/octahedron.mesh -o 1
 //               mpirun -np 4 ex3p -m ../data/star-surf.mesh -o 2
 //               mpirun -np 4 ex3p -m ../data/mobius-strip.mesh -o 2 -f 0.1
 //               mpirun -np 4 ex3p -m ../data/klein-bottle.mesh -o 2 -f 0.1
@@ -105,7 +103,6 @@ int main(int argc, char *argv[])
      {
         args.PrintUsage(cout);
      }
-      // HYPRE_Finalize();
      MPI_Finalize();
      return 1;
   }
@@ -141,7 +138,9 @@ int main(int argc, char *argv[])

   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
+   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
+   //    meshes need to be reoriented before we can define high-order Nedelec
+   //    spaces on them.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
   delete mesh;
   {
@@ -151,6 +150,7 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
+   pmesh->ReorientTetMesh();

   // 7. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Nedelec finite elements of the specified order.
@@ -19,8 +19,6 @@
 //               ex4 -m ../data/amr-hex.mesh
 //               ex4 -m ../data/amr-hex.mesh -o 2 -hb
 //               ex4 -m ../data/fichera-amr.mesh -o 2 -sc
-//               ex4 -m ../data/ref-prism.mesh -o 1
-//               ex4 -m ../data/octahedron.mesh -o 1
 //               ex4 -m ../data/star-surf.mesh -o 1
 //
 // Device sample runs:
@@ -19,8 +19,6 @@
 //               mpirun -np 3 ex4p -m ../data/amr-quad.mesh -o 2 -hb
 //               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -sc
 //               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -hb
-//               mpirun -np 4 ex4p -m ../data/ref-prism.mesh -o 1
-//               mpirun -np 4 ex4p -m ../data/octahedron.mesh -o 1
 //               mpirun -np 4 ex4p -m ../data/star-surf.mesh -o 3 -hb
 //
 // Device sample runs:
@@ -137,7 +135,9 @@ int main(int argc, char *argv[])

   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
+   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
+   //    meshes need to be reoriented before we can define high-order Nedelec
+   //    spaces on them (this is needed in the ADS solver below).
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
   delete mesh;
   {
@@ -147,6 +147,7 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
+   pmesh->ReorientTetMesh();

   // 7. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Raviart-Thomas finite elements of the specified order.
@@ -197,7 +197,6 @@ int main(int argc, char *argv[])
      SparseMatrix &M(mVarf->SpMat());
      SparseMatrix &B(bVarf->SpMat());
      B *= -1.;
-      if (Device::IsEnabled()) { B.BuildTranspose(); }
      Bt = new TransposeOperator(&B);

      darcyOp.SetBlock(0,0, &M);
@@ -241,7 +240,6 @@ int main(int argc, char *argv[])
   {
      SparseMatrix &M(mVarf->SpMat());
      M.GetDiag(Md);
-      Md.HostReadWrite();

      SparseMatrix &B(bVarf->SpMat());
      MinvBt = Transpose(B);
@@ -289,18 +287,12 @@ int main(int argc, char *argv[])
   chrono.Stop();

   if (solver.GetConverged())
-   {
      std::cout << "MINRES converged in " << solver.GetNumIterations()
-                << " iterations with a residual norm of "
-                << solver.GetFinalNorm() << ".\n";
-   }
+                << " iterations with a residual norm of " << solver.GetFinalNorm() << ".\n";
   else
-   {
      std::cout << "MINRES did not converge in " << solver.GetNumIterations()
-                << " iterations. Residual norm is " << solver.GetFinalNorm()
-                << ".\n";
-   }
-   std::cout << "MINRES solver took " << chrono.RealTime() << "s.\n";
+                << " iterations. Residual norm is " << solver.GetFinalNorm() << ".\n";
+   std::cout << "MINRES solver took " << chrono.RealTime() << "s. \n";

   // 12. Create the grid functions u and p. Compute the L2 error norms.
   GridFunction u, p;
@@ -47,7 +47,6 @@ int main(int argc, char *argv[])
   int order = 2;
   bool always_snap = false;
   bool visualization = 1;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&elem_type, "-e", "--elem",
@@ -66,8 +65,6 @@ int main(int argc, char *argv[])
                  "--snap-at-the-end",
                  "If true, snap nodes to the sphere initially and after each refinement "
                  "otherwise, snap only after the last refinement");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -83,12 +80,7 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Generate an initial high-order (surface) mesh on the unit sphere. The
+   // 3. Generate an initial high-order (surface) mesh on the unit sphere. The
   //    Mesh object represents a 2D mesh in 3 spatial dimensions. We first add
   //    the elements and the vertices of the mesh, and then make it high-order
   //    by specifying a finite element space for its nodes.
@@ -154,7 +146,7 @@ int main(int argc, char *argv[])
   FiniteElementSpace nodal_fes(mesh, &fec, mesh->SpaceDimension());
   mesh->SetNodalFESpace(&nodal_fes);

-   // 5. Refine the mesh while snapping nodes to the sphere. Number of parallel
+   // 4. Refine the mesh while snapping nodes to the sphere. Number of parallel
   //    refinements is fixed to 2.
   for (int l = 0; l <= ref_levels; l++)
   {
@@ -226,7 +218,7 @@ int main(int argc, char *argv[])
      SnapNodes(*pmesh);
   }

-   // 6. Define a finite element space on the mesh. Here we use isoparametric
+   // 5. Define a finite element space on the mesh. Here we use isoparametric
   //    finite elements -- the same as the mesh nodes.
   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, &fec);
   HYPRE_BigInt size = fespace->GlobalTrueVSize();
@@ -235,7 +227,7 @@ int main(int argc, char *argv[])
      cout << "Number of unknowns: " << size << endl;
   }

-   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
+   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
   //    the basis functions in the finite element fespace.
   ParLinearForm *b = new ParLinearForm(fespace);
@@ -245,27 +237,27 @@ int main(int argc, char *argv[])
   b->AddDomainIntegrator(new DomainLFIntegrator(rhs_coef));
   b->Assemble();

-   // 8. Define the solution vector x as a finite element grid function
+   // 7. Define the solution vector x as a finite element grid function
   //    corresponding to fespace. Initialize x with initial guess of zero.
   ParGridFunction x(fespace);
   x = 0.0;

-   // 9. Set up the bilinear form a(.,.) on the finite element space
+   // 8. Set up the bilinear form a(.,.) on the finite element space
   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
   //    and Mass domain integrators.
   ParBilinearForm *a = new ParBilinearForm(fespace);
   a->AddDomainIntegrator(new DiffusionIntegrator(one));
   a->AddDomainIntegrator(new MassIntegrator(one));

-   // 10. Assemble the parallel linear system, applying any transformations
-   //     such as: parallel assembly, applying conforming constraints, etc.
+   // 9. Assemble the parallel linear system, applying any transformations
+   //    such as: parallel assembly, applying conforming constraints, etc.
   a->Assemble();
   HypreParMatrix A;
   Vector B, X;
   Array<int> empty_tdof_list;
   a->FormLinearSystem(empty_tdof_list, x, *b, A, X, B);

-   // 11. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
+   // 10. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
   //     preconditioner from hypre. Extract the parallel grid function x
   //     corresponding to the finite element approximation X. This is the local
   //     solution on each processor.
@@ -281,14 +273,14 @@ int main(int argc, char *argv[])
   delete a;
   delete b;

-   // 12. Compute and print the L^2 norm of the error.
+   // 11. Compute and print the L^2 norm of the error.
   double err = x.ComputeL2Error(sol_coef);
   if (myid == 0)
   {
      cout << "\nL2 norm of error: " << err << endl;
   }

-   // 13. Save the refined mesh and the solution. This output can be viewed
+   // 12. Save the refined mesh and the solution. This output can be viewed
   //     later using GLVis: "glvis -np <np> -m sphere_refined -g sol".
   {
      ostringstream mesh_name, sol_name;
@@ -304,7 +296,7 @@ int main(int argc, char *argv[])
      x.Save(sol_ofs);
   }

-   // 14. Send the solution by socket to a GLVis server.
+   // 13. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -315,7 +307,7 @@ int main(int argc, char *argv[])
      sol_sock << "solution\n" << *pmesh << x << flush;
   }

-   // 15. Free the used memory.
+   // 14. Free the used memory.
   delete pcg;
   delete amg;
   delete fespace;
@@ -106,6 +106,7 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
+   pmesh->ReorientTetMesh();

   // 6. Define the trial, interfacial (trace) and test DPG spaces:
   //    - The trial space, x0_space, contains the non-interfacial unknowns and
@@ -69,8 +69,7 @@ int main(int argc, char *argv[])
   bool pa = false;
   const char *device_config = "cpu";
   bool visualization = true;
-   int solver_config = 0;
-   int print_lvl = 1;
+   bool use_ginkgo_solver= true;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -87,14 +86,9 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
-   args.AddOption(&solver_config, "-s", "--solver-config",
-                  "Solver and preconditioner combination: \n\t"
-                  "   0 - Ginkgo solver and Ginkgo preconditioner, \n\t"
-                  "   1 - Ginkgo solver and MFEM preconditioner, \n\t"
-                  "   2 - MFEM solver and Ginkgo preconditioner, \n\t"
-                  "   3 - MFEM solver and MFEM preconditioner.");
-   args.AddOption(&print_lvl, "-pl", "--print-level",
-                  "Print level for iterative solver (1 prints every iteration).");
+   args.AddOption(&use_ginkgo_solver, "-gko", "--use_gko_solver", "-no-gko",
+                  "--no-gko-solver",
+                  "Solve using ginkgo.");
   args.Parse();
   if (!args.Good())
   {
@@ -197,123 +191,39 @@ int main(int argc, char *argv[])
   // 11. Solve the linear system A X = B.
   if (!pa)
   {
-      switch (solver_config)
+      if (use_ginkgo_solver)
      {
-         // Solve the linear system with CG + IC from Ginkgo
-         case 0:
-         {
-            cout << "Using Ginkgo solver + preconditioner...\n";
-            Ginkgo::GinkgoExecutor exec(device);
-            Ginkgo::IcPreconditioner ginkgo_precond(exec, "paric", 30);
-            Ginkgo::CGSolver ginkgo_solver(exec, ginkgo_precond);
-            ginkgo_solver.SetPrintLevel(print_lvl);
-            ginkgo_solver.SetRelTol(1e-12);
-            ginkgo_solver.SetAbsTol(0.0);
-            ginkgo_solver.SetMaxIter(400);
-            ginkgo_solver.SetOperator(*(A.Ptr()));
-            ginkgo_solver.Mult(B, X);
-            break;
-         }
-
-         // Solve the linear system with CG from Ginkgo + MFEM preconditioner
-         case 1:
-         {
-            cout << "Using Ginkgo solver + MFEM preconditioner...\n";
-            Ginkgo::GinkgoExecutor exec(device);
-            //Create MFEM preconditioner and wrap it for Ginkgo's use.
-            DSmoother M((SparseMatrix&)(*A));
-            Ginkgo::MFEMPreconditioner gko_M(exec, M);
-            Ginkgo::CGSolver ginkgo_solver(exec, gko_M);
-            ginkgo_solver.SetPrintLevel(print_lvl);
-            ginkgo_solver.SetRelTol(1e-12);
-            ginkgo_solver.SetAbsTol(0.0);
-            ginkgo_solver.SetMaxIter(400);
-            ginkgo_solver.SetOperator(*(A.Ptr()));
-            ginkgo_solver.Mult(B, X);
-            break;
-         }
-
-         // Ginkgo IC preconditioner +  MFEM CG solver
-         case 2:
-         {
-            cout << "Using MFEM solver + Ginkgo preconditioner...\n";
-            Ginkgo::GinkgoExecutor exec(device);
-            Ginkgo::IcPreconditioner M(exec, "paric", 30);
-            M.SetOperator(*(A.Ptr()));  // Generate the preconditioner for the matrix A.
-            PCG(*A, M, B, X, print_lvl, 400, 1e-12, 0.0);
-            break;
-         }
-
-         // MFEM solver + MFEM preconditioner
-         case 3:
-         {
-            cout << "Using MFEM solver + MFEM preconditioner...\n";
-            // Use a simple Jacobi preconditioner with PCG.
-            DSmoother M((SparseMatrix&)(*A));
-            PCG(*A, M, B, X, print_lvl, 400, 1e-12, 0.0);
-            break;
-         }
-      } // End switch on solver_config
+#ifdef MFEM_USE_GINKGO
+         // Solve the linear system with CG + ILU from Ginkgo.
+         std::string executor = "reference";
+         auto exec = gko::ReferenceExecutor::create();
+         auto ilu_precond =
+            gko::preconditioner::Ilu<gko::solver::LowerTrs<>,
+            gko::solver::UpperTrs<>, false>::build()
+            .on(exec);
+         GinkgoWrappers::CGSolver ginkgo_solver(executor, 1, 2000, 1e-12, 0.0,
+                                                ilu_precond.release() );
+         ginkgo_solver.solve(&((SparseMatrix&)(*A)), X, B);
+#endif
+      }
+      else
+      {
+#ifndef MFEM_USE_SUITESPARSE
+         // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
+         GSSmoother M((SparseMatrix&)(*A));
+         PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
+#else
+         // If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+         UMFPackSolver umf_solver;
+         umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+         umf_solver.SetOperator(*A);
+         umf_solver.Mult(B, X);
+#endif
+      }
   }
-   // Partial assembly mode.  Cannot use Ginkgo preconditioners, but can use Ginkgo
-   // solvers.
-   else
+   else // No preconditioning for now in partial assembly mode.
   {
-      if (UsesTensorBasis(*fespace))
-      {
-         // Use Jacobi preconditioning in partial assembly mode.
-         OperatorJacobiSmoother M(*a, ess_tdof_list);
-         switch (solver_config)
-         {
-            // No Ginkgo preconditioners work with matrix-free; error
-            case 0:
-            {
-               cout << "Using Ginkgo solver + preconditioner...\n";
-               MFEM_ABORT("Cannot use Ginkgo preconditioner in partial assembly mode.\n"
-                          "            Try -s 1 to test Ginkgo solver with an MFEM preconditioner.");
-               break;
-            }
-
-            // Use Ginkgo solver with MFEM preconditioner
-            case 1:
-            {
-               cout << "Using Ginkgo solver + MFEM preconditioner...\n";
-               Ginkgo::GinkgoExecutor exec(device);
-               // Wrap MFEM preconditioner for Ginkgo's use.
-               Ginkgo::MFEMPreconditioner gko_M(exec, M);
-               Ginkgo::CGSolver ginkgo_solver(exec, gko_M);
-               ginkgo_solver.SetPrintLevel(print_lvl);
-               ginkgo_solver.SetRelTol(1e-12);
-               ginkgo_solver.SetAbsTol(0.0);
-               ginkgo_solver.SetMaxIter(400);
-               ginkgo_solver.SetOperator(*(A.Ptr()));
-               ginkgo_solver.Mult(B, X);
-               break;
-            }
-
-            // No Ginkgo preconditioners work with matrix-free; error
-            case 2:
-            {
-               cout << "Using MFEM solver + Ginkgo preconditioner...\n";
-               MFEM_ABORT("Cannot use Ginkgo preconditioner in partial assembly mode.\n"
-                          "            Try -s 1 to test Ginkgo solver with an MFEM preconditioner.");
-               break;
-            }
-
-            // Use MFEM solver and preconditioner
-            case 3:
-            {
-               cout << "Using MFEM solver + MFEM preconditioner...\n";
-               PCG(*A, M, B, X, print_lvl, 400, 1e-12, 0.0);
-               break;
-            }
-         } // End switch on solver_config
-      }
-      else // CG with no preconditioning
-      {
-         cout << "Using MFEM solver + no preconditioner...\n";
-         CG(*A, B, X, print_lvl, 400, 1e-12, 0.0);
-      }
+      CG(*A, B, X, 1, 2000, 1e-12, 0.0);
   }

   // 12. Recover the solution as a finite element grid function.
@@ -216,7 +216,7 @@ int main(int argc, char *argv[])
 {
   // 1. Parse command-line options.
   problem = 0;
-   optimizer_type = 2;
+   optimizer_type = 1;
   const char *mesh_file = "../../data/periodic-hexagon.mesh";
   int ref_levels = 2;
   int order = 3;
@@ -247,7 +247,7 @@ int main(int argc, char *argv[])

   // 2. Parse command-line options.
   problem = 0;
-   optimizer_type = 2;
+   optimizer_type = 1;
   const char *mesh_file = "../../data/periodic-hexagon.mesh";
   int ser_ref_levels = 2;
   int par_ref_levels = 0;
@@ -50,13 +50,6 @@ endif
 MFEM_TESTS = EXAMPLES
 include $(MFEM_TEST_MK)

-# Testing: Parallel vs. serial runs
-RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
-%-test-par: %
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
-%-test-seq: %
-	@$(call mfem-test,$<,, Serial example)
-
 # Testing: "test" target and mfem-test* variables are defined in config/test.mk

 # Generate an error message if the MFEM library is not built and exit
@@ -26,9 +26,6 @@ SEQ_EXAMPLES = ex0 ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 \
 PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
  ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
  ex25p ex26p ex27p ex28p ex29p
-SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
-PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p \
-  ex24p ex25p ex26p

 ifeq ($(MFEM_USE_MPI),NO)
   EXAMPLES = $(SEQ_EXAMPLES)
@@ -102,14 +99,6 @@ RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
 	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
 %-test-seq: %
 	@$(call mfem-test,$<,, Serial example)
-%-test-par-cuda: %
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel CUDA example,-d cuda)
-%-test-seq-cuda: %
-	@$(call mfem-test,$<,, Serial CUDA example,-d cuda)
-%-test-par-hip: %
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel HIP example,-d hip)
-%-test-seq-hip: %
-	@$(call mfem-test,$<,, Serial HIP example,-d hip)

 # Testing: Specific execution options
 ex0-test-seq: ex0
@@ -121,7 +121,9 @@ int main(int argc, char *argv[])

   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
+   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
+   //    meshes need to be reoriented before we can define high-order Nedelec
+   //    spaces on them.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
   delete mesh;
   {
@@ -131,6 +133,7 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
+   pmesh->ReorientTetMesh();

   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Nedelec finite elements of the specified order.
@@ -122,7 +122,9 @@ int main(int argc, char *argv[])

   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
+   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
+   //    meshes need to be reoriented before we can define high-order Nedelec
+   //    spaces on them (this is needed in the ADS solver below).
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
   delete mesh;
   {
@@ -132,6 +134,7 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
+   pmesh->ReorientTetMesh();

   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Raviart-Thomas finite elements of the specified order.
@@ -282,10 +282,6 @@ int main(int argc, char *argv[])
   superlu->SetOperator(*SLU_A);
   superlu->SetPrintStatistics(true);
   superlu->Mult(B, X);
-   superlu->DismantleGrid();
-
-   delete SLU_A;
-   delete superlu;

   // 14. Recover the parallel grid function corresponding to X. This is the
   //     local finite element solution on each processor.
@@ -39,7 +39,6 @@ set(SRCS
  complex_fem.cpp
  convergence.cpp
  datacollection.cpp
-  doftrans.cpp
  eltrans.cpp
  estimators.cpp
  fe.cpp
@@ -106,7 +105,6 @@ set(SRCS
  tmop/tmop_pa_w3.cpp
  tmop/tmop_pa_w3_c0.cpp
  tmop_tools.cpp
-  tmop_amr.cpp
  gslib.cpp
  transfer.cpp
  lor.cpp
@@ -120,7 +118,6 @@ set(HDRS
  complex_fem.hpp
  convergence.hpp
  datacollection.hpp
-  doftrans.hpp
  eltrans.hpp
  estimators.hpp
  fe.hpp
@@ -167,7 +164,6 @@ set(HDRS
  tmop.hpp
  tmop/tmop_pa.hpp
  tmop_tools.hpp
-  tmop_amr.hpp
  gslib.hpp
  transfer.hpp
  lor.hpp
@@ -188,11 +184,6 @@ if (MFEM_USE_ADIOS2)
  list(APPEND HDRS adios2datacollection.hpp)
 endif()

-if (MFEM_USE_FMS)
-  list(APPEND SRCS fmsdatacollection.cpp fmsconvert.cpp)
-  list(APPEND HDRS fmsdatacollection.hpp fmsconvert.hpp)
-endif()
-
 if (MFEM_USE_MPI)
  list(APPEND SRCS
    pbilinearform.cpp
@@ -31,7 +31,7 @@ void BilinearForm::AllocMat()
   const Table &elem_dof = fes->GetElementToDofTable();
   Table dof_dof;

-   if (interior_face_integs.Size() > 0)
+   if (fbfi.Size() > 0)
   {
      // the sparsity pattern is defined from the map: face->element->dof
      Table face_dof, dof_face;
@@ -99,15 +99,15 @@ BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)
   ext = NULL;

   // Copy the pointers to the integrators
-   domain_integs = bf->domain_integs;
+   dbfi = bf->dbfi;

-   boundary_integs = bf->boundary_integs;
-   boundary_integs_marker = bf->boundary_integs_marker;
+   bbfi = bf->bbfi;
+   bbfi_marker = bf->bbfi_marker;

-   interior_face_integs = bf->interior_face_integs;
+   fbfi = bf->fbfi;

-   boundary_face_integs = bf->boundary_face_integs;
-   boundary_face_integs_marker = bf->boundary_face_integs_marker;
+   bfbfi = bf->bfbfi;
+   bfbfi_marker = bf->bfbfi_marker;

   AllocMat();
 }
@@ -234,47 +234,38 @@ void BilinearForm::Finalize (int skip_zeros)

 void BilinearForm::AddDomainIntegrator(BilinearFormIntegrator *bfi)
 {
-   domain_integs.Append(bfi);
-   domain_integs_marker.Append(NULL); // NULL marker means apply everywhere
-}
-
-void BilinearForm::AddDomainIntegrator(BilinearFormIntegrator *bfi,
-                                       Array<int> &elem_marker)
-{
-   domain_integs.Append(bfi);
-   domain_integs_marker.Append(&elem_marker);
+   dbfi.Append(bfi);
 }

 void BilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi)
 {
-   boundary_integs.Append (bfi);
-   boundary_integs_marker.Append(NULL); // NULL marker means apply everywhere
+   bbfi.Append (bfi);
+   bbfi_marker.Append(NULL); // NULL marker means apply everywhere
 }

 void BilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi,
                                          Array<int> &bdr_marker)
 {
-   boundary_integs.Append (bfi);
-   boundary_integs_marker.Append(&bdr_marker);
+   bbfi.Append (bfi);
+   bbfi_marker.Append(&bdr_marker);
 }

-void BilinearForm::AddInteriorFaceIntegrator(BilinearFormIntegrator * bfi)
+void BilinearForm::AddInteriorFaceIntegrator (BilinearFormIntegrator * bfi)
 {
-   interior_face_integs.Append (bfi);
+   fbfi.Append (bfi);
 }

 void BilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator *bfi)
 {
-   boundary_face_integs.Append(bfi);
-   // NULL marker means apply everywhere
-   boundary_face_integs_marker.Append(NULL);
+   bfbfi.Append(bfi);
+   bfbfi_marker.Append(NULL); // NULL marker means apply everywhere
 }

 void BilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator *bfi,
                                        Array<int> &bdr_marker)
 {
-   boundary_face_integs.Append(bfi);
-   boundary_face_integs_marker.Append(&bdr_marker);
+   bfbfi.Append(bfi);
+   bfbfi_marker.Append(&bdr_marker);
 }

 void BilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
@@ -286,14 +277,14 @@ void BilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
      return;
   }

-   if (domain_integs.Size())
+   if (dbfi.Size())
   {
      const FiniteElement &fe = *fes->GetFE(i);
      ElementTransformation *eltrans = fes->GetElementTransformation(i);
-      domain_integs[0]->AssembleElementMatrix(fe, *eltrans, elmat);
-      for (int k = 1; k < domain_integs.Size(); k++)
+      dbfi[0]->AssembleElementMatrix(fe, *eltrans, elmat);
+      for (int k = 1; k < dbfi.Size(); k++)
      {
-         domain_integs[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
+         dbfi[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
         elmat += elemmat;
      }
   }
@@ -307,14 +298,14 @@ void BilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)

 void BilinearForm::ComputeBdrElementMatrix(int i, DenseMatrix &elmat)
 {
-   if (boundary_integs.Size())
+   if (bbfi.Size())
   {
      const FiniteElement &be = *fes->GetBE(i);
      ElementTransformation *eltrans = fes->GetBdrElementTransformation(i);
-      boundary_integs[0]->AssembleElementMatrix(be, *eltrans, elmat);
-      for (int k = 1; k < boundary_integs.Size(); k++)
+      bbfi[0]->AssembleElementMatrix(be, *eltrans, elmat);
+      for (int k = 1; k < bbfi.Size(); k++)
      {
-         boundary_integs[k]->AssembleElementMatrix(be, *eltrans, elemmat);
+         bbfi[k]->AssembleElementMatrix(be, *eltrans, elemmat);
         elmat += elemmat;
      }
   }
@@ -391,7 +382,6 @@ void BilinearForm::Assemble(int skip_zeros)
   }

   ElementTransformation *eltrans;
-   DofTransformation * doftrans;
   Mesh *mesh = fes -> GetMesh();
   DenseMatrix elmat, *elmat_p;

@@ -409,59 +399,24 @@ void BilinearForm::Assemble(int skip_zeros)
   }
 #endif

-   if (domain_integs.Size())
+   if (dbfi.Size())
   {
-      for (int k = 0; k < domain_integs.Size(); k++)
-      {
-         if (domain_integs_marker[k] != NULL)
-         {
-            MFEM_VERIFY(mesh->attributes.Size() ==
-                        domain_integs_marker[k]->Size(),
-                        "invalid element marker for domain integrator #"
-                        << k << ", counting from zero");
-         }
-      }
-
      for (int i = 0; i < fes -> GetNE(); i++)
      {
-         int elem_attr = fes->GetMesh()->GetAttribute(i);
-         doftrans = fes->GetElementVDofs(i, vdofs);
+         fes->GetElementVDofs(i, vdofs);
         if (element_matrices)
         {
            elmat_p = &(*element_matrices)(i);
         }
         else
         {
-            elmat.SetSize(0);
-            for (int k = 0; k < domain_integs.Size(); k++)
+            const FiniteElement &fe = *fes->GetFE(i);
+            eltrans = fes->GetElementTransformation(i);
+            dbfi[0]->AssembleElementMatrix(fe, *eltrans, elmat);
+            for (int k = 1; k < dbfi.Size(); k++)
            {
-               if ( domain_integs_marker[k] == NULL ||
-                    (*(domain_integs_marker[k]))[elem_attr-1] == 1)
-               {
-                  const FiniteElement &fe = *fes->GetFE(i);
-                  eltrans = fes->GetElementTransformation(i);
-                  domain_integs[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
-                  if (elmat.Size() == 0)
-                  {
-                     elmat = elemmat;
-                  }
-                  else
-                  {
-                     elmat += elemmat;
-                  }
-               }
-            }
-            if (elmat.Size() == 0)
-            {
-               continue;
-            }
-            else
-            {
-               elmat_p = &elmat;
-            }
-            if (doftrans)
-            {
-               doftrans->TransformDual(elmat);
+               dbfi[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
+               elmat += elemmat;
            }
            elmat_p = &elmat;
         }
@@ -480,20 +435,20 @@ void BilinearForm::Assemble(int skip_zeros)
      }
   }

-   if (boundary_integs.Size())
+   if (bbfi.Size())
   {
      // Which boundary attributes need to be processed?
      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
                                 mesh->bdr_attributes.Max() : 0);
      bdr_attr_marker = 0;
-      for (int k = 0; k < boundary_integs.Size(); k++)
+      for (int k = 0; k < bbfi.Size(); k++)
      {
-         if (boundary_integs_marker[k] == NULL)
+         if (bbfi_marker[k] == NULL)
         {
            bdr_attr_marker = 1;
            break;
         }
-         Array<int> &bdr_marker = *boundary_integs_marker[k];
+         Array<int> &bdr_marker = *bbfi_marker[k];
         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
                     "invalid boundary marker for boundary integrator #"
                     << k << ", counting from zero");
@@ -509,47 +464,42 @@ void BilinearForm::Assemble(int skip_zeros)
         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }

         const FiniteElement &be = *fes->GetBE(i);
-         doftrans = fes -> GetBdrElementVDofs (i, vdofs);
+         fes -> GetBdrElementVDofs (i, vdofs);
         eltrans = fes -> GetBdrElementTransformation (i);
         int k = 0;
-         for (; k < boundary_integs.Size(); k++)
+         for (; k < bbfi.Size(); k++)
         {
-            if (boundary_integs_marker[k] &&
-                (*boundary_integs_marker[k])[bdr_attr-1] == 0) { continue; }
+            if (bbfi_marker[k] &&
+                (*bbfi_marker[k])[bdr_attr-1] == 0) { continue; }

-            boundary_integs[k]->AssembleElementMatrix(be, *eltrans, elmat);
+            bbfi[k]->AssembleElementMatrix(be, *eltrans, elmat);
            k++;
            break;
         }
-         for (; k < boundary_integs.Size(); k++)
+         for (; k < bbfi.Size(); k++)
         {
-            if (boundary_integs_marker[k] &&
-                (*boundary_integs_marker[k])[bdr_attr-1] == 0) { continue; }
+            if (bbfi_marker[k] &&
+                (*bbfi_marker[k])[bdr_attr-1] == 0) { continue; }

-            boundary_integs[k]->AssembleElementMatrix(be, *eltrans, elemmat);
+            bbfi[k]->AssembleElementMatrix(be, *eltrans, elemmat);
            elmat += elemmat;
         }
-         if (doftrans)
-         {
-            doftrans->TransformDual(elmat);
-         }
-         elmat_p = &elmat;
         if (!static_cond)
         {
-            mat->AddSubMatrix(vdofs, vdofs, *elmat_p, skip_zeros);
+            mat->AddSubMatrix(vdofs, vdofs, elmat, skip_zeros);
            if (hybridization)
            {
-               hybridization->AssembleBdrMatrix(i, *elmat_p);
+               hybridization->AssembleBdrMatrix(i, elmat);
            }
         }
         else
         {
-            static_cond->AssembleBdrMatrix(i, *elmat_p);
+            static_cond->AssembleBdrMatrix(i, elmat);
         }
      }
   }

-   if (interior_face_integs.Size())
+   if (fbfi.Size())
   {
      FaceElementTransformations *tr;
      Array<int> vdofs2;
@@ -563,19 +513,18 @@ void BilinearForm::Assemble(int skip_zeros)
            fes -> GetElementVDofs (tr -> Elem1No, vdofs);
            fes -> GetElementVDofs (tr -> Elem2No, vdofs2);
            vdofs.Append (vdofs2);
-            for (int k = 0; k < interior_face_integs.Size(); k++)
+            for (int k = 0; k < fbfi.Size(); k++)
            {
-               interior_face_integs[k]->
-               AssembleFaceMatrix(*fes->GetFE(tr->Elem1No),
-                                  *fes->GetFE(tr->Elem2No),
-                                  *tr, elemmat);
+               fbfi[k] -> AssembleFaceMatrix (*fes -> GetFE (tr -> Elem1No),
+                                              *fes -> GetFE (tr -> Elem2No),
+                                              *tr, elemmat);
               mat -> AddSubMatrix (vdofs, vdofs, elemmat, skip_zeros);
            }
         }
      }
   }

-   if (boundary_face_integs.Size())
+   if (bfbfi.Size())
   {
      FaceElementTransformations *tr;
      const FiniteElement *fe1, *fe2;
@@ -584,14 +533,14 @@ void BilinearForm::Assemble(int skip_zeros)
      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
                                 mesh->bdr_attributes.Max() : 0);
      bdr_attr_marker = 0;
-      for (int k = 0; k < boundary_face_integs.Size(); k++)
+      for (int k = 0; k < bfbfi.Size(); k++)
      {
-         if (boundary_face_integs_marker[k] == NULL)
+         if (bfbfi_marker[k] == NULL)
         {
            bdr_attr_marker = 1;
            break;
         }
-         Array<int> &bdr_marker = *boundary_face_integs_marker[k];
+         Array<int> &bdr_marker = *bfbfi_marker[k];
         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
                     "invalid boundary marker for boundary face integrator #"
                     << k << ", counting from zero");
@@ -615,14 +564,12 @@ void BilinearForm::Assemble(int skip_zeros)
            // but we can't dereference a NULL pointer, and we don't want to
            // actually make a fake element.
            fe2 = fe1;
-            for (int k = 0; k < boundary_face_integs.Size(); k++)
+            for (int k = 0; k < bfbfi.Size(); k++)
            {
-               if (boundary_face_integs_marker[k] &&
-                   (*boundary_face_integs_marker[k])[bdr_attr-1] == 0)
-               { continue; }
+               if (bfbfi_marker[k] &&
+                   (*bfbfi_marker[k])[bdr_attr-1] == 0) { continue; }

-               boundary_face_integs[k] -> AssembleFaceMatrix (*fe1, *fe2, *tr,
-                                                              elemmat);
+               bfbfi[k] -> AssembleFaceMatrix (*fe1, *fe2, *tr, elemmat);
               mat -> AddSubMatrix (vdofs, vdofs, elemmat, skip_zeros);
            }
         }
@@ -736,8 +683,8 @@ void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
      {
         // A, X and B point to the same data as mat, x and b
         EliminateVDofsInRHS(ess_tdof_list, x, b);
-         X.MakeRef(x, 0, x.Size());
-         B.MakeRef(b, 0, b.Size());
+         X.NewMemoryAndSize(x.GetMemory(), x.Size(), false);
+         B.NewMemoryAndSize(b.GetMemory(), b.Size(), false);
         if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
      }
   }
@@ -873,7 +820,7 @@ void BilinearForm::RecoverFEMSolution(const Vector &X,

 void BilinearForm::ComputeElementMatrices()
 {
-   if (element_matrices || domain_integs.Size() == 0 || fes->GetNE() == 0)
+   if (element_matrices || dbfi.Size() == 0 || fes->GetNE() == 0)
   {
      return;
   }
@@ -902,11 +849,11 @@ void BilinearForm::ComputeElementMatrices()
 #endif
      fes->GetElementTransformation(i, &eltrans);

-      domain_integs[0]->AssembleElementMatrix(fe, eltrans, elmat);
-      for (int k = 1; k < domain_integs.Size(); k++)
+      dbfi[0]->AssembleElementMatrix(fe, eltrans, elmat);
+      for (int k = 1; k < dbfi.Size(); k++)
      {
         // note: some integrators may not be thread-safe
-         domain_integs[k]->AssembleElementMatrix(fe, eltrans, tmp);
+         dbfi[k]->AssembleElementMatrix(fe, eltrans, tmp);
         elmat += tmp;
      }
      elmat.ClearExternalData();
@@ -1121,12 +1068,10 @@ BilinearForm::~BilinearForm()
   if (!extern_bfs)
   {
      int k;
-      for (k=0; k < domain_integs.Size(); k++) { delete domain_integs[k]; }
-      for (k=0; k < boundary_integs.Size(); k++) { delete boundary_integs[k]; }
-      for (k=0; k < interior_face_integs.Size(); k++)
-      { delete interior_face_integs[k]; }
-      for (k=0; k < boundary_face_integs.Size(); k++)
-      { delete boundary_face_integs[k]; }
+      for (k=0; k < dbfi.Size(); k++) { delete dbfi[k]; }
+      for (k=0; k < bbfi.Size(); k++) { delete bbfi[k]; }
+      for (k=0; k < fbfi.Size(); k++) { delete fbfi[k]; }
+      for (k=0; k < bfbfi.Size(); k++) { delete bfbfi[k]; }
   }

   delete ext;
@@ -1159,13 +1104,13 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
   ext = NULL;

   // Copy the pointers to the integrators
-   domain_integs = mbf->domain_integs;
-   boundary_integs = mbf->boundary_integs;
-   trace_face_integs = mbf->trace_face_integs;
-   boundary_trace_face_integs = mbf->boundary_trace_face_integs;
+   dbfi = mbf->dbfi;
+   bbfi = mbf->bbfi;
+   tfbfi = mbf->tfbfi;
+   btfbfi = mbf->btfbfi;

-   boundary_integs_marker = mbf->boundary_integs_marker;
-   boundary_trace_face_integs_marker = mbf->boundary_trace_face_integs_marker;
+   bbfi_marker = mbf->bbfi_marker;
+   btfbfi_marker = mbf->btfbfi_marker;

   assembly = AssemblyLevel::LEGACY;
   ext = NULL;
@@ -1254,8 +1199,7 @@ MatrixInverse * MixedBilinearForm::Inverse() const
 {
   if (assembly != AssemblyLevel::LEGACY)
   {
-      MFEM_WARNING("MixedBilinearForm::Inverse not possible with this "
-                   "assembly level!");
+      MFEM_WARNING("MixedBilinearForm::Inverse not possible with this assembly level!");
      return NULL;
   }
   else
@@ -1286,39 +1230,38 @@ void MixedBilinearForm::GetBlocks(Array2D<SparseMatrix *> &blocks) const

 void MixedBilinearForm::AddDomainIntegrator (BilinearFormIntegrator * bfi)
 {
-   domain_integs.Append (bfi);
+   dbfi.Append (bfi);
 }

 void MixedBilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi)
 {
-   boundary_integs.Append (bfi);
-   boundary_integs_marker.Append(NULL); // NULL marker means apply everywhere
+   bbfi.Append (bfi);
+   bbfi_marker.Append(NULL); // NULL marker means apply everywhere
 }

 void MixedBilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi,
                                               Array<int> &bdr_marker)
 {
-   boundary_integs.Append (bfi);
-   boundary_integs_marker.Append(&bdr_marker);
+   bbfi.Append (bfi);
+   bbfi_marker.Append(&bdr_marker);
 }

 void MixedBilinearForm::AddTraceFaceIntegrator (BilinearFormIntegrator * bfi)
 {
-   trace_face_integs.Append (bfi);
+   tfbfi.Append (bfi);
 }

 void MixedBilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator *bfi)
 {
-   boundary_trace_face_integs.Append(bfi);
-   // NULL marker means apply everywhere
-   boundary_trace_face_integs_marker.Append(NULL);
+   btfbfi.Append(bfi);
+   btfbfi_marker.Append(NULL); // NULL marker means apply everywhere
 }

 void MixedBilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator *bfi,
                                                  Array<int> &bdr_marker)
 {
-   boundary_trace_face_integs.Append(bfi);
-   boundary_trace_face_integs_marker.Append(&bdr_marker);
+   btfbfi.Append(bfi);
+   btfbfi_marker.Append(&bdr_marker);
 }

 void MixedBilinearForm::Assemble (int skip_zeros)
@@ -1329,10 +1272,9 @@ void MixedBilinearForm::Assemble (int skip_zeros)
      return;
   }

+   Array<int> tr_vdofs, te_vdofs;
   ElementTransformation *eltrans;
-   DofTransformation * dom_dof_trans;
-   DofTransformation * ran_dof_trans;
-   DenseMatrix elmat;
+   DenseMatrix elemmat;

   Mesh *mesh = test_fes -> GetMesh();

@@ -1341,45 +1283,37 @@ void MixedBilinearForm::Assemble (int skip_zeros)
      mat = new SparseMatrix(height, width);
   }

-   if (domain_integs.Size())
+   if (dbfi.Size())
   {
      for (int i = 0; i < test_fes -> GetNE(); i++)
      {
-         dom_dof_trans = trial_fes -> GetElementVDofs (i, trial_vdofs);
-         ran_dof_trans = test_fes  -> GetElementVDofs (i, test_vdofs);
+         trial_fes -> GetElementVDofs (i, tr_vdofs);
+         test_fes  -> GetElementVDofs (i, te_vdofs);
         eltrans = test_fes -> GetElementTransformation (i);
-
-         elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
-         elmat = 0.0;
-         for (int k = 0; k < domain_integs.Size(); k++)
+         for (int k = 0; k < dbfi.Size(); k++)
         {
-            domain_integs[k] -> AssembleElementMatrix2 (*trial_fes -> GetFE(i),
-                                                        *test_fes  -> GetFE(i),
-                                                        *eltrans, elemmat);
-            elmat += elemmat;
+            dbfi[k] -> AssembleElementMatrix2 (*trial_fes -> GetFE(i),
+                                               *test_fes  -> GetFE(i),
+                                               *eltrans, elemmat);
+            mat -> AddSubMatrix (te_vdofs, tr_vdofs, elemmat, skip_zeros);
         }
-         if (ran_dof_trans || dom_dof_trans)
-         {
-            TransformDual(ran_dof_trans, dom_dof_trans, elmat);
-         }
-         mat -> AddSubMatrix (test_vdofs, trial_vdofs, elmat, skip_zeros);
      }
   }

-   if (boundary_integs.Size())
+   if (bbfi.Size())
   {
      // Which boundary attributes need to be processed?
      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
                                 mesh->bdr_attributes.Max() : 0);
      bdr_attr_marker = 0;
-      for (int k = 0; k < boundary_integs.Size(); k++)
+      for (int k = 0; k < bbfi.Size(); k++)
      {
-         if (boundary_integs_marker[k] == NULL)
+         if (bbfi_marker[k] == NULL)
         {
            bdr_attr_marker = 1;
            break;
         }
-         Array<int> &bdr_marker = *boundary_integs_marker[k];
+         Array<int> &bdr_marker = *bbfi_marker[k];
         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
                     "invalid boundary marker for boundary integrator #"
                     << k << ", counting from zero");
@@ -1394,48 +1328,40 @@ void MixedBilinearForm::Assemble (int skip_zeros)
         const int bdr_attr = mesh->GetBdrAttribute(i);
         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }

-         dom_dof_trans = trial_fes -> GetBdrElementVDofs (i, trial_vdofs);
-         ran_dof_trans = test_fes  -> GetBdrElementVDofs (i, test_vdofs);
+         trial_fes -> GetBdrElementVDofs (i, tr_vdofs);
+         test_fes  -> GetBdrElementVDofs (i, te_vdofs);
         eltrans = test_fes -> GetBdrElementTransformation (i);
-
-         elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
-         elmat = 0.0;
-         for (int k = 0; k < boundary_integs.Size(); k++)
+         for (int k = 0; k < bbfi.Size(); k++)
         {
-            if (boundary_integs_marker[k] &&
-                (*boundary_integs_marker[k])[bdr_attr-1] == 0) { continue; }
+            if (bbfi_marker[k] &&
+                (*bbfi_marker[k])[bdr_attr-1] == 0) { continue; }

-            boundary_integs[k]->AssembleElementMatrix2 (*trial_fes -> GetBE(i),
-                                                        *test_fes  -> GetBE(i),
-                                                        *eltrans, elemmat);
-            elmat += elemmat;
+            bbfi[k] -> AssembleElementMatrix2 (*trial_fes -> GetBE(i),
+                                               *test_fes  -> GetBE(i),
+                                               *eltrans, elemmat);
+            mat -> AddSubMatrix (te_vdofs, tr_vdofs, elemmat, skip_zeros);
         }
-         if (ran_dof_trans || dom_dof_trans)
-         {
-            TransformDual(ran_dof_trans, dom_dof_trans, elmat);
-         }
-         mat -> AddSubMatrix (test_vdofs, trial_vdofs, elmat, skip_zeros);
      }
   }

-   if (trace_face_integs.Size())
+   if (tfbfi.Size())
   {
      FaceElementTransformations *ftr;
-      Array<int> test_vdofs2;
+      Array<int> te_vdofs2;
      const FiniteElement *trial_face_fe, *test_fe1, *test_fe2;

      int nfaces = mesh->GetNumFaces();
      for (int i = 0; i < nfaces; i++)
      {
         ftr = mesh->GetFaceElementTransformations(i);
-         trial_fes->GetFaceVDofs(i, trial_vdofs);
-         test_fes->GetElementVDofs(ftr->Elem1No, test_vdofs);
+         trial_fes->GetFaceVDofs(i, tr_vdofs);
+         test_fes->GetElementVDofs(ftr->Elem1No, te_vdofs);
         trial_face_fe = trial_fes->GetFaceElement(i);
         test_fe1 = test_fes->GetFE(ftr->Elem1No);
         if (ftr->Elem2No >= 0)
         {
-            test_fes->GetElementVDofs(ftr->Elem2No, test_vdofs2);
-            test_vdofs.Append(test_vdofs2);
+            test_fes->GetElementVDofs(ftr->Elem2No, te_vdofs2);
+            te_vdofs.Append(te_vdofs2);
            test_fe2 = test_fes->GetFE(ftr->Elem2No);
         }
         else
@@ -1445,16 +1371,16 @@ void MixedBilinearForm::Assemble (int skip_zeros)
            // want to actually make a fake element.
            test_fe2 = test_fe1;
         }
-         for (int k = 0; k < trace_face_integs.Size(); k++)
+         for (int k = 0; k < tfbfi.Size(); k++)
         {
-            trace_face_integs[k]->AssembleFaceMatrix(*trial_face_fe, *test_fe1,
-                                                     *test_fe2, *ftr, elemmat);
-            mat->AddSubMatrix(test_vdofs, trial_vdofs, elemmat, skip_zeros);
+            tfbfi[k]->AssembleFaceMatrix(*trial_face_fe, *test_fe1, *test_fe2,
+                                         *ftr, elemmat);
+            mat->AddSubMatrix(te_vdofs, tr_vdofs, elemmat, skip_zeros);
         }
      }
   }

-   if (boundary_trace_face_integs.Size())
+   if (btfbfi.Size())
   {
      FaceElementTransformations *ftr;
      Array<int> te_vdofs2;
@@ -1464,17 +1390,17 @@ void MixedBilinearForm::Assemble (int skip_zeros)
      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
                                 mesh->bdr_attributes.Max() : 0);
      bdr_attr_marker = 0;
-      for (int k = 0; k < boundary_trace_face_integs.Size(); k++)
+      for (int k = 0; k < btfbfi.Size(); k++)
      {
-         if (boundary_trace_face_integs_marker[k] == NULL)
+         if (btfbfi_marker[k] == NULL)
         {
            bdr_attr_marker = 1;
            break;
         }
-         Array<int> &bdr_marker = *boundary_trace_face_integs_marker[k];
+         Array<int> &bdr_marker = *btfbfi_marker[k];
         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
-                     "invalid boundary marker for boundary trace face"
-                     "integrator #" << k << ", counting from zero");
+                     "invalid boundary marker for boundary trace face integrator #"
+                     << k << ", counting from zero");
         for (int i = 0; i < bdr_attr_marker.Size(); i++)
         {
            bdr_attr_marker[i] |= bdr_marker[i];
@@ -1489,25 +1415,22 @@ void MixedBilinearForm::Assemble (int skip_zeros)
         ftr = mesh->GetBdrFaceTransformations(i);
         if (ftr)
         {
-            trial_fes->GetFaceVDofs(ftr->ElementNo, trial_vdofs);
-            test_fes->GetElementVDofs(ftr->Elem1No, test_vdofs);
+            trial_fes->GetFaceVDofs(ftr->ElementNo, tr_vdofs);
+            test_fes->GetElementVDofs(ftr->Elem1No, te_vdofs);
            trial_face_fe = trial_fes->GetFaceElement(ftr->ElementNo);
            test_fe1 = test_fes->GetFE(ftr->Elem1No);
            // The test_fe2 object is really a dummy and not used on the
            // boundaries, but we can't dereference a NULL pointer, and we don't
            // want to actually make a fake element.
            test_fe2 = test_fe1;
-            for (int k = 0; k < boundary_trace_face_integs.Size(); k++)
+            for (int k = 0; k < btfbfi.Size(); k++)
            {
-               if (boundary_trace_face_integs_marker[k] &&
-                   (*boundary_trace_face_integs_marker[k])[bdr_attr-1] == 0)
-               { continue; }
+               if (btfbfi_marker[k] &&
+                   (*btfbfi_marker[k])[bdr_attr-1] == 0) { continue; }

-               boundary_trace_face_integs[k]->AssembleFaceMatrix(*trial_face_fe,
-                                                                 *test_fe1,
-                                                                 *test_fe2,
-                                                                 *ftr, elemmat);
-               mat->AddSubMatrix(test_vdofs, trial_vdofs, elemmat, skip_zeros);
+               btfbfi[k]->AssembleFaceMatrix(*trial_face_fe, *test_fe1, *test_fe2,
+                                             *ftr, elemmat);
+               mat->AddSubMatrix(te_vdofs, tr_vdofs, elemmat, skip_zeros);
            }
         }
      }
@@ -1597,17 +1520,15 @@ void MixedBilinearForm::ConformingAssemble()

 void MixedBilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
 {
-   if (domain_integs.Size())
+   if (dbfi.Size())
   {
      const FiniteElement &trial_fe = *trial_fes->GetFE(i);
      const FiniteElement &test_fe = *test_fes->GetFE(i);
      ElementTransformation *eltrans = test_fes->GetElementTransformation(i);
-      domain_integs[0]->AssembleElementMatrix2(trial_fe, test_fe, *eltrans,
-                                               elmat);
-      for (int k = 1; k < domain_integs.Size(); k++)
+      dbfi[0]->AssembleElementMatrix2(trial_fe, test_fe, *eltrans, elmat);
+      for (int k = 1; k < dbfi.Size(); k++)
      {
-         domain_integs[k]->AssembleElementMatrix2(trial_fe, test_fe, *eltrans,
-                                                  elemmat);
+         dbfi[k]->AssembleElementMatrix2(trial_fe, test_fe, *eltrans, elemmat);
         elmat += elemmat;
      }
   }
@@ -1622,17 +1543,15 @@ void MixedBilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)

 void MixedBilinearForm::ComputeBdrElementMatrix(int i, DenseMatrix &elmat)
 {
-   if (boundary_integs.Size())
+   if (bbfi.Size())
   {
      const FiniteElement &trial_be = *trial_fes->GetBE(i);
      const FiniteElement &test_be = *test_fes->GetBE(i);
      ElementTransformation *eltrans = test_fes->GetBdrElementTransformation(i);
-      boundary_integs[0]->AssembleElementMatrix2(trial_be, test_be, *eltrans,
-                                                 elmat);
-      for (int k = 1; k < boundary_integs.Size(); k++)
+      bbfi[0]->AssembleElementMatrix2(trial_be, test_be, *eltrans, elmat);
+      for (int k = 1; k < bbfi.Size(); k++)
      {
-         boundary_integs[k]->AssembleElementMatrix2(trial_be, test_be, *eltrans,
-                                                    elemmat);
+         bbfi[k]->AssembleElementMatrix2(trial_be, test_be, *eltrans, elemmat);
         elmat += elemmat;
      }
   }
@@ -1732,10 +1651,10 @@ void MixedBilinearForm::EliminateTestDofs (const Array<int> &bdr_attr_is_ess)
      }
 }

-void MixedBilinearForm::FormRectangularSystemMatrix(
-   const Array<int> &trial_tdof_list,
-   const Array<int> &test_tdof_list,
-   OperatorHandle &A)
+void MixedBilinearForm::FormRectangularSystemMatrix(const Array<int>
+                                                    &trial_tdof_list,
+                                                    const Array<int> &test_tdof_list,
+                                                    OperatorHandle &A)

 {
   if (ext)
@@ -1773,17 +1692,17 @@ void MixedBilinearForm::FormRectangularSystemMatrix(
   A.Reset(mat, false);
 }

-void MixedBilinearForm::FormRectangularLinearSystem(
-   const Array<int> &trial_tdof_list,
-   const Array<int> &test_tdof_list,
-   Vector &x, Vector &b,
-   OperatorHandle &A,
-   Vector &X, Vector &B)
+void MixedBilinearForm::FormRectangularLinearSystem(const Array<int>
+                                                    &trial_tdof_list,
+                                                    const Array<int> &test_tdof_list,
+                                                    Vector &x, Vector &b,
+                                                    OperatorHandle &A,
+                                                    Vector &X, Vector &B)
 {
   if (ext)
   {
-      ext->FormRectangularLinearSystem(trial_tdof_list, test_tdof_list,
-                                       x, b, A, X, B);
+      ext->FormRectangularLinearSystem(trial_tdof_list, test_tdof_list, x, b, A, X,
+                                       B);
      return;
   }

@@ -1821,13 +1740,10 @@ MixedBilinearForm::~MixedBilinearForm()
   if (!extern_bfs)
   {
      int i;
-      for (i = 0; i < domain_integs.Size(); i++) { delete domain_integs[i]; }
-      for (i = 0; i < boundary_integs.Size(); i++)
-      { delete boundary_integs[i]; }
-      for (i = 0; i < trace_face_integs.Size(); i++)
-      { delete trace_face_integs[i]; }
-      for (i = 0; i < boundary_trace_face_integs.Size(); i++)
-      { delete boundary_trace_face_integs[i]; }
+      for (i = 0; i < dbfi.Size(); i++) { delete dbfi[i]; }
+      for (i = 0; i < bbfi.Size(); i++) { delete bbfi[i]; }
+      for (i = 0; i < tfbfi.Size(); i++) { delete tfbfi[i]; }
+      for (i = 0; i < btfbfi.Size(); i++) { delete btfbfi[i]; }
   }
   delete ext;
 }
@@ -1869,8 +1785,6 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)

   Array<int> dom_vdofs, ran_vdofs;
   ElementTransformation *T;
-   DofTransformation * dom_dof_trans;
-   DofTransformation * ran_dof_trans;
   const FiniteElement *dom_fe, *ran_fe;
   DenseMatrix totelmat, elmat;

@@ -1879,33 +1793,27 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
      mat = new SparseMatrix(height, width);
   }

-   if (domain_integs.Size() > 0)
+   if (dbfi.Size() > 0)
   {
      for (int i = 0; i < test_fes->GetNE(); i++)
      {
-         dom_dof_trans = trial_fes->GetElementVDofs(i, dom_vdofs);
-         ran_dof_trans = test_fes->GetElementVDofs(i, ran_vdofs);
+         trial_fes->GetElementVDofs(i, dom_vdofs);
+         test_fes->GetElementVDofs(i, ran_vdofs);
         T = test_fes->GetElementTransformation(i);
         dom_fe = trial_fes->GetFE(i);
         ran_fe = test_fes->GetFE(i);

-         domain_integs[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
-                                                  totelmat);
-         for (int j = 1; j < domain_integs.Size(); j++)
+         dbfi[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, totelmat);
+         for (int j = 1; j < dbfi.Size(); j++)
         {
-            domain_integs[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
-                                                     elmat);
+            dbfi[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, elmat);
            totelmat += elmat;
         }
-         if (ran_dof_trans || dom_dof_trans)
-         {
-            TransformPrimal(ran_dof_trans, dom_dof_trans, totelmat);
-         }
         mat->SetSubMatrix(ran_vdofs, dom_vdofs, totelmat, skip_zeros);
      }
   }

-   if (trace_face_integs.Size())
+   if (tfbfi.Size())
   {
      const int nfaces = test_fes->GetMesh()->GetNumFaces();
      for (int i = 0; i < nfaces; i++)
@@ -1916,12 +1824,10 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
         dom_fe = trial_fes->GetFaceElement(i);
         ran_fe = test_fes->GetFaceElement(i);

-         trace_face_integs[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
-                                                      totelmat);
-         for (int j = 1; j < trace_face_integs.Size(); j++)
+         tfbfi[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, totelmat);
+         for (int j = 1; j < tfbfi.Size(); j++)
         {
-            trace_face_integs[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
-                                                         elmat);
+            tfbfi[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, elmat);
            totelmat += elmat;
         }
         mat->SetSubMatrix(ran_vdofs, dom_vdofs, totelmat, skip_zeros);
@@ -84,29 +84,23 @@ protected:
       the BilinearForm. */
   long sequence;

-   /** @brief Indicates the BilinearFormIntegrator%s stored in #domain_integs,
-       #boundary_integs, #interior_face_integs, and #boundary_face_integs are
-       owned by another BilinearForm. */
+   /** @brief Indicates the BilinearFormIntegrator%s stored in #dbfi, #bbfi,
+       #fbfi, and #bfbfi are owned by another BilinearForm. */
   int extern_bfs;

   /// Set of Domain Integrators to be applied.
-   Array<BilinearFormIntegrator*> domain_integs;
-   /// Element attribute marker (should be of length mesh->attributes)
-   /// Includes all by default.
-   /// 0 - ignore attribute
-   /// 1 - include attribute
-   Array<Array<int>*>             domain_integs_marker;
+   Array<BilinearFormIntegrator*> dbfi;

   /// Set of Boundary Integrators to be applied.
-   Array<BilinearFormIntegrator*> boundary_integs;
-   Array<Array<int>*> boundary_integs_marker; ///< Entries are not owned.
+   Array<BilinearFormIntegrator*> bbfi;
+   Array<Array<int>*>             bbfi_marker; ///< Entries are not owned.

   /// Set of interior face Integrators to be applied.
-   Array<BilinearFormIntegrator*> interior_face_integs;
+   Array<BilinearFormIntegrator*> fbfi;

   /// Set of boundary face Integrators to be applied.
-   Array<BilinearFormIntegrator*> boundary_face_integs;
-   Array<Array<int>*> boundary_face_integs_marker; ///< Entries are not owned.
+   Array<BilinearFormIntegrator*> bfbfi;
+   Array<Array<int>*>             bfbfi_marker; ///< Entries are not owned.

   DenseMatrix elemmat;
   Array<int>  vdofs;
@@ -232,25 +226,24 @@ public:
   void AllocateMatrix() { if (mat == NULL) { AllocMat(); } }

   /// Access all the integrators added with AddDomainIntegrator().
-   Array<BilinearFormIntegrator*> *GetDBFI() { return &domain_integs; }
+   Array<BilinearFormIntegrator*> *GetDBFI() { return &dbfi; }

   /// Access all the integrators added with AddBoundaryIntegrator().
-   Array<BilinearFormIntegrator*> *GetBBFI() { return &boundary_integs; }
+   Array<BilinearFormIntegrator*> *GetBBFI() { return &bbfi; }
   /** @brief Access all boundary markers added with AddBoundaryIntegrator().
       If no marker was specified when the integrator was added, the
       corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetBBFI_Marker() { return &boundary_integs_marker; }
+   Array<Array<int>*> *GetBBFI_Marker() { return &bbfi_marker; }

   /// Access all integrators added with AddInteriorFaceIntegrator().
-   Array<BilinearFormIntegrator*> *GetFBFI() { return &interior_face_integs; }
+   Array<BilinearFormIntegrator*> *GetFBFI() { return &fbfi; }

   /// Access all integrators added with AddBdrFaceIntegrator().
-   Array<BilinearFormIntegrator*> *GetBFBFI() { return &boundary_face_integs; }
+   Array<BilinearFormIntegrator*> *GetBFBFI() { return &bfbfi; }
   /** @brief Access all boundary markers added with AddBdrFaceIntegrator().
       If no marker was specified when the integrator was added, the
       corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetBFBFI_Marker()
-   { return &boundary_face_integs_marker; }
+   Array<Array<int>*> *GetBFBFI_Marker() { return &bfbfi_marker; }

   /// Returns a reference to: \f$ M_{ij} \f$
   const double &operator()(int i, int j) { return (*mat)(i,j); }
@@ -339,10 +332,6 @@ public:

   /// Adds new Domain Integrator. Assumes ownership of @a bfi.
   void AddDomainIntegrator(BilinearFormIntegrator *bfi);
-   /// Adds new Domain Integrator restricted to certain elements specified by
-   /// the @a elem_attr_marker.
-   void AddDomainIntegrator(BilinearFormIntegrator *bfi,
-                            Array<int> &elem_marker);

   /// Adds new Boundary Integrator. Assumes ownership of @a bfi.
   void AddBoundaryIntegrator(BilinearFormIntegrator *bfi);
@@ -654,25 +643,23 @@ protected:
       Partial Assembly (PA), or Matrix Free assembly (MF). */
   MixedBilinearFormExtension *ext;

-   /** @brief Indicates the BilinearFormIntegrator%s stored in #domain_integs,
-       #boundary_integs, #trace_face_integs and #boundary_trace_face_integs
-       are owned by another MixedBilinearForm. */
+   /** @brief Indicates the BilinearFormIntegrator%s stored in #dbfi, #bbfi,
+       #tfbfi and #btfbfi are owned by another MixedBilinearForm. */
   int extern_bfs;

   /// Domain integrators.
-   Array<BilinearFormIntegrator*> domain_integs;
+   Array<BilinearFormIntegrator*> dbfi;

   /// Boundary integrators.
-   Array<BilinearFormIntegrator*> boundary_integs;
-   Array<Array<int>*> boundary_integs_marker; ///< Entries are not owned.
+   Array<BilinearFormIntegrator*> bbfi;
+   Array<Array<int>*>             bbfi_marker;///< Entries are not owned.

   /// Trace face (skeleton) integrators.
-   Array<BilinearFormIntegrator*> trace_face_integs;
+   Array<BilinearFormIntegrator*> tfbfi;

   /// Boundary trace face (skeleton) integrators.
-   Array<BilinearFormIntegrator*> boundary_trace_face_integs;
-   /// Entries are not owned.
-   Array<Array<int>*> boundary_trace_face_integs_marker;
+   Array<BilinearFormIntegrator*> btfbfi;
+   Array<Array<int>*>             btfbfi_marker;///< Entries are not owned.

   DenseMatrix elemmat;
   Array<int>  trial_vdofs, test_vdofs;
@@ -766,26 +753,24 @@ public:
                                   Array<int> &bdr_marker);

   /// Access all integrators added with AddDomainIntegrator().
-   Array<BilinearFormIntegrator*> *GetDBFI() { return &domain_integs; }
+   Array<BilinearFormIntegrator*> *GetDBFI() { return &dbfi; }

   /// Access all integrators added with AddBoundaryIntegrator().
-   Array<BilinearFormIntegrator*> *GetBBFI() { return &boundary_integs; }
+   Array<BilinearFormIntegrator*> *GetBBFI() { return &bbfi; }
   /** @brief Access all boundary markers added with AddBoundaryIntegrator().
       If no marker was specified when the integrator was added, the
       corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetBBFI_Marker() { return &boundary_integs_marker; }
+   Array<Array<int>*> *GetBBFI_Marker() { return &bbfi_marker; }

   /// Access all integrators added with AddTraceFaceIntegrator().
-   Array<BilinearFormIntegrator*> *GetTFBFI() { return &trace_face_integs; }
+   Array<BilinearFormIntegrator*> *GetTFBFI() { return &tfbfi; }

   /// Access all integrators added with AddBdrTraceFaceIntegrator().
-   Array<BilinearFormIntegrator*> *GetBTFBFI()
-   { return &boundary_trace_face_integs; }
+   Array<BilinearFormIntegrator*> *GetBTFBFI() { return &btfbfi; }
   /** @brief Access all boundary markers added with AddBdrTraceFaceIntegrator().
       If no marker was specified when the integrator was added, the
       corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetBTFBFI_Marker()
-   { return &boundary_trace_face_integs_marker; }
+   Array<Array<int>*> *GetBTFBFI_Marker() { return &btfbfi_marker; }

   /// Sets all sparse values of \f$ M \f$ to @a a.
   void operator=(const double a) { *mat = a; }
@@ -1010,7 +995,7 @@ public:
   { AddTraceFaceIntegrator(di); }

   /// Access all interpolators added with AddDomainInterpolator().
-   Array<BilinearFormIntegrator*> *GetDI() { return &domain_integs; }
+   Array<BilinearFormIntegrator*> *GetDI() { return &dbfi; }

   /// Set the desired assembly level. The default is AssemblyLevel::FULL.
   /** This method must be called before assembly. */
@@ -160,7 +160,7 @@ void MFBilinearFormExtension::Mult(const Vector &x, Vector &y) const
         {
            intFaceIntegrators[i]->AddMultMF(faceIntX, faceIntY);
         }
-         int_face_restrict_lex->AddMultTranspose(faceIntY, y);
+         int_face_restrict_lex->MultTranspose(faceIntY, y);
      }
   }

@@ -176,7 +176,7 @@ void MFBilinearFormExtension::Mult(const Vector &x, Vector &y) const
         {
            bdrFaceIntegrators[i]->AddMultMF(faceBdrX, faceBdrY);
         }
-         bdr_face_restrict_lex->AddMultTranspose(faceBdrY, y);
+         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
      }
   }
 }
@@ -217,7 +217,7 @@ void MFBilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         {
            intFaceIntegrators[i]->AddMultTransposeMF(faceIntX, faceIntY);
         }
-         int_face_restrict_lex->AddMultTranspose(faceIntY, y);
+         int_face_restrict_lex->MultTranspose(faceIntY, y);
      }
   }

@@ -233,7 +233,7 @@ void MFBilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         {
            bdrFaceIntegrators[i]->AddMultTransposeMF(faceBdrX, faceBdrY);
         }
-         bdr_face_restrict_lex->AddMultTranspose(faceBdrY, y);
+         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
      }
   }
 }
@@ -417,7 +417,7 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
         {
            intFaceIntegrators[i]->AddMultPA(faceIntX, faceIntY);
         }
-         int_face_restrict_lex->AddMultTranspose(faceIntY, y);
+         int_face_restrict_lex->MultTranspose(faceIntY, y);
      }
   }

@@ -433,7 +433,7 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
         {
            bdrFaceIntegrators[i]->AddMultPA(faceBdrX, faceBdrY);
         }
-         bdr_face_restrict_lex->AddMultTranspose(faceBdrY, y);
+         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
      }
   }
 }
@@ -474,7 +474,7 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         {
            intFaceIntegrators[i]->AddMultTransposePA(faceIntX, faceIntY);
         }
-         int_face_restrict_lex->AddMultTranspose(faceIntY, y);
+         int_face_restrict_lex->MultTranspose(faceIntY, y);
      }
   }

@@ -490,7 +490,7 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         {
            bdrFaceIntegrators[i]->AddMultTransposePA(faceBdrX, faceBdrY);
         }
-         bdr_face_restrict_lex->AddMultTranspose(faceBdrY, y);
+         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
      }
   }
 }
@@ -657,7 +657,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
            Y(j, 0, f) += res;
         });
         // Apply the Interior Face Restriction transposed
-         int_face_restrict_lex->AddMultTranspose(faceIntY, y);
+         int_face_restrict_lex->MultTranspose(faceIntY, y);
      }
   }

@@ -688,7 +688,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
            Y(j, f) += res;
         });
         // Apply the Boundary Face Restriction transposed
-         bdr_face_restrict_lex->AddMultTranspose(faceBdrY, y);
+         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
      }
   }
 }
@@ -783,7 +783,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
            Y(j, 0, f) += res;
         });
         // Apply the Interior Face Restriction transposed
-         int_face_restrict_lex->AddMultTranspose(faceIntY, y);
+         int_face_restrict_lex->MultTranspose(faceIntY, y);
      }
   }

@@ -814,7 +814,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
            Y(j, f) += res;
         });
         // Apply the Boundary Face Restriction transposed
-         bdr_face_restrict_lex->AddMultTranspose(faceBdrY, y);
+         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
      }
   }
 }
@@ -72,8 +72,8 @@ protected:
   mutable Vector faceIntX, faceIntY;
   mutable Vector faceBdrX, faceBdrY;
   const Operator *elem_restrict; // Not owned
-   const FaceRestriction *int_face_restrict_lex; // Not owned
-   const FaceRestriction *bdr_face_restrict_lex; // Not owned
+   const Operator *int_face_restrict_lex; // Not owned
+   const Operator *bdr_face_restrict_lex; // Not owned

 public:
   PABilinearFormExtension(BilinearForm*);
@@ -143,8 +143,8 @@ protected:
   mutable Vector faceIntX, faceIntY;
   mutable Vector faceBdrX, faceBdrY;
   const Operator *elem_restrict; // Not owned
-   const FaceRestriction *int_face_restrict_lex; // Not owned
-   const FaceRestriction *bdr_face_restrict_lex; // Not owned
+   const Operator *int_face_restrict_lex; // Not owned
+   const Operator *bdr_face_restrict_lex; // Not owned

 public:
   MFBilinearFormExtension(BilinearForm *form);
@@ -175,11 +175,6 @@ void BilinearFormIntegrator::AssembleFaceVector(
   elmat.Mult(elfun, elvect);
 }

-void TransposeIntegrator::SetIntRule(const IntegrationRule *ir)
-{
-   IntRule = ir;
-   bfi->SetIntRule(ir);
-}

 void TransposeIntegrator::AssembleElementMatrix (
   const FiniteElement &el, ElementTransformation &Trans, DenseMatrix &elmat)
@@ -207,12 +202,6 @@ void TransposeIntegrator::AssembleFaceMatrix (
   elmat.Transpose (bfi_elmat);
 }

-void LumpedIntegrator::SetIntRule(const IntegrationRule *ir)
-{
-   IntRule = ir;
-   bfi->SetIntRule(ir);
-}
-
 void LumpedIntegrator::AssembleElementMatrix (
   const FiniteElement &el, ElementTransformation &Trans, DenseMatrix &elmat)
 {
@@ -220,12 +209,6 @@ void LumpedIntegrator::AssembleElementMatrix (
   elmat.Lump();
 }

-void InverseIntegrator::SetIntRule(const IntegrationRule *ir)
-{
-   IntRule = ir;
-   integrator->SetIntRule(ir);
-}
-
 void InverseIntegrator::AssembleElementMatrix(
   const FiniteElement &el, ElementTransformation &Trans, DenseMatrix &elmat)
 {
@@ -233,15 +216,6 @@ void InverseIntegrator::AssembleElementMatrix(
   elmat.Invert();
 }

-void SumIntegrator::SetIntRule(const IntegrationRule *ir)
-{
-   IntRule = ir;
-   for (int i = 0; i < integrators.Size(); i++)
-   {
-      integrators[i]->SetIntRule(ir);
-   }
-}
-
 void SumIntegrator::AssembleElementMatrix(
   const FiniteElement &el, ElementTransformation &Trans, DenseMatrix &elmat)
 {
@@ -1777,16 +1751,15 @@ void DerivativeIntegrator::AssembleElementMatrix2 (
   int dim = trial_fe.GetDim();
   int trial_nd = trial_fe.GetDof();
   int test_nd = test_fe.GetDof();
-   int spaceDim = Trans.GetSpaceDim();

   int i, l;
   double det;

   elmat.SetSize (test_nd,trial_nd);
   dshape.SetSize (trial_nd,dim);
-   dshapedxt.SetSize(trial_nd, spaceDim);
+   dshapedxt.SetSize(trial_nd,dim);
   dshapedxi.SetSize(trial_nd);
-   invdfdx.SetSize(dim, spaceDim);
+   invdfdx.SetSize(dim);
   shape.SetSize (test_nd);

   const IntegrationRule *ir = IntRule;
@@ -21,11 +21,7 @@ namespace mfem

 // Local maximum size of dofs and quads in 1D
 constexpr int HCURL_MAX_D1D = 5;
-#ifdef MFEM_USE_HIP
-constexpr int HCURL_MAX_Q1D = 5;
-#else
 constexpr int HCURL_MAX_Q1D = 6;
-#endif

 constexpr int HDIV_MAX_D1D = 5;
 constexpr int HDIV_MAX_Q1D = 6;
@@ -261,8 +257,6 @@ public:
   TransposeIntegrator (BilinearFormIntegrator *bfi_, int own_bfi_ = 1)
   { bfi = bfi_; own_bfi = own_bfi_; }

-   virtual void SetIntRule(const IntegrationRule *ir);
-
   virtual void AssembleElementMatrix(const FiniteElement &el,
                                      ElementTransformation &Trans,
                                      DenseMatrix &elmat);
@@ -330,8 +324,6 @@ public:
   LumpedIntegrator (BilinearFormIntegrator *bfi_, int own_bfi_ = 1)
   { bfi = bfi_; own_bfi = own_bfi_; }

-   virtual void SetIntRule(const IntegrationRule *ir);
-
   virtual void AssembleElementMatrix(const FiniteElement &el,
                                      ElementTransformation &Trans,
                                      DenseMatrix &elmat);
@@ -350,8 +342,6 @@ public:
   InverseIntegrator(BilinearFormIntegrator *integ, int own_integ = 1)
   { integrator = integ; own_integrator = own_integ; }

-   virtual void SetIntRule(const IntegrationRule *ir);
-
   virtual void AssembleElementMatrix(const FiniteElement &el,
                                      ElementTransformation &Trans,
                                      DenseMatrix &elmat);
@@ -370,8 +360,6 @@ private:
 public:
   SumIntegrator(int own_integs = 1) { own_integrators = own_integs; }

-   virtual void SetIntRule(const IntegrationRule *ir);
-
   void AddIntegrator(BilinearFormIntegrator *integ)
   { integrators.Append(integ); }

@@ -711,7 +699,7 @@ protected:
   {
      return "MixedScalarDerivativeIntegrator:  "
             "Trial and test spaces must both be scalar fields in 1D "
-             "and the trial space must implement CalcDShape.";
+             "and the trial space must implement CaldDShape.";
   }

   inline virtual void CalcTrialShape(const FiniteElement & trial_fe,
@@ -2936,11 +2924,11 @@ public:

    - F. Bassi and S. Rebay. A high order discontinuous Galerkin method for
      compressible turbulent flows. In B. Cockburn, G. E. Karniadakis, and
-      C.-W. Shu, editors, Discontinuous Galerkin Methods, pages 77-88. Springer
+      C.-W. Shu, editors, Discontinuous Galerkin Methods, pages 77–88. Springer
      Berlin Heidelberg, 2000.
    - D. N. Arnold, F. Brezzi, B. Cockburn, and L. D. Marini. Unified analysis
      of discontinuous Galerkin methods for elliptic problems. SIAM Journal on
-      Numerical Analysis, 39(5):1749-1779, 2002.
+      Numerical Analysis, 39(5):1749–1779, 2002.
 */
 class DGDiffusionBR2Integrator : public BilinearFormIntegrator
 {
@@ -17,14 +17,14 @@ namespace mfem
 {

 template<int T_D1D = 0, int T_Q1D = 0>
-void EAConvectionAssemble1D(const int NE,
-                            const Array<double> &b,
-                            const Array<double> &g,
-                            const Vector &padata,
-                            Vector &eadata,
-                            const bool add,
-                            const int d1d = 0,
-                            const int q1d = 0)
+static void EAConvectionAssemble1D(const int NE,
+                                   const Array<double> &b,
+                                   const Array<double> &g,
+                                   const Vector &padata,
+                                   Vector &eadata,
+                                   const bool add,
+                                   const int d1d = 0,
+                                   const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -69,14 +69,14 @@ void EAConvectionAssemble1D(const int NE,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-void EAConvectionAssemble2D(const int NE,
-                            const Array<double> &b,
-                            const Array<double> &g,
-                            const Vector &padata,
-                            Vector &eadata,
-                            const bool add,
-                            const int d1d = 0,
-                            const int q1d = 0)
+static void EAConvectionAssemble2D(const int NE,
+                                   const Array<double> &b,
+                                   const Array<double> &g,
+                                   const Vector &padata,
+                                   Vector &eadata,
+                                   const bool add,
+                                   const int d1d = 0,
+                                   const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -146,14 +146,14 @@ void EAConvectionAssemble2D(const int NE,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-void EAConvectionAssemble3D(const int NE,
-                            const Array<double> &b,
-                            const Array<double> &g,
-                            const Vector &padata,
-                            Vector &eadata,
-                            const bool add,
-                            const int d1d = 0,
-                            const int q1d = 0)
+static void EAConvectionAssemble3D(const int NE,
+                                   const Array<double> &b,
+                                   const Array<double> &g,
+                                   const Vector &padata,
+                                   Vector &eadata,
+                                   const bool add,
+                                   const int d1d = 0,
+                                   const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -21,13 +21,13 @@ namespace mfem
 // PA Convection Integrator

 // PA Convection Assemble 2D kernel
-void PAConvectionSetup2D(const int NQ,
-                         const int NE,
-                         const Array<double> &w,
-                         const Vector &j,
-                         const Vector &vel,
-                         const double alpha,
-                         Vector &op)
+static void PAConvectionSetup2D(const int NQ,
+                                const int NE,
+                                const Array<double> &w,
+                                const Vector &j,
+                                const Vector &vel,
+                                const double alpha,
+                                Vector &op)
 {
   constexpr int DIM = 2;

@@ -60,13 +60,13 @@ void PAConvectionSetup2D(const int NQ,
 }

 // PA Convection Assemble 3D kernel
-void PAConvectionSetup3D(const int NQ,
-                         const int NE,
-                         const Array<double> &w,
-                         const Vector &j,
-                         const Vector &vel,
-                         const double alpha,
-                         Vector &op)
+static void PAConvectionSetup3D(const int NQ,
+                                const int NE,
+                                const Array<double> &w,
+                                const Vector &j,
+                                const Vector &vel,
+                                const double alpha,
+                                Vector &op)
 {
   constexpr int DIM = 3;
   constexpr int SDIM = DIM;
@@ -135,7 +135,7 @@ static void PAConvectionSetup(const int dim,
 }

 // PA Convection Apply 2D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
+template<int T_D1D = 0, int T_Q1D = 0> static
 void PAConvectionApply2D(const int ne,
                         const Array<double> &b,
                         const Array<double> &g,
@@ -254,7 +254,7 @@ void PAConvectionApply2D(const int ne,
 }

 // Optimized PA Convection Apply 2D kernel
-template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
+template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0> static
 void SmemPAConvectionApply2D(const int ne,
                             const Array<double> &b,
                             const Array<double> &g,
@@ -382,7 +382,7 @@ void SmemPAConvectionApply2D(const int ne,
 }

 // PA Convection Apply 3D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
+template<int T_D1D = 0, int T_Q1D = 0> static
 void PAConvectionApply3D(const int ne,
                         const Array<double> &b,
                         const Array<double> &g,
@@ -563,7 +563,7 @@ void PAConvectionApply3D(const int ne,
 }

 // Optimized PA Convection Apply 3D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
+template<int T_D1D = 0, int T_Q1D = 0> static
 void SmemPAConvectionApply3D(const int ne,
                             const Array<double> &b,
                             const Array<double> &g,
@@ -768,8 +768,6 @@ void SmemPAConvectionApply3D(const int ne,

 void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
-   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
-                         Device::GetDeviceMemoryType() : pa_mt;
   // Assumes tensor-product elements
   Mesh *mesh = fes.GetMesh();
   const FiniteElement &el = *fes.GetFE(0);
@@ -786,11 +784,11 @@ void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   const int nq = ir->GetNPoints();
   dim = mesh->Dimension();
   ne = fes.GetNE();
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mt);
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;
-   pa_data.SetSize(symmDims * nq * ne, mt);
+   pa_data.SetSize(symmDims * nq * ne, Device::GetMemoryType());
   Vector vel;
   if (VectorConstantCoefficient *cQ =
          dynamic_cast<VectorConstantCoefficient*>(Q))
@@ -800,7 +798,7 @@ void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   else if (VectorGridFunctionCoefficient *vgfQ =
               dynamic_cast<VectorGridFunctionCoefficient*>(Q))
   {
-      vel.SetSize(dim * nq * ne, mt);
+      vel.SetSize(dim * nq * ne);

      const GridFunction *gf = vgfQ->GetGridFunction();
      const FiniteElementSpace &gf_fes = *gf->FESpace();
@@ -811,7 +809,7 @@ void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
                                          ElementDofOrdering::NATIVE;
      const Operator *R = gf_fes.GetElementRestriction(ordering);

-      Vector xe(R->Height(), mt);
+      Vector xe(R->Height(), Device::GetMemoryType());
      xe.UseDevice(true);

      R->Mult(*gf, xe);
@@ -16,12 +16,12 @@
 namespace mfem
 {

-void EADGTraceAssemble1DInt(const int NF,
-                            const Array<double> &basis,
-                            const Vector &padata,
-                            Vector &eadata_int,
-                            Vector &eadata_ext,
-                            const bool add)
+static void EADGTraceAssemble1DInt(const int NF,
+                                   const Array<double> &basis,
+                                   const Vector &padata,
+                                   Vector &eadata_int,
+                                   Vector &eadata_ext,
+                                   const bool add)
 {
   auto D = Reshape(padata.Read(), 2, 2, NF);
   auto A_int = Reshape(eadata_int.ReadWrite(), 2, NF);
@@ -50,11 +50,11 @@ void EADGTraceAssemble1DInt(const int NF,
   });
 }

-void EADGTraceAssemble1DBdr(const int NF,
-                            const Array<double> &basis,
-                            const Vector &padata,
-                            Vector &eadata_bdr,
-                            const bool add)
+static void EADGTraceAssemble1DBdr(const int NF,
+                                   const Array<double> &basis,
+                                   const Vector &padata,
+                                   Vector &eadata_bdr,
+                                   const bool add)
 {
   auto D = Reshape(padata.Read(), 2, 2, NF);
   auto A_bdr = Reshape(eadata_bdr.ReadWrite(), NF);
@@ -72,14 +72,14 @@ void EADGTraceAssemble1DBdr(const int NF,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-void EADGTraceAssemble2DInt(const int NF,
-                            const Array<double> &basis,
-                            const Vector &padata,
-                            Vector &eadata_int,
-                            Vector &eadata_ext,
-                            const bool add,
-                            const int d1d = 0,
-                            const int q1d = 0)
+static void EADGTraceAssemble2DInt(const int NF,
+                                   const Array<double> &basis,
+                                   const Vector &padata,
+                                   Vector &eadata_int,
+                                   Vector &eadata_ext,
+                                   const bool add,
+                                   const int d1d = 0,
+                                   const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -128,13 +128,13 @@ void EADGTraceAssemble2DInt(const int NF,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-void EADGTraceAssemble2DBdr(const int NF,
-                            const Array<double> &basis,
-                            const Vector &padata,
-                            Vector &eadata_bdr,
-                            const bool add,
-                            const int d1d = 0,
-                            const int q1d = 0)
+static void EADGTraceAssemble2DBdr(const int NF,
+                                   const Array<double> &basis,
+                                   const Vector &padata,
+                                   Vector &eadata_bdr,
+                                   const bool add,
+                                   const int d1d = 0,
+                                   const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -170,14 +170,14 @@ void EADGTraceAssemble2DBdr(const int NF,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-void EADGTraceAssemble3DInt(const int NF,
-                            const Array<double> &basis,
-                            const Vector &padata,
-                            Vector &eadata_int,
-                            Vector &eadata_ext,
-                            const bool add,
-                            const int d1d = 0,
-                            const int q1d = 0)
+static void EADGTraceAssemble3DInt(const int NF,
+                                   const Array<double> &basis,
+                                   const Vector &padata,
+                                   Vector &eadata_int,
+                                   Vector &eadata_ext,
+                                   const bool add,
+                                   const int d1d = 0,
+                                   const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -268,13 +268,13 @@ void EADGTraceAssemble3DInt(const int NF,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-void EADGTraceAssemble3DBdr(const int NF,
-                            const Array<double> &basis,
-                            const Vector &padata,
-                            Vector &eadata_bdr,
-                            const bool add,
-                            const int d1d = 0,
-                            const int q1d = 0)
+static void EADGTraceAssemble3DBdr(const int NF,
+                                   const Array<double> &basis,
+                                   const Vector &padata,
+                                   Vector &eadata_bdr,
+                                   const bool add,
+                                   const int d1d = 0,
+                                   const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -19,16 +19,16 @@ using namespace std;
 namespace mfem
 {
 // PA DG Trace Integrator
-void PADGTraceSetup2D(const int Q1D,
-                      const int NF,
-                      const Array<double> &w,
-                      const Vector &det,
-                      const Vector &nor,
-                      const Vector &rho,
-                      const Vector &vel,
-                      const double alpha,
-                      const double beta,
-                      Vector &op)
+static void PADGTraceSetup2D(const int Q1D,
+                             const int NF,
+                             const Array<double> &w,
+                             const Vector &det,
+                             const Vector &nor,
+                             const Vector &rho,
+                             const Vector &vel,
+                             const double alpha,
+                             const double beta,
+                             Vector &op)
 {
   const int VDIM = 2;

@@ -61,16 +61,16 @@ void PADGTraceSetup2D(const int Q1D,
   });
 }

-void PADGTraceSetup3D(const int Q1D,
-                      const int NF,
-                      const Array<double> &w,
-                      const Vector &det,
-                      const Vector &nor,
-                      const Vector &rho,
-                      const Vector &vel,
-                      const double alpha,
-                      const double beta,
-                      Vector &op)
+static void PADGTraceSetup3D(const int Q1D,
+                             const int NF,
+                             const Array<double> &w,
+                             const Vector &det,
+                             const Vector &nor,
+                             const Vector &rho,
+                             const Vector &vel,
+                             const double alpha,
+                             const double beta,
+                             Vector &op)
 {
   const int VDIM = 3;

@@ -301,7 +301,7 @@ void DGTraceIntegrator::AssemblePABoundaryFaces(const FiniteElementSpace& fes)
 }

 // PA DGTrace Apply 2D kernel for Gauss-Lobatto/Bernstein
-template<int T_D1D = 0, int T_Q1D = 0>
+template<int T_D1D = 0, int T_Q1D = 0> static
 void PADGTraceApply2D(const int NF,
                      const Array<double> &b,
                      const Array<double> &bt,
@@ -392,7 +392,7 @@ void PADGTraceApply2D(const int NF,
 }

 // PA DGTrace Apply 3D kernel for Gauss-Lobatto/Bernstein
-template<int T_D1D = 0, int T_Q1D = 0>
+template<int T_D1D = 0, int T_Q1D = 0> static
 void PADGTraceApply3D(const int NF,
                      const Array<double> &b,
                      const Array<double> &bt,
@@ -537,7 +537,7 @@ void PADGTraceApply3D(const int NF,
 }

 // Optimized PA DGTrace Apply 3D kernel for Gauss-Lobatto/Bernstein
-template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
+template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0> static
 void SmemPADGTraceApply3D(const int NF,
                          const Array<double> &b,
                          const Array<double> &bt,
@@ -701,7 +701,7 @@ static void PADGTraceApply(const int dim,
 }

 // PA DGTrace Apply 2D kernel for Gauss-Lobatto/Bernstein
-template<int T_D1D = 0, int T_Q1D = 0>
+template<int T_D1D = 0, int T_Q1D = 0> static
 void PADGTraceApplyTranspose2D(const int NF,
                               const Array<double> &b,
                               const Array<double> &bt,
@@ -797,7 +797,7 @@ void PADGTraceApplyTranspose2D(const int NF,
 }

 // PA DGTrace Apply Transpose 3D kernel for Gauss-Lobatto/Bernstein
-template<int T_D1D = 0, int T_Q1D = 0>
+template<int T_D1D = 0, int T_Q1D = 0> static
 void PADGTraceApplyTranspose3D(const int NF,
                               const Array<double> &b,
                               const Array<double> &bt,
@@ -953,7 +953,7 @@ void PADGTraceApplyTranspose3D(const int NF,
 }

 // Optimized PA DGTrace Apply Transpose 3D kernel for Gauss-Lobatto/Bernstein
-template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
+template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0> static
 void SmemPADGTraceApplyTranspose3D(const int NF,
                                   const Array<double> &b,
                                   const Array<double> &bt,
@@ -17,14 +17,14 @@ namespace mfem
 {

 template<int T_D1D = 0, int T_Q1D = 0>
-void EADiffusionAssemble1D(const int NE,
-                           const Array<double> &b,
-                           const Array<double> &g,
-                           const Vector &padata,
-                           Vector &eadata,
-                           const bool add,
-                           const int d1d = 0,
-                           const int q1d = 0)
+static void EADiffusionAssemble1D(const int NE,
+                                  const Array<double> &b,
+                                  const Array<double> &g,
+                                  const Vector &padata,
+                                  Vector &eadata,
+                                  const bool add,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -68,14 +68,14 @@ void EADiffusionAssemble1D(const int NE,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-void EADiffusionAssemble2D(const int NE,
-                           const Array<double> &b,
-                           const Array<double> &g,
-                           const Vector &padata,
-                           Vector &eadata,
-                           const bool add,
-                           const int d1d = 0,
-                           const int q1d = 0)
+static void EADiffusionAssemble2D(const int NE,
+                                  const Array<double> &b,
+                                  const Array<double> &g,
+                                  const Vector &padata,
+                                  Vector &eadata,
+                                  const bool add,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -145,14 +145,14 @@ void EADiffusionAssemble2D(const int NE,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-void EADiffusionAssemble3D(const int NE,
-                           const Array<double> &b,
-                           const Array<double> &g,
-                           const Vector &padata,
-                           Vector &eadata,
-                           const bool add,
-                           const int d1d = 0,
-                           const int q1d = 0)
+static void EADiffusionAssemble3D(const int NE,
+                                  const Array<double> &b,
+                                  const Array<double> &g,
+                                  const Vector &padata,
+                                  Vector &eadata,
+                                  const bool add,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -351,8 +351,6 @@ static void PADiffusionSetup(const int dim,

 void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
-   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
-                         Device::GetDeviceMemoryType() : pa_mt;
   // Assuming the same element type
   fespace = &fes;
   Mesh *mesh = fes.GetMesh();
@@ -373,7 +371,7 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   const int nq = ir->GetNPoints();
   dim = mesh->Dimension();
   ne = fes.GetNE();
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS, mt);
+   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
   const int sdim = mesh->SpaceDimension();
   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
   dofs1D = maps->ndof;
@@ -490,20 +488,21 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
         }
      }
   }
-   pa_data.SetSize((symmetric ? symmDims : MQfullDim) * nq * ne, mt);
+   pa_data.SetSize((symmetric ? symmDims : MQfullDim) * nq * ne,
+                   Device::GetDeviceMemoryType());
   PADiffusionSetup(dim, sdim, dofs1D, quad1D, coeffDim, ne, ir->GetWeights(),
                    geom->J, coeff, pa_data);
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-void PADiffusionDiagonal2D(const int NE,
-                           const bool symmetric,
-                           const Array<double> &b,
-                           const Array<double> &g,
-                           const Vector &d,
-                           Vector &y,
-                           const int d1d = 0,
-                           const int q1d = 0)
+static void PADiffusionDiagonal2D(const int NE,
+                                  const bool symmetric,
+                                  const Array<double> &b,
+                                  const Array<double> &g,
+                                  const Vector &d,
+                                  Vector &y,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -562,14 +561,14 @@ void PADiffusionDiagonal2D(const int NE,

 // Shared memory PA Diffusion Diagonal 2D kernel
 template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
-void SmemPADiffusionDiagonal2D(const int NE,
-                               const bool symmetric,
-                               const Array<double> &b_,
-                               const Array<double> &g_,
-                               const Vector &d_,
-                               Vector &y_,
-                               const int d1d = 0,
-                               const int q1d = 0)
+static void SmemPADiffusionDiagonal2D(const int NE,
+                                      const bool symmetric,
+                                      const Array<double> &b_,
+                                      const Array<double> &g_,
+                                      const Vector &d_,
+                                      Vector &y_,
+                                      const int d1d = 0,
+                                      const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -656,14 +655,14 @@ void SmemPADiffusionDiagonal2D(const int NE,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-void PADiffusionDiagonal3D(const int NE,
-                           const bool symmetric,
-                           const Array<double> &b,
-                           const Array<double> &g,
-                           const Vector &d,
-                           Vector &y,
-                           const int d1d = 0,
-                           const int q1d = 0)
+static void PADiffusionDiagonal3D(const int NE,
+                                  const bool symmetric,
+                                  const Array<double> &b,
+                                  const Array<double> &g,
+                                  const Vector &d,
+                                  Vector &y,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
 {
   constexpr int DIM = 3;
   const int D1D = T_D1D ? T_D1D : d1d;
@@ -757,14 +756,14 @@ void PADiffusionDiagonal3D(const int NE,

 // Shared memory PA Diffusion Diagonal 3D kernel
 template<int T_D1D = 0, int T_Q1D = 0>
-void SmemPADiffusionDiagonal3D(const int NE,
-                               const bool symmetric,
-                               const Array<double> &b_,
-                               const Array<double> &g_,
-                               const Vector &d_,
-                               Vector &y_,
-                               const int d1d = 0,
-                               const int q1d = 0)
+static void SmemPADiffusionDiagonal3D(const int NE,
+                                      const bool symmetric,
+                                      const Array<double> &b_,
+                                      const Array<double> &g_,
+                                      const Vector &d_,
+                                      Vector &y_,
+                                      const int d1d = 0,
+                                      const int q1d = 0)
 {
   constexpr int DIM = 3;
   const int D1D = T_D1D ? T_D1D : d1d;
@@ -1034,17 +1033,17 @@ static void OccaPADiffusionApply3D(const int D1D,

 // PA Diffusion Apply 2D kernel
 template<int T_D1D = 0, int T_Q1D = 0>
-void PADiffusionApply2D(const int NE,
-                        const bool symmetric,
-                        const Array<double> &b_,
-                        const Array<double> &g_,
-                        const Array<double> &bt_,
-                        const Array<double> &gt_,
-                        const Vector &d_,
-                        const Vector &x_,
-                        Vector &y_,
-                        const int d1d = 0,
-                        const int q1d = 0)
+static void PADiffusionApply2D(const int NE,
+                               const bool symmetric,
+                               const Array<double> &b_,
+                               const Array<double> &g_,
+                               const Array<double> &bt_,
+                               const Array<double> &gt_,
+                               const Vector &d_,
+                               const Vector &x_,
+                               Vector &y_,
+                               const int d1d = 0,
+                               const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -1156,15 +1155,15 @@ void PADiffusionApply2D(const int NE,

 // Shared memory PA Diffusion Apply 2D kernel
 template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
-void SmemPADiffusionApply2D(const int NE,
-                            const bool symmetric,
-                            const Array<double> &b_,
-                            const Array<double> &g_,
-                            const Vector &d_,
-                            const Vector &x_,
-                            Vector &y_,
-                            const int d1d = 0,
-                            const int q1d = 0)
+static void SmemPADiffusionApply2D(const int NE,
+                                   const bool symmetric,
+                                   const Array<double> &b_,
+                                   const Array<double> &g_,
+                                   const Vector &d_,
+                                   const Vector &x_,
+                                   Vector &y_,
+                                   const int d1d = 0,
+                                   const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -1314,16 +1313,16 @@ void SmemPADiffusionApply2D(const int NE,

 // PA Diffusion Apply 3D kernel
 template<int T_D1D = 0, int T_Q1D = 0>
-void PADiffusionApply3D(const int NE,
-                        const bool symmetric,
-                        const Array<double> &b,
-                        const Array<double> &g,
-                        const Array<double> &bt,
-                        const Array<double> &gt,
-                        const Vector &d_,
-                        const Vector &x_,
-                        Vector &y_,
-                        int d1d = 0, int q1d = 0)
+static void PADiffusionApply3D(const int NE,
+                               const bool symmetric,
+                               const Array<double> &b,
+                               const Array<double> &g,
+                               const Array<double> &bt,
+                               const Array<double> &gt,
+                               const Vector &d_,
+                               const Vector &x_,
+                               Vector &y_,
+                               int d1d = 0, int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -1533,15 +1532,15 @@ static MFEM_HOST_DEVICE inline double sign(const int q, const int d)
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-void SmemPADiffusionApply3D(const int NE,
-                            const bool symmetric,
-                            const Array<double> &b_,
-                            const Array<double> &g_,
-                            const Vector &d_,
-                            const Vector &x_,
-                            Vector &y_,
-                            const int d1d = 0,
-                            const int q1d = 0)
+static void SmemPADiffusionApply3D(const int NE,
+                                   const bool symmetric,
+                                   const Array<double> &b_,
+                                   const Array<double> &g_,
+                                   const Vector &d_,
+                                   const Vector &x_,
+                                   Vector &y_,
+                                   const int d1d = 0,
+                                   const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -21,12 +21,12 @@ namespace mfem
 // PA Divergence Integrator

 // PA Divergence Assemble 2D kernel
-void PADivergenceSetup2D(const int Q1D,
-                         const int NE,
-                         const Array<double> &w,
-                         const Vector &j,
-                         const double COEFF,
-                         Vector &op)
+static void PADivergenceSetup2D(const int Q1D,
+                                const int NE,
+                                const Array<double> &w,
+                                const Vector &j,
+                                const double COEFF,
+                                Vector &op)
 {
   const int NQ = Q1D*Q1D;
   auto W = w.Read();
@@ -51,12 +51,12 @@ void PADivergenceSetup2D(const int Q1D,
 }

 // PA Divergence Assemble 3D kernel
-void PADivergenceSetup3D(const int Q1D,
-                         const int NE,
-                         const Array<double> &w,
-                         const Vector &j,
-                         const double COEFF,
-                         Vector &op)
+static void PADivergenceSetup3D(const int Q1D,
+                                const int NE,
+                                const Array<double> &w,
+                                const Vector &j,
+                                const double COEFF,
+                                Vector &op)
 {
   const int NQ = Q1D*Q1D*Q1D;
   auto W = w.Read();
@@ -160,16 +160,16 @@ void VectorDivergenceIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,

 // PA Divergence Apply 2D kernel
 template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0>
-void PADivergenceApply2D(const int NE,
-                         const Array<double> &b,
-                         const Array<double> &g,
-                         const Array<double> &bt,
-                         const Vector &op_,
-                         const Vector &x_,
-                         Vector &y_,
-                         const int tr_d1d = 0,
-                         const int te_d1d = 0,
-                         const int q1d = 0)
+static void PADivergenceApply2D(const int NE,
+                                const Array<double> &b,
+                                const Array<double> &g,
+                                const Array<double> &bt,
+                                const Vector &op_,
+                                const Vector &x_,
+                                Vector &y_,
+                                const int tr_d1d = 0,
+                                const int te_d1d = 0,
+                                const int q1d = 0)
 {
   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -281,16 +281,16 @@ void PADivergenceApply2D(const int NE,
 // Shared memory PA Divergence Apply 2D kernel
 template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0,
         const int T_NBZ = 0>
-void SmemPADivergenceApply2D(const int NE,
-                             const Array<double> &b_,
-                             const Array<double> &g_,
-                             const Array<double> &bt_,
-                             const Vector &op_,
-                             const Vector &x_,
-                             Vector &y_,
-                             const int tr_d1d = 0,
-                             const int te_d1d = 0,
-                             const int q1d = 0)
+static void SmemPADivergenceApply2D(const int NE,
+                                    const Array<double> &b_,
+                                    const Array<double> &g_,
+                                    const Array<double> &bt_,
+                                    const Vector &op_,
+                                    const Vector &x_,
+                                    Vector &y_,
+                                    const int tr_d1d = 0,
+                                    const int te_d1d = 0,
+                                    const int q1d = 0)
 {
   // TODO
   MFEM_ASSERT(false, "SHARED MEM NOT PROGRAMMED YET");
@@ -298,16 +298,16 @@ void SmemPADivergenceApply2D(const int NE,

 // PA Divergence Apply 2D kernel transpose
 template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0>
-void PADivergenceApplyTranspose2D(const int NE,
-                                  const Array<double> &bt,
-                                  const Array<double> &gt,
-                                  const Array<double> &b,
-                                  const Vector &op_,
-                                  const Vector &x_,
-                                  Vector &y_,
-                                  const int tr_d1d = 0,
-                                  const int te_d1d = 0,
-                                  const int q1d = 0)
+static void PADivergenceApplyTranspose2D(const int NE,
+                                         const Array<double> &bt,
+                                         const Array<double> &gt,
+                                         const Array<double> &b,
+                                         const Vector &op_,
+                                         const Vector &x_,
+                                         Vector &y_,
+                                         const int tr_d1d = 0,
+                                         const int te_d1d = 0,
+                                         const int q1d = 0)
 {
   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -414,16 +414,16 @@ void PADivergenceApplyTranspose2D(const int NE,

 // PA Vector Divergence Apply 3D kernel
 template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0>
-void PADivergenceApply3D(const int NE,
-                         const Array<double> &b,
-                         const Array<double> &g,
-                         const Array<double> &bt,
-                         const Vector &op_,
-                         const Vector &x_,
-                         Vector &y_,
-                         int tr_d1d = 0,
-                         int te_d1d = 0,
-                         int q1d = 0)
+static void PADivergenceApply3D(const int NE,
+                                const Array<double> &b,
+                                const Array<double> &g,
+                                const Array<double> &bt,
+                                const Vector &op_,
+                                const Vector &x_,
+                                Vector &y_,
+                                int tr_d1d = 0,
+                                int te_d1d = 0,
+                                int q1d = 0)
 {
   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -597,16 +597,16 @@ void PADivergenceApply3D(const int NE,

 // PA Vector Divergence Apply 3D kernel
 template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0>
-void PADivergenceApplyTranspose3D(const int NE,
-                                  const Array<double> &bt,
-                                  const Array<double> &gt,
-                                  const Array<double> &b,
-                                  const Vector &op_,
-                                  const Vector &x_,
-                                  Vector &y_,
-                                  int tr_d1d = 0,
-                                  int te_d1d = 0,
-                                  int q1d = 0)
+static void PADivergenceApplyTranspose3D(const int NE,
+                                         const Array<double> &bt,
+                                         const Array<double> &gt,
+                                         const Array<double> &b,
+                                         const Vector &op_,
+                                         const Vector &x_,
+                                         Vector &y_,
+                                         int tr_d1d = 0,
+                                         int te_d1d = 0,
+                                         int q1d = 0)
 {
   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -775,16 +775,16 @@ void PADivergenceApplyTranspose3D(const int NE,

 // Shared memory PA Vector Divergence Apply 3D kernel
 template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0>
-void SmemPADivergenceApply3D(const int NE,
-                             const Array<double> &b_,
-                             const Array<double> &g_,
-                             const Array<double> &bt_,
-                             const Vector &q_,
-                             const Vector &x_,
-                             Vector &y_,
-                             const int tr_d1d = 0,
-                             const int te_d1d = 0,
-                             const int q1d = 0)
+static void SmemPADivergenceApply3D(const int NE,
+                                    const Array<double> &b_,
+                                    const Array<double> &g_,
+                                    const Array<double> &bt_,
+                                    const Vector &q_,
+                                    const Vector &x_,
+                                    Vector &y_,
+                                    const int tr_d1d = 0,
+                                    const int te_d1d = 0,
+                                    const int q1d = 0)
 {
   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -25,7 +25,7 @@ namespace mfem
   \b Q1D number of quadrature points in one dimension.
   \b w quadrature weights.
   \b j element Jacobians.
-   \b c coefficient at quadrature points.
+   \b COEFF coefficient at quadrature points.

   The function is used precompute data needed at quadrature points during
   the action. */
@@ -70,22 +70,18 @@ namespace mfem
   the \b MFEM_SHARED keyword for local arrays. */

 // PA Gradient Assemble 2D kernel
-void PAGradientSetup2D(const int Q1D,
-                       const int NE,
-                       const Array<double> &w,
-                       const Vector &j,
-                       const Vector &c,
-                       Vector &op)
+static void PAGradientSetup2D(const int Q1D,
+                              const int NE,
+                              const Array<double> &w,
+                              const Vector &j,
+                              const double COEFF,
+                              Vector &op)
 {
   const int NQ = Q1D*Q1D;
   auto W = w.Read();
   auto J = Reshape(j.Read(), NQ, 2, 2, NE);
   auto y = Reshape(op.Write(), NQ, 2, 2, NE);

-   const bool const_c = c.Size() == 1;
-   const auto C = const_c ? Reshape(c.Read(), 1,1) :
-                  Reshape(c.Read(), NQ, NE);
-
   MFEM_FORALL(e, NE,
   {
      for (int q = 0; q < NQ; ++q)
@@ -95,38 +91,30 @@ void PAGradientSetup2D(const int Q1D,
         const double J21 = J(q,1,0,e);
         const double J22 = J(q,1,1,e);
         // Store wq * Q * adj(J)
-         const double Co = const_c ? C(0,0) : C(q,e);
-         y(q,0,0,e) = W[q] * Co *  J22; // 1,1
-         y(q,0,1,e) = W[q] * Co * -J12; // 1,2
-         y(q,1,0,e) = W[q] * Co * -J21; // 2,1
-         y(q,1,1,e) = W[q] * Co *  J11; // 2,2
+         y(q,0,0,e) = W[q] * COEFF *  J22; // 1,1
+         y(q,0,1,e) = W[q] * COEFF * -J12; // 1,2
+         y(q,1,0,e) = W[q] * COEFF * -J21; // 2,1
+         y(q,1,1,e) = W[q] * COEFF *  J11; // 2,2
      }
   });
 }

 // PA Gradient Assemble 3D kernel
-void PAGradientSetup3D(const int Q1D,
-                       const int NE,
-                       const Array<double> &w,
-                       const Vector &j,
-                       const Vector &c,
-                       Vector &op)
+static void PAGradientSetup3D(const int Q1D,
+                              const int NE,
+                              const Array<double> &w,
+                              const Vector &j,
+                              const double COEFF,
+                              Vector &op)
 {
   const int NQ = Q1D*Q1D*Q1D;
   auto W = w.Read();
   auto J = Reshape(j.Read(), NQ, 3, 3, NE);
   auto y = Reshape(op.Write(), NQ, 3, 3, NE);
-
-   const bool const_c = c.Size() == 1;
-   const auto C = const_c ? Reshape(c.Read(), 1,1) :
-                  Reshape(c.Read(), NQ,NE);
-
   MFEM_FORALL(e, NE,
   {
      for (int q = 0; q < NQ; ++q)
      {
-         const double Co = const_c ? C(0,0) : C(q,e);
-
         const double J11 = J(q,0,0,e);
         const double J21 = J(q,1,0,e);
         const double J31 = J(q,2,0,e);
@@ -136,7 +124,7 @@ void PAGradientSetup3D(const int Q1D,
         const double J13 = J(q,0,2,e);
         const double J23 = J(q,1,2,e);
         const double J33 = J(q,2,2,e);
-         const double cw  = W[q] * Co;
+         const double cw  = W[q] * COEFF;
         // adj(J)
         const double A11 = (J22 * J33) - (J23 * J32);
         const double A12 = (J32 * J13) - (J12 * J33);
@@ -168,7 +156,7 @@ static void PAGradientSetup(const int dim,
                            const int NE,
                            const Array<double> &W,
                            const Vector &J,
-                            const Vector &COEFF,
+                            const double COEFF,
                            Vector &op)
 {
   if (dim == 1) { MFEM_ABORT("dim==1 not supported in PAGradientSetup"); }
@@ -208,62 +196,29 @@ void GradientIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
   MFEM_ASSERT(quad1D == test_maps->nqpt,
               "PA requires test and trial space to have same number of quadrature points!");
   pa_data.SetSize(nq * dimsToStore * ne, Device::GetMemoryType());
-
-   Vector coeff;
-
-   if (Q == nullptr)
+   double coeff = 1.0;
+   if (Q)
   {
-      coeff.SetSize(1);
-      coeff(0) = 1.0;
+      ConstantCoefficient *cQ = dynamic_cast<ConstantCoefficient*>(Q);
+      MFEM_VERIFY(cQ != NULL, "only ConstantCoefficient is supported!");
+      coeff = cQ->constant;
   }
-   else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
-   {
-      coeff.SetSize(1);
-      coeff(0) = cQ->constant;
-   }
-   else if (QuadratureFunctionCoefficient* cQ =
-               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
-   {
-      const QuadratureFunction &qFun = cQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == ne*nq,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
-   else
-   {
-      coeff.SetSize(nq * ne);
-      auto C = Reshape(coeff.HostWrite(), nq, ne);
-      for (int e = 0; e < ne; ++e)
-      {
-         ElementTransformation& T = *trial_fes.GetElementTransformation(e);
-         for (int q = 0; q < nq; ++q)
-         {
-            C(q,e) = Q->Eval(T, ir->IntPoint(q));
-         }
-      }
-   }
-
   PAGradientSetup(dim, trial_dofs1D, test_dofs1D, quad1D,
                   ne, ir->GetWeights(), geom->J, coeff, pa_data);
 }

 // PA Gradient Apply 2D kernel
 template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
-void PAGradientApply2D(const int NE,
-                       const Array<double> &b,
-                       const Array<double> &g,
-                       const Array<double> &bt,
-                       const Vector &op_,
-                       const Vector &x_,
-                       Vector &y_,
-                       const int tr_d1d = 0,
-                       const int te_d1d = 0,
-                       const int q1d = 0)
+static void PAGradientApply2D(const int NE,
+                              const Array<double> &b,
+                              const Array<double> &g,
+                              const Array<double> &bt,
+                              const Vector &op_,
+                              const Vector &x_,
+                              Vector &y_,
+                              const int tr_d1d = 0,
+                              const int te_d1d = 0,
+                              const int q1d = 0)
 {
   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -384,16 +339,16 @@ static void PAGradientApplyTranspose2D(const int NE,

 // PA Gradient Apply 3D kernel
 template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0>
-void PAGradientApply3D(const int NE,
-                       const Array<double> &b,
-                       const Array<double> &g,
-                       const Array<double> &bt,
-                       const Vector &op_,
-                       const Vector &x_,
-                       Vector &y_,
-                       int tr_d1d = 0,
-                       int te_d1d = 0,
-                       int q1d = 0)
+static void PAGradientApply3D(const int NE,
+                              const Array<double> &b,
+                              const Array<double> &g,
+                              const Array<double> &bt,
+                              const Vector &op_,
+                              const Vector &x_,
+                              Vector &y_,
+                              int tr_d1d = 0,
+                              int te_d1d = 0,
+                              int q1d = 0)
 {
   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -579,16 +534,16 @@ static void PAGradientApplyTranspose3D(const int NE,

 // Shared memory PA Gradient Apply 3D kernel
 template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0>
-void SmemPAGradientApply3D(const int NE,
-                           const Array<double> &b_,
-                           const Array<double> &g_,
-                           const Array<double> &bt_,
-                           const Vector &d_,
-                           const Vector &x_,
-                           Vector &y_,
-                           const int tr_d1d = 0,
-                           const int te_d1d = 0,
-                           const int q1d = 0)
+static void SmemPAGradientApply3D(const int NE,
+                                  const Array<double> &b_,
+                                  const Array<double> &g_,
+                                  const Array<double> &bt_,
+                                  const Vector &d_,
+                                  const Vector &x_,
+                                  Vector &y_,
+                                  const int tr_d1d = 0,
+                                  const int te_d1d = 0,
+                                  const int q1d = 0)
 {
   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -791,12 +791,12 @@ void SmemPAHcurlMassApply3D(const int D1D,
 }

 // PA H(curl) curl-curl assemble 2D kernel
-void PACurlCurlSetup2D(const int Q1D,
-                       const int NE,
-                       const Array<double> &w,
-                       const Vector &j,
-                       Vector &coeff,
-                       Vector &op)
+static void PACurlCurlSetup2D(const int Q1D,
+                              const int NE,
+                              const Array<double> &w,
+                              const Vector &j,
+                              Vector &coeff,
+                              Vector &op)
 {
   const int NQ = Q1D*Q1D;
   auto W = w.Read();
@@ -818,13 +818,13 @@ void PACurlCurlSetup2D(const int Q1D,
 }

 // PA H(curl) curl-curl assemble 3D kernel
-void PACurlCurlSetup3D(const int Q1D,
-                       const int coeffDim,
-                       const int NE,
-                       const Array<double> &w,
-                       const Vector &j,
-                       Vector &coeff,
-                       Vector &op)
+static void PACurlCurlSetup3D(const int Q1D,
+                              const int coeffDim,
+                              const int NE,
+                              const Array<double> &w,
+                              const Vector &j,
+                              Vector &coeff,
+                              Vector &op)
 {
   const int NQ = Q1D*Q1D*Q1D;
   const bool symmetric = (coeffDim != 9);
@@ -1045,16 +1045,16 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
   }
 }

-void PACurlCurlApply2D(const int D1D,
-                       const int Q1D,
-                       const int NE,
-                       const Array<double> &bo,
-                       const Array<double> &bot,
-                       const Array<double> &gc,
-                       const Array<double> &gct,
-                       const Vector &pa_data,
-                       const Vector &x,
-                       Vector &y)
+static void PACurlCurlApply2D(const int D1D,
+                              const int Q1D,
+                              const int NE,
+                              const Array<double> &bo,
+                              const Array<double> &bot,
+                              const Array<double> &gc,
+                              const Array<double> &gct,
+                              const Vector &pa_data,
+                              const Vector &x,
+                              Vector &y)
 {
   constexpr static int VDIM = 2;
   constexpr static int MAX_D1D = HCURL_MAX_D1D;
@@ -1166,19 +1166,19 @@ void PACurlCurlApply2D(const int D1D,
 }

 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-void PACurlCurlApply3D(const int D1D,
-                       const int Q1D,
-                       const bool symmetric,
-                       const int NE,
-                       const Array<double> &bo,
-                       const Array<double> &bc,
-                       const Array<double> &bot,
-                       const Array<double> &bct,
-                       const Array<double> &gc,
-                       const Array<double> &gct,
-                       const Vector &pa_data,
-                       const Vector &x,
-                       Vector &y)
+static void PACurlCurlApply3D(const int D1D,
+                              const int Q1D,
+                              const bool symmetric,
+                              const int NE,
+                              const Array<double> &bo,
+                              const Array<double> &bc,
+                              const Array<double> &bot,
+                              const Array<double> &bct,
+                              const Array<double> &gc,
+                              const Array<double> &gct,
+                              const Vector &pa_data,
+                              const Vector &x,
+                              Vector &y)
 {
   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
@@ -1677,19 +1677,19 @@ void PACurlCurlApply3D(const int D1D,
 }

 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-void SmemPACurlCurlApply3D(const int D1D,
-                           const int Q1D,
-                           const bool symmetric,
-                           const int NE,
-                           const Array<double> &bo,
-                           const Array<double> &bc,
-                           const Array<double> &bot,
-                           const Array<double> &bct,
-                           const Array<double> &gc,
-                           const Array<double> &gct,
-                           const Vector &pa_data,
-                           const Vector &x,
-                           Vector &y)
+static void SmemPACurlCurlApply3D(const int D1D,
+                                  const int Q1D,
+                                  const bool symmetric,
+                                  const int NE,
+                                  const Array<double> &bo,
+                                  const Array<double> &bc,
+                                  const Array<double> &bot,
+                                  const Array<double> &bct,
+                                  const Array<double> &gc,
+                                  const Array<double> &gct,
+                                  const Vector &pa_data,
+                                  const Vector &x,
+                                  Vector &y)
 {
   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
@@ -2032,13 +2032,13 @@ void CurlCurlIntegrator::AddMultPA(const Vector &x, Vector &y) const
   }
 }

-void PACurlCurlAssembleDiagonal2D(const int D1D,
-                                  const int Q1D,
-                                  const int NE,
-                                  const Array<double> &bo,
-                                  const Array<double> &gc,
-                                  const Vector &pa_data,
-                                  Vector &diag)
+static void PACurlCurlAssembleDiagonal2D(const int D1D,
+                                         const int Q1D,
+                                         const int NE,
+                                         const Array<double> &bo,
+                                         const Array<double> &gc,
+                                         const Vector &pa_data,
+                                         Vector &diag)
 {
   constexpr static int VDIM = 2;
   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
@@ -2087,16 +2087,16 @@ void PACurlCurlAssembleDiagonal2D(const int D1D,
 }

 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-void PACurlCurlAssembleDiagonal3D(const int D1D,
-                                  const int Q1D,
-                                  const bool symmetric,
-                                  const int NE,
-                                  const Array<double> &bo,
-                                  const Array<double> &bc,
-                                  const Array<double> &go,
-                                  const Array<double> &gc,
-                                  const Vector &pa_data,
-                                  Vector &diag)
+static void PACurlCurlAssembleDiagonal3D(const int D1D,
+                                         const int Q1D,
+                                         const bool symmetric,
+                                         const int NE,
+                                         const Array<double> &bo,
+                                         const Array<double> &bc,
+                                         const Array<double> &go,
+                                         const Array<double> &gc,
+                                         const Vector &pa_data,
+                                         Vector &diag)
 {
   constexpr static int VDIM = 3;
   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
@@ -2273,16 +2273,16 @@ void PACurlCurlAssembleDiagonal3D(const int D1D,
 }

 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-void SmemPACurlCurlAssembleDiagonal3D(const int D1D,
-                                      const int Q1D,
-                                      const bool symmetric,
-                                      const int NE,
-                                      const Array<double> &bo,
-                                      const Array<double> &bc,
-                                      const Array<double> &go,
-                                      const Array<double> &gc,
-                                      const Vector &pa_data,
-                                      Vector &diag)
+static void SmemPACurlCurlAssembleDiagonal3D(const int D1D,
+                                             const int Q1D,
+                                             const bool symmetric,
+                                             const int NE,
+                                             const Array<double> &bo,
+                                             const Array<double> &bc,
+                                             const Array<double> &go,
+                                             const Array<double> &gc,
+                                             const Vector &pa_data,
+                                             Vector &diag)
 {
   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
@@ -2955,18 +2955,18 @@ void MixedVectorCurlIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
 // Apply to x corresponding to DOF's in H(curl) (trial), whose curl is
 // integrated against H(curl) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-void PAHcurlL2Apply3D(const int D1D,
-                      const int Q1D,
-                      const int coeffDim,
-                      const int NE,
-                      const Array<double> &bo,
-                      const Array<double> &bc,
-                      const Array<double> &bot,
-                      const Array<double> &bct,
-                      const Array<double> &gc,
-                      const Vector &pa_data,
-                      const Vector &x,
-                      Vector &y)
+static void PAHcurlL2Apply3D(const int D1D,
+                             const int Q1D,
+                             const int coeffDim,
+                             const int NE,
+                             const Array<double> &bo,
+                             const Array<double> &bc,
+                             const Array<double> &bot,
+                             const Array<double> &bct,
+                             const Array<double> &gc,
+                             const Vector &pa_data,
+                             const Vector &x,
+                             Vector &y)
 {
   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
@@ -3297,16 +3297,16 @@ void PAHcurlL2Apply3D(const int D1D,
 // Apply to x corresponding to DOF's in H(curl) (trial), whose curl is
 // integrated against H(curl) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-void SmemPAHcurlL2Apply3D(const int D1D,
-                          const int Q1D,
-                          const int coeffDim,
-                          const int NE,
-                          const Array<double> &bo,
-                          const Array<double> &bc,
-                          const Array<double> &gc,
-                          const Vector &pa_data,
-                          const Vector &x,
-                          Vector &y)
+static void SmemPAHcurlL2Apply3D(const int D1D,
+                                 const int Q1D,
+                                 const int coeffDim,
+                                 const int NE,
+                                 const Array<double> &bo,
+                                 const Array<double> &bc,
+                                 const Array<double> &gc,
+                                 const Vector &pa_data,
+                                 const Vector &x,
+                                 Vector &y)
 {
   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
@@ -3585,18 +3585,18 @@ void SmemPAHcurlL2Apply3D(const int D1D,
 // Apply to x corresponding to DOF's in H(curl) (trial), whose curl is
 // integrated against H(div) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-void PAHcurlHdivApply3D(const int D1D,
-                        const int D1Dtest,
-                        const int Q1D,
-                        const int NE,
-                        const Array<double> &bo,
-                        const Array<double> &bc,
-                        const Array<double> &bot,
-                        const Array<double> &bct,
-                        const Array<double> &gc,
-                        const Vector &pa_data,
-                        const Vector &x,
-                        Vector &y)
+static void PAHcurlHdivApply3D(const int D1D,
+                               const int D1Dtest,
+                               const int Q1D,
+                               const int NE,
+                               const Array<double> &bo,
+                               const Array<double> &bc,
+                               const Array<double> &bot,
+                               const Array<double> &bct,
+                               const Array<double> &gc,
+                               const Vector &pa_data,
+                               const Vector &x,
+                               Vector &y)
 {
   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
@@ -4071,18 +4071,18 @@ void MixedVectorWeakCurlIntegrator::AssemblePA(const FiniteElementSpace
 // Apply to x corresponding to DOF's in H(curl) (trial), integrated against curl
 // of H(curl) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-void PAHcurlL2Apply3DTranspose(const int D1D,
-                               const int Q1D,
-                               const int coeffDim,
-                               const int NE,
-                               const Array<double> &bo,
-                               const Array<double> &bc,
-                               const Array<double> &bot,
-                               const Array<double> &bct,
-                               const Array<double> &gct,
-                               const Vector &pa_data,
-                               const Vector &x,
-                               Vector &y)
+static void PAHcurlL2Apply3DTranspose(const int D1D,
+                                      const int Q1D,
+                                      const int coeffDim,
+                                      const int NE,
+                                      const Array<double> &bo,
+                                      const Array<double> &bc,
+                                      const Array<double> &bot,
+                                      const Array<double> &bct,
+                                      const Array<double> &gct,
+                                      const Vector &pa_data,
+                                      const Vector &x,
+                                      Vector &y)
 {
   // See PAHcurlL2Apply3D for comments.

@@ -4413,16 +4413,16 @@ void PAHcurlL2Apply3DTranspose(const int D1D,
 }

 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-void SmemPAHcurlL2Apply3DTranspose(const int D1D,
-                                   const int Q1D,
-                                   const int coeffDim,
-                                   const int NE,
-                                   const Array<double> &bo,
-                                   const Array<double> &bc,
-                                   const Array<double> &gc,
-                                   const Vector &pa_data,
-                                   const Vector &x,
-                                   Vector &y)
+static void SmemPAHcurlL2Apply3DTranspose(const int D1D,
+                                          const int Q1D,
+                                          const int coeffDim,
+                                          const int NE,
+                                          const Array<double> &bo,
+                                          const Array<double> &bc,
+                                          const Array<double> &gc,
+                                          const Vector &pa_data,
+                                          const Vector &x,
+                                          Vector &y)
 {
   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
@@ -4675,13 +4675,13 @@ void MixedVectorWeakCurlIntegrator::AddMultPA(const Vector &x, Vector &y) const
 // Apply to x corresponding to DOFs in H^1 (domain) the (topological) gradient
 // to get a dof in H(curl) (range). You can think of the range as the "test" space
 // and the domain as the "trial" space, but there's no integration.
-void PAHcurlApplyGradient2D(const int c_dofs1D,
-                            const int o_dofs1D,
-                            const int NE,
-                            const Array<double> &B_,
-                            const Array<double> &G_,
-                            const Vector &x_,
-                            Vector &y_)
+static void PAHcurlApplyGradient2D(const int c_dofs1D,
+                                   const int o_dofs1D,
+                                   const int NE,
+                                   const Array<double> &B_,
+                                   const Array<double> &G_,
+                                   const Vector &x_,
+                                   Vector &y_)
 {
   auto B = Reshape(B_.Read(), c_dofs1D, c_dofs1D);
   auto G = Reshape(G_.Read(), o_dofs1D, c_dofs1D);
@@ -4753,12 +4753,12 @@ void PAHcurlApplyGradient2D(const int c_dofs1D,
 }

 // Specialization of PAHcurlApplyGradient2D to the case where B is identity
-void PAHcurlApplyGradient2DBId(const int c_dofs1D,
-                               const int o_dofs1D,
-                               const int NE,
-                               const Array<double> &G_,
-                               const Vector &x_,
-                               Vector &y_)
+static void PAHcurlApplyGradient2DBId(const int c_dofs1D,
+                                      const int o_dofs1D,
+                                      const int NE,
+                                      const Array<double> &G_,
+                                      const Vector &x_,
+                                      Vector &y_)
 {
   auto G = Reshape(G_.Read(), o_dofs1D, c_dofs1D);

@@ -4822,7 +4822,7 @@ void PAHcurlApplyGradient2DBId(const int c_dofs1D,
   });
 }

-void PAHcurlApplyGradientTranspose2D(
+static void PAHcurlApplyGradientTranspose2D(
   const int c_dofs1D, const int o_dofs1D, const int NE,
   const Array<double> &B_, const Array<double> &G_,
   const Vector &x_, Vector &y_)
@@ -4898,7 +4898,7 @@ void PAHcurlApplyGradientTranspose2D(

 // Specialization of PAHcurlApplyGradientTranspose2D to the case where
 // B is identity
-void PAHcurlApplyGradientTranspose2DBId(
+static void PAHcurlApplyGradientTranspose2DBId(
   const int c_dofs1D, const int o_dofs1D, const int NE,
   const Array<double> &G_,
   const Vector &x_, Vector &y_)
@@ -4965,13 +4965,13 @@ void PAHcurlApplyGradientTranspose2DBId(
   });
 }

-void PAHcurlApplyGradient3D(const int c_dofs1D,
-                            const int o_dofs1D,
-                            const int NE,
-                            const Array<double> &B_,
-                            const Array<double> &G_,
-                            const Vector &x_,
-                            Vector &y_)
+static void PAHcurlApplyGradient3D(const int c_dofs1D,
+                                   const int o_dofs1D,
+                                   const int NE,
+                                   const Array<double> &B_,
+                                   const Array<double> &G_,
+                                   const Vector &x_,
+                                   Vector &y_)
 {
   auto B = Reshape(B_.Read(), c_dofs1D, c_dofs1D);
   auto G = Reshape(G_.Read(), o_dofs1D, c_dofs1D);
@@ -5154,12 +5154,12 @@ void PAHcurlApplyGradient3D(const int c_dofs1D,
 }

 // Specialization of PAHcurlApplyGradient3D to the case where
-void PAHcurlApplyGradient3DBId(const int c_dofs1D,
-                               const int o_dofs1D,
-                               const int NE,
-                               const Array<double> &G_,
-                               const Vector &x_,
-                               Vector &y_)
+static void PAHcurlApplyGradient3DBId(const int c_dofs1D,
+                                      const int o_dofs1D,
+                                      const int NE,
+                                      const Array<double> &G_,
+                                      const Vector &x_,
+                                      Vector &y_)
 {
   auto G = Reshape(G_.Read(), o_dofs1D, c_dofs1D);

@@ -5322,7 +5322,7 @@ void PAHcurlApplyGradient3DBId(const int c_dofs1D,
   });
 }

-void PAHcurlApplyGradientTranspose3D(
+static void PAHcurlApplyGradientTranspose3D(
   const int c_dofs1D, const int o_dofs1D, const int NE,
   const Array<double> &B_, const Array<double> &G_,
   const Vector &x_, Vector &y_)
@@ -5507,7 +5507,7 @@ void PAHcurlApplyGradientTranspose3D(
 }

 // Specialization of PAHcurlApplyGradientTranspose3D to the case where
-void PAHcurlApplyGradientTranspose3DBId(
+static void PAHcurlApplyGradientTranspose3DBId(
   const int c_dofs1D, const int o_dofs1D, const int NE,
   const Array<double> &G_,
   const Vector &x_, Vector &y_)
@@ -5789,14 +5789,14 @@ void GradientInterpolator::AddMultTransposePA(const Vector &x, Vector &y) const
   }
 }

-void PAHcurlVecH1IdentityApply3D(const int c_dofs1D,
-                                 const int o_dofs1D,
-                                 const int NE,
-                                 const Array<double> &Bclosed,
-                                 const Array<double> &Bopen,
-                                 const Vector &pa_data,
-                                 const Vector &x_,
-                                 Vector &y_)
+static void PAHcurlVecH1IdentityApply3D(const int c_dofs1D,
+                                        const int o_dofs1D,
+                                        const int NE,
+                                        const Array<double> &Bclosed,
+                                        const Array<double> &Bopen,
+                                        const Vector &pa_data,
+                                        const Vector &x_,
+                                        Vector &y_)
 {
   auto Bc = Reshape(Bclosed.Read(), c_dofs1D, c_dofs1D);
   auto Bo = Reshape(Bopen.Read(), o_dofs1D, c_dofs1D);
@@ -6002,14 +6002,14 @@ void PAHcurlVecH1IdentityApply3D(const int c_dofs1D,
   });
 }

-void PAHcurlVecH1IdentityApplyTranspose3D(const int c_dofs1D,
-                                          const int o_dofs1D,
-                                          const int NE,
-                                          const Array<double> &Bclosed,
-                                          const Array<double> &Bopen,
-                                          const Vector &pa_data,
-                                          const Vector &x_,
-                                          Vector &y_)
+static void PAHcurlVecH1IdentityApplyTranspose3D(const int c_dofs1D,
+                                                 const int o_dofs1D,
+                                                 const int NE,
+                                                 const Array<double> &Bclosed,
+                                                 const Array<double> &Bopen,
+                                                 const Vector &pa_data,
+                                                 const Vector &x_,
+                                                 Vector &y_)
 {
   auto Bc = Reshape(Bclosed.Read(), c_dofs1D, c_dofs1D);
   auto Bo = Reshape(Bopen.Read(), o_dofs1D, c_dofs1D);
@@ -6228,14 +6228,14 @@ void PAHcurlVecH1IdentityApplyTranspose3D(const int c_dofs1D,
   });
 }

-void PAHcurlVecH1IdentityApply2D(const int c_dofs1D,
-                                 const int o_dofs1D,
-                                 const int NE,
-                                 const Array<double> &Bclosed,
-                                 const Array<double> &Bopen,
-                                 const Vector &pa_data,
-                                 const Vector &x_,
-                                 Vector &y_)
+static void PAHcurlVecH1IdentityApply2D(const int c_dofs1D,
+                                        const int o_dofs1D,
+                                        const int NE,
+                                        const Array<double> &Bclosed,
+                                        const Array<double> &Bopen,
+                                        const Vector &pa_data,
+                                        const Vector &x_,
+                                        Vector &y_)
 {
   auto Bc = Reshape(Bclosed.Read(), c_dofs1D, c_dofs1D);
   auto Bo = Reshape(Bopen.Read(), o_dofs1D, c_dofs1D);
@@ -6327,14 +6327,14 @@ void PAHcurlVecH1IdentityApply2D(const int c_dofs1D,
   });
 }

-void PAHcurlVecH1IdentityApplyTranspose2D(const int c_dofs1D,
-                                          const int o_dofs1D,
-                                          const int NE,
-                                          const Array<double> &Bclosed,
-                                          const Array<double> &Bopen,
-                                          const Vector &pa_data,
-                                          const Vector &x_,
-                                          Vector &y_)
+static void PAHcurlVecH1IdentityApplyTranspose2D(const int c_dofs1D,
+                                                 const int o_dofs1D,
+                                                 const int NE,
+                                                 const Array<double> &Bclosed,
+                                                 const Array<double> &Bopen,
+                                                 const Vector &pa_data,
+                                                 const Vector &x_,
+                                                 Vector &y_)
 {
   auto Bc = Reshape(Bclosed.Read(), c_dofs1D, c_dofs1D);
   auto Bo = Reshape(Bopen.Read(), o_dofs1D, c_dofs1D);
@@ -539,12 +539,12 @@ void PAHdivMassApply3D(const int D1D,

 // PA H(div) div-div assemble 2D kernel
 // NOTE: this is identical to PACurlCurlSetup3D
-void PADivDivSetup2D(const int Q1D,
-                     const int NE,
-                     const Array<double> &w,
-                     const Vector &j,
-                     Vector &coeff_,
-                     Vector &op)
+static void PADivDivSetup2D(const int Q1D,
+                            const int NE,
+                            const Array<double> &w,
+                            const Vector &j,
+                            Vector &coeff_,
+                            Vector &op)
 {
   const int NQ = Q1D*Q1D;
   auto W = w.Read();
@@ -565,12 +565,12 @@ void PADivDivSetup2D(const int Q1D,
   });
 }

-void PADivDivSetup3D(const int Q1D,
-                     const int NE,
-                     const Array<double> &w,
-                     const Vector &j,
-                     Vector &coeff_,
-                     Vector &op)
+static void PADivDivSetup3D(const int Q1D,
+                            const int NE,
+                            const Array<double> &w,
+                            const Vector &j,
+                            Vector &coeff_,
+                            Vector &op)
 {
   const int NQ = Q1D*Q1D*Q1D;
   auto W = w.Read();
@@ -599,16 +599,16 @@ void PADivDivSetup3D(const int Q1D,
   });
 }

-void PADivDivApply2D(const int D1D,
-                     const int Q1D,
-                     const int NE,
-                     const Array<double> &Bo_,
-                     const Array<double> &Gc_,
-                     const Array<double> &Bot_,
-                     const Array<double> &Gct_,
-                     const Vector &op_,
-                     const Vector &x_,
-                     Vector &y_)
+static void PADivDivApply2D(const int D1D,
+                            const int Q1D,
+                            const int NE,
+                            const Array<double> &Bo_,
+                            const Array<double> &Gc_,
+                            const Array<double> &Bot_,
+                            const Array<double> &Gct_,
+                            const Vector &op_,
+                            const Vector &x_,
+                            Vector &y_)
 {
   constexpr static int VDIM = 2;
   constexpr static int MAX_D1D = HDIV_MAX_D1D;
@@ -718,16 +718,16 @@ void PADivDivApply2D(const int D1D,
   }); // end of element loop
 }

-void PADivDivApply3D(const int D1D,
-                     const int Q1D,
-                     const int NE,
-                     const Array<double> &Bo_,
-                     const Array<double> &Gc_,
-                     const Array<double> &Bot_,
-                     const Array<double> &Gct_,
-                     const Vector &op_,
-                     const Vector &x_,
-                     Vector &y_)
+static void PADivDivApply3D(const int D1D,
+                            const int Q1D,
+                            const int NE,
+                            const Array<double> &Bo_,
+                            const Array<double> &Gc_,
+                            const Array<double> &Bot_,
+                            const Array<double> &Gct_,
+                            const Vector &op_,
+                            const Vector &x_,
+                            Vector &y_)
 {
   MFEM_VERIFY(D1D <= HDIV_MAX_D1D, "Error: D1D > HDIV_MAX_D1D");
   MFEM_VERIFY(Q1D <= HDIV_MAX_Q1D, "Error: Q1D > HDIV_MAX_Q1D");
@@ -967,13 +967,13 @@ void DivDivIntegrator::AddMultPA(const Vector &x, Vector &y) const
   }
 }

-void PADivDivAssembleDiagonal2D(const int D1D,
-                                const int Q1D,
-                                const int NE,
-                                const Array<double> &Bo_,
-                                const Array<double> &Gc_,
-                                const Vector &op_,
-                                Vector &diag_)
+static void PADivDivAssembleDiagonal2D(const int D1D,
+                                       const int Q1D,
+                                       const int NE,
+                                       const Array<double> &Bo_,
+                                       const Array<double> &Gc_,
+                                       const Vector &op_,
+                                       Vector &diag_)
 {
   constexpr static int VDIM = 2;
   constexpr static int MAX_Q1D = HDIV_MAX_Q1D;
@@ -1023,13 +1023,13 @@ void PADivDivAssembleDiagonal2D(const int D1D,
   });
 }

-void PADivDivAssembleDiagonal3D(const int D1D,
-                                const int Q1D,
-                                const int NE,
-                                const Array<double> &Bo_,
-                                const Array<double> &Gc_,
-                                const Vector &op_,
-                                Vector &diag_)
+static void PADivDivAssembleDiagonal3D(const int D1D,
+                                       const int Q1D,
+                                       const int NE,
+                                       const Array<double> &Bo_,
+                                       const Array<double> &Gc_,
+                                       const Vector &op_,
+                                       Vector &diag_)
 {
   MFEM_VERIFY(D1D <= HDIV_MAX_D1D, "Error: D1D > HDIV_MAX_D1D");
   MFEM_VERIFY(Q1D <= HDIV_MAX_Q1D, "Error: Q1D > HDIV_MAX_Q1D");
@@ -1104,11 +1104,11 @@ void DivDivIntegrator::AssembleDiagonalPA(Vector& diag)
 }

 // PA H(div)-L2 (div u, p) assemble 2D kernel
-void PADivL2Setup2D(const int Q1D,
-                    const int NE,
-                    const Array<double> &w,
-                    Vector &coeff_,
-                    Vector &op)
+static void PADivL2Setup2D(const int Q1D,
+                           const int NE,
+                           const Array<double> &w,
+                           Vector &coeff_,
+                           Vector &op)
 {
   const int NQ = Q1D*Q1D;
   auto W = w.Read();
@@ -1123,11 +1123,11 @@ void PADivL2Setup2D(const int Q1D,
   });
 }

-void PADivL2Setup3D(const int Q1D,
-                    const int NE,
-                    const Array<double> &w,
-                    Vector &coeff_,
-                    Vector &op)
+static void PADivL2Setup3D(const int Q1D,
+                           const int NE,
+                           const Array<double> &w,
+                           Vector &coeff_,
+                           Vector &op)
 {
   const int NQ = Q1D*Q1D*Q1D;
   auto W = w.Read();
@@ -1225,16 +1225,16 @@ VectorFEDivergenceIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,

 // Apply to x corresponding to DOF's in H(div) (trial), whose divergence is
 // integrated against L_2 test functions corresponding to y.
-void PAHdivL2Apply3D(const int D1D,
-                     const int Q1D,
-                     const int L2D1D,
-                     const int NE,
-                     const Array<double> &Bo_,
-                     const Array<double> &Gc_,
-                     const Array<double> &L2Bot_,
-                     const Vector &op_,
-                     const Vector &x_,
-                     Vector &y_)
+static void PAHdivL2Apply3D(const int D1D,
+                            const int Q1D,
+                            const int L2D1D,
+                            const int NE,
+                            const Array<double> &Bo_,
+                            const Array<double> &Gc_,
+                            const Array<double> &L2Bot_,
+                            const Vector &op_,
+                            const Vector &x_,
+                            Vector &y_)
 {
   MFEM_VERIFY(D1D <= HDIV_MAX_D1D, "Error: D1D > HDIV_MAX_D1D");
   MFEM_VERIFY(Q1D <= HDIV_MAX_Q1D, "Error: Q1D > HDIV_MAX_Q1D");
@@ -1388,16 +1388,16 @@ void PAHdivL2Apply3D(const int D1D,

 // Apply to x corresponding to DOF's in H(div) (trial), whose divergence is
 // integrated against L_2 test functions corresponding to y.
-void PAHdivL2Apply2D(const int D1D,
-                     const int Q1D,
-                     const int L2D1D,
-                     const int NE,
-                     const Array<double> &Bo_,
-                     const Array<double> &Gc_,
-                     const Array<double> &L2Bot_,
-                     const Vector &op_,
-                     const Vector &x_,
-                     Vector &y_)
+static void PAHdivL2Apply2D(const int D1D,
+                            const int Q1D,
+                            const int L2D1D,
+                            const int NE,
+                            const Array<double> &Bo_,
+                            const Array<double> &Gc_,
+                            const Array<double> &L2Bot_,
+                            const Vector &op_,
+                            const Vector &x_,
+                            Vector &y_)
 {
   constexpr static int VDIM = 2;
   constexpr static int MAX_D1D = HDIV_MAX_D1D;
@@ -1494,16 +1494,16 @@ void PAHdivL2Apply2D(const int D1D,
   }); // end of element loop
 }

-void PAHdivL2ApplyTranspose3D(const int D1D,
-                              const int Q1D,
-                              const int L2D1D,
-                              const int NE,
-                              const Array<double> &L2Bo_,
-                              const Array<double> &Gct_,
-                              const Array<double> &Bot_,
-                              const Vector &op_,
-                              const Vector &x_,
-                              Vector &y_)
+static void PAHdivL2ApplyTranspose3D(const int D1D,
+                                     const int Q1D,
+                                     const int L2D1D,
+                                     const int NE,
+                                     const Array<double> &L2Bo_,
+                                     const Array<double> &Gct_,
+                                     const Array<double> &Bot_,
+                                     const Vector &op_,
+                                     const Vector &x_,
+                                     Vector &y_)
 {
   MFEM_VERIFY(D1D <= HDIV_MAX_D1D, "Error: D1D > HDIV_MAX_D1D");
   MFEM_VERIFY(Q1D <= HDIV_MAX_Q1D, "Error: Q1D > HDIV_MAX_Q1D");
@@ -1656,16 +1656,16 @@ void PAHdivL2ApplyTranspose3D(const int D1D,
   }); // end of element loop
 }

-void PAHdivL2ApplyTranspose2D(const int D1D,
-                              const int Q1D,
-                              const int L2D1D,
-                              const int NE,
-                              const Array<double> &L2Bo_,
-                              const Array<double> &Gct_,
-                              const Array<double> &Bot_,
-                              const Vector &op_,
-                              const Vector &x_,
-                              Vector &y_)
+static void PAHdivL2ApplyTranspose2D(const int D1D,
+                                     const int Q1D,
+                                     const int L2D1D,
+                                     const int NE,
+                                     const Array<double> &L2Bo_,
+                                     const Array<double> &Gct_,
+                                     const Array<double> &Bot_,
+                                     const Vector &op_,
+                                     const Vector &x_,
+                                     Vector &y_)
 {
   constexpr static int VDIM = 2;
   constexpr static int MAX_D1D = HDIV_MAX_D1D;
@@ -1791,16 +1791,16 @@ void VectorFEDivergenceIntegrator::AddMultTransposePA(const Vector &x,
   }
 }

-void PAHdivL2AssembleDiagonal_ADAt_3D(const int D1D,
-                                      const int Q1D,
-                                      const int L2D1D,
-                                      const int NE,
-                                      const Array<double> &L2Bo_,
-                                      const Array<double> &Gct_,
-                                      const Array<double> &Bot_,
-                                      const Vector &op_,
-                                      const Vector &D_,
-                                      Vector &diag_)
+static void PAHdivL2AssembleDiagonal_ADAt_3D(const int D1D,
+                                             const int Q1D,
+                                             const int L2D1D,
+                                             const int NE,
+                                             const Array<double> &L2Bo_,
+                                             const Array<double> &Gct_,
+                                             const Array<double> &Bot_,
+                                             const Vector &op_,
+                                             const Vector &D_,
+                                             Vector &diag_)
 {
   MFEM_VERIFY(D1D <= HDIV_MAX_D1D, "Error: D1D > HDIV_MAX_D1D");
   MFEM_VERIFY(Q1D <= HDIV_MAX_Q1D, "Error: Q1D > HDIV_MAX_Q1D");
@@ -1916,16 +1916,16 @@ void PAHdivL2AssembleDiagonal_ADAt_3D(const int D1D,
   }); // end of element loop
 }

-void PAHdivL2AssembleDiagonal_ADAt_2D(const int D1D,
-                                      const int Q1D,
-                                      const int L2D1D,
-                                      const int NE,
-                                      const Array<double> &L2Bo_,
-                                      const Array<double> &Gct_,
-                                      const Array<double> &Bot_,
-                                      const Vector &op_,
-                                      const Vector &D_,
-                                      Vector &diag_)
+static void PAHdivL2AssembleDiagonal_ADAt_2D(const int D1D,
+                                             const int Q1D,
+                                             const int L2D1D,
+                                             const int NE,
+                                             const Array<double> &L2Bo_,
+                                             const Array<double> &Gct_,
+                                             const Array<double> &Bot_,
+                                             const Vector &op_,
+                                             const Vector &D_,
+                                             Vector &diag_)
 {
   constexpr static int VDIM = 2;

@@ -17,13 +17,13 @@ namespace mfem
 {

 template<int T_D1D = 0, int T_Q1D = 0>
-void EAMassAssemble1D(const int NE,
-                      const Array<double> &basis,
-                      const Vector &padata,
-                      Vector &eadata,
-                      const bool add,
-                      const int d1d = 0,
-                      const int q1d = 0)
+static void EAMassAssemble1D(const int NE,
+                             const Array<double> &basis,
+                             const Vector &padata,
+                             Vector &eadata,
+                             const bool add,
+                             const int d1d = 0,
+                             const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -67,13 +67,13 @@ void EAMassAssemble1D(const int NE,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-void EAMassAssemble2D(const int NE,
-                      const Array<double> &basis,
-                      const Vector &padata,
-                      Vector &eadata,
-                      const bool add,
-                      const int d1d = 0,
-                      const int q1d = 0)
+static void EAMassAssemble2D(const int NE,
+                             const Array<double> &basis,
+                             const Vector &padata,
+                             Vector &eadata,
+                             const bool add,
+                             const int d1d = 0,
+                             const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -139,13 +139,13 @@ void EAMassAssemble2D(const int NE,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-void EAMassAssemble3D(const int NE,
-                      const Array<double> &basis,
-                      const Vector &padata,
-                      Vector &eadata,
-                      const bool add,
-                      const int d1d = 0,
-                      const int q1d = 0)
+static void EAMassAssemble3D(const int NE,
+                             const Array<double> &basis,
+                             const Vector &padata,
+                             Vector &eadata,
+                             const bool add,
+                             const int d1d = 0,
+                             const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -25,9 +25,6 @@ namespace mfem

 void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
-   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
-                         Device::GetDeviceMemoryType() : pa_mt;
-
   // Assuming the same element type
   fespace = &fes;
   Mesh *mesh = fes.GetMesh();
@@ -45,11 +42,11 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
   ne = fes.GetMesh()->GetNE();
   nq = ir->GetNPoints();
   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::COORDINATES |
-                                    GeometricFactors::JACOBIANS, mt);
+                                    GeometricFactors::JACOBIANS);
   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;
-   pa_data.SetSize(ne*nq, mt);
+   pa_data.SetSize(ne*nq, Device::GetDeviceMemoryType());
   Vector coeff;
   if (Q == nullptr)
   {
@@ -155,12 +152,12 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-void PAMassAssembleDiagonal2D(const int NE,
-                              const Array<double> &b,
-                              const Vector &d,
-                              Vector &y,
-                              const int d1d = 0,
-                              const int q1d = 0)
+static void PAMassAssembleDiagonal2D(const int NE,
+                                     const Array<double> &b,
+                                     const Vector &d,
+                                     Vector &y,
+                                     const int d1d = 0,
+                                     const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -201,12 +198,12 @@ void PAMassAssembleDiagonal2D(const int NE,
 }

 template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
-void SmemPAMassAssembleDiagonal2D(const int NE,
-                                  const Array<double> &b_,
-                                  const Vector &d_,
-                                  Vector &y_,
-                                  const int d1d = 0,
-                                  const int q1d = 0)
+static void SmemPAMassAssembleDiagonal2D(const int NE,
+                                         const Array<double> &b_,
+                                         const Vector &d_,
+                                         Vector &y_,
+                                         const int d1d = 0,
+                                         const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -267,12 +264,12 @@ void SmemPAMassAssembleDiagonal2D(const int NE,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-void PAMassAssembleDiagonal3D(const int NE,
-                              const Array<double> &b,
-                              const Vector &d,
-                              Vector &y,
-                              const int d1d = 0,
-                              const int q1d = 0)
+static void PAMassAssembleDiagonal3D(const int NE,
+                                     const Array<double> &b,
+                                     const Vector &d,
+                                     Vector &y,
+                                     const int d1d = 0,
+                                     const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -336,12 +333,12 @@ void PAMassAssembleDiagonal3D(const int NE,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-void SmemPAMassAssembleDiagonal3D(const int NE,
-                                  const Array<double> &b_,
-                                  const Vector &d_,
-                                  Vector &y_,
-                                  const int d1d = 0,
-                                  const int q1d = 0)
+static void SmemPAMassAssembleDiagonal3D(const int NE,
+                                         const Array<double> &b_,
+                                         const Vector &d_,
+                                         Vector &y_,
+                                         const int d1d = 0,
+                                         const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -569,14 +566,14 @@ static void OccaPAMassApply3D(const int D1D,
 #endif // MFEM_USE_OCCA

 template<int T_D1D = 0, int T_Q1D = 0>
-void PAMassApply2D(const int NE,
-                   const Array<double> &b_,
-                   const Array<double> &bt_,
-                   const Vector &d_,
-                   const Vector &x_,
-                   Vector &y_,
-                   const int d1d = 0,
-                   const int q1d = 0)
+static void PAMassApply2D(const int NE,
+                          const Array<double> &b_,
+                          const Array<double> &bt_,
+                          const Vector &d_,
+                          const Vector &x_,
+                          Vector &y_,
+                          const int d1d = 0,
+                          const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -661,14 +658,14 @@ void PAMassApply2D(const int NE,
 }

 template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
-void SmemPAMassApply2D(const int NE,
-                       const Array<double> &b_,
-                       const Array<double> &bt_,
-                       const Vector &d_,
-                       const Vector &x_,
-                       Vector &y_,
-                       const int d1d = 0,
-                       const int q1d = 0)
+static void SmemPAMassApply2D(const int NE,
+                              const Array<double> &b_,
+                              const Array<double> &bt_,
+                              const Vector &d_,
+                              const Vector &x_,
+                              Vector &y_,
+                              const int d1d = 0,
+                              const int q1d = 0)
 {
   MFEM_CONTRACT_VAR(bt_);
   const int D1D = T_D1D ? T_D1D : d1d;
@@ -784,14 +781,14 @@ void SmemPAMassApply2D(const int NE,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-void PAMassApply3D(const int NE,
-                   const Array<double> &b_,
-                   const Array<double> &bt_,
-                   const Vector &d_,
-                   const Vector &x_,
-                   Vector &y_,
-                   const int d1d = 0,
-                   const int q1d = 0)
+static void PAMassApply3D(const int NE,
+                          const Array<double> &b_,
+                          const Array<double> &bt_,
+                          const Vector &d_,
+                          const Vector &x_,
+                          Vector &y_,
+                          const int d1d = 0,
+                          const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -925,14 +922,14 @@ void PAMassApply3D(const int NE,
 }

 template<int T_D1D = 0, int T_Q1D = 0>
-void SmemPAMassApply3D(const int NE,
-                       const Array<double> &b_,
-                       const Array<double> &bt_,
-                       const Vector &d_,
-                       const Vector &x_,
-                       Vector &y_,
-                       const int d1d = 0,
-                       const int q1d = 0)
+static void SmemPAMassApply3D(const int NE,
+                              const Array<double> &b_,
+                              const Array<double> &bt_,
+                              const Vector &d_,
+                              const Vector &x_,
+                              Vector &y_,
+                              const int d1d = 0,
+                              const int q1d = 0)
 {
   MFEM_CONTRACT_VAR(bt_);
   const int D1D = T_D1D ? T_D1D : d1d;
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
blaz	af84ca1ef5	wip	2021-07-10 10:33:09 -07:00
blaz	c8a8562ea4	modification and examples	2021-07-02 11:21:48 -07:00
blaz	a32ca0cb89	Another Lambda example	2021-05-26 15:53:39 -07:00
blaz	2e92b44070	Lambda function differentiation	2021-05-26 15:11:05 -07:00
blaz	2aa374efe8	Example of auto diff	2021-05-25 21:00:57 -07:00
blaz	476e642935	added cmake and make files for Eigen and CoDiPack	2021-05-25 17:53:33 -07:00
blaz	0da14179f7	initial copy frm fadg branch	2021-05-25 16:38:32 -07:00