Merge branch 'master' into DST-maxwell-solver-dev

ParDST first commit in new (clean) branch
2021-06-16 10:30:33 -07:00 · 2020-12-21 11:36:56 -08:00
407 changed files with 28367 additions and 54847 deletions
@@ -1,12 +0,0 @@
-# extends https://github.com/jupyterhub/repo2docker/blob/main/repo2docker/buildpacks/conda/environment.yml
-# see https://mybinder.readthedocs.io/en/latest/using/config_files.html#environment-yml-install-a-conda-environment
-channels:
- conda-forge
-dependencies:
- xeus-cling=0.13.0
- xwidgets=0.26.0
-# NOTE: it's possible these aren't needed for the lab frontend
- widgetsnbextension=3.5.1
- pip
- pip:
-  - glvis==0.3.2
@@ -1,26 +0,0 @@
-#!/bin/bash
-
-set -e
-
-# cling is installed here (in bin) and will look in {dir}/include and {dir}/lib
-# without extra intervention (jk it doesn't look in {dir}/lib unless something
-# has been #included from {dir}/include first...)
-install_dir=/srv/conda/envs/notebook
-mkdir -p $install_dir
-
-# build and install mfem, which is the directory we start in
-make serial SHARED=YES -j8
-make install PREFIX=$install_dir
-
-# install xeus-glvis
-git clone https://github.com/GLVis/xeus-glvis.git
-pushd xeus-glvis
-make install prefix=$install_dir
-popd
-
-# install jupyter-lab extension
-jupyter labextension install @jupyter-widgets/jupyterlab-manager --no-build
-jupyter labextension install glvis-jupyter
-
-# fixup kernelspec, we could probably do this from sh but ¯\_(ツ)_/¯
-python .binder/update_kernel_env.py
@@ -1,14 +0,0 @@
-# Update the LD_LIBRARY_PATH of the C++14 kernel so it can find mfem without
-# extra pragma cling statements
-
-import json
-
-kernelspec = "/srv/conda/envs/notebook/share/jupyter/kernels/xcpp14/kernel.json"
-
-with open(kernelspec, "r") as f:
-    obj = json.load(f)
-
-obj["env"] = {"LD_LIBRARY_PATH": "/srv/conda/envs/notebook/lib"}
-
-with open(kernelspec, "w") as f:
-    json.dump(obj, f)
@@ -47,36 +47,26 @@ jobs:
    strategy:
      matrix:
        os: [ubuntu-18.04, macos-10.15]
-        target: [dbg, opt]
-        mpi: [seq, par]
+        target: [debug, optim]
+        mpi: [sequential, parallel]
        build-system: [make]
-        hypre-target: [int32]
-        # 'include' allows us to:
-        # - Add a variable to all jobs without creating a new matrix dimension.
-        #   Codecov is defined that way.
-        # - Add a new combination.
-        #   'build-system: cmake' and 'hypre-target: int64'
+        # 'include' allows us to
+        # - add a variable without creating a new matrix dimension.
+        # - add a new combination ('build-system: cmake' case here)
        #
        # note: we will gather coverage info for any non-debug run except the
        # CMake build.
        include:
-          - target: dbg
+          - target: debug
            codecov: NO
-          - target: opt
+          - target: optim
            codecov: YES
          - os: ubuntu-18.04
-            target: opt
+            target: optim
            codecov: NO
-            mpi: par
+            mpi: parallel
            build-system: cmake
-            hypre-target: int32
-          - os: ubuntu-18.04
-            target: opt
-            codecov: NO
-            mpi: par
-            build-system: make
-            hypre-target: int64
-    name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
+    name: ${{ matrix.os }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.build-system }}

    runs-on: ${{ matrix.os }}

@@ -102,7 +92,7 @@ jobs:
    # TODO: It would be nice to have only one step, e.g. with a dedicated
    # action, but I (@adrienbernede) don't see how at the moment.
    - name: get MPI (Linux)
-      if: matrix.mpi == 'par' && matrix.os == 'ubuntu-18.04'
+      if: matrix.mpi == 'parallel' && matrix.os == 'ubuntu-18.04'
      run: |
        sudo apt-get install mpich libmpich-dev
        export MAKE_CXX_FLAG="MPICXX=mpic++"
@@ -113,11 +103,11 @@ jobs:
        sudo apt-get install lcov

    - name: Set up Homebrew
-      if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-10.15'
+      if: ( matrix.mpi == 'parallel' || matrix.codecov == 'YES' ) && matrix.os == 'macos-10.15'
      uses: Homebrew/actions/setup-homebrew@c4aafe8c4620bf08883dd4679c374f11e73329d3

    - name: get MPI (MacOS)
-      if: matrix.mpi == 'par' && matrix.os == 'macos-10.15'
+      if: matrix.mpi == 'parallel' && matrix.os == 'macos-10.15'
      run: |
        export HOMEBREW_NO_INSTALL_CLEANUP=1
        brew install openmpi
@@ -133,40 +123,39 @@ jobs:
    # Install will only run on cache miss.
    - name: cache hypre
      id: hypre-cache
-      if: matrix.mpi == 'par'
+      if: matrix.mpi == 'parallel'
      uses: actions/cache@v2
      with:
        path: ${{ env.HYPRE_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.0
+        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2

    - name: get hypre
-      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-hypre@v2.0
+      if: matrix.mpi == 'parallel' && steps.hypre-cache.outputs.cache-hit != 'true'
+      uses: mfem/github-actions/build-hypre@v1.0
      with:
-        archive: ${{ env.HYPRE_ARCHIVE }}
-        dir: ${{ env.HYPRE_TOP_DIR }}
-        target: ${{ matrix.hypre-target }}
+        hypre-archive: ${{ env.HYPRE_ARCHIVE }}
+        hypre-dir: ${{ env.HYPRE_TOP_DIR }}

    # Get Metis through cache, or build it.
    # Install will only run on cache miss.
    - name: cache metis
      id: metis-cache
-      if: matrix.mpi == 'par'
+      if: matrix.mpi == 'parallel'
      uses: actions/cache@v2
      with:
        path: ${{ env.METIS_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.0
+        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2

    - name: install metis
-      if: matrix.mpi == 'par' && steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@v2.0
+      if: matrix.mpi == 'parallel' && steps.metis-cache.outputs.cache-hit != 'true'
+      uses: mfem/github-actions/build-metis@v1.0
      with:
-        archive: ${{ env.METIS_ARCHIVE }}
-        dir: ${{ env.METIS_TOP_DIR }}
+        metis-archive: ${{ env.METIS_ARCHIVE }}
+        metis-dir: ${{ env.METIS_TOP_DIR }}

    # MFEM build and test
    - name: build
-      uses: mfem/github-actions/build-mfem@v2.0
+      uses: mfem/github-actions/build-mfem@v1.0
      with:
        os: ${{ matrix.os }}
        target: ${{ matrix.target }}
@@ -179,17 +168,17 @@ jobs:

    # Run checks (and only checks) on debug targets
    - name: checks
-      if: matrix.build-system == 'make' && matrix.target == 'dbg'
+      if: matrix.build-system == 'make' && matrix.target == 'debug'
      run: |
        cd ${{ env.MFEM_TOP_DIR }}  && make check

    - name: unit tests
-      if: matrix.build-system == 'make' && matrix.target == 'opt'
+      if: matrix.build-system == 'make' && matrix.target == 'optim'
      run: |
        cd ${{ env.MFEM_TOP_DIR }} && make unittest

    - name: tests
-      if: matrix.build-system == 'make' && matrix.target == 'opt'
+      if: matrix.build-system == 'make' && matrix.target == 'optim'
      run: |
        cd ${{ env.MFEM_TOP_DIR }} && make test

@@ -201,8 +190,8 @@ jobs:
    # Code coverage (process and upload reports)
    - name: codecov
      if: matrix.codecov == 'YES'
-      uses: mfem/github-actions/upload-coverage@v2.0
+      uses: mfem/github-actions/upload-coverage@v1.0
      with:
-        name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
+        name: ${{ matrix.os }}-${{ matrix.mpi }}
        project_dir: ${{ env.MFEM_TOP_DIR }}
        directories: "fem general linalg mesh"
@@ -53,38 +53,37 @@ jobs:
      uses: actions/cache@v2
      with:
        path: ${{ env.HYPRE_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.0
+        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2

    - name: Get Hypre
      if: steps.hypre-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-hypre@v2.0
+      uses: mfem/github-actions/build-hypre@master
      with:
-        archive: ${{ env.HYPRE_ARCHIVE }}
-        dir: ${{ env.HYPRE_TOP_DIR }}
-        target: int32
+        hypre-archive: ${{ env.HYPRE_ARCHIVE }}
+        hypre-dir: ${{ env.HYPRE_TOP_DIR }}

    - name: Cache Metis Install
      id: metis-cache
      uses: actions/cache@v2
      with:
        path: ${{ env.METIS_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.0
+        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2

    - name: Install Metis
      if: steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@v2.0
+      uses: mfem/github-actions/build-metis@master
      with:
-        archive: ${{ env.METIS_ARCHIVE }}
-        dir: ${{ env.METIS_TOP_DIR }}
+        metis-archive: ${{ env.METIS_ARCHIVE }}
+        metis-dir: ${{ env.METIS_TOP_DIR }}

    # MFEM build and test
    - name: build-mfem
-      uses: mfem/github-actions/build-mfem@v2.0
+      uses: mfem/github-actions/build-mfem@master
      with:
        os: ${{ runner.os }}
-        target: opt
+        target: optim
        codecov: NO
-        mpi: par
+        mpi: parallel
        build-system: make
        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
        metis-dir: ${{ env.METIS_TOP_DIR }}
@@ -28,24 +28,49 @@ jobs:
        access_token: ${{ github.token }}
    - name: checkout mfem
      uses: actions/checkout@v2
+      with:
+        path: mfem

    - name: copyright check
      id: copyright
      run: |
-        ./config/githooks/pre-push --copyright
-
+        cd mfem
+        if git grep -l "^\(#\|//\).*\(\-2020\|\ 2010,\)" > matches.txt
+        then
+          echo "Please update the following files to Copyright (c) 2010-2021:"
+          cat matches.txt
+          exit 1
+        else
+          echo "No outdated copyright found."
+        fi
      continue-on-error: true

    - name: license check
      id: license
      run: |
-        ./config/githooks/pre-push --license
+        cd mfem
+        if git grep -li "^\(#\|//\).*GNU\ Lesser\ General\ Public\ License" > matches.txt
+        then
+          echo "Please update the following files to the BSD-3 license:"
+          cat matches.txt
+          exit 1
+        else
+          echo "No GNU GPL license found."
+        fi
      continue-on-error: true

    - name: release check
      id: release
      run: |
-        ./config/githooks/pre-push --release
+        cd mfem
+        if git grep -l "^\(#\|//\).*LLNL\-CODE\-443211" > matches.txt
+        then
+          echo "Please update the following files to LLNL-CODE-806117:"
+          cat matches.txt
+          exit 1
+        else
+          echo "No outdated release number found."
+        fi
      continue-on-error: true

    - name: wrap-up
@@ -63,7 +88,7 @@ jobs:
        exit 1

  code-style:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-16.04 # needed for astyle 2.05.1

    steps:
    - name: checkout mfem
@@ -71,11 +96,12 @@ jobs:

    - name: get astyle
      run: |
-        sudo apt-get install astyle=3.1-1ubuntu2
+        sudo apt-get install astyle=2.05.1-0ubuntu1

    - name: style check
      run: |
-        ./config/githooks/pre-push --style
+        cd tests/scripts
+        ./runtest code-style

  documentation:
    runs-on: ubuntu-18.04
@@ -105,9 +131,7 @@ jobs:

    - name: branch-history
      run: |
-        # We override origin to make sure we point to the main repo.
-        # This is to have consistent test results on PRs from forks.
-        git remote remove origin
-        git remote add origin https://github.com/mfem/mfem.git
+        git fetch origin master:master
        git checkout -b gh-actions-branch-history
-        ./config/githooks/pre-push --history
+        cd tests/scripts
+        ./runtest branch-history
@@ -26,7 +26,6 @@ CMakeFiles/
 config/_config.hpp
 config/config.mk
 config/sample-runs-build.log
-config/user.mk
 doc/CodeDocumentation.conf
 doc/CodeDocumentation.html
 doc/CodeDocumentation
@@ -51,8 +50,6 @@ examples/ex1[04-9]
 examples/ex1[0-9]p
 examples/ex2[0-9]
 examples/ex2[0-9]p
-examples/ex30
-examples/ex30p

 examples/refined.mesh
 examples/displaced.mesh
@@ -225,14 +222,6 @@ miniapps/mtop/ParHeat*
 miniapps/mtop/seqheat
 miniapps/mtop/SeqHeat*

-miniapps/autodiff/paradiff
-miniapps/autodiff/seqadiff
-miniapps/autodiff/seqtest
-miniapps/autodiff/par_example
-miniapps/autodiff/seq_example
-miniapps/autodiff/seq_test
-miniapps/autodiff/Exampl*
-
 miniapps/navier/navier_mms
 miniapps/navier/navier_kovasznay
 miniapps/navier/navier_kovasznay_vs
@@ -262,7 +251,6 @@ miniapps/shifted/ParaViewDistance
 miniapps/shifted/diffusion
 miniapps/shifted/diffusion.mesh
 miniapps/shifted/diffusion.gf
-miniapps/shifted/ParaViewDiffusion

 miniapps/tools/display-basis
 miniapps/tools/load-dc
@@ -298,27 +286,15 @@ miniapps/solvers/ParaView
 miniapps/solvers/mesh.*
 miniapps/solvers/sol.*

-miniapps/parelag/MultilevelHcurlHdivSolver
-miniapps/parelag/*.mesh
-
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
 tests/unit/punit_tests
-tests/unit/cunit_tests
-tests/unit/pcunit_tests
 tests/unit/sedov_tests_*
 tests/unit/psedov_tests_*
 tests/unit/tmop_pa_tests_*
 tests/unit/ptmop_pa_tests_*
 tests/unit/ceed_tests
-tests/unit/debug_device_tests
-
-# Benchmark binaries
-tests/benchmarks/bench_ceed
-tests/benchmarks/bench_tmop
-tests/benchmarks/bench_vector
-tests/benchmarks/bench_virtuals

 # Test script output
 tests/scripts/*.err
@@ -336,6 +312,3 @@ build-*/*
 # PETSc automated build
 petsc-build/*
 pkg.gitcommit
-
-# Jupyter Notebook Checkpoints
-.ipynb_checkpoints
@@ -13,84 +13,223 @@
 # at Lawrence Livermore National Laboratory (LLNL). This entire pipeline is
 # LLNL-specific!

+# We define the following GitLab pipeline variables:
+#
+# BUILD_ROOT:
+# The path to the shared resources between all jobs. For example, external
+# repositories like 'tests' and 'tpls' are cloned here. Also, 'tpls' is built
+# once for all targets, so that build happen here. The BUILD_ROOT is unique to
+# the pipeline, preventing any form of concurrency with other pipelines. This
+# also means that the BUILD_ROOT directory will never be cleaned.
+# TODO: add a clean-up mechanism
+#
+# REBASELINE:
+# Defines the default choice for updating the saved baseline results. By default
+# the baseline can only be updated from the master branch. This variable offers
+# the option to manually ask for rebaselining from another branch if necessary.
+#
+# MFEM_ALLOC_NAME:
+# On LLNL's quartz, there is only one allocation shared among jobs in order to
+# save time and resources. This allocation has to be uniquely named so that we
+# are sure to retrieve it.
+#
+# TPLS_REPO & TESTS_REPO:
+# Git repositories used in the pipeline
+#
+# ARTIFACTS_DIR:
+# Directory used to place artifacts.

-# The pipeline is divided into stages. Usually, jobs in a given stage wait for
-# the preceding stages to complete before to start. However, we sometimes use
-# the "needs" keyword and express the DAG of jobs for more efficiency.
-# - We use setup and setup_baseline phases to download content outside of mfem
-#   directory.
-# - Allocate/Release is where quartz resource are allocated/released once for all.
+variables:
+  BUILD_ROOT: ${CI_BUILDS_DIR}/MFEM/${CI_PROJECT_NAME}_${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID}
+  AUTOTEST_ROOT: ${CI_BUILDS_DIR}/MFEM
+  REBASELINE: "NO"
+  AUTOTEST: "NO"
+  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
+  TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
+  TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
+  AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
+  ARTIFACTS_DIR: artifacts
+
+# The pipeline is divided into stages. Usually, these are also synchronization
+# points, however, we use "needs" keyword to express the DAG of jobs for more
+# efficiency.
+# - We use setup phase to download content outside of mfem directory.
+# - Allocate/Release is where quartz resources are allocated/released once for all.
 # - Build and Test is where we build and MFEM for multiple toolchains.
 # - Baseline_checks gathers baseline-type test suites execution
 # - Baseline_publish, only available on master, allows to update baseline
 #   results
 stages:
-  - sub-pipelines
+  - q_allocate_resources
+  - q_build_and_test
+  - q_release_resources
+  - l_build_and_test
+  - c_build_and_test
+  - setup
+  - baseline_check
+  - baseline_to_autotest
+  - baseline_publish

-variables:
-  CUSTOM_CI_BUILDS_DIR: "/usr/workspace/mfem/gitlab-runner"
-  USER_CI_TOP_DIR: "${CUSTOM_CI_BUILDS_DIR}/${GITLAB_USER_LOGIN}"
-  SHARED_REPOS_DIR: "${USER_CI_TOP_DIR}/repos"
-  AUTOTEST_ROOT: "${SHARED_REPOS_DIR}"
-  # MFEM_DATA_DIR is setup in '.gitlab/configs/setup-build-and-test.yml' and
-  # used in '.gitlab/configs/<machine>-config.yml':
-  MFEM_DATA_DIR: "${SHARED_REPOS_DIR}/mfem-data"
-
-# Defines the default choice for updating the saved baseline results. By default
-# the baseline can only be updated from the master branch. This variable offers
-# the option to manually ask for rebaselining from another branch if necessary.
-  REBASELINE: "NO"
-  AUTOTEST: "NO"
-  # AUTOTEST_COMMIT: used only when AUTOTEST is set to YES.
-  # * If AUTOTEST_COMMIT is NOT set to NO, reporting jobs will commit their
-  #   files to the MFEM/autotest repo.
-  # * If AUTOTEST_COMMIT is set to NO, reporting jobs will NOT commit their
-  #   files to the MFEM/autotest repo. Instead they will just show the contents
-  #   of the report files and remove them.
-  AUTOTEST_COMMIT: "YES"
-
-# Trigger subpipelines:
-quartz-build-and-test:
-  stage: sub-pipelines
+# The setup job in setup stage don't rely on MFEM git repo. It prepares a
+# pipeline-wide working directory downloading/updating external repos.
+# TODO: updating tests and tpls is not necessary anymore since pipelines are
+# now using unique directories so repo are never shared with another pipeline.
+# This is not memory efficient (we keep a lot of data), hence this reminder.
+# Setup
+setup:
+  tags:
+    - shell
+    - quartz
+  stage: setup
  variables:
-    # Explicitly pass down values that we want to be able to set when triggering
-    # pipelines manually or using scheduling
-    AUTOTEST: "${AUTOTEST}"
-    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
-  trigger:
-    include: .gitlab/quartz-build-and-test.yml
-    strategy: depend
+    GIT_STRATEGY: none
+  script:
+    - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
+    - if [ ! -d "tpls" ]; then git clone ${TPLS_REPO}; fi
+    - if [ ! -d "tests" ]; then git clone ${TESTS_REPO}; fi
+    - cd tpls && git pull && cd ..
+    - cd tests && git pull && cd ..
+    - cd ${AUTOTEST_ROOT}
+    - if [ ! -d "autotest" ]; then git clone ${AUTOTEST_REPO}; fi
+    - cd autotest && git pull && cd ..
+  needs: []

-quartz-baseline:
-  stage: sub-pipelines
-  variables:
-    # Explicitly pass down values that we want to be able to set when triggering
-    # pipelines manually or using scheduling
-    REBASELINE: "${REBASELINE}"
-    AUTOTEST: "${AUTOTEST}"
-    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
-  trigger:
-    include: .gitlab/quartz-baseline.yml
-    strategy: depend
+.build_toss_3_x86_64_ib_script:
+  script:
+    - export THREADS=12
+    - echo ${ALLOC_NAME}
+    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
+    - echo ${JOBID}
+    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 30 -N 1 tests/gitlab/build_and_test

-lassen-build-and-test:
-  stage: sub-pipelines
-  variables:
-    # Explicitly pass down values that we want to be able to set when triggering
-    # pipelines manually or using scheduling
-    AUTOTEST: "${AUTOTEST}"
-    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
-  trigger:
-    include: .gitlab/lassen-build-and-test.yml
-    strategy: depend
+.build_toss_3_x86_64_ib_corona_script:
+  script:
+    - srun -p mi60 -t 15 -N 1 tests/gitlab/build_and_test

-corona-build-and-test:
-  stage: sub-pipelines
+# Lassen uses a different job scheduler (spectrum lsf) that does not
+# allow pre-allocation the same way slurm does.
+# We use pdebug queue on lassen to speed-up the allocation.
+# However this would not be scalable to multiple builds.
+.build_blueos_3_ppc64le_ib_script:
+  script:
+    - lalloc 1 -W 30 -q pdebug tests/gitlab/build_and_test
+
+# Shared script for baseline and sample-run-baseline, the value of BASELINE_TEST
+# differentiates between the two tests.
+.baseline_script: &baseline_script |
+  # locals
+  _glob_err=${BASELINE_TEST}.err
+  _base_diff=${BASELINE_TEST}-${SYS_TYPE}.diff
+  _base_patch=${BASELINE_TEST}-${SYS_TYPE}.patch
+  _base_out=${BASELINE_TEST}-${SYS_TYPE}.out
+  # prepare
+  cd ${BUILD_ROOT}
+  ln -snf ${CI_PROJECT_DIR} mfem
+  cd tests
+  mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
+  # run
+  srun --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${ADDITIONAL_DIR}"
+  # post
+  mkdir ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}
+  if [[ -s ${_glob_err} ]]
+  then
+    echo "ERROR during ${BASELINE_TEST} execution";
+    echo "Here is the ${_glob_err} file content";
+    cat ${_glob_err}
+    cp ${_glob_err} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_glob_err}
+    exit 1;
+  elif [[ ! -f ${_base_patch} && ! -f ${_base_out} ]]
+  then
+    echo "Something went WRONG in ${BASELINE_TEST}:";
+    echo "Either ${_base_patch} or ${_base_out} should exists";
+    exit 1;
+  elif [[ -f ${_base_patch} ]]
+  then
+    echo "${BASELINE_TEST}: Differences found, patch generated"
+    cp ${_base_patch} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_patch}
+  elif [[ -f ${_base_out} ]]
+  then
+    echo "${BASELINE_TEST}: Differences found, replacement file generated"
+    cp ${_base_out} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_out}
+  fi
+  # _base_diff won't even exist if there is no difference.
+  if [[ -f ${_base_diff} ]]
+  then
+    echo "${BASELINE_TEST}: Relevant differences (filtered diff) ..."
+    cat ${_base_diff}
+    cp ${_base_diff} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_diff}
+    # We create a .err file, because that's how we signal that there was a diff.
+    cp ${_base_diff} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/gitlab-${BASELINE_TEST}-${SYS_TYPE}.err
+  fi
+  if [[ ! -s ${_base_diff} ]]
+  then
+    echo "${BASELINE_TEST}: PASSED"
+    true
+  else
+    echo "${BASELINE_TEST}: FAILED"
+    false
+  fi
+
+# Actual templates for baseline checks
+.baselinecheck_mfem:
+  stage: baseline_check
  variables:
-    # Explicitly pass down values that we want to be able to set when triggering
-    # pipelines manually or using scheduling
-    AUTOTEST: "${AUTOTEST}"
-    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
-  trigger:
-    include: .gitlab/corona-build-and-test.yml
-    strategy: depend
+    BASELINE_TEST: baseline
+    ADDITIONAL_DIR: ${BUILD_ROOT}/tpls
+  script:
+    - *baseline_script
+  artifacts:
+    when: always
+    paths:
+      - ${ARTIFACTS_DIR}
+  allow_failure: true
+
+.samplebaselinecheck_mfem:
+  stage: baseline_check
+  variables:
+    BASELINE_TEST: sample-runs-baseline
+    ADDITIONAL_DIR: ""
+  script:
+    - *baseline_script
+  timeout: 4h
+  artifacts:
+    when: always
+    paths:
+      - ${ARTIFACTS_DIR}
+  allow_failure: true
+
+# This job can only be manually triggered on a pipeline for master branch, or if
+# the pipeline was triggered with REBASELINE="YES"
+.rebaseline_mfem:
+  stage: baseline_publish
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "YES"'
+      when: manual
+  script:
+    - export PATCH_FILE=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}.patch
+    - export FULL_FILE=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}.out
+    - export DIFF_FILE=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}.diff
+    - cd ${BUILD_ROOT}/tests
+    - |
+      if [[ ! -f "${DIFF_FILE}" ]]
+      then
+          echo "Nothing to be done: no relevant change in baseline"
+          exit 0
+      elif [[ -f "${PATCH_FILE}" ]]
+      then
+          patch "./baseline-${SYS_TYPE}.saved" < "${PATCH_FILE}"
+      elif [[ -f "${FULL_FILE}t" ]]
+      then
+          cp "${FULL_FILE}" "./baseline-${SYS_TYPE}.saved"
+      else
+          echo "File missing: expected ${PATCH_FILE} or ${FULL_FILE}"
+          exit 1
+      fi
+    - git add baseline-${SYS_TYPE}.saved
+    - git commit -m "${SYS_TYPE} rebaselined in GitLab pipeline ${CI_PIPELINE_ID}"
+    - git push origin master
+
+# The list on jobs is defined in machine-specific files.
+include:
+  - local: .gitlab/quartz.yml
+  - local: .gitlab/lassen.yml
@@ -1,94 +0,0 @@
-                    Finite Element Discretization Library
-                                   __
-                       _ __ ___   / _|  ___  _ __ ___
-                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
-                      | | | | | ||  _||  __/| | | | | |
-                      |_| |_| |_||_|   \___||_| |_| |_|
-
-                               https://mfem.org
-
-
-This directory contains most of the GitLab CI configuration. MFEM runs both PR
-and nightly testing on GitLab.
-
-# Structure
-
-## Top level
-
-The root configuration file is `.gitlab-ci.yml` at the root of MFEM repo.
-This file only defines one stage, in which we trigger several
-sub-pipelines.
-
-We use sub-pipelines to isolate the test for one combination of `machine`
-and `test type`.
-
-Machines typically include:
-
-* Quartz: Intel bi-socket x86
-* Lassen: Power9 + Nvidia GPU
-* Corona: AMD GPU
-
-Test types include:
-
-* Build and test: Spack driven build of dependencies, mfem build, mfem
-  test
-* Baseline: Script driven build of dependencies, thorough testing
-
-⚠️ The sub-pipeline design allows to add a new machine or a new test type without
-altering the scheduling, execution and displaying of the others.
-
-## Sub-pipelines
-
-Each file is this directory is the root configuration file for one
-sub-pipeline. The naming reflects the corresponding couple (`machine`,
-`test_type`).
-
-Those files define the *stages* and the *jobs* for the sub-pipeline. They
-also contain any configuration that cannot be shared. For the most part
-though, the configuration is shared and is placed in `.gitlab/configs`.
-
-We try to keep scripts out of the CI config and share them among similar
-jobs. They are gathered in `.gitlab/scripts`.
-
-## Scripts
-
-Scripts specific to the CI only are in `.gitlab/scripts`. It is best practice
-to keep scripts outside the CI configuration (no bash scripts embedded in a
-yaml file) because it helps with readability, maintenance and also with
-transition to another CI system.
-
-⚠️ Most of the scripts there are driven by environment variables and do not have a
-usage function. This should be improved.
-
-
-# More testing
-
-## Adding a new target to a build_and_test pipeline
-
-`build_and_test` pipelines rely on Spack to install dependencies. Spack is
-driven by Uberenv which helps freezing Spack configuration: the goal being to
-point to specific commit in Spack and isolate its configuration so that it is
-not influenced by the user environment. More documentation about this can be
-found in `tests/gitlab`.
-
-In the end, the MFEM target for which to build the dependencies is expressed
-with a spack spec of MFEM, within the limits permitted by the MFEM spack
-package.
-
-In any build-and-test sub-pipeline a job basically consists in defining the
-spack spec to use. Adding a job on quartz for example resumes to:
-
-```yaml
-<job_name>:
-  variables:
-    SPEC: "<spack_spec>"
-  extends: .build_and_test_on_quartz
-```
-
-The remaining and non trivial work is to make sure this spec is working. To
-test a spec before adding it, or reproduce a CI configuration, please refer to
-`tests/gitlab/reproduce-ci-jobs-interactively.md`.
-
-⚠️ It is assumed that the spack spec applies to `mfem@develop`. That's why in the
-CI all the specs start with the compiler or the variants to apply to mfem. The
-mechanism still works with a full spec.
@@ -1,36 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# We define the following GitLab pipeline variables:
-variables:
-
-# The path to the shared resource between all jobs. For example, external
-# repositories like 'tests' and 'tpls' are cloned here. Also, 'tpls' is built
-# once for all targets, so that build happen here. The BUILD_ROOT is unique to
-# the pipeline, preventing any form of concurrency with other pipelines. This
-# also means that the BUILD_ROOT directory will never be cleaned.
-# TODO: add a clean-up mechanism
-  BUILD_ROOT: ${USER_CI_TOP_DIR}/${CI_PROJECT_NAME}-${MACHINE_NAME}-pipeline-${CI_PIPELINE_ID}
-
-# On LLNL's quartz, there is only one allocation shared among jobs in order to
-# save time and resource. This allocation has to be uniquely named so that we
-# are sure to retrieve it.
-  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
-
-# Git repositories used in the pipeline
-  TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
-  TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
-  AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
-  MFEM_DATA_REPO: https://github.com/mfem/data.git
-
-# Directory used to place artifacts.
-  ARTIFACTS_DIR: artifacts
-  SLURM_OVERLAP: 1
@@ -1,59 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# GitLab pipeline configuration for the Corona machine at LLNL
-variables:
-  MACHINE_NAME: corona
-
-.on_corona:
-  tags:
-    - shell
-    - corona
-  rules:
-    # Don’t run corona jobs if...
-    # Note: This makes corona an "opt-in" machine. To activate builds on corona
-    # for a given GitLab clone of MFEM, go to Setting/CI-CD/variables, and set
-    # "ON_CORONA" to "ON". An LC account on for corona is required to trigger a
-    # pipeline there.
-    - if: '$CI_COMMIT_BRANCH =~ /_cnone/ || $ON_CORONA != "ON"'
-      when: never
-    # Don’t run autotest update if...
-    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
-      when: never
-    # Report success on success status
-    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
-      when: on_success
-    # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
-      when: on_failure
-    # Always release resource
-    - if: '$CI_JOB_NAME =~ /release_resource/'
-      when: always
-    # Always cleanup
-    - if: '$CI_JOB_NAME =~ /cleanup/'
-      when: always
-    # Default is to run if previous stage succeeded
-    - when: on_success
-
-# Spack helped builds
-# Generic corona build job, extending build script
-.build_and_test_on_corona:
-  extends: [.on_corona]
-  stage: build_and_test
-  script:
-    # THREADS is used by 'tests/gitlab/build_and_test', run below
-    - export THREADS=12
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - echo ${MFEM_DATA_DIR}
-    - echo ${SPEC}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 15 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
@@ -1,49 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# GitLab pipelines configurations for the Lassen machine at LLNL
-variables:
-  MACHINE_NAME: lassen
-
-.on_lassen:
-  tags:
-    - shell
-    - lassen
-  rules:
-    - if: '$CI_COMMIT_BRANCH =~ /_lnone/ || $ON_LASSEN == "OFF"' #run except if ...
-      when: never
-    # Don't run autotest update if...
-    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
-      when: never
-    # Report success on success status
-    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
-      when: on_success
-    # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
-      when: on_failure
-    # Always cleanup
-    - if: '$CI_JOB_NAME =~ /cleanup/'
-      when: always
-    - when: on_success
-
-# Lassen uses a different job scheduler (spectrum lsf) that does not allow
-# pre-allocation the same way slurm does.  We use pdebug queue on lassen
-# to speed-up the allocation.  However this would not be scalable to
-# multiple builds.
-.build_and_test_on_lassen:
-  extends: [.on_lassen]
-  stage: build_and_test
-  script:
-    - echo ${MFEM_DATA_DIR}
-    - echo ${SPEC}
-    # Next script uses 'THREADS': leaving it empty --> it uses 'make all -j'
-    - lalloc 1 -W 30 -q pdebug tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
-  needs: [setup]
@@ -1,55 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# GitLab pipelines configurations for the Quartz machine at LLNL
-variables:
-  MACHINE_NAME: quartz
-
-.on_quartz:
-  tags:
-    - shell
-    - quartz
-  rules:
-    # Don't run quartz jobs if...
-    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"'
-      when: never
-    # Don't run autotest update if...
-    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
-      when: never
-    # Report success on success status
-    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
-      when: on_success
-    # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
-      when: on_failure
-    # Always release resource
-    - if: '$CI_JOB_NAME =~ /release_resource/'
-      when: always
-    # Always cleanup
-    - if: '$CI_JOB_NAME =~ /cleanup/'
-      when: always
-    # Default is to run if previous stage succeeded
-    - when: on_success
-
-# Spack helped builds
-# Generic quartz build job, extending build script
-.build_and_test_on_quartz:
-  extends: [.on_quartz]
-  stage: build_and_test
-  script:
-    # THREADS is used by 'tests/gitlab/build_and_test', run below
-    - export THREADS=12
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - echo ${MFEM_DATA_DIR}
-    - echo ${SPEC}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 30 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
@@ -1,81 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Jobs report
-.report_job_success:
-  script:
-    - echo ${MACHINE_NAME}
-    - echo ${AUTOTEST}
-    - echo ${AUTOTEST_COMMIT}
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - cd ${AUTOTEST_ROOT}
-    - |
-      (
-        date
-        echo "Waiting to aquire lock on '$PWD/autotest.lock' ..."
-        # try to get an excusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
-        done
-        echo "Aquired lock on '$PWD/autotest.lock'"
-        date
-        # Report SUCCESS while holding the file lock on 'autotest.lock'.
-        # The next script uses the following environment variables:
-        # - MACHINE_NAME, AUTOTEST_ROOT, AUTOTEST_COMMIT
-        # - CI_COMMIT_REF_SLUG, CI_PROJECT_DIR, CI_PIPELINE_URL
-        # It also calls the script '.gitlab/scripts/safe_create_rundir'.
-        ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test_success
-        err=$?
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock
-
-.report_job_failure:
-  script:
-    - echo ${MACHINE_NAME}
-    - echo ${AUTOTEST}
-    - echo ${AUTOTEST_COMMIT}
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - cd ${AUTOTEST_ROOT}
-    - |
-      (
-        date
-        echo "Waiting to aquire lock on '$PWD/autotest.lock' ..."
-        # try to get an excusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
-        done
-        echo "Aquired lock on '$PWD/autotest.lock'"
-        date
-        # Report FAILURE while holding the file lock on 'autotest.lock'.
-        # The next script uses the following environment variables:
-        # - MACHINE_NAME, AUTOTEST_ROOT, AUTOTEST_COMMIT
-        # - CI_COMMIT_REF_SLUG, CI_PROJECT_DIR, CI_PIPELINE_URL
-        # It also calls the script '.gitlab/scripts/safe_create_rundir'.
-        ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test_failure
-        err=$?
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock
@@ -1,72 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# The setup_baseline job doesn't rely on MFEM git repo. It prepares a
-# pipeline-wide working directory downloading/updating external repos.
-# TODO:
-# updating tests and tpls is not necessary anymore since pipelines are now
-# using unique directories so repo are never shared with another pipeline. This
-# is not memory efficient (we keep a lot of data), hence this reminder.
-setup_baseline:
-  tags:
-    - shell
-    - quartz
-  stage: setup
-  variables:
-    GIT_STRATEGY: none
-  script:
-    #
-    # Setup ${BUILD_ROOT}/tpls and ${BUILD_ROOT}/tests:
-    #
-    - echo "MACHINE_NAME = ${MACHINE_NAME}"
-    - echo "REBASELINE = ${REBASELINE}"
-    - echo "AUTOTEST = ${AUTOTEST}"
-    - echo "AUTOTEST_COMMIT = ${AUTOTEST_COMMIT}"
-    - echo "BUILD_ROOT ${BUILD_ROOT}"
-    - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
-    - if [ ! -d "tpls" ]; then git clone ${TPLS_REPO}; fi
-    - if [ ! -d "tests" ]; then git clone ${TESTS_REPO}; fi
-    - cd tpls && git pull && cd ..
-    - cd tests && git pull origin && cd ..
-    #
-    # Setup ${AUTOTEST_ROOT}/autotest:
-    #
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - mkdir -p ${AUTOTEST_ROOT} && cd ${AUTOTEST_ROOT}
-    - command -v flock || echo "Required command 'flock' not found"
-    - |
-      (
-        date
-        echo "Waiting to aquire lock on '$PWD/autotest.lock' ..."
-        # try to get an excusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
-        done
-        echo "Aquired lock on '$PWD/autotest.lock'"
-        date
-        # clone/update the autotest repo while holding the file lock on
-        # 'autotest.lock'
-        err=0
-        if [[ ! -d "autotest" ]]; then
-          git clone ${AUTOTEST_REPO}
-        else
-          cd autotest && git pull && cd ..
-        fi || err=1
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock
@@ -1,94 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test
-# script then symlinks the repo to the parent directory of the MFEM source
-# directory. Unit tests that depend on the mfem/data repo will then detect that
-# this directory is present and be enabled.
-setup:
-  tags:
-    - shell
-    - quartz
-  stage: setup
-  variables:
-    GIT_STRATEGY: none
-  script:
-    #
-    # Setup MFEM_DATA_DIR=${SHARED_REPOS_DIR}/mfem-data, see '.gitlab-ci.yml'
-    # and '.gitlab/configs/<machine>-config.yml'
-    #
-    - echo "MACHINE_NAME = ${MACHINE_NAME}"
-    - echo "AUTOTEST = ${AUTOTEST}"
-    - echo "AUTOTEST_COMMIT = ${AUTOTEST_COMMIT}"
-    - echo "SHARED_REPOS_DIR ${SHARED_REPOS_DIR}"
-    - mkdir -p ${SHARED_REPOS_DIR} && cd ${SHARED_REPOS_DIR}
-    - command -v flock || echo "Required command 'flock' not found"
-    - |
-      (
-        date
-        echo "Waiting to aquire lock on '$PWD/mfem-data.lock' ..."
-        # try to get an excusive lock on fd 9 (mfem-data.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
-        done
-        echo "Aquired lock on '$PWD/mfem-data.lock'"
-        date
-        # clone/update the mfem/data repo while holding the file lock on
-        # 'mfem-data.lock'
-        err=0
-        if [[ ! -d "mfem-data" ]]; then
-          git clone ${MFEM_DATA_REPO} "mfem-data"
-        else
-          cd "mfem-data" && git pull && cd ..
-        fi || err=1
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> mfem-data.lock
-    #
-    # Setup ${AUTOTEST_ROOT}/autotest:
-    #
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - mkdir -p ${AUTOTEST_ROOT} && cd ${AUTOTEST_ROOT}
-    - |
-      (
-        date
-        echo "Waiting to aquire lock on '$PWD/autotest.lock' ..."
-        # try to get an excusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
-        done
-        echo "Aquired lock on '$PWD/autotest.lock'"
-        date
-        # clone/update the autotest repo while holding the file lock on
-        # 'autotest.lock'
-        err=0
-        if [[ ! -d "autotest" ]]; then
-          git clone ${AUTOTEST_REPO}
-        else
-          cd autotest && git pull && cd ..
-        fi || err=1
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock
@@ -1,67 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-stages:
-  - setup
-  - allocate_resource
-  - build_and_test
-  - release_resource_and_report
-
-# Slurm shared allocation
-allocate_resource:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_corona
-  stage: allocate_resource
-  script:
-    - echo ${ALLOC_NAME}
-    - salloc --exclusive --nodes=1 --partition=mi60 --time=30 --no-shell --job-name=${ALLOC_NAME}
-  timeout: 6h
-  needs: [setup]
-
-# Build and test jobs, simply provide a spec
-rocm_gcc_8.3.1:
-  variables:
-    SPEC: "@develop%gcc@8.3.1+rocm amdgpu_target=gfx906"
-  extends: .build_and_test_on_corona
-  needs: [allocate_resource]
-
-# Release slurm allocation
-release_resource:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_corona
-  stage: release_resource_and_report
-  script:
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
-  needs: [rocm_gcc_8.3.1]
-
-# Jobs report
-report_job_success:
-  stage: release_resource_and_report
-  extends:
-    - .on_corona
-    - .report_job_success
-
-report_job_failure:
-  stage: release_resource_and_report
-  extends:
-    - .on_corona
-    - .report_job_failure
-
-include:
-  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/corona-config.yml
-  - local: .gitlab/configs/setup-build-and-test.yml
-  - local: .gitlab/configs/report-build-and-test.yml
@@ -9,31 +9,25 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

-stages:
-  - setup
-  - build_and_test
-  - report
+# GitLab pipelines configurations for the Lassen machine at LLNL
+
+.on_lassen:
+  tags:
+    - shell
+    - lassen
+  rules:
+    - if: '$CI_COMMIT_BRANCH =~ /_lnone/ || $ON_LASSEN == "OFF"' #run except if ...
+      when: never
+    - when: on_success
+
+# Spack helped builds
+# Generic lassen build job, extending build script
+.build_and_test_on_lassen:
+  extends: [.build_blueos_3_ppc64le_ib_script, .on_lassen]
+  stage: l_build_and_test
+  needs: []

 opt_mpi_cuda_xl_16_1_1_8:
  variables:
-    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=70"
+    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=sm_70"
  extends: .build_and_test_on_lassen
-
-# Jobs report
-report_job_success:
-  stage: report
-  extends:
-    - .on_lassen
-    - .report_job_success
-
-report_job_failure:
-  stage: report
-  extends:
-    - .on_lassen
-    - .report_job_failure
-
-include:
-  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/lassen-config.yml
-  - local: .gitlab/configs/setup-build-and-test.yml
-  - local: .gitlab/configs/report-build-and-test.yml
@@ -1,133 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-variables:
-  BASELINE_TEST: baseline
-
-stages:
-  - setup
-  - baseline_check
-  - baseline_report
-  - cleanup
-  - baseline_publish
-
-baselinecheck_mfem_intel_quartz:
-  extends: [.on_quartz]
-  stage: baseline_check
-  variables:
-    # TPLS_DIR is used in .gitlab/scripts/baseline to provide the tpls location
-    # when call the runtest script in MFEM test repo.
-    # Note: the value must be consistent with the setup performed in
-    #       .gitlab/configs/setup-baseline.yml.
-    TPLS_DIR: ${BUILD_ROOT}/tpls
-  script:
-    - echo ${BUILD_ROOT}
-    - echo ${TPLS_DIR}
-    # Used by the tests in MFEM/tests:
-    - export MFEM_TEST_NP=32
-    # The next script uses the following environment variables:
-    # * BASELINE_TEST, SYS_TYPE, CI_PROJECT_DIR, ARTIFACTS_DIR,
-    # * BUILD_ROOT, TPLS_DIR, MACHINE_NAME
-    - .gitlab/scripts/baseline
-  artifacts:
-    when: always
-    paths:
-      - ${ARTIFACTS_DIR}
-  allow_failure: true
-
-cleanup:
-  extends: .on_quartz
-  stage: cleanup
-  variables:
-    GIT_STRATEGY: none
-  script:
-    - echo "BUILD_ROOT=${BUILD_ROOT}"
-    - rm -rf "${BUILD_ROOT}" || true
-
-report_baseline:
-  extends: [.on_quartz]
-  stage: baseline_report
-  script:
-    - echo ${MACHINE_NAME}
-    - echo ${AUTOTEST}
-    - echo ${AUTOTEST_COMMIT}
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - cd ${AUTOTEST_ROOT}
-    - |
-      (
-        date
-        echo "Waiting to aquire lock on '$PWD/autotest.lock' ..."
-        # try to get an excusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
-        done
-        echo "Aquired lock on '$PWD/autotest.lock'"
-        date
-        # ----------------------
-        cd ${AUTOTEST_ROOT}/autotest || \
-          { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
-        mkdir -p ${MACHINE_NAME}
-        rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-${BASELINE_TEST}-${CI_COMMIT_REF_SLUG}"
-        rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir ${rundir})
-        cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
-        # We create an autotest-email.html file, because that's how we signal that there was a diff (temporary).
-        if [[ -f ${rundir}/${BASELINE_TEST}.err ]]; then
-          cp ${rundir}/${BASELINE_TEST}.err ${rundir}/autotest-email.html
-        fi
-        printf "%s\n" "" "Pipeline URL:" "$CI_PIPELINE_URL" \
-          >> ${rundir}/pipeline.txt
-        msg="GitLab CI log for ${BASELINE_TEST} on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
-        if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
-          git pull && \
-          git add ${rundir} && \
-          git commit -m "${msg}" && \
-          git push origin master
-        else
-          for file in ${rundir}/*; do
-            echo "------------------------------"
-            echo "Content of '$file'"
-            echo "******************************"
-            cat $file
-            echo "******************************"
-          done
-          rm -rf ${rundir} || true
-        fi
-        err=$?
-        # ----------------------
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock
-
-baselinepublish_mfem_quartz:
-  extends: [.on_quartz]
-  stage: baseline_publish
-  rules:
-    # - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "YES"'
-    - if: '$REBASELINE == "YES"'
-      when: manual
-  script:
-    - echo ${BUILD_ROOT}
-    - echo ${PWD}
-    - echo ${ARTIFACTS_DIR}
-    - ls -lA ${ARTIFACTS_DIR}
-    - .gitlab/scripts/rebaseline
-
-include:
-  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/quartz-config.yml
-  - local: .gitlab/configs/setup-baseline.yml
@@ -1,99 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-stages:
-  - setup
-  - allocate_resource
-  - build_and_test
-  - release_resource_and_report
-
-# Allocate
-allocate_resource:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_quartz
-  stage: allocate_resource
-  script:
-    - echo ${ALLOC_NAME}
-    - salloc --exclusive --nodes=1 --partition=pdebug --time=30 --no-shell --job-name=${ALLOC_NAME}
-  timeout: 6h
-
-# GitLab jobs for the Quartz machine at LLNL
-debug_ser_gcc_4_9_3:
-  variables:
-    SPEC: "%gcc@4.9.3 +debug~mpi"
-  extends: .build_and_test_on_quartz
-
-debug_ser_gcc_6_1_0:
-  variables:
-    SPEC: "%gcc@6.1.0 +debug~mpi"
-  extends: .build_and_test_on_quartz
-
-debug_par_gcc_6_1_0:
-  variables:
-    SPEC: "%gcc@6.1.0 +debug+mpi"
-  extends: .build_and_test_on_quartz
-
-opt_ser_gcc_6_1_0:
-  variables:
-    SPEC: "%gcc@6.1.0 ~mpi"
-  extends: .build_and_test_on_quartz
-
-opt_par_gcc_6_1_0:
-  variables:
-    SPEC: "%gcc@6.1.0"
-  extends: .build_and_test_on_quartz
-
-opt_par_gcc_6_1_0_sundials:
-  variables:
-    SPEC: "%gcc@6.1.0 +sundials"
-  extends: .build_and_test_on_quartz
-
-opt_par_gcc_6_1_0_petsc:
-  variables:
-    SPEC: "%gcc@6.1.0 +petsc ^petsc+mumps"
-  extends: .build_and_test_on_quartz
-
-opt_par_gcc_6_1_0_pumi:
-  variables:
-    SPEC: "%gcc@6.1.0 +pumi"
-  extends: .build_and_test_on_quartz
-
-# Release
-release_resource:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_quartz
-  stage: release_resource_and_report
-  script:
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
-
-# Jobs report
-report_job_success:
-  stage: release_resource_and_report
-  extends:
-    - .on_quartz
-    - .report_job_success
-
-report_job_failure:
-  stage: release_resource_and_report
-  extends:
-    - .on_quartz
-    - .report_job_failure
-
-include:
-  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/quartz-config.yml
-  - local: .gitlab/configs/setup-build-and-test.yml
-  - local: .gitlab/configs/report-build-and-test.yml
@@ -0,0 +1,165 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# GitLab pipelines configurations for the Quartz machine at LLNL
+
+.on_quartz:
+  tags:
+    - shell
+    - quartz
+  rules:
+    # Don’t run quartz jobs if...
+    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"'
+      when: never
+    # Don’t run autotest update if...
+    - if: '$CI_JOB_NAME =~ /update_autotest/ && $AUTOTEST != "YES"'
+      when: never
+    # Don’t run autotest update if...
+    - if: '$CI_JOB_NAME =~ /q_report/ && $AUTOTEST != "YES"'
+      when: never
+    # Report success on success status
+    - if: '$CI_JOB_NAME =~ /q_report_success/ && $AUTOTEST == "YES"'
+      when: on_success
+    # Report failure on failure status
+    - if: '$CI_JOB_NAME =~ /q_report_failure/ && $AUTOTEST == "YES"'
+      when: on_failure
+    # Always release resources
+    - if: '$CI_JOB_NAME =~ /release_resources/'
+      when: always
+    # Default is to run if previous stage succeeded
+    - when: on_success
+
+# Allocate
+q_allocate_resources:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_quartz
+  stage: q_allocate_resources
+  script:
+    - salloc --exclusive --nodes=1 --partition=pdebug --time=30 --no-shell --job-name=${ALLOC_NAME}
+  timeout: 6h
+
+# Release
+q_release_resources:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_quartz
+  stage: q_release_resources
+  script:
+    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
+    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
+
+# Release
+q_report_success:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_quartz
+  stage: q_release_resources
+  script:
+    - echo "Can only run if all the quartz jobs passed"
+    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
+    - cd ${AUTOTEST_ROOT}/autotest && git pull
+    - mkdir -p ${rundir}
+    - echo "The Quartz jobs were successful" > ${rundir}/gitlab.out
+    - git add ${rundir}
+    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
+    - git push origin master
+
+q_report_failure:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_quartz
+  stage: q_release_resources
+  script:
+    - echo "Runs if there was at least one failure on quartz"
+    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
+    - cd ${AUTOTEST_ROOT}/autotest && git pull
+    - mkdir -p ${rundir}
+    - echo "There was an error while running CI on Quartz" > ${rundir}/gitlab.err
+    - cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
+    - git add ${rundir}
+    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
+    - git push origin master
+
+# Spack helped builds
+# Generic quartz build job, extending build script
+.build_and_test_on_quartz:
+  extends: [.build_toss_3_x86_64_ib_script, .on_quartz]
+  stage: q_build_and_test
+
+# Build MFEM
+debug_ser_gcc_4_9_3:
+  variables:
+    SPEC: "%gcc@4.9.3 +debug~mpi"
+  extends: .build_and_test_on_quartz
+
+debug_ser_gcc_6_1_0:
+  variables:
+    SPEC: "%gcc@6.1.0 +debug~mpi"
+  extends: .build_and_test_on_quartz
+
+debug_par_gcc_6_1_0:
+  variables:
+    SPEC: "%gcc@6.1.0 +debug+mpi"
+  extends: .build_and_test_on_quartz
+
+opt_ser_gcc_6_1_0:
+  variables:
+    SPEC: "%gcc@6.1.0 ~mpi"
+  extends: .build_and_test_on_quartz
+
+opt_par_gcc_6_1_0:
+  variables:
+    SPEC: "%gcc@6.1.0"
+  extends: .build_and_test_on_quartz
+
+opt_par_gcc_6_1_0_sundials:
+  variables:
+    SPEC: "%gcc@6.1.0 +sundials"
+  extends: .build_and_test_on_quartz
+
+opt_par_gcc_6_1_0_petsc:
+  variables:
+    SPEC: "%gcc@6.1.0 +petsc ^petsc+mumps"
+  extends: .build_and_test_on_quartz
+
+opt_par_gcc_6_1_0_pumi:
+  variables:
+    SPEC: "%gcc@6.1.0 +pumi"
+  extends: .build_and_test_on_quartz
+
+# Baseline
+baselinecheck_mfem_intel_quartz:
+  extends: [.baselinecheck_mfem, .on_quartz]
+  needs: [setup]
+
+update_autotest:
+  extends: [.on_quartz]
+  needs: [baselinecheck_mfem_intel_quartz]
+  stage: baseline_to_autotest
+  script:
+    - rundir="quartz/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
+    - cd ${AUTOTEST_ROOT}/autotest && git pull
+    - mkdir -p ${rundir}
+    - cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
+    # We create an autotest-email.html file, because that's how we signal that there was a diff (temporary).
+    - |
+      if [[ -f ${rundir}/*.err ]]
+      then
+        cp ${rundir}/*.err ${rundir}/autotest-email.html
+      fi
+    - git add ${rundir}
+    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
+    - git push origin master
+
+baselinepublish_mfem_quartz:
+  extends: [.on_quartz, .rebaseline_mfem]
+  needs: [baselinecheck_mfem_intel_quartz]
@@ -1,88 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# locals
-glob_err=${BASELINE_TEST}.err
-base=${BASELINE_TEST}-${SYS_TYPE}
-base_diff=${base}.diff
-base_patch=${base}.patch
-base_out=${base}.out
-artifacts_path=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}
-
-# prepare
-cd ${BUILD_ROOT} || \
-  { echo "Invalid BUILD_ROOT=$BUILD_ROOT"; exit 1; }
-ln -snf ${CI_PROJECT_DIR} mfem
-cd tests
-[[ -d _${BASELINE_TEST} ]] && rm -rf _${BASELINE_TEST}
-mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
-
-# run
-if [[ "${MACHINE_NAME}" == "quartz" || "${MACHINE_NAME}" == "ruby" ]]; then
-  srun --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
-elif [[ ${MACHINE_NAME} == "corona" ]]; then
-  srun --nodes=1 -t 60 -p mi60 ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
-elif [[ ${MACHINE_NAME} == "lassen" ]]; then
-  lalloc 1 -q pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
-else
-  echo "Unknown machine: MACHINE_NAME=$MACHINE_NAME"
-  exit 1
-fi
-
-# post
-mkdir ${artifacts_path}
-
-if [[ -s ${glob_err} ]]
-then
-  echo "ERROR during ${BASELINE_TEST} execution";
-  echo "Here is the ${glob_err} file content";
-  cat ${glob_err}
-  cp ${glob_err} ${artifacts_path}/${glob_err}
-  exit 1;
-elif [[ ! -f ${base_patch} && ! -f ${base_out} ]]
-then
-  echo "Something went WRONG in ${BASELINE_TEST}:";
-  echo "Either ${base_patch} or ${base_out} should exists";
-  exit 1;
-elif [[ -f ${base_patch} ]]
-then
-  echo "${BASELINE_TEST}: Differences found, patch generated"
-  cp ${base_patch} ${artifacts_path}/${base_patch}
-elif [[ -f ${base_out} ]]
-then
-  echo "${BASELINE_TEST}: Differences found, replacement file generated"
-  cp ${base_out} ${artifacts_path}/${base_out}
-fi
-
-if [[ -f ${BASELINE_TEST}.out ]]; then
-  cp ${BASELINE_TEST}.out ${artifacts_path}
-fi
-
-# base_diff won't even exist if there is no difference.
-if [[ -f ${base_diff} ]]
-then
-  echo "${BASELINE_TEST}: Relevant differences (filtered diff) ..."
-  cat ${base_diff}
-  cp ${base_diff} ${artifacts_path}/${base_diff}
-  # We create a .err file, because that's how we signal that there was a diff.
-  cp ${base_diff} ${artifacts_path}/gitlab-${BASELINE_TEST}-${MACHINE_NAME}.err
-fi
-
-if [[ ! -s ${base_diff} ]]
-then
-  echo "${BASELINE_TEST}: PASSED"
-  true
-else
-  echo "${BASELINE_TEST}: FAILED"
-  false
-fi
@@ -1,49 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# There will be collision between corona and quartz baselines.
-# Once the corresponding files have been generated, we can switch to machine
-# specific ref.
-ARTIFACT_PATH=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}
-#ARTIFACT_PATH=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}-${MACHINE_NAME}
-
-PATCH_FILE=${ARTIFACT_PATH}.patch
-FULL_FILE=${ARTIFACT_PATH}.out
-DIFF_FILE=${ARTIFACT_PATH}.diff
-
-# There will be collision between corona and quartz baselines.
-# Once the corresponding files have been generated, we can switch to machine
-# specific ref.
-SAVED_NAME=baseline-${SYS_TYPE}.saved
-#SAVED_NAME=baseline-${SYS_TYPE}-${MACHINE_NAME}.saved
-
-cd ${BUILD_ROOT}/tests
-
-if [[ ! -f "${DIFF_FILE}" ]]
-then
-    echo "Nothing to be done: no relevant change in baseline"
-    exit 0
-elif [[ -f "${PATCH_FILE}" ]]
-then
-    patch "${SAVED_NAME}" < "${PATCH_FILE}"
-elif [[ -f "${FULL_FILE}" ]]
-then
-    cp "${FULL_FILE}" "${SAVED_NAME}"
-else
-    echo "File missing: expected ${PATCH_FILE} or ${FULL_FILE}"
-    exit 1
-fi
-
-git add "${SAVED_NAME}"
-git commit -m "${SYS_TYPE} (${MACHINE_NAME}) rebaselined in GitLab pipeline ${CI_PIPELINE_ID}"
-git push origin master
@@ -1,45 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-echo "Runs if there was at least one failure on ${MACHINE_NAME}"
-
-cd ${AUTOTEST_ROOT}/autotest || \
-  { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
-mkdir -p ${MACHINE_NAME}
-
-rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
-rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir)
-
-printf "%s\n" "Some 'build-and-test' jobs on ${MACHINE_NAME} FAILED." \
-  "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.err
-
-msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
-
-# Create 'autotest-email.html' to indicate failure:
-cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
-
-if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
-  git pull && \
-  git add ${rundir} && \
-  git commit -m "${msg}" && \
-  git push origin master
-else
-  for file in ${rundir}/*; do
-    echo "------------------------------"
-    echo "Content of '$file'"
-    echo "******************************"
-    cat $file
-    echo "******************************"
-  done
-  rm -rf ${rundir} || true
-fi
@@ -1,42 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-echo "Can only run if all the ${MACHINE_NAME} jobs passed"
-
-cd ${AUTOTEST_ROOT}/autotest || \
-  { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
-mkdir -p ${MACHINE_NAME}
-
-rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
-rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir)
-
-printf "%s\n" "The 'build-and-test' jobs on ${MACHINE_NAME} were SUCCESSFUL." \
-  "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.out
-
-msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
-
-if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
-  git pull && \
-  git add ${rundir} && \
-  git commit -m "${msg}" && \
-  git push origin master
-else
-  for file in ${rundir}/*; do
-    echo "------------------------------"
-    echo "Content of '$file'"
-    echo "******************************"
-    cat $file
-    echo "******************************"
-  done
-  rm -rf ${rundir} || true
-fi
@@ -1,42 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# This script takes a seed for a directory name and appends it with a counter
-# incremented until it can create a new directory with it.
-
-# Usage:
-#
-# Expects 1 argument: a string that is use as a seed for the directory name.
-#
-# > rundir="desired_name"
-# > rundir=$(./safe_create_rundir $rundir)
-
-set -o errexit
-set -o nounset
-
-rundir=${1:-""}
-if [[ -z ${rundir} ]]; then
-  >&2 echo "The script expects a string as argument for directory creation."
-  exit 1
-fi
-
-if ! mkdir ${rundir}; then
-  n=1
-  while ! mkdir ${rundir}_${n}
-  do
-    n=$((n+1))
-  done
-  rundir=${rundir}_${n}
-fi
-
-echo $rundir
@@ -0,0 +1,469 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+language: cpp
+
+os: linux
+dist: bionic
+
+stages:
+  - checks
+  - tests
+  - optional
+
+env:
+  global:
+    - HYPRE_ARCHIVE=v2.19.0.tar.gz
+      HYPRE_URL=https://github.com/hypre-space/hypre/archive/$HYPRE_ARCHIVE
+      HYPRE_TOP_DIR=hypre-2.19.0
+
+jobs:
+  include:
+
+    # ========================
+    #         Checks
+    # ========================
+    # - code-style
+    # - documentation
+    # - gitignore
+
+    - stage: checks
+      os: linux
+      dist: xenial
+      name: "code-style"
+      addons:
+        apt:
+          packages:
+            - astyle=2.05.1-0ubuntu1
+      script:
+        - cd ${TRAVIS_BUILD_DIR}
+        - cd tests/scripts
+        - ./runtest code-style
+
+    - stage: checks
+      os: linux
+      name: "documentation"
+      addons:
+        apt:
+          packages:
+            - doxygen
+            - graphviz
+      script:
+        - cd ${TRAVIS_BUILD_DIR}
+        - cd tests/scripts
+        - ./runtest documentation
+
+    - stage: checks
+      os: linux
+      name: "gitignore"
+      addons:
+        apt:
+          packages:
+            - mpich
+            - libmpich-dev
+      env: MPI=YES
+      before_script:
+        - cd ${TRAVIS_BUILD_DIR}
+        - mpicxx -v
+        - make config MFEM_USE_MPI=YES MFEM_MPI_NP=2
+        - make all -j3
+        - make test-noclean
+      script:
+        - cd tests/scripts
+        - ./runtest gitignore
+      cache:
+        ccache: true
+        directories:
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
+          rm -f Lib/*.{c,o}
+
+    # ========================
+    #  Optional Checks/Tests
+    # ========================
+    # - branch-history
+
+    - stage: optional
+      name: "branch-history"
+      if: branch != next
+      # need full git history for the binary/big files check
+      git:
+        depth: false
+      script:
+        - cd ${TRAVIS_BUILD_DIR}
+        # update master
+        - git fetch origin master:master
+        # checkout a branch (otherwise Travis works in detached head)
+        - git checkout -b travis_tests
+        - cd tests/scripts
+        - ./runtest branch-history
+
+    # ========================
+    #       Linux tests
+    # ========================
+    # - serial + debug
+    # - serial
+    # - parallel + debug
+    # - parallel
+
+    - stage: tests
+      os: linux
+      compiler: gcc
+      name: "Linux: Serial + Debug"
+      env: DEBUG=YES
+           MPI=NO
+           CODECOV=NO
+           MFEM_TEST_TARGET=check
+      cache:
+        ccache: true
+
+    - os: linux
+      compiler: gcc
+      name: "Linux: Serial"
+      env: DEBUG=NO
+           MPI=NO
+           CODECOV=NO
+           MFEM_TEST_TARGET=test
+      cache:
+        ccache: true
+
+    - os: linux
+      compiler: gcc
+      name: "Linux: Parallel + Debug"
+      addons:
+        apt:
+          # sources:
+            # - ubuntu-toolchain-r-test
+          packages:
+            # GCC 4.9
+            # - g++-4.9
+            # MPICH
+            - mpich
+            - libmpich-dev
+            # OpenMPI
+            # - openmpi-bin
+            # - libopenmpi-dev
+      env: DEBUG=YES
+           MPI=YES
+           CODECOV=NO
+           MFEM_TEST_TARGET=check
+           NPROCS=2
+      cache:
+        ccache: true
+        directories:
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
+          rm -f Lib/*.{c,o}
+
+    - os: linux
+      compiler: gcc
+      name: "Linux: Parallel"
+      addons:
+        apt:
+          # sources:
+            # - ubuntu-toolchain-r-test
+          packages:
+            # GCC 4.9
+            # - g++-4.9
+            # MPICH
+            - mpich
+            - libmpich-dev
+            # OpenMPI
+            # - openmpi-bin
+            # - libopenmpi-dev
+      env: DEBUG=NO
+           MPI=YES
+           CODECOV=YES
+           MFEM_TEST_TARGET=test
+           NPROCS=2
+      cache:
+        ccache: true
+        directories:
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
+          rm -f Lib/*.{c,o}
+
+    - os: linux
+      compiler: gcc
+      name: "Linux: Parallel (cmake)"
+      addons:
+        apt:
+          packages:
+            - mpich
+            - libmpich-dev
+      env: MPI=YES
+           NPROCS=2
+      script:
+        - cd ${TRAVIS_BUILD_DIR}
+        - mkdir ${TRAVIS_BUILD_DIR}/build
+        - cd ${TRAVIS_BUILD_DIR}/build
+        - cmake ..
+             -DMFEM_USE_MPI=ON
+             -DHYPRE_DIR=${TRAVIS_BUILD_DIR}/../$HYPRE_TOP_DIR/src/hypre
+             -DMFEM_MPI_NP=$NPROCS
+        - make -j3 mfem examples
+        - cd ${TRAVIS_BUILD_DIR}/build/tests/unit
+        - make -j3
+        - ctest --output-on-failure
+      cache:
+        ccache: true
+        directories:
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
+          rm -f Lib/*.{c,o}
+
+    # ========================
+    #      Mac OS X tests
+    # ========================
+    # - serial + debug
+    # - serial
+    # - parallel + debug
+    # - parallel
+
+    - os: osx
+      osx_image: xcode11.2
+      compiler: clang
+      name: "Mac: Serial + Debug"
+      addons:
+        homebrew:
+          packages:
+            - ccache
+      env: DEBUG=YES
+           MPI=NO
+           CODECOV=NO
+           MFEM_TEST_TARGET=check
+      cache:
+        ccache: true
+
+    - os: osx
+      osx_image: xcode11.2
+      compiler: clang
+      name: "Mac: Serial"
+      addons:
+        homebrew:
+          packages:
+            - ccache
+      env: DEBUG=NO
+           MPI=NO
+           CODECOV=NO
+           MFEM_TEST_TARGET=test
+      cache:
+        ccache: true
+
+    - os: osx
+      osx_image: xcode11.2
+      compiler: clang
+      name: "Mac: Parallel + Debug"
+      addons:
+        homebrew:
+          packages:
+            - ccache
+      env: DEBUG=YES
+           MPI=YES
+           CODECOV=NO
+           MFEM_TEST_TARGET=check
+           NPROCS=4
+           TMPDIR=/tmp
+      cache:
+        ccache: true
+        directories:
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+          - $HOME/local-cached
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
+          rm -f Lib/*.{c,o}
+
+    - os: osx
+      osx_image: xcode11.2
+      compiler: clang
+      name: "Mac: Parallel"
+      addons:
+        homebrew:
+          packages:
+            - ccache
+      env: DEBUG=NO
+           MPI=YES
+           CODECOV=YES
+           MFEM_TEST_TARGET=test
+           NPROCS=4
+           TMPDIR=/tmp
+      cache:
+        ccache: true
+        directories:
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+          - $HOME/local-cached
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
+          rm -f Lib/*.{c,o}
+
+before_install:
+   # No addon for brew yet, have to install OSX packages this way.
+   # - if [ $TRAVIS_OS_NAME == "osx" ] && [ $MPI == "YES" ]; then
+   #      brew install open-mpi;
+   #   fi
+
+   # Disable ccache while building dependencies that are cached:
+   - echo "before \$PATH = $PATH";
+     export PATH=${PATH//\/usr\/lib\/ccache:/};
+     echo "after  \$PATH = $PATH"
+
+   # On Mac OS X, build and cache OpenMPI 2.1.6:
+   - if [ $TRAVIS_OS_NAME == "osx" ] && [ $MPI == "YES" ]; then
+        if [ ! -e $HOME/local-cached/bin/mpicc ]; then
+           mkdir -p $HOME/builds && cd $HOME/builds &&
+           wget https://download.open-mpi.org/release/open-mpi/v2.1/openmpi-2.1.6.tar.bz2 &&
+           tar jxf openmpi-2.1.6.tar.bz2 &&
+           mkdir openmpi-build && cd openmpi-build &&
+           ../openmpi-2.1.6/configure --prefix=$HOME/local-cached &&
+           make -j3 all && make install;
+        fi;
+        PATH=$HOME/local-cached/bin:$PATH;
+        cd $TRAVIS_BUILD_DIR;
+     fi
+
+   # Update environment to find g++ 4.9 installation first.
+   # - if [ $TRAVIS_OS_NAME == "linux" ]; then
+   #      mkdir -p latest-gcc-symlinks;
+   #      ln -s /usr/bin/g++-4.9 latest-gcc-symlinks/g++;
+   #      ln -s /usr/bin/gcc-4.9 latest-gcc-symlinks/gcc;
+   #      ln -s /usr/bin/gcov-4.9 latest-gcc-symlinks/gcov;
+   #      export PATH=$PWD/latest-gcc-symlinks:$PATH;
+   #   fi
+
+   # Install tool to upload code coverage reports to coveralls.io
+   - if [ "$CODECOV" == "YES" ]; then
+        export PYTHONUSERBASE=$HOME/local;
+        pip install --user cpp-coveralls;
+        pip install --user pyyaml;
+        PATH=$HOME/local/bin:$PATH;
+     fi
+
+install:
+   # Set MPI compilers, print compiler version
+   - if [ $MPI == "YES" ]; then
+        if [ "$TRAVIS_OS_NAME" == "linux" ]; then
+           export MPICH_CC="$CC";
+           export MPICH_CXX="$CXX";
+        else
+           export OMPI_CC="$CC";
+           export OMPI_CXX="$CXX";
+           mpic++ --showme:version;
+        fi;
+        mpic++ -v;
+     else
+        $CXX -v;
+     fi
+
+   # Back out of the mfem directory to install the libraries
+   - cd ..
+
+   # hypre
+   - if [ $MPI == "YES" ]; then
+        if [ ! -e $HYPRE_TOP_DIR/src/hypre/lib/libHYPRE.a ]; then
+           wget $HYPRE_URL;
+           rm -rf $HYPRE_TOP_DIR;
+           tar xvzf $HYPRE_ARCHIVE;
+           cd $HYPRE_TOP_DIR/src;
+           ./configure --disable-fortran CC=mpicc CXX=mpic++;
+           make -j3;
+           cd ../..;
+        else
+            echo "Reusing cached $HYPRE_TOP_DIR/";
+        fi;
+        ln -s $HYPRE_TOP_DIR hypre;
+     else
+        echo "Serial build, not using hypre";
+     fi
+
+   # METIS, use a mirror because the original source server is not always up.
+   # Original url:
+   #    http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/OLD/metis-4.0.3.tar.gz
+   - if [ $MPI == "YES" ]; then
+        if [ ! -e metis-4.0/libmetis.a ]; then
+           wget https://mfem.github.io/tpls/metis-4.0.3.tar.gz;
+           tar xvzf metis-4.0.3.tar.gz;
+           make -j3 -C metis-4.0.3/Lib CC="$CC" OPTFLAGS="-O2";
+           rm -rf metis-4.0;
+           mv metis-4.0.3 metis-4.0;
+        else
+           echo "Reusing cached metis-4.0/";
+        fi;
+     fi
+
+   # Re-enable ccache on linux; enable ccache on mac os:
+   - if [ $TRAVIS_OS_NAME == "linux" ]; then
+        export PATH="/usr/lib/ccache:$PATH";
+     else
+        if [ $TRAVIS_OS_NAME == "osx" ]; then
+           export PATH="/usr/local/opt/ccache/libexec:$PATH";
+        fi;
+     fi
+
+   - printf "which \$CC  = "; which $CC;
+     printf "which \$CXX = "; which $CXX
+
+script:
+   # Compiler
+   - if [ $MPI == "YES" ]; then
+        export MYCXX=mpic++;
+        export MAKE_CXX_FLAG=MPICXX=$MYCXX;
+     else
+        export MYCXX="$CXX";
+        export MAKE_CXX_FLAG=CXX=$MYCXX;
+     fi
+
+   # Print the compiler version
+   - $MYCXX -v
+
+   # Set some variables
+   - cd $TRAVIS_BUILD_DIR;
+     CPPFLAGS="";
+     SKIP_TEST_DIRS="";
+     if [ "$CODECOV" == "YES" ]; then
+        CPPFLAGS="--coverage -g";
+     fi;
+     if [ "$TRAVIS_OS_NAME" != "linux" ] || [ "$DEBUG" == "YES" ]; then
+        CPPFLAGS+=" -pedantic -Wall -Werror";
+     fi
+
+   # Configure the library
+   - make config MFEM_USE_MPI=$MPI MFEM_DEBUG=$DEBUG $MAKE_CXX_FLAG
+        MFEM_MPI_NP=$NPROCS CPPFLAGS="$CPPFLAGS"
+   # Show the configuration
+   - make info
+   # Build the library
+   - make -j3
+   # Build the examples and the miniapps
+   - make -j3 all
+   # Run tests
+   - make $MFEM_TEST_TARGET SKIP_TEST_DIRS="$SKIP_TEST_DIRS"
+
+after_success:
+   - if [ "$CODECOV" == "YES" ]; then
+        coveralls --include fem --include general --include linalg --include
+           mesh --exclude /usr --gcov-options '\-lp' --root $TRAVIS_BUILD_DIR;
+     fi
@@ -8,138 +8,40 @@
                               https://mfem.org


-Version 4.3.1 (development)
+Version 4.2.1 (development)
 ===========================
- Added support for automatic differentiation. Users can select between 
-  native implementation and external library implementation at the 
-  configuration phase. A parallel and two serial examples are implemented
-  in the autodiff miniapp directory.
+- Added initial support for GPU-accelerated versions of PETSc that works with
+  MFEM_USE_CUDA if PETSc has been configured with CUDA support. Examples 1 and 9
+  in the examples/petsc directory have been modified to work with --device cuda.
+  Examples with GAMG (ex1p) and SLEPc (ex11p) are also provided.

- Added support for mesh preprocessing to resolve fine scale problem data
-  before simulation. This feature uses adaptive mesh refinement to control the
-  associated data oscillation error. See the new Example 30/30p.
+- Memory management:
+  * Added method Device::SetMemoryTypes that can be used to change the default
+    host and device MemoryTypes before Device setup.
+  * In class MemoryManager, added methods GetDualMemoryType and
+    SetDualMemoryType; dual MemoryTypes are used to determine the second
+    MemoryType (host or device) when only one MemoryType is specified in methods
+    of class Memory.
+  * Added Memory constructor for setting both the host and device MemoryTypes.
+  * Switched the default behavior of device memory allocations so that they
+    are deferred until the device pointer is needed.
+  * Added a second Umpire device MemoryType, DEVICE_UMPIRE_2, with
+    corresponding allocator that can be set with the method
+    MemoryManager::SetUmpireDevice2AllocatorName.
+  * Added HOST_PINNED MemoryType and a pinned host allocator for CUDA and HIP.

- Switched from Artistic Style (astyle) version 2.05.1 to version 3.1 for code
-  formatting. See the "make style" target.
+- Added support for Caliper: a library to integrate performance profiling
+  capabilities into applications. See examples/caliper for more details.

- Split the fem/fe.?pp files into separate files in the new fem/fe/ directory
-  to simplify and clarify the organization of FiniteElement classes.
+- Added support for explicit vectorization in the high-performance templated
+  code for Fujitsu's A64FX ARM microprocessor architecture.

- Added support for hr-adaptivity using TMOP-based error estimator.
+- Added AlgebraicCeedSolver that does matrix-free algebraic p-multigrid for
+  diffusion problems with the Ceed backend.

- Coefficient::SetTime now propagates the new time into internally stored
-  Coefficient objects.
-
- Added initial support for google-benchmarks in the tests/benchmarks directory.
-  It can be enabled with MFEM_USE_BENCHMARK=YES.
-
- Added Binder (mybinder.org) configuration files for C++ MFEM Jupyter Notebooks
-  with inline GLVis visualization as well as a new examples/jupyter/ directory
-  with a sample notebook based on Example 1. Implementation based on xeus-cling,
-  github.com/jupyter-xeus/xeus-cling + xeus-glvis, github.com/GLVis/xeus-glvis.
-
- Added 'double' atomicAdd implementation for previous versions of CUDA.
-
- Adding lowest order Nedelec and Raviart-Thomas basis functions on wedge
-  shaped elements.
-
- Added initial support for meshes with pyramidal elements, including several
-  pyramidal meshes in the data/ directory and support for the lowest order H1,
-  Nedelec, Raviart-Thomas, and L2 basis functions on pyramids.
-
- Updated the hypre interface according to changes in hypre-2.22.1. The ADS
-  solver is now fully working on GPUs.
-
- Tetrahedral meshes no longer need to be reordered to support high order
-  Nedelec basis functions. This will allow future support for Nedelec basis
-  functions on wedges and pyramids which are not amenable to reordering. The
-  ReorientTetMesh method of the Mesh and ParMesh classes has been deprecated.
-
- Gmsh meshes where all elements have zero physical tag (the default Gmsh
-  output format if no physical groups are defined) are now successfully loaded,
-  and elements are reassigned attribute number 1.
-
- Added new miniapps that use the ParELAG library, its hybrid smoothers, and the
-  hierarchy of spaces created by the element-based AMG (AMGe) methodology in
-  ParELAG to build multigrid solvers for H(curl) and H(div) forms. See the
-  miniapps/parelag directory for more details.
-
- Fixed several MinGW build issues on Windows.
-
- Remove the 'u' flag in the ar command, to update all files in the archive,
-  avoiding file name collisions from different subdirectories.
-
- Added initial TMOP-based capabilities for surface fitting and tangential
-  relaxation in the mesh-optimizer and pmesh-optimizer miniapps.
- 
- Added ParMesh Adjaceny Set (adjset) creation support to the Conduit Mesh
-  Blueprint MFEM wrapper functions in ConduitDataCollection.
-
- `HypreParVector` and `Vector` now support move semantics, and the copy
-  constructor for `HypreParVector` now copies the local vector data.
-
-
-Version 4.3, released on July 29, 2021
-======================================
-
-Discretization improvements
---------------------------
- Variable order spaces, p- and hp-refinement. This is the initial (serial)
-  support for variable-order FiniteElementCollection and FiniteElementSpace.
-  The new method FiniteElementSpace::SetElementOrder can be called to set an
-  arbitrary order for each mesh element. The conforming interpolation matrix
-  will now automatically constrain p- and hp- interfaces, enabling general
-  hp-refinement in both 2D and 3D, on uniform or mixed NC meshes. Support for
-  parallel variable-order spaces will follow shortly.
-
- Extended the support for field transfer between high-order and low-order
-  refined finite element spaces to include: dual fields and H1 fields (both
-  primary and dual). These are illustrated in the lor-transfer miniapp.
-
- Improved libCEED integration, including support for VectorCoefficient,
-  ConvectionIntegrator, and VectorConvectionNLFIntegrator with libCEED backends.
-
- Extending support for L2 basis functions using MapTypes VALUE and INTEGRAL in
-  linear interpolators and GridFunction "GetValue" methods.
-
- Changed the interface for the error estimator and implemented the Kelly error
-  indicator for scalar-valued problems, supported in serial and parallel builds.
-
- Added support for the "BR2" discontinuous Galerkin discretization for
-  diffusion via DGDiffusionBR2Integrator (see Example 14/14p).
-
- Added convective and skew-symmetric integrators for the nonlinear term in the
-  Navier-Stokes equations.
-
- Added new classes DenseSymmetricMatrix and SymmetricMatrixCoefficient for
-  efficient evaluation of symmetric matrix coefficients. This replaces the now
-  deprecated EvalSymmetric in MatrixCoefficient. Added DiagonalMatrixCoefficient
-  for clarity, which is a typedef of VectorCoefficient.
-
- Added support for nonscalar coefficient with VectorDiffusionIntegrator.
-
-Linear and nonlinear solvers
----------------------------
- Added support for AMG preconditioners on GPUs based on the hypre library
-  (version 2.22.0 or later). These include BoomerAMG, AMS and ADS and most
-  MFEM examples that use hypre have been ported to support this functionality.
-  The GPU preconditioners require that both hypre and MFEM are built with CUDA
-  support. Hypre builds with CUDA and unified memory are also supported and
-  can be used with `-d cuda:uvm` as a command-line option.
-
- Added support for AMG preconditioners for non-symmetric systems (e.g.
-  advection-dominated problems) using hypre's approximate ideal restriction
-  (AIR) AMG. Requires hypre version 2.14.0 or newer. Usage is illustrated in
-  example 9/9p.
-
- Added new functionality for constructing low-order refined discretizations and
-  solvers, see the LORDiscretization and LORSolver classes. A new basis type for
-  H(curl) and H(div) spaces is introduced to give spectral equivalence. This
-  functionality is illustrated in the LOR solvers miniapp in miniapps/solvers.
-
- Generalized the Multigrid class to support non-geometric multigrid. Previous
-  functionality, based on FiniteElementSpaceHierarchy, is now available in the
-  derived class GeometricMultigrid.
+- Introduced new options for the mesh-explorer miniapp to visualize the actual
+  element attributes in parallel meshes while retaining the visualization of
+  the domain decomposition.

 - Introduced solver interface for linear problems with constraints, a few
  concrete solvers that implement the interface, and a demonstration of their
@@ -150,18 +52,19 @@ Linear and nonlinear solvers
  as described in Barker and Kolev 2020 (https://doi.org/10.1002/nla.2348). See
  Example 3p and linalg/auxiliary.?pp.

- Improved interface for using the Ginkgo library, including: support for matrix-
-  free operators in Ginkgo solvers, new wrappers for Ginkgo preconditioners, HIP
-  support, and reduction of unnecessary data copies.
+- Added a new miniapp block-solvers that compares the performance of various
+  solvers for mixed finite element discretization of the second order scalar
+  elliptic equations. Currently available solvers in the miniapp include a
+  block-diagonal preconditioner that is based on approximate Schur complement
+  (implemented in ex5p), and a newly implemented solver DivFreeSolver, which
+  exploits a multilevel decomposition of the Raviart-Thomas space and its
+  divergence-free subspace. See the miniapps/solvers directory for more details.

- Added initial support for hypre's mixed integer (mixedint) capability, which
-  uses different data types for local and global indices in order to save memory
-  in large problems. This capability requires that hypre was configured with the
-  --enable-mixedint option. Note that this option is currently tested only in
-  ex1p, ex3p, and ex4p, and may not work in more general settings.
+- Added a new miniapp for computing (signed) distance functions to a point
+  source or zero level set. See miniapps/shifted/distance.cpp.

- Added AlgebraicCeedSolver that does matrix-free algebraic p-multigrid for
-  diffusion problems with the Ceed backend.
+- Added matrix-free GPU-enabled implementations of GradientInterpolator and
+  IdentityInterpolator.

 - Added interface to MUMPS direct solver. Its usage is demonstrated in ex25p.
  See http://mumps.enseeiht.fr/ for more details. Supported versions >= 5.1.1.
@@ -169,17 +72,6 @@ Linear and nonlinear solvers
 - Added three ESDIRK time integrators: implicit trapezoid rule, L-stable
  ESDIRK-32, and A-stable ESDIRK-33.

- Implemented a variable step-size IMEX (VSSIMEX) method for the Navier miniapp.
-
- Implemented an adaptive linear solver tolerance option for NewtonSolver based
-  on the algorithm of Eisenstat and Walker.
-
-Meshing improvements
--------------------
- Added support for reading high-order Lagrange meshes in VTK format. Arbitrary-
-  orders and all element types are supported. See the VTK blog for more info:
-  https://blog.kitware.com/wp-content/uploads/2018/09/Source_Issue_43.pdf.
-
 - Introduced a new non-conforming mesh format that fixes known inconsistencies
  of legacy "MFEM mesh v1.1" NC format and works consistently in both serial and
  parallel. ParMesh::ParPrint can now print non-conforming AMR meshes that can
@@ -188,26 +80,109 @@ Meshing improvements
  NC data files are compatible with serial code, e.g., can be viewed with serial
  GLVis. Loading of legacy NC mesh files is still supported.

- Added FMS support (https://github.com/CEED/FMS) to mfem. FMS can represent
-  unstructured high-order meshes with general high-order finite element fields
-  on them. When enabled, mfem can convert data collections to/from FMS data
-  collections in memory. In addition, an FMS data collection class was added so
-  the convert-dc miniapp can read and generate data files in FMS format.
+- Added support for 1D non-conforming meshes (which can be useful for parallel
+  load balancing and derefinement).
+
+- Added a "scaled Jacobian" visualization option in the Mesh Explorer miniapp to
+  help identify elements with poor mesh quality.
+
+- Added support for the "BR2" discontinuous Galerkin discretization for
+  diffusion via DGDiffusionBR2Integrator (see Example 14/14p).
+
+- Generalized the Multigrid class to support non-geometric multigrid. The
+  previous functionality, based on FiniteElementSpaceHierarchy, is now available
+  in the derived class GeometricMultigrid.
+
+- Upgraded the Catch unit test framework from version 2.13.0 to version 2.13.2.
+
+- The TMOP mesh optimization algorithms were extended to GPU:
+   - QualityMetric #1, #2, #7 and #77 are available in 2D, #302, #303, #315
+     and #321 in 3D
+   - Both AnalyticAdaptTC and DiscreteAdaptTC TargetConstructor are available
+   - Kernels for normalization and limiting have been added
+   - The AdvectorCG now also supports AssemblyLevel::PARTIAL
+
+- Added a new command line boolean option (`--all`) to the unit tests to launch
+  *all* non-regression tests.
+
+- Added support for different modes of QuadratureInterpolator on GPU.
+  The layout (QVectorLayout::byNODES|byVDIM) and the tensor products modes can
+  be enabled before calling the Mult, Values, Derivatives, PhysDerivatives and
+  Determinants methods.
+
+- Implemented a filter method for the Navier miniapp to stabilize highly
+  turbulent flows in direct numerical simulation.
+
+- Added HIP support to the CMake build system.
+
+- Added support for reading high-order Lagrange meshes in VTK format. Arbitrary-
+  orders and all element types are supported. See the VTK blog for more info:
+  https://blog.kitware.com/wp-content/uploads/2018/09/Source_Issue_43.pdf.
+
+- Added support for reading VTK meshes in XML format.
+
+- Added partial assembly and device support to Example 25/25p, with diagonal
+  preconditioning.
+
+- Implemented a variable step-size IMEX (VSSIMEX) method for the Navier miniapp.

 - Added new mesh quality metrics and improved the untangling capabilities of the
  TMOP-based mesh optimization algorithms.

- The TMOP mesh optimization algorithms were extended to GPU:
-  * QualityMetric 1, 2, 7, 77 are available in 2D, 302, 303, 315, 321 in 3D
-  * Both AnalyticAdaptTC and DiscreteAdaptTC TargetConstructor are available
-  * Kernels for normalization and limiting have been added
-  * The AdvectorCG now also supports AssemblyLevel::PARTIAL
+- Added convective and skew-symmetric integrators for the nonlinear term in the
+  Navier-Stokes equations.
+
+- Added new miniapp directory mtop/ with optimization-oriented block parametric
+  non-linear form and abstract integrators. Two new miniapps, ParHeat and
+  SeqHeat, demonstrate parallel and sequential implementation of gradients
+  evaluation for linear diffusion with discrete density.
+
+- Changed the interface for the error estimator.
+
+- Implemented the Kelly error indicator for scalar-valued problems, supported
+  in serial and parallel builds.
+
+- Added new classes DenseSymmetricMatrix and SymmetricMatrixCoefficient for
+  efficient evaluation of symmetric matrix coefficients. This replaces the now
+  deprecated EvalSymmetric in MatrixCoefficient. Added DiagonalMatrixCoefficient
+  for clarity, which is a typedef of VectorCoefficient.
+
+- Added support for AMG preconditioners for non-symmetric systems (e.g.
+  advection-dominated problems) using hypre's approximate ideal restriction
+  (AIR) AMG. Requires hypre version 2.14.0 or newer. Usage is illustrated in
+  example 9/9p.
+
+- Implemented an adaptive linear solver tolerance option for NewtonSolver based
+  on the algorithm of Eisenstat and Walker.
+
+- Added support for nonscalar coefficient with VectorDiffusionIntegrator.
+
+- Extending support for L2 basis functions using MapTypes VALUE and INTEGRAL in
+  linear interpolators and GridFunction "GetValue" methods.
+
+- Variable order spaces, p- and hp-refinement. This is the initial (serial)
+  support for variable-order FiniteElementCollection and FiniteElementSpace.
+  The new method FiniteElementSpace::SetElementOrder can be called to set an
+  arbitrary order for each mesh element. The conforming interpolation matrix
+  will now automatically constrain p- and hp- interfaces, enabling general
+  hp-refinement in both 2D and 3D, on uniform or mixed NC meshes. Support for
+  parallel variable-order spaces will follow shortly.

 - Added support for creating refined meshes for all element types (e.g. by
  splitting high-order elements into low-order refined elements), including
  mixed meshes. The LOR Transfer miniapp (miniapps/tools/lor-transfer.cpp) now
  supports meshes with any element geometry.

+- Gitlab CI: use Spack (and Uberenv) to automate the build of TPLs.
+
+- Added new miniapps demonstrating: 1) the use of GSLIB for overlapping grids,
+  see gslib/schwarz_ex1, and 2) coupling different physics in different domains,
+  see navier/cht. Note that gslib v1.0.7 is require (see INSTALL for details).
+
+- Added a new, very simple example (ex0 and parallel version ex0p). This
+  example solves a simple Poisson problem using H1 elements (the same problem as
+  ex1), but is intended to be extremely simple and approachable for new users.
+
 - Meshes consisting of any type of elements (including mixed meshes) can be
  converted to all-simplex meshes using Mesh::MakeSimplicial.

@@ -220,147 +195,56 @@ Meshing improvements
  requisite periodic vertex mappings can be created with
  Mesh::CreatePeriodicVertexMapping.

- Added support for 1D non-conforming meshes (which can be useful for parallel
-  load balancing and derefinement).
+- Added support for transferring dual fields between high-order and low-order
+  refined finite element spaces using the transposed versions of the
+  L2ProjectionGridTransfer operators. This functionality is illustrated in the
+  lor-transfer miniapp.
+
+- Improved interface for using the Ginkgo library, including: support for matrix-
+  free operators in Ginkgo solvers, new wrappers for Ginkgo preconditioners, HIP
+  support, and reduction of unnecessary data copies.
+
+- Added initial support for hypre's mixed integer (mixedint) capability, which
+  uses different data types for local and global indices in order to save memory
+  in large problems. This capability requires that hypre was configured with the
+  --enable-mixedint option. Note that this option is currently tested only in
+  ex1p and may not work in more general settings.
+
+- Added support for transferring fields (primary and dual) between high-order
+  and low-order refined H1 finite element spaces using the
+  L2ProjectionH1GridTransfer operators. This functionality is demonstrated
+  through the lor-transfer miniapp when run with the -h1 option.
+
+- Added new functionality for constructing low-order refined discretizations and
+  solvers, see the LORDiscretization and LORSolver classes. A new basis type for
+  H(curl) and H(div) spaces is introduced to give spectral equivalence. This
+  functionality is illustrated in the LOR solvers miniapp in miniapps/solvers.

 - Added sample meshes in the `data` subdirectory showing the reference elements
  of the six currently supported element types; ref-segment.mesh,
  ref-triangle.mesh, ref-square.mesh, ref-tetrahedron.mesh, ref-cube.mesh, and
  ref-prism.mesh.

-High-performance computing
--------------------------
- Added initial support for GPU-accelerated versions of PETSc that works with
-  MFEM_USE_CUDA if PETSc has been configured with CUDA support. Examples 1 and 9
-  in the examples/petsc directory have been modified to work with --device cuda.
-  Examples with GAMG (ex1p) and SLEPc (ex11p) are also provided.
-
- Added support for explicit vectorization in the high-performance templated
-  code for Fujitsu's A64FX ARM microprocessor architecture.
-
- Added support for different modes of QuadratureInterpolator on GPU.
-  The layout (QVectorLayout::byNODES|byVDIM) and the tensor products modes can
-  be enabled before calling the Mult, Values, Derivatives, PhysDerivatives and
-  Determinants methods.
-
- Added method Device::SetMemoryTypes that can be used to change the default
-  host and device MemoryTypes before Device setup.
-
- In class MemoryManager, added methods GetDualMemoryType and SetDualMemoryType;
-  dual MemoryTypes are used to determine the second MemoryType (host or device)
-  when only one MemoryType is specified in methods of class Memory.
-
- Added Memory constructor for setting both the host and device MemoryTypes.
-
- Switched the default behavior of device memory allocations so that they are
-  deferred until the device pointer is needed.
-
- Added a second Umpire device MemoryType, DEVICE_UMPIRE_2, with corresponding
-  allocator that can be set with the method SetUmpireDevice2AllocatorName.
-
- Added HOST_PINNED MemoryType and a pinned host allocator for CUDA and HIP.
-
- Added matrix-free GPU-enabled implementations of GradientInterpolator and
-  IdentityInterpolator.
-
-New and updated examples and miniapps
-------------------------------------
- Added a new, very simple example (ex0 and parallel version ex0p). This example
-  solves a simple Poisson problem using H1 elements (the same problem as ex1),
-  but is intended to be extremely simple and approachable for new users.
-
- Added new miniapps demonstrating: 1) the use of GSLIB for overlapping grids,
-  see gslib/schwarz_ex1, and 2) coupling different physics in different domains,
-  see navier/cht. Note that gslib v1.0.7 is require (see INSTALL for details).
-
- Added a new miniapp for computing (signed) distance functions to a point
-  source or zero level set. See miniapps/shifted/distance.cpp.
-
 - Added a high-order extension of the shifted boundary method to solve PDEs on
  non body-fitted meshes. This is illustrated in the new Shifted Diffusion
  miniapp, see miniapps/shifted/diffusion.cpp.

- Added new miniapp directory mtop/ with optimization-oriented block parametric
-  non-linear form and abstract integrators. Two new miniapps, ParHeat and
-  SeqHeat, demonstrate parallel and sequential implementation of gradients
-  evaluation for linear diffusion with discrete density.
-
- Added a new miniapp block-solvers that compares the performance of various
-  solvers for mixed finite element discretization of the second order scalar
-  elliptic equations. Currently available solvers in the miniapp include a
-  block-diagonal preconditioner that is based on approximate Schur complement
-  (implemented in ex5p), and a newly implemented solver DivFreeSolver, which
-  exploits a multilevel decomposition of the Raviart-Thomas space and its
-  divergence-free subspace. See the miniapps/solvers directory for more details.
-
- Introduced new options for the mesh-explorer miniapp to visualize the actual
-  element attributes in parallel meshes while retaining the visualization of the
-  domain decomposition.
-
- Added partial assembly and device support to Example 25/25p, with diagonal
-  preconditioning.
-
- Implemented a filter method for the Navier miniapp to stabilize highly
-  turbulent flows in direct numerical simulation.
-
-Improved testing
----------------
- Transitioned from Travis to GitHub Action for testing/CI on GitHub.
-
- Use Spack (and Uberenv) to automate TPL building in LLNL GitLab tests.
-
- Extended `make test` to include GPU tests when MFEM is built with CUDA or HIP
-  support.
-
- Added a set of suggested git hooks for developers in config/githooks.
-
- Added support for Caliper: a library to integrate performance profiling
-  capabilities into applications. See examples/caliper for more details.
-
- Added a new command line boolean option (`--all`) to the unit tests to launch
-  *all* non-regression tests.
-
- Upgraded the Catch unit test framework from version 2.13.0 to version 2.13.2.
-
-Miscellaneous
-------------
- The following integrations have updated minimum version requirements:
-  * CUDA >= 10.1.168
-  * Ginkgo >= 1.4.0
-  * GSLIB >= 1.0.7
-  * HIOP >= 0.4
-  * HYPRE >= 2.20.0 for mixedint support
-  * HYPRE >= 2.22.0 for CUDA support
-  * libCEED >= 0.8
-  * PETSc >= 3.15.0 for CUDA support
-  * RAJA >= 0.13.0
-  see INSTALL for more details.
-
- Added a "scaled Jacobian" visualization option in the Mesh Explorer miniapp to
-  help identify elements with poor mesh quality.
-
- Added support for reading VTK meshes in XML format.
-
 - Added makefile rule to generate TAGS table for vi or Emacs users.

- Added HIP support to the CMake build system.
+libCEED integration improvements
+--------------------------------
+- Refactor the libCEED integration

- Various other simplifications, extensions, and bugfixes in the code.
+- Add support for VectorCoefficient with libCEED backends.

-API changes
-----------
- Added an abstract interface `mfem::FaceRestriction` for `H1FaceRestriction`
-  and `L2FaceRestriction`.
-  In order to conform with the semantic of `MultTranspose` in `mfem::Operator`,
-  `mfem::FaceRestriction::MultTranspose` now sets instead of adding values, and
-  `mfem::FaceRestriction::AddMultTranspose` should replace previous calls to
-  `mfem::FaceRestriction::MultTranspose`.
+- Add support for ConvectionIntegrator, and VectorConvectionNLFIntegrator with
+  libCEED backends.


 Version 4.2, released on October 30, 2020
 =========================================

-High-performance computing
+High-Performance Computing
 --------------------------
 - Added support for explicit vectorization in the high-performance templated
  code, which can now take advantage of specific classes on the following
@@ -442,6 +326,9 @@ Linear and nonlinear solvers
  matrix with the function HypreParMatrixFromBlocks. This could be useful for
  solving block systems with parallel direct solvers such as STRUMPACK.

+- Added CUDA support for SUNDIALS ODE integrators. See the updated SUNDIALS
+  modification of Example 9/9p.
+
 - Added wrappers for hypre's flexible GMRES solver and the new parallel ILU
  preconditioner. The latter requires hypre version 2.19.0 or later.

@@ -552,7 +439,7 @@ New and updated examples and miniapps
    L2, with partial assembly support in Example 24/24p.
  * Weak Dirichlet boundary conditions (Nitsche) to the NURBS miniapp.

-Data management and visualization
+Data management and Visualization
 ---------------------------------
 - Added support for ADIOS2 for parallel I/O with ParaView visualization. See
  Examples 5, 9, 12, 16. The classes adios2stream and ADIOS2DataCollection
@@ -1,66 +0,0 @@
-cff-version: 1.2.0
-message: "If you use MFEM, please cite it as follows."
-authors:
- family-names: "MFEM Team"
-title: "MFEM: Modular Finite Element Methods [Software]"
-doi: 10.11578/dc.20171025.1248
-url: "https://mfem.org"
-preferred-citation:
-  type: article
-  authors:
-  - family-names: "Anderson"
-    given-names: "Robert"
-    orcid: "https://orcid.org/0000-0002-3508-9944"
-  - family-names: "Andrej"
-    given-names: "Julian"
-    orcid: "https://orcid.org/0000-0001-7661-4840"
-  - family-names: "Barker"
-    given-names: "Andrew"
-    orcid: "https://orcid.org/0000-0003-3572-911X"
-  - family-names: "Bramwell"
-    given-names: "Jamie"
-  - family-names: "Camier"
-    given-names: "Jean-Sylvain"
-    orcid: "https://orcid.org/0000-0003-2421-1999"
-  - family-names: "Cerveny"
-    given-names: "Jakub"
-    orcid: "https://orcid.org/0000-0003-4231-2531"
-  - family-names: "Dobrev"
-    given-names: "Veselin"
-    orcid: "https://orcid.org/0000-0003-1793-5622"
-  - family-names: "Dudouit"
-    given-names: "Yohann"
-    orcid: "https://orcid.org/0000-0001-5831-561X"
-  - family-names: "Fisher"
-    given-names: "Aaron"
-  - family-names: "Kolev"
-    given-names: "Tzanio"
-    orcid: "https://orcid.org/0000-0002-2810-3090"
-  - family-names: "Pazner"
-    given-names: "Will"
-    orcid: "https://orcid.org/0000-0003-4885-2934"
-  - family-names: "Stowell"
-    given-names: "Mark"
-    orcid: "https://orcid.org/0000-0002-5389-7435"
-  - family-names: "Tomov"
-    given-names: "Vladimir"
-    orcid: "https://orcid.org/0000-0002-1846-6816"
-  - family-names: "Akkerman"
-    given-names: "Ido"
-    orcid: "https://orcid.org/0000-0002-5937-0300"
-  - family-names: "Dahm"
-    given-names: "Johann"
-    orcid: "https://orcid.org/0000-0001-9657-3564"
-  - family-names: "Medina"
-    given-names: "David"
-  - family-names: "Zampini"
-    given-names: "Stefano"
-    orcid: "https://orcid.org/0000-0002-0435-0433"
-  doi: "10.1016/j.camwa.2020.06.009"
-  journal: "Computers \\& Mathematics with Applications"
-  month: 1
-  start: 42 # First page number
-  end: 74 # Last page number
-  title: "MFEM: A Modular Finite Element Methods Library"
-  volume: 81
-  year: 2021
@@ -16,7 +16,7 @@ set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH

 # Require C++11 and disable compiler-specific extensions
 set(CMAKE_CXX_STANDARD 11)
-if (MFEM_USE_GINKGO)
+if (MFEM_USE_GINKGO) 
  set(CMAKE_CXX_STANDARD 14)
 endif()
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -54,7 +54,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.3.1)
+set(${PROJECT_NAME}_VERSION 4.2.1)

 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -90,11 +90,6 @@ include("${CMAKE_CURRENT_SOURCE_DIR}/config/XSDKDefaults.cmake")

 # Enable languages.
 enable_language(CXX)
-if (MINGW)
-   # MinGW GCC does not expose the functions jn/_jn, yn/_yn (used in Example
-   # 25/25p) unless we use '-std=gnu++11':
-   set(CMAKE_CXX_EXTENSIONS ON)
-endif()
 if (MFEM_USE_CUDA)
   if (MFEM_USE_HIP)
      message(FATAL_ERROR " *** MFEM_USE_HIP cannot be combined with MFEM_USE_CUDA.")
@@ -107,7 +102,7 @@ if (MFEM_USE_CUDA)
   endif()
   enable_language(CUDA)
   set(CMAKE_CUDA_STANDARD 11)
-   if (MFEM_USE_GINKGO)
+   if (MFEM_USE_GINKGO) 
     set(CMAKE_CUDA_STANDARD 14)
   endif()
   set(CMAKE_CUDA_STANDARD_REQUIRED ON)
@@ -251,12 +246,6 @@ if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
    message(FATAL_ERROR " *** MFEM_USE_LEGACY_OPENMP requires MFEM_THREAD_SAFE=ON.")
  endif()
  find_package(OpenMP REQUIRED)
-  set(OPENMP_LIBRARIES ${OpenMP_CXX_LIBRARIES})
-  if(APPLE)
-    # On macOS, the compiler needs additional help to find the <omp.h> header.
-    # See issue #2642 for more information.
-    include_directories(${OpenMP_CXX_INCLUDE_DIRS})
-  endif(APPLE)
 endif()

 # SuiteSparse (before SUNDIALS which may depend on KLU)
@@ -338,11 +327,7 @@ if (MFEM_USE_AMGX)
 endif()

 if (MFEM_USE_CONDUIT)
-    find_package(Conduit REQUIRED conduit relay blueprint)
-endif()
-
-if (MFEM_USE_FMS)
-    find_package(FMS REQUIRED fms)
+    find_package(Conduit REQUIRED conduit relay blueprint )
 endif()

 # Axom/Sidre
@@ -372,12 +357,6 @@ if (MFEM_USE_HIOP)
  # find_package updates HIOP_FOUND, HIOP_INCLUDE_DIRS, HIOP_LIBRARIES
 endif()

-# CoDiPack package
-if (MFEM_USE_CODIPACK)
-  find_package(CODIPACK REQUIRED)
-  # find_package updates CODIPACK_FOUND, CODIPACK_INCLUDE_DIRS, CODIPACK_LIBRARIES
-endif()
-
 # OCCA
 if (MFEM_USE_OCCA)
   find_package(OCCA REQUIRED)
@@ -393,11 +372,6 @@ if (MFEM_USE_UMPIRE)
   find_package(UMPIRE REQUIRED)
 endif()

-# GOOGLE-BENCHMARK
-if (MFEM_USE_BENCHMARK)
-   find_package(Benchmark REQUIRED)
-endif()
-
 # Caliper
 if (MFEM_USE_CALIPER)
  find_package(Caliper REQUIRED)
@@ -426,11 +400,6 @@ if (MFEM_USE_MKL_CPARDISO)
  endif()
 endif()

-# PARELAG
-if (MFEM_USE_PARELAG)
-   find_package(PARELAG REQUIRED)
-endif()
-
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
  if (APPLE)
@@ -454,11 +423,10 @@ endif()
 #    With newer versions of SuiteSparse which include METIS header using 64-bit
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
-set(MFEM_TPLS OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS PETSC
-    SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB NETCDF
+set(MFEM_TPLS MPI_CXX OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS PETSC
+    SLEPC MESQUITE MUMPS STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
-    CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG MPI_CXX)
-
+    CUSPARSE MKL_CPARDISO AMGX CALIPER)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")
@@ -477,9 +445,6 @@ include_directories(${TPL_INCLUDE_DIRS})
 if (OPENMP_FOUND)
  message(STATUS "MFEM: using package OpenMP")
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
-  if (MFEM_USE_CUDA)
-    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=${OpenMP_CXX_FLAGS}")
-  endif()
 endif()

 message(STATUS "MFEM build type: CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}")
@@ -522,9 +487,13 @@ set(MFEM_INSTALL_DIR ${CMAKE_INSTALL_PREFIX} CACHE PATH
 # Declaring the library
 mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
 # message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
-target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES})
+if (CMAKE_VERSION VERSION_GREATER 2.8.11)
+  target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES})
+else()
+  target_link_libraries(mfem ${TPL_LIBRARIES})
+endif()
 if (MINGW)
-  target_link_libraries(mfem PRIVATE ws2_32)
+  target_link_libraries(mfem ws2_32)
 endif()
 set_target_properties(mfem PROPERTIES VERSION "${mfem_VERSION}")
 set_target_properties(mfem PROPERTIES SOVERSION "${mfem_VERSION}")
@@ -563,21 +532,15 @@ endif()
 set(MFEM_CUSTOM_TARGET_PREFIX CACHE STRING "")

 #-------------------------------------------------------------------------------
-# Examples, miniapps, benchmarks and testing
+# Examples, miniapps, and testing
 #-------------------------------------------------------------------------------

-# Enable testing and benchmarks if required
+# Enable testing if required
 if (MFEM_ENABLE_TESTING)
  enable_testing()
  set(MFEM_ALL_TESTS_TARGET_NAME tests)
  add_mfem_target(${MFEM_ALL_TESTS_TARGET_NAME} OFF)
  add_subdirectory(tests EXCLUDE_FROM_ALL)
-
-  # Create a target for all benchmarks and, optionally, enable it.
-  set(MFEM_ALL_BENCHMARKS_TARGET_NAME benchmarks)
-  add_mfem_target(${MFEM_ALL_BENCHMARKS_TARGET_NAME}
-                  ${MFEM_ENABLE_GOOGLE_BENCHMARKS})
-  add_subdirectory(tests/benchmarks EXCLUDE_FROM_ALL)
 endif()

 # Define a target that all examples and miniapps will depend on.
@@ -587,11 +550,7 @@ add_custom_target(${MFEM_EXEC_PREREQUISITES_TARGET_NAME})
 # Create a target for all examples and, optionally, enable it.
 set(MFEM_ALL_EXAMPLES_TARGET_NAME examples)
 add_mfem_target(${MFEM_ALL_EXAMPLES_TARGET_NAME} ${MFEM_ENABLE_EXAMPLES})
-if (MFEM_ENABLE_EXAMPLES)
-  add_subdirectory(examples) #install examples if enabled
-else()
-  add_subdirectory(examples EXCLUDE_FROM_ALL)
-endif()
+add_subdirectory(examples EXCLUDE_FROM_ALL)

 # Create a target for all miniapps and, optionally, enable it.
 set(MFEM_ALL_MINIAPPS_TARGET_NAME miniapps)
@@ -601,7 +560,6 @@ add_subdirectory(miniapps EXCLUDE_FROM_ALL)
 # Target to build all executables, i.e. everything.
 add_custom_target(exec)
 add_dependencies(exec
-  ${MFEM_ALL_BENCHMARKS_TARGET_NAME}
  ${MFEM_ALL_EXAMPLES_TARGET_NAME}
  ${MFEM_ALL_MINIAPPS_TARGET_NAME}
  ${MFEM_ALL_TESTS_TARGET_NAME})
@@ -4,9 +4,7 @@

 <p align="center">
 <a href="https://github.com/mfem/mfem/blob/master/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-BSD-brightgreen.svg"></a>
-<a href="https://github.com/mfem/mfem/actions?query=workflow%3Arepo-check+branch%3Amaster"><img alt="Repo check" src="https://github.com/mfem/mfem/actions/workflows/repo-check.yml/badge.svg?branch=master"></a>
-<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuild-analysis+branch%3Amaster"><img alt="Build Analysis" src="https://github.com/mfem/mfem/actions/workflows/mfem-analysis.yml/badge.svg?branch=master"></a>
-<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuilds-and-tests+branch%3Amaster"><img alt="Builds and Tests" src="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml/badge.svg?branch=master"></a>
+<a href="https://travis-ci.org/mfem/mfem"><img alt="Build Status" src="https://travis-ci.org/mfem/mfem.svg?branch=master"></a>
 <a href="https://ci.appveyor.com/project/mfem/mfem"><img alt="Build Status" src="https://ci.appveyor.com/api/projects/status/19non9sqm6msi2wy?svg=true"></a>
 <a href="https://mfem.github.io/doxygen/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
 </p>
@@ -42,7 +40,6 @@ back to them before issuing pull requests:
  - [New Feature Development](#new-feature-development)
  - [Developer Guidelines](#developer-guidelines)
  - [Pull Requests](#pull-requests)
-  - [MFEM PR Rules](#mfem-pr-rules)
  - [Pull Request Checklist](#pull-request-checklist)
  - [Master/Next Workflow](#masternext-workflow)
  - [Releases](#releases)
@@ -66,18 +63,14 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
  development branches off `mfem:master`.
 - Please follow the [developer guidelines](#developer-guidelines), in particular
  with regards to documentation and code styling.
- Please do not commit large/binary files to the central repository (use a fork
-  instead).
- Pull requests should be issued toward `mfem:master`. Make sure
-  to check the items off the [Pull Request Checklist](#pull-request-checklist) and
-  follow the [MFEM PR Rules](#mfem-pr-rules).
+- Pull requests  should be issued toward `mfem:master`. Make sure
+  to check the items off the [Pull Request Checklist](#pull-request-checklist).
 - When your contribution is fully working and ready to be reviewed, add
  the `ready-for-review` label.
 - PRs are treated similarly to journal submission with an "editor" assigning two
  reviewers to evaluate the changes.
 - The reviewers have 3 weeks to evaluate the PR and work with the author to
  fix issues and implement improvements.
- During review there should be no force pushes/rewriting history in the branch.
 - After approval, MFEM developers merge the PR manually in the [mfem:next branch](#masternext-workflow).
 - After a week of testing in `mfem:next`, the original PR is merged in `mfem:master`.
 - We use [milestones](https://github.com/mfem/mfem/milestones) to coordinate the
@@ -98,8 +91,8 @@ The MFEM source code has the following structure:
 ```
  .
  ├── config
-  │   ├── cmake
-  │   └── githooks
+  │   └── cmake
+  │       └── ...
  ├── data
  ├── doc
  ├── examples
@@ -107,14 +100,12 @@ The MFEM source code has the following structure:
  │   ├── caliper
  │   ├── ginkgo
  │   ├── hiop
-  │   ├── jupyter
  │   ├── petsc
  │   ├── pumi
  │   ├── sundials
  |   └── superlu
  ├── fem
  │   ├── ceed
-  │   ├── fe
  │   ├── qinterp
  │   └── tmop
  ├── general
@@ -123,7 +114,6 @@ The MFEM source code has the following structure:
  ├── mesh
  ├── miniapps
  │   ├── adjoint
-  │   ├── autodiff
  │   ├── common
  │   ├── electromagnetics
  │   ├── gslib
@@ -131,7 +121,6 @@ The MFEM source code has the following structure:
  │   ├── mtop
  │   ├── navier
  │   ├── nurbs
-  │   ├── parelag
  │   ├── performance
  │   ├── shifted
  │   ├── solvers
@@ -140,10 +129,10 @@ The MFEM source code has the following structure:
  └── tests
      ├── convergence
      ├── gitlab
-      ├── mem_manager
      ├── par-mesh-format
      ├── scripts
      └── unit
+          └── ...
 ```

 #### Main directories and classes
@@ -329,22 +318,15 @@ Before you can start, you need a GitHub account, here are a few suggestions:
    change the code by default.

 - Code specifics
-  - All new public, protected, and private classes, methods, data members, and
-    functions have Doxygen-style documentation in source comments.
-  - In addition to arguments and functionality, documentation should include the
-    current limitations of the code, any background information that is
-    implicitly assumed in the implementation, and the ownership and lifetime
-    of data.
+  - All significant new classes, methods and functions have Doxygen-style
+    documentation in source comments.
  - Consistent code styling is enforced with `make style` in the top-level
    directory. This requires [Artistic Style](http://astyle.sourceforge.net) (we
-    specifically use version 3.1). See also the file `config/mfem.astylerc`.
+    specifically use version 2.05.1). See also the file `config/mfem.astylerc`.
  - Use `mfem::out` and `mfem::err` instead of `std::cout` and `std::cerr` in
    internal library code. (You can use `std` in examples and miniapps.)
  - When manually resolving conflicts during a merge, make sure to mention the
    conflicted files in the commit message.
-  - All significant new features and changes should be documented in CHANGELOG.
-  - New examples and miniapps should have documentation on the MFEM webpage.
-

 ### Pull Requests

@@ -381,10 +363,6 @@ Before you can start, you need a GitHub account, here are a few suggestions:
  two reviewers to evaluate the changes. The reviewers have 3 weeks to evaluate
  the PR and work with the author to implement improvements and fix issues.

- Once the `ready-for-review` label has been applied and reviewers have been
-  assigned, the PR is considered under review. To help with the review process
-  there should be no force pushes/rewriting history in the branch.
-
 - After approval, the PR is [tested](#masternext-workflow) for a week with
  other approved PRs in the `mfem:next` branch.

@@ -392,101 +370,20 @@ Before you can start, you need a GitHub account, here are a few suggestions:
  `mfem:next`, see the [README](tests/scripts/README) file in that directory
  for more details.

- Track the GitHub Actions and Appveyor [continuous integration](#automated-testing)
+- Track the Travis CI, Github Actions and Appveyor [continuous integration](#automated-testing)
  builds at the end of the PR. These should generally run clean, so address any
  errors as soon as possible. Please ask if you are unsure how to do that.

- Note that some tests, such as the `branch-history` check in GitHub Actions
-  are safeguards that are allowed to fail in certain cases.
+- Note that some tests, such as the `branch-history` check in Travis and Github
+  Actions are safeguards that are allowed to fail in certain cases.

 - Other tests, such as the `code-style`, `documentation` and `gitignore`
-  checks in GitHub Actions enforce MFEM-specific rules which are explained in
-  the error messages and the `tests/scripts` directory.
-
- Also note that the tests `branch-history` and `repos-checks` found in GitHub
-  Actions can be triggered automatically before each push using git hooks. See
-  the [git hooks README](config/githooks/README.md) for a detailed explanation.
+  checks in Travis and Github Actions enforce MFEM-specific rules which are
+  explained in the error messages and the `tests/scripts` directory.

 - If triggered, track the status of the LLNL GitLab tests. If failing, ask
  one of the _LLNL developers_ for details.

-
-### MFEM PR Rules
-
-The Pull Request (PR) approval process in MFEM is similar to the approval of papers in a peer-reviewed journal. In particular:
-
-1. There is an MFEM board of "editors" that evaluates new PRs and assigns "reviewers" for each PR.
-
-2. The assigned reviewers are responsible to carefully review and test the proposed PR.
-
-3. A PR can be (manually) merged in the *next* branch only if 2 of the assigned reviewers have approved it and it has passed internal testing. This merge can be performed by any of the assigned reviewers or by any of the editors.
-
-4. A PR can be merged in the *master* branch only if it has been tested successfully for a week in *next* and an editor has (optionally) taken a final look. This merge can be performed only by one of the editors.
-
-#### Responsibilities of Editors
-
-The current list of MFEM editors is:
-
- @v-dobrev (Veselin Dobrev)
- @tzanio (Tzanio Kolev)
- @pazner (Will Pazner)
- @mlstowell (Mark Stowell)
-
-**The responsibilities of the editors are:**
-
-1. To assign appropriate milestone and labels for new PRs, e.g. *bugfix*, *minor*, *api-change*, *high-impact*, etc.
-
-2. To assign at least 2 reviewers for new PRs. An editor can also be a reviewer. The editor, reviewers, and author should be listed as "Assignees" on the GitHub PR page. After assignment, the `in-review` label should be added.
-
-3. To complete the initial PR evaluation and assignments in a timely manner: 1 week from submission.
-
-4. To assist reviewers when they need help with their reviews (but also to stay out of the way when they don't).
-
-5. To remind the reviewers about timely completion of their review.
-
-6. To take a final look and complete the PR merge in *master*. The final look step is optional and shouldn't take more than 3 days.
-
-7. The assignment of bugfixes should be expedited proportional to their importance, e.g. in some cases the editor can assign much shorter review window.
-
-#### Responsibilities of Reviewers
-
-Everyone on the MFEM team can be asked to serve as a reviewer on a PR in their area of expertise.
-
-**The responsibilities of the reviewers are:**
-
-1. To let the editors know if the proposed assignment is not a good match for them.
-
-2. To communicate with the PR author, provide feedback and work with them to resolve issues.
-
-3. To ensure the quality of the PR by making sure that the code adheres to the [Developer Guidelines](#developer-guidelines), e.g. all methods, data members, and functions have documentation, including data ownership and lifetime, new examples/miniapps have a corresponding PR in mfem/web, major features have `CHANGELOG` entries, etc.
-
-3. To seek help from the editors in case of difficulties.
-
-4. To complete the review in a timely manner: 3 weeks from assignment.
-
-5. To test the PR thoroughly before merging in *next*. The PR author is also encouraged to perform testing and inform the reviewers about the results.
-
-6. To monitor the PR impact on the testing in the *next* branch and alert the editors that the PR is ready for merging in *master*.
-
-7. The review of bugfixes should be expedited proportional to their importance. The review window can be much less than three weeks in such cases.
-
-#### Responsibilities of Authors
-
-Authors should clearly indicate when a PR is ready for review (before that the PR should be marked as `Draft` or `[WIP]`).
-
-**The responsibilities of the authors are:**
-
-1. To follow the instructions and PR checklist in the `CONTRIBUTING.md` document in the MFEM repository.
-
-2. To respond to reviewer feedback in a timely manner.
-
-3. Authors are encouraged to perform testing and inform the reviewers about the results.
-
-4. Authors can use the "Reviewers" section of the GitHub PR page to suggest reviewers, but the "Assignees" section will show who the editor has assigned to do the reviews.
-
-5. To indicate when the PR is ready for review by adding the `ready-for-review` label.
-
-
 ### Pull Request Checklist

 Before a PR can be merged, it should satisfy the following:
@@ -502,7 +399,7 @@ Before a PR can be merged, it should satisfy the following:
    - [ ] Does `make` or `cmake` have a new target?
    - [ ] Did the requirements or the installation process change? *(rare)*
 - [ ] Update continuous integration server configurations if necessary (e.g. with new version requirements for each of MFEM's dependencies)
-    - [ ] `.github`
+    - [ ] `.travis.yml`
    - [ ] `.appveyor.yml`
 - [ ] Update `.gitignore`:
    - [ ] Check if `make distclean; git status` shows any files that were generated from the source by the project (not an IDE) but we don't want to track in the repository.
@@ -540,9 +437,7 @@ Before a PR can be merged, it should satisfy the following:
     - [ ] The miniapps go at the end of the page, and are usually listed only under a specific "Application (PDE)" category.
     - [ ] Add a short description of the miniapp in the "Extensive Examples" section of `features.md`.
 - [ ] New capability:
-   - [ ] All new public, protected, and private classes, methods, data members, and functions have full Doxygen-style documentation in source comments. Documentation should include descriptions of member data, function arguments and return values, template parameters, and prerequisites for calling new functions.
-   - [ ] Pointer arguments and return values must specify whether ownership is being transferred or lent with the call.
-   - [ ] Any new functions should include descriptions of their intended use e.g. for internal use only, user-facing, etc., along with references to example code whenever possible/appropriate.
+   - [ ] All significant new classes, methods and functions have Doxygen-style documentation in source comments.
   - [ ] Consider adding new sample runs in existing examples to highlight the new capability.
   - [ ] Consider saving cool simulation pictures with the new capability in the Confluence gallery (LLNL only) or submitting them, via pull request, to the gallery section of the `mfem/web` repo.
   - [ ] If this is a major new feature, consider mentioning it in the short summary inside `README` *(rare)*.
@@ -553,7 +448,6 @@ Before a PR can be merged, it should satisfy the following:
 - [ ] (LLNL only) After merging:
   - [ ] Update internal tests to include the new features.

-
 ### Master/Next Workflow

 MFEM uses a `master`/`next`-branch workflow as described below:
@@ -622,7 +516,7 @@ MFEM uses a `master`/`next`-branch workflow as described below:
    - [ ] `doc/CodeDocumentation.conf.in`
 - [ ] Check that version requirements for each of MFEM's dependencies are documented in `INSTALL` and up-to-date
 - [ ] Check that continuous integration server configurations reflect the dependency version requirements of the new release
-    - [ ] `.github`
+    - [ ] `.travis.yml`
    - [ ] `.appveyor.yml`
 - [ ] Update the `CHANGELOG` to organize all release contributions
 - [ ] Review the whole source code once over
@@ -645,10 +539,8 @@ MFEM uses a `master`/`next`-branch workflow as described below:
    - Update version and shortlinks in `src/index.md` and `src/download.md`.
    -  Use [cloc-1.62.pl](http://cloc.sourceforge.net/) and `ls -lh` to estimate the SLOC and the tarball size in `src/download.md`.

-
 ## LLNL Workflow

-
 ### Mirroring on Bitbucket

 - The GitHub `master` and `next` branches are mirrored to the LLNL institutional
@@ -668,17 +560,16 @@ MFEM uses a `master`/`next`-branch workflow as described below:
  - `mfem:gh-next` -- Bleeding-edge development version, may be broken, use at
     your own risk.

-
 ### Mirroring on GitLab

 - MFEM repository is also mirrored on the LLNL GitLab instance, in a
  semi-automated manner.

 - This instance is meant to complete CI testing with tests on Livermore
-  Computing systems. GitLab pipeline status is reported in the corresponding
+  Computing systems. Gitlab pipeline status is reported in the corresponding
  GitHub pull request.

- In GitLab pipelines, TPLs (dependencies) are built using Spack, driven by Uberenv.
+- In Gitlab pipelines, TPLs (dependencies) are built using Spack, driven by Uberenv.

 - No change to the MFEM repo can be made on this instance.

@@ -687,23 +578,18 @@ MFEM uses a `master`/`next`-branch workflow as described below:
 MFEM has several levels of automated testing running on GitHub, as well as on
 local Mac and Linux workstations, and Livermore Computing clusters at LLNL.

-In addition, developers can set local git hooks to run some quick checks on
-commit or push, see the [README](config/githooks/README.md) in the `config/githooks`
-directory.
-
-
 ### Linux and Mac smoke tests
-We use GitHub Actions to drive the default tests on the `master` and `next`
-branches. See the `.github/workflows` files and the logs at
-[https://github.com/mfem/mfem/actions](https://github.com/mfem/mfem/actions).
+We use Travis CI and Github Actions to drive the default tests on the `master`
+and `next` branches. See the `.travis` file and the logs at
+[https://travis-ci.org/mfem/mfem](https://travis-ci.org/mfem/mfem).

-Testing using GitHub Actions should be kept lightweight, as there is a time
-constraint on jobs. Two virtual machines are configured - Mac (OS X) and Linux.
+Testing using Travis CI and Github Actions should be kept lightweight, as there
+is a time constraint on jobs. Two virtual machines are configured - Mac (OS X)
+and Linux.

 - Tests on the `master` branch are triggered whenever a PR is issued on this branch.
 - Tests on the `next` branch are currently scheduled to run each night.

-
 ### Windows smoke test
 We use Appveyor to test building with the MS Visual C++ compiler in a Windows
 environment, as well as to test the CMake build. See the `.appveyor` file and the
@@ -713,7 +599,6 @@ build logs at
 CMake is used to generate the MSVC Project files and drive the build.  A release
 and debug build is performed with a simple run of `ex1` to verify the executable.

-
 ### Tests at LLNL

 - We mirror the `master` and `next` branches internally (to `gh-master` and
@@ -459,22 +459,10 @@ MFEM_USE_UMPIRE = YES/NO
   discovery, provision, and management of memory on machines with multiple
   memory devices like NUMA and GPUs.

-MFEM_USE_BENCHMARK = YES/NO
-   Enables support for Google Benchmark, a library to support the benchmarking
-   of functions, in the tests/benchmarks directory.
-
 MFEM_USE_HIOP = YES/NO
   Enable the usage of HiOp (https://github.com/LLNL/hiop) in MFEM. HiOp is an
   HPC solver for nonlinear optimization problems.

-MFEM_USE_CODIPACK = YES/NO
-  Enable automatic differentiation using the CoDiPack library.
-  www.scicomp.uni-kl.de/codi/
-
-MFEM_USE_ADFORWARD = YES/NO
-  Enable forward mode for AD packages. This option is valid
-  only if the AD package supports two modes (backward/forward).
-
 MFEM_USE_CUDA = YES/NO
   Enables support for CUDA devices in MFEM. CUDA is a parallel computing
   platform and programming model for general computing on graphical processing
@@ -486,7 +474,7 @@ MFEM_USE_HIP = YES/NO
   Enables support for AMD devices in MFEM. HIP is a heterogeneous-compute
   interface for portability developed by AMD that can target both AMD and
   NVIDIA GPUs. The variable HIP_ARCH is used to specify the AMD GPU processor
-   used during compilation (by default, HIP_ARCH=gfx900). When enabled, this
+   used during compilation (by default, HIP_ARCH=gfx900).  When enabled, this
   option uses the HIP_* build options, see below.

 MFEM_USE_RAJA = YES/NO
@@ -528,18 +516,6 @@ MFEM_USE_CALIPER = YES/NO
   profiling at runtime with Caliper's configuration API. Alternatively, one
   can configure Caliper through environment variables or config files.

-MFEM_USE_FMS = YES/NO
-   Enables support for the FMS library which consists of the DataCollection
-   sub-class mfem::FMSDataCollection for I/O in FMS formats, see the header file
-   fem/fmsdatacollection.hpp. In addition, this option enables in-memory
-   convetion routines between FMS's FmsDataCollection structure and MFEM's
-   DataCollection class, see the header file fem/fmsconvert.hpp.
-
-MFEM_USE_PARELAG = YES/NO
-   Enables the miniapps that use the ParELAG library. MFEM does not currently
-   use ParELAG. In fact, ParELAG is dependent on MFEM. Therefore, this option
-   currently only concerns the miniapps.
-
 MFEM_BUILD_TAG = (any value)
   An optional tag to characterize the build. Exported to config/config.mk.
   Can be used to identify the MFEM build from other makefiles.
@@ -564,9 +540,8 @@ The specific libraries and their options are:
  See also the "Specific options for hypre" section at the end of this file.
  URL: https://github.com/hypre-space/hypre and https://www.llnl.gov/casc/hypre
  Options: HYPRE_OPT, HYPRE_LIB.
-  Versions: HYPRE >= 2.10.0b  (HYPRE built without CUDA)
-            HYPRE >= 2.20.0   (HYPRE built with '--enable-mixedint')
-            HYPRE >= 2.22.1   (HYPRE built with CUDA)
+  Versions: HYPRE >= 2.10.0b,
+            HYPRE >= 2.20.0 for '--enable-mixedint' support.

 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
  MFEM_USE_METIS_5 = YES (default is to use METIS 4).
@@ -640,7 +615,7 @@ The specific libraries and their options are:
  and dependencies of specific modules, see the Ginkgo webpage below.
  URL: https://ginkgo-project.github.io
  Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or Debug).
-  Versions: Ginkgo >= 1.4.0.
+  Versions: Ginkgo >= 1.4.0. 

 - AmgX (optional), used when MFEM_USE_AMGX = YES.
  URL: https://github.com/NVIDIA/AMGX
@@ -709,12 +684,7 @@ The specific libraries and their options are:
 - HiOp (optional), used when MFEM_USE_HIOP = YES.
  URL: https://github.com/LLNL/hiop
  Options: HIOP_OPT, HIOP_LIB.
-  Versions: HIOP >= 0.4.6.
-
- CoDiPack (optiobal), used with MFEM_USE_CODIPACK = YES
-  URL: https://www.scicomp.uni-kl.de/codi/
-  Options: CODIPACK_OPT
-  Versions: 1.9.3
+  Versions: HIOP >= 0.4.

 - GSLIB (optional), used when MFEM_USE_GSLIB = YES. The gslib library must be
  built prior to the MFEM build, as follows: download gslib-1.0.7, untar it at
@@ -752,10 +722,10 @@ The specific libraries and their options are:
  Versions: libCEED >= 0.8.

 - RAJA (optional), used when MFEM_USE_RAJA = YES.
-  Beginning with MFEM v4.3, only RAJA v0.14.0+ is supported.
+  Beginning with MFEM v4.3, only RAJA v0.13.0+ is supported.
  URL: https://github.com/LLNL/RAJA
  Options: RAJA_DIR, RAJA_OPT, RAJA_LIB.
-  Versions: RAJA >= 0.14.0.
+  Versions: RAJA >= 0.13.0.

 - Caliper (optional), used when MFEM_USE_CALIPER = YES.
  URL: https://github.com/LLNL/Caliper
@@ -766,12 +736,7 @@ The specific libraries and their options are:
  Umpire requires camp when the Umpire version is >= 3.0.0.
  URL: https://github.com/LLNL/Umpire
  Options: UMPIRE_DIR, UMPIRE_OPT, UMPIRE_LIB.
-  Versions: Umpire >= 3.0.0.
-
- Benchmark, used when MFEM_USE_BENCHMARK = YES.
-  URL: https://github.com/google/benchmark
-  Options: BENCHMARK_DIR, BENCHMARK_LIB.
-  Versions: Benchmark >= 1.5.6.
+  Versions: Umpire >= 2.0.0.

 - MPFR (optional), used when MFEM_USE_MPFR = YES.
  URL: http://mpfr.org, it depends on the GMP library: https://gmplib.org
@@ -788,15 +753,6 @@ The specific libraries and their options are:
  URL: https://zlib.net
  Options: ZLIB_OPT, ZLIB_LIB.

- FMS (optional), used when MFEM_USE_FMS = YES.
-  URL: https://github.com/CEED/FMS
-  Options: FMS_OPT, FMS_LIB.
-  Versions: FMS >= 0.2.
-
- ParELAG, used when MFEM_USE_PARELAG = YES.
-  URL: https://github.com/LLNL/parelag
-  Options: PARELAG_DIR, PARELAG_OPT, PARELAG_LIB.
-
 Building with CMake
 ===================
 The MFEM build system consists of two steps: configuration and compilation.
@@ -921,8 +877,6 @@ MFEM_USE_MPFR
 MFEM_USE_ZLIB
 MFEM_USE_PUMI
 MFEM_USE_HIOP
-MFEM_USE_CODIPACK
-MFEM_USE_ADFORWARD
 MFEM_USE_CUDA
 MFEM_USE_OCCA
 MFEM_USE_CEED
@@ -930,9 +884,6 @@ MFEM_USE_RAJA
 MFEM_USE_UMPIRE
 MFEM_USE_SIDRE
 MFEM_USE_CALIPER
-MFEM_USE_FMS
-MFEM_USE_BENCHMARK
-MFEM_USE_PARELAG

 The following options are CMake specific:

@@ -982,15 +933,11 @@ The CMake build system adds auto-detection for the following packages/libraries:
 - POSIXCLOCKS
 - PUMI
 - HIOP
- - CoDiPack
 - OCCA
 - RAJA
 - UMPIRE
 - AXOM - Used when MFEM_USE_SIDRE is enabled
 - CALIPER
- - FMS
- - BENCHMARK
- - ParELAG

 The following built-in CMake packages are also used:

@@ -1008,7 +955,7 @@ config/config.hpp.in:

   cp config/config.hpp.in config/_config.hpp

-The file config/_config.hpp can then be edited to enable desired options. The
+The file config/_config.hpp can then be edited to enable desired options.  The
 MFEM library is simply a combination of all object files obtained by compiling
 the .cpp source files in the source directories: general, linalg, mesh, and fem.

@@ -1016,7 +963,7 @@ the .cpp source files in the source directories: general, linalg, mesh, and fem.
 Specifying an MPI job launcher
 ==============================
 By default, MFEM will use 'mpirun -np #' to launch any of its parallel tests or
-miniapps, where # is the number of MPI tasks. An alternate MPI launcher can be
+miniapps, where # is the number of MPI tasks.  An alternate MPI launcher can be
 provided by setting the MFEM_MPIEXEC and MFEM_MPIEXEC_NP config variables.

 MFEM will expect the launcher command, plus the command line option to allow it
@@ -256,10 +256,6 @@ IF (DEFINED TPL_ENABLE_SIDRE)
  SET(MFEM_USE_SIDRE ${TPL_ENABLE_SIDRE} CACHE BOOL "Enable Axom/Sidre usage" FORCE)
 ENDIF()

-IF (DEFINED TPL_ENABLE_FMS)
-  SET(MFEM_USE_FMS ${TPL_ENABLE_FMS} CACHE BOOL "Enable FMS usage" FORCE)
-ENDIF()
-
 IF (DEFINED TPL_ENABLE_CONDUIT)
  SET(MFEM_USE_CONDUIT ${TPL_ENABLE_CONDUIT} CACHE BOOL "Enable Conduit usage" FORCE)
 ENDIF()
@@ -283,11 +279,3 @@ ENDIF()
 IF (DEFINED TPL_ENABLE_UMPIRE)
  SET(MFEM_USE_UMPIRE ${TPL_ENABLE_UMPIRE} CACHE BOOL "Enable Umpire" FORCE)
 ENDIF()
-
-IF (DEFINED TPL_ENABLE_BENCHMARK)
-  SET(MFEM_USE_BENCHMARK ${TPL_ENABLE_BENCHMARK} CACHE BOOL "Enable Google-Benchmark" FORCE)
-ENDIF()
-
-IF (DEFINED TPL_ENABLE_PARELAG)
-  SET(MFEM_USE_PARELAG ${TPL_ENABLE_PARELAG} CACHE BOOL "Enable ParELAG" FORCE)
-ENDIF()
@@ -44,7 +44,6 @@ set(MFEM_USE_PETSC @MFEM_USE_PETSC@)
 set(MFEM_USE_SLEPC @MFEM_USE_SLEPC@)
 set(MFEM_USE_MPFR @MFEM_USE_MPFR@)
 set(MFEM_USE_SIDRE @MFEM_USE_SIDRE@)
-set(MFEM_USE_FMS @MFEM_USE_FMS@)
 set(MFEM_USE_CONDUIT @MFEM_USE_CONDUIT@)
 set(MFEM_USE_PUMI @MFEM_USE_PUMI@)
 set(MFEM_USE_CUDA @MFEM_USE_CUDA@)
@@ -54,11 +53,7 @@ set(MFEM_USE_CEED @MFEM_USE_CEED@)
 set(MFEM_USE_UMPIRE @MFEM_USE_UMPIRE@)
 set(MFEM_USE_SIMD @MFEM_USE_SIMD@)
 set(MFEM_USE_ADIOS2 @MFEM_USE_ADIOS2@)
-set(MFEM_USE_CODIPACK @MFEM_USE_CODIPACK@)
-set(MFEM_USE_ADFORWARD @MFEM_USE_ADFORWARD@)
 set(MFEM_USE_CALIPER @MFEM_USE_CALIPER@)
-set(MFEM_USE_BENCHMARK @MFEM_USE_BENCHMARK@)
-set(MFEM_USE_PARELAG @MFEM_USE_PARELAG@)

 set(MFEM_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
 set(MFEM_CXX_FLAGS "@CMAKE_CXX_FLAGS@")
@@ -119,9 +119,6 @@
 // Enable the use of SIMD in the high performance templated classes
 #cmakedefine MFEM_USE_SIMD

-// Enable MFEM functionality based on the FMS library
-#cmakedefine MFEM_USE_FMS
-
 // Enable MFEM functionality based on Conduit
 #cmakedefine MFEM_USE_CONDUIT

@@ -175,13 +172,4 @@
 // Enable interface to the MKL CPardiso library.
 #cmakedefine MFEM_USE_MKL_CPARDISO

-// Use forward mode for automatic differentiation
-#cmakedefine MFEM_USE_ADFORWARD
-
-// Enable the use of the CoDiPack library for AD
-#cmakedefine MFEM_USE_CODIPACK
-
-// Enable MFEM functionality based on the Google Benchmark library.
-#cmakedefine MFEM_USE_BENCHMARK
-
 #endif // MFEM_CONFIG_HEADER
@@ -1,22 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - BENCHMARK_FOUND
-#   - BENCHMARK_LIBRARIES
-#   - BENCHMARK_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(Benchmark BENCHMARK BENCHMARK_DIR
-        "include" "benchmark/benchmark.h"
-        "lib" "benchmark"
-        "Paths to headers required by Google Benchmark."
-        "Libraries required by Google Benchmark.")
@@ -1,24 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Automatic differentiation using the CoDiPack library.
-#  www.scicomp.uni-kl.de/codi/
-# Sets the following variables:
-#   - CODIPACK_FOUND
-#   - CODIPACK_INCLUDE_DIRS
-#   - CODIPACK_LIBRARIES
-
-include(MfemCmakeUtilities)
-mfem_find_package(CODIPACK CODIPACK CODIPACK_DIR
-  "include" "codi.h"
-  "lib" ""
-  "Paths to headers required by CODIPACK."
-  "Libraries required by CODIPACK.")
@@ -1,20 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - FMS_FOUND
-#   - FMS_LIBRARIES
-#   - FMS_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(FMS FMS FMS_DIR
-  "include" fms.h "lib" fms
-  "Paths to headers required by FMS." "Libraries required by FMS.")
@@ -1,19 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - PARELAG_FOUND
-#   - PARELAG_LIBRARIES
-#   - PARELAG_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(PARELAG PARELAG PARELAG_DIR "" "" "" ""
-  "Paths to headers required by ParELAG." "Libraries required by ParELAG.")
@@ -100,8 +100,6 @@ macro(add_mfem_examples EXE_SRCS)

    string(REPLACE ".cpp" "" EXE_NAME "${EXE_PREFIX}${SRC_FILENAME}")
    mfem_add_executable(${EXE_NAME} ${SRC_FILE})
-    install(TARGETS ${EXE_NAME}
-            RUNTIME DESTINATION examples)
    add_dependencies(${MFEM_ALL_EXAMPLES_TARGET_NAME} ${EXE_NAME})
    if (EXE_NEEDED_BY)
      add_dependencies(${EXE_NEEDED_BY} ${EXE_NAME})
@@ -766,7 +764,7 @@ function(mfem_export_mk_files)
      MFEM_USE_GNUTLS MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC
      MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_CONDUIT MFEM_USE_PUMI
      MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA MFEM_USE_UMPIRE MFEM_USE_SIMD
-      MFEM_USE_ADIOS2 MFEM_USE_BENCHMARK MFEM_USE_PARELAG)
+      MFEM_USE_ADIOS2)
  foreach(var ${CONFIG_MK_BOOL_VARS})
    if (${var})
      set(${var} YES)
@@ -117,9 +117,6 @@
 // Enable the use of SIMD in the high performance templated classes
 // #define MFEM_USE_SIMD

-// Enable FMS support
-// #define MFEM_USE_FMS
-
 // Enable Conduit support
 // #define MFEM_USE_CONDUIT

@@ -180,13 +177,4 @@
 // Enable interface to the MKL CPardiso library.
 // #define MFEM_USE_MKL_CPARDISO

-// Use forward mode for automatic differentiation
-// #define MFEM_USE_ADFORWARD
-
-// Enable the use of the CoDiPack library for AD
-// #define MFEM_USE_CODIPACK
-
-// Enable functionality based on the Google Benchmark library.
-// #define MFEM_USE_BENCHMARK
-
 #endif // MFEM_CONFIG_HEADER
@@ -43,7 +43,6 @@ MFEM_USE_PETSC         = @MFEM_USE_PETSC@
 MFEM_USE_SLEPC         = @MFEM_USE_SLEPC@
 MFEM_USE_MPFR          = @MFEM_USE_MPFR@
 MFEM_USE_SIDRE         = @MFEM_USE_SIDRE@
-MFEM_USE_FMS           = @MFEM_USE_FMS@
 MFEM_USE_CONDUIT       = @MFEM_USE_CONDUIT@
 MFEM_USE_PUMI          = @MFEM_USE_PUMI@
 MFEM_USE_HIOP          = @MFEM_USE_HIOP@
@@ -58,10 +57,6 @@ MFEM_USE_UMPIRE        = @MFEM_USE_UMPIRE@
 MFEM_USE_SIMD          = @MFEM_USE_SIMD@
 MFEM_USE_ADIOS2        = @MFEM_USE_ADIOS2@
 MFEM_USE_MKL_CPARDISO  = @MFEM_USE_MKL_CPARDISO@
-MFEM_USE_ADFORWARD     = @MFEM_USE_ADFORWARD@
-MFEM_USE_CODIPACK      = @MFEM_USE_CODIPACK@
-MFEM_USE_BENCHMARK     = @MFEM_USE_BENCHMARK@
-MFEM_USE_PARELAG       = @MFEM_USE_PARELAG@

 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@
@@ -45,7 +45,6 @@ option(MFEM_USE_PETSC "Enable PETSc support." OFF)
 option(MFEM_USE_SLEPC "Enable SLEPc support." OFF)
 option(MFEM_USE_MPFR "Enable MPFR usage." OFF)
 option(MFEM_USE_SIDRE "Enable Axom/Sidre usage" OFF)
-option(MFEM_USE_FMS "Enable FMS usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
 option(MFEM_USE_PUMI "Enable PUMI" OFF)
 option(MFEM_USE_HIOP "Enable HiOp" OFF)
@@ -58,10 +57,6 @@ option(MFEM_USE_SIMD "Enable use of SIMD intrinsics" OFF)
 option(MFEM_USE_ADIOS2 "Enable ADIOS2" OFF)
 option(MFEM_USE_CALIPER "Enable Caliper support" OFF)
 option(MFEM_USE_MKL_CPARDISO "Enable MKL CPardiso" OFF)
-option(MFEM_USE_ADFORWARD "Enable forward mode for AD" OFF)
-option(MFEM_USE_CODIPACK "Enable automatic differentiation (AD) using CoDiPack" OFF)
-option(MFEM_USE_BENCHMARK "Enable Google Benchmark" OFF)
-option(MFEM_USE_PARELAG "Enable ParELAG" OFF)

 # Optional overrides for autodetected MPIEXEC and MPIEXEC_NUMPROC_FLAG
 # set(MFEM_MPIEXEC "mpirun" CACHE STRING "Command for running MPI tests")
@@ -78,7 +73,6 @@ set(MFEM_MPI_NP 4 CACHE STRING "Number of processes used for MPI tests")
 option(MFEM_ENABLE_TESTING "Enable the ctest framework for testing" ON)
 option(MFEM_ENABLE_EXAMPLES "Build all of the examples" OFF)
 option(MFEM_ENABLE_MINIAPPS "Build all of the miniapps" OFF)
-option(MFEM_ENABLE_GOOGLE_BENCHMARKS "Build all of the Google benchmarks" OFF)

 # Setting CXX/MPICXX on the command line or in user.cmake will overwrite the
 # autodetected C++ compiler.
@@ -102,11 +96,6 @@ set(HYPRE_DIR "${MFEM_DIR}/../hypre/src/hypre" CACHE PATH
 # If hypre was compiled to depend on BLAS and LAPACK:
 # set(HYPRE_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
 #     "Packages that HYPRE depends on.")
-if (MFEM_USE_CUDA)
-   # This is only necessary when hypre is built with cuda:
-   set(HYPRE_REQUIRED_LIBRARIES "-lcusparse" "-lcurand" CACHE STRING
-       "Libraries that HYPRE depends on.")
-endif()

 set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library.")

@@ -143,10 +132,10 @@ set(MUMPS_DIR "${MFEM_DIR}/../MUMPS_5.2.0" CACHE PATH
    "Path to the MUMPS library.")
 # Packages required by MUMPS, depending on how it was compiled.
 set(MUMPS_REQUIRED_PACKAGES "MPI" "BLAS" "METIS" "ScaLAPACK" CACHE STRING
-    "Additional packages required by MUMPS.")
+    "Additional packages required by MUMPS.")    
 # If the MPI package does not find all required Fortran libraries:
 # set(MUMPS_REQUIRED_LIBRARIES "gfortran" "mpi_mpifh" CACHE STRING
-#     "Additional libraries required by MUMPS.")
+#     "Additional libraries required by MUMPS.")    

 set(STRUMPACK_DIR "${MFEM_DIR}/../STRUMPACK-build" CACHE PATH
    "Path to the STRUMPACK library.")
@@ -198,12 +187,6 @@ set(SLEPC_ARCH "arch-linux2-c-debug" CACHE STRING "SLEPC build architecture.")

 set(MPFR_DIR "" CACHE PATH "Path to the MPFR library.")

-set(FMS_DIR "${MFEM_DIR}/../fms" CACHE PATH
-    "Path to the FMS library.")
-# If FMS is built with Conduit:
-# set(FMS_REQUIRED_PACKAGES "Conduit/relay" CACHE STRING
-#     "Additional packages required by FMS.")
-
 set(CONDUIT_DIR "${MFEM_DIR}/../conduit" CACHE PATH
    "Path to the Conduit library.")

@@ -229,25 +212,12 @@ set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")
 set(UMPIRE_DIR "${MFEM_DIR}/../umpire" CACHE PATH "Path to Umpire")
 set(CALIPER_DIR "${MFEM_DIR}/../caliper" CACHE PATH "Path to Caliper")
-set(BENCHMARK_DIR "${MFEM_DIR}/../google-benchmark" CACHE PATH
-    "Path to Google Benchmark")
-
-# Provide paths, since ParELAG is dependent on MFEM and MFEM needs to be
-# compiled (or at least cmake needs to succeed) before compiling ParELAG.
-set(PARELAG_DIR "${MFEM_DIR}/../parelag" CACHE PATH "Path to ParELAG")
-set(PARELAG_INCLUDE_DIRS "${PARELAG_DIR}/src;${PARELAG_DIR}/build/src" CACHE
-    STRING "Path to ParELAG headers.")
-set(PARELAG_LIBRARIES "${PARELAG_DIR}/build/src/libParELAG.a" CACHE STRING
-    "The ParELAG library.")

 set(BLAS_INCLUDE_DIRS "" CACHE STRING "Path to BLAS headers.")
 set(BLAS_LIBRARIES "" CACHE STRING "The BLAS library.")
 set(LAPACK_INCLUDE_DIRS "" CACHE STRING "Path to LAPACK headers.")
 set(LAPACK_LIBRARIES "" CACHE STRING "The LAPACK library.")

-set(CODIPACK_INCLUDE_DIRS "${MFEM_DIR}/../CoDiPack/inlude" CACHE STRING "Path to CoDiPack headers.")
-set(CODIPACK_LIBRARIES "")
-
 # Some useful variables:
 set(CMAKE_SKIP_PREPROCESSED_SOURCE_RULES ON) # Skip *.i rules
 set(CMAKE_SKIP_ASSEMBLY_SOURCE_RULES  ON)    # Skip *.s rules
@@ -59,12 +59,9 @@ HIP_FLAGS = --amdgpu-target=$(HIP_ARCH)
 HIP_XCOMPILER =
 HIP_XLINKER   = -Wl,

-# Flags for generating dependencies.
-DEP_FLAGS = -MM -MT
-
 ifneq ($(NOTMAC),)
   AR      = ar
-   ARFLAGS = crv
+   ARFLAGS = cruv
   RANLIB  = ranlib
   PICFLAG = $(XCOMPILER)-fPIC
   SO_EXT  = so
@@ -76,7 +73,7 @@ ifneq ($(NOTMAC),)
 else
   # Silence "has no symbols" warnings on Mac OS X
   AR      = ar
-   ARFLAGS = Scrv
+   ARFLAGS = Scruv
   RANLIB  = ranlib -no_warning_for_no_symbols
   PICFLAG = $(XCOMPILER)-fPIC
   SO_EXT  = dylib
@@ -89,9 +86,6 @@ else
   BUILD_RPATH = $(XLINKER)-undefined,dynamic_lookup
   INSTALL_SOFLAGS = $(subst $1 ,,$(call MAKE_SOFLAGS,$(MFEM_LIB_DIR)))
   INSTALL_RPATH = $(XLINKER)-undefined,dynamic_lookup
-   # Silence unused command line argument warnings when generating dependencies
-   # with mpicxx and clang
-   DEP_FLAGS := -Wno-unused-command-line-argument $(DEP_FLAGS)
 endif

 # Set CXXFLAGS to overwrite the default selection of DEBUG_FLAGS/OPTIM_FLAGS
@@ -142,7 +136,6 @@ MFEM_USE_PETSC         = NO
 MFEM_USE_SLEPC         = NO
 MFEM_USE_MPFR          = NO
 MFEM_USE_SIDRE         = NO
-MFEM_USE_FMS           = NO
 MFEM_USE_CONDUIT       = NO
 MFEM_USE_PUMI          = NO
 MFEM_USE_HIOP          = NO
@@ -157,10 +150,6 @@ MFEM_USE_UMPIRE        = NO
 MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
 MFEM_USE_MKL_CPARDISO  = NO
-MFEM_USE_ADFORWARD     = NO
-MFEM_USE_CODIPACK      = NO
-MFEM_USE_BENCHMARK     = NO
-MFEM_USE_PARELAG       = NO

 # MPI library compile and link flags
 # These settings are used only when building MFEM with MPI + HIP
@@ -185,10 +174,6 @@ LIBUNWIND_LIB = $(if $(NOTMAC),-lunwind -ldl,)
 HYPRE_DIR = @MFEM_DIR@/../hypre/src/hypre
 HYPRE_OPT = -I$(HYPRE_DIR)/include
 HYPRE_LIB = -L$(HYPRE_DIR)/lib -lHYPRE
-ifeq (YES,$(MFEM_USE_CUDA))
-   # This is only necessary when hypre is built with cuda:
-   HYPRE_LIB += -lcusparse -lcurand
-endif

 # METIS library configuration
 ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK)$(MFEM_USE_MUMPS),NONONO)
@@ -372,11 +357,6 @@ endif
 MPFR_OPT =
 MPFR_LIB = -lmpfr

-# FMS and required libraries configuration
-FMS_DIR = $(MFEM_DIR)/../fms
-FMS_OPT = -I$(FMS_DIR)/include
-FMS_LIB = -Wl,-rpath,$(FMS_DIR)/lib -L$(FMS_DIR)/lib -lfms
-
 # Conduit and required libraries configuration
 CONDUIT_DIR = @MFEM_DIR@/../conduit
 CONDUIT_OPT = -I$(CONDUIT_DIR)/include/conduit
@@ -416,11 +396,6 @@ HIOP_DIR = @MFEM_DIR@/../hiop/install
 HIOP_OPT = -I$(HIOP_DIR)/include
 HIOP_LIB = -L$(HIOP_DIR)/lib -lhiop $(LAPACK_LIB)

-# CoDiPack
-CODIPACK_DIR = @MFEM_DIR@/../CoDiPack
-CODIPACK_OPT = -I$(CODIPACK_DIR)
-CODIPACK_LIB =
-
 # GSLIB library
 GSLIB_DIR = @MFEM_DIR@/../gslib/build
 GSLIB_OPT = -I$(GSLIB_DIR)/include
@@ -444,11 +419,6 @@ CALIPER_DIR = @MFEM_DIR@/../caliper
 CALIPER_OPT = -I$(CALIPER_DIR)/include
 CALIPER_LIB = $(XLINKER)-rpath,$(CALIPER_DIR)/lib64 -L$(CALIPER_DIR)/lib64 -lcaliper

-# BENCHMARK library configuration
-BENCHMARK_DIR = @MFEM_DIR@/../google-benchmark
-BENCHMARK_OPT = -I$(BENCHMARK_DIR)/include
-BENCHMARK_LIB = -L$(BENCHMARK_DIR)/lib -lbenchmark -lpthread
-
 # libCEED library configuration
 CEED_DIR ?= @MFEM_DIR@/../libCEED
 CEED_OPT = -I$(CEED_DIR)/include
@@ -479,11 +449,6 @@ MKL_CPARDISO_LIB = $(XLINKER)-rpath,$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
   -L$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR) -l$(MKL_MPI_WRAPPER)\
   -lmkl_intel_lp64 -lmkl_sequential -lmkl_core

-# PARELAG library configuration
-PARELAG_DIR = @MFEM_DIR@/../parelag
-PARELAG_OPT = -I$(PARELAG_DIR)/src -I$(PARELAG_DIR)/build/src
-PARELAG_LIB = -L$(PARELAG_DIR)/build/src -lParELAG
-
 # If YES, enable some informational messages
 VERBOSE = NO

@@ -1,41 +0,0 @@
-                    Finite Element Discretization Library
-                                   __
-                       _ __ ___   / _|  ___  _ __ ___
-                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
-                      | | | | | ||  _||  __/| | | | | |
-                      |_| |_| |_||_|   \___||_| |_| |_|
-
-                               https://mfem.org
-
-
-This directory contains recommended git hooks, which are scripts that can be
-used to improve your development experience with MFEM:
-
-### The hooks
-
-* `pre-commit` is a hook that will be applied before each commit and run
-`astyle` on the code. This will ensure that your changes comply with the MFEM
-code styling guidelines.
-
-* `pre-push` is a hook that will be applied before each push to run a quick set
-of tests that verify that your files headers are in compliance, and that you did
-not add any large files to the repo.
-
-### Setup
-
-To setup the git hooks, run `make hooks`, which creates symlinks to the hooks in
-the `.git/hooks` directory. Individual hooks can be enabled by manually creating
-symlinks.
-
-(You may also copy the scripts directly and customize them further, but this way
-you may miss additional updates in the future.)
-
-### Failures
-
-The `branch-history` check can fail in some cases when the history is OK. For
-example, when a large number of files were modified for a legitimate reason, or
-when a picture was added for documentation.
-
-If that is the case, make sure the failure is indeed justified, and rerun the
-push command with the `--no-verify` option. This will skip the hooks, allowing
-you to push those changes.
@@ -1,4 +0,0 @@
-#!/bin/sh
-
-# Apply automated code formatting
-make -C $(git rev-parse --show-toplevel) style
@@ -1,107 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-option=${1:-""}
-
-if [[ "${option}" == "--help" ]]; then
-  echo "This script runs checks on the repository."
-  echo "It has 2 modes: with and without an option."
-  echo ""
-  echo "Options are used in GitHub Actions and can be:"
-  echo "  --copyright"
-  echo "  --license"
-  echo "  --release"
-  echo "  --style"
-  echo "  --history"
-  echo ""
-  echo "As a githook, the script is used without options."
-  echo "In that case, it will run all the checks except style."
-  echo ""
-  echo "Use --help to print this help message."
-fi
-
-cd $(git rev-parse --show-toplevel)
-
-# copyright check
-copyright=true
-if [[ "${option}" == "--copyright" || "${option}" == "" ]]; then
-  if git grep -l "^\(#\|//\).*\(\-2020\|\ 2010,\)" > matches.txt; then
-    echo "Please update the following files to Copyright (c) 2010-2021:"
-    cat matches.txt
-    copyright=false
-  fi
-fi
-
-# license check
-license=true
-if [[ "${option}" == "--license" || "${option}" == "" ]]; then
-  if git grep -li "^\(#\|//\).*GNU\ Lesser\ General\ Public\ License" > matches.txt; then
-    echo "Please update the following files to the BSD-3 license:"
-    cat matches.txt
-    license=false
-  fi
-fi
-
-# release check
-release=true
-if [[ "${option}" == "--release" || "${option}" == "" ]]; then
-  if git grep -l "^\(#\|//\).*LLNL\-CODE\-443211" > matches.txt
-  then
-    echo "Please update the following files to LLNL-CODE-806117:"
-    cat matches.txt
-    release=false
-  fi
-fi
-
-# wrap-up
-code=0
-if ! $copyright ; then
-  echo "copyright check failed, unroll log for details"
-  code=1
-fi
-if ! $license ; then
-  echo "license check failed, unroll log for details"
-  code=1
-fi
-if ! $release ; then
-  echo "release check failed, unroll log for details"
-  code=1
-fi
-
-# `code-style` is not just a check, it will actually reformat the code if
-# necessary. This means that if one pushes while the repo is in dirty state
-# (changes not staged), those changes may be mixed with format changes.
-# To activate this, you will need to hard-copy this hook script in the hook
-# directory and uncomment only then. (See README.md)
-#
-## style check
-#if [[ "${option}" == "--style" || "${option}" == "" ]]; then
-if [[ "${option}" == "--style" ]]; then
-  if which astyle && [[ "$(astyle --version)" == "Artistic Style Version 3.1" ]]; then
-    cd tests/scripts
-    if ! ./runtest code-style; then code=1; fi
-    cd -
-  else
-    echo "Warning: astyle not found or version is not 3.1"
-  fi
-fi
-
-# branch-history
-if [[ "${option}" == "--history" || "${option}" == "" ]]; then
-  git fetch origin master:master
-  cd tests/scripts
-  if ! ./runtest branch-history; then code=1; fi
-  cd -
-fi
-
-exit $code
@@ -57,27 +57,22 @@ TIMECMD := $(word 1,$(TIMECMD))
 ifneq (,$(filter test%,$(MAKECMDGOALS)))
   MAKEFLAGS += -k
 endif
-# Test runs of the examples/miniapps with parameters - check exit code:
-# 0 means success, 255 means the test was skipped, anything else means error
+# Test runs of the examples/miniapps with parameters - check exit code
 mfem-test = \
   printf "   $(3) [$(2) $(1) ... ]: "; \
   $(call $(TIMEFUN),$(TIMECMD),$(2) ./$(1) $(if $(5),,-no-vis )$(4) \
     > $(1).stderr 2>&1); \
-   err="$$3"; \
-   if [ "$$3" = 0 ]; then $(PRINT_OK); \
-   else if [ "$$3" = 255 ]; then $(PRINT_SKIP); err=0; \
-   else $(PRINT_FAILED); cat $(1).stderr; fi; fi; \
-   rm -f $(1).stderr; exit $$err
+   if [ "$$3" = 0 ]; \
+   then $(PRINT_OK); else $(PRINT_FAILED); cat $(1).stderr; fi; \
+   rm -f $(1).stderr; exit $$3

 # Test runs of the examples/miniapps - check exit code and if a file exists
-# See mfem-test for the interpretation of the error code
 mfem-test-file = \
   printf "   $(3) [$(2) $(1) ... ]: "; \
   $(call $(TIMEFUN),$(TIMECMD),$(2) ./$(1) -no-vis > $(1).stderr 2>&1); \
   err="$$3"; \
-   if [ "$$3" = 0 ] && [ -e $(4) ]; then $(PRINT_OK); \
-   else if [ "$$3" = 255 ] && [ -e $(4) ]; then $(PRINT_SKIP); err=0; \
-   else $(PRINT_FAILED); cat $(1).stderr; err=64; fi; fi; \
+   if [ "$$3" = 0 ] && [ -e $(4) ]; \
+   then $(PRINT_OK); else $(PRINT_FAILED); cat $(1).stderr; err=64; fi; \
   rm -f $(1).stderr; exit $$err

 .PHONY: test test-par-YES test-par-NO test-ser test-par test-clean test-print
@@ -85,26 +80,6 @@ mfem-test-file = \
 # What sets of tests to run in serial and parallel
 test-par-YES: $(PAR_$(MFEM_TESTS):=-test-par) $(SEQ_$(MFEM_TESTS):=-test-seq)
 test-par-NO:  $(SEQ_$(MFEM_TESTS):=-test-seq)
-ifeq ($(MFEM_USE_CUDA),YES)
-.PHONY: test-par-YES-cuda test-par-NO-cuda test-ser-cuda test-par-cuda test-cuda
-test-par-YES: test-par-YES-cuda
-test-par-NO:  test-par-NO-cuda
-test-par-YES-cuda: test-par-cuda test-ser-cuda
-test-par-NO-cuda:  test-ser-cuda
-test-ser-cuda: $(SEQ_DEVICE_$(MFEM_TESTS):=-test-seq-cuda)
-test-par-cuda: $(PAR_DEVICE_$(MFEM_TESTS):=-test-par-cuda)
-test-cuda: test-par-$(MFEM_USE_MPI)-cuda clean-exec
-endif
-ifeq ($(MFEM_USE_HIP),YES)
-.PHONY: test-par-YES-hip test-par-NO-hip test-ser-hip test-par-hip test-hip
-test-par-YES: test-par-YES-hip
-test-par-NO:  test-par-NO-hip
-test-par-YES-hip: test-par-hip test-ser-hip
-test-par-NO-hip:  test-ser-hip
-test-ser-hip: $(SEQ_DEVICE_$(MFEM_TESTS):=-test-seq-hip)
-test-par-hip: $(PAR_DEVICE_$(MFEM_TESTS):=-test-par-hip)
-test-hip: test-par-$(MFEM_USE_MPI)-hip clean-exec
-endif
 test-ser:     test-par-NO
 test-par:     test-par-YES
 test:         all test-par-$(MFEM_USE_MPI) clean-exec
@@ -1,9 +0,0 @@
-MFEM INLINE mesh v1.0
-
-type = pyramid
-nx = 4
-ny = 4
-nz = 4
-sx = 1.0
-sy = 1.0
-sz = 1.0
@@ -1,43 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-# PYRAMID     = 7
-#
-
-dimension
-3
-
-elements
-2
-1 7 4 3 2 1 0
-1 7 1 2 3 4 5
-
-boundary
-8
-1 2 0 2 1
-2 2 0 3 2
-3 2 0 4 3
-4 2 0 1 4
-5 2 1 2 5
-6 2 2 3 5
-7 2 3 4 5
-8 2 4 1 5
-
-vertices
-6
-3
-0 0 -1
-1 0 0
-0 1 0
-1 0 0
-0 -1 0
-0 0 1
@@ -1,38 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-# PYRAMID     = 7
-#
-
-dimension
-3
-
-elements
-1
-1 7 0 1 2 3 4
-
-boundary
-5
-1 3 3 2 1 0
-2 2 0 1 4
-3 2 1 2 4
-4 2 2 3 4
-5 2 3 0 4
-
-vertices
-5
-3
-0 0 0
-1 0 0
-1 1 0
-0 1 0
-0 0 1
@@ -1,246 +0,0 @@
-FMS: 100
-DataCollection/Name: star
-DataCollection/NumberOfFieldDescriptors: 1
-DataCollection/FieldDescriptors/0/Name: CoordsDescriptor
-DataCollection/FieldDescriptors/0/ComponentName: volume
-DataCollection/FieldDescriptors/0/Type: 0
-DataCollection/FieldDescriptors/0/FixedOrder/Size: 3
-DataCollection/FieldDescriptors/0/FixedOrder/Type: FMS_UINT64
-DataCollection/FieldDescriptors/0/FixedOrder/Values: [0, 1, 3]
-DataCollection/FieldDescriptors/0/NumDofs: 211
-DataCollection/NumberOfFields: 1
-DataCollection/Fields/0/Name: Coords
-DataCollection/Fields/0/LayoutType: 0
-DataCollection/Fields/0/NumberOfVectorComponents: 2
-DataCollection/Fields/0/FieldDescriptorName: CoordsDescriptor
-DataCollection/Fields/0/Data/Size: 422
-DataCollection/Fields/0/Data/Type: FMS_DOUBLE
-DataCollection/Fields/0/Data/Values: [-0.016886, 1.000000, 0.309017, 
-1.309020, -0.809017, -0.500000, 
-0.809017, -1.618030, 0.309017, 
-0.500000, 1.309020, 0.519420, 
-1.154510, 0.809019, 0.147680, 
-0.095492, -0.654508, -0.415586, 
-1.213520, -1.213520, -0.392210, 
-0.654508, -0.095492, 0.139949, 
-0.809019, 1.154510, 0.660184, 
-0.264063, -0.800064, -0.231060, 
-0.663691, 0.183114, 0.317639, 
-0.543082, 0.598483, 0.345112, 
-0.478298, 0.027703, 0.095229, 
-0.012368, -0.092534, -0.334412, 
-0.313767, -0.140526, -0.293881, 
-0.534056, -0.660290, -0.537646, 
-0.655590, -0.121396, -0.274504, 
-0.346497, -0.296570, 0.004737, 
-0.098835, 0.069287, 0.082675, 
-0.318799, 0.467183, 0.564505, 
-0.595190, 0.846237, 0.671735, 
-1.051500, 1.103010, 0.964008, 
-0.821603, 1.257520, 1.206010, 
-1.142350, 0.975686, 0.781273, 
-0.717257, 0.475684, 0.642352, 
-0.268930, 0.211049, 0.174181, 
-0.039345, -0.147746, -0.177481, 
-0.365164, -0.230328, -0.551503, 
-0.603005, -0.497587, -0.389864, 
-0.757514, -0.706011, -0.675487, 
-0.528946, -0.943851, -1.078690, 
-1.087600, -0.955467, -1.483190, 
-1.348360, -1.483190, -1.348360, 
-1.085930, -0.938010, -0.943851, 
-1.078690, -0.681476, -0.540944, 
-0.757514, -0.706011, -0.540614, 
-0.367058, -0.551503, -0.603005, 
-0.365164, -0.230328, -0.138552, 
-0.206896, 0.174181, 0.039345, 
-0.268468, 0.222269, 0.475684, 
-0.642352, 0.759791, 0.719381, 
-1.142350, 0.975686, 1.257520, 
-1.206010, 0.972837, 0.836119, 
-1.051500, 1.103010, 0.214572, 
-0.407449, 0.288323, 0.449827, 
-0.086700, -0.027358, -0.200560, 
-0.166595, -0.271802, -0.418426, 
-0.426131, -0.551441, -0.096117, 
-0.206969, -0.027946, -0.184969, 
-0.211136, 0.260131, 0.407172, 
-0.430781, 0.718277, 0.885068, 
-0.753103, 0.957692, 0.866273, 
-1.024530, 0.934099, 1.093820, 
-0.348422, 0.524463, 0.404903, 
-0.587376, 0.054525, 0.146431, 
-0.078026, -0.007795, -0.329488, 
-0.302967, -0.488115, -0.439332, 
-0.498515, -0.453527, -0.633059, 
-0.570251, -0.655787, -0.791132, 
-0.802013, -0.956872, -1.094720, 
-1.208560, -1.207720, -1.339910, 
-0.693371, -0.795404, -0.803594, 
-0.945068, -0.464668, -0.631721, 
-0.458968, -0.547876, -0.332878, 
-0.485696, -0.286385, -0.408481, 
-0.053021, -0.058200, 0.110846, 
-0.002086, 0.381384, 0.416784, 
-0.551133, 0.613261, 0.872474, 
-0.901208, 1.038300, 1.084660, 
-0.737459, 0.751250, 0.890002, 
-0.915210, 0.010915, 0.000000, 
-0.951057, 0.951057, 0.587785, 
-1.538840, -0.587785, 0.000000, 
-0.951057, -1.538840, -0.951057, 
-0.015847, 0.475529, 0.951057, 
-0.492248, 1.244950, 1.063310, 
-0.274399, 0.293893, -0.293892, 
-0.296404, -1.063310, -1.244950, 
-0.453865, -0.951057, -0.475529, 
-0.466620, 0.792932, -0.013913, 
-0.748783, -0.497528, 0.021382, 
-0.017158, 0.172591, 0.330125, 
-0.458568, 0.457971, 0.137740, 
-0.299049, 0.588394, 0.667324, 
-0.432341, 0.634346, 0.117322, 
-0.193603, 0.211702, 0.098278, 
-0.199438, -0.077304, -0.082243, 
-0.216296, -0.458634, -0.592374, 
-0.563926, -0.680404, -0.135751, 
-0.302942, -0.469005, -0.453640, 
-0.182727, -0.314240, 0.024270, 
-0.021546, 0.158510, 0.317019, 
-0.485799, 0.492951, 0.792548, 
-0.634038, 0.951057, 0.951057, 
-0.777915, 0.613430, 0.951057, 
-0.951057, 0.793994, 0.635800, 
-1.049020, 1.146990, 1.084480, 
-0.924310, 1.440880, 1.342910, 
-1.380330, 1.221820, 0.948209, 
-0.856297, 0.746293, 0.904802, 
-0.476242, 0.393234, 0.489821, 
-0.391857, 0.194471, 0.075751, 
-0.097964, 0.195929, -0.097964, 
-0.195928, -0.173234, -0.078922, 
-0.489821, -0.391856, -0.467007, 
-0.397859, -0.746293, -0.904802, 
-0.945206, -0.849559, -1.380330, 
-1.221820, -1.440880, -1.342910, 
-1.100830, -0.923191, -1.049020, 
-1.146990, -0.774515, -0.621542, 
-0.951057, -0.951057, -0.803055, 
-0.635255, -0.951057, -0.951057, 
-0.792548, -0.634038, -0.454301, 
-0.479369, -0.158510, -0.317019, 
-0.149331, 0.178643, 0.295860, 
-0.306275, 0.246225, 0.406610, 
-0.367954, 0.498458, -0.014929, 
-0.084917, -0.094272, -0.000726, 
-0.245374, -0.340755, -0.435351, 
-0.490564, -0.176355, -0.294974, 
-0.148010, -0.328246, 0.136248, 
-0.182883, 0.328957, 0.309903, 
-0.646446, 0.622546, 0.800859, 
-0.801803, 0.643087, 0.654473, 
-0.796963, 0.816799, 0.748189, 
-0.898148, 0.819092, 0.970033, 
-1.049540, 1.162660, 1.115310, 
-1.266700, 0.539960, 0.728396, 
-0.656318, 0.783152, 0.302716, 
-0.403858, 0.219006, 0.309252, 
-0.009667, 0.107161, -0.118211, 
-0.011786, -0.284704, -0.179858, 
-0.411567, -0.296676, -0.570168, 
-0.635348, -0.726784, -0.793211, 
-1.016860, -1.112970, -1.170910, 
-1.281530, -0.737860, -0.851723, 
-0.883732, -0.995070, -0.653153, 
-0.769939, -0.631918, -0.797688, 
-0.616289, -0.806819, -0.638485, 
-0.790356, -0.136399, -0.322769, 
-0.165339, -0.309622]
-DataCollection/Mesh/PartitionInfo/Size: 2
-DataCollection/Mesh/PartitionInfo/Type: FMS_UINT64
-DataCollection/Mesh/PartitionInfo/Values: [0, 1]
-DataCollection/Mesh/NumDomainNames: 1
-DataCollection/Mesh/NumComponents: 1
-DataCollection/Mesh/NumTags: 0
-DataCollection/Mesh/DomainNames/0/Name: Domain
-DataCollection/Mesh/DomainNames/0/NumDomains: 1
-DataCollection/Mesh/DomainNames/0/Domains/0/Dimension: 2
-DataCollection/Mesh/DomainNames/0/Domains/0/NumVertices: 31
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/EntityType: FMS_EDGE
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/NumEntities: 50
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Size: 100
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Type: FMS_INT32
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Values: [11, 0, 26, 
-11, 26, 14, 
-14, 0, 27, 
-14, 27, 17, 
-17, 0, 28, 
-17, 28, 20, 
-20, 0, 29, 
-20, 29, 23, 
-23, 0, 30, 
-23, 30, 11, 
-11, 1, 12, 
-1, 26, 12, 
-12, 3, 13, 
-3, 26, 13, 
-13, 2, 14, 
-2, 15, 2, 
-27, 15, 15, 
-5, 16, 5, 
-27, 16, 16, 
-4, 17, 4, 
-18, 4, 28, 
-18, 18, 7, 
-19, 7, 28, 
-19, 19, 6, 
-20, 6, 21, 
-6, 29, 21, 
-21, 9, 22, 
-9, 29, 22, 
-22, 8, 23, 
-8, 24, 8, 
-30, 24, 24, 
-10, 25, 10, 
-30, 25, 25, 1]
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/EntityType: FMS_QUADRILATERAL
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/NumEntities: 20
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Size: 80
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Type: FMS_INT32
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Values: [0, 1, 2, 
-3, 3, 4, 
-5, 6, 6, 
-7, 8, 9, 
-9, 10, 11, 
-12, 12, 13, 
-14, 0, 15, 
-16, 17, 1, 
-17, 18, 19, 
-20, 2, 20, 
-21, 22, 22, 
-23, 24, 4, 
-24, 25, 26, 
-27, 5, 27, 
-28, 29, 29, 
-30, 31, 7, 
-31, 32, 33, 
-34, 8, 34, 
-35, 36, 36, 
-37, 38, 10, 
-38, 39, 40, 
-41, 11, 41, 
-42, 43, 43, 
-44, 45, 13, 
-45, 46, 47, 
-48, 14, 48, 
-49, 15]
-DataCollection/Mesh/Components/0/Name: volume
-DataCollection/Mesh/Components/0/Dimension: 2
-DataCollection/Mesh/Components/0/NumEntities: 20
-DataCollection/Mesh/Components/0/Coordinates: Coords
-DataCollection/Mesh/Components/0/NumParts: 1
-DataCollection/Mesh/Components/0/Parts/0/DomainName: Domain
-DataCollection/Mesh/Components/0/Parts/0/DomainID: 0
-DataCollection/Mesh/Components/0/Parts/0/FullDomain: Yes
-DataCollection/Mesh/Components/0/Relations/Size: 0
-DataCollection/Mesh/Components/0/Relations/Type: FMS_UINT64
@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.

-PROJECT_NUMBER         = v4.3.1
+PROJECT_NUMBER         = v4.2.1

 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -765,7 +765,6 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/linalg \
                         @MFEM_SOURCE_DIR@/mesh \
                         @MFEM_SOURCE_DIR@/fem \
-                         @MFEM_SOURCE_DIR@/fem/fe \
                         @MFEM_SOURCE_DIR@/examples \
                         @MFEM_SOURCE_DIR@/examples/caliper \
                         @MFEM_SOURCE_DIR@/examples/amgx \
@@ -781,15 +780,13 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/miniapps/gslib \
                         @MFEM_SOURCE_DIR@/miniapps/meshing \
                         @MFEM_SOURCE_DIR@/miniapps/mtop \
-                         @MFEM_SOURCE_DIR@/miniapps/autodiff \
                         @MFEM_SOURCE_DIR@/miniapps/navier \
                         @MFEM_SOURCE_DIR@/miniapps/nurbs \
                         @MFEM_SOURCE_DIR@/miniapps/performance \
                         @MFEM_SOURCE_DIR@/miniapps/shifted \
                         @MFEM_SOURCE_DIR@/miniapps/solvers \
                         @MFEM_SOURCE_DIR@/miniapps/tools \
-                         @MFEM_SOURCE_DIR@/miniapps/toys \
-                         @MFEM_SOURCE_DIR@/miniapps/parelag
+                         @MFEM_SOURCE_DIR@/miniapps/toys

 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
@@ -194,8 +194,6 @@ namespace mfem {
 * - <a class="el" href="parheat_8cpp_source.html">Optimization gradients</a>: Gradients of PDE-constrained function
 * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
 * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
- * - <a class="el" href="par__example_8cpp_source.html">Parallel pLaplacian example using AD</a>: Parallel pLaplacian example
- * - <a class="el" href="seq__example_8cpp_source.html">Serial pLaplacian example using AD</a>: Serial pLaplacian example
 *
 * See also the <a class="el" href="https://mfem.org/examples/">examples documentation</a> online.
 */
@@ -37,7 +37,6 @@ list(APPEND ALL_EXE_SRCS
  ex27.cpp
  ex28.cpp
  ex29.cpp
-  ex30.cpp
  )

 if (MFEM_USE_MPI)
@@ -71,7 +70,6 @@ if (MFEM_USE_MPI)
    ex27p.cpp
    ex28p.cpp
    ex29p.cpp
-    ex30p.cpp
    )
 endif()

@@ -82,80 +80,49 @@ include_directories(BEFORE ${PROJECT_BINARY_DIR})
 add_mfem_examples(ALL_EXE_SRCS)

 # Add a test for each example
-if (MFEM_ENABLE_TESTING)
-  foreach(SRC_FILE ${ALL_EXE_SRCS})
-    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+foreach(SRC_FILE ${ALL_EXE_SRCS})
+  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})

+  if (NOT (${TEST_NAME} MATCHES "ex0p?"))
    set(THIS_TEST_OPTIONS "-no-vis")
-    if (${TEST_NAME} MATCHES "ex0p?")
-      set(THIS_TEST_OPTIONS)
-    endif()
-    if (${TEST_NAME} MATCHES "ex10p*")
-      list(APPEND THIS_TEST_OPTIONS "-tf" "5")
-    elseif(${TEST_NAME} MATCHES "ex15p*")
-      list(APPEND THIS_TEST_OPTIONS "-e" "1")
-    elseif(${TEST_NAME} MATCHES "ex27p*")
-      list(APPEND THIS_TEST_OPTIONS "-dg")
-    endif()
-
-    if (NOT (${TEST_NAME} MATCHES ".*p$"))
-      add_test(NAME ${TEST_NAME}_ser
-        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-    else()
-      add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
-        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-        ${MPIEXEC_PREFLAGS}
-        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-        ${MPIEXEC_POSTFLAGS})
-    endif()
-  endforeach()
-
-  # Add CUDA/HIP tests.
-  set(DEVICE_EXAMPLES
-      # serial examples with device support:
-      ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
-      # parallel examples with device support:
-      ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p ex24p ex25p ex26p)
-  set(MFEM_TEST_DEVICE)
-  if (MFEM_USE_CUDA)
-    set(MFEM_TEST_DEVICE "cuda")
-  elseif (MFEM_USE_HIP)
-    set(MFEM_TEST_DEVICE "hip")
  endif()
-  if (MFEM_TEST_DEVICE)
-    foreach(TEST_NAME ${DEVICE_EXAMPLES})
-      set(THIS_TEST_OPTIONS "-no-vis" "-d" "${MFEM_TEST_DEVICE}")
-      if (NOT (${TEST_NAME} MATCHES ".*p$"))
-        add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_ser
-          COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-      elseif (MFEM_USE_MPI)
-        add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_np=${MFEM_MPI_NP}
-          COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-          ${MPIEXEC_PREFLAGS}
-          $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-          ${MPIEXEC_POSTFLAGS})
-      endif()
-    endforeach()
+  if (${TEST_NAME} MATCHES "ex10p*")
+    list(APPEND THIS_TEST_OPTIONS "-tf" "5")
+  elseif(${TEST_NAME} MATCHES "ex15p*")
+    list(APPEND THIS_TEST_OPTIONS "-e" "1")
+  elseif(${TEST_NAME} MATCHES "ex27p*")
+    list(APPEND THIS_TEST_OPTIONS "-dg")
  endif()

-  # If STRUMPACK is enabled, add a test run that uses it.
-  if (MFEM_USE_STRUMPACK)
-    add_test(NAME ex11p_strumpack_np=${MFEM_MPI_NP}
+  if (NOT (${TEST_NAME} MATCHES ".*p$"))
+    add_test(NAME ${TEST_NAME}_ser
+      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+  else()
+    add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:ex11p> "-no-vis" "--strumpack"
+      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
      ${MPIEXEC_POSTFLAGS})
  endif()
+endforeach()

-  # If SuperLU_DIST is enabled, add a test run that uses it.
-  if (MFEM_USE_SUPERLU)
-    add_test(NAME ex11p_superlu_np=${MFEM_MPI_NP}
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:ex11p> "-no-vis" "--superlu"
-      ${MPIEXEC_POSTFLAGS})
-  endif()
+# If STRUMPACK is enabled, add a test run that uses it.
+if (MFEM_USE_STRUMPACK)
+  add_test(NAME ex11p_strumpack_np=${MFEM_MPI_NP}
+    COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+    ${MPIEXEC_PREFLAGS}
+    $<TARGET_FILE:ex11p> "-no-vis" "--strumpack"
+    ${MPIEXEC_POSTFLAGS})
+endif()
+
+# If SuperLU_DIST is enabled, add a test run that uses it.
+if (MFEM_USE_SUPERLU)
+  add_test(NAME ex11p_superlu_np=${MFEM_MPI_NP}
+    COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+    ${MPIEXEC_PREFLAGS}
+    $<TARGET_FILE:ex11p> "-no-vis" "--superlu"
+    ${MPIEXEC_POSTFLAGS})
 endif()

 # Include the examples/amgx directory if AmgX is enabled
@@ -50,32 +50,30 @@ add_mfem_examples(AMGX_EXAMPLES_SRCS ${PFX} copy_amgx_json_files test_amgx)
 # which builds the examples and runs:
 #   ctest -R amgx

-if (MFEM_ENABLE_TESTING)
-  # Command line options for the tests.
-  # Example 1/1p:
-  set(EX1_TEST_OPTS)
-  set(EX1P_TEST_OPTS)
+# Command line options for the tests.
+# Example 1/1p:
+set(EX1_TEST_OPTS)
+set(EX1P_TEST_OPTS)

-  # Add the tests: one test per source file.
-  foreach(SRC_FILE ${AMGX_EXAMPLES_SRCS})
-    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-    string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-    set(TEST_NAME ${PFX}${TEST_NAME})
+# Add the tests: one test per source file.
+foreach(SRC_FILE ${AMGX_EXAMPLES_SRCS})
+  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+  set(TEST_NAME ${PFX}${TEST_NAME})

-    set(THIS_TEST_OPTIONS "-no-vis")
-    list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-    # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+  set(THIS_TEST_OPTIONS "-no-vis")
+  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")

-    if (NOT (${TEST_NAME} MATCHES ".*p$"))
-      add_test(NAME ${TEST_NAME}_ser
-        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-    else()
-      add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
-        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-        ${MPIEXEC_PREFLAGS}
-        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-        ${MPIEXEC_POSTFLAGS})
-    endif()
-  endforeach()
-endif()
+  if (NOT (${TEST_NAME} MATCHES ".*p$"))
+    add_test(NAME ${TEST_NAME}_ser
+      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+  else()
+    add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+      ${MPIEXEC_POSTFLAGS})
+  endif()
+endforeach()
@@ -30,22 +30,22 @@ set(PREFIX caliper_)
 add_mfem_examples(CALIPER_EXE_SRCS ${PREFIX})

 # Add a test for each example
-if (MFEM_ENABLE_TESTING)
-  foreach(SRC_FILE ${CALIPER_EXE_SRCS})
-    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+foreach(SRC_FILE ${CALIPER_EXE_SRCS})
+  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+
+  set(THIS_TEST_OPTIONS "-no-vis")
+
+  if (NOT (${TEST_NAME} MATCHES ".*p$"))
+    add_test(NAME ${TEST_NAME}_ser
+      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+  else()
+    add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+      ${MPIEXEC_POSTFLAGS})
+  endif()
+endforeach()

-    set(THIS_TEST_OPTIONS "-no-vis")

-    if (NOT (${TEST_NAME} MATCHES ".*p$"))
-      add_test(NAME ${TEST_NAME}_ser
-        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-    else()
-      add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
-        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-        ${MPIEXEC_PREFLAGS}
-        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-        ${MPIEXEC_POSTFLAGS})
-    endif()
-  endforeach()
-endif()
@@ -1,18 +0,0 @@
-                    Finite Element Discretization Library
-                                   __
-                       _ __ ___   / _|  ___  _ __ ___
-                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
-                      | | | | | ||  _||  __/| | | | | |
-                      |_| |_| |_||_|   \___||_| |_| |_|
-
-                               https://mfem.org
-
-This directory contains modifications of the example codes that illustrate the
-use of MFEM features based on the Caliper performance profiling library.
-
-To build these examples, make sure that MFEM is configured with the option
-"MFEM_USE_CALIPER = YES", see the top-level INSTALL file for details (version
-2.5.0 of Caliper is recommended, though older versions may work too).
-
-We recommend comparing the original example codes with the corresponding files
-in the current directory.
@@ -206,9 +206,9 @@ int main(int argc, char *argv[])
   cout << "Size of linear system: " << A->Height() << endl;

   // 11. Solve the linear system A X = B.
+   MFEM_PERF_BEGIN("Solve A X=B");
   if (!pa)
   {
-      MFEM_PERF_SCOPE("Solve A X=B (FA)");
 #ifndef MFEM_USE_SUITESPARSE
      // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
      GSSmoother M((SparseMatrix&)(*A));
@@ -223,7 +223,6 @@ int main(int argc, char *argv[])
   }
   else // Jacobi preconditioning in partial assembly mode
   {
-      MFEM_PERF_SCOPE("Solve A X=B (PA)");
      if (UsesTensorBasis(fespace))
      {
         OperatorJacobiSmoother M(a, ess_tdof_list);
@@ -234,6 +233,7 @@ int main(int argc, char *argv[])
         CG(*A, B, X, 1, 400, 1e-12, 0.0);
      }
   }
+   MFEM_PERF_END("Solve A X=B");
   // 12. Recover the solution as a finite element grid function.
   a.RecoverFEMSolution(X, b, x);

@@ -231,29 +231,28 @@ int main(int argc, char *argv[])
   // 13. Solve the linear system A X = B.
   //     * With full assembly, use the BoomerAMG preconditioner from hypre.
   //     * With partial assembly, use Jacobi smoothing, for now.
+   MFEM_PERF_BEGIN("Solve A X = B");
+   Solver *prec = NULL;
+   if (pa)
   {
-      MFEM_PERF_SCOPE("Solve A X=B");
-      Solver *prec = NULL;
-      if (pa)
+      if (UsesTensorBasis(fespace))
      {
-         if (UsesTensorBasis(fespace))
-         {
-            prec = new OperatorJacobiSmoother(a, ess_tdof_list);
-         }
+         prec = new OperatorJacobiSmoother(a, ess_tdof_list);
      }
-      else
-      {
-         prec = new HypreBoomerAMG;
-      }
-      CGSolver cg(MPI_COMM_WORLD);
-      cg.SetRelTol(1e-12);
-      cg.SetMaxIter(2000);
-      cg.SetPrintLevel(1);
-      if (prec) { cg.SetPreconditioner(*prec); }
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete prec;
   }
+   else
+   {
+      prec = new HypreBoomerAMG;
+   }
+   CGSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(1);
+   if (prec) { cg.SetPreconditioner(*prec); }
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+   delete prec;
+   MFEM_PERF_END("Solve A X = B");
   // 14. Recover the parallel grid function corresponding to X. This is the
   //     local finite element solution on each processor.
   a.RecoverFEMSolution(X, b, x);
@@ -9,7 +9,6 @@
 //               ex1 -m ../data/fichera.mesh
 //               ex1 -m ../data/fichera-mixed.mesh
 //               ex1 -m ../data/toroid-wedge.mesh
-//               ex1 -m ../data/octahedron.mesh -o 1
 //               ex1 -m ../data/periodic-annulus-sector.msh
 //               ex1 -m ../data/periodic-torus-sector.msh
 //               ex1 -m ../data/square-disc-p2.vtk -o 2
@@ -55,7 +55,6 @@ int main(int argc, char *argv[])
   int order = 1;
   int nev = 5;
   bool visualization = 1;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -72,8 +71,6 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -89,18 +86,13 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors. We
+   // 3. Read the (serial) mesh from the given mesh file on all processors. We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 5. Refine the serial mesh on all processors to increase the resolution. In
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ref_levels' of uniform refinement (2 by default, or
   //    specified on the command line with -rs).
   for (int lev = 0; lev < ser_ref_levels; lev++)
@@ -108,7 +100,7 @@ int main(int argc, char *argv[])
      mesh->UniformRefinement();
   }

-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution (1 time by
   //    default, or specified on the command line with -rp). Once the parallel
   //    mesh is defined, the serial mesh can be deleted.
@@ -118,8 +110,9 @@ int main(int argc, char *argv[])
   {
      pmesh->UniformRefinement();
   }
+   pmesh->ReorientTetMesh();

-   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Nedelec finite elements of the specified order.
   FiniteElementCollection *fec = new ND_FECollection(order, dim);
   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
@@ -129,7 +122,7 @@ int main(int argc, char *argv[])
      cout << "Number of unknowns: " << size << endl;
   }

-   // 8. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
+   // 7. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
   //    element space. The first corresponds to the curl curl, while the second
   //    is a simple mass matrix needed on the right hand side of the
   //    generalized eigenvalue problem below. The boundary conditions are
@@ -171,7 +164,7 @@ int main(int argc, char *argv[])
   delete a;
   delete m;

-   // 9. Define and configure the AME eigensolver and the AMS preconditioner for
+   // 8. Define and configure the AME eigensolver and the AMS preconditioner for
   //    A to be used within the solver. Set the matrices which define the
   //    generalized eigenproblem A x = lambda M x.
   HypreAMS *ams = new HypreAMS(*A,fespace);
@@ -187,15 +180,15 @@ int main(int argc, char *argv[])
   ame->SetMassMatrix(*M);
   ame->SetOperator(*A);

-   // 10. Compute the eigenmodes and extract the array of eigenvalues. Define a
-   //     parallel grid function to represent each of the eigenmodes returned by
-   //     the solver.
+   // 9. Compute the eigenmodes and extract the array of eigenvalues. Define a
+   //    parallel grid function to represent each of the eigenmodes returned by
+   //    the solver.
   Array<double> eigenvalues;
   ame->Solve();
   ame->GetEigenvalues(eigenvalues);
   ParGridFunction x(fespace);

-   // 11. Save the refined mesh and the modes in parallel. This output can be
+   // 10. Save the refined mesh and the modes in parallel. This output can be
   //     viewed later using GLVis: "glvis -np <np> -m mesh -g mode".
   {
      ostringstream mesh_name, mode_name;
@@ -220,7 +213,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 12. Send the solution by socket to a GLVis server.
+   // 11. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -260,7 +253,7 @@ int main(int argc, char *argv[])
      mode_sock.close();
   }

-   // 13. Free the used memory.
+   // 12. Free the used memory.
   delete ame;
   delete ams;
   delete M;
@@ -135,8 +135,8 @@ int main(int argc, char *argv[])
   a->AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
   if (eta > 0)
   {
-      a->AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
-      a->AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
+      a->AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
+      a->AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
   }
   a->Assemble();
   a->Finalize();
@@ -199,8 +199,8 @@ int main(int argc, char *argv[])
   a->AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
   if (eta > 0)
   {
-      a->AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
-      a->AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(*fespace, eta));
+      a->AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
+      a->AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
   }
   a->Assemble();
   a->Finalize();
@@ -221,7 +221,7 @@ int main(int argc, char *argv[])
   {
      HyprePCG pcg(*A);
      pcg.SetTol(1e-12);
-      pcg.SetMaxIter(500);
+      pcg.SetMaxIter(200);
      pcg.SetPrintLevel(2);
      pcg.SetPreconditioner(*amg);
      pcg.Mult(*B, *X);
@@ -232,7 +232,7 @@ int main(int argc, char *argv[])
      GMRESSolver gmres(MPI_COMM_WORLD);
      gmres.SetAbsTol(0.0);
      gmres.SetRelTol(1e-12);
-      gmres.SetMaxIter(500);
+      gmres.SetMaxIter(200);
      gmres.SetKDim(10);
      gmres.SetPrintLevel(1);
      gmres.SetOperator(*A);
@@ -24,10 +24,7 @@
 //               class ConductionOperator defining C(u)), as well as their
 //               implicit time integration. Note that implementing the method
 //               ConductionOperator::ImplicitSolve is the only requirement for
-//               high-order implicit (SDIRK) time integration. In this example,
-//               the diffusion operator is linearized by evaluating with the
-//               lagged solution from the previous timestep, so there is only
-//               a linear solve.
+//               high-order implicit (SDIRK) time integration.
 //
 //               We recommend viewing examples 2, 9 and 10 before viewing this
 //               example.
@@ -329,8 +326,8 @@ ConductionOperator::ConductionOperator(FiniteElementSpace &f, double al,
 void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
 {
   // Compute:
-   //    du_dt = M^{-1}*-Ku
-   // for du_dt, where K is linearized by using u from the previous timestep
+   //    du_dt = M^{-1}*-K(u)
+   // for du_dt
   Kmat.Mult(u, z);
   z.Neg(); // z = -z
   M_solver.Mult(z, du_dt);
@@ -341,7 +338,7 @@ void ConductionOperator::ImplicitSolve(const double dt,
 {
   // Solve the equation:
   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
-   // for du_dt, where K is linearized by using u from the previous timestep
+   // for du_dt
   if (!T)
   {
      T = Add(1.0, Mmat, dt, Kmat);
@@ -24,11 +24,8 @@
 //               class ConductionOperator defining C(u)), as well as their
 //               implicit time integration. Note that implementing the method
 //               ConductionOperator::ImplicitSolve is the only requirement for
-//               high-order implicit (SDIRK) time integration. In this example,
-//               the diffusion operator is linearized by evaluating with the
-//               lagged solution from the previous timestep, so there is only
-//               a linear solve. Optional saving with ADIOS2
-//               (adios2.readthedocs.io) is also illustrated.
+//               high-order implicit (SDIRK) time integration. Optional saving
+//               with ADIOS2 (adios2.readthedocs.io) is also illustrated.
 //
 //               We recommend viewing examples 2, 9 and 10 before viewing this
 //               example.
@@ -423,8 +420,8 @@ ConductionOperator::ConductionOperator(ParFiniteElementSpace &f, double al,
 void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
 {
   // Compute:
-   //    du_dt = M^{-1}*-Ku
-   // for du_dt, where K is linearized by using u from the previous timestep
+   //    du_dt = M^{-1}*-K(u)
+   // for du_dt
   Kmat.Mult(u, z);
   z.Neg(); // z = -z
   M_solver.Mult(z, du_dt);
@@ -435,7 +432,7 @@ void ConductionOperator::ImplicitSolve(const double dt,
 {
   // Solve the equation:
   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
-   // for du_dt, where K is linearized by using u from the previous timestep
+   // for du_dt
   if (!T)
   {
      T = Add(1.0, Mmat, dt, Kmat);
@@ -196,12 +196,6 @@ void InitialDeformation(const Vector &x, Vector &y);

 int main(int argc, char *argv[])
 {
-#ifdef HYPRE_USING_CUDA
-   cout << "\nAs of mfem-4.3 and hypre-2.22.0 (July 2021) this example\n"
-        << "is NOT supported with the CUDA version of hypre.\n\n";
-   return 255;
-#endif
-
   // 1. Initialize MPI
   MPI_Session mpi;
   const int myid = mpi.WorldRank();
@@ -444,19 +438,15 @@ JacobianPreconditioner::JacobianPreconditioner(Array<ParFiniteElementSpace *>
 void JacobianPreconditioner::Mult(const Vector &k, Vector &y) const
 {
   // Extract the blocks from the input and output vectors
-   Vector disp_in;
-   disp_in.MakeRef(const_cast<Vector&>(k), block_trueOffsets[0],
-                   block_trueOffsets[1]-block_trueOffsets[0]);
-   Vector pres_in;
-   pres_in.MakeRef(const_cast<Vector&>(k), block_trueOffsets[1],
-                   block_trueOffsets[2]-block_trueOffsets[1]);
+   Vector disp_in(k.GetData() + block_trueOffsets[0],
+                  block_trueOffsets[1]-block_trueOffsets[0]);
+   Vector pres_in(k.GetData() + block_trueOffsets[1],
+                  block_trueOffsets[2]-block_trueOffsets[1]);

-   Vector disp_out;
-   disp_out.MakeRef(y, block_trueOffsets[0],
-                    block_trueOffsets[1]-block_trueOffsets[0]);
-   Vector pres_out;
-   pres_out.MakeRef(y, block_trueOffsets[1],
-                    block_trueOffsets[2]-block_trueOffsets[1]);
+   Vector disp_out(y.GetData() + block_trueOffsets[0],
+                   block_trueOffsets[1]-block_trueOffsets[0]);
+   Vector pres_out(y.GetData() + block_trueOffsets[1],
+                   block_trueOffsets[2]-block_trueOffsets[1]);

   Vector temp(block_trueOffsets[1]-block_trueOffsets[0]);
   Vector temp2(block_trueOffsets[1]-block_trueOffsets[0]);
@@ -469,9 +459,6 @@ void JacobianPreconditioner::Mult(const Vector &k, Vector &y) const
   subtract(disp_in, temp, temp2);

   stiff_pcg->Mult(temp2, disp_out);
-
-   disp_out.SyncAliasMemory(y);
-   pres_out.SyncAliasMemory(y);
 }

 void JacobianPreconditioner::SetOperator(const Operator &op)
@@ -486,10 +473,7 @@ void JacobianPreconditioner::SetOperator(const Operator &op)

      if (!spaces[0]->GetParMesh()->Nonconforming())
      {
-#ifndef HYPRE_USING_CUDA
-         // Not available yet when hypre is built with CUDA
         stiff_prec_amg->SetElasticityOptions(spaces[0]);
-#endif
      }

      stiff_prec = stiff_prec_amg;
@@ -1,266 +0,0 @@
-//                                MFEM Example 1
-//
-// Compile with: make ex1
-//
-// Sample runs:  ex1 -m ../data/square-disc.mesh
-//               ex1 -m ../data/star.mesh
-//               ex1 -m ../data/escher.mesh
-//               ex1 -m ../data/fichera.mesh
-//               ex1 -m ../data/square-disc-p2.vtk -o 2
-//               ex1 -m ../data/square-disc-p3.mesh -o 3
-//               ex1 -m ../data/square-disc-nurbs.mesh -o -1
-//               ex1 -m ../data/disc-nurbs.mesh -o -1
-//               ex1 -m ../data/pipe-nurbs.mesh -o -1
-//               ex1 -m ../data/star-surf.mesh
-//               ex1 -m ../data/square-disc-surf.mesh
-//               ex1 -m ../data/inline-segment.mesh
-//               ex1 -m ../data/amr-quad.mesh
-//               ex1 -m ../data/amr-hex.mesh
-//               ex1 -m ../data/fichera-amr.mesh
-//               ex1 -m ../data/mobius-strip.mesh
-//               ex1 -m ../data/mobius-strip.mesh -o -1 -sc
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order, or if order < 1 using an isoparametric/isogeometric
-//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example highlights the use of mesh refinement, finite
-//               element grid functions, as well as linear and bilinear forms
-//               corresponding to the left-hand side and right-hand side of the
-//               discrete linear system. We also cover the explicit elimination
-//               of essential boundary conditions, static condensation, and the
-//               optional connection to the GLVis tool for visualization.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "./star-set.mesh";
-   int order = 1;
-   int rs = -1;
-   int ra = 0;
-   int bt = EntitySets::INVALID;
-   const char *bs = "Origin";
-   bool static_cond = false;
-   bool visualization = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&rs, "-rs", "--refine-serial",
-                  "Number of serial refinement levels");
-   args.AddOption(&ra, "-ra", "--refine-adaptive",
-                  "Number of adaptive refinement levels");
-   args.AddOption(&bt, "-bt", "--bc-entity-type",
-                  "");
-   args.AddOption(&bs, "-bs", "--bc-entity-set-name",
-                  "");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // 2. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
-   //    the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 3. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
-   //    largest number that gives a final mesh with no more than 50,000
-   //    elements.
-   {
-      int ref_levels = ( rs >= 0 ) ? rs :
-                       (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh->UniformRefinement();
-      }
-   }
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-      mesh->ent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "mesh->ent_sets is NULL" << endl;
-   }
-
-   if ( ra > 0 )
-   {
-      cout << "calling EnsureNCMesh" << endl;
-      mesh->EnsureNCMesh();
-      cout << "back from EnsureNCMesh" << endl;
-   }
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-   }
-   else
-   {
-      cout << "mesh->ent_sets is NULL" << endl;
-   }
-   cout << "Calling RandomRefinement " << ra << " times." << endl;
-   for (int l = 0; l < ra; l++)
-   {
-      mesh->RandomRefinement(0.2);
-   }
-   cout << "Done with refinement" << endl;
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-      mesh->ent_sets->PrintSetInfo(cout);
-   }
-   if ( mesh->ncmesh )
-   {
-      mesh->ncmesh->PrintStats(cout);
-
-      ofstream ofsV("vp.out");
-      ofstream ofsE("ce.out");
-      mesh->ncmesh->PrintVertexParents(ofsV);
-      mesh->ncmesh->PrintCoarseElements(ofsE);
-   }
-
-   // 4. Define a finite element space on the mesh. Here we use continuous
-   //    Lagrange finite elements of the specified order. If order < 1, we
-   //    instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-   }
-   else if (mesh->GetNodes())
-   {
-      fec = mesh->GetNodes()->OwnFEC();
-      cout << "Using isoparametric FEs: " << fec->Name() << endl;
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-   }
-   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
-   cout << "Number of finite element unknowns: "
-        << fespace->GetTrueVSize() << endl;
-
-   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
-   //    In this example, the boundary conditions are defined by marking all
-   //    the boundary attributes from the mesh as essential (Dirichlet) and
-   //    converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if ( bt == EntitySets::INVALID )
-   {
-      if (mesh->bdr_attributes.Size())
-      {
-         Array<int> ess_bdr(mesh->bdr_attributes.Max());
-         ess_bdr = 1;
-         fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-   }
-   else
-   {
-      fespace->GetEssentialTrueDofs((EntitySets::EntityType)bt, bs,
-                                    ess_tdof_list);
-   }
-
-   cout << "Number of Dirichlet dofs: " << ess_tdof_list.Size() << endl;
-
-   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
-   //    the basis functions in the finite element fespace.
-   LinearForm *b = new LinearForm(fespace);
-   ConstantCoefficient one(1.0);
-   b->AddDomainIntegrator(new DomainLFIntegrator(one));
-   b->Assemble();
-
-   // 7. Define the solution vector x as a finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero,
-   //    which satisfies the boundary conditions.
-   GridFunction x(fespace);
-   x = 0.0;
-
-   // 8. Set up the bilinear form a(.,.) on the finite element space
-   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //    domain integrator.
-   BilinearForm *a = new BilinearForm(fespace);
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   // 9. Assemble the bilinear form and the corresponding linear system,
-   //    applying any necessary transformations such as: eliminating boundary
-   //    conditions, applying conforming constraints for non-conforming AMR,
-   //    static condensation, etc.
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
-
-   SparseMatrix A;
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-
-   cout << "Size of linear system: " << A.Height() << endl;
-
-#ifndef MFEM_USE_SUITESPARSE
-   // 10. Define a simple symmetric Gauss-Seidel preconditioner and use it to
-   //     solve the system A X = B with PCG.
-   GSSmoother M(A);
-   PCG(A, M, B, X, 1, 200, 1e-12, 0.0);
-#else
-   // 10. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-   UMFPackSolver umf_solver;
-   umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-   umf_solver.SetOperator(A);
-   umf_solver.Mult(B, X);
-#endif
-
-   // 11. Recover the solution as a finite element grid function.
-   a->RecoverFEMSolution(X, *b, x);
-
-   // 12. Save the refined mesh and the solution. This output can be viewed later
-   //     using GLVis: "glvis -m refined.mesh -g sol.gf".
-   ofstream mesh_ofs("refined.mesh");
-   mesh_ofs.precision(8);
-   mesh->Print(mesh_ofs);
-   ofstream sol_ofs("sol.gf");
-   sol_ofs.precision(8);
-   x.Save(sol_ofs);
-
-   // 13. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *mesh << x << flush;
-   }
-
-   // 14. Free the used memory.
-   delete a;
-   delete b;
-   delete fespace;
-   if (order > 0) { delete fec; }
-   delete mesh;
-
-   return 0;
-}
@@ -9,7 +9,6 @@
 //               mpirun -np 4 ex1p -m ../data/fichera.mesh
 //               mpirun -np 4 ex1p -m ../data/fichera-mixed.mesh
 //               mpirun -np 4 ex1p -m ../data/toroid-wedge.mesh
-//               mpirun -np 4 ex1p -m ../data/octahedron.mesh -o 1
 //               mpirun -np 4 ex1p -m ../data/periodic-annulus-sector.msh
 //               mpirun -np 4 ex1p -m ../data/periodic-torus-sector.msh
 //               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
@@ -90,8 +89,7 @@ int main(int argc, char *argv[])
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
 #ifdef MFEM_USE_CEED
-   args.AddOption(&algebraic_ceed, "-a", "--algebraic",
-                  "-no-a", "--no-algebraic",
+   args.AddOption(&algebraic_ceed, "-a", "--algebraic", "-no-a", "--no-algebraic",
                  "Use algebraic Ceed solver");
 #endif
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -199,15 +197,15 @@ int main(int argc, char *argv[])
   b.AddDomainIntegrator(new DomainLFIntegrator(one));
   b.Assemble();

-   // 10. Define the solution vector x as a parallel finite element grid
-   //     function corresponding to fespace. Initialize x with initial guess of
-   //     zero, which satisfies the boundary conditions.
+   // 10. Define the solution vector x as a parallel finite element grid function
+   //     corresponding to fespace. Initialize x with initial guess of zero,
+   //     which satisfies the boundary conditions.
   ParGridFunction x(&fespace);
   x = 0.0;

   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the
-   //     Diffusion domain integrator.
+   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
+   //     domain integrator.
   ParBilinearForm a(&fespace);
   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   a.AddDomainIntegrator(new DiffusionIntegrator(one));
@@ -1,388 +0,0 @@
-//                       MFEM Example 1 - Parallel Version
-//
-// Compile with: make ex1p
-//
-// Sample runs:  mpirun -np 4 ex1p -m ../data/square-disc.mesh
-//               mpirun -np 4 ex1p -m ../data/star.mesh
-//               mpirun -np 4 ex1p -m ../data/escher.mesh
-//               mpirun -np 4 ex1p -m ../data/fichera.mesh
-//               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
-//               mpirun -np 4 ex1p -m ../data/square-disc-p3.mesh -o 3
-//               mpirun -np 4 ex1p -m ../data/square-disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/pipe-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/ball-nurbs.mesh -o 2
-//               mpirun -np 4 ex1p -m ../data/star-surf.mesh
-//               mpirun -np 4 ex1p -m ../data/square-disc-surf.mesh
-//               mpirun -np 4 ex1p -m ../data/inline-segment.mesh
-//               mpirun -np 4 ex1p -m ../data/amr-quad.mesh
-//               mpirun -np 4 ex1p -m ../data/amr-hex.mesh
-//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh
-//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh -o -1 -sc
-//
-//               The following are examples of using EntitySets to define
-//               homogeneous Dirichlet boundary condition.  These examples
-//               require a modified mesh file and a specialized version of
-//               example 1 called "ex1p_es".
-//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh -bt 0 -bs Origin
-//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh -bt 1 -bs Axes
-//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh
-//                                    -bt 1 -bs "Negative Axes"
-//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh
-//                                    -bt 2 -bs "Interior Corner"
-//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh
-//                                    -bt 2 -bs "Exterior Corner"
-//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh
-//                                    -bt 3 -bs "Interior Corner"
-//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh
-//                                    -bt 3 -bs "Exterior Corner"
-//               mpirun -np 4 ex1p_es -m ./fichera-set.mesh -bt 3 -bs "Steps"
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order, or if order < 1 using an isoparametric/isogeometric
-//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example highlights the use of mesh refinement, finite
-//               element grid functions, as well as linear and bilinear forms
-//               corresponding to the left-hand side and right-hand side of the
-//               discrete linear system. We also cover the explicit elimination
-//               of essential boundary conditions, static condensation, and the
-//               optional connection to the GLVis tool for visualization.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "./star-set.mesh";
-   int order = 1;
-   int rs = -1;
-   int rp = 2;
-   int ra = 0;
-   int bt = EntitySets::INVALID;
-   const char *bs = "Origin";
-   bool static_cond = false;
-   bool visualization = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&rs, "-rs", "--refine-serial",
-                  "Number of serial refinement levels");
-   args.AddOption(&rp, "-rp", "--refine-parallel",
-                  "Number of parallel refinement levels");
-   args.AddOption(&ra, "-ra", "--refine-adaptive",
-                  "Number of adaptive refinement levels");
-   args.AddOption(&bt, "-bt", "--bc-entity-type",
-                  "");
-   args.AddOption(&bs, "-bs", "--bc-entity-set-name",
-                  "");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 4. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 10,000 elements.
-   {
-      int ref_levels = ( rs >= 0 ) ? rs :
-                       (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         if ( myid == 0 ) { cout << "Uniform refinement in serial..."; }
-         mesh->UniformRefinement();
-      }
-      MPI_Barrier(MPI_COMM_WORLD);
-      if ( myid == 0 && rs > 0 ) { cout << "Done" << endl; }
-   }
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-      mesh->ent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "mesh->ent_sets is NULL" << endl;
-   }
-   /*
-     At this point we have a serial mesh containing an EntitySets
-     object which stores the current node/edge/face/element indices
-     for each entity in each set.  This data is duplicated on each MPI
-     rank.
-    */
-   if ( ra > 0 )
-   {
-      cout << "calling EnsureNCMesh" << endl;
-      mesh->EnsureNCMesh();
-      cout << "back from EnsureNCMesh" << endl;
-   }
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-      mesh->ent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "mesh->ent_sets is NULL" << endl;
-   }
-   /*
-     We now have an NCEntitySets object which stores the node indices
-     describing each enity in each node/edge/face set and the element
-     indices for the elements in each element set.  This data is
-     duplicated on each MPI rank.
-    */
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   cout << "creating ParMesh from serial mesh" << endl;
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   cout << "done creating ParMesh from serial mesh" << endl;
-   delete mesh;
-   if ( pmesh->pent_sets )
-   {
-      cout << "pmesh->pent_sets is non NULL" << endl;
-      pmesh->pent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "pmesh->pent_sets is NULL" << endl;
-   }
-   /*
-     We now have a ParEntitySets object which marshals the data stored
-     in EntitySets objects.  The data has now been pruned so that each
-     rank only contains indices of local entities.
-
-     The NCEntitySets object remains unchanged...
-
-     If we have an NC mesh a different path is taken and the
-     EntitySets are ignored.
-
-     1) ParNCMesh is created from NCMesh
-       a) Creates a ParNCEntitySets object from ncmesh (every rank contains
-          information to find every entity)
-     2) ParNCMesh is pruned which involves renumbering elements and vertices
-     3) ParMesh is initialized from ParNCMesh
-     4) ParNCMesh::OnMeshUpdated is called
-     5) Mesh::GenerateNCFaceInfo is called
-    */
-   {
-      int par_ref_levels = rp;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         if ( myid == 0 ) { cout << "Uniform refinement in parallel..."; }
-         pmesh->UniformRefinement();
-      }
-      MPI_Barrier(MPI_COMM_WORLD);
-      if ( myid == 0 && rs > 0 ) { cout << "Done" << endl; }
-   }
-   /*
-     RandomRefinement will end up calling
-     ParMesh::NonconformingRefinement which will create a new ParMesh
-     object using the ParNCMesh object and then call
-     ParMesh::OnMeshUpdated on this new mesh.
-    */
-
-   for (int l = 0; l < ra; l++)
-   {
-      pmesh->RandomRefinement(0.2);
-   }
-   if ( ra > 0 )
-   {
-      if ( pmesh->pent_sets )
-      {
-         cout << "pmesh->pent_sets is non NULL post random refinement" << endl;
-         pmesh->pent_sets->PrintSetInfo(cout);
-      }
-      else
-      {
-         cout << "pmesh->pent_sets is NULL post random refinement" << endl;
-      }
-   }
-
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
-   //    use continuous Lagrange finite elements of the specified order. If
-   //    order < 1, we instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-   }
-   else if (pmesh->GetNodes())
-   {
-      fec = pmesh->GetNodes()->OwnFEC();
-      if (myid == 0)
-      {
-         cout << "Using isoparametric FEs: " << fec->Name() << endl;
-      }
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-   }
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_Int size = fespace->GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-   }
-
-   // 7. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet) and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if ( bt == EntitySets::INVALID )
-   {
-      if (pmesh->bdr_attributes.Size())
-      {
-         Array<int> ess_bdr(pmesh->bdr_attributes.Max());
-         ess_bdr = 1;
-         fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-   }
-   else
-   {
-      fespace->GetEssentialTrueDofs((EntitySets::EntityType)bt, bs,
-                                    ess_tdof_list);
-   }
-   for (int i=0; i<num_procs; i++)
-   {
-      if (myid == i)
-      {
-         cout << "Number of Dirichlet dofs on proc " << i << ": "
-              << ess_tdof_list.Size() << endl;
-      }
-   }
-
-   // 8. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (1,phi_i) where phi_i are the basis functions in fespace.
-   ParLinearForm *b = new ParLinearForm(fespace);
-   ConstantCoefficient one(1.0);
-   b->AddDomainIntegrator(new DomainLFIntegrator(one));
-   b->Assemble();
-
-   // 9. Define the solution vector x as a parallel finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero,
-   //    which satisfies the boundary conditions.
-   ParGridFunction x(fespace);
-   x = 0.0;
-
-   // 10. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //     domain integrator.
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   // 11. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
-
-   HypreParMatrix A;
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-
-   if (myid == 0)
-   {
-      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
-   }
-
-   // 12. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
-   //     preconditioner from hypre.
-   HypreSolver *amg = new HypreBoomerAMG(A);
-   HyprePCG *pcg = new HyprePCG(A);
-   pcg->SetTol(1e-12);
-   pcg->SetMaxIter(200);
-   pcg->SetPrintLevel(2);
-   pcg->SetPreconditioner(*amg);
-   pcg->Mult(B, X);
-
-   // 13. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a->RecoverFEMSolution(X, *b, x);
-
-   // 14. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 15. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << x << flush;
-   }
-
-   // 16. Free the used memory.
-   delete pcg;
-   delete amg;
-   delete a;
-   delete b;
-   delete fespace;
-   if (order > 0) { delete fec; }
-   delete pmesh;
-
-   MPI_Finalize();
-
-   return 0;
-}
@@ -13,8 +13,6 @@
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 1
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 2
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 2 -pa
-//               ex22 -m ../data/inline-wedge.mesh -o 1
-//               ex22 -m ../data/inline-pyramid.mesh -o 1
 //               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0
 //
 // Device sample runs:
@@ -13,8 +13,6 @@
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 1
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 2
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 1 -p 2 -pa
-//               mpirun -np 4 ex22p -m ../data/inline-wedge.mesh -o 1
-//               mpirun -np 4 ex22p -m ../data/inline-pyramid.mesh -o 1
 //               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0
 //
 // Device sample runs:
@@ -113,6 +113,7 @@ int main(int argc, char *argv[])
         mesh->UniformRefinement();
      }
   }
+   mesh->ReorientTetMesh();

   // 5. Define a finite element space on the mesh. Here we use Nedelec or
   //    Raviart-Thomas finite elements of the specified order.
@@ -141,6 +141,7 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
+   pmesh->ReorientTetMesh();

   // 7. Define a parallel finite element space on the parallel mesh. Here we
   //    use Nedelec or Raviart-Thomas finite elements of the specified order.
@@ -92,7 +92,7 @@ class PMLDiagMatrixCoefficient : public VectorCoefficient
 {
 private:
   CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML *, Vector &);
+   void (*Function)(const Vector &, CartesianPML * , Vector &);
 public:
   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
                                              Vector &),
@@ -277,8 +277,10 @@ int main(int argc, char *argv[])
      mesh->UniformRefinement();
   }

-   // 6. Set element attributes in order to distinguish elements in the
-   //    PML region
+   // 6. Reorient mesh in case of a tet mesh
+   mesh->ReorientTetMesh();
+
+   // Set element attributes in order to distinguish elements in the PML region
   pml->SetAttributes(mesh);

   // 7. Define a finite element space on the mesh. Here we use the Nedelec
@@ -92,7 +92,7 @@ class PMLDiagMatrixCoefficient : public VectorCoefficient
 {
 private:
   CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML *, Vector &);
+   void (*Function)(const Vector &, CartesianPML * , Vector &);
 public:
   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
                                              Vector &),
@@ -316,6 +316,9 @@ int main(int argc, char *argv[])
      }
   }

+   // 7a. Reorient mesh in case of a tet mesh
+   pmesh->ReorientTetMesh();
+
   // 8. Set element attributes in order to distinguish elements in the PML
   pml->SetAttributes(pmesh);

@@ -105,7 +105,7 @@ private:
      Vector diag(fespace.GetTrueVSize());
      bfs.Last()->AssembleDiagonal(diag);

-      Solver* smoother = new OperatorChebyshevSmoother(*opr, diag,
+      Solver* smoother = new OperatorChebyshevSmoother(opr.Ptr(), diag,
                                                       *essentialTrueDofs.Last(), 2);
      AddLevel(opr.Ptr(), smoother, true, true);
   }
@@ -115,7 +115,7 @@ private:
      Vector diag(fespace.GetTrueVSize());
      bfs.Last()->AssembleDiagonal(diag);

-      Solver* smoother = new OperatorChebyshevSmoother(*opr, diag,
+      Solver* smoother = new OperatorChebyshevSmoother(opr.Ptr(), diag,
                                                       *essentialTrueDofs.Last(), 2, fespace.GetParMesh()->GetComm());

      AddLevel(opr.Ptr(), smoother, true, true);
@@ -295,7 +295,17 @@ int main(int argc, char *argv[])
   //     element solution.
   a.RecoverFEMSolution(X, b, u);

-   // 13. Compute the various boundary integrals.
+   // 13. Build a mass matrix to help solve for n.Grad(u) where 'n' is a surface
+   //     normal.
+   BilinearForm m(&fespace);
+   m.AddDomainIntegrator(new MassIntegrator);
+   m.Assemble();
+
+   ess_tdof_list.SetSize(0);
+   OperatorPtr M;
+   m.FormSystemMatrix(ess_tdof_list, M);
+
+   // 14. Compute the various boundary integrals.
   mfem::out << endl
             << "Verifying boundary conditions" << endl
             << "=============================" << endl;
@@ -351,7 +361,7 @@ int main(int argc, char *argv[])
                << " error " << err << endl;
   }

-   // 14. Save the refined mesh and the solution. This output can be viewed
+   // 15. Save the refined mesh and the solution. This output can be viewed
   //     later using GLVis: "glvis -m refined.mesh -g sol.gf".
   {
      ofstream mesh_ofs("refined.mesh");
@@ -362,7 +372,7 @@ int main(int argc, char *argv[])
      u.Save(sol_ofs);
   }

-   // 15. Send the solution by socket to a GLVis server.
+   // 16. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      string title_str = h1 ? "H1" : "DG";
@@ -375,7 +385,7 @@ int main(int argc, char *argv[])
               << " keys 'mmc'" << flush;
   }

-   // 16. Free the used memory.
+   // 17. Free the used memory.
   delete fec;
   delete mesh;

@@ -314,7 +314,17 @@ int main(int argc, char *argv[])
   //     local finite element solution on each processor.
   a.RecoverFEMSolution(X, b, u);

-   // 14. Compute the various boundary integrals.
+   // 14. Build a mass matrix to help solve for n.Grad(u) where 'n' is a surface
+   //     normal.
+   ParBilinearForm m(&fespace);
+   m.AddDomainIntegrator(new MassIntegrator);
+   m.Assemble();
+
+   ess_tdof_list.SetSize(0);
+   OperatorPtr M;
+   m.FormSystemMatrix(ess_tdof_list, M);
+
+   // 15. Compute the various boundary integrals.
   mfem::out << endl
             << "Verifying boundary conditions" << endl
             << "=============================" << endl;
@@ -370,7 +380,7 @@ int main(int argc, char *argv[])
                << " error " << err << endl;
   }

-   // 15. Save the refined mesh and the solution in parallel. This output can be
+   // 16. Save the refined mesh and the solution in parallel. This output can be
   //     viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
   {
      ostringstream mesh_name, sol_name;
@@ -386,7 +396,7 @@ int main(int argc, char *argv[])
      u.Save(sol_ofs);
   }

-   // 16. Send the solution by socket to a GLVis server.
+   // 17. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      string title_str = h1 ? "H1" : "DG";
@@ -401,7 +411,7 @@ int main(int argc, char *argv[])
               << " keys 'mmc'" << flush;
   }

-   // 17. Free the used memory.
+   // 18. Free the used memory.
   delete fec;

   return 0;
@@ -81,12 +81,6 @@ Mesh * build_trapezoid_mesh(double offset)

 int main(int argc, char *argv[])
 {
-#ifdef HYPRE_USING_CUDA
-   cout << "\nAs of mfem-4.3 and hypre-2.22.0 (July 2021) this example\n"
-        << "is NOT supported with the CUDA version of hypre.\n\n";
-   return 255;
-#endif
-
   // 1. Initialize MPI.
   int num_procs, myid;
   MPI_Init(&argc, &argv);
@@ -366,7 +360,6 @@ int main(int argc, char *argv[])
   }
   delete pmesh;

-   // HYPRE_Finalize();
   MPI_Finalize();

   return 0;
@@ -61,7 +61,6 @@ int main(int argc, char *argv[])
   bool visualization = 1;
   bool amg_elast = 0;
   bool reorder_space = false;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -79,8 +78,6 @@ int main(int argc, char *argv[])
                  "Enable or disable GLVis visualization.");
   args.AddOption(&reorder_space, "-nodes", "--by-nodes", "-vdim", "--by-vdim",
                  "Use byNODES ordering of vector space instead of byVDIM");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -96,12 +93,7 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
@@ -117,14 +109,14 @@ int main(int argc, char *argv[])
      return 3;
   }

-   // 5. Select the order of the finite element discretization space. For NURBS
+   // 4. Select the order of the finite element discretization space. For NURBS
   //    meshes, we increase the order by degree elevation.
   if (mesh->NURBSext)
   {
      mesh->DegreeElevate(order, order);
   }

-   // 6. Refine the serial mesh on all processors to increase the resolution. In
+   // 5. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ref_levels' of uniform refinement. We choose
   //    'ref_levels' to be the largest number that gives a final mesh with no
   //    more than 1,000 elements.
@@ -137,7 +129,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 7. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -150,7 +142,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 8. Define a parallel finite element space on the parallel mesh. Here we
+   // 7. Define a parallel finite element space on the parallel mesh. Here we
   //    use vector finite elements, i.e. dim copies of a scalar finite element
   //    space. We use the ordering by vector dimension (the last argument of
   //    the FiniteElementSpace constructor) which is expected in the systems
@@ -183,7 +175,7 @@ int main(int argc, char *argv[])
           << "Assembling: " << flush;
   }

-   // 9. Determine the list of true (i.e. parallel conforming) essential
+   // 8. Determine the list of true (i.e. parallel conforming) essential
   //    boundary dofs. In this example, the boundary conditions are defined by
   //    marking only boundary attribute 1 from the mesh as essential and
   //    converting it to a list of true dofs.
@@ -192,14 +184,14 @@ int main(int argc, char *argv[])
   ess_bdr[0] = 1;
   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);

-   // 10. Set up the parallel linear form b(.) which corresponds to the
-   //     right-hand side of the FEM linear system. In this case, b_i equals the
-   //     boundary integral of f*phi_i where f represents a "pull down" force on
-   //     the Neumann part of the boundary and phi_i are the basis functions in
-   //     the finite element fespace. The force is defined by the object f, which
-   //     is a vector of Coefficient objects. The fact that f is non-zero on
-   //     boundary attribute 2 is indicated by the use of piece-wise constants
-   //     coefficient for its last component.
+   // 9. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system. In this case, b_i equals the
+   //    boundary integral of f*phi_i where f represents a "pull down" force on
+   //    the Neumann part of the boundary and phi_i are the basis functions in
+   //    the finite element fespace. The force is defined by the object f, which
+   //    is a vector of Coefficient objects. The fact that f is non-zero on
+   //    boundary attribute 2 is indicated by the use of piece-wise constants
+   //    coefficient for its last component.
   VectorArrayCoefficient f(dim);
   for (int i = 0; i < dim-1; i++)
   {
@@ -220,13 +212,13 @@ int main(int argc, char *argv[])
   }
   b->Assemble();

-   // 11. Define the solution vector x as a parallel finite element grid
+   // 10. Define the solution vector x as a parallel finite element grid
   //     function corresponding to fespace. Initialize x with initial guess of
   //     zero, which satisfies the boundary conditions.
   ParGridFunction x(fespace);
   x = 0.0;

-   // 12. Set up the parallel bilinear form a(.,.) on the finite element space
+   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
   //     corresponding to the linear elasticity integrator with piece-wise
   //     constants coefficient lambda and mu.
   Vector lambda(pmesh->attributes.Max());
@@ -241,7 +233,7 @@ int main(int argc, char *argv[])
   ParBilinearForm *a = new ParBilinearForm(fespace);
   a->AddDomainIntegrator(new ElasticityIntegrator(lambda_func, mu_func));

-   // 13. Assemble the parallel bilinear form and the corresponding linear
+   // 12. Assemble the parallel bilinear form and the corresponding linear
   //     system, applying any necessary transformations such as: parallel
   //     assembly, eliminating boundary conditions, applying conforming
   //     constraints for non-conforming AMR, static condensation, etc.
@@ -258,7 +250,7 @@ int main(int argc, char *argv[])
      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
   }

-   // 14. Define and apply a parallel PCG solver for A X = B with the BoomerAMG
+   // 13. Define and apply a parallel PCG solver for A X = B with the BoomerAMG
   //     preconditioner from hypre.
   HypreBoomerAMG *amg = new HypreBoomerAMG(A);
   if (amg_elast && !a->StaticCondensationIsEnabled())
@@ -276,11 +268,11 @@ int main(int argc, char *argv[])
   pcg->SetPreconditioner(*amg);
   pcg->Mult(B, X);

-   // 15. Recover the parallel grid function corresponding to X. This is the
+   // 14. Recover the parallel grid function corresponding to X. This is the
   //     local finite element solution on each processor.
   a->RecoverFEMSolution(X, *b, x);

-   // 16. For non-NURBS meshes, make the mesh curved based on the finite element
+   // 15. For non-NURBS meshes, make the mesh curved based on the finite element
   //     space. This means that we define the mesh elements through a fespace
   //     based transformation of the reference element.  This allows us to save
   //     the displaced mesh as a curved mesh when using high-order finite
@@ -292,7 +284,7 @@ int main(int argc, char *argv[])
      pmesh->SetNodalFESpace(fespace);
   }

-   // 17. Save in parallel the displaced mesh and the inverted solution (which
+   // 16. Save in parallel the displaced mesh and the inverted solution (which
   //     gives the backward displacements to the original grid). This output
   //     can be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
   {
@@ -313,7 +305,7 @@ int main(int argc, char *argv[])
      x.Save(sol_ofs);
   }

-   // 18. Send the above data by socket to a GLVis server.  Use the "n" and "b"
+   // 17. Send the above data by socket to a GLVis server.  Use the "n" and "b"
   //     keys in GLVis to visualize the displacements.
   if (visualization)
   {
@@ -325,7 +317,7 @@ int main(int argc, char *argv[])
      sol_sock << "solution\n" << *pmesh << x << flush;
   }

-   // 19. Free the used memory.
+   // 18. Free the used memory.
   delete pcg;
   delete amg;
   delete a;
@@ -16,8 +16,6 @@
 //               ex3 -m ../data/beam-hex-nurbs.mesh
 //               ex3 -m ../data/amr-hex.mesh
 //               ex3 -m ../data/fichera-amr.mesh
-//               ex3 -m ../data/ref-prism.mesh -o 1
-//               ex3 -m ../data/octahedron.mesh -o 1
 //               ex3 -m ../data/star-surf.mesh -o 1
 //               ex3 -m ../data/mobius-strip.mesh -f 0.1
 //               ex3 -m ../data/klein-bottle.mesh -f 0.1
@@ -115,6 +113,7 @@ int main(int argc, char *argv[])
         mesh->UniformRefinement();
      }
   }
+   mesh->ReorientTetMesh();

   // 5. Define a finite element space on the mesh. Here we use the Nedelec
   //    finite elements of the specified order.
@@ -1,195 +0,0 @@
-//                            MFEM Example 30
-//
-// Compile with: make ex30
-//
-// Sample runs:  ex30 -m ../data/square-disc.mesh -o 1
-//               ex30 -m ../data/square-disc.mesh -o 2
-//               ex30 -m ../data/square-disc.mesh -o 2 -me 1e3
-//               ex30 -m ../data/square-disc-nurbs.mesh -o 2
-//               ex30 -m ../data/star.mesh -o 2 -eo 4
-//               ex30 -m ../data/fichera.mesh -o 2 -me 1e4
-//               ex30 -m ../data/disc-nurbs.mesh -o 2
-//               ex30 -m ../data/ball-nurbs.mesh -o 2 -eo 3 -e 1e-2 -me 1e4
-//               ex30 -m ../data/star-surf.mesh -o 2
-//               ex30 -m ../data/square-disc-surf.mesh -o 2
-//               ex30 -m ../data/amr-quad.mesh -l 2
-//
-// Description:  This is an example of adaptive mesh refinement preprocessing
-//               which lowers the data oscillation [1] to a user-defined
-//               relative threshold. There is no PDE being solved.
-//
-//               MFEM's capability to work with both conforming and
-//               nonconforming meshes is demonstrated in example 6. In some
-//               problems, the material data or loading data is not sufficiently
-//               resolved on the initial mesh. This missing fine scale data
-//               reduces the accuracy of the solution as well as the accuracy
-//               of some local error estimators. By preprocessing the mesh
-//               before solving the PDE, many issues can be avoided.
-//
-//               [1] Morin, P., Nochetto, R. H., & Siebert, K. G. (2000).
-//                   Data oscillation and convergence of adaptive FEM. SIAM
-//                   Journal on Numerical Analysis, 38(2), 466-488.
-//
-//               [2] Mitchell, W. F. (2013). A collection of 2D elliptic
-//                   problems for testing adaptive grid refinement algorithms.
-//                   Applied mathematics and computation, 220, 350-364.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Piecewise-affine function which is sometimes mesh-conforming
-double affine_function(const Vector &p)
-{
-   double x = p(0), y = p(1);
-   if (x < 0.0)
-   {
-      return 1.0 + x + y;
-   }
-   else
-   {
-      return 1.0;
-   }
-}
-
-// Piecewise-constant function which is never mesh-conforming
-double jump_function(const Vector &p)
-{
-   if (p.Normlp(2.0) > 0.4 && p.Normlp(2.0) < 0.6) { return 1.0; }
-   return 5.0;
-}
-
-// Singular function derived from the Laplacian of the "steep wavefront"
-// problem in [2].
-double singular_function(const Vector &p)
-{
-   double x = p(0), y = p(1);
-   double alpha = 1000.0;
-   double xc = 0.75, yc = 0.5;
-   double r0 = 0.7;
-   double r = sqrt(pow(x - xc,2.0) + pow(y - yc,2.0));
-   double num = - ( alpha - pow(alpha,3) * (pow(r,2) - pow(r0,2)) );
-   double denom = pow(r * ( pow(alpha,2) * pow(r0,2) + pow(alpha,2) * pow(r,2) \
-                            - 2 * pow(alpha,2) * r0 * r + 1.0 ),2);
-   denom = max(denom,1e-8);
-   return num / denom;
-}
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../data/star.mesh";
-   int order = 1;
-   int nc_limit = 1;
-   int max_elems = 1e5;
-   double double_max_elems = double(max_elems);
-   bool visualization = true;
-   double osc_threshold = 1e-3;
-   int enriched_order = 5;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&nc_limit, "-l", "--nc-limit",
-                  "Maximum level of hanging nodes.");
-   args.AddOption(&double_max_elems, "-me", "--max-elems",
-                  "Stop after reaching this many elements.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&osc_threshold, "-e", "--error",
-                  "relative data oscillation threshold.");
-   args.AddOption(&enriched_order, "-eo", "--enriched_order",
-                  "Enriched quadrature order.");
-
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   max_elems = int(double_max_elems);
-   Mesh mesh(mesh_file, 1, 1);
-
-   // 2. Since a NURBS mesh can currently only be refined uniformly, we need to
-   //    convert it to a piecewise-polynomial curved mesh. First we refine the
-   //    NURBS mesh a bit more and then project the curvature to quadratic Nodes.
-   if (mesh.NURBSext)
-   {
-      for (int i = 0; i < 2; i++)
-      {
-         mesh.UniformRefinement();
-      }
-      mesh.SetCurvature(2);
-   }
-
-   // 3. Define functions and refiner.
-   FunctionCoefficient affine_coeff(affine_function);
-   FunctionCoefficient jump_coeff(jump_function);
-   FunctionCoefficient singular_coeff(singular_function);
-   CoefficientRefiner coeffrefiner(affine_coeff, order);
-
-   // 4. Connect to GLVis.
-   char vishost[] = "localhost";
-   int  visport   = 19916;
-   socketstream sol_sock;
-   if (visualization)
-   {
-      sol_sock.open(vishost, visport);
-   }
-
-   // 5. Define custom integration rule (optional).
-   const IntegrationRule *irs[Geometry::NumGeom];
-   int order_quad = 2*order + enriched_order;
-   for (int i = 0; i < Geometry::NumGeom; ++i)
-   {
-      irs[i] = &(IntRules.Get(i, order_quad));
-   }
-
-   // 6. Apply custom refiner settings.
-   coeffrefiner.SetIntRule(irs);
-   coeffrefiner.SetMaxElements(max_elems);
-   coeffrefiner.SetThreshold(osc_threshold);
-   coeffrefiner.SetNCLimit(nc_limit);
-   coeffrefiner.PrintWarnings();
-
-   // 7. Preprocess mesh to control osc (piecewise-affine function).
-   //    This is mostly just a verification check. The oscillation should
-   //    be zero if the function is mesh-conforming and order > 0.
-   coeffrefiner.PreprocessMesh(mesh);
-
-   mfem::out  << "\n";
-   mfem::out << "Function 0 (affine) \n";
-   mfem::out << "Number of Elements " << mesh.GetNE() << "\n";
-   mfem::out << "Osc error " << coeffrefiner.GetOsc() << "\n";
-
-   // 8. Preprocess mesh to control osc (jump function).
-   coeffrefiner.ResetCoefficient(jump_coeff);
-   coeffrefiner.PreprocessMesh(mesh);
-
-   mfem::out  << "\n";
-   mfem::out << "Function 1 (discontinuous) \n";
-   mfem::out << "Number of Elements " << mesh.GetNE() << "\n";
-   mfem::out << "Osc error " << coeffrefiner.GetOsc() << "\n";
-
-   // 9. Preprocess mesh to control osc (singular function).
-   coeffrefiner.ResetCoefficient(singular_coeff);
-   coeffrefiner.PreprocessMesh(mesh);
-
-   mfem::out  << "\n";
-   mfem::out << "Function 2 (singular) \n";
-   mfem::out << "Number of Elements " << mesh.GetNE() << "\n";
-   mfem::out << "Osc error " << coeffrefiner.GetOsc() << "\n";
-
-   sol_sock.precision(8);
-   sol_sock << "mesh\n" << mesh << flush;
-
-   return 0;
-}
@@ -1,241 +0,0 @@
-//                        MFEM Example 30 - Parallel Version
-//
-// Compile with: make ex30p
-//
-// Sample runs:  mpirun -np 4 ex30p -m ../data/square-disc.mesh -o 1
-//               mpirun -np 4 ex30p -m ../data/square-disc.mesh -o 2
-//               mpirun -np 4 ex30p -m ../data/square-disc.mesh -o 2 -me 1e3
-//               mpirun -np 4 ex30p -m ../data/square-disc-nurbs.mesh -o 2
-//               mpirun -np 4 ex30p -m ../data/star.mesh -o 2 -eo 4
-//               mpirun -np 4 oscp -m ../data/fichera.mesh -o 2 -me 1e4
-//               mpirun -np 4 ex30p -m ../data/disc-nurbs.mesh -o 2
-//               mpirun -np 4 ex30p -m ../data/ball-nurbs.mesh -o 2 -eo 3 -e 1e-2
-//               mpirun -np 4 ex30p -m ../data/star-surf.mesh -o 2
-//               mpirun -np 4 ex30p -m ../data/square-disc-surf.mesh -o 2
-//               mpirun -np 4 ex30p -m ../data/amr-quad.mesh -l 2
-//
-// Description:  This is an example of adaptive mesh refinement preprocessing
-//               which lowers the data oscillation [1] to a user-defined
-//               relative threshold. There is no PDE being solved.
-//
-//               MFEM's capability to work with both conforming and
-//               nonconforming meshes is demonstrated in example 6. In some
-//               problems, the material data or loading data is not sufficiently
-//               resolved on the initial mesh. This missing fine scale data
-//               reduces the accuracy of the solution as well as the accuracy
-//               of some local error estimators. By preprocessing the mesh
-//               before solving the PDE, many issues can be avoided.
-//
-//               [1] Morin, P., Nochetto, R. H., & Siebert, K. G. (2000).
-//                   Data oscillation and convergence of adaptive FEM. SIAM
-//                   Journal on Numerical Analysis, 38(2), 466-488.
-//
-//               [2] Mitchell, W. F. (2013). A collection of 2D elliptic
-//                   problems for testing adaptive grid refinement algorithms.
-//                   Applied mathematics and computation, 220, 350-364.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Piecewise-affine function which is sometimes mesh-conforming
-double affine_function(const Vector &p)
-{
-   double x = p(0), y = p(1);
-   if (x < 0.0)
-   {
-      return 1.0 + x + y;
-   }
-   else
-   {
-      return 1.0;
-   }
-}
-
-// Piecewise-constant function which is never mesh-conforming
-double jump_function(const Vector &p)
-{
-   if (p.Normlp(2.0) > 0.4 && p.Normlp(2.0) < 0.6) { return 1.0; }
-   return 5.0;
-}
-
-// Singular function derived from the Laplacian of the "steep wavefront"
-// problem in [2].
-double singular_function(const Vector &p)
-{
-   double x = p(0), y = p(1);
-   double alpha = 1000.0;
-   double xc = 0.75, yc = 0.5;
-   double r0 = 0.7;
-   double r = sqrt(pow(x - xc,2.0) + pow(y - yc,2.0));
-   double num = - ( alpha - pow(alpha,3) * (pow(r,2) - pow(r0,2)) );
-   double denom = pow(r * ( pow(alpha,2) * pow(r0,2) + pow(alpha,2) * pow(r,2) \
-                            - 2 * pow(alpha,2) * r0 * r + 1.0 ),2);
-   denom = max(denom,1e-8);
-   return num / denom;
-}
-
-int main(int argc, char *argv[])
-{
-   // 0. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 1. Parse command-line options.
-   const char *mesh_file = "../data/star.mesh";
-   int order = 1;
-   int nc_limit = 1;
-   int max_elems = 1e5;
-   double double_max_elems = double(max_elems);
-   bool visualization = true;
-   bool nc_simplices = true;
-   double osc_threshold = 1e-3;
-   int enriched_order = 5;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&nc_limit, "-l", "--nc-limit",
-                  "Maximum level of hanging nodes.");
-   args.AddOption(&double_max_elems, "-me", "--max-elems",
-                  "Stop after reaching this many elements.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&osc_threshold, "-e", "--error",
-                  "relative data oscillation threshold.");
-   args.AddOption(&enriched_order, "-eo", "--enriched_order",
-                  "Enriched quadrature order.");
-   args.AddOption(&nc_simplices, "-ns", "--nonconforming-simplices",
-                  "-cs", "--conforming-simplices",
-                  "For simplicial meshes, enable/disable nonconforming"
-                  " refinement");
-
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   max_elems = int(double_max_elems);
-   Mesh mesh(mesh_file, 1, 1);
-
-   // 2. Since a NURBS mesh can currently only be refined uniformly, we need to
-   //    convert it to a piecewise-polynomial curved mesh. First we refine the
-   //    NURBS mesh a bit more and then project the curvature to quadratic Nodes.
-   if (mesh.NURBSext)
-   {
-      for (int i = 0; i < 2; i++)
-      {
-         mesh.UniformRefinement();
-      }
-      mesh.SetCurvature(2);
-   }
-
-   // 3. Make sure the mesh is in the non-conforming mode to enable local
-   //    refinement of quadrilaterals/hexahedra. Simplices can be refined
-   //    either in conforming or in non-conforming mode. The conforming
-   //    mode however does not support dynamic partitioning.
-   mesh.EnsureNCMesh(nc_simplices);
-
-   // 4. Define a parallel mesh by partitioning the serial mesh.
-   //    Once the parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
-
-   // 5. Define functions and refiner.
-   FunctionCoefficient affine_coeff(affine_function);
-   FunctionCoefficient jump_coeff(jump_function);
-   FunctionCoefficient singular_coeff(singular_function);
-   CoefficientRefiner coeffrefiner(affine_coeff,order);
-
-   // 6. Connect to GLVis.
-   char vishost[] = "localhost";
-   int  visport   = 19916;
-   socketstream sol_sock;
-   if (visualization)
-   {
-      sol_sock.open(vishost, visport);
-   }
-
-   // 7. Define custom integration rule (optional).
-   const IntegrationRule *irs[Geometry::NumGeom];
-   int order_quad = 2*order + enriched_order;
-   for (int i=0; i < Geometry::NumGeom; ++i)
-   {
-      irs[i] = &(IntRules.Get(i, order_quad));
-   }
-
-   // 8. Apply custom refiner settings.
-   coeffrefiner.SetIntRule(irs);
-   coeffrefiner.SetMaxElements(max_elems);
-   coeffrefiner.SetThreshold(osc_threshold);
-   coeffrefiner.SetNCLimit(nc_limit);
-   coeffrefiner.PrintWarnings();
-
-   // 9. Preprocess mesh to control osc (piecewise-affine function).
-   //    This is mostly just a verification check. The oscillation should
-   //    be zero if the function is mesh-conforming and order > 0.
-   coeffrefiner.PreprocessMesh(pmesh);
-
-   int globalNE = pmesh.GetGlobalNE();
-   double osc = coeffrefiner.GetOsc();
-   if (myid == 0)
-   {
-      mfem::out  << "\n";
-      mfem::out << "Function 0 (affine) \n";
-      mfem::out << "Number of Elements " << globalNE << "\n";
-      mfem::out << "Osc error " << osc << "\n";
-   }
-
-   // 10. Preprocess mesh to control osc (jump function).
-   coeffrefiner.ResetCoefficient(jump_coeff);
-   coeffrefiner.PreprocessMesh(pmesh);
-
-   globalNE = pmesh.GetGlobalNE();
-   osc = coeffrefiner.GetOsc();
-   if (myid == 0)
-   {
-      mfem::out  << "\n";
-      mfem::out << "Function 1 (discontinuous) \n";
-      mfem::out << "Number of Elements " << globalNE << "\n";
-      mfem::out << "Osc error " << osc << "\n";
-   }
-
-   // 11. Preprocess mesh to control osc (singular function).
-   coeffrefiner.ResetCoefficient(singular_coeff);
-   coeffrefiner.PreprocessMesh(pmesh);
-
-   globalNE = pmesh.GetGlobalNE();
-   osc = coeffrefiner.GetOsc();
-   if (myid == 0)
-   {
-      mfem::out  << "\n";
-      mfem::out << "Function 2 (singular) \n";
-      mfem::out << "Number of Elements " << globalNE << "\n";
-      mfem::out << "Osc error " << osc << "\n";
-   }
-
-   sol_sock.precision(8);
-   sol_sock << "parallel " << num_procs << " " << myid << "\n";
-   sol_sock << "mesh\n" << pmesh << flush;
-
-   MPI_Finalize();
-   return 0;
-}
@@ -16,8 +16,6 @@
 //               mpirun -np 4 ex3p -m ../data/beam-hex-nurbs.mesh
 //               mpirun -np 4 ex3p -m ../data/amr-quad.mesh -o 2
 //               mpirun -np 4 ex3p -m ../data/amr-hex.mesh
-//               mpirun -np 4 ex3p -m ../data/ref-prism.mesh -o 1
-//               mpirun -np 4 ex3p -m ../data/octahedron.mesh -o 1
 //               mpirun -np 4 ex3p -m ../data/star-surf.mesh -o 2
 //               mpirun -np 4 ex3p -m ../data/mobius-strip.mesh -o 2 -f 0.1
 //               mpirun -np 4 ex3p -m ../data/klein-bottle.mesh -o 2 -f 0.1
@@ -105,7 +103,6 @@ int main(int argc, char *argv[])
      {
         args.PrintUsage(cout);
      }
-      // HYPRE_Finalize();
      MPI_Finalize();
      return 1;
   }
@@ -141,7 +138,9 @@ int main(int argc, char *argv[])

   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
+   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
+   //    meshes need to be reoriented before we can define high-order Nedelec
+   //    spaces on them.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
   delete mesh;
   {
@@ -151,6 +150,7 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
+   pmesh->ReorientTetMesh();

   // 7. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Nedelec finite elements of the specified order.
@@ -1,411 +0,0 @@
-//                       MFEM Example 3 - Parallel Version
-//
-// Compile with: make ex3p
-//
-// Sample runs:  mpirun -np 4 ex3p -m ../data/star.mesh
-//               mpirun -np 4 ex3p -m ../data/square-disc.mesh -o 2
-//               mpirun -np 4 ex3p -m ../data/beam-tet.mesh
-//               mpirun -np 4 ex3p -m ../data/beam-hex.mesh
-//               mpirun -np 4 ex3p -m ../data/escher.mesh
-//               mpirun -np 4 ex3p -m ../data/fichera.mesh
-//               mpirun -np 4 ex3p -m ../data/fichera-q2.vtk
-//               mpirun -np 4 ex3p -m ../data/fichera-q3.mesh
-//               mpirun -np 4 ex3p -m ../data/square-disc-nurbs.mesh
-//               mpirun -np 4 ex3p -m ../data/beam-hex-nurbs.mesh
-//               mpirun -np 4 ex3p -m ../data/amr-quad.mesh -o 2
-//               mpirun -np 4 ex3p -m ../data/amr-hex.mesh
-//               mpirun -np 4 ex3p -m ../data/star-surf.mesh -o 2
-//               mpirun -np 4 ex3p -m ../data/mobius-strip.mesh -o 2 -f 0.1
-//               mpirun -np 4 ex3p -m ../data/klein-bottle.mesh -o 2 -f 0.1
-//
-// Description:  This example code solves a simple electromagnetic diffusion
-//               problem corresponding to the second order definite Maxwell
-//               equation curl curl E + E = f with boundary condition
-//               E x n = <given tangential field>. Here, we use a given exact
-//               solution E and compute the corresponding r.h.s. f.
-//               We discretize with Nedelec finite elements in 2D or 3D.
-//
-//               The example demonstrates the use of H(curl) finite element
-//               spaces with the curl-curl and the (vector finite element) mass
-//               bilinear form, as well as the computation of discretization
-//               error when the exact solution is known. Static condensation is
-//               also illustrated.
-//
-//               We recommend viewing examples 1-2 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Exact solution, E, and r.h.s., f. See below for implementation.
-//void E_exact(const Vector &, Vector &);
-//void f_exact(const Vector &, Vector &);
-//double freq = 1.0, kappa;
-void f_const(const Vector &, Vector &);
-
-int dim;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../data/beam-tet.mesh";
-   int order = 1;
-   int rs = -1;
-   int rp = 2;
-   int ra = 0;
-   int bt = EntitySets::INVALID;
-   const char *bs = "Origin";
-   bool static_cond = false;
-   bool visualization = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   /*
-   args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
-                  " solution.");
-   */
-   args.AddOption(&rs, "-rs", "--refine-serial",
-                  "Number of serial refinement levels");
-   args.AddOption(&rp, "-rp", "--refine-parallel",
-                  "Number of parallel refinement levels");
-   args.AddOption(&ra, "-ra", "--refine-adaptive",
-                  "Number of adaptive refinement levels");
-   args.AddOption(&bt, "-bt", "--bc-entity-type",
-                  "");
-   args.AddOption(&bs, "-bs", "--bc-entity-set-name",
-                  "");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-   // kappa = freq * M_PI;
-
-   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   dim = mesh->Dimension();
-   int sdim = mesh->SpaceDimension();
-
-   // 4. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 1,000 elements.
-   {
-      int ref_levels = ( rs >= 0 ) ? rs :
-                       (int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         if ( myid == 0 ) { cout << "Uniform refinement in serial..."; }
-         mesh->UniformRefinement();
-      }
-      MPI_Barrier(MPI_COMM_WORLD);
-      if ( myid == 0 && rs > 0 ) { cout << "Done" << endl; }
-   }
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-      mesh->ent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "mesh->ent_sets is NULL" << endl;
-   }
-   /*
-     At this point we have a serial mesh containing an EntitySets
-     object which stores the current node/edge/face/element indices
-     for each entity in each set.  This data is duplicated on each MPI
-     rank.
-    */
-   if ( ra > 0 )
-   {
-      cout << "calling EnsureNCMesh" << endl;
-      mesh->EnsureNCMesh();
-      cout << "back from EnsureNCMesh" << endl;
-   }
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-      mesh->ent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "mesh->ent_sets is NULL" << endl;
-   }
-   /*
-     We now have an NCEntitySets object which stores the node indices
-     describing each enity in each node/edge/face set and the element
-     indices for the elements in each element set.  This data is
-     duplicated on each MPI rank.
-    */
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
-   //    meshes need to be reoriented before we can define high-order Nedelec
-   //    spaces on them.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   if ( pmesh->pent_sets )
-   {
-      cout << "pmesh->pent_sets is non NULL" << endl;
-      pmesh->pent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "pmesh->pent_sets is NULL" << endl;
-   }
-   {
-      int par_ref_levels = rp;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         if ( myid == 0 ) { cout << "Uniform refinement in parallel..."; }
-         pmesh->UniformRefinement();
-      }
-      MPI_Barrier(MPI_COMM_WORLD);
-      if ( myid == 0 && rs > 0 ) { cout << "Done" << endl; }
-   }
-   pmesh->ReorientTetMesh();
-   pmesh->ent_sets->PrintSetInfo(cout);
-
-   for (int l = 0; l < ra; l++)
-   {
-      pmesh->RandomRefinement(0.2);
-   }
-   if ( ra > 0 )
-   {
-      if ( pmesh->pent_sets )
-      {
-         cout << "pmesh->pent_sets is non NULL post random refinement" << endl;
-         pmesh->pent_sets->PrintSetInfo(cout);
-      }
-      else
-      {
-         cout << "pmesh->pent_sets is NULL post random refinement" << endl;
-      }
-   }
-
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
-   //    use the Nedelec finite elements of the specified order.
-   FiniteElementCollection *fec = new ND_FECollection(order, dim);
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_Int size = fespace->GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-   }
-
-   // 7. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet) and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if ( bt == EntitySets::INVALID )
-   {
-      if (pmesh->bdr_attributes.Size())
-      {
-         Array<int> ess_bdr(pmesh->bdr_attributes.Max());
-         ess_bdr = 1;
-         fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-   }
-   else
-   {
-      fespace->GetEssentialTrueDofs((EntitySets::EntityType)bt, bs,
-                                    ess_tdof_list);
-   }
-   if (myid == 0)
-   {
-      cout << "Number of Dirichlet dofs: " << ess_tdof_list.Size() << endl;
-   }
-
-
-   // 8. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
-   //    basis functions in the finite element fespace.
-   VectorFunctionCoefficient f(sdim, f_const);
-   ParLinearForm *b = new ParLinearForm(fespace);
-   b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
-   b->Assemble();
-
-   // 9. Define the solution vector x as a parallel finite element grid function
-   //    corresponding to fespace. Initialize x by projecting the exact
-   //    solution. Note that only values from the boundary edges will be used
-   //    when eliminating the non-homogeneous boundary condition to modify the
-   //    r.h.s. vector b.
-   ParGridFunction x(fespace);
-   // VectorFunctionCoefficient E(sdim, E_exact);
-   // x.ProjectCoefficient(E);
-   x = 0.0;
-
-   // 10. Set up the parallel bilinear form corresponding to the EM diffusion
-   //     operator curl muinv curl + sigma I, by adding the curl-curl and the
-   //     mass domain integrators.
-   Coefficient *muinv = new ConstantCoefficient(1.0);
-   Coefficient *sigma = new ConstantCoefficient(1.0);
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
-   a->AddDomainIntegrator(new VectorFEMassIntegrator(*sigma));
-
-   // 11. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
-
-   HypreParMatrix A;
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-
-   if (myid == 0)
-   {
-      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
-   }
-
-   // 12. Define and apply a parallel PCG solver for AX=B with the AMS
-   //     preconditioner from hypre.
-   ParFiniteElementSpace *prec_fespace =
-      (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
-   HypreSolver *ams = new HypreAMS(A, prec_fespace);
-   HyprePCG *pcg = new HyprePCG(A);
-   pcg->SetTol(1e-12);
-   pcg->SetMaxIter(500);
-   pcg->SetPrintLevel(2);
-   pcg->SetPreconditioner(*ams);
-   pcg->Mult(B, X);
-
-   // 13. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a->RecoverFEMSolution(X, *b, x);
-
-   /*
-   // 14. Compute and print the L^2 norm of the error.
-   {
-      double err = x.ComputeL2Error(E);
-      if (myid == 0)
-      {
-         cout << "\n|| E_h - E ||_{L^2} = " << err << '\n' << endl;
-      }
-   }
-   */
-   // 15. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 16. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << x << flush;
-   }
-
-   // 17. Free the used memory.
-   delete pcg;
-   delete ams;
-   delete a;
-   delete sigma;
-   delete muinv;
-   delete b;
-   delete fespace;
-   delete fec;
-   delete pmesh;
-
-   MPI_Finalize();
-
-   return 0;
-}
-
-/*
-void E_exact(const Vector &x, Vector &E)
-{
-   if (dim == 3)
-   {
-      E(0) = sin(kappa * x(1));
-      E(1) = sin(kappa * x(2));
-      E(2) = sin(kappa * x(0));
-   }
-   else
-   {
-      E(0) = sin(kappa * x(1));
-      E(1) = sin(kappa * x(0));
-      if (x.Size() == 3) { E(2) = 0.0; }
-   }
-}
-
-void f_exact(const Vector &x, Vector &f)
-{
-   if (dim == 3)
-   {
-      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
-      f(1) = (1. + kappa * kappa) * sin(kappa * x(2));
-      f(2) = (1. + kappa * kappa) * sin(kappa * x(0));
-   }
-   else
-   {
-      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
-      f(1) = (1. + kappa * kappa) * sin(kappa * x(0));
-      if (x.Size() == 3) { f(2) = 0.0; }
-   }
-}
-*/
-void f_const(const Vector &x, Vector &f)
-{
-   if (dim == 3)
-   {
-      f(0) = 1.0;
-      f(1) = 1.0;
-      f(2) = 1.0;
-   }
-   else
-   {
-      f(0) = 1.0;
-      f(1) = 1.0;
-      if (x.Size() == 3) { f(2) = 0.0; }
-   }
-}
@@ -19,8 +19,6 @@
 //               ex4 -m ../data/amr-hex.mesh
 //               ex4 -m ../data/amr-hex.mesh -o 2 -hb
 //               ex4 -m ../data/fichera-amr.mesh -o 2 -sc
-//               ex4 -m ../data/ref-prism.mesh -o 1
-//               ex4 -m ../data/octahedron.mesh -o 1
 //               ex4 -m ../data/star-surf.mesh -o 1
 //
 // Device sample runs:
@@ -19,8 +19,6 @@
 //               mpirun -np 3 ex4p -m ../data/amr-quad.mesh -o 2 -hb
 //               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -sc
 //               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -hb
-//               mpirun -np 4 ex4p -m ../data/ref-prism.mesh -o 1
-//               mpirun -np 4 ex4p -m ../data/octahedron.mesh -o 1
 //               mpirun -np 4 ex4p -m ../data/star-surf.mesh -o 3 -hb
 //
 // Device sample runs:
@@ -137,7 +135,9 @@ int main(int argc, char *argv[])

   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
+   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
+   //    meshes need to be reoriented before we can define high-order Nedelec
+   //    spaces on them (this is needed in the ADS solver below).
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
   delete mesh;
   {
@@ -147,6 +147,7 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
+   pmesh->ReorientTetMesh();

   // 7. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Raviart-Thomas finite elements of the specified order.
@@ -1,438 +0,0 @@
-//                       MFEM Example 4 - Parallel Version
-//
-// Compile with: make ex4p
-//
-// Sample runs:  mpirun -np 4 ex4p -m ../data/square-disc.mesh
-//               mpirun -np 4 ex4p -m ../data/star.mesh
-//               mpirun -np 4 ex4p -m ../data/beam-tet.mesh
-//               mpirun -np 4 ex4p -m ../data/beam-hex.mesh
-//               mpirun -np 4 ex4p -m ../data/escher.mesh -o 2 -sc
-//               mpirun -np 4 ex4p -m ../data/fichera.mesh -o 2 -hb
-//               mpirun -np 4 ex4p -m ../data/fichera-q2.vtk
-//               mpirun -np 4 ex4p -m ../data/fichera-q3.mesh -o 2 -sc
-//               mpirun -np 4 ex4p -m ../data/square-disc-nurbs.mesh -o 3
-//               mpirun -np 4 ex4p -m ../data/beam-hex-nurbs.mesh -o 3
-//               mpirun -np 4 ex4p -m ../data/periodic-square.mesh -no-bc
-//               mpirun -np 4 ex4p -m ../data/periodic-cube.mesh -no-bc
-//               mpirun -np 4 ex4p -m ../data/amr-quad.mesh
-//               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -sc
-//               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -hb
-//               mpirun -np 4 ex4p -m ../data/star-surf.mesh -o 3 -hb
-//
-// Description:  This example code solves a simple 2D/3D H(div) diffusion
-//               problem corresponding to the second order definite equation
-//               -grad(alpha div F) + beta F = f with boundary condition F dot n
-//               = <given normal field>. Here, we use a given exact solution F
-//               and compute the corresponding r.h.s. f.  We discretize with
-//               Raviart-Thomas finite elements.
-//
-//               The example demonstrates the use of H(div) finite element
-//               spaces with the grad-div and H(div) vector finite element mass
-//               bilinear form, as well as the computation of discretization
-//               error when the exact solution is known. Bilinear form
-//               hybridization and static condensation are also illustrated.
-//
-//               We recommend viewing examples 1-3 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Exact solution, F, and r.h.s., f. See below for implementation.
-//void F_exact(const Vector &, Vector &);
-//void f_exact(const Vector &, Vector &);
-//double freq = 1.0, kappa;
-void f_const(const Vector &, Vector &);
-
-int dim;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../data/star.mesh";
-   int order = 1;
-   int rs = -1;
-   int rp = 2;
-   int ra = 0;
-   int bt = EntitySets::INVALID;
-   const char *bs = "Origin";
-   bool set_bc = true;
-   bool static_cond = false;
-   bool hybridization = false;
-   bool visualization = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&set_bc, "-bc", "--impose-bc", "-no-bc", "--dont-impose-bc",
-                  "Impose or not essential boundary conditions.");
-   args.AddOption(&rs, "-rs", "--refine-serial",
-                  "Number of serial refinement levels");
-   args.AddOption(&rp, "-rp", "--refine-parallel",
-                  "Number of parallel refinement levels");
-   args.AddOption(&ra, "-ra", "--refine-adaptive",
-                  "Number of adaptive refinement levels");
-   args.AddOption(&bt, "-bt", "--bc-entity-type",
-                  "");
-   args.AddOption(&bs, "-bs", "--bc-entity-set-name",
-                  "");
-   // args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
-   //                 " solution.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&hybridization, "-hb", "--hybridization", "-no-hb",
-                  "--no-hybridization", "Enable hybridization.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-   // kappa = freq * M_PI;
-
-   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume, as well as periodic meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   dim = mesh->Dimension();
-   int sdim = mesh->SpaceDimension();
-
-   // 4. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 1,000 elements.
-   {
-      int ref_levels = ( rs >= 0 ) ? rs :
-                       (int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         if ( myid == 0 ) { cout << "Uniform refinement in serial..."; }
-         mesh->UniformRefinement();
-      }
-      MPI_Barrier(MPI_COMM_WORLD);
-      if ( myid == 0 && rs > 0 ) { cout << "Done" << endl; }
-   }
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-      mesh->ent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "mesh->ent_sets is NULL" << endl;
-   }
-   /*
-     At this point we have a serial mesh containing an EntitySets
-     object which stores the current node/edge/face/element indices
-     for each entity in each set.  This data is duplicated on each MPI
-     rank.
-    */
-   if ( ra > 0 )
-   {
-      cout << "calling EnsureNCMesh" << endl;
-      mesh->EnsureNCMesh();
-      cout << "back from EnsureNCMesh" << endl;
-   }
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-      mesh->ent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "mesh->ent_sets is NULL" << endl;
-   }
-   /*
-     We now have an NCEntitySets object which stores the node indices
-     describing each enity in each node/edge/face set and the element
-     indices for the elements in each element set.  This data is
-     duplicated on each MPI rank.
-    */
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
-   //    meshes need to be reoriented before we can define high-order Nedelec
-   //    spaces on them (this is needed in the ADS solver below).
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   if ( pmesh->pent_sets )
-   {
-      cout << "pmesh->pent_sets is non NULL" << endl;
-      pmesh->pent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "pmesh->pent_sets is NULL" << endl;
-   }
-   {
-      int par_ref_levels = rp;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         if ( myid == 0 ) { cout << "Uniform refinement in parallel..."; }
-         pmesh->UniformRefinement();
-      }
-   }
-   pmesh->ReorientTetMesh();
-
-   for (int l = 0; l < ra; l++)
-   {
-      pmesh->RandomRefinement(0.2);
-   }
-   if ( ra > 0 )
-   {
-      if ( pmesh->pent_sets )
-      {
-         cout << "pmesh->pent_sets is non NULL post random refinement" << endl;
-         pmesh->pent_sets->PrintSetInfo(cout);
-      }
-      else
-      {
-         cout << "pmesh->pent_sets is NULL post random refinement" << endl;
-      }
-   }
-
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
-   //    use the Raviart-Thomas finite elements of the specified order.
-   FiniteElementCollection *fec = new RT_FECollection(order-1, dim);
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_Int size = fespace->GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-   }
-
-   // 7. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet) and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if ( bt == EntitySets::INVALID )
-   {
-      if (pmesh->bdr_attributes.Size())
-      {
-         Array<int> ess_bdr(pmesh->bdr_attributes.Max());
-         ess_bdr = set_bc ? 1 : 0;
-         fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-   }
-   else
-   {
-      fespace->GetEssentialTrueDofs((EntitySets::EntityType)bt, bs,
-                                    ess_tdof_list);
-   }
-   if (myid == 0)
-   {
-      cout << "Number of Dirichlet dofs: " << ess_tdof_list.Size() << endl;
-   }
-
-   // 8. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
-   //    basis functions in the finite element fespace.
-   VectorFunctionCoefficient f(sdim, f_const);
-   ParLinearForm *b = new ParLinearForm(fespace);
-   b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
-   b->Assemble();
-
-   // 9. Define the solution vector x as a parallel finite element grid function
-   //    corresponding to fespace. Initialize x by projecting the exact
-   //    solution. Note that only values from the boundary faces will be used
-   //    when eliminating the non-homogeneous boundary condition to modify the
-   //    r.h.s. vector b.
-   ParGridFunction x(fespace);
-   // VectorFunctionCoefficient F(sdim, F_exact);
-   // x.ProjectCoefficient(F);
-   x = 0.0;
-
-   // 10. Set up the parallel bilinear form corresponding to the H(div)
-   //     diffusion operator grad alpha div + beta I, by adding the div-div and
-   //     the mass domain integrators.
-   Coefficient *alpha = new ConstantCoefficient(1.0);
-   Coefficient *beta  = new ConstantCoefficient(1.0);
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new DivDivIntegrator(*alpha));
-   a->AddDomainIntegrator(new VectorFEMassIntegrator(*beta));
-
-   // 11. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation,
-   //     hybridization, etc.
-   FiniteElementCollection *hfec = NULL;
-   ParFiniteElementSpace *hfes = NULL;
-   if (static_cond)
-   {
-      a->EnableStaticCondensation();
-   }
-   else if (hybridization)
-   {
-      hfec = new DG_Interface_FECollection(order-1, dim);
-      hfes = new ParFiniteElementSpace(pmesh, hfec);
-      a->EnableHybridization(hfes, new NormalTraceJumpIntegrator(),
-                             ess_tdof_list);
-   }
-   a->Assemble();
-
-   HypreParMatrix A;
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-
-   HYPRE_Int glob_size = A.GetGlobalNumRows();
-   if (myid == 0)
-   {
-      cout << "Size of linear system: " << glob_size << endl;
-   }
-
-   // 12. Define and apply a parallel PCG solver for A X = B with the 2D AMS or
-   //     the 3D ADS preconditioners from hypre. If using hybridization, the
-   //     system is preconditioned with hypre's BoomerAMG.
-   HypreSolver *prec = NULL;
-   CGSolver *pcg = new CGSolver(A.GetComm());
-   pcg->SetOperator(A);
-   pcg->SetRelTol(1e-12);
-   pcg->SetMaxIter(500);
-   pcg->SetPrintLevel(1);
-   if (hybridization) { prec = new HypreBoomerAMG(A); }
-   else
-   {
-      ParFiniteElementSpace *prec_fespace =
-         (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
-      if (dim == 2)   { prec = new HypreAMS(A, prec_fespace); }
-      else            { prec = new HypreADS(A, prec_fespace); }
-   }
-   pcg->SetPreconditioner(*prec);
-   pcg->Mult(B, X);
-
-   // 13. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a->RecoverFEMSolution(X, *b, x);
-   /*
-   // 14. Compute and print the L^2 norm of the error.
-   {
-      double err = x.ComputeL2Error(F);
-      if (myid == 0)
-      {
-         cout << "\n|| F_h - F ||_{L^2} = " << err << '\n' << endl;
-      }
-   }
-   */
-   // 15. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 16. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << x << flush;
-   }
-
-   // 17. Free the used memory.
-   delete pcg;
-   delete prec;
-   delete hfes;
-   delete hfec;
-   delete a;
-   delete alpha;
-   delete beta;
-   delete b;
-   delete fespace;
-   delete fec;
-   delete pmesh;
-
-   MPI_Finalize();
-
-   return 0;
-}
-
-/*
-// The exact solution (for non-surface meshes)
-void F_exact(const Vector &p, Vector &F)
-{
-   int dim = p.Size();
-
-   double x = p(0);
-   double y = p(1);
-   // double z = (dim == 3) ? p(2) : 0.0;
-
-   F(0) = cos(kappa*x)*sin(kappa*y);
-   F(1) = cos(kappa*y)*sin(kappa*x);
-   if (dim == 3)
-   {
-      F(2) = 0.0;
-   }
-}
-
-// The right hand side
-void f_exact(const Vector &p, Vector &f)
-{
-   int dim = p.Size();
-
-   double x = p(0);
-   double y = p(1);
-   // double z = (dim == 3) ? p(2) : 0.0;
-
-   double temp = 1 + 2*kappa*kappa;
-
-   f(0) = temp*cos(kappa*x)*sin(kappa*y);
-   f(1) = temp*cos(kappa*y)*sin(kappa*x);
-   if (dim == 3)
-   {
-      f(2) = 0;
-   }
-}
-*/
-void f_const(const Vector &x, Vector &f)
-{
-   if (dim == 3)
-   {
-      f(0) = 1.0;
-      f(1) = 1.0;
-      f(2) = 1.0;
-   }
-   else
-   {
-      f(0) = 1.0;
-      f(1) = 1.0;
-      if (x.Size() == 3) { f(2) = 0.0; }
-   }
-}
@@ -197,7 +197,6 @@ int main(int argc, char *argv[])
      SparseMatrix &M(mVarf->SpMat());
      SparseMatrix &B(bVarf->SpMat());
      B *= -1.;
-      if (Device::IsEnabled()) { B.BuildTranspose(); }
      Bt = new TransposeOperator(&B);

      darcyOp.SetBlock(0,0, &M);
@@ -241,7 +240,6 @@ int main(int argc, char *argv[])
   {
      SparseMatrix &M(mVarf->SpMat());
      M.GetDiag(Md);
-      Md.HostReadWrite();

      SparseMatrix &B(bVarf->SpMat());
      MinvBt = Transpose(B);
@@ -289,18 +287,12 @@ int main(int argc, char *argv[])
   chrono.Stop();

   if (solver.GetConverged())
-   {
      std::cout << "MINRES converged in " << solver.GetNumIterations()
-                << " iterations with a residual norm of "
-                << solver.GetFinalNorm() << ".\n";
-   }
+                << " iterations with a residual norm of " << solver.GetFinalNorm() << ".\n";
   else
-   {
      std::cout << "MINRES did not converge in " << solver.GetNumIterations()
-                << " iterations. Residual norm is " << solver.GetFinalNorm()
-                << ".\n";
-   }
-   std::cout << "MINRES solver took " << chrono.RealTime() << "s.\n";
+                << " iterations. Residual norm is " << solver.GetFinalNorm() << ".\n";
+   std::cout << "MINRES solver took " << chrono.RealTime() << "s. \n";

   // 12. Create the grid functions u and p. Compute the L2 error norms.
   GridFunction u, p;
@@ -1,325 +0,0 @@
-//                       MFEM Example 6 - Parallel Version
-//
-// Compile with: make ex6p
-//
-// Sample runs:  mpirun -np 4 ex6p -m ../data/square-disc.mesh -o 1
-//               mpirun -np 4 ex6p -m ../data/square-disc.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/square-disc-nurbs.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/star.mesh -o 3
-//               mpirun -np 4 ex6p -m ../data/escher.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/fichera.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/disc-nurbs.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/ball-nurbs.mesh
-//               mpirun -np 4 ex6p -m ../data/pipe-nurbs.mesh
-//               mpirun -np 4 ex6p -m ../data/star-surf.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/square-disc-surf.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/amr-quad.mesh
-//
-// Description:  This is a version of Example 1 with a simple adaptive mesh
-//               refinement loop. The problem being solved is again the Laplace
-//               equation -Delta u = 1 with homogeneous Dirichlet boundary
-//               conditions. The problem is solved on a sequence of meshes which
-//               are locally refined in a conforming (triangles, tetrahedrons)
-//               or non-conforming (quadrilaterals, hexahedra) manner according
-//               to a simple ZZ error estimator.
-//
-//               The example demonstrates MFEM's capability to work with both
-//               conforming and nonconforming refinements, in 2D and 3D, on
-//               linear, curved and surface meshes. Interpolation of functions
-//               from coarse to fine meshes, as well as persistent GLVis
-//               visualization are also illustrated.
-//
-//               We recommend viewing Example 1 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-static int max_dofs = 100000;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "./star-set.mesh";
-   int order = 1;
-   int bt = EntitySets::INVALID;
-   const char *bs = "";
-   bool visualization = true;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&max_dofs, "-md", "--max-dofs",
-                  "Maximum number of degrees of freedom.");
-   args.AddOption(&bt, "-bt", "--bc-entity-type",
-                  "");
-   args.AddOption(&bs, "-bs", "--bc-entity-set-name",
-                  "");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-   int sdim = mesh->SpaceDimension();
-
-   // 4. Refine the serial mesh on all processors to increase the resolution.
-   //    Also project a NURBS mesh to a piecewise-quadratic curved mesh. Make
-   //    sure that the mesh is non-conforming.
-   if (mesh->NURBSext)
-   {
-      mesh->UniformRefinement();
-      mesh->SetCurvature(2);
-   }
-   mesh->EnsureNCMesh();
-   if ( mesh->ent_sets )
-   {
-      cout << "mesh->ent_sets is non NULL" << endl;
-      mesh->ent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "mesh->ent_sets is NULL" << endl;
-   }
-
-   // 5. Define a parallel mesh by partitioning the serial mesh.
-   //    Once the parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   if ( pmesh.pent_sets )
-   {
-      cout << "pmesh->pent_sets is non NULL" << endl;
-      pmesh.pent_sets->PrintSetInfo(cout);
-   }
-   else
-   {
-      cout << "pmesh->pent_sets is NULL" << endl;
-   }
-
-   // 6. Define a finite element space on the mesh. The polynomial order is
-   //    one (linear) by default, but this can be changed on the command line.
-   H1_FECollection fec(order, dim);
-   ParFiniteElementSpace fespace(&pmesh, &fec);
-
-   Array<int> ess_tdof_list;
-   if ( bt == EntitySets::INVALID )
-   {
-      if (pmesh.bdr_attributes.Size())
-      {
-         Array<int> ess_bdr(pmesh.bdr_attributes.Max());
-         ess_bdr = 1;
-         fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-   }
-   else
-   {
-      fespace.GetEssentialTrueDofs((EntitySets::EntityType)bt, bs,
-                                   ess_tdof_list);
-   }
-
-   // 7. As in Example 1p, we set up bilinear and linear forms corresponding to
-   //    the Laplace problem -\Delta u = 1. We don't assemble the discrete
-   //    problem yet, this will be done in the main loop.
-   ParBilinearForm a(&fespace);
-   ParLinearForm b(&fespace);
-
-   ConstantCoefficient one(1.0);
-
-   BilinearFormIntegrator *integ = new DiffusionIntegrator(one);
-   a.AddDomainIntegrator(integ);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-
-   // 8. The solution vector x and the associated finite element grid function
-   //    will be maintained over the AMR iterations. We initialize it to zero.
-   ParGridFunction x(&fespace);
-   x = 0;
-
-   // 9. Connect to GLVis.
-   char vishost[] = "localhost";
-   int  visport   = 19916;
-
-   socketstream sout;
-   if (visualization)
-   {
-      sout.open(vishost, visport);
-      if (!sout)
-      {
-         if (myid == 0)
-         {
-            cout << "Unable to connect to GLVis server at "
-                 << vishost << ':' << visport << endl;
-            cout << "GLVis visualization disabled.\n";
-         }
-         visualization = false;
-      }
-
-      sout.precision(8);
-   }
-
-   // 10. Set up an error estimator. Here we use the Zienkiewicz-Zhu estimator
-   //     with L2 projection in the smoothing step to better handle hanging
-   //     nodes and parallel partitioning. We need to supply a space for the
-   //     discontinuous flux (L2) and a space for the smoothed flux (H(div) is
-   //     used here).
-   L2_FECollection flux_fec(order, dim);
-   ParFiniteElementSpace flux_fes(&pmesh, &flux_fec, sdim);
-   RT_FECollection smooth_flux_fec(order-1, dim);
-   ParFiniteElementSpace smooth_flux_fes(&pmesh, &smooth_flux_fec);
-   // Another possible option for the smoothed flux space:
-   // H1_FECollection smooth_flux_fec(order, dim);
-   // ParFiniteElementSpace smooth_flux_fes(&pmesh, &smooth_flux_fec, dim);
-   L2ZienkiewiczZhuEstimator estimator(*integ, x, flux_fes, smooth_flux_fes);
-
-   // 11. A refiner selects and refines elements based on a refinement strategy.
-   //     The strategy here is to refine elements with errors larger than a
-   //     fraction of the maximum element error. Other strategies are possible.
-   //     The refiner will call the given error estimator.
-   ThresholdRefiner refiner(estimator);
-   refiner.SetTotalErrorFraction(0.7);
-
-   // 12. The main AMR loop. In each iteration we solve the problem on the
-   //     current mesh, visualize the solution, and refine the mesh.
-   // const int max_dofs = 100000;
-   for (int it = 0; ; it++)
-   {
-      HYPRE_Int global_dofs = fespace.GlobalTrueVSize();
-      if (myid == 0)
-      {
-         cout << "\nAMR iteration " << it << endl;
-         cout << "Number of unknowns: " << global_dofs << endl;
-      }
-
-      // 13. Assemble the stiffness matrix and the right-hand side. Note that
-      //     MFEM doesn't care at this point that the mesh is nonconforming
-      //     and parallel. The FE space is considered 'cut' along hanging
-      //     edges/faces, and also across processor boundaries.
-      a.Assemble();
-      b.Assemble();
-
-      // 14. Create the parallel linear system: eliminate boundary conditions,
-      //     constrain hanging nodes and nodes across processor boundaries.
-      //     The system will be solved for true (unconstrained/unique) DOFs only.
-      // Array<int> ess_tdof_list;
-      if ( bt == EntitySets::INVALID )
-      {
-         if (pmesh.bdr_attributes.Size())
-         {
-            Array<int> ess_bdr(pmesh.bdr_attributes.Max());
-            ess_bdr = 1;
-            fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-         }
-      }
-      else
-      {
-         fespace.GetEssentialTrueDofs((EntitySets::EntityType)bt, bs,
-                                      ess_tdof_list);
-      }
-
-      HypreParMatrix A;
-      Vector B, X;
-      const int copy_interior = 1;
-      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
-
-      // 15. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
-      //     preconditioner from hypre.
-      HypreBoomerAMG amg;
-      amg.SetPrintLevel(0);
-      CGSolver pcg(A.GetComm());
-      pcg.SetPreconditioner(amg);
-      pcg.SetOperator(A);
-      pcg.SetRelTol(1e-6);
-      pcg.SetMaxIter(200);
-      pcg.SetPrintLevel(3); // print the first and the last iterations only
-      pcg.Mult(B, X);
-
-      // 16. Extract the parallel grid function corresponding to the finite element
-      //     approximation X. This is the local solution on each processor.
-      a.RecoverFEMSolution(X, b, x);
-
-      // 17. Send the solution by socket to a GLVis server.
-      if (visualization)
-      {
-         sout << "parallel " << num_procs << " " << myid << "\n";
-         sout << "solution\n" << pmesh << x << flush;
-      }
-
-      if (global_dofs > max_dofs)
-      {
-         if (myid == 0)
-         {
-            cout << "Reached the maximum number of dofs. Stop." << endl;
-         }
-         break;
-      }
-
-      // 18. Call the refiner to modify the mesh. The refiner calls the error
-      //     estimator to obtain element errors, then it selects elements to be
-      //     refined and finally it modifies the mesh. The Stop() method can be
-      //     used to determine if a stopping criterion was met.
-      refiner.Apply(pmesh);
-      if (refiner.Stop())
-      {
-         if (myid == 0)
-         {
-            cout << "Stopping criterion satisfied. Stop." << endl;
-         }
-         break;
-      }
-
-      // 19. Update the finite element space (recalculate the number of DOFs,
-      //     etc.) and create a grid function update matrix. Apply the matrix
-      //     to any GridFunctions over the space. In this case, the update
-      //     matrix is an interpolation matrix so the updated GridFunction will
-      //     still represent the same function as before refinement.
-      fespace.Update();
-      x.Update();
-
-      // 20. Load balance the mesh, and update the space and solution. Currently
-      //     available only for nonconforming meshes.
-      if (pmesh.Nonconforming())
-      {
-         pmesh.Rebalance();
-
-         // Update the space and the GridFunction. This time the update matrix
-         // redistributes the GridFunction among the processors.
-         fespace.Update();
-         x.Update();
-      }
-
-      // 21. Inform also the bilinear and linear forms that the space has
-      //     changed.
-      a.Update();
-      b.Update();
-   }
-
-   MPI_Finalize();
-   return 0;
-}
@@ -47,7 +47,6 @@ int main(int argc, char *argv[])
   int order = 2;
   bool always_snap = false;
   bool visualization = 1;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&elem_type, "-e", "--elem",
@@ -66,8 +65,6 @@ int main(int argc, char *argv[])
                  "--snap-at-the-end",
                  "If true, snap nodes to the sphere initially and after each refinement "
                  "otherwise, snap only after the last refinement");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -83,12 +80,7 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Generate an initial high-order (surface) mesh on the unit sphere. The
+   // 3. Generate an initial high-order (surface) mesh on the unit sphere. The
   //    Mesh object represents a 2D mesh in 3 spatial dimensions. We first add
   //    the elements and the vertices of the mesh, and then make it high-order
   //    by specifying a finite element space for its nodes.
@@ -154,7 +146,7 @@ int main(int argc, char *argv[])
   FiniteElementSpace nodal_fes(mesh, &fec, mesh->SpaceDimension());
   mesh->SetNodalFESpace(&nodal_fes);

-   // 5. Refine the mesh while snapping nodes to the sphere. Number of parallel
+   // 4. Refine the mesh while snapping nodes to the sphere. Number of parallel
   //    refinements is fixed to 2.
   for (int l = 0; l <= ref_levels; l++)
   {
@@ -226,7 +218,7 @@ int main(int argc, char *argv[])
      SnapNodes(*pmesh);
   }

-   // 6. Define a finite element space on the mesh. Here we use isoparametric
+   // 5. Define a finite element space on the mesh. Here we use isoparametric
   //    finite elements -- the same as the mesh nodes.
   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, &fec);
   HYPRE_BigInt size = fespace->GlobalTrueVSize();
@@ -235,7 +227,7 @@ int main(int argc, char *argv[])
      cout << "Number of unknowns: " << size << endl;
   }

-   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
+   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
   //    the basis functions in the finite element fespace.
   ParLinearForm *b = new ParLinearForm(fespace);
@@ -245,27 +237,27 @@ int main(int argc, char *argv[])
   b->AddDomainIntegrator(new DomainLFIntegrator(rhs_coef));
   b->Assemble();

-   // 8. Define the solution vector x as a finite element grid function
+   // 7. Define the solution vector x as a finite element grid function
   //    corresponding to fespace. Initialize x with initial guess of zero.
   ParGridFunction x(fespace);
   x = 0.0;

-   // 9. Set up the bilinear form a(.,.) on the finite element space
+   // 8. Set up the bilinear form a(.,.) on the finite element space
   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
   //    and Mass domain integrators.
   ParBilinearForm *a = new ParBilinearForm(fespace);
   a->AddDomainIntegrator(new DiffusionIntegrator(one));
   a->AddDomainIntegrator(new MassIntegrator(one));

-   // 10. Assemble the parallel linear system, applying any transformations
-   //     such as: parallel assembly, applying conforming constraints, etc.
+   // 9. Assemble the parallel linear system, applying any transformations
+   //    such as: parallel assembly, applying conforming constraints, etc.
   a->Assemble();
   HypreParMatrix A;
   Vector B, X;
   Array<int> empty_tdof_list;
   a->FormLinearSystem(empty_tdof_list, x, *b, A, X, B);

-   // 11. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
+   // 10. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
   //     preconditioner from hypre. Extract the parallel grid function x
   //     corresponding to the finite element approximation X. This is the local
   //     solution on each processor.
@@ -281,14 +273,14 @@ int main(int argc, char *argv[])
   delete a;
   delete b;

-   // 12. Compute and print the L^2 norm of the error.
+   // 11. Compute and print the L^2 norm of the error.
   double err = x.ComputeL2Error(sol_coef);
   if (myid == 0)
   {
      cout << "\nL2 norm of error: " << err << endl;
   }

-   // 13. Save the refined mesh and the solution. This output can be viewed
+   // 12. Save the refined mesh and the solution. This output can be viewed
   //     later using GLVis: "glvis -np <np> -m sphere_refined -g sol".
   {
      ostringstream mesh_name, sol_name;
@@ -304,7 +296,7 @@ int main(int argc, char *argv[])
      x.Save(sol_ofs);
   }

-   // 14. Send the solution by socket to a GLVis server.
+   // 13. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -315,7 +307,7 @@ int main(int argc, char *argv[])
      sol_sock << "solution\n" << *pmesh << x << flush;
   }

-   // 15. Free the used memory.
+   // 14. Free the used memory.
   delete pcg;
   delete amg;
   delete fespace;
@@ -106,6 +106,7 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
+   pmesh->ReorientTetMesh();

   // 6. Define the trial, interfacial (trace) and test DPG spaces:
   //    - The trial space, x0_space, contains the non-interfacial unknowns and
@@ -1,162 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-3
-
-elements
-14
-1 4 13 15 21 25
-1 4 12 13 15 21
-1 4 13 21 22 25
-1 4 15 24 21 25
-1 4 13 15 25 16
-1 5  0  1  4  3  9 10 13 12
-1 5  8  9 12 11 17 18 21 20
-1 5  2  3  6  5 11 12 15 14
-1 6  3  4  6 12 13 15
-1 6  4  7  6 13 16 15
-1 6 12 13 21  9 10 18
-1 6 13 22 21 10 19 18
-1 6 11 14 20 12 15 21
-1 6 15 21 24 14 20 23
-
-boundary
-30
- 1 3  5  6  3  2
- 2 2  3  6  4
- 2 2  4  6  7
- 3 3  3  4  1  0
- 4 3 11 12  9  8
- 5 3  2  3 12 11
- 6 3  0  1 10  9
- 7 2  9 10 18
- 7 2 10 19 18
- 8 3  8  9 18 17
- 9 3  1  4 13 10
-10 3  4  7 16 13
-11 2 13 16 25
-11 2 13 25 22
-12 3 10 13 22 19
-13 3  7  6 15 16
-14 3  6  5 14 15
-15 3 15 14 23 24
-16 2 16 15 25
-16 2 15 24 25
-17 3  5  2 11 14
-18 3  3  0  9 12
-19 3 11  8 17 20
-20 2 11 20 14
-20 2 14 20 23
-21 3 17 18 21 20
-22 3 18 19 22 21
-23 2 21 22 25
-23 2 21 25 24
-24 3 20 21 24 23
-
-vertices
-26
-3
-0 -1 -1
-1 -1 -1
-1 0 -1
-0 0 -1
-1 0 -1
-1 1 -1
-0 1 -1
-1 1 -1
-1 -1 0
-0 -1 0
-1 -1 0
-1 0 0
-0 0 0
-1 0 0
-1 1 0
-0 1 0
-1 1 0
-1 -1 1
-0 -1 1
-1 -1 1
-1 0 1
-0 0 1
-1 0 1
-1 1 1
-0 1 1
-1 1 1
-
-MFEM sets v1.0
-
-vertex_sets
-1
-
-Origin
-1
-12
-
-edge_sets
-2
-
-Axes
-3
-12 13
-12 15
-12 21
-
-Negative Axes
-3
-12 9
-12 11
-12 3
-
-face_sets
-2
-
-Interior Corner
-3
-3 11 12  9 8
-3  2  3 12 11
-3  3  0  9 12
-
-Exterior Corner
-15
-2 13 16 25
-2 13 25 22
-2 16 15 25
-2 15 24 25
-2 21 22 25
-2 21 25 24
-3 10 13 22 19
-3 4 7 16 13
-3 1 4 13 10
-3 7 6 15 16
-3 6 5 14 15
-3 15 14 23 24
-3 20 21 24 23
-3 18 19 22 21
-3 17 18 21 20
-
-element_sets
-3
-
-Interior Corner
-3
-5 6 7
-
-Exterior Corner
-5
-0 1 2 3 4
-
-Steps
-3
-6 8 9
@@ -1,145 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-#
-
-dimension
-3
-
-elements
-7
-1 5 0 1 4 3 9 10 13 12
-1 5 3 4 7 6 12 13 16 15
-1 5 2 3 6 5 11 12 15 14
-1 5 8 9 12 11 17 18 21 20
-1 5 9 10 13 12 18 19 22 21
-1 5 12 13 16 15 21 22 25 24
-1 5 11 12 15 14 20 21 24 23
-
-boundary
-24
-1 3 5 6 3 2
-2 3 6 7 4 3
-3 3 3 4 1 0
-4 3 11 12 9 8
-5 3 2 3 12 11
-6 3 0 1 10 9
-7 3 9 10 19 18
-8 3 8 9 18 17
-9 3 1 4 13 10
-10 3 4 7 16 13
-11 3 13 16 25 22
-12 3 10 13 22 19
-13 3 7 6 15 16
-14 3 6 5 14 15
-15 3 15 14 23 24
-16 3 16 15 24 25
-17 3 5 2 11 14
-18 3 3 0 9 12
-19 3 11 8 17 20
-20 3 14 11 20 23
-21 3 17 18 21 20
-22 3 18 19 22 21
-23 3 21 22 25 24
-24 3 20 21 24 23
-
-vertices
-26
-3
-0 -1 -1
-1 -1 -1
-1 0 -1
-0 0 -1
-1 0 -1
-1 1 -1
-0 1 -1
-1 1 -1
-1 -1 0
-0 -1 0
-1 -1 0
-1 0 0
-0 0 0
-1 0 0
-1 1 0
-0 1 0
-1 1 0
-1 -1 1
-0 -1 1
-1 -1 1
-1 0 1
-0 0 1
-1 0 1
-1 1 1
-0 1 1
-1 1 1
-
-MFEM sets v1.0
-
-vertex_sets
-1
-
-Origin
-1
-12
-
-edge_sets
-2
-
-Axes
-3
-12 13
-12 15
-12 21
-
-Negative Axes
-3
-12 9
-12 11
-12 3
-
-face_sets
-2
-
-Interior Corner
-3
-3 11 12  9 8
-3  2  3 12 11
-3  3  0  9 12
-
-Exterior Corner
-12
-3 13 16 25 22
-3 16 15 24 25
-3 21 22 25 24
-3 10 13 22 19
-3 4 7 16 13
-3 1 4 13 10
-3 7 6 15 16
-3 6 5 14 15
-3 15 14 23 24
-3 20 21 24 23
-3 18 19 22 21
-3 17 18 21 20
-
-element_sets
-3
-
-Interior Corner
-3
-0 2 3
-
-Exterior Corner
-1
-5
-
-Steps
-2
-1 3
@@ -31,31 +31,29 @@ add_mfem_examples(GINKGO_EXAMPLES_SRCS ${PFX} "" test_ginkgo)
 # which builds the examples and runs:
 #   ctest -R ginkgo

-if (MFEM_ENABLE_TESTING)
-  # Command line options for the tests.
-  set(EX1_COMMON_OPTS ex1 -m ../data/star.mesh --use_gko_solver)
-  set(EX1_TEST_OPTS ${EX9_COMMON_OPTS})
+# Command line options for the tests.
+set(EX1_COMMON_OPTS ex1 -m ../data/star.mesh --use_gko_solver)
+set(EX1_TEST_OPTS ${EX9_COMMON_OPTS})

-  # Add the tests: one test per source file.
-  foreach(SRC_FILE ${GINKGO_EXAMPLES_SRCS})
-    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-    string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-    set(TEST_NAME ${PFX}${TEST_NAME})
+# Add the tests: one test per source file.
+foreach(SRC_FILE ${GINKGO_EXAMPLES_SRCS})
+  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+  set(TEST_NAME ${PFX}${TEST_NAME})

-    set(THIS_TEST_OPTIONS "-no-vis")
-    list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-    # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+  set(THIS_TEST_OPTIONS "-no-vis")
+  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")

-    if (NOT (${TEST_NAME} MATCHES ".*p$"))
-      add_test(NAME ${TEST_NAME}_ser
-        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-    else()
-      add_test(NAME ${TEST_NAME}_np=4
-        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-        ${MPIEXEC_PREFLAGS}
-        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-        ${MPIEXEC_POSTFLAGS})
-    endif()
-  endforeach()
-endif()
+  if (NOT (${TEST_NAME} MATCHES ".*p$"))
+    add_test(NAME ${TEST_NAME}_ser
+      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+  else()
+    add_test(NAME ${TEST_NAME}_np=4
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+      ${MPIEXEC_POSTFLAGS})
+  endif()
+endforeach()
@@ -33,33 +33,31 @@ add_mfem_examples(HIOP_EXAMPLES_SRCS ${PFX} "" test_hiop)
 # which builds the examples and runs:
 #   ctest -R hiop

-if (MFEM_ENABLE_TESTING)
-  # Command line options for the tests.
-  # Example 9:
-  set(EX9_COMMON_OPTS -m ../../data/periodic-segment.mesh -p 0 -dt 0.005)
-  set(EX9_TEST_OPTS ${EX9_COMMON_OPTS}  -r  2 )
-  set(EX9P_TEST_OPTS ${EX9_COMMON_OPTS})
+# Command line options for the tests.
+# Example 9:
+set(EX9_COMMON_OPTS -m ../../data/periodic-segment.mesh -p 0 -dt 0.005)
+set(EX9_TEST_OPTS ${EX9_COMMON_OPTS}  -r  2 )
+set(EX9P_TEST_OPTS ${EX9_COMMON_OPTS})

-  # Add the tests: one test per source file.
-  foreach(SRC_FILE ${HIOP_EXAMPLES_SRCS})
-    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-    string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-    set(TEST_NAME ${PFX}${TEST_NAME})
+# Add the tests: one test per source file.
+foreach(SRC_FILE ${HIOP_EXAMPLES_SRCS})
+  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+  set(TEST_NAME ${PFX}${TEST_NAME})

-    set(THIS_TEST_OPTIONS "-no-vis")
-    list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-    # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+  set(THIS_TEST_OPTIONS "-no-vis")
+  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")

-    if (NOT (${TEST_NAME} MATCHES ".*p$"))
-      add_test(NAME ${TEST_NAME}_ser
-        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-    else()
-      add_test(NAME ${TEST_NAME}_np=4
-        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4
-        ${MPIEXEC_PREFLAGS}
-        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-        ${MPIEXEC_POSTFLAGS})
-    endif()
-  endforeach()
-endif()
+  if (NOT (${TEST_NAME} MATCHES ".*p$"))
+    add_test(NAME ${TEST_NAME}_ser
+      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+  else()
+    add_test(NAME ${TEST_NAME}_np=4
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+      ${MPIEXEC_POSTFLAGS})
+  endif()
+endforeach()
@@ -1,31 +0,0 @@
-# Jupyter Notebooks using xeus-cling
-
-[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/mfem/mfem/master?filepath=examples%2Fjupyter%2Fex.ipynb)
-
-[xeus-cling](https://github.com/jupyter-xeus/xeus-cling) is a C++ Jupyter Kernel based on [cling](https://github.com/root-project/cling),
-which can be used to create interactive C++ MFEM and GLVis notebooks.
-
-Click on the `binder` button above for an interactive example.
-
-## Installing Locally
-
-In order to run notebooks locally you will need `xeus-cling` along with `mfem` and `xglvis`. We recommend you use
-[miniconda](https://docs.conda.io/en/latest/miniconda.html) or, if you already have it installed,
-[conda](https://docs.conda.io/projects/conda/en/latest/).
-
-1. Follow the install steps on https://github.com/jupyter-xeus/xeus-cling to install the C++ kernels
-2. Build and install a _shared_ version of mfem
-    * for example: `make serial SHARED=YES`
-3. Install [pyglvis](https://github.com/glvis/pyglvis)
-    * for the widget frontend
-4. Get [xeus-glvis](https://github.com/glvis/xeus-glvis) and `cp` the header to `{PREFIX}/glvis/xglvis.hpp`
-    * (this could be improved)
-
-## Running Locally
-
-Once you've installed Jupyter, the C++ Kernel, mfem, and glvis start the notebook server (`jupyter-notebook`)
-and open an existing example or a new `C++ 1x` kernel.
-
-You will _always_ need to `#pragma cling load("mfem")` and you may need to point the `cling` runtime at your
-mfem and/or glvis installs, do this with the
-`#pragma cling` [statements](https://xeus-cling.readthedocs.io/en/latest/build_options.html#using-third-party-libraries).
@@ -1,155 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "owned-extraction",
-   "metadata": {},
-   "source": [
-    "## Load the MFEM library\n",
-    "\n",
-    "Any non-default libraries must be loaded before you can `#include` files that use them. For more info see the [xeus-cling help](https://xeus-cling.readthedocs.io/en/latest/build_options.html)."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "waiting-portrait",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "#pragma cling load(\"mfem\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "foreign-recycling",
-   "metadata": {},
-   "source": [
-    "## MFEM Example 1"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "public-white",
-   "metadata": {},
-   "source": [
-    "This is the simplest MFEM example and a good starting point for new users. The example demonstrates the use of MFEM to define and solve an $H^1$ finite element discretization of the Laplace problem\n",
-    "\n",
-    "$$\n",
-    "-\\Delta u = 1\n",
-    "$$\n",
-    "\n",
-    "with homogeneous Dirichlet boundary conditions $u=0$.\n",
-    "\n",
-    "The example illustrates the use of the basic MFEM classes for defining the mesh, finite element space, as well as linear and bilinear forms corresponding to the left-hand side and right-hand side of the discrete linear system.\n",
-    "\n",
-    "Compare with MFEM's [ex1.cpp](https://github.com/mfem/mfem/blob/master/examples/ex1.cpp) and PyMFEM's [ex1.py](https://github.com/mfem/PyMFEM/blob/master/examples/ex1.py)."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "protective-darkness",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "#include <fstream>\n",
-    "#include <iostream>\n",
-    "#include <sstream>\n",
-    "\n",
-    "#include <mfem.hpp>\n",
-    "#include <glvis/xglvis.hpp>"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "falling-monkey",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "using namespace std;\n",
-    "using namespace mfem;\n",
-    "\n",
-    "Mesh mesh = Mesh::MakeCartesian2D(5, 5, Element::TRIANGLE);\n",
-    "mesh.UniformRefinement();\n",
-    "\n",
-    "H1_FECollection fec(2, mesh.Dimension());\n",
-    "\n",
-    "FiniteElementSpace fespace(&mesh, &fec);\n",
-    "cout << \"Number of finite element unknowns: \" << fespace.GetTrueVSize() << endl;\n",
-    "\n",
-    "Array<int> ess_tdof_list;\n",
-    "if (mesh.bdr_attributes.Size())\n",
-    "{\n",
-    "  Array<int> ess_bdr(mesh.bdr_attributes.Max());\n",
-    "  ess_bdr = 1;\n",
-    "  fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);\n",
-    "}\n",
-    "\n",
-    "LinearForm b(&fespace);\n",
-    "ConstantCoefficient one(1.0);\n",
-    "b.AddDomainIntegrator(new DomainLFIntegrator(one));\n",
-    "b.Assemble();\n",
-    "\n",
-    "GridFunction x(&fespace);\n",
-    "x = 0.0;\n",
-    "\n",
-    "BilinearForm a(&fespace);\n",
-    "a.AddDomainIntegrator(new DiffusionIntegrator(one));\n",
-    "a.Assemble();\n",
-    "\n",
-    "OperatorPtr A;\n",
-    "Vector B, X;\n",
-    "a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);\n",
-    "\n",
-    "cout << \"Size of linear system: \" << A->Height() << endl;\n",
-    "\n",
-    "GSSmoother M((SparseMatrix&)(*A));\n",
-    "PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);\n",
-    "a.RecoverFEMSolution(X, b, x);"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "hawaiian-republican",
-   "metadata": {},
-   "source": [
-    "## GLVis Visualization\n",
-    "\n",
-    "For now we save the computational mesh and finite element solution in a string and pass that to the glvis widget, see https://github.com/glvis/xeus-glvis for the widget backend and https://github.com/GLVis/pyglvis/tree/master/js for the widget frontend."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ordinary-equation",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "std::stringstream ss;\n",
-    "ss << \"solution\\n\" << mesh << x << flush;\n",
-    "\n",
-    "auto glv = glvis::glvis();\n",
-    "glv.plot(ss.str() + \"keys Rjml\"); // the `+ \"keys ....\"' is optional\n",
-    "glv"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "C++14",
-   "language": "C++14",
-   "name": "xcpp14"
-  },
-  "language_info": {
-   "codemirror_mode": "text/x-c++src",
-   "file_extension": ".cpp",
-   "mimetype": "text/x-c++src",
-   "name": "c++",
-   "version": "14"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
psocratis	39150a110c	Merge branch 'master' into DST-maxwell-solver-dev	2021-06-16 10:30:33 -07:00
psocratis	1bd031a948	ParDST first commit in new (clean) branch	2020-12-21 11:36:56 -08:00