Href ex edits

Michalewicz function coded as exact=3
Updated case 3
2021-03-19 15:12:45 -07:00 · 2021-03-17 15:01:57 -07:00 · 2021-03-17 11:06:11 -07:00 · 2021-03-17 10:17:48 -07:00 · 2021-03-15 15:40:22 -07:00 · 2021-03-15 13:50:44 -07:00
514 changed files with 10788 additions and 54087 deletions
@@ -1,32 +0,0 @@
-codecov:
-  require_ci_to_pass: yes
-
-coverage:
-  precision: 2
-  round: nearest
-  range: "0...100"
-  status:
-    patch:
-      default:
-        target: auto
-        threshold: 0%
-        base: auto
-        branches:
-          - master
-        if_ci_failed: error
-        informational: true
-        only_pulls: true
-    project:
-      default:
-        target: auto   # compares coverage to the previous base commit
-        threshold: 1%  # allows variations around the target
-        base: auto
-        branches:
-          - master
-        if_ci_failed: error
-        only_pulls: true
-
-github_checks:
-  annotations: false
-
-comment: false
@@ -1,208 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# In this CI section, we build different variants of mfem and run test on them.
-name: builds-and-tests
-
-# Github actions can use the default "GITHUB_TOKEN". By default, this token
-# is set to have permissive access. However, this is not a good practice
-# security-wise. Here we use an external action, so we restrict the
-# permission to the minimum required.
-# When the 'permissions' is set, all the scopes not mentioned are set to the
-# most restrictive setting. So the following is enough.
-permissions:
-  actions: write
-
-on:
-  push:
-    branches:
-      - master
-      - next
-  pull_request:
-
-env:
-  HYPRE_ARCHIVE: v2.19.0.tar.gz
-  HYPRE_TOP_DIR: hypre-2.19.0
-  METIS_ARCHIVE: metis-4.0.3.tar.gz
-  METIS_TOP_DIR: metis-4.0.3
-  MFEM_TOP_DIR: mfem
-
-# Note for future improvements:
-#
-# We cannot reuse cached dependencies and have to build them for each target
-# although they could be shared sometimes. That's because Github cache Action
-# has no read-only mode. But there is a PR ready for this
-# (https://github.com/actions/cache/pull/489)
-
-jobs:
-  builds-and-tests:
-    strategy:
-      matrix:
-        os: [ubuntu-18.04, macos-10.15]
-        target: [dbg, opt]
-        mpi: [seq, par]
-        build-system: [make]
-        hypre-target: [int32]
-        # 'include' allows us to:
-        # - Add a variable to all jobs without creating a new matrix dimension.
-        #   Codecov is defined that way.
-        # - Add a new combination.
-        #   'build-system: cmake' and 'hypre-target: int64'
-        #
-        # note: we will gather coverage info for any non-debug run except the
-        # CMake build.
-        include:
-          - target: dbg
-            codecov: NO
-          - target: opt
-            codecov: YES
-          - os: ubuntu-18.04
-            target: opt
-            codecov: NO
-            mpi: par
-            build-system: cmake
-            hypre-target: int32
-          - os: ubuntu-18.04
-            target: opt
-            codecov: NO
-            mpi: par
-            build-system: make
-            hypre-target: int64
-    name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
-
-    runs-on: ${{ matrix.os }}
-
-    steps:
-    # This external action allows to interrupt a workflow already running on
-    # the same branch to save resource
-    - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.9.0
-      with:
-        access_token: ${{ github.token }}
-
-    # Checkout MFEM in "mfem" subdirectory. Final path:
-    # /home/runner/work/mfem/mfem/mfem
-    # Note: Done now to access "install-hypre" and "install-metis" actions.
-    - name: checkout mfem
-      uses: actions/checkout@v2
-      with:
-        path: ${{ env.MFEM_TOP_DIR }}
-        # Fetch the complete history for codecov to access commits ID
-        fetch-depth: 0
-
-    # Only get MPI if defined for the job.
-    # TODO: It would be nice to have only one step, e.g. with a dedicated
-    # action, but I (@adrienbernede) don't see how at the moment.
-    - name: get MPI (Linux)
-      if: matrix.mpi == 'par' && matrix.os == 'ubuntu-18.04'
-      run: |
-        sudo apt-get install mpich libmpich-dev
-        export MAKE_CXX_FLAG="MPICXX=mpic++"
-
-    - name: get lcov (Linux)
-      if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-18.04'
-      run: |
-        sudo apt-get install lcov
-
-    - name: Set up Homebrew
-      if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-10.15'
-      uses: Homebrew/actions/setup-homebrew@c4aafe8c4620bf08883dd4679c374f11e73329d3
-
-    - name: get MPI (MacOS)
-      if: matrix.mpi == 'par' && matrix.os == 'macos-10.15'
-      run: |
-        export HOMEBREW_NO_INSTALL_CLEANUP=1
-        brew install openmpi
-        export MAKE_CXX_FLAG="MPICXX=mpic++"
-
-    - name: get MPI (MacOS)
-      if: matrix.codecov == 'YES' && matrix.os == 'macos-10.15'
-      run: |
-        export HOMEBREW_NO_INSTALL_CLEANUP=1
-        brew install lcov
-
-    # Get Hypre through cache, or build it.
-    # Install will only run on cache miss.
-    - name: cache hypre
-      id: hypre-cache
-      if: matrix.mpi == 'par'
-      uses: actions/cache@v2
-      with:
-        path: ${{ env.HYPRE_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.0
-
-    - name: get hypre
-      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-hypre@v2.0
-      with:
-        archive: ${{ env.HYPRE_ARCHIVE }}
-        dir: ${{ env.HYPRE_TOP_DIR }}
-        target: ${{ matrix.hypre-target }}
-
-    # Get Metis through cache, or build it.
-    # Install will only run on cache miss.
-    - name: cache metis
-      id: metis-cache
-      if: matrix.mpi == 'par'
-      uses: actions/cache@v2
-      with:
-        path: ${{ env.METIS_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.0
-
-    - name: install metis
-      if: matrix.mpi == 'par' && steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@v2.0
-      with:
-        archive: ${{ env.METIS_ARCHIVE }}
-        dir: ${{ env.METIS_TOP_DIR }}
-
-    # MFEM build and test
-    - name: build
-      uses: mfem/github-actions/build-mfem@v2.0
-      with:
-        os: ${{ matrix.os }}
-        target: ${{ matrix.target }}
-        codecov: ${{ matrix.codecov }}
-        mpi: ${{ matrix.mpi }}
-        build-system: ${{ matrix.build-system }}
-        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
-        metis-dir: ${{ env.METIS_TOP_DIR }}
-        mfem-dir: ${{ env.MFEM_TOP_DIR }}
-
-    # Run checks (and only checks) on debug targets
-    - name: checks
-      if: matrix.build-system == 'make' && matrix.target == 'dbg'
-      run: |
-        cd ${{ env.MFEM_TOP_DIR }}  && make check
-
-    - name: unit tests
-      if: matrix.build-system == 'make' && matrix.target == 'opt'
-      run: |
-        cd ${{ env.MFEM_TOP_DIR }} && make unittest
-
-    - name: tests
-      if: matrix.build-system == 'make' && matrix.target == 'opt'
-      run: |
-        cd ${{ env.MFEM_TOP_DIR }} && make test
-
-    - name: cmake unit tests
-      if: matrix.build-system == 'cmake'
-      run: |
-        cd ${{ env.MFEM_TOP_DIR }}/build/tests/unit && ctest --output-on-failure
-
-    # Code coverage (process and upload reports)
-    - name: codecov
-      if: matrix.codecov == 'YES'
-      uses: mfem/github-actions/upload-coverage@v2.0
-      with:
-        name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
-        project_dir: ${{ env.MFEM_TOP_DIR }}
-        directories: "fem general linalg mesh"
@@ -1,100 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-name: build-analysis
-
-permissions:
-  actions: write
-
-on:
-  push:
-    branches:
-      - master
-      - next
-  pull_request:
-
-env:
-  HYPRE_ARCHIVE: v2.19.0.tar.gz
-  HYPRE_TOP_DIR: hypre-2.19.0
-  METIS_ARCHIVE: metis-4.0.3.tar.gz
-  METIS_TOP_DIR: metis-4.0.3
-  COVERAGE_ENV: mfem-coverage
-
-jobs:
-  gitignore:
-    runs-on: ubuntu-18.04
-
-    steps:
-    - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.9.0
-      with:
-        access_token: ${{ github.token }}
-
-    - name: checkout MFEM
-      uses: actions/checkout@v2
-      with:
-        path: mfem
-
-    - name: Get MPI (Linux)
-      run: |
-        sudo apt-get install mpich libmpich-dev
-        export MAKE_CXX_FLAG="MPICXX=mpic++"
-
-    - name: Cache Hypre Install
-      id: hypre-cache
-      uses: actions/cache@v2
-      with:
-        path: ${{ env.HYPRE_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.0
-
-    - name: Get Hypre
-      if: steps.hypre-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-hypre@v2.0
-      with:
-        archive: ${{ env.HYPRE_ARCHIVE }}
-        dir: ${{ env.HYPRE_TOP_DIR }}
-        target: int32
-
-    - name: Cache Metis Install
-      id: metis-cache
-      uses: actions/cache@v2
-      with:
-        path: ${{ env.METIS_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.0
-
-    - name: Install Metis
-      if: steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@v2.0
-      with:
-        archive: ${{ env.METIS_ARCHIVE }}
-        dir: ${{ env.METIS_TOP_DIR }}
-
-    # MFEM build and test
-    - name: build-mfem
-      uses: mfem/github-actions/build-mfem@v2.0
-      with:
-        os: ${{ runner.os }}
-        target: optim
-        codecov: NO
-        mpi: parallel
-        build-system: make
-        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
-        metis-dir: ${{ env.METIS_TOP_DIR }}
-        mfem-dir: mfem
-
-    - name: test (no clean)
-      run: |
-        cd mfem && make test-noclean
-
-    - name: gitignore
-      run: |
-        cd mfem/tests/scripts
-        ./runtest gitignore
@@ -11,100 +11,32 @@

 name: repo-check

-permissions:
-  actions: write
-
 on:
  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master

 jobs:
-  file-headers-check:
-    runs-on: ubuntu-18.04
+  copyright-check:
+    runs-on: ubuntu-latest

    steps:
-    - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.9.0
-      with:
-        access_token: ${{ github.token }}
    - name: checkout mfem
      uses: actions/checkout@v2
+      with:
+        path: mfem

    - name: copyright check
-      id: copyright
      run: |
-        ./config/githooks/pre-push --copyright
-
-      continue-on-error: true
-
-    - name: license check
-      id: license
-      run: |
-        ./config/githooks/pre-push --license
-      continue-on-error: true
-
-    - name: release check
-      id: release
-      run: |
-        ./config/githooks/pre-push --release
-      continue-on-error: true
-
-    - name: wrap-up
-      if: steps.copyright.outcome != 'success' || steps.license.outcome != 'success' || steps.release.outcome != 'success'
-      run: |
-        if [[ "${{ steps.copyright.outcome }}" != "success" ]]; then
-          echo "copyright check failed, unroll log for details"
+        cd mfem
+        if git grep -l "^#.*\-2020" > matches.txt
+        then
+          echo "Please update the following files to Copyright (c) 2010-2021:"
+          cat matches.txt
+          exit 1
+        else
+          echo "No outdated copyright found."
        fi
-        if [[ "${{ steps.license.outcome }}" != "success" ]]; then
-          echo "license check failed, unroll log for details"
-        fi
-        if [[ "${{ steps.release.outcome }}" != "success" ]]; then
-          echo "release check failed, unroll log for details"
-        fi
-        exit 1
-
-  code-style:
-    runs-on: ubuntu-16.04 # needed for astyle 2.05.1
-
-    steps:
-    - name: checkout mfem
-      uses: actions/checkout@v2
-
-    - name: get astyle
-      run: |
-        sudo apt-get install astyle=2.05.1-0ubuntu1
-
-    - name: style check
-      run: |
-        ./config/githooks/pre-push --style
-
-  documentation:
-    runs-on: ubuntu-18.04
-
-    steps:
-    - name: checkout mfem
-      uses: actions/checkout@v2
-
-    - name: get doxygen and graphviz
-      run: |
-        sudo apt-get install doxygen graphviz
-
-    - name: build documentation
-      run: |
-        cd tests/scripts
-        ./runtest documentation
-
-  branch-history:
-    if: github.ref != 'refs/heads/next' && github.ref != 'refs/heads/master'
-    runs-on: ubuntu-18.04
-
-    steps:
-    - name: checkout mfem
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-
-    - name: branch-history
-      run: |
-        git fetch origin master:master
-        git checkout -b gh-actions-branch-history
-        ./config/githooks/pre-push --history
@@ -26,7 +26,6 @@ CMakeFiles/
 config/_config.hpp
 config/config.mk
 config/sample-runs-build.log
-config/user.mk
 doc/CodeDocumentation.conf
 doc/CodeDocumentation.html
 doc/CodeDocumentation
@@ -45,8 +44,8 @@ doc/warnings.log

 # Example and miniapp binaries and outputs

-examples/ex[0-9]
-examples/ex[0-9]p
+examples/ex[1-9]
+examples/ex[1-9]p
 examples/ex1[04-9]
 examples/ex1[0-9]p
 examples/ex2[0-9]
@@ -103,9 +102,6 @@ examples/Example23*
 examples/ex25.mesh
 examples/ex25-*.gf
 examples/ex25p-*.*
-examples/ex28_*
-examples/ex28p_*
-examples/flux.*

 examples/amgx/ex1
 examples/amgx/ex1p
@@ -218,11 +214,6 @@ miniapps/meshing/optimized*
 miniapps/meshing/perturbed*
 miniapps/meshing/polar-nc.mesh

-miniapps/mtop/parheat
-miniapps/mtop/ParHeat*
-miniapps/mtop/seqheat
-miniapps/mtop/SeqHeat*
-
 miniapps/navier/navier_mms
 miniapps/navier/navier_kovasznay
 miniapps/navier/navier_kovasznay_vs
@@ -249,10 +240,6 @@ miniapps/performance/sol.*

 miniapps/shifted/distance
 miniapps/shifted/ParaViewDistance
-miniapps/shifted/diffusion
-miniapps/shifted/diffusion.mesh
-miniapps/shifted/diffusion.gf
-miniapps/shifted/ParaViewDiffusion

 miniapps/tools/display-basis
 miniapps/tools/load-dc
@@ -282,11 +269,6 @@ miniapps/toys/lissajous.gf
 miniapps/toys/mondrian.mesh

 miniapps/solvers/block-solvers
-miniapps/solvers/lor_solvers
-miniapps/solvers/plor_solvers
-miniapps/solvers/ParaView
-miniapps/solvers/mesh.*
-miniapps/solvers/sol.*

 # Unit test binary and outputs
 tests/unit/output_meshes
@@ -294,10 +276,7 @@ tests/unit/unit_tests
 tests/unit/punit_tests
 tests/unit/sedov_tests_*
 tests/unit/psedov_tests_*
-tests/unit/tmop_pa_tests_*
-tests/unit/ptmop_pa_tests_*
 tests/unit/ceed_tests
-tests/unit/debug_device_tests

 # Test script output
 tests/scripts/*.err
@@ -311,7 +290,3 @@ tests/par-mesh-format/ex1p

 # VPATH builds
 build-*/*
-
-# PETSc automated build
-petsc-build/*
-pkg.gitcommit
@@ -40,66 +40,43 @@
 # Directory used to place artifacts.

 variables:
-  BUILD_ROOT: ${CI_BUILDS_DIR}/MFEM/${CI_PROJECT_NAME}_${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID}
-  AUTOTEST_ROOT: ${CI_BUILDS_DIR}/MFEM
+  BUILD_ROOT: ${CI_BUILDS_DIR}/${CI_PROJECT_NAME}_${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID}
  REBASELINE: "NO"
-  AUTOTEST: "NO"
  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
  TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
  TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
-  AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
-  MFEM_DATA_REPO: https://github.com/mfem/data.git
  ARTIFACTS_DIR: artifacts

-# The pipeline is divided into stages. Usually, jobs in a given stage wait for
-# the preceding stages to complete before to start. However, we sometimes use
-# the "needs" keyword and express the DAG of jobs for more efficiency.
-# - We use setup and setup_baseline phases to download content outside of mfem
-#   directory.
+# The pipeline is divided into stages. Usually, these are also synchronization
+# points, however, we use "needs" keyword to express the DAG of jobs for more
+# efficiency.
+# - We use setup phase to download content outside of mfem directory.
 # - Allocate/Release is where quartz resources are allocated/released once for all.
 # - Build and Test is where we build and MFEM for multiple toolchains.
 # - Baseline_checks gathers baseline-type test suites execution
 # - Baseline_publish, only available on master, allows to update baseline
 #   results
 stages:
-  - setup
  - q_allocate_resources
  - q_build_and_test
  - q_release_resources
  - l_build_and_test
  - c_build_and_test
-  - setup_baseline
+  - setup
  - baseline_check
-  - baseline_to_autotest
  - baseline_publish

-# setup clones the mfem/data repo in ${BUILD_ROOT}. The build_and_test script
-# then symlinks the repo to the parent directory of the MFEM source directory.
-# Unit tests that depend on the mfem/data repo will then detect that this
-# directory is present and be enabled.
+# The setup job in setup stage don't rely on MFEM git repo. It prepares a
+# pipeline-wide working directory downloading/updating external repos.
+# TODO: updating tests and tpls is not necessary anymore since pipelines are
+# now using unique directories so repo are never shared with another pipeline.
+# This is not memory efficient (we keep a lot of data), hence this reminder.
+# Setup
 setup:
  tags:
    - shell
    - quartz
  stage: setup
-  variables:
-    GIT_STRATEGY: none
-  script:
-    - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
-    - if [ ! -d data ]; then git clone ${MFEM_DATA_REPO}; fi
-
-# The setup_baseline job in setup stage_baseline doesn't rely on MFEM git repo.
-# It prepares a pipeline-wide working directory downloading/updating external
-# repos. TODO: updating tests and tpls is not necessary anymore since pipelines
-# are now using unique directories so repo are never shared with another
-# pipeline. This is not memory efficient (we keep a lot of data), hence this
-# reminder.
-# Note: This job can start immediately.
-setup_baseline:
-  tags:
-    - shell
-    - quartz
-  stage: setup_baseline
  variables:
    GIT_STRATEGY: none
  script:
@@ -108,9 +85,6 @@ setup_baseline:
    - if [ ! -d "tests" ]; then git clone ${TESTS_REPO}; fi
    - cd tpls && git pull && cd ..
    - cd tests && git pull && cd ..
-    - cd ${AUTOTEST_ROOT}
-    - if [ ! -d "autotest" ]; then git clone ${AUTOTEST_REPO}; fi
-    - cd autotest && git pull && cd ..
  needs: []

 .build_toss_3_x86_64_ib_script:
@@ -125,22 +99,26 @@ setup_baseline:
  script:
    - srun -p mi60 -t 15 -N 1 tests/gitlab/build_and_test

-# Lassen uses a different job scheduler (spectrum lsf) that does not allow
-# pre-allocation the same way slurm does.  We use pdebug queue on lassen to
-# speed-up the allocation.  However this would not be scalable to multiple
-# builds.
+# Lassen and Butte use a different job scheduler (spectrum lsf) that does not
+# allow pre-allocation the same way slurm does.
 .build_blueos_3_ppc64le_ib_script:
  script:
-    - lalloc 1 -W 30 -q pdebug tests/gitlab/build_and_test
+    - lalloc 1 -W 15 tests/gitlab/build_and_test

 # Shared script for baseline and sample-run-baseline, the value of BASELINE_TEST
 # differentiates between the two tests.
 .baseline_script: &baseline_script |
  # locals
+  _glob_out=${BASELINE_TEST}.out
  _glob_err=${BASELINE_TEST}.err
  _base_diff=${BASELINE_TEST}-${SYS_TYPE}.diff
  _base_patch=${BASELINE_TEST}-${SYS_TYPE}.patch
  _base_out=${BASELINE_TEST}-${SYS_TYPE}.out
+
+  _out=${BASELINE_TEST}-${SYS_TYPE}.out
+  _ref=../${BASELINE_TEST}-${SYS_TYPE}.saved
+  _out_txt=${BASELINE_TEST}.txt
+  _diff=${BASELINE_TEST}-diff.txt
  # prepare
  cd ${BUILD_ROOT}
  ln -snf ${CI_PROJECT_DIR} mfem
@@ -155,7 +133,7 @@ setup_baseline:
    echo "ERROR during ${BASELINE_TEST} execution";
    echo "Here is the ${_glob_err} file content";
    cat ${_glob_err}
-    cp ${_glob_err} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_glob_err}
+    cp ${_glob_err} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_glob_err}.txt
    exit 1;
  elif [[ ! -f ${_base_patch} && ! -f ${_base_out} ]]
  then
@@ -165,20 +143,18 @@ setup_baseline:
  elif [[ -f ${_base_patch} ]]
  then
    echo "${BASELINE_TEST}: Differences found, patch generated"
-    cp ${_base_patch} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_patch}
+    cp ${_base_patch} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_patch}.txt
  elif [[ -f ${_base_out} ]]
  then
    echo "${BASELINE_TEST}: Differences found, replacement file generated"
-    cp ${_base_out} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_out}
+    cp ${_base_out} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_out}.txt
  fi
  # _base_diff won't even exist if there is no difference.
  if [[ -f ${_base_diff} ]]
  then
    echo "${BASELINE_TEST}: Relevant differences (filtered diff) ..."
    cat ${_base_diff}
-    cp ${_base_diff} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_diff}
-    # We create a .err file, because that's how we signal that there was a diff.
-    cp ${_base_diff} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/gitlab-${BASELINE_TEST}-${SYS_TYPE}.err
+    cp ${_base_diff} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_diff}.txt
  fi
  if [[ ! -s ${_base_diff} ]]
  then
@@ -217,7 +193,7 @@ setup_baseline:
      - ${ARTIFACTS_DIR}
  allow_failure: true

-# This job can only be manually triggered on a pipeline for master branch, or if
+# This job can only be manually triggers on a pipeline for master branch, or if
 # the pipeline was triggered with REBASELINE="YES"
 .rebaseline_mfem:
  stage: baseline_publish
@@ -230,18 +206,19 @@ setup_baseline:
    - export DIFF_FILE=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}.diff
    - cd ${BUILD_ROOT}/tests
    - |
-      if [[ ! -f "${DIFF_FILE}" ]]
+      if [[ ! -f "${DIFF_FILE}.txt" ]]
      then
          echo "Nothing to be done: no relevant change in baseline"
          exit 0
-      elif [[ -f "${PATCH_FILE}" ]]
+      elif [[ -f "${PATCH_FILE}.txt" ]]
      then
+          mv ${PATCH_FILE}.txt ${PATCH_FILE}
          patch "./baseline-${SYS_TYPE}.saved" < "${PATCH_FILE}"
-      elif [[ -f "${FULL_FILE}t" ]]
+      elif [[ -f "${FULL_FILE}.txt" ]]
      then
-          cp "${FULL_FILE}" "./baseline-${SYS_TYPE}.saved"
+          cp "${FULL_FILE}.txt" "./baseline-${SYS_TYPE}.saved"
      else
-          echo "File missing: expected ${PATCH_FILE} or ${FULL_FILE}"
+          echo "File missing: expected ${PATCH_FILE}.txt or ${FULL_FILE}.txt"
          exit 1
      fi
    - git add baseline-${SYS_TYPE}.saved
@@ -251,4 +228,4 @@ setup_baseline:
 # The list on jobs is defined in machine-specific files.
 include:
  - local: .gitlab/quartz.yml
-  - local: .gitlab/lassen.yml
+    #  - local: .gitlab/lassen.yml
@@ -15,20 +15,43 @@
  tags:
    - shell
    - lassen
-  rules:
-    - if: '$CI_COMMIT_BRANCH =~ /_lnone/ || $ON_LASSEN == "OFF"' #run except if ...
-      when: never
-    - when: on_success
-
-# Spack helped builds
-# Generic lassen build job, extending build script
-# Note: Lassen jobs can start as soon as the setup job is complete.
-.build_and_test_on_lassen:
-  extends: [.build_blueos_3_ppc64le_ib_script, .on_lassen]
-  stage: l_build_and_test
-  needs: [setup]
-
-opt_mpi_cuda_xl_16_1_1_8:
  variables:
-    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=sm_70"
-  extends: .build_and_test_on_lassen
+    PLAT: lassen
+
+# Build MFEM
+build_mfem_ser_lassen:
+  extends: [.with_gcc_8_3_1, .on_lassen]
+  needs: [setup]
+  stage: lassen_build
+  script:
+    - mkdir -p ${BUILD_PATH}
+    - cp -r ${CI_PROJECT_DIR} ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser
+    - lalloc 1 -W 5 -q pdebug make -j cuda CUDA_ARCH=sm_70
+
+build_mfem_debug_ser_lassen:
+  extends: [.with_gcc_8_3_1, .on_lassen]
+  needs: [setup]
+  stage: lassen_build
+  script:
+    - mkdir -p ${BUILD_PATH}
+    - cp -r ${CI_PROJECT_DIR} ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
+    - lalloc 1 -W 5 -q pdebug make -j cuda MFEM_DEBUG="YES" CPPFLAGS=-O2 CUDA_ARCH=sm_70
+
+# Sanity check
+sanitycheck_mfem_ser_lassen:
+  extends: [.with_gcc_8_3_1, .on_lassen]
+  stage: lassen_test
+  needs: [build_mfem_ser_lassen]
+  script:
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser
+    - lalloc 1 -W 15 -q pdebug make -j test
+
+sanitycheck_mfem_debug_ser_lassen:
+  extends: [.with_gcc_8_3_1, .on_lassen]
+  stage: lassen_test
+  needs: [build_mfem_debug_ser_lassen]
+  script:
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
+    - lalloc 1 -W 30 -q pdebug make -j test
@@ -16,39 +16,12 @@
    - shell
    - quartz
  rules:
-    # Don't run quartz jobs if...
-    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"'
+    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"' #run except if ...
      when: never
-    # Don't run autotest update if...
-    - if: '$CI_JOB_NAME =~ /update_autotest/ && $AUTOTEST != "YES"'
-      when: never
-    # Don't run autotest update if...
-    - if: '$CI_JOB_NAME =~ /q_report/ && $AUTOTEST != "YES"'
-      when: never
-    # Report success on success status
-    - if: '$CI_JOB_NAME =~ /q_report_success/ && $AUTOTEST == "YES"'
-      when: on_success
-    # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /q_report_failure/ && $AUTOTEST == "YES"'
-      when: on_failure
-    # Always release resources
    - if: '$CI_JOB_NAME =~ /release_resources/'
      when: always
-    # Default is to run if previous stage succeeded
    - when: on_success

-# This is a yaml anchor, it can be used to avoid duplication like here.
-# The code below will simply be pasted wherever the anchor is placed.
-.safe_create_rundir: &safe_create_rundir |
-  if ! mkdir ${rundir}; then
-    n=1
-    while ! mkdir ${rundir}_${n}
-    do
-      n=$((n+1))
-    done
-    rundir=${rundir}_${n}
-  fi
-
 # Allocate
 q_allocate_resources:
  variables:
@@ -69,40 +42,6 @@ q_release_resources:
    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})

-# Release
-q_report_success:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_quartz
-  stage: q_release_resources
-  script:
-    - echo "Can only run if all the quartz jobs passed"
-    - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
-    - *safe_create_rundir
-    - echo "The Quartz jobs were successful" > ${rundir}/gitlab.out
-    - echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
-    - git add ${rundir}
-    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
-    - git push origin master
-
-q_report_failure:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_quartz
-  stage: q_release_resources
-  script:
-    - echo "Runs if there was at least one failure on quartz"
-    - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
-    - *safe_create_rundir
-    - echo "There was an error while running CI on Quartz" > ${rundir}/gitlab.err
-    - echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
-    - cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
-    - git add ${rundir}
-    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
-    - git push origin master
-
 # Spack helped builds
 # Generic quartz build job, extending build script
 .build_and_test_on_quartz:
@@ -150,34 +89,10 @@ opt_par_gcc_6_1_0_pumi:
    SPEC: "%gcc@6.1.0 +pumi"
  extends: .build_and_test_on_quartz

-# Baseline jobs form an independent set of jobs. We use `needs:[]` to specify
-# that "setup-baseline" can start immediately. Then, we have to use needs for
-# each one of the baseline jobs, otherwise they will wait for the rest of the
-# pipeline.
-
+# Baseline
 baselinecheck_mfem_intel_quartz:
  extends: [.baselinecheck_mfem, .on_quartz]
-  needs: [setup_baseline]
-
-update_autotest:
-  extends: [.on_quartz]
-  needs: [baselinecheck_mfem_intel_quartz]
-  stage: baseline_to_autotest
-  script:
-    - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - rundir="quartz/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
-    - *safe_create_rundir
-    - cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
-    # We create an autotest-email.html file, because that's how we signal that there was a diff (temporary).
-    - |
-      if [[ -f ${rundir}/*.err ]]
-      then
-        echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/*.err
-        cp ${rundir}/*.err ${rundir}/autotest-email.html
-      fi
-    - git add ${rundir}
-    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
-    - git push origin master
+  needs: [setup]

 baselinepublish_mfem_quartz:
  extends: [.on_quartz, .rebaseline_mfem]
@@ -0,0 +1,469 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+language: cpp
+
+os: linux
+dist: bionic
+
+stages:
+  - checks
+  - tests
+  - optional
+
+env:
+  global:
+    - HYPRE_ARCHIVE=v2.19.0.tar.gz
+      HYPRE_URL=https://github.com/hypre-space/hypre/archive/$HYPRE_ARCHIVE
+      HYPRE_TOP_DIR=hypre-2.19.0
+
+jobs:
+  include:
+
+    # ========================
+    #         Checks
+    # ========================
+    # - code-style
+    # - documentation
+    # - gitignore
+
+    - stage: checks
+      os: linux
+      dist: xenial
+      name: "code-style"
+      addons:
+        apt:
+          packages:
+            - astyle=2.05.1-0ubuntu1
+      script:
+        - cd ${TRAVIS_BUILD_DIR}
+        - cd tests/scripts
+        - ./runtest code-style
+
+    - stage: checks
+      os: linux
+      name: "documentation"
+      addons:
+        apt:
+          packages:
+            - doxygen
+            - graphviz
+      script:
+        - cd ${TRAVIS_BUILD_DIR}
+        - cd tests/scripts
+        - ./runtest documentation
+
+    - stage: checks
+      os: linux
+      name: "gitignore"
+      addons:
+        apt:
+          packages:
+            - mpich
+            - libmpich-dev
+      env: MPI=YES
+      before_script:
+        - cd ${TRAVIS_BUILD_DIR}
+        - mpicxx -v
+        - make config MFEM_USE_MPI=YES MFEM_MPI_NP=2
+        - make all -j3
+        - make test-noclean
+      script:
+        - cd tests/scripts
+        - ./runtest gitignore
+      cache:
+        ccache: true
+        directories:
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
+          rm -f Lib/*.{c,o}
+
+    # ========================
+    #  Optional Checks/Tests
+    # ========================
+    # - branch-history
+
+    - stage: optional
+      name: "branch-history"
+      if: branch != next
+      # need full git history for the binary/big files check
+      git:
+        depth: false
+      script:
+        - cd ${TRAVIS_BUILD_DIR}
+        # update master
+        - git fetch origin master:master
+        # checkout a branch (otherwise Travis works in detached head)
+        - git checkout -b travis_tests
+        - cd tests/scripts
+        - ./runtest branch-history
+
+    # ========================
+    #       Linux tests
+    # ========================
+    # - serial + debug
+    # - serial
+    # - parallel + debug
+    # - parallel
+
+    - stage: tests
+      os: linux
+      compiler: gcc
+      name: "Linux: Serial + Debug"
+      env: DEBUG=YES
+           MPI=NO
+           CODECOV=NO
+           MFEM_TEST_TARGET=check
+      cache:
+        ccache: true
+
+    - os: linux
+      compiler: gcc
+      name: "Linux: Serial"
+      env: DEBUG=NO
+           MPI=NO
+           CODECOV=NO
+           MFEM_TEST_TARGET=test
+      cache:
+        ccache: true
+
+    - os: linux
+      compiler: gcc
+      name: "Linux: Parallel + Debug"
+      addons:
+        apt:
+          # sources:
+            # - ubuntu-toolchain-r-test
+          packages:
+            # GCC 4.9
+            # - g++-4.9
+            # MPICH
+            - mpich
+            - libmpich-dev
+            # OpenMPI
+            # - openmpi-bin
+            # - libopenmpi-dev
+      env: DEBUG=YES
+           MPI=YES
+           CODECOV=NO
+           MFEM_TEST_TARGET=check
+           NPROCS=2
+      cache:
+        ccache: true
+        directories:
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
+          rm -f Lib/*.{c,o}
+
+    - os: linux
+      compiler: gcc
+      name: "Linux: Parallel"
+      addons:
+        apt:
+          # sources:
+            # - ubuntu-toolchain-r-test
+          packages:
+            # GCC 4.9
+            # - g++-4.9
+            # MPICH
+            - mpich
+            - libmpich-dev
+            # OpenMPI
+            # - openmpi-bin
+            # - libopenmpi-dev
+      env: DEBUG=NO
+           MPI=YES
+           CODECOV=YES
+           MFEM_TEST_TARGET=test
+           NPROCS=2
+      cache:
+        ccache: true
+        directories:
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
+          rm -f Lib/*.{c,o}
+
+    - os: linux
+      compiler: gcc
+      name: "Linux: Parallel (cmake)"
+      addons:
+        apt:
+          packages:
+            - mpich
+            - libmpich-dev
+      env: MPI=YES
+           NPROCS=2
+      script:
+        - cd ${TRAVIS_BUILD_DIR}
+        - mkdir ${TRAVIS_BUILD_DIR}/build
+        - cd ${TRAVIS_BUILD_DIR}/build
+        - cmake ..
+             -DMFEM_USE_MPI=ON
+             -DHYPRE_DIR=${TRAVIS_BUILD_DIR}/../$HYPRE_TOP_DIR/src/hypre
+             -DMFEM_MPI_NP=$NPROCS
+        - make -j3 mfem examples
+        - cd ${TRAVIS_BUILD_DIR}/build/tests/unit
+        - make -j3
+        - ctest --output-on-failure
+      cache:
+        ccache: true
+        directories:
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
+          rm -f Lib/*.{c,o}
+
+    # ========================
+    #      Mac OS X tests
+    # ========================
+    # - serial + debug
+    # - serial
+    # - parallel + debug
+    # - parallel
+
+    - os: osx
+      osx_image: xcode11.2
+      compiler: clang
+      name: "Mac: Serial + Debug"
+      addons:
+        homebrew:
+          packages:
+            - ccache
+      env: DEBUG=YES
+           MPI=NO
+           CODECOV=NO
+           MFEM_TEST_TARGET=check
+      cache:
+        ccache: true
+
+    - os: osx
+      osx_image: xcode11.2
+      compiler: clang
+      name: "Mac: Serial"
+      addons:
+        homebrew:
+          packages:
+            - ccache
+      env: DEBUG=NO
+           MPI=NO
+           CODECOV=NO
+           MFEM_TEST_TARGET=test
+      cache:
+        ccache: true
+
+    - os: osx
+      osx_image: xcode11.2
+      compiler: clang
+      name: "Mac: Parallel + Debug"
+      addons:
+        homebrew:
+          packages:
+            - ccache
+      env: DEBUG=YES
+           MPI=YES
+           CODECOV=NO
+           MFEM_TEST_TARGET=check
+           NPROCS=4
+           TMPDIR=/tmp
+      cache:
+        ccache: true
+        directories:
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+          - $HOME/local-cached
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
+          rm -f Lib/*.{c,o}
+
+    - os: osx
+      osx_image: xcode11.2
+      compiler: clang
+      name: "Mac: Parallel"
+      addons:
+        homebrew:
+          packages:
+            - ccache
+      env: DEBUG=NO
+           MPI=YES
+           CODECOV=YES
+           MFEM_TEST_TARGET=test
+           NPROCS=4
+           TMPDIR=/tmp
+      cache:
+        ccache: true
+        directories:
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+          - $HOME/local-cached
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
+          rm -f Lib/*.{c,o}
+
+before_install:
+   # No addon for brew yet, have to install OSX packages this way.
+   # - if [ $TRAVIS_OS_NAME == "osx" ] && [ $MPI == "YES" ]; then
+   #      brew install open-mpi;
+   #   fi
+
+   # Disable ccache while building dependencies that are cached:
+   - echo "before \$PATH = $PATH";
+     export PATH=${PATH//\/usr\/lib\/ccache:/};
+     echo "after  \$PATH = $PATH"
+
+   # On Mac OS X, build and cache OpenMPI 2.1.6:
+   - if [ $TRAVIS_OS_NAME == "osx" ] && [ $MPI == "YES" ]; then
+        if [ ! -e $HOME/local-cached/bin/mpicc ]; then
+           mkdir -p $HOME/builds && cd $HOME/builds &&
+           wget https://download.open-mpi.org/release/open-mpi/v2.1/openmpi-2.1.6.tar.bz2 &&
+           tar jxf openmpi-2.1.6.tar.bz2 &&
+           mkdir openmpi-build && cd openmpi-build &&
+           ../openmpi-2.1.6/configure --prefix=$HOME/local-cached &&
+           make -j3 all && make install;
+        fi;
+        PATH=$HOME/local-cached/bin:$PATH;
+        cd $TRAVIS_BUILD_DIR;
+     fi
+
+   # Update environment to find g++ 4.9 installation first.
+   # - if [ $TRAVIS_OS_NAME == "linux" ]; then
+   #      mkdir -p latest-gcc-symlinks;
+   #      ln -s /usr/bin/g++-4.9 latest-gcc-symlinks/g++;
+   #      ln -s /usr/bin/gcc-4.9 latest-gcc-symlinks/gcc;
+   #      ln -s /usr/bin/gcov-4.9 latest-gcc-symlinks/gcov;
+   #      export PATH=$PWD/latest-gcc-symlinks:$PATH;
+   #   fi
+
+   # Install tool to upload code coverage reports to coveralls.io
+   - if [ "$CODECOV" == "YES" ]; then
+        export PYTHONUSERBASE=$HOME/local;
+        pip install --user cpp-coveralls;
+        pip install --user pyyaml;
+        PATH=$HOME/local/bin:$PATH;
+     fi
+
+install:
+   # Set MPI compilers, print compiler version
+   - if [ $MPI == "YES" ]; then
+        if [ "$TRAVIS_OS_NAME" == "linux" ]; then
+           export MPICH_CC="$CC";
+           export MPICH_CXX="$CXX";
+        else
+           export OMPI_CC="$CC";
+           export OMPI_CXX="$CXX";
+           mpic++ --showme:version;
+        fi;
+        mpic++ -v;
+     else
+        $CXX -v;
+     fi
+
+   # Back out of the mfem directory to install the libraries
+   - cd ..
+
+   # hypre
+   - if [ $MPI == "YES" ]; then
+        if [ ! -e $HYPRE_TOP_DIR/src/hypre/lib/libHYPRE.a ]; then
+           wget $HYPRE_URL;
+           rm -rf $HYPRE_TOP_DIR;
+           tar xvzf $HYPRE_ARCHIVE;
+           cd $HYPRE_TOP_DIR/src;
+           ./configure --disable-fortran CC=mpicc CXX=mpic++;
+           make -j3;
+           cd ../..;
+        else
+            echo "Reusing cached $HYPRE_TOP_DIR/";
+        fi;
+        ln -s $HYPRE_TOP_DIR hypre;
+     else
+        echo "Serial build, not using hypre";
+     fi
+
+   # METIS, use a mirror because the original source server is not always up.
+   # Original url:
+   #    http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/OLD/metis-4.0.3.tar.gz
+   - if [ $MPI == "YES" ]; then
+        if [ ! -e metis-4.0/libmetis.a ]; then
+           wget https://mfem.github.io/tpls/metis-4.0.3.tar.gz;
+           tar xvzf metis-4.0.3.tar.gz;
+           make -j3 -C metis-4.0.3/Lib CC="$CC" OPTFLAGS="-O2";
+           rm -rf metis-4.0;
+           mv metis-4.0.3 metis-4.0;
+        else
+           echo "Reusing cached metis-4.0/";
+        fi;
+     fi
+
+   # Re-enable ccache on linux; enable ccache on mac os:
+   - if [ $TRAVIS_OS_NAME == "linux" ]; then
+        export PATH="/usr/lib/ccache:$PATH";
+     else
+        if [ $TRAVIS_OS_NAME == "osx" ]; then
+           export PATH="/usr/local/opt/ccache/libexec:$PATH";
+        fi;
+     fi
+
+   - printf "which \$CC  = "; which $CC;
+     printf "which \$CXX = "; which $CXX
+
+script:
+   # Compiler
+   - if [ $MPI == "YES" ]; then
+        export MYCXX=mpic++;
+        export MAKE_CXX_FLAG=MPICXX=$MYCXX;
+     else
+        export MYCXX="$CXX";
+        export MAKE_CXX_FLAG=CXX=$MYCXX;
+     fi
+
+   # Print the compiler version
+   - $MYCXX -v
+
+   # Set some variables
+   - cd $TRAVIS_BUILD_DIR;
+     CPPFLAGS="";
+     SKIP_TEST_DIRS="";
+     if [ "$CODECOV" == "YES" ]; then
+        CPPFLAGS="--coverage -g";
+     fi;
+     if [ "$TRAVIS_OS_NAME" != "linux" ] || [ "$DEBUG" == "YES" ]; then
+        CPPFLAGS+=" -pedantic -Wall -Werror";
+     fi
+
+   # Configure the library
+   - make config MFEM_USE_MPI=$MPI MFEM_DEBUG=$DEBUG $MAKE_CXX_FLAG
+        MFEM_MPI_NP=$NPROCS CPPFLAGS="$CPPFLAGS"
+   # Show the configuration
+   - make info
+   # Build the library
+   - make -j3
+   # Build the examples and the miniapps
+   - make -j3 all
+   # Run tests
+   - make $MFEM_TEST_TARGET SKIP_TEST_DIRS="$SKIP_TEST_DIRS"
+
+after_success:
+   - if [ "$CODECOV" == "YES" ]; then
+        coveralls --include fem --include general --include linalg --include
+           mesh --exclude /usr --gcov-options '\-lp' --root $TRAVIS_BUILD_DIR;
+     fi
@@ -8,216 +8,13 @@
                               https://mfem.org


-Version 4.3.1 (development)
+Version 4.2.1 (development)
 ===========================

-
-Version 4.3, released on July 29, 2021
-======================================
-
-Discretization improvements
---------------------------
- Variable order spaces, p- and hp-refinement. This is the initial (serial)
-  support for variable-order FiniteElementCollection and FiniteElementSpace.
-  The new method FiniteElementSpace::SetElementOrder can be called to set an
-  arbitrary order for each mesh element. The conforming interpolation matrix
-  will now automatically constrain p- and hp- interfaces, enabling general
-  hp-refinement in both 2D and 3D, on uniform or mixed NC meshes. Support for
-  parallel variable-order spaces will follow shortly.
-
- Extended the support for field transfer between high-order and low-order
-  refined finite element spaces to include: dual fields and H1 fields (both
-  primary and dual). These are illustrated in the lor-transfer miniapp.
-
- Improved libCEED integration, including support for VectorCoefficient,
-  ConvectionIntegrator, and VectorConvectionNLFIntegrator with libCEED backends.
-
- Extending support for L2 basis functions using MapTypes VALUE and INTEGRAL in
-  linear interpolators and GridFunction "GetValue" methods.
-
- Changed the interface for the error estimator and implemented the Kelly error
-  indicator for scalar-valued problems, supported in serial and parallel builds.
-
- Added support for the "BR2" discontinuous Galerkin discretization for
-  diffusion via DGDiffusionBR2Integrator (see Example 14/14p).
-
- Added convective and skew-symmetric integrators for the nonlinear term in the
-  Navier-Stokes equations.
-
- Added new classes DenseSymmetricMatrix and SymmetricMatrixCoefficient for
-  efficient evaluation of symmetric matrix coefficients. This replaces the now
-  deprecated EvalSymmetric in MatrixCoefficient. Added DiagonalMatrixCoefficient
-  for clarity, which is a typedef of VectorCoefficient.
-
- Added support for nonscalar coefficient with VectorDiffusionIntegrator.
-
-Linear and nonlinear solvers
----------------------------
- Added support for AMG preconditioners on GPUs based on the hypre library
-  (version 2.22.0 or later). These include BoomerAMG, AMS and ADS and most
-  MFEM examples that use hypre have been ported to support this functionality.
-  The GPU preconditioners require that both hypre and MFEM are built with CUDA
-  support. Hypre builds with CUDA and unified memory are also supported and
-  can be used with `-d cuda:uvm` as a command-line option.
-
- Added support for AMG preconditioners for non-symmetric systems (e.g.
-  advection-dominated problems) using hypre's approximate ideal restriction
-  (AIR) AMG. Requires hypre version 2.14.0 or newer. Usage is illustrated in
-  example 9/9p.
-
- Added new functionality for constructing low-order refined discretizations and
-  solvers, see the LORDiscretization and LORSolver classes. A new basis type for
-  H(curl) and H(div) spaces is introduced to give spectral equivalence. This
-  functionality is illustrated in the LOR solvers miniapp in miniapps/solvers.
-
- Generalized the Multigrid class to support non-geometric multigrid. Previous
-  functionality, based on FiniteElementSpaceHierarchy, is now available in the
-  derived class GeometricMultigrid.
-
- Introduced solver interface for linear problems with constraints, a few
-  concrete solvers that implement the interface, and a demonstration of their
-  use in Example 28(p), which solves an elasticity problem with zero normal
-  displacement (but allowed tangential displacement) on two boundaries.
-
 - Added high-order matrix-free auxiliary Maxwell solver for H(curl) problems,
  as described in Barker and Kolev 2020 (https://doi.org/10.1002/nla.2348). See
  Example 3p and linalg/auxiliary.?pp.

- Improved interface for using the Ginkgo library, including: support for matrix-
-  free operators in Ginkgo solvers, new wrappers for Ginkgo preconditioners, HIP
-  support, and reduction of unnecessary data copies.
-
- Added initial support for hypre's mixed integer (mixedint) capability, which
-  uses different data types for local and global indices in order to save memory
-  in large problems. This capability requires that hypre was configured with the
-  --enable-mixedint option. Note that this option is currently tested only in
-  ex1p, ex3p, and ex4p, and may not work in more general settings.
-
- Added AlgebraicCeedSolver that does matrix-free algebraic p-multigrid for
-  diffusion problems with the Ceed backend.
-
- Added interface to MUMPS direct solver. Its usage is demonstrated in ex25p.
-  See http://mumps.enseeiht.fr/ for more details. Supported versions >= 5.1.1.
-
- Added three ESDIRK time integrators: implicit trapezoid rule, L-stable
-  ESDIRK-32, and A-stable ESDIRK-33.
-
- Implemented a variable step-size IMEX (VSSIMEX) method for the Navier miniapp.
-
- Implemented an adaptive linear solver tolerance option for NewtonSolver based
-  on the algorithm of Eisenstat and Walker.
-
-Meshing improvements
--------------------
- Added support for reading high-order Lagrange meshes in VTK format. Arbitrary-
-  orders and all element types are supported. See the VTK blog for more info:
-  https://blog.kitware.com/wp-content/uploads/2018/09/Source_Issue_43.pdf.
-
- Introduced a new non-conforming mesh format that fixes known inconsistencies
-  of legacy "MFEM mesh v1.1" NC format and works consistently in both serial and
-  parallel. ParMesh::ParPrint can now print non-conforming AMR meshes that can
-  be used to restart a parallel AMR computation. Example 6p has been extended to
-  demonstrate restarting from a previously saved checkpoint. Note that parallel
-  NC data files are compatible with serial code, e.g., can be viewed with serial
-  GLVis. Loading of legacy NC mesh files is still supported.
-
- Added FMS support (https://github.com/CEED/FMS) to mfem. FMS can represent
-  unstructured high-order meshes with general high-order finite element fields
-  on them. When enabled, mfem can convert data collections to/from FMS data
-  collections in memory. In addition, an FMS data collection class was added so
-  the convert-dc miniapp can read and generate data files in FMS format.
-
- Added new mesh quality metrics and improved the untangling capabilities of the
-  TMOP-based mesh optimization algorithms.
-
- The TMOP mesh optimization algorithms were extended to GPU:
-  * QualityMetric 1, 2, 7, 77 are available in 2D, 302, 303, 315, 321 in 3D
-  * Both AnalyticAdaptTC and DiscreteAdaptTC TargetConstructor are available
-  * Kernels for normalization and limiting have been added
-  * The AdvectorCG now also supports AssemblyLevel::PARTIAL
-
- Added support for creating refined meshes for all element types (e.g. by
-  splitting high-order elements into low-order refined elements), including
-  mixed meshes. The LOR Transfer miniapp (miniapps/tools/lor-transfer.cpp) now
-  supports meshes with any element geometry.
-
- Meshes consisting of any type of elements (including mixed meshes) can be
-  converted to all-simplex meshes using Mesh::MakeSimplicial.
-
- Several of the mesh constructors (creating Cartesian meshes, refined (LOR)
-  meshes, simplex meshes, etc.) are now available as "named constructors", e.g.
-  Mesh::MakeCartesian2D or Mesh::MakeRefined. The legacy constructors are marked
-  as deprecated.
-
- Added support for creating periodic meshes with Mesh::MakePeriodic. The
-  requisite periodic vertex mappings can be created with
-  Mesh::CreatePeriodicVertexMapping.
-
- Added support for 1D non-conforming meshes (which can be useful for parallel
-  load balancing and derefinement).
-
- Added sample meshes in the `data` subdirectory showing the reference elements
-  of the six currently supported element types; ref-segment.mesh,
-  ref-triangle.mesh, ref-square.mesh, ref-tetrahedron.mesh, ref-cube.mesh, and
-  ref-prism.mesh.
-
-High-performance computing
--------------------------
- Added initial support for GPU-accelerated versions of PETSc that works with
-  MFEM_USE_CUDA if PETSc has been configured with CUDA support. Examples 1 and 9
-  in the examples/petsc directory have been modified to work with --device cuda.
-  Examples with GAMG (ex1p) and SLEPc (ex11p) are also provided.
-
- Added support for explicit vectorization in the high-performance templated
-  code for Fujitsu's A64FX ARM microprocessor architecture.
-
- Added support for different modes of QuadratureInterpolator on GPU.
-  The layout (QVectorLayout::byNODES|byVDIM) and the tensor products modes can
-  be enabled before calling the Mult, Values, Derivatives, PhysDerivatives and
-  Determinants methods.
-
- Added method Device::SetMemoryTypes that can be used to change the default
-  host and device MemoryTypes before Device setup.
-
- In class MemoryManager, added methods GetDualMemoryType and SetDualMemoryType;
-  dual MemoryTypes are used to determine the second MemoryType (host or device)
-  when only one MemoryType is specified in methods of class Memory.
-
- Added Memory constructor for setting both the host and device MemoryTypes.
-
- Switched the default behavior of device memory allocations so that they are
-  deferred until the device pointer is needed.
-
- Added a second Umpire device MemoryType, DEVICE_UMPIRE_2, with corresponding
-  allocator that can be set with the method SetUmpireDevice2AllocatorName.
-
- Added HOST_PINNED MemoryType and a pinned host allocator for CUDA and HIP.
-
- Added matrix-free GPU-enabled implementations of GradientInterpolator and
-  IdentityInterpolator.
-
-New and updated examples and miniapps
-------------------------------------
- Added a new, very simple example (ex0 and parallel version ex0p). This example
-  solves a simple Poisson problem using H1 elements (the same problem as ex1),
-  but is intended to be extremely simple and approachable for new users.
-
- Added new miniapps demonstrating: 1) the use of GSLIB for overlapping grids,
-  see gslib/schwarz_ex1, and 2) coupling different physics in different domains,
-  see navier/cht. Note that gslib v1.0.7 is require (see INSTALL for details).
-
- Added a new miniapp for computing (signed) distance functions to a point
-  source or zero level set. See miniapps/shifted/distance.cpp.
-
- Added a high-order extension of the shifted boundary method to solve PDEs on
-  non body-fitted meshes. This is illustrated in the new Shifted Diffusion
-  miniapp, see miniapps/shifted/diffusion.cpp.
-
- Added new miniapp directory mtop/ with optimization-oriented block parametric
-  non-linear form and abstract integrators. Two new miniapps, ParHeat and
-  SeqHeat, demonstrate parallel and sequential implementation of gradients
-  evaluation for linear diffusion with discrete density.
-
 - Added a new miniapp block-solvers that compares the performance of various
  solvers for mixed finite element discretization of the second order scalar
  elliptic equations. Currently available solvers in the miniapp include a
@@ -226,74 +23,110 @@ New and updated examples and miniapps
  exploits a multilevel decomposition of the Raviart-Thomas space and its
  divergence-free subspace. See the miniapps/solvers directory for more details.

- Introduced new options for the mesh-explorer miniapp to visualize the actual
-  element attributes in parallel meshes while retaining the visualization of the
-  domain decomposition.
+- Added a new miniapp for computing (signed) distance functions to a point
+  source or zero level set. See miniapps/shifted/distance.cpp.

- Added partial assembly and device support to Example 25/25p, with diagonal
-  preconditioning.
+- Added matrix-free GPU-enabled implementations of GradientInterpolator and
+  IdentityInterpolator.

- Implemented a filter method for the Navier miniapp to stabilize highly
-  turbulent flows in direct numerical simulation.
+- Added interface to MUMPS direct solver. Its usage is demonstrated in ex25p.
+  See http://mumps.enseeiht.fr/ for more details. Supported versions >= 5.1.1.

-Improved testing
----------------
- Transitioned from Travis to GitHub Action for testing/CI on GitHub.
+- Added three ESDIRK time integrators: implicit trapezoid rule, L-stable
+  ESDIRK-32, and A-stable ESDIRK-33.

- Use Spack (and Uberenv) to automate TPL building in LLNL GitLab tests.
-
- Extended `make test` to include GPU tests when MFEM is built with CUDA or HIP
-  support.
-
- Added a set of suggested git hooks for developers in config/githooks.
-
- Added support for Caliper: a library to integrate performance profiling
-  capabilities into applications. See examples/caliper for more details.
-
- Added a new command line boolean option (`--all`) to the unit tests to launch
-  *all* non-regression tests.
-
- Upgraded the Catch unit test framework from version 2.13.0 to version 2.13.2.
-
-Miscellaneous
-------------
- The following integrations have updated minimum version requirements:
-  * CUDA >= 10.1.168
-  * Ginkgo >= 1.4.0
-  * GSLIB >= 1.0.7
-  * HIOP >= 0.4
-  * HYPRE >= 2.20.0 for mixedint support
-  * HYPRE >= 2.22.0 for CUDA support
-  * libCEED >= 0.8
-  * PETSc >= 3.15.0 for CUDA support
-  * RAJA >= 0.13.0
-  see INSTALL for more details.
+- Introduced a new non-conforming mesh format that fixes known inconsistencies
+  of legacy "MFEM mesh v1.1" NC format and works consistently in both serial and
+  parallel. ParMesh::ParPrint can now print non-conforming AMR meshes that can
+  be used to restart a parallel AMR computation. Example 6p has been extended to
+  demonstrate restarting from a previously saved checkpoint. Note that parallel
+  NC data files are compatible with serial code, e.g., can be viewed with serial
+  GLVis. Loading of legacy NC mesh files is still supported.

 - Added a "scaled Jacobian" visualization option in the Mesh Explorer miniapp to
  help identify elements with poor mesh quality.

- Added support for reading VTK meshes in XML format.
+- Added support for the "BR2" discontinuous Galerkin discretization for
+  diffusion via DGDiffusionBR2Integrator (see Example 14/14p).

- Added makefile rule to generate TAGS table for vi or Emacs users.
+- Generalized the Multigrid class to support non-geometric multigrid. The
+  previous functionality, based on FiniteElementSpaceHierarchy, is now available
+  in the derived class GeometricMultigrid.
+
+- Upgraded the Catch unit test framework from version 2.13.0 to version 2.13.2.
+
+- Implemented a filter method for the Navier miniapp to stabilize highly
+  turbulent flows in direct numerical simulation.

 - Added HIP support to the CMake build system.

- Various other simplifications, extensions, and bugfixes in the code.
+- Added support for reading high-order Lagrange meshes in VTK format. Arbitrary-
+  orders and all element types are supported. See the VTK blog for more info:
+  https://blog.kitware.com/wp-content/uploads/2018/09/Source_Issue_43.pdf.

-API changes
-----------
- Added an abstract interface `mfem::FaceRestriction` for `H1FaceRestriction`
-  and `L2FaceRestriction`.
-  In order to conform with the semantic of `MultTranspose` in `mfem::Operator`,
-  `mfem::FaceRestriction::MultTranspose` now sets instead of adding values, and
-  `mfem::FaceRestriction::AddMultTranspose` should replace previous calls to
-  `mfem::FaceRestriction::MultTranspose`.
+- Added support for reading VTK meshes in XML format.
+
+- Added partial assembly and device support to Example 25/25p, with diagonal
+  preconditioning.
+
+- Implemented a variable step-size IMEX (VSSIMEX) method for the Navier miniapp.
+
+- Added new mesh quality metrics and improved the untangling capabilities of the
+  TMOP-based mesh optimization algorithms.
+
+- Added convective and skew-symmetric integrators for the nonlinear term in the
+  Navier-Stokes equations.
+
+- Changed the interface for the error estimator.
+
+- Implemented the parallel Kelly error indicator for scalar-valued problems.
+
+- Added new classes DenseSymmetricMatrix and SymmetricMatrixCoefficient for
+  efficient evaluation of symmetric matrix coefficients. This replaces the now
+  deprecated EvalSymmetric in MatrixCoefficient. Added DiagonalMatrixCoefficient
+  for clarity, which is a typedef of VectorCoefficient.
+
+- Added support for AMG preconditioners for non-symmetric systems (e.g.
+  advection-dominated problems) using hypre's approximate ideal restriction
+  (AIR) AMG. Requires hypre version 2.14.0 or newer. Usage is illustrated in
+  example 9/9p.
+
+- Implemented an adaptive linear solver tolerance option for NewtonSolver based
+  on the algorithm of Eisenstat and Walker.
+
+- Added support for nonscalar coefficient with VectorDiffusionIntegrator.
+
+- Extending support for L2 basis functions using MapTypes VALUE and INTEGRAL in
+  linear interpolators and GridFunction "GetValue" methods.
+
+- Variable order spaces, p- and hp-refinement. This is the initial (serial)
+  support for variable-order FiniteElementCollection and FiniteElementSpace.
+  The new method FiniteElementSpace::SetElementOrder can be called to set an
+  arbitrary order for each mesh element. The conforming interpolation matrix
+  will now automatically constrain p- and hp- interfaces, enabling general
+  hp-refinement in both 2D and 3D, on uniform or mixed NC meshes. Support for
+  parallel variable-order spaces will follow shortly.
+
+- Added support for creating refined meshes for all element types (e.g. by
+  splitting high-order elements into low-order refined elements), including
+  mixed meshes. The LOR Transfer miniapp (miniapps/tools/lor-transfer.cpp) now
+  supports meshes with any element geometry.
+
+- Gitlab CI: use Spack (and Uberenv) to automate the build of TPLs.
+
+libCEED integration improvements
+--------------------------------
+- Refactor the libCEED integration
+
+- Add support for VectorCoefficient with libCEED backends.
+
+- Add support for ConvectionIntegrator, and VectorConvectionNLFIntegrator with libCEED backends.


 Version 4.2, released on October 30, 2020
 =========================================

-High-performance computing
+High-Performance Computing
 --------------------------
 - Added support for explicit vectorization in the high-performance templated
  code, which can now take advantage of specific classes on the following
@@ -485,7 +318,7 @@ New and updated examples and miniapps
    L2, with partial assembly support in Example 24/24p.
  * Weak Dirichlet boundary conditions (Nitsche) to the NURBS miniapp.

-Data management and visualization
+Data management and Visualization
 ---------------------------------
 - Added support for ADIOS2 for parallel I/O with ParaView visualization. See
  Examples 5, 9, 12, 16. The classes adios2stream and ADIOS2DataCollection
@@ -16,9 +16,6 @@ set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH

 # Require C++11 and disable compiler-specific extensions
 set(CMAKE_CXX_STANDARD 11)
-if (MFEM_USE_GINKGO)
-  set(CMAKE_CXX_STANDARD 14)
-endif()
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)

@@ -54,7 +51,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.3.1)
+set(${PROJECT_NAME}_VERSION 4.2.1)

 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -102,9 +99,6 @@ if (MFEM_USE_CUDA)
   endif()
   enable_language(CUDA)
   set(CMAKE_CUDA_STANDARD 11)
-   if (MFEM_USE_GINKGO)
-     set(CMAKE_CUDA_STANDARD 14)
-   endif()
   set(CMAKE_CUDA_STANDARD_REQUIRED ON)
   set(CMAKE_CUDA_EXTENSIONS OFF)
   set(CUDA_FLAGS "--expt-extended-lambda")
@@ -179,12 +173,6 @@ endif()
 if (MFEM_USE_MPI)
  find_package(MPI REQUIRED)
  set(MPI_CXX_INCLUDE_DIRS ${MPI_CXX_INCLUDE_PATH})
-  if (MFEM_MPIEXEC)
-    set(MPIEXEC ${MFEM_MPIEXEC})
-  endif()
-  if (MFEM_MPIEXEC_NP)
-    set(MPIEXEC_NUMPROC_FLAG ${MFEM_MPIEXEC_NP})
-  endif()
  # Parallel MFEM depends on hypre
  find_package(HYPRE REQUIRED)
  set(MFEM_HYPRE_VERSION ${HYPRE_VERSION})
@@ -246,7 +234,6 @@ if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
    message(FATAL_ERROR " *** MFEM_USE_LEGACY_OPENMP requires MFEM_THREAD_SAFE=ON.")
  endif()
  find_package(OpenMP REQUIRED)
-  set(OPENMP_LIBRARIES ${OpenMP_CXX_LIBRARIES})
 endif()

 # SuiteSparse (before SUNDIALS which may depend on KLU)
@@ -267,15 +254,6 @@ if (MFEM_USE_SUNDIALS)
  find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS})
 endif()

-# EPIC
-if (MFEM_USE_EPIC)
-  if (NOT (MFEM_USE_MPI AND MFEM_USE_SUNDIALS AND MFEM_USE_LAPACK) )
-    message(FATAL_ERROR " *** EPIC requires that MPI, SUNDIALS and LAPACK be enabled.")
-  else()
-    find_package(EPIC REQUIRED SUNDIALS NVector_Serial NVector_Parallel BLAS LAPACK)
-  endif()
-endif()
-
 # Mesquite
 if (MFEM_USE_MESQUITE)
  find_package(Mesquite REQUIRED)
@@ -340,10 +318,6 @@ if (MFEM_USE_CONDUIT)
    find_package(Conduit REQUIRED conduit relay blueprint )
 endif()

-if (MFEM_USE_FMS)
-    find_package(FMS REQUIRED fms )
-endif()
-
 # Axom/Sidre
 if (MFEM_USE_SIDRE)
  find_package(Axom REQUIRED Axom)
@@ -386,20 +360,12 @@ if (MFEM_USE_UMPIRE)
   find_package(UMPIRE REQUIRED)
 endif()

-# Caliper
-if (MFEM_USE_CALIPER)
-  find_package(Caliper REQUIRED)
-endif()
-
 # AMD HIP
 if (MFEM_USE_HIP)
  find_package(HIP REQUIRED)
  if (HIP_ARCH)
    message(STATUS "Using HIP architecture: ${HIP_ARCH}")
    list(APPEND HIP_HIPCC_FLAGS "--amdgpu-target=${HIP_ARCH}")
-    if (MFEM_USE_GINKGO)
-      list(APPEND HIP_HIPCC_FLAGS "-std=c++14")
-    endif()
  endif()
 endif()

@@ -437,11 +403,10 @@ endif()
 #    With newer versions of SuiteSparse which include METIS header using 64-bit
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
-set(MFEM_TPLS MPI_CXX OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS EPIC PETSC
-    SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB NETCDF
+set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
+    SLEPC MESQUITE SuperLUDist MUMPS STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
-    CUSPARSE MKL_CPARDISO AMGX CALIPER)
-
+    CUSPARSE MKL_CPARDISO AMGX)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")
@@ -460,9 +425,6 @@ include_directories(${TPL_INCLUDE_DIRS})
 if (OPENMP_FOUND)
  message(STATUS "MFEM: using package OpenMP")
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
-  if (MFEM_USE_CUDA)
-    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=${OpenMP_CXX_FLAGS}")
-  endif()
 endif()

 message(STATUS "MFEM build type: CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}")
@@ -547,8 +509,6 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
  endforeach()
 endif()

-set(MFEM_CUSTOM_TARGET_PREFIX CACHE STRING "")
-
 #-------------------------------------------------------------------------------
 # Examples, miniapps, and testing
 #-------------------------------------------------------------------------------
@@ -599,17 +559,16 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
 endif()

 # Add 'check' target - quick test
-set(MFEM_CHECK_TARGET_NAME ${MFEM_CUSTOM_TARGET_PREFIX}check)
 if (NOT MFEM_USE_MPI)
-  add_custom_target(${MFEM_CHECK_TARGET_NAME}
+  add_custom_target(check
    ${CMAKE_CTEST_COMMAND} -R \"^ex1_ser\" -C ${CMAKE_CFG_INTDIR}
    USES_TERMINAL)
-  add_dependencies(${MFEM_CHECK_TARGET_NAME} ex1)
+  add_dependencies(check ex1)
 else()
-  add_custom_target(${MFEM_CHECK_TARGET_NAME}
+  add_custom_target(check
    ${CMAKE_CTEST_COMMAND} -R \"^ex1p\" -C ${CMAKE_CFG_INTDIR}
    USES_TERMINAL)
-  add_dependencies(${MFEM_CHECK_TARGET_NAME} ex1p)
+  add_dependencies(check ex1p)
 endif()

 #-------------------------------------------------------------------------------
@@ -4,9 +4,7 @@

 <p align="center">
 <a href="https://github.com/mfem/mfem/blob/master/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-BSD-brightgreen.svg"></a>
-<a href="https://github.com/mfem/mfem/actions?query=workflow%3Arepo-check+branch%3Amaster"><img alt="Repo check" src="https://github.com/mfem/mfem/actions/workflows/repo-check.yml/badge.svg?branch=master"></a>
-<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuild-analysis+branch%3Amaster"><img alt="Build Analysis" src="https://github.com/mfem/mfem/actions/workflows/mfem-analysis.yml/badge.svg?branch=master"></a>
-<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuilds-and-tests+branch%3Amaster"><img alt="Builds and Tests" src="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml/badge.svg?branch=master"></a>
+<a href="https://travis-ci.org/mfem/mfem"><img alt="Build Status" src="https://travis-ci.org/mfem/mfem.svg?branch=master"></a>
 <a href="https://ci.appveyor.com/project/mfem/mfem"><img alt="Build Status" src="https://ci.appveyor.com/api/projects/status/19non9sqm6msi2wy?svg=true"></a>
 <a href="https://mfem.github.io/doxygen/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
 </p>
@@ -65,8 +63,6 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
  development branches off `mfem:master`.
 - Please follow the [developer guidelines](#developer-guidelines), in particular
  with regards to documentation and code styling.
- Please do not commit large/binary files to the central repository (use a fork
-  instead).
 - Pull requests  should be issued toward `mfem:master`. Make sure
  to check the items off the [Pull Request Checklist](#pull-request-checklist).
 - When your contribution is fully working and ready to be reviewed, add
@@ -75,7 +71,6 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
  reviewers to evaluate the changes.
 - The reviewers have 3 weeks to evaluate the PR and work with the author to
  fix issues and implement improvements.
- During review there should be no force pushes/rewriting history in the branch.
 - After approval, MFEM developers merge the PR manually in the [mfem:next branch](#masternext-workflow).
 - After a week of testing in `mfem:next`, the original PR is merged in `mfem:master`.
 - We use [milestones](https://github.com/mfem/mfem/milestones) to coordinate the
@@ -96,13 +91,12 @@ The MFEM source code has the following structure:
 ```
  .
  ├── config
-  │   ├── cmake
-  │   └── githooks
+  │   └── cmake
+  │       └── ...
  ├── data
  ├── doc
  ├── examples
  │   ├── amgx
-  │   ├── caliper
  │   ├── ginkgo
  │   ├── hiop
  │   ├── petsc
@@ -110,9 +104,7 @@ The MFEM source code has the following structure:
  │   ├── sundials
  |   └── superlu
  ├── fem
-  │   ├── ceed
-  │   ├── qinterp
-  │   └── tmop
+  │   └── ceed
  ├── general
  ├── linalg
  │   └── simd
@@ -123,7 +115,6 @@ The MFEM source code has the following structure:
  │   ├── electromagnetics
  │   ├── gslib
  │   ├── meshing
-  │   ├── mtop
  │   ├── navier
  │   ├── nurbs
  │   ├── performance
@@ -134,10 +125,10 @@ The MFEM source code has the following structure:
  └── tests
      ├── convergence
      ├── gitlab
-      ├── mem_manager
      ├── par-mesh-format
      ├── scripts
      └── unit
+          └── ...
 ```

 #### Main directories and classes
@@ -368,10 +359,6 @@ Before you can start, you need a GitHub account, here are a few suggestions:
  two reviewers to evaluate the changes. The reviewers have 3 weeks to evaluate
  the PR and work with the author to implement improvements and fix issues.

- Once the `ready-for-review` label has been applied and reviewers have been
-  assigned, the PR is considered under review. To help with the review process
-  there should be no force pushes/rewriting history in the branch.
-
 - After approval, the PR is [tested](#masternext-workflow) for a week with
  other approved PRs in the `mfem:next` branch.

@@ -379,20 +366,16 @@ Before you can start, you need a GitHub account, here are a few suggestions:
  `mfem:next`, see the [README](tests/scripts/README) file in that directory
  for more details.

- Track the GitHub Actions and Appveyor [continuous integration](#automated-testing)
+- Track the Travis CI and Appveyor [continuous integration](#automated-testing)
  builds at the end of the PR. These should generally run clean, so address any
  errors as soon as possible. Please ask if you are unsure how to do that.

- Note that some tests, such as the `branch-history` check in GitHub Actions
-  are safeguards that are allowed to fail in certain cases.
+- Note that some tests, such as the `branch-history` check in Travis are
+  safeguards that are allowed to fail in certain cases.

 - Other tests, such as the `code-style`, `documentation` and `gitignore`
-  checks in GitHub Actions enforce MFEM-specific rules which are explained in
-  the error messages and the `tests/scripts` directory.
-
- Also note that the tests `branch-history` and `repos-checks` found in GitHub
-  Actions can be triggered automatically before each push using git hooks. See
-  the [git hooks README](config/githooks/README.md) for a detailed explanation.
+  checks in Travis enforce MFEM-specific rules which are explained in the
+  error messages and the `tests/scripts` directory.

 - If triggered, track the status of the LLNL GitLab tests. If failing, ask
  one of the _LLNL developers_ for details.
@@ -412,7 +395,7 @@ Before a PR can be merged, it should satisfy the following:
    - [ ] Does `make` or `cmake` have a new target?
    - [ ] Did the requirements or the installation process change? *(rare)*
 - [ ] Update continuous integration server configurations if necessary (e.g. with new version requirements for each of MFEM's dependencies)
-    - [ ] `.github`
+    - [ ] `.travis.yml`
    - [ ] `.appveyor.yml`
 - [ ] Update `.gitignore`:
    - [ ] Check if `make distclean; git status` shows any files that were generated from the source by the project (not an IDE) but we don't want to track in the repository.
@@ -442,7 +425,6 @@ Before a PR can be merged, it should satisfy the following:
      - [ ] Add/update the `CMakeLists.txt` file in the new miniapp directory.
      - [ ] Consider adding a new test for the new miniapp.
   - [ ] List the new miniapp in `doc/CodeDocumentation.dox`
-   - [ ] If new miniapps directory (e.g.`miniapps/nurbs`), add it to `MINIAPP_SUBDIRS` in the `makefile`.
   - [ ] If new miniapps directory (e.g.`miniapps/nurbs`), list it in `doc/CodeDocumentation.conf.in`
   - [ ] Companion pull request for documentation in [mfem/web](https://github.com/mfem/web) repo:
     - [ ] Update or add miniapp-specific documentation, see e.g. the `src/meshing.md` and `src/electromagnetics.md` files.
@@ -529,7 +511,7 @@ MFEM uses a `master`/`next`-branch workflow as described below:
    - [ ] `doc/CodeDocumentation.conf.in`
 - [ ] Check that version requirements for each of MFEM's dependencies are documented in `INSTALL` and up-to-date
 - [ ] Check that continuous integration server configurations reflect the dependency version requirements of the new release
-    - [ ] `.github`
+    - [ ] `.travis.yml`
    - [ ] `.appveyor.yml`
 - [ ] Update the `CHANGELOG` to organize all release contributions
 - [ ] Review the whole source code once over
@@ -591,16 +573,12 @@ MFEM uses a `master`/`next`-branch workflow as described below:
 MFEM has several levels of automated testing running on GitHub, as well as on
 local Mac and Linux workstations, and Livermore Computing clusters at LLNL.

-In addition, developers can set local git hooks to run some quick checks on
-commit or push, see the [README](config/githooks/README.md) in the `config/githooks`
-directory.
-
 ### Linux and Mac smoke tests
-We use GitHub Actions to drive the default tests on the `master` and `next`
-branches. See the `.github/workflows` files and the logs at
-[https://github.com/mfem/mfem/actions](https://github.com/mfem/mfem/actions).
+We use Travis CI to drive the default tests on the `master` and `next`
+branches. See the `.travis` file and the logs at
+[https://travis-ci.org/mfem/mfem](https://travis-ci.org/mfem/mfem).

-Testing using GitHub Actions should be kept lightweight, as there is a time
+Testing using Travis CI should be kept lightweight, as there is a 50 minute time
 constraint on jobs. Two virtual machines are configured - Mac (OS X) and Linux.

 - Tests on the `master` branch are triggered whenever a PR is issued on this branch.
@@ -58,7 +58,6 @@ following package managers:

 - Spack, https://github.com/spack/spack
 - OpenHPC, http://openhpc.community
- Conda-forge, https://conda-forge.org (pre-built binaries linked with OpenMPI/MPICH, hypre, and METIS)
 - Homebrew/Science, https://github.com/Homebrew/homebrew-science (deprecated)

 We also recommend downloading and building the MFEM-based GLVis visualization
@@ -351,9 +350,10 @@ MFEM_USE_SUPERLU5 = YES/NO

 MFEM_USE_MUMPS = YES/NO
   Enable MFEM functionality based on the MUMPS library. Currently, this
-   option adds the class MUMPSSolver (a parallel sparse direct solver).
-   When enabled, this option uses the MUMPS_* library options, see below.
-
+   option adds the class MUMPSSolver (a parallel sparse direct solver). 
+   When enabled, this option uses the MUMPS_* library options, see
+   below.
+   
 MFEM_USE_STRUMPACK = YES/NO
   Enable MFEM functionality based on the STRUMPACK sparse direct solver and
   preconditioner through the STRUMPACKSolver and STRUMPACKRowLocMatrix
@@ -460,8 +460,8 @@ MFEM_USE_UMPIRE = YES/NO
   memory devices like NUMA and GPUs.

 MFEM_USE_HIOP = YES/NO
-   Enable the usage of HiOp (https://github.com/LLNL/hiop) in MFEM. HiOp is an
-   HPC solver for nonlinear optimization problems.
+  Enable the usage of HiOp (https://github.com/LLNL/hiop) in MFEM. HiOp is an
+  HPC solver for nonlinear optimization problems.

 MFEM_USE_CUDA = YES/NO
   Enables support for CUDA devices in MFEM. CUDA is a parallel computing
@@ -474,7 +474,7 @@ MFEM_USE_HIP = YES/NO
   Enables support for AMD devices in MFEM. HIP is a heterogeneous-compute
   interface for portability developed by AMD that can target both AMD and
   NVIDIA GPUs. The variable HIP_ARCH is used to specify the AMD GPU processor
-   used during compilation (by default, HIP_ARCH=gfx900). When enabled, this
+   used during compilation (by default, HIP_ARCH=gfx900).  When enabled, this
   option uses the HIP_* build options, see below.

 MFEM_USE_RAJA = YES/NO
@@ -508,21 +508,6 @@ MFEM_USE_MKL_CPARDISO = YES/NO
   MFEM_USE_LAPACK=YES, verify that the MKL LAPACK libraries are used. The
   OpenMP capabilities are disabled at link time.

-MFEM_USE_CALIPER = YES/NO
-   Enables the interface to Caliper. Caliper is a library to integrate
-   performance profiling capabilities into applications. To use Caliper,
-   developers mark code regions of interest using either Caliper's annotation
-   API or their equivalent in MFEM. Applications can then enable performance
-   profiling at runtime with Caliper's configuration API. Alternatively, one
-   can configure Caliper through environment variables or config files.
-
-MFEM_USE_FMS = YES/NO
-   Enables support for the FMS library which consists of the DataCollection
-   sub-class mfem::FMSDataCollection for I/O in FMS formats, see the header file
-   fem/fmsdatacollection.hpp. In addition, this option enables in-memory
-   convetion routines between FMS's FmsDataCollection structure and MFEM's
-   DataCollection class, see the header file fem/fmsconvert.hpp.
-
 MFEM_BUILD_TAG = (any value)
   An optional tag to characterize the build. Exported to config/config.mk.
   Can be used to identify the MFEM build from other makefiles.
@@ -544,12 +529,9 @@ directory and use the string @MFEM_DIR@, e.g. HYPRE_OPT = -I@MFEM_DIR@/../hypre.
 The specific libraries and their options are:

 - HYPRE, required for the parallel build, i.e. when MFEM_USE_MPI = YES.
-  See also the "Specific options for hypre" section at the end of this file.
  URL: https://github.com/hypre-space/hypre and https://www.llnl.gov/casc/hypre
  Options: HYPRE_OPT, HYPRE_LIB.
-  Versions: HYPRE >= 2.10.0b  (HYPRE built without CUDA)
-            HYPRE >= 2.20.0   (HYPRE built with '--enable-mixedint')
-            HYPRE >= 2.22.0   (HYPRE built with CUDA)
+  Versions: HYPRE >= 2.10.0b.

 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
  MFEM_USE_METIS_5 = YES (default is to use METIS 4).
@@ -619,11 +601,10 @@ The specific libraries and their options are:
  Versions: STRUMPACK >= 3.0.0.

 - Ginkgo (optional), used when MFEM_USE_GINKGO = YES. Note that Ginkgo needs a
-  C++ compiler that supports the C++-14 standard. For additional requirements
-  and dependencies of specific modules, see the Ginkgo webpage below.
+  C++ compiler that supports the C++-11 standard. For additional requirements
+  and dependencies of specific modules see the Ginkgo webpage below.
  URL: https://ginkgo-project.github.io
-  Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or Debug).
-  Versions: Ginkgo >= 1.4.0.
+  Options: GINKGO_OPT (Not used), GINKGO_LIB.

 - AmgX (optional), used when MFEM_USE_AMGX = YES.
  URL: https://github.com/NVIDIA/AMGX
@@ -655,8 +636,7 @@ The specific libraries and their options are:
                --with-shared-libraries=0
  URL: https://www.mcs.anl.gov/petsc
  Options: PETSC_OPT, PETSC_LIB.
-  Versions: PETSc >= 3.8.0   (PETSc build without CUDA)
-            PETSc >= 3.15.0  (PETSc built with CUDA)
+  Versions: PETSc >= 3.8.0.

 - SLEPc (optional), used when MFEM_USE_SLEPC = YES. SLEPc depends on PETSc and
  uses some of the PETSc options when compiled.
@@ -692,17 +672,17 @@ The specific libraries and their options are:
 - HiOp (optional), used when MFEM_USE_HIOP = YES.
  URL: https://github.com/LLNL/hiop
  Options: HIOP_OPT, HIOP_LIB.
-  Versions: HIOP >= 0.4.
+  Versions: HIOP >= 0.1.

 - GSLIB (optional), used when MFEM_USE_GSLIB = YES. The gslib library must be
-  built prior to the MFEM build, as follows: download gslib-1.0.7, untar it at
-  the same level as MFEM and create a symbolic link: "ln -s gslib-1.0.7 gslib".
+  built prior to the MFEM build, as follows: download gslib-1.0.5, untar it at
+  the same level as MFEM and create a symbolic link: "ln -s gslib-1.0.5 gslib".
  Build gslib in parallel or in serial based on the desired MFEM build: "make
  clean; make CC=mpicc" or "make clean; make CC=gcc MPI=0". Build MFEM with
  MFEM_USE_GSLIB=YES.
-  URL: https://github.com/gslib/gslib/archive/v1.0.7.tar.gz
+  URL: https://github.com/gslib/gslib/archive/v1.0.5.tar.gz
  Options: GSLIB_OPT, GSLIB_LIB.
-  Versions: GSLIB >= 1.0.7.
+  Versions: GSLIB >= 1.0.5.

 - MKL CPardiso (optional), used when MFEM_USE_MKL_CPARDISO = YES.
  URL: https://software.intel.com/content/www/us/en/develop/tools/math-kernel-library.html
@@ -727,7 +707,7 @@ The specific libraries and their options are:
  URL: https://github.com/CEED/libCEED
       https://ceed.exascaleproject.org/libceed
  Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED >= 0.8.
+  Versions: libCEED >= 0.7.

 - RAJA (optional), used when MFEM_USE_RAJA = YES.
  Beginning with MFEM v4.3, only RAJA v0.13.0+ is supported.
@@ -735,13 +715,7 @@ The specific libraries and their options are:
  Options: RAJA_DIR, RAJA_OPT, RAJA_LIB.
  Versions: RAJA >= 0.13.0.

- Caliper (optional), used when MFEM_USE_CALIPER = YES.
-  URL: https://github.com/LLNL/Caliper
-  Options: CALIPER_DIR
-  Versions: CALIPER >= 2.5.0, older versions may work too.
-
 - Umpire, used when MFEM_USE_UMPIRE = YES.
-  Umpire requires camp when the Umpire version is >= 3.0.0.
  URL: https://github.com/LLNL/Umpire
  Options: UMPIRE_DIR, UMPIRE_OPT, UMPIRE_LIB.
  Versions: Umpire >= 2.0.0.
@@ -761,11 +735,6 @@ The specific libraries and their options are:
  URL: https://zlib.net
  Options: ZLIB_OPT, ZLIB_LIB.

- FMS (optional), used when MFEM_USE_FMS = YES.
-  URL: https://github.com/CEED/FMS
-  Options: FMS_OPT, FMS_LIB.
-  Versions: FMS >= 0.2.
-
 Building with CMake
 ===================
 The MFEM build system consists of two steps: configuration and compilation.
@@ -896,8 +865,6 @@ MFEM_USE_CEED
 MFEM_USE_RAJA
 MFEM_USE_UMPIRE
 MFEM_USE_SIDRE
-MFEM_USE_CALIPER
-MFEM_USE_FMS

 The following options are CMake specific:

@@ -951,8 +918,6 @@ The CMake build system adds auto-detection for the following packages/libraries:
 - RAJA
 - UMPIRE
 - AXOM - Used when MFEM_USE_SIDRE is enabled
- - CALIPER
- - FMS

 The following built-in CMake packages are also used:

@@ -970,7 +935,7 @@ config/config.hpp.in:

   cp config/config.hpp.in config/_config.hpp

-The file config/_config.hpp can then be edited to enable desired options. The
+The file config/_config.hpp can then be edited to enable desired options.  The
 MFEM library is simply a combination of all object files obtained by compiling
 the .cpp source files in the source directories: general, linalg, mesh, and fem.

@@ -978,7 +943,7 @@ the .cpp source files in the source directories: general, linalg, mesh, and fem.
 Specifying an MPI job launcher
 ==============================
 By default, MFEM will use 'mpirun -np #' to launch any of its parallel tests or
-miniapps, where # is the number of MPI tasks. An alternate MPI launcher can be
+miniapps, where # is the number of MPI tasks.  An alternate MPI launcher can be
 provided by setting the MFEM_MPIEXEC and MFEM_MPIEXEC_NP config variables.

 MFEM will expect the launcher command, plus the command line option to allow it
@@ -988,19 +953,3 @@ MFEM_MPIEXEC    = mpirun # default
 MFEM_MPIEXEC_NP = -np    # default
 MFEM_MPIEXEC    = srun   # example for platforms using SLURM
 MFEM_MPIEXEC_NP = -n     # example for platforms using SLURM
-
-
-Specific options for hypre
-==========================
-The hypre library has multiple options to define local and global index storage
-sizes. By default, all indices are stored as an architecture aware integer. For
-most platforms, this will be 32-bit. This limits the maximum number of global
-degrees of freedom in a vector or matrix to about 2 billion. In order to solve
-larger problems, there are two options:
-
-1. Building hypre with '--enable-bigint' defines the local and global indices to
-   be 64-bit. This is convenient, but requires more memory than necessary.
-
-2. Building hypre with '--enable-mixedint' defines the local indiced to be
-   32-bit, while using a 64-bit storage for global indices. This option is
-   currently tested only in ex1p, and may not work in more general settings.
@@ -256,10 +256,6 @@ IF (DEFINED TPL_ENABLE_SIDRE)
  SET(MFEM_USE_SIDRE ${TPL_ENABLE_SIDRE} CACHE BOOL "Enable Axom/Sidre usage" FORCE)
 ENDIF()

-IF (DEFINED TPL_ENABLE_FMS)
-  SET(MFEM_USE_FMS ${TPL_ENABLE_FMS} CACHE BOOL "Enable FMS usage" FORCE)
-ENDIF()
-
 IF (DEFINED TPL_ENABLE_CONDUIT)
  SET(MFEM_USE_CONDUIT ${TPL_ENABLE_CONDUIT} CACHE BOOL "Enable Conduit usage" FORCE)
 ENDIF()
@@ -29,7 +29,6 @@ set(MFEM_USE_LEGACY_OPENMP @MFEM_USE_LEGACY_OPENMP@)
 set(MFEM_USE_MEMALLOC @MFEM_USE_MEMALLOC@)
 set(MFEM_TIMER_TYPE @MFEM_TIMER_TYPE@)
 set(MFEM_USE_SUNDIALS @MFEM_USE_SUNDIALS@)
-set(MFEM_USE_EPIC @MFEM_USE_EPIC@)
 set(MFEM_USE_MESQUITE @MFEM_USE_MESQUITE@)
 set(MFEM_USE_SUITESPARSE @MFEM_USE_SUITESPARSE@)
 set(MFEM_USE_SUPERLU @MFEM_USE_SUPERLU@)
@@ -45,7 +44,6 @@ set(MFEM_USE_PETSC @MFEM_USE_PETSC@)
 set(MFEM_USE_SLEPC @MFEM_USE_SLEPC@)
 set(MFEM_USE_MPFR @MFEM_USE_MPFR@)
 set(MFEM_USE_SIDRE @MFEM_USE_SIDRE@)
-set(MFEM_USE_FMS @MFEM_USE_FMS@)
 set(MFEM_USE_CONDUIT @MFEM_USE_CONDUIT@)
 set(MFEM_USE_PUMI @MFEM_USE_PUMI@)
 set(MFEM_USE_CUDA @MFEM_USE_CUDA@)
@@ -55,7 +53,6 @@ set(MFEM_USE_CEED @MFEM_USE_CEED@)
 set(MFEM_USE_UMPIRE @MFEM_USE_UMPIRE@)
 set(MFEM_USE_SIMD @MFEM_USE_SIMD@)
 set(MFEM_USE_ADIOS2 @MFEM_USE_ADIOS2@)
-set(MFEM_USE_CALIPER @MFEM_USE_CALIPER@)

 set(MFEM_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
 set(MFEM_CXX_FLAGS "@CMAKE_CXX_FLAGS@")
@@ -119,9 +119,6 @@
 // Enable the use of SIMD in the high performance templated classes
 #cmakedefine MFEM_USE_SIMD

-// Enable MFEM functionality based on the FMS library
-#cmakedefine MFEM_USE_FMS
-
 // Enable MFEM functionality based on Conduit
 #cmakedefine MFEM_USE_CONDUIT

@@ -154,9 +151,6 @@
 // Enable MFEM functionality based on the ADIOS2 library
 #cmakedefine MFEM_USE_ADIOS2

-// Enable MFEM functionality based on the Caliper library
-#cmakedefine MFEM_USE_CALIPER
-
 // Which library functions to use in class StopWatch for measuring time.
 // For a list of the available options, see INSTALL.
 // If not defined, an option is selected automatically.
@@ -165,9 +159,6 @@
 // Enable MFEM functionality based on the SUNDIALS libraries.
 #cmakedefine MFEM_USE_SUNDIALS

-// Enable MFEM functionality based on the EPIC libraries.
-#cmakedefine MFEM_USE_EPIC
-
 // Version of HYPRE used for building MFEM.
 #cmakedefine MFEM_HYPRE_VERSION @MFEM_HYPRE_VERSION@

@@ -15,10 +15,6 @@
 #   - AMGX_INCLUDE_DIRS

 include(MfemCmakeUtilities)
-set(AMGX_REQUIRED_LIBRARIES cusparse cusolver cublas cublasLt nvToolsExt)
+set(AMGX_REQUIRED_LIBRARIES cusparse cusolver cublas nvToolsExt)
 mfem_find_package(AMGX AMGX AMGX_DIR "include" "amgx_c.h" "lib" "amgx"
  "Paths to headers required by AMGX." "Libraries required by AMGX.")
-# Make sure the library location is locked down
-foreach(lib ${AMGX_REQUIRED_LIBRARIES})
-  list(APPEND AMGX_LIBRARIES ${CUDA_TOOLKIT_ROOT_DIR}/lib64/lib${lib}${CMAKE_SHARED_LIBRARY_SUFFIX})
-endforeach()
@@ -1,22 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - CALIPER_FOUND
-#   - CALIPER_LIBRARIES
-#   - CALIPER_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(Caliper CALIPER CALIPER_DIR
-       	"include" "caliper/cali.h"
-       	"lib" "caliper"
-  	"Paths to headers required by Caliper."
-       	"Libraries required by Caliper.")
@@ -1,21 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - EPIC_FOUND
-#   - EPIC_LIBRARIES
-#   - EPIC_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(EPIC EPIC EPIC_DIR
-       "include" Epic.h "lib" epic1.0.0
-  "Paths to headers required by EPIC." "Libraries required by EPIC.")
-
@@ -1,20 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - FMS_FOUND
-#   - FMS_LIBRARIES
-#   - FMS_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(FMS FMS FMS_DIR
-  "include" fms.h "lib" fms
-  "Paths to headers required by FMS." "Libraries required by FMS.")
@@ -15,20 +15,5 @@
 #   - NETCDF_INCLUDE_DIRS

 include(MfemCmakeUtilities)
-
-# FindHDF5.cmake uses HDF5_ROOT, so we "translate" from the MFEM convention
-set(HDF5_ROOT ${HDF5_DIR} CACHE PATH "")
-# We need to guard against the case where HDF5 was already found but without
-# the HL extensions (in which case mfem_find_package will treat the package
-# as already having been found), so we reset the variable to force FindHDF5.cmake
-# to be called for a second time
-set(HDF5_FOUND OFF)
-enable_language(C) # FindHDF5.cmake uses the C compiler
 mfem_find_package(NetCDF NETCDF NETCDF_DIR "include" netcdf.h "lib" netcdf
  "Paths to headers required by NetCDF." "Libraries required by NetCDF.")
-# The HL extension libraries are in a separate variable and must precede
-# the "regular" hdf5 library, as hdf5_hl depends on hdf5
-# The netcdf library will always be the first element of NETCDF_LIBRARIES
-# and we need to insert after that library but before the hdf5 library, so
-# position 1 is used
-list(INSERT NETCDF_LIBRARIES 1 ${HDF5_C_LIBRARY_hdf5_hl})
@@ -47,7 +47,6 @@ endfunction()
 macro(mfem_add_executable NAME)
  if (MFEM_USE_HIP)
    hip_add_executable(${NAME} ${ARGN})
-    set_target_properties(${NAME} PROPERTIES LINKER_LANGUAGE CXX)
  else()
    add_executable(${NAME} ${ARGN})
  endif()
@@ -759,7 +758,7 @@ function(mfem_export_mk_files)
  set(CONFIG_MK_BOOL_VARS MFEM_USE_MPI MFEM_USE_METIS MFEM_USE_METIS_5
      MFEM_DEBUG MFEM_USE_EXCEPTIONS MFEM_USE_ZLIB MFEM_USE_LIBUNWIND
      MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP MFEM_USE_LEGACY_OPENMP
-      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_EPIC MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
+      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX
      MFEM_USE_GNUTLS MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC
      MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_CONDUIT MFEM_USE_PUMI
@@ -85,9 +85,6 @@
 // Enable MFEM functionality based on the SUNDIALS libraries.
 // #define MFEM_USE_SUNDIALS

-// Enable MFEM functionality based on the EPIC libraries.
-// #define MFEM_USE_EPIC
-
 // Enable MFEM functionality based on the Mesquite library.
 // #define MFEM_USE_MESQUITE

@@ -120,9 +117,6 @@
 // Enable the use of SIMD in the high performance templated classes
 // #define MFEM_USE_SIMD

-// Enable FMS support
-// #define MFEM_USE_FMS
-
 // Enable Conduit support
 // #define MFEM_USE_CONDUIT

@@ -164,9 +158,6 @@
 // Enable functionality based on the libCEED library.
 // #define MFEM_USE_CEED

-// Enable functionality based on the Caliper library.
-// #define MFEM_USE_CALIPER
-
 // Enable functionality based on the Umpire library.
 // #define MFEM_USE_UMPIRE

@@ -29,7 +29,6 @@ MFEM_USE_OPENMP        = @MFEM_USE_OPENMP@
 MFEM_USE_MEMALLOC      = @MFEM_USE_MEMALLOC@
 MFEM_TIMER_TYPE        = @MFEM_TIMER_TYPE@
 MFEM_USE_SUNDIALS      = @MFEM_USE_SUNDIALS@
-MFEM_USE_EPIC          = @MFEM_USE_EPIC@
 MFEM_USE_MESQUITE      = @MFEM_USE_MESQUITE@
 MFEM_USE_SUITESPARSE   = @MFEM_USE_SUITESPARSE@
 MFEM_USE_SUPERLU       = @MFEM_USE_SUPERLU@
@@ -44,7 +43,6 @@ MFEM_USE_PETSC         = @MFEM_USE_PETSC@
 MFEM_USE_SLEPC         = @MFEM_USE_SLEPC@
 MFEM_USE_MPFR          = @MFEM_USE_MPFR@
 MFEM_USE_SIDRE         = @MFEM_USE_SIDRE@
-MFEM_USE_FMS           = @MFEM_USE_FMS@
 MFEM_USE_CONDUIT       = @MFEM_USE_CONDUIT@
 MFEM_USE_PUMI          = @MFEM_USE_PUMI@
 MFEM_USE_HIOP          = @MFEM_USE_HIOP@
@@ -54,7 +52,6 @@ MFEM_USE_HIP           = @MFEM_USE_HIP@
 MFEM_USE_RAJA          = @MFEM_USE_RAJA@
 MFEM_USE_OCCA          = @MFEM_USE_OCCA@
 MFEM_USE_CEED          = @MFEM_USE_CEED@
-MFEM_USE_CALIPER       = @MFEM_USE_CALIPER@
 MFEM_USE_UMPIRE        = @MFEM_USE_UMPIRE@
 MFEM_USE_SIMD          = @MFEM_USE_SIMD@
 MFEM_USE_ADIOS2        = @MFEM_USE_ADIOS2@
@@ -30,7 +30,6 @@ option(MFEM_USE_OPENMP "Enable the OpenMP backend" OFF)
 option(MFEM_USE_LEGACY_OPENMP "Enable legacy OpenMP usage" OFF)
 option(MFEM_USE_MEMALLOC "Enable the internal MEMALLOC option." ON)
 option(MFEM_USE_SUNDIALS "Enable SUNDIALS usage" OFF)
-option(MFEM_USE_EPIC "Enable EPIC usage" OFF)
 option(MFEM_USE_MESQUITE "Enable MESQUITE usage" OFF)
 option(MFEM_USE_SUITESPARSE "Enable SuiteSparse usage" OFF)
 option(MFEM_USE_SUPERLU "Enable SuperLU_DIST usage" OFF)
@@ -46,7 +45,6 @@ option(MFEM_USE_PETSC "Enable PETSc support." OFF)
 option(MFEM_USE_SLEPC "Enable SLEPc support." OFF)
 option(MFEM_USE_MPFR "Enable MPFR usage." OFF)
 option(MFEM_USE_SIDRE "Enable Axom/Sidre usage" OFF)
-option(MFEM_USE_FMS "Enable FMS usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
 option(MFEM_USE_PUMI "Enable PUMI" OFF)
 option(MFEM_USE_HIOP "Enable HiOp" OFF)
@@ -57,14 +55,8 @@ option(MFEM_USE_CEED "Enable CEED" OFF)
 option(MFEM_USE_UMPIRE "Enable Umpire" OFF)
 option(MFEM_USE_SIMD "Enable use of SIMD intrinsics" OFF)
 option(MFEM_USE_ADIOS2 "Enable ADIOS2" OFF)
-option(MFEM_USE_CALIPER "Enable Caliper support" OFF)
 option(MFEM_USE_MKL_CPARDISO "Enable MKL CPardiso" OFF)

-# Optional overrides for autodetected MPIEXEC and MPIEXEC_NUMPROC_FLAG
-# set(MFEM_MPIEXEC "mpirun" CACHE STRING "Command for running MPI tests")
-# set(MFEM_MPIEXEC_NP "-np" CACHE STRING
-#     "Flag for setting the number of MPI tasks")
-
 set(MFEM_MPI_NP 4 CACHE STRING "Number of processes used for MPI tests")

 # Allow a user to disable testing, examples, and/or miniapps at CONFIGURE TIME
@@ -98,11 +90,6 @@ set(HYPRE_DIR "${MFEM_DIR}/../hypre/src/hypre" CACHE PATH
 # If hypre was compiled to depend on BLAS and LAPACK:
 # set(HYPRE_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
 #     "Packages that HYPRE depends on.")
-if (MFEM_USE_CUDA)
-   # This is only necessary when hypre is built with cuda:
-   set(HYPRE_REQUIRED_LIBRARIES "-lcusparse" "-lcurand" CACHE STRING
-       "Libraries that HYPRE depends on.")
-endif()

 set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library.")

@@ -116,9 +103,6 @@ set(SUNDIALS_DIR "${MFEM_DIR}/../sundials-5.0.0/instdir" CACHE PATH
 # set(SUNDIALS_REQUIRED_PACKAGES "SuiteSparse/KLU/AMD/BTF/COLAMD/config"
 #     CACHE STRING "Additional packages required by SUNDIALS.")

-set(EPIC_DIR "${MFEM_DIR}/../epic-cpp/instdir" CACHE PATH
-    "Path to the EPIC library.")
-
 set(MESQUITE_DIR "${MFEM_DIR}/../mesquite-2.99" CACHE PATH
    "Path to the Mesquite library.")

@@ -142,10 +126,10 @@ set(MUMPS_DIR "${MFEM_DIR}/../MUMPS_5.2.0" CACHE PATH
    "Path to the MUMPS library.")
 # Packages required by MUMPS, depending on how it was compiled.
 set(MUMPS_REQUIRED_PACKAGES "MPI" "BLAS" "METIS" "ScaLAPACK" CACHE STRING
-    "Additional packages required by MUMPS.")
+    "Additional packages required by MUMPS.")    
 # If the MPI package does not find all required Fortran libraries:
 # set(MUMPS_REQUIRED_LIBRARIES "gfortran" "mpi_mpifh" CACHE STRING
-#     "Additional libraries required by MUMPS.")
+#     "Additional libraries required by MUMPS.")    

 set(STRUMPACK_DIR "${MFEM_DIR}/../STRUMPACK-build" CACHE PATH
    "Path to the STRUMPACK library.")
@@ -184,7 +168,8 @@ set(GNUTLS_DIR "" CACHE PATH "Path to the GnuTLS library.")
 set(GSLIB_DIR "" CACHE PATH "Path to the GSLIB library.")

 set(NETCDF_DIR "" CACHE PATH "Path to the NetCDF library.")
-set(NetCDF_REQUIRED_PACKAGES "HDF5/C/HL" CACHE STRING
+# May need to add "HDF5" as requirement.
+set(NetCDF_REQUIRED_PACKAGES "" CACHE STRING
    "Additional packages required by NetCDF.")

 set(PETSC_DIR "${MFEM_DIR}/../petsc" CACHE PATH
@@ -197,12 +182,6 @@ set(SLEPC_ARCH "arch-linux2-c-debug" CACHE STRING "SLEPC build architecture.")

 set(MPFR_DIR "" CACHE PATH "Path to the MPFR library.")

-set(FMS_DIR "${MFEM_DIR}/../fms" CACHE PATH
-    "Path to the FMS library.")
-# If FMS is built with Conduit:
-# set(FMS_REQUIRED_PACKAGES "Conduit/relay" CACHE STRING
-#     "Additional packages required by FMS.")
-
 set(CONDUIT_DIR "${MFEM_DIR}/../conduit" CACHE PATH
    "Path to the Conduit library.")

@@ -227,7 +206,6 @@ set(OCCA_DIR "${MFEM_DIR}/../occa" CACHE PATH "Path to OCCA")
 set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")
 set(UMPIRE_DIR "${MFEM_DIR}/../umpire" CACHE PATH "Path to Umpire")
-set(CALIPER_DIR "${MFEM_DIR}/../caliper" CACHE PATH "Path to Caliper")

 set(BLAS_INCLUDE_DIRS "" CACHE STRING "Path to BLAS headers.")
 set(BLAS_LIBRARIES "" CACHE STRING "The BLAS library.")
@@ -18,9 +18,6 @@
 # Some choices below are based on the OS type:
 NOTMAC := $(subst Darwin,,$(shell uname -s))

-ETAGS_BIN = $(shell command -v etags 2> /dev/null)
-EGREP_BIN = $(shell command -v egrep 2> /dev/null)
-
 CXX = g++
 MPICXX = mpicxx

@@ -122,7 +119,6 @@ MFEM_USE_LEGACY_OPENMP = NO
 MFEM_USE_MEMALLOC      = YES
 MFEM_TIMER_TYPE        = $(if $(NOTMAC),2,4)
 MFEM_USE_SUNDIALS      = NO
-MFEM_USE_EPIC          = NO
 MFEM_USE_MESQUITE      = NO
 MFEM_USE_SUITESPARSE   = NO
 MFEM_USE_SUPERLU       = NO
@@ -137,7 +133,6 @@ MFEM_USE_PETSC         = NO
 MFEM_USE_SLEPC         = NO
 MFEM_USE_MPFR          = NO
 MFEM_USE_SIDRE         = NO
-MFEM_USE_FMS           = NO
 MFEM_USE_CONDUIT       = NO
 MFEM_USE_PUMI          = NO
 MFEM_USE_HIOP          = NO
@@ -147,7 +142,6 @@ MFEM_USE_HIP           = NO
 MFEM_USE_RAJA          = NO
 MFEM_USE_OCCA          = NO
 MFEM_USE_CEED          = NO
-MFEM_USE_CALIPER       = NO
 MFEM_USE_UMPIRE        = NO
 MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
@@ -176,10 +170,6 @@ LIBUNWIND_LIB = $(if $(NOTMAC),-lunwind -ldl,)
 HYPRE_DIR = @MFEM_DIR@/../hypre/src/hypre
 HYPRE_OPT = -I$(HYPRE_DIR)/include
 HYPRE_LIB = -L$(HYPRE_DIR)/lib -lHYPRE
-ifeq (YES,$(MFEM_USE_CUDA))
-   # This is only necessary when hypre is built with cuda:
-   HYPRE_LIB += -lcusparse -lcurand
-endif

 # METIS library configuration
 ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK)$(MFEM_USE_MUMPS),NONONO)
@@ -232,11 +222,6 @@ endif
 # If SUNDIALS was built with KLU:
 # MFEM_USE_SUITESPARSE = YES

-# EPIC library configuration
-MESQUITE_DIR = @MFEM_DIR@/../epic-cpp/instdir
-MESQUITE_OPT = -I$(EPIC_DIR)/include
-MESQUITE_LIB = -L$(EPIC_DIR)/lib -lepic1.0.0
-
 # MESQUITE library configuration
 MESQUITE_DIR = @MFEM_DIR@/../mesquite-2.99
 MESQUITE_OPT = -I$(MESQUITE_DIR)/include
@@ -299,23 +284,9 @@ STRUMPACK_LIB = -L$(STRUMPACK_DIR)/lib -lstrumpack $(MPI_FORTRAN_LIB)\

 # Ginkgo library configuration (currently not needed)
 GINKGO_DIR = @MFEM_DIR@/../ginkgo/install
-GINKGO_BUILD_TYPE=Release
-ifeq ($(MFEM_USE_GINKGO),YES)
-   BASE_FLAGS = -std=c++14
-endif
 GINKGO_OPT = -isystem $(GINKGO_DIR)/include
-GINKGO_LIB_DIR = $(sort $(dir $(wildcard $(GINKGO_DIR)/lib*/libginkgo*.a  $(GINKGO_DIR)/lib*/libginkgo*.so $(GINKGO_DIR)/lib*/libginkgo*.dylib $(GINKGO_DIR)/lib*/libginkgo*.dll)))
-ALL_GINKGO_LIBS_DEBUG = $(notdir $(basename $(wildcard $(GINKGO_DIR)/lib*/libginkgo*d.a  $(GINKGO_DIR)/lib*/libginkgo*d.so $(GINKGO_DIR)/lib*/libginkgo*d.dylib $(GINKGO_DIR)/lib*/libginkgo*d.dll)))
-ALL_GINKGO_LIBS = $(notdir $(basename $(wildcard $(GINKGO_DIR)/lib*/libginkgo*.a  $(GINKGO_DIR)/lib*/libginkgo*.so $(GINKGO_DIR)/lib*/libginkgo*.dylib $(GINKGO_DIR)/lib*/libginkgo*.dll)))
-ALL_GINKGO_LIBS_RELEASE = $(filter-out $(ALL_GINKGO_LIBS_DEBUG),$(ALL_GINKGO_LIBS))
-GINKGO_LINK = $(subst libginkgo,-lginkgo,$(ALL_GINKGO_LIBS_RELEASE))
-ifeq ($(GINKGO_BUILD_TYPE),Debug)
-  ifneq (,$(ALL_GINKGO_LIBS_DEBUG))
-    GINKGO_LINK = $(subst libginkgo,-lginkgo,$(ALL_GINKGO_LIBS_DEBUG))
-  endif
-else
-endif
-GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_LIB_DIR) -L$(GINKGO_LIB_DIR) $(GINKGO_LINK)
+GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_DIR)/lib -L$(GINKGO_DIR)/lib -lginkgo\
+ -lginkgo_omp -lginkgo_cuda -lginkgo_reference

 # AmgX library configuration
 AMGX_DIR = @MFEM_DIR@/../amgx
@@ -368,11 +339,6 @@ endif
 MPFR_OPT =
 MPFR_LIB = -lmpfr

-# FMS and required libraries configuration
-FMS_DIR = $(MFEM_DIR)/../fms
-FMS_OPT = -I$(FMS_DIR)/include
-FMS_LIB = -Wl,-rpath,$(FMS_DIR)/lib -L$(FMS_DIR)/lib -lfms
-
 # Conduit and required libraries configuration
 CONDUIT_DIR = @MFEM_DIR@/../conduit
 CONDUIT_OPT = -I$(CONDUIT_DIR)/include/conduit
@@ -430,11 +396,6 @@ OCCA_DIR = @MFEM_DIR@/../occa
 OCCA_OPT = -I$(OCCA_DIR)/include
 OCCA_LIB = $(XLINKER)-rpath,$(OCCA_DIR)/lib -L$(OCCA_DIR)/lib -locca

-# CALIPER library configuration
-CALIPER_DIR = @MFEM_DIR@/../caliper
-CALIPER_OPT = -I$(CALIPER_DIR)/include
-CALIPER_LIB = $(XLINKER)-rpath,$(CALIPER_DIR)/lib64 -L$(CALIPER_DIR)/lib64 -lcaliper
-
 # libCEED library configuration
 CEED_DIR ?= @MFEM_DIR@/../libCEED
 CEED_OPT = -I$(CEED_DIR)/include
@@ -446,14 +407,11 @@ RAJA_OPT = -I$(RAJA_DIR)/include
 ifdef CUB_DIR
   RAJA_OPT += -I$(CUB_DIR)
 endif
-ifdef CAMP_DIR
-   RAJA_OPT += -I$(CAMP_DIR)/include
-endif
 RAJA_LIB = $(XLINKER)-rpath,$(RAJA_DIR)/lib -L$(RAJA_DIR)/lib -lRAJA

 # UMPIRE library configuration
 UMPIRE_DIR = @MFEM_DIR@/../umpire
-UMPIRE_OPT = -I$(UMPIRE_DIR)/include $(if $(CAMP_DIR), -I$(CAMP_DIR)/include)
+UMPIRE_OPT = -I$(UMPIRE_DIR)/include
 UMPIRE_LIB = -L$(UMPIRE_DIR)/lib -lumpire

 # MKL CPardiso library configuration
@@ -1,41 +0,0 @@
-                    Finite Element Discretization Library
-                                   __
-                       _ __ ___   / _|  ___  _ __ ___
-                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
-                      | | | | | ||  _||  __/| | | | | |
-                      |_| |_| |_||_|   \___||_| |_| |_|
-
-                               https://mfem.org
-
-
-This directory contains recommended git hooks, which are scripts that can be
-used to improve your development experience with MFEM:
-
-### The hooks
-
-* `pre-commit` is a hook that will be applied before each commit and run
-`astyle` on the code. This will ensure that your changes comply with the MFEM
-code styling guidelines.
-
-* `pre-push` is a hook that will be applied before each push to run a quick set
-of tests that verify that your files headers are in compliance, and that you did
-not add any large files to the repo.
-
-### Setup
-
-To setup the git hooks, run `make hooks`, which creates symlinks to the hooks in
-the `.git/hooks` directory. Individual hooks can be enabled by manually creating
-symlinks.
-
-(You may also copy the scripts directly and customize them further, but this way
-you may miss additional updates in the future.)
-
-### Failures
-
-The `branch-history` check can fail in some cases when the history is OK. For
-example, when a large number of files were modified for a legitimate reason, or
-when a picture was added for documentation.
-
-If that is the case, make sure the failure is indeed justified, and rerun the
-push command with the `--no-verify` option. This will skip the hooks, allowing
-you to push those changes.
@@ -1,4 +0,0 @@
-#!/bin/sh
-
-# Apply automated code formatting
-make -C $(git rev-parse --show-toplevel) style
@@ -1,107 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-option=${1:-""}
-
-if [[ "${option}" == "--help" ]]; then
-  echo "This script runs checks on the repository."
-  echo "It has 2 modes: with and without an option."
-  echo ""
-  echo "Options are used in GitHub Actions and can be:"
-  echo "  --copyright"
-  echo "  --license"
-  echo "  --release"
-  echo "  --style"
-  echo "  --history"
-  echo ""
-  echo "As a githook, the script is used without options."
-  echo "In that case, it will run all the checks except style."
-  echo ""
-  echo "Use --help to print this help message."
-fi
-
-cd $(git rev-parse --show-toplevel)
-
-# copyright check
-copyright=true
-if [[ "${option}" == "--copyright" || "${option}" == "" ]]; then
-  if git grep -l "^\(#\|//\).*\(\-2020\|\ 2010,\)" > matches.txt; then
-    echo "Please update the following files to Copyright (c) 2010-2021:"
-    cat matches.txt
-    copyright=false
-  fi
-fi
-
-# license check
-license=true
-if [[ "${option}" == "--license" || "${option}" == "" ]]; then
-  if git grep -li "^\(#\|//\).*GNU\ Lesser\ General\ Public\ License" > matches.txt; then
-    echo "Please update the following files to the BSD-3 license:"
-    cat matches.txt
-    license=false
-  fi
-fi
-
-# release check
-release=true
-if [[ "${option}" == "--release" || "${option}" == "" ]]; then
-  if git grep -l "^\(#\|//\).*LLNL\-CODE\-443211" > matches.txt
-  then
-    echo "Please update the following files to LLNL-CODE-806117:"
-    cat matches.txt
-    release=false
-  fi
-fi
-
-# wrap-up
-code=0
-if ! $copyright ; then
-  echo "copyright check failed, unroll log for details"
-  code=1
-fi
-if ! $license ; then
-  echo "license check failed, unroll log for details"
-  code=1
-fi
-if ! $release ; then
-  echo "release check failed, unroll log for details"
-  code=1
-fi
-
-# `code-style` is not just a check, it will actually reformat the code if
-# necessary. This means that if one pushes while the repo is in dirty state
-# (changes not staged), those changes may be mixed with format changes.
-# To activate this, you will need to hard-copy this hook script in the hook
-# directory and uncomment only then. (See README.md)
-#
-## style check
-#if [[ "${option}" == "--style" || "${option}" == "" ]]; then
-if [[ "${option}" == "--style" ]]; then
-  if which astyle && [[ "$(astyle --version)" == "Artistic Style Version 2.05.1" ]]; then
-    cd tests/scripts
-    if ! ./runtest code-style; then code=1; fi
-    cd -
-  else
-    echo "Warning: astyle not found or version is not 2.05.1"
-  fi
-fi
-
-# branch-history
-if [[ "${option}" == "--history" || "${option}" == "" ]]; then
-  git fetch origin master:master
-  cd tests/scripts
-  if ! ./runtest branch-history; then code=1; fi
-  cd -
-fi
-
-exit $code
@@ -42,52 +42,12 @@ groups_serial=(
   "Performance miniapps:"
   "miniapps/performance"
   "ex1.cpp"'
-'"amgx"
-   "AmgX examples:"
-   "examples/amgx"
-   "ex1.cpp"'
-'"caliper"
-   "Caliper examples:"
-   "examples/caliper"
-   "ex1.cpp"'
-'"ginkgo"
-   "Ginkgo examples:"
-   "examples/ginkgo"
-   "ex1.cpp"'
-'"hiop"
-   "HiOp examples:"
-   "examples/hiop"
-   "ex9.cpp"'
-'"pumi"
-   "PUMI examples:"
-   "examples/pumi"
-   "ex1.cpp ex2.cpp"'
 #   ""'
 '"meshing"
   "Meshing miniapps:"
   "miniapps/meshing"
   "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp
    mesh-optimizer.cpp minimal-surface.cpp"'
-'"adjoint"
-   "Adjoint miniapps:"
-   "miniapps/adjoint"
-   "cvsRoberts_ASAi_dns.cpp"'
-'"gslib"
-   "GSLIB miniapps:"
-   "miniapps/gslib"
-   "field-diff.cpp field-interp.cpp findpts.cpp schwarz_ex1.cpp "'
-'"nurbs"
-   "NURBS miniapps:"
-   "miniapps/nurbs"
-   "nurbs_ex1.cpp"'
-'"tools"
-   "Tools miniapps:"
-   "miniapps/tools"
-   "convert-dc.cpp display-basis.cpp get-values.cpp load-dc.cpp lor-transfer.cpp"'
-'"toys"
-   "Toys miniapps:"
-   "miniapps/toys"
-   "automata.cpp life.cpp lissajous.cpp mandel.cpp mondrian.cpp rubik.cpp snake.cpp"'
 '"convergence"
   "Convergence tests:"
   "tests/convergence"
@@ -112,26 +72,6 @@ groups_parallel=(
   "Performance miniapps:"
   "miniapps/performance"
   "ex1p.cpp"'
-'"amgx"
-   "AmgX examples:"
-   "examples/amgx"
-   "ex1p.cpp"'
-'"caliper"
-   "Caliper examples:"
-   "examples/caliper"
-   "ex1p.cpp"'
-'"hiop"
-   "HiOp examples:"
-   "examples/hiop"
-   "ex9p.cpp"'
-'"pumi"
-   "PUMI examples:"
-   "examples/pumi"
-   "ex1p.cpp ex6p.cpp"'
-'"superlu"
-   "Superlu examples:"
-   "examples/superlu"
-   "ex1p.cpp"'
 #   ""'
 '"meshing"
   "Meshing miniapps:"
@@ -142,34 +82,6 @@ groups_parallel=(
   "miniapps/electromagnetics"
   "joule.cpp"'
 #   "{volta,tesla,joule}.cpp"' # todo: multiline sample runs
-'"adjoint"
-   "Adjoint miniapps:"
-   "miniapps/adjoint"
-   "adjoint_advection_diffusion.cpp"'
-'"gslib"
-   "GSLIB miniapps:"
-   "miniapps/gslib"
-   "pfindpts.cpp schwarz_ex1p.cpp"'
-'"navier"
-   "Navier miniapps:"
-   "miniapps/navier"
-   "navier_cht.cpp"'
-'"nurbs"
-   "NURBS miniapps:"
-   "miniapps/nurbs"
-   "nurbs_ex1p.cpp nurbs_ex11p.cpp"'
-'"shifted"
-   "Shifted miniapps:"
-   "miniapps/shifted"
-   "distance.cpp"'
-'"solvers"
-   "Solvers miniapps:"
-   "miniapps/solvers"
-   "block-solvers.cpp"'
-'"tools"
-   "Tools miniapps:"
-   "miniapps/tools"
-   "convert-cd.cpp get-values.cpp load-dc.cpp"'
 '"convergence"
   "Convergence tests:"
   "tests/convergence"
@@ -197,30 +109,6 @@ groups_all=(
   "Performance miniapps:"
   "miniapps/performance"
   "ex1{,p}.cpp"'
-'"amgx"
-   "AmgX examples:"
-   "examples/amgx"
-   "ex1.cpp ex1p.cpp"'
-'"caliper"
-   "Caliper examples:"
-   "examples/caliper"
-   "ex1.cpp ex1p.cpp"'
-'"ginkgo"
-   "Ginkgo examples:"
-   "examples/ginkgo"
-   "ex1.cpp"'
-'"hiop"
-   "HiOp examples:"
-   "examples/hiop"
-   "ex9.cpp ex9p.cpp"'
-'"pumi"
-   "PUMI examples:"
-   "examples/pumi"
-   "ex1.cpp ex1p.cpp ex2.cpp ex6p.cpp"'
-'"superlu"
-   "Superlu examples:"
-   "examples/superlu"
-   "ex1p.cpp"'
 '"meshing"
   "Meshing miniapps:"
   "miniapps/meshing"
@@ -231,38 +119,6 @@ groups_all=(
   "miniapps/electromagnetics"
   "joule.cpp"'
 #   "{volta,tesla,joule}.cpp"' # todo: multiline sample runs
-'"adjoint"
-   "Adjoint miniapps:"
-   "miniapps/adjoint"
-   "adjoint_advection_diffusion.cpp cvsRoberts_ASAi_dns.cpp"'
-'"gslib"
-   "GSLIB miniapps:"
-   "miniapps/gslib"
-   "field-diff.cpp field-interp.cpp findpts.cpp schwarz_ex1.cpp pfindpts.cpp schwarz_ex1p.cpp"'
-'"navier"
-   "Navier miniapps:"
-   "miniapps/navier"
-   "navier_cht.cpp"'
-'"nurbs"
-   "NURBS miniapps:"
-   "miniapps/nurbs"
-   "nurbs_ex1.cpp nurbs_ex1p.cpp nurbs_ex11p.cpp"'
-'"shifted"
-   "Shifted miniapps:"
-   "miniapps/shifted"
-   "distance.cpp"'
-'"solvers"
-   "Solvers miniapps:"
-   "miniapps/solvers"
-   "block-solvers.cpp"'
-'"tools"
-   "Tools miniapps:"
-   "miniapps/tools"
-   "convert-dc.cpp display-basis.cpp get-values.cpp load-dc.cpp lor-transfer.cpp"'
-'"toys"
-   "Toys miniapps:"
-   "miniapps/toys"
-   "automata.cpp life.cpp lissajous.cpp mandel.cpp mondrian.cpp rubik.cpp snake.cpp"'
 '"convergence"
   "Convergence tests:"
   "tests/convergence"
@@ -588,8 +444,6 @@ function go_group()
      mkdir -p "${group_output_dir}" || exit 1
   fi
   for src in "$@"; do
-      ex_run_suffix=${run_suffix} && [[ $src =~ ex0p?\.cpp ]] \
-         && ex_run_suffix=""
      cd "${mfem_dir}/${group_dir}" || exit 1
      extract_sample_runs "${src}" || continue
      [ "${#runs[@]}" -eq 0 ] && continue
@@ -609,7 +463,7 @@ function go_group()
      fi
      for run in "${runs[@]}"; do
         if [ "${run}" == "" ]; then continue; fi
-         eval go \"\${run_prefix} \${run} \${ex_run_suffix}\" $output
+         eval go \"\${run_prefix} \${run} \${run_suffix}\" $output
      done
   done
   ${make} clean-exec
@@ -39,7 +39,7 @@ set -- $$($(1) $(SHELL) -c "$(2)" 2>&1); while [ "$$#" -gt 3 ]; do shift; done
 endef
 define TIMECMD.NOTGNU
 set -- $$($(1) -l $(SHELL) -c "{ $(2); } > /dev/null 2>&1" 2>&1; echo $$?); \
-set -- "$$1"s "$$(($$7/1024))"kB "$${!#}"
+set -- "$$1"s "$$(($$7/1024))"kB "$${60}"
 endef
 define TIMECMD.BASH
 TIMEFORMAT=$$'%3Rs'; \
@@ -57,27 +57,22 @@ TIMECMD := $(word 1,$(TIMECMD))
 ifneq (,$(filter test%,$(MAKECMDGOALS)))
   MAKEFLAGS += -k
 endif
-# Test runs of the examples/miniapps with parameters - check exit code:
-# 0 means success, 255 means the test was skipped, anything else means error
+# Test runs of the examples/miniapps with parameters - check exit code
 mfem-test = \
   printf "   $(3) [$(2) $(1) ... ]: "; \
   $(call $(TIMEFUN),$(TIMECMD),$(2) ./$(1) $(if $(5),,-no-vis )$(4) \
     > $(1).stderr 2>&1); \
-   err="$$3"; \
-   if [ "$$3" = 0 ]; then $(PRINT_OK); \
-   else if [ "$$3" = 255 ]; then $(PRINT_SKIP); err=0; \
-   else $(PRINT_FAILED); cat $(1).stderr; fi; fi; \
-   rm -f $(1).stderr; exit $$err
+   if [ "$$3" = 0 ]; \
+   then $(PRINT_OK); else $(PRINT_FAILED); cat $(1).stderr; fi; \
+   rm -f $(1).stderr; exit $$3

 # Test runs of the examples/miniapps - check exit code and if a file exists
-# See mfem-test for the interpretation of the error code
 mfem-test-file = \
   printf "   $(3) [$(2) $(1) ... ]: "; \
   $(call $(TIMEFUN),$(TIMECMD),$(2) ./$(1) -no-vis > $(1).stderr 2>&1); \
   err="$$3"; \
-   if [ "$$3" = 0 ] && [ -e $(4) ]; then $(PRINT_OK); \
-   else if [ "$$3" = 255 ] && [ -e $(4) ]; then $(PRINT_SKIP); err=0; \
-   else $(PRINT_FAILED); cat $(1).stderr; err=64; fi; fi; \
+   if [ "$$3" = 0 ] && [ -e $(4) ]; \
+   then $(PRINT_OK); else $(PRINT_FAILED); cat $(1).stderr; err=64; fi; \
   rm -f $(1).stderr; exit $$err

 .PHONY: test test-par-YES test-par-NO test-ser test-par test-clean test-print
@@ -85,26 +80,6 @@ mfem-test-file = \
 # What sets of tests to run in serial and parallel
 test-par-YES: $(PAR_$(MFEM_TESTS):=-test-par) $(SEQ_$(MFEM_TESTS):=-test-seq)
 test-par-NO:  $(SEQ_$(MFEM_TESTS):=-test-seq)
-ifeq ($(MFEM_USE_CUDA),YES)
-.PHONY: test-par-YES-cuda test-par-NO-cuda test-ser-cuda test-par-cuda test-cuda
-test-par-YES: test-par-YES-cuda
-test-par-NO:  test-par-NO-cuda
-test-par-YES-cuda: test-par-cuda test-ser-cuda
-test-par-NO-cuda:  test-ser-cuda
-test-ser-cuda: $(SEQ_DEVICE_$(MFEM_TESTS):=-test-seq-cuda)
-test-par-cuda: $(PAR_DEVICE_$(MFEM_TESTS):=-test-par-cuda)
-test-cuda: test-par-$(MFEM_USE_MPI)-cuda clean-exec
-endif
-ifeq ($(MFEM_USE_HIP),YES)
-.PHONY: test-par-YES-hip test-par-NO-hip test-ser-hip test-par-hip test-hip
-test-par-YES: test-par-YES-hip
-test-par-NO:  test-par-NO-hip
-test-par-YES-hip: test-par-hip test-ser-hip
-test-par-NO-hip:  test-ser-hip
-test-ser-hip: $(SEQ_DEVICE_$(MFEM_TESTS):=-test-seq-hip)
-test-par-hip: $(PAR_DEVICE_$(MFEM_TESTS):=-test-par-hip)
-test-hip: test-par-$(MFEM_USE_MPI)-hip clean-exec
-endif
 test-ser:     test-par-NO
 test-par:     test-par-YES
 test:         all test-par-$(MFEM_USE_MPI) clean-exec
@@ -1,7 +1,6 @@
 MFEM NC mesh v1.0

 # NCMesh supported geometry types:
-# SEGMENT     = 1
 # TRIANGLE    = 2
 # SQUARE      = 3
 # TETRAHEDRON = 4
@@ -1,7 +1,6 @@
 MFEM NC mesh v1.0

 # NCMesh supported geometry types:
-# SEGMENT     = 1
 # TRIANGLE    = 2
 # SQUARE      = 3
 # TETRAHEDRON = 4
@@ -1,7 +1,6 @@
 MFEM NC mesh v1.0

 # NCMesh supported geometry types:
-# SEGMENT     = 1
 # TRIANGLE    = 2
 # SQUARE      = 3
 # TETRAHEDRON = 4
@@ -1,7 +1,6 @@
 MFEM NC mesh v1.0

 # NCMesh supported geometry types:
-# SEGMENT     = 1
 # TRIANGLE    = 2
 # SQUARE      = 3
 # TETRAHEDRON = 4
@@ -0,0 +1,7 @@
+MFEM INLINE mesh v1.0
+
+type = tri
+nx = 1
+ny = 1
+sx = 3.14
+sy = 3.14
@@ -1,38 +0,0 @@
-// 0 for tetrahedra, 1 for hexahedra
-tet_or_hex = 1;
-
-Point(1) = {0, 0, 0, 1.0};
-Point(2) = {1, 0, 0, 1.0};
-Point(3) = {1, 1, 0, 1.0};
-Point(4) = {0, 1, 0, 1.0};
-
-Characteristic Length {:} = 0.25;
-
-Line(1) = {1, 2};
-Line(2) = {2, 3};
-Line(3) = {3, 4};
-Line(4) = {4, 1};
-
-Periodic Curve {1} = {-3};
-Periodic Curve {2} = {-4};
-
-Curve Loop(1) = {1, 2, 3, 4};
-Plane Surface(1) = {1};
-Transfinite Surface {1};
-
-If (tet_or_hex == 1)
-   Recombine Surface {1};
-   out[] = Extrude {0, 0, 1} { Surface{1}; Layers{4}; Recombine; };
-Else
-   out[] = Extrude {0, 0, 1} { Surface{1}; Layers{4}; }
-EndIf
-
-Physical Volume(1) = {out[1]};
-Physical Surface(1) = {1,out[0],out[2],out[3],out[4],out[5]};
-
-Mesh 3;
-Mesh.MshFileVersion = 2.2;
-
-Periodic Surface {out[0]} = {1} Translate {0, 0, 1};
-Periodic Surface {out[4]} = {out[2]} Translate {0, 1, 0};
-Periodic Surface {out[3]} = {out[5]} Translate {1, 0, 0};
@@ -1,381 +0,0 @@
-$MeshFormat
-2.2 0 8
-$EndMeshFormat
-$Nodes
-125
-1 0 0 0
-2 1 0 0
-3 1 1 0
-4 0 1 0
-5 0 0 1
-6 1 0 1
-7 1 1 1
-8 0 1 1
-9 0.2500000000010404 0 0
-10 0.5000000000020591 0 0
-11 0.7500000000003465 0 0
-12 1 0.2500000000010404 0
-13 1 0.5000000000020591 0
-14 1 0.7500000000003465 0
-15 0.7500000000003465 1 0
-16 0.5000000000020591 1 0
-17 0.2500000000010404 1 0
-18 0 0.7500000000003465 0
-19 0 0.5000000000020591 0
-20 0 0.2500000000010404 0
-21 0.2500000000010404 0 1
-22 0.5000000000020591 0 1
-23 0.7500000000003465 0 1
-24 1 0.2500000000010404 1
-25 1 0.5000000000020591 1
-26 1 0.7500000000003465 1
-27 0.7500000000003465 1 1
-28 0.5000000000020591 1 1
-29 0.2500000000010404 1 1
-30 0 0.7500000000003465 1
-31 0 0.5000000000020591 1
-32 0 0.2500000000010404 1
-33 0 0 0.25
-34 0 0 0.5
-35 0 0 0.75
-36 1 0 0.25
-37 1 0 0.5
-38 1 0 0.75
-39 1 1 0.25
-40 1 1 0.5
-41 1 1 0.75
-42 0 1 0.25
-43 0 1 0.5
-44 0 1 0.75
-45 0.2500000000010404 0.2500000000010404 0
-46 0.2500000000010404 0.5000000000020591 0
-47 0.2500000000010404 0.7500000000003464 0
-48 0.5000000000020591 0.2500000000010404 0
-49 0.5000000000020591 0.5000000000020591 0
-50 0.5000000000020591 0.7500000000003465 0
-51 0.7500000000003464 0.2500000000010404 0
-52 0.7500000000003467 0.5000000000020591 0
-53 0.7500000000003464 0.7500000000003466 0
-54 0.2500000000010404 0 0.25
-55 0.2500000000010404 0 0.5
-56 0.2500000000010404 0 0.75
-57 0.5000000000020591 0 0.25
-58 0.5000000000020591 0 0.5
-59 0.5000000000020591 0 0.75
-60 0.7500000000003465 0 0.25
-61 0.7500000000003465 0 0.5
-62 0.7500000000003465 0 0.75
-63 1 0.2500000000010404 0.25
-64 1 0.2500000000010404 0.5
-65 1 0.2500000000010404 0.75
-66 1 0.5000000000020591 0.25
-67 1 0.5000000000020591 0.5
-68 1 0.5000000000020591 0.75
-69 1 0.7500000000003465 0.25
-70 1 0.7500000000003465 0.5
-71 1 0.7500000000003465 0.75
-72 0.7500000000003465 1 0.25
-73 0.7500000000003465 1 0.5
-74 0.7500000000003465 1 0.75
-75 0.5000000000020591 1 0.25
-76 0.5000000000020591 1 0.5
-77 0.5000000000020591 1 0.75
-78 0.2500000000010404 1 0.25
-79 0.2500000000010404 1 0.5
-80 0.2500000000010404 1 0.75
-81 0 0.7500000000003465 0.25
-82 0 0.7500000000003465 0.5
-83 0 0.7500000000003465 0.75
-84 0 0.5000000000020591 0.25
-85 0 0.5000000000020591 0.5
-86 0 0.5000000000020591 0.75
-87 0 0.2500000000010404 0.25
-88 0 0.2500000000010404 0.5
-89 0 0.2500000000010404 0.75
-90 0.2500000000010404 0.2500000000010404 1
-91 0.2500000000010404 0.5000000000020591 1
-92 0.2500000000010404 0.7500000000003464 1
-93 0.5000000000020591 0.2500000000010404 1
-94 0.5000000000020591 0.5000000000020591 1
-95 0.5000000000020591 0.7500000000003465 1
-96 0.7500000000003464 0.2500000000010404 1
-97 0.7500000000003467 0.5000000000020591 1
-98 0.7500000000003464 0.7500000000003466 1
-99 0.2500000000010404 0.2500000000010404 0.25
-100 0.2500000000010404 0.2500000000010404 0.5
-101 0.2500000000010404 0.2500000000010404 0.75
-102 0.2500000000010404 0.5000000000020591 0.25
-103 0.2500000000010404 0.5000000000020591 0.5
-104 0.2500000000010404 0.5000000000020591 0.75
-105 0.2500000000010404 0.7500000000003464 0.25
-106 0.2500000000010404 0.7500000000003464 0.5
-107 0.2500000000010404 0.7500000000003464 0.75
-108 0.5000000000020591 0.2500000000010404 0.25
-109 0.5000000000020591 0.2500000000010404 0.5
-110 0.5000000000020591 0.2500000000010404 0.75
-111 0.5000000000020591 0.5000000000020591 0.25
-112 0.5000000000020591 0.5000000000020591 0.5
-113 0.5000000000020591 0.5000000000020591 0.75
-114 0.5000000000020591 0.7500000000003465 0.25
-115 0.5000000000020591 0.7500000000003465 0.5
-116 0.5000000000020591 0.7500000000003465 0.75
-117 0.7500000000003464 0.2500000000010404 0.25
-118 0.7500000000003464 0.2500000000010404 0.5
-119 0.7500000000003464 0.2500000000010404 0.75
-120 0.7500000000003467 0.5000000000020591 0.25
-121 0.7500000000003467 0.5000000000020591 0.5
-122 0.7500000000003467 0.5000000000020591 0.75
-123 0.7500000000003464 0.7500000000003466 0.25
-124 0.7500000000003464 0.7500000000003466 0.5
-125 0.7500000000003464 0.7500000000003466 0.75
-$EndNodes
-$Elements
-160
-1 3 2 1 1 1 9 45 20
-2 3 2 1 1 20 45 46 19
-3 3 2 1 1 19 46 47 18
-4 3 2 1 1 18 47 17 4
-5 3 2 1 1 9 10 48 45
-6 3 2 1 1 45 48 49 46
-7 3 2 1 1 46 49 50 47
-8 3 2 1 1 47 50 16 17
-9 3 2 1 1 10 11 51 48
-10 3 2 1 1 48 51 52 49
-11 3 2 1 1 49 52 53 50
-12 3 2 1 1 50 53 15 16
-13 3 2 1 1 11 2 12 51
-14 3 2 1 1 51 12 13 52
-15 3 2 1 1 52 13 14 53
-16 3 2 1 1 53 14 3 15
-17 3 2 1 13 1 9 54 33
-18 3 2 1 13 33 54 55 34
-19 3 2 1 13 34 55 56 35
-20 3 2 1 13 35 56 21 5
-21 3 2 1 13 9 10 57 54
-22 3 2 1 13 54 57 58 55
-23 3 2 1 13 55 58 59 56
-24 3 2 1 13 56 59 22 21
-25 3 2 1 13 10 11 60 57
-26 3 2 1 13 57 60 61 58
-27 3 2 1 13 58 61 62 59
-28 3 2 1 13 59 62 23 22
-29 3 2 1 13 11 2 36 60
-30 3 2 1 13 60 36 37 61
-31 3 2 1 13 61 37 38 62
-32 3 2 1 13 62 38 6 23
-33 3 2 1 17 2 12 63 36
-34 3 2 1 17 36 63 64 37
-35 3 2 1 17 37 64 65 38
-36 3 2 1 17 38 65 24 6
-37 3 2 1 17 12 13 66 63
-38 3 2 1 17 63 66 67 64
-39 3 2 1 17 64 67 68 65
-40 3 2 1 17 65 68 25 24
-41 3 2 1 17 13 14 69 66
-42 3 2 1 17 66 69 70 67
-43 3 2 1 17 67 70 71 68
-44 3 2 1 17 68 71 26 25
-45 3 2 1 17 14 3 39 69
-46 3 2 1 17 69 39 40 70
-47 3 2 1 17 70 40 41 71
-48 3 2 1 17 71 41 7 26
-49 3 2 1 21 3 15 72 39
-50 3 2 1 21 39 72 73 40
-51 3 2 1 21 40 73 74 41
-52 3 2 1 21 41 74 27 7
-53 3 2 1 21 15 16 75 72
-54 3 2 1 21 72 75 76 73
-55 3 2 1 21 73 76 77 74
-56 3 2 1 21 74 77 28 27
-57 3 2 1 21 16 17 78 75
-58 3 2 1 21 75 78 79 76
-59 3 2 1 21 76 79 80 77
-60 3 2 1 21 77 80 29 28
-61 3 2 1 21 17 4 42 78
-62 3 2 1 21 78 42 43 79
-63 3 2 1 21 79 43 44 80
-64 3 2 1 21 80 44 8 29
-65 3 2 1 25 4 18 81 42
-66 3 2 1 25 42 81 82 43
-67 3 2 1 25 43 82 83 44
-68 3 2 1 25 44 83 30 8
-69 3 2 1 25 18 19 84 81
-70 3 2 1 25 81 84 85 82
-71 3 2 1 25 82 85 86 83
-72 3 2 1 25 83 86 31 30
-73 3 2 1 25 19 20 87 84
-74 3 2 1 25 84 87 88 85
-75 3 2 1 25 85 88 89 86
-76 3 2 1 25 86 89 32 31
-77 3 2 1 25 20 1 33 87
-78 3 2 1 25 87 33 34 88
-79 3 2 1 25 88 34 35 89
-80 3 2 1 25 89 35 5 32
-81 3 2 1 26 5 21 90 32
-82 3 2 1 26 32 90 91 31
-83 3 2 1 26 31 91 92 30
-84 3 2 1 26 30 92 29 8
-85 3 2 1 26 21 22 93 90
-86 3 2 1 26 90 93 94 91
-87 3 2 1 26 91 94 95 92
-88 3 2 1 26 92 95 28 29
-89 3 2 1 26 22 23 96 93
-90 3 2 1 26 93 96 97 94
-91 3 2 1 26 94 97 98 95
-92 3 2 1 26 95 98 27 28
-93 3 2 1 26 23 6 24 96
-94 3 2 1 26 96 24 25 97
-95 3 2 1 26 97 25 26 98
-96 3 2 1 26 98 26 7 27
-97 5 2 1 1 1 9 45 20 33 54 99 87
-98 5 2 1 1 33 54 99 87 34 55 100 88
-99 5 2 1 1 34 55 100 88 35 56 101 89
-100 5 2 1 1 35 56 101 89 5 21 90 32
-101 5 2 1 1 20 45 46 19 87 99 102 84
-102 5 2 1 1 87 99 102 84 88 100 103 85
-103 5 2 1 1 88 100 103 85 89 101 104 86
-104 5 2 1 1 89 101 104 86 32 90 91 31
-105 5 2 1 1 19 46 47 18 84 102 105 81
-106 5 2 1 1 84 102 105 81 85 103 106 82
-107 5 2 1 1 85 103 106 82 86 104 107 83
-108 5 2 1 1 86 104 107 83 31 91 92 30
-109 5 2 1 1 18 47 17 4 81 105 78 42
-110 5 2 1 1 81 105 78 42 82 106 79 43
-111 5 2 1 1 82 106 79 43 83 107 80 44
-112 5 2 1 1 83 107 80 44 30 92 29 8
-113 5 2 1 1 9 10 48 45 54 57 108 99
-114 5 2 1 1 54 57 108 99 55 58 109 100
-115 5 2 1 1 55 58 109 100 56 59 110 101
-116 5 2 1 1 56 59 110 101 21 22 93 90
-117 5 2 1 1 45 48 49 46 99 108 111 102
-118 5 2 1 1 99 108 111 102 100 109 112 103
-119 5 2 1 1 100 109 112 103 101 110 113 104
-120 5 2 1 1 101 110 113 104 90 93 94 91
-121 5 2 1 1 46 49 50 47 102 111 114 105
-122 5 2 1 1 102 111 114 105 103 112 115 106
-123 5 2 1 1 103 112 115 106 104 113 116 107
-124 5 2 1 1 104 113 116 107 91 94 95 92
-125 5 2 1 1 47 50 16 17 105 114 75 78
-126 5 2 1 1 105 114 75 78 106 115 76 79
-127 5 2 1 1 106 115 76 79 107 116 77 80
-128 5 2 1 1 107 116 77 80 92 95 28 29
-129 5 2 1 1 10 11 51 48 57 60 117 108
-130 5 2 1 1 57 60 117 108 58 61 118 109
-131 5 2 1 1 58 61 118 109 59 62 119 110
-132 5 2 1 1 59 62 119 110 22 23 96 93
-133 5 2 1 1 48 51 52 49 108 117 120 111
-134 5 2 1 1 108 117 120 111 109 118 121 112
-135 5 2 1 1 109 118 121 112 110 119 122 113
-136 5 2 1 1 110 119 122 113 93 96 97 94
-137 5 2 1 1 49 52 53 50 111 120 123 114
-138 5 2 1 1 111 120 123 114 112 121 124 115
-139 5 2 1 1 112 121 124 115 113 122 125 116
-140 5 2 1 1 113 122 125 116 94 97 98 95
-141 5 2 1 1 50 53 15 16 114 123 72 75
-142 5 2 1 1 114 123 72 75 115 124 73 76
-143 5 2 1 1 115 124 73 76 116 125 74 77
-144 5 2 1 1 116 125 74 77 95 98 27 28
-145 5 2 1 1 11 2 12 51 60 36 63 117
-146 5 2 1 1 60 36 63 117 61 37 64 118
-147 5 2 1 1 61 37 64 118 62 38 65 119
-148 5 2 1 1 62 38 65 119 23 6 24 96
-149 5 2 1 1 51 12 13 52 117 63 66 120
-150 5 2 1 1 117 63 66 120 118 64 67 121
-151 5 2 1 1 118 64 67 121 119 65 68 122
-152 5 2 1 1 119 65 68 122 96 24 25 97
-153 5 2 1 1 52 13 14 53 120 66 69 123
-154 5 2 1 1 120 66 69 123 121 67 70 124
-155 5 2 1 1 121 67 70 124 122 68 71 125
-156 5 2 1 1 122 68 71 125 97 25 26 98
-157 5 2 1 1 53 14 3 15 123 69 39 72
-158 5 2 1 1 123 69 39 72 124 70 40 73
-159 5 2 1 1 124 70 40 73 125 71 41 74
-160 5 2 1 1 125 71 41 74 98 26 7 27
-$EndElements
-$Periodic
-3
-2 17 25
-Affine 1 0 0 1 0 1 0 0 0 0 1 0 0 0 0 1
-25
-2 1
-3 4
-6 5
-7 8
-63 87
-64 88
-65 89
-66 84
-67 85
-68 86
-69 81
-70 82
-71 83
-14 18
-24 32
-25 31
-26 30
-36 33
-37 34
-38 35
-39 42
-40 43
-41 44
-12 20
-13 19
-2 21 13
-Affine 1 0 0 0 0 1 0 1 0 0 1 0 0 0 0 1
-25
-3 2
-4 1
-7 6
-8 5
-15 11
-16 10
-17 9
-72 60
-73 61
-74 62
-75 57
-76 58
-77 59
-78 54
-79 55
-80 56
-27 23
-28 22
-29 21
-39 36
-40 37
-41 38
-42 33
-43 34
-44 35
-2 26 1
-Affine 1 0 0 0 0 1 0 0 0 0 1 1 0 0 0 1
-25
-5 1
-6 2
-7 3
-8 4
-90 45
-91 46
-92 47
-93 48
-94 49
-95 50
-96 51
-97 52
-98 53
-21 9
-22 10
-23 11
-24 12
-25 13
-26 14
-27 15
-28 16
-29 17
-30 18
-31 19
-32 20
-$EndPeriodic
@@ -1,31 +0,0 @@
-// 0 for triangles, 1 for quads
-tri_or_quad = 1;
-
-Point(1) = {0, 0, 0, 1.0};
-Point(2) = {1, 0, 0, 1.0};
-Point(3) = {1, 1, 0, 1.0};
-Point(4) = {0, 1, 0, 1.0};
-
-Characteristic Length {:} = 0.25;
-
-Line(1) = {1, 2};
-Line(2) = {2, 3};
-Line(3) = {3, 4};
-Line(4) = {4, 1};
-
-Periodic Line {3} = {-1};
-Periodic Line {2} = {-4};
-
-Curve Loop(1) = {1, 2, 3, 4};
-Plane Surface(1) = {1};
-Transfinite Surface {1};
-
-If (tri_or_quad == 1)
-   Recombine Surface {1};
-EndIf
-
-Physical Surface(1) = {1};
-Physical Curve(1) = {1, 2, 3, 4};
-
-Mesh.MshFileVersion = 2.2;
-Mesh 2;
@@ -1,83 +0,0 @@
-$MeshFormat
-2.2 0 8
-$EndMeshFormat
-$Nodes
-25
-1 0 0 0
-2 1 0 0
-3 1 1 0
-4 0 1 0
-5 0.2499999999994121 0 0
-6 0.499999999998694 0 0
-7 0.7499999999993416 0 0
-8 1 0.2500000000010404 0
-9 1 0.5000000000020591 0
-10 1 0.7500000000003465 0
-11 0.7499999999993416 1 0
-12 0.4999999999986939 1 0
-13 0.249999999999412 1 0
-14 0 0.7500000000003465 0
-15 0 0.5000000000020591 0
-16 0 0.2500000000010404 0
-17 0.2499999999994121 0.2500000000010404 0
-18 0.249999999999412 0.5000000000020591 0
-19 0.249999999999412 0.7500000000003466 0
-20 0.4999999999986939 0.2500000000010404 0
-21 0.4999999999986939 0.5000000000020591 0
-22 0.4999999999986939 0.7500000000003466 0
-23 0.7499999999993416 0.2500000000010404 0
-24 0.7499999999993416 0.5000000000020591 0
-25 0.7499999999993416 0.7500000000003465 0
-$EndNodes
-$Elements
-32
-1 1 2 1 1 1 5
-2 1 2 1 1 5 6
-3 1 2 1 1 6 7
-4 1 2 1 1 7 2
-5 1 2 1 2 2 8
-6 1 2 1 2 8 9
-7 1 2 1 2 9 10
-8 1 2 1 2 10 3
-9 1 2 1 3 3 11
-10 1 2 1 3 11 12
-11 1 2 1 3 12 13
-12 1 2 1 3 13 4
-13 1 2 1 4 4 14
-14 1 2 1 4 14 15
-15 1 2 1 4 15 16
-16 1 2 1 4 16 1
-17 3 2 1 1 1 5 17 16
-18 3 2 1 1 16 17 18 15
-19 3 2 1 1 15 18 19 14
-20 3 2 1 1 14 19 13 4
-21 3 2 1 1 5 6 20 17
-22 3 2 1 1 17 20 21 18
-23 3 2 1 1 18 21 22 19
-24 3 2 1 1 19 22 12 13
-25 3 2 1 1 6 7 23 20
-26 3 2 1 1 20 23 24 21
-27 3 2 1 1 21 24 25 22
-28 3 2 1 1 22 25 11 12
-29 3 2 1 1 7 2 8 23
-30 3 2 1 1 23 8 9 24
-31 3 2 1 1 24 9 10 25
-32 3 2 1 1 25 10 3 11
-$EndElements
-$Periodic
-2
-1 2 4
-5
-8 16
-9 15
-10 14
-2 1
-3 4
-1 3 1
-5
-11 7
-12 6
-13 5
-3 2
-4 1
-$EndPeriodic
@@ -1,41 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-3
-
-elements
-1
-1 5 0 1 2 3 4 5 6 7
-
-boundary
-6
-1 3 3 2 1 0
-2 3 0 1 5 4
-3 3 1 2 6 5
-4 3 2 3 7 6
-5 3 3 0 4 7
-6 3 4 5 6 7
-
-vertices
-8
-3
-0 0 0
-1 0 0
-1 1 0
-0 1 0
-0 0 1
-1 0 1
-1 1 1
-0 1 1
@@ -1,38 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-3
-
-elements
-1
-1 6 0 1 2 3 4 5
-
-boundary
-5
-1 2 0 2 1
-2 2 3 4 5
-3 3 0 1 4 3
-4 3 1 2 5 4
-5 3 2 0 3 5
-
-vertices
-6
-3
-0 0 0
-1 0 0
-0 1 0
-0 0 1
-1 0 1
-0 1 1
@@ -1,31 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-1
-
-elements
-1
-1 1 0 1
-
-boundary
-2
-1 0 0
-2 0 1
-
-vertices
-2
-1
-0
-1
@@ -1,35 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-2
-
-elements
-1
-1 3 0 1 2 3
-
-boundary
-4
-1 1 0 1
-2 1 1 2
-3 1 2 3
-4 1 3 0
-
-vertices
-4
-2
-0 0
-1 0
-1 1
-0 1
@@ -1,35 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-3
-
-elements
-1
-1 4 0 1 2 3
-
-boundary
-4
-1 2 1 2 3
-2 2 0 3 2
-3 2 0 1 3
-4 2 0 2 1
-
-vertices
-4
-3
-0 0 0
-1 0 0
-0 1 0
-0 0 1
@@ -1,33 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-2
-
-elements
-1
-1 2 0 1 2
-
-boundary
-3
-1 1 0 1
-2 1 1 2
-3 1 2 0
-
-vertices
-3
-2
-0 0
-1 0
-0 1
@@ -1,246 +0,0 @@
-FMS: 100
-DataCollection/Name: star
-DataCollection/NumberOfFieldDescriptors: 1
-DataCollection/FieldDescriptors/0/Name: CoordsDescriptor
-DataCollection/FieldDescriptors/0/ComponentName: volume
-DataCollection/FieldDescriptors/0/Type: 0
-DataCollection/FieldDescriptors/0/FixedOrder/Size: 3
-DataCollection/FieldDescriptors/0/FixedOrder/Type: FMS_UINT64
-DataCollection/FieldDescriptors/0/FixedOrder/Values: [0, 1, 3]
-DataCollection/FieldDescriptors/0/NumDofs: 211
-DataCollection/NumberOfFields: 1
-DataCollection/Fields/0/Name: Coords
-DataCollection/Fields/0/LayoutType: 0
-DataCollection/Fields/0/NumberOfVectorComponents: 2
-DataCollection/Fields/0/FieldDescriptorName: CoordsDescriptor
-DataCollection/Fields/0/Data/Size: 422
-DataCollection/Fields/0/Data/Type: FMS_DOUBLE
-DataCollection/Fields/0/Data/Values: [-0.016886, 1.000000, 0.309017, 
-1.309020, -0.809017, -0.500000, 
-0.809017, -1.618030, 0.309017, 
-0.500000, 1.309020, 0.519420, 
-1.154510, 0.809019, 0.147680, 
-0.095492, -0.654508, -0.415586, 
-1.213520, -1.213520, -0.392210, 
-0.654508, -0.095492, 0.139949, 
-0.809019, 1.154510, 0.660184, 
-0.264063, -0.800064, -0.231060, 
-0.663691, 0.183114, 0.317639, 
-0.543082, 0.598483, 0.345112, 
-0.478298, 0.027703, 0.095229, 
-0.012368, -0.092534, -0.334412, 
-0.313767, -0.140526, -0.293881, 
-0.534056, -0.660290, -0.537646, 
-0.655590, -0.121396, -0.274504, 
-0.346497, -0.296570, 0.004737, 
-0.098835, 0.069287, 0.082675, 
-0.318799, 0.467183, 0.564505, 
-0.595190, 0.846237, 0.671735, 
-1.051500, 1.103010, 0.964008, 
-0.821603, 1.257520, 1.206010, 
-1.142350, 0.975686, 0.781273, 
-0.717257, 0.475684, 0.642352, 
-0.268930, 0.211049, 0.174181, 
-0.039345, -0.147746, -0.177481, 
-0.365164, -0.230328, -0.551503, 
-0.603005, -0.497587, -0.389864, 
-0.757514, -0.706011, -0.675487, 
-0.528946, -0.943851, -1.078690, 
-1.087600, -0.955467, -1.483190, 
-1.348360, -1.483190, -1.348360, 
-1.085930, -0.938010, -0.943851, 
-1.078690, -0.681476, -0.540944, 
-0.757514, -0.706011, -0.540614, 
-0.367058, -0.551503, -0.603005, 
-0.365164, -0.230328, -0.138552, 
-0.206896, 0.174181, 0.039345, 
-0.268468, 0.222269, 0.475684, 
-0.642352, 0.759791, 0.719381, 
-1.142350, 0.975686, 1.257520, 
-1.206010, 0.972837, 0.836119, 
-1.051500, 1.103010, 0.214572, 
-0.407449, 0.288323, 0.449827, 
-0.086700, -0.027358, -0.200560, 
-0.166595, -0.271802, -0.418426, 
-0.426131, -0.551441, -0.096117, 
-0.206969, -0.027946, -0.184969, 
-0.211136, 0.260131, 0.407172, 
-0.430781, 0.718277, 0.885068, 
-0.753103, 0.957692, 0.866273, 
-1.024530, 0.934099, 1.093820, 
-0.348422, 0.524463, 0.404903, 
-0.587376, 0.054525, 0.146431, 
-0.078026, -0.007795, -0.329488, 
-0.302967, -0.488115, -0.439332, 
-0.498515, -0.453527, -0.633059, 
-0.570251, -0.655787, -0.791132, 
-0.802013, -0.956872, -1.094720, 
-1.208560, -1.207720, -1.339910, 
-0.693371, -0.795404, -0.803594, 
-0.945068, -0.464668, -0.631721, 
-0.458968, -0.547876, -0.332878, 
-0.485696, -0.286385, -0.408481, 
-0.053021, -0.058200, 0.110846, 
-0.002086, 0.381384, 0.416784, 
-0.551133, 0.613261, 0.872474, 
-0.901208, 1.038300, 1.084660, 
-0.737459, 0.751250, 0.890002, 
-0.915210, 0.010915, 0.000000, 
-0.951057, 0.951057, 0.587785, 
-1.538840, -0.587785, 0.000000, 
-0.951057, -1.538840, -0.951057, 
-0.015847, 0.475529, 0.951057, 
-0.492248, 1.244950, 1.063310, 
-0.274399, 0.293893, -0.293892, 
-0.296404, -1.063310, -1.244950, 
-0.453865, -0.951057, -0.475529, 
-0.466620, 0.792932, -0.013913, 
-0.748783, -0.497528, 0.021382, 
-0.017158, 0.172591, 0.330125, 
-0.458568, 0.457971, 0.137740, 
-0.299049, 0.588394, 0.667324, 
-0.432341, 0.634346, 0.117322, 
-0.193603, 0.211702, 0.098278, 
-0.199438, -0.077304, -0.082243, 
-0.216296, -0.458634, -0.592374, 
-0.563926, -0.680404, -0.135751, 
-0.302942, -0.469005, -0.453640, 
-0.182727, -0.314240, 0.024270, 
-0.021546, 0.158510, 0.317019, 
-0.485799, 0.492951, 0.792548, 
-0.634038, 0.951057, 0.951057, 
-0.777915, 0.613430, 0.951057, 
-0.951057, 0.793994, 0.635800, 
-1.049020, 1.146990, 1.084480, 
-0.924310, 1.440880, 1.342910, 
-1.380330, 1.221820, 0.948209, 
-0.856297, 0.746293, 0.904802, 
-0.476242, 0.393234, 0.489821, 
-0.391857, 0.194471, 0.075751, 
-0.097964, 0.195929, -0.097964, 
-0.195928, -0.173234, -0.078922, 
-0.489821, -0.391856, -0.467007, 
-0.397859, -0.746293, -0.904802, 
-0.945206, -0.849559, -1.380330, 
-1.221820, -1.440880, -1.342910, 
-1.100830, -0.923191, -1.049020, 
-1.146990, -0.774515, -0.621542, 
-0.951057, -0.951057, -0.803055, 
-0.635255, -0.951057, -0.951057, 
-0.792548, -0.634038, -0.454301, 
-0.479369, -0.158510, -0.317019, 
-0.149331, 0.178643, 0.295860, 
-0.306275, 0.246225, 0.406610, 
-0.367954, 0.498458, -0.014929, 
-0.084917, -0.094272, -0.000726, 
-0.245374, -0.340755, -0.435351, 
-0.490564, -0.176355, -0.294974, 
-0.148010, -0.328246, 0.136248, 
-0.182883, 0.328957, 0.309903, 
-0.646446, 0.622546, 0.800859, 
-0.801803, 0.643087, 0.654473, 
-0.796963, 0.816799, 0.748189, 
-0.898148, 0.819092, 0.970033, 
-1.049540, 1.162660, 1.115310, 
-1.266700, 0.539960, 0.728396, 
-0.656318, 0.783152, 0.302716, 
-0.403858, 0.219006, 0.309252, 
-0.009667, 0.107161, -0.118211, 
-0.011786, -0.284704, -0.179858, 
-0.411567, -0.296676, -0.570168, 
-0.635348, -0.726784, -0.793211, 
-1.016860, -1.112970, -1.170910, 
-1.281530, -0.737860, -0.851723, 
-0.883732, -0.995070, -0.653153, 
-0.769939, -0.631918, -0.797688, 
-0.616289, -0.806819, -0.638485, 
-0.790356, -0.136399, -0.322769, 
-0.165339, -0.309622]
-DataCollection/Mesh/PartitionInfo/Size: 2
-DataCollection/Mesh/PartitionInfo/Type: FMS_UINT64
-DataCollection/Mesh/PartitionInfo/Values: [0, 1]
-DataCollection/Mesh/NumDomainNames: 1
-DataCollection/Mesh/NumComponents: 1
-DataCollection/Mesh/NumTags: 0
-DataCollection/Mesh/DomainNames/0/Name: Domain
-DataCollection/Mesh/DomainNames/0/NumDomains: 1
-DataCollection/Mesh/DomainNames/0/Domains/0/Dimension: 2
-DataCollection/Mesh/DomainNames/0/Domains/0/NumVertices: 31
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/EntityType: FMS_EDGE
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/NumEntities: 50
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Size: 100
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Type: FMS_INT32
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Values: [11, 0, 26, 
-11, 26, 14, 
-14, 0, 27, 
-14, 27, 17, 
-17, 0, 28, 
-17, 28, 20, 
-20, 0, 29, 
-20, 29, 23, 
-23, 0, 30, 
-23, 30, 11, 
-11, 1, 12, 
-1, 26, 12, 
-12, 3, 13, 
-3, 26, 13, 
-13, 2, 14, 
-2, 15, 2, 
-27, 15, 15, 
-5, 16, 5, 
-27, 16, 16, 
-4, 17, 4, 
-18, 4, 28, 
-18, 18, 7, 
-19, 7, 28, 
-19, 19, 6, 
-20, 6, 21, 
-6, 29, 21, 
-21, 9, 22, 
-9, 29, 22, 
-22, 8, 23, 
-8, 24, 8, 
-30, 24, 24, 
-10, 25, 10, 
-30, 25, 25, 1]
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/EntityType: FMS_QUADRILATERAL
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/NumEntities: 20
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Size: 80
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Type: FMS_INT32
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Values: [0, 1, 2, 
-3, 3, 4, 
-5, 6, 6, 
-7, 8, 9, 
-9, 10, 11, 
-12, 12, 13, 
-14, 0, 15, 
-16, 17, 1, 
-17, 18, 19, 
-20, 2, 20, 
-21, 22, 22, 
-23, 24, 4, 
-24, 25, 26, 
-27, 5, 27, 
-28, 29, 29, 
-30, 31, 7, 
-31, 32, 33, 
-34, 8, 34, 
-35, 36, 36, 
-37, 38, 10, 
-38, 39, 40, 
-41, 11, 41, 
-42, 43, 43, 
-44, 45, 13, 
-45, 46, 47, 
-48, 14, 48, 
-49, 15]
-DataCollection/Mesh/Components/0/Name: volume
-DataCollection/Mesh/Components/0/Dimension: 2
-DataCollection/Mesh/Components/0/NumEntities: 20
-DataCollection/Mesh/Components/0/Coordinates: Coords
-DataCollection/Mesh/Components/0/NumParts: 1
-DataCollection/Mesh/Components/0/Parts/0/DomainName: Domain
-DataCollection/Mesh/Components/0/Parts/0/DomainID: 0
-DataCollection/Mesh/Components/0/Parts/0/FullDomain: Yes
-DataCollection/Mesh/Components/0/Relations/Size: 0
-DataCollection/Mesh/Components/0/Relations/Type: FMS_UINT64
@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.

-PROJECT_NUMBER         = v4.3.1
+PROJECT_NUMBER         = v4.2.1

 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -766,7 +766,6 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/mesh \
                         @MFEM_SOURCE_DIR@/fem \
                         @MFEM_SOURCE_DIR@/examples \
-                         @MFEM_SOURCE_DIR@/examples/caliper \
                         @MFEM_SOURCE_DIR@/examples/amgx \
                         @MFEM_SOURCE_DIR@/examples/ginkgo \
                         @MFEM_SOURCE_DIR@/examples/hiop \
@@ -779,7 +778,6 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/miniapps/electromagnetics \
                         @MFEM_SOURCE_DIR@/miniapps/gslib \
                         @MFEM_SOURCE_DIR@/miniapps/meshing \
-                         @MFEM_SOURCE_DIR@/miniapps/mtop \
                         @MFEM_SOURCE_DIR@/miniapps/navier \
                         @MFEM_SOURCE_DIR@/miniapps/nurbs \
                         @MFEM_SOURCE_DIR@/miniapps/performance \
@@ -42,10 +42,8 @@ namespace mfem {
 * - MFEM_FORALL macro in forall.hpp
 *
 * <H3>Example codes</H3>
- * - <a class="el" href="ex0_8cpp_source.html">Example 0</a>: simplest example, nodal H1 FEM for the Laplace problem
- * - <a class="el" href="ex0p_8cpp_source.html">Example 0p</a>: simplest parallel example, nodal H1 FEM for the Laplace problem
- * - <a class="el" href="examples_2ex1_8cpp_source.html">Example 1</a>: nodal H1 FEM for the Laplace problem (same discretization as ex0 but with more sophisticated options)
- * - <a class="el" href="examples_2ex1p_8cpp_source.html">Example 1p</a>: parallel nodal H1 FEM for the Laplace problem (same discretization as ex0p but with more sophisticated options)
+ * - <a class="el" href="examples_2ex1_8cpp_source.html">Example 1</a>: nodal H1 FEM for the Laplace problem
+ * - <a class="el" href="examples_2ex1p_8cpp_source.html">Example 1p</a>: parallel nodal H1 FEM for the Laplace problem
 * - <a class="el" href="ex2_8cpp_source.html">Example 2</a>: vector FEM for linear elasticity
 * - <a class="el" href="ex2p_8cpp_source.html">Example 2p</a>: parallel vector FEM for linear elasticity
 * - <a class="el" href="ex3_8cpp_source.html">Example 3</a>: Nedelec H(curl) FEM for the definite Maxwell problem
@@ -94,10 +92,6 @@ namespace mfem {
 * - <a class="el" href="ex26p_8cpp_source.html">Example 26p</a>: parallel multigrid preconditioner for the Laplace problem using nodal H1 FEM
 * - <a class="el" href="ex27_8cpp_source.html">Example 27</a>: boundary conditions for the Laplace problem
 * - <a class="el" href="ex27p_8cpp_source.html">Example 27p</a>: parallel boundary conditions for the Laplace problem
- * - <a class="el" href="ex28_8cpp_source.html">Example 28</a>: sliding contact in elasticity
- * - <a class="el" href="ex28p_8cpp_source.html">Example 28p</a>: parallel sliding contact in elasticity
- * - <a class="el" href="ex29_8cpp_source.html">Example 29</a>: Laplace solve on a 3D-embedded surface
- * - <a class="el" href="ex29p_8cpp_source.html">Example 29p</a>: parallel Laplace solve on a 3D-embedded surface
 *
 * <H4>AmgX Examples</H4>
 * - Variants of Examples
@@ -105,12 +99,6 @@ namespace mfem {
 *   <a class="el" href="examples_2amgx_2ex1p_8cpp_source.html">1p</a>,
 *   demonstrating the use of MFEM's \link amgxsolver.hpp AmgX integration\endlink.
 *
- * <H4>Caliper Examples</H4>
- * - Variants of Example
- *   <a class="el" href="examples_2caliper_2ex1_8cpp_source.html">1</a> and
- *   <a class="el" href="examples_2caliper_2ex1p_8cpp_source.html">1p</a>,
- *   demonstrating the use of MFEM's \link annotation.hpp Ginkgo integration\endlink.
- *
 * <H4>Ginkgo Examples</H4>
 * - Variants of Example
 *   <a class="el" href="examples_2ginkgo_2ex1_8cpp_source.html">1</a>,
@@ -189,9 +177,7 @@ namespace mfem {
 * - <a class="el" href="field-diff_8cpp_source.html">Field Diff</a>: compare grid functions on different meshes
 * - <a class="el" href="field-interp_8cpp_source.html">Field Interp</a>: transfer a grid functions between meshes
 * - <a class="el" href="distance_8cpp_source.html">Distance</a>: finite element distance function solver
- * - <a class="el" href="diffusion_8cpp_source.html">Shifted Diffusion</a>: shifted boundary diffusion solver
 * - <a class="el" href="distance_8cpp_source.html">Block Solvers</a>: comparison of saddle point system solvers
- * - <a class="el" href="parheat_8cpp_source.html">Optimization gradients</a>: Gradients of PDE-constrained function
 * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
 * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
 *
@@ -10,7 +10,6 @@
 # CONTRIBUTING.md for details.

 list(APPEND ALL_EXE_SRCS
-  ex0.cpp
  ex1.cpp
  ex2.cpp
  ex3.cpp
@@ -35,13 +34,10 @@ list(APPEND ALL_EXE_SRCS
  ex25.cpp
  ex26.cpp
  ex27.cpp
-  ex28.cpp
-  ex29.cpp
  )

 if (MFEM_USE_MPI)
  list(APPEND ALL_EXE_SRCS
-    ex0p.cpp
    ex1p.cpp
    ex2p.cpp
    ex3p.cpp
@@ -68,8 +64,6 @@ if (MFEM_USE_MPI)
    ex25p.cpp
    ex26p.cpp
    ex27p.cpp
-    ex28p.cpp
-    ex29p.cpp
    )
 endif()

@@ -85,9 +79,6 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})

  set(THIS_TEST_OPTIONS "-no-vis")
-  if (${TEST_NAME} MATCHES "ex0p?")
-    set(THIS_TEST_OPTIONS)
-  endif()
  if (${TEST_NAME} MATCHES "ex10p*")
    list(APPEND THIS_TEST_OPTIONS "-tf" "5")
  elseif(${TEST_NAME} MATCHES "ex15p*")
@@ -108,34 +99,6 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
  endif()
 endforeach()

-# Add CUDA/HIP tests.
-set(DEVICE_EXAMPLES
-    # serial examples with device support:
-    ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
-    # parallel examples with device support:
-    ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p ex24p ex25p ex26p)
-set(MFEM_TEST_DEVICE)
-if (MFEM_USE_CUDA)
-  set(MFEM_TEST_DEVICE "cuda")
-elseif (MFEM_USE_HIP)
-  set(MFEM_TEST_DEVICE "hip")
-endif()
-if (MFEM_TEST_DEVICE)
-  foreach(TEST_NAME ${DEVICE_EXAMPLES})
-    set(THIS_TEST_OPTIONS "-no-vis" "-d" "${MFEM_TEST_DEVICE}")
-    if (NOT (${TEST_NAME} MATCHES ".*p$"))
-      add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_ser
-        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-    elseif (MFEM_USE_MPI)
-      add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_np=${MFEM_MPI_NP}
-        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-        ${MPIEXEC_PREFLAGS}
-        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-        ${MPIEXEC_POSTFLAGS})
-    endif()
-  endforeach()
-endif()
-
 # If STRUMPACK is enabled, add a test run that uses it.
 if (MFEM_USE_STRUMPACK)
  add_test(NAME ex11p_strumpack_np=${MFEM_MPI_NP}
@@ -159,11 +122,6 @@ if (MFEM_USE_AMGX)
  add_subdirectory(amgx)
 endif()

-# Include the examples/epic directory if EPIC is enabled.
-if (MFEM_USE_EPIC)
-  add_subdirectory(epic)
-endif()
-
 # Include the examples/ginkgo directory if GINKGO is enabled.
 if (MFEM_USE_GINKGO)
  add_subdirectory(ginkgo)
@@ -189,10 +147,6 @@ if (MFEM_USE_SUNDIALS)
  add_subdirectory(sundials)
 endif()

-if(MFEM_USE_CALIPER)
-  add_subdirectory(caliper)
-endif()
-
 # Include the examples/superlu directory if SUPERLU is enabled.
 if (MFEM_USE_SUPERLU)
  add_subdirectory(superlu)
@@ -157,7 +157,7 @@ int main(int argc, char *argv[])
      delete_fec = true;
   }
   ParFiniteElementSpace fespace(&pmesh, fec);
-   HYPRE_BigInt size = fespace.GlobalTrueVSize();
+   HYPRE_Int size = fespace.GlobalTrueVSize();
   if (myid == 0)
   {
      cout << "Number of finite element unknowns: " << size << endl;
@@ -1,51 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-set(CALIPER_EXAMPLES_SRCS)
-
-list(APPEND CALIPER_EXE_SRCS
-    ex1.cpp
-    )
-
-if (MFEM_USE_MPI)
-  list(APPEND CALIPER_EXE_SRCS
-      ex1p.cpp
-      )
-endif()
-
-
-# Include the source directory where mfem.hpp and mfem-performance.hpp are.
-include_directories(BEFORE ${PROJECT_BINARY_DIR})
-
-# Add one executable per cpp file
-set(PREFIX caliper_)
-add_mfem_examples(CALIPER_EXE_SRCS ${PREFIX})
-
-# Add a test for each example
-foreach(SRC_FILE ${CALIPER_EXE_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-
-  set(THIS_TEST_OPTIONS "-no-vis")
-
-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
-
-
@@ -1,18 +0,0 @@
-                    Finite Element Discretization Library
-                                   __
-                       _ __ ___   / _|  ___  _ __ ___
-                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
-                      | | | | | ||  _||  __/| | | | | |
-                      |_| |_| |_||_|   \___||_| |_| |_|
-
-                               https://mfem.org
-
-This directory contains modifications of the example codes that illustrate the
-use of MFEM features based on the Caliper performance profiling library.
-
-To build these examples, make sure that MFEM is configured with the option
-"MFEM_USE_CALIPER = YES", see the top-level INSTALL file for details (version
-2.5.0 of Caliper is recommended, though older versions may work too).
-
-We recommend comparing the original example codes with the corresponding files
-in the current directory.
@@ -1,270 +0,0 @@
-//                                MFEM Example 1
-//                             Caliper Modification
-//
-// Compile with: make ex1
-//
-// Sample runs:  ex1 -m ../data/square-disc.mesh
-//               ex1 -m ../data/star.mesh
-//               ex1 -m ../data/star-mixed.mesh
-//               ex1 -m ../data/escher.mesh
-//               ex1 -m ../data/fichera.mesh
-//               ex1 -m ../data/fichera-mixed.mesh
-//               ex1 -m ../data/toroid-wedge.mesh
-//               ex1 -m ../data/periodic-annulus-sector.msh
-//               ex1 -m ../data/periodic-torus-sector.msh
-//               ex1 -m ../data/square-disc-p2.vtk -o 2
-//               ex1 -m ../data/square-disc-p3.mesh -o 3
-//               ex1 -m ../data/square-disc-nurbs.mesh -o -1
-//               ex1 -m ../data/star-mixed-p2.mesh -o 2
-//               ex1 -m ../data/disc-nurbs.mesh -o -1
-//               ex1 -m ../data/pipe-nurbs.mesh -o -1
-//               ex1 -m ../data/fichera-mixed-p2.mesh -o 2
-//               ex1 -m ../data/star-surf.mesh
-//               ex1 -m ../data/square-disc-surf.mesh
-//               ex1 -m ../data/inline-segment.mesh
-//               ex1 -m ../data/amr-quad.mesh
-//               ex1 -m ../data/amr-hex.mesh
-//               ex1 -m ../data/fichera-amr.mesh
-//               ex1 -m ../data/mobius-strip.mesh
-//               ex1 -m ../data/mobius-strip.mesh -o -1 -sc
-//
-// Device sample runs:
-//               ex1 -pa -d cuda
-//               ex1 -pa -d raja-cuda
-//               ex1 -pa -d occa-cuda
-//               ex1 -pa -d raja-omp
-//               ex1 -pa -d occa-omp
-//               ex1 -pa -d ceed-cpu
-//             * ex1 -pa -d ceed-cuda
-//               ex1 -pa -d ceed-cuda:/gpu/cuda/shared
-//               ex1 -m ../data/beam-hex.mesh -pa -d cuda
-//               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cpu
-//               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cuda:/gpu/cuda/ref
-//
-// Description: This example is a copy of Example 1 instrumented with the
-//              Caliper performance profilinh library. Any option supported by
-//              the Caliper ConfigManager can be passed to the code using a
-//              configuration string after -p or --caliper flag. For more
-//              information, see the Caliper documentation.
-//
-// Examples: ex1 --caliper runtime-report
-//           ex1 --caliper runtime-report,mem.highwatermark
-//
-// The first run will return the default report. The second run will also output
-// the memory high-water mark and time spent in MPI routines.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // Define Caliper ConfigManager
-   cali::ConfigManager mgr;
-   // Caliper instrumentation
-   MFEM_PERF_FUNCTION;
-
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../data/star.mesh";
-   int order = 1;
-   bool static_cond = false;
-   bool pa = false;
-   const char *device_config = "cpu";
-   bool visualization = true;
-   const char* cali_config = "runtime-report";
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&cali_config, "-p", "--caliper",
-                  "Caliper configuration string.");
-
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // Caliper configuration
-   mgr.add(cali_config);
-   mgr.start();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
-   //    the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   // 4. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
-   //    largest number that gives a final mesh with no more than 50,000
-   //    elements.
-   {
-      int ref_levels =
-         (int)floor(log(50000./mesh.GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh.UniformRefinement();
-      }
-   }
-
-   // 5. Define a finite element space on the mesh. Here we use continuous
-   //    Lagrange finite elements of the specified order. If order < 1, we
-   //    instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   bool delete_fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
-   }
-   else if (mesh.GetNodes())
-   {
-      fec = mesh.GetNodes()->OwnFEC();
-      delete_fec = false;
-      cout << "Using isoparametric FEs: " << fec->Name() << endl;
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
-   }
-   FiniteElementSpace fespace(&mesh, fec);
-   cout << "Number of finite element unknowns: "
-        << fespace.GetTrueVSize() << endl;
-
-   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
-   //    In this example, the boundary conditions are defined by marking all
-   //    the boundary attributes from the mesh as essential (Dirichlet) and
-   //    converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (mesh.bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(mesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
-   //    the basis functions in the finite element fespace.
-   MFEM_PERF_BEGIN("Set up the linear form");
-   LinearForm b(&fespace);
-   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
-   MFEM_PERF_END("Set up the linear form");
-
-   // 8. Define the solution vector x as a finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero,
-   //    which satisfies the boundary conditions.
-   GridFunction x(&fespace);
-   x = 0.0;
-
-   // 9. Set up the bilinear form a(.,.) on the finite element space
-   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //    domain integrator.
-   MFEM_PERF_BEGIN("Set up the bilinear form");
-   BilinearForm a(&fespace);
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   // 10. Assemble the bilinear form and the corresponding linear system,
-   //     applying any necessary transformations such as: eliminating boundary
-   //     conditions, applying conforming constraints for non-conforming AMR,
-   //     static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
-   a.Assemble();
-
-   OperatorPtr A;
-   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-   MFEM_PERF_END("Set up the bilinear form");
-
-   cout << "Size of linear system: " << A->Height() << endl;
-
-   // 11. Solve the linear system A X = B.
-   MFEM_PERF_BEGIN("Solve A X=B");
-   if (!pa)
-   {
-#ifndef MFEM_USE_SUITESPARSE
-      // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
-      GSSmoother M((SparseMatrix&)(*A));
-      PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
-#else
-      // If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-      UMFPackSolver umf_solver;
-      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      umf_solver.SetOperator(*A);
-      umf_solver.Mult(B, X);
-#endif
-   }
-   else // Jacobi preconditioning in partial assembly mode
-   {
-      if (UsesTensorBasis(fespace))
-      {
-         OperatorJacobiSmoother M(a, ess_tdof_list);
-         PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
-      }
-      else
-      {
-         CG(*A, B, X, 1, 400, 1e-12, 0.0);
-      }
-   }
-   MFEM_PERF_END("Solve A X=B");
-   // 12. Recover the solution as a finite element grid function.
-   a.RecoverFEMSolution(X, b, x);
-
-   // 13. Save the refined mesh and the solution. This output can be viewed later
-   //     using GLVis: "glvis -m refined.mesh -g sol.gf".
-   MFEM_PERF_BEGIN("Save the results");
-   ofstream mesh_ofs("refined.mesh");
-   mesh_ofs.precision(8);
-   mesh.Print(mesh_ofs);
-   ofstream sol_ofs("sol.gf");
-   sol_ofs.precision(8);
-   x.Save(sol_ofs);
-   MFEM_PERF_END("Save the results");
-   // 14. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << mesh << x << flush;
-   }
-
-   // 15. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
-
-   // Flush output
-   mgr.flush();
-
-   return 0;
-}
@@ -1,298 +0,0 @@
-//                       MFEM Example 1 - Parallel Version
-//                             Caliper Modification
-//
-// Compile with: make ex1p
-//
-// Sample runs:  mpirun -np 4 ex1p -m ../data/square-disc.mesh
-//               mpirun -np 4 ex1p -m ../data/star.mesh
-//               mpirun -np 4 ex1p -m ../data/star-mixed.mesh
-//               mpirun -np 4 ex1p -m ../data/escher.mesh
-//               mpirun -np 4 ex1p -m ../data/fichera.mesh
-//               mpirun -np 4 ex1p -m ../data/fichera-mixed.mesh
-//               mpirun -np 4 ex1p -m ../data/toroid-wedge.mesh
-//               mpirun -np 4 ex1p -m ../data/periodic-annulus-sector.msh
-//               mpirun -np 4 ex1p -m ../data/periodic-torus-sector.msh
-//               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
-//               mpirun -np 4 ex1p -m ../data/square-disc-p3.mesh -o 3
-//               mpirun -np 4 ex1p -m ../data/square-disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/star-mixed-p2.mesh -o 2
-//               mpirun -np 4 ex1p -m ../data/disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/pipe-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/ball-nurbs.mesh -o 2
-//               mpirun -np 4 ex1p -m ../data/fichera-mixed-p2.mesh -o 2
-//               mpirun -np 4 ex1p -m ../data/star-surf.mesh
-//               mpirun -np 4 ex1p -m ../data/square-disc-surf.mesh
-//               mpirun -np 4 ex1p -m ../data/inline-segment.mesh
-//               mpirun -np 4 ex1p -m ../data/amr-quad.mesh
-//               mpirun -np 4 ex1p -m ../data/amr-hex.mesh
-//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh
-//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh -o -1 -sc
-//
-// Device sample runs:
-//               mpirun -np 4 ex1p -pa -d cuda
-//               mpirun -np 4 ex1p -pa -d occa-cuda
-//               mpirun -np 4 ex1p -pa -d raja-omp
-//               mpirun -np 4 ex1p -pa -d ceed-cpu
-//             * mpirun -np 4 ex1p -pa -d ceed-cuda
-//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
-//               mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
-//
-// Description: This example is a copy of Example 1 instrumented with the
-//              Caliper performance profilinh library. Any option supported by
-//              the Caliper ConfigManager can be passed to the code using a
-//              configuration string after -p or --caliper flag. For more
-//              information, see the Caliper documentation.
-//
-// Examples:  mpirun -np 4 ex1p --caliper runtime-report
-//            mpirun -np 4 ex1p --caliper runtime-report,mem.highwatermark,mpi-report
-//
-// The first run will return the default report. The second run will also output
-// the memory high-water mark and time spent in MPI routines.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-   // Define Caliper ConfigManager
-   cali::ConfigManager mgr;
-   // Caliper instrumentation
-   MFEM_PERF_FUNCTION;
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../../data/star.mesh";
-   int order = 1;
-   bool static_cond = false;
-   bool pa = false;
-   const char *device_config = "cpu";
-   bool visualization = true;
-   const char* cali_config = "runtime-report";
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&cali_config, "-p", "--caliper",
-                  "Caliper configuration string.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // Caliper configuration
-   mgr.add(cali_config);
-   mgr.start();
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   // 5. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 10,000 elements.
-   {
-      int ref_levels =
-         (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh.UniformRefinement();
-      }
-   }
-
-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
-   {
-      int par_ref_levels = 2;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         pmesh.UniformRefinement();
-      }
-   }
-
-   // 7. Define a parallel finite element space on the parallel mesh. Here we
-   //    use continuous Lagrange finite elements of the specified order. If
-   //    order < 1, we instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   bool delete_fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
-   }
-   else if (pmesh.GetNodes())
-   {
-      fec = pmesh.GetNodes()->OwnFEC();
-      delete_fec = false;
-      if (myid == 0)
-      {
-         cout << "Using isoparametric FEs: " << fec->Name() << endl;
-      }
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
-   }
-   ParFiniteElementSpace fespace(&pmesh, fec);
-   HYPRE_BigInt size = fespace.GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-   }
-
-   // 8. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet) and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (pmesh.bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 9. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (1,phi_i) where phi_i are the basis functions in fespace.
-   MFEM_PERF_BEGIN("Set up the linear form");
-   ParLinearForm b(&fespace);
-   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
-   MFEM_PERF_END("Set up the linear form");
-
-   // 10. Define the solution vector x as a parallel finite element grid function
-   //     corresponding to fespace. Initialize x with initial guess of zero,
-   //     which satisfies the boundary conditions.
-   ParGridFunction x(&fespace);
-   x = 0.0;
-
-   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //     domain integrator.
-   MFEM_PERF_BEGIN("Set up the bilinear form");
-   ParBilinearForm a(&fespace);
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   // 12. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
-   a.Assemble();
-
-   OperatorPtr A;
-   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-   MFEM_PERF_END("Set up the bilinear form");
-   // 13. Solve the linear system A X = B.
-   //     * With full assembly, use the BoomerAMG preconditioner from hypre.
-   //     * With partial assembly, use Jacobi smoothing, for now.
-   MFEM_PERF_BEGIN("Solve A X = B");
-   Solver *prec = NULL;
-   if (pa)
-   {
-      if (UsesTensorBasis(fespace))
-      {
-         prec = new OperatorJacobiSmoother(a, ess_tdof_list);
-      }
-   }
-   else
-   {
-      prec = new HypreBoomerAMG;
-   }
-   CGSolver cg(MPI_COMM_WORLD);
-   cg.SetRelTol(1e-12);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(1);
-   if (prec) { cg.SetPreconditioner(*prec); }
-   cg.SetOperator(*A);
-   cg.Mult(B, X);
-   delete prec;
-   MFEM_PERF_END("Solve A X = B");
-   // 14. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a.RecoverFEMSolution(X, b, x);
-
-   // 15. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   MFEM_PERF_BEGIN("Save the results");
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh.Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-   MFEM_PERF_END("Save the results");
-   // 16. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << pmesh << x << flush;
-   }
-
-   // 17. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
-   // Flush output before MPI_finalize
-   mgr.flush();
-   MPI_Finalize();
-
-   return 0;
-}
@@ -1,76 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Use the MFEM build directory
-MFEM_DIR ?= ../..
-MFEM_BUILD_DIR ?= ../..
-SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/caliper,)
-CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
-# Use the MFEM install directory
-# MFEM_INSTALL_DIR = ../../mfem
-# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
-
-MFEM_LIB_FILE = mfem_is_not_built
-include $(CONFIG_MK)
-
-SEQ_EXAMPLES = ex1
-PAR_EXAMPLES = ex1p
-
-ifeq ($(MFEM_USE_MPI),NO)
-   EXAMPLES = $(SEQ_EXAMPLES)
-else
-   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
-endif
-
-.SUFFIXES:
-.SUFFIXES: .o .cpp .mk
-.PHONY: all clean clean-build clean-exec
-
-# Remove built-in rule
-%: %.cpp
-
-# Replace the default implicit rule for *.cpp files
-%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
-
-all: $(EXAMPLES)
-
-MFEM_TESTS = EXAMPLES
-include $(MFEM_TEST_MK)
-
-# Testing: Parallel vs. serial runs
-RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
-%-test-par: %
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
-%-test-seq: %
-	@$(call mfem-test,$<,, Serial example)
-
-# Testing: Specific execution options
-ex1-test-seq: ex1
-	@$(call mfem-test,$<,, Caliper serial example)
-ex1p-test-par: ex1p
-	@$(call mfem-test,$<, $(RUN_MPI), Caliper parallel example)
-
-# Testing: "test" target and mfem-test* variables are defined in config/test.mk
-
-# Generate an error message if the MFEM library is not built and exit
-$(MFEM_LIB_FILE):
-	$(error The MFEM library is not built)
-
-clean: clean-build clean-exec $(SUBDIRS_CLEAN)
-
-clean-build:
-	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
-	rm -rf *.dSYM *.TVD.*breakpoints
-
-clean-exec:
-	@rm -f refined.mesh displaced.mesh mesh.* ex5.mesh
-	@rm -f sphere_refined.* sol.* sol_u.* sol_p.* sol_r.* sol_i.*
@@ -1,64 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-set(EPIC_EXAMPLES_SRCS)
-list(APPEND EPIC_EXAMPLES_SRCS
-  ex16.cpp
-  )
-
-if (MFEM_USE_MPI)
-list(APPEND EPIC_EXAMPLES_SRCS
-    ex16p.cpp
-    )
-endif()
-
-# Include the source directory where mfem.hpp and mfem-performance.hpp are.
-include_directories(BEFORE ${PROJECT_BINARY_DIR})
-
-# Add "test_epic" target, see below.
-add_custom_target(test_epic
-  ${CMAKE_CTEST_COMMAND} -R epic USES_TERMINAL)
-
-# Add one executable per cpp file, adding "epic_" as prefix. Sets
-# "test_epic" as a target that depends on the given examples.
-set(PFX epic_)
-add_mfem_examples(EPIC_EXAMPLES_SRCS ${PFX} "" test_epic)
-
-# Testing.
-# The EPIC tests can be run separately using the target "test_epic"
-# which builds the examples and runs:
-#   ctest -R epic
-
-# Example 16: use the default options
-
-# Add the tests: one test per source file.
-foreach(SRC_FILE ${EPIC_EXAMPLES_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-  set(TEST_NAME ${PFX}${TEST_NAME})
-
-  set(THIS_TEST_OPTIONS "-no-vis")
-  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
-
-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=4
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
-
@@ -1,17 +0,0 @@
-                    Finite Element Discretization Library
-                                   __
-                       _ __ ___   / _|  ___  _ __ ___
-                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
-                      | | | | | ||  _||  __/| | | | | |
-                      |_| |_| |_||_|   \___||_| |_| |_|
-
-                               http://mfem.org
-
-This directory contains modifications of the example codes that illustrate the
-use of MFEM features based on the EPIC suite of time integration.
-
-To build these examples, make sure that MFEM is configured with the option
-"MFEM_USE_EPIC = YES".
-
-We recommend comparing the original example codes with the corresponding files
-in the current directory.
@@ -1,610 +0,0 @@
-//                                MFEM Example 16
-//                               EPIC Modification
-//
-// Compile with: make ex16
-//
-// Sample runs:  ex16
-//               ex16 -m ../../data/inline-tri.mesh
-//               ex16 -m ../../data/disc-nurbs.mesh -tf 2
-//               ex16 -s 8 -a 1.0 -k 0.0 -dt 1e-4 -tf 5e-2 -vs 25
-//               ex16 -m ../../data/fichera-q2.mesh
-//               ex16 -m ../../data/escher.mesh
-//               ex16 -m ../../data/beam-tet.mesh -tf 10 -dt 0.1
-//               ex16 -m ../../data/amr-quad.mesh -o 4 -r 0
-//               ex16 -m ../../data/amr-hex.mesh -o 2 -r 0
-//
-// Description:  This example solves a time dependent nonlinear heat equation
-//               problem of the form du/dt = C(u), with a non-linear diffusion
-//               operator C(u) = \nabla \cdot (\kappa + \alpha u) \nabla u.
-//
-//               We recommend viewing examples 2, 9 and 10 before viewing this
-//               example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-class ImplicitSolveOperator;
-class JacobianOperator;
-
-/** After spatial discretization, the conduction model can be written as:
- *
- *     du/dt = M^{-1}(-K(u) u)
- *
- *  where u is the vector representing the temperature, M is the mass matrix,
- *  and K is the diffusion operator with diffusivity depending on u:
- *  (\kappa + \alpha u).
- *
- *  Class ConductionOperator represents the right-hand side of the above ODE.
- */
-class ConductionOperator : public TimeDependentOperator
-{
-protected:
-   FiniteElementSpace &fespace;
-   Array<int> ess_tdof_list; // this list remains empty for pure Neumann b.c.
-
-   BilinearForm *M;
-   mutable BilinearForm *K;
-   mutable BilinearForm *dK;
-   mutable BilinearForm *J_K;
-
-   SparseMatrix Mmat;
-   mutable SparseMatrix J_K_mat;
-
-   mutable CGSolver M_solver; // Krylov solver for inverting the mass matrix M
-   DSmoother M_prec;  // Preconditioner for the mass matrix M
-
-   CGSolver Jg_solver; // Krylov solver for inverting the Jacobian in the nonlinear solve
-   DSmoother Jg_prec;  // Preconditioner for the Jacobian Jg
-
-   NewtonSolver newton_solver;
-   mutable JacobianOperator *jac;
-
-   double alpha, kappa;
-
-   mutable Vector z; // auxiliary vector
-
-   mutable int nRhsMult, nSetJac, nJacMult, nImpSolve, nImpIter, nImpMult, nImpSet;
-
-public:
-   Vector u0;
-
-   ConductionOperator(FiniteElementSpace &f, double alpha, double kappa, const Vector &u);
-
-   void UpdateStats();
-   void PrintStats(ostream& out);
-
-   void ExtractJacobians(const Vector& x, std::ostream &out, std::ostream &out2);
-
-   BilinearForm& GetKLambda(const Vector& u) const;
-   BilinearForm& GetdKLambda(const Vector& u) const;
-
-   virtual void Mult(const Vector &u, Vector &du_dt) const;
-   virtual Operator& GetGradient(const Vector &k) const;
-
-   virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);
-
-   virtual ~ConductionOperator();
-};
-
-class ImplicitSolveOperator : public Operator
-{
-private:
-   double dt;
-   const Vector* x;
-   ConductionOperator* oper;
-
-   const SparseMatrix* M;
-   mutable SparseMatrix* Jg;
-
-   mutable Vector u, z;
-   mutable int nMult, nSet;
-
-public:
-   ImplicitSolveOperator(ConductionOperator* oper, const SparseMatrix* M, double dt, const Vector* x);
-
-   int GetnMult() { return nMult; }
-   int GetnSet() { return nSet; }
-   virtual void Mult(const Vector &k, Vector &gk) const;
-   virtual Operator &GetGradient(const Vector &k) const;
-};
-
-class JacobianOperator : public Operator
-{
-private:
-   Operator* J;
-   Operator* M_solver;
-
-   mutable int nMult;
-   mutable Vector z;
-public:
-   JacobianOperator(Operator* J, Operator* M_solver);
-
-   int GetnMult() { return nMult; }
-
-   void ExtractJacobian(const Vector& x, std::ostream &out);
-   virtual void Mult(const Vector &k, Vector &gk) const;
-};
-
-double InitialTemperature(const Vector &x);
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../data/star.mesh";
-   int ref_levels = 2;
-   int order = 2;
-   int ode_solver_type = 8; // Exponential Euler
-   double t_final = 0.5;
-   double dt = 1.0e-2;
-   double alpha = 1.0e-2;
-   double kappa = 0.5;
-   bool visualization = true;
-   bool visit = false;
-   int vis_steps = 5;
-
-   int precision = 8;
-   cout.precision(precision);
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&ref_levels, "-r", "--refine",
-                  "Number of times to refine the mesh uniformly.");
-   args.AddOption(&order, "-o", "--order",
-                  "Order (degree) of the finite elements.");
-   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver:\n\t"
-                  "1  - Forward Euler,\n\t"
-                  "2  - RK2,\n\t"
-                  "3  - RK3 SSP,\n\t"
-                  "4  - RK4,\n\t"
-                  "5  - Backward Euler,\n\t"
-                  "6  - SDIRK 2,\n\t"
-                  "7  - SDIRK 3,\n\t"
-                  "8  - EPIC (exponential euler)\n\t");
-   args.AddOption(&t_final, "-tf", "--t-final",
-                  "Final time; start time is 0.");
-   args.AddOption(&dt, "-dt", "--time-step",
-                  "Time step.");
-   args.AddOption(&alpha, "-a", "--alpha",
-                  "Alpha coefficient.");
-   args.AddOption(&kappa, "-k", "--kappa",
-                  "Kappa coefficient offset.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
-                  "--no-visit-datafiles",
-                  "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
-                  "Visualize every n-th timestep.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   if (ode_solver_type < 1 || ode_solver_type > 9)
-   {
-      cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-      return 3;
-   }
-   args.PrintOptions(cout);
-
-   // 2. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral and hexahedral meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 3. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement, where 'ref_levels' is a
-   //    command-line parameter.
-   for (int lev = 0; lev < ref_levels; lev++)
-   {
-      mesh->UniformRefinement();
-   }
-
-   // 4. Define the vector finite element space representing the current and the
-   //    initial temperature, u_ref.
-   H1_FECollection fe_coll(order, dim);
-   FiniteElementSpace fespace(mesh, &fe_coll);
-
-   int fe_size = fespace.GetTrueVSize();
-   cout << "Number of temperature unknowns: " << fe_size << endl;
-
-   GridFunction u_gf(&fespace);
-
-   // 5. Set the initial conditions for u. All boundaries are considered
-   //    natural.
-   FunctionCoefficient u_0(InitialTemperature);
-   u_gf.ProjectCoefficient(u_0);
-   Vector u;
-   u_gf.GetTrueDofs(u);
-
-   // 6. Initialize the conduction operator and the visualization.
-   ConductionOperator oper(fespace, alpha, kappa, u);
-
-   u_gf.SetFromTrueDofs(u);
-   {
-      ofstream omesh("ex16.mesh");
-      omesh.precision(precision);
-      mesh->Print(omesh);
-      ofstream osol("ex16-init.gf");
-      osol.precision(precision);
-      u_gf.Save(osol);
-   }
-
-   VisItDataCollection visit_dc("Example16", mesh);
-   visit_dc.RegisterField("temperature", &u_gf);
-   if (visit)
-   {
-      visit_dc.SetCycle(0);
-      visit_dc.SetTime(0.0);
-      visit_dc.Save();
-   }
-
-   socketstream sout;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      sout.open(vishost, visport);
-      if (!sout)
-      {
-         cout << "Unable to connect to GLVis server at "
-              << vishost << ':' << visport << endl;
-         visualization = false;
-         cout << "GLVis visualization disabled.\n";
-      }
-      else
-      {
-         sout.precision(precision);
-         sout << "solution\n" << *mesh << u_gf;
-         sout << "pause\n";
-         sout << flush;
-         cout << "GLVis visualization paused."
-              << " Press space (in the GLVis window) to resume it.\n";
-      }
-   }
-
-   // 7. Define the ODE solver used for time integration.
-   double t = 0.0;
-   ODESolver *ode_solver = NULL;
-   switch (ode_solver_type)
-   {
-      // MFEM explicit methods
-      case 1: ode_solver = new ForwardEulerSolver; break;
-      case 2: ode_solver = new RK2Solver(0.5); break; // midpoint method
-      case 3: ode_solver = new RK3SSPSolver; break;
-      case 4: ode_solver = new RK4Solver; break;
-      // MFEM implicit L-stable methods
-      case 5: ode_solver = new BackwardEulerSolver; break;
-      case 6: ode_solver = new SDIRK23Solver(2); break;
-      case 7: ode_solver = new SDIRK33Solver; break;
-      // EPIC 
-      case 8: ode_solver = new EPI2();break;
-      case 9: ode_solver = new EPIRK4(); break;
-   }
-
-   // Initialize integrators
-   ode_solver->Init(oper);
-
-   // 8. Perform time-integration (looping over the time iterations, ti, with a
-   //    time-step dt).
-   cout << "Integrating the ODE ..." << endl;
-   tic_toc.Clear();
-   tic_toc.Start();
-
-   /*ofstream out_jac_an("jacobian_an.txt");
-   ofstream out_jac_fd("jacobian_fd.txt");
-   oper.ExtractJacobians(u, out_jac_fd, out_jac_an);*/
-
-   bool last_step = false;
-   int ti;
-   for (ti = 1; !last_step; ti++)
-   {
-      double dt_real = min(dt, t_final - t);
-
-      // Note that since we are using the "one-step" mode of the SUNDIALS
-      // solvers, they will, generally, step over the final time and will not
-      // explicitly perform the interpolation to t_final as they do in the
-      // "normal" step mode.
-      ode_solver->Step(u, t, dt_real);
-
-      oper.UpdateStats();
-
-      last_step = (t >= t_final - 1e-8*dt);
-
-      if (last_step || (ti % vis_steps) == 0) {
-         cout << "step " << ti << ", t = " << t << endl;
-
-         u_gf.SetFromTrueDofs(u);
-         if (visualization) {
-            sout << "solution\n" << *mesh << u_gf << flush;
-         }
-
-         if (visit) {
-            visit_dc.SetCycle(ti);
-            visit_dc.SetTime(t);
-            visit_dc.Save();
-         }
-      }
-   }
-   tic_toc.Stop();
-   double comp_time = tic_toc.RealTime();
-   cout << "Done, " << comp_time << "s." << endl;
-
-   // 9. Save the final solution. This output can be viewed later using GLVis:
-   //    "glvis -m ex16.mesh -g ex16-final.gf".
-   {
-      ofstream osol("ex16-final.gf");
-      osol.precision(precision);
-      u_gf.Save(osol);
-
-      ofstream ostats("ex16-stats.txt");
-      ostats << "time " << comp_time << endl;
-      oper.PrintStats(ostats);
-   }
-
-   // 10. Free the used memory.
-   delete ode_solver;
-   delete mesh;
-
-   return 0;
-}
-
-ConductionOperator::ConductionOperator(FiniteElementSpace &f, double al, double kap, const Vector &u)
-   : TimeDependentOperator(f.GetTrueVSize(), 0.0), fespace(f), M(NULL), K(NULL), dK(NULL), J_K(NULL), jac(NULL), z(height), u0(height),
-     nRhsMult(0), nSetJac(0), nJacMult(0), nImpSolve(0), nImpIter(0), nImpMult(0), nImpSet(0)
-{
-   const double rel_tol = 1e-8;
-
-   M = new BilinearForm(&fespace);
-   M->AddDomainIntegrator(new MassIntegrator());
-   M->Assemble();
-   M->FormSystemMatrix(ess_tdof_list, Mmat);
-
-   M_solver.iterative_mode = false;
-   M_solver.SetRelTol(rel_tol);
-   M_solver.SetAbsTol(0.0);
-   M_solver.SetMaxIter(50);
-   M_solver.SetPrintLevel(0);
-   M_solver.SetPreconditioner(M_prec);
-   M_solver.SetOperator(Mmat);
-
-   Jg_solver.SetRelTol(rel_tol);
-   Jg_solver.SetAbsTol(0.0);
-   Jg_solver.SetMaxIter(50);
-   Jg_solver.SetPrintLevel(0);
-   Jg_solver.SetPreconditioner(Jg_prec);
-
-   newton_solver.SetMaxIter(10);
-   newton_solver.SetRelTol(rel_tol);
-   newton_solver.SetPrintLevel(-1);
-   newton_solver.SetSolver(Jg_solver);
-   newton_solver.SetMaxIter(100);
-   newton_solver.iterative_mode = false;
-
-   alpha = al;
-   kappa = kap;
-}
-
-void ConductionOperator::UpdateStats()
-{
-   if (jac)
-   {
-      nJacMult += jac->GetnMult();
-   }
-}
-
-void ConductionOperator::PrintStats(ostream &out)
-{
-   out  << "nRhsMult " << nRhsMult << endl
-        << "nSetJac " << nSetJac << endl
-        << "nJacMult " << nJacMult  << endl
-        << "nImplicitSolve " << nImpSolve  << endl
-        << "nImplicitIter " << nImpIter << endl
-        << "nImplicitMult " << nImpMult << endl
-        << "nImplicitSet " << nImpSet << endl;
-}
-
-BilinearForm& ConductionOperator::GetKLambda(const Vector &u) const
-{
-   GridFunction conductivity_gf(&fespace);
-   conductivity_gf.SetFromTrueDofs(u);
-   for (int i = 0; i < conductivity_gf.Size(); i++)
-   {
-      conductivity_gf(i) = kappa + alpha*conductivity_gf(i);
-   }
-
-   GridFunctionCoefficient conductivity_coeff(&conductivity_gf);
-
-   delete K;
-   K = new BilinearForm(&fespace);
-   K->AddDomainIntegrator(new DiffusionIntegrator(conductivity_coeff));
-   K->Assemble();
-
-   return *K;
-}
-
-BilinearForm& ConductionOperator::GetdKLambda(const Vector &u) const
-{
-   GridFunction conductivity_gf(&fespace);
-   conductivity_gf.SetFromTrueDofs(u);
-   for (int i = 0; i < conductivity_gf.Size(); i++)
-   {
-      conductivity_gf(i) = kappa + alpha*conductivity_gf(i);
-   }
-
-   // Define diffusion form with conductivity = kappa(u0)
-   GridFunctionCoefficient conductivity_coeff(&conductivity_gf);
-
-   // Define advection form with velocity = grad kappa(u0)
-   GridFunction neg_cond_gf(conductivity_gf);
-   neg_cond_gf.Neg();
-   GradientGridFunctionCoefficient velocity_coeff(&neg_cond_gf);
-
-   delete dK;
-   dK = new BilinearForm(&fespace);
-
-   dK->AddDomainIntegrator(new DiffusionIntegrator(conductivity_coeff));
-   dK->AddDomainIntegrator(new MixedScalarWeakDivergenceIntegrator(velocity_coeff));
-   dK->Assemble();
-
-   return *dK;
-}
-
-void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
-{
-   // Compute:
-   //    du_dt = M^{-1}*-K(u)
-   // for du_dt
-   GetKLambda(u);
-   K->Mult(u, z);
-   z.Neg(); // z = -z
-   M_solver.Mult(z, du_dt);
-   nRhsMult++;
-}
-
-void ConductionOperator::ImplicitSolve(const double dt, const Vector &x, Vector &k)
-{
-   ImplicitSolveOperator imp_oper(this, &this->Mmat, dt, &x);
-   newton_solver.SetOperator(imp_oper);
-
-   Vector zero; // empty vector is interpreted as zero r.h.s. by NewtonSolver
-   newton_solver.Mult(zero, k);
-   MFEM_VERIFY(newton_solver.GetConverged(), "Newton solver did not converge.");
-
-   nImpSolve++;
-   nImpMult += imp_oper.GetnMult();
-   nImpSet  += imp_oper.GetnSet();
-   nImpIter += newton_solver.GetNumIterations();
-}
-
-Operator &ConductionOperator::GetGradient(const Vector &u) const
-{
-   delete jac;
-   GetdKLambda(u);
-   jac = new JacobianOperator(dK, &M_solver);
-
-   nSetJac++;
-
-   return *jac;
-}
-
-ConductionOperator::~ConductionOperator()
-{
-   delete M;
-   delete K;
-   delete dK;
-   delete J_K;
-   delete jac;
-}
-
-ImplicitSolveOperator::ImplicitSolveOperator(ConductionOperator *oper_, const SparseMatrix* M_, double dt_, const Vector* x_):
-   Operator(oper_->Height()), oper(oper_), M(M_), dt(dt_), x(x_), u(height), z(height), Jg(NULL), nMult(0), nSet(0)
-{ }
-
-
-void ImplicitSolveOperator::Mult(const Vector& y, Vector& gy) const
-{
-   // Compute gy = g(y) = My + dt K(lambda(u)) u
-   // with u = x + dt y
-   add(*x, dt, y, u);
-   BilinearForm& K = oper->GetKLambda(u);
-   K.Mult(u, gy);
-
-   M->AddMult(y, gy);
-
-   nMult++;
-}
-
-Operator& ImplicitSolveOperator::GetGradient(const Vector &k) const
-{
-   add(*x, dt, k, u);
-
-   BilinearForm& dK = oper->GetdKLambda(u);
-   Array<int> ess_tdof_list;
-   SparseMatrix dK_mat;
-   dK.FormSystemMatrix(ess_tdof_list, dK_mat);
-
-   delete Jg;
-   Jg = Add(1.0, *M, dt, dK_mat);
-
-   nSet++;
-   return *Jg;
-}
-
-JacobianOperator::JacobianOperator(Operator* J_, Operator* M_solver_):
-   Operator(M_solver_->Height()), J(J_), M_solver(M_solver_), z(height), nMult(0)
-{ }
-
-void JacobianOperator::Mult(const Vector &v, Vector &Jv) const
-{
-   Vector temp(v);
-   J->Mult(v, z);
-   z.Neg(); // z = -z
-   M_solver->Mult(z, Jv);
-   nMult++;
-}
-
-
-void ConductionOperator::ExtractJacobians(const Vector& x, std::ostream &out, std::ostream &out2)
-{
-   int n = x.Size();
-
-   Vector e(n);
-   e = 0.0;
-
-   double eps = 1e-8;
-   Vector fx(n), fx_eps(n), x_eps(n);
-   Mult(x, fx);
-
-   DenseMatrix J(n);
-
-   for (int i = 0; i < n; i++)
-   {
-      e[i] = 1.0;
-      add(x, eps, e, x_eps);
-      Mult(x_eps, fx_eps);
-      fx_eps -= fx;
-      fx_eps /= eps;
-      J.SetCol(i, fx_eps);
-      e[i] = 0.0;
-   }
-
-   J.PrintMatlab(out);
-   GetGradient(x);
-   jac->ExtractJacobian(x, out2);
-}
-
-void JacobianOperator::ExtractJacobian(const Vector& x, std::ostream &out)
-{
-   int n = z.Size();
-
-   Vector e(n);
-   e= 0.0;
-
-   Vector J_i(n);
-   DenseMatrix J(n);
-
-   for (int i = 0; i < n; i++)
-   {
-      e[i] = 1.0;
-      Mult(e, J_i);
-      J.SetCol(i, J_i);
-      e[i] = 0.0;
-   }
-
-   J.PrintMatlab(out);
-}
-
-double InitialTemperature(const Vector &x)
-{
-   if (x.Norml2() < 0.5) { return 2.0; }
-   else                  { return 1.0; }
-}
@@ -1,494 +0,0 @@
-//                         MFEM Example 16 - Parallel Version
-//                             SUNDIALS Modification
-//
-// Compile with: make ex16p
-//
-// Sample runs:
-//     mpirun -np 4 ex16p
-//     mpirun -np 4 ex16p -m ../../data/inline-tri.mesh
-//     mpirun -np 4 ex16p -m ../../data/disc-nurbs.mesh -tf 2
-//     mpirun -np 4 ex16p -s 12 -a 0.0 -k 1.0
-//     mpirun -np 4 ex16p -s 8 -a 1.0 -k 0.0 -dt 4e-6 -tf 2e-2 -vs 50
-//     mpirun -np 8 ex16p -s 9 -a 0.5 -k 0.5 -o 4 -dt 8e-6 -tf 2e-2 -vs 50
-//     mpirun -np 4 ex16p -s 10 -dt 2.0e-4 -tf 4.0e-2
-//     mpirun -np 16 ex16p -m ../../data/fichera-q2.mesh
-//     mpirun -np 16 ex16p -m ../../data/escher-p2.mesh
-//     mpirun -np 8 ex16p -m ../../data/beam-tet.mesh -tf 10 -dt 0.1
-//     mpirun -np 4 ex16p -m ../../data/amr-quad.mesh -o 4 -rs 0 -rp 0
-//     mpirun -np 4 ex16p -m ../../data/amr-hex.mesh -o 2 -rs 0 -rp 0
-//
-// Description:  This example solves a time dependent nonlinear heat equation
-//               problem of the form du/dt = C(u), with a non-linear diffusion
-//               operator C(u) = \nabla \cdot (\kappa + \alpha u) \nabla u.
-//
-//               We recommend viewing examples 2, 9 and 10 before viewing this
-//               example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-/** After spatial discretization, the conduction model can be written as:
- *
- *     du/dt = M^{-1}(-Ku)
- *
- *  where u is the vector representing the temperature, M is the mass matrix,
- *  and K is the diffusion operator with diffusivity depending on u:
- *  (\kappa + \alpha u).
- *
- *  Class ConductionOperator represents the right-hand side of the above ODE.
- */
-class ConductionOperator : public TimeDependentOperator
-{
-protected:
-   ParFiniteElementSpace &fespace;
-   Array<int> ess_tdof_list; // this list remains empty for pure Neumann b.c.
-
-   ParBilinearForm *M;
-   ParBilinearForm *K;
-
-   HypreParMatrix Mmat;
-   HypreParMatrix Kmat;
-   HypreParMatrix *T; // T = M + dt K
-   double current_dt;
-
-   CGSolver M_solver;    // Krylov solver for inverting the mass matrix M
-   HypreSmoother M_prec; // Preconditioner for the mass matrix M
-
-   CGSolver T_solver;    // Implicit solver for T = M + dt K
-   HypreSmoother T_prec; // Preconditioner for the implicit solver
-
-   double alpha, kappa;
-
-   mutable Vector z; // auxiliary vector
-
-public:
-   ConductionOperator(ParFiniteElementSpace &f, double alpha, double kappa,
-                      const Vector &u);
-
-   virtual void Mult(const Vector &u, Vector &du_dt) const;
-
-   /** Solve the Backward-Euler equation: k = f(u + dt*k, t), for the unknown k.
-       This is the only requirement for high-order SDIRK implicit integration.*/
-   virtual void ImplicitSolve(const double dt, const Vector &u, Vector &k);
-
-   /** Setup the system (M + dt K) x = M b. This method is used by the implicit
-       SUNDIALS solvers. */
-   virtual int SUNImplicitSetup(const Vector &x, const Vector &fx,
-                                int jok, int *jcur, double gamma);
-
-   /** Solve the system (M + dt K) x = M b. This method is used by the implicit
-       SUNDIALS solvers. */
-   virtual int SUNImplicitSolve(const Vector &b, Vector &x, double tol);
-
-   /// Update the diffusion BilinearForm K using the given true-dof vector `u`.
-   void SetParameters(const Vector &u);
-
-   virtual ~ConductionOperator();
-};
-
-double InitialTemperature(const Vector &x);
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../../data/star.mesh";
-   int ser_ref_levels = 2;
-   int par_ref_levels = 1;
-   int order = 2;
-   int ode_solver_type = 8; // Exponential Euler 
-   double t_final = 0.5;
-   double dt = 1.0e-2;
-   double alpha = 1.0e-2;
-   double kappa = 0.5;
-   bool visualization = true;
-   bool visit = false;
-   int vis_steps = 5;
-
-   int precision = 8;
-   cout.precision(precision);
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
-                  "Number of times to refine the mesh uniformly in parallel.");
-   args.AddOption(&order, "-o", "--order",
-                  "Order (degree) of the finite elements.");
-   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver:\n\t"
-                  "1  - Forward Euler,\n\t"
-                  "2  - RK2,\n\t"
-                  "3  - RK3 SSP,\n\t"
-                  "4  - RK4,\n\t"
-                  "5  - Backward Euler,\n\t"
-                  "6  - SDIRK 2,\n\t"
-                  "7  - SDIRK 3,\n\t"
-                  "8  - Exponential Euler,\n\t");
-   args.AddOption(&t_final, "-tf", "--t-final",
-                  "Final time; start time is 0.");
-   args.AddOption(&dt, "-dt", "--time-step",
-                  "Time step.");
-   args.AddOption(&alpha, "-a", "--alpha",
-                  "Alpha coefficient.");
-   args.AddOption(&kappa, "-k", "--kappa",
-                  "Kappa coefficient offset.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
-                  "--no-visit-datafiles",
-                  "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
-                  "Visualize every n-th timestep.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      MPI_Finalize();
-      return 1;
-   }
-
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // check for vaild ODE solver option
-   if (ode_solver_type < 1 || ode_solver_type > 8)
-   {
-      if (myid == 0)
-      {
-         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-      }
-      MPI_Finalize();
-      return 1;
-   }
-
-   // 3. Read the serial mesh from the given mesh file on all processors. We can
-   //    handle triangular, quadrilateral, tetrahedral and hexahedral meshes
-   //    with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 4. Refine the mesh in serial to increase the resolution. In this example
-   //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
-   //    a command-line parameter.
-   for (int lev = 0; lev < ser_ref_levels; lev++)
-   {
-      mesh->UniformRefinement();
-   }
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   for (int lev = 0; lev < par_ref_levels; lev++)
-   {
-      pmesh->UniformRefinement();
-   }
-
-   // 6. Define the vector finite element space representing the current and the
-   //    initial temperature, u_ref.
-   H1_FECollection fe_coll(order, dim);
-   ParFiniteElementSpace fespace(pmesh, &fe_coll);
-
-   int fe_size = fespace.GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of temperature unknowns: " << fe_size << endl;
-   }
-
-   ParGridFunction u_gf(&fespace);
-
-   // 7. Set the initial conditions for u. All boundaries are considered
-   //    natural.
-   FunctionCoefficient u_0(InitialTemperature);
-   u_gf.ProjectCoefficient(u_0);
-   Vector u;
-   u_gf.GetTrueDofs(u);
-
-   // 8. Initialize the conduction operator and the VisIt visualization.
-   ConductionOperator oper(fespace, alpha, kappa, u);
-
-   u_gf.SetFromTrueDofs(u);
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "ex16-mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "ex16-init." << setfill('0') << setw(6) << myid;
-      ofstream omesh(mesh_name.str().c_str());
-      omesh.precision(precision);
-      pmesh->Print(omesh);
-      ofstream osol(sol_name.str().c_str());
-      osol.precision(precision);
-      u_gf.Save(osol);
-   }
-
-   VisItDataCollection visit_dc("Example16-Parallel", pmesh);
-   visit_dc.RegisterField("temperature", &u_gf);
-   if (visit)
-   {
-      visit_dc.SetCycle(0);
-      visit_dc.SetTime(0.0);
-      visit_dc.Save();
-   }
-
-   socketstream sout;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      sout.open(vishost, visport);
-      sout << "parallel " << num_procs << " " << myid << endl;
-      int good = sout.good(), all_good;
-      MPI_Allreduce(&good, &all_good, 1, MPI_INT, MPI_MIN, pmesh->GetComm());
-      if (!all_good)
-      {
-         sout.close();
-         visualization = false;
-         if (myid == 0)
-         {
-            cout << "Unable to connect to GLVis server at "
-                 << vishost << ':' << visport << endl;
-            cout << "GLVis visualization disabled.\n";
-         }
-      }
-      else
-      {
-         sout.precision(precision);
-         sout << "solution\n" << *pmesh << u_gf;
-         sout << "pause\n";
-         sout << flush;
-         if (myid == 0)
-         {
-            cout << "GLVis visualization paused."
-                 << " Press space (in the GLVis window) to resume it.\n";
-         }
-      }
-   }
-
-   // 9. Define the ODE solver used for time integration.
-   double t = 0.0;
-   ODESolver *ode_solver = NULL;
-   EPICSolver *epic_solver = NULL;
-   switch (ode_solver_type)
-   {
-      // MFEM explicit methods
-      case 1: ode_solver = new ForwardEulerSolver; break;
-      case 2: ode_solver = new RK2Solver(0.5); break; // midpoint method
-      case 3: ode_solver = new RK3SSPSolver; break;
-      case 4: ode_solver = new RK4Solver; break;
-      // MFEM implicit L-stable methods
-      case 5: ode_solver = new BackwardEulerSolver; break;
-      case 6: ode_solver = new SDIRK23Solver(2); break;
-      case 7: ode_solver = new SDIRK33Solver; break;
-      // EPIC
-      case 8:
-         epic_solver = new EPICSolver();
-	 epic_solver->Init(oper);
-	 ode_solver = epic_solver;
-	 break;
-   }
-
-   // Initialize MFEM integrators
-   ode_solver->Init(oper);
-
-   // 10. Perform time-integration (looping over the time iterations, ti, with a
-   //     time-step dt).
-   if (myid == 0)
-   {
-      cout << "Integrating the ODE ..." << endl;
-   }
-   tic_toc.Clear();
-   tic_toc.Start();
-
-   bool last_step = false;
-   for (int ti = 1; !last_step; ti++)
-   {
-      double dt_real = min(dt, t_final - t);
-
-      // Note that since we are using the "one-step" mode of the SUNDIALS
-      // solvers, they will, generally, step over the final time and will not
-      // explicitly perform the interpolation to t_final as they do in the
-      // "normal" step mode.
-
-      ode_solver->Step(u, t, dt_real);
-
-      last_step = (t >= t_final - 1e-8*dt);
-
-      if (last_step || (ti % vis_steps) == 0)
-      {
-         if (myid == 0)
-         {
-            cout << "step " << ti << ", t = " << t << endl;
-         }
-
-         u_gf.SetFromTrueDofs(u);
-         if (visualization)
-         {
-            sout << "parallel " << num_procs << " " << myid << "\n";
-            sout << "solution\n" << *pmesh << u_gf << flush;
-         }
-
-         if (visit)
-         {
-            visit_dc.SetCycle(ti);
-            visit_dc.SetTime(t);
-            visit_dc.Save();
-         }
-      }
-      oper.SetParameters(u);
-   }
-   tic_toc.Stop();
-   if (myid == 0)
-   {
-      cout << "Done, " << tic_toc.RealTime() << "s." << endl;
-   }
-
-   // 11. Save the final solution in parallel. This output can be viewed later
-   //     using GLVis: "glvis -np <np> -m ex16-mesh -g ex16-final".
-   {
-      ostringstream sol_name;
-      sol_name << "ex16-final." << setfill('0') << setw(6) << myid;
-      ofstream osol(sol_name.str().c_str());
-      osol.precision(precision);
-      u_gf.Save(osol);
-   }
-
-   // 12. Free the used memory.
-   delete ode_solver;
-   delete pmesh;
-
-   MPI_Finalize();
-
-   return 0;
-}
-
-ConductionOperator::ConductionOperator(ParFiniteElementSpace &f, double al,
-                                       double kap, const Vector &u)
-   : TimeDependentOperator(f.GetTrueVSize(), 0.0), fespace(f), M(NULL), K(NULL),
-     T(NULL),
-     M_solver(f.GetComm()), T_solver(f.GetComm()), z(height)
-{
-   const double rel_tol = 1e-8;
-
-   M = new ParBilinearForm(&fespace);
-   M->AddDomainIntegrator(new MassIntegrator());
-   M->Assemble(0); // keep sparsity pattern of M and K the same
-   M->FormSystemMatrix(ess_tdof_list, Mmat);
-
-   M_solver.iterative_mode = false;
-   M_solver.SetRelTol(rel_tol);
-   M_solver.SetAbsTol(0.0);
-   M_solver.SetMaxIter(100);
-   M_solver.SetPrintLevel(0);
-   M_prec.SetType(HypreSmoother::Jacobi);
-   M_solver.SetPreconditioner(M_prec);
-   M_solver.SetOperator(Mmat);
-
-   alpha = al;
-   kappa = kap;
-
-   T_solver.iterative_mode = false;
-   T_solver.SetRelTol(rel_tol);
-   T_solver.SetAbsTol(0.0);
-   T_solver.SetMaxIter(100);
-   T_solver.SetPrintLevel(0);
-   T_solver.SetPreconditioner(T_prec);
-
-   SetParameters(u);
-}
-
-void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
-{
-   // Compute:
-   //    du_dt = M^{-1}*-K(u)
-   // for du_dt
-   Kmat.Mult(u, z);
-   z.Neg(); // z = -z
-   M_solver.Mult(z, du_dt);
-}
-
-void ConductionOperator::ImplicitSolve(const double dt,
-                                       const Vector &u, Vector &du_dt)
-{
-   // Solve the equation:
-   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
-   // for du_dt
-   if (T) { delete T; }
-   T = Add(1.0, Mmat, dt, Kmat);
-   T_solver.SetOperator(*T);
-   Kmat.Mult(u, z);
-   z.Neg();
-   T_solver.Mult(z, du_dt);
-}
-
-int ConductionOperator::SUNImplicitSetup(const Vector &x,
-                                         const Vector &fx, int jok, int *jcur,
-                                         double gamma)
-{
-   // Setup the ODE Jacobian T = M + gamma K.
-   if (T) { delete T; }
-   T = Add(1.0, Mmat, gamma, Kmat);
-   T_solver.SetOperator(*T);
-   *jcur = 1;
-   return (0);
-}
-
-int ConductionOperator::SUNImplicitSolve(const Vector &b, Vector &x, double tol)
-{
-   // Solve the system A x = z => (M - gamma K) x = M b.
-   Mmat.Mult(b, z);
-   T_solver.Mult(z, x);
-   return (0);
-}
-
-void ConductionOperator::SetParameters(const Vector &u)
-{
-   ParGridFunction u_alpha_gf(&fespace);
-   u_alpha_gf.SetFromTrueDofs(u);
-   for (int i = 0; i < u_alpha_gf.Size(); i++)
-   {
-      u_alpha_gf(i) = kappa + alpha*u_alpha_gf(i);
-   }
-
-   delete K;
-   K = new ParBilinearForm(&fespace);
-
-   GridFunctionCoefficient u_coeff(&u_alpha_gf);
-
-   K->AddDomainIntegrator(new DiffusionIntegrator(u_coeff));
-   K->Assemble(0); // keep sparsity pattern of M and K the same
-   K->FormSystemMatrix(ess_tdof_list, Kmat);
-}
-
-ConductionOperator::~ConductionOperator()
-{
-   delete T;
-   delete M;
-   delete K;
-}
-
-double InitialTemperature(const Vector &x)
-{
-   if (x.Norml2() < 0.5)
-   {
-      return 2.0;
-   }
-   else
-   {
-      return 1.0;
-   }
-}
@@ -1,76 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Use the MFEM build directory
-MFEM_DIR ?= ../..
-MFEM_BUILD_DIR ?= ../..
-SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/epic/,)
-CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
-# Use the MFEM install directory
-# MFEM_INSTALL_DIR = ../../mfem
-# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
-
-MFEM_LIB_FILE = mfem_is_not_built
-include $(CONFIG_MK)
-
-SEQ_EXAMPLES = ex16
-PAR_EXAMPLES = ex16p
-ifeq ($(MFEM_USE_MPI),NO)
-   EXAMPLES = $(SEQ_EXAMPLES)
-else
-   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
-endif
-
-.SUFFIXES:
-.SUFFIXES: .o .cpp .mk
-.PHONY: all clean clean-build clean-exec
-
-# Remove built-in rule
-%: %.cpp
-
-# Replace the default implicit rule for *.cpp files
-%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
-
-all: $(EXAMPLES)
-
-ifeq ($(MFEM_USE_EPIC),NO)
-$(EXAMPLES):
-	$(error MFEM is not configured with EPIC)
-endif
-
-MFEM_TESTS = EXAMPLES
-include $(MFEM_TEST_MK)
-
-# Testing: Parallel vs. serial runs
-RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
-SERIAL_NAME := Serial EPIC example
-PARALLEL_NAME := Parallel EPIC example
-%-test-par: %
-	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME))
-%-test-seq: %
-	@$(call mfem-test,$<,, $(SERIAL_NAME))
-
-# Testing: "test" target and mfem-test* variables are defined in config/test.mk
-
-# Generate an error message if the MFEM library is not built and exit
-$(MFEM_LIB_FILE):
-	$(error The MFEM library is not built)
-
-clean: clean-build clean-exec
-
-clean-build:
-	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
-	rm -rf *.dSYM *.TVD.*breakpoints
-
-clean-exec:
-	@rm -f deformed.* velocity.* elastic_energy.*
-	@rm -f ex16.mesh ex16-mesh.* ex16-init.* ex16-final.* Example16*
@@ -1,81 +0,0 @@
-//                                MFEM Example 0
-//
-// Compile with: make ex0
-//
-// Sample runs:  ex0
-//               ex0 -m ../data/fichera.mesh
-//               ex0 -m ../data/square-disc.mesh -o 2
-//
-// Description: This example code demonstrates the most basic usage of MFEM to
-//              define a simple finite element discretization of the Laplace
-//              problem -Delta u = 1 with zero Dirichlet boundary conditions.
-//              General 2D/3D mesh files and finite element polynomial degrees
-//              can be specified by command line options.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command line options
-   const char *mesh_file = "../data/star.mesh";
-   int order = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order", "Finite element polynomial degree");
-   args.ParseCheck();
-
-   // 2. Read the mesh from the given mesh file, and refine once uniformly.
-   Mesh mesh(mesh_file);
-   mesh.UniformRefinement();
-
-   // 3. Define a finite element space on the mesh. Here we use H1 continuous
-   //    high-order Lagrange finite elements of the given order.
-   H1_FECollection fec(order, mesh.Dimension());
-   FiniteElementSpace fespace(&mesh, &fec);
-   cout << "Number of unknowns: " << fespace.GetTrueVSize() << endl;
-
-   // 4. Extract the list of all the boundary DOFs. These will be marked as
-   //    Dirichlet in order to enforce zero boundary conditions.
-   Array<int> boundary_dofs;
-   fespace.GetBoundaryTrueDofs(boundary_dofs);
-
-   // 5. Define the solution x as a finite element grid function in fespace. Set
-   //    the initial guess to zero, which also sets the boundary conditions.
-   GridFunction x(&fespace);
-   x = 0.0;
-
-   // 6. Set up the linear form b(.) corresponding to the right-hand side.
-   ConstantCoefficient one(1.0);
-   LinearForm b(&fespace);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
-
-   // 7. Set up the bilinear form a(.,.) corresponding to the -Delta operator.
-   BilinearForm a(&fespace);
-   a.AddDomainIntegrator(new DiffusionIntegrator);
-   a.Assemble();
-
-   // 8. Form the linear system A X = B. This includes eliminating boundary
-   //    conditions, applying AMR constraints, and other transformations.
-   SparseMatrix A;
-   Vector B, X;
-   a.FormLinearSystem(boundary_dofs, x, b, A, X, B);
-
-   // 9. Solve the system using PCG with symmetric Gauss-Seidel preconditioner.
-   GSSmoother M(A);
-   PCG(A, M, B, X, 1, 200, 1e-12, 0.0);
-
-   // 10. Recover the solution x as a grid function and save to file. The output
-   //     can be viewed using GLVis as follows: "glvis -m mesh.mesh -g sol.gf"
-   a.RecoverFEMSolution(X, b, x);
-   x.Save("sol.gf");
-   mesh.Save("mesh.mesh");
-
-   return 0;
-}
@@ -1,96 +0,0 @@
-//                       MFEM Example 0 - Parallel Version
-//
-// Compile with: make ex0p
-//
-// Sample runs:  mpirun -np 4 ex0p
-//               mpirun -np 4 ex0p -m ../data/fichera.mesh
-//               mpirun -np 4 ex0p -m ../data/square-disc.mesh -o 2
-//
-// Description: This example code demonstrates the most basic parallel usage of
-//              MFEM to define a simple finite element discretization of the
-//              Laplace problem -Delta u = 1 with zero Dirichlet boundary
-//              conditions. General 2D/3D serial mesh files and finite element
-//              polynomial degrees can be specified by command line options.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI
-   MPI_Session mpi(argc, argv);
-
-   // 2. Parse command line options
-   const char *mesh_file = "../data/star.mesh";
-   int order = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order", "Finite element polynomial degree");
-   args.ParseCheck();
-
-   // 3. Read the serial mesh from the given mesh file.
-   Mesh serial_mesh(mesh_file);
-
-   // 4. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh once in parallel to increase the resolution.
-   ParMesh mesh(MPI_COMM_WORLD, serial_mesh);
-   serial_mesh.Clear(); // the serial mesh is no longer needed
-   mesh.UniformRefinement();
-
-   // 5. Define a finite element space on the mesh. Here we use H1 continuous
-   //    high-order Lagrange finite elements of the given order.
-   H1_FECollection fec(order, mesh.Dimension());
-   ParFiniteElementSpace fespace(&mesh, &fec);
-   HYPRE_BigInt total_num_dofs = fespace.GlobalTrueVSize();
-   if (mpi.Root()) { cout << "Number of unknowns: " << total_num_dofs << endl; }
-
-   // 6. Extract the list of all the boundary DOFs. These will be marked as
-   //    Dirichlet in order to enforce zero boundary conditions.
-   Array<int> boundary_dofs;
-   fespace.GetBoundaryTrueDofs(boundary_dofs);
-
-   // 7. Define the solution x as a finite element grid function in fespace. Set
-   //    the initial guess to zero, which also sets the boundary conditions.
-   ParGridFunction x(&fespace);
-   x = 0.0;
-
-   // 8. Set up the linear form b(.) corresponding to the right-hand side.
-   ConstantCoefficient one(1.0);
-   ParLinearForm b(&fespace);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
-
-   // 9. Set up the bilinear form a(.,.) corresponding to the -Delta operator.
-   ParBilinearForm a(&fespace);
-   a.AddDomainIntegrator(new DiffusionIntegrator);
-   a.Assemble();
-
-   // 10. Form the linear system A X = B. This includes eliminating boundary
-   //     conditions, applying AMR constraints, parallel assembly, etc.
-   HypreParMatrix A;
-   Vector B, X;
-   a.FormLinearSystem(boundary_dofs, x, b, A, X, B);
-
-   // 11. Solve the system using PCG with hypre's BoomerAMG preconditioner.
-   HypreBoomerAMG M(A);
-   CGSolver cg(MPI_COMM_WORLD);
-   cg.SetRelTol(1e-12);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(1);
-   cg.SetPreconditioner(M);
-   cg.SetOperator(A);
-   cg.Mult(B, X);
-
-   // 12. Recover the solution x as a grid function and save to file. The output
-   //     can be viewed using GLVis as follows: "glvis -np <np> -m mesh -g sol"
-   a.RecoverFEMSolution(X, b, x);
-   x.Save("sol");
-   mesh.Save("mesh");
-
-   return 0;
-}
@@ -35,7 +35,6 @@
 //               ex1 -pa -d raja-omp
 //               ex1 -pa -d occa-omp
 //               ex1 -pa -d ceed-cpu
-//               ex1 -pa -d ceed-cpu -o 4 -a
 //             * ex1 -pa -d ceed-cuda
 //             * ex1 -pa -d ceed-hip
 //               ex1 -pa -d ceed-cuda:/gpu/cuda/shared
@@ -74,7 +73,6 @@ int main(int argc, char *argv[])
   bool pa = false;
   const char *device_config = "cpu";
   bool visualization = true;
-   bool algebraic_ceed = false;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -88,10 +86,6 @@ int main(int argc, char *argv[])
                  "--no-partial-assembly", "Enable Partial Assembly.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
-#ifdef MFEM_USE_CEED
-   args.AddOption(&algebraic_ceed, "-a", "--algebraic", "-no-a", "--no-algebraic",
-                  "Use algebraic Ceed solver");
-#endif
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -213,20 +207,12 @@ int main(int argc, char *argv[])
      umf_solver.Mult(B, X);
 #endif
   }
-   else
+   else // Jacobi preconditioning in partial assembly mode
   {
      if (UsesTensorBasis(fespace))
      {
-         if (algebraic_ceed)
-         {
-            ceed::AlgebraicSolver M(a, ess_tdof_list);
-            PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
-         }
-         else
-         {
-            OperatorJacobiSmoother M(a, ess_tdof_list);
-            PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
-         }
+         OperatorJacobiSmoother M(a, ess_tdof_list);
+         PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
      }
      else
      {
@@ -293,7 +293,7 @@ int main(int argc, char *argv[])
   H1_FECollection fe_coll(order, dim);
   ParFiniteElementSpace fespace(pmesh, &fe_coll, dim);

-   HYPRE_BigInt glob_size = fespace.GlobalTrueVSize();
+   HYPRE_Int glob_size = fespace.GlobalTrueVSize();
   if (myid == 0)
   {
      cout << "Number of velocity/deformation unknowns: " << glob_size << endl;
@@ -174,7 +174,7 @@ int main(int argc, char *argv[])
      fec = new H1_FECollection(order = 1, dim);
   }
   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_BigInt size = fespace->GlobalTrueVSize();
+   HYPRE_Int size = fespace->GlobalTrueVSize();
   if (myid == 0)
   {
      cout << "Number of unknowns: " << size << endl;
@@ -165,7 +165,7 @@ int main(int argc, char *argv[])
      fec = new H1_FECollection(order, dim);
      fespace = new ParFiniteElementSpace(pmesh, fec, dim, Ordering::byVDIM);
   }
-   HYPRE_BigInt size = fespace->GlobalTrueVSize();
+   HYPRE_Int size = fespace->GlobalTrueVSize();
   if (myid == 0)
   {
      cout << "Number of unknowns: " << size << endl
@@ -55,7 +55,6 @@ int main(int argc, char *argv[])
   int order = 1;
   int nev = 5;
   bool visualization = 1;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -72,8 +71,6 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -89,18 +86,13 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors. We
+   // 3. Read the (serial) mesh from the given mesh file on all processors. We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 5. Refine the serial mesh on all processors to increase the resolution. In
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ref_levels' of uniform refinement (2 by default, or
   //    specified on the command line with -rs).
   for (int lev = 0; lev < ser_ref_levels; lev++)
@@ -108,7 +100,7 @@ int main(int argc, char *argv[])
      mesh->UniformRefinement();
   }

-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution (1 time by
   //    default, or specified on the command line with -rp). Once the parallel
   //    mesh is defined, the serial mesh can be deleted.
@@ -120,17 +112,17 @@ int main(int argc, char *argv[])
   }
   pmesh->ReorientTetMesh();

-   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Nedelec finite elements of the specified order.
   FiniteElementCollection *fec = new ND_FECollection(order, dim);
   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_BigInt size = fespace->GlobalTrueVSize();
+   HYPRE_Int size = fespace->GlobalTrueVSize();
   if (myid == 0)
   {
      cout << "Number of unknowns: " << size << endl;
   }

-   // 8. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
+   // 7. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
   //    element space. The first corresponds to the curl curl, while the second
   //    is a simple mass matrix needed on the right hand side of the
   //    generalized eigenvalue problem below. The boundary conditions are
@@ -172,7 +164,7 @@ int main(int argc, char *argv[])
   delete a;
   delete m;

-   // 9. Define and configure the AME eigensolver and the AMS preconditioner for
+   // 8. Define and configure the AME eigensolver and the AMS preconditioner for
   //    A to be used within the solver. Set the matrices which define the
   //    generalized eigenproblem A x = lambda M x.
   HypreAMS *ams = new HypreAMS(*A,fespace);
@@ -188,15 +180,15 @@ int main(int argc, char *argv[])
   ame->SetMassMatrix(*M);
   ame->SetOperator(*A);

-   // 10. Compute the eigenmodes and extract the array of eigenvalues. Define a
-   //     parallel grid function to represent each of the eigenmodes returned by
-   //     the solver.
+   // 9. Compute the eigenmodes and extract the array of eigenvalues. Define a
+   //    parallel grid function to represent each of the eigenmodes returned by
+   //    the solver.
   Array<double> eigenvalues;
   ame->Solve();
   ame->GetEigenvalues(eigenvalues);
   ParGridFunction x(fespace);

-   // 11. Save the refined mesh and the modes in parallel. This output can be
+   // 10. Save the refined mesh and the modes in parallel. This output can be
   //     viewed later using GLVis: "glvis -np <np> -m mesh -g mode".
   {
      ostringstream mesh_name, mode_name;
@@ -221,7 +213,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 12. Send the solution by socket to a GLVis server.
+   // 11. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -261,7 +253,7 @@ int main(int argc, char *argv[])
      mode_sock.close();
   }

-   // 13. Free the used memory.
+   // 12. Free the used memory.
   delete ame;
   delete ams;
   delete M;
@@ -166,7 +166,7 @@ int main(int argc, char *argv[])
   //    use discontinuous finite elements of the specified order >= 0.
   FiniteElementCollection *fec = new DG_FECollection(order, dim);
   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_BigInt size = fespace->GlobalTrueVSize();
+   HYPRE_Int size = fespace->GlobalTrueVSize();
   if (myid == 0)
   {
      cout << "Number of unknowns: " << size << endl;
@@ -19,14 +19,6 @@
 //               ex15 -m ../data/square-disc.mesh
 //               ex15 -m ../data/escher.mesh -r 2 -tf 0.3
 //
-//               Kelly estimator:
-//
-//               ex15 -est 1 -e 0.0001
-//               ex15 -est 1 -o 1 -y 0.4
-//               ex15 -est 1 -o 4 -y 0.1
-//               ex15 -est 1 -n 5
-//               ex15 -est 1 -p 1 -n 3
-//
 // Description:  Building on Example 6, this example demonstrates dynamic AMR.
 //               The mesh is adapted to a time-dependent solution by refinement
 //               as well as by derefinement. For simplicity, the solution is
@@ -36,10 +28,10 @@
 //               At each outer iteration the right hand side function is changed
 //               to mimic a time dependent problem.  Within each inner iteration
 //               the problem is solved on a sequence of meshes which are locally
-//               refined according to a simple ZZ or Kelly error estimator.  At
-//               the end of the inner iteration the error estimates are also
-//               used to identify any elements which may be over-refined and a
-//               single derefinement step is performed.
+//               refined according to a simple ZZ error estimator.  At the end
+//               of the inner iteration the error estimates are also used to
+//               identify any elements which may be over-refined and a single
+//               derefinement step is performed.
 //
 //               The example demonstrates MFEM's capability to refine and
 //               derefine nonconforming meshes, in 2D and 3D, and on linear,
@@ -86,7 +78,6 @@ int main(int argc, char *argv[])
   int nc_limit = 3;         // maximum level of hanging nodes
   bool visualization = true;
   bool visit = false;
-   int which_estimator = 0;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -107,9 +98,6 @@ int main(int argc, char *argv[])
                  "Maximum level of hanging nodes.");
   args.AddOption(&t_final, "-tf", "--t-final",
                  "Final time; start time is 0.");
-   args.AddOption(&which_estimator, "-est", "--estimator",
-                  "Which estimator to use: "
-                  "0 = ZZ, 1 = Kelly. Defaults to ZZ.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -198,38 +186,19 @@ int main(int argc, char *argv[])
   visit_dc.RegisterField("solution", &x);
   int vis_cycle = 0;

-   // 9. As in Example 6, we set up an estimator that will be used to obtain
-   //    element error indicators. The integrator needs to provide the method
-   //    ComputeElementFlux. The smoothed flux space is a vector valued H1 (ZZ)
-   //    or L2 (Kelly) space here.
-   L2_FECollection flux_fec(order, dim);
-   ErrorEstimator* estimator{nullptr};
-
-   switch (which_estimator)
-   {
-      case 1:
-      {
-         auto flux_fes = new FiniteElementSpace(&mesh, &flux_fec, sdim);
-         estimator = new KellyErrorEstimator(*integ, x, flux_fes);
-         break;
-      }
-
-      default:
-         std::cout << "Unknown estimator. Falling back to ZZ." << std::endl;
-      case 0:
-      {
-         auto flux_fes = new FiniteElementSpace(&mesh, &fec, sdim);
-         estimator = new ZienkiewiczZhuEstimator(*integ, x, flux_fes);
-         break;
-      }
-   }
+   // 9. As in Example 6, we set up a Zienkiewicz-Zhu estimator that will be
+   //    used to obtain element error indicators. The integrator needs to
+   //    provide the method ComputeElementFlux. The smoothed flux space is a
+   //    vector valued H1 space here.
+   FiniteElementSpace flux_fespace(&mesh, &fec, sdim);
+   ZienkiewiczZhuEstimator estimator(*integ, x, flux_fespace);

   // 10. As in Example 6, we also need a refiner. This time the refinement
   //     strategy is based on a fixed threshold that is applied locally to each
   //     element. The global threshold is turned off by setting the total error
   //     fraction to zero. We also enforce a maximum refinement ratio between
   //     adjacent elements.
-   ThresholdRefiner refiner(*estimator);
+   ThresholdRefiner refiner(estimator);
   refiner.SetTotalErrorFraction(0.0); // use purely local threshold
   refiner.SetLocalErrorGoal(max_elem_error);
   refiner.PreferConformingRefinement();
@@ -238,7 +207,7 @@ int main(int argc, char *argv[])
   // 11. A derefiner selects groups of elements that can be coarsened to form
   //     a larger element. A conservative enough threshold needs to be set to
   //     prevent derefining elements that would immediately be refined again.
-   ThresholdDerefiner derefiner(*estimator);
+   ThresholdDerefiner derefiner(estimator);
   derefiner.SetThreshold(hysteresis * max_elem_error);
   derefiner.SetNCLimit(nc_limit);

@@ -339,8 +308,6 @@ int main(int argc, char *argv[])
      b.Update();
   }

-   delete estimator;
-
   return 0;
 }

@@ -223,14 +223,13 @@ int main(int argc, char *argv[])
   visit_dc.RegisterField("solution", &x);
   int vis_cycle = 0;

-   // 10. As in Example 6p, we set up an estimator that will be used to obtain
-   //     element error indicators. The integrator needs to provide the method
-   //     ComputeElementFlux. We supply an L2 space for the discontinuous flux
-   //     and an H(div) space for the smoothed flux.
+   // 10. As in Example 6p, we set up a Zienkiewicz-Zhu estimator that will be
+   //     used to obtain element error indicators. The integrator needs to
+   //     provide the method ComputeElementFlux. We supply an L2 space for the
+   //     discontinuous flux and an H(div) space for the smoothed flux.
   L2_FECollection flux_fec(order, dim);
   RT_FECollection smooth_flux_fec(order-1, dim);
-   ErrorEstimator* estimator{nullptr};
-
+   ErrorEstimator* estimator;
   switch (which_estimator)
   {
      case 1:
@@ -249,7 +248,7 @@ int main(int argc, char *argv[])
      default:
         if (myid == 0)
         {
-            std::cout << "Unknown estimator. Falling back to L2ZZ." << std::endl;
+            std::cout << "Unkown estimator. Falling back to L2ZZ." << std::endl;
         }
      case 0:
      {
@@ -301,7 +300,7 @@ int main(int argc, char *argv[])
      //     time step resolved to the prescribed tolerance in each element.
      for (int ref_it = 1; ; ref_it++)
      {
-         HYPRE_BigInt global_dofs = fespace.GlobalTrueVSize();
+         HYPRE_Int global_dofs = fespace.GlobalTrueVSize();
         if (myid == 0)
         {
            cout << "Iteration: " << ref_it << ", number of unknowns: "
@@ -213,7 +213,7 @@ int main(int argc, char *argv[])
   H1_FECollection fe_coll(order, dim);
   ParFiniteElementSpace fespace(pmesh, &fe_coll);

-   HYPRE_BigInt fe_size = fespace.GlobalTrueVSize();
+   int fe_size = fespace.GlobalTrueVSize();
   if (myid == 0)
   {
      cout << "Number of temperature unknowns: " << fe_size << endl;
@@ -190,7 +190,7 @@ int main(int argc, char *argv[])
   DG_FECollection fec(order, dim, BasisType::GaussLobatto);
   ParFiniteElementSpace fespace(&pmesh, &fec, dim, Ordering::byVDIM);

-   HYPRE_BigInt glob_size = fespace.GlobalTrueVSize();
+   HYPRE_Int glob_size = fespace.GlobalTrueVSize();
   if (mpi.Root())
   {
      cout << "Number of finite element unknowns: " << glob_size
@@ -35,8 +35,8 @@ private:
   void GetFlux(const DenseMatrix &state, DenseTensor &flux) const;

 public:
-   FE_Evolution(FiniteElementSpace &vfes_,
-                Operator &A_, SparseMatrix &Aflux_);
+   FE_Evolution(FiniteElementSpace &_vfes,
+                Operator &_A, SparseMatrix &_Aflux);

   virtual void Mult(const Vector &x, Vector &y) const;

@@ -99,13 +99,13 @@ public:
 };

 // Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(FiniteElementSpace &vfes_,
-                           Operator &A_, SparseMatrix &Aflux_)
-   : TimeDependentOperator(A_.Height()),
-     dim(vfes_.GetFE(0)->GetDim()),
-     vfes(vfes_),
-     A(A_),
-     Aflux(Aflux_),
+FE_Evolution::FE_Evolution(FiniteElementSpace &_vfes,
+                           Operator &_A, SparseMatrix &_Aflux)
+   : TimeDependentOperator(_A.Height()),
+     dim(_vfes.GetFE(0)->GetDim()),
+     vfes(_vfes),
+     A(_A),
+     Aflux(_Aflux),
     Me_inv(vfes.GetFE(0)->GetDof(), vfes.GetFE(0)->GetDof(), vfes.GetNE()),
     state(num_equation),
     f(num_equation, dim),
@@ -171,7 +171,7 @@ int main(int argc, char *argv[])
   // This example depends on this ordering of the space.
   MFEM_ASSERT(fes.GetOrdering() == Ordering::byNODES, "");

-   HYPRE_BigInt glob_size = vfes.GlobalTrueVSize();
+   HYPRE_Int glob_size = vfes.GlobalTrueVSize();
   if (mpi.Root()) { cout << "Number of unknowns: " << glob_size << endl; }

   // 8. Define the initial conditions, save the corresponding mesh and grid
@@ -196,12 +196,6 @@ void InitialDeformation(const Vector &x, Vector &y);

 int main(int argc, char *argv[])
 {
-#ifdef HYPRE_USING_CUDA
-   cout << "\nAs of mfem-4.3 and hypre-2.22.0 (July 2021) this example\n"
-        << "is NOT supported with the CUDA version of hypre.\n\n";
-   return 255;
-#endif
-
   // 1. Initialize MPI
   MPI_Session mpi;
   const int myid = mpi.WorldRank();
@@ -291,8 +285,8 @@ int main(int argc, char *argv[])
   spaces[0] = &R_space;
   spaces[1] = &W_space;

-   HYPRE_BigInt glob_R_size = R_space.GlobalTrueVSize();
-   HYPRE_BigInt glob_W_size = W_space.GlobalTrueVSize();
+   HYPRE_Int glob_R_size = R_space.GlobalTrueVSize();
+   HYPRE_Int glob_W_size = W_space.GlobalTrueVSize();

   // 8. Define the Dirichlet conditions (set to boundary attribute 1 and 2)
   Array<Array<int> *> ess_bdr(2);
@@ -444,19 +438,15 @@ JacobianPreconditioner::JacobianPreconditioner(Array<ParFiniteElementSpace *>
 void JacobianPreconditioner::Mult(const Vector &k, Vector &y) const
 {
   // Extract the blocks from the input and output vectors
-   Vector disp_in;
-   disp_in.MakeRef(const_cast<Vector&>(k), block_trueOffsets[0],
-                   block_trueOffsets[1]-block_trueOffsets[0]);
-   Vector pres_in;
-   pres_in.MakeRef(const_cast<Vector&>(k), block_trueOffsets[1],
-                   block_trueOffsets[2]-block_trueOffsets[1]);
+   Vector disp_in(k.GetData() + block_trueOffsets[0],
+                  block_trueOffsets[1]-block_trueOffsets[0]);
+   Vector pres_in(k.GetData() + block_trueOffsets[1],
+                  block_trueOffsets[2]-block_trueOffsets[1]);

-   Vector disp_out;
-   disp_out.MakeRef(y, block_trueOffsets[0],
-                    block_trueOffsets[1]-block_trueOffsets[0]);
-   Vector pres_out;
-   pres_out.MakeRef(y, block_trueOffsets[1],
-                    block_trueOffsets[2]-block_trueOffsets[1]);
+   Vector disp_out(y.GetData() + block_trueOffsets[0],
+                   block_trueOffsets[1]-block_trueOffsets[0]);
+   Vector pres_out(y.GetData() + block_trueOffsets[1],
+                   block_trueOffsets[2]-block_trueOffsets[1]);

   Vector temp(block_trueOffsets[1]-block_trueOffsets[0]);
   Vector temp2(block_trueOffsets[1]-block_trueOffsets[0]);
@@ -469,9 +459,6 @@ void JacobianPreconditioner::Mult(const Vector &k, Vector &y) const
   subtract(disp_in, temp, temp2);

   stiff_pcg->Mult(temp2, disp_out);
-
-   disp_out.SyncAliasMemory(y);
-   pres_out.SyncAliasMemory(y);
 }

 void JacobianPreconditioner::SetOperator(const Operator &op)
@@ -486,10 +473,7 @@ void JacobianPreconditioner::SetOperator(const Operator &op)

      if (!spaces[0]->GetParMesh()->Nonconforming())
      {
-#ifndef HYPRE_USING_CUDA
-         // Not available yet when hypre is built with CUDA
         stiff_prec_amg->SetElasticityOptions(spaces[0]);
-#endif
      }

      stiff_prec = stiff_prec_amg;
@@ -32,7 +32,6 @@
 //               mpirun -np 4 ex1p -pa -d occa-cuda
 //               mpirun -np 4 ex1p -pa -d raja-omp
 //               mpirun -np 4 ex1p -pa -d ceed-cpu
-//               mpirun -np 4 ex1p -pa -d ceed-cpu -o 4 -a
 //             * mpirun -np 4 ex1p -pa -d ceed-cuda
 //             * mpirun -np 4 ex1p -pa -d ceed-hip
 //               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
@@ -63,9 +62,10 @@ using namespace mfem;
 int main(int argc, char *argv[])
 {
   // 1. Initialize MPI.
-   MPI_Session mpi;
-   int num_procs = mpi.WorldSize();
-   int myid = mpi.WorldRank();
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);

   // 2. Parse command-line options.
   const char *mesh_file = "../data/star.mesh";
@@ -74,7 +74,6 @@ int main(int argc, char *argv[])
   bool pa = false;
   const char *device_config = "cpu";
   bool visualization = true;
-   bool algebraic_ceed = false;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -88,11 +87,6 @@ int main(int argc, char *argv[])
                  "--no-partial-assembly", "Enable Partial Assembly.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
-#ifdef MFEM_USE_CEED
-   args.AddOption(&algebraic_ceed, "-a", "--algebraic",
-                  "-no-a", "--no-algebraic",
-                  "Use algebraic Ceed solver");
-#endif
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -103,6 +97,7 @@ int main(int argc, char *argv[])
      {
         args.PrintUsage(cout);
      }
+      MPI_Finalize();
      return 1;
   }
   if (myid == 0)
@@ -172,7 +167,7 @@ int main(int argc, char *argv[])
      delete_fec = true;
   }
   ParFiniteElementSpace fespace(&pmesh, fec);
-   HYPRE_BigInt size = fespace.GlobalTrueVSize();
+   HYPRE_Int size = fespace.GlobalTrueVSize();
   if (myid == 0)
   {
      cout << "Number of finite element unknowns: " << size << endl;
@@ -198,15 +193,15 @@ int main(int argc, char *argv[])
   b.AddDomainIntegrator(new DomainLFIntegrator(one));
   b.Assemble();

-   // 10. Define the solution vector x as a parallel finite element grid
-   //     function corresponding to fespace. Initialize x with initial guess of
-   //     zero, which satisfies the boundary conditions.
+   // 10. Define the solution vector x as a parallel finite element grid function
+   //     corresponding to fespace. Initialize x with initial guess of zero,
+   //     which satisfies the boundary conditions.
   ParGridFunction x(&fespace);
   x = 0.0;

   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the
-   //     Diffusion domain integrator.
+   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
+   //     domain integrator.
   ParBilinearForm a(&fespace);
   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   a.AddDomainIntegrator(new DiffusionIntegrator(one));
@@ -230,14 +225,7 @@ int main(int argc, char *argv[])
   {
      if (UsesTensorBasis(fespace))
      {
-         if (algebraic_ceed)
-         {
-            prec = new ceed::AlgebraicSolver(a, ess_tdof_list);
-         }
-         else
-         {
-            prec = new OperatorJacobiSmoother(a, ess_tdof_list);
-         }
+         prec = new OperatorJacobiSmoother(a, ess_tdof_list);
      }
   }
   else
@@ -289,6 +277,7 @@ int main(int argc, char *argv[])
   {
      delete fec;
   }
+   MPI_Finalize();

   return 0;
 }
@@ -3,10 +3,6 @@
 // Compile with: make ex20
 //
 // Sample runs:  ex20
-//               ex20 -p 1 -o 1 -n 120 -dt 0.1
-//               ex20 -p 1 -o 2 -n 60 -dt 0.2
-//               ex20 -p 1 -o 3 -n 40 -dt 0.3
-//               ex20 -p 1 -o 4 -n 30 -dt 0.4
 //
 // Description: This example demonstrates the use of the variable order,
 //              symplectic ODE integration algorithm.  Symplectic integration
@@ -235,7 +231,6 @@ int main(int argc, char *argv[])
   // 9. Finalize the GLVis output
   if (visualization)
   {
-      mesh.FinalizeQuadMesh(1);
      H1_FECollection fec(order = 1, 2);
      FiniteElementSpace fespace(&mesh, &fec);
      GridFunction energy(&fespace);
@@ -3,10 +3,6 @@
 // Compile with: make ex20p
 //
 // Sample runs:  mpirun -np 4 ex20p
-//               mpirun -np 4 ex20p -p 1 -o 1 -n 120 -dt 0.1
-//               mpirun -np 4 ex20p -p 1 -o 2 -n 60 -dt 0.2
-//               mpirun -np 4 ex20p -p 1 -o 3 -n 40 -dt 0.3
-//               mpirun -np 4 ex20p -p 1 -o 4 -n 30 -dt 0.4
 //
 // Description: This example demonstrates the use of the variable order,
 //              symplectic ODE integration algorithm.  Symplectic integration
@@ -172,7 +168,7 @@ int main(int argc, char *argv[])
   }

   // 6. Create a Mesh for visualization in phase space
-   int nverts = (visualization) ? 2*num_procs*(nsteps+1) : 0;
+   int nverts = (visualization) ? (num_procs+1)*(nsteps+1) : 0;
   int nelems = (visualization) ? (nsteps * num_procs) : 0;
   Mesh mesh(2, nverts, nelems, 0, 3);

@@ -194,9 +190,9 @@ int main(int argc, char *argv[])

         if (visualization)
         {
+            mesh.AddVertex(x0);
            for (int j = 0; j < num_procs; j++)
            {
-               mesh.AddVertex(x0);
               x1[0] = q(0);
               x1[1] = p(0);
               x1[2] = 0.0;
@@ -220,17 +216,17 @@ int main(int argc, char *argv[])
      if (visualization)
      {
         x0[2] = t;
+         mesh.AddVertex(x0);
         for (int j = 0; j < num_procs; j++)
         {
-            mesh.AddVertex(x0);
            x1[0] = q(0);
            x1[1] = p(0);
            x1[2] = t;
            mesh.AddVertex(x1);
-            v[0] = 2 * num_procs * i + 2 * j;
-            v[1] = 2 * num_procs * (i + 1) + 2 * j;
-            v[2] = 2 * num_procs * (i + 1) + 2 * j + 1;
-            v[3] = 2 * num_procs * i + 2 * j + 1;
+            v[0] = (num_procs + 1) * i;
+            v[1] = (num_procs + 1) * (i + 1);
+            v[2] = (num_procs + 1) * (i + 1) + j + 1;
+            v[3] = (num_procs + 1) * i + j + 1;
            mesh.AddQuad(v);
            part[num_procs * i + j] = j;
         }
@@ -211,7 +211,7 @@ int main(int argc, char *argv[])
   const int max_amr_itr = 20;
   for (int it = 0; it <= max_amr_itr; it++)
   {
-      HYPRE_BigInt global_dofs = fespace.GlobalTrueVSize();
+      HYPRE_Int global_dofs = fespace.GlobalTrueVSize();
      if (myid == 0)
      {
         cout << "\nAMR iteration " << it << endl;
@@ -213,7 +213,7 @@ int main(int argc, char *argv[])
      default: break; // This should be unreachable
   }
   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_BigInt size = fespace->GlobalTrueVSize();
+   HYPRE_Int size = fespace->GlobalTrueVSize();
   if (myid == 0)
   {
      cout << "Number of finite element unknowns: " << size << endl;
@@ -167,8 +167,8 @@ int main(int argc, char *argv[])
   ParFiniteElementSpace trial_fes(pmesh, trial_fec);
   ParFiniteElementSpace test_fes(pmesh, test_fec);

-   HYPRE_BigInt trial_size = trial_fes.GlobalTrueVSize();
-   HYPRE_BigInt test_size = test_fes.GlobalTrueVSize();
+   HYPRE_Int trial_size = trial_fes.GlobalTrueVSize();
+   HYPRE_Int test_size = test_fes.GlobalTrueVSize();

   if (myid == 0)
   {
@@ -326,7 +326,7 @@ int main(int argc, char *argv[])
   //    use the Nedelec finite elements of the specified order.
   FiniteElementCollection *fec = new ND_FECollection(order, dim);
   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_BigInt size = fespace->GlobalTrueVSize();
+   HYPRE_Int size = fespace->GlobalTrueVSize();
   if (myid == 0)
   {
      cout << "Number of finite element unknowns: " << size << endl;
@@ -105,7 +105,7 @@ private:
      Vector diag(fespace.GetTrueVSize());
      bfs.Last()->AssembleDiagonal(diag);

-      Solver* smoother = new OperatorChebyshevSmoother(*opr, diag,
+      Solver* smoother = new OperatorChebyshevSmoother(opr.Ptr(), diag,
                                                       *essentialTrueDofs.Last(), 2);
      AddLevel(opr.Ptr(), smoother, true, true);
   }
@@ -115,7 +115,7 @@ private:
      Vector diag(fespace.GetTrueVSize());
      bfs.Last()->AssembleDiagonal(diag);

-      Solver* smoother = new OperatorChebyshevSmoother(*opr, diag,
+      Solver* smoother = new OperatorChebyshevSmoother(opr.Ptr(), diag,
                                                       *essentialTrueDofs.Last(), 2, fespace.GetParMesh()->GetComm());

      AddLevel(opr.Ptr(), smoother, true, true);
@@ -224,7 +224,7 @@ int main(int argc, char *argv[])
      fespaces->AddOrderRefinedLevel(collections.Last());
   }

-   HYPRE_BigInt size = fespaces->GetFinestFESpace().GlobalTrueVSize();
+   HYPRE_Int size = fespaces->GetFinestFESpace().GlobalTrueVSize();
   if (myid == 0)
   {
      cout << "Number of finite element unknowns: " << size << endl;
@@ -176,7 +176,7 @@ int main(int argc, char *argv[])
      h1 ? (FiniteElementCollection*)new H1_FECollection(order, dim) :
      (FiniteElementCollection*)new DG_FECollection(order, dim);
   ParFiniteElementSpace fespace(&pmesh, fec);
-   HYPRE_BigInt size = fespace.GlobalTrueVSize();
+   HYPRE_Int size = fespace.GlobalTrueVSize();
   mfem::out << "Number of finite element unknowns: " << size << endl;

   // 6. Create "marker arrays" to define the portions of boundary associated
@@ -1,267 +0,0 @@
-//                                MFEM Example 28
-//
-// Compile with: make ex28
-//
-// Sample runs:  ex28
-//               ex28 --visit-datafiles
-//               ex28 --order 2
-//
-// Description:  Demonstrates a sliding boundary condition in an elasticity
-//               problem. A trapezoid, roughly as pictured below, is pushed
-//               from the right into a rigid notch. Normal displacement is
-//               restricted, but tangential movement is allowed, so the
-//               trapezoid compresses into the notch.
-//
-//                                       /-------+
-//               normal constrained --->/        | <--- boundary force (2)
-//               boundary (4)          /---------+
-//                                          ^
-//                                          |
-//                                normal constrained boundary (1)
-//
-//               This example demonstrates the use of the ConstrainedSolver
-//               framework.
-//
-//               We recommend viewing Example 2 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-#include <set>
-
-using namespace std;
-using namespace mfem;
-
-// Return a mesh with a single element with vertices (0, 0), (1, 0), (1, 1),
-// (offset, 1) to demonstrate boundary conditions on a surface that is not
-// axis-aligned.
-Mesh * build_trapezoid_mesh(double offset)
-{
-   MFEM_VERIFY(offset < 0.9, "offset is too large!");
-
-   const int dimension = 2;
-   const int nvt = 4; // vertices
-   const int nbe = 4; // num boundary elements
-   Mesh * mesh = new Mesh(dimension, nvt, 1, nbe);
-
-   // vertices
-   double vc[dimension];
-   vc[0] = 0.0; vc[1] = 0.0;
-   mesh->AddVertex(vc);
-   vc[0] = 1.0; vc[1] = 0.0;
-   mesh->AddVertex(vc);
-   vc[0] = offset; vc[1] = 1.0;
-   mesh->AddVertex(vc);
-   vc[0] = 1.0; vc[1] = 1.0;
-   mesh->AddVertex(vc);
-
-   // element
-   Array<int> vert(4);
-   vert[0] = 0; vert[1] = 1; vert[2] = 3; vert[3] = 2;
-   mesh->AddQuad(vert, 1);
-
-   // boundary
-   Array<int> sv(2);
-   sv[0] = 0; sv[1] = 1;
-   mesh->AddBdrSegment(sv, 1);
-   sv[0] = 1; sv[1] = 3;
-   mesh->AddBdrSegment(sv, 2);
-   sv[0] = 2; sv[1] = 3;
-   mesh->AddBdrSegment(sv, 3);
-   sv[0] = 0; sv[1] = 2;
-   mesh->AddBdrSegment(sv, 4);
-
-   mesh->FinalizeQuadMesh(1, 0, true);
-
-   return mesh;
-}
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   int order = 1;
-   bool visualization = 1;
-   double offset = 0.3;
-   bool visit = false;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&offset, "--offset", "--offset",
-                  "How much to offset the trapezoid.");
-   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
-                  "--no-visit-datafiles",
-                  "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // 2. Build a trapezoidal mesh with a single quadrilateral element, where
-   //    'offset' determines how far off it is from a rectangle.
-   Mesh *mesh = build_trapezoid_mesh(offset);
-   int dim = mesh->Dimension();
-
-   // 3. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
-   //    largest number that gives a final mesh with no more than 1,000
-   //    elements.
-   {
-      int ref_levels =
-         (int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh->UniformRefinement();
-      }
-   }
-
-   // 4. Define a finite element space on the mesh. Here we use vector finite
-   //    elements, i.e. dim copies of a scalar finite element space. The vector
-   //    dimension is specified by the last argument of the FiniteElementSpace
-   //    constructor.
-   FiniteElementCollection *fec = new H1_FECollection(order, dim);
-   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec, dim);
-   cout << "Number of finite element unknowns: " << fespace->GetTrueVSize()
-        << endl;
-   cout << "Assembling matrix and r.h.s... " << flush;
-
-   // 5. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, there are no essential boundary
-   //    conditions in the usual sense, but we leave the machinery here for
-   //    users to modify if they wish.
-   Array<int> ess_tdof_list, ess_bdr(mesh->bdr_attributes.Max());
-   ess_bdr = 0;
-   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-
-   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system. In this case, b_i equals the boundary integral
-   //    of f*phi_i where f represents a "push" force on the right side of the
-   //    trapezoid.
-   VectorArrayCoefficient f(dim);
-   for (int i = 0; i < dim-1; i++)
-   {
-      f.Set(i, new ConstantCoefficient(0.0));
-   }
-   {
-      Vector push_force(mesh->bdr_attributes.Max());
-      push_force = 0.0;
-      push_force(1) = -5.0e-2; // index 1 attribute 2
-      f.Set(0, new PWConstCoefficient(push_force));
-   }
-   LinearForm *b = new LinearForm(fespace);
-   b->AddBoundaryIntegrator(new VectorBoundaryLFIntegrator(f));
-   b->Assemble();
-
-   // 7. Define the solution vector x as a finite element grid function
-   //    corresponding to fespace.
-   GridFunction x(fespace);
-   x = 0.0;
-
-   // 8. Set up the bilinear form a(.,.) on the finite element space
-   //    corresponding to the linear elasticity integrator with piece-wise
-   //    constants coefficient lambda and mu. We use constant coefficients,
-   //    but see ex2 for how to set up piecewise constant coefficients based
-   //    on attribute.
-   Vector lambda(mesh->attributes.Max());
-   lambda = 1.0;
-   PWConstCoefficient lambda_func(lambda);
-   Vector mu(mesh->attributes.Max());
-   mu = 1.0;
-   PWConstCoefficient mu_func(mu);
-
-   BilinearForm *a = new BilinearForm(fespace);
-   a->AddDomainIntegrator(new ElasticityIntegrator(lambda_func, mu_func));
-
-   // 9. Assemble the bilinear form and the corresponding linear system,
-   //    applying any necessary transformations such as: eliminating boundary
-   //    conditions, applying conforming constraints for non-conforming AMR,
-   //    static condensation, etc.
-   a->Assemble();
-
-   SparseMatrix A;
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-   cout << "done." << endl;
-   cout << "Size of linear system: " << A.Height() << endl;
-
-   // 10. Set up constraint matrix to constrain normal displacement (but
-   //     allow tangential displacement) on specified boundaries.
-   Array<int> constraint_atts(2);
-   constraint_atts[0] = 1;  // attribute 1 bottom
-   constraint_atts[1] = 4;  // attribute 4 left side
-   Array<int> lagrange_rowstarts;
-   SparseMatrix* local_constraints =
-      BuildNormalConstraints(*fespace, constraint_atts, lagrange_rowstarts);
-
-   // 11. Define and apply an iterative solver for the constrained system
-   //     in saddle-point form with a Gauss-Seidel smoother for the
-   //     displacement block.
-   GSSmoother M(A);
-   SchurConstrainedSolver * solver =
-      new SchurConstrainedSolver(A, *local_constraints, M);
-   solver->SetRelTol(1e-5);
-   solver->SetMaxIter(2000);
-   solver->SetPrintLevel(1);
-   solver->Mult(B, X);
-
-   // 12. Recover the solution as a finite element grid function. Move the
-   //     mesh to reflect the displacement of the elastic body being
-   //     simulated, for purposes of output.
-   a->RecoverFEMSolution(X, *b, x);
-   mesh->SetNodalFESpace(fespace);
-   GridFunction *nodes = mesh->GetNodes();
-   *nodes += x;
-
-   // 13. Save the refined mesh and the solution in VisIt format.
-   if (visit)
-   {
-      VisItDataCollection visit_dc("ex28", mesh);
-      visit_dc.SetLevelsOfDetail(4);
-      visit_dc.RegisterField("displacement", &x);
-      visit_dc.Save();
-   }
-
-   // 14. Save the displaced mesh and the inverted solution (which gives the
-   //     backward displacements to the original grid). This output can be
-   //     viewed later using GLVis: "glvis -m displaced.mesh -g sol.gf".
-   {
-      x *= -1; // sign convention for GLVis displacements
-      ofstream mesh_ofs("displaced.mesh");
-      mesh_ofs.precision(8);
-      mesh->Print(mesh_ofs);
-      ofstream sol_ofs("sol.gf");
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 15. Send the above data by socket to a GLVis server.  Use the "n" and "b"
-   //     keys in GLVis to visualize the displacements.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *mesh << x << flush;
-   }
-
-   // 16. Free the used memory.
-   delete local_constraints;
-   delete solver;
-   delete a;
-   delete b;
-   if (fec)
-   {
-      delete fespace;
-      delete fec;
-   }
-   delete mesh;
-
-   return 0;
-}
@@ -1,373 +0,0 @@
-//                       MFEM Example 28 - Parallel Version
-//
-// Compile with: make ex28p
-//
-// Sample runs:  ex28p
-//               ex28p --visit-datafiles
-//               ex28p --order 4
-//               ex28p --penalty 1e+5
-//
-//               mpirun -np 4 ex28p
-//               mpirun -np 4 ex28p --penalty 1e+5
-//
-// Description:  Demonstrates a sliding boundary condition in an elasticity
-//               problem. A trapezoid, roughly as pictured below, is pushed
-//               from the right into a rigid notch. Normal displacement is
-//               restricted, but tangential movement is allowed, so the
-//               trapezoid compresses into the notch.
-//
-//                                       /-------+
-//               normal constrained --->/        | <--- boundary force (2)
-//               boundary (4)          /---------+
-//                                          ^
-//                                          |
-//                                normal constrained boundary (1)
-//
-//               This example demonstrates the use of the ConstrainedSolver
-//               framework.
-//
-//               We recommend viewing Example 2 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Return a mesh with a single element with vertices (0, 0), (1, 0), (1, 1),
-// (offset, 1) to demonstrate boundary conditions on a surface that is not
-// axis-aligned.
-Mesh * build_trapezoid_mesh(double offset)
-{
-   MFEM_VERIFY(offset < 0.9, "offset is too large!");
-
-   const int dimension = 2;
-   const int nvt = 4; // vertices
-   const int nbe = 4; // num boundary elements
-   Mesh * mesh = new Mesh(dimension, nvt, 1, nbe);
-
-   // vertices
-   double vc[dimension];
-   vc[0] = 0.0; vc[1] = 0.0;
-   mesh->AddVertex(vc);
-   vc[0] = 1.0; vc[1] = 0.0;
-   mesh->AddVertex(vc);
-   vc[0] = offset; vc[1] = 1.0;
-   mesh->AddVertex(vc);
-   vc[0] = 1.0; vc[1] = 1.0;
-   mesh->AddVertex(vc);
-
-   // element
-   Array<int> vert(4);
-   vert[0] = 0; vert[1] = 1; vert[2] = 3; vert[3] = 2;
-   mesh->AddQuad(vert, 1);
-
-   // boundary
-   Array<int> sv(2);
-   sv[0] = 0; sv[1] = 1;
-   mesh->AddBdrSegment(sv, 1);
-   sv[0] = 1; sv[1] = 3;
-   mesh->AddBdrSegment(sv, 2);
-   sv[0] = 2; sv[1] = 3;
-   mesh->AddBdrSegment(sv, 3);
-   sv[0] = 0; sv[1] = 2;
-   mesh->AddBdrSegment(sv, 4);
-
-   mesh->FinalizeQuadMesh(1, 0, true);
-
-   return mesh;
-}
-
-int main(int argc, char *argv[])
-{
-#ifdef HYPRE_USING_CUDA
-   cout << "\nAs of mfem-4.3 and hypre-2.22.0 (July 2021) this example\n"
-        << "is NOT supported with the CUDA version of hypre.\n\n";
-   return 255;
-#endif
-
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   int order = 1;
-   bool visualization = 1;
-   bool reorder_space = false;
-   double offset = 0.3;
-   bool visit = false;
-   double penalty = 0.0;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&reorder_space, "-nodes", "--by-nodes", "-vdim", "--by-vdim",
-                  "Use byNODES ordering of vector space instead of byVDIM");
-   args.AddOption(&offset, "--offset", "--offset",
-                  "How much to offset the trapezoid.");
-   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
-                  "--no-visit-datafiles",
-                  "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.AddOption(&penalty, "-p", "--penalty",
-                  "Penalty parameter; 0 means use elimination solver.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 3. Build a trapezoidal mesh with a single quadrilateral element, where
-   //    'offset' determines how far off it is from a rectangle.
-   Mesh *mesh = build_trapezoid_mesh(offset);
-   int dim = mesh->Dimension();
-
-   // 4. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 1,000 elements.
-   {
-      int ref_levels =
-         (int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh->UniformRefinement();
-      }
-   }
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   {
-      int par_ref_levels = 1;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         pmesh->UniformRefinement();
-      }
-   }
-
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
-   //    use vector finite elements, i.e. dim copies of a scalar finite element
-   //    space. We use the ordering by vector dimension (the last argument of
-   //    the FiniteElementSpace constructor) which is expected in the systems
-   //    version of BoomerAMG preconditioner. For NURBS meshes, we use the
-   //    (degree elevated) NURBS space associated with the mesh nodes.
-   FiniteElementCollection *fec;
-   ParFiniteElementSpace *fespace;
-   const bool use_nodal_fespace = pmesh->NURBSext;
-   if (use_nodal_fespace)
-   {
-      fec = NULL;
-      fespace = (ParFiniteElementSpace *)pmesh->GetNodes()->FESpace();
-   }
-   else
-   {
-      fec = new H1_FECollection(order, dim);
-      if (reorder_space)
-      {
-         fespace = new ParFiniteElementSpace(pmesh, fec, dim, Ordering::byNODES);
-      }
-      else
-      {
-         fespace = new ParFiniteElementSpace(pmesh, fec, dim, Ordering::byVDIM);
-      }
-   }
-   HYPRE_BigInt size = fespace->GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl
-           << "Assembling matrix and r.h.s... " << flush;
-   }
-
-   // 7. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, there are no essential boundary
-   //    conditions in the usual sense, but we leave the machinery here for
-   //    users to modify if they wish.
-   Array<int> ess_tdof_list, ess_bdr(pmesh->bdr_attributes.Max());
-   ess_bdr = 0;
-   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-
-   // 8. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system. In this case, b_i equals the
-   //    boundary integral of f*phi_i where f represents a "pull down" force on
-   //    the Neumann part of the boundary and phi_i are the basis functions in
-   //    the finite element fespace. The force is defined by the object f, which
-   //    is a vector of Coefficient objects. The fact that f is non-zero on
-   //    boundary attribute 2 is indicated by the use of piece-wise constants
-   //    coefficient for its last component.
-   VectorArrayCoefficient f(dim);
-   for (int i = 0; i < dim-1; i++)
-   {
-      f.Set(i, new ConstantCoefficient(0.0));
-   }
-
-   // 9. Put a leftward force on the right side of the trapezoid
-   {
-      Vector push_force(pmesh->bdr_attributes.Max());
-      push_force = 0.0;
-      push_force(1) = -5.0e-2; // index 1 attribute 2
-      f.Set(0, new PWConstCoefficient(push_force));
-   }
-
-   ParLinearForm *b = new ParLinearForm(fespace);
-   b->AddBoundaryIntegrator(new VectorBoundaryLFIntegrator(f));
-   b->Assemble();
-
-   // 10. Define the solution vector x as a parallel finite element grid
-   //     function corresponding to fespace. Initialize x with initial guess of
-   //     zero, which satisfies the boundary conditions.
-   ParGridFunction x(fespace);
-   x = 0.0;
-
-   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the linear elasticity integrator with piece-wise
-   //     constants coefficient lambda and mu. We use constant coefficients,
-   //     but see ex2 for how to set up piecewise constant coefficients based
-   //     on attribute.
-   Vector lambda(pmesh->attributes.Max());
-   lambda = 1.0;
-   PWConstCoefficient lambda_func(lambda);
-   Vector mu(pmesh->attributes.Max());
-   mu = 1.0;
-   PWConstCoefficient mu_func(mu);
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new ElasticityIntegrator(lambda_func, mu_func));
-
-   // 12. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, etc.
-   a->Assemble();
-
-   HypreParMatrix A;
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-   if (myid == 0)
-   {
-      cout << "done." << endl;
-      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
-   }
-
-   // 13. Set up constraint matrix to constrain normal displacement (but
-   //     allow tangential displacement) on specified boundaries.
-   Array<int> constraint_atts(2);
-   constraint_atts[0] = 1;  // attribute 1 bottom
-   constraint_atts[1] = 4;  // attribute 4 left side
-   Array<int> constraint_rowstarts;
-   SparseMatrix* local_constraints =
-      ParBuildNormalConstraints(*fespace, constraint_atts,
-                                constraint_rowstarts);
-
-   // 14. Define and apply a parallel PCG solver for the constrained system
-   //     where the normal boundary constraints have been separately eliminated
-   //     from the system.
-   ConstrainedSolver * solver;
-   if (penalty == 0.0)
-   {
-      solver = new EliminationCGSolver(A, *local_constraints,
-                                       constraint_rowstarts, dim,
-                                       reorder_space);
-   }
-   else
-   {
-      solver = new PenaltyPCGSolver(A, *local_constraints, penalty,
-                                    dim, reorder_space);
-   }
-
-   solver->SetRelTol(1e-8);
-   solver->SetMaxIter(500);
-   solver->SetPrintLevel(1);
-   solver->Mult(B, X);
-
-   // 15. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a->RecoverFEMSolution(X, *b, x);
-
-   // 16. For non-NURBS meshes, make the mesh curved based on the finite element
-   //     space. This means that we define the mesh elements through a fespace
-   //     based transformation of the reference element.  This allows us to save
-   //     the displaced mesh as a curved mesh when using high-order finite
-   //     element displacement field. We assume that the initial mesh (read from
-   //     the file) is not higher order curved mesh compared to the chosen FE
-   //     space.
-   if (!use_nodal_fespace)
-   {
-      pmesh->SetNodalFESpace(fespace);
-   }
-
-   GridFunction *nodes = pmesh->GetNodes();
-   *nodes += x;
-
-   // 17. Save the refined mesh and the solution in VisIt format.
-   if (visit)
-   {
-      VisItDataCollection visit_dc(MPI_COMM_WORLD, "ex28p", pmesh);
-      visit_dc.SetLevelsOfDetail(4);
-      visit_dc.RegisterField("displacement", &x);
-      visit_dc.Save();
-   }
-
-   // 18. Save in parallel the displaced mesh and the inverted solution (which
-   //     gives the backward displacements to the original grid). This output
-   //     can be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      x *= -1; // sign convention for GLVis displacements
-
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 19. Send the above data by socket to a GLVis server.  Use the "n" and "b"
-   //     keys in GLVis to visualize the displacements.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << x << flush;
-   }
-
-   // 20. Free the used memory.
-   delete local_constraints;
-   delete solver;
-   delete a;
-   delete b;
-   if (fec)
-   {
-      delete fespace;
-      delete fec;
-   }
-   delete pmesh;
-
-   // HYPRE_Finalize();
-   MPI_Finalize();
-
-   return 0;
-}
@@ -1,350 +0,0 @@
-//                                MFEM Example 29
-//
-// Compile with: make ex29
-//
-// Sample runs:  ex29
-//               ex29 -r 2 -sc
-//               ex29 -mt 3 -o 4 -sc
-//               ex29 -mt 3 -r 2 -o 4 -sc
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               finite element discretization of a PDE on a 2 dimensional
-//               surface embedded in a 3 dimensional domain. In this case we
-//               solve the Laplace problem -Div(sigma Grad u) = 1, with
-//               homogeneous Dirichlet boundary conditions, where sigma is an
-//               anisotropic diffusion constant defined as a 3x3 matrix
-//               coefficient.
-//
-//               This example demonstrates the use of finite element integrators
-//               on 2D domains with 3D coefficients.
-//
-//               We recommend viewing examples 1 and 7 before viewing this
-//               example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-Mesh * GetMesh(int type);
-
-void trans(const Vector &x, Vector &r);
-
-void sigmaFunc(const Vector &x, DenseMatrix &s);
-
-double uExact(const Vector &x)
-{
-   return (0.25 * (2.0 + x[0]) - x[2]) * (x[2] + 0.25 * (2.0 + x[0]));
-}
-
-void duExact(const Vector &x, Vector &du)
-{
-   du.SetSize(3);
-   du[0] = 0.125 * (2.0 + x[0]) * x[1] * x[1];
-   du[1] = -0.125 * (2.0 + x[0]) * x[0] * x[1];
-   du[2] = -2.0 * x[2];
-}
-
-void fluxExact(const Vector &x, Vector &f)
-{
-   f.SetSize(3);
-
-   DenseMatrix s(3);
-   sigmaFunc(x, s);
-
-   Vector du(3);
-   duExact(x, du);
-
-   s.Mult(du, f);
-   f *= -1.0;
-}
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   int order = 3;
-   int mesh_type = 4; // Default to Quadrilateral mesh
-   int mesh_order = 3;
-   int ref_levels = 0;
-   bool static_cond = false;
-   bool visualization = true;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_type, "-mt", "--mesh-type",
-                  "Mesh type: 3 - Triangular, 4 - Quadrilateral.");
-   args.AddOption(&mesh_order, "-mo", "--mesh-order",
-                  "Geometric order of the curved mesh.");
-   args.AddOption(&ref_levels, "-r", "--refine",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.ParseCheck();
-
-   // 2. Construct a quadrilateral or triangular mesh with the topology of a
-   //    cylindrical surface.
-   Mesh *mesh = GetMesh(mesh_type);
-   int dim = mesh->Dimension();
-
-   // 3. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement.
-   for (int l = 0; l < ref_levels; l++)
-   {
-      mesh->UniformRefinement();
-   }
-
-   // 4. Transform the mesh so that it has a more interesting geometry.
-   mesh->SetCurvature(mesh_order);
-   mesh->Transform(trans);
-
-   // 5. Define a finite element space on the mesh. Here we use continuous
-   //    Lagrange finite elements of the specified order.
-   H1_FECollection fec(order, dim);
-   FiniteElementSpace fespace(mesh, &fec);
-   cout << "Number of finite element unknowns: "
-        << fespace.GetTrueVSize() << endl;
-
-   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
-   //    In this example, the boundary conditions are defined by marking all
-   //    the boundary attributes from the mesh as essential (Dirichlet) and
-   //    converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (mesh->bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(mesh->bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
-   //    the basis functions in the finite element fespace.
-   LinearForm b(&fespace);
-   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
-
-   // 8. Define the solution vector x as a finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero,
-   //    which satisfies the boundary conditions.
-   GridFunction x(&fespace);
-   x = 0.0;
-
-   // 9. Set up the bilinear form a(.,.) on the finite element space
-   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //    domain integrator.
-   BilinearForm a(&fespace);
-   MatrixFunctionCoefficient sigma(3, sigmaFunc);
-   BilinearFormIntegrator *integ = new DiffusionIntegrator(sigma);
-   a.AddDomainIntegrator(integ);
-
-   // 10. Assemble the bilinear form and the corresponding linear system,
-   //     applying any necessary transformations such as: eliminating boundary
-   //     conditions, applying conforming constraints for non-conforming AMR,
-   //     static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
-   a.Assemble();
-
-   OperatorPtr A;
-   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-
-   cout << "Size of linear system: " << A->Height() << endl;
-
-   // 11. Solve the linear system A X = B.
-   //     Use a simple symmetric Gauss-Seidel preconditioner with PCG.
-   GSSmoother M((SparseMatrix&)(*A));
-   PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
-
-   // 12. Recover the solution as a finite element grid function.
-   a.RecoverFEMSolution(X, b, x);
-
-   // 13. Compute error in the solution and its flux
-   FunctionCoefficient uCoef(uExact);
-   double err = x.ComputeL2Error(uCoef);
-
-   cout << "|u - u_h|_2 = " << err << endl;
-
-   FiniteElementSpace flux_fespace(mesh, &fec, 3);
-   GridFunction flux(&flux_fespace);
-   x.ComputeFlux(*integ, flux); flux *= -1.0;
-
-   VectorFunctionCoefficient fluxCoef(3, fluxExact);
-   double flux_err = flux.ComputeL2Error(fluxCoef);
-
-   cout << "|f - f_h|_2 = " << flux_err << endl;
-
-   // 14. Save the refined mesh and the solution. This output can be viewed
-   //     later using GLVis: "glvis -m refined.mesh -g sol.gf".
-   ofstream mesh_ofs("refined.mesh");
-   mesh_ofs.precision(8);
-   mesh->Print(mesh_ofs);
-   ofstream sol_ofs("sol.gf");
-   sol_ofs.precision(8);
-   x.Save(sol_ofs);
-
-   // 15. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *mesh << x
-               << "window_title 'Solution'\n" << flush;
-
-      socketstream flux_sock(vishost, visport);
-      flux_sock.precision(8);
-      flux_sock << "solution\n" << *mesh << flux
-                << "keys vvv\n"
-                << "window_geometry 402 0 400 350\n"
-                << "window_title 'Flux'\n"  << flush;
-   }
-
-   // 16. Free the used memory.
-   delete mesh;
-
-   return 0;
-}
-
-// Defines a mesh consisting of four flat rectangular surfaces connected to form
-// a loop.
-Mesh * GetMesh(int type)
-{
-   Mesh * mesh = NULL;
-
-   if (type == 3)
-   {
-      mesh = new Mesh(2, 12, 16, 8, 3);
-
-      mesh->AddVertex(-1.0, -1.0, 0.0);
-      mesh->AddVertex( 1.0, -1.0, 0.0);
-      mesh->AddVertex( 1.0,  1.0, 0.0);
-      mesh->AddVertex(-1.0,  1.0, 0.0);
-      mesh->AddVertex(-1.0, -1.0, 1.0);
-      mesh->AddVertex( 1.0, -1.0, 1.0);
-      mesh->AddVertex( 1.0,  1.0, 1.0);
-      mesh->AddVertex(-1.0,  1.0, 1.0);
-      mesh->AddVertex( 0.0, -1.0, 0.5);
-      mesh->AddVertex( 1.0,  0.0, 0.5);
-      mesh->AddVertex( 0.0,  1.0, 0.5);
-      mesh->AddVertex(-1.0,  0.0, 0.5);
-
-      mesh->AddTriangle(0, 1, 8);
-      mesh->AddTriangle(1, 5, 8);
-      mesh->AddTriangle(5, 4, 8);
-      mesh->AddTriangle(4, 0, 8);
-      mesh->AddTriangle(1, 2, 9);
-      mesh->AddTriangle(2, 6, 9);
-      mesh->AddTriangle(6, 5, 9);
-      mesh->AddTriangle(5, 1, 9);
-      mesh->AddTriangle(2, 3, 10);
-      mesh->AddTriangle(3, 7, 10);
-      mesh->AddTriangle(7, 6, 10);
-      mesh->AddTriangle(6, 2, 10);
-      mesh->AddTriangle(3, 0, 11);
-      mesh->AddTriangle(0, 4, 11);
-      mesh->AddTriangle(4, 7, 11);
-      mesh->AddTriangle(7, 3, 11);
-
-      mesh->AddBdrSegment(0, 1, 1);
-      mesh->AddBdrSegment(1, 2, 1);
-      mesh->AddBdrSegment(2, 3, 1);
-      mesh->AddBdrSegment(3, 0, 1);
-      mesh->AddBdrSegment(5, 4, 2);
-      mesh->AddBdrSegment(6, 5, 2);
-      mesh->AddBdrSegment(7, 6, 2);
-      mesh->AddBdrSegment(4, 7, 2);
-   }
-   else if (type == 4)
-   {
-      mesh = new Mesh(2, 8, 4, 8, 3);
-
-      mesh->AddVertex(-1.0, -1.0, 0.0);
-      mesh->AddVertex( 1.0, -1.0, 0.0);
-      mesh->AddVertex( 1.0,  1.0, 0.0);
-      mesh->AddVertex(-1.0,  1.0, 0.0);
-      mesh->AddVertex(-1.0, -1.0, 1.0);
-      mesh->AddVertex( 1.0, -1.0, 1.0);
-      mesh->AddVertex( 1.0,  1.0, 1.0);
-      mesh->AddVertex(-1.0,  1.0, 1.0);
-
-      mesh->AddQuad(0, 1, 5, 4);
-      mesh->AddQuad(1, 2, 6, 5);
-      mesh->AddQuad(2, 3, 7, 6);
-      mesh->AddQuad(3, 0, 4, 7);
-
-      mesh->AddBdrSegment(0, 1, 1);
-      mesh->AddBdrSegment(1, 2, 1);
-      mesh->AddBdrSegment(2, 3, 1);
-      mesh->AddBdrSegment(3, 0, 1);
-      mesh->AddBdrSegment(5, 4, 2);
-      mesh->AddBdrSegment(6, 5, 2);
-      mesh->AddBdrSegment(7, 6, 2);
-      mesh->AddBdrSegment(4, 7, 2);
-   }
-   else
-   {
-      MFEM_ABORT("Unrecognized mesh type " << type << "!");
-   }
-   mesh->FinalizeTopology();
-
-   return mesh;
-}
-
-// Transforms the four-sided loop into a curved cylinder with skewed top and
-// base.
-void trans(const Vector &x, Vector &r)
-{
-   r.SetSize(3);
-
-   double tol = 1e-6;
-   double theta = 0.0;
-   if (fabs(x[1] + 1.0) < tol)
-   {
-      theta = 0.25 * M_PI * (x[0] - 2.0);
-   }
-   else if (fabs(x[0] - 1.0) < tol)
-   {
-      theta = 0.25 * M_PI * x[1];
-   }
-   else if (fabs(x[1] - 1.0) < tol)
-   {
-      theta = 0.25 * M_PI * (2.0 - x[0]);
-   }
-   else if (fabs(x[0] + 1.0) < tol)
-   {
-      theta = 0.25 * M_PI * (4.0 - x[1]);
-   }
-   else
-   {
-      cout << "side not recognized "
-           << x[0] << " " << x[1] << " " << x[2] << endl;
-   }
-
-   r[0] = cos(theta);
-   r[1] = sin(theta);
-   r[2] = 0.25 * (2.0 * x[2] - 1.0) * (r[0] + 2.0);
-}
-
-// Anisotropic diffusion coefficient
-void sigmaFunc(const Vector &x, DenseMatrix &s)
-{
-   s.SetSize(3);
-   double a = 17.0 - 2.0 * x[0] * (1.0 + x[0]);
-   s(0,0) = 0.5 + x[0] * x[0] * (8.0 / a - 0.5);
-   s(0,1) = x[0] * x[1] * (8.0 / a - 0.5);
-   s(0,2) = 0.0;
-   s(1,0) = s(0,1);
-   s(1,1) = 0.5 * x[0] * x[0] + 8.0 * x[1] * x[1] / a;
-   s(1,2) = 0.0;
-   s(2,0) = 0.0;
-   s(2,1) = 0.0;
-   s(2,2) = a / 32.0;
-}
@@ -1,391 +0,0 @@
-//                      MFEM Example 29 - Parallel Version
-//
-// Compile with: make ex29p
-//
-// Sample runs:  mpirun -np 4 ex29p
-//               mpirun -np 4 ex29p -sc
-//               mpirun -np 4 ex29p -mt 3 -o 3 -sc
-//               mpirun -np 4 ex29p -mt 3 -rs 1 -o 4 -sc
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               finite element discretization of a PDE on a 2 dimensional
-//               surface embedded in a 3 dimensional domain. In this case we
-//               solve the Laplace problem -Div(sigma Grad u) = 1, with
-//               homogeneous Dirichlet boundary conditions, where sigma is an
-//               anisotropic diffusion constant defined as a 3x3 matrix
-//               coefficient.
-//
-//               This example demonstrates the use of finite element integrators
-//               on 2D domains with 3D coefficients.
-//
-//               We recommend viewing examples 1 and 7 before viewing this
-//               example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-Mesh * GetMesh(int type);
-
-void trans(const Vector &x, Vector &r);
-
-void sigmaFunc(const Vector &x, DenseMatrix &s);
-
-double uExact(const Vector &x)
-{
-   return (0.25 * (2.0 + x[0]) - x[2]) * (x[2] + 0.25 * (2.0 + x[0]));
-}
-
-void duExact(const Vector &x, Vector &du)
-{
-   du.SetSize(3);
-   du[0] = 0.125 * (2.0 + x[0]) * x[1] * x[1];
-   du[1] = -0.125 * (2.0 + x[0]) * x[0] * x[1];
-   du[2] = -2.0 * x[2];
-}
-
-void fluxExact(const Vector &x, Vector &f)
-{
-   f.SetSize(3);
-
-   DenseMatrix s(3);
-   sigmaFunc(x, s);
-
-   Vector du(3);
-   duExact(x, du);
-
-   s.Mult(du, f);
-   f *= -1.0;
-}
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   MPI_Session mpi(argc, argv);
-   int num_procs = mpi.WorldSize();
-   int myid = mpi.WorldRank();
-
-   // 2. Parse command-line options.
-   int order = 3;
-   int mesh_type = 4; // Default to Quadrilateral mesh
-   int mesh_order = 3;
-   int ser_ref_levels = 2;
-   int par_ref_levels = 1;
-   bool static_cond = false;
-   bool visualization = true;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_type, "-mt", "--mesh-type",
-                  "Mesh type: 3 - Triangular, 4 - Quadrilateral.");
-   args.AddOption(&mesh_order, "-mo", "--mesh-order",
-                  "Geometric order of the curved mesh.");
-   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
-                  "Number of times to refine the mesh uniformly in parallel.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.ParseCheck();
-
-   // 3. Construct a quadrilateral or triangular mesh with the topology of a
-   //    cylindrical surface.
-   Mesh *mesh = GetMesh(mesh_type);
-   int dim = mesh->Dimension();
-
-   // 4. Refine the mesh to increase the resolution. In this example we do
-   //    'ser_ref_levels' of uniform refinement.
-   for (int l = 0; l < ser_ref_levels; l++)
-   {
-      mesh->UniformRefinement();
-   }
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   for (int l = 0; l < par_ref_levels; l++)
-   {
-      pmesh.UniformRefinement();
-   }
-
-   // 6. Transform the mesh so that it has a more interesting geometry.
-   pmesh.SetCurvature(mesh_order);
-   pmesh.Transform(trans);
-
-   // 7. Define a finite element space on the mesh. Here we use continuous
-   //    Lagrange finite elements of the specified order.
-   H1_FECollection fec(order, dim);
-   ParFiniteElementSpace fespace(&pmesh, &fec);
-   HYPRE_Int total_num_dofs = fespace.GlobalTrueVSize();
-   if (mpi.Root()) { cout << "Number of unknowns: " << total_num_dofs << endl; }
-
-   // 8. Determine the list of true (i.e. conforming) essential boundary dofs.
-   //    In this example, the boundary conditions are defined by marking all
-   //    the boundary attributes from the mesh as essential (Dirichlet) and
-   //    converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (pmesh.bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 9. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
-   //    the basis functions in the finite element fespace.
-   ParLinearForm b(&fespace);
-   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
-
-   // 10. Define the solution vector x as a finite element grid function
-   //     corresponding to fespace. Initialize x with initial guess of zero,
-   //     which satisfies the boundary conditions.
-   ParGridFunction x(&fespace);
-   x = 0.0;
-
-   // 11. Set up the bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the
-   //     Diffusion domain integrator.
-   ParBilinearForm a(&fespace);
-   MatrixFunctionCoefficient sigma(3, sigmaFunc);
-   BilinearFormIntegrator *integ = new DiffusionIntegrator(sigma);
-   a.AddDomainIntegrator(integ);
-
-   // 12. Assemble the bilinear form and the corresponding linear system,
-   //     applying any necessary transformations such as: eliminating boundary
-   //     conditions, applying conforming constraints for non-conforming AMR,
-   //     static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
-   a.Assemble();
-
-   OperatorPtr A;
-   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-
-   if (myid == 0)
-   {
-      cout << "Size of linear system: "
-           << A.As<HypreParMatrix>()->GetGlobalNumRows() << endl;
-   }
-
-   // 13. Define and apply a parallel PCG solver for A X = B with the BoomerAMG
-   //     preconditioner from hypre.
-   HypreBoomerAMG *amg = new HypreBoomerAMG;
-   CGSolver cg(MPI_COMM_WORLD);
-   cg.SetRelTol(1e-12);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(1);
-   cg.SetPreconditioner(*amg);
-   cg.SetOperator(*A);
-   cg.Mult(B, X);
-   delete amg;
-
-   // 14. Recover the solution as a finite element grid function.
-   a.RecoverFEMSolution(X, b, x);
-
-   // 15. Compute error in the solution and its flux
-   FunctionCoefficient uCoef(uExact);
-   double err = x.ComputeL2Error(uCoef);
-
-   if (myid == 0) { cout << "|u - u_h|_2 = " << err << endl; }
-
-   ParFiniteElementSpace flux_fespace(&pmesh, &fec, 3);
-   ParGridFunction flux(&flux_fespace);
-   x.ComputeFlux(*integ, flux); flux *= -1.0;
-
-   VectorFunctionCoefficient fluxCoef(3, fluxExact);
-   double flux_err = flux.ComputeL2Error(fluxCoef);
-
-   if (myid == 0) { cout << "|f - f_h|_2 = " << flux_err << endl; }
-
-   // 16. Save the refined mesh and the solution. This output can be viewed
-   //     later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name, flux_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-      flux_name << "flux." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh.Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-
-      ofstream flux_ofs(flux_name.str().c_str());
-      flux_ofs.precision(8);
-      flux.Save(flux_ofs);
-   }
-
-   // 17. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << pmesh << x
-               << "window_title 'Solution'\n" << flush;
-
-      socketstream flux_sock(vishost, visport);
-      flux_sock << "parallel " << num_procs << " " << myid << "\n";
-      flux_sock.precision(8);
-      flux_sock << "solution\n" << pmesh << flux
-                << "keys vvv\n"
-                << "window_geometry 402 0 400 350\n"
-                << "window_title 'Flux'\n"  << flush;
-   }
-
-   return 0;
-}
-
-// Defines a mesh consisting of four flat rectangular surfaces connected to form
-// a loop.
-Mesh * GetMesh(int type)
-{
-   Mesh * mesh = NULL;
-
-   if (type == 3)
-   {
-      mesh = new Mesh(2, 12, 16, 8, 3);
-
-      mesh->AddVertex(-1.0, -1.0, 0.0);
-      mesh->AddVertex( 1.0, -1.0, 0.0);
-      mesh->AddVertex( 1.0,  1.0, 0.0);
-      mesh->AddVertex(-1.0,  1.0, 0.0);
-      mesh->AddVertex(-1.0, -1.0, 1.0);
-      mesh->AddVertex( 1.0, -1.0, 1.0);
-      mesh->AddVertex( 1.0,  1.0, 1.0);
-      mesh->AddVertex(-1.0,  1.0, 1.0);
-      mesh->AddVertex( 0.0, -1.0, 0.5);
-      mesh->AddVertex( 1.0,  0.0, 0.5);
-      mesh->AddVertex( 0.0,  1.0, 0.5);
-      mesh->AddVertex(-1.0,  0.0, 0.5);
-
-      mesh->AddTriangle(0, 1, 8);
-      mesh->AddTriangle(1, 5, 8);
-      mesh->AddTriangle(5, 4, 8);
-      mesh->AddTriangle(4, 0, 8);
-      mesh->AddTriangle(1, 2, 9);
-      mesh->AddTriangle(2, 6, 9);
-      mesh->AddTriangle(6, 5, 9);
-      mesh->AddTriangle(5, 1, 9);
-      mesh->AddTriangle(2, 3, 10);
-      mesh->AddTriangle(3, 7, 10);
-      mesh->AddTriangle(7, 6, 10);
-      mesh->AddTriangle(6, 2, 10);
-      mesh->AddTriangle(3, 0, 11);
-      mesh->AddTriangle(0, 4, 11);
-      mesh->AddTriangle(4, 7, 11);
-      mesh->AddTriangle(7, 3, 11);
-
-      mesh->AddBdrSegment(0, 1, 1);
-      mesh->AddBdrSegment(1, 2, 1);
-      mesh->AddBdrSegment(2, 3, 1);
-      mesh->AddBdrSegment(3, 0, 1);
-      mesh->AddBdrSegment(5, 4, 2);
-      mesh->AddBdrSegment(6, 5, 2);
-      mesh->AddBdrSegment(7, 6, 2);
-      mesh->AddBdrSegment(4, 7, 2);
-   }
-   else if (type == 4)
-   {
-      mesh = new Mesh(2, 8, 4, 8, 3);
-
-      mesh->AddVertex(-1.0, -1.0, 0.0);
-      mesh->AddVertex( 1.0, -1.0, 0.0);
-      mesh->AddVertex( 1.0,  1.0, 0.0);
-      mesh->AddVertex(-1.0,  1.0, 0.0);
-      mesh->AddVertex(-1.0, -1.0, 1.0);
-      mesh->AddVertex( 1.0, -1.0, 1.0);
-      mesh->AddVertex( 1.0,  1.0, 1.0);
-      mesh->AddVertex(-1.0,  1.0, 1.0);
-
-      mesh->AddQuad(0, 1, 5, 4);
-      mesh->AddQuad(1, 2, 6, 5);
-      mesh->AddQuad(2, 3, 7, 6);
-      mesh->AddQuad(3, 0, 4, 7);
-
-      mesh->AddBdrSegment(0, 1, 1);
-      mesh->AddBdrSegment(1, 2, 1);
-      mesh->AddBdrSegment(2, 3, 1);
-      mesh->AddBdrSegment(3, 0, 1);
-      mesh->AddBdrSegment(5, 4, 2);
-      mesh->AddBdrSegment(6, 5, 2);
-      mesh->AddBdrSegment(7, 6, 2);
-      mesh->AddBdrSegment(4, 7, 2);
-   }
-   else
-   {
-      MFEM_ABORT("Unrecognized mesh type " << type << "!");
-   }
-   mesh->FinalizeTopology();
-
-   return mesh;
-}
-
-// Transforms the four-sided loop into a curved cylinder with skewed top and
-// base.
-void trans(const Vector &x, Vector &r)
-{
-   r.SetSize(3);
-
-   double tol = 1e-6;
-   double theta = 0.0;
-   if (fabs(x[1] + 1.0) < tol)
-   {
-      theta = 0.25 * M_PI * (x[0] - 2.0);
-   }
-   else if (fabs(x[0] - 1.0) < tol)
-   {
-      theta = 0.25 * M_PI * x[1];
-   }
-   else if (fabs(x[1] - 1.0) < tol)
-   {
-      theta = 0.25 * M_PI * (2.0 - x[0]);
-   }
-   else if (fabs(x[0] + 1.0) < tol)
-   {
-      theta = 0.25 * M_PI * (4.0 - x[1]);
-   }
-   else
-   {
-      cerr << "side not recognized "
-           << x[0] << " " << x[1] << " " << x[2] << endl;
-   }
-
-   r[0] = cos(theta);
-   r[1] = sin(theta);
-   r[2] = 0.25 * (2.0 * x[2] - 1.0) * (r[0] + 2.0);
-}
-
-// Anisotropic diffusion coefficient
-void sigmaFunc(const Vector &x, DenseMatrix &s)
-{
-   s.SetSize(3);
-   double a = 17.0 - 2.0 * x[0] * (1.0 + x[0]);
-   s(0,0) = 0.5 + x[0] * x[0] * (8.0 / a - 0.5);
-   s(0,1) = x[0] * x[1] * (8.0 / a - 0.5);
-   s(0,2) = 0.0;
-   s(1,0) = s(0,1);
-   s(1,1) = 0.5 * x[0] * x[0] + 8.0 * x[1] * x[1] / a;
-   s(1,2) = 0.0;
-   s(2,0) = 0.0;
-   s(2,1) = 0.0;
-   s(2,2) = a / 32.0;
-}
@@ -61,7 +61,6 @@ int main(int argc, char *argv[])
   bool visualization = 1;
   bool amg_elast = 0;
   bool reorder_space = false;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -79,8 +78,6 @@ int main(int argc, char *argv[])
                  "Enable or disable GLVis visualization.");
   args.AddOption(&reorder_space, "-nodes", "--by-nodes", "-vdim", "--by-vdim",
                  "Use byNODES ordering of vector space instead of byVDIM");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -96,12 +93,7 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
@@ -117,14 +109,14 @@ int main(int argc, char *argv[])
      return 3;
   }

-   // 5. Select the order of the finite element discretization space. For NURBS
+   // 4. Select the order of the finite element discretization space. For NURBS
   //    meshes, we increase the order by degree elevation.
   if (mesh->NURBSext)
   {
      mesh->DegreeElevate(order, order);
   }

-   // 6. Refine the serial mesh on all processors to increase the resolution. In
+   // 5. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ref_levels' of uniform refinement. We choose
   //    'ref_levels' to be the largest number that gives a final mesh with no
   //    more than 1,000 elements.
@@ -137,7 +129,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 7. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -150,7 +142,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 8. Define a parallel finite element space on the parallel mesh. Here we
+   // 7. Define a parallel finite element space on the parallel mesh. Here we
   //    use vector finite elements, i.e. dim copies of a scalar finite element
   //    space. We use the ordering by vector dimension (the last argument of
   //    the FiniteElementSpace constructor) which is expected in the systems
@@ -176,14 +168,14 @@ int main(int argc, char *argv[])
         fespace = new ParFiniteElementSpace(pmesh, fec, dim, Ordering::byVDIM);
      }
   }
-   HYPRE_BigInt size = fespace->GlobalTrueVSize();
+   HYPRE_Int size = fespace->GlobalTrueVSize();
   if (myid == 0)
   {
      cout << "Number of finite element unknowns: " << size << endl
           << "Assembling: " << flush;
   }

-   // 9. Determine the list of true (i.e. parallel conforming) essential
+   // 8. Determine the list of true (i.e. parallel conforming) essential
   //    boundary dofs. In this example, the boundary conditions are defined by
   //    marking only boundary attribute 1 from the mesh as essential and
   //    converting it to a list of true dofs.
@@ -192,14 +184,14 @@ int main(int argc, char *argv[])
   ess_bdr[0] = 1;
   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);

-   // 10. Set up the parallel linear form b(.) which corresponds to the
-   //     right-hand side of the FEM linear system. In this case, b_i equals the
-   //     boundary integral of f*phi_i where f represents a "pull down" force on
-   //     the Neumann part of the boundary and phi_i are the basis functions in
-   //     the finite element fespace. The force is defined by the object f, which
-   //     is a vector of Coefficient objects. The fact that f is non-zero on
-   //     boundary attribute 2 is indicated by the use of piece-wise constants
-   //     coefficient for its last component.
+   // 9. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system. In this case, b_i equals the
+   //    boundary integral of f*phi_i where f represents a "pull down" force on
+   //    the Neumann part of the boundary and phi_i are the basis functions in
+   //    the finite element fespace. The force is defined by the object f, which
+   //    is a vector of Coefficient objects. The fact that f is non-zero on
+   //    boundary attribute 2 is indicated by the use of piece-wise constants
+   //    coefficient for its last component.
   VectorArrayCoefficient f(dim);
   for (int i = 0; i < dim-1; i++)
   {
@@ -220,13 +212,13 @@ int main(int argc, char *argv[])
   }
   b->Assemble();

-   // 11. Define the solution vector x as a parallel finite element grid
+   // 10. Define the solution vector x as a parallel finite element grid
   //     function corresponding to fespace. Initialize x with initial guess of
   //     zero, which satisfies the boundary conditions.
   ParGridFunction x(fespace);
   x = 0.0;

-   // 12. Set up the parallel bilinear form a(.,.) on the finite element space
+   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
   //     corresponding to the linear elasticity integrator with piece-wise
   //     constants coefficient lambda and mu.
   Vector lambda(pmesh->attributes.Max());
@@ -241,7 +233,7 @@ int main(int argc, char *argv[])
   ParBilinearForm *a = new ParBilinearForm(fespace);
   a->AddDomainIntegrator(new ElasticityIntegrator(lambda_func, mu_func));

-   // 13. Assemble the parallel bilinear form and the corresponding linear
+   // 12. Assemble the parallel bilinear form and the corresponding linear
   //     system, applying any necessary transformations such as: parallel
   //     assembly, eliminating boundary conditions, applying conforming
   //     constraints for non-conforming AMR, static condensation, etc.
@@ -258,7 +250,7 @@ int main(int argc, char *argv[])
      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
   }

-   // 14. Define and apply a parallel PCG solver for A X = B with the BoomerAMG
+   // 13. Define and apply a parallel PCG solver for A X = B with the BoomerAMG
   //     preconditioner from hypre.
   HypreBoomerAMG *amg = new HypreBoomerAMG(A);
   if (amg_elast && !a->StaticCondensationIsEnabled())
@@ -276,11 +268,11 @@ int main(int argc, char *argv[])
   pcg->SetPreconditioner(*amg);
   pcg->Mult(B, X);

-   // 15. Recover the parallel grid function corresponding to X. This is the
+   // 14. Recover the parallel grid function corresponding to X. This is the
   //     local finite element solution on each processor.
   a->RecoverFEMSolution(X, *b, x);

-   // 16. For non-NURBS meshes, make the mesh curved based on the finite element
+   // 15. For non-NURBS meshes, make the mesh curved based on the finite element
   //     space. This means that we define the mesh elements through a fespace
   //     based transformation of the reference element.  This allows us to save
   //     the displaced mesh as a curved mesh when using high-order finite
@@ -292,7 +284,7 @@ int main(int argc, char *argv[])
      pmesh->SetNodalFESpace(fespace);
   }

-   // 17. Save in parallel the displaced mesh and the inverted solution (which
+   // 16. Save in parallel the displaced mesh and the inverted solution (which
   //     gives the backward displacements to the original grid). This output
   //     can be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
   {
@@ -313,7 +305,7 @@ int main(int argc, char *argv[])
      x.Save(sol_ofs);
   }

-   // 18. Send the above data by socket to a GLVis server.  Use the "n" and "b"
+   // 17. Send the above data by socket to a GLVis server.  Use the "n" and "b"
   //     keys in GLVis to visualize the displacements.
   if (visualization)
   {
@@ -325,7 +317,7 @@ int main(int argc, char *argv[])
      sol_sock << "solution\n" << *pmesh << x << flush;
   }

-   // 19. Free the used memory.
+   // 18. Free the used memory.
   delete pcg;
   delete amg;
   delete a;
@@ -103,7 +103,6 @@ int main(int argc, char *argv[])
      {
         args.PrintUsage(cout);
      }
-      // HYPRE_Finalize();
      MPI_Finalize();
      return 1;
   }
@@ -157,7 +156,7 @@ int main(int argc, char *argv[])
   //    use the Nedelec finite elements of the specified order.
   FiniteElementCollection *fec = new ND_FECollection(order, dim);
   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_BigInt size = fespace->GlobalTrueVSize();
+   HYPRE_Int size = fespace->GlobalTrueVSize();
   if (myid == 0)
   {
      cout << "Number of finite element unknowns: " << size << endl;
@@ -153,7 +153,7 @@ int main(int argc, char *argv[])
   //    use the Raviart-Thomas finite elements of the specified order.
   FiniteElementCollection *fec = new RT_FECollection(order-1, dim);
   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_BigInt size = fespace->GlobalTrueVSize();
+   HYPRE_Int size = fespace->GlobalTrueVSize();
   if (myid == 0)
   {
      cout << "Number of finite element unknowns: " << size << endl;
@@ -197,7 +197,6 @@ int main(int argc, char *argv[])
      SparseMatrix &M(mVarf->SpMat());
      SparseMatrix &B(bVarf->SpMat());
      B *= -1.;
-      if (Device::IsEnabled()) { B.BuildTranspose(); }
      Bt = new TransposeOperator(&B);

      darcyOp.SetBlock(0,0, &M);
@@ -241,7 +240,6 @@ int main(int argc, char *argv[])
   {
      SparseMatrix &M(mVarf->SpMat());
      M.GetDiag(Md);
-      Md.HostReadWrite();

      SparseMatrix &B(bVarf->SpMat());
      MinvBt = Transpose(B);
@@ -289,18 +287,12 @@ int main(int argc, char *argv[])
   chrono.Stop();

   if (solver.GetConverged())
-   {
      std::cout << "MINRES converged in " << solver.GetNumIterations()
-                << " iterations with a residual norm of "
-                << solver.GetFinalNorm() << ".\n";
-   }
+                << " iterations with a residual norm of " << solver.GetFinalNorm() << ".\n";
   else
-   {
      std::cout << "MINRES did not converge in " << solver.GetNumIterations()
-                << " iterations. Residual norm is " << solver.GetFinalNorm()
-                << ".\n";
-   }
-   std::cout << "MINRES solver took " << chrono.RealTime() << "s.\n";
+                << " iterations. Residual norm is " << solver.GetFinalNorm() << ".\n";
+   std::cout << "MINRES solver took " << chrono.RealTime() << "s. \n";

   // 12. Create the grid functions u and p. Compute the L2 error norms.
   GridFunction u, p;
@@ -155,8 +155,8 @@ int main(int argc, char *argv[])
   ParFiniteElementSpace *R_space = new ParFiniteElementSpace(pmesh, hdiv_coll);
   ParFiniteElementSpace *W_space = new ParFiniteElementSpace(pmesh, l2_coll);

-   HYPRE_BigInt dimR = R_space->GlobalTrueVSize();
-   HYPRE_BigInt dimW = W_space->GlobalTrueVSize();
+   HYPRE_Int dimR = R_space->GlobalTrueVSize();
+   HYPRE_Int dimW = W_space->GlobalTrueVSize();

   if (verbose)
   {
@@ -14,7 +14,6 @@
 //               ex6 -m ../data/star-surf.mesh -o 2
 //               ex6 -m ../data/square-disc-surf.mesh -o 2
 //               ex6 -m ../data/amr-quad.mesh
-//               ex6 -m ../data/inline-segment.mesh -o 1 -md 100
 //
 // Device sample runs:
 //               ex6 -pa -d cuda
@@ -54,7 +53,6 @@ int main(int argc, char *argv[])
   int order = 1;
   bool pa = false;
   const char *device_config = "cpu";
-   int max_dofs = 50000;
   bool visualization = true;

   OptionsParser args(argc, argv);
@@ -66,8 +64,6 @@ int main(int argc, char *argv[])
                  "--no-partial-assembly", "Enable Partial Assembly.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&max_dofs, "-md", "--max-dofs",
-                  "Stop after reaching this many degrees of freedom.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -164,6 +160,7 @@ int main(int argc, char *argv[])

   // 12. The main AMR loop. In each iteration we solve the problem on the
   //     current mesh, visualize the solution, and refine the mesh.
+   const int max_dofs = 50000;
   for (int it = 0; ; it++)
   {
      int cdofs = fespace.GetTrueVSize();
@@ -2,20 +2,19 @@
 //
 // Compile with: make ex6p
 //
-// Sample runs:  mpirun -np 4 ex6p -m ../data/star-hilbert.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/square-disc.mesh -rm 1 -o 1
-//               mpirun -np 4 ex6p -m ../data/square-disc.mesh -rm 1 -o 2 -h1
-//               mpirun -np 4 ex6p -m ../data/square-disc.mesh -o 2 -cs
+// Sample runs:  mpirun -np 4 ex6p -m ../data/square-disc.mesh -o 1
+//               mpirun -np 4 ex6p -m ../data/square-disc.mesh -o 2
+//               mpirun -np 4 ex6p -m ../data/square-disc.mesh -o 2 -ns
 //               mpirun -np 4 ex6p -m ../data/square-disc-nurbs.mesh -o 2
+//               mpirun -np 4 ex6p -m ../data/star.mesh -o 3
+//               mpirun -np 4 ex6p -m ../data/escher.mesh -o 2
+//               mpirun -np 4 ex6p -m ../data/escher.mesh -o 2 -ns
 //               mpirun -np 4 ex6p -m ../data/fichera.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/escher.mesh -rm 2 -o 2
-//               mpirun -np 4 ex6p -m ../data/escher.mesh -o 2 -cs
 //               mpirun -np 4 ex6p -m ../data/disc-nurbs.mesh -o 2
 //               mpirun -np 4 ex6p -m ../data/ball-nurbs.mesh
 //               mpirun -np 4 ex6p -m ../data/pipe-nurbs.mesh
 //               mpirun -np 4 ex6p -m ../data/star-surf.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/square-disc-surf.mesh -rm 2 -o 2
-//               mpirun -np 4 ex6p -m ../data/inline-segment.mesh -o 1 -md 200
+//               mpirun -np 4 ex6p -m ../data/square-disc-surf.mesh -o 2
 //               mpirun -np 4 ex6p -m ../data/amr-quad.mesh
 //               mpirun -np 4 ex6p --restart
 //
@@ -63,10 +62,8 @@ int main(int argc, char *argv[])
   int order = 1;
   bool pa = false;
   const char *device_config = "cpu";
-   bool nc_simplices = true;
-   int reorder_mesh = 0;
+   bool nc_simplices = false;
   int max_dofs = 100000;
-   bool smooth_rt = true;
   bool restart = false;
   bool visualization = true;

@@ -79,17 +76,12 @@ int main(int argc, char *argv[])
                  "--no-partial-assembly", "Enable Partial Assembly.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&reorder_mesh, "-rm", "--reorder-mesh",
-                  "Reorder elements of the coarse mesh to improve "
-                  "dynamic partitioning: 0=none, 1=hilbert, 2=gecko.");
   args.AddOption(&nc_simplices, "-ns", "--nonconforming-simplices",
                  "-cs", "--conforming-simplices",
                  "For simplicial meshes, enable/disable nonconforming"
                  " refinement");
   args.AddOption(&max_dofs, "-md", "--max-dofs",
                  "Stop after reaching this many degrees of freedom.");
-   args.AddOption(&smooth_rt, "-rt", "--smooth-rt", "-h1", "--smooth-h1",
-                  "Represent the smooth flux in RT or vector H1 space.");
   args.AddOption(&restart, "-res", "--restart", "-no-res", "--no-restart",
                  "Restart computation from the last checkpoint.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -123,49 +115,23 @@ int main(int argc, char *argv[])
      //    surface and volume meshes with the same code.
      Mesh mesh(mesh_file, 1, 1);

-      // 5. A NURBS mesh cannot be refined locally so we refine it uniformly
-      //    and project it to a standard curvilinear mesh of order 2.
+      // 5. Refine the serial mesh on all processors to increase the resolution.
+      //    Also project a NURBS mesh to a piecewise-quadratic curved mesh. Make
+      //    sure that the mesh is non-conforming.
      if (mesh.NURBSext)
      {
         mesh.UniformRefinement();
         mesh.SetCurvature(2);
      }
-
-      // 6. MFEM supports dynamic partitioning (load balancing) of parallel non-
-      //    conforming meshes based on space-filling curve (SFC) partitioning.
-      //    SFC partitioning is extremely fast and scales to hundreds of
-      //    thousands of processors, but requires the coarse mesh to be ordered,
-      //    ideally as a sequence of face-neighbors. The mesh may already be
-      //    ordered (like star-hilbert.mesh) or we can order it here. Ordering
-      //    type 1 is a fast spatial sort of the mesh, type 2 is a high quality
-      //    optimization algorithm suitable for ordering general unstructured
-      //    meshes.
-      if (reorder_mesh)
-      {
-         Array<int> ordering;
-         switch (reorder_mesh)
-         {
-            case 1: mesh.GetHilbertElementOrdering(ordering); break;
-            case 2: mesh.GetGeckoElementOrdering(ordering); break;
-            default: MFEM_ABORT("Unknown mesh reodering type " << reorder_mesh);
-         }
-         mesh.ReorderElements(ordering);
-      }
-
-      // 7. Make sure the mesh is in the non-conforming mode to enable local
-      //    refinement of quadrilaterals/hexahedra, and the above partitioning
-      //    algorithm. Simplices can be refined either in conforming or in non-
-      //    conforming mode. The conforming mode however does not support
-      //    dynamic partitioning.
      mesh.EnsureNCMesh(nc_simplices);

-      // 8. Define a parallel mesh by partitioning the serial mesh.
+      // 6. Define a parallel mesh by partitioning the serial mesh.
      //    Once the parallel mesh is defined, the serial mesh can be deleted.
      pmesh = new ParMesh(MPI_COMM_WORLD, mesh);
   }
   else
   {
-      // 9. We can also restart the computation by loading the mesh from a
+      // 7. We can also restart the computation by loading the mesh from a
      //    previously saved check-point.
      string fname(MakeParFilename("ex6p-checkpoint.", myid));
      ifstream ifs(fname);
@@ -181,14 +147,14 @@ int main(int argc, char *argv[])
   Array<int> ess_bdr(pmesh->bdr_attributes.Max());
   ess_bdr = 1;

-   // 10. Define a finite element space on the mesh. The polynomial order is
-   //     one (linear) by default, but this can be changed on the command line.
+   // 8. Define a finite element space on the mesh. The polynomial order is
+   //    one (linear) by default, but this can be changed on the command line.
   H1_FECollection fec(order, dim);
   ParFiniteElementSpace fespace(pmesh, &fec);

-   // 11. As in Example 1p, we set up bilinear and linear forms corresponding to
-   //     the Laplace problem -\Delta u = 1. We don't assemble the discrete
-   //     problem yet, this will be done in the main loop.
+   // 9. As in Example 1p, we set up bilinear and linear forms corresponding to
+   //    the Laplace problem -\Delta u = 1. We don't assemble the discrete
+   //    problem yet, this will be done in the main loop.
   ParBilinearForm a(&fespace);
   if (pa)
   {
@@ -203,12 +169,12 @@ int main(int argc, char *argv[])
   a.AddDomainIntegrator(integ);
   b.AddDomainIntegrator(new DomainLFIntegrator(one));

-   // 12. The solution vector x and the associated finite element grid function
+   // 10. The solution vector x and the associated finite element grid function
   //     will be maintained over the AMR iterations. We initialize it to zero.
   ParGridFunction x(&fespace);
   x = 0;

-   // 13. Connect to GLVis.
+   // 11. Connect to GLVis.
   char vishost[] = "localhost";
   int  visport   = 19916;

@@ -230,59 +196,51 @@ int main(int argc, char *argv[])
      sout.precision(8);
   }

-   // 14. Set up an error estimator. Here we use the Zienkiewicz-Zhu estimator
+   // 12. Set up an error estimator. Here we use the Zienkiewicz-Zhu estimator
   //     with L2 projection in the smoothing step to better handle hanging
   //     nodes and parallel partitioning. We need to supply a space for the
-   //     discontinuous flux (L2) and a space for the smoothed flux.
+   //     discontinuous flux (L2) and a space for the smoothed flux (H(div) is
+   //     used here).
   L2_FECollection flux_fec(order, dim);
   ParFiniteElementSpace flux_fes(pmesh, &flux_fec, sdim);
-   FiniteElementCollection *smooth_flux_fec = NULL;
-   ParFiniteElementSpace *smooth_flux_fes = NULL;
-   if (smooth_rt && dim > 1)
-   {
-      // Use an H(div) space for the smoothed flux (this is the default).
-      smooth_flux_fec = new RT_FECollection(order-1, dim);
-      smooth_flux_fes = new ParFiniteElementSpace(pmesh, smooth_flux_fec, 1);
-   }
-   else
-   {
-      // Another possible option for the smoothed flux space: H1^dim space
-      smooth_flux_fec = new H1_FECollection(order, dim);
-      smooth_flux_fes = new ParFiniteElementSpace(pmesh, smooth_flux_fec, dim);
-   }
-   L2ZienkiewiczZhuEstimator estimator(*integ, x, flux_fes, *smooth_flux_fes);
+   RT_FECollection smooth_flux_fec(order-1, dim);
+   ParFiniteElementSpace smooth_flux_fes(pmesh, &smooth_flux_fec);
+   // Another possible option for the smoothed flux space:
+   // H1_FECollection smooth_flux_fec(order, dim);
+   // ParFiniteElementSpace smooth_flux_fes(pmesh, &smooth_flux_fec, dim);
+   L2ZienkiewiczZhuEstimator estimator(*integ, x, flux_fes, smooth_flux_fes);

-   // 15. A refiner selects and refines elements based on a refinement strategy.
+   // 13. A refiner selects and refines elements based on a refinement strategy.
   //     The strategy here is to refine elements with errors larger than a
   //     fraction of the maximum element error. Other strategies are possible.
   //     The refiner will call the given error estimator.
   ThresholdRefiner refiner(estimator);
   refiner.SetTotalErrorFraction(0.7);

-   // 16. The main AMR loop. In each iteration we solve the problem on the
+   // 14. The main AMR loop. In each iteration we solve the problem on the
   //     current mesh, visualize the solution, and refine the mesh.
   for (int it = 0; ; it++)
   {
-      HYPRE_BigInt global_dofs = fespace.GlobalTrueVSize();
+      HYPRE_Int global_dofs = fespace.GlobalTrueVSize();
      if (myid == 0)
      {
         cout << "\nAMR iteration " << it << endl;
         cout << "Number of unknowns: " << global_dofs << endl;
      }

-      // 17. Assemble the right-hand side and determine the list of true
+      // 15. Assemble the right-hand side and determine the list of true
      //     (i.e. parallel conforming) essential boundary dofs.
      Array<int> ess_tdof_list;
      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
      b.Assemble();

-      // 18. Assemble the stiffness matrix. Note that MFEM doesn't care at this
+      // 16. Assemble the stiffness matrix. Note that MFEM doesn't care at this
      //     point that the mesh is nonconforming and parallel.  The FE space is
      //     considered 'cut' along hanging edges/faces, and also across
      //     processor boundaries.
      a.Assemble();

-      // 19. Create the parallel linear system: eliminate boundary conditions.
+      // 17. Create the parallel linear system: eliminate boundary conditions.
      //     The system will be solved for true (unconstrained/unique) DOFs only.
      OperatorPtr A;
      Vector B, X;
@@ -290,7 +248,7 @@ int main(int argc, char *argv[])
      const int copy_interior = 1;
      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);

-      // 20. Solve the linear system A X = B.
+      // 18. Solve the linear system A X = B.
      //     * With full assembly, use the BoomerAMG preconditioner from hypre.
      //     * With partial assembly, use a diagonal preconditioner.
      Solver *M = NULL;
@@ -313,12 +271,12 @@ int main(int argc, char *argv[])
      cg.Mult(B, X);
      delete M;

-      // 21. Switch back to the host and extract the parallel grid function
+      // 19. Switch back to the host and extract the parallel grid function
      //     corresponding to the finite element approximation X. This is the
      //     local solution on each processor.
      a.RecoverFEMSolution(X, b, x);

-      // 22. Send the solution by socket to a GLVis server.
+      // 20. Send the solution by socket to a GLVis server.
      if (visualization)
      {
         sout << "parallel " << num_procs << " " << myid << "\n";
@@ -334,7 +292,7 @@ int main(int argc, char *argv[])
         break;
      }

-      // 23. Call the refiner to modify the mesh. The refiner calls the error
+      // 21. Call the refiner to modify the mesh. The refiner calls the error
      //     estimator to obtain element errors, then it selects elements to be
      //     refined and finally it modifies the mesh. The Stop() method can be
      //     used to determine if a stopping criterion was met.
@@ -348,7 +306,7 @@ int main(int argc, char *argv[])
         break;
      }

-      // 24. Update the finite element space (recalculate the number of DOFs,
+      // 22. Update the finite element space (recalculate the number of DOFs,
      //     etc.) and create a grid function update matrix. Apply the matrix
      //     to any GridFunctions over the space. In this case, the update
      //     matrix is an interpolation matrix so the updated GridFunction will
@@ -356,7 +314,7 @@ int main(int argc, char *argv[])
      fespace.Update();
      x.Update();

-      // 25. Load balance the mesh, and update the space and solution. Currently
+      // 23. Load balance the mesh, and update the space and solution. Currently
      //     available only for nonconforming meshes.
      if (pmesh->Nonconforming())
      {
@@ -368,12 +326,12 @@ int main(int argc, char *argv[])
         x.Update();
      }

-      // 26. Inform also the bilinear and linear forms that the space has
+      // 24. Inform also the bilinear and linear forms that the space has
      //     changed.
      a.Update();
      b.Update();

-      // 27. Save the current state of the mesh every 5 iterations. The
+      // 25. Save the current state of the mesh every 5 iterations. The
      //     computation can be restarted from this point. Note that unlike in
      //     visualization, we need to use the 'ParPrint' method to save all
      //     internal parallel data structures.
@@ -390,8 +348,6 @@ int main(int argc, char *argv[])
      }
   }

-   delete smooth_flux_fes;
-   delete smooth_flux_fec;
   delete pmesh;

   MPI_Finalize();
@@ -47,7 +47,6 @@ int main(int argc, char *argv[])
   int order = 2;
   bool always_snap = false;
   bool visualization = 1;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&elem_type, "-e", "--elem",
@@ -66,8 +65,6 @@ int main(int argc, char *argv[])
                  "--snap-at-the-end",
                  "If true, snap nodes to the sphere initially and after each refinement "
                  "otherwise, snap only after the last refinement");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -83,12 +80,7 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Generate an initial high-order (surface) mesh on the unit sphere. The
+   // 3. Generate an initial high-order (surface) mesh on the unit sphere. The
   //    Mesh object represents a 2D mesh in 3 spatial dimensions. We first add
   //    the elements and the vertices of the mesh, and then make it high-order
   //    by specifying a finite element space for its nodes.
@@ -154,7 +146,7 @@ int main(int argc, char *argv[])
   FiniteElementSpace nodal_fes(mesh, &fec, mesh->SpaceDimension());
   mesh->SetNodalFESpace(&nodal_fes);

-   // 5. Refine the mesh while snapping nodes to the sphere. Number of parallel
+   // 4. Refine the mesh while snapping nodes to the sphere. Number of parallel
   //    refinements is fixed to 2.
   for (int l = 0; l <= ref_levels; l++)
   {
@@ -226,16 +218,16 @@ int main(int argc, char *argv[])
      SnapNodes(*pmesh);
   }

-   // 6. Define a finite element space on the mesh. Here we use isoparametric
+   // 5. Define a finite element space on the mesh. Here we use isoparametric
   //    finite elements -- the same as the mesh nodes.
   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, &fec);
-   HYPRE_BigInt size = fespace->GlobalTrueVSize();
+   HYPRE_Int size = fespace->GlobalTrueVSize();
   if (myid == 0)
   {
      cout << "Number of unknowns: " << size << endl;
   }

-   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
+   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
   //    the basis functions in the finite element fespace.
   ParLinearForm *b = new ParLinearForm(fespace);
@@ -245,27 +237,27 @@ int main(int argc, char *argv[])
   b->AddDomainIntegrator(new DomainLFIntegrator(rhs_coef));
   b->Assemble();

-   // 8. Define the solution vector x as a finite element grid function
+   // 7. Define the solution vector x as a finite element grid function
   //    corresponding to fespace. Initialize x with initial guess of zero.
   ParGridFunction x(fespace);
   x = 0.0;

-   // 9. Set up the bilinear form a(.,.) on the finite element space
+   // 8. Set up the bilinear form a(.,.) on the finite element space
   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
   //    and Mass domain integrators.
   ParBilinearForm *a = new ParBilinearForm(fespace);
   a->AddDomainIntegrator(new DiffusionIntegrator(one));
   a->AddDomainIntegrator(new MassIntegrator(one));

-   // 10. Assemble the parallel linear system, applying any transformations
-   //     such as: parallel assembly, applying conforming constraints, etc.
+   // 9. Assemble the parallel linear system, applying any transformations
+   //    such as: parallel assembly, applying conforming constraints, etc.
   a->Assemble();
   HypreParMatrix A;
   Vector B, X;
   Array<int> empty_tdof_list;
   a->FormLinearSystem(empty_tdof_list, x, *b, A, X, B);

-   // 11. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
+   // 10. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
   //     preconditioner from hypre. Extract the parallel grid function x
   //     corresponding to the finite element approximation X. This is the local
   //     solution on each processor.
@@ -281,14 +273,14 @@ int main(int argc, char *argv[])
   delete a;
   delete b;

-   // 12. Compute and print the L^2 norm of the error.
+   // 11. Compute and print the L^2 norm of the error.
   double err = x.ComputeL2Error(sol_coef);
   if (myid == 0)
   {
      cout << "\nL2 norm of error: " << err << endl;
   }

-   // 13. Save the refined mesh and the solution. This output can be viewed
+   // 12. Save the refined mesh and the solution. This output can be viewed
   //     later using GLVis: "glvis -np <np> -m sphere_refined -g sol".
   {
      ostringstream mesh_name, sol_name;
@@ -304,7 +296,7 @@ int main(int argc, char *argv[])
      x.Save(sol_ofs);
   }

-   // 14. Send the solution by socket to a GLVis server.
+   // 13. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -315,7 +307,7 @@ int main(int argc, char *argv[])
      sol_sock << "solution\n" << *pmesh << x << flush;
   }

-   // 15. Free the used memory.
+   // 14. Free the used memory.
   delete pcg;
   delete amg;
   delete fespace;
@@ -145,9 +145,9 @@ int main(int argc, char *argv[])
   xhat_space = new ParFiniteElementSpace(pmesh, xhat_fec);
   test_space = new ParFiniteElementSpace(pmesh, test_fec);

-   HYPRE_BigInt glob_true_s0     =   x0_space->GlobalTrueVSize();
-   HYPRE_BigInt glob_true_s1     = xhat_space->GlobalTrueVSize();
-   HYPRE_BigInt glob_true_s_test = test_space->GlobalTrueVSize();
+   HYPRE_Int glob_true_s0     =   x0_space->GlobalTrueVSize();
+   HYPRE_Int glob_true_s1     = xhat_space->GlobalTrueVSize();
+   HYPRE_Int glob_true_s_test = test_space->GlobalTrueVSize();
   if (myid == 0)
   {
      cout << "\nNumber of Unknowns:\n"
@@ -16,8 +16,6 @@
 //    ex9 -m ../data/disc-nurbs.mesh -p 2 -r 3 -dt 0.005 -tf 9
 //    ex9 -m ../data/periodic-square.mesh -p 3 -r 4 -dt 0.0025 -tf 9 -vs 20
 //    ex9 -m ../data/periodic-cube.mesh -p 0 -r 2 -o 2 -dt 0.02 -tf 8
-//    ex9 -m ../data/periodic-square.msh -p 0 -r 2 -dt 0.005 -tf 2
-//    ex9 -m ../data/periodic-cube.msh -p 0 -r 1 -o 2 -tf 2
 //
 // Device sample runs:
 //    ex9 -pa
@@ -131,7 +129,7 @@ private:
   mutable Vector z;

 public:
-   FE_Evolution(BilinearForm &M_, BilinearForm &K_, const Vector &b_);
+   FE_Evolution(BilinearForm &_M, BilinearForm &_K, const Vector &_b);

   virtual void Mult(const Vector &x, Vector &y) const;
   virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);
@@ -448,8 +446,8 @@ int main(int argc, char *argv[])


 // Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(BilinearForm &M_, BilinearForm &K_, const Vector &b_)
-   : TimeDependentOperator(M_.Height()), M(M_), K(K_), b(b_), z(M_.Height())
+FE_Evolution::FE_Evolution(BilinearForm &_M, BilinearForm &_K, const Vector &_b)
+   : TimeDependentOperator(_M.Height()), M(_M), K(_K), b(_b), z(_M.Height())
 {
   Array<int> ess_tdof_list;
   if (M.GetAssemblyLevel() == AssemblyLevel::LEGACY)
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Andrew Gillette	801f3dd5d7	Href ex edits	2021-03-19 15:12:45 -07:00
Andrew Gillette	9f0989b578	Michalewicz function coded as exact=3	2021-03-17 15:01:57 -07:00
Andrew Gillette	00277b5f5b	Updated case 3	2021-03-17 11:06:11 -07:00
Andrew Gillette	7fba38a3e3	New case for h ref study	2021-03-17 10:17:48 -07:00
Andrew Gillette	70b3cc2586	Adding content from convergence test	2021-03-15 15:40:22 -07:00
Andrew Gillette	71aa900a77	Starting branch from master - ha	2021-03-15 13:50:44 -07:00