Improve #include logic.

Update the 'raja-cuda' and 'occa-cuda' backends to use the
default stream 0. Optimize OperatorJacobiSmoother::Mult by fusing kernels as much as possible.
2020-10-29 05:20:58 -07:00 · 2020-10-29 04:28:15 -07:00
1231 changed files with 65648 additions and 264694 deletions
@@ -1,13 +1,10 @@
 version: '{build}'

 # https://www.appveyor.com/docs/build-environment/#build-worker-images
-image: Visual Studio 2019
+image: Visual Studio 2017

 install:

-# Start from outside clone directory
- cd ..
-
 # Install MS-MPI
 - ps: Start-FileDownload 'https://download.microsoft.com/download/B/2/E/B2EB83FE-98C2-4156-834A-E1711E6884FB/MSMpiSetup.exe'
 - MSMpiSetup.exe -unattend
@@ -18,51 +15,37 @@ install:
 - msmpisdk.msi /passive
 - set PATH=C:\Program Files\Microsoft MPI\Bin;%PATH%

-# Set MSMPI environment variables needed for CMake detection
- set MSMPI_LIB32=C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86
- set MSMPI_LIB64=C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x64
- set MSMPI_INC=C:\Program Files (x86)\Microsoft SDKs\MPI\Include
-
-# Install METIS, use MFEM's mirror because the original source server is often
-# down and we don't support yet the new repo https://github.com/KarypisLab/METIS
+# Install METIS, use a mirror because the original source server is not always
+# up. Original url:
+#    http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz
 - ps: Start-FileDownload 'https://mfem.github.io/tpls/metis-5.1.0.tar.gz'
 - 7z x metis-5.1.0.tar.gz -so | 7z x -si -ttar > nul
 - cd metis-5.1.0
 - ps: ( get-content "GKlib\gk_arch.h") | % { If ($_.ReadCount -ge 52) {$_ -replace "#ifdef __MSC__","#ifdef DISABLE_THIS_ANCIENT_MSC_CHECK"} Else {$_} } | set-content "GKlib\gk_arch.h"
 - cmake -H. -Bbuild
 # -DCMAKE_BUILD_TYPE=Release
- cmake --build build --target metis -j 4
- set METIS_PATH=%cd%
+- cmake --build build
 - cd ..

 # Install hypre
 - ps: Start-FileDownload 'https://github.com/hypre-space/hypre/archive/v2.19.0.tar.gz'
 - 7z x v2.19.0.tar.gz -so | 7z x -si -ttar > nul
 - cd hypre-2.19.0/src
- cmake -H. -Bbuild
- cmake --build build -j 4
+- cmake -H. -Bbuild -DMPI_C_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DMPI_C_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include"
+- cmake --build build
 - cmake --build build --target install
 - cd ../..

-# Return to clone directory
- cd %APPVEYOR_BUILD_FOLDER%
-
 # MFEM
 before_build:
- cmake . -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DHYPRE_DIR=%cd%\..\hypre-2.19.0\src\hypre -DMETIS_LIBRARIES=%METIS_PATH%\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%METIS_PATH%\include
-# - cmake . -Bbuild_serial -DMFEM_USE_MPI=FALSE
- cmake . -Bbuild_shared_serial -DMFEM_USE_MPI=FALSE -DBUILD_SHARED_LIBS=ON
+- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DHYPRE_DIR=%cd%\hypre-2.19.0\src\hypre -DMETIS_LIBRARIES=%cd%\metis-5.1.0\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%cd%\metis-5.1.0\include
+- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_serial -DMFEM_USE_MPI=FALSE

 build_script:
- cmake --build build_parallel --config Debug -j 4
-# - cmake --build build_serial --config Release -j 4
-# - cmake --build build_serial --target exec --config Release -j 4
- cmake --build build_shared_serial --config Release -j 4
- cmake --build build_shared_serial --target exec --config Release -j 4
+- cmake --build build_parallel --config Release -j 4
+- cmake --build build_serial --config Release -j 4
+- cmake --build build_serial --target exec --config Release -j 4

 after_build:
-# - cd build_serial
-# - ctest -C Release --output-on-failure
-# - cd ..
- cd build_shared_serial
+- cd build_serial
 - ctest -C Release --output-on-failure
@@ -1,12 +0,0 @@
-# extends https://github.com/jupyterhub/repo2docker/blob/main/repo2docker/buildpacks/conda/environment.yml
-# see https://mybinder.readthedocs.io/en/latest/using/config_files.html#environment-yml-install-a-conda-environment
-channels:
- conda-forge
-dependencies:
- xeus-cling=0.13.0
- xwidgets=0.26.0
-# NOTE: it is possible that these are not needed for the lab frontend
- widgetsnbextension=3.5.1
- pip
- pip:
-  - glvis==0.3.2
@@ -1,26 +0,0 @@
-#!/bin/bash
-
-set -e
-
-# cling is installed here (in bin) and will look in {dir}/include and {dir}/lib
-# without extra intervention (jk it doesn't look in {dir}/lib unless something
-# has been #included from {dir}/include first...)
-install_dir=/srv/conda/envs/notebook
-mkdir -p $install_dir
-
-# build and install mfem, which is the directory we start in
-make serial SHARED=YES -j8
-make install PREFIX=$install_dir
-
-# install xeus-glvis
-git clone https://github.com/GLVis/xeus-glvis.git
-pushd xeus-glvis
-make install prefix=$install_dir
-popd
-
-# install jupyter-lab extension
-jupyter labextension install @jupyter-widgets/jupyterlab-manager --no-build
-jupyter labextension install glvis-jupyter
-
-# fixup kernelspec, we could probably do this from sh but ¯\_(ツ)_/¯
-python .binder/update_kernel_env.py
@@ -1,14 +0,0 @@
-# Update the LD_LIBRARY_PATH of the C++14 kernel so it can find mfem without
-# extra pragma cling statements
-
-import json
-
-kernelspec = "/srv/conda/envs/notebook/share/jupyter/kernels/xcpp14/kernel.json"
-
-with open(kernelspec, "r") as f:
-    obj = json.load(f)
-
-obj["env"] = {"LD_LIBRARY_PATH": "/srv/conda/envs/notebook/lib"}
-
-with open(kernelspec, "w") as f:
-    json.dump(obj, f)
@@ -1,32 +0,0 @@
-codecov:
-  require_ci_to_pass: yes
-
-coverage:
-  precision: 2
-  round: nearest
-  range: "0...100"
-  status:
-    patch:
-      default:
-        target: auto
-        threshold: 0%
-        base: auto
-        branches:
-          - master
-        if_ci_failed: error
-        informational: true
-        only_pulls: true
-    project:
-      default:
-        target: auto   # compares coverage to the previous base commit
-        threshold: 1%  # allows variations around the target
-        base: auto
-        branches:
-          - master
-        if_ci_failed: error
-        only_pulls: true
-
-github_checks:
-  annotations: false
-
-comment: false
@@ -36,7 +36,7 @@ staleLabel: stale
 # Comment to post when marking an issue as stale. Set to `false` to disable
 markComment: >
  :warning: This issue or PR has been automatically marked as stale because it has not
-  had any activity in the last month. *If no activity occurs in the next week, it will
+  had any activity in the last month. *If no activity occurs in the next week, it will 
  be automatically closed.* Thank you for your contributions.

 # Comment to post when closing a stale issue. Set to `false` to disable
@@ -1,48 +0,0 @@
-                    Finite Element Discretization Library
-                                   __
-                       _ __ ___   / _|  ___  _ __ ___
-                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
-                      | | | | | ||  _||  __/| | | | | |
-                      |_| |_| |_||_|   \___||_| |_| |_|
-
-                               https://mfem.org
-
-
-This directory contains the GitHub CI scripts for MFEM.
-
-Note that some of these scripts use the shared MFEM GitHub Actions from the external mfem/github-actions repository:
-
-  https://github.com/mfem/github-actions
-
-For a particular action, e.g. `mfem/github-actions/build-mfem@v2.1`, the `v2.1` suffix denotes the branch in the above from which the action is taken.
-
-The current CI workflows are:
-
-### `repo-check.yml`
-
-Runs a number of static repository-level sanity checks.
-
- `file-headers-check` checks copyright date, license, etc. using the `--copyright`, `--license` and `--release` options of the `config/githooks/pre-push` script.
-
- `code-style` checks the code style using the `--style` option of the `config/githooks/pre-push` script.
-
- `documentation` checks the documentation build using the `tests/scripts/documentation` script.
-
- `branch-history` guards against accidental commits of large files using the `--history` option of the `config/githooks/pre-push` script.
-
-### `mfem-analysis.yml` (`build-analysis`)
-
-Checks if the code builds and satisfies minimal requirements.
-
- `gitignore` builds hypre, METIS, and MFEM using `mfem/github-actions/build-hypre`, `mfem/github-actions/build-metis`, and `mfem/github-actions/build-mfem` and checks for correct `.gitignore` settings by running the `tests/scripts/gitignore` script.
-
-### `builds-and-tests.yml`
-
-Runs a matrix of builds and tests runs with different compilers, OS, mfem/hypre settings, etc. Also processes and upload Codecov reports.
-
-Uses the following GitHub Actions from https://github.com/mfem/github-actions:
-
- `mfem/github-actions/build-hypre`
- `mfem/github-actions/build-metis`
- `mfem/github-actions/build-mfem`
- `mfem/github-actions/upload-coverage`
@@ -1,85 +0,0 @@
-name: "Docker"
-
-on:
-  # Always have a base image ready to go - this is a nightly build
-  schedule:
-    - cron: 0 3 * * *
-
-  # Allow manual trigger of a build
-  workflow_dispatch:
-
-  # On push to main we build and deploy images
-  push:
-    branches:
-      - master
-
-  # Publish packages on release
-  release:
-    types: [published]
-
-jobs:
-  build:
-    if: github.repository == 'mfem/mfem' # Don't run in forks
-    permissions:
-      packages: write
-    strategy:
-      fail-fast: false
-      matrix:
-        # Dockerfiles to build, a matrix supports future expanded builds
-        container: [["config/docker/Dockerfile.base", "ghcr.io/mfem/mfem-ubuntu-base"],
-                    ["config/docker/Dockerfile", "ghcr.io/mfem/mfem-ubuntu"]]
-
-    runs-on: ubuntu-latest
-    name: Build
-    steps:
-      - name: Run Actions Cleaner
-        uses: easimon/maximize-build-space@v8
-        with:
-          overprovision-lvm: 'true'
-          remove-dotnet: 'true'
-          remove-android: 'true'
-          remove-haskell: 'true'
-          remove-codeql: 'true'
-          remove-docker-images: 'true'
-
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      # It's easier to reference named variables than indexes of the matrix
-      - name: Set Environment
-        env:
-          dockerfile: ${{ matrix.container[0] }}
-          uri: ${{ matrix.container[1] }}
-        run: |
-          echo "dockerfile=$dockerfile" >> $GITHUB_ENV
-          echo "uri=$uri" >> $GITHUB_ENV
-
-      - name: Pull previous layers for cache
-        run: docker pull ${uri}:latest || echo "No container to pull"
-
-      - name: Build Container
-        run: |
-           container=$uri:latest
-           docker build -f ${dockerfile} -t ${container} .
-           echo "container=$container" >> $GITHUB_ENV
-
-      - name: GHCR Login
-        if: (github.event_name != 'pull_request')
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Deploy
-        if: (github.event_name != 'pull_request')
-        run: |
-            docker push ${container}
-
-      - name: Tag and Push Release
-        if: (github.event_name == 'release')
-        run: |
-            tag=${GITHUB_REF#refs/tags/}
-            echo "Tagging and releasing ${uri}:${tag}"
-            docker tag ${uri}:latest ${uri}:${tag}
-            docker push ${uri}:${tag}
@@ -1,289 +0,0 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# In this CI section, we build different variants of mfem and run test on them.
-name: "Tests"
-
-# Github actions can use the default "GITHUB_TOKEN". By default, this token
-# is set to have permissive access. However, this is not a good practice
-# security-wise. Here we use an external action, so we restrict the
-# permission to the minimum required.
-# When the 'permissions' is set, all the scopes not mentioned are set to the
-# most restrictive setting. So the following is enough.
-permissions:
-  actions: write
-
-on:
-  push:
-    branches:
-      - master
-      - next
-  pull_request:
-  workflow_dispatch:
-
-env:
-  HYPRE_ARCHIVE: v2.19.0.tar.gz
-  HYPRE_TOP_DIR: hypre-2.19.0
-  METIS_ARCHIVE: metis-4.0.3.tar.gz
-  METIS_TOP_DIR: metis-4.0.3
-  MFEM_TOP_DIR: mfem
-
-# Note for future improvements:
-#
-# We cannot reuse cached dependencies and have to build them for each target
-# although they could be shared sometimes. That's because Github cache Action
-# has no read-only mode. But there is a PR ready for this
-# (https://github.com/actions/cache/pull/489)
-
-jobs:
-  builds-and-tests:
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
-        target: [dbg, opt]
-        mpi: [seq, par]
-        build-system: [make, cmake]
-        hypre-target: [int32]
-        exclude:
-          - os: ubuntu-latest
-            build-system: cmake
-          - os: macos-latest
-            build-system: cmake
-          - os: windows-latest
-            build-system: make
-        # 'include' allows us to:
-        # - Add a variable to all jobs without creating a new matrix dimension.
-        #   Codecov is defined that way.
-        # - Add a new combination.
-        #   'build-system: cmake' and 'hypre-target: int64'
-        #
-        # Note: we will gather coverage info for any non-debug run except the
-        # CMake build.
-        include:
-          - target: dbg
-            codecov: NO
-          - target: opt
-            codecov: YES
-          - os: ubuntu-latest
-            target: dbg
-            config-opts: 'CPPFLAGS+=-Og'
-          - os: windows-latest
-            codecov: NO
-          - os: windows-latest
-            target: opt
-            mpi: par
-            config-opts: '-DBUILD_SHARED_LIBS=ON'
-          - os: ubuntu-latest
-            target: opt
-            codecov: NO
-            mpi: par
-            build-system: cmake
-            hypre-target: int32
-            # This option can be set to pass additional configuration options to
-            # the MFEM configuration command.
-            # config-opts: '-DCMAKE_VERBOSE_MAKEFILE=ON'
-          - os: ubuntu-latest
-            target: opt
-            codecov: NO
-            mpi: par
-            build-system: make
-            hypre-target: int64
-    name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
-
-    runs-on: ${{ matrix.os }}
-
-    steps:
-      # This external action allows to interrupt a workflow already running on
-      # the same branch to save resources.
-      - name: Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.11.0
-        with:
-          access_token: ${{ github.token }}
-
-      # Fix 'No space left on device' errors for Ubuntu builds.
-      - name: Run Actions Cleaner
-        if: matrix.os == 'ubuntu-latest'
-        uses: easimon/maximize-build-space@v8
-        with:
-          overprovision-lvm: 'true'
-          remove-android: 'true'
-
-      # Checkout MFEM in "mfem" subdirectory. Final path:
-      # /home/runner/work/mfem/mfem/mfem
-      # Note: Done now to access "install-hypre" and "install-metis" actions.
-      - name: checkout mfem
-        uses: actions/checkout@v3
-        with:
-          path: ${{ env.MFEM_TOP_DIR }}
-          # Fetch the complete history for codecov to access commits ID
-          fetch-depth: 0
-
-      # Only get MPI if defined for the job.
-      # TODO: It would be nice to have only one step, e.g. with a dedicated
-      # action, but I (@adrienbernede) don't see how at the moment.
-      - name: get MPI (Linux)
-        if: matrix.mpi == 'par' && matrix.os == 'ubuntu-latest'
-        run: |
-          sudo apt-get install mpich libmpich-dev
-
-      - name: get lcov (Linux)
-        if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-latest'
-        run: |
-          sudo apt-get install lcov
-
-      # Keep the following section in case we need it again in the future,
-      # see: https://github.com/mfem/mfem/pull/3385#discussion_r1058013032
-      # - name: Set up Homebrew
-      #   if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-latest'
-      #   uses: Homebrew/actions/setup-homebrew@master
-
-      - name: get MPI (MacOS)
-        if: matrix.mpi == 'par' && matrix.os == 'macos-latest'
-        run: |
-          export HOMEBREW_NO_INSTALL_CLEANUP=1
-          brew install openmpi
-
-      - name: get lcov (MacOS)
-        if: matrix.codecov == 'YES' && matrix.os == 'macos-latest'
-        run: |
-          export HOMEBREW_NO_INSTALL_CLEANUP=1
-          brew install lcov
-
-      - name: get MPI (Windows)
-        if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
-        uses: mpi4py/setup-mpi@v1.1.4
-
-      # Get Hypre through cache, or build it.
-      # Install will only run on cache miss.
-      - name: cache hypre
-        id: hypre-cache
-        if: matrix.mpi == 'par'
-        uses: actions/cache@v3
-        with:
-          path: ${{ env.HYPRE_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.2
-
-      - name: get hypre
-        if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
-        uses: mfem/github-actions/build-hypre@v2.4
-        with:
-          archive: ${{ env.HYPRE_ARCHIVE }}
-          dir: ${{ env.HYPRE_TOP_DIR }}
-          target: ${{ matrix.hypre-target }}
-          build-system: make
-
-      - name: get hypre (Windows)
-        if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
-        uses: mfem/github-actions/build-hypre@v2.4
-        with:
-          archive: ${{ env.HYPRE_ARCHIVE }}
-          dir: ${{ env.HYPRE_TOP_DIR }}
-          target: ${{ matrix.hypre-target }}
-          build-system: cmake
-
-      # Get Metis through cache, or build it.
-      # Install will only run on cache miss.
-      - name: cache metis
-        id: metis-cache
-        if: matrix.mpi == 'par' && matrix.os != 'windows-latest'
-        uses: actions/cache@v3
-        with:
-          path: ${{ env.METIS_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
-
-      - name: install metis
-        if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.4
-        with:
-          archive: ${{ env.METIS_ARCHIVE }}
-          dir: ${{ env.METIS_TOP_DIR }}
-
-      - name: cache vcpkg (Windows)
-        id: vcpkg-cache
-        if: matrix.os == 'windows-latest'
-        uses: actions/cache@v3
-        with:
-          path: vcpkg_cache
-          key: ${{ runner.os }}-${{ matrix.mpi }}-vcpkg-v1
-
-      - name: prepare vcpkg binary cache location (Windows)
-        if: matrix.os == 'windows-latest' && steps.vcpkg-cache.outputs.cache-hit != 'true'
-        run: |
-          mkdir -p vcpkg_cache
-
-      - name: install metis (Windows)
-        if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
-        env:
-          VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
-        run: |
-          vcpkg install metis-mfem --triplet=x64-windows-static --overlay-ports=${{ env.MFEM_TOP_DIR }}/config/vcpkg/ports
-
-      # MFEM build and test
-      - name: build
-        uses: mfem/github-actions/build-mfem@v2.4
-        env:
-          VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
-        with:
-          os: ${{ matrix.os }}
-          target: ${{ matrix.target }}
-          codecov: ${{ matrix.codecov }}
-          mpi: ${{ matrix.mpi }}
-          build-system: ${{ matrix.build-system }}
-          hypre-dir: ${{ env.HYPRE_TOP_DIR }}
-          metis-dir: ${{ env.METIS_TOP_DIR }}
-          mfem-dir: ${{ env.MFEM_TOP_DIR }}
-          config-options: ${{ matrix.config-opts }}
-          library-only: ${{ matrix.target == 'dbg' && matrix.os != 'ubuntu-latest' }}
-
-      # Run checks (and only checks) on debug targets
-      - name: checks
-        if: matrix.build-system == 'make' && matrix.target == 'dbg'
-        run: |
-          cd ${{ env.MFEM_TOP_DIR }}  && make check
-
-      # Note: 'tests' include the unit tests
-      - name: tests
-        if: matrix.build-system == 'make' && (matrix.target == 'opt' || matrix.os == 'ubuntu-latest')
-        run: |
-          cd ${{ env.MFEM_TOP_DIR }} && make test
-
-      - name: cmake checks
-        if: matrix.build-system == 'cmake' && matrix.target == 'dbg'
-        run: |
-          CTEST_CONFIG="Debug"
-          cd ${{ env.MFEM_TOP_DIR }} && cmake --build build --target check --config ${CTEST_CONFIG}
-        shell: bash
-
-      - name: cmake unit tests (Ubuntu)
-        if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os == 'ubuntu-latest'
-        run: |
-          CTEST_CONFIG="Release"
-          [[ ${{ matrix.target }} == 'dbg' ]] && CTEST_CONFIG="Debug"
-          cd ${{ env.MFEM_TOP_DIR }}/build/tests/unit && ctest --output-on-failure -C ${CTEST_CONFIG}
-        shell: bash
-
-      - name: cmake tests
-        if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os != 'ubuntu-latest'
-        run: |
-          CTEST_CONFIG="Release"
-          cd ${{ env.MFEM_TOP_DIR }}/build && \
-          ctest --output-on-failure -C ${CTEST_CONFIG} || \
-          ctest --rerun-failed --output-on-failure -C ${CTEST_CONFIG}
-        shell: bash
-
-      # Code coverage (process and upload reports)
-      - name: codecov
-        if: matrix.codecov == 'YES'
-        uses: mfem/github-actions/upload-coverage@v2.4
-        with:
-          name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
-          project_dir: ${{ env.MFEM_TOP_DIR }}
-          directories: "fem general linalg mesh"
@@ -1,69 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "Static Analysis"
-
-on:
-  push:
-    branches: ["master", "next"]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: ["master"]
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'cpp' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
-        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-
-      # Initializes the CodeQL tools for scanning.
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
-        with:
-          languages: ${{ matrix.language }}
-          # If you wish to specify custom queries, you can do so here or in a config file.
-          # By default, queries listed here will override any specified in a config file.
-          # Prefix the list here with "+" to use these queries and those in the config file.
-
-          # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-          # queries: security-extended,security-and-quality
-
-      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-      # If this step fails, then you should remove it and run the build manually (see below)
-      - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
-
-      # ℹ️ Command-line programs to run using the OS shell.
-      # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
-
-      #   If the Autobuild fails above, remove it and uncomment the following three lines.
-      #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
-
-      # - run: |
-      #   echo "Run, Build Application using script"
-      #   ./location_of_script_within_repo/buildscript.sh
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
@@ -1,100 +0,0 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-name: "Build Analysis"
-
-permissions:
-  actions: write
-
-on:
-  push:
-    branches:
-      - master
-      - next
-  pull_request:
-  workflow_dispatch:
-
-env:
-  HYPRE_ARCHIVE: v2.19.0.tar.gz
-  HYPRE_TOP_DIR: hypre-2.19.0
-  METIS_ARCHIVE: metis-4.0.3.tar.gz
-  METIS_TOP_DIR: metis-4.0.3
-  COVERAGE_ENV: mfem-coverage
-
-jobs:
-  gitignore:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.11.0
-        with:
-          access_token: ${{ github.token }}
-
-      - name: checkout MFEM
-        uses: actions/checkout@v3
-        with:
-          path: mfem
-
-      - name: Get MPI (Linux)
-        run: |
-          sudo apt-get install mpich libmpich-dev
-
-      - name: Cache Hypre Install
-        id: hypre-cache
-        uses: actions/cache@v3
-        with:
-          path: ${{ env.HYPRE_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.2
-
-      - name: Get Hypre
-        if: steps.hypre-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-hypre@v2.4
-        with:
-          archive: ${{ env.HYPRE_ARCHIVE }}
-          dir: ${{ env.HYPRE_TOP_DIR }}
-          target: int32
-
-      - name: Cache Metis Install
-        id: metis-cache
-        uses: actions/cache@v3
-        with:
-          path: ${{ env.METIS_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
-
-      - name: Install Metis
-        if: steps.metis-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.4
-        with:
-          archive: ${{ env.METIS_ARCHIVE }}
-          dir: ${{ env.METIS_TOP_DIR }}
-
-      # MFEM build and test
-      - name: build-mfem
-        uses: mfem/github-actions/build-mfem@v2.4
-        with:
-          os: ${{ runner.os }}
-          target: opt
-          codecov: NO
-          mpi: par
-          build-system: make
-          hypre-dir: ${{ env.HYPRE_TOP_DIR }}
-          metis-dir: ${{ env.METIS_TOP_DIR }}
-          mfem-dir: mfem
-
-      - name: test (no clean)
-        run: |
-          cd mfem && make test-noclean
-
-      - name: gitignore
-        run: |
-          cd mfem/tests/scripts
-          ./runtest gitignore
@@ -1,70 +0,0 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-name: "Sanitizer"
-
-permissions:
-  actions: write
-
-on:
-  push:
-    branches:
-      - master
-      - next
-  pull_request:
-  workflow_dispatch:
-
-jobs:
-  Serial:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.11.0
-        with:
-          access_token: ${{ github.token }}
-
-      - name: MFEM Checkout
-        uses: actions/checkout@v3
-        with:
-          path: mfem
-
-      - name: MFEM Build
-        uses: mfem/github-actions/build-mfem@v2.4
-        with:
-          os: ${{ runner.os }}
-          target: opt
-          mpi: seq
-          hypre-dir: unused-hypre-dir
-          metis-dir: unused-metis-dir
-          mfem-dir: mfem
-          build-system: make
-          library-only: false
-          config-options:
-            CXX="clang++-14"
-            CXXFLAGS="-g -O1 -std=c++11
-                      -fsanitize=address
-                      -fno-omit-frame-pointer
-                      -fsanitize-address-use-after-scope"
-
-      - name: MFEM Info
-        working-directory: mfem
-        run: make info
-
-      - name: MFEM Sanitize
-        working-directory: mfem
-        run:
-          ASAN_OPTIONS="detect_leaks=1,
-                        strict_init_order=1,
-                        strict_string_checks=1,
-                        check_initialization_order=1,
-                        detect_stack_use_after_return=1"
-          make test
@@ -1,140 +0,0 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-name: "Checks"
-
-permissions:
-  actions: write
-
-on:
-  push:
-  pull_request:
-  workflow_dispatch:
-
-# This workflow is run on pushes to any branch in the MFEM repo (with or without
-# PRs), as well as on updates to PRs from forks. In particular, we do not
-# duplicate work by running on both pushes and updates to local PRs. We do that
-# by checking if the workflow trigger is 'push' ("github.event_name == 'push'")
-# and if we are in a fork ("github.event.pull_request.head.repo.full_name !=
-# github.repository").
-
-jobs:
-  file-headers-check:
-    runs-on: ubuntu-latest
-    if: |
-      (github.event_name == 'push' ||
-       github.event.pull_request.head.repo.full_name != github.repository)
-    steps:
-      - name: Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.11.0
-        with:
-          access_token: ${{ github.token }}
-
-      - name: checkout mfem
-        uses: actions/checkout@v3
-
-      - name: copyright check
-        id: copyright
-        run: |
-          ./config/githooks/pre-push --copyright
-
-        continue-on-error: true
-
-      - name: license check
-        id: license
-        run: |
-          ./config/githooks/pre-push --license
-        continue-on-error: true
-
-      - name: release check
-        id: release
-        run: |
-          ./config/githooks/pre-push --release
-        continue-on-error: true
-
-      - name: wrap-up
-        if: |
-          steps.copyright.outcome != 'success' ||
-          steps.license.outcome != 'success' ||
-          steps.release.outcome != 'success'
-        run: |
-          if [[ "${{ steps.copyright.outcome }}" != "success" ]]; then
-            echo "copyright check failed, unroll log for details"
-          fi
-          if [[ "${{ steps.license.outcome }}" != "success" ]]; then
-            echo "license check failed, unroll log for details"
-          fi
-          if [[ "${{ steps.release.outcome }}" != "success" ]]; then
-            echo "release check failed, unroll log for details"
-          fi
-          exit 1
-
-  code-style:
-    runs-on: ubuntu-latest
-    if: |
-      (github.event_name == 'push' ||
-       github.event.pull_request.head.repo.full_name != github.repository)
-    steps:
-      - name: checkout mfem
-        uses: actions/checkout@v3
-
-      - name: get astyle
-        run: |
-          sudo apt-get install astyle
-
-      - name: style check
-        run: |
-          ./config/githooks/pre-push --style
-
-  documentation:
-    runs-on: ubuntu-latest
-    if: |
-      (github.event_name == 'push' ||
-       github.event.pull_request.head.repo.full_name != github.repository)
-    steps:
-      - name: checkout mfem
-        uses: actions/checkout@v3
-
-      - name: get doxygen and graphviz
-        run: |
-          sudo apt-get install doxygen graphviz
-
-      - name: update doxygen config file
-        run: |
-          cd doc
-          doxygen -u CodeDocumentation.conf.in
-
-      - name: build documentation
-        run: |
-          cd tests/scripts
-          ./runtest documentation
-
-  branch-history:
-    if: |
-      github.ref != 'refs/heads/next' &&
-      github.ref != 'refs/heads/master' &&
-      (github.event_name == 'push' ||
-       github.event.pull_request.head.repo.full_name != github.repository)
-    runs-on: ubuntu-latest
-    steps:
-      - name: checkout mfem
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: branch-history
-        run: |
-          # We override origin to make sure we point to the main repo.
-          # This is to have consistent test results on PRs from forks.
-          git remote remove origin
-          git remote add origin https://github.com/mfem/mfem.git
-          git checkout -b gh-actions-branch-history
-          ./config/githooks/pre-push --history
@@ -21,16 +21,11 @@ CMakeFiles/
 # Default install location
 /mfem/

-# Typical build directory
-/build/
-
 # Generated files in main directory, config/ and docs/
 /deps.mk
 config/_config.hpp
 config/config.mk
 config/sample-runs-build.log
-config/user.cmake
-config/user.mk
 doc/CodeDocumentation.conf
 doc/CodeDocumentation.html
 doc/CodeDocumentation
@@ -49,14 +44,12 @@ doc/warnings.log

 # Example and miniapp binaries and outputs

-examples/ex[0-9]
-examples/ex[0-9]p
+examples/ex[1-9]
+examples/ex[1-9]p
 examples/ex1[04-9]
 examples/ex1[0-9]p
 examples/ex2[0-9]
 examples/ex2[0-9]p
-examples/ex3[0-9]
-examples/ex3[0-9]p

 examples/refined.mesh
 examples/displaced.mesh
@@ -73,7 +66,6 @@ examples/sol_u.*
 examples/sol_p.*
 examples/sol_r.*
 examples/sol_i.*
-examples/ex6p-checkpoint.*
 examples/ex9.mesh
 examples/ex9-mesh.*
 examples/ex9-init.*
@@ -82,7 +74,6 @@ examples/deformed.*
 examples/velocity.*
 examples/elastic_energy.*
 examples/mode_*
-examples/mode_deriv_*
 examples/ex5-p-*.bp
 examples/ex9-p-*.bp
 examples/ex12-p-*.bp
@@ -110,15 +101,6 @@ examples/Example23*
 examples/ex25.mesh
 examples/ex25-*.gf
 examples/ex25p-*.*
-examples/ex28_*
-examples/ex28p_*
-examples/flux.*
-examples/dsol.*
-examples/cond.*
-examples/cond_j.*
-examples/cond_mesh.*
-examples/port_mesh.*
-examples/port_mode.*

 examples/amgx/ex1
 examples/amgx/ex1p
@@ -128,9 +110,11 @@ examples/amgx/sol.gf
 examples/amgx/mesh.*
 examples/amgx/sol.*

-examples/ginkgo/ex1
-examples/ginkgo/refined.mesh
-examples/ginkgo/sol.gf
+examples/gingko/ex1
+examples/gingko/refined.mesh
+examples/gingko/sol.gf
+examples/gingko/mesh.*
+examples/gingko/sol.*

 examples/hiop/ex9
 examples/hiop/ex9p
@@ -156,8 +140,8 @@ examples/petsc/velocity.*
 examples/petsc/elastic_energy.*
 examples/petsc/mode_*

-examples/pumi/ex[12]
-examples/pumi/ex[16]p
+examples/pumi/ex1
+examples/pumi/ex[126]p
 examples/pumi/refined.mesh
 examples/pumi/sol.gf
 examples/pumi/mesh.*
@@ -202,9 +186,6 @@ miniapps/gslib/field-diff
 miniapps/gslib/field-interp
 miniapps/gslib/findpts
 miniapps/gslib/pfindpts
-miniapps/gslib/schwarz_ex1
-miniapps/gslib/schwarz_ex1p
-miniapps/gslib/interpolated.gf

 miniapps/meshing/mobius-strip
 miniapps/meshing/klein-bottle
@@ -213,69 +194,41 @@ miniapps/meshing/twist
 miniapps/meshing/mesh-explorer
 miniapps/meshing/shaper
 miniapps/meshing/extruder
-miniapps/meshing/fit-node-position
 miniapps/meshing/trimmer
-miniapps/meshing/reflector
 miniapps/meshing/mesh-optimizer
 miniapps/meshing/pmesh-optimizer
-miniapps/meshing/pmesh-fitting
 miniapps/meshing/minimal-surface
 miniapps/meshing/pminimal-surface
 miniapps/meshing/polar-nc
-miniapps/meshing/mesh-quality
 miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
 miniapps/meshing/toroid-*.mesh
 miniapps/meshing/twist-*.mesh
 miniapps/meshing/mesh-explorer.mesh
 miniapps/meshing/partitioning.txt
-miniapps/meshing/mesh-explorer-visit*
-miniapps/meshing/mesh-explorer-paraview/
 miniapps/meshing/shaper.mesh
 miniapps/meshing/extruder.mesh
 miniapps/meshing/trimmer.mesh
-miniapps/meshing/reflected.mesh
 miniapps/meshing/optimized*
 miniapps/meshing/perturbed*
 miniapps/meshing/polar-nc.mesh

-miniapps/mtop/parheat
-miniapps/mtop/ParHeat*
-miniapps/mtop/seqheat
-miniapps/mtop/SeqHeat*
-
-miniapps/autodiff/paradiff
-miniapps/autodiff/seqadiff
-miniapps/autodiff/seqtest
-miniapps/autodiff/par_example
-miniapps/autodiff/seq_example
-miniapps/autodiff/seq_test
-miniapps/autodiff/Exampl*
-
 miniapps/navier/navier_mms
 miniapps/navier/navier_kovasznay
-miniapps/navier/navier_kovasznay_vs
 miniapps/navier/navier_tgv
 miniapps/navier/navier_shear
 miniapps/navier/navier_3dfoc
-miniapps/navier/navier_turbchan
-miniapps/navier/navier_cht
 miniapps/navier/tgv_out*.txt
 miniapps/navier/*_output

 miniapps/nurbs/nurbs_ex1
 miniapps/nurbs/nurbs_ex1p
 miniapps/nurbs/nurbs_ex11p
-miniapps/nurbs/nurbs_printfunc
-miniapps/nurbs/nurbs_patch_ex1
-miniapps/nurbs/nurbs_curveint
 miniapps/nurbs/refined.mesh
 miniapps/nurbs/mesh.*
 miniapps/nurbs/sol.*
 miniapps/nurbs/mode_*
 miniapps/nurbs/Example1*
-miniapps/nurbs/sin-fit.mesh
-miniapps/nurbs/CurveInt

 miniapps/performance/ex1
 miniapps/performance/ex1p
@@ -283,29 +236,11 @@ miniapps/performance/refined.mesh
 miniapps/performance/mesh.*
 miniapps/performance/sol.*

-miniapps/shifted/distance
-miniapps/shifted/ParaViewDistance
-miniapps/shifted/ParaViewLSF
-miniapps/shifted/extrapolate
-miniapps/shifted/ParaViewExtrapolate
-miniapps/shifted/diffusion
-miniapps/shifted/diffusion.mesh
-miniapps/shifted/diffusion.gf
-miniapps/shifted/ParaViewDiffusion
-miniapps/shifted/lsf_integral
-
 miniapps/tools/display-basis
 miniapps/tools/load-dc
 miniapps/tools/convert-dc
 miniapps/tools/lor-transfer
-miniapps/tools/plor-transfer
 miniapps/tools/get-values
-miniapps/tools/check-tmop-metric
-miniapps/tools/tmop-metric-magnitude
-miniapps/tools/nodal-transfer
-miniapps/tools/ParaView
-miniapps/tools/gridfunc_*
-miniapps/tools/mesh_*

 miniapps/toys/automata
 miniapps/toys/life
@@ -328,55 +263,13 @@ miniapps/toys/lissajous.mesh
 miniapps/toys/lissajous.gf
 miniapps/toys/mondrian.mesh

-miniapps/solvers/block-solvers
-miniapps/solvers/lor_solvers
-miniapps/solvers/plor_solvers
-miniapps/solvers/ParaView
-miniapps/solvers/mesh.*
-miniapps/solvers/sol.*
-
-miniapps/hdiv-linear-solver/darcy
-miniapps/hdiv-linear-solver/grad_div
-
-miniapps/parelag/MultilevelHcurlHdivSolver
-miniapps/parelag/*.mesh
-
-miniapps/multidomain/multidomain
-miniapps/hooke/hooke
-
-miniapps/dpg/diffusion
-miniapps/dpg/pdiffusion
-miniapps/dpg/convection-diffusion
-miniapps/dpg/pconvection-diffusion
-miniapps/dpg/acoustics
-miniapps/dpg/pacoustics
-miniapps/dpg/maxwell
-miniapps/dpg/pmaxwell
-miniapps/dpg/ParaView
-
-miniapps/spde/generate_random_field
-miniapps/spde/ParaView
-
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
 tests/unit/punit_tests
-tests/unit/cunit_tests
-tests/unit/pcunit_tests
 tests/unit/sedov_tests_*
 tests/unit/psedov_tests_*
-tests/unit/tmop_pa_tests_*
-tests/unit/ptmop_pa_tests_*
 tests/unit/ceed_tests
-tests/unit/debug_device_tests
-tests/unit/parallel_in_serial.mesh
-tests/unit/parallel_in_serial.gf
-
-# Benchmark binaries
-tests/benchmarks/bench_ceed
-tests/benchmarks/bench_tmop
-tests/benchmarks/bench_vector
-tests/benchmarks/bench_virtuals

 # Test script output
 tests/scripts/*.err
@@ -390,10 +283,3 @@ tests/par-mesh-format/ex1p

 # VPATH builds
 build-*/*
-
-# PETSc automated build
-petsc-build/*
-pkg.gitcommit
-
-# Jupyter Notebook Checkpoints
-.ipynb_checkpoints
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -13,84 +13,296 @@
 # at Lawrence Livermore National Laboratory (LLNL). This entire pipeline is
 # LLNL-specific!

-
-# The pipeline is divided into stages. Usually, jobs in a given stage wait for
-# the preceding stages to complete before to start. However, we sometimes use
-# the "needs" keyword and express the DAG of jobs for more efficiency.
-# - We use setup and setup_baseline phases to download content outside of mfem
-#   directory.
-# - Allocate/Release is where quartz resource are allocated/released once for all.
-# - Build and Test is where we build and MFEM for multiple toolchains.
-# - Baseline_checks gathers baseline-type test suites execution
-# - Baseline_publish, only available on master, allows to update baseline
-#   results
-stages:
-  - sub-pipelines
-
-variables:
-  CUSTOM_CI_BUILDS_DIR: "/usr/workspace/mfem/gitlab-runner"
-  USER_CI_TOP_DIR: "${CUSTOM_CI_BUILDS_DIR}/${GITLAB_USER_LOGIN}"
-  SHARED_REPOS_DIR: "${USER_CI_TOP_DIR}/repos"
-  AUTOTEST_ROOT: "${SHARED_REPOS_DIR}"
-  # MFEM_DATA_DIR is setup in '.gitlab/configs/setup-build-and-test.yml' and
-  # used in '.gitlab/configs/<machine>-config.yml':
-  MFEM_DATA_DIR: "${SHARED_REPOS_DIR}/mfem-data"
-
+# We define the following GitLab pipeline variables:
+#
+# BUILD_ROOT:
+# The path to the shared resources between all jobs. For example, external
+# repositories like 'tests' and 'tpls' are cloned here. Also, 'tpls' is built
+# once for all targets, so that build happen here. The BUILD_ROOT is unique to
+# the pipeline, preventing any form of concurrency with other pipelines. This
+# also means that the BUILD_ROOT directory will never be cleaned.
+# TODO: add a clean-up mechanism
+#
+# BUILD_PATH:
+# In BUILD_ROOT, we want to separate builds depending on the machine used,
+# typically because we build on a set of dependencies per machine.
+#
+# REBASELINE:
 # Defines the default choice for updating the saved baseline results. By default
 # the baseline can only be updated from the master branch. This variable offers
 # the option to manually ask for rebaselining from another branch if necessary.
+#
+# MFEM_ALLOC_NAME:
+# On LLNL's quartz, there is only one allocation shared among jobs in order to
+# save time and resources. This allocation has to be uniquely named so that we
+# are sure to retrieve it.
+#
+# TPLS_REPO & TESTS_REPO:
+# Git repositories used in the pipeline
+#
+# ARTIFACTS_DIR:
+# Directory used to place artifacts.
+
+variables:
+  BUILD_ROOT: ${CI_BUILDS_DIR}/${CI_PROJECT_NAME}_${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID}
+  BUILD_PATH: ${CI_BUILDS_DIR}/${CI_PROJECT_NAME}_${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID}/${PLAT}/${TOOLCHAIN}
  REBASELINE: "NO"
-  AUTOTEST: "NO"
-  # AUTOTEST_COMMIT: used only when AUTOTEST is set to YES.
-  # * If AUTOTEST_COMMIT is NOT set to NO, reporting jobs will commit their
-  #   files to the MFEM/autotest repo.
-  # * If AUTOTEST_COMMIT is set to NO, reporting jobs will NOT commit their
-  #   files to the MFEM/autotest repo. Instead they will just show the contents
-  #   of the report files and remove them.
-  AUTOTEST_COMMIT: "YES"
+  MFEM_ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
+  TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
+  TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
+  ARTIFACTS_DIR: artifacts

-# Trigger subpipelines:
-quartz-build-and-test:
-  stage: sub-pipelines
-  variables:
-    # Explicitly pass down values that we want to be able to set when triggering
-    # pipelines manually or using scheduling
-    AUTOTEST: "${AUTOTEST}"
-    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
-  trigger:
-    include: .gitlab/quartz-build-and-test.yml
-    strategy: depend
+# The pipeline is divided into stages. Usually, these are also synchronization
+# points, however, we use "needs" keyword to express the DAG of jobs for more
+# efficiency.
+# - We use setup phase to download content outside of mfem directory.
+# - Allocate is where quartz resources are allocated once for all.
+# - Libs is where dependencies are built.
+# - Build is where we build MFEM for multiple toolchains.
+# - Test is where we perform a first set of tests on MFEM build.
+# - Baseline_checks gathers baseline-type test suites execution
+# - Baseline_publish, only available on master, allows to update baseline
+#   results
+# - Deallocate releases quartz resources
+stages:
+  - setup
+  - allocate
+  - libs
+  - build
+  - test
+  - deallocate
+  - lassen_build
+  - lassen_test
+  - baseline_check
+  - baseline_publish

-quartz-baseline:
-  stage: sub-pipelines
+# The setup job in setup stage don’t rely on MFEM git repo. It prepares a
+# pipeline-wide working directory downloading/updating external repos.
+# TODO: updating tests and tpls is not necessary anymore since pipelines are
+# now using unique directories so repo are never shared with another pipeline.
+# This is not memory efficient (we keep a lot of data), hence this reminder.
+# Setup
+setup:
+  tags:
+    - shell
+    - quartz
+  stage: setup
  variables:
-    # Explicitly pass down values that we want to be able to set when triggering
-    # pipelines manually or using scheduling
-    REBASELINE: "${REBASELINE}"
-    AUTOTEST: "${AUTOTEST}"
-    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
-  trigger:
-    include: .gitlab/quartz-baseline.yml
-    strategy: depend
+    GIT_STRATEGY: none
+  script:
+    - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
+    - if [ ! -d "tpls" ]; then git clone ${TPLS_REPO}; fi
+    - if [ ! -d "tests" ]; then git clone ${TESTS_REPO}; fi
+    - cd tpls && git pull && cd ..
+    - cd tests && git pull && cd ..

-lassen-build-and-test:
-  stage: sub-pipelines
+# Share configurations (templates) for toolchain configuration.
+# Note: Any job using this template shouldn't redefine "before_script" key.
+.with_gcc_6_1_0:
  variables:
-    # Explicitly pass down values that we want to be able to set when triggering
-    # pipelines manually or using scheduling
-    AUTOTEST: "${AUTOTEST}"
-    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
-  trigger:
-    include: .gitlab/lassen-build-and-test.yml
-    strategy: depend
+    TOOLCHAIN: gcc_6_1_0
+    CXX: g++
+    CC: gcc
+  before_script:
+    - module load gcc/6.1.0

-corona-build-and-test:
-  stage: sub-pipelines
+# On lassen
+.with_gcc_8_3_1:
  variables:
-    # Explicitly pass down values that we want to be able to set when triggering
-    # pipelines manually or using scheduling
-    AUTOTEST: "${AUTOTEST}"
-    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
-  trigger:
-    include: .gitlab/corona-build-and-test.yml
-    strategy: depend
+    TOOLCHAIN: gcc_8_3_1
+    CXX: g++
+    CC: gcc
+  before_script:
+    - module load gcc/8.3.1
+
+.with_gcc_4_9_3:
+  variables:
+    TOOLCHAIN: gcc_4_9_3
+    CXX: g++
+    CC: gcc
+  before_script:
+    - module load gcc/4.9.3
+
+.if_srun_alloc_for_make: &if_srun_alloc_for_make |
+  if [[ "${MPIEXEC}" == "srun" ]]
+  then
+    export JOB_ID=$(squeue -h --name=${MFEM_ALLOC_NAME} --format=%A)
+    [[ -n "JOB_ID" ]] && export JOB_ID="--jobid=${JOB_ID}"
+    RESOURCES="--extra-node-info=1:${MAKE_PAR}:1 --cpus-per-task=${MAKE_PAR} --ntasks=1"
+    # Allocation prefix:
+    export EXEC_PREFIX="srun ${JOB_ID} ${RESOURCES}"
+  fi
+
+# Build of dependencies and mfem:
+# if on quartz, uses a part of preallocated resources
+# Note: Those jobs don't rely on MFEM repository, hence the use of
+# "GIT_STRATEGY: none" preventing its clone.
+.build_hypre:
+  variables:
+    EXEC_PREFIX: ""
+    GIT_STRATEGY: none
+  stage: libs
+  script:
+    - cd ${BUILD_ROOT}/tpls
+    - mkdir -p ${BUILD_PATH}
+    - make clean
+    - *if_srun_alloc_for_make
+    - ${EXEC_PREFIX} make -j ${MAKE_PAR} hypre PREFIX=${BUILD_PATH}/hypre/src/ BUILD_DIR=${BUILD_PATH} LOG_DIR=${BUILD_PATH}
+
+.build_metis:
+  variables:
+    EXEC_PREFIX: ""
+    GIT_STRATEGY: none
+  stage: libs
+  script:
+    - cd ${BUILD_ROOT}/tpls
+    - mkdir -p ${BUILD_PATH}
+    - *if_srun_alloc_for_make
+    - ${EXEC_PREFIX} make -j ${MAKE_PAR} ${METIS} PREFIX=${BUILD_PATH}/ BUILD_DIR=${BUILD_PATH} LOG_DIR=${BUILD_PATH} CC=${CC} CXX=${CXX}
+
+# Building MFEM
+.build_mfem:
+  variables:
+    EXEC_PREFIX: ""
+  stage: build
+  script:
+    - mkdir -p ${BUILD_PATH}
+    - cp -r ${CI_PROJECT_DIR} ${BUILD_PATH}/${CI_PROJECT_NAME}_${MFEM_USE_MPI}_${MFEM_DEBUG}
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_${MFEM_USE_MPI}_${MFEM_DEBUG}
+    - *if_srun_alloc_for_make
+    - make config CXX=${CXX} MPICXX=mpicxx MFEM_USE_MPI=${MFEM_USE_MPI} MFEM_DEBUG=${MFEM_DEBUG} MFEM_MPIEXEC="${EXEC_PREFIX}" MFEM_MPIEXEC_NP=${MPIEXEC_NP}
+    - ${EXEC_PREFIX} make -j ${MAKE_PAR} all
+
+# TODO: Could this phase use a parallel run?
+.sanitycheck_mfem:
+  stage: test
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_${MFEM_USE_MPI}_${MFEM_DEBUG}
+    - make test
+
+# Shared script for baseline and sample-run-baseline, the value of BASELINE_TEST
+# differentiates between the two tests.
+.baseline_script: &baseline_script |
+  # locals
+  _glob_out=${BASELINE_TEST}.out
+  _glob_err=${BASELINE_TEST}.err
+  _base_diff=${BASELINE_TEST}-${SYS_TYPE}.diff
+  _base_patch=${BASELINE_TEST}-${SYS_TYPE}.patch
+  _base_out=${BASELINE_TEST}-${SYS_TYPE}.out
+
+  _out=${BASELINE_TEST}-${SYS_TYPE}.out
+  _ref=../${BASELINE_TEST}-${SYS_TYPE}.saved
+  _out_txt=${BASELINE_TEST}.txt
+  _diff=${BASELINE_TEST}-diff.txt
+  # prepare
+  cd ${BUILD_ROOT}
+  ln -snf ${CI_PROJECT_DIR} mfem
+  cd tests
+  mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
+  # run
+  salloc --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${ADDITIONAL_DIR}"
+  # post
+  mkdir ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}
+  if [[ -s ${_glob_err} ]]
+  then
+    echo "ERROR during ${BASELINE_TEST} execution";
+    echo "Here is the ${_glob_err} file content";
+    cat ${_glob_err}
+    cp ${_glob_err} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_glob_err}.txt
+    exit 1;
+  elif [[ ! -f ${_base_patch} && ! -f ${_base_out} ]]
+  then
+    echo "Something went WRONG in ${BASELINE_TEST}:";
+    echo "Either ${_base_patch} or ${_base_out} should exists";
+    exit 1;
+  elif [[ -f ${_base_patch} ]]
+  then
+    echo "${BASELINE_TEST}: Differences found, patch generated"
+    cp ${_base_patch} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_patch}.txt
+  elif [[ -f ${_base_out} ]]
+  then
+    echo "${BASELINE_TEST}: Differences found, replacement file generated"
+    cp ${_base_out} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_out}.txt
+  fi
+  # _base_diff won’t even exist if there is no difference.
+  if [[ -f ${_base_diff} ]]
+  then
+    echo "${BASELINE_TEST}: Relevant differences (filtered diff) ..."
+    cat ${_base_diff}
+    cp ${_base_diff} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_diff}.txt
+  fi
+  if [[ ! -s ${_base_diff} ]]
+  then
+    echo "${BASELINE_TEST}: PASSED"
+    true
+  else
+    echo "${BASELINE_TEST}: FAILED"
+    false
+  fi
+
+# Actual templates for baseline checks
+.baselinecheck_mfem:
+  stage: baseline_check
+  variables:
+    BASELINE_TEST: baseline
+    ADDITIONAL_DIR: ${BUILD_ROOT}/tpls
+  script:
+    - *baseline_script
+  artifacts:
+    when: always
+    paths:
+      - ${ARTIFACTS_DIR}
+  allow_failure: true
+
+.samplebaselinecheck_mfem:
+  stage: baseline_check
+  variables:
+    BASELINE_TEST: sample-runs-baseline
+    ADDITIONAL_DIR: ""
+  script:
+    - *baseline_script
+  timeout: 4h
+  artifacts:
+    when: always
+    paths:
+      - ${ARTIFACTS_DIR}
+  allow_failure: true
+
+# This job can only be manually triggers on a pipeline for master branch, or if
+# the pipeline was triggered with REBASELINE="YES"
+.rebaseline_mfem:
+  stage: baseline_publish
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "YES"'
+      when: manual
+  script:
+    - export PATCH_FILE=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}.patch
+    - export FULL_FILE=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}.out
+    - export DIFF_FILE=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}.diff
+    - cd ${BUILD_ROOT}/tests
+    - |
+      if [[ ! -f "${DIFF_FILE}.txt" ]]
+      then
+          echo "Nothing to be done: no relevant change in baseline"
+          exit 0
+      elif [[ -f "${PATCH_FILE}.txt" ]]
+      then
+          mv ${PATCH_FILE}.txt ${PATCH_FILE}
+          patch "./baseline-${SYS_TYPE}.saved" < "${PATCH_FILE}"
+      elif [[ -f "${FULL_FILE}.txt" ]]
+      then
+          cp "${FULL_FILE}.txt" "./baseline-${SYS_TYPE}.saved"
+      else
+          echo "File missing: expected ${PATCH_FILE}.txt or ${FULL_FILE}.txt"
+          exit 1
+      fi
+    - git add baseline-${SYS_TYPE}.saved
+    - git commit -m "${SYS_TYPE} rebaselined in GitLab pipeline ${CI_PIPELINE_ID}"
+    - git push origin master
+
+# The list on jobs is defined in machine-specific files.
+include:
+  - local: .gitlab/quartz.yml
+  - local: .gitlab/lassen.yml
@@ -1,94 +0,0 @@
-                    Finite Element Discretization Library
-                                   __
-                       _ __ ___   / _|  ___  _ __ ___
-                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
-                      | | | | | ||  _||  __/| | | | | |
-                      |_| |_| |_||_|   \___||_| |_| |_|
-
-                               https://mfem.org
-
-
-This directory contains most of the GitLab CI configuration. MFEM runs both PR
-and nightly testing on GitLab.
-
-# Structure
-
-## Top level
-
-The root configuration file is `.gitlab-ci.yml` at the root of MFEM repo.
-This file only defines one stage, in which we trigger several
-sub-pipelines.
-
-We use sub-pipelines to isolate the test for one combination of `machine`
-and `test type`.
-
-Machines typically include:
-
-* Quartz: Intel bi-socket x86
-* Lassen: Power9 + Nvidia GPU
-* Corona: AMD GPU
-
-Test types include:
-
-* Build and test: Spack driven build of dependencies, mfem build, mfem
-  test
-* Baseline: Script driven build of dependencies, thorough testing
-
-⚠️ The sub-pipeline design allows to add a new machine or a new test type without
-altering the scheduling, execution and displaying of the others.
-
-## Sub-pipelines
-
-Each file is this directory is the root configuration file for one
-sub-pipeline. The naming reflects the corresponding couple (`machine`,
-`test_type`).
-
-Those files define the *stages* and the *jobs* for the sub-pipeline. They
-also contain any configuration that cannot be shared. For the most part
-though, the configuration is shared and is placed in `.gitlab/configs`.
-
-We try to keep scripts out of the CI config and share them among similar
-jobs. They are gathered in `.gitlab/scripts`.
-
-## Scripts
-
-Scripts specific to the CI only are in `.gitlab/scripts`. It is best practice
-to keep scripts outside the CI configuration (no bash scripts embedded in a
-yaml file) because it helps with readability, maintenance and also with
-transition to another CI system.
-
-⚠️ Most of the scripts there are driven by environment variables and do not have a
-usage function. This should be improved.
-
-
-# More testing
-
-## Adding a new target to a build_and_test pipeline
-
-`build_and_test` pipelines rely on Spack to install dependencies. Spack is
-driven by Uberenv which helps freezing Spack configuration: the goal being to
-point to specific commit in Spack and isolate its configuration so that it is
-not influenced by the user environment. More documentation about this can be
-found in `tests/gitlab`.
-
-In the end, the MFEM target for which to build the dependencies is expressed
-with a spack spec of MFEM, within the limits permitted by the MFEM spack
-package.
-
-In any build-and-test sub-pipeline a job basically consists in defining the
-spack spec to use. Adding a job on quartz for example resumes to:
-
-```yaml
-<job_name>:
-  variables:
-    SPEC: "<spack_spec>"
-  extends: .build_and_test_on_quartz
-```
-
-The remaining and non trivial work is to make sure this spec is working. To
-test a spec before adding it, or reproduce a CI configuration, please refer to
-`tests/gitlab/reproduce-ci-jobs-interactively.md`.
-
-⚠️ It is assumed that the spack spec applies to `mfem@develop`. That's why in the
-CI all the specs start with the compiler or the variants to apply to mfem. The
-mechanism still works with a full spec.
@@ -1,36 +0,0 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# We define the following GitLab pipeline variables:
-variables:
-
-# The path to the shared resource between all jobs. For example, external
-# repositories like 'tests' and 'tpls' are cloned here. Also, 'tpls' is built
-# once for all targets, so that build happen here. The BUILD_ROOT is unique to
-# the pipeline, preventing any form of concurrency with other pipelines. This
-# also means that the BUILD_ROOT directory will never be cleaned.
-# TODO: add a clean-up mechanism
-  BUILD_ROOT: ${USER_CI_TOP_DIR}/${CI_PROJECT_NAME}-${MACHINE_NAME}-pipeline-${CI_PIPELINE_ID}
-
-# On LLNL's quartz, there is only one allocation shared among jobs in order to
-# save time and resource. This allocation has to be uniquely named so that we
-# are sure to retrieve it.
-  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
-
-# Git repositories used in the pipeline
-  TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
-  TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
-  AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
-  MFEM_DATA_REPO: https://github.com/mfem/data.git
-
-# Directory used to place artifacts.
-  ARTIFACTS_DIR: artifacts
-  SLURM_OVERLAP: 1
@@ -1,59 +0,0 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# GitLab pipeline configuration for the Corona machine at LLNL
-variables:
-  MACHINE_NAME: corona
-
-.on_corona:
-  tags:
-    - shell
-    - corona
-  rules:
-    # Don't run corona jobs if...
-    # Note: This makes corona an "opt-in" machine. To activate builds on corona
-    # for a given GitLab clone of MFEM, go to Setting/CI-CD/variables, and set
-    # "ON_CORONA" to "ON". An LC account on for corona is required to trigger a
-    # pipeline there.
-    - if: '$CI_COMMIT_BRANCH =~ /_cnone/ || $ON_CORONA != "ON"'
-      when: never
-    # Don't run autotest update if...
-    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
-      when: never
-    # Report success on success status
-    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
-      when: on_success
-    # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
-      when: on_failure
-    # Always release resource
-    - if: '$CI_JOB_NAME =~ /release_resource/'
-      when: always
-    # Always cleanup
-    - if: '$CI_JOB_NAME =~ /cleanup/'
-      when: always
-    # Default is to run if previous stage succeeded
-    - when: on_success
-
-# Spack helped builds
-# Generic corona build job, extending build script
-.build_and_test_on_corona:
-  extends: [.on_corona]
-  stage: build_and_test
-  script:
-    # THREADS is used by 'tests/gitlab/build_and_test', run below
-    - export THREADS=12
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - echo ${MFEM_DATA_DIR}
-    - echo ${SPEC}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 15 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
@@ -1,49 +0,0 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# GitLab pipelines configurations for the Lassen machine at LLNL
-variables:
-  MACHINE_NAME: lassen
-
-.on_lassen:
-  tags:
-    - shell
-    - lassen
-  rules:
-    - if: '$CI_COMMIT_BRANCH =~ /_lnone/ || $ON_LASSEN == "OFF"' #run except if ...
-      when: never
-    # Don't run autotest update if...
-    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
-      when: never
-    # Report success on success status
-    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
-      when: on_success
-    # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
-      when: on_failure
-    # Always cleanup
-    - if: '$CI_JOB_NAME =~ /cleanup/'
-      when: always
-    - when: on_success
-
-# Lassen uses a different job scheduler (spectrum lsf) that does not allow
-# pre-allocation the same way slurm does.  We use pdebug queue on lassen
-# to speed-up the allocation.  However this would not be scalable to
-# multiple builds.
-.build_and_test_on_lassen:
-  extends: [.on_lassen]
-  stage: build_and_test
-  script:
-    - echo ${MFEM_DATA_DIR}
-    - echo ${SPEC}
-    # Next script uses 'THREADS': leaving it empty --> it uses 'make all -j'
-    - lalloc 1 -W 45 -q pdebug --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
-  needs: [setup]
@@ -1,55 +0,0 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# GitLab pipelines configurations for the Quartz machine at LLNL
-variables:
-  MACHINE_NAME: quartz
-
-.on_quartz:
-  tags:
-    - shell
-    - quartz
-  rules:
-    # Don't run quartz jobs if...
-    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"'
-      when: never
-    # Don't run autotest update if...
-    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
-      when: never
-    # Report success on success status
-    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
-      when: on_success
-    # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
-      when: on_failure
-    # Always release resource
-    - if: '$CI_JOB_NAME =~ /release_resource/'
-      when: always
-    # Always cleanup
-    - if: '$CI_JOB_NAME =~ /cleanup/'
-      when: always
-    # Default is to run if previous stage succeeded
-    - when: on_success
-
-# Spack helped builds
-# Generic quartz build job, extending build script
-.build_and_test_on_quartz:
-  extends: [.on_quartz]
-  stage: build_and_test
-  script:
-    # THREADS is used by 'tests/gitlab/build_and_test', run below
-    - export THREADS=12
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - echo ${MFEM_DATA_DIR}
-    - echo ${SPEC}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 45 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
@@ -1,77 +0,0 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Jobs report
-.report_job_success:
-  script:
-    - echo ${MACHINE_NAME}
-    - echo ${AUTOTEST}
-    - echo ${AUTOTEST_COMMIT}
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - cd ${AUTOTEST_ROOT}
-    - |
-      (
-        date
-        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
-        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -n 9; do
-          sleep 5
-        done
-        echo "Acquired lock on '$PWD/autotest.lock'"
-        date
-        # Report SUCCESS while holding the file lock on 'autotest.lock'.
-        # The next script uses the following environment variables:
-        # - MACHINE_NAME, AUTOTEST_ROOT, AUTOTEST_COMMIT
-        # - CI_COMMIT_REF_SLUG, CI_PROJECT_DIR, CI_PIPELINE_URL
-        # It also calls the script '.gitlab/scripts/safe_create_rundir'.
-        ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test_success
-        err=$?
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock
-
-.report_job_failure:
-  script:
-    - echo ${MACHINE_NAME}
-    - echo ${AUTOTEST}
-    - echo ${AUTOTEST_COMMIT}
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - cd ${AUTOTEST_ROOT}
-    - |
-      (
-        date
-        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
-        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -n 9; do
-          sleep 5
-        done
-        echo "Acquired lock on '$PWD/autotest.lock'"
-        date
-        # Report FAILURE while holding the file lock on 'autotest.lock'.
-        # The next script uses the following environment variables:
-        # - MACHINE_NAME, AUTOTEST_ROOT, AUTOTEST_COMMIT
-        # - CI_COMMIT_REF_SLUG, CI_PROJECT_DIR, CI_PIPELINE_URL
-        # It also calls the script '.gitlab/scripts/safe_create_rundir'.
-        ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test_failure
-        err=$?
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock
@@ -1,70 +0,0 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# The setup_baseline job doesn't rely on MFEM git repo. It prepares a
-# pipeline-wide working directory downloading/updating external repos.
-# TODO:
-# updating tests and tpls is not necessary anymore since pipelines are now
-# using unique directories so repo are never shared with another pipeline. This
-# is not memory efficient (we keep a lot of data), hence this reminder.
-setup_baseline:
-  tags:
-    - shell
-    - quartz
-  stage: setup
-  variables:
-    GIT_STRATEGY: none
-  script:
-    #
-    # Setup ${BUILD_ROOT}/tpls and ${BUILD_ROOT}/tests:
-    #
-    - echo "MACHINE_NAME = ${MACHINE_NAME}"
-    - echo "REBASELINE = ${REBASELINE}"
-    - echo "AUTOTEST = ${AUTOTEST}"
-    - echo "AUTOTEST_COMMIT = ${AUTOTEST_COMMIT}"
-    - echo "BUILD_ROOT ${BUILD_ROOT}"
-    - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
-    - if [ ! -d "tpls" ]; then git clone ${TPLS_REPO}; fi
-    - if [ ! -d "tests" ]; then git clone ${TESTS_REPO}; fi
-    - cd tpls && git pull && cd ..
-    - cd tests && git pull origin && cd ..
-    #
-    # Setup ${AUTOTEST_ROOT}/autotest:
-    #
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - mkdir -p ${AUTOTEST_ROOT} && cd ${AUTOTEST_ROOT}
-    - command -v flock || echo "Required command 'flock' not found"
-    - |
-      (
-        date
-        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
-        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -n 9; do
-          sleep 5
-        done
-        echo "Acquired lock on '$PWD/autotest.lock'"
-        date
-        # clone/update the autotest repo while holding the file lock on
-        # 'autotest.lock'
-        err=0
-        if [[ ! -d "autotest" ]]; then
-          git clone ${AUTOTEST_REPO}
-        else
-          cd autotest && git pull && cd ..
-        fi || err=1
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock
@@ -1,90 +0,0 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test
-# script then symlinks the repo to the parent directory of the MFEM source
-# directory. Unit tests that depend on the mfem/data repo will then detect that
-# this directory is present and be enabled.
-setup:
-  tags:
-    - shell
-    - quartz
-  stage: setup
-  variables:
-    GIT_STRATEGY: none
-  script:
-    #
-    # Setup MFEM_DATA_DIR=${SHARED_REPOS_DIR}/mfem-data, see '.gitlab-ci.yml'
-    # and '.gitlab/configs/<machine>-config.yml'
-    #
-    - echo "MACHINE_NAME = ${MACHINE_NAME}"
-    - echo "AUTOTEST = ${AUTOTEST}"
-    - echo "AUTOTEST_COMMIT = ${AUTOTEST_COMMIT}"
-    - echo "SHARED_REPOS_DIR ${SHARED_REPOS_DIR}"
-    - mkdir -p ${SHARED_REPOS_DIR} && cd ${SHARED_REPOS_DIR}
-    - command -v flock || echo "Required command 'flock' not found"
-    - |
-      (
-        date
-        echo "Waiting to acquire lock on '$PWD/mfem-data.lock' ..."
-        # try to get an exclusive lock on fd 9 (mfem-data.lock) repeating the
-        # try every 5 seconds; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -n 9; do
-          sleep 5
-        done
-        echo "Acquired lock on '$PWD/mfem-data.lock'"
-        date
-        # clone/update the mfem/data repo while holding the file lock on
-        # 'mfem-data.lock'
-        err=0
-        if [[ ! -d "mfem-data" ]]; then
-          git clone ${MFEM_DATA_REPO} "mfem-data"
-        else
-          cd "mfem-data" && git pull && cd ..
-        fi || err=1
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> mfem-data.lock
-    #
-    # Setup ${AUTOTEST_ROOT}/autotest:
-    #
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - mkdir -p ${AUTOTEST_ROOT} && cd ${AUTOTEST_ROOT}
-    - |
-      (
-        date
-        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
-        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -n 9; do
-          sleep 5
-        done
-        echo "Acquired lock on '$PWD/autotest.lock'"
-        date
-        # clone/update the autotest repo while holding the file lock on
-        # 'autotest.lock'
-        err=0
-        if [[ ! -d "autotest" ]]; then
-          git clone ${AUTOTEST_REPO}
-        else
-          cd autotest && git pull && cd ..
-        fi || err=1
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock
@@ -1,67 +0,0 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-stages:
-  - setup
-  - allocate_resource
-  - build_and_test
-  - release_resource_and_report
-
-# Slurm shared allocation
-allocate_resource:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_corona
-  stage: allocate_resource
-  script:
-    - echo ${ALLOC_NAME}
-    - salloc --exclusive --nodes=1 --partition=mi60 --time=45 --no-shell --job-name=${ALLOC_NAME}
-  timeout: 6h
-  needs: [setup]
-
-# Build and test jobs, simply provide a spec
-rocm_gcc_8.3.1:
-  variables:
-    SPEC: "@develop%gcc@8.3.1+rocm amdgpu_target=gfx906"
-  extends: .build_and_test_on_corona
-  needs: [allocate_resource]
-
-# Release slurm allocation
-release_resource:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_corona
-  stage: release_resource_and_report
-  script:
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
-  needs: [rocm_gcc_8.3.1]
-
-# Jobs report
-report_job_success:
-  stage: release_resource_and_report
-  extends:
-    - .on_corona
-    - .report_job_success
-
-report_job_failure:
-  stage: release_resource_and_report
-  extends:
-    - .on_corona
-    - .report_job_failure
-
-include:
-  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/corona-config.yml
-  - local: .gitlab/configs/setup-build-and-test.yml
-  - local: .gitlab/configs/report-build-and-test.yml
@@ -1,44 +0,0 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-stages:
-  - setup
-  - build_and_test
-  - report
-
-opt_mpi_cuda_xl_16_1_1_12:
-  variables:
-    SPEC: "%xl@16.1.1.12 +mpi +cuda cuda_arch=70"
-  extends: .build_and_test_on_lassen
-
-opt_mpi_cuda_hypre_cuda_xl:
-  variables:
-    SPEC: "%xl@16.1.1.12 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
-  extends: .build_and_test_on_lassen
-
-# Jobs report
-report_job_success:
-  stage: report
-  extends:
-    - .on_lassen
-    - .report_job_success
-
-report_job_failure:
-  stage: report
-  extends:
-    - .on_lassen
-    - .report_job_failure
-
-include:
-  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/lassen-config.yml
-  - local: .gitlab/configs/setup-build-and-test.yml
-  - local: .gitlab/configs/report-build-and-test.yml
@@ -0,0 +1,57 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# GitLab pipelines configurations for the Lassen machine at LLNL
+
+.on_lassen:
+  tags:
+    - shell
+    - lassen
+  variables:
+    PLAT: lassen
+
+# Build MFEM
+build_mfem_ser_lassen:
+  extends: [.with_gcc_8_3_1, .on_lassen]
+  needs: [setup]
+  stage: lassen_build
+  script:
+    - mkdir -p ${BUILD_PATH}
+    - cp -r ${CI_PROJECT_DIR} ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser
+    - lalloc 1 -W 5 -q pdebug make -j cuda CUDA_ARCH=sm_70
+
+build_mfem_debug_ser_lassen:
+  extends: [.with_gcc_8_3_1, .on_lassen]
+  needs: [setup]
+  stage: lassen_build
+  script:
+    - mkdir -p ${BUILD_PATH}
+    - cp -r ${CI_PROJECT_DIR} ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
+    - lalloc 1 -W 5 -q pdebug make -j cuda MFEM_DEBUG="YES" CPPFLAGS=-O2 CUDA_ARCH=sm_70
+
+# Sanity check
+sanitycheck_mfem_ser_lassen:
+  extends: [.with_gcc_8_3_1, .on_lassen]
+  stage: lassen_test
+  needs: [build_mfem_ser_lassen]
+  script:
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser
+    - lalloc 1 -W 15 -q pdebug make -j test
+
+sanitycheck_mfem_debug_ser_lassen:
+  extends: [.with_gcc_8_3_1, .on_lassen]
+  stage: lassen_test
+  needs: [build_mfem_debug_ser_lassen]
+  script:
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
+    - lalloc 1 -W 30 -q pdebug make -j test
@@ -1,135 +0,0 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-variables:
-  BASELINE_TEST: baseline
-
-stages:
-  - setup
-  - baseline_check
-  - baseline_report
-  - cleanup
-  - baseline_publish
-
-baselinecheck_mfem_intel_quartz:
-  extends: [.on_quartz]
-  stage: baseline_check
-  variables:
-    # TPLS_DIR is used in .gitlab/scripts/baseline to provide the tpls location
-    # when call the runtest script in MFEM test repo.
-    # Note: the value must be consistent with the setup performed in
-    #       .gitlab/configs/setup-baseline.yml.
-    TPLS_DIR: ${BUILD_ROOT}/tpls
-  script:
-    - echo ${BUILD_ROOT}
-    - echo ${TPLS_DIR}
-    # Used by the tests in MFEM/tests:
-    - export MFEM_TEST_NP=32
-    # The next script uses the following environment variables:
-    # * BASELINE_TEST, SYS_TYPE, CI_PROJECT_DIR, ARTIFACTS_DIR,
-    # * BUILD_ROOT, TPLS_DIR, MACHINE_NAME
-    - .gitlab/scripts/baseline
-  artifacts:
-    when: always
-    paths:
-      - ${ARTIFACTS_DIR}
-  allow_failure: true
-
-cleanup:
-  extends: .on_quartz
-  stage: cleanup
-  variables:
-    GIT_STRATEGY: none
-  script:
-    - echo "BUILD_ROOT=${BUILD_ROOT}"
-    - rm -rf "${BUILD_ROOT}" || true
-    - echo "CI_PROJECT_DIR=${CI_PROJECT_DIR}"
-    - make -C "${CI_PROJECT_DIR}" distclean
-
-report_baseline:
-  extends: [.on_quartz]
-  stage: baseline_report
-  script:
-    - echo ${MACHINE_NAME}
-    - echo ${AUTOTEST}
-    - echo ${AUTOTEST_COMMIT}
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - cd ${AUTOTEST_ROOT}
-    - |
-      (
-        date
-        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
-        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -n 9; do
-          sleep 5
-        done
-        echo "Acquired lock on '$PWD/autotest.lock'"
-        date
-        # ----------------------
-        cd ${AUTOTEST_ROOT}/autotest || \
-          { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
-        mkdir -p ${MACHINE_NAME}
-        rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-${BASELINE_TEST}-${CI_COMMIT_REF_SLUG}"
-        rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir ${rundir})
-        cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
-        printf "%s\n" "" "Pipeline URL:" "$CI_PIPELINE_URL" \
-          >> ${rundir}/pipeline.txt
-        # We create an autotest-email.html file, because that's how we signal
-        # that there was an error / diff (temporary).
-        if [[ -f ${rundir}/${BASELINE_TEST}.err ]] || \
-           [[ -f ${rundir}/${BASELINE_TEST}-${SYS_TYPE}.diff ]]; then
-          cp ${rundir}/pipeline.txt ${rundir}/autotest-email.html
-        fi
-        msg="GitLab CI log for ${BASELINE_TEST} on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
-        if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
-          git pull && \
-          git add ${rundir} && \
-          git commit -m "${msg}" && \
-          ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
-        else
-          for file in ${rundir}/*; do
-            echo "------------------------------"
-            echo "Content of '$file'"
-            echo "******************************"
-            cat $file
-            echo "******************************"
-          done
-          rm -rf ${rundir} || true
-        fi
-        err=$?
-        # ----------------------
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock
-
-baselinepublish_mfem_quartz:
-  extends: [.on_quartz]
-  stage: baseline_publish
-  rules:
-    # - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "YES"'
-    - if: '$REBASELINE == "YES"'
-      when: manual
-  script:
-    - echo ${BUILD_ROOT}
-    - echo ${PWD}
-    - echo ${ARTIFACTS_DIR}
-    - ls -lA ${ARTIFACTS_DIR}
-    - .gitlab/scripts/rebaseline
-
-include:
-  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/quartz-config.yml
-  - local: .gitlab/configs/setup-baseline.yml
@@ -1,94 +0,0 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-stages:
-  - setup
-  - allocate_resource
-  - build_and_test
-  - release_resource_and_report
-
-# Allocate
-allocate_resource:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_quartz
-  stage: allocate_resource
-  script:
-    - echo ${ALLOC_NAME}
-    - salloc --exclusive --nodes=1 --partition=pdebug --time=45 --no-shell --job-name=${ALLOC_NAME}
-  timeout: 6h
-
-# GitLab jobs for the Quartz machine at LLNL
-debug_ser_gcc_10:
-  variables:
-    SPEC: "%gcc@10.3.1 +debug~mpi"
-  extends: .build_and_test_on_quartz
-
-debug_par_gcc_10:
-  variables:
-    SPEC: "%gcc@10.3.1 +debug+mpi"
-  extends: .build_and_test_on_quartz
-
-opt_ser_gcc_10:
-  variables:
-    SPEC: "%gcc@10.3.1 ~mpi"
-  extends: .build_and_test_on_quartz
-
-opt_par_gcc_10:
-  variables:
-    SPEC: "%gcc@10.3.1"
-  extends: .build_and_test_on_quartz
-
-opt_par_gcc_10_sundials:
-  variables:
-    SPEC: "%gcc@10.3.1 +sundials"
-  extends: .build_and_test_on_quartz
-
-opt_par_gcc_10_petsc:
-  variables:
-    SPEC: "%gcc@10.3.1 +petsc ^petsc+mumps~superlu-dist"
-  extends: .build_and_test_on_quartz
-
-opt_par_gcc_10_pumi:
-  variables:
-    SPEC: "%gcc@10.3.1 +pumi"
-  extends: .build_and_test_on_quartz
-
-# Release
-release_resource:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_quartz
-  stage: release_resource_and_report
-  script:
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
-
-# Jobs report
-report_job_success:
-  stage: release_resource_and_report
-  extends:
-    - .on_quartz
-    - .report_job_success
-
-report_job_failure:
-  stage: release_resource_and_report
-  extends:
-    - .on_quartz
-    - .report_job_failure
-
-include:
-  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/quartz-config.yml
-  - local: .gitlab/configs/setup-build-and-test.yml
-  - local: .gitlab/configs/report-build-and-test.yml
@@ -0,0 +1,145 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# GitLab pipelines configurations for the Quartz machine at LLNL
+
+.on_quartz:
+  tags:
+    - shell
+    - quartz
+  variables:
+    PLAT: quartz
+    MPIEXEC: srun
+    MPIEXEC_NP: "-n"
+    MAKE_PAR: 6
+    BASELINE_PAR: 18
+
+# Allocate
+allocate_quartz:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_quartz
+  stage: allocate
+  script:
+    - |
+      if [[ "${MPIEXEC}" == "srun" ]]
+      then
+      salloc --nodes=1 --extra-node-info=2:18:1 --partition=pdebug --time=30 --no-shell --job-name=${MFEM_ALLOC_NAME}
+      fi
+  timeout: 6h
+
+# Release
+deallocate_quartz:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_quartz
+  stage: deallocate
+  script:
+    - |
+      if [[ "${MPIEXEC}" == "srun" ]]
+      then
+      export JOB_ID=$(squeue -h --name=${MFEM_ALLOC_NAME} --format=%A)
+      ([[ -n "${JOB_ID}" ]] && scancel ${JOB_ID})
+      fi
+  when: always
+
+# Build external libraries (tpls)
+build_hypre_gcc_6_1_0_quartz:
+  extends: [.build_hypre, .with_gcc_6_1_0, .on_quartz]
+
+build_parmetis_gcc_6_1_0_quartz:
+  extends: [.build_metis, .with_gcc_6_1_0, .on_quartz]
+  variables:
+    METIS: parmetis
+
+build_metis4_gcc_6_1_0_quartz:
+  extends: [.build_metis, .with_gcc_6_1_0, .on_quartz]
+  variables:
+    METIS: metis4
+
+build_metis5_gcc_6_1_0_quartz:
+  extends: [.build_metis, .with_gcc_6_1_0, .on_quartz]
+  variables:
+    METIS: metis5
+  allow_failure: true
+
+# Build MFEM
+build_mfem_debug_ser_gcc_4_9_3_quartz:
+  extends: [.build_mfem, .with_gcc_4_9_3, .on_quartz]
+  variables:
+    MFEM_DEBUG: "YES"
+    MFEM_USE_MPI: "NO"
+
+build_mfem_debug_ser_gcc_6_1_0_quartz:
+  extends: [.build_mfem, .with_gcc_6_1_0, .on_quartz]
+  variables:
+    MFEM_DEBUG: "YES"
+    MFEM_USE_MPI: "NO"
+
+build_mfem_debug_par_gcc_6_1_0_quartz:
+  extends: [.build_mfem, .with_gcc_6_1_0, .on_quartz]
+  variables:
+    MFEM_DEBUG: "YES"
+    MFEM_USE_MPI: "YES"
+  needs: [build_hypre_gcc_6_1_0_quartz, build_parmetis_gcc_6_1_0_quartz]
+
+build_mfem_opt_ser_gcc_6_1_0_quartz:
+  extends: [.build_mfem, .with_gcc_6_1_0, .on_quartz]
+  variables:
+    MFEM_DEBUG: "NO"
+    MFEM_USE_MPI: "NO"
+
+build_mfem_opt_par_gcc_6_1_0_quartz:
+  extends: [.build_mfem, .with_gcc_6_1_0, .on_quartz]
+  variables:
+    MFEM_DEBUG: "NO"
+    MFEM_USE_MPI: "YES"
+  needs: [build_hypre_gcc_6_1_0_quartz, build_parmetis_gcc_6_1_0_quartz]
+
+### Tests
+
+# Sanity check
+sanitycheck_mfem_debug_ser_gcc_4_9_3_quartz:
+  extends: [.sanitycheck_mfem, .with_gcc_4_9_3, .on_quartz]
+  variables:
+    MFEM_DEBUG: "YES"
+    MFEM_USE_MPI: "NO"
+  needs: [build_mfem_debug_ser_gcc_4_9_3_quartz, allocate_quartz]
+
+sanitycheck_mfem_debug_ser_gcc_6_1_0_quartz:
+  extends: [.sanitycheck_mfem, .with_gcc_6_1_0, .on_quartz]
+  variables:
+    MFEM_DEBUG: "YES"
+    MFEM_USE_MPI: "NO"
+  needs: [build_mfem_debug_ser_gcc_6_1_0_quartz, allocate_quartz]
+
+sanitycheck_opt_ser_mfem_gcc_6_1_0_quartz:
+  extends: [.sanitycheck_mfem, .with_gcc_6_1_0, .on_quartz]
+  variables:
+    MFEM_DEBUG: "NO"
+    MFEM_USE_MPI: "NO"
+  needs: [build_mfem_opt_ser_gcc_6_1_0_quartz, allocate_quartz]
+
+sanitycheck_opt_par_mfem_gcc_6_1_0_quartz:
+  extends: [.sanitycheck_mfem, .with_gcc_6_1_0, .on_quartz]
+  variables:
+    MFEM_DEBUG: "NO"
+    MFEM_USE_MPI: "YES"
+  needs: [build_mfem_opt_par_gcc_6_1_0_quartz, allocate_quartz]
+
+# Baseline
+baselinecheck_mfem_intel_quartz:
+  extends: [.baselinecheck_mfem, .on_quartz]
+  needs: [allocate_quartz]
+
+baselinepublish_mfem_quartz:
+  extends: [.rebaseline_mfem, .on_quartz]
+  needs: [baselinecheck_mfem_intel_quartz]
@@ -1,75 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# locals
-glob_err=${BASELINE_TEST}.err
-base=${BASELINE_TEST}-${SYS_TYPE}
-base_diff=${base}.diff
-base_patch=${base}.patch
-base_out=${base}.out
-artifacts_path=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}
-
-# prepare
-cd ${BUILD_ROOT} || \
-  { echo "Invalid BUILD_ROOT=$BUILD_ROOT"; exit 1; }
-ln -snf ${CI_PROJECT_DIR} mfem
-cd tests
-[[ -d _${BASELINE_TEST} ]] && rm -rf _${BASELINE_TEST}
-mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
-
-# run
-if [[ "${MACHINE_NAME}" == "quartz" || "${MACHINE_NAME}" == "ruby" ]]; then
-  salloc --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
-elif [[ ${MACHINE_NAME} == "corona" ]]; then
-  salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
-elif [[ ${MACHINE_NAME} == "lassen" ]]; then
-  lalloc 1 -q pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
-else
-  echo "Unknown machine: MACHINE_NAME=$MACHINE_NAME"
-  exit 1
-fi
-
-# post
-mkdir ${artifacts_path}
-
-status=0
-if [[ -f ${BASELINE_TEST}.out ]]; then
-  cp ${BASELINE_TEST}.out ${artifacts_path}
-fi
-if [[ -s ${glob_err} ]]; then
-  echo "ERROR during ${BASELINE_TEST} execution"
-  echo "Here is the ${glob_err} file content"
-  cat ${glob_err}
-  cp ${glob_err} ${artifacts_path}/${glob_err}
-  status=1
-fi
-if [[ -f ${base_patch} ]]; then
-  echo "${BASELINE_TEST}: Differences found, patch generated"
-  cp ${base_patch} ${artifacts_path}/${base_patch}
-elif [[ -f ${base_out} ]]; then
-  echo "${BASELINE_TEST}: Differences found, replacement file generated"
-  cp ${base_out} ${artifacts_path}/${base_out}
-fi
-# base_diff won't even exist if there is no difference.
-if [[ -f ${base_diff} ]]; then
-  echo "${BASELINE_TEST}: Relevant differences (filtered diff) ..."
-  cat ${base_diff}
-  cp ${base_diff} ${artifacts_path}/${base_diff}
-  status=1
-fi
-if [[ $status -eq 0 ]]; then
-  echo "${BASELINE_TEST}: PASSED"
-else
-  echo "${BASELINE_TEST}: FAILED"
-fi
-exit $status
@@ -1,41 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Try to push to the remote 5 times. If the push fails, and the local and remote
-# have diverged, then pull from the remote to merge changes, and try pushing
-# again. If some other failure happens
-for i in {1..5}; do
-   git push origin master && exit 0
-   # Wait for 20 seconds in case someone else is pushing to the remote
-   # concurrently
-   sleep 20
-   # Fetch any updates from the remote
-   git remote update
-   # Get the latest commit on the local branch
-   LOCAL=$(git rev-parse @)
-   # Get the latest commit on the remote
-   REMOTE=$(git rev-parse @{u})
-   # Get the common ancestor
-   BASE=$(git merge-base @ @{u})
-   # Have the local and remote diverged?
-   if [[ $LOCAL != $REMOTE && $LOCAL != $BASE && $REMOTE != $BASE ]]; then
-      git pull
-      if [[ $? == 0 ]]; then
-         continue
-      else
-         exit 1 # Something else went wrong trying to pull
-      fi
-   fi
-done
-
-exit 1 # Did not succeed in 5 attempts
@@ -1,49 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# There will be collision between corona and quartz baselines.
-# Once the corresponding files have been generated, we can switch to machine
-# specific ref.
-ARTIFACT_PATH=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}
-#ARTIFACT_PATH=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}-${MACHINE_NAME}
-
-PATCH_FILE=${ARTIFACT_PATH}.patch
-FULL_FILE=${ARTIFACT_PATH}.out
-DIFF_FILE=${ARTIFACT_PATH}.diff
-
-# There will be collision between corona and quartz baselines.
-# Once the corresponding files have been generated, we can switch to machine
-# specific ref.
-SAVED_NAME=baseline-${SYS_TYPE}.saved
-#SAVED_NAME=baseline-${SYS_TYPE}-${MACHINE_NAME}.saved
-
-cd ${BUILD_ROOT}/tests
-
-if [[ ! -f "${DIFF_FILE}" ]]
-then
-    echo "Nothing to be done: no relevant change in baseline"
-    exit 0
-elif [[ -f "${PATCH_FILE}" ]]
-then
-    patch "${SAVED_NAME}" < "${PATCH_FILE}"
-elif [[ -f "${FULL_FILE}" ]]
-then
-    cp "${FULL_FILE}" "${SAVED_NAME}"
-else
-    echo "File missing: expected ${PATCH_FILE} or ${FULL_FILE}"
-    exit 1
-fi
-
-git add "${SAVED_NAME}"
-git commit -m "${SYS_TYPE} (${MACHINE_NAME}) rebaselined in GitLab pipeline ${CI_PIPELINE_ID}"
-git push origin master
@@ -1,45 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-echo "Runs if there was at least one failure on ${MACHINE_NAME}"
-
-cd ${AUTOTEST_ROOT}/autotest || \
-  { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
-mkdir -p ${MACHINE_NAME}
-
-rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
-rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir)
-
-printf "%s\n" "Some 'build-and-test' jobs on ${MACHINE_NAME} FAILED." \
-  "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.err
-
-msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
-
-# Create 'autotest-email.html' to indicate failure:
-cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
-
-if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
-  git pull && \
-  git add ${rundir} && \
-  git commit -m "${msg}" && \
-  ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
-else
-  for file in ${rundir}/*; do
-    echo "------------------------------"
-    echo "Content of '$file'"
-    echo "******************************"
-    cat $file
-    echo "******************************"
-  done
-  rm -rf ${rundir} || true
-fi
@@ -1,42 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-echo "Can only run if all the ${MACHINE_NAME} jobs passed"
-
-cd ${AUTOTEST_ROOT}/autotest || \
-  { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
-mkdir -p ${MACHINE_NAME}
-
-rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
-rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir)
-
-printf "%s\n" "The 'build-and-test' jobs on ${MACHINE_NAME} were SUCCESSFUL." \
-  "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.out
-
-msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
-
-if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
-  git pull && \
-  git add ${rundir} && \
-  git commit -m "${msg}" && \
-  ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
-else
-  for file in ${rundir}/*; do
-    echo "------------------------------"
-    echo "Content of '$file'"
-    echo "******************************"
-    cat $file
-    echo "******************************"
-  done
-  rm -rf ${rundir} || true
-fi
@@ -1,42 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# This script takes a seed for a directory name and appends it with a counter
-# incremented until it can create a new directory with it.
-
-# Usage:
-#
-# Expects 1 argument: a string that is use as a seed for the directory name.
-#
-# > rundir="desired_name"
-# > rundir=$(./safe_create_rundir $rundir)
-
-set -o errexit
-set -o nounset
-
-rundir=${1:-""}
-if [[ -z ${rundir} ]]; then
-  >&2 echo "The script expects a string as argument for directory creation."
-  exit 1
-fi
-
-if ! mkdir ${rundir}; then
-  n=1
-  while ! mkdir ${rundir}_${n}
-  do
-    n=$((n+1))
-  done
-  rundir=${rundir}_${n}
-fi
-
-echo $rundir
@@ -0,0 +1,469 @@
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+language: cpp
+
+os: linux
+dist: bionic
+
+stages:
+  - checks
+  - tests
+  - optional
+
+env:
+  global:
+    - HYPRE_ARCHIVE=v2.19.0.tar.gz
+      HYPRE_URL=https://github.com/hypre-space/hypre/archive/$HYPRE_ARCHIVE
+      HYPRE_TOP_DIR=hypre-2.19.0
+
+jobs:
+  include:
+
+    # ========================
+    #         Checks
+    # ========================
+    # - code-style
+    # - documentation
+    # - gitignore
+
+    - stage: checks
+      os: linux
+      dist: xenial
+      name: "code-style"
+      addons:
+        apt:
+          packages:
+            - astyle=2.05.1-0ubuntu1
+      script:
+        - cd ${TRAVIS_BUILD_DIR}
+        - cd tests/scripts
+        - ./runtest code-style
+
+    - stage: checks
+      os: linux
+      name: "documentation"
+      addons:
+        apt:
+          packages:
+            - doxygen
+            - graphviz
+      script:
+        - cd ${TRAVIS_BUILD_DIR}
+        - cd tests/scripts
+        - ./runtest documentation
+
+    - stage: checks
+      os: linux
+      name: "gitignore"
+      addons:
+        apt:
+          packages:
+            - mpich
+            - libmpich-dev
+      env: MPI=YES
+      before_script:
+        - cd ${TRAVIS_BUILD_DIR}
+        - mpicxx -v
+        - make config MFEM_USE_MPI=YES MFEM_MPI_NP=2
+        - make all -j3
+        - make test-noclean
+      script:
+        - cd tests/scripts
+        - ./runtest gitignore
+      cache:
+        ccache: true
+        directories:
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
+          rm -f Lib/*.{c,o}
+
+    # ========================
+    #  Optional Checks/Tests
+    # ========================
+    # - branch-history
+
+    - stage: optional
+      name: "branch-history"
+      if: branch != next
+      # need full git history for the binary/big files check
+      git:
+        depth: false
+      script:
+        - cd ${TRAVIS_BUILD_DIR}
+        # update master
+        - git fetch origin master:master
+        # checkout a branch (otherwise Travis works in detached head)
+        - git checkout -b travis_tests
+        - cd tests/scripts
+        - ./runtest branch-history
+
+    # ========================
+    #       Linux tests
+    # ========================
+    # - serial + debug
+    # - serial
+    # - parallel + debug
+    # - parallel
+
+    - stage: tests
+      os: linux
+      compiler: gcc
+      name: "Linux: Serial + Debug"
+      env: DEBUG=YES
+           MPI=NO
+           CODECOV=NO
+           MFEM_TEST_TARGET=check
+      cache:
+        ccache: true
+
+    - os: linux
+      compiler: gcc
+      name: "Linux: Serial"
+      env: DEBUG=NO
+           MPI=NO
+           CODECOV=NO
+           MFEM_TEST_TARGET=test
+      cache:
+        ccache: true
+
+    - os: linux
+      compiler: gcc
+      name: "Linux: Parallel + Debug"
+      addons:
+        apt:
+          # sources:
+            # - ubuntu-toolchain-r-test
+          packages:
+            # GCC 4.9
+            # - g++-4.9
+            # MPICH
+            - mpich
+            - libmpich-dev
+            # OpenMPI
+            # - openmpi-bin
+            # - libopenmpi-dev
+      env: DEBUG=YES
+           MPI=YES
+           CODECOV=NO
+           MFEM_TEST_TARGET=check
+           NPROCS=2
+      cache:
+        ccache: true
+        directories:
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
+          rm -f Lib/*.{c,o}
+
+    - os: linux
+      compiler: gcc
+      name: "Linux: Parallel"
+      addons:
+        apt:
+          # sources:
+            # - ubuntu-toolchain-r-test
+          packages:
+            # GCC 4.9
+            # - g++-4.9
+            # MPICH
+            - mpich
+            - libmpich-dev
+            # OpenMPI
+            # - openmpi-bin
+            # - libopenmpi-dev
+      env: DEBUG=NO
+           MPI=YES
+           CODECOV=YES
+           MFEM_TEST_TARGET=test
+           NPROCS=2
+      cache:
+        ccache: true
+        directories:
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
+          rm -f Lib/*.{c,o}
+
+    - os: linux
+      compiler: gcc
+      name: "Linux: Parallel (cmake)"
+      addons:
+        apt:
+          packages:
+            - mpich
+            - libmpich-dev
+      env: MPI=YES
+           NPROCS=2
+      script:
+        - cd ${TRAVIS_BUILD_DIR}
+        - mkdir ${TRAVIS_BUILD_DIR}/build
+        - cd ${TRAVIS_BUILD_DIR}/build
+        - cmake ..
+             -DMFEM_USE_MPI=ON
+             -DHYPRE_DIR=${TRAVIS_BUILD_DIR}/../$HYPRE_TOP_DIR/src/hypre
+             -DMFEM_MPI_NP=$NPROCS
+        - make -j3 mfem examples
+        - cd ${TRAVIS_BUILD_DIR}/build/tests/unit
+        - make -j3
+        - ctest --output-on-failure
+      cache:
+        ccache: true
+        directories:
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
+          rm -f Lib/*.{c,o}
+
+    # ========================
+    #      Mac OS X tests
+    # ========================
+    # - serial + debug
+    # - serial
+    # - parallel + debug
+    # - parallel
+
+    - os: osx
+      osx_image: xcode11.2
+      compiler: clang
+      name: "Mac: Serial + Debug"
+      addons:
+        homebrew:
+          packages:
+            - ccache
+      env: DEBUG=YES
+           MPI=NO
+           CODECOV=NO
+           MFEM_TEST_TARGET=check
+      cache:
+        ccache: true
+
+    - os: osx
+      osx_image: xcode11.2
+      compiler: clang
+      name: "Mac: Serial"
+      addons:
+        homebrew:
+          packages:
+            - ccache
+      env: DEBUG=NO
+           MPI=NO
+           CODECOV=NO
+           MFEM_TEST_TARGET=test
+      cache:
+        ccache: true
+
+    - os: osx
+      osx_image: xcode11.2
+      compiler: clang
+      name: "Mac: Parallel + Debug"
+      addons:
+        homebrew:
+          packages:
+            - ccache
+      env: DEBUG=YES
+           MPI=YES
+           CODECOV=NO
+           MFEM_TEST_TARGET=check
+           NPROCS=4
+           TMPDIR=/tmp
+      cache:
+        ccache: true
+        directories:
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+          - $HOME/local-cached
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
+          rm -f Lib/*.{c,o}
+
+    - os: osx
+      osx_image: xcode11.2
+      compiler: clang
+      name: "Mac: Parallel"
+      addons:
+        homebrew:
+          packages:
+            - ccache
+      env: DEBUG=NO
+           MPI=YES
+           CODECOV=YES
+           MFEM_TEST_TARGET=test
+           NPROCS=4
+           TMPDIR=/tmp
+      cache:
+        ccache: true
+        directories:
+          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../metis-4.0
+          - $HOME/local-cached
+      before_cache:
+        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
+          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
+          rm -f Lib/*.{c,o}
+
+before_install:
+   # No addon for brew yet, have to install OSX packages this way.
+   # - if [ $TRAVIS_OS_NAME == "osx" ] && [ $MPI == "YES" ]; then
+   #      brew install open-mpi;
+   #   fi
+
+   # Disable ccache while building dependencies that are cached:
+   - echo "before \$PATH = $PATH";
+     export PATH=${PATH//\/usr\/lib\/ccache:/};
+     echo "after  \$PATH = $PATH"
+
+   # On Mac OS X, build and cache OpenMPI 2.1.6:
+   - if [ $TRAVIS_OS_NAME == "osx" ] && [ $MPI == "YES" ]; then
+        if [ ! -e $HOME/local-cached/bin/mpicc ]; then
+           mkdir -p $HOME/builds && cd $HOME/builds &&
+           wget https://download.open-mpi.org/release/open-mpi/v2.1/openmpi-2.1.6.tar.bz2 &&
+           tar jxf openmpi-2.1.6.tar.bz2 &&
+           mkdir openmpi-build && cd openmpi-build &&
+           ../openmpi-2.1.6/configure --prefix=$HOME/local-cached &&
+           make -j3 all && make install;
+        fi;
+        PATH=$HOME/local-cached/bin:$PATH;
+        cd $TRAVIS_BUILD_DIR;
+     fi
+
+   # Update environment to find g++ 4.9 installation first.
+   # - if [ $TRAVIS_OS_NAME == "linux" ]; then
+   #      mkdir -p latest-gcc-symlinks;
+   #      ln -s /usr/bin/g++-4.9 latest-gcc-symlinks/g++;
+   #      ln -s /usr/bin/gcc-4.9 latest-gcc-symlinks/gcc;
+   #      ln -s /usr/bin/gcov-4.9 latest-gcc-symlinks/gcov;
+   #      export PATH=$PWD/latest-gcc-symlinks:$PATH;
+   #   fi
+
+   # Install tool to upload code coverage reports to coveralls.io
+   - if [ "$CODECOV" == "YES" ]; then
+        export PYTHONUSERBASE=$HOME/local;
+        pip install --user cpp-coveralls;
+        pip install --user pyyaml;
+        PATH=$HOME/local/bin:$PATH;
+     fi
+
+install:
+   # Set MPI compilers, print compiler version
+   - if [ $MPI == "YES" ]; then
+        if [ "$TRAVIS_OS_NAME" == "linux" ]; then
+           export MPICH_CC="$CC";
+           export MPICH_CXX="$CXX";
+        else
+           export OMPI_CC="$CC";
+           export OMPI_CXX="$CXX";
+           mpic++ --showme:version;
+        fi;
+        mpic++ -v;
+     else
+        $CXX -v;
+     fi
+
+   # Back out of the mfem directory to install the libraries
+   - cd ..
+
+   # hypre
+   - if [ $MPI == "YES" ]; then
+        if [ ! -e $HYPRE_TOP_DIR/src/hypre/lib/libHYPRE.a ]; then
+           wget $HYPRE_URL;
+           rm -rf $HYPRE_TOP_DIR;
+           tar xvzf $HYPRE_ARCHIVE;
+           cd $HYPRE_TOP_DIR/src;
+           ./configure --disable-fortran CC=mpicc CXX=mpic++;
+           make -j3;
+           cd ../..;
+        else
+            echo "Reusing cached $HYPRE_TOP_DIR/";
+        fi;
+        ln -s $HYPRE_TOP_DIR hypre;
+     else
+        echo "Serial build, not using hypre";
+     fi
+
+   # METIS, use a mirror because the original source server is not always up.
+   # Original url:
+   #    http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/OLD/metis-4.0.3.tar.gz
+   - if [ $MPI == "YES" ]; then
+        if [ ! -e metis-4.0/libmetis.a ]; then
+           wget https://mfem.github.io/tpls/metis-4.0.3.tar.gz;
+           tar xvzf metis-4.0.3.tar.gz;
+           make -j3 -C metis-4.0.3/Lib CC="$CC" OPTFLAGS="-O2";
+           rm -rf metis-4.0;
+           mv metis-4.0.3 metis-4.0;
+        else
+           echo "Reusing cached metis-4.0/";
+        fi;
+     fi
+
+   # Re-enable ccache on linux; enable ccache on mac os:
+   - if [ $TRAVIS_OS_NAME == "linux" ]; then
+        export PATH="/usr/lib/ccache:$PATH";
+     else
+        if [ $TRAVIS_OS_NAME == "osx" ]; then
+           export PATH="/usr/local/opt/ccache/libexec:$PATH";
+        fi;
+     fi
+
+   - printf "which \$CC  = "; which $CC;
+     printf "which \$CXX = "; which $CXX
+
+script:
+   # Compiler
+   - if [ $MPI == "YES" ]; then
+        export MYCXX=mpic++;
+        export MAKE_CXX_FLAG=MPICXX=$MYCXX;
+     else
+        export MYCXX="$CXX";
+        export MAKE_CXX_FLAG=CXX=$MYCXX;
+     fi
+
+   # Print the compiler version
+   - $MYCXX -v
+
+   # Set some variables
+   - cd $TRAVIS_BUILD_DIR;
+     CPPFLAGS="";
+     SKIP_TEST_DIRS="";
+     if [ "$CODECOV" == "YES" ]; then
+        CPPFLAGS="--coverage -g";
+     fi;
+     if [ "$TRAVIS_OS_NAME" != "linux" ] || [ "$DEBUG" == "YES" ]; then
+        CPPFLAGS+=" -pedantic -Wall -Werror";
+     fi
+
+   # Configure the library
+   - make config MFEM_USE_MPI=$MPI MFEM_DEBUG=$DEBUG $MAKE_CXX_FLAG
+        MFEM_MPI_NP=$NPROCS CPPFLAGS="$CPPFLAGS"
+   # Show the configuration
+   - make info
+   # Build the library
+   - make -j3
+   # Build the examples and the miniapps
+   - make -j3 all
+   # Run tests
+   - make $MFEM_TEST_TARGET SKIP_TEST_DIRS="$SKIP_TEST_DIRS"
+
+after_success:
+   - if [ "$CODECOV" == "YES" ]; then
+        coveralls --include fem --include general --include linalg --include
+           mesh --exclude /usr --gcov-options '\-lp' --root $TRAVIS_BUILD_DIR;
+     fi
@@ -1,66 +0,0 @@
-cff-version: 1.2.0
-message: "If you use MFEM, please cite it as follows."
-authors:
- family-names: "MFEM Team"
-title: "MFEM: Modular Finite Element Methods [Software]"
-doi: 10.11578/dc.20171025.1248
-url: "https://mfem.org"
-preferred-citation:
-  type: article
-  authors:
-  - family-names: "Anderson"
-    given-names: "Robert"
-    orcid: "https://orcid.org/0000-0002-3508-9944"
-  - family-names: "Andrej"
-    given-names: "Julian"
-    orcid: "https://orcid.org/0000-0001-7661-4840"
-  - family-names: "Barker"
-    given-names: "Andrew"
-    orcid: "https://orcid.org/0000-0003-3572-911X"
-  - family-names: "Bramwell"
-    given-names: "Jamie"
-  - family-names: "Camier"
-    given-names: "Jean-Sylvain"
-    orcid: "https://orcid.org/0000-0003-2421-1999"
-  - family-names: "Cerveny"
-    given-names: "Jakub"
-    orcid: "https://orcid.org/0000-0003-4231-2531"
-  - family-names: "Dobrev"
-    given-names: "Veselin"
-    orcid: "https://orcid.org/0000-0003-1793-5622"
-  - family-names: "Dudouit"
-    given-names: "Yohann"
-    orcid: "https://orcid.org/0000-0001-5831-561X"
-  - family-names: "Fisher"
-    given-names: "Aaron"
-  - family-names: "Kolev"
-    given-names: "Tzanio"
-    orcid: "https://orcid.org/0000-0002-2810-3090"
-  - family-names: "Pazner"
-    given-names: "Will"
-    orcid: "https://orcid.org/0000-0003-4885-2934"
-  - family-names: "Stowell"
-    given-names: "Mark"
-    orcid: "https://orcid.org/0000-0002-5389-7435"
-  - family-names: "Tomov"
-    given-names: "Vladimir"
-    orcid: "https://orcid.org/0000-0002-1846-6816"
-  - family-names: "Akkerman"
-    given-names: "Ido"
-    orcid: "https://orcid.org/0000-0002-5937-0300"
-  - family-names: "Dahm"
-    given-names: "Johann"
-    orcid: "https://orcid.org/0000-0001-9657-3564"
-  - family-names: "Medina"
-    given-names: "David"
-  - family-names: "Zampini"
-    given-names: "Stefano"
-    orcid: "https://orcid.org/0000-0002-0435-0433"
-  doi: "10.1016/j.camwa.2020.06.009"
-  journal: "Computers \\& Mathematics with Applications"
-  month: 1
-  start: 42 # First page number
-  end: 74 # Last page number
-  title: "MFEM: A Modular Finite Element Methods Library"
-  volume: 81
-  year: 2021
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -9,11 +9,7 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

-# The variable CMAKE_CXX_STANDARD and related were introduced in CMake v3.1
-# Version 3.8 fixes the handling of CMAKE_CXX_STANDARD for try_compile.
-# Version 3.8 or newer is required for direct CUDA support.
-cmake_minimum_required(VERSION 3.8)
-message(STATUS "CMake version: ${CMAKE_VERSION}")
+cmake_minimum_required(VERSION 2.8.11)
 set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH
  "Path to optional user configuration file.")

@@ -25,11 +21,8 @@ set(CMAKE_CXX_EXTENSIONS OFF)
 # Load user settings before the defaults - this way the defaults will not
 # overwrite the user set options. If the user has not set all options, we still
 # have the defaults.
-include("${USER_CONFIG}" OPTIONAL RESULT_VARIABLE USER_CONFIG_LOADED)
-if (USER_CONFIG_LOADED)
-  set(USER_CONFIG_LOADED "LOADED")
-endif()
-message(STATUS "Loading USER_CONFIG = ${USER_CONFIG} (${USER_CONFIG_LOADED})")
+message(STATUS "(optional) USER_CONFIG = ${USER_CONFIG}")
+include("${USER_CONFIG}" OPTIONAL)
 include("${CMAKE_CURRENT_SOURCE_DIR}/config/defaults.cmake")

 # Allow overwriting of the compiler by setting CXX/MPICXX on the command line or
@@ -57,7 +50,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.6.1)
+set(${PROJECT_NAME}_VERSION 4.1.1)

 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -72,7 +65,8 @@ set(XSDK_ENABLE_C OFF)
 set(XSDK_ENABLE_Fortran OFF)

 # Check if we need to enable C or Fortran.
-if (MFEM_USE_CONDUIT OR
+if (CMAKE_VERSION VERSION_LESS 3.2 OR
+    MFEM_USE_CONDUIT OR
    MFEM_USE_SIDRE OR
    MFEM_USE_PETSC)
   # This seems to be needed by:
@@ -82,40 +76,25 @@ if (MFEM_USE_CONDUIT OR
   #  * find_package(PETSc REQUIRED)
   set(XSDK_ENABLE_C ON)
 endif()
-if (MFEM_USE_STRUMPACK OR MFEM_USE_MUMPS)
+if (MFEM_USE_STRUMPACK)
  # Just needed to find the MPI_Fortran libraries to link with
  set(XSDK_ENABLE_Fortran ON)
 endif()
-# SUNDIALS, STRUMPACK, Ginkgo, RAJA and Umpire require C++14:
-if ((MFEM_USE_SUNDIALS OR
-     MFEM_USE_STRUMPACK OR
-     MFEM_USE_GINKGO OR
-     MFEM_USE_RAJA OR
-     MFEM_USE_UMPIRE) AND
-    ("${CMAKE_CXX_STANDARD}" LESS "14"))
-  set(CMAKE_CXX_STANDARD 14)
-endif()

 # Include xSDK default CMake file.
 include("${CMAKE_CURRENT_SOURCE_DIR}/config/XSDKDefaults.cmake")

 # Enable languages.
 enable_language(CXX)
-if (MINGW)
-   # MinGW GCC does not expose the functions jn/_jn, yn/_yn (used in Example
-   # 25/25p) unless we use '-std=gnu++11':
-   set(CMAKE_CXX_EXTENSIONS ON)
-endif()
 if (MFEM_USE_CUDA)
-   if (MFEM_USE_HIP)
-      message(FATAL_ERROR " *** MFEM_USE_HIP cannot be combined with MFEM_USE_CUDA.")
-   endif()
+   # MFEM_USE_CUDA requires CMake 3.8 or newer (for direct CUDA support)
+   cmake_minimum_required(VERSION 3.8 FATAL_ERROR)
   # Use ${CMAKE_CXX_COMPILER} as the cuda host compiler.
   if (NOT CMAKE_CUDA_HOST_COMPILER)
      set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
   endif()
   enable_language(CUDA)
-   set(CMAKE_CUDA_STANDARD ${CMAKE_CXX_STANDARD})
+   set(CMAKE_CUDA_STANDARD 11)
   set(CMAKE_CUDA_STANDARD_REQUIRED ON)
   set(CMAKE_CUDA_EXTENSIONS OFF)
   set(CUDA_FLAGS "--expt-extended-lambda")
@@ -138,11 +117,10 @@ if (MFEM_USE_CUDA)
      set(CUDA_FLAGS "-ccbin=${CMAKE_CXX_COMPILER} ${CUDA_FLAGS}")
      set(CMAKE_CUDA_HOST_LINK_LAUNCHER ${CMAKE_CXX_COMPILER})
   endif()
-   set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${CUDA_FLAGS}")
+   set(CMAKE_CUDA_FLAGS "${CUDA_FLAGS}" CACHE STRING
+       "CUDA flags set for MFEM" FORCE)
   set(CUSPARSE_FOUND TRUE)
   set(CUSPARSE_LIBRARIES "cusparse")
-   set(CUBLAS_FOUND TRUE)
-   set(CUBLAS_LIBRARIES "cublas")
 endif()

 if (XSDK_ENABLE_C)
@@ -187,66 +165,10 @@ else()
  set(MFEM_DEBUG OFF)
 endif()

-# Shared build on Windows
-if (WIN32 AND BUILD_SHARED_LIBS)
-  # CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS works only with MSVC?
-  set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON CACHE BOOL
-      "Automatically export symbols for shared Windows DLL build")
-  # Place all .dll and .exe files in the same sub-directory, 'bin/<config>', of
-  # the build directory, so that the .exe files can find the .dll files at
-  # runtime:
-  set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin")
-  # Note: CMake tests are run from the build-tree sub-directory where the test
-  # is defined, e.g. <build-dir>/tests/unit, so hard coded paths to meshes still
-  # work fine.
-  # Not strictly necessary:
-  # set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
-  # set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
-endif()
-
-# AMD HIP
-if (MFEM_USE_HIP)
-  if (HIP_ARCH)
-    message(STATUS "Using HIP architecture: ${HIP_ARCH}")
-    set(GPU_TARGETS "${HIP_ARCH}" CACHE STRING "HIP targets to compile for")
-  endif()
-  if (ROCM_PATH)
-    list(INSERT CMAKE_PREFIX_PATH 0 ${ROCM_PATH})
-  endif()
-  find_package(HIP REQUIRED)
-  find_package(HIPSPARSE REQUIRED)
-endif()
-
-# OpenMP
-if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
-  if (NOT MFEM_THREAD_SAFE AND MFEM_USE_LEGACY_OPENMP)
-    message(FATAL_ERROR " *** MFEM_USE_LEGACY_OPENMP requires MFEM_THREAD_SAFE=ON.")
-  endif()
-  find_package(OpenMP REQUIRED)
-  set(OPENMP_LIBRARIES ${OpenMP_CXX_LIBRARIES})
-  if(APPLE)
-    # On macOS, the compiler needs additional help to find the <omp.h> header.
-    # See issue #2642 for more information.
-    set(OPENMP_INCLUDE_DIRS ${OpenMP_CXX_INCLUDE_DIRS})
-  endif(APPLE)
-  if (OPENMP_FOUND)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
-    if (MFEM_USE_CUDA)
-      set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=${OpenMP_CXX_FLAGS}")
-    endif()
-  endif()
-endif()
-
 # MPI -> hypre; PETSc (optional)
 if (MFEM_USE_MPI)
  find_package(MPI REQUIRED)
  set(MPI_CXX_INCLUDE_DIRS ${MPI_CXX_INCLUDE_PATH})
-  if (MFEM_MPIEXEC)
-    string(REPLACE " " ";" MPIEXEC ${MFEM_MPIEXEC})
-  endif()
-  if (MFEM_MPIEXEC_NP)
-    string(REPLACE " " ";" MPIEXEC_NUMPROC_FLAG ${MFEM_MPIEXEC_NP})
-  endif()
  # Parallel MFEM depends on hypre
  find_package(HYPRE REQUIRED)
  set(MFEM_HYPRE_VERSION ${HYPRE_VERSION})
@@ -263,7 +185,7 @@ if (MFEM_USE_MPI)
    endif()
  endif()
 else()
-  set(PKGS_NEED_MPI SUPERLU MUMPS PETSC SLEPC STRUMPACK PUMI)
+  set(PKGS_NEED_MPI SUPERLU PETSC SLEPC STRUMPACK PUMI)
  foreach(PKG IN LISTS PKGS_NEED_MPI)
    if (MFEM_USE_${PKG})
      message(STATUS "Disabling package ${PKG} - requires MPI")
@@ -302,6 +224,14 @@ if (MFEM_USE_LAPACK)
  find_package(LAPACK REQUIRED)
 endif()

+# OpenMP
+if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
+  if (NOT MFEM_THREAD_SAFE AND MFEM_USE_LEGACY_OPENMP)
+    message(FATAL_ERROR " *** MFEM_USE_LEGACY_OPENMP requires MFEM_THREAD_SAFE=ON.")
+  endif()
+  find_package(OpenMP REQUIRED)
+endif()
+
 # SuiteSparse (before SUNDIALS which may depend on KLU)
 if (MFEM_USE_SUITESPARSE)
  find_package(SuiteSparse REQUIRED
@@ -317,12 +247,14 @@ if (MFEM_USE_SUNDIALS)
  if (MFEM_USE_CUDA)
    list(APPEND SUNDIALS_COMPONENTS NVector_Cuda)
  endif()
-  if (MFEM_USE_HIP)
-    list(APPEND SUNDIALS_COMPONENTS NVector_Hip)
-  endif()
  find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS})
 endif()

+# Mesquite
+if (MFEM_USE_MESQUITE)
+  find_package(Mesquite REQUIRED)
+endif()
+
 # SuperLU_DIST can only be enabled in parallel
 if (MFEM_USE_SUPERLU)
  if (MFEM_USE_MPI)
@@ -332,16 +264,6 @@ if (MFEM_USE_SUPERLU)
  endif()
 endif()

-# MUMPS can only be enabled in parallel
-if (MFEM_USE_MUMPS)
-  if (MFEM_USE_MPI)
-    find_package(MUMPS REQUIRED mumps_common pord)
-    set(MFEM_MUMPS_VERSION ${MUMPS_VERSION})
-  else()
-    message(FATAL_ERROR " *** MUMPS requires that MPI be enabled.")
-  endif()
-endif()
-
 # STRUMPACK can only be enabled in parallel
 if (MFEM_USE_STRUMPACK)
  if (MFEM_USE_MPI)
@@ -380,11 +302,7 @@ if (MFEM_USE_AMGX)
 endif()

 if (MFEM_USE_CONDUIT)
-    find_package(Conduit REQUIRED conduit relay blueprint)
-endif()
-
-if (MFEM_USE_FMS)
-    find_package(FMS REQUIRED fms)
+    find_package(Conduit REQUIRED conduit relay blueprint )
 endif()

 # Axom/Sidre
@@ -396,7 +314,7 @@ endif()
 if (MFEM_USE_PUMI)
  # If PUMI_DIR was specified, only link to that directory,
  # i.e. don't link to another installation in /usr/lib by mistake
-  find_package(SCOREC 2.2.6 REQUIRED OPTIONAL_COMPONENTS gmi_sim
+  find_package(SCOREC 2.1.0 REQUIRED OPTIONAL_COMPONENTS gmi_sim
    CONFIG PATHS ${PUMI_DIR} NO_DEFAULT_PATH)
  if (SCOREC_FOUND)
    # Define a header file with the MFEM_USE_SIMMETRIX preprocessor variable
@@ -408,33 +326,12 @@ if (MFEM_USE_PUMI)
  endif()
 endif()

-# Moonolith
-if(MFEM_USE_MOONOLITH)
-  find_package(ParMoonolith REQUIRED)
-  if(ParMoonolith_FOUND)
-    get_target_property(
-      MOONOLITH_INCLUDE_DIRS ParMoonolith::par_moonolith
-      INTERFACE_INCLUDE_DIRECTORIES)
-    set(MOONOLITH_FOUND TRUE)
-    set(MOONOLITH_LIBRARIES ParMoonolith::par_moonolith)
-    message(
-      STATUS
-      "MOONOLITH_LIBRARIES=${MOONOLITH_LIBRARIES}, MOONOLITH_INCLUDE_DIRS=${MOONOLITH_INCLUDE_DIRS}")
-  endif()
-endif()
-
 # HiOp optimizer
 if (MFEM_USE_HIOP)
  find_package(HIOP REQUIRED)
  # find_package updates HIOP_FOUND, HIOP_INCLUDE_DIRS, HIOP_LIBRARIES
 endif()

-# CoDiPack package
-if (MFEM_USE_CODIPACK)
-  find_package(CODIPACK REQUIRED)
-  # find_package updates CODIPACK_FOUND, CODIPACK_INCLUDE_DIRS, CODIPACK_LIBRARIES
-endif()
-
 # OCCA
 if (MFEM_USE_OCCA)
   find_package(OCCA REQUIRED)
@@ -450,48 +347,17 @@ if (MFEM_USE_UMPIRE)
   find_package(UMPIRE REQUIRED)
 endif()

-# GOOGLE-BENCHMARK
-if (MFEM_USE_BENCHMARK)
-   find_package(Benchmark REQUIRED)
-endif()
-
-# Caliper
-if (MFEM_USE_CALIPER)
-  find_package(Caliper REQUIRED)
-endif()
-
-# Algoim
-if (MFEM_USE_ALGOIM)
-  find_package(Algoim REQUIRED)
-endif()
-
 # ADIOS2 for parallel I/O
 if (MFEM_USE_ADIOS2)
   find_package(ADIOS2 REQUIRED)
 endif()

-# MKL CPardiso
 if (MFEM_USE_MKL_CPARDISO)
  if (MFEM_USE_MPI)
   find_package(MKL_CPARDISO REQUIRED MKL_SEQUENTIAL MKL_LP64 MKL_MPI_WRAPPER)
  endif()
 endif()

-# MKL Pardiso
-if (MFEM_USE_MKL_PARDISO)
-   find_package(MKL_PARDISO REQUIRED MKL_SEQUENTIAL MKL_LP64)
-endif()
-
-# PARELAG
-if (MFEM_USE_PARELAG)
-   find_package(PARELAG REQUIRED)
-endif()
-
-# Enzyme
-if (MFEM_USE_ENZYME)
-   find_package(ENZYME REQUIRED)
-endif()
-
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
  if (APPLE)
@@ -510,30 +376,15 @@ if (NOT DEFINED MFEM_TIMER_TYPE)
  endif()
 endif()

-# Without this, CMake 3.21.1 (and 3.20.2) run into CMake Errors like the following:
-# CMake Error at config/cmake/modules/MfemCmakeUtilities.cmake:60 (add_library):
-#   Target "mfem" links to target "Threads::Threads" but the target was not
-#   found.  Perhaps a find_package() call is missing for an IMPORTED target, or
-#   an ALIAS target is missing?
-# Call Stack (most recent call first):
-#   CMakeLists.txt:474 (mfem_add_library)
-#
-# NOTE: We need to figure out which TPL library adds the dependency on
-#       "Threads::Threads" and call the next line only when that TPL library is
-#       enabled.  -V. Dobrev
-find_package(Threads REQUIRED)
-
 # List all possible libraries in order of dependencies.
 # [METIS < SuiteSparse]:
 #    With newer versions of SuiteSparse which include METIS header using 64-bit
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
-set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
-    SUNDIALS PETSC SLEPC MUMPS AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
-    NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
-    ADIOS2 CUBLAS CUSPARSE MKL_CPARDISO MKL_PARDISO AMGX CALIPER CODIPACK
-    BENCHMARK PARELAG MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)
-
+set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
+    SLEPC MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
+    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
+    CUSPARSE MKL_CPARDISO AMGX)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")
@@ -544,13 +395,16 @@ foreach(TPL IN LISTS MFEM_TPLS)
    list(APPEND TPL_INCLUDE_DIRS ${${TPL}_INCLUDE_DIRS})
  endif()
 endforeach(TPL)
-list(REVERSE TPL_LIBRARIES)
 list(REMOVE_DUPLICATES TPL_LIBRARIES)
-list(REVERSE TPL_LIBRARIES)
 list(REMOVE_DUPLICATES TPL_INCLUDE_DIRS)
 # message(STATUS "TPL_INCLUDE_DIRS = ${TPL_INCLUDE_DIRS}")
+include_directories(${TPL_INCLUDE_DIRS})
+
+if (OPENMP_FOUND)
+  message(STATUS "MFEM: using package OpenMP")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+endif()

-message(STATUS "MFEM shared library: BUILD_SHARED_LIBS = ${BUILD_SHARED_LIBS}")
 message(STATUS "MFEM build type: CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}")
 message(STATUS "MFEM version: v${MFEM_VERSION_STRING}")
 message(STATUS "MFEM git string: ${MFEM_GIT_STRING}")
@@ -563,17 +417,14 @@ message(STATUS "MFEM git string: ${MFEM_GIT_STRING}")
 set(SOURCES "")
 set(HEADERS "")
 set(MFEM_SOURCE_DIRS general linalg mesh fem)
-
-if(MFEM_USE_MOONOLITH)
-  set(MFEM_SOURCE_DIRS ${MFEM_SOURCE_DIRS} fem/moonolith)
-endif()
-
 foreach(DIR IN LISTS MFEM_SOURCE_DIRS)
  add_subdirectory(${DIR})
 endforeach()

 if (MFEM_USE_CUDA)
-  set_source_files_properties(${SOURCES} PROPERTIES LANGUAGE CUDA)
+   foreach(file IN LISTS SOURCES)
+      set_property(SOURCE ${file} PROPERTY LANGUAGE CUDA)
+   endforeach()
 endif()

 add_subdirectory(config)
@@ -592,33 +443,16 @@ set(MFEM_INSTALL_DIR ${CMAKE_INSTALL_PREFIX} CACHE PATH
    "The MFEM install directory" FORCE)

 # Declaring the library
-mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
+add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
 # message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
-target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES})
+if (CMAKE_VERSION VERSION_GREATER 2.8.11)
+  target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES})
+else()
+  target_link_libraries(mfem ${TPL_LIBRARIES})
+endif()
 if (MINGW)
-  target_link_libraries(mfem PRIVATE ws2_32)
+  target_link_libraries(mfem ws2_32)
 endif()
-if (MSVC)
-  target_compile_options(mfem PUBLIC "/wd4819")
-endif()
-message(STATUS "TPL_INCLUDE_DIRS = ${TPL_INCLUDE_DIRS}")
-target_include_directories(mfem
-  PUBLIC
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-    # The INSTALL_INTERFACE is handled below
-    ${TPL_INCLUDE_DIRS})
-if (MFEM_USE_MPI)
-  if (MPI_CXX_COMPILE_FLAGS)
-    separate_arguments(MPI_CXX_COMPILE_ARGS UNIX_COMMAND
-      "${MPI_CXX_COMPILE_FLAGS}")
-    target_compile_options(mfem PUBLIC ${MPI_CXX_COMPILE_ARGS})
-  endif()
-  if (MPI_CXX_LINK_FLAGS)
-    target_link_libraries(mfem PUBLIC ${MPI_CXX_LINK_FLAGS})
-  endif()
-endif()
-
 set_target_properties(mfem PROPERTIES VERSION "${mfem_VERSION}")
 set_target_properties(mfem PROPERTIES SOVERSION "${mfem_VERSION}")

@@ -630,7 +464,6 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
 endif()

 # Generate configuration file in the build directory: config/_config.hpp.
-set(MFEM_SHARED_BUILD ${BUILD_SHARED_LIBS})
 configure_file(
  "${PROJECT_SOURCE_DIR}/config/cmake/config.hpp.in"
  "${PROJECT_BINARY_DIR}/config/_config.hpp")
@@ -654,24 +487,16 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
  endforeach()
 endif()

-set(MFEM_CUSTOM_TARGET_PREFIX CACHE STRING "")
-
 #-------------------------------------------------------------------------------
-# Examples, miniapps, benchmarks and testing
+# Examples, miniapps, and testing
 #-------------------------------------------------------------------------------

-# Enable testing and benchmarks if required
+# Enable testing if required
 if (MFEM_ENABLE_TESTING)
  enable_testing()
  set(MFEM_ALL_TESTS_TARGET_NAME tests)
  add_mfem_target(${MFEM_ALL_TESTS_TARGET_NAME} OFF)
  add_subdirectory(tests EXCLUDE_FROM_ALL)
-
-  # Create a target for all benchmarks and, optionally, enable it.
-  set(MFEM_ALL_BENCHMARKS_TARGET_NAME benchmarks)
-  add_mfem_target(${MFEM_ALL_BENCHMARKS_TARGET_NAME}
-                  ${MFEM_ENABLE_GOOGLE_BENCHMARKS})
-  add_subdirectory(tests/benchmarks EXCLUDE_FROM_ALL)
 endif()

 # Define a target that all examples and miniapps will depend on.
@@ -681,11 +506,7 @@ add_custom_target(${MFEM_EXEC_PREREQUISITES_TARGET_NAME})
 # Create a target for all examples and, optionally, enable it.
 set(MFEM_ALL_EXAMPLES_TARGET_NAME examples)
 add_mfem_target(${MFEM_ALL_EXAMPLES_TARGET_NAME} ${MFEM_ENABLE_EXAMPLES})
-if (MFEM_ENABLE_EXAMPLES)
-  add_subdirectory(examples) #install examples if enabled
-else()
-  add_subdirectory(examples EXCLUDE_FROM_ALL)
-endif()
+add_subdirectory(examples EXCLUDE_FROM_ALL)

 # Create a target for all miniapps and, optionally, enable it.
 set(MFEM_ALL_MINIAPPS_TARGET_NAME miniapps)
@@ -695,7 +516,6 @@ add_subdirectory(miniapps EXCLUDE_FROM_ALL)
 # Target to build all executables, i.e. everything.
 add_custom_target(exec)
 add_dependencies(exec
-  ${MFEM_ALL_BENCHMARKS_TARGET_NAME}
  ${MFEM_ALL_EXAMPLES_TARGET_NAME}
  ${MFEM_ALL_MINIAPPS_TARGET_NAME}
  ${MFEM_ALL_TESTS_TARGET_NAME})
@@ -717,17 +537,16 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
 endif()

 # Add 'check' target - quick test
-set(MFEM_CHECK_TARGET_NAME ${MFEM_CUSTOM_TARGET_PREFIX}check)
 if (NOT MFEM_USE_MPI)
-  add_custom_target(${MFEM_CHECK_TARGET_NAME}
+  add_custom_target(check
    ${CMAKE_CTEST_COMMAND} -R \"^ex1_ser\" -C ${CMAKE_CFG_INTDIR}
    USES_TERMINAL)
-  add_dependencies(${MFEM_CHECK_TARGET_NAME} ex1)
+  add_dependencies(check ex1)
 else()
-  add_custom_target(${MFEM_CHECK_TARGET_NAME}
+  add_custom_target(check
    ${CMAKE_CTEST_COMMAND} -R \"^ex1p\" -C ${CMAKE_CFG_INTDIR}
    USES_TERMINAL)
-  add_dependencies(${MFEM_CHECK_TARGET_NAME} ex1p)
+  add_dependencies(check ex1p)
 endif()

 #-------------------------------------------------------------------------------
@@ -742,18 +561,12 @@ add_subdirectory(doc)
 message(STATUS "CMAKE_INSTALL_PREFIX = ${CMAKE_INSTALL_PREFIX}")
 set(INSTALL_INCLUDE_DIR include
  CACHE PATH "Relative path for installing header files.")
-set(INSTALL_BIN_DIR bin
-  CACHE PATH "Relative path for installing the binaries.")
 set(INSTALL_LIB_DIR lib
  CACHE PATH "Relative path for installing the library.")
 # other options: "share/mfem/cmake", "lib/mfem/cmake"
 set(INSTALL_CMAKE_DIR lib/cmake/mfem
  CACHE PATH "Relative path for installing cmake config files.")

-target_include_directories(mfem BEFORE
-  PUBLIC
-    $<INSTALL_INTERFACE:${INSTALL_INCLUDE_DIR}>)
-
 # The 'install' target will not depend on 'all'.
 # set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY TRUE)

@@ -762,9 +575,7 @@ set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME Development)
 # Install the library
 install(TARGETS ${PROJECT_NAME}
  EXPORT ${PROJECT_NAME_UC}Targets
-  RUNTIME DESTINATION ${INSTALL_BIN_DIR}
-  LIBRARY DESTINATION ${INSTALL_LIB_DIR}
-  ARCHIVE DESTINATION ${INSTALL_LIB_DIR})
+  DESTINATION ${INSTALL_LIB_DIR})

 # Install the master headers
 foreach(Header mfem.hpp mfem-performance.hpp)
@@ -789,7 +600,7 @@ endif()
 if (MFEM_USE_CEED)
  install(DIRECTORY ${MFEM_SOURCE_DIRS}
    DESTINATION ${INSTALL_INCLUDE_DIR}/mfem
-    FILES_MATCHING PATTERN "fem/ceed/integrators/*/*.h")
+    FILES_MATCHING PATTERN "fem/libceed/*.h")
 endif()

 # Install ${HEADERS}
@@ -802,8 +613,10 @@ endif()

 # Install the configuration header files
 install(FILES ${PROJECT_BINARY_DIR}/config/_config.hpp
-  ${PROJECT_SOURCE_DIR}/config/config.hpp
-  ${PROJECT_SOURCE_DIR}/config/tconfig.hpp
+  DESTINATION ${INSTALL_INCLUDE_DIR}/mfem/config
+  RENAME config.hpp)
+
+install(FILES ${PROJECT_SOURCE_DIR}/config/tconfig.hpp
  DESTINATION ${INSTALL_INCLUDE_DIR}/mfem/config)

 # Package the whole thing up nicely
@@ -818,8 +631,14 @@ export(TARGETS ${PROJECT_NAME}
 # TODO: How do we register the install-tree? Replacing the build-tree?
 export(PACKAGE ${PROJECT_NAME})

+# Extract the include directories required to use MFEM
+get_target_property(MFEM_TPL_INCLUDE_DIRS mfem INCLUDE_DIRECTORIES)
+if (NOT MFEM_TPL_INCLUDE_DIRS)
+  set(MFEM_TPL_INCLUDE_DIRS "")
+endif()
+
 # This is the build-tree version
-set(INCLUDE_INSTALL_DIRS ${PROJECT_BINARY_DIR} ${TPL_INCLUDE_DIRS})
+set(INCLUDE_INSTALL_DIRS ${PROJECT_BINARY_DIR} ${MFEM_TPL_INCLUDE_DIRS})
 set(LIB_INSTALL_DIR ${PROJECT_BINARY_DIR})
 configure_package_config_file(config/cmake/MFEMConfig.cmake.in
  ${CMAKE_CURRENT_BINARY_DIR}/MFEMConfig.cmake
@@ -827,7 +646,7 @@ configure_package_config_file(config/cmake/MFEMConfig.cmake.in
  PATH_VARS INCLUDE_INSTALL_DIRS LIB_INSTALL_DIR)

 # This is the version that will be installed
-set(INCLUDE_INSTALL_DIRS ${INSTALL_INCLUDE_DIR}  ${TPL_INCLUDE_DIRS})
+set(INCLUDE_INSTALL_DIRS ${INSTALL_INCLUDE_DIR}  ${MFEM_TPL_INCLUDE_DIRS})
 set(LIB_INSTALL_DIR ${INSTALL_LIB_DIR})
 configure_package_config_file(config/cmake/MFEMConfig.cmake.in
  ${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/MFEMConfig.cmake
@@ -1,133 +0,0 @@
-# MFEM Code of Conduct
-
-## Our Pledge
-
-We as members, contributors, and leaders pledge to make participation in our
-community a harassment-free experience for everyone, regardless of age, body
-size, visible or invisible disability, ethnicity, sex characteristics, gender
-identity and expression, level of experience, education, socio-economic status,
-nationality, personal appearance, race, caste, color, religion, or sexual
-identity and orientation.
-
-We pledge to act and interact in ways that contribute to an open, welcoming,
-diverse, inclusive, and healthy community.
-
-## Our Standards
-
-Examples of behavior that contributes to a positive environment for our
-community include:
-
-* Demonstrating empathy and kindness toward other people
-* Being respectful of differing opinions, viewpoints, and experiences
-* Giving and gracefully accepting constructive feedback
-* Accepting responsibility and apologizing to those affected by our mistakes,
-  and learning from the experience
-* Focusing on what is best not just for us as individuals, but for the overall
-  community
-
-Examples of unacceptable behavior include:
-
-* The use of sexualized language or imagery, and sexual attention or advances of
-  any kind
-* Trolling, insulting or derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or email address,
-  without their explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-## Enforcement Responsibilities
-
-Community leaders are responsible for clarifying and enforcing our standards of
-acceptable behavior and will take appropriate and fair corrective action in
-response to any behavior that they deem inappropriate, threatening, offensive,
-or harmful.
-
-Community leaders have the right and responsibility to remove, edit, or reject
-comments, commits, code, wiki edits, issues, and other contributions that are
-not aligned to this Code of Conduct, and will communicate reasons for moderation
-decisions when appropriate.
-
-## Scope
-
-This Code of Conduct applies within all community spaces, and also applies when
-an individual is officially representing the community in public spaces.
-Examples of representing our community include using an official e-mail address,
-posting via an official social media account, or acting as an appointed
-representative at an online or offline event.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported to the community leaders responsible for enforcement at mfem@llnl.gov.
-All complaints will be reviewed and investigated promptly and fairly.
-
-All community leaders are obligated to respect the privacy and security of the
-reporter of any incident. Anyone involved in the reported behavior will recuse
-themselves from the investigation and decision making about the resolution of
-the complaint.
-
-## Enforcement Guidelines
-
-Community leaders will follow these Community Impact Guidelines in determining
-the consequences for any action they deem in violation of this Code of Conduct:
-
-### 1. Correction
-
-**Community Impact**: Use of inappropriate language or other behavior deemed
-unprofessional or unwelcome in the community.
-
-**Consequence**: A private, written warning from community leaders, providing
-clarity around the nature of the violation and an explanation of why the
-behavior was inappropriate. A public apology may be requested.
-
-### 2. Warning
-
-**Community Impact**: A violation through a single incident or series of
-actions.
-
-**Consequence**: A warning with consequences for continued behavior. No
-interaction with the people involved, including unsolicited interaction with
-those enforcing the Code of Conduct, for a specified period of time. This
-includes avoiding interactions in community spaces as well as external channels
-like social media. Violating these terms may lead to a temporary or permanent
-ban.
-
-### 3. Temporary Ban
-
-**Community Impact**: A serious violation of community standards, including
-sustained inappropriate behavior.
-
-**Consequence**: A temporary ban from any sort of interaction or public
-communication with the community for a specified period of time. No public or
-private interaction with the people involved, including unsolicited interaction
-with those enforcing the Code of Conduct, is allowed during this period.
-Violating these terms may lead to a permanent ban.
-
-### 4. Permanent Ban
-
-**Community Impact**: Demonstrating a pattern of violation of community
-standards, including sustained inappropriate behavior, harassment of an
-individual, or aggression toward or disparagement of classes of individuals.
-
-**Consequence**: A permanent ban from any sort of public interaction within the
-community.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage],
-version 2.1, available at
-[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
-
-Community Impact Guidelines were inspired by
-[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
-
-For answers to common questions about this code of conduct, see the FAQ at
-[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
-[https://www.contributor-covenant.org/translations][translations].
-
-[homepage]: https://www.contributor-covenant.org
-[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
-[Mozilla CoC]: https://github.com/mozilla/diversity
-[FAQ]: https://www.contributor-covenant.org/faq
-[translations]: https://www.contributor-covenant.org/translations
@@ -1,14 +1,12 @@
 <p align="center">
-<a href="https://mfem.org/"><img alt="mfem" src="https://mfem.org/img/logo-300.png"></a>
+<a href="http://mfem.org/"><img alt="mfem" src="http://mfem.org/img/logo-300.png"></a>
 </p>

 <p align="center">
 <a href="https://github.com/mfem/mfem/blob/master/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-BSD-brightgreen.svg"></a>
-<a href="https://github.com/mfem/mfem/actions?query=workflow%3Arepo-check+branch%3Amaster"><img alt="Repo check" src="https://github.com/mfem/mfem/actions/workflows/repo-check.yml/badge.svg?branch=master"></a>
-<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuild-analysis+branch%3Amaster"><img alt="Build Analysis" src="https://github.com/mfem/mfem/actions/workflows/mfem-analysis.yml/badge.svg?branch=master"></a>
-<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuilds-and-tests+branch%3Amaster"><img alt="Builds and Tests" src="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml/badge.svg?branch=master"></a>
+<a href="https://travis-ci.org/mfem/mfem"><img alt="Build Status" src="https://travis-ci.org/mfem/mfem.svg?branch=master"></a>
 <a href="https://ci.appveyor.com/project/mfem/mfem"><img alt="Build Status" src="https://ci.appveyor.com/api/projects/status/19non9sqm6msi2wy?svg=true"></a>
-<a href="https://docs.mfem.org/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
+<a href="http://mfem.github.io/doxygen/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
 </p>


@@ -21,9 +19,6 @@ documentation; new examples and miniapps; HPC performance improvements; etc.
 MFEM is distributed under the terms of the BSD-3 license. All new contributions
 must be made under this license.

-Note also that MFEM has a [Code of Conduct](CODE_OF_CONDUCT.md). By participating
-in the MFEM community, you agree to abide by its rules.
-
 If you plan on contributing to MFEM, consider reviewing the
 [issue tracker](https://github.com/mfem/mfem/issues) first to check if a thread
 already exists for your desired feature or the bug you ran into. Use a pull
@@ -31,7 +26,7 @@ request (PR) toward the `mfem:master` branch to propose your contribution. If
 you are planning significant code changes or have questions, you may want to
 open an [issue](https://github.com/mfem/mfem/issues) before issuing a PR. In
 addition to technical contributions, we are also interested in your results and
-[simulation images](https://mfem.org/gallery/), which you can share via a pull
+[simulation images](http://mfem.org/gallery/), which you can share via a pull
 request in the [mfem/web](https://github.com/mfem/web) repo.

 See the [Quick Summary](#quick-summary) section for the main highlights of our
@@ -45,7 +40,6 @@ back to them before issuing pull requests:
  - [New Feature Development](#new-feature-development)
  - [Developer Guidelines](#developer-guidelines)
  - [Pull Requests](#pull-requests)
-  - [MFEM PR Rules](#mfem-pr-rules)
  - [Pull Request Checklist](#pull-request-checklist)
  - [Master/Next Workflow](#masternext-workflow)
  - [Releases](#releases)
@@ -57,7 +51,7 @@ back to them before issuing pull requests:
 Contributing to MFEM requires knowledge of Git and, likely, finite elements. If
 you are new to Git, see the [GitHub learning
 resources](https://help.github.com/articles/git-and-github-learning-resources/).
-To learn more about the finite element method, see our [FEM page](https://mfem.org/fem).
+To learn more about the finite element method, see our [FEM page](http://mfem.org/fem).

 *By submitting a pull request, you are affirming the [Developer's Certificate of
 Origin](#developers-certificate-of-origin-11) at the end of this file.*
@@ -69,18 +63,14 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
  development branches off `mfem:master`.
 - Please follow the [developer guidelines](#developer-guidelines), in particular
  with regards to documentation and code styling.
- Please do not commit large/binary files to the central repository (use a fork
-  instead).
- Pull requests should be issued toward `mfem:master`. Make sure
-  to check the items off the [Pull Request Checklist](#pull-request-checklist) and
-  follow the [MFEM PR Rules](#mfem-pr-rules).
+- Pull requests  should be issued toward `mfem:master`. Make sure
+  to check the items off the [Pull Request Checklist](#pull-request-checklist).
 - When your contribution is fully working and ready to be reviewed, add
  the `ready-for-review` label.
 - PRs are treated similarly to journal submission with an "editor" assigning two
  reviewers to evaluate the changes.
 - The reviewers have 3 weeks to evaluate the PR and work with the author to
  fix issues and implement improvements.
- During review there should be no force pushes/rewriting history in the branch.
 - After approval, MFEM developers merge the PR manually in the [mfem:next branch](#masternext-workflow).
 - After a week of testing in `mfem:next`, the original PR is merged in `mfem:master`.
 - We use [milestones](https://github.com/mfem/mfem/milestones) to coordinate the
@@ -101,65 +91,44 @@ The MFEM source code has the following structure:
 ```
  .
  ├── config
-  │   ├── cmake
-  │   ├── docker
-  │   ├── githooks
-  │   └── vcpkg
+  │   └── cmake
+  │       └── modules
  ├── data
  ├── doc
+  │   └── web
+  │       └── examples
  ├── examples
  │   ├── amgx
-  │   ├── caliper
  │   ├── ginkgo
  │   ├── hiop
-  │   ├── jupyter
-  │   ├── moonolith
  │   ├── petsc
  │   ├── pumi
-  │   ├── sundials
+  │   └── sundials
  |   └── superlu
  ├── fem
-  │   ├── ceed
-  │   ├── fe
-  │   ├── integ
-  │   ├── lor
-  │   ├── moonolith
-  │   ├── qinterp
-  │   └── tmop
+  │   └── libceed
  ├── general
  ├── linalg
  │   └── simd
  ├── mesh
-  │   └── submesh
  ├── miniapps
  │   ├── adjoint
-  │   ├── autodiff
  │   ├── common
-  │   ├── dpg
  │   ├── electromagnetics
  │   ├── gslib
-  │   ├── hdiv-linear-solver
-  │   ├── hooke
  │   ├── meshing
-  │   ├── mtop
-  │   ├── multidomain
  │   ├── navier
  │   ├── nurbs
-  │   ├── parelag
  │   ├── performance
-  │   ├── shifted
-  │   ├── solvers
-  │   ├── spde
  │   ├── tools
  │   └── toys
  └── tests
-      ├── benchmarks
      ├── convergence
-      ├── gitlab
-      ├── mem_manager
      ├── par-mesh-format
      ├── scripts
-      └── unit
+      ├── unit
+      │   ├── ...
+      └── ...
 ```

 #### Main directories and classes
@@ -169,24 +138,24 @@ classes implementing the finite element, mesh and linear algebra concepts
 respectively.

 - The main mesh classes are:
-  + [`Mesh`](https://docs.mfem.org/html/classmfem_1_1Mesh.html)
-  + [`NCMesh`](https://docs.mfem.org/html/classmfem_1_1NCMesh.html)
-  + [`Element`](https://docs.mfem.org/html/classmfem_1_1Element.html)
-  + [`ElementTransformation`](https://docs.mfem.org/html/classmfem_1_1ElementTransformation.html)
+  + [`Mesh`](http://mfem.github.io/doxygen/html/classmfem_1_1Mesh.html)
+  + [`NCMesh`](http://mfem.github.io/doxygen/html/classmfem_1_1NCMesh.html)
+  + [`Element`](http://mfem.github.io/doxygen/html/classmfem_1_1Element.html)
+  + [`ElementTransformation`](http://mfem.github.io/doxygen/html/classmfem_1_1ElementTransformation.html)

 - The main finite element classes are:
-  + [`FiniteElement`](https://docs.mfem.org/html/classmfem_1_1FiniteElement.html)
-  + [`FiniteElementCollection`](https://docs.mfem.org/html/classmfem_1_1FiniteElement.html)
-  + [`FiniteElementSpace`](https://docs.mfem.org/html/classmfem_1_1FiniteElementSpace.html)
-  + [`GridFunction`](https://docs.mfem.org/html/classmfem_1_1GridFunction.html)
-  + [`BilinearFormIntegrator`](https://docs.mfem.org/html/classmfem_1_1BilinearFormIntegrator.html) and [`LinearFormIntegrator`](https://docs.mfem.org/html/classmfem_1_1LinearFormIntegrator.html)
-  + [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearFormIntegrator.html), [`BilinearForm`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html) and [`MixedBilinearForm`](https://docs.mfem.org/html/classmfem_1_1MixedBilinearForm.html)
+  + [`FiniteElement`](http://mfem.github.io/doxygen/html/classmfem_1_1FiniteElement.html)
+  + [`FiniteElementCollection`](http://mfem.github.io/doxygen/html/classmfem_1_1FiniteElement.html)
+  + [`FiniteElementSpace`](http://mfem.github.io/doxygen/html/classmfem_1_1FiniteElementSpace.html)
+  + [`GridFunction`](http://mfem.github.io/doxygen/html/classmfem_1_1GridFunction.html)
+  + [`BilinearFormIntegrator`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearFormIntegrator.html) and [`LinearFormIntegrator`](http://mfem.github.io/doxygen/html/classmfem_1_1LinearFormIntegrator.html)
+  + [`LinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1LinearFormIntegrator.html), [`BilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html) and [`MixedBilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1MixedBilinearForm.html)

 - The main linear algebra classes and sources are
-  + [`Operator`](https://docs.mfem.org/html/classmfem_1_1Operator.html) and [`BilinearForm`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html)
-  + [`Vector`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html) and [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearForm.html)
-  + [`DenseMatrix`](https://docs.mfem.org/html/classmfem_1_1DenseMatrix.html) and [`SparseMatrix`](https://docs.mfem.org/html/classmfem_1_1SparseMatrix.html)
-  + Sparse [smoothers](https://docs.mfem.org/html/sparsesmoothers_8hpp.html) and linear [solvers](https://docs.mfem.org/html/solvers_8hpp.html)
+  + [`Operator`](http://mfem.github.io/doxygen/html/classmfem_1_1Operator.html) and [`BilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html)
+  + [`Vector`](http://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html) and [`LinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1LinearForm.html)
+  + [`DenseMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1DenseMatrix.html) and [`SparseMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1SparseMatrix.html)
+  + Sparse [smoothers](http://mfem.github.io/doxygen/html/sparsesmoothers_8hpp.html) and linear [solvers](http://mfem.github.io/doxygen/html/solvers_8hpp.html)

 #### Parallel implementation

@@ -196,13 +165,13 @@ shared geometric entities between different tasks. The parallel source files
 have a `p` prefix, e.g. `pmesh.cpp` vs. the serial `mesh.cpp`.

 - The main parallel classes are
-  + [`ParMesh`](https://docs.mfem.org/html/solvers_8hpp.html)
-  + [`ParNCMesh`](https://docs.mfem.org/html/classmfem_1_1ParMesh.html)
-  + [`ParFiniteElementSpace`](https://docs.mfem.org/html/classmfem_1_1ParFiniteElementSpace.html)
-  + [`ParGridFunction`](https://docs.mfem.org/html/classmfem_1_1ParGridFunction.html)
-  + [`ParBilinearForm`](https://docs.mfem.org/html/classmfem_1_1ParBilinearForm.html) and [`ParLinearForm`](https://docs.mfem.org/html/classmfem_1_1ParLinearForm.html)
-  + [`HypreParMatrix`](https://docs.mfem.org/html/classmfem_1_1HypreParMatrix.html) and [`HypreParVector`](https://docs.mfem.org/html/classmfem_1_1HypreParVector.html)
-  + [`HypreSolver`](https://docs.mfem.org/html/classmfem_1_1HypreSolver.html) and other [hypre classes](https://docs.mfem.org/html/hypre_8hpp.html)
+  + [`ParMesh`](http://mfem.github.io/doxygen/html/solvers_8hpp.html)
+  + [`ParNCMesh`](http://mfem.github.io/doxygen/html/classmfem_1_1ParMesh.html)
+  + [`ParFiniteElementSpace`](http://mfem.github.io/doxygen/html/classmfem_1_1ParFiniteElementSpace.html)
+  + [`ParGridFunction`](http://mfem.github.io/doxygen/html/classmfem_1_1ParGridFunction.html)
+  + [`ParBilinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1ParBilinearForm.html) and [`ParLinearForm`](http://mfem.github.io/doxygen/html/classmfem_1_1ParLinearForm.html)
+  + [`HypreParMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreParMatrix.html) and [`HypreParVector`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreParVector.html)
+  + [`HypreSolver`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreSolver.html) and other [hypre classes](http://mfem.github.io/doxygen/html/hypre_8hpp.html)

 #### GPU and general device support

@@ -211,10 +180,10 @@ backends (CUDA, OCCA, RAJA, OpenMP, etc.) and an internal lightweight
 device/host memory manager.

 - The main device-relevant classes and sources are:
-  + [`Device`](https://docs.mfem.org/html/device_8hpp.html)
-  + [`MemoryManager`](https://docs.mfem.org/html/mem_manager_8hpp.html)
-  + the [`mfem::forall`](https://docs.mfem.org/html/forall_8hpp.html) function
-  + the [`cuda.hpp`](https://docs.mfem.org/html/cuda_8hpp.html) and [`occa.hpp`](https://docs.mfem.org/html/occa_8hpp.html) files
+  + [`Device`](http://mfem.github.io/doxygen/html/device_8hpp.html)
+  + [`MemoryManager`](http://mfem.github.io/doxygen/html/mem_manager_8hpp.html)
+  + the [`MFEM_FORALL`](http://mfem.github.io/doxygen/html/forall_8hpp.html) macro
+  + the [`cuda.hpp`](http://mfem.github.io/doxygen/html/cuda_8hpp.html) and [`occa.hpp`](http://mfem.github.io/doxygen/html/occa_8hpp.html) files

 #### Utilities, building and documentation
 - The `general/` directory contains C++ classes that serve as utilities for
@@ -222,7 +191,8 @@ device/host memory manager.
 - The `config/` directory contains build-related files, both for the plain
  Makefile and the CMake build options.
 - The `doc/` directory contains configuration for the Doxygen code documentation
-  that can either be built locally or browsed online at https://docs.mfem.org.
+  that can either be built locally or browsed online at
+  http://mfem.github.io/doxygen/html/index.html.

 #### Examples and tests
 - `examples` and `miniapps` respectively gather simple and more fully-featured
@@ -231,7 +201,7 @@ device/host memory manager.
 - The `tests/` directory contains a unit test suite and will later contain more
  tests that run example codes.

-See also the [code overview](https://mfem.org/code-overview/) section on the MFEM
+See also the [code overview](http://mfem.org/code-overview/) section on the MFEM
 website.

 ## GitHub Workflow
@@ -344,22 +314,15 @@ Before you can start, you need a GitHub account, here are a few suggestions:
    change the code by default.

 - Code specifics
-  - All new public, protected, and private classes, methods, data members, and
-    functions have Doxygen-style documentation in source comments.
-  - In addition to arguments and functionality, documentation should include the
-    current limitations of the code, any background information that is
-    implicitly assumed in the implementation, and the ownership and lifetime
-    of data.
+  - All significant new classes, methods and functions have Doxygen-style
+    documentation in source comments.
  - Consistent code styling is enforced with `make style` in the top-level
    directory. This requires [Artistic Style](http://astyle.sourceforge.net) (we
-    specifically use version 3.1). See also the file `config/mfem.astylerc`.
+    specifically use version 2.05.1). See also the file `config/mfem.astylerc`.
  - Use `mfem::out` and `mfem::err` instead of `std::cout` and `std::cerr` in
    internal library code. (You can use `std` in examples and miniapps.)
  - When manually resolving conflicts during a merge, make sure to mention the
    conflicted files in the commit message.
-  - All significant new features and changes should be documented in CHANGELOG.
-  - New examples and miniapps should have documentation on the MFEM webpage.
-

 ### Pull Requests

@@ -396,10 +359,6 @@ Before you can start, you need a GitHub account, here are a few suggestions:
  two reviewers to evaluate the changes. The reviewers have 3 weeks to evaluate
  the PR and work with the author to implement improvements and fix issues.

- Once the `ready-for-review` label has been applied and reviewers have been
-  assigned, the PR is considered under review. To help with the review process
-  there should be no force pushes/rewriting history in the branch.
-
 - After approval, the PR is [tested](#masternext-workflow) for a week with
  other approved PRs in the `mfem:next` branch.

@@ -407,101 +366,20 @@ Before you can start, you need a GitHub account, here are a few suggestions:
  `mfem:next`, see the [README](tests/scripts/README) file in that directory
  for more details.

- Track the GitHub Actions and Appveyor [continuous integration](#automated-testing)
+- Track the Travis CI and Appveyor [continuous integration](#automated-testing)
  builds at the end of the PR. These should generally run clean, so address any
  errors as soon as possible. Please ask if you are unsure how to do that.

- Note that some tests, such as the `branch-history` check in GitHub Actions
-  are safeguards that are allowed to fail in certain cases.
+- Note that some tests, such as the `branch-history` check in Travis are
+  safeguards that are allowed to fail in certain cases.

 - Other tests, such as the `code-style`, `documentation` and `gitignore`
-  checks in GitHub Actions enforce MFEM-specific rules which are explained in
-  the error messages and the `tests/scripts` directory.
-
- Also note that the tests `branch-history` and `repos-checks` found in GitHub
-  Actions can be triggered automatically before each push using git hooks. See
-  the [git hooks README](config/githooks/README.md) for a detailed explanation.
+  checks in Travis enforce MFEM-specific rules which are explained in the
+  error messages and the `tests/scripts` directory.

 - If triggered, track the status of the LLNL GitLab tests. If failing, ask
  one of the _LLNL developers_ for details.

-
-### MFEM PR Rules
-
-The Pull Request (PR) approval process in MFEM is similar to the approval of papers in a peer-reviewed journal. In particular:
-
-1. There is an MFEM board of "editors" that evaluates new PRs and assigns "reviewers" for each PR.
-
-2. The assigned reviewers are responsible to carefully review and test the proposed PR.
-
-3. A PR can be (manually) merged in the *next* branch only if 2 of the assigned reviewers have approved it and it has passed internal testing. This merge can be performed by any of the assigned reviewers or by any of the editors.
-
-4. A PR can be merged in the *master* branch only if it has been tested successfully for a week in *next* and an editor has (optionally) taken a final look. This merge can be performed only by one of the editors.
-
-#### Responsibilities of Editors
-
-The current list of MFEM editors is:
-
- [@v-dobrev](https://github.com/v-dobrev) (Veselin Dobrev)
- [@tzanio](https://github.com/tzanio) (Tzanio Kolev)
- [@pazner](https://github.com/pazner) (Will Pazner)
- [@mlstowell](https://github.com/mlstowell) (Mark Stowell)
-
-**The responsibilities of the editors are:**
-
-1. To assign appropriate milestone and labels for new PRs, e.g. *bugfix*, *minor*, *api-change*, *high-impact*, etc.
-
-2. To assign at least 2 reviewers for new PRs. An editor can also be a reviewer. The editor, reviewers, and author should be listed as "Assignees" on the GitHub PR page. After assignment, the `in-review` label should be added.
-
-3. To complete the initial PR evaluation and assignments in a timely manner: 1 week from submission.
-
-4. To assist reviewers when they need help with their reviews (but also to stay out of the way when they don't).
-
-5. To remind the reviewers about timely completion of their review.
-
-6. To take a final look and complete the PR merge in *master*. The final look step is optional and shouldn't take more than 3 days.
-
-7. The assignment of bugfixes should be expedited proportional to their importance, e.g. in some cases the editor can assign much shorter review window.
-
-#### Responsibilities of Reviewers
-
-Everyone on the MFEM team can be asked to serve as a reviewer on a PR in their area of expertise.
-
-**The responsibilities of the reviewers are:**
-
-1. To let the editors know if the proposed assignment is not a good match for them.
-
-2. To communicate with the PR author, provide feedback and work with them to resolve issues.
-
-3. To ensure the quality of the PR by making sure that the code adheres to the [Developer Guidelines](#developer-guidelines), e.g. all methods, data members, and functions have documentation, including data ownership and lifetime, new examples/miniapps have a corresponding PR in mfem/web, major features have `CHANGELOG` entries, etc.
-
-3. To seek help from the editors in case of difficulties.
-
-4. To complete the review in a timely manner: 3 weeks from assignment.
-
-5. To test the PR thoroughly before merging in *next*. The PR author is also encouraged to perform testing and inform the reviewers about the results.
-
-6. To monitor the PR impact on the testing in the *next* branch and alert the editors that the PR is ready for merging in *master*.
-
-7. The review of bugfixes should be expedited proportional to their importance. The review window can be much less than three weeks in such cases.
-
-#### Responsibilities of Authors
-
-Authors should clearly indicate when a PR is ready for review (before that the PR should be marked as `Draft` or `[WIP]`).
-
-**The responsibilities of the authors are:**
-
-1. To follow the instructions and PR checklist in the `CONTRIBUTING.md` document in the MFEM repository.
-
-2. To respond to reviewer feedback in a timely manner.
-
-3. Authors are encouraged to perform testing and inform the reviewers about the results.
-
-4. Authors can use the "Reviewers" section of the GitHub PR page to suggest reviewers, but the "Assignees" section will show who the editor has assigned to do the reviews.
-
-5. To indicate when the PR is ready for review by adding the `ready-for-review` label.
-
-
 ### Pull Request Checklist

 Before a PR can be merged, it should satisfy the following:
@@ -517,7 +395,7 @@ Before a PR can be merged, it should satisfy the following:
    - [ ] Does `make` or `cmake` have a new target?
    - [ ] Did the requirements or the installation process change? *(rare)*
 - [ ] Update continuous integration server configurations if necessary (e.g. with new version requirements for each of MFEM's dependencies)
-    - [ ] `.github`
+    - [ ] `.travis.yml`
    - [ ] `.appveyor.yml`
 - [ ] Update `.gitignore`:
    - [ ] Check if `make distclean; git status` shows any files that were generated from the source by the project (not an IDE) but we don't want to track in the repository.
@@ -547,7 +425,6 @@ Before a PR can be merged, it should satisfy the following:
      - [ ] Add/update the `CMakeLists.txt` file in the new miniapp directory.
      - [ ] Consider adding a new test for the new miniapp.
   - [ ] List the new miniapp in `doc/CodeDocumentation.dox`
-   - [ ] If new miniapps directory (e.g.`miniapps/nurbs`), add it to `MINIAPP_SUBDIRS` in the `makefile`.
   - [ ] If new miniapps directory (e.g.`miniapps/nurbs`), list it in `doc/CodeDocumentation.conf.in`
   - [ ] Companion pull request for documentation in [mfem/web](https://github.com/mfem/web) repo:
     - [ ] Update or add miniapp-specific documentation, see e.g. the `src/meshing.md` and `src/electromagnetics.md` files.
@@ -555,9 +432,7 @@ Before a PR can be merged, it should satisfy the following:
     - [ ] The miniapps go at the end of the page, and are usually listed only under a specific "Application (PDE)" category.
     - [ ] Add a short description of the miniapp in the "Extensive Examples" section of `features.md`.
 - [ ] New capability:
-   - [ ] All new public, protected, and private classes, methods, data members, and functions have full Doxygen-style documentation in source comments. Documentation should include descriptions of member data, function arguments and return values, template parameters, and prerequisites for calling new functions.
-   - [ ] Pointer arguments and return values must specify whether ownership is being transferred or lent with the call.
-   - [ ] Any new functions should include descriptions of their intended use e.g. for internal use only, user-facing, etc., along with references to example code whenever possible/appropriate.
+   - [ ] All significant new classes, methods and functions have Doxygen-style documentation in source comments.
   - [ ] Consider adding new sample runs in existing examples to highlight the new capability.
   - [ ] Consider saving cool simulation pictures with the new capability in the Confluence gallery (LLNL only) or submitting them, via pull request, to the gallery section of the `mfem/web` repo.
   - [ ] If this is a major new feature, consider mentioning it in the short summary inside `README` *(rare)*.
@@ -566,9 +441,9 @@ Before a PR can be merged, it should satisfy the following:
 - [ ] Run `make unittest` to make sure all unit tests pass.
 - [ ] Run the tests in `tests/scripts`.
 - [ ] (LLNL only) After merging:
+   - [ ] Regenerate `README.html` files from companion documentation pull requests.
   - [ ] Update internal tests to include the new features.

-
 ### Master/Next Workflow

 MFEM uses a `master`/`next`-branch workflow as described below:
@@ -637,12 +512,9 @@ MFEM uses a `master`/`next`-branch workflow as described below:
    - [ ] `doc/CodeDocumentation.conf.in`
 - [ ] Check that version requirements for each of MFEM's dependencies are documented in `INSTALL` and up-to-date
 - [ ] Check that continuous integration server configurations reflect the dependency version requirements of the new release
-    - [ ] `.github`
+    - [ ] `.travis.yml`
    - [ ] `.appveyor.yml`
- [ ] Update the `CHANGELOG` to organize all release contributions
- [ ] Review the whole source code once over
- [ ] Ask MFEM-based applications to test the pre-release branch
- [ ] Test on additional platforms and compilers
+- [ ] (LLNL only) Make sure all `README.html` files in the source repo are up to date.
 - [ ] Tag the repository:

  ```
@@ -651,9 +523,9 @@ MFEM uses a `master`/`next`-branch workflow as described below:
  ```
 - [ ] Create the release tarball and push to `mfem/releases`.
 - [ ] Recreate the `next` branch as described in previous section.
- [ ] Update and push documentation to `mfem/doxygen`. Update the `README.md` file and the `html` link in the `mfem/doxygen` repo.
+- [ ] Update and push documentation  to `mfem/doxygen`.
 - [ ] Update URL shortlinks:
-    - [ ] Create a shortlink at [http://bit.ly/](http://bit.ly/) for the release tarball, e.g. https://mfem.github.io/releases/mfem-3.1.tgz.
+    - [ ] Create a shortlink at [http://bit.ly/](http://bit.ly/) for the release tarball, e.g. http://mfem.github.io/releases/mfem-3.1.tgz.
    - [ ] (LLNL only) Add and commit the new shortlink in the `links` and `links-mfem` files of the internal `mfem/downloads` repo.
    - [ ] Add the new shortlinks to the MFEM packages in `spack`, `homebrew/science`, `VisIt`, etc.
 - [ ] Update website in `mfem/web` repo:
@@ -663,7 +535,6 @@ MFEM uses a `master`/`next`-branch workflow as described below:

 ## LLNL Workflow

-
 ### Mirroring on Bitbucket

 - The GitHub `master` and `next` branches are mirrored to the LLNL institutional
@@ -683,42 +554,33 @@ MFEM uses a `master`/`next`-branch workflow as described below:
  - `mfem:gh-next` -- Bleeding-edge development version, may be broken, use at
     your own risk.

-
 ### Mirroring on GitLab

 - MFEM repository is also mirrored on the LLNL GitLab instance, in a
  semi-automated manner.

 - This instance is meant to complete CI testing with tests on Livermore
-  Computing systems. GitLab pipeline status is reported in the corresponding
+  Computing systems. Gitlab pipeline status is reported in the corresponding
  GitHub pull request.

- In GitLab pipelines, TPLs (dependencies) are built using Spack, driven by Uberenv.
-
- No change to the MFEM repo can be made on this instance.
+- No change can be made on this instance.

 ## Automated Testing

 MFEM has several levels of automated testing running on GitHub, as well as on
 local Mac and Linux workstations, and Livermore Computing clusters at LLNL.

-In addition, developers can set local git hooks to run some quick checks on
-commit or push, see the [README](config/githooks/README.md) in the `config/githooks`
-directory.
-
-
 ### Linux and Mac smoke tests
-We use GitHub Actions to drive the default tests on the `master` and `next`
-branches. See the `.github/workflows` files and the logs at
-[https://github.com/mfem/mfem/actions](https://github.com/mfem/mfem/actions).
+We use Travis CI to drive the default tests on the `master` and `next`
+branches. See the `.travis` file and the logs at
+[https://travis-ci.org/mfem/mfem](https://travis-ci.org/mfem/mfem).

-Testing using GitHub Actions should be kept lightweight, as there is a time
+Testing using Travis CI should be kept lightweight, as there is a 50 minute time
 constraint on jobs. Two virtual machines are configured - Mac (OS X) and Linux.

 - Tests on the `master` branch are triggered whenever a PR is issued on this branch.
 - Tests on the `next` branch are currently scheduled to run each night.

-
 ### Windows smoke test
 We use Appveyor to test building with the MS Visual C++ compiler in a Windows
 environment, as well as to test the CMake build. See the `.appveyor` file and the
@@ -728,7 +590,6 @@ build logs at
 CMake is used to generate the MSVC Project files and drive the build.  A release
 and debug build is performed with a simple run of `ex1` to verify the executable.

-
 ### Tests at LLNL

 - We mirror the `master` and `next` branches internally (to `gh-master` and
@@ -5,11 +5,7 @@
                      | | | | | ||  _||  __/| | | | | |
                      |_| |_| |_||_|   \___||_| |_| |_|

-                               https://mfem.org
-
-This file provides a detailed description of how to build and install the MFEM
-library. For a simple build, see the step-by-step instructions on the website
-at https://mfem.org/building.
+                               http://mfem.org

 The MFEM library has a serial and an MPI-based parallel version, which largely
 share the same code base. The only prerequisite for building the serial version
@@ -20,17 +16,13 @@ requires an MPI C++ compiler, as well as the following external libraries:
  https://github.com/hypre-space/hypre

 - METIS (a family of multilevel partitioning algorithms)
-  https://github.com/mfem/tpls
-
-  Note: We recommend our mirror of metis-4.0.3/5.1.0 above because the METIS
-  webpage, http://glaros.dtc.umn.edu/gkhome/metis/metis/overview, is often down
-  and we don't support yet the new repo https://github.com/KarypisLab/METIS.
+  http://glaros.dtc.umn.edu/gkhome/metis/metis/overview

 The hypre dependency can be downloaded as a tarball from GitHub or from the
-project webpage https://www.llnl.gov/casc/hypre. For example, the 2.24.0 release
+project webpage https://www.llnl.gov/casc/hypre. For example, the 2.16.0 release
 of hypre is available at

-   https://github.com/hypre-space/hypre/archive/v2.24.0.tar.gz
+   https://github.com/hypre-space/hypre/archive/v2.16.0.tar.gz

 The METIS dependency can be disabled but that is not generally recommended, see
 the option MFEM_USE_METIS.
@@ -45,7 +37,7 @@ as CUDA, HIP, OCCA, OpenMP and RAJA.
  https://developer.nvidia.com/cuda-toolkit

 - HIP support requires an AMD GPU and an installation of the ROCm software stack
-  https://rocmdocs.amd.com
+  https://rocm.github.io/ROCmInstall.html#installing-from-amd-rocm-repositories

 - OCCA support requires the OCCA library
  https://libocca.org
@@ -66,12 +58,11 @@ following package managers:

 - Spack, https://github.com/spack/spack
 - OpenHPC, http://openhpc.community
- Conda-forge, https://conda-forge.org (pre-built binaries linked with
-  OpenMPI/MPICH, hypre, and METIS)
+- Homebrew/Science, https://github.com/Homebrew/homebrew-science (deprecated)

 We also recommend downloading and building the MFEM-based GLVis visualization
 tool which can be used to visualize the meshes and solution in MFEM's examples
-and miniapps. See https://glvis.org and https://mfem.org/building.
+and miniapps. See http://glvis.org and http://mfem.org/building.

 Quick start with GNU make
 =========================
@@ -86,7 +77,7 @@ Parallel build:

 CUDA build:
   make cuda -j 4
-   (build for a specific compute capability: 'make cuda -j 4 CUDA_ARCH=sm_70')
+   (build for a specific compute capability: 'make cuda -j 4 CUDA_ARCH=sm_30')

 HIP build:
   make hip -j 4
@@ -337,6 +328,10 @@ MFEM_USE_SUNDIALS = YES/NO
   library. When enabled, this option uses the SUNDIALS_* library options,
   see below.

+MFEM_USE_MESQUITE = YES/NO
+   Enable MFEM functionality based on the Mesquite library. When enabled, this
+   option uses the MESQUITE_* library options, see below.
+
 MFEM_USE_SUITESPARSE = YES/NO
   Enable MFEM functionality based on the SuiteSparse library. Currently, this
   option adds the classes UMFPackSolver and KLUSolver (both sparse serial
@@ -353,11 +348,6 @@ MFEM_USE_SUPERLU5 = YES/NO
   If SuperLU functionality is enabled, use the older 5.1.0 version rather than
   the more recent 6+ versions.

-MFEM_USE_MUMPS = YES/NO
-   Enable MFEM functionality based on the MUMPS library. Currently, this
-   option adds the class MUMPSSolver (a parallel sparse direct solver).
-   When enabled, this option uses the MUMPS_* library options, see below.
-
 MFEM_USE_STRUMPACK = YES/NO
   Enable MFEM functionality based on the STRUMPACK sparse direct solver and
   preconditioner through the STRUMPACKSolver and STRUMPACKRowLocMatrix
@@ -463,29 +453,9 @@ MFEM_USE_UMPIRE = YES/NO
   discovery, provision, and management of memory on machines with multiple
   memory devices like NUMA and GPUs.

-MFEM_USE_BENCHMARK = YES/NO
-   Enables support for Google Benchmark, a library to support the benchmarking
-   of functions, in the tests/benchmarks directory.
-
 MFEM_USE_HIOP = YES/NO
-   Enable the usage of HiOp (https://github.com/LLNL/hiop) in MFEM. HiOp is an
-   HPC solver for nonlinear optimization problems.
-
-MFEM_USE_CODIPACK = YES/NO
-  Enable automatic differentiation using the CoDiPack library.
-  www.scicomp.uni-kl.de/codi/
-
-MFEM_USE_ALGOIM = YES/NO
-  Enable the usage of Algoim - a collection of high-order accurate numerical
-  methods and C++ algorithms for working with implicitly-defined geometry and
-  level set methods. The Algoim library requires the Blitz++ library. The MFEM
-  provides interface to Algoim v1. Thus, to check out the specific state use:
-  git checkout 9c9ca0ef094d8ab0390ed36367a1151b459bbe0a
-  https://algoim.github.io
-
-MFEM_USE_ADFORWARD = YES/NO
-  Enable forward mode for AD packages. This option is valid
-  only if the AD package supports two modes (backward/forward).
+  Enable the usage of HiOp (https://github.com/LLNL/hiop) in MFEM. HiOp is an
+  HPC solver for nonlinear optimization problems.

 MFEM_USE_CUDA = YES/NO
   Enables support for CUDA devices in MFEM. CUDA is a parallel computing
@@ -498,7 +468,7 @@ MFEM_USE_HIP = YES/NO
   Enables support for AMD devices in MFEM. HIP is a heterogeneous-compute
   interface for portability developed by AMD that can target both AMD and
   NVIDIA GPUs. The variable HIP_ARCH is used to specify the AMD GPU processor
-   used during compilation (by default, HIP_ARCH=gfx900). When enabled, this
+   used during compilation (by default, HIP_ARCH=gfx900).  When enabled, this
   option uses the HIP_* build options, see below.

 MFEM_USE_RAJA = YES/NO
@@ -532,44 +502,6 @@ MFEM_USE_MKL_CPARDISO = YES/NO
   MFEM_USE_LAPACK=YES, verify that the MKL LAPACK libraries are used. The
   OpenMP capabilities are disabled at link time.

-MFEM_USE_MOONOLITH = YES/NO
-   Enables the ParMoonolith interface for parallel non-conforming, non-matching,
-   variational, volumetric mesh information transfer. It requires the variable
-   MOONOLITH_DIR=<path to installation> to be defined in the environment in
-   order to be used with the Makefile. Makefile users are also required to
-   install moonolith using the command `make install_all`, see
-   https://bitbucket.org/zulianp/par_moonolith for details.
-   Although Moonolith is an MPI-based library, both serial (MFEM_USE_MPI=NO) and
-   parallel (MFEM_USE_MPI=YES) versions of MFEM are supported.
-
-MFEM_USE_CALIPER = YES/NO
-   Enables the interface to Caliper. Caliper is a library to integrate
-   performance profiling capabilities into applications. To use Caliper,
-   developers mark code regions of interest using either Caliper's annotation
-   API or their equivalent in MFEM. Applications can then enable performance
-   profiling at runtime with Caliper's configuration API. Alternatively, one
-   can configure Caliper through environment variables or config files.
-
-MFEM_USE_FMS = YES/NO
-   Enables support for the FMS library which consists of the DataCollection
-   sub-class mfem::FMSDataCollection for I/O in FMS formats, see the header file
-   fem/fmsdatacollection.hpp. In addition, this option enables in-memory
-   conversion routines between FMS's FmsDataCollection structure and MFEM's
-   DataCollection class, see the header file fem/fmsconvert.hpp.
-
-MFEM_USE_PARELAG = YES/NO
-   Enables the miniapps that use the ParELAG library. MFEM does not currently
-   use ParELAG. In fact, ParELAG is dependent on MFEM. Therefore, this option
-   currently only concerns the miniapps.
-
-MFEM_USE_ENZYME = YES/NO
-   Enables automatic differentiation support through the LLVM plugin Enzyme.
-   This requires the compiler to be set to clang (>=14.0.0). We also advise to
-   use the link time optimization (LTO) plugin, to enable functions that you
-   define over multiple files (compilation units) and want to be differentiated
-   automatically, to work. This requires to also use LLVM/LLD for linking.
-   Recommended options are in config/defaults.mk.
-
 MFEM_BUILD_TAG = (any value)
   An optional tag to characterize the build. Exported to config/config.mk.
   Can be used to identify the MFEM build from other makefiles.
@@ -591,17 +523,13 @@ directory and use the string @MFEM_DIR@, e.g. HYPRE_OPT = -I@MFEM_DIR@/../hypre.
 The specific libraries and their options are:

 - HYPRE, required for the parallel build, i.e. when MFEM_USE_MPI = YES.
-  See also the "Specific options for hypre" section at the end of this file.
  URL: https://github.com/hypre-space/hypre and https://www.llnl.gov/casc/hypre
  Options: HYPRE_OPT, HYPRE_LIB.
-  Versions: HYPRE >= 2.10.0b  (HYPRE built without CUDA)
-            HYPRE >= 2.20.0   (HYPRE built with '--enable-mixedint')
-            HYPRE >= 2.22.1   (HYPRE built with CUDA)
-            HYPRE >= 2.23.0   (HYPRE built with HIP)
+  Versions: HYPRE >= 2.10.0b.

 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
  MFEM_USE_METIS_5 = YES (default is to use METIS 4).
-  URL: https://github.com/mfem/tpls (MFEM mirror, see above)
+  URL: http://glaros.dtc.umn.edu/gkhome/metis/metis/overview
  Options: METIS_OPT, METIS_LIB.
  Versions: METIS 4.0.3 or 5.1.0.

@@ -628,13 +556,13 @@ The specific libraries and their options are:
  both MPI and hypre.
  If MFEM_USE_CUDA is enabled, we expect that SUNDIALS is built with support
  for CUDA.
-  If MFEM_USE_HIP is enabled, we expect that SUNDIALS is built with support
-  for HIP.
-  URL: http://computing.llnl.gov/projects/sundials/sundials-software
+  URL: http://computation.llnl.gov/projects/sundials/sundials-software
  Options: SUNDIALS_OPT, SUNDIALS_LIB.
-  Versions: SUNDIALS >= 5.0.0,
-            SUNDIALS >= 5.4.0 for CUDA support, and
-            SUNDIALS >= 5.7.0 for HIP support.
+  Versions: SUNDIALS >= 5.0.0, SUNDIALS >= 5.4.0 for CUDA suppport.
+
+- Mesquite (optional), used when MFEM_USE_MESQUITE = YES.
+  URL: http://trilinos.org/oldsite/packages/mesquite
+  Options: MESQUITE_OPT, MESQUITE_LIB.

 - SuiteSparse (optional), used when MFEM_USE_SUITESPARSE = YES.
  URL: http://faculty.cse.tamu.edu/davis/suitesparse.html
@@ -644,17 +572,11 @@ The specific libraries and their options are:
 - SuperLU_DIST (optional), used when MFEM_USE_SUPERLU = YES. Note that
  SuperLU_DIST requires ParMETIS, which includes METIS 5 in its distribution.
  Both ParMETIS and the included METIS 5 should be built and installed in the
-  same location. If using SuperLU_Dist v5, set MFEM_USE_SUPERLU5=YES.
+  same location.
  URL: http://crd-legacy.lbl.gov/~xiaoye/SuperLU
  Options: SUPERLU_OPT, SUPERLU_LIB.
  Versions: SuperLU_DIST >= 5.1.0.

- MUMPS (optional), used when MFEM_USE_MUMPS = YES. Note that MUMPS
-  requires LAPACK, SCALAPACK and a reordering package such as PORD or METIS.
-  URL: http://mumps.enseeiht.fr
-  Options: MUMPS_OPT, MUMPS_LIB.
-  Versions: MUMPS >= 5.1.1
-
 - STRUMPACK (optional), used when MFEM_USE_STRUMPACK = YES. Note that STRUMPACK
  requires the PT-Scotch and Scalapack libraries as well as ParMETIS, which
  includes METIS 5 in its distribution. Starting with STRUMPACK v2.2.0, ParMETIS
@@ -663,15 +585,13 @@ The specific libraries and their options are:
  2.0.0 or later.
  URL: http://portal.nersc.gov/project/sparse/strumpack
  Options: STRUMPACK_OPT, STRUMPACK_LIB.
-  Versions: STRUMPACK >= 3.0.0.
+  Versions: STRUMPACK >= 3.0.0, requires HYPRE < 2.16.0.

 - Ginkgo (optional), used when MFEM_USE_GINKGO = YES. Note that Ginkgo needs a
-  C++ compiler that supports the C++-14 standard. For additional requirements
-  and dependencies of specific modules, see the Ginkgo webpage below.
+  C++ compiler that supports the C++-11 standard. For additional requirements
+  and dependencies of specific modules see the Ginkgo webpage below.
  URL: https://ginkgo-project.github.io
-  Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or
-           Debug).
-  Versions: Ginkgo >= 1.4.0.
+  Options: GINKGO_OPT (Not used), GINKGO_LIB.

 - AmgX (optional), used when MFEM_USE_AMGX = YES.
  URL: https://github.com/NVIDIA/AMGX
@@ -699,15 +619,11 @@ The specific libraries and their options are:
  PETSc has been cloned on the same level as mfem and hypre:
    ./configure --download-fblaslapack=yes --download-scalapack=yes \
                --download-mumps=yes --download-suitesparse=yes \
-                --with-hypre-dir=../hypre/src/hypre \
+                --with-hypre-dir=../hypre-2.10.0b/src/hypre \
                --with-shared-libraries=0
-  When building PETSc with HIP, one may need to add a flag like -std=c2x to
-  CFLAGS to allow proper parsing of the hipsparse header under C.
  URL: https://www.mcs.anl.gov/petsc
  Options: PETSC_OPT, PETSC_LIB.
-  Versions: PETSc >= 3.8.0   (PETSc build without CUDA/HIP)
-            PETSc >= 3.15.0  (PETSc built with CUDA)
-            PETSc >= 3.19.0  (PETSc built with HIP, older versions may work too)
+  Versions: PETSc >= 3.8.0.

 - SLEPc (optional), used when MFEM_USE_SLEPC = YES. SLEPc depends on PETSc and
  uses some of the PETSc options when compiled.
@@ -732,47 +648,26 @@ The specific libraries and their options are:

 - ADIOS2 (optional) used when MFEM_USE_ADIOS2 = YES.
  URL: https://adios2.readthedocs.io/
-  Versions: ADIOS >= 2.5.0.

 - PUMI (optional), used when MFEM_USE_PUMI = YES.
  URL: https://scorec.rpi.edu/pumi
       https://github.com/SCOREC/core
  Options: PUMI_OPT, PUMI_LIB.
-  Versions: PUMI >= 2.2.6.
+  Versions: PUMI >= 2.2.3.

 - HiOp (optional), used when MFEM_USE_HIOP = YES.
  URL: https://github.com/LLNL/hiop
  Options: HIOP_OPT, HIOP_LIB.
-  Versions: HIOP >= 0.4.6.
-
- CoDiPack (optional), used with MFEM_USE_CODIPACK = YES
-  URL: https://www.scicomp.uni-kl.de/codi/
-  Options: CODIPACK_OPT
-  Versions: 1.9.3

 - GSLIB (optional), used when MFEM_USE_GSLIB = YES. The gslib library must be
-  built prior to the MFEM build, as follows: download gslib-1.0.7, untar it at
-  the same level as MFEM and create a symbolic link: "ln -s gslib-1.0.7 gslib".
+  built prior to the MFEM build, as follows: download gslib-1.0.5, untar it at
+  the same level as MFEM and create a symbolic link: "ln -s gslib-1.0.5 gslib".
  Build gslib in parallel or in serial based on the desired MFEM build: "make
  clean; make CC=mpicc" or "make clean; make CC=gcc MPI=0". Build MFEM with
  MFEM_USE_GSLIB=YES.
-  URL: https://github.com/gslib/gslib/archive/v1.0.7.tar.gz
+  URL: https://github.com/gslib/gslib/archive/v1.0.5.tar.gz
  Options: GSLIB_OPT, GSLIB_LIB.
-  Versions: GSLIB >= 1.0.7.
-
- ALGOIM (optional), used when MFEM_USE_ALGOIM=YES. The library provides only
-  headers so it just needs to be downloaded at the same level as MFEM. Download
-  the specific version we use as:
-  "git clone https://github.com/algoim/algoim.git;
-   git checkout 9c9ca0ef094d8ab0390ed36367a1151b459bbe0a"
-  ALGOIM depends on BLITZ and the library must be built prior to the MFEM build.
-  Download v1.0.2, untar it at the same level as MFEM and create a symbolic link:
-  "ln -s blitz-1.0.2 blitz".
-  Build Blitz using CMake as:
-  "cmake . -DCMAKE_INSTALL_PREFIX=.; make lib; make install"
-  URL: https://github.com/blitzpp/blitz/archive/refs/tags/1.0.2.tar.gz
-  Options: BLITZ_OPT, BLITZ_LIB
-  Versions: BLITZ = 1.0.2
+  Versions: GSLIB >= 1.0.5.

 - MKL CPardiso (optional), used when MFEM_USE_MKL_CPARDISO = YES.
  URL: https://software.intel.com/content/www/us/en/develop/tools/math-kernel-library.html
@@ -782,10 +677,10 @@ The specific libraries and their options are:
 - CUDA (optional), used when MFEM_USE_CUDA = YES.
  URL: https://developer.nvidia.com/cuda-toolkit
  Options: CUDA_CXX, CUDA_ARCH, CUDA_OPT, CUDA_LIB.
-  Versions: CUDA >= 10.1.168.
+  Versions: CUDA >= 9.1, older versions may work too.

 - HIP (optional), used when MFEM_USE_HIP = YES.
-  URL: https://rocmdocs.amd.com
+  URL: https://rocm.github.io/ROCmInstall.html
  Options: HIP_CXX, HIP_ARCH, HIP_OPT, HIP_LIB.

 - OCCA (optional), used when MFEM_USE_OCCA = YES.
@@ -797,34 +692,18 @@ The specific libraries and their options are:
  URL: https://github.com/CEED/libCEED
       https://ceed.exascaleproject.org/libceed
  Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED >= 0.10.
+  Versions: libCEED >= 0.7.

 - RAJA (optional), used when MFEM_USE_RAJA = YES.
-  Beginning with MFEM v4.5.1, only RAJA v2022.10.3+ is supported.
+  Beginning with MFEM v4.1, only RAJA v0.10.0+ is supported.
  URL: https://github.com/LLNL/RAJA
  Options: RAJA_DIR, RAJA_OPT, RAJA_LIB.
-  Versions: RAJA >= 2022.10.3.
-
- Moonolith (optional), use when MFEM_USE_MOONOLITH = YES.
-  URL: https://bitbucket.org/zulianp/par_moonolith
-  Options: MOONOLITH_DIR
-  Versions: MOONOLITH >= 1.1.0.
-
- Caliper (optional), used when MFEM_USE_CALIPER = YES.
-  URL: https://github.com/LLNL/Caliper
-  Options: CALIPER_DIR
-  Versions: CALIPER >= 2.5.0, older versions may work too.
+  Versions: RAJA >= 0.10.0.

 - Umpire, used when MFEM_USE_UMPIRE = YES.
-  Umpire requires camp when the Umpire version is >= 3.0.0.
  URL: https://github.com/LLNL/Umpire
  Options: UMPIRE_DIR, UMPIRE_OPT, UMPIRE_LIB.
-  Versions: Umpire >= 3.0.0.
-
- Benchmark, used when MFEM_USE_BENCHMARK = YES.
-  URL: https://github.com/google/benchmark
-  Options: BENCHMARK_DIR, BENCHMARK_LIB.
-  Versions: Benchmark >= 1.5.6.
+  Versions: Umpire >= 2.0.0.

 - MPFR (optional), used when MFEM_USE_MPFR = YES.
  URL: http://mpfr.org, it depends on the GMP library: https://gmplib.org
@@ -841,21 +720,6 @@ The specific libraries and their options are:
  URL: https://zlib.net
  Options: ZLIB_OPT, ZLIB_LIB.

- FMS (optional), used when MFEM_USE_FMS = YES.
-  URL: https://github.com/CEED/FMS
-  Options: FMS_OPT, FMS_LIB.
-  Versions: FMS >= 0.2.
-
- ParELAG, used when MFEM_USE_PARELAG = YES.
-  URL: https://github.com/LLNL/parelag
-  Options: PARELAG_DIR, PARELAG_OPT, PARELAG_LIB.
-
- Enzyme, used when MFEM_USE_ENZYME = YES. Requires LLVM/Clang >= 14.0.0.
-  URL: https://github.com/EnzymeAD/Enzyme
-  Options: ENZYME_DIR, ENZYME_OPT, ENZYME_LIB.
-  Versions: Enzyme >= v0.0.33.
-
-
 Building with CMake
 ===================
 The MFEM build system consists of two steps: configuration and compilation.
@@ -967,9 +831,9 @@ MFEM_USE_LEGACY_OPENMP
 MFEM_USE_OPENMP
 MFEM_USE_MEMALLOC
 MFEM_TIMER_TYPE - Set automatically, can be overwritten.
+MFEM_USE_MESQUITE
 MFEM_USE_SUITESPARSE
 MFEM_USE_SUPERLU
-MFEM_USE_MUMPS
 MFEM_USE_STRUMPACK
 MFEM_USE_GINKGO
 MFEM_USE_AMGX
@@ -979,21 +843,12 @@ MFEM_USE_MPFR
 MFEM_USE_ZLIB
 MFEM_USE_PUMI
 MFEM_USE_HIOP
-MFEM_USE_CODIPACK
-MFEM_USE_ADFORWARD
 MFEM_USE_CUDA
-MFEM_USE_HIP
 MFEM_USE_OCCA
 MFEM_USE_CEED
 MFEM_USE_RAJA
 MFEM_USE_UMPIRE
 MFEM_USE_SIDRE
-MFEM_USE_MOONOLITH
-MFEM_USE_CALIPER
-MFEM_USE_FMS
-MFEM_USE_BENCHMARK
-MFEM_USE_PARELAG
-MFEM_USE_ENZYME

 The following options are CMake specific:

@@ -1031,6 +886,7 @@ The CMake build system adds auto-detection for the following packages/libraries:
 - HYPRE
 - METIS - The option MFEM_USE_METIS_5 is auto-detected.
 - ParMETIS
+ - MESQUITE
 - SuiteSparse
 - SuperLUDist, STRUMPACK
 - Ginkgo
@@ -1042,17 +898,10 @@ The CMake build system adds auto-detection for the following packages/libraries:
 - POSIXCLOCKS
 - PUMI
 - HIOP
- - CoDiPack
 - OCCA
 - RAJA
 - UMPIRE
 - AXOM - Used when MFEM_USE_SIDRE is enabled
- - MOONOLITH
- - CALIPER
- - FMS
- - BENCHMARK
- - ParELAG
- - Enzyme

 The following built-in CMake packages are also used:

@@ -1070,7 +919,7 @@ config/config.hpp.in:

   cp config/config.hpp.in config/_config.hpp

-The file config/_config.hpp can then be edited to enable desired options. The
+The file config/_config.hpp can then be edited to enable desired options.  The
 MFEM library is simply a combination of all object files obtained by compiling
 the .cpp source files in the source directories: general, linalg, mesh, and fem.

@@ -1078,7 +927,7 @@ the .cpp source files in the source directories: general, linalg, mesh, and fem.
 Specifying an MPI job launcher
 ==============================
 By default, MFEM will use 'mpirun -np #' to launch any of its parallel tests or
-miniapps, where # is the number of MPI tasks. An alternate MPI launcher can be
+miniapps, where # is the number of MPI tasks.  An alternate MPI launcher can be
 provided by setting the MFEM_MPIEXEC and MFEM_MPIEXEC_NP config variables.

 MFEM will expect the launcher command, plus the command line option to allow it
@@ -1088,19 +937,3 @@ MFEM_MPIEXEC    = mpirun # default
 MFEM_MPIEXEC_NP = -np    # default
 MFEM_MPIEXEC    = srun   # example for platforms using SLURM
 MFEM_MPIEXEC_NP = -n     # example for platforms using SLURM
-
-
-Specific options for hypre
-==========================
-The hypre library has multiple options to define local and global index storage
-sizes. By default, all indices are stored as an architecture aware integer. For
-most platforms, this will be 32-bit. This limits the maximum number of global
-degrees of freedom in a vector or matrix to about 2 billion. In order to solve
-larger problems, there are two options:
-
-1. Building hypre with '--enable-bigint' defines the local and global indices to
-   be 64-bit. This is convenient, but requires more memory than necessary.
-
-2. Building hypre with '--enable-mixedint' defines the local indiced to be
-   32-bit, while using a 64-bit storage for global indices. This option is
-   currently tested only in ex1p, and may not work in more general settings.
@@ -1,6 +1,6 @@
 BSD 3-Clause License

-Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC
+Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC
 All rights reserved.

 Redistribution and use in source and binary forms, with or without
@@ -29,5 +29,4 @@ license files. These software products and their licenses are as follows:
 * Catch++ (tests/unit/catch.hpp) -- Boost 1.0 license
 * Gecko (general/gecko.{cpp,hpp}) -- BSD 3-clause license
 * Picojson (fem/picojson.h) -- Custom 2-clause license
-* TinyXML2 (general/tinyxml2.{cpp,h}) -- zlib license
 * Zstr (general/zstr.hpp) -- MIT license
@@ -5,25 +5,22 @@
                      | | | | | ||  _||  __/| | | | | |
                      |_| |_| |_||_|   \___||_| |_| |_|

-                               https://mfem.org
+                               http://mfem.org

-[MFEM](https://mfem.org) is a modular parallel C++ library for finite element
-methods. Its goal is to enable high-performance scalable finite element
-discretization research and application development on a wide variety of
-platforms, ranging from laptops to supercomputers.
+MFEM is a modular parallel C++ library for finite element methods. Its goal is
+to enable high-performance scalable finite element discretization research and
+application development on a wide variety of platforms, ranging from laptops to
+supercomputers.

-We welcome contributions and feedback from the community. Please see the file
-[CONTRIBUTING.md](CONTRIBUTING.md) for additional details about our development
-process.
+* For building instructions, see the file INSTALL, or type "make help".

-* For building instructions, see the file [INSTALL](INSTALL), or type "make help".
+* Copyright and licensing information can be found in files LICENSE and NOTICE.

-* Copyright and licensing information can be found in files [LICENSE](LICENSE) and [NOTICE](NOTICE).
+* The best starting point for new users interested in MFEM's features is the
+  interactive documentation in examples/README.html.

-* The best starting point for new users interested in MFEM's features is to
-  review the examples and miniapps at https://mfem.org/examples.
-
-* Instructions for learning with Docker are in [config/docker](config/docker).
+* Developers interested in contributing to the library, should read the
+  instructions and documentation in the CONTRIBUTING.md file.

 Conceptually, MFEM can be viewed as a finite element toolbox that provides the
 building blocks for developing finite element algorithms in a manner similar to
@@ -52,25 +49,21 @@ library. Nonlinear solvers (the Newton method), eigensolvers (LOBPCG), and
 several explicit and implicit Runge-Kutta time integrators are also available.

 MFEM supports MPI-based parallelism throughout the library, and can readily be
-used as a scalable unstructured finite element problem generator. Starting with
-version 4.0, MFEM offers support for GPU acceleration, and programming models,
-such as CUDA, HIP, OCCA, RAJA and OpenMP. MFEM-based applications require
-minimal changes to switch from a serial to a highly-performant MPI-parallel
-version of the code, where they can take advantage of the integrated linear
-solvers from the hypre library. Comprehensive support for other external
-packages, e.g. PETSc, SUNDIALS and libCEED is also included, giving access to
-additional linear and nonlinear solvers, preconditioners, time integrators, etc.
+used as a scalable unstructured finite element problem generator. As of version
+4.0, MFEM offers initial support for GPU acceleration, and programming models,
+such as CUDA, OCCA, RAJA and OpenMP. MFEM-based applications require minimal
+changes to switch from a serial to a high-performing MPI-parallel version of the
+code, where they can take advantage of the integrated linear solvers from the
+hypre library.  Comprehensive support for other external packages, e.g. PETSc
+and SUNDIALS is also included, giving access to many additional linear and
+nonlinear solvers, preconditioners, time integrators, etc.

-For examples of using MFEM, see the [examples/](examples) and [miniapps/](miniapps)
-directories, as well as the OpenGL visualization tool GLVis which is available
-at https://glvis.org.
-
-## License
+For examples of using MFEM, see the examples/ and miniapps/ directories, as well
+as the OpenGL visualization tool GLVis which is available at http://glvis.org.

 MFEM is distributed under the terms of the BSD-3 license. All new contributions
-must be made under this license. See [LICENSE](LICENSE) and [NOTICE](NOTICE) for
-details.
+must be made under this license. See LICENSE and NOTICE for details.

-SPDX-License-Identifier: BSD-3-Clause <br>
-LLNL Release Number: LLNL-CODE-806117 <br>
+SPDX-License-Identifier: BSD-3-Clause
+LLNL Release Number: LLNL-CODE-806117
 DOI: 10.11578/dc.20171025.1248
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -212,6 +212,10 @@ IF (DEFINED TPL_ENABLE_SUNDIALS)
  SET(MFEM_USE_SUNDIALS ${TPL_ENABLE_SUNDIALS} CACHE BOOL "Enable SUNDIALS usage" FORCE)
 ENDIF()

+IF (DEFINED TPL_ENABLE_MESQUITE)
+  SET(MFEM_USE_MESQUITE ${TPL_ENABLE_MESQUITE} CACHE BOOL "Enable MESQUITE usage" FORCE)
+ENDIF()
+
 IF (DEFINED TPL_ENABLE_SUITESPARSE)
  SET(MFEM_USE_SUITESPARSE ${TPL_ENABLE_SUITESPARSE} CACHE BOOL "Enable SuiteSparse usage" FORCE)
 ENDIF()
@@ -252,10 +256,6 @@ IF (DEFINED TPL_ENABLE_SIDRE)
  SET(MFEM_USE_SIDRE ${TPL_ENABLE_SIDRE} CACHE BOOL "Enable Axom/Sidre usage" FORCE)
 ENDIF()

-IF (DEFINED TPL_ENABLE_FMS)
-  SET(MFEM_USE_FMS ${TPL_ENABLE_FMS} CACHE BOOL "Enable FMS usage" FORCE)
-ENDIF()
-
 IF (DEFINED TPL_ENABLE_CONDUIT)
  SET(MFEM_USE_CONDUIT ${TPL_ENABLE_CONDUIT} CACHE BOOL "Enable Conduit usage" FORCE)
 ENDIF()
@@ -279,11 +279,3 @@ ENDIF()
 IF (DEFINED TPL_ENABLE_UMPIRE)
  SET(MFEM_USE_UMPIRE ${TPL_ENABLE_UMPIRE} CACHE BOOL "Enable Umpire" FORCE)
 ENDIF()
-
-IF (DEFINED TPL_ENABLE_BENCHMARK)
-  SET(MFEM_USE_BENCHMARK ${TPL_ENABLE_BENCHMARK} CACHE BOOL "Enable Google-Benchmark" FORCE)
-ENDIF()
-
-IF (DEFINED TPL_ENABLE_PARELAG)
-  SET(MFEM_USE_PARELAG ${TPL_ENABLE_PARELAG} CACHE BOOL "Enable ParELAG" FORCE)
-ENDIF()
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -29,13 +29,12 @@ set(MFEM_USE_LEGACY_OPENMP @MFEM_USE_LEGACY_OPENMP@)
 set(MFEM_USE_MEMALLOC @MFEM_USE_MEMALLOC@)
 set(MFEM_TIMER_TYPE @MFEM_TIMER_TYPE@)
 set(MFEM_USE_SUNDIALS @MFEM_USE_SUNDIALS@)
+set(MFEM_USE_MESQUITE @MFEM_USE_MESQUITE@)
 set(MFEM_USE_SUITESPARSE @MFEM_USE_SUITESPARSE@)
 set(MFEM_USE_SUPERLU @MFEM_USE_SUPERLU@)
-set(MFEM_USE_MUMPS @MFEM_USE_MUMPS@)
 set(MFEM_USE_STRUMPACK @MFEM_USE_STRUMPACK@)
 set(MFEM_USE_GINKGO @MFEM_USE_GINKGO@)
 set(MFEM_USE_AMGX @MFEM_USE_AMGX@)
-set(MFEM_USE_HIOP @MFEM_USE_HIOP@)
 set(MFEM_USE_GNUTLS @MFEM_USE_GNUTLS@)
 set(MFEM_USE_GSLIB @MFEM_USE_GSLIB@)
 set(MFEM_USE_NETCDF @MFEM_USE_NETCDF@)
@@ -43,7 +42,6 @@ set(MFEM_USE_PETSC @MFEM_USE_PETSC@)
 set(MFEM_USE_SLEPC @MFEM_USE_SLEPC@)
 set(MFEM_USE_MPFR @MFEM_USE_MPFR@)
 set(MFEM_USE_SIDRE @MFEM_USE_SIDRE@)
-set(MFEM_USE_FMS @MFEM_USE_FMS@)
 set(MFEM_USE_CONDUIT @MFEM_USE_CONDUIT@)
 set(MFEM_USE_PUMI @MFEM_USE_PUMI@)
 set(MFEM_USE_CUDA @MFEM_USE_CUDA@)
@@ -53,16 +51,6 @@ set(MFEM_USE_CEED @MFEM_USE_CEED@)
 set(MFEM_USE_UMPIRE @MFEM_USE_UMPIRE@)
 set(MFEM_USE_SIMD @MFEM_USE_SIMD@)
 set(MFEM_USE_ADIOS2 @MFEM_USE_ADIOS2@)
-set(MFEM_USE_MOONOLITH @MFEM_USE_MOONOLITH@)
-set(MFEM_USE_CODIPACK @MFEM_USE_CODIPACK@)
-set(MFEM_USE_MKL_CPARDISO @MFEM_USE_MKL_CPARDISO@)
-set(MFEM_USE_MKL_PARDISO @MFEM_USE_MKL_PARDISO@)
-set(MFEM_USE_ADFORWARD @MFEM_USE_ADFORWARD@)
-set(MFEM_USE_CALIPER @MFEM_USE_CALIPER@)
-set(MFEM_USE_ALGOIM @MFEM_USE_ALGOIM@)
-set(MFEM_USE_BENCHMARK @MFEM_USE_BENCHMARK@)
-set(MFEM_USE_PARELAG @MFEM_USE_PARELAG@)
-set(MFEM_USE_ENZYME @MFEM_USE_ENZYME@)

 set(MFEM_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
 set(MFEM_CXX_FLAGS "@CMAKE_CXX_FLAGS@")
@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -39,9 +39,6 @@
 // Description of the git commit used to build MFEM.
 #cmakedefine MFEM_GIT_STRING "@MFEM_GIT_STRING@"

-// Enable shared library build of MFEM.
-#cmakedefine MFEM_SHARED_BUILD
-
 // Build the parallel MFEM library.
 // Requires an MPI compiler, and the libraries HYPRE and METIS.
 #cmakedefine MFEM_USE_MPI
@@ -80,102 +77,81 @@
 // Internal MFEM option: enable group/batch allocation for some small objects.
 #cmakedefine MFEM_USE_MEMALLOC

-// Which library functions to use in class StopWatch for measuring time.
-// For a list of the available options, see INSTALL.
-// If not defined, an option is selected automatically.
-#cmakedefine MFEM_TIMER_TYPE @MFEM_TIMER_TYPE@
-
-// Enable MFEM functionality based on the SUNDIALS libraries.
-#cmakedefine MFEM_USE_SUNDIALS
+// Enable MFEM functionality based on the Mesquite library.
+#cmakedefine MFEM_USE_MESQUITE

 // Enable MFEM functionality based on the SuiteSparse library.
 #cmakedefine MFEM_USE_SUITESPARSE

 // Enable MFEM functionality based on the SuperLU_DIST library.
 #cmakedefine MFEM_USE_SUPERLU
-#cmakedefine MFEM_USE_SUPERLU5
-
-// Enable MFEM functionality based on the MUMPS library.
-#cmakedefine MFEM_USE_MUMPS
-#cmakedefine MFEM_MUMPS_VERSION @MFEM_MUMPS_VERSION@

 // Enable MFEM functionality based on the STRUMPACK library.
 #cmakedefine MFEM_USE_STRUMPACK

-// Enable functionality based on the Ginkgo library.
+// Enable functionality based on the Ginkgo library
 #cmakedefine MFEM_USE_GINKGO

-// Enable MFEM functionality based on the AmgX library.
+// Enable MFEM functionality based on the AmgX library
 #cmakedefine MFEM_USE_AMGX

-// Enable secure socket streams based on the GNUTLS library.
+// Enable MFEM functionality based on the GnuTLS library
 #cmakedefine MFEM_USE_GNUTLS

-// Enable Sidre support.
-#cmakedefine MFEM_USE_SIDRE
-
-// Enable the use of SIMD in the high performance templated classes.
-#cmakedefine MFEM_USE_SIMD
-
-// Enable FMS support.
-#cmakedefine MFEM_USE_FMS
-
-// Enable Conduit support.
-#cmakedefine MFEM_USE_CONDUIT
-
-// Enable functionality based on the NetCDF library (reading CUBIT files).
-#cmakedefine MFEM_USE_NETCDF
-
-// Enable functionality based on the PETSc library.
-#cmakedefine MFEM_USE_PETSC
-
-// Enable functionality based on the SLEPc library.
-#cmakedefine MFEM_USE_SLEPC
-
-// Enable functionality based on the MPFR library.
-#cmakedefine MFEM_USE_MPFR
-
-// Enable MFEM functionality based on the PUMI library.
-#cmakedefine MFEM_USE_PUMI
-
-// Enable Moonolith-based general interpolation between finite element spaces.
-#cmakedefine MFEM_USE_MOONOLITH
-
-// Enable MFEM functionality based on the HIOP library.
-#cmakedefine MFEM_USE_HIOP
-
-// Enable MFEM functionality based on the GSLIB library.
+// Enable MFEM functionality based on the GSLIB library
 #cmakedefine MFEM_USE_GSLIB

-// Build the NVIDIA GPU/CUDA-enabled version of the MFEM library.
+// Enable MFEM functionality based on the NetCDF library
+#cmakedefine MFEM_USE_NETCDF
+
+// Enable MFEM functionality based on the PETSc library
+#cmakedefine MFEM_USE_PETSC
+
+// Enable MFEM functionality based on the SLEPc library
+#cmakedefine MFEM_USE_SLEPC
+
+// Enable MFEM functionality based on the Sidre library
+#cmakedefine MFEM_USE_SIDRE
+
+// Enable the use of SIMD in the high performance templated classes
+#cmakedefine MFEM_USE_SIMD
+
+// Enable MFEM functionality based on Conduit
+#cmakedefine MFEM_USE_CONDUIT
+
+// Enable MFEM functionality based on the PUMI library
+#cmakedefine MFEM_USE_PUMI
+
+// Enable MFEM functionality based on the HiOp library
+#cmakedefine MFEM_USE_HIOP
+
+// Build the GPU/CUDA-enabled version of the MFEM library.
 // Requires a CUDA compiler (nvcc).
 #cmakedefine MFEM_USE_CUDA

-// Build the AMD GPU/HIP-enabled version of the MFEM library.
-// Requires a HIP compiler (hipcc).
-#cmakedefine MFEM_USE_HIP
-
-// Enable functionality based on the RAJA library.
+// Enable MFEM functionality based on the RAJA library
 #cmakedefine MFEM_USE_RAJA

-// Enable functionality based on the OCCA library.
+// Enable MFEM functionality based on the OCCA library
 #cmakedefine MFEM_USE_OCCA

-// Enable functionality based on the libCEED library.
+// Enable MFEM functionality based on the libCEED library
 #cmakedefine MFEM_USE_CEED

-// Enable functionality based on the Caliper library.
-#cmakedefine MFEM_USE_CALIPER
-
-// Enable functionality based on the Algoim library.
-#cmakedefine MFEM_USE_ALGOIM
-
-// Enable functionality based on the Umpire library.
+// Enable MFEM functionality based on the Umpire library
 #cmakedefine MFEM_USE_UMPIRE

-// Enable IO functionality based on the ADIOS2 library.
+// Enable MFEM functionality based on the ADIOS2 library
 #cmakedefine MFEM_USE_ADIOS2

+// Which library functions to use in class StopWatch for measuring time.
+// For a list of the available options, see INSTALL.
+// If not defined, an option is selected automatically.
+#define MFEM_TIMER_TYPE @MFEM_TIMER_TYPE@
+
+// Enable MFEM functionality based on the SUNDIALS libraries.
+#cmakedefine MFEM_USE_SUNDIALS
+
 // Version of HYPRE used for building MFEM.
 #cmakedefine MFEM_HYPRE_VERSION @MFEM_HYPRE_VERSION@

@@ -186,19 +162,4 @@
 // Enable interface to the MKL CPardiso library.
 #cmakedefine MFEM_USE_MKL_CPARDISO

-// Enable interface to the MKL Pardiso library.
-#cmakedefine MFEM_USE_MKL_PARDISO
-
-// Use forward mode for automatic differentiation.
-#cmakedefine MFEM_USE_ADFORWARD
-
-// Enable the use of the CoDiPack library for AD.
-#cmakedefine MFEM_USE_CODIPACK
-
-// Enable functionality based on the Google Benchmark library.
-#cmakedefine MFEM_USE_BENCHMARK
-
-// Enable Enzyme for AD
-#cmakedefine MFEM_USE_ENZYME
-
 #endif // MFEM_CONFIG_HEADER
@@ -38,19 +38,19 @@ if(NOT ADIOS2_FOUND)
  endif()

  find_path(ADIOS2_INCLUDE_DIR adios2.h ${ADIOS2_INCLUDE_OPTS})
-
-  # adios2 version 2.5.0
+  
+  # adios2 version 2.5.0 
  find_library(ADIOS2_LIBRARY NAMES adios2 ${ADIOS2_LIBRARY_OPTS})
-
+  
  # adios2 version 2.6.0 and onwards
  if(NOT ADIOS2_LIBRARY)
-	find_library(ADIOS2_CXX11_MPI_LIBRARY NAMES adios2_cxx11_mpi ${ADIOS2_LIBRARY_OPTS})
-	find_library(ADIOS2_CXX11_LIBRARY NAMES adios2_cxx11 ${ADIOS2_LIBRARY_OPTS})
-	set(ADIOS2_LIBRARY ${ADIOS2_CXX11_MPI_LIBRARY} ${ADIOS2_CXX11_LIBRARY})
-	if(MFEM_USE_MPI)
-	  add_definitions(-DADIOS2_USE_MPI)
-	endif()
-  endif()
+  	find_library(ADIOS2_CXX11_MPI_LIBRARY NAMES adios2_cxx11_mpi ${ADIOS2_LIBRARY_OPTS})
+  	find_library(ADIOS2_CXX11_LIBRARY NAMES adios2_cxx11 ${ADIOS2_LIBRARY_OPTS})
+  	set(ADIOS2_LIBRARY ${ADIOS2_CXX11_MPI_LIBRARY} ${ADIOS2_CXX11_LIBRARY})
+  	if(MFEM_USE_MPI)
+  	  add_definitions(-DADIOS2_USE_MPI)
+  	endif()
+  endif() 

  include(FindPackageHandleStandardArgs)
  find_package_handle_standard_args(ADIOS2
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -15,10 +15,6 @@
 #   - AMGX_INCLUDE_DIRS

 include(MfemCmakeUtilities)
-set(AMGX_REQUIRED_LIBRARIES cusparse cusolver cublas cublasLt nvToolsExt)
+set(AMGX_REQUIRED_LIBRARIES cusparse cusolver cublas nvToolsExt)
 mfem_find_package(AMGX AMGX AMGX_DIR "include" "amgx_c.h" "lib" "amgx"
  "Paths to headers required by AMGX." "Libraries required by AMGX.")
-# Make sure the library location is locked down
-foreach(lib ${AMGX_REQUIRED_LIBRARIES})
-  list(APPEND AMGX_LIBRARIES ${CUDA_TOOLKIT_ROOT_DIR}/lib64/lib${lib}${CMAKE_SHARED_LIBRARY_SUFFIX})
-endforeach()
@@ -1,22 +0,0 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - ALGOIM_FOUND
-#   - ALGOIM_LIBRARIES
-#   - ALGOIM_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(Algoim ALGOIM ALGOIM_DIR
-        "include;src" "algoim_quad.hpp"
-       	"" ""
-        "Paths to headers required by Algoim."
-        "Libraries required by Algoim.")
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,22 +0,0 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - BENCHMARK_FOUND
-#   - BENCHMARK_LIBRARIES
-#   - BENCHMARK_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(Benchmark BENCHMARK BENCHMARK_DIR
-        "include" "benchmark/benchmark.h"
-        "lib" "benchmark"
-        "Paths to headers required by Google Benchmark."
-        "Libraries required by Google Benchmark.")
@@ -1,24 +0,0 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Automatic differentiation using the CoDiPack library.
-#  www.scicomp.uni-kl.de/codi/
-# Sets the following variables:
-#   - CODIPACK_FOUND
-#   - CODIPACK_INCLUDE_DIRS
-#   - CODIPACK_LIBRARIES
-
-include(MfemCmakeUtilities)
-mfem_find_package(CODIPACK CODIPACK CODIPACK_DIR
-  "include" "codi.h"
-  "lib" ""
-  "Paths to headers required by CODIPACK."
-  "Libraries required by CODIPACK.")
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,37 +0,0 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - CALIPER_FOUND
-#   - CALIPER_LIBRARIES
-#   - CALIPER_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(Caliper CALIPER CALIPER_DIR
-      "include" "caliper/cali.h"
-      "lib" "caliper"
-      "Paths to headers required by Caliper."
-      "Libraries required by Caliper.")
-
-# Append adiak path/lib if the user provided ADIAK_DIR
-if(ADIAK_DIR AND EXISTS ${ADIAK_DIR})
-    find_package(adiak NO_DEFAULT_PATH REQUIRED PATHS ${ADIAK_DIR}/lib/cmake/adiak ${ADIAK_DIR})
-    list(APPEND CALIPER_INCLUDE_DIRS ${adiak_INCLUDE_DIRS})
-    list(APPEND CALIPER_LIBRARIES ${adiak_LIBRARIES})
-endif()
-
-# Append gotcha path/lib if the user provided GOTCHA_DIR
-if(GOTCHA_DIR AND EXISTS ${GOTCHA_DIR})
-    find_package(gotcha NO_DEFAULT_PATH REQUIRED PATHS ${GOTCHA_DIR}/lib/cmake/gotcha ${GOTCHA_DIR})
-    list(APPEND CALIPER_INCLUDE_DIRS ${gotcha_INCLUDE_DIRS})
-    list(APPEND CALIPER_LIBRARIES ${gotcha_LIBRARIES})
-endif()
-
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -21,15 +21,8 @@ if(EXISTS ${CONDUIT_DIR}/include/conduit/conduit_relay_hdf5.hpp)
  # we only need  HDF5 if Conduit was built with HDF5 support
  set(Conduit_REQUIRED_PACKAGES "HDF5" CACHE STRING
      "Additional packages required by Conduit.")
-  # Suppress warning about HDF5_ROOT being set
-  if (POLICY CMP0074)
-    cmake_policy(SET CMP0074 NEW)
-  endif()
-  # HDF5_ROOT is needed in some cases, e.g. when HDF5_TARGET_NAMES is set and
-  # MFEM's FindHDF5.cmake is not used.
-  set(HDF5_ROOT ${HDF5_DIR} CACHE PATH "")
-else()
-  message(STATUS "Conduit Relay HDF5 Support is DISABLED")
+  else()
+    message(STATUS "Conduit Relay HDF5 Support is DISABLED")
 endif()

 include(MfemCmakeUtilities)
@@ -39,4 +32,4 @@ mfem_find_package(Conduit CONDUIT CONDUIT_DIR
  ADD_COMPONENT relay
    "include;include/conduit" conduit_relay.hpp "lib" conduit_relay
  ADD_COMPONENT blueprint
-    "include;include/conduit" conduit_blueprint.hpp "lib" conduit_blueprint)
+      "include;include/conduit" conduit_blueprint.hpp "lib" conduit_blueprint)
@@ -1,27 +0,0 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-message(STATUS "Looking for ENZYME ...")
-message(STATUS "   in ENZYME_DIR = ${ENZYME_DIR}")
-
-# Make sure the directory and version combination works. Do nothing otherwise.
-if(EXISTS "${ENZYME_DIR}/ClangEnzyme-${ENZYME_VERSION}.so")
-  message(STATUS "Found ENZYME: ${ENZYME_DIR}/ClangEnzyme-${ENZYME_VERSION}.so")
-
-  # Set ENZYME_FOUND
-  set(ENZYME_FOUND TRUE CACHE BOOL "ENZYME was found." FORCE)
-
-  # Set CXX flags to accommodate the Enzyme Clang plugin
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -load -Xclang ${ENZYME_DIR}/ClangEnzyme-${ENZYME_VERSION}.so -mllvm -enzyme-loose-types=1")
-  set(MFEM_USE_ENZYME YES)
-else()
-
-endif()
@@ -1,20 +0,0 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - FMS_FOUND
-#   - FMS_LIBRARIES
-#   - FMS_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(FMS FMS FMS_DIR
-  "include" fms.h "lib" fms
-  "Paths to headers required by FMS." "Libraries required by FMS.")
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -15,5 +15,5 @@
 #   - GSLIB_INCLUDE_DIRS

 include(MfemCmakeUtilities)
-mfem_find_package(GSLIB GSLIB GSLIB_DIR "include" gslib.h "lib" gs
+mfem_find_package(gslib GSLIB GSLIB_DIR "include" gslib.h "lib" gs
  "Paths to headers required by GSLIB." "Libraries required by GSLIB.")
@@ -1,70 +0,0 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - HDF5_FOUND - If HDF5 was found
-#   - HDF5_LIBRARIES - The HDF5 libraries
-#   - HDF5_INCLUDE_DIRS - The HDF5 include directories
-
-# NOTE: Using this FindHDF5.cmake instead of the CMake provided version may lead
-#       to issues with some TPL libraries that depend (or may depend) on HDF5.
-#       For this reason, we should consider removing this file, or at least
-#       making it use the CMake provided version by default and apply the logic
-#       below only when specifically requested by a user.  -V. Dobrev
-
-# First Check for HDF5_DIR
-if(NOT HDF5_DIR)
-  message(FATAL_ERROR
-    "Could not find HDF5. HDF5 support needs explicit HDF5_DIR")
-endif()
-
-if (NOT HDF5_FIND_QUIETLY)
-  message(STATUS "Looking for HDF5 ...")
-  message(STATUS "   in HDF5_DIR = ${HDF5_DIR}")
-endif()
-
-# Find includes
-find_path( HDF5_INCLUDE_DIRS hdf5.h
-           PATHS  ${HDF5_DIR}/include/
-           NO_DEFAULT_PATH
-           NO_CMAKE_ENVIRONMENT_PATH
-           NO_CMAKE_PATH
-           NO_SYSTEM_ENVIRONMENT_PATH
-           NO_CMAKE_SYSTEM_PATH)
-
-find_library( __HDF5_LIBRARY NAMES hdf5 libhdf5 libhdf5_D libhdf5_debug
-              PATHS ${HDF5_DIR}/lib
-              NO_DEFAULT_PATH
-              NO_CMAKE_ENVIRONMENT_PATH
-              NO_CMAKE_PATH
-              NO_SYSTEM_ENVIRONMENT_PATH
-              NO_CMAKE_SYSTEM_PATH)
-
-find_library( __HDF5_HL_LIBRARY NAMES hdf5_hl libhdf5_hl libhdf5_hl_D libhdf5_hl_debug
-              PATHS ${HDF5_DIR}/lib
-              NO_DEFAULT_PATH
-              NO_CMAKE_ENVIRONMENT_PATH
-              NO_CMAKE_PATH
-              NO_SYSTEM_ENVIRONMENT_PATH
-              NO_CMAKE_SYSTEM_PATH)
-
-set(HDF5_LIBRARIES ${__HDF5_HL_LIBRARY} ${__HDF5_LIBRARY})
-
-include(FindPackageHandleStandardArgs)
-
-# Handle the QUIETLY and REQUIRED arguments and set HDF5_FOUND to TRUE if all
-# listed variables are TRUE
-find_package_handle_standard_args(HDF5
-                                  " *** HDF5 not found. Please set HDF5_DIR."
-                                  HDF5_LIBRARIES
-                                  HDF5_INCLUDE_DIRS
-                                  __HDF5_LIBRARY
-                                  __HDF5_HL_LIBRARY)
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -14,33 +14,10 @@
 #   - HYPRE_LIBRARIES
 #   - HYPRE_INCLUDE_DIRS
 #   - HYPRE_VERSION
-#   - HYPRE_USING_HIP (internal)
-
-if (HYPRE_FOUND)
-  if (HYPRE_USING_HIP)
-    find_package(rocsparse REQUIRED)
-    find_package(rocrand REQUIRED)
-  endif()
-  return()
-endif()

 include(MfemCmakeUtilities)
 mfem_find_package(HYPRE HYPRE HYPRE_DIR "include" "HYPRE.h" "lib" "HYPRE"
-  "Paths to headers required by HYPRE." "Libraries required by HYPRE."
-  CHECK_BUILD HYPRE_USING_HIP FALSE
-  "
-#undef HYPRE_USING_HIP
-#include <HYPRE_config.h>
-
-#ifndef HYPRE_USING_HIP
-#error HYPRE is built without HIP.
-#endif
-
-int main()
-{
-   return 0;
-}
-")
+  "Paths to headers required by HYPRE." "Libraries required by HYPRE.")

 if (HYPRE_FOUND AND (NOT HYPRE_VERSION))
  try_run(HYPRE_VERSION_RUN_RESULT HYPRE_VERSION_COMPILE_RESULT
@@ -56,12 +33,3 @@ if (HYPRE_FOUND AND (NOT HYPRE_VERSION))
    message(FATAL_ERROR "Unable to determine HYPRE version.")
  endif()
 endif()
-
-if (HYPRE_FOUND AND HYPRE_USING_HIP)
-  find_package(rocsparse REQUIRED)
-  find_package(rocrand REQUIRED)
-  list(APPEND HYPRE_LIBRARIES ${rocsparse_LIBRARIES} ${rocrand_LIBRARIES})
-  set(HYPRE_LIBRARIES ${HYPRE_LIBRARIES} CACHE STRING
-      "HYPRE libraries + dependencies." FORCE)
-  message(STATUS "Updated HYPRE_LIBRARIES: ${HYPRE_LIBRARIES}")
-endif()
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -25,16 +25,16 @@ mfem_find_package(METIS METIS METIS_DIR "include;Lib" "metis.h"

 int main()
 {
-    idx_t n = 10;
-    idx_t nparts = 5;
-    idx_t edgecut;
-    idx_t* partitioning = new idx_t[10];
-    idx_t* I = partitioning,
+    int n = 10;
+    int nparts = 5;
+    int edgecut;
+    int* partitioning = new int[10];
+    int* I = partitioning,
       * J = partitioning;

-    idx_t ncon = 1;
+    int ncon = 1;
    int err;
-    idx_t options[40];
+    int options[40];

    METIS_SetDefaultOptions(options);
    options[10] = 1; // set METIS_OPTION_CONTIG
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,27 +0,0 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - MKL_PARDISO_FOUND
-#   - MKL_PARDISO_LIBRARIES
-#   - MKL_PARDISO_INCLUDE_DIRS
-
-if(NOT MKL_LIBRARY_DIR)
-  message(WARNING "Using default MKL library path. Double check the variable MKL_LIBRARY_DIR")
-  set(MKL_LIBRARY_DIR "lib/intel64")
-endif()
-
-include(MfemCmakeUtilities)
-mfem_find_package(MKL_PARDISO MKL_PARDISO
-    MKL_PARDISO_DIR "include" mkl_pardiso.h ${MKL_LIBRARY_DIR} mkl_core
-  "Paths to headers required by MKL Pardiso." "Libraries required by MKL PARDISO."
-  ADD_COMPONENT MKL_LP64 "include" "" ${MKL_LIBRARY_DIR} mkl_intel_lp64
-  ADD_COMPONENT MKL_SEQUENTIAL "include" "" ${MKL_LIBRARY_DIR} mkl_sequential)
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,39 +0,0 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Sets the following variables:
-#   - MUMPS_FOUND
-#   - MUMPS_LIBRARIES
-#   - MUMPS_INCLUDE_DIRS
-#   - MUMPS_VERSION
-
-include(MfemCmakeUtilities)
-mfem_find_package(MUMPS MUMPS MUMPS_DIR
-  "include" dmumps_c.h "lib" dmumps
-  "Paths to headers required by MUMPS."
-  "Libraries required by MUMPS."
-  ADD_COMPONENT mumps_common "include" dmumps_c.h "lib" mumps_common
-  ADD_COMPONENT pord "include" dmumps_c.h "lib" pord)
-
-if (MUMPS_FOUND AND (NOT MUMPS_VERSION))
-  try_run(MUMPS_VERSION_RUN_RESULT MUMPS_VERSION_COMPILE_RESULT
-          ${CMAKE_CURRENT_BINARY_DIR}/config
-          ${CMAKE_CURRENT_SOURCE_DIR}/config/get_mumps_version.cpp
-          CMAKE_FLAGS -DINCLUDE_DIRECTORIES:STRING=${MUMPS_INCLUDE_DIRS}
-          RUN_OUTPUT_VARIABLE MUMPS_VERSION_OUTPUT)
-  if ((MUMPS_VERSION_RUN_RESULT EQUAL 0) AND MUMPS_VERSION_OUTPUT)
-    string(STRIP "${MUMPS_VERSION_OUTPUT}" MUMPS_VERSION)
-    set(MUMPS_VERSION ${MUMPS_VERSION} CACHE STRING "MUMPS version." FORCE)
-    message(STATUS "Found MUMPS version ${MUMPS_VERSION}")
-  else()
-    message(FATAL_ERROR "Unable to determine MUMPS version.")
-  endif()
-endif()
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -10,13 +10,11 @@
 # CONTRIBUTING.md for details.

 # Defines the following variables:
-#   - BLITZ_FOUND
-#   - BLITZ_LIBRARIES
-#   - BLITZ_INCLUDE_DIRS
+#   - MESQUITE_FOUND
+#   - MESQUITE_LIBRARIES
+#   - MESQUITE_INCLUDE_DIRS

 include(MfemCmakeUtilities)
-mfem_find_package(Blitz BLITZ BLITZ_DIR
-       	"include" "blitz/blitz.h"
-       	"lib" "blitz"
-  	"Paths to headers required by Blitz."
-       	"Libraries required by Blitz.")
+mfem_find_package(Mesquite MESQUITE MESQUITE_DIR
+  "include" "Mesquite_all_headers.hpp" "lib" "mesquite"
+  "Paths to headers required by Mesquite." "Libraries required by Mesquite.")
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -14,38 +14,6 @@
 #   - NETCDF_LIBRARIES
 #   - NETCDF_INCLUDE_DIRS

-# Suppress warning about HDF5_ROOT being set
-if (POLICY CMP0074)
-  cmake_policy(SET CMP0074 NEW)
-endif()
-
 include(MfemCmakeUtilities)
-
-# FindHDF5.cmake uses HDF5_ROOT, so we "translate" from the MFEM convention
-# (Needed in some cases, e.g. when HDF5_TARGET_NAMES is set and MFEM's
-# FindHDF5.cmake is not used.)
-set(HDF5_ROOT ${HDF5_DIR} CACHE PATH "")
-
-# We need to guard against the case where HDF5 was already found but without
-# the HL extensions (in which case mfem_find_package will treat the package
-# as already having been found), so we reset the variable to force FindHDF5.cmake
-# to be called for a second time
-set(HDF5_FOUND OFF)
-enable_language(C) # FindHDF5.cmake uses the C compiler
-
 mfem_find_package(NetCDF NETCDF NETCDF_DIR "include" netcdf.h "lib" netcdf
  "Paths to headers required by NetCDF." "Libraries required by NetCDF.")
-
-# The HL extension libraries are in a separate variable and must precede
-# the "regular" hdf5 library, as hdf5_hl depends on hdf5
-# The netcdf library will always be the first element of NETCDF_LIBRARIES
-# and we need to insert after that library but before the hdf5 library, so
-# position 1 is used
-# (Needed in some cases, e.g. when HDF5_TARGET_NAMES is set and MFEM's
-# FindHDF5.cmake is not used.)
-if (HDF5_C_LIBRARY_hdf5_hl)
-  list(INSERT NETCDF_LIBRARIES 1 ${HDF5_C_LIBRARY_hdf5_hl})
-  if (NOT NetCDF_FIND_QUIETLY)
-    message(STATUS "Updated NetCDF libraries: ${NETCDF_LIBRARIES}")
-  endif()
-endif()
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,19 +0,0 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - PARELAG_FOUND
-#   - PARELAG_LIBRARIES
-#   - PARELAG_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(PARELAG PARELAG PARELAG_DIR "" "" "" ""
-  "Paths to headers required by ParELAG." "Libraries required by ParELAG.")
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -14,27 +14,17 @@
 #   - RAJA_LIBRARIES
 #   - RAJA_INCLUDE_DIRS

-if (RAJA_FOUND)
-   return()
+include(MfemCmakeUtilities)
+mfem_find_package(RAJA RAJA RAJA_DIR "include" "RAJA/RAJA.hpp" "lib" "RAJA"
+  "Paths to headers required by RAJA." "Libraries required by RAJA.")
+
+if (NOT RAJA_CONFIG_CMAKE)
+   set(RAJA_CONFIG_CMAKE "${RAJA_DIR}/share/raja/cmake/raja-config.cmake")
 endif()
-message(STATUS "Looking for RAJA ...")
-if (RAJA_DIR)
-   message(STATUS "   in RAJA_DIR = ${RAJA_DIR}")
-   find_package(RAJA CONFIG NO_DEFAULT_PATH PATHS "${RAJA_DIR}")
-endif()
-if (NOT RAJA_FOUND)
-   message(STATUS "   in standard CMake locations")
-   find_package(RAJA CONFIG)
-endif()
-if (RAJA_FOUND)
-   set(RAJA_LIBRARIES "RAJA" CACHE STRING "RAJA imported target." FORCE)
-   set(RAJA_INCLUDE_DIRS "" CACHE STRING "RAJA include dirs (not used)" FORCE)
-   message(STATUS
-      "Found RAJA target: ${RAJA_LIBRARIES} (version: ${RAJA_VERSION})")
-else()
-   set(msg STATUS)
-   if (RAJA_FIND_REQUIRED)
-      set(msg FATAL_ERROR)
+if (EXISTS "${RAJA_CONFIG_CMAKE}")
+   include("${RAJA_CONFIG_CMAKE}")
+   if (ENABLE_CUDA AND NOT MFEM_USE_CUDA)
+      message(FATAL_ERROR
+         "RAJA is built with CUDA: MFEM_USE_CUDA=YES is required")
   endif()
-   message(${msg} "RAJA not found. Please set RAJA_DIR to the RAJA prefix.")
 endif()
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -22,8 +22,8 @@ mfem_find_package(SUNDIALS SUNDIALS SUNDIALS_DIR
    "include" nvector/nvector_serial.h "lib" sundials_nvecserial
  ADD_COMPONENT NVector_Cuda
    "include" nvector/nvector_cuda.h "lib" sundials_nveccuda
-  ADD_COMPONENT NVector_Hip
-    "include" nvector/nvector_hip.h "lib" sundials_nvechip
+  ADD_COMPONENT NVector_ParHyp
+    "include" nvector/nvector_parhyp.h "lib" sundials_nvecparhyp
  ADD_COMPONENT NVector_Parallel
    "include" nvector/nvector_parallel.h "lib" sundials_nvecparallel
  ADD_COMPONENT NVector_MPIPlusX
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -14,23 +14,6 @@
 #   - UMPIRE_LIBRARIES
 #   - UMPIRE_INCLUDE_DIRS

-if (NOT umpire_DIR AND UMPIRE_DIR)
-  set(umpire_DIR ${UMPIRE_DIR}/lib/cmake/umpire)
-endif()
-message(STATUS "Looking for UMPIRE ...")
-message(STATUS "   in UMPIRE_DIR = ${UMPIRE_DIR}")
-message(STATUS "      umpire_DIR = ${umpire_DIR}")
-find_package(umpire CONFIG)
-set(UMPIRE_FOUND ${umpire_FOUND})
-set(UMPIRE_LIBRARIES "umpire")
-if (UMPIRE_FOUND)
-  message(STATUS
-    "Found UMPIRE target: ${UMPIRE_LIBRARIES} (version: ${umpire_VERSION})")
-else()
-  set(msg STATUS)
-  if (UMPIRE_FIND_REQUIRED)
-    set(msg FATAL_ERROR)
-  endif()
-  message(${msg}
-    "UMPIRE not found. Please set UMPIRE_DIR to the install prefix.")
-endif()
+include(MfemCmakeUtilities)
+mfem_find_package(UMPIRE UMPIRE UMPIRE_DIR "include" "umpire/Umpire.hpp" "lib" "umpire"
+  "Paths to headers required by UMPIRE." "Libraries required by UMPIRE.")
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -43,20 +43,6 @@ function(convert_filenames_to_full_paths NAMES)
  set(${NAMES} ${tmp_names} PARENT_SCOPE)
 endfunction()

-# Wrapper for add_executable
-macro(mfem_add_executable NAME)
-  add_executable(${NAME} ${ARGN})
-  if (MFEM_USE_CUDA)
-    set_target_properties(${NAME} PROPERTIES
-      CUDA_RESOLVE_DEVICE_SYMBOLS ON)
-  endif()
-endmacro()
-
-# Wrapper for add_library
-macro(mfem_add_library NAME)
-  add_library(${NAME} ${ARGN})
-endmacro()
-
 # Simple shortcut to add_custom_target() with option to add the target to the
 # main target.
 function(add_mfem_target TARGET_NAME ADD_TO_ALL)
@@ -70,7 +56,7 @@ function(add_mfem_target TARGET_NAME ADD_TO_ALL)
 endfunction()

 # Add mfem examples
-macro(add_mfem_examples EXE_SRCS)
+function(add_mfem_examples EXE_SRCS)
  set(EXE_PREFIX "")
  set(EXE_PREREQUISITE "")
  set(EXE_NEEDED_BY "")
@@ -86,15 +72,13 @@ macro(add_mfem_examples EXE_SRCS)
  foreach(SRC_FILE IN LISTS ${EXE_SRCS})
    # If CUDA is enabled, tag source files to be compiled with nvcc.
    if (MFEM_USE_CUDA)
-      set_source_files_properties(${SRC_FILE} PROPERTIES LANGUAGE CUDA)
+      set_property(SOURCE ${SRC_FILE} PROPERTY LANGUAGE CUDA)
    endif()

    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)

    string(REPLACE ".cpp" "" EXE_NAME "${EXE_PREFIX}${SRC_FILENAME}")
-    mfem_add_executable(${EXE_NAME} ${SRC_FILE})
-    install(TARGETS ${EXE_NAME}
-            RUNTIME DESTINATION examples)
+    add_executable(${EXE_NAME} ${SRC_FILE})
    add_dependencies(${MFEM_ALL_EXAMPLES_TARGET_NAME} ${EXE_NAME})
    if (EXE_NEEDED_BY)
      add_dependencies(${EXE_NEEDED_BY} ${EXE_NAME})
@@ -103,11 +87,30 @@ macro(add_mfem_examples EXE_SRCS)
      ${MFEM_EXEC_PREREQUISITES_TARGET_NAME} ${EXE_PREREQUISITE})

    target_link_libraries(${EXE_NAME} mfem)
+    if (MFEM_USE_MPI)
+      # Not needed: (mfem already links with MPI_CXX_LIBRARIES)
+      # target_link_libraries(${EXE_NAME} ${MPI_CXX_LIBRARIES})
+
+      # Language-specific include directories:
+      if (MPI_CXX_INCLUDE_PATH)
+        target_include_directories(${EXE_NAME} PRIVATE "${MPI_CXX_INCLUDE_PATH}")
+      endif()
+      if (MPI_CXX_COMPILE_FLAGS)
+        separate_arguments(MPI_CXX_COMPILE_ARGS UNIX_COMMAND
+          "${MPI_CXX_COMPILE_FLAGS}")
+        target_compile_options(${EXE_NAME} PRIVATE ${MPI_CXX_COMPILE_ARGS})
+      endif()
+
+      if (MPI_CXX_LINK_FLAGS)
+        set_target_properties(${EXE_NAME} PROPERTIES
+          LINK_FLAGS "${MPI_CXX_LINK_FLAGS}")
+      endif()
+    endif()
  endforeach(SRC_FILE)
-endmacro()
+endfunction()

 # A slightly more versatile function for adding miniapps to MFEM
-macro(add_mfem_miniapp MFEM_EXE_NAME)
+function(add_mfem_miniapp MFEM_EXE_NAME)
  # Parse the input arguments looking for the things we need
  set(POSSIBLE_ARGS "MAIN" "EXTRA_SOURCES" "EXTRA_HEADERS" "EXTRA_OPTIONS" "EXTRA_DEFINES" "LIBRARIES")
  set(CURRENT_ARG)
@@ -123,7 +126,8 @@ macro(add_mfem_miniapp MFEM_EXE_NAME)

  # If CUDA is enabled, tag source files to be compiled with nvcc.
  if (MFEM_USE_CUDA)
-    set_source_files_properties(${MAIN_LIST} ${EXTRA_SOURCES_LIST} PROPERTIES LANGUAGE CUDA)
+    set_property(SOURCE ${MAIN_LIST} ${EXTRA_SOURCES_LIST}
+      PROPERTY LANGUAGE CUDA)
    if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.12.0)
      list(TRANSFORM EXTRA_OPTIONS_LIST PREPEND "-Xcompiler=")
    else()
@@ -136,24 +140,61 @@ macro(add_mfem_miniapp MFEM_EXE_NAME)
  endif()

  # Actually add the executable
-  mfem_add_executable(${MFEM_EXE_NAME} ${MAIN_LIST}
-      ${EXTRA_SOURCES_LIST} ${EXTRA_HEADERS_LIST})
+  add_executable(${MFEM_EXE_NAME} ${MAIN_LIST}
+    ${EXTRA_SOURCES_LIST} ${EXTRA_HEADERS_LIST})
  add_dependencies(${MFEM_ALL_MINIAPPS_TARGET_NAME} ${MFEM_EXE_NAME})
  add_dependencies(${MFEM_EXE_NAME} ${MFEM_EXEC_PREREQUISITES_TARGET_NAME})

  # Append the additional libraries and options
  if (LIBRARIES_LIST)
-    target_link_libraries(${MFEM_EXE_NAME} PRIVATE ${LIBRARIES_LIST})
+    if(CMAKE_VERSION VERSION_GREATER 2.8.11)
+      target_link_libraries(${MFEM_EXE_NAME} PRIVATE ${LIBRARIES_LIST})
+    else()
+      target_link_libraries(${MFEM_EXE_NAME} ${LIBRARIES_LIST})
+    endif()
  endif()
  if (EXTRA_OPTIONS_LIST)
    string(REPLACE ";" " " EXTRA_OPTIONS_STRING "${EXTRA_OPTIONS_LIST}")
    message(STATUS "${MFEM_EXE_NAME}: add flags \"${EXTRA_OPTIONS_STRING}\"")
-    target_compile_options(${MFEM_EXE_NAME} PRIVATE ${EXTRA_OPTIONS_LIST})
+    if(CMAKE_VERSION VERSION_GREATER 2.8.11)
+      target_compile_options(${MFEM_EXE_NAME} PRIVATE ${EXTRA_OPTIONS_LIST})
+    else()
+      get_target_property(THIS_COMPILE_FLAGS ${MFEM_EXE_NAME} COMPILE_FLAGS)
+      if (THIS_COMPILE_FLAGS)
+        set(THIS_COMPILE_FLAGS "${THIS_COMPILE_FLAGS} ${EXTRA_OPTIONS_STRING}")
+      else()
+        set(THIS_COMPILE_FLAGS "${EXTRA_OPTIONS_STRING}")
+      endif()
+      set_target_properties(${MFEM_EXE_NAME}
+        PROPERTIES COMPILE_FLAGS ${THIS_COMPILE_FLAGS})
+    endif()
  endif()
  if (EXTRA_DEFINES_LIST)
    target_compile_definitions(${MFEM_EXE_NAME} PRIVATE ${EXTRA_DEFINES_LIST})
  endif()
-endmacro()
+
+  # Handle the MPI separately
+  if (MFEM_USE_MPI)
+    # Add MPI_CXX_LIBRARIES, in case this target does not link with mfem.
+    if(CMAKE_VERSION VERSION_GREATER 2.8.11)
+      target_link_libraries(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_LIBRARIES})
+    else()
+      target_link_libraries(${MFEM_EXE_NAME} ${MPI_CXX_LIBRARIES})
+    endif()
+
+    if (MPI_CXX_INCLUDE_PATH)
+      target_include_directories(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_INCLUDE_PATH})
+    endif()
+    if (MPI_CXX_COMPILE_FLAGS)
+      target_compile_options(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_COMPILE_FLAGS})
+    endif()
+
+    if (MPI_CXX_LINK_FLAGS)
+      set_target_properties(${MFEM_EXE_NAME} PROPERTIES
+        LINK_FLAGS "${MPI_CXX_LINK_FLAGS}")
+    endif()
+  endif()
+endfunction()


 # Auxiliary function, used in mfem_find_package().
@@ -194,7 +235,7 @@ endfunction(mfem_find_component)
 #   code snippets. Additionally, a list of required/optional/alternative
 #   packages (given by ${Name}_REQUIRED_PACKAGES) are searched for and added to
 #   the ${Prefix}_INCLUDE_DIRS and ${Prefix}_LIBRARIES lists. The variable
-#   ${Name}_REQUIRED_LIBRARIES can be set to specify any additional libraries
+#   ${Name}_REQUIRED_LIBRARIES can be set to spcecify any additional libraries
 #   that are needed. This function defines the following CACHE variables:
 #
 #      ${Prefix}_FOUND
@@ -397,9 +438,9 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
              break()
            endif()
          endforeach()
-          if (Required AND NOT ${ReqPack}_FOUND)
+          if (${Required} AND NOT ${ReqPack}_FOUND)
            message(FATAL_ERROR " *** Required package ${ReqPack} not found."
-              " Checked target names: ${ReqPack} ${${ReqPack}_TARGET_NAMES}")
+              "Checked target names: ${ReqPack} ${${ReqPack}_TARGET_NAMES}")
          endif()
        endif()
      endif()
@@ -465,23 +506,16 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
          endif()
          get_target_property(IsImported ${TargetName} IMPORTED)
          if (IsImported)
-            get_target_property(ImpConfigs ${TargetName} IMPORTED_CONFIGURATIONS)
-            # message(STATUS "${ReqPack} imported configs: ${ImpConfigs}")
            set(ImportConfig ${${ReqPack}_IMPORT_CONFIG})
            if (NOT ImportConfig)
              set(ImportConfig RELEASE)
-              list(FIND ImpConfigs ${ImportConfig} _Index)
-              if (_Index EQUAL -1)
-                list(GET ImpConfigs 0 ImportConfig)
-              endif()
            endif()
-            set(ImportConfigSuffix "_${ImportConfig}")
-            if ((NOT ImportConfig) OR ("${ImportConfig}" STREQUAL "NO_CONFIG"))
-              set(ImportConfig "NO_CONFIG")
-              set(ImportConfigSuffix "")
-              # message(FATAL_ERROR " *** ${ReqPack}: configuration "
-              #   "${ImportConfig} not found. Set ${ReqPack}_IMPORT_CONFIG "
-              #   "from the list: ${ImpConfigs}.")
+            get_target_property(ImpConfigs ${TargetName} IMPORTED_CONFIGURATIONS)
+            list(FIND ImpConfigs ${ImportConfig} _Index)
+            if (_Index EQUAL -1)
+              message(FATAL_ERROR " *** ${ReqPack}: configuration "
+                "${ImportConfig} not found. Set ${ReqPack}_IMPORT_CONFIG "
+                "from the list: ${ImpConfigs}.")
            endif()
          endif()
          # Set _Pack_LIBS
@@ -493,9 +527,8 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
            endif()
          else()
            # Set _Pack_LIBS from the target properties for ImportConfig
-            foreach (_prop IMPORTED_LOCATION${ImportConfigSuffix}
-                IMPORTED_LINK_INTERFACE_LIBRARIES${ImportConfigSuffix}
-                INTERFACE_LINK_LIBRARIES)
+            foreach (_prop IMPORTED_LOCATION_${ImportConfig}
+                IMPORTED_LINK_INTERFACE_LIBRARIES_${ImportConfig})
              get_target_property(_value ${TargetName} ${_prop})
              if (_value)
                list(APPEND _Pack_LIBS ${_value})
@@ -507,7 +540,7 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
            endif()
          endif()
          # Set _Pack_INCS
-          foreach (_prop INCLUDE_DIRECTORIES INTERFACE_INCLUDE_DIRECTORIES)
+          foreach (_prop INCLUDE_DIRECTORIES)
            get_target_property(_value ${TargetName} ${_prop})
            if (_value)
              list(APPEND _Pack_INCS ${_value})
@@ -685,146 +718,6 @@ function(mfem_find_library Name Prefix Lib LibDoc CheckVar CheckSrc)
 endfunction(mfem_find_library)


-#
-# Extract compile and link options needed by the given target.
-#
-function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
-
-  if (NOT TARGET ${Target})
-    return()
-  endif()
-
-  # CMAKE_SHARED_LIBRARY_RUNTIME_C_FLAG -> '-Wl,-rpath,'
-  set(shared_link_flag ${CMAKE_SHARED_LIBRARY_RUNTIME_C_FLAG})
-  if (NOT shared_link_flag)
-    set(shared_link_flag "-Wl,-rpath,")
-  endif()
-
-  set(tgt "${Target}")
-  unset(CompileOpts)
-  unset(LinkOpts)
-  get_target_property(IsImported ${tgt} IMPORTED)
-  # message(STATUS "${tgt}[IMPORTED]: ${IsImported}")
-  # Generally, the possible target types are: STATIC_LIBRARY, MODULE_LIBRARY,
-  # SHARED_LIBRARY, INTERFACE_LIBRARY, EXECUTABLE.
-  get_target_property(type ${tgt} TYPE)
-  # message(STATUS "${tgt}[TYPE]: ${type}")
-  unset(ImportConfig)
-  get_target_property(ImportConfigs ${tgt} IMPORTED_CONFIGURATIONS)
-  if (ImportConfigs)
-    list(GET ImportConfigs 0 ImportConfig)
-  endif()
-  if (NOT ImportConfig)
-    set(ImportConfig RELEASE)
-  endif()
-  # message(STATUS "${tgt}[ImportConfig]: ${ImportConfig}")
-  # List all properties with: cmake --help-property-list
-  get_target_property(Defs ${tgt} INTERFACE_COMPILE_DEFINITIONS)
-  if (Defs)
-    list(REMOVE_DUPLICATES Defs)
-    foreach(Def ${Defs})
-      list(APPEND CompileOpts "-D${Def}")
-    endforeach()
-  endif()
-  get_target_property(Opts ${tgt} INTERFACE_COMPILE_OPTIONS)
-  if (Opts)
-    foreach(Opt ${Opts})
-      list(APPEND CompileOpts "${Opt}")
-    endforeach()
-  endif()
-  get_target_property(Dirs ${tgt} INTERFACE_INCLUDE_DIRECTORIES)
-  if (Dirs)
-    list(REMOVE_DUPLICATES Dirs)
-    foreach(Dir ${Dirs})
-      list(APPEND CompileOpts "-I\"${Dir}\"")
-    endforeach()
-  endif()
-  get_target_property(SysDirs ${tgt} INTERFACE_SYSTEM_INCLUDE_DIRECTORIES)
-  if (SysDirs)
-    list(REMOVE_DUPLICATES SysDirs)
-    foreach(SysDir ${SysDirs})
-      list(APPEND CompileOpts "-isystem \"${SysDir}\"")
-    endforeach()
-  endif()
-  set(AddInterfaceLinkLibs TRUE)
-  if ("${type}" STREQUAL "STATIC_LIBRARY")
-    get_target_property(Location ${tgt} LOCATION)
-    if (Location)
-      list(APPEND LinkOpts "\"${Location}\"")
-    else()
-      message(STATUS " *** Warning: [${tgt}] LOCATION not defined!")
-    endif()
-  elseif ("${type}" STREQUAL "SHARED_LIBRARY")
-    get_target_property(Location ${tgt} LOCATION)
-    if (Location)
-      get_filename_component(Dir ${Location} DIRECTORY)
-      get_filename_component(NameWE ${Location} NAME_WE)
-      string(REGEX REPLACE "^lib" "" LibName ${NameWE})
-      list(APPEND LinkOpts
-        "-L\"${Dir}\""
-        "${shared_link_flag}\"${Dir}\""
-        "-l${LibName}")
-    else()
-      message(STATUS " *** Warning: [${tgt}] LOCATION not defined!")
-    endif()
-  elseif ("${type}" STREQUAL "INTERFACE_LIBRARY")
-    # The INTERFACE_LINK_LIBRARIES property is handled below.
-    # Other properties we may need to handle:
-    # INTERFACE_LINK_DEPENDS
-    # INTERFACE_LINK_DIRECTORIES
-    # INTERFACE_LINK_OPTIONS
-  else()
-    message(STATUS " *** Warning: [${tgt}] uses target type '${type}'"
-      " which is not supported!")
-    set(AddInterfaceLinkLibs FALSE)
-  endif()
-  if (AddInterfaceLinkLibs)
-    get_target_property(Libs ${tgt} INTERFACE_LINK_LIBRARIES)
-    # message(STATUS "${tgt}[INTERFACE_LINK_LIBRARIES]: ${Libs}")
-    if (Libs)
-      foreach(Lib ${Libs})
-        if (NOT (TARGET ${Lib}))
-          # message(STATUS "Lib = ${Lib}")
-          # Filter-out generator expressions
-          if (NOT ("${Lib}" MATCHES "^\\$"))
-            list(APPEND LinkOpts "${Lib}")
-          endif()
-        else()
-          mfem_get_target_options(${Lib} COpts LOpts)
-          list(APPEND CompileOpts ${COpts})
-          list(APPEND LinkOpts ${LOpts})
-        endif()
-      endforeach()
-    endif()
-  endif()
-
-  # Other potentially relevant properties:
-  # - For all target types:
-  # IMPORTED_LIBNAME
-  # IMPORTED_LIBNAME_${ImportConfig}
-  # INTERFACE_AUTOUIC_OPTIONS
-  # INTERFACE_COMPILE_FEATURES
-  # INTERFACE_POSITION_INDEPENDENT_CODE
-  # INTERFACE_SOURCES
-  # INTERFACE_SYSTEM_INCLUDE_DIRECTORIES)
-  # - For non-"INTERFACE_LIBRARY" target types only:
-  # IMPORTED_LOCATION
-  # IMPORTED_LOCATION_${ImportConfig}
-  # IMPORTED_LINK_INTERFACE_LIBRARIES
-  # IMPORTED_LINK_INTERFACE_LIBRARIES_${ImportConfig}
-  # LINK_FLAGS
-  # LINK_FLAGS_${ImportConfig}
-  # LOCATION_${ImportConfig})
-
-  set(${CompileOptsVar} "${CompileOpts}" PARENT_SCOPE)
-  set(${LinkOptsVar} "${LinkOpts}" PARENT_SCOPE)
-
-  # message(STATUS "${tgt}[CompileOpts]: ${CompileOpts}")
-  # message(STATUS "${tgt}[LinkOpts]: ${LinkOpts}")
-
-endfunction(mfem_get_target_options)
-
-
 #
 #   Function that creates 'config.mk' from 'config.mk.in' for the both the
 #   build- and the install-locations and define install rules for 'config.mk'
@@ -840,19 +733,16 @@ function(mfem_export_mk_files)
    set(shared_link_flag "-Wl,-rpath,")
  endif()

-  # Convert Boolean vars to YES/NO without writing the values to cache
+  # Convert Boolean vars to YES/NO without writting the values to cache
  set(CONFIG_MK_BOOL_VARS MFEM_USE_MPI MFEM_USE_METIS MFEM_USE_METIS_5
      MFEM_DEBUG MFEM_USE_EXCEPTIONS MFEM_USE_ZLIB MFEM_USE_LIBUNWIND
-      MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_LEGACY_OPENMP MFEM_USE_OPENMP
-      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_SUITESPARSE
-      MFEM_USE_SUPERLU MFEM_USE_SUPERLU5 MFEM_USE_MUMPS MFEM_USE_STRUMPACK
-      MFEM_USE_GINKGO MFEM_USE_AMGX MFEM_USE_GNUTLS MFEM_USE_NETCDF
-      MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_FMS
-      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA
-      MFEM_USE_HIP MFEM_USE_RAJA MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_CALIPER
-      MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO
-      MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG
-      MFEM_USE_MOONOLITH MFEM_USE_ALGOIM MFEM_USE_ENZYME)
+      MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP MFEM_USE_LEGACY_OPENMP
+      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
+      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX
+      MFEM_USE_GNUTLS MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC
+      MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_CONDUIT MFEM_USE_PUMI
+      MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA MFEM_USE_UMPIRE MFEM_USE_SIMD
+      MFEM_USE_ADIOS2)
  foreach(var ${CONFIG_MK_BOOL_VARS})
    if (${var})
      set(${var} YES)
@@ -864,16 +754,10 @@ function(mfem_export_mk_files)
  set(MFEM_CXX ${CMAKE_CXX_COMPILER})
  set(MFEM_HOST_CXX ${MFEM_CXX})
  set(MFEM_CPPFLAGS "")
-  get_target_property(cxx_std mfem CXX_STANDARD)
-  # For now, we ignore the setting of the CXX_EXTENSIONS property. If this
-  # property is set, then we need to use a variable like:
-  #    CMAKE_CXX11_EXTENSION_COMPILE_OPTION
-  set(cxx_std_flag ${CMAKE_CXX${cxx_std}_STANDARD_COMPILE_OPTION})
-  string(STRIP
-         "${cxx_std_flag} ${CMAKE_CXX_FLAGS_${BUILD_TYPE}} ${CMAKE_CXX_FLAGS}"
+  string(STRIP "${CMAKE_CXX_FLAGS_${BUILD_TYPE}} ${CMAKE_CXX_FLAGS}"
         MFEM_CXXFLAGS)
  set(MFEM_TPLFLAGS "")
-  foreach(dir ${TPL_INCLUDE_DIRS})
+  foreach(dir ${MFEM_TPL_INCLUDE_DIRS})
    set(MFEM_TPLFLAGS "${MFEM_TPLFLAGS} -I${dir}")
  endforeach()
  # TODO: MFEM_TPLFLAGS: add other TPL flags, in addition to the -I flags.
@@ -936,11 +820,9 @@ function(mfem_export_mk_files)
      string(REGEX REPLACE "^SCOREC::" "" libname ${pumilib})
      string(FIND "${pumilib}" ".a" staticlib)
      string(FIND "${pumilib}" ".so" sharedlib)
-      string(FIND "${pumilib}" ".dylib" dynamiclib)
-      find_library(lib ${libname} PATHS ${PUMI_DIR}/lib NO_DEFAULT_PATH)
+      find_library(lib ${libname} PATHS ${PUMI_DIR}/lib NO_DEFUALT_PATH)
      if (NOT "${sharedlib}" MATCHES "-1" OR
-          NOT "${staticlib}" MATCHES "-1" OR
-          NOT "${dynamiclib}" MATCHES "-1"  )
+          NOT "${staticlib}" MATCHES "-1"   )
        set(MFEM_EXT_LIBS "${pumilib} ${MFEM_EXT_LIBS}")
      elseif (NOT "${lib}" MATCHES "lib-NOTFOUND")
        set(MFEM_EXT_LIBS "${lib} ${MFEM_EXT_LIBS}")
@@ -955,19 +837,9 @@ function(mfem_export_mk_files)
  foreach(lib ${TPL_LIBRARIES})
    get_filename_component(suffix ${lib} EXT)
    # handle interfaces (e.g., SCOREC::apf)
-    if ("${lib}" MATCHES "SCOREC::.*" OR "${lib}" MATCHES "Ginkgo::.*" OR "${lib}" MATCHES "ParMoonolith::.*")
-    elseif (TARGET "${lib}")
-      mfem_get_target_options(${lib} CompileOpts LinkOpts)
-      # Removing duplicates may lead to issues:
-      # list(REMOVE_DUPLICATES CompileOpts)
-      # list(REMOVE_DUPLICATES LinkOpts)
-      string(REPLACE ";" " " COpts "${CompileOpts}")
-      string(REPLACE ";" " " LOpts "${LinkOpts}")
-      # message(STATUS "${lib}[COpts]: '${COpts}'")
-      # message(STATUS "${lib}[LOpts]: '${LOpts}'")
-      set(MFEM_TPLFLAGS "${MFEM_TPLFLAGS} ${COpts}")
-      set(MFEM_EXT_LIBS "${MFEM_EXT_LIBS} ${LOpts}")
-      # message(FATAL_ERROR "***** interface lib found ... exiting *****")
+    if ("${lib}" MATCHES "SCOREC::.*" OR "${lib}" MATCHES "Ginkgo::.*")
+    elseif (NOT "${lib}" MATCHES "SCOREC::.*" AND "${lib}" MATCHES ".*::.*")
+      message(FATAL_ERROR "***** interface lib found ... exiting *****")
      # handle static and shared libs
    elseif ("${suffix}" STREQUAL "${CMAKE_SHARED_LIBRARY_SUFFIX}")
      get_filename_component(dir ${lib} DIRECTORY)
@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -30,37 +30,10 @@
 #endif

 // Windows specific options
-#if defined(_WIN32) && !defined(_USE_MATH_DEFINES)
+#ifdef _WIN32
 // Macro needed to get defines like M_PI from <cmath>. (Visual Studio C++ only?)
 #define _USE_MATH_DEFINES
 #endif
-// Macro MFEM_EXPORT: this macro is used when declaring exported global
-// variables and static class variables in public header files, e.g.:
-//    extern MFEM_EXPORT Geometry Geometries;
-//    static MFEM_EXPORT Device device_singleton;
-// In cases where a class contains multiple static variables, instead of marking
-// all such variables with MFEM_EXPORT, one can mark the class with MFEM_EXPORT,
-// e.g.:
-//    class MFEM_EXPORT MemoryManager ...
-// Note: MFEM's GitHub CI includes a shared MSVC build that will fail if a
-// variable that needs MFEM_EXPORT does not have it. However, builds with
-// optional external libraries are not tested and may require separate checks to
-// determine the necessity of MFEM_EXPORT.
-#if defined(_MSC_VER) && defined(MFEM_SHARED_BUILD)
-#ifdef mfem_EXPORTS
-#define MFEM_EXPORT __declspec(dllexport)
-#else
-#define MFEM_EXPORT __declspec(dllimport)
-#endif
-#else
-#define MFEM_EXPORT
-#endif
-// On Cygwin the option -std=c++11 prevents the definition of M_PI. Defining
-// the following macro allows us to get M_PI and some needed functions, e.g.
-// posix_memalign(), strdup(), strerror_r().
-#ifdef __CYGWIN__
-#define _XOPEN_SOURCE 600
-#endif

 // Check dependencies:

@@ -69,9 +42,6 @@
 #ifdef MFEM_USE_SUPERLU
 #error Building with SuperLU_DIST (MFEM_USE_SUPERLU=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
-#ifdef MFEM_USE_MUMPS
-#error Building with MUMPS (MFEM_USE_MUMPS=YES) requires MPI (MFEM_USE_MPI=YES)
-#endif
 #ifdef MFEM_USE_STRUMPACK
 #error Building with STRUMPACK (MFEM_USE_STRUMPACK=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -30,18 +30,15 @@
 #define MFEM_VERSION_MINOR (((MFEM_VERSION)/100)%100)
 #define MFEM_VERSION_PATCH ((MFEM_VERSION)%100)

-// The absolute path of the MFEM source prefix.
+// The absolute path of the MFEM source prefix
 // #define MFEM_SOURCE_DIR "@MFEM_SOURCE_DIR@"

-// The absolute path of the MFEM installation prefix.
+// The absolute path of the MFEM installation prefix
 // #define MFEM_INSTALL_DIR "@MFEM_INSTALL_DIR@"

 // Description of the git commit used to build MFEM.
 // #define MFEM_GIT_STRING "@MFEM_GIT_STRING@"

-// Enable shared library build of MFEM.
-// #define MFEM_SHARED_BUILD
-
 // Build the parallel MFEM library.
 // Requires an MPI compiler, and the libraries HYPRE and METIS.
 // #define MFEM_USE_MPI
@@ -88,63 +85,56 @@
 // Enable MFEM functionality based on the SUNDIALS libraries.
 // #define MFEM_USE_SUNDIALS

+// Enable MFEM functionality based on the Mesquite library.
+// #define MFEM_USE_MESQUITE
+
 // Enable MFEM functionality based on the SuiteSparse library.
 // #define MFEM_USE_SUITESPARSE

-// Enable MFEM functionality based on the SuperLU_DIST library.
+// Enable MFEM functionality based on the SuperLU library.
 // #define MFEM_USE_SUPERLU
 // #define MFEM_USE_SUPERLU5

-// Enable MFEM functionality based on the MUMPS library.
-// #define MFEM_USE_MUMPS
-// #define MFEM_MUMPS_VERSION @MFEM_MUMPS_VERSION@
-
 // Enable MFEM functionality based on the STRUMPACK library.
 // #define MFEM_USE_STRUMPACK

-// Enable MFEM features based on the Ginkgo library.
+// Enable MFEM features based on the Ginkgo library
 // #define MFEM_USE_GINKGO

 // Enable MFEM functionality based on the AmgX library.
 // #define MFEM_USE_AMGX

-// Enable secure socket streams based on the GNUTLS library.
+// Enable secure socket streams based on the GNUTLS library
 // #define MFEM_USE_GNUTLS

-// Enable Sidre support.
+// Enable Sidre support
 // #define MFEM_USE_SIDRE

-// Enable the use of SIMD in the high performance templated classes.
+// Enable the use of SIMD in the high performance templated classes
 // #define MFEM_USE_SIMD

-// Enable FMS support.
-// #define MFEM_USE_FMS
-
-// Enable Conduit support.
+// Enable Conduit support
 // #define MFEM_USE_CONDUIT

-// Enable functionality based on the NetCDF library (reading CUBIT files).
+// Enable functionality based on the NetCDF library (reading CUBIT files)
 // #define MFEM_USE_NETCDF

-// Enable functionality based on the PETSc library.
+// Enable functionality based on the PETSc library
 // #define MFEM_USE_PETSC

-// Enable functionality based on the SLEPc library.
+// Enable functionality based on the SLEPc library
 // #define MFEM_USE_SLEPC

 // Enable functionality based on the MPFR library.
 // #define MFEM_USE_MPFR

-// Enable MFEM functionality based on the PUMI library.
+// Enable MFEM functionality based on the PUMI library
 // #define MFEM_USE_PUMI

-// Enable Moonolith-based general interpolation between finite element spaces.
-// #define MFEM_USE_MOONOLITH
-
 // Enable MFEM functionality based on the HIOP library.
 // #define MFEM_USE_HIOP

-// Enable MFEM functionality based on the GSLIB library.
+// Enable MFEM functionality based on the GSLIB library
 // #define MFEM_USE_GSLIB

 // Build the NVIDIA GPU/CUDA-enabled version of the MFEM library.
@@ -164,12 +154,6 @@
 // Enable functionality based on the libCEED library.
 // #define MFEM_USE_CEED

-// Enable functionality based on the Caliper library.
-// #define MFEM_USE_CALIPER
-
-// Enable functionality based on the Algoim library.
-// #define MFEM_USE_ALGOIM
-
 // Enable functionality based on the Umpire library.
 // #define MFEM_USE_UMPIRE

@@ -186,19 +170,4 @@
 // Enable interface to the MKL CPardiso library.
 // #define MFEM_USE_MKL_CPARDISO

-// Enable interface to the MKL Pardiso library.
-// #define MFEM_USE_MKL_PARDISO
-
-// Use forward mode for automatic differentiation.
-// #define MFEM_USE_ADFORWARD
-
-// Enable the use of the CoDiPack library for AD.
-// #define MFEM_USE_CODIPACK
-
-// Enable functionality based on the Google Benchmark library.
-// #define MFEM_USE_BENCHMARK
-
-// Enable the Enzyme LLVM plugin
-// #define MFEM_USE_ENZYME
-
 #endif // MFEM_CONFIG_HEADER
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -29,10 +29,10 @@ MFEM_USE_OPENMP        = @MFEM_USE_OPENMP@
 MFEM_USE_MEMALLOC      = @MFEM_USE_MEMALLOC@
 MFEM_TIMER_TYPE        = @MFEM_TIMER_TYPE@
 MFEM_USE_SUNDIALS      = @MFEM_USE_SUNDIALS@
+MFEM_USE_MESQUITE      = @MFEM_USE_MESQUITE@
 MFEM_USE_SUITESPARSE   = @MFEM_USE_SUITESPARSE@
 MFEM_USE_SUPERLU       = @MFEM_USE_SUPERLU@
 MFEM_USE_SUPERLU5      = @MFEM_USE_SUPERLU5@
-MFEM_USE_MUMPS         = @MFEM_USE_MUMPS@
 MFEM_USE_STRUMPACK     = @MFEM_USE_STRUMPACK@
 MFEM_USE_GINKGO        = @MFEM_USE_GINKGO@
 MFEM_USE_AMGX          = @MFEM_USE_AMGX@
@@ -42,7 +42,6 @@ MFEM_USE_PETSC         = @MFEM_USE_PETSC@
 MFEM_USE_SLEPC         = @MFEM_USE_SLEPC@
 MFEM_USE_MPFR          = @MFEM_USE_MPFR@
 MFEM_USE_SIDRE         = @MFEM_USE_SIDRE@
-MFEM_USE_FMS           = @MFEM_USE_FMS@
 MFEM_USE_CONDUIT       = @MFEM_USE_CONDUIT@
 MFEM_USE_PUMI          = @MFEM_USE_PUMI@
 MFEM_USE_HIOP          = @MFEM_USE_HIOP@
@@ -52,18 +51,10 @@ MFEM_USE_HIP           = @MFEM_USE_HIP@
 MFEM_USE_RAJA          = @MFEM_USE_RAJA@
 MFEM_USE_OCCA          = @MFEM_USE_OCCA@
 MFEM_USE_CEED          = @MFEM_USE_CEED@
-MFEM_USE_CALIPER       = @MFEM_USE_CALIPER@
 MFEM_USE_UMPIRE        = @MFEM_USE_UMPIRE@
 MFEM_USE_SIMD          = @MFEM_USE_SIMD@
 MFEM_USE_ADIOS2        = @MFEM_USE_ADIOS2@
 MFEM_USE_MKL_CPARDISO  = @MFEM_USE_MKL_CPARDISO@
-MFEM_USE_MKL_PARDISO   = @MFEM_USE_MKL_PARDISO@
-MFEM_USE_MOONOLITH     = @MFEM_USE_MOONOLITH@
-MFEM_USE_ADFORWARD     = @MFEM_USE_ADFORWARD@
-MFEM_USE_CODIPACK      = @MFEM_USE_CODIPACK@
-MFEM_USE_BENCHMARK     = @MFEM_USE_BENCHMARK@
-MFEM_USE_PARELAG       = @MFEM_USE_PARELAG@
-MFEM_USE_ENZYME        = @MFEM_USE_ENZYME@

 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@
@@ -93,7 +84,7 @@ MFEM_MPIEXEC_NP = @MFEM_MPIEXEC_NP@
 MFEM_MPI_NP     = @MFEM_MPI_NP@

 # The NVCC compiler cannot link with -x=cu
-MFEM_LINK_FLAGS := $(filter-out -x=cu -xhip, $(MFEM_FLAGS))
+MFEM_LINK_FLAGS := $(filter-out -x=cu, $(MFEM_FLAGS))

 # Optional extra configuration
@MFEM_CONFIG_EXTRA@
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -19,7 +19,6 @@ if (NOT CMAKE_BUILD_TYPE)
 endif()

 # MFEM options. Set to mimic the default "defaults.mk" file.
-option(BUILD_SHARED_LIBS "Enable shared library build of MFEM" OFF)
 option(MFEM_USE_MPI "Enable MPI parallel build" OFF)
 option(MFEM_USE_METIS "Enable METIS usage" ${MFEM_USE_MPI})
 option(MFEM_USE_EXCEPTIONS "Enable the use of exceptions" OFF)
@@ -31,10 +30,10 @@ option(MFEM_USE_OPENMP "Enable the OpenMP backend" OFF)
 option(MFEM_USE_LEGACY_OPENMP "Enable legacy OpenMP usage" OFF)
 option(MFEM_USE_MEMALLOC "Enable the internal MEMALLOC option." ON)
 option(MFEM_USE_SUNDIALS "Enable SUNDIALS usage" OFF)
+option(MFEM_USE_MESQUITE "Enable MESQUITE usage" OFF)
 option(MFEM_USE_SUITESPARSE "Enable SuiteSparse usage" OFF)
 option(MFEM_USE_SUPERLU "Enable SuperLU_DIST usage" OFF)
 option(MFEM_USE_SUPERLU5 "Use the old SuperLU_DIST 5.1 version" OFF)
-option(MFEM_USE_MUMPS "Enable MUMPS usage" OFF)
 option(MFEM_USE_STRUMPACK "Enable STRUMPACK usage" OFF)
 option(MFEM_USE_GINKGO "Enable Ginkgo usage" OFF)
 option(MFEM_USE_AMGX "Enable AmgX usage" OFF)
@@ -45,32 +44,17 @@ option(MFEM_USE_PETSC "Enable PETSc support." OFF)
 option(MFEM_USE_SLEPC "Enable SLEPc support." OFF)
 option(MFEM_USE_MPFR "Enable MPFR usage." OFF)
 option(MFEM_USE_SIDRE "Enable Axom/Sidre usage" OFF)
-option(MFEM_USE_FMS "Enable FMS usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
 option(MFEM_USE_PUMI "Enable PUMI" OFF)
 option(MFEM_USE_HIOP "Enable HiOp" OFF)
 option(MFEM_USE_CUDA "Enable CUDA" OFF)
-option(MFEM_USE_HIP "Enable HIP" OFF)
 option(MFEM_USE_OCCA "Enable OCCA" OFF)
 option(MFEM_USE_RAJA "Enable RAJA" OFF)
 option(MFEM_USE_CEED "Enable CEED" OFF)
 option(MFEM_USE_UMPIRE "Enable Umpire" OFF)
 option(MFEM_USE_SIMD "Enable use of SIMD intrinsics" OFF)
 option(MFEM_USE_ADIOS2 "Enable ADIOS2" OFF)
-option(MFEM_USE_CALIPER "Enable Caliper support" OFF)
-option(MFEM_USE_ALGOIM "Enable Algoim support" OFF)
 option(MFEM_USE_MKL_CPARDISO "Enable MKL CPardiso" OFF)
-option(MFEM_USE_MKL_PARDISO "Enable MKL Pardiso" OFF)
-option(MFEM_USE_ADFORWARD "Enable forward mode for AD" OFF)
-option(MFEM_USE_CODIPACK "Enable automatic differentiation (AD) using CoDiPack" OFF)
-option(MFEM_USE_BENCHMARK "Enable Google Benchmark" OFF)
-option(MFEM_USE_PARELAG "Enable ParELAG" OFF)
-option(MFEM_USE_ENZYME "Enable Enzyme" OFF)
-
-# Optional overrides for autodetected MPIEXEC and MPIEXEC_NUMPROC_FLAG
-# set(MFEM_MPIEXEC "mpirun" CACHE STRING "Command for running MPI tests")
-# set(MFEM_MPIEXEC_NP "-np" CACHE STRING
-#     "Flag for setting the number of MPI tasks")

 set(MFEM_MPI_NP 4 CACHE STRING "Number of processes used for MPI tests")

@@ -82,7 +66,6 @@ set(MFEM_MPI_NP 4 CACHE STRING "Number of processes used for MPI tests")
 option(MFEM_ENABLE_TESTING "Enable the ctest framework for testing" ON)
 option(MFEM_ENABLE_EXAMPLES "Build all of the examples" OFF)
 option(MFEM_ENABLE_MINIAPPS "Build all of the miniapps" OFF)
-option(MFEM_ENABLE_GOOGLE_BENCHMARKS "Build all of the Google benchmarks" OFF)

 # Setting CXX/MPICXX on the command line or in user.cmake will overwrite the
 # autodetected C++ compiler.
@@ -92,9 +75,6 @@ option(MFEM_ENABLE_GOOGLE_BENCHMARKS "Build all of the Google benchmarks" OFF)
 # Set the target CUDA architecture
 set(CUDA_ARCH "sm_60" CACHE STRING "Target CUDA architecture.")

-# Set the target HIP architecture
-set(HIP_ARCH "gfx900" CACHE STRING "Target HIP architecture.")
-
 set(MFEM_DIR ${CMAKE_CURRENT_SOURCE_DIR})

 # The *_DIR paths below will be the first place searched for the corresponding
@@ -106,12 +86,6 @@ set(HYPRE_DIR "${MFEM_DIR}/../hypre/src/hypre" CACHE PATH
 # If hypre was compiled to depend on BLAS and LAPACK:
 # set(HYPRE_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
 #     "Packages that HYPRE depends on.")
-if (MFEM_USE_CUDA)
-   # This is only necessary when hypre is built with cuda:
-   set(HYPRE_REQUIRED_LIBRARIES "-lcusparse" "-lcurand" CACHE STRING
-       "Libraries that HYPRE depends on.")
-endif()
-# HIP dependency for HYPRE is handled in FindHYPRE.cmake.

 set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library.")

@@ -125,6 +99,9 @@ set(SUNDIALS_DIR "${MFEM_DIR}/../sundials-5.0.0/instdir" CACHE PATH
 # set(SUNDIALS_REQUIRED_PACKAGES "SuiteSparse/KLU/AMD/BTF/COLAMD/config"
 #     CACHE STRING "Additional packages required by SUNDIALS.")

+set(MESQUITE_DIR "${MFEM_DIR}/../mesquite-2.99" CACHE PATH
+    "Path to the Mesquite library.")
+
 set(SuiteSparse_DIR "${MFEM_DIR}/../SuiteSparse" CACHE PATH
    "Path to the SuiteSparse library.")
 set(SuiteSparse_REQUIRED_PACKAGES "BLAS" "METIS"
@@ -135,23 +112,12 @@ set(ParMETIS_DIR "${MFEM_DIR}/../parmetis-4.0.3" CACHE PATH
 set(ParMETIS_REQUIRED_PACKAGES "METIS" CACHE STRING
    "Additional packages required by ParMETIS.")

-set(SuperLUDist_DIR "${MFEM_DIR}/../SuperLU_DIST_8.1.2" CACHE PATH
+set(SuperLUDist_DIR "${MFEM_DIR}/../SuperLU_DIST_6.3.1" CACHE PATH
    "Path to the SuperLU_DIST library.")
 # SuperLU_DIST may also depend on "OpenMP", depending on how it was compiled.
-set(SuperLUDist_REQUIRED_PACKAGES "MPI" "ParMETIS" "METIS"
-    "LAPACK" "BLAS" CACHE STRING
+set(SuperLUDist_REQUIRED_PACKAGES "MPI" "BLAS" "ParMETIS" CACHE STRING
    "Additional packages required by SuperLU_DIST.")

-set(MUMPS_DIR "${MFEM_DIR}/../MUMPS_5.5.0" CACHE PATH
-    "Path to the MUMPS library.")
-# MUMPS may also depend on "OpenMP", depending on how it was compiled.
-set(MUMPS_REQUIRED_PACKAGES "MPI" "MPI_Fortran" "ParMETIS" "METIS"
-    "ScaLAPACK" "LAPACK" "BLAS" CACHE STRING
-    "Additional packages required by MUMPS.")
-# If the MPI package does not find all required Fortran libraries:
-# set(MUMPS_REQUIRED_LIBRARIES "gfortran" "mpi_mpifh" CACHE STRING
-#     "Additional libraries required by MUMPS.")
-
 set(STRUMPACK_DIR "${MFEM_DIR}/../STRUMPACK-build" CACHE PATH
    "Path to the STRUMPACK library.")
 # STRUMPACK may also depend on "OpenMP", depending on how it was compiled.
@@ -188,9 +154,9 @@ set(GNUTLS_DIR "" CACHE PATH "Path to the GnuTLS library.")

 set(GSLIB_DIR "" CACHE PATH "Path to the GSLIB library.")

-set(HDF5_DIR "/usr" CACHE PATH "Path to the HDF5 library.")
 set(NETCDF_DIR "" CACHE PATH "Path to the NetCDF library.")
-set(NetCDF_REQUIRED_PACKAGES "HDF5/C/HL" CACHE STRING
+# May need to add "HDF5" as requirement.
+set(NetCDF_REQUIRED_PACKAGES "" CACHE STRING
    "Additional packages required by NetCDF.")

 set(PETSC_DIR "${MFEM_DIR}/../petsc" CACHE PATH
@@ -203,12 +169,6 @@ set(SLEPC_ARCH "arch-linux2-c-debug" CACHE STRING "SLEPC build architecture.")

 set(MPFR_DIR "" CACHE PATH "Path to the MPFR library.")

-set(FMS_DIR "${MFEM_DIR}/../fms" CACHE PATH
-    "Path to the FMS library.")
-# If FMS is built with Conduit:
-# set(FMS_REQUIRED_PACKAGES "Conduit/relay" CACHE STRING
-#     "Additional packages required by FMS.")
-
 set(CONDUIT_DIR "${MFEM_DIR}/../conduit" CACHE PATH
    "Path to the Conduit library.")

@@ -229,38 +189,16 @@ set(MKL_CPARDISO_DIR "" CACHE STRING "MKL installation path.")
 set(MKL_MPI_WRAPPER_LIB "mkl_blacs_mpich_lp64" CACHE STRING "MKL MPI wrapper library")
 set(MKL_LIBRARY_DIR "" CACHE STRING "Custom library subdirectory")

-set(MKL_PARDISO_DIR "" CACHE STRING "MKL installation path.")
-
 set(OCCA_DIR "${MFEM_DIR}/../occa" CACHE PATH "Path to OCCA")
 set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")
 set(UMPIRE_DIR "${MFEM_DIR}/../umpire" CACHE PATH "Path to Umpire")
-set(CALIPER_DIR "${MFEM_DIR}/../caliper" CACHE PATH "Path to Caliper")
-set(BLITZ_DIR "${MFEM_DIR}/../blitz" CACHE PATH "Path to Blitz")
-set(ALGOIM_DIR "${MFEM_DIR}/../algoim" CACHE PATH "Path to Algoim")
-set(Algoim_REQUIRED_PACKAGES "Blitz" CACHE STRING
-    "Packages that ALGOIM depends on.")
-
-set(BENCHMARK_DIR "${MFEM_DIR}/../google-benchmark" CACHE PATH
-    "Path to Google Benchmark")
-
-# Provide paths, since ParELAG is dependent on MFEM and MFEM needs to be
-# compiled (or at least cmake needs to succeed) before compiling ParELAG.
-set(PARELAG_DIR "${MFEM_DIR}/../parelag" CACHE PATH "Path to ParELAG")
-set(PARELAG_INCLUDE_DIRS "${PARELAG_DIR}/src;${PARELAG_DIR}/build/src" CACHE
-    STRING "Path to ParELAG headers.")
-set(PARELAG_LIBRARIES "${PARELAG_DIR}/build/src/libParELAG.a" CACHE STRING
-    "The ParELAG library.")

 set(BLAS_INCLUDE_DIRS "" CACHE STRING "Path to BLAS headers.")
 set(BLAS_LIBRARIES "" CACHE STRING "The BLAS library.")
 set(LAPACK_INCLUDE_DIRS "" CACHE STRING "Path to LAPACK headers.")
 set(LAPACK_LIBRARIES "" CACHE STRING "The LAPACK library.")

-set(CODIPACK_INCLUDE_DIRS "${MFEM_DIR}/../CoDiPack/include" CACHE STRING
-    "Path to CoDiPack headers.")
-set(CODIPACK_LIBRARIES "")
-
 # Some useful variables:
 set(CMAKE_SKIP_PREPROCESSED_SOURCE_RULES ON) # Skip *.i rules
 set(CMAKE_SKIP_ASSEMBLY_SOURCE_RULES  ON)    # Skip *.s rules
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -18,9 +18,6 @@
 # Some choices below are based on the OS type:
 NOTMAC := $(subst Darwin,,$(shell uname -s))

-ETAGS_BIN = $(shell command -v etags 2> /dev/null)
-EGREP_BIN = $(shell command -v egrep 2> /dev/null)
-
 CXX = g++
 MPICXX = mpicxx

@@ -42,9 +39,6 @@ STATIC = YES
 SHARED = NO

 # CUDA configuration options
-#
-# If you set MFEM_USE_ENZYME=YES, CUDA_CXX has to be configured to use cuda with
-# clang as its host compiler.
 CUDA_CXX = nvcc
 CUDA_ARCH = sm_60
 CUDA_FLAGS = -x=cu --expt-extended-lambda -arch=$(CUDA_ARCH)
@@ -58,16 +52,11 @@ HIP_CXX = hipcc
 # example: gfx600 (tahiti), gfx700 (kaveri), gfx701 (hawaii), gfx801 (carrizo),
 # gfx900, gfx1010, etc.
 HIP_ARCH = gfx900
-HIP_FLAGS = --offload-arch=$(HIP_ARCH)
-HIP_XCOMPILER =
-HIP_XLINKER   = -Wl,
-
-# Flags for generating dependencies.
-DEP_FLAGS = -MM -MT
+HIP_FLAGS = --amdgpu-target=$(HIP_ARCH)

 ifneq ($(NOTMAC),)
   AR      = ar
-   ARFLAGS = crv
+   ARFLAGS = cruv
   RANLIB  = ranlib
   PICFLAG = $(XCOMPILER)-fPIC
   SO_EXT  = so
@@ -79,7 +68,7 @@ ifneq ($(NOTMAC),)
 else
   # Silence "has no symbols" warnings on Mac OS X
   AR      = ar
-   ARFLAGS = Scrv
+   ARFLAGS = Scruv
   RANLIB  = ranlib -no_warning_for_no_symbols
   PICFLAG = $(XCOMPILER)-fPIC
   SO_EXT  = dylib
@@ -92,9 +81,6 @@ else
   BUILD_RPATH = $(XLINKER)-undefined,dynamic_lookup
   INSTALL_SOFLAGS = $(subst $1 ,,$(call MAKE_SOFLAGS,$(MFEM_LIB_DIR)))
   INSTALL_RPATH = $(XLINKER)-undefined,dynamic_lookup
-   # Silence unused command line argument warnings when generating dependencies
-   # with mpicxx and clang
-   DEP_FLAGS := -Wno-unused-command-line-argument $(DEP_FLAGS)
 endif

 # Set CXXFLAGS to overwrite the default selection of DEBUG_FLAGS/OPTIM_FLAGS
@@ -131,10 +117,10 @@ MFEM_USE_LEGACY_OPENMP = NO
 MFEM_USE_MEMALLOC      = YES
 MFEM_TIMER_TYPE        = $(if $(NOTMAC),2,4)
 MFEM_USE_SUNDIALS      = NO
+MFEM_USE_MESQUITE      = NO
 MFEM_USE_SUITESPARSE   = NO
 MFEM_USE_SUPERLU       = NO
 MFEM_USE_SUPERLU5      = NO
-MFEM_USE_MUMPS         = NO
 MFEM_USE_STRUMPACK     = NO
 MFEM_USE_GINKGO        = NO
 MFEM_USE_AMGX          = NO
@@ -144,7 +130,6 @@ MFEM_USE_PETSC         = NO
 MFEM_USE_SLEPC         = NO
 MFEM_USE_MPFR          = NO
 MFEM_USE_SIDRE         = NO
-MFEM_USE_FMS           = NO
 MFEM_USE_CONDUIT       = NO
 MFEM_USE_PUMI          = NO
 MFEM_USE_HIOP          = NO
@@ -154,49 +139,16 @@ MFEM_USE_HIP           = NO
 MFEM_USE_RAJA          = NO
 MFEM_USE_OCCA          = NO
 MFEM_USE_CEED          = NO
-MFEM_USE_CALIPER       = NO
-MFEM_USE_ALGOIM        = NO
 MFEM_USE_UMPIRE        = NO
 MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
 MFEM_USE_MKL_CPARDISO  = NO
-MFEM_USE_MKL_PARDISO   = NO
-MFEM_USE_MOONOLITH     = NO
-MFEM_USE_ADFORWARD     = NO
-MFEM_USE_CODIPACK      = NO
-MFEM_USE_BENCHMARK     = NO
-MFEM_USE_PARELAG       = NO
-MFEM_USE_ENZYME        = NO
-
-# MPI library compile and link flags
-# These settings are used only when building MFEM with MPI + HIP
-ifeq ($(MFEM_USE_MPI)$(MFEM_USE_HIP),YESYES)
-   # We determine MPI_DIR assuming $(MPICXX) is in $(MPI_DIR)/bin
-   MPI_DIR := $(patsubst %/,%,$(dir $(shell which $(MPICXX))))
-   MPI_DIR := $(patsubst %/,%,$(dir $(MPI_DIR)))
-   MPI_OPT = -I$(MPI_DIR)/include
-   MPI_LIB = -L$(MPI_DIR)/lib $(XLINKER)-rpath,$(MPI_DIR)/lib -lmpi
-endif
-
-# ROCM/HIP directory such that ROCM/HIP libraries like rocsparse and rocrand are
-# found in $(HIP_DIR)/lib, usually as links. Typically, this directory is of
-# the form /opt/rocm-X.Y.Z which is called ROCM_PATH by hipconfig.
-ifeq ($(MFEM_USE_HIP),YES)
-   HIP_DIR := $(patsubst %/,%,$(dir $(shell which $(HIP_CXX))))
-   HIP_DIR := $(patsubst %/,%,$(dir $(HIP_DIR)))
-   ifeq (,$(wildcard $(HIP_DIR)/lib/librocsparse.*))
-      HIP_DIR := $(shell hipconfig --rocmpath 2> /dev/null)
-      ifeq (,$(wildcard $(HIP_DIR)/lib/librocsparse.*))
-         $(error Unable to determine HIP_DIR. Please set it manually.)
-      endif
-   endif
-endif

 # Compile and link options for zlib.
 ZLIB_DIR =
 ZLIB_OPT = $(if $(ZLIB_DIR),-I$(ZLIB_DIR)/include)
 ZLIB_LIB = $(if $(ZLIB_DIR),$(ZLIB_RPATH) -L$(ZLIB_DIR)/lib ,)-lz
-ZLIB_RPATH = $(XLINKER)-rpath,$(ZLIB_DIR)/lib
+ZLIB_RPATH = -Wl,-rpath,$(ZLIB_DIR)/lib

 LIBUNWIND_OPT = -g
 LIBUNWIND_LIB = $(if $(NOTMAC),-lunwind -ldl,)
@@ -205,18 +157,9 @@ LIBUNWIND_LIB = $(if $(NOTMAC),-lunwind -ldl,)
 HYPRE_DIR = @MFEM_DIR@/../hypre/src/hypre
 HYPRE_OPT = -I$(HYPRE_DIR)/include
 HYPRE_LIB = -L$(HYPRE_DIR)/lib -lHYPRE
-ifeq (YES,$(MFEM_USE_CUDA))
-   # This is only necessary when hypre is built with cuda:
-   HYPRE_LIB += -lcusparse -lcurand -lcublas
-endif
-ifeq (YES,$(MFEM_USE_HIP))
-   # This is only necessary when hypre is built with hip:
-   HYPRE_LIB += -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib\
- -lrocsparse -lrocrand
-endif

 # METIS library configuration
-ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK)$(MFEM_USE_MUMPS),NONONO)
+ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK),NONO)
   ifeq ($(MFEM_USE_METIS_5),NO)
     METIS_DIR = @MFEM_DIR@/../metis-4.0
     METIS_OPT =
@@ -251,47 +194,43 @@ POSIX_CLOCKS_LIB = -lrt
 # SUNDIALS library configuration
 # For sundials_nvecmpiplusx and nvecparallel remember to build with MPI_ENABLE=ON
 # and modify cmake variables for hypre for sundials
-SUNDIALS_DIR = @MFEM_DIR@/../sundials-5.0.0/instdir
-# SUNDIALS >= 6.4.0 requires C++14:
-ifeq ($(MFEM_USE_SUNDIALS),YES)
-   BASE_FLAGS = -std=c++14
-endif
-SUNDIALS_OPT = -I$(SUNDIALS_DIR)/include
-SUNDIALS_LIB = $(XLINKER)-rpath,$(SUNDIALS_DIR)/lib64\
- $(XLINKER)-rpath,$(SUNDIALS_DIR)/lib\
- -L$(SUNDIALS_DIR)/lib64 -L$(SUNDIALS_DIR)/lib\
+SUNDIALS_DIR    = @MFEM_DIR@/../sundials-5.0.0/instdir
+SUNDIALS_OPT    = -I$(SUNDIALS_DIR)/include
+SUNDIALS_LIBDIR = $(wildcard $(SUNDIALS_DIR)/lib*)
+SUNDIALS_LIB    = $(XLINKER)-rpath,$(SUNDIALS_LIBDIR) -L$(SUNDIALS_LIBDIR)\
 -lsundials_arkode -lsundials_cvodes -lsundials_nvecserial -lsundials_kinsol
+
 ifeq ($(MFEM_USE_MPI),YES)
   SUNDIALS_LIB += -lsundials_nvecparallel -lsundials_nvecmpiplusx
 endif
 ifeq ($(MFEM_USE_CUDA),YES)
   SUNDIALS_LIB += -lsundials_nveccuda
 endif
-ifeq ($(MFEM_USE_HIP),YES)
-   SUNDIALS_LIB += -lsundials_nvechip
-endif
 # If SUNDIALS was built with KLU:
 # MFEM_USE_SUITESPARSE = YES

+# MESQUITE library configuration
+MESQUITE_DIR = @MFEM_DIR@/../mesquite-2.99
+MESQUITE_OPT = -I$(MESQUITE_DIR)/include
+MESQUITE_LIB = -L$(MESQUITE_DIR)/lib -lmesquite
+
 # SuiteSparse library configuration
 LIB_RT = $(if $(NOTMAC),-lrt,)
 SUITESPARSE_DIR = @MFEM_DIR@/../SuiteSparse
 SUITESPARSE_OPT = -I$(SUITESPARSE_DIR)/include
-SUITESPARSE_LIB = $(XLINKER)-rpath,$(SUITESPARSE_DIR)/lib\
- -L$(SUITESPARSE_DIR)/lib -lklu -lbtf -lumfpack -lcholmod -lcolamd -lamd -lcamd\
- -lccolamd -lsuitesparseconfig $(LIB_RT) $(METIS_LIB) $(LAPACK_LIB)
+SUITESPARSE_LIB = -Wl,-rpath,$(SUITESPARSE_DIR)/lib -L$(SUITESPARSE_DIR)/lib\
+ -lklu -lbtf -lumfpack -lcholmod -lcolamd -lamd -lcamd -lccolamd\
+ -lsuitesparseconfig $(LIB_RT) $(METIS_LIB) $(LAPACK_LIB)

 # SuperLU library configuration
 ifeq ($(MFEM_USE_SUPERLU5),YES)
   SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_5.1.0
   SUPERLU_OPT = -I$(SUPERLU_DIR)/include
-   SUPERLU_LIB = $(XLINKER)-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib\
-      -lsuperlu_dist_5.1.0
+   SUPERLU_LIB = -Wl,-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib -lsuperlu_dist_5.1.0   
 else
-   SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_8.1.2
+   SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_6.3.1
   SUPERLU_OPT = -I$(SUPERLU_DIR)/include
-   SUPERLU_LIB = $(XLINKER)-rpath,$(SUPERLU_DIR)/lib64 -L$(SUPERLU_DIR)/lib64\
-      -lsuperlu_dist $(LAPACK_LIB)
+   SUPERLU_LIB = -Wl,-rpath,$(SUPERLU_DIR)/lib64 -L$(SUPERLU_DIR)/lib64 -lsuperlu_dist -lblas
 endif

 # SCOTCH library configuration (required by STRUMPACK <= v2.1.0, optional in
@@ -301,70 +240,32 @@ SCOTCH_OPT = -I$(SCOTCH_DIR)/include
 SCOTCH_LIB = -L$(SCOTCH_DIR)/lib -lptscotch -lptscotcherr -lscotch -lscotcherr\
 -lpthread

-# SCALAPACK library configuration (required by STRUMPACK and MUMPS)
+# SCALAPACK library configuration (required by STRUMPACK)
 SCALAPACK_DIR = @MFEM_DIR@/../scalapack-2.0.2
 SCALAPACK_OPT = -I$(SCALAPACK_DIR)/SRC
 SCALAPACK_LIB = -L$(SCALAPACK_DIR)/lib -lscalapack $(LAPACK_LIB)

-# MPI Fortran library, needed e.g. by STRUMPACK or MUMPS
+# MPI Fortran library, needed e.g. by STRUMPACK
 # MPICH:
 MPI_FORTRAN_LIB = -lmpifort
 # OpenMPI:
 # MPI_FORTRAN_LIB = -lmpi_mpifh
-# Additional Fortran library:
+# Additional Fortan library:
 # MPI_FORTRAN_LIB += -lgfortran

-# MUMPS library configuration
-MUMPS_DIR = @MFEM_DIR@/../MUMPS_5.5.0
-MUMPS_OPT = -I$(MUMPS_DIR)/include
-MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib -ldmumps\
- -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB)
-
 # STRUMPACK library configuration
 STRUMPACK_DIR = @MFEM_DIR@/../STRUMPACK-build
-ifeq ($(MFEM_USE_STRUMPACK),YES)
-   BASE_FLAGS = -std=c++14
-endif
 STRUMPACK_OPT = -I$(STRUMPACK_DIR)/include $(SCOTCH_OPT)
 # If STRUMPACK was build with OpenMP support, the following may be need:
 # STRUMPACK_OPT += $(OPENMP_OPT)
 STRUMPACK_LIB = -L$(STRUMPACK_DIR)/lib -lstrumpack $(MPI_FORTRAN_LIB)\
 $(SCOTCH_LIB) $(SCALAPACK_LIB)

-# Ginkgo library configuration
+# Ginkgo library configuration (currently not needed)
 GINKGO_DIR = @MFEM_DIR@/../ginkgo/install
-GINKGO_SEARCH_DIR = $(subst @MFEM_DIR@,$(MFEM_DIR),$(GINKGO_DIR))
-GINKGO_BUILD_TYPE=Release
-ifeq ($(MFEM_USE_GINKGO),YES)
-   BASE_FLAGS = -std=c++14
-endif
 GINKGO_OPT = -isystem $(GINKGO_DIR)/include
-GINKGO_LIB_DIR = $(sort $(dir $(wildcard\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.a\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.so\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.dylib\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.dll)))
-GINKGO_LINK_LIB_DIR = $(GINKGO_DIR)$(subst $(GINKGO_SEARCH_DIR),,$(GINKGO_LIB_DIR))
-ALL_GINKGO_LIBS_DEBUG = $(notdir $(basename $(wildcard\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*d.a\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*d.so\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*d.dylib\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*d.dll)))
-ALL_GINKGO_LIBS = $(notdir $(basename $(wildcard\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.a\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.so\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.dylib\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.dll)))
-ALL_GINKGO_LIBS_RELEASE = $(filter-out $(ALL_GINKGO_LIBS_DEBUG),$(ALL_GINKGO_LIBS))
-GINKGO_LINK = $(subst libginkgo,-lginkgo,$(ALL_GINKGO_LIBS_RELEASE))
-ifeq ($(GINKGO_BUILD_TYPE),Debug)
-  ifneq (,$(ALL_GINKGO_LIBS_DEBUG))
-    GINKGO_LINK = $(subst libginkgo,-lginkgo,$(ALL_GINKGO_LIBS_DEBUG))
-  endif
-else
-endif
-GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_LINK_LIB_DIR) -L$(GINKGO_LINK_LIB_DIR)\
- $(GINKGO_LINK)
+GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_DIR)/lib -L$(GINKGO_DIR)/lib -lginkgo\
+ -lginkgo_omp -lginkgo_cuda -lginkgo_reference

 # AmgX library configuration
 AMGX_DIR = @MFEM_DIR@/../amgx
@@ -379,8 +280,8 @@ GNUTLS_LIB = -lgnutls
 NETCDF_DIR = $(HOME)/local
 HDF5_DIR   = $(HOME)/local
 NETCDF_OPT = -I$(NETCDF_DIR)/include -I$(HDF5_DIR)/include $(ZLIB_OPT)
-NETCDF_LIB = $(XLINKER)-rpath,$(NETCDF_DIR)/lib -L$(NETCDF_DIR)/lib\
- $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib\
+NETCDF_LIB = -Wl,-rpath,$(NETCDF_DIR)/lib -L$(NETCDF_DIR)/lib\
+ -Wl,-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib\
 -lnetcdf -lhdf5_hl -lhdf5 $(ZLIB_LIB)

 # PETSc library configuration (version greater or equal to 3.8 or the dev branch)
@@ -392,10 +293,9 @@ PETSC_INC_VAR = PETSC_CC_INCLUDES
 PETSC_LIB_VAR = PETSC_EXTERNAL_LIB_BASIC
 ifeq ($(PETSC_FOUND),YES)
   PETSC_OPT := $(shell sed -n "s/$(PETSC_INC_VAR) = *//p" $(PETSC_VARS))
-   PETSC_DEP := $(shell sed -n "s/$(PETSC_LIB_VAR) = *//p" $(PETSC_VARS))
-   PETSC_LIB = $(XLINKER)-rpath,$(abspath $(PETSC_DIR))/lib\
-      -L$(abspath $(PETSC_DIR))/lib -lpetsc\
-      $(subst $(CXX_XLINKER),$(XLINKER),$(PETSC_DEP))
+   PETSC_LIB := $(shell sed -n "s/$(PETSC_LIB_VAR) = *//p" $(PETSC_VARS))
+   PETSC_LIB := -Wl,-rpath,$(abspath $(PETSC_DIR))/lib\
+      -L$(abspath $(PETSC_DIR))/lib -lpetsc $(PETSC_LIB)
 endif

 SLEPC_DIR := $(MFEM_DIR)/../slepc
@@ -407,31 +307,20 @@ ifeq ($(SLEPC_FOUND),YES)
   SLEPC_OPT := $(shell sed -n "s/$(SLEPC_INC_VAR) *= *//p" $(SLEPC_VARS))
   # Some additional external libraries might be defined in this file
   -include ${SLEPC_DIR}/${PETSC_ARCH}/lib/slepc/conf/slepcvariables
-   SLEPC_DEP := $(shell sed -n "s/$(SLEPC_LIB_VAR) *= *//p" $(SLEPC_VARS))
-   SLEPC_LIB = $(XLINKER)-rpath,$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib\
-      -L$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib -lslepc\
-      $(subst $(CXX_XLINKER),$(XLINKER),$(SLEPC_DEP))
-endif
-
-ifeq ($(MFEM_USE_MOONOLITH),YES)
-  include $(MOONOLITH_DIR)/config/moonolith-config.makefile
-  MOONOLITH_LIB=$(MOONOLITH_LIBRARIES)
+   SLEPC_LIB := $(shell sed -n "s/$(SLEPC_LIB_VAR) *= *//p" $(SLEPC_VARS))
+   SLEPC_LIB := -Wl,-rpath,$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib\
+      -L$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib -lslepc $(SLEPC_LIB)
 endif

 # MPFR library configuration
 MPFR_OPT =
 MPFR_LIB = -lmpfr

-# FMS and required libraries configuration
-FMS_DIR = $(MFEM_DIR)/../fms
-FMS_OPT = -I$(FMS_DIR)/include
-FMS_LIB = -Wl,-rpath,$(FMS_DIR)/lib -L$(FMS_DIR)/lib -lfms
-
 # Conduit and required libraries configuration
 CONDUIT_DIR = @MFEM_DIR@/../conduit
 CONDUIT_OPT = -I$(CONDUIT_DIR)/include/conduit
 CONDUIT_LIB = \
-   $(XLINKER)-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \
+   -Wl,-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \
   -lconduit -lconduit_relay -lconduit_blueprint  -ldl

 # Check if Conduit was built with hdf5 support, by looking
@@ -439,7 +328,7 @@ CONDUIT_LIB = \
 CONDUIT_HDF5_HEADER=$(CONDUIT_DIR)/include/conduit/conduit_relay_hdf5.hpp
 ifneq (,$(wildcard $(CONDUIT_HDF5_HEADER)))
   CONDUIT_OPT += -I$(HDF5_DIR)/include
-   CONDUIT_LIB += $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \
+   CONDUIT_LIB += -Wl,-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \
                  -lhdf5 $(ZLIB_LIB)
 endif

@@ -449,9 +338,9 @@ SIDRE_DIR = @MFEM_DIR@/../axom
 SIDRE_OPT = -I$(SIDRE_DIR)/include -I$(CONDUIT_DIR)/include/conduit\
 -I$(HDF5_DIR)/include
 SIDRE_LIB = \
-   $(XLINKER)-rpath,$(SIDRE_DIR)/lib -L$(SIDRE_DIR)/lib \
-   $(XLINKER)-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \
-   $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \
+   -Wl,-rpath,$(SIDRE_DIR)/lib -L$(SIDRE_DIR)/lib \
+   -Wl,-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \
+   -Wl,-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \
   -laxom -lconduit -lconduit_relay -lconduit_blueprint -lhdf5 $(ZLIB_LIB) -ldl

 # PUMI
@@ -466,11 +355,6 @@ HIOP_DIR = @MFEM_DIR@/../hiop/install
 HIOP_OPT = -I$(HIOP_DIR)/include
 HIOP_LIB = -L$(HIOP_DIR)/lib -lhiop $(LAPACK_LIB)

-# CoDiPack
-CODIPACK_DIR = @MFEM_DIR@/../CoDiPack
-CODIPACK_OPT = -I$(CODIPACK_DIR)
-CODIPACK_LIB =
-
 # GSLIB library
 GSLIB_DIR = @MFEM_DIR@/../gslib/build
 GSLIB_OPT = -I$(GSLIB_DIR)/include
@@ -480,111 +364,41 @@ GSLIB_LIB = -L$(GSLIB_DIR)/lib -lgs
 CUDA_OPT =
 CUDA_LIB = -lcusparse

-# HIP library configuration
+# HIP library configuration (currently not needed)
 HIP_OPT =
-HIP_LIB = -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib -lhipsparse
+HIP_LIB =

 # OCCA library configuration
 OCCA_DIR = @MFEM_DIR@/../occa
 OCCA_OPT = -I$(OCCA_DIR)/include
 OCCA_LIB = $(XLINKER)-rpath,$(OCCA_DIR)/lib -L$(OCCA_DIR)/lib -locca

-# CALIPER library configuration
-CALIPER_DIR = @MFEM_DIR@/../caliper
-CALIPER_OPT = -I$(CALIPER_DIR)/include
-CALIPER_LIB = $(XLINKER)-rpath,$(CALIPER_DIR)/lib64 $(XLINKER)-rpath,$(CALIPER_DIR)/lib -L$(CALIPER_DIR)/lib64 -L$(CALIPER_DIR)/lib -lcaliper
-
-ifdef ADIAK_DIR
-   CALIPER_OPT += -I$(ADIAK_DIR)/include
-   CALIPER_LIB += $(XLINKER)-rpath,$(ADIAK_DIR)/lib64 $(XLINKER)-rpath,$(ADIAK_DIR)/lib -L$(ADIAK_DIR)/lib64 -L$(ADIAK_DIR)/lib -ladiak
-endif
-ifdef GOTCHA_DIR
-   CALIPER_OPT += -I$(GOTCHA_DIR)/include
-   CALIPER_LIB += $(XLINKER)-rpath,$(GOTCHA_DIR)/lib64 $(XLINKER)-rpath,$(GOTCHA_DIR)/lib -L$(GOTCHA_DIR)/lib64 -L$(GOTCHA_DIR)/lib -lgotcha
-endif
-
-# BLITZ library configuration
-BLITZ_DIR = @MFEM_DIR@/../blitz
-BLITZ_OPT = -I$(BLITZ_DIR)/include
-BLITZ_LIB = $(XLINKER)-rpath,$(BLITZ_DIR)/lib -L$(BLITZ_DIR)/lib -lblitz
-
-# ALGOIM library configuration
-ALGOIM_DIR = @MFEM_DIR@/../algoim
-ALGOIM_OPT = -I$(ALGOIM_DIR)/src $(BLITZ_OPT)
-ALGOIM_LIB = $(BLITZ_LIB)
-
-# BENCHMARK library configuration
-BENCHMARK_DIR = @MFEM_DIR@/../google-benchmark
-BENCHMARK_OPT = -I$(BENCHMARK_DIR)/include
-BENCHMARK_LIB = -L$(BENCHMARK_DIR)/lib -lbenchmark -lpthread
-
 # libCEED library configuration
 CEED_DIR ?= @MFEM_DIR@/../libCEED
 CEED_OPT = -I$(CEED_DIR)/include
 CEED_LIB = $(XLINKER)-rpath,$(CEED_DIR)/lib -L$(CEED_DIR)/lib -lceed

 # RAJA library configuration
-ifeq ($(MFEM_USE_RAJA),YES)
-   BASE_FLAGS = -std=c++14
-endif
 RAJA_DIR = @MFEM_DIR@/../raja
 RAJA_OPT = -I$(RAJA_DIR)/include
 ifdef CUB_DIR
   RAJA_OPT += -I$(CUB_DIR)
 endif
-
-CAMP_LIB = -lcamp
-ifdef CAMP_DIR
-   RAJA_OPT += -I$(CAMP_DIR)/include
-   CAMP_LIB = $(XLINKER)-rpath,$(CAMP_DIR)/lib -L$(CAMP_DIR)/lib -lcamp
-endif
-RAJA_LIB = $(XLINKER)-rpath,$(RAJA_DIR)/lib -L$(RAJA_DIR)/lib -lRAJA $(CAMP_LIB)
+RAJA_LIB = $(XLINKER)-rpath,$(RAJA_DIR)/lib -L$(RAJA_DIR)/lib -lRAJA

 # UMPIRE library configuration
-ifeq ($(MFEM_USE_UMPIRE),YES)
-   BASE_FLAGS = -std=c++14
-endif
 UMPIRE_DIR = @MFEM_DIR@/../umpire
-UMPIRE_OPT = -I$(UMPIRE_DIR)/include $(if $(CAMP_DIR), -I$(CAMP_DIR)/include)
-UMPIRE_LIB = -L$(UMPIRE_DIR)/lib -lumpire $(CAMP_LIB)
+UMPIRE_OPT = -I$(UMPIRE_DIR)/include
+UMPIRE_LIB = -L$(UMPIRE_DIR)/lib -lumpire

 # MKL CPardiso library configuration
 MKL_CPARDISO_DIR ?=
 MKL_MPI_WRAPPER ?= mkl_blacs_mpich_lp64
 MKL_LIBRARY_SUBDIR ?= lib
 MKL_CPARDISO_OPT = -I$(MKL_CPARDISO_DIR)/include
-MKL_CPARDISO_LIB = $(XLINKER)-rpath,$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
-   -L$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR) -l$(MKL_MPI_WRAPPER)\
-   -lmkl_intel_lp64 -lmkl_sequential -lmkl_core
-
-# MKL Pardiso library configuration
-MKL_PARDISO_DIR ?=
-MKL_LIBRARY_SUBDIR ?= lib
-MKL_PARDISO_OPT = -I$(MKL_PARDISO_DIR)/include
-MKL_PARDISO_LIB = $(XLINKER)-rpath,$(MKL_PARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
-   -L$(MKL_PARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
-   -lmkl_intel_lp64 -lmkl_sequential -lmkl_core
-
-# PARELAG library configuration
-PARELAG_DIR = @MFEM_DIR@/../parelag
-PARELAG_OPT = -I$(PARELAG_DIR)/src -I$(PARELAG_DIR)/build/src
-PARELAG_LIB = -L$(PARELAG_DIR)/build/src -lParELAG
-
-# Enzyme configuration
-
-# If you want to enable automatic differentiation at compile time, use the
-# options below, adapted to your configuration. To be more flexible, we
-# recommend using the Enzyme plugin during link time optimization. One option is
-# to add your options to the global compiler/linker flags like
-#
-# BASE_FLAGS += -flto
-# CXX_XLINKER += -fuse-ld=lld -Wl,--lto-legacy-pass-manager\
-#                -Wl,-mllvm=-load=$(ENZYME_DIR)/LLDEnzyme-$(ENZYME_VERSION).so -Wl,
-#
-ENZYME_DIR ?= @MFEM_DIR@/../enzyme
-ENZYME_VERSION ?= 14
-ENZYME_OPT = -fno-experimental-new-pass-manager -Xclang -load -Xclang $(ENZYME_DIR)/ClangEnzyme-$(ENZYME_VERSION).so
-ENZYME_LIB = ""
+MKL_CPARDISO_LIB = -Wl,-rpath,$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
+                   -L$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR) -l$(MKL_MPI_WRAPPER)\
+                   -lmkl_intel_lp64 -lmkl_sequential -lmkl_core

 # If YES, enable some informational messages
 VERBOSE = NO
@@ -1,25 +0,0 @@
-FROM ghcr.io/mfem/mfem-ubuntu-base:latest as builder
-
-# docker build -t ghcr.io/mfem/mfem-ubuntu .
-
-COPY ./config/docker/spack.yaml /opt/mfem-env/spack.yaml
-RUN apt-get install -y python3 && \
-    cd /opt/mfem-env && \
-    . /opt/spack/share/spack/setup-env.sh && \
-    spack env activate . && \
-    spack env view regenerate
-
-FROM ubuntu:22.04
-
-COPY --from=builder /opt/view /opt/view
-COPY --from=builder /opt/mfem-view /opt/mfem-view
-
-RUN apt-get update && \
-    apt-get install -y unzip gfortran && \
-    apt-get install -y libcurl4-openssl-dev libssl-dev
-
-ENV PATH=$PATH:/opt/mfem-view/bin
-ENV DEBIAN_FRONTEND=noninteractive
-
-# The user will see the view on shell into the container
-WORKDIR /opt/mfem-view
@@ -1,47 +0,0 @@
-FROM ghcr.io/rse-ops/cuda-ubuntu-20.04:cuda-11.0.3
-
-# docker build -f Dockerfile.base -t ghcr.io/mfem/mfem-ubuntu-base .
-
-RUN apt-get update && \
-    apt-get install -y unzip gfortran && \
-    spack compiler find && \
-    apt-get install -y libcurl4-openssl-dev libssl-dev
-
-# /code is the working directory for code
-WORKDIR /code
-COPY . /code
-
-# This is for a spack environment/view to install from there
-RUN mkdir -p /opt/mfem-env \
-&&  (echo "spack:" \
-&&   echo "  view:" \
-&&   echo "    mfem:" \
-&&   echo "      root: /opt/mfem-view" \
-&&   echo "      link_type: copy" \
-&&   echo "  packages:" \
-&&   echo "    all:" \
-&&   echo "      target:" \
-&&   echo "      - x86_64_v3" \
-&&   echo "  config:" \
-&&   echo "    concretizer: clingo" \
-&&   echo "    compiler:" \
-&&   echo "      target:" \
-&&   echo "      - x86_64_v3" \
-&&   echo "    install_missing_compilers: true" \
-&&   echo "  concretization: together") > /opt/mfem-env/spack.yaml
-
-RUN cd /opt/mfem-env && \
-    . /opt/spack/share/spack/setup-env.sh && \
-    spack env activate . && \
-    spack develop --path /code mfem@master+examples+miniapps && \
-    spack add mfem@master+examples+miniapps && \
-    spack install
-
-# ensure mfem always on various paths
-RUN cd /opt/mfem-env && \
-    spack env activate --sh -d . >> /etc/profile.d/z10_spack_environment.sh
-
-# Present the software install when we shell in
-# The view is at /opt/mfem-env/.spack-env/view
-WORKDIR /opt/software
-ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l", "-c"]
@@ -1,209 +0,0 @@
-# mfem Docker
-
-We provide a [Dockerfile.base](Dockerfile.base) to build an ubuntu base image,
-and a [Dockerfile](Dockerfile) to build a smaller one with a multi-stage build.
-You can use this image for a demo of using mfem! 🎉️
-
-Updated containers are built and deployed on merges to the main branch and releases.
-If you want to request a build on demand, you can [manually run the workflow](https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow) thanks to the workflow dispatch event.
-
-## Usage
-
-We provide two containers, which you can either build or use directly from
-[GitHub packages](https://github.com/orgs/mfem/packages?repo_name=mfem).
-
- - `ghcr.io/mfem/mfem-ubuntu-base`: a "build from scratch" for mfem
- - `ghcr.io/mfem/mfem-ubuntu`: a quick build that uses the base container
-
-In the above, "ghcr.io" means "GitHub Container Registry" and
-is the [GitHub packages](https://github.com/features/packages) registry that supports
-Docker images and other OCI artifacts. 
-
-### Ubuntu
-
-> Use or build this container for a multi-stage, slimmer base to develop on top of mfem
-
-Note that this container is provided on GitHub packages [here](https://github.com/mfem/mfem/pkgs/container/mfem-ubuntu)
-so you don't need to build it. However, if you want to, you can do the following:
-
-```bash
-$ docker build -f config/docker/Dockerfile -t ghcr.io/mfem/mfem-ubuntu .
-```
-
-Note that this will pull the base image. If you want to rebuild it, see [ubuntu base](#ubuntu-base)
-below. Once you have built (or prefer to pull) you can shell into the container as follows:
-
-```bash
-$ docker run -it ghcr.io/mfem/mfem-ubuntu
-```
-
-This smaller image has a view where everything is installed.
-
-```bash
-$ ls
-bin  etc  include  lib  libexec  sbin  share  var
-```
-
- - Examples are in share/mfem/examples
- - Examples are in share/mfem/miniapps
-
-Using this container, if you want to develop a tool that _uses_ mfem, you can find the libraries / includes in:
-
-```bash
-$ ls include/ | grep mfem
-mfem
-mfem-performance.hpp
-mfem.hpp
-```
-
-And yes, this is the working directory when you shell into the container!
-You can find the examples here:
-
-
-```bash
-cd share/mfem/examples
-```
-
-Try quickly setting the `LD_LIBRARY_PATH` so we can see the shared libraries
-we need:
-
-```bash
-export LD_LIBRARY_PATH=/opt/mfem-view/lib:$LD_LIBRARY_PATH
-```
-
-And then run:
-
-```bash
-$ ./ex0
-Options used:
-   --mesh ../data/star.mesh
-   --order 1
-Number of unknowns: 101
-   Iteration :   0  (B r, r) = 0.184259
-   Iteration :   1  (B r, r) = 0.102754
-   Iteration :   2  (B r, r) = 0.00558141
-   Iteration :   3  (B r, r) = 1.5247e-05
-   Iteration :   4  (B r, r) = 1.13807e-07
-   Iteration :   5  (B r, r) = 6.27231e-09
-   Iteration :   6  (B r, r) = 3.76268e-11
-   Iteration :   7  (B r, r) = 6.07423e-13
-   Iteration :   8  (B r, r) = 4.10615e-15
-Average reduction factor = 0.140201
-```
-
-You can also explore the "mini apps," also in share/mfem, but under miniapps.
-
-```bash
-# This is run from the examples directory
-$ cd ../miniapps
-```
-```bash
-$ ls
-CMakeLists.txt  common            meshing  nurbs        shifted  toys
-adjoint         electromagnetics  mtop     parelag      solvers
-autodiff        gslib             navier   performance  tools
-```
-
-And an example in "toys"
-
-```bash
-cd toys
-```
-```bash
-$ ./automata -no-vis
-Options used:
-   --num-steps 16
-   --rule 90
-   --no-visualization
-
-Rule:
- 111 110 101 100 011 010 001 000
-  0   1   0   1   1   0   1   0
-
-Applying rule...done.
-```
-
-Have fun! As a reminder, this container is ideal for developing your own
-applications that might use mfem, or having a nice environment to test out
-examples.
-
-
-### Ubuntu Base
-
-> Use this build for a development environment with spack and mfem
-
-This container is also [provided on GitHub packages](https://github.com/mfem/mfem/pkgs/container/mfem-ubuntu-base), 
-however you can build it locally too:
-
-```bash
-$ docker build -f config/docker/Dockerfile.base -t ghcr.io/mfem/mfem-ubuntu-base .
-```
-
-To shell into the container:
-
-```bash
-$ docker run -it ghcr.io/mfem/mfem-ubuntu-base bash
-```
-
-Change directory to the mfem environment, setup spack, and activate the environment:
-
-```bash
-source /opt/spack/share/spack/setup-env.sh
-cd /opt/mfem-env/
-spack env activate .
-```
-
-Note that this environment is installing to the view at `/opt/view`. Since the environment
-knows to install mfem from `/code` this means that you could make changes in the container (or bind
-`/code` to your container) and then update spack:
-
-```bash
-# Note that concretization takes a hot minute!
-$ spack install
-```
-
-And if you want to load mfem:
-
-```bash
-$ spack load mfem
-$ env | grep mfem
-```
-
-In this development container, you can find the examples and miniapps alongside
-mfem under `/code`:
-
-```bash
-cd /code/examples
-```
-```bash
-$ ./ex0
-```
-```console
-Options used:
-   --mesh ../data/star.mesh
-   --order 1
-Number of unknowns: 101
-   Iteration :   0  (B r, r) = 0.184259
-   Iteration :   1  (B r, r) = 0.102754
-   Iteration :   2  (B r, r) = 0.00558141
-   Iteration :   3  (B r, r) = 1.5247e-05
-   Iteration :   4  (B r, r) = 1.13807e-07
-   Iteration :   5  (B r, r) = 6.27231e-09
-   Iteration :   6  (B r, r) = 3.76268e-11
-   Iteration :   7  (B r, r) = 6.07423e-13
-   Iteration :   8  (B r, r) = 4.10615e-15
-Average reduction factor = 0.140201
-```
-
-This container is likely ideal for someone that wants to develop mfem itself.
-For other use cases, we recommend using the slimmer image. As an example,
-if you want to develop with your own code base (and mfem as is in the container) 
-you can bind to somewhere else in the container (e.g., src)
-
-```bash
-$ docker run -it ghcr.io/mfem/mfem-ubuntu-base -v $PWD:/code bash
-```
-
-In the above, we can pretend your project is in the present working directory (PWD) and we are
-binding to source. You can then use the mfem in the container for development, and if you
-want to distribute your library or app in a container, you can use the mfem container as the base.
@@ -1,11 +0,0 @@
-spack:
-  specs: [mfem@master+examples+miniapps]
-  view: 
-    mfem:
-      root: /opt/mfem-view 
-      link_type: copy
-  concretization: together
-  develop:
-    mfem:
-      path: /code
-      spec: mfem@master+examples+miniapps
@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -1,33 +0,0 @@
-// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "dmumps_c.h"
-#include <string>
-#include <iostream>
-#include <algorithm>
-
-// Macros to expand a macro as a string
-#define STR_EXPAND(s) #s
-#define STR(s) STR_EXPAND(s)
-
-int main()
-{
-#ifdef MUMPS_VERSION
-   const char *ptr = STR(MUMPS_VERSION);
-   std::string s(ptr);
-   s.erase(std::remove(s.begin(), s.end(), '"'), s.end());
-   s.erase(std::remove(s.begin(), s.end(), '.'), s.end());
-   std::cout << s << "\n";
-   return 0;
-#else
-   return -1;
-#endif
-}
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Veselin Dobrev	e61f04062f	Improve #include logic.	2020-10-29 05:20:58 -07:00
Veselin Dobrev	040a2e1a90	Update the 'raja-cuda' and 'occa-cuda' backends to use the default stream 0. Optimize OperatorJacobiSmoother::Mult by fusing kernels as much as possible.	2020-10-29 04:28:15 -07:00