make style

2. Add _USE_MATH_DEFINES to CmakeLists.txt (without this change VS doesn't "see" this preprocessor in config.hpp)
3. Remove asm definition - asm is not allowed in VS anymore
4. Convert "const double Epsilon" to constexpr (fixes CUDA compilation error)

.appveyor.yml

@@ -1,13 +1,10 @@
 version: '{build}'
 
 # https://www.appveyor.com/docs/build-environment/#build-worker-images
-image: Visual Studio 2019
+image: Visual Studio 2017
 
 install:
 
-# Start from outside clone directory
-- cd ..
-
 # Install MS-MPI
 - ps: Start-FileDownload 'https://download.microsoft.com/download/B/2/E/B2EB83FE-98C2-4156-834A-E1711E6884FB/MSMpiSetup.exe'
 - MSMpiSetup.exe -unattend
@@ -18,51 +15,37 @@ install:
 - msmpisdk.msi /passive
 - set PATH=C:\Program Files\Microsoft MPI\Bin;%PATH%
 
-# Set MSMPI environment variables needed for CMake detection
-- set MSMPI_LIB32=C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86
-- set MSMPI_LIB64=C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x64
-- set MSMPI_INC=C:\Program Files (x86)\Microsoft SDKs\MPI\Include
-
-# Install METIS, use MFEM's mirror because the original source server is often
-# down and we don't support yet the new repo https://github.com/KarypisLab/METIS
+# Install METIS, use a mirror because the original source server is not always
+# up. Original url:
+#    http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz
 - ps: Start-FileDownload 'https://mfem.github.io/tpls/metis-5.1.0.tar.gz'
 - 7z x metis-5.1.0.tar.gz -so | 7z x -si -ttar > nul
 - cd metis-5.1.0
 - ps: ( get-content "GKlib\gk_arch.h") | % { If ($_.ReadCount -ge 52) {$_ -replace "#ifdef __MSC__","#ifdef DISABLE_THIS_ANCIENT_MSC_CHECK"} Else {$_} } | set-content "GKlib\gk_arch.h"
 - cmake -H. -Bbuild
 # -DCMAKE_BUILD_TYPE=Release
-- cmake --build build --target metis -j 4
-- set METIS_PATH=%cd%
+- cmake --build build
 - cd ..
 
 # Install hypre
 - ps: Start-FileDownload 'https://github.com/hypre-space/hypre/archive/v2.19.0.tar.gz'
 - 7z x v2.19.0.tar.gz -so | 7z x -si -ttar > nul
 - cd hypre-2.19.0/src
-- cmake -H. -Bbuild
-- cmake --build build -j 4
+- cmake -H. -Bbuild -DMPI_C_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DMPI_C_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include"
+- cmake --build build
 - cmake --build build --target install
 - cd ../..
 
-# Return to clone directory
-- cd %APPVEYOR_BUILD_FOLDER%
-
 # MFEM
 before_build:
-- cmake . -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DHYPRE_DIR=%cd%\..\hypre-2.19.0\src\hypre -DMETIS_LIBRARIES=%METIS_PATH%\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%METIS_PATH%\include
-# - cmake . -Bbuild_serial -DMFEM_USE_MPI=FALSE
-- cmake . -Bbuild_shared_serial -DMFEM_USE_MPI=FALSE -DBUILD_SHARED_LIBS=ON
+- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DHYPRE_DIR=%cd%\hypre-2.19.0\src\hypre -DMETIS_LIBRARIES=%cd%\metis-5.1.0\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%cd%\metis-5.1.0\include
+- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_serial -DMFEM_USE_MPI=FALSE
 
 build_script:
-- cmake --build build_parallel --config Debug -j 4
-# - cmake --build build_serial --config Release -j 4
-# - cmake --build build_serial --target exec --config Release -j 4
-- cmake --build build_shared_serial --config Release -j 4
-- cmake --build build_shared_serial --target exec --config Release -j 4
+- cmake --build build_parallel --config Release -j 4
+- cmake --build build_serial --config Release -j 4
+- cmake --build build_serial --target exec --config Release -j 4
 
 after_build:
-# - cd build_serial
-# - ctest -C Release --output-on-failure
-# - cd ..
-- cd build_shared_serial
+- cd build_serial
 - ctest -C Release --output-on-failure

.binder/environment.yml

@@ -1,12 +0,0 @@
-# extends https://github.com/jupyterhub/repo2docker/blob/main/repo2docker/buildpacks/conda/environment.yml
-# see https://mybinder.readthedocs.io/en/latest/using/config_files.html#environment-yml-install-a-conda-environment
-channels:
-- conda-forge
-dependencies:
-- xeus-cling=0.13.0
-- xwidgets=0.26.0
-# NOTE: it is possible that these are not needed for the lab frontend
-- widgetsnbextension=3.5.1
-- pip
-- pip:
-  - glvis==0.3.2

.binder/postBuild

@@ -1,26 +0,0 @@
-#!/bin/bash
-
-set -e
-
-# cling is installed here (in bin) and will look in {dir}/include and {dir}/lib
-# without extra intervention (jk it doesn't look in {dir}/lib unless something
-# has been #included from {dir}/include first...)
-install_dir=/srv/conda/envs/notebook
-mkdir -p $install_dir
-
-# build and install mfem, which is the directory we start in
-make serial SHARED=YES -j8
-make install PREFIX=$install_dir
-
-# install xeus-glvis
-git clone https://github.com/GLVis/xeus-glvis.git
-pushd xeus-glvis
-make install prefix=$install_dir
-popd
-
-# install jupyter-lab extension
-jupyter labextension install @jupyter-widgets/jupyterlab-manager --no-build
-jupyter labextension install glvis-jupyter
-
-# fixup kernelspec, we could probably do this from sh but ¯\_(ツ)_/¯
-python .binder/update_kernel_env.py

.binder/update_kernel_env.py

@@ -1,14 +0,0 @@
-# Update the LD_LIBRARY_PATH of the C++14 kernel so it can find mfem without
-# extra pragma cling statements
-
-import json
-
-kernelspec = "/srv/conda/envs/notebook/share/jupyter/kernels/xcpp14/kernel.json"
-
-with open(kernelspec, "r") as f:
-    obj = json.load(f)
-
-obj["env"] = {"LD_LIBRARY_PATH": "/srv/conda/envs/notebook/lib"}
-
-with open(kernelspec, "w") as f:
-    json.dump(obj, f)

.github/workflows/README.md

@@ -1,48 +0,0 @@
-                    Finite Element Discretization Library
-                                   __
-                       _ __ ___   / _|  ___  _ __ ___
-                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
-                      | | | | | ||  _||  __/| | | | | |
-                      |_| |_| |_||_|   \___||_| |_| |_|
-
-                               https://mfem.org
-
-
-This directory contains the GitHub CI scripts for MFEM.
-
-Note that some of these scripts use the shared MFEM GitHub Actions from the external mfem/github-actions repository:
-
-  https://github.com/mfem/github-actions
-
-For a particular action, e.g. `mfem/github-actions/build-mfem@v2.1`, the `v2.1` suffix denotes the branch in the above from which the action is taken.
-
-The current CI workflows are:
-
-### `repo-check.yml`
-
-Runs a number of static repository-level sanity checks.
-
-- `file-headers-check` checks copyright date, license, etc. using the `--copyright`, `--license` and `--release` options of the `config/githooks/pre-push` script.
-
-- `code-style` checks the code style using the `--style` option of the `config/githooks/pre-push` script.
-
-- `documentation` checks the documentation build using the `tests/scripts/documentation` script.
-
-- `branch-history` guards against accidental commits of large files using the `--history` option of the `config/githooks/pre-push` script.
-
-### `mfem-analysis.yml` (`build-analysis`)
-
-Checks if the code builds and satisfies minimal requirements.
-
-- `gitignore` builds hypre, METIS, and MFEM using `mfem/github-actions/build-hypre`, `mfem/github-actions/build-metis`, and `mfem/github-actions/build-mfem` and checks for correct `.gitignore` settings by running the `tests/scripts/gitignore` script.
-
-### `builds-and-tests.yml`
-
-Runs a matrix of builds and tests runs with different compilers, OS, mfem/hypre settings, etc. Also processes and upload Codecov reports.
-
-Uses the following GitHub Actions from https://github.com/mfem/github-actions:
-
-- `mfem/github-actions/build-hypre`
-- `mfem/github-actions/build-metis`
-- `mfem/github-actions/build-mfem`
-- `mfem/github-actions/upload-coverage`

.github/workflows/build-container.yml

@@ -1,85 +0,0 @@
-name: "Docker"
-
-on:
-  # Always have a base image ready to go - this is a nightly build
-  schedule:
-    - cron: 0 3 * * *
-
-  # Allow manual trigger of a build
-  workflow_dispatch:
-
-  # On push to main we build and deploy images
-  push:
-    branches:
-      - master
-
-  # Publish packages on release
-  release:
-    types: [published]
-
-jobs:
-  build:
-    if: github.repository == 'mfem/mfem' # Don't run in forks
-    permissions:
-      packages: write
-    strategy:
-      fail-fast: false
-      matrix:
-        # Dockerfiles to build, a matrix supports future expanded builds
-        container: [["config/docker/Dockerfile.base", "ghcr.io/mfem/mfem-ubuntu-base"],
-                    ["config/docker/Dockerfile", "ghcr.io/mfem/mfem-ubuntu"]]
-
-    runs-on: ubuntu-latest
-    name: Build
-    steps:
-      - name: Run Actions Cleaner
-        uses: easimon/maximize-build-space@v8
-        with:
-          overprovision-lvm: 'true'
-          remove-dotnet: 'true'
-          remove-android: 'true'
-          remove-haskell: 'true'
-          remove-codeql: 'true'
-          remove-docker-images: 'true'
-
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      # It's easier to reference named variables than indexes of the matrix
-      - name: Set Environment
-        env:
-          dockerfile: ${{ matrix.container[0] }}
-          uri: ${{ matrix.container[1] }}
-        run: |
-          echo "dockerfile=$dockerfile" >> $GITHUB_ENV
-          echo "uri=$uri" >> $GITHUB_ENV
-
-      - name: Pull previous layers for cache
-        run: docker pull ${uri}:latest || echo "No container to pull"
-
-      - name: Build Container
-        run: |
-           container=$uri:latest
-           docker build -f ${dockerfile} -t ${container} .
-           echo "container=$container" >> $GITHUB_ENV
-
-      - name: GHCR Login
-        if: (github.event_name != 'pull_request')
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Deploy
-        if: (github.event_name != 'pull_request')
-        run: |
-            docker push ${container}
-
-      - name: Tag and Push Release
-        if: (github.event_name == 'release')
-        run: |
-            tag=${GITHUB_REF#refs/tags/}
-            echo "Tagging and releasing ${uri}:${tag}"
-            docker tag ${uri}:latest ${uri}:${tag}
-            docker push ${uri}:${tag}

.github/workflows/builds-and-tests.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -10,7 +10,7 @@
 # CONTRIBUTING.md for details.
 
 # In this CI section, we build different variants of mfem and run test on them.
-name: "Tests"
+name: builds-and-tests
 
 # Github actions can use the default "GITHUB_TOKEN". By default, this token
 # is set to have permissive access. However, this is not a good practice
@@ -27,7 +27,6 @@ on:
       - master
       - next
   pull_request:
-  workflow_dispatch:
 
 env:
   HYPRE_ARCHIVE: v2.19.0.tar.gz
@@ -47,50 +46,31 @@ jobs:
   builds-and-tests:
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu-18.04, macos-10.15]
         target: [dbg, opt]
         mpi: [seq, par]
-        build-system: [make, cmake]
+        build-system: [make]
         hypre-target: [int32]
-        exclude:
-          - os: ubuntu-latest
-            build-system: cmake
-          - os: macos-latest
-            build-system: cmake
-          - os: windows-latest
-            build-system: make
         # 'include' allows us to:
         # - Add a variable to all jobs without creating a new matrix dimension.
         #   Codecov is defined that way.
         # - Add a new combination.
         #   'build-system: cmake' and 'hypre-target: int64'
         #
-        # Note: we will gather coverage info for any non-debug run except the
+        # note: we will gather coverage info for any non-debug run except the
         # CMake build.
         include:
           - target: dbg
             codecov: NO
           - target: opt
             codecov: YES
-          - os: ubuntu-latest
-            target: dbg
-            config-opts: 'CPPFLAGS+=-Og'
-          - os: windows-latest
-            codecov: NO
-          - os: windows-latest
-            target: opt
-            mpi: par
-            config-opts: '-DBUILD_SHARED_LIBS=ON'
-          - os: ubuntu-latest
+          - os: ubuntu-18.04
             target: opt
             codecov: NO
             mpi: par
             build-system: cmake
             hypre-target: int32
-            # This option can be set to pass additional configuration options to
-            # the MFEM configuration command.
-            # config-opts: '-DCMAKE_VERBOSE_MAKEFILE=ON'
-          - os: ubuntu-latest
+          - os: ubuntu-18.04
             target: opt
             codecov: NO
             mpi: par
@@ -101,189 +81,128 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     steps:
-      # This external action allows to interrupt a workflow already running on
-      # the same branch to save resources.
-      - name: Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.12.1
-        with:
-          access_token: ${{ github.token }}
+    # This external action allows to interrupt a workflow already running on
+    # the same branch to save resource
+    - name: Cancel Previous Runs
+      uses: styfle/cancel-workflow-action@0.9.0
+      with:
+        access_token: ${{ github.token }}
 
-      # Fix 'No space left on device' errors for Ubuntu builds.
-      - name: Run Actions Cleaner
-        if: matrix.os == 'ubuntu-latest'
-        uses: easimon/maximize-build-space@v8
-        with:
-          overprovision-lvm: 'true'
-          remove-android: 'true'
+    # Checkout MFEM in "mfem" subdirectory. Final path:
+    # /home/runner/work/mfem/mfem/mfem
+    # Note: Done now to access "install-hypre" and "install-metis" actions.
+    - name: checkout mfem
+      uses: actions/checkout@v2
+      with:
+        path: ${{ env.MFEM_TOP_DIR }}
+        # Fetch the complete history for codecov to access commits ID
+        fetch-depth: 0
 
-      # Checkout MFEM in "mfem" subdirectory. Final path:
-      # /home/runner/work/mfem/mfem/mfem
-      # Note: Done now to access "install-hypre" and "install-metis" actions.
-      - name: checkout mfem
-        uses: actions/checkout@v4
-        with:
-          path: ${{ env.MFEM_TOP_DIR }}
-          # Fetch the complete history for codecov to access commits ID
-          fetch-depth: 0
+    # Only get MPI if defined for the job.
+    # TODO: It would be nice to have only one step, e.g. with a dedicated
+    # action, but I (@adrienbernede) don't see how at the moment.
+    - name: get MPI (Linux)
+      if: matrix.mpi == 'par' && matrix.os == 'ubuntu-18.04'
+      run: |
+        sudo apt-get install mpich libmpich-dev
+        export MAKE_CXX_FLAG="MPICXX=mpic++"
 
-      # Only get MPI if defined for the job.
-      # TODO: It would be nice to have only one step, e.g. with a dedicated
-      # action, but I (@adrienbernede) don't see how at the moment.
-      - name: get MPI (Linux)
-        if: matrix.mpi == 'par' && matrix.os == 'ubuntu-latest'
-        run: |
-          sudo apt-get install mpich libmpich-dev
+    - name: get lcov (Linux)
+      if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-18.04'
+      run: |
+        sudo apt-get install lcov
 
-      - name: get lcov (Linux)
-        if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-latest'
-        run: |
-          sudo apt-get install lcov
+    - name: Set up Homebrew
+      if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-10.15'
+      uses: Homebrew/actions/setup-homebrew@c4aafe8c4620bf08883dd4679c374f11e73329d3
 
-      # Keep the following section in case we need it again in the future,
-      # see: https://github.com/mfem/mfem/pull/3385#discussion_r1058013032
-      # - name: Set up Homebrew
-      #   if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-latest'
-      #   uses: Homebrew/actions/setup-homebrew@master
+    - name: get MPI (MacOS)
+      if: matrix.mpi == 'par' && matrix.os == 'macos-10.15'
+      run: |
+        export HOMEBREW_NO_INSTALL_CLEANUP=1
+        brew install openmpi
+        export MAKE_CXX_FLAG="MPICXX=mpic++"
 
-      - name: get MPI (MacOS)
-        if: matrix.mpi == 'par' && matrix.os == 'macos-latest'
-        run: |
-          export HOMEBREW_NO_INSTALL_CLEANUP=1
-          brew install openmpi
+    - name: get MPI (MacOS)
+      if: matrix.codecov == 'YES' && matrix.os == 'macos-10.15'
+      run: |
+        export HOMEBREW_NO_INSTALL_CLEANUP=1
+        brew install lcov
 
-      - name: get lcov (MacOS)
-        if: matrix.codecov == 'YES' && matrix.os == 'macos-latest'
-        run: |
-          export HOMEBREW_NO_INSTALL_CLEANUP=1
-          brew install lcov
+    # Get Hypre through cache, or build it.
+    # Install will only run on cache miss.
+    - name: cache hypre
+      id: hypre-cache
+      if: matrix.mpi == 'par'
+      uses: actions/cache@v2
+      with:
+        path: ${{ env.HYPRE_TOP_DIR }}
+        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.0
 
-      - name: get MPI (Windows)
-        if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
-        uses: mpi4py/setup-mpi@v1.1.4
+    - name: get hypre
+      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true'
+      uses: mfem/github-actions/build-hypre@v2.0
+      with:
+        archive: ${{ env.HYPRE_ARCHIVE }}
+        dir: ${{ env.HYPRE_TOP_DIR }}
+        target: ${{ matrix.hypre-target }}
 
-      # Get Hypre through cache, or build it.
-      # Install will only run on cache miss.
-      - name: cache hypre
-        id: hypre-cache
-        if: matrix.mpi == 'par'
-        uses: actions/cache@v4
-        with:
-          path: ${{ env.HYPRE_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.2
+    # Get Metis through cache, or build it.
+    # Install will only run on cache miss.
+    - name: cache metis
+      id: metis-cache
+      if: matrix.mpi == 'par'
+      uses: actions/cache@v2
+      with:
+        path: ${{ env.METIS_TOP_DIR }}
+        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.0
 
-      - name: get hypre
-        if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
-        uses: mfem/github-actions/build-hypre@v2.4
-        with:
-          archive: ${{ env.HYPRE_ARCHIVE }}
-          dir: ${{ env.HYPRE_TOP_DIR }}
-          target: ${{ matrix.hypre-target }}
-          build-system: make
+    - name: install metis
+      if: matrix.mpi == 'par' && steps.metis-cache.outputs.cache-hit != 'true'
+      uses: mfem/github-actions/build-metis@v2.0
+      with:
+        archive: ${{ env.METIS_ARCHIVE }}
+        dir: ${{ env.METIS_TOP_DIR }}
 
-      - name: get hypre (Windows)
-        if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
-        uses: mfem/github-actions/build-hypre@v2.4
-        with:
-          archive: ${{ env.HYPRE_ARCHIVE }}
-          dir: ${{ env.HYPRE_TOP_DIR }}
-          target: ${{ matrix.hypre-target }}
-          build-system: cmake
+    # MFEM build and test
+    - name: build
+      uses: mfem/github-actions/build-mfem@v2.0
+      with:
+        os: ${{ matrix.os }}
+        target: ${{ matrix.target }}
+        codecov: ${{ matrix.codecov }}
+        mpi: ${{ matrix.mpi }}
+        build-system: ${{ matrix.build-system }}
+        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
+        metis-dir: ${{ env.METIS_TOP_DIR }}
+        mfem-dir: ${{ env.MFEM_TOP_DIR }}
 
-      # Get Metis through cache, or build it.
-      # Install will only run on cache miss.
-      - name: cache metis
-        id: metis-cache
-        if: matrix.mpi == 'par' && matrix.os != 'windows-latest'
-        uses: actions/cache@v4
-        with:
-          path: ${{ env.METIS_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
+    # Run checks (and only checks) on debug targets
+    - name: checks
+      if: matrix.build-system == 'make' && matrix.target == 'dbg'
+      run: |
+        cd ${{ env.MFEM_TOP_DIR }}  && make check
 
-      - name: install metis
-        if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.4
-        with:
-          archive: ${{ env.METIS_ARCHIVE }}
-          dir: ${{ env.METIS_TOP_DIR }}
+    - name: unit tests
+      if: matrix.build-system == 'make' && matrix.target == 'opt'
+      run: |
+        cd ${{ env.MFEM_TOP_DIR }} && make unittest
 
-      - name: cache vcpkg (Windows)
-        id: vcpkg-cache
-        if: matrix.os == 'windows-latest'
-        uses: actions/cache@v4
-        with:
-          path: vcpkg_cache
-          key: ${{ runner.os }}-${{ matrix.mpi }}-vcpkg-v1
+    - name: tests
+      if: matrix.build-system == 'make' && matrix.target == 'opt'
+      run: |
+        cd ${{ env.MFEM_TOP_DIR }} && make test
 
-      - name: prepare vcpkg binary cache location (Windows)
-        if: matrix.os == 'windows-latest' && steps.vcpkg-cache.outputs.cache-hit != 'true'
-        run: |
-          mkdir -p vcpkg_cache
+    - name: cmake unit tests
+      if: matrix.build-system == 'cmake'
+      run: |
+        cd ${{ env.MFEM_TOP_DIR }}/build/tests/unit && ctest --output-on-failure
 
-      - name: install metis (Windows)
-        if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
-        env:
-          VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
-        run: |
-          vcpkg install metis-mfem --triplet=x64-windows-static --overlay-ports=${{ env.MFEM_TOP_DIR }}/config/vcpkg/ports
-
-      # MFEM build and test
-      - name: build
-        uses: mfem/github-actions/build-mfem@v2.4
-        env:
-          VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
-        with:
-          os: ${{ matrix.os }}
-          target: ${{ matrix.target }}
-          codecov: ${{ matrix.codecov }}
-          mpi: ${{ matrix.mpi }}
-          build-system: ${{ matrix.build-system }}
-          hypre-dir: ${{ env.HYPRE_TOP_DIR }}
-          metis-dir: ${{ env.METIS_TOP_DIR }}
-          mfem-dir: ${{ env.MFEM_TOP_DIR }}
-          config-options: ${{ matrix.config-opts }}
-          library-only: ${{ matrix.target == 'dbg' && matrix.os != 'ubuntu-latest' }}
-
-      # Run checks (and only checks) on debug targets
-      - name: checks
-        if: matrix.build-system == 'make' && matrix.target == 'dbg'
-        run: |
-          cd ${{ env.MFEM_TOP_DIR }}  && make check
-
-      # Note: 'tests' include the unit tests
-      - name: tests
-        if: matrix.build-system == 'make' && (matrix.target == 'opt' || matrix.os == 'ubuntu-latest')
-        run: |
-          cd ${{ env.MFEM_TOP_DIR }} && make test
-
-      - name: cmake checks
-        if: matrix.build-system == 'cmake' && matrix.target == 'dbg'
-        run: |
-          CTEST_CONFIG="Debug"
-          cd ${{ env.MFEM_TOP_DIR }} && cmake --build build --target check --config ${CTEST_CONFIG}
-        shell: bash
-
-      - name: cmake unit tests (Ubuntu)
-        if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os == 'ubuntu-latest'
-        run: |
-          CTEST_CONFIG="Release"
-          [[ ${{ matrix.target }} == 'dbg' ]] && CTEST_CONFIG="Debug"
-          cd ${{ env.MFEM_TOP_DIR }}/build/tests/unit && ctest --output-on-failure -C ${CTEST_CONFIG}
-        shell: bash
-
-      - name: cmake tests
-        if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os != 'ubuntu-latest'
-        run: |
-          CTEST_CONFIG="Release"
-          cd ${{ env.MFEM_TOP_DIR }}/build && \
-          ctest --output-on-failure -C ${CTEST_CONFIG} || \
-          ctest --rerun-failed --output-on-failure -C ${CTEST_CONFIG}
-        shell: bash
-
-      # Code coverage (process and upload reports)
-      - name: codecov
-        if: matrix.codecov == 'YES'
-        uses: mfem/github-actions/upload-coverage@v2.4
-        with:
-          name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
-          project_dir: ${{ env.MFEM_TOP_DIR }}
-          directories: "fem general linalg mesh"
+    # Code coverage (process and upload reports)
+    - name: codecov
+      if: matrix.codecov == 'YES'
+      uses: mfem/github-actions/upload-coverage@v2.0
+      with:
+        name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
+        project_dir: ${{ env.MFEM_TOP_DIR }}
+        directories: "fem general linalg mesh"

.github/workflows/codeql-analysis.yml

@@ -1,69 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "Static Analysis"
-
-on:
-  push:
-    branches: ["master", "next"]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: ["master"]
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'cpp' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
-        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      # Initializes the CodeQL tools for scanning.
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
-        with:
-          languages: ${{ matrix.language }}
-          # If you wish to specify custom queries, you can do so here or in a config file.
-          # By default, queries listed here will override any specified in a config file.
-          # Prefix the list here with "+" to use these queries and those in the config file.
-
-          # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-          # queries: security-extended,security-and-quality
-
-      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-      # If this step fails, then you should remove it and run the build manually (see below)
-      - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
-
-      # ℹ️ Command-line programs to run using the OS shell.
-      # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
-
-      #   If the Autobuild fails above, remove it and uncomment the following three lines.
-      #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
-
-      # - run: |
-      #   echo "Run, Build Application using script"
-      #   ./location_of_script_within_repo/buildscript.sh
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2

.github/workflows/mfem-analysis.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -9,7 +9,7 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-name: "Build Analysis"
+name: build-analysis
 
 permissions:
   actions: write
@@ -20,7 +20,6 @@ on:
       - master
       - next
   pull_request:
-  workflow_dispatch:
 
 env:
   HYPRE_ARCHIVE: v2.19.0.tar.gz
@@ -31,70 +30,71 @@ env:
 
 jobs:
   gitignore:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
 
     steps:
-      - name: Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.12.1
-        with:
-          access_token: ${{ github.token }}
+    - name: Cancel Previous Runs
+      uses: styfle/cancel-workflow-action@0.9.0
+      with:
+        access_token: ${{ github.token }}
 
-      - name: checkout MFEM
-        uses: actions/checkout@v4
-        with:
-          path: mfem
+    - name: checkout MFEM
+      uses: actions/checkout@v2
+      with:
+        path: mfem
 
-      - name: Get MPI (Linux)
-        run: |
-          sudo apt-get install mpich libmpich-dev
+    - name: Get MPI (Linux)
+      run: |
+        sudo apt-get install mpich libmpich-dev
+        export MAKE_CXX_FLAG="MPICXX=mpic++"
 
-      - name: Cache Hypre Install
-        id: hypre-cache
-        uses: actions/cache@v4
-        with:
-          path: ${{ env.HYPRE_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.2
+    - name: Cache Hypre Install
+      id: hypre-cache
+      uses: actions/cache@v2
+      with:
+        path: ${{ env.HYPRE_TOP_DIR }}
+        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.0
 
-      - name: Get Hypre
-        if: steps.hypre-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-hypre@v2.4
-        with:
-          archive: ${{ env.HYPRE_ARCHIVE }}
-          dir: ${{ env.HYPRE_TOP_DIR }}
-          target: int32
+    - name: Get Hypre
+      if: steps.hypre-cache.outputs.cache-hit != 'true'
+      uses: mfem/github-actions/build-hypre@v2.0
+      with:
+        archive: ${{ env.HYPRE_ARCHIVE }}
+        dir: ${{ env.HYPRE_TOP_DIR }}
+        target: int32
 
-      - name: Cache Metis Install
-        id: metis-cache
-        uses: actions/cache@v4
-        with:
-          path: ${{ env.METIS_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
+    - name: Cache Metis Install
+      id: metis-cache
+      uses: actions/cache@v2
+      with:
+        path: ${{ env.METIS_TOP_DIR }}
+        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.0
 
-      - name: Install Metis
-        if: steps.metis-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.4
-        with:
-          archive: ${{ env.METIS_ARCHIVE }}
-          dir: ${{ env.METIS_TOP_DIR }}
+    - name: Install Metis
+      if: steps.metis-cache.outputs.cache-hit != 'true'
+      uses: mfem/github-actions/build-metis@v2.0
+      with:
+        archive: ${{ env.METIS_ARCHIVE }}
+        dir: ${{ env.METIS_TOP_DIR }}
 
-      # MFEM build and test
-      - name: build-mfem
-        uses: mfem/github-actions/build-mfem@v2.4
-        with:
-          os: ${{ runner.os }}
-          target: opt
-          codecov: NO
-          mpi: par
-          build-system: make
-          hypre-dir: ${{ env.HYPRE_TOP_DIR }}
-          metis-dir: ${{ env.METIS_TOP_DIR }}
-          mfem-dir: mfem
+    # MFEM build and test
+    - name: build-mfem
+      uses: mfem/github-actions/build-mfem@v2.0
+      with:
+        os: ${{ runner.os }}
+        target: optim
+        codecov: NO
+        mpi: parallel
+        build-system: make
+        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
+        metis-dir: ${{ env.METIS_TOP_DIR }}
+        mfem-dir: mfem
 
-      - name: test (no clean)
-        run: |
-          cd mfem && make test-noclean
+    - name: test (no clean)
+      run: |
+        cd mfem && make test-noclean
 
-      - name: gitignore
-        run: |
-          cd mfem/tests/scripts
-          ./runtest gitignore
+    - name: gitignore
+      run: |
+        cd mfem/tests/scripts
+        ./runtest gitignore

.github/workflows/mfem-sanitizer.yml

@@ -1,70 +0,0 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-name: "Sanitizer"
-
-permissions:
-  actions: write
-
-on:
-  push:
-    branches:
-      - master
-      - next
-  pull_request:
-  workflow_dispatch:
-
-jobs:
-  Serial:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.12.1
-        with:
-          access_token: ${{ github.token }}
-
-      - name: MFEM Checkout
-        uses: actions/checkout@v4
-        with:
-          path: mfem
-
-      - name: MFEM Build
-        uses: mfem/github-actions/build-mfem@v2.4
-        with:
-          os: ${{ runner.os }}
-          target: opt
-          mpi: seq
-          hypre-dir: unused-hypre-dir
-          metis-dir: unused-metis-dir
-          mfem-dir: mfem
-          build-system: make
-          library-only: false
-          config-options:
-            CXX="clang++-14"
-            CXXFLAGS="-g -O1 -std=c++11
-                      -fsanitize=address
-                      -fno-omit-frame-pointer
-                      -fsanitize-address-use-after-scope"
-
-      - name: MFEM Info
-        working-directory: mfem
-        run: make info
-
-      - name: MFEM Sanitize
-        working-directory: mfem
-        run:
-          ASAN_OPTIONS="detect_leaks=1,
-                        strict_init_order=1,
-                        strict_string_checks=1,
-                        check_initialization_order=1,
-                        detect_stack_use_after_return=1"
-          make test

.github/workflows/repo-check.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -9,142 +9,102 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-name: "Checks"
+name: repo-check
 
 permissions:
   actions: write
 
 on:
   push:
-  pull_request:
-  workflow_dispatch:
-
-# This workflow is run on pushes to any branch in the MFEM repo (with or without
-# PRs), as well as on updates to PRs from forks. In particular, we do not
-# duplicate work by running on both pushes and updates to local PRs. We do that
-# by checking if the workflow trigger is 'push' ("github.event_name == 'push'")
-# and if we are in a fork ("github.event.pull_request.head.repo.full_name !=
-# github.repository").
 
 jobs:
   file-headers-check:
-    runs-on: ubuntu-latest
-    if: |
-      (github.event_name == 'push' ||
-       github.event.pull_request.head.repo.full_name != github.repository)
+    runs-on: ubuntu-18.04
+
     steps:
-      - name: Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.12.1
-        with:
-          access_token: ${{ github.token }}
+    - name: Cancel Previous Runs
+      uses: styfle/cancel-workflow-action@0.9.0
+      with:
+        access_token: ${{ github.token }}
+    - name: checkout mfem
+      uses: actions/checkout@v2
 
-      - name: checkout mfem
-        uses: actions/checkout@v4
+    - name: copyright check
+      id: copyright
+      run: |
+        ./config/githooks/pre-push --copyright
 
-      - name: copyright check
-        id: copyright
-        run: |
-          ./config/githooks/pre-push --copyright
+      continue-on-error: true
 
-        continue-on-error: true
+    - name: license check
+      id: license
+      run: |
+        ./config/githooks/pre-push --license
+      continue-on-error: true
 
-      - name: license check
-        id: license
-        run: |
-          ./config/githooks/pre-push --license
-        continue-on-error: true
+    - name: release check
+      id: release
+      run: |
+        ./config/githooks/pre-push --release
+      continue-on-error: true
 
-      - name: release check
-        id: release
-        run: |
-          ./config/githooks/pre-push --release
-        continue-on-error: true
-
-      - name: math check
-        id: math
-        run: |
-          ./config/githooks/pre-push --math
-        continue-on-error: true
-
-      - name: wrap-up
-        if: |
-          steps.copyright.outcome != 'success' ||
-          steps.license.outcome != 'success' ||
-          steps.release.outcome != 'success' ||
-          steps.math.outcome != 'success'
-        run: |
-          if [[ "${{ steps.copyright.outcome }}" != "success" ]]; then
-            echo "copyright check failed, unroll log for details"
-          fi
-          if [[ "${{ steps.license.outcome }}" != "success" ]]; then
-            echo "license check failed, unroll log for details"
-          fi
-          if [[ "${{ steps.release.outcome }}" != "success" ]]; then
-            echo "release check failed, unroll log for details"
-          fi
-          if [[ "${{ steps.math.outcome }}" != "success" ]]; then
-            echo "math check failed, unroll log for details"
-          fi
-          exit 1
+    - name: wrap-up
+      if: steps.copyright.outcome != 'success' || steps.license.outcome != 'success' || steps.release.outcome != 'success'
+      run: |
+        if [[ "${{ steps.copyright.outcome }}" != "success" ]]; then
+          echo "copyright check failed, unroll log for details"
+        fi
+        if [[ "${{ steps.license.outcome }}" != "success" ]]; then
+          echo "license check failed, unroll log for details"
+        fi
+        if [[ "${{ steps.release.outcome }}" != "success" ]]; then
+          echo "release check failed, unroll log for details"
+        fi
+        exit 1
 
   code-style:
-    runs-on: ubuntu-latest
-    if: |
-      (github.event_name == 'push' ||
-       github.event.pull_request.head.repo.full_name != github.repository)
+    runs-on: ubuntu-16.04 # needed for astyle 2.05.1
+
     steps:
-      - name: checkout mfem
-        uses: actions/checkout@v4
+    - name: checkout mfem
+      uses: actions/checkout@v2
 
-      - name: get astyle
-        run: |
-          sudo apt-get install astyle
+    - name: get astyle
+      run: |
+        sudo apt-get install astyle=2.05.1-0ubuntu1
 
-      - name: style check
-        run: |
-          ./config/githooks/pre-push --style
+    - name: style check
+      run: |
+        ./config/githooks/pre-push --style
 
   documentation:
-    runs-on: ubuntu-latest
-    if: |
-      (github.event_name == 'push' ||
-       github.event.pull_request.head.repo.full_name != github.repository)
+    runs-on: ubuntu-18.04
+
     steps:
-      - name: checkout mfem
-        uses: actions/checkout@v4
+    - name: checkout mfem
+      uses: actions/checkout@v2
 
-      - name: get doxygen and graphviz
-        run: |
-          sudo apt-get install doxygen graphviz
+    - name: get doxygen and graphviz
+      run: |
+        sudo apt-get install doxygen graphviz
 
-      - name: update doxygen config file
-        run: |
-          cd doc
-          doxygen -u CodeDocumentation.conf.in
-
-      - name: build documentation
-        run: |
-          cd tests/scripts
-          ./runtest documentation
+    - name: build documentation
+      run: |
+        cd tests/scripts
+        ./runtest documentation
 
   branch-history:
-    if: |
-      github.ref != 'refs/heads/next' &&
-      github.ref != 'refs/heads/master' &&
-      (github.event_name == 'push' ||
-       github.event.pull_request.head.repo.full_name != github.repository)
-    runs-on: ubuntu-latest
-    steps:
-      - name: checkout mfem
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
+    if: github.ref != 'refs/heads/next' && github.ref != 'refs/heads/master'
+    runs-on: ubuntu-18.04
 
-      - name: branch-history
-        run: |
-          # We override origin to make sure we point to the main repo.
-          # This is to have consistent test results on PRs from forks.
-          git remote remove origin
-          git remote add origin https://github.com/mfem/mfem.git
-          git checkout -b gh-actions-branch-history
-          ./config/githooks/pre-push --history
+    steps:
+    - name: checkout mfem
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: branch-history
+      run: |
+        git fetch origin master:master
+        git checkout -b gh-actions-branch-history
+        ./config/githooks/pre-push --history

.gitignore

@@ -21,15 +21,11 @@ CMakeFiles/
 # Default install location
 /mfem/
 
-# Typical build directory
-/build/
-
 # Generated files in main directory, config/ and docs/
 /deps.mk
 config/_config.hpp
 config/config.mk
 config/sample-runs-build.log
-config/user.cmake
 config/user.mk
 doc/CodeDocumentation.conf
 doc/CodeDocumentation.html
@@ -55,8 +51,6 @@ examples/ex1[04-9]
 examples/ex1[0-9]p
 examples/ex2[0-9]
 examples/ex2[0-9]p
-examples/ex3[0-9]
-examples/ex3[0-9]p
 
 examples/refined.mesh
 examples/displaced.mesh
@@ -82,7 +76,6 @@ examples/deformed.*
 examples/velocity.*
 examples/elastic_energy.*
 examples/mode_*
-examples/mode_deriv_*
 examples/ex5-p-*.bp
 examples/ex9-p-*.bp
 examples/ex12-p-*.bp
@@ -113,12 +106,6 @@ examples/ex25p-*.*
 examples/ex28_*
 examples/ex28p_*
 examples/flux.*
-examples/dsol.*
-examples/cond.*
-examples/cond_j.*
-examples/cond_mesh.*
-examples/port_mesh.*
-examples/port_mode.*
 
 examples/amgx/ex1
 examples/amgx/ex1p
@@ -128,16 +115,11 @@ examples/amgx/sol.gf
 examples/amgx/mesh.*
 examples/amgx/sol.*
 
-examples/caliper/ex1
-examples/caliper/ex1p
-examples/caliper/refined.mesh
-examples/caliper/sol.gf
-examples/caliper/mesh.*
-examples/caliper/sol.*
-
-examples/ginkgo/ex1
-examples/ginkgo/refined.mesh
-examples/ginkgo/sol.gf
+examples/gingko/ex1
+examples/gingko/refined.mesh
+examples/gingko/sol.gf
+examples/gingko/mesh.*
+examples/gingko/sol.*
 
 examples/hiop/ex9
 examples/hiop/ex9p
@@ -163,8 +145,8 @@ examples/petsc/velocity.*
 examples/petsc/elastic_energy.*
 examples/petsc/mode_*
 
-examples/pumi/ex[12]
-examples/pumi/ex[16]p
+examples/pumi/ex1
+examples/pumi/ex[126]p
 examples/pumi/refined.mesh
 examples/pumi/sol.gf
 examples/pumi/mesh.*
@@ -209,9 +191,6 @@ miniapps/gslib/field-diff
 miniapps/gslib/field-interp
 miniapps/gslib/findpts
 miniapps/gslib/pfindpts
-miniapps/gslib/schwarz_ex1
-miniapps/gslib/schwarz_ex1p
-miniapps/gslib/interpolated.gf
 
 miniapps/meshing/mobius-strip
 miniapps/meshing/klein-bottle
@@ -220,28 +199,21 @@ miniapps/meshing/twist
 miniapps/meshing/mesh-explorer
 miniapps/meshing/shaper
 miniapps/meshing/extruder
-miniapps/meshing/fit-node-position
 miniapps/meshing/trimmer
-miniapps/meshing/reflector
 miniapps/meshing/mesh-optimizer
 miniapps/meshing/pmesh-optimizer
-miniapps/meshing/pmesh-fitting
 miniapps/meshing/minimal-surface
 miniapps/meshing/pminimal-surface
 miniapps/meshing/polar-nc
-miniapps/meshing/mesh-quality
 miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
 miniapps/meshing/toroid-*.mesh
 miniapps/meshing/twist-*.mesh
 miniapps/meshing/mesh-explorer.mesh
 miniapps/meshing/partitioning.txt
-miniapps/meshing/mesh-explorer-visit*
-miniapps/meshing/mesh-explorer-paraview/
 miniapps/meshing/shaper.mesh
 miniapps/meshing/extruder.mesh
 miniapps/meshing/trimmer.mesh
-miniapps/meshing/reflected.mesh
 miniapps/meshing/optimized*
 miniapps/meshing/perturbed*
 miniapps/meshing/polar-nc.mesh
@@ -251,42 +223,23 @@ miniapps/mtop/ParHeat*
 miniapps/mtop/seqheat
 miniapps/mtop/SeqHeat*
 
-miniapps/autodiff/paradiff
-miniapps/autodiff/seqadiff
-miniapps/autodiff/seqtest
-miniapps/autodiff/par_example
-miniapps/autodiff/seq_example
-miniapps/autodiff/seq_test
-miniapps/autodiff/Exampl*
-
 miniapps/navier/navier_mms
 miniapps/navier/navier_kovasznay
 miniapps/navier/navier_kovasznay_vs
 miniapps/navier/navier_tgv
 miniapps/navier/navier_shear
 miniapps/navier/navier_3dfoc
-miniapps/navier/navier_turbchan
-miniapps/navier/navier_cht
 miniapps/navier/tgv_out*.txt
 miniapps/navier/*_output
 
 miniapps/nurbs/nurbs_ex1
 miniapps/nurbs/nurbs_ex1p
 miniapps/nurbs/nurbs_ex11p
-miniapps/nurbs/nurbs_printfunc
-miniapps/nurbs/nurbs_patch_ex1
-miniapps/nurbs/nurbs_curveint
 miniapps/nurbs/refined.mesh
 miniapps/nurbs/mesh.*
 miniapps/nurbs/sol.*
 miniapps/nurbs/mode_*
 miniapps/nurbs/Example1*
-miniapps/nurbs/sin-fit.mesh
-miniapps/nurbs/CurveInt
-miniapps/nurbs/nurbs_naca_cmesh
-miniapps/nurbs/naca-cmesh.mesh
-miniapps/nurbs/glvis_naca-cmesh.mesh
-miniapps/nurbs/Naca_cmesh
 
 miniapps/performance/ex1
 miniapps/performance/ex1p
@@ -296,27 +249,16 @@ miniapps/performance/sol.*
 
 miniapps/shifted/distance
 miniapps/shifted/ParaViewDistance
-miniapps/shifted/ParaViewLSF
-miniapps/shifted/extrapolate
-miniapps/shifted/ParaViewExtrapolate
 miniapps/shifted/diffusion
 miniapps/shifted/diffusion.mesh
 miniapps/shifted/diffusion.gf
 miniapps/shifted/ParaViewDiffusion
-miniapps/shifted/lsf_integral
 
 miniapps/tools/display-basis
 miniapps/tools/load-dc
 miniapps/tools/convert-dc
 miniapps/tools/lor-transfer
-miniapps/tools/plor-transfer
 miniapps/tools/get-values
-miniapps/tools/tmop-check-metric
-miniapps/tools/tmop-metric-magnitude
-miniapps/tools/nodal-transfer
-miniapps/tools/ParaView
-miniapps/tools/gridfunc_*
-miniapps/tools/mesh_*
 
 miniapps/toys/automata
 miniapps/toys/life
@@ -342,53 +284,20 @@ miniapps/toys/mondrian.mesh
 miniapps/solvers/block-solvers
 miniapps/solvers/lor_solvers
 miniapps/solvers/plor_solvers
-miniapps/solvers/lor_elast
 miniapps/solvers/ParaView
 miniapps/solvers/mesh.*
 miniapps/solvers/sol.*
 
-miniapps/hdiv-linear-solver/darcy
-miniapps/hdiv-linear-solver/grad_div
-
-miniapps/parelag/MultilevelHcurlHdivSolver
-miniapps/parelag/*.mesh
-
-miniapps/multidomain/multidomain
-miniapps/hooke/hooke
-
-miniapps/dpg/diffusion
-miniapps/dpg/pdiffusion
-miniapps/dpg/convection-diffusion
-miniapps/dpg/pconvection-diffusion
-miniapps/dpg/acoustics
-miniapps/dpg/pacoustics
-miniapps/dpg/maxwell
-miniapps/dpg/pmaxwell
-miniapps/dpg/ParaView
-
-miniapps/spde/generate_random_field
-miniapps/spde/ParaView
-
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
 tests/unit/punit_tests
-tests/unit/cunit_tests
-tests/unit/pcunit_tests
 tests/unit/sedov_tests_*
 tests/unit/psedov_tests_*
 tests/unit/tmop_pa_tests_*
 tests/unit/ptmop_pa_tests_*
 tests/unit/ceed_tests
 tests/unit/debug_device_tests
-tests/unit/parallel_in_serial.mesh
-tests/unit/parallel_in_serial.gf
-
-# Benchmark binaries
-tests/benchmarks/bench_ceed
-tests/benchmarks/bench_tmop
-tests/benchmarks/bench_vector
-tests/benchmarks/bench_virtuals
 
 # Test script output
 tests/scripts/*.err
@@ -406,6 +315,3 @@ build-*/*
 # PETSc automated build
 petsc-build/*
 pkg.gitcommit
-
-# Jupyter Notebook Checkpoints
-.ipynb_checkpoints

.gitlab-ci.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -13,84 +13,243 @@
 # at Lawrence Livermore National Laboratory (LLNL). This entire pipeline is
 # LLNL-specific!
 
+# We define the following GitLab pipeline variables:
+#
+# BUILD_ROOT:
+# The path to the shared resources between all jobs. For example, external
+# repositories like 'tests' and 'tpls' are cloned here. Also, 'tpls' is built
+# once for all targets, so that build happen here. The BUILD_ROOT is unique to
+# the pipeline, preventing any form of concurrency with other pipelines. This
+# also means that the BUILD_ROOT directory will never be cleaned.
+# TODO: add a clean-up mechanism
+#
+# REBASELINE:
+# Defines the default choice for updating the saved baseline results. By default
+# the baseline can only be updated from the master branch. This variable offers
+# the option to manually ask for rebaselining from another branch if necessary.
+#
+# MFEM_ALLOC_NAME:
+# On LLNL's quartz, there is only one allocation shared among jobs in order to
+# save time and resources. This allocation has to be uniquely named so that we
+# are sure to retrieve it.
+#
+# TPLS_REPO & TESTS_REPO:
+# Git repositories used in the pipeline
+#
+# ARTIFACTS_DIR:
+# Directory used to place artifacts.
+
+variables:
+  BUILD_ROOT: ${CI_BUILDS_DIR}/MFEM/${CI_PROJECT_NAME}_${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID}
+  AUTOTEST_ROOT: ${CI_BUILDS_DIR}/MFEM
+  REBASELINE: "NO"
+  AUTOTEST: "NO"
+  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
+  TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
+  TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
+  AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
+  MFEM_DATA_REPO: https://github.com/mfem/data.git
+  ARTIFACTS_DIR: artifacts
+  SLURM_OVERLAP: 1
 
 # The pipeline is divided into stages. Usually, jobs in a given stage wait for
 # the preceding stages to complete before to start. However, we sometimes use
 # the "needs" keyword and express the DAG of jobs for more efficiency.
 # - We use setup and setup_baseline phases to download content outside of mfem
 #   directory.
-# - Allocate/Release is where quartz resource are allocated/released once for all.
+# - Allocate/Release is where quartz resources are allocated/released once for all.
 # - Build and Test is where we build and MFEM for multiple toolchains.
 # - Baseline_checks gathers baseline-type test suites execution
 # - Baseline_publish, only available on master, allows to update baseline
 #   results
 stages:
-  - sub-pipelines
+  - setup
+  - q_allocate_resources
+  - q_build_and_test
+  - q_release_resources
+  - l_build_and_test
+  - c_build_and_test
+  - setup_baseline
+  - baseline_check
+  - baseline_to_autotest
+  - baseline_publish
 
-variables:
-  CUSTOM_CI_BUILDS_DIR: "/usr/workspace/mfem/gitlab-runner"
-  USER_CI_TOP_DIR: "${CUSTOM_CI_BUILDS_DIR}/${GITLAB_USER_LOGIN}"
-  SHARED_REPOS_DIR: "${USER_CI_TOP_DIR}/repos"
-  AUTOTEST_ROOT: "${SHARED_REPOS_DIR}"
-  # MFEM_DATA_DIR is setup in '.gitlab/configs/setup-build-and-test.yml' and
-  # used in '.gitlab/configs/<machine>-config.yml':
-  MFEM_DATA_DIR: "${SHARED_REPOS_DIR}/mfem-data"
-
-# Defines the default choice for updating the saved baseline results. By default
-# the baseline can only be updated from the master branch. This variable offers
-# the option to manually ask for rebaselining from another branch if necessary.
-  REBASELINE: "NO"
-  AUTOTEST: "NO"
-  # AUTOTEST_COMMIT: used only when AUTOTEST is set to YES.
-  # * If AUTOTEST_COMMIT is NOT set to NO, reporting jobs will commit their
-  #   files to the MFEM/autotest repo.
-  # * If AUTOTEST_COMMIT is set to NO, reporting jobs will NOT commit their
-  #   files to the MFEM/autotest repo. Instead they will just show the contents
-  #   of the report files and remove them.
-  AUTOTEST_COMMIT: "YES"
-
-# Trigger subpipelines:
-quartz-build-and-test:
-  stage: sub-pipelines
+# setup clones the mfem/data repo in ${BUILD_ROOT}. The build_and_test script
+# then symlinks the repo to the parent directory of the MFEM source directory.
+# Unit tests that depend on the mfem/data repo will then detect that this
+# directory is present and be enabled.
+setup:
+  tags:
+    - shell
+    - quartz
+  stage: setup
   variables:
-    # Explicitly pass down values that we want to be able to set when triggering
-    # pipelines manually or using scheduling
-    AUTOTEST: "${AUTOTEST}"
-    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
-  trigger:
-    include: .gitlab/quartz-build-and-test.yml
-    strategy: depend
+    GIT_STRATEGY: none
+  script:
+    - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
+    - if [ ! -d data ]; then git clone ${MFEM_DATA_REPO}; fi
 
-quartz-baseline:
-  stage: sub-pipelines
+# The setup_baseline job in setup stage_baseline doesn't rely on MFEM git repo.
+# It prepares a pipeline-wide working directory downloading/updating external
+# repos. TODO: updating tests and tpls is not necessary anymore since pipelines
+# are now using unique directories so repo are never shared with another
+# pipeline. This is not memory efficient (we keep a lot of data), hence this
+# reminder.
+# Note: This job can start immediately.
+setup_baseline:
+  tags:
+    - shell
+    - quartz
+  stage: setup_baseline
   variables:
-    # Explicitly pass down values that we want to be able to set when triggering
-    # pipelines manually or using scheduling
-    REBASELINE: "${REBASELINE}"
-    AUTOTEST: "${AUTOTEST}"
-    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
-  trigger:
-    include: .gitlab/quartz-baseline.yml
-    strategy: depend
+    GIT_STRATEGY: none
+  script:
+    - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
+    - if [ ! -d "tpls" ]; then git clone ${TPLS_REPO}; fi
+    - if [ ! -d "tests" ]; then git clone ${TESTS_REPO}; fi
+    - cd tpls && git pull && cd ..
+    - cd tests && git pull && cd ..
+    - cd ${AUTOTEST_ROOT}
+    - if [ ! -d "autotest" ]; then git clone ${AUTOTEST_REPO}; fi
+    - cd autotest && git pull && cd ..
+  needs: []
 
-lassen-build-and-test:
-  stage: sub-pipelines
-  variables:
-    # Explicitly pass down values that we want to be able to set when triggering
-    # pipelines manually or using scheduling
-    AUTOTEST: "${AUTOTEST}"
-    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
-  trigger:
-    include: .gitlab/lassen-build-and-test.yml
-    strategy: depend
+.build_toss_3_x86_64_ib_script:
+  script:
+    - export THREADS=12
+    - echo ${ALLOC_NAME}
+    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
+    - echo ${JOBID}
+    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 30 -N 1 tests/gitlab/build_and_test
 
-corona-build-and-test:
-  stage: sub-pipelines
+.build_toss_3_x86_64_ib_corona_script:
+  script:
+    - srun -p mi60 -t 15 -N 1 tests/gitlab/build_and_test
+
+# Lassen uses a different job scheduler (spectrum lsf) that does not allow
+# pre-allocation the same way slurm does.  We use pdebug queue on lassen to
+# speed-up the allocation.  However this would not be scalable to multiple
+# builds.
+.build_blueos_3_ppc64le_ib_script:
+  script:
+    - lalloc 1 -W 30 -q pdebug tests/gitlab/build_and_test
+
+# Shared script for baseline and sample-run-baseline, the value of BASELINE_TEST
+# differentiates between the two tests.
+.baseline_script: &baseline_script |
+  # locals
+  _glob_err=${BASELINE_TEST}.err
+  _base_diff=${BASELINE_TEST}-${SYS_TYPE}.diff
+  _base_patch=${BASELINE_TEST}-${SYS_TYPE}.patch
+  _base_out=${BASELINE_TEST}-${SYS_TYPE}.out
+  # prepare
+  cd ${BUILD_ROOT}
+  ln -snf ${CI_PROJECT_DIR} mfem
+  cd tests
+  mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
+  # run
+  srun --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${ADDITIONAL_DIR}"
+  # post
+  mkdir ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}
+  if [[ -s ${_glob_err} ]]
+  then
+    echo "ERROR during ${BASELINE_TEST} execution";
+    echo "Here is the ${_glob_err} file content";
+    cat ${_glob_err}
+    cp ${_glob_err} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_glob_err}
+    exit 1;
+  elif [[ ! -f ${_base_patch} && ! -f ${_base_out} ]]
+  then
+    echo "Something went WRONG in ${BASELINE_TEST}:";
+    echo "Either ${_base_patch} or ${_base_out} should exists";
+    exit 1;
+  elif [[ -f ${_base_patch} ]]
+  then
+    echo "${BASELINE_TEST}: Differences found, patch generated"
+    cp ${_base_patch} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_patch}
+  elif [[ -f ${_base_out} ]]
+  then
+    echo "${BASELINE_TEST}: Differences found, replacement file generated"
+    cp ${_base_out} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_out}
+  fi
+  # _base_diff won't even exist if there is no difference.
+  if [[ -f ${_base_diff} ]]
+  then
+    echo "${BASELINE_TEST}: Relevant differences (filtered diff) ..."
+    cat ${_base_diff}
+    cp ${_base_diff} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_diff}
+    # We create a .err file, because that's how we signal that there was a diff.
+    cp ${_base_diff} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/gitlab-${BASELINE_TEST}-${SYS_TYPE}.err
+  fi
+  if [[ ! -s ${_base_diff} ]]
+  then
+    echo "${BASELINE_TEST}: PASSED"
+    true
+  else
+    echo "${BASELINE_TEST}: FAILED"
+    false
+  fi
+
+# Actual templates for baseline checks
+.baselinecheck_mfem:
+  stage: baseline_check
   variables:
-    # Explicitly pass down values that we want to be able to set when triggering
-    # pipelines manually or using scheduling
-    AUTOTEST: "${AUTOTEST}"
-    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
-  trigger:
-    include: .gitlab/corona-build-and-test.yml
-    strategy: depend
+    BASELINE_TEST: baseline
+    ADDITIONAL_DIR: ${BUILD_ROOT}/tpls
+  script:
+    - *baseline_script
+  artifacts:
+    when: always
+    paths:
+      - ${ARTIFACTS_DIR}
+  allow_failure: true
+
+.samplebaselinecheck_mfem:
+  stage: baseline_check
+  variables:
+    BASELINE_TEST: sample-runs-baseline
+    ADDITIONAL_DIR: ""
+  script:
+    - *baseline_script
+  timeout: 4h
+  artifacts:
+    when: always
+    paths:
+      - ${ARTIFACTS_DIR}
+  allow_failure: true
+
+# This job can only be manually triggered on a pipeline for master branch, or if
+# the pipeline was triggered with REBASELINE="YES"
+.rebaseline_mfem:
+  stage: baseline_publish
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "YES"'
+      when: manual
+  script:
+    - export PATCH_FILE=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}.patch
+    - export FULL_FILE=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}.out
+    - export DIFF_FILE=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}.diff
+    - cd ${BUILD_ROOT}/tests
+    - |
+      if [[ ! -f "${DIFF_FILE}" ]]
+      then
+          echo "Nothing to be done: no relevant change in baseline"
+          exit 0
+      elif [[ -f "${PATCH_FILE}" ]]
+      then
+          patch "./baseline-${SYS_TYPE}.saved" < "${PATCH_FILE}"
+      elif [[ -f "${FULL_FILE}t" ]]
+      then
+          cp "${FULL_FILE}" "./baseline-${SYS_TYPE}.saved"
+      else
+          echo "File missing: expected ${PATCH_FILE} or ${FULL_FILE}"
+          exit 1
+      fi
+    - git add baseline-${SYS_TYPE}.saved
+    - git commit -m "${SYS_TYPE} rebaselined in GitLab pipeline ${CI_PIPELINE_ID}"
+    - git push origin master
+
+# The list on jobs is defined in machine-specific files.
+include:
+  - local: .gitlab/quartz.yml
+  - local: .gitlab/lassen.yml

.gitlab/README.md

@@ -1,94 +0,0 @@
-                    Finite Element Discretization Library
-                                   __
-                       _ __ ___   / _|  ___  _ __ ___
-                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
-                      | | | | | ||  _||  __/| | | | | |
-                      |_| |_| |_||_|   \___||_| |_| |_|
-
-                               https://mfem.org
-
-
-This directory contains most of the GitLab CI configuration. MFEM runs both PR
-and nightly testing on GitLab.
-
-# Structure
-
-## Top level
-
-The root configuration file is `.gitlab-ci.yml` at the root of MFEM repo.
-This file only defines one stage, in which we trigger several
-sub-pipelines.
-
-We use sub-pipelines to isolate the test for one combination of `machine`
-and `test type`.
-
-Machines typically include:
-
-* Quartz: Intel bi-socket x86
-* Lassen: Power9 + Nvidia GPU
-* Corona: AMD GPU
-
-Test types include:
-
-* Build and test: Spack driven build of dependencies, mfem build, mfem
-  test
-* Baseline: Script driven build of dependencies, thorough testing
-
-⚠️ The sub-pipeline design allows to add a new machine or a new test type without
-altering the scheduling, execution and displaying of the others.
-
-## Sub-pipelines
-
-Each file is this directory is the root configuration file for one
-sub-pipeline. The naming reflects the corresponding couple (`machine`,
-`test_type`).
-
-Those files define the *stages* and the *jobs* for the sub-pipeline. They
-also contain any configuration that cannot be shared. For the most part
-though, the configuration is shared and is placed in `.gitlab/configs`.
-
-We try to keep scripts out of the CI config and share them among similar
-jobs. They are gathered in `.gitlab/scripts`.
-
-## Scripts
-
-Scripts specific to the CI only are in `.gitlab/scripts`. It is best practice
-to keep scripts outside the CI configuration (no bash scripts embedded in a
-yaml file) because it helps with readability, maintenance and also with
-transition to another CI system.
-
-⚠️ Most of the scripts there are driven by environment variables and do not have a
-usage function. This should be improved.
-
-
-# More testing
-
-## Adding a new target to a build_and_test pipeline
-
-`build_and_test` pipelines rely on Spack to install dependencies. Spack is
-driven by Uberenv which helps freezing Spack configuration: the goal being to
-point to specific commit in Spack and isolate its configuration so that it is
-not influenced by the user environment. More documentation about this can be
-found in `tests/gitlab`.
-
-In the end, the MFEM target for which to build the dependencies is expressed
-with a spack spec of MFEM, within the limits permitted by the MFEM spack
-package.
-
-In any build-and-test sub-pipeline a job basically consists in defining the
-spack spec to use. Adding a job on quartz for example resumes to:
-
-```yaml
-<job_name>:
-  variables:
-    SPEC: "<spack_spec>"
-  extends: .build_and_test_on_quartz
-```
-
-The remaining and non trivial work is to make sure this spec is working. To
-test a spec before adding it, or reproduce a CI configuration, please refer to
-`tests/gitlab/reproduce-ci-jobs-interactively.md`.
-
-⚠️ It is assumed that the spack spec applies to `mfem@develop`. That's why in the
-CI all the specs start with the compiler or the variants to apply to mfem. The
-mechanism still works with a full spec.

.gitlab/configs/common.yml

@@ -1,36 +0,0 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# We define the following GitLab pipeline variables:
-variables:
-
-# The path to the shared resource between all jobs. For example, external
-# repositories like 'tests' and 'tpls' are cloned here. Also, 'tpls' is built
-# once for all targets, so that build happen here. The BUILD_ROOT is unique to
-# the pipeline, preventing any form of concurrency with other pipelines. This
-# also means that the BUILD_ROOT directory will never be cleaned.
-# TODO: add a clean-up mechanism
-  BUILD_ROOT: ${USER_CI_TOP_DIR}/${CI_PROJECT_NAME}-${MACHINE_NAME}-pipeline-${CI_PIPELINE_ID}
-
-# On LLNL's quartz, there is only one allocation shared among jobs in order to
-# save time and resource. This allocation has to be uniquely named so that we
-# are sure to retrieve it.
-  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
-
-# Git repositories used in the pipeline
-  TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
-  TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
-  AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
-  MFEM_DATA_REPO: https://github.com/mfem/data.git
-
-# Directory used to place artifacts.
-  ARTIFACTS_DIR: artifacts
-  SLURM_OVERLAP: 1

.gitlab/configs/corona-config.yml

@@ -1,59 +0,0 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# GitLab pipeline configuration for the Corona machine at LLNL
-variables:
-  MACHINE_NAME: corona
-
-.on_corona:
-  tags:
-    - shell
-    - corona
-  rules:
-    # Don't run corona jobs if...
-    # Note: This makes corona an "opt-in" machine. To activate builds on corona
-    # for a given GitLab clone of MFEM, go to Setting/CI-CD/variables, and set
-    # "ON_CORONA" to "ON". An LC account on for corona is required to trigger a
-    # pipeline there.
-    - if: '$CI_COMMIT_BRANCH =~ /_cnone/ || $ON_CORONA != "ON"'
-      when: never
-    # Don't run autotest update if...
-    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
-      when: never
-    # Report success on success status
-    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
-      when: on_success
-    # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
-      when: on_failure
-    # Always release resource
-    - if: '$CI_JOB_NAME =~ /release_resource/'
-      when: always
-    # Always cleanup
-    - if: '$CI_JOB_NAME =~ /cleanup/'
-      when: always
-    # Default is to run if previous stage succeeded
-    - when: on_success
-
-# Spack helped builds
-# Generic corona build job, extending build script
-.build_and_test_on_corona:
-  extends: [.on_corona]
-  stage: build_and_test
-  script:
-    # THREADS is used by 'tests/gitlab/build_and_test', run below
-    - export THREADS=12
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - echo ${MFEM_DATA_DIR}
-    - echo ${SPEC}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 15 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data

.gitlab/configs/lassen-config.yml

@@ -1,49 +0,0 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# GitLab pipelines configurations for the Lassen machine at LLNL
-variables:
-  MACHINE_NAME: lassen
-
-.on_lassen:
-  tags:
-    - shell
-    - lassen
-  rules:
-    - if: '$CI_COMMIT_BRANCH =~ /_lnone/ || $ON_LASSEN == "OFF"' #run except if ...
-      when: never
-    # Don't run autotest update if...
-    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
-      when: never
-    # Report success on success status
-    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
-      when: on_success
-    # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
-      when: on_failure
-    # Always cleanup
-    - if: '$CI_JOB_NAME =~ /cleanup/'
-      when: always
-    - when: on_success
-
-# Lassen uses a different job scheduler (spectrum lsf) that does not allow
-# pre-allocation the same way slurm does.  We use pdebug queue on lassen
-# to speed-up the allocation.  However this would not be scalable to
-# multiple builds.
-.build_and_test_on_lassen:
-  extends: [.on_lassen]
-  stage: build_and_test
-  script:
-    - echo ${MFEM_DATA_DIR}
-    - echo ${SPEC}
-    # Next script uses 'THREADS': leaving it empty --> it uses 'make all -j'
-    - lalloc 1 -W 45 -q pdebug --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
-  needs: [setup]

.gitlab/configs/quartz-config.yml

@@ -1,55 +0,0 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# GitLab pipelines configurations for the Quartz machine at LLNL
-variables:
-  MACHINE_NAME: quartz
-
-.on_quartz:
-  tags:
-    - shell
-    - quartz
-  rules:
-    # Don't run quartz jobs if...
-    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"'
-      when: never
-    # Don't run autotest update if...
-    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
-      when: never
-    # Report success on success status
-    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
-      when: on_success
-    # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
-      when: on_failure
-    # Always release resource
-    - if: '$CI_JOB_NAME =~ /release_resource/'
-      when: always
-    # Always cleanup
-    - if: '$CI_JOB_NAME =~ /cleanup/'
-      when: always
-    # Default is to run if previous stage succeeded
-    - when: on_success
-
-# Spack helped builds
-# Generic quartz build job, extending build script
-.build_and_test_on_quartz:
-  extends: [.on_quartz]
-  stage: build_and_test
-  script:
-    # THREADS is used by 'tests/gitlab/build_and_test', run below
-    - export THREADS=12
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - echo ${MFEM_DATA_DIR}
-    - echo ${SPEC}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 45 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data

.gitlab/configs/report-build-and-test.yml

@@ -1,77 +0,0 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Jobs report
-.report_job_success:
-  script:
-    - echo ${MACHINE_NAME}
-    - echo ${AUTOTEST}
-    - echo ${AUTOTEST_COMMIT}
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - cd ${AUTOTEST_ROOT}
-    - |
-      (
-        date
-        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
-        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -n 9; do
-          sleep 5
-        done
-        echo "Acquired lock on '$PWD/autotest.lock'"
-        date
-        # Report SUCCESS while holding the file lock on 'autotest.lock'.
-        # The next script uses the following environment variables:
-        # - MACHINE_NAME, AUTOTEST_ROOT, AUTOTEST_COMMIT
-        # - CI_COMMIT_REF_SLUG, CI_PROJECT_DIR, CI_PIPELINE_URL
-        # It also calls the script '.gitlab/scripts/safe_create_rundir'.
-        ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test_success
-        err=$?
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock
-
-.report_job_failure:
-  script:
-    - echo ${MACHINE_NAME}
-    - echo ${AUTOTEST}
-    - echo ${AUTOTEST_COMMIT}
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - cd ${AUTOTEST_ROOT}
-    - |
-      (
-        date
-        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
-        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -n 9; do
-          sleep 5
-        done
-        echo "Acquired lock on '$PWD/autotest.lock'"
-        date
-        # Report FAILURE while holding the file lock on 'autotest.lock'.
-        # The next script uses the following environment variables:
-        # - MACHINE_NAME, AUTOTEST_ROOT, AUTOTEST_COMMIT
-        # - CI_COMMIT_REF_SLUG, CI_PROJECT_DIR, CI_PIPELINE_URL
-        # It also calls the script '.gitlab/scripts/safe_create_rundir'.
-        ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test_failure
-        err=$?
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock

.gitlab/configs/setup-baseline.yml

@@ -1,70 +0,0 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# The setup_baseline job doesn't rely on MFEM git repo. It prepares a
-# pipeline-wide working directory downloading/updating external repos.
-# TODO:
-# updating tests and tpls is not necessary anymore since pipelines are now
-# using unique directories so repo are never shared with another pipeline. This
-# is not memory efficient (we keep a lot of data), hence this reminder.
-setup_baseline:
-  tags:
-    - shell
-    - quartz
-  stage: setup
-  variables:
-    GIT_STRATEGY: none
-  script:
-    #
-    # Setup ${BUILD_ROOT}/tpls and ${BUILD_ROOT}/tests:
-    #
-    - echo "MACHINE_NAME = ${MACHINE_NAME}"
-    - echo "REBASELINE = ${REBASELINE}"
-    - echo "AUTOTEST = ${AUTOTEST}"
-    - echo "AUTOTEST_COMMIT = ${AUTOTEST_COMMIT}"
-    - echo "BUILD_ROOT ${BUILD_ROOT}"
-    - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
-    - if [ ! -d "tpls" ]; then git clone ${TPLS_REPO}; fi
-    - if [ ! -d "tests" ]; then git clone ${TESTS_REPO}; fi
-    - cd tpls && git pull && cd ..
-    - cd tests && git pull origin && cd ..
-    #
-    # Setup ${AUTOTEST_ROOT}/autotest:
-    #
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - mkdir -p ${AUTOTEST_ROOT} && cd ${AUTOTEST_ROOT}
-    - command -v flock || echo "Required command 'flock' not found"
-    - |
-      (
-        date
-        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
-        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -n 9; do
-          sleep 5
-        done
-        echo "Acquired lock on '$PWD/autotest.lock'"
-        date
-        # clone/update the autotest repo while holding the file lock on
-        # 'autotest.lock'
-        err=0
-        if [[ ! -d "autotest" ]]; then
-          git clone ${AUTOTEST_REPO}
-        else
-          cd autotest && git pull && cd ..
-        fi || err=1
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock

.gitlab/configs/setup-build-and-test.yml

@@ -1,90 +0,0 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test
-# script then symlinks the repo to the parent directory of the MFEM source
-# directory. Unit tests that depend on the mfem/data repo will then detect that
-# this directory is present and be enabled.
-setup:
-  tags:
-    - shell
-    - quartz
-  stage: setup
-  variables:
-    GIT_STRATEGY: none
-  script:
-    #
-    # Setup MFEM_DATA_DIR=${SHARED_REPOS_DIR}/mfem-data, see '.gitlab-ci.yml'
-    # and '.gitlab/configs/<machine>-config.yml'
-    #
-    - echo "MACHINE_NAME = ${MACHINE_NAME}"
-    - echo "AUTOTEST = ${AUTOTEST}"
-    - echo "AUTOTEST_COMMIT = ${AUTOTEST_COMMIT}"
-    - echo "SHARED_REPOS_DIR ${SHARED_REPOS_DIR}"
-    - mkdir -p ${SHARED_REPOS_DIR} && cd ${SHARED_REPOS_DIR}
-    - command -v flock || echo "Required command 'flock' not found"
-    - |
-      (
-        date
-        echo "Waiting to acquire lock on '$PWD/mfem-data.lock' ..."
-        # try to get an exclusive lock on fd 9 (mfem-data.lock) repeating the
-        # try every 5 seconds; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -n 9; do
-          sleep 5
-        done
-        echo "Acquired lock on '$PWD/mfem-data.lock'"
-        date
-        # clone/update the mfem/data repo while holding the file lock on
-        # 'mfem-data.lock'
-        err=0
-        if [[ ! -d "mfem-data" ]]; then
-          git clone ${MFEM_DATA_REPO} "mfem-data"
-        else
-          cd "mfem-data" && git pull && cd ..
-        fi || err=1
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> mfem-data.lock
-    #
-    # Setup ${AUTOTEST_ROOT}/autotest:
-    #
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - mkdir -p ${AUTOTEST_ROOT} && cd ${AUTOTEST_ROOT}
-    - |
-      (
-        date
-        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
-        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -n 9; do
-          sleep 5
-        done
-        echo "Acquired lock on '$PWD/autotest.lock'"
-        date
-        # clone/update the autotest repo while holding the file lock on
-        # 'autotest.lock'
-        err=0
-        if [[ ! -d "autotest" ]]; then
-          git clone ${AUTOTEST_REPO}
-        else
-          cd autotest && git pull && cd ..
-        fi || err=1
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock

.gitlab/corona-build-and-test.yml

@@ -1,67 +0,0 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-stages:
-  - setup
-  - allocate_resource
-  - build_and_test
-  - release_resource_and_report
-
-# Slurm shared allocation
-allocate_resource:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_corona
-  stage: allocate_resource
-  script:
-    - echo ${ALLOC_NAME}
-    - salloc --exclusive --nodes=1 --partition=mi60 --time=45 --no-shell --job-name=${ALLOC_NAME}
-  timeout: 6h
-  needs: [setup]
-
-# Build and test jobs, simply provide a spec
-rocm_gcc_8.3.1:
-  variables:
-    SPEC: "@develop%gcc@8.3.1+rocm amdgpu_target=gfx906"
-  extends: .build_and_test_on_corona
-  needs: [allocate_resource]
-
-# Release slurm allocation
-release_resource:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_corona
-  stage: release_resource_and_report
-  script:
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
-  needs: [rocm_gcc_8.3.1]
-
-# Jobs report
-report_job_success:
-  stage: release_resource_and_report
-  extends:
-    - .on_corona
-    - .report_job_success
-
-report_job_failure:
-  stage: release_resource_and_report
-  extends:
-    - .on_corona
-    - .report_job_failure
-
-include:
-  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/corona-config.yml
-  - local: .gitlab/configs/setup-build-and-test.yml
-  - local: .gitlab/configs/report-build-and-test.yml

.gitlab/lassen-build-and-test.yml

@@ -1,44 +0,0 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-stages:
-  - setup
-  - build_and_test
-  - report
-
-opt_mpi_cuda_xl_16_1_1_12:
-  variables:
-    SPEC: "%xl@16.1.1.12 +mpi +cuda cuda_arch=70"
-  extends: .build_and_test_on_lassen
-
-opt_mpi_cuda_hypre_cuda_xl:
-  variables:
-    SPEC: "%xl@16.1.1.12 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
-  extends: .build_and_test_on_lassen
-
-# Jobs report
-report_job_success:
-  stage: report
-  extends:
-    - .on_lassen
-    - .report_job_success
-
-report_job_failure:
-  stage: report
-  extends:
-    - .on_lassen
-    - .report_job_failure
-
-include:
-  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/lassen-config.yml
-  - local: .gitlab/configs/setup-build-and-test.yml
-  - local: .gitlab/configs/report-build-and-test.yml

.gitlab/lassen.yml

@@ -0,0 +1,34 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# GitLab pipelines configurations for the Lassen machine at LLNL
+
+.on_lassen:
+  tags:
+    - shell
+    - lassen
+  rules:
+    - if: '$CI_COMMIT_BRANCH =~ /_lnone/ || $ON_LASSEN == "OFF"' #run except if ...
+      when: never
+    - when: on_success
+
+# Spack helped builds
+# Generic lassen build job, extending build script
+# Note: Lassen jobs can start as soon as the setup job is complete.
+.build_and_test_on_lassen:
+  extends: [.build_blueos_3_ppc64le_ib_script, .on_lassen]
+  stage: l_build_and_test
+  needs: [setup]
+
+opt_mpi_cuda_xl_16_1_1_8:
+  variables:
+    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=70"
+  extends: .build_and_test_on_lassen

.gitlab/quartz-baseline.yml

@@ -1,135 +0,0 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-variables:
-  BASELINE_TEST: baseline
-
-stages:
-  - setup
-  - baseline_check
-  - baseline_report
-  - cleanup
-  - baseline_publish
-
-baselinecheck_mfem_intel_quartz:
-  extends: [.on_quartz]
-  stage: baseline_check
-  variables:
-    # TPLS_DIR is used in .gitlab/scripts/baseline to provide the tpls location
-    # when call the runtest script in MFEM test repo.
-    # Note: the value must be consistent with the setup performed in
-    #       .gitlab/configs/setup-baseline.yml.
-    TPLS_DIR: ${BUILD_ROOT}/tpls
-  script:
-    - echo ${BUILD_ROOT}
-    - echo ${TPLS_DIR}
-    # Used by the tests in MFEM/tests:
-    - export MFEM_TEST_NP=32
-    # The next script uses the following environment variables:
-    # * BASELINE_TEST, SYS_TYPE, CI_PROJECT_DIR, ARTIFACTS_DIR,
-    # * BUILD_ROOT, TPLS_DIR, MACHINE_NAME
-    - .gitlab/scripts/baseline
-  artifacts:
-    when: always
-    paths:
-      - ${ARTIFACTS_DIR}
-  allow_failure: true
-
-cleanup:
-  extends: .on_quartz
-  stage: cleanup
-  variables:
-    GIT_STRATEGY: none
-  script:
-    - echo "BUILD_ROOT=${BUILD_ROOT}"
-    - rm -rf "${BUILD_ROOT}" || true
-    - echo "CI_PROJECT_DIR=${CI_PROJECT_DIR}"
-    - make -C "${CI_PROJECT_DIR}" distclean
-
-report_baseline:
-  extends: [.on_quartz]
-  stage: baseline_report
-  script:
-    - echo ${MACHINE_NAME}
-    - echo ${AUTOTEST}
-    - echo ${AUTOTEST_COMMIT}
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - cd ${AUTOTEST_ROOT}
-    - |
-      (
-        date
-        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
-        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -n 9; do
-          sleep 5
-        done
-        echo "Acquired lock on '$PWD/autotest.lock'"
-        date
-        # ----------------------
-        cd ${AUTOTEST_ROOT}/autotest || \
-          { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
-        mkdir -p ${MACHINE_NAME}
-        rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-${BASELINE_TEST}-${CI_COMMIT_REF_SLUG}"
-        rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir ${rundir})
-        cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
-        printf "%s\n" "" "Pipeline URL:" "$CI_PIPELINE_URL" \
-          >> ${rundir}/pipeline.txt
-        # We create an autotest-email.html file, because that's how we signal
-        # that there was an error / diff (temporary).
-        if [[ -f ${rundir}/${BASELINE_TEST}.err ]] || \
-           [[ -f ${rundir}/${BASELINE_TEST}-${SYS_TYPE}.diff ]]; then
-          cp ${rundir}/pipeline.txt ${rundir}/autotest-email.html
-        fi
-        msg="GitLab CI log for ${BASELINE_TEST} on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
-        if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
-          git pull && \
-          git add ${rundir} && \
-          git commit -m "${msg}" && \
-          ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
-        else
-          for file in ${rundir}/*; do
-            echo "------------------------------"
-            echo "Content of '$file'"
-            echo "******************************"
-            cat $file
-            echo "******************************"
-          done
-          rm -rf ${rundir} || true
-        fi
-        err=$?
-        # ----------------------
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock
-
-baselinepublish_mfem_quartz:
-  extends: [.on_quartz]
-  stage: baseline_publish
-  rules:
-    # - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "YES"'
-    - if: '$REBASELINE == "YES"'
-      when: manual
-  script:
-    - echo ${BUILD_ROOT}
-    - echo ${PWD}
-    - echo ${ARTIFACTS_DIR}
-    - ls -lA ${ARTIFACTS_DIR}
-    - .gitlab/scripts/rebaseline
-
-include:
-  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/quartz-config.yml
-  - local: .gitlab/configs/setup-baseline.yml

.gitlab/quartz-build-and-test.yml

@@ -1,94 +0,0 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-stages:
-  - setup
-  - allocate_resource
-  - build_and_test
-  - release_resource_and_report
-
-# Allocate
-allocate_resource:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_quartz
-  stage: allocate_resource
-  script:
-    - echo ${ALLOC_NAME}
-    - salloc --exclusive --nodes=1 --reservation=ci --time=60 --no-shell --job-name=${ALLOC_NAME}
-  timeout: 6h
-
-# GitLab jobs for the Quartz machine at LLNL
-debug_ser_gcc_10:
-  variables:
-    SPEC: "%gcc@10.3.1 +debug~mpi"
-  extends: .build_and_test_on_quartz
-
-debug_par_gcc_10:
-  variables:
-    SPEC: "%gcc@10.3.1 +debug+mpi"
-  extends: .build_and_test_on_quartz
-
-opt_ser_gcc_10:
-  variables:
-    SPEC: "%gcc@10.3.1 ~mpi"
-  extends: .build_and_test_on_quartz
-
-opt_par_gcc_10:
-  variables:
-    SPEC: "%gcc@10.3.1"
-  extends: .build_and_test_on_quartz
-
-opt_par_gcc_10_sundials:
-  variables:
-    SPEC: "%gcc@10.3.1 +sundials"
-  extends: .build_and_test_on_quartz
-
-opt_par_gcc_10_petsc:
-  variables:
-    SPEC: "%gcc@10.3.1 +petsc ^petsc+mumps~superlu-dist"
-  extends: .build_and_test_on_quartz
-
-opt_par_gcc_10_pumi:
-  variables:
-    SPEC: "%gcc@10.3.1 +pumi"
-  extends: .build_and_test_on_quartz
-
-# Release
-release_resource:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_quartz
-  stage: release_resource_and_report
-  script:
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
-
-# Jobs report
-report_job_success:
-  stage: release_resource_and_report
-  extends:
-    - .on_quartz
-    - .report_job_success
-
-report_job_failure:
-  stage: release_resource_and_report
-  extends:
-    - .on_quartz
-    - .report_job_failure
-
-include:
-  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/quartz-config.yml
-  - local: .gitlab/configs/setup-build-and-test.yml
-  - local: .gitlab/configs/report-build-and-test.yml

.gitlab/quartz.yml

@@ -0,0 +1,184 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# GitLab pipelines configurations for the Quartz machine at LLNL
+
+.on_quartz:
+  tags:
+    - shell
+    - quartz
+  rules:
+    # Don't run quartz jobs if...
+    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"'
+      when: never
+    # Don't run autotest update if...
+    - if: '$CI_JOB_NAME =~ /update_autotest/ && $AUTOTEST != "YES"'
+      when: never
+    # Don't run autotest update if...
+    - if: '$CI_JOB_NAME =~ /q_report/ && $AUTOTEST != "YES"'
+      when: never
+    # Report success on success status
+    - if: '$CI_JOB_NAME =~ /q_report_success/ && $AUTOTEST == "YES"'
+      when: on_success
+    # Report failure on failure status
+    - if: '$CI_JOB_NAME =~ /q_report_failure/ && $AUTOTEST == "YES"'
+      when: on_failure
+    # Always release resources
+    - if: '$CI_JOB_NAME =~ /release_resources/'
+      when: always
+    # Default is to run if previous stage succeeded
+    - when: on_success
+
+# This is a yaml anchor, it can be used to avoid duplication like here.
+# The code below will simply be pasted wherever the anchor is placed.
+.safe_create_rundir: &safe_create_rundir |
+  if ! mkdir ${rundir}; then
+    n=1
+    while ! mkdir ${rundir}_${n}
+    do
+      n=$((n+1))
+    done
+    rundir=${rundir}_${n}
+  fi
+
+# Allocate
+q_allocate_resources:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_quartz
+  stage: q_allocate_resources
+  script:
+    - salloc --exclusive --nodes=1 --partition=pdebug --time=30 --no-shell --job-name=${ALLOC_NAME}
+  timeout: 6h
+
+# Release
+q_release_resources:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_quartz
+  stage: q_release_resources
+  script:
+    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
+    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
+
+# Release
+q_report_success:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_quartz
+  stage: q_release_resources
+  script:
+    - echo "Can only run if all the quartz jobs passed"
+    - cd ${AUTOTEST_ROOT}/autotest && git pull
+    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
+    - *safe_create_rundir
+    - echo "The Quartz jobs were successful" > ${rundir}/gitlab.out
+    - echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
+    - git add ${rundir}
+    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
+    - git push origin master
+
+q_report_failure:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_quartz
+  stage: q_release_resources
+  script:
+    - echo "Runs if there was at least one failure on quartz"
+    - cd ${AUTOTEST_ROOT}/autotest && git pull
+    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
+    - *safe_create_rundir
+    - echo "There was an error while running CI on Quartz" > ${rundir}/gitlab.err
+    - echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
+    - cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
+    - git add ${rundir}
+    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
+    - git push origin master
+
+# Spack helped builds
+# Generic quartz build job, extending build script
+.build_and_test_on_quartz:
+  extends: [.build_toss_3_x86_64_ib_script, .on_quartz]
+  stage: q_build_and_test
+
+# Build MFEM
+debug_ser_gcc_4_9_3:
+  variables:
+    SPEC: "%gcc@4.9.3 +debug~mpi"
+  extends: .build_and_test_on_quartz
+
+debug_ser_gcc_6_1_0:
+  variables:
+    SPEC: "%gcc@6.1.0 +debug~mpi"
+  extends: .build_and_test_on_quartz
+
+debug_par_gcc_6_1_0:
+  variables:
+    SPEC: "%gcc@6.1.0 +debug+mpi"
+  extends: .build_and_test_on_quartz
+
+opt_ser_gcc_6_1_0:
+  variables:
+    SPEC: "%gcc@6.1.0 ~mpi"
+  extends: .build_and_test_on_quartz
+
+opt_par_gcc_6_1_0:
+  variables:
+    SPEC: "%gcc@6.1.0"
+  extends: .build_and_test_on_quartz
+
+opt_par_gcc_6_1_0_sundials:
+  variables:
+    SPEC: "%gcc@6.1.0 +sundials"
+  extends: .build_and_test_on_quartz
+
+opt_par_gcc_6_1_0_petsc:
+  variables:
+    SPEC: "%gcc@6.1.0 +petsc ^petsc+mumps"
+  extends: .build_and_test_on_quartz
+
+opt_par_gcc_6_1_0_pumi:
+  variables:
+    SPEC: "%gcc@6.1.0 +pumi"
+  extends: .build_and_test_on_quartz
+
+# Baseline jobs form an independent set of jobs. We use `needs:[]` to specify
+# that "setup-baseline" can start immediately. Then, we have to use needs for
+# each one of the baseline jobs, otherwise they will wait for the rest of the
+# pipeline.
+
+baselinecheck_mfem_intel_quartz:
+  extends: [.baselinecheck_mfem, .on_quartz]
+  needs: [setup_baseline]
+
+update_autotest:
+  extends: [.on_quartz]
+  needs: [baselinecheck_mfem_intel_quartz]
+  stage: baseline_to_autotest
+  script:
+    - cd ${AUTOTEST_ROOT}/autotest && git pull
+    - rundir="quartz/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
+    - *safe_create_rundir
+    - cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
+    # We create an autotest-email.html file, because that's how we signal that there was a diff (temporary).
+    - |
+      if [[ -f ${rundir}/*.err ]]
+      then
+        echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/*.err
+        cp ${rundir}/*.err ${rundir}/autotest-email.html
+      fi
+    - git add ${rundir}
+    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
+    - git push origin master
+
+baselinepublish_mfem_quartz:
+  extends: [.on_quartz, .rebaseline_mfem]
+  needs: [baselinecheck_mfem_intel_quartz]

.gitlab/scripts/baseline

@@ -1,75 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# locals
-glob_err=${BASELINE_TEST}.err
-base=${BASELINE_TEST}-${SYS_TYPE}
-base_diff=${base}.diff
-base_patch=${base}.patch
-base_out=${base}.out
-artifacts_path=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}
-
-# prepare
-cd ${BUILD_ROOT} || \
-  { echo "Invalid BUILD_ROOT=$BUILD_ROOT"; exit 1; }
-ln -snf ${CI_PROJECT_DIR} mfem
-cd tests
-[[ -d _${BASELINE_TEST} ]] && rm -rf _${BASELINE_TEST}
-mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
-
-# run
-if [[ "${MACHINE_NAME}" == "quartz" || "${MACHINE_NAME}" == "ruby" ]]; then
-  salloc --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
-elif [[ ${MACHINE_NAME} == "corona" ]]; then
-  salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
-elif [[ ${MACHINE_NAME} == "lassen" ]]; then
-  lalloc 1 -q pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
-else
-  echo "Unknown machine: MACHINE_NAME=$MACHINE_NAME"
-  exit 1
-fi
-
-# post
-mkdir ${artifacts_path}
-
-status=0
-if [[ -f ${BASELINE_TEST}.out ]]; then
-  cp ${BASELINE_TEST}.out ${artifacts_path}
-fi
-if [[ -s ${glob_err} ]]; then
-  echo "ERROR during ${BASELINE_TEST} execution"
-  echo "Here is the ${glob_err} file content"
-  cat ${glob_err}
-  cp ${glob_err} ${artifacts_path}/${glob_err}
-  status=1
-fi
-if [[ -f ${base_patch} ]]; then
-  echo "${BASELINE_TEST}: Differences found, patch generated"
-  cp ${base_patch} ${artifacts_path}/${base_patch}
-elif [[ -f ${base_out} ]]; then
-  echo "${BASELINE_TEST}: Differences found, replacement file generated"
-  cp ${base_out} ${artifacts_path}/${base_out}
-fi
-# base_diff won't even exist if there is no difference.
-if [[ -f ${base_diff} ]]; then
-  echo "${BASELINE_TEST}: Relevant differences (filtered diff) ..."
-  cat ${base_diff}
-  cp ${base_diff} ${artifacts_path}/${base_diff}
-  status=1
-fi
-if [[ $status -eq 0 ]]; then
-  echo "${BASELINE_TEST}: PASSED"
-else
-  echo "${BASELINE_TEST}: FAILED"
-fi
-exit $status

.gitlab/scripts/git_try_to_push

@@ -1,41 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Try to push to the remote 5 times. If the push fails, and the local and remote
-# have diverged, then pull from the remote to merge changes, and try pushing
-# again. If some other failure happens
-for i in {1..5}; do
-   git push origin master && exit 0
-   # Wait for 20 seconds in case someone else is pushing to the remote
-   # concurrently
-   sleep 20
-   # Fetch any updates from the remote
-   git remote update
-   # Get the latest commit on the local branch
-   LOCAL=$(git rev-parse @)
-   # Get the latest commit on the remote
-   REMOTE=$(git rev-parse @{u})
-   # Get the common ancestor
-   BASE=$(git merge-base @ @{u})
-   # Have the local and remote diverged?
-   if [[ $LOCAL != $REMOTE && $LOCAL != $BASE && $REMOTE != $BASE ]]; then
-      git pull
-      if [[ $? == 0 ]]; then
-         continue
-      else
-         exit 1 # Something else went wrong trying to pull
-      fi
-   fi
-done
-
-exit 1 # Did not succeed in 5 attempts

.gitlab/scripts/rebaseline

@@ -1,49 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# There will be collision between corona and quartz baselines.
-# Once the corresponding files have been generated, we can switch to machine
-# specific ref.
-ARTIFACT_PATH=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}
-#ARTIFACT_PATH=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}-${MACHINE_NAME}
-
-PATCH_FILE=${ARTIFACT_PATH}.patch
-FULL_FILE=${ARTIFACT_PATH}.out
-DIFF_FILE=${ARTIFACT_PATH}.diff
-
-# There will be collision between corona and quartz baselines.
-# Once the corresponding files have been generated, we can switch to machine
-# specific ref.
-SAVED_NAME=baseline-${SYS_TYPE}.saved
-#SAVED_NAME=baseline-${SYS_TYPE}-${MACHINE_NAME}.saved
-
-cd ${BUILD_ROOT}/tests
-
-if [[ ! -f "${DIFF_FILE}" ]]
-then
-    echo "Nothing to be done: no relevant change in baseline"
-    exit 0
-elif [[ -f "${PATCH_FILE}" ]]
-then
-    patch "${SAVED_NAME}" < "${PATCH_FILE}"
-elif [[ -f "${FULL_FILE}" ]]
-then
-    cp "${FULL_FILE}" "${SAVED_NAME}"
-else
-    echo "File missing: expected ${PATCH_FILE} or ${FULL_FILE}"
-    exit 1
-fi
-
-git add "${SAVED_NAME}"
-git commit -m "${SYS_TYPE} (${MACHINE_NAME}) rebaselined in GitLab pipeline ${CI_PIPELINE_ID}"
-git push origin master

.gitlab/scripts/report_build_and_test_failure

@@ -1,45 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-echo "Runs if there was at least one failure on ${MACHINE_NAME}"
-
-cd ${AUTOTEST_ROOT}/autotest || \
-  { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
-mkdir -p ${MACHINE_NAME}
-
-rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
-rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir)
-
-printf "%s\n" "Some 'build-and-test' jobs on ${MACHINE_NAME} FAILED." \
-  "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.err
-
-msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
-
-# Create 'autotest-email.html' to indicate failure:
-cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
-
-if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
-  git pull && \
-  git add ${rundir} && \
-  git commit -m "${msg}" && \
-  ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
-else
-  for file in ${rundir}/*; do
-    echo "------------------------------"
-    echo "Content of '$file'"
-    echo "******************************"
-    cat $file
-    echo "******************************"
-  done
-  rm -rf ${rundir} || true
-fi

.gitlab/scripts/report_build_and_test_success

@@ -1,42 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-echo "Can only run if all the ${MACHINE_NAME} jobs passed"
-
-cd ${AUTOTEST_ROOT}/autotest || \
-  { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
-mkdir -p ${MACHINE_NAME}
-
-rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
-rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir)
-
-printf "%s\n" "The 'build-and-test' jobs on ${MACHINE_NAME} were SUCCESSFUL." \
-  "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.out
-
-msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
-
-if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
-  git pull && \
-  git add ${rundir} && \
-  git commit -m "${msg}" && \
-  ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
-else
-  for file in ${rundir}/*; do
-    echo "------------------------------"
-    echo "Content of '$file'"
-    echo "******************************"
-    cat $file
-    echo "******************************"
-  done
-  rm -rf ${rundir} || true
-fi

.gitlab/scripts/safe_create_rundir

@@ -1,42 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# This script takes a seed for a directory name and appends it with a counter
-# incremented until it can create a new directory with it.
-
-# Usage:
-#
-# Expects 1 argument: a string that is use as a seed for the directory name.
-#
-# > rundir="desired_name"
-# > rundir=$(./safe_create_rundir $rundir)
-
-set -o errexit
-set -o nounset
-
-rundir=${1:-""}
-if [[ -z ${rundir} ]]; then
-  >&2 echo "The script expects a string as argument for directory creation."
-  exit 1
-fi
-
-if ! mkdir ${rundir}; then
-  n=1
-  while ! mkdir ${rundir}_${n}
-  do
-    n=$((n+1))
-  done
-  rundir=${rundir}_${n}
-fi
-
-echo $rundir

CHANGELOG

@@ -8,525 +8,24 @@
                                https://mfem.org
 
 
-Version 4.6.1 (development)
+Version 4.3.1 (development)
 ===========================
+- Added support for hr-adaptivity using TMOP-based error estimator.
 
-Meshing improvements
---------------------
-- Introduced formulas for refinement of patches in NURBS meshes. Refinement by
-  arbitrary integer factors is also enabled, e.g. in the mesh-explorer miniapp.
-  NURBS coarsening and knot removal are also introduced.
-
-Discretization improvements
----------------------------
-- Introduced support for higher order non conformal Nedelec elements on
-  simplices in ParMesh.
-- Introduced support for internal boundary elements in nonconformal adapted
-  meshes.
-
-- Added functionality for construction of cut-surface and cut-volume
-  IntegrationRules through a moment-fitting approach. The cut is specified by
-  the zero level set of a Coefficient. See fem/intrules_cut.hpp and Example 38.
-
-GPU support
-----------------------------
-- Added support for full assembly on simplices.
-- Added functionality for BilinearFormIntegrators to use kernels that work for both
-  tensor and unstructured elements.
-- Added partial assembly for linear elasticity. Does not use sum factorization for now.
-
-New and updated examples and miniapps
--------------------------------------
-- Added a new block solver in miniapp/solvers for the Darcy problem.
-  The new solver is based on a Bramble-Pasciak preconditioning. User can
-  use and implement their own preconditioner for the mass matrix.
-
-- Added miniapp to demonstrate new elasticity integrator and unstructured element GPU support,
-  and a block diagonal preconditioner using low order refinement. Allows comparison with
-  currently existing legacy mode integrator. See miniapps/solvers/lor_elast.
-
-Miscellaneous
--------------
-- The ReadCubit Genesis mesh importer has been rewritten to improve readability.
-
-- Updated the Doxygen documentation style, which now requires Doxygen version
-  1.9.8 or later. See the doc/ directory.
-
-- Improved thread safety for global variables in the library, for example
-  IntegrationRules IntRules, RefinedIntRules, GeometryRefiner
-  GlobGeometryRefiner, and FiniteElement::dof2quad_array.
-
-
-Version 4.6, released on September 27, 2023
-===========================================
-
-- MFEM is now available in Homebrew and can be installed on a Mac with just
-  "brew install mfem". See https://formulae.brew.sh/formula/mfem.
-
-Meshing improvements
---------------------
-- Added asymptotically-balanced TMOP compound metrics 90, 94, 328, 338. A new
-  tool, tmop-metric-magnitude, can be used to track how metrics change under
-  geometric perturbations. See miniapps/tools.
-
-- Several NURBS meshing improvements:
-  * Support for free connectivity of NURBS patches allowing for more complex
-    patch configurations such as C-meshes.
-  * New methods to set and get attributes on NURBS patches and patch boundaries.
-  * The edge to knot map for NURBS meshes can be determined automatically. It is
-    no longer needed to specify this in the NURBS mesh.
-  * Added curve interpolation method for NURBS.
-  * Added new small miniapp for printing of shape functions of a KnotVector
-  * See miniapps/nurbs for example meshes and miniapps.
-
-Discretization improvements
----------------------------
-- SubMesh and ParSubMesh have been extended to support the transfer of
-  Nedelec and Raviart-Thomas finite element spaces.
-
-- Added support for partial assembly on NURBS patches, and NURBS-patch sparse
-  matrix assembly. Patch matrix assembly includes the option to use reduced
-  approximate integration rules, computed by the newly implemented non-negative
-  least-squares (NNLS) solver.
-
-- Support for parallel transfer of H1 fields using the low-order refined (LOR)
-  transfer operators in L2ProjectionGridTransfer
-
-- Added KDTree class for 2D/3D set of points, which is then utilized in the new
-  KDTreeNodalProjection class to project a function defined on an arbitrary set
-  of points onto an MFEM grid function. This functionality is demonstrated in
-  the nodal-transfer miniapp. The current implementation is serial only. Further
-  extensions can include search in arbitrary dimensional spaces.
-
-- Added support for p-refined meshes in GSLIB-FindPoints.
-
-- Device kernels can now access device-specific DOF and quadrature limits using
-  the DofQuadLimits structure, allowing increased limits when executing on CPU.
-  The limits for the runtime selected device can be accessed in host code using
-  DeviceDofQuadLimits::Get(). The global constants MAX_D1D and MAX_Q1D are no
-  longer available.
-
-- Face restriction operators for Nedelec and Raviart-Thomas finite element
-  spaces are now supported through the ConformingFaceRestriction class.
-
-- VectorFEBoundaryFluxLFIntegrator is now supported on device/GPU.
-
-Linear and nonlinear solvers
-----------------------------
-- Updated the MUMPS interface to support multiple right-hand sides, block
-  low-rank compression, builds using 64-bit integers, and other improvements.
-
-- Added an interface to the MKL Pardiso sparse direct solver developed by Intel.
-  The interface provides a serial (OpenMP shared memory) version of Pardiso for
-  use with SparseMatrix. This complements the existing parallel (MPI distributed
-  memory) version already available through the CPardiso MFEM integration.
-
-- Added HIP support to the PETSc and SUNDIALS interfaces.
-
-- Efficient GPU-accelerated LOR assembly now supports surface meshes.
-
-New and updated examples and miniapps
--------------------------------------
-- Added a new H(div) solver miniapp demonstrating the use of a matrix-free
-  saddle-point solver methodology, suitable for high-order discretizations and
-  for GPU acceleration. Examples illustrating the solution of Darcy and grad-div
-  problems are included. See miniapps/hdiv-linear-solver.
-
-- Added new Discontinuous Petrov-Galerkin (DPG) miniapp which includes serial
-  and parallel examples for diffusion, convection-diffusion, acoustics and
-  Maxwell equations. The miniapp includes new classes such as (Par)DPGWeakForm,
-  (Par)ComplexDPGWeakForm and (Complex)BlockStaticCondensation. Three new
-  integrators are added in support of DPG systems: TraceIntegrator,
-  NormalTraceIntegrator and TangentTraceIntegrator. See miniapps/dpg.
-
-- Added a new miniapp that implements the SPDE method for generating Gaussian
-  random fields of Matern covariance. The resulting random field can be used,
-  e.g., to model material uncertainties. See miniapps/spde.
-
-- Added a new parallel LOR transfer miniapp, plor-transfer, which mirrors the
-  functionality of the serial LOR transfer miniapp. See miniapps/tools.
-
-- New serial miniapp, nodal-transfer, demonstrating the use of KDTree to map a
-  parallel grid function to a different parallel partitioning of the same mesh.
-
-- Added 3 additional TMOP miniapps in miniapps/meshing:
-  * Mesh-Quality evaluates quality using size, skewness, and aspect-ratio
-    computed from the Jacobian of the transformation.
-  * Mesh-Fitting can be used for interface and boundary fitting to implicit
-    domains defined using level-set functions.
-  * Fit-Node-Position fits selected mesh nodes to specified positions, while
-    maintaining overall mesh quality.
-
-- Added 4 new example codes:
-  * Example 34/34p solves a simple magnetostatic problem where source terms and
-    boundary conditions are transferred with SubMesh objects.
-  * Example 35p implements H1, H(curl) and H(div) variants of a damped harmonic
-    oscillator with field transfer using SubMesh objects.
-  * Example 36/36p demonstrates the solution of the obstacle problem with a new
-    finite element method (proximal Galerkin).
-  * Example 37/37p demonstrates topology optimization with MFEM.
-
-- Added a random refinement option to the mesh-explorer miniapp to assist users
-  in experimenting with nonconforming meshes.
-
-- Moved the distance solver methods from miniapps/shifted to miniapps/common.
-
-Miscellaneous
--------------
-- Improved lambda body debugging with the addition of mfem::forall functions.
-  These functions can take the place of the MFEM_FORALL macros, which have been
-  preserved for backwards compatibility.
-
-- Added an address sanitizer GitHub action for a serial build/test on Ubuntu,
-  based on Clang/LLVM (https://clang.llvm.org/docs/AddressSanitizer.html).
-
-- Reorganized files for bilinear form, linear form, and nonlinear form integrators
-  in the fem/integ/ subdirectory.
-
-- FiniteElementSpace::GetFE has been updated to abort instead of returning NULL for
-  an empty partition.
-
-- Various other simplifications, extensions, and bugfixes in the code.
-
-
-Version 4.5.2, released on March 23, 2023
-=========================================
-
-- Added support for pyramids in non-conforming meshes. Currently only isotropic
-  refinement is supported in this case.
-
-- Removed the support for the Mesquite toolkit. We recommend using MFEM's TMOP
-  functionality instead for mesh optimization. See the mesh-optimizer miniapp.
-
-- Added a fast normalization-based distance solver, see the Distance miniapp
-  in the miniapps/shifted/ directory.
-
-- Added a new meshing miniapp, Reflector, which reflects a high-order or NURBS
-  hexahedral mesh about a plane.
-
-- Updated logic in FindPointsGSLIB to ignore points found near (but outside) the
-  domain boundary.
-
-- Added an option to auto-balance compound TMOP metrics.
-
-- Fixed a bug in TMOP metric 301.
-
-- When using discontinuous (L2) spaces, use local (element-wise) L2 projection
-  as the coarsening operator for non-conforming AMR meshes.
-
-- Added support for GridFunction::GetGradients() and GetVectorGradient() on
-  face-neighbor elements.
-
-- Added support for pyramids in Gmsh meshes.
-
-- The Mesh Explorer miniapp can now save mesh files in the VisIt or ParaView
-  formats using the corresponding DataCollection. See option 'D' in the menu.
-
-- VisItDataCollection now correctly handles data collection names containing
-  underscores.
-
-- Added support for shared Windows builds with MSVC through CMake.
-  Developers note: this enhancement is facilitated by the addition of the macro
-  MFEM_EXPORT, see config.hpp for more details on its usage.
-
-- The following integrations have updated minimum version requirements:
-  * RAJA >= 2022.10.3
-
-API changes
------------
-- The implicit cast methods of class Vector to 'double *' and 'const double *'
-  have been deprecated and generate deprecation warnings if used. They will be
-  removed in a future release.
-
-- The methods Mesh::GetFaceBaseGeometry and Mesh::GetFaceGeometryType have been
-  deprecated, and Mesh::GetFaceGeometry (which provides identical functionality)
-  should be used instead.
-
-- VisItDataCollection::SetPadDigits() no longer alters the number of digits
-  used to represent the MPI rank because VisIt seems to require 6 digits.
-  This parameter can still be explicitly overridden with SetPadDigitsRank().
-
-
-Version 4.5, released on October 22, 2022
-=========================================
-
-Meshing improvements
---------------------
-- Added new SubMesh and ParSubMesh classes that can be used to extract a subset
-  of a given Mesh. These classes have the same functionality as Mesh and ParMesh
-  and work with all existing MFEM interfaces like finite element spaces etc.
-
-- Added a method, ParMesh::GetSerialMesh(), that reconstructs a partitioned
-  parallel mesh on a given single rank. Also, added ParMesh::PrintAsSerial(),
-  which saves the reconstructed serial mesh to a C++ stream on rank 0.
-
-- Added more 3D TMOP metrics, as well as specialized metrics for mesh
-  untangling and worst-case quality improvement.
-
-- Added a new method, Mesh::NodesUpdated, which should be called after the mesh
-  node coordinates have changed, e.g. after the mesh has moved. This is
-  necessary, for example, with device assembly of linear and bilinear forms.
-
-- Added support for mixed meshes and pyramids in GSLIB-FindPoints.
-
-Discretization improvements
----------------------------
-- Added full assembly and device support for several LinearForm integrators:
-  * DomainLF: (f, v)
-  * VectorDomainLF: ((f1,...,fn), (v1,...,vn))
-  * DomainLFGrad: (f, grad(v))
-  * VectorDomainLFGrad: ((f1x,f1y,f1z,...,fnx,fny,fnz), grad(v1,...,vn))
-  The device assembly of linear forms has to be explicitly enabled by calling
-  LinearForm::UseFastAssembly(true), otherwise the legacy linear form assembly
-  is used by default.
-
-- Added support for assembling low-order-refined matrices using a GPU-enabled
-  "batched" algorithm. The lor_solvers and plor_solvers now fully support GPU
-  acceleration with arbitrary user-supplied coefficients.
-
-- Added a new class FaceQuadratureSpace that allows for the construction of
-  QuadratureFunctions on the interior or boundary faces of a mesh.
-
-- Added a class CoefficientVector for efficient access of variable coefficient
-  values at quadrature points (in particular for GPU/device kernels).
-
-- Added WhiteGaussianNoiseDomainLFIntegrator: a LinearFormIntegrator class for
-  spatial Gaussian white noise.
-
-- Added a new Zienkiewicz-Zhu patch recovery-based a posteriori error estimator.
-  See fem/estimators.hpp.
-
-- Various fixes and improvements in LinearFormExtension.
-
-Linear and nonlinear solvers
-----------------------------
-- Added a new class DGMassInverse that performs a local element-wise CG
-  iteration to solve systems involving the discontinuous Galerkin mass matrix,
-  including support for device/GPU acceleration.
-
-- Added more flexibility to the constrained solver classes:
-  * PenaltyConstrainedSolver now allows for a vector of penalty parameters
-    (necessary for penalty contact)
-  * PenaltyConstrainedSolver and EliminationSolver can use GMRES or PCG
-  * All constraint solver classes can take a user-defined preconditioner
-
- - Added functions to toggle additional options for the SuperLU_Dist and Hypre
-   preconditioners (ParaSails, Euclid, ILU).
-
-- Added boundary elimination with device support for `SparseMatrix` and
-  `HypreParMatrix`.
-
-New and updated examples and miniapps
--------------------------------------
-- Added a new elasticity miniapp, Hooke, that showcases a low-level approach of
-  using MFEM to solve a nonlinear elasticity problem based on the fundamental
-  finite element operator decomposition. The miniapp also integrates with
-  automatic differentiation tools like a native dual number implementation or a
-  third party library such as Enzyme. See miniapps/elasticity for more details.
-
-- Added example for body-fitted volumetric and shape integration using the
-  Algoim library in miniapps/shifted.
-
-- Add a new example code, Example 33/33p, to demonstrate the solution of
-  spectral fractional PDEs with MFEM.
-
-Integrations, testing and documentation
----------------------------------------
-- Added a Dockerfile for a simple MFEM container, see config/docker/README.md.
-  More sophisticated developer containers are available in the new repo
-  https://github.com/mfem/containers.
-
-- Added support for the LLVM-based automatic differentiation tool Enzyme, see
-  https://github.com/EnzymeAD/Enzyme. Build system flags and a convenience
-  header are provided. The functionality and interaction are demonstrated in a
-  new miniapp in miniapps/elasticity.
-
-- Added support for partial assembly and fully matrix-free operators on mixed
-  meshes (different element types and p-adaptivity) through libCEED, including
-  device acceleration, e.g. with NVIDIA and AMD GPUs. The p-adaptivity is
-  currently limited to 2D serial meshes. All mixed element topologies are
-  supported in both serial and parallel.
-
-- Added support for ParMoonolith, https://bitbucket.org/zulianp/par_moonolith,
-  which provides parallel non-conforming, non-matching, variational, volumetric
-  mesh information transfer. With ParMortarAssember, fields can be exchanged
-  between arbitrarily distributed and unrelated finite element meshes in a
-  variationally consistent way.
-
-- Fully encapsulated SUNDIALS `N_Vector` object within the `SundialsNVector`
-  class by removing deprecated (e.g. `HypreParVector::ToNVector`) and
-  non-deprecated (e.g. `Vector::ToNVector`) functions in other classes.
-
-- New benchmark for the different assembly levels inspired by the CEED
-  Bake-Off Problems, see tests/benchmarks/bench_assembly_levels.cpp.
-
-- Added Windows 2022 CI testing with GitHub actions.
-
-Miscellaneous
--------------
-- The method SparseMatrix::EnsureMultTranspose() is now automatically called
-  by the methods AddMultTranspose(), MultTranspose(), and AbsMultTranspose().
-  Added a method with the same name to class HypreParMatrix which is also called
-  automatically by the HypreParMatrix::MultTranspose() methods.
-
-- Updated various MemoryUsage methods to return 'std::size_t' instead of 'long'
-  since the latter is 32-bit in Win64 builds.
-
-- When using `AssemblyLevel::FULL`, `FABilinearFormExtension::FormSystemMatrix`
-  outputs an `OperatorHandle` containing a `SparseMatrix` in serial, and an
-  `HypreParMatrix` in parallel (instead of a `ConstrainedOperator`).
-
-- In various places in the library, replace the use of 'long' with 'long long'
-  to better support Win64 builds where 'long' is 32-bit and 'long long' is
-  64-bit. On Linux and MacOS, both types are typically 64-bit.
-
-- The behavior of GridFunction::GetTrueVector() has been changed to not return
-  an empty true vector.
-
-- Added support for ordering search points byVDIM in FindPointsGSLIB.
-
-- Various other simplifications, extensions, and bugfixes in the code.
-
-
-Version 4.4, released on March 21, 2022
-=======================================
-
-Linear and nonlinear solvers
-----------------------------
-- Added support for using the hypre library built with HIP support. Similar to
-  the existing hypre + CUDA support, most of the MFEM examples and miniapps work
-  transparently with hypre + HIP builds. This includes the BoomerAMG, AMS, and
-  ADS solvers.
-
-- Added a simple singleton class, Hypre, to automatically set hypre's global
-  parameters, particularly GPU-relevant options. Updated parallel example codes
-  and miniapps to call Hypre::Init() where appropriate.
-
-- Added hipSPARSE support for sparse matrix-vector multiplications.
-
-- More explicit and consistent formatting of the output of iterative solvers
-  with the new IterativeSolver::PrintLevel options. See linalg/solvers.hpp.
-
-Meshing improvements
---------------------
-- New TMOP-based methods for hr-adaptivity, interface fitting, and tangential
-  relaxation of high-order meshes.
+- Adding lowest order Nedelec and Raviart-Thomas basis functions on wedge
+  shaped elements.
 
 - Added initial support for meshes with pyramidal elements, including several
   pyramidal meshes in the data/ directory and support for the lowest order H1,
   Nedelec, Raviart-Thomas, and L2 basis functions on pyramids.
 
-- Added a simpler interface to access mesh face information, see FaceInformation
-  and GetFaceInformation in the Mesh class.
+- Updated the hypre interface according to changes in hypre-2.22.1. The ADS
+  solver is now fully working on GPUs.
 
-- Gmsh meshes where all elements have zero physical tag (the default Gmsh output
-  format if no physical groups are defined) are now successfully loaded, and
-  elements are reassigned attribute number 1.
-
-- Added ParMesh adjacency set (adjset) creation support to the Conduit Mesh
-  Blueprint MFEM wrapper functions in ConduitDataCollection.
-
-Discretization improvements
----------------------------
-- Added general dof transformation to support high order Nedelec basis functions
-  on tetrahedral meshes without reordering. The ReorientTetMesh method of the
-  Mesh and ParMesh classes has been deprecated. See the new DofTransformation
-  class in fem/doftrans.hpp.
-
-- GPU-enabled partial (PA) and element (EA) assembly for discontinuous Galerkin
-  methods on nonconforming AMR meshes.
-
-- Support for arbitrary order Nedelec and Raviart-Thomas elements on wedges.
-
-- Added special Nedelec and Raviart-Thomas basis functions for modeling three
-  dimensional vector fields in 1D and 2D domains, see the new Example 31/31p.
-
-- GridFunctionCoefficient (and the related vector, gradient, divergence, and
-  curl classes) now work properly with LORDiscretization and LORSolver.
-
-- Added PA support for the action of MixedScalarCurlIntegrator in 2D and
-  MixedVectorGradientIntegrator in 2D and 3D, as well as their transposes.
-
-- Coefficient::SetTime now propagates the new time into internally stored
-  Coefficient objects.
-
-- Split the fem/fe.?pp files into separate files in the new fem/fe/ directory to
-  simplify and clarify the organization of FiniteElement classes.
-
-New and updated examples and miniapps
--------------------------------------
-- Added two new miniapps with initial support for automatic differentiation (AD)
-  in the miniapps/autodiff/ directory. Users can select between external library
-  and native implementation during configuration. The support for AD will be
-  extended in future releases of MFEM.
-
-- Added Binder (mybinder.org) configuration files for C++ MFEM Jupyter Notebooks
-  with inline GLVis visualization in the new examples/jupyter/ directory with a
-  sample notebook based on Example 1. The implementation is based on xeus-cling,
-  see github.com/jupyter-xeus/xeus-cling and github.com/GLVis/xeus-glvis.
-
-- Added a new miniapp (Extrapolation) for PDE-based extrapolation of finite
-  element functions from known values in a set of elements to the rest of the
-  computational domain. See miniapps/shifted/extrapolate.cpp.
-
-- Added new miniapp that uses the ParELAG library, its hybrid smoothers, and the
-  hierarchy of spaces created by the element-based AMG (AMGe) methodology in
-  ParELAG to build multigrid solvers for H(curl) and H(div) forms. See the
-  miniapps/parelag directory for more details.
-
-- Added a new Example 30/30p demonstrating support for mesh preprocessing to
-  resolve fine scale problem data before simulation. This feature uses adaptive
-  mesh refinement to control the associated data oscillation error.
-
-- Added new Examples 31, 31p and 32p showing anisotropic definite Maxwell
-  serial/parallel solver and parallel eigensolver 1D, 2D, or 3D.
-
-- Updated the mesh-optimizer and pmesh-optimizer miniapps to demonstrate the
-  hr-adaptivity and interface fitting capability.
-
-- The HPC versions of ex1 and ex1p (in miniapps/performance) now support runtime
-  selection of either 2D or 3D meshes.
-
-Integrations, testing and documentation
----------------------------------------
-- Doxygen documentation for all releases is now available at docs.mfem.org.
-
-- The following integrations have updated minimum version requirements:
-  * HIOP >= 0.4.6
-  * HYPRE >= 2.23.0 for HIP support
-  * libCEED >= 0.10
-  * PUMI >= 2.2.6
-  * RAJA >= 0.14.0
-  * Umpire >= 3.0.0
-  see INSTALL for more details.
-
-- Added new optional integrations with ParELAG and CoDiPack (version >= 1.9.3+).
-
-- Added initial support for Google Benchmark (version >= 1.5.6) in the
-  tests/benchmarks directory. It can be enabled with MFEM_USE_BENCHMARK=YES.
-
-- Switched from Artistic Style (astyle) version 2.05.1 to version 3.1 for code
-  formatting. See the "make style" target.
-
-Miscellaneous
--------------
-- Added a simple singleton class, Mpi, as a replacement for MPI_Session. New
-  code should use Mpi::Init() and other Mpi methods instead of MPI_Session.
-
-- Added ParaView visualization of QuadratureFunction fields, through both
-  QuadratureFunction::SaveVTU and ParaViewDataCollection::RegisterQField.
-
-- Fixed several MinGW build issues on Windows.
-
-- Added 'double' atomicAdd implementation for previous versions of CUDA.
-
-- HypreParVector and Vector now support C++ move semantics, and the copy
-  constructor for HypreParVector now copies the local vector data.
-
-- Removed the 'u' flag in the ar command, to update all files in the archive,
-  avoiding file name collisions from different subdirectories.
-
-- Various other simplifications, extensions, and bugfixes in the code.
+- Tetrahedral meshes no longer need to be reordered to support high order
+  Nedelec basis functions. This will allow future support for Nedelec basis
+  functions on wedges and pyramids which are not amenable to reordering. The
+  ReorientTetMesh method of the Mesh and ParMesh classes has been deprecated.
 
 
 Version 4.3, released on July 29, 2021

CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -10,27 +10,23 @@
 # CONTRIBUTING.md for details.
 
 # The variable CMAKE_CXX_STANDARD and related were introduced in CMake v3.1
-# Version 3.8 fixes the handling of CMAKE_CXX_STANDARD for try_compile.
-# Version 3.8 or newer is required for direct CUDA support.
-cmake_minimum_required(VERSION 3.8)
-message(STATUS "CMake version: ${CMAKE_VERSION}")
+cmake_minimum_required(VERSION 3.1)
 set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH
   "Path to optional user configuration file.")
 
 # Require C++11 and disable compiler-specific extensions
-set(CMAKE_CXX_STANDARD 11 CACHE STRING "C++ standard to use.")
-set(CMAKE_CXX_STANDARD_REQUIRED ON CACHE BOOL
-  "Force the use of the chosen C++ standard.")
-set(CMAKE_CXX_EXTENSIONS OFF CACHE BOOL "Enable C++ standard extensions.")
+set(CMAKE_CXX_STANDARD 11)
+if (MFEM_USE_GINKGO)
+  set(CMAKE_CXX_STANDARD 14)
+endif()
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
 
 # Load user settings before the defaults - this way the defaults will not
 # overwrite the user set options. If the user has not set all options, we still
 # have the defaults.
-include("${USER_CONFIG}" OPTIONAL RESULT_VARIABLE USER_CONFIG_LOADED)
-if (USER_CONFIG_LOADED)
-  set(USER_CONFIG_LOADED "LOADED")
-endif()
-message(STATUS "Loading USER_CONFIG = ${USER_CONFIG} (${USER_CONFIG_LOADED})")
+message(STATUS "(optional) USER_CONFIG = ${USER_CONFIG}")
+include("${USER_CONFIG}" OPTIONAL)
 include("${CMAKE_CURRENT_SOURCE_DIR}/config/defaults.cmake")
 
 # Allow overwriting of the compiler by setting CXX/MPICXX on the command line or
@@ -58,7 +54,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.6.1)
+set(${PROJECT_NAME}_VERSION 4.3.1)
 
 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -73,7 +69,8 @@ set(XSDK_ENABLE_C OFF)
 set(XSDK_ENABLE_Fortran OFF)
 
 # Check if we need to enable C or Fortran.
-if (MFEM_USE_CONDUIT OR
+if (CMAKE_VERSION VERSION_LESS 3.2 OR
+    MFEM_USE_CONDUIT OR
     MFEM_USE_SIDRE OR
     MFEM_USE_PETSC)
    # This seems to be needed by:
@@ -83,45 +80,33 @@ if (MFEM_USE_CONDUIT OR
    #  * find_package(PETSc REQUIRED)
    set(XSDK_ENABLE_C ON)
 endif()
-if (MFEM_USE_STRUMPACK OR MFEM_USE_MUMPS)
+if (MFEM_USE_STRUMPACK)
   # Just needed to find the MPI_Fortran libraries to link with
   set(XSDK_ENABLE_Fortran ON)
 endif()
-# SUNDIALS, STRUMPACK, Ginkgo, RAJA and Umpire require C++14:
-if ((MFEM_USE_SUNDIALS OR
-     MFEM_USE_STRUMPACK OR
-     MFEM_USE_GINKGO OR
-     MFEM_USE_RAJA OR
-     MFEM_USE_UMPIRE) AND
-    ("${CMAKE_CXX_STANDARD}" LESS "14"))
-  set(CMAKE_CXX_STANDARD 14 CACHE STRING "C++ standard to use." FORCE)
-endif()
 
 # Include xSDK default CMake file.
 include("${CMAKE_CURRENT_SOURCE_DIR}/config/XSDKDefaults.cmake")
 
 # Enable languages.
 enable_language(CXX)
-if (MINGW)
-   # MinGW GCC does not expose the functions jn/_jn, yn/_yn (used in Example
-   # 25/25p) unless we use '-std=gnu++11':
-   set(CMAKE_CXX_EXTENSIONS ON
-      CACHE BOOL "Enable C++ standard extensions." FORCE)
-endif()
 if (MFEM_USE_CUDA)
    if (MFEM_USE_HIP)
       message(FATAL_ERROR " *** MFEM_USE_HIP cannot be combined with MFEM_USE_CUDA.")
    endif()
+   # MFEM_USE_CUDA requires CMake 3.8 or newer (for direct CUDA support)
+   cmake_minimum_required(VERSION 3.8 FATAL_ERROR)
    # Use ${CMAKE_CXX_COMPILER} as the cuda host compiler.
    if (NOT CMAKE_CUDA_HOST_COMPILER)
       set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
    endif()
    enable_language(CUDA)
-   set(CMAKE_CUDA_STANDARD ${CMAKE_CXX_STANDARD} CACHE STRING
-      "CUDA standard to use.")
-   set(CMAKE_CUDA_STANDARD_REQUIRED ON CACHE BOOL
-      "Force the use of the chosen CUDA standard.")
-   set(CMAKE_CUDA_EXTENSIONS OFF CACHE BOOL "Enable CUDA standard extensions.")
+   set(CMAKE_CUDA_STANDARD 11)
+   if (MFEM_USE_GINKGO)
+     set(CMAKE_CUDA_STANDARD 14)
+   endif()
+   set(CMAKE_CUDA_STANDARD_REQUIRED ON)
+   set(CMAKE_CUDA_EXTENSIONS OFF)
    set(CUDA_FLAGS "--expt-extended-lambda")
    if (CMAKE_VERSION VERSION_LESS 3.18.0)
       set(CUDA_FLAGS "-arch=${CUDA_ARCH} ${CUDA_FLAGS}")
@@ -142,10 +127,10 @@ if (MFEM_USE_CUDA)
       set(CUDA_FLAGS "-ccbin=${CMAKE_CXX_COMPILER} ${CUDA_FLAGS}")
       set(CMAKE_CUDA_HOST_LINK_LAUNCHER ${CMAKE_CXX_COMPILER})
    endif()
-   set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${CUDA_FLAGS}")
-   find_package(CUDAToolkit REQUIRED)
+   set(CMAKE_CUDA_FLAGS "${CUDA_FLAGS}" CACHE STRING
+       "CUDA flags set for MFEM" FORCE)
    set(CUSPARSE_FOUND TRUE)
-   get_target_property(CUSPARSE_LIBRARIES CUDA::cusparse LOCATION)
+   set(CUSPARSE_LIBRARIES "cusparse")
 endif()
 
 if (XSDK_ENABLE_C)
@@ -190,54 +175,8 @@ else()
   set(MFEM_DEBUG OFF)
 endif()
 
-# Shared build on Windows
-if (WIN32 AND BUILD_SHARED_LIBS)
-  # CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS works only with MSVC?
-  set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON CACHE BOOL
-      "Automatically export symbols for shared Windows DLL build")
-  # Place all .dll and .exe files in the same sub-directory, 'bin/<config>', of
-  # the build directory, so that the .exe files can find the .dll files at
-  # runtime:
-  set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin")
-  # Note: CMake tests are run from the build-tree sub-directory where the test
-  # is defined, e.g. <build-dir>/tests/unit, so hard coded paths to meshes still
-  # work fine.
-  # Not strictly necessary:
-  # set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
-  # set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
-endif()
-
-# AMD HIP
-if (MFEM_USE_HIP)
-  if (HIP_ARCH)
-    message(STATUS "Using HIP architecture: ${HIP_ARCH}")
-    set(GPU_TARGETS "${HIP_ARCH}" CACHE STRING "HIP targets to compile for")
-  endif()
-  if (ROCM_PATH)
-    list(INSERT CMAKE_PREFIX_PATH 0 ${ROCM_PATH})
-  endif()
-  find_package(HIP REQUIRED)
-  find_package(HIPSPARSE REQUIRED)
-endif()
-
-# OpenMP
-if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
-  if (NOT MFEM_THREAD_SAFE AND MFEM_USE_LEGACY_OPENMP)
-    message(FATAL_ERROR " *** MFEM_USE_LEGACY_OPENMP requires MFEM_THREAD_SAFE=ON.")
-  endif()
-  find_package(OpenMP REQUIRED)
-  set(OPENMP_LIBRARIES ${OpenMP_CXX_LIBRARIES})
-  if(APPLE)
-    # On macOS, the compiler needs additional help to find the <omp.h> header.
-    # See issue #2642 for more information.
-    set(OPENMP_INCLUDE_DIRS ${OpenMP_CXX_INCLUDE_DIRS})
-  endif(APPLE)
-  if (OPENMP_FOUND)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
-    if (MFEM_USE_CUDA)
-      set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=${OpenMP_CXX_FLAGS}")
-    endif()
-  endif()
+if (WIN32)
+	add_definitions(-D_USE_MATH_DEFINES)
 endif()
 
 # MPI -> hypre; PETSc (optional)
@@ -245,10 +184,10 @@ if (MFEM_USE_MPI)
   find_package(MPI REQUIRED)
   set(MPI_CXX_INCLUDE_DIRS ${MPI_CXX_INCLUDE_PATH})
   if (MFEM_MPIEXEC)
-    string(REPLACE " " ";" MPIEXEC ${MFEM_MPIEXEC})
+    set(MPIEXEC ${MFEM_MPIEXEC})
   endif()
   if (MFEM_MPIEXEC_NP)
-    string(REPLACE " " ";" MPIEXEC_NUMPROC_FLAG ${MFEM_MPIEXEC_NP})
+    set(MPIEXEC_NUMPROC_FLAG ${MFEM_MPIEXEC_NP})
   endif()
   # Parallel MFEM depends on hypre
   find_package(HYPRE REQUIRED)
@@ -305,6 +244,15 @@ if (MFEM_USE_LAPACK)
   find_package(LAPACK REQUIRED)
 endif()
 
+# OpenMP
+if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
+  if (NOT MFEM_THREAD_SAFE AND MFEM_USE_LEGACY_OPENMP)
+    message(FATAL_ERROR " *** MFEM_USE_LEGACY_OPENMP requires MFEM_THREAD_SAFE=ON.")
+  endif()
+  find_package(OpenMP REQUIRED)
+  set(OPENMP_LIBRARIES ${OpenMP_CXX_LIBRARIES})
+endif()
+
 # SuiteSparse (before SUNDIALS which may depend on KLU)
 if (MFEM_USE_SUITESPARSE)
   find_package(SuiteSparse REQUIRED
@@ -320,12 +268,14 @@ if (MFEM_USE_SUNDIALS)
   if (MFEM_USE_CUDA)
     list(APPEND SUNDIALS_COMPONENTS NVector_Cuda)
   endif()
-  if (MFEM_USE_HIP)
-    list(APPEND SUNDIALS_COMPONENTS NVector_Hip)
-  endif()
   find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS})
 endif()
 
+# Mesquite
+if (MFEM_USE_MESQUITE)
+  find_package(Mesquite REQUIRED)
+endif()
+
 # SuperLU_DIST can only be enabled in parallel
 if (MFEM_USE_SUPERLU)
   if (MFEM_USE_MPI)
@@ -339,7 +289,6 @@ endif()
 if (MFEM_USE_MUMPS)
   if (MFEM_USE_MPI)
     find_package(MUMPS REQUIRED mumps_common pord)
-    set(MFEM_MUMPS_VERSION ${MUMPS_VERSION})
   else()
     message(FATAL_ERROR " *** MUMPS requires that MPI be enabled.")
   endif()
@@ -383,11 +332,11 @@ if (MFEM_USE_AMGX)
 endif()
 
 if (MFEM_USE_CONDUIT)
-    find_package(Conduit REQUIRED conduit relay blueprint)
+    find_package(Conduit REQUIRED conduit relay blueprint )
 endif()
 
 if (MFEM_USE_FMS)
-    find_package(FMS REQUIRED fms)
+    find_package(FMS REQUIRED fms )
 endif()
 
 # Axom/Sidre
@@ -399,7 +348,7 @@ endif()
 if (MFEM_USE_PUMI)
   # If PUMI_DIR was specified, only link to that directory,
   # i.e. don't link to another installation in /usr/lib by mistake
-  find_package(SCOREC 2.2.6 REQUIRED OPTIONAL_COMPONENTS gmi_sim
+  find_package(SCOREC 2.1.0 REQUIRED OPTIONAL_COMPONENTS gmi_sim
     CONFIG PATHS ${PUMI_DIR} NO_DEFAULT_PATH)
   if (SCOREC_FOUND)
     # Define a header file with the MFEM_USE_SIMMETRIX preprocessor variable
@@ -411,33 +360,12 @@ if (MFEM_USE_PUMI)
   endif()
 endif()
 
-# Moonolith
-if(MFEM_USE_MOONOLITH)
-  find_package(ParMoonolith REQUIRED)
-  if(ParMoonolith_FOUND)
-    get_target_property(
-      MOONOLITH_INCLUDE_DIRS ParMoonolith::par_moonolith
-      INTERFACE_INCLUDE_DIRECTORIES)
-    set(MOONOLITH_FOUND TRUE)
-    set(MOONOLITH_LIBRARIES ParMoonolith::par_moonolith)
-    message(
-      STATUS
-      "MOONOLITH_LIBRARIES=${MOONOLITH_LIBRARIES}, MOONOLITH_INCLUDE_DIRS=${MOONOLITH_INCLUDE_DIRS}")
-  endif()
-endif()
-
 # HiOp optimizer
 if (MFEM_USE_HIOP)
   find_package(HIOP REQUIRED)
   # find_package updates HIOP_FOUND, HIOP_INCLUDE_DIRS, HIOP_LIBRARIES
 endif()
 
-# CoDiPack package
-if (MFEM_USE_CODIPACK)
-  find_package(CODIPACK REQUIRED)
-  # find_package updates CODIPACK_FOUND, CODIPACK_INCLUDE_DIRS, CODIPACK_LIBRARIES
-endif()
-
 # OCCA
 if (MFEM_USE_OCCA)
    find_package(OCCA REQUIRED)
@@ -453,19 +381,21 @@ if (MFEM_USE_UMPIRE)
    find_package(UMPIRE REQUIRED)
 endif()
 
-# GOOGLE-BENCHMARK
-if (MFEM_USE_BENCHMARK)
-   find_package(Benchmark REQUIRED)
-endif()
-
 # Caliper
 if (MFEM_USE_CALIPER)
   find_package(Caliper REQUIRED)
 endif()
 
-# Algoim
-if (MFEM_USE_ALGOIM)
-  find_package(Algoim REQUIRED)
+# AMD HIP
+if (MFEM_USE_HIP)
+  find_package(HIP REQUIRED)
+  if (HIP_ARCH)
+    message(STATUS "Using HIP architecture: ${HIP_ARCH}")
+    list(APPEND HIP_HIPCC_FLAGS "--amdgpu-target=${HIP_ARCH}")
+    if (MFEM_USE_GINKGO)
+      list(APPEND HIP_HIPCC_FLAGS "-std=c++14")
+    endif()
+  endif()
 endif()
 
 # ADIOS2 for parallel I/O
@@ -473,28 +403,12 @@ if (MFEM_USE_ADIOS2)
    find_package(ADIOS2 REQUIRED)
 endif()
 
-# MKL CPardiso
 if (MFEM_USE_MKL_CPARDISO)
   if (MFEM_USE_MPI)
    find_package(MKL_CPARDISO REQUIRED MKL_SEQUENTIAL MKL_LP64 MKL_MPI_WRAPPER)
   endif()
 endif()
 
-# MKL Pardiso
-if (MFEM_USE_MKL_PARDISO)
-   find_package(MKL_PARDISO REQUIRED MKL_SEQUENTIAL MKL_LP64)
-endif()
-
-# PARELAG
-if (MFEM_USE_PARELAG)
-   find_package(PARELAG REQUIRED)
-endif()
-
-# Enzyme
-if (MFEM_USE_ENZYME)
-   find_package(ENZYME REQUIRED)
-endif()
-
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
   if (APPLE)
@@ -513,29 +427,15 @@ if (NOT DEFINED MFEM_TIMER_TYPE)
   endif()
 endif()
 
-# Without this, CMake 3.21.1 (and 3.20.2) run into CMake Errors like the following:
-# CMake Error at config/cmake/modules/MfemCmakeUtilities.cmake:60 (add_library):
-#   Target "mfem" links to target "Threads::Threads" but the target was not
-#   found.  Perhaps a find_package() call is missing for an IMPORTED target, or
-#   an ALIAS target is missing?
-# Call Stack (most recent call first):
-#   CMakeLists.txt:474 (mfem_add_library)
-#
-# NOTE: We need to figure out which TPL library adds the dependency on
-#       "Threads::Threads" and call the next line only when that TPL library is
-#       enabled.  -V. Dobrev
-find_package(Threads REQUIRED)
-
 # List all possible libraries in order of dependencies.
 # [METIS < SuiteSparse]:
 #    With newer versions of SuiteSparse which include METIS header using 64-bit
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
-set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
-    SUNDIALS PETSC SLEPC MUMPS AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
-    NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
-    ADIOS2 CUSPARSE MKL_CPARDISO MKL_PARDISO AMGX CALIPER CODIPACK
-    BENCHMARK PARELAG MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)
+set(MFEM_TPLS MPI_CXX OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS PETSC
+    SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB NETCDF
+    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
+    CUSPARSE MKL_CPARDISO AMGX CALIPER)
 
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
@@ -547,13 +447,19 @@ foreach(TPL IN LISTS MFEM_TPLS)
     list(APPEND TPL_INCLUDE_DIRS ${${TPL}_INCLUDE_DIRS})
   endif()
 endforeach(TPL)
-list(REVERSE TPL_LIBRARIES)
 list(REMOVE_DUPLICATES TPL_LIBRARIES)
-list(REVERSE TPL_LIBRARIES)
 list(REMOVE_DUPLICATES TPL_INCLUDE_DIRS)
 # message(STATUS "TPL_INCLUDE_DIRS = ${TPL_INCLUDE_DIRS}")
+include_directories(${TPL_INCLUDE_DIRS})
+
+if (OPENMP_FOUND)
+  message(STATUS "MFEM: using package OpenMP")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+  if (MFEM_USE_CUDA)
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=${OpenMP_CXX_FLAGS}")
+  endif()
+endif()
 
-message(STATUS "MFEM shared library: BUILD_SHARED_LIBS = ${BUILD_SHARED_LIBS}")
 message(STATUS "MFEM build type: CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}")
 message(STATUS "MFEM version: v${MFEM_VERSION_STRING}")
 message(STATUS "MFEM git string: ${MFEM_GIT_STRING}")
@@ -566,17 +472,14 @@ message(STATUS "MFEM git string: ${MFEM_GIT_STRING}")
 set(SOURCES "")
 set(HEADERS "")
 set(MFEM_SOURCE_DIRS general linalg mesh fem)
-
-if(MFEM_USE_MOONOLITH)
-  set(MFEM_SOURCE_DIRS ${MFEM_SOURCE_DIRS} fem/moonolith)
-endif()
-
 foreach(DIR IN LISTS MFEM_SOURCE_DIRS)
   add_subdirectory(${DIR})
 endforeach()
 
 if (MFEM_USE_CUDA)
   set_source_files_properties(${SOURCES} PROPERTIES LANGUAGE CUDA)
+elseif(MFEM_USE_HIP)
+  set_source_files_properties(${SOURCES} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
 endif()
 
 add_subdirectory(config)
@@ -597,31 +500,14 @@ set(MFEM_INSTALL_DIR ${CMAKE_INSTALL_PREFIX} CACHE PATH
 # Declaring the library
 mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
 # message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
-target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES})
+if (CMAKE_VERSION VERSION_GREATER 2.8.11)
+  target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES})
+else()
+  target_link_libraries(mfem ${TPL_LIBRARIES})
+endif()
 if (MINGW)
-  target_link_libraries(mfem PRIVATE ws2_32)
+  target_link_libraries(mfem ws2_32)
 endif()
-if (MSVC)
-  target_compile_options(mfem PUBLIC "/wd4819")
-endif()
-message(STATUS "TPL_INCLUDE_DIRS = ${TPL_INCLUDE_DIRS}")
-target_include_directories(mfem
-  PUBLIC
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-    # The INSTALL_INTERFACE is handled below
-    ${TPL_INCLUDE_DIRS})
-if (MFEM_USE_MPI)
-  if (MPI_CXX_COMPILE_FLAGS)
-    separate_arguments(MPI_CXX_COMPILE_ARGS UNIX_COMMAND
-      "${MPI_CXX_COMPILE_FLAGS}")
-    target_compile_options(mfem PUBLIC ${MPI_CXX_COMPILE_ARGS})
-  endif()
-  if (MPI_CXX_LINK_FLAGS)
-    target_link_libraries(mfem PUBLIC ${MPI_CXX_LINK_FLAGS})
-  endif()
-endif()
-
 set_target_properties(mfem PROPERTIES VERSION "${mfem_VERSION}")
 set_target_properties(mfem PROPERTIES SOVERSION "${mfem_VERSION}")
 
@@ -633,7 +519,6 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
 endif()
 
 # Generate configuration file in the build directory: config/_config.hpp.
-set(MFEM_SHARED_BUILD ${BUILD_SHARED_LIBS})
 configure_file(
   "${PROJECT_SOURCE_DIR}/config/cmake/config.hpp.in"
   "${PROJECT_BINARY_DIR}/config/_config.hpp")
@@ -644,55 +529,31 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
   foreach(Header mfem.hpp mfem-performance.hpp)
     message(STATUS
       "Writing substitute header --> \"${Header}\"")
-    file(WRITE "${PROJECT_BINARY_DIR}/${Header}.tmp"
+    file(WRITE "${PROJECT_BINARY_DIR}/${Header}"
 "// Auto-generated file.
 #define MFEM_CONFIG_FILE \"${PROJECT_BINARY_DIR}/config/_config.hpp\"
 #include \"${PROJECT_SOURCE_DIR}/${Header}\"
 ")
-
-    execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different 
-      "${PROJECT_BINARY_DIR}/${Header}.tmp"
-      "${PROJECT_BINARY_DIR}/${Header}"
-    )
-    execute_process(COMMAND ${CMAKE_COMMAND} -E remove
-      "${PROJECT_BINARY_DIR}/${Header}.tmp"
-    )
-
     # This version will be installed in the top include directory:
-    file(WRITE "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}.tmp"
+    file(WRITE "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}"
 "// Auto-generated file.
 #include \"mfem/${Header}\"
 ")
-
-    execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different 
-      "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}.tmp"
-      "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}"
-    )
-    execute_process(COMMAND ${CMAKE_COMMAND} -E remove
-      "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}.tmp"
-    )
-
   endforeach()
 endif()
 
 set(MFEM_CUSTOM_TARGET_PREFIX CACHE STRING "")
 
 #-------------------------------------------------------------------------------
-# Examples, miniapps, benchmarks and testing
+# Examples, miniapps, and testing
 #-------------------------------------------------------------------------------
 
-# Enable testing and benchmarks if required
+# Enable testing if required
 if (MFEM_ENABLE_TESTING)
   enable_testing()
   set(MFEM_ALL_TESTS_TARGET_NAME tests)
   add_mfem_target(${MFEM_ALL_TESTS_TARGET_NAME} OFF)
   add_subdirectory(tests EXCLUDE_FROM_ALL)
-
-  # Create a target for all benchmarks and, optionally, enable it.
-  set(MFEM_ALL_BENCHMARKS_TARGET_NAME benchmarks)
-  add_mfem_target(${MFEM_ALL_BENCHMARKS_TARGET_NAME}
-                  ${MFEM_ENABLE_GOOGLE_BENCHMARKS})
-  add_subdirectory(tests/benchmarks EXCLUDE_FROM_ALL)
 endif()
 
 # Define a target that all examples and miniapps will depend on.
@@ -702,11 +563,7 @@ add_custom_target(${MFEM_EXEC_PREREQUISITES_TARGET_NAME})
 # Create a target for all examples and, optionally, enable it.
 set(MFEM_ALL_EXAMPLES_TARGET_NAME examples)
 add_mfem_target(${MFEM_ALL_EXAMPLES_TARGET_NAME} ${MFEM_ENABLE_EXAMPLES})
-if (MFEM_ENABLE_EXAMPLES)
-  add_subdirectory(examples) #install examples if enabled
-else()
-  add_subdirectory(examples EXCLUDE_FROM_ALL)
-endif()
+add_subdirectory(examples EXCLUDE_FROM_ALL)
 
 # Create a target for all miniapps and, optionally, enable it.
 set(MFEM_ALL_MINIAPPS_TARGET_NAME miniapps)
@@ -716,7 +573,6 @@ add_subdirectory(miniapps EXCLUDE_FROM_ALL)
 # Target to build all executables, i.e. everything.
 add_custom_target(exec)
 add_dependencies(exec
-  ${MFEM_ALL_BENCHMARKS_TARGET_NAME}
   ${MFEM_ALL_EXAMPLES_TARGET_NAME}
   ${MFEM_ALL_MINIAPPS_TARGET_NAME}
   ${MFEM_ALL_TESTS_TARGET_NAME})
@@ -763,18 +619,12 @@ add_subdirectory(doc)
 message(STATUS "CMAKE_INSTALL_PREFIX = ${CMAKE_INSTALL_PREFIX}")
 set(INSTALL_INCLUDE_DIR include
   CACHE PATH "Relative path for installing header files.")
-set(INSTALL_BIN_DIR bin
-  CACHE PATH "Relative path for installing the binaries.")
 set(INSTALL_LIB_DIR lib
   CACHE PATH "Relative path for installing the library.")
 # other options: "share/mfem/cmake", "lib/mfem/cmake"
 set(INSTALL_CMAKE_DIR lib/cmake/mfem
   CACHE PATH "Relative path for installing cmake config files.")
 
-target_include_directories(mfem BEFORE
-  PUBLIC
-    $<INSTALL_INTERFACE:${INSTALL_INCLUDE_DIR}>)
-
 # The 'install' target will not depend on 'all'.
 # set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY TRUE)
 
@@ -783,9 +633,7 @@ set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME Development)
 # Install the library
 install(TARGETS ${PROJECT_NAME}
   EXPORT ${PROJECT_NAME_UC}Targets
-  RUNTIME DESTINATION ${INSTALL_BIN_DIR}
-  LIBRARY DESTINATION ${INSTALL_LIB_DIR}
-  ARCHIVE DESTINATION ${INSTALL_LIB_DIR})
+  DESTINATION ${INSTALL_LIB_DIR})
 
 # Install the master headers
 foreach(Header mfem.hpp mfem-performance.hpp)
@@ -810,7 +658,7 @@ endif()
 if (MFEM_USE_CEED)
   install(DIRECTORY ${MFEM_SOURCE_DIRS}
     DESTINATION ${INSTALL_INCLUDE_DIR}/mfem
-    FILES_MATCHING PATTERN "fem/ceed/integrators/*/*.h")
+    FILES_MATCHING PATTERN "fem/ceed/*.h")
 endif()
 
 # Install ${HEADERS}
@@ -823,8 +671,10 @@ endif()
 
 # Install the configuration header files
 install(FILES ${PROJECT_BINARY_DIR}/config/_config.hpp
-  ${PROJECT_SOURCE_DIR}/config/config.hpp
-  ${PROJECT_SOURCE_DIR}/config/tconfig.hpp
+  DESTINATION ${INSTALL_INCLUDE_DIR}/mfem/config
+  RENAME config.hpp)
+
+install(FILES ${PROJECT_SOURCE_DIR}/config/tconfig.hpp
   DESTINATION ${INSTALL_INCLUDE_DIR}/mfem/config)
 
 # Package the whole thing up nicely
@@ -839,8 +689,14 @@ export(TARGETS ${PROJECT_NAME}
 # TODO: How do we register the install-tree? Replacing the build-tree?
 export(PACKAGE ${PROJECT_NAME})
 
+# Extract the include directories required to use MFEM
+get_target_property(MFEM_TPL_INCLUDE_DIRS mfem INCLUDE_DIRECTORIES)
+if (NOT MFEM_TPL_INCLUDE_DIRS)
+  set(MFEM_TPL_INCLUDE_DIRS "")
+endif()
+
 # This is the build-tree version
-set(INCLUDE_INSTALL_DIRS ${PROJECT_BINARY_DIR} ${TPL_INCLUDE_DIRS})
+set(INCLUDE_INSTALL_DIRS ${PROJECT_BINARY_DIR} ${MFEM_TPL_INCLUDE_DIRS})
 set(LIB_INSTALL_DIR ${PROJECT_BINARY_DIR})
 configure_package_config_file(config/cmake/MFEMConfig.cmake.in
   ${CMAKE_CURRENT_BINARY_DIR}/MFEMConfig.cmake
@@ -848,7 +704,7 @@ configure_package_config_file(config/cmake/MFEMConfig.cmake.in
   PATH_VARS INCLUDE_INSTALL_DIRS LIB_INSTALL_DIR)
 
 # This is the version that will be installed
-set(INCLUDE_INSTALL_DIRS ${INSTALL_INCLUDE_DIR}  ${TPL_INCLUDE_DIRS})
+set(INCLUDE_INSTALL_DIRS ${INSTALL_INCLUDE_DIR}  ${MFEM_TPL_INCLUDE_DIRS})
 set(LIB_INSTALL_DIR ${INSTALL_LIB_DIR})
 configure_package_config_file(config/cmake/MFEMConfig.cmake.in
   ${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/MFEMConfig.cmake

CODE_OF_CONDUCT.md

@@ -1,133 +0,0 @@
-# MFEM Code of Conduct
-
-## Our Pledge
-
-We as members, contributors, and leaders pledge to make participation in our
-community a harassment-free experience for everyone, regardless of age, body
-size, visible or invisible disability, ethnicity, sex characteristics, gender
-identity and expression, level of experience, education, socio-economic status,
-nationality, personal appearance, race, caste, color, religion, or sexual
-identity and orientation.
-
-We pledge to act and interact in ways that contribute to an open, welcoming,
-diverse, inclusive, and healthy community.
-
-## Our Standards
-
-Examples of behavior that contributes to a positive environment for our
-community include:
-
-* Demonstrating empathy and kindness toward other people
-* Being respectful of differing opinions, viewpoints, and experiences
-* Giving and gracefully accepting constructive feedback
-* Accepting responsibility and apologizing to those affected by our mistakes,
-  and learning from the experience
-* Focusing on what is best not just for us as individuals, but for the overall
-  community
-
-Examples of unacceptable behavior include:
-
-* The use of sexualized language or imagery, and sexual attention or advances of
-  any kind
-* Trolling, insulting or derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or email address,
-  without their explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-## Enforcement Responsibilities
-
-Community leaders are responsible for clarifying and enforcing our standards of
-acceptable behavior and will take appropriate and fair corrective action in
-response to any behavior that they deem inappropriate, threatening, offensive,
-or harmful.
-
-Community leaders have the right and responsibility to remove, edit, or reject
-comments, commits, code, wiki edits, issues, and other contributions that are
-not aligned to this Code of Conduct, and will communicate reasons for moderation
-decisions when appropriate.
-
-## Scope
-
-This Code of Conduct applies within all community spaces, and also applies when
-an individual is officially representing the community in public spaces.
-Examples of representing our community include using an official e-mail address,
-posting via an official social media account, or acting as an appointed
-representative at an online or offline event.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported to the community leaders responsible for enforcement at mfem@llnl.gov.
-All complaints will be reviewed and investigated promptly and fairly.
-
-All community leaders are obligated to respect the privacy and security of the
-reporter of any incident. Anyone involved in the reported behavior will recuse
-themselves from the investigation and decision making about the resolution of
-the complaint.
-
-## Enforcement Guidelines
-
-Community leaders will follow these Community Impact Guidelines in determining
-the consequences for any action they deem in violation of this Code of Conduct:
-
-### 1. Correction
-
-**Community Impact**: Use of inappropriate language or other behavior deemed
-unprofessional or unwelcome in the community.
-
-**Consequence**: A private, written warning from community leaders, providing
-clarity around the nature of the violation and an explanation of why the
-behavior was inappropriate. A public apology may be requested.
-
-### 2. Warning
-
-**Community Impact**: A violation through a single incident or series of
-actions.
-
-**Consequence**: A warning with consequences for continued behavior. No
-interaction with the people involved, including unsolicited interaction with
-those enforcing the Code of Conduct, for a specified period of time. This
-includes avoiding interactions in community spaces as well as external channels
-like social media. Violating these terms may lead to a temporary or permanent
-ban.
-
-### 3. Temporary Ban
-
-**Community Impact**: A serious violation of community standards, including
-sustained inappropriate behavior.
-
-**Consequence**: A temporary ban from any sort of interaction or public
-communication with the community for a specified period of time. No public or
-private interaction with the people involved, including unsolicited interaction
-with those enforcing the Code of Conduct, is allowed during this period.
-Violating these terms may lead to a permanent ban.
-
-### 4. Permanent Ban
-
-**Community Impact**: Demonstrating a pattern of violation of community
-standards, including sustained inappropriate behavior, harassment of an
-individual, or aggression toward or disparagement of classes of individuals.
-
-**Consequence**: A permanent ban from any sort of public interaction within the
-community.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage],
-version 2.1, available at
-[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
-
-Community Impact Guidelines were inspired by
-[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
-
-For answers to common questions about this code of conduct, see the FAQ at
-[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
-[https://www.contributor-covenant.org/translations][translations].
-
-[homepage]: https://www.contributor-covenant.org
-[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
-[Mozilla CoC]: https://github.com/mozilla/diversity
-[FAQ]: https://www.contributor-covenant.org/faq
-[translations]: https://www.contributor-covenant.org/translations

CONTRIBUTING.md

@@ -8,7 +8,7 @@
 <a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuild-analysis+branch%3Amaster"><img alt="Build Analysis" src="https://github.com/mfem/mfem/actions/workflows/mfem-analysis.yml/badge.svg?branch=master"></a>
 <a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuilds-and-tests+branch%3Amaster"><img alt="Builds and Tests" src="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml/badge.svg?branch=master"></a>
 <a href="https://ci.appveyor.com/project/mfem/mfem"><img alt="Build Status" src="https://ci.appveyor.com/api/projects/status/19non9sqm6msi2wy?svg=true"></a>
-<a href="https://docs.mfem.org/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
+<a href="https://mfem.github.io/doxygen/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
 </p>
 
 
@@ -21,9 +21,6 @@ documentation; new examples and miniapps; HPC performance improvements; etc.
 MFEM is distributed under the terms of the BSD-3 license. All new contributions
 must be made under this license.
 
-Note also that MFEM has a [Code of Conduct](CODE_OF_CONDUCT.md). By participating
-in the MFEM community, you agree to abide by its rules.
-
 If you plan on contributing to MFEM, consider reviewing the
 [issue tracker](https://github.com/mfem/mfem/issues) first to check if a thread
 already exists for your desired feature or the bug you ran into. Use a pull
@@ -45,7 +42,6 @@ back to them before issuing pull requests:
   - [New Feature Development](#new-feature-development)
   - [Developer Guidelines](#developer-guidelines)
   - [Pull Requests](#pull-requests)
-  - [MFEM PR Rules](#mfem-pr-rules)
   - [Pull Request Checklist](#pull-request-checklist)
   - [Master/Next Workflow](#masternext-workflow)
   - [Releases](#releases)
@@ -71,9 +67,8 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
   with regards to documentation and code styling.
 - Please do not commit large/binary files to the central repository (use a fork
   instead).
-- Pull requests should be issued toward `mfem:master`. Make sure
-  to check the items off the [Pull Request Checklist](#pull-request-checklist) and
-  follow the [MFEM PR Rules](#mfem-pr-rules).
+- Pull requests  should be issued toward `mfem:master`. Make sure
+  to check the items off the [Pull Request Checklist](#pull-request-checklist).
 - When your contribution is fully working and ready to be reviewed, add
   the `ready-for-review` label.
 - PRs are treated similarly to journal submission with an "editor" assigning two
@@ -102,9 +97,7 @@ The MFEM source code has the following structure:
   .
   ├── config
   │   ├── cmake
-  │   ├── docker
-  │   ├── githooks
-  │   └── vcpkg
+  │   └── githooks
   ├── data
   ├── doc
   ├── examples
@@ -112,48 +105,33 @@ The MFEM source code has the following structure:
   │   ├── caliper
   │   ├── ginkgo
   │   ├── hiop
-  │   ├── jupyter
-  │   ├── moonolith
   │   ├── petsc
   │   ├── pumi
   │   ├── sundials
   |   └── superlu
   ├── fem
   │   ├── ceed
-  │   ├── fe
-  │   ├── integ
-  │   ├── lor
-  │   ├── moonolith
   │   ├── qinterp
   │   └── tmop
   ├── general
   ├── linalg
   │   └── simd
   ├── mesh
-  │   └── submesh
   ├── miniapps
   │   ├── adjoint
-  │   ├── autodiff
   │   ├── common
-  │   ├── dpg
   │   ├── electromagnetics
   │   ├── gslib
-  │   ├── hdiv-linear-solver
-  │   ├── hooke
   │   ├── meshing
   │   ├── mtop
-  │   ├── multidomain
   │   ├── navier
   │   ├── nurbs
-  │   ├── parelag
   │   ├── performance
   │   ├── shifted
   │   ├── solvers
-  │   ├── spde
   │   ├── tools
   │   └── toys
   └── tests
-      ├── benchmarks
       ├── convergence
       ├── gitlab
       ├── mem_manager
@@ -169,24 +147,24 @@ classes implementing the finite element, mesh and linear algebra concepts
 respectively.
 
 - The main mesh classes are:
-  + [`Mesh`](https://docs.mfem.org/html/classmfem_1_1Mesh.html)
-  + [`NCMesh`](https://docs.mfem.org/html/classmfem_1_1NCMesh.html)
-  + [`Element`](https://docs.mfem.org/html/classmfem_1_1Element.html)
-  + [`ElementTransformation`](https://docs.mfem.org/html/classmfem_1_1ElementTransformation.html)
+  + [`Mesh`](https://mfem.github.io/doxygen/html/classmfem_1_1Mesh.html)
+  + [`NCMesh`](https://mfem.github.io/doxygen/html/classmfem_1_1NCMesh.html)
+  + [`Element`](https://mfem.github.io/doxygen/html/classmfem_1_1Element.html)
+  + [`ElementTransformation`](https://mfem.github.io/doxygen/html/classmfem_1_1ElementTransformation.html)
 
 - The main finite element classes are:
-  + [`FiniteElement`](https://docs.mfem.org/html/classmfem_1_1FiniteElement.html)
-  + [`FiniteElementCollection`](https://docs.mfem.org/html/classmfem_1_1FiniteElement.html)
-  + [`FiniteElementSpace`](https://docs.mfem.org/html/classmfem_1_1FiniteElementSpace.html)
-  + [`GridFunction`](https://docs.mfem.org/html/classmfem_1_1GridFunction.html)
-  + [`BilinearFormIntegrator`](https://docs.mfem.org/html/classmfem_1_1BilinearFormIntegrator.html) and [`LinearFormIntegrator`](https://docs.mfem.org/html/classmfem_1_1LinearFormIntegrator.html)
-  + [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearFormIntegrator.html), [`BilinearForm`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html) and [`MixedBilinearForm`](https://docs.mfem.org/html/classmfem_1_1MixedBilinearForm.html)
+  + [`FiniteElement`](https://mfem.github.io/doxygen/html/classmfem_1_1FiniteElement.html)
+  + [`FiniteElementCollection`](https://mfem.github.io/doxygen/html/classmfem_1_1FiniteElement.html)
+  + [`FiniteElementSpace`](https://mfem.github.io/doxygen/html/classmfem_1_1FiniteElementSpace.html)
+  + [`GridFunction`](https://mfem.github.io/doxygen/html/classmfem_1_1GridFunction.html)
+  + [`BilinearFormIntegrator`](https://mfem.github.io/doxygen/html/classmfem_1_1BilinearFormIntegrator.html) and [`LinearFormIntegrator`](https://mfem.github.io/doxygen/html/classmfem_1_1LinearFormIntegrator.html)
+  + [`LinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1LinearFormIntegrator.html), [`BilinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html) and [`MixedBilinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1MixedBilinearForm.html)
 
 - The main linear algebra classes and sources are
-  + [`Operator`](https://docs.mfem.org/html/classmfem_1_1Operator.html) and [`BilinearForm`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html)
-  + [`Vector`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html) and [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearForm.html)
-  + [`DenseMatrix`](https://docs.mfem.org/html/classmfem_1_1DenseMatrix.html) and [`SparseMatrix`](https://docs.mfem.org/html/classmfem_1_1SparseMatrix.html)
-  + Sparse [smoothers](https://docs.mfem.org/html/sparsesmoothers_8hpp.html) and linear [solvers](https://docs.mfem.org/html/solvers_8hpp.html)
+  + [`Operator`](https://mfem.github.io/doxygen/html/classmfem_1_1Operator.html) and [`BilinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html)
+  + [`Vector`](https://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html) and [`LinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1LinearForm.html)
+  + [`DenseMatrix`](https://mfem.github.io/doxygen/html/classmfem_1_1DenseMatrix.html) and [`SparseMatrix`](https://mfem.github.io/doxygen/html/classmfem_1_1SparseMatrix.html)
+  + Sparse [smoothers](https://mfem.github.io/doxygen/html/sparsesmoothers_8hpp.html) and linear [solvers](https://mfem.github.io/doxygen/html/solvers_8hpp.html)
 
 #### Parallel implementation
 
@@ -196,13 +174,13 @@ shared geometric entities between different tasks. The parallel source files
 have a `p` prefix, e.g. `pmesh.cpp` vs. the serial `mesh.cpp`.
 
 - The main parallel classes are
-  + [`ParMesh`](https://docs.mfem.org/html/solvers_8hpp.html)
-  + [`ParNCMesh`](https://docs.mfem.org/html/classmfem_1_1ParMesh.html)
-  + [`ParFiniteElementSpace`](https://docs.mfem.org/html/classmfem_1_1ParFiniteElementSpace.html)
-  + [`ParGridFunction`](https://docs.mfem.org/html/classmfem_1_1ParGridFunction.html)
-  + [`ParBilinearForm`](https://docs.mfem.org/html/classmfem_1_1ParBilinearForm.html) and [`ParLinearForm`](https://docs.mfem.org/html/classmfem_1_1ParLinearForm.html)
-  + [`HypreParMatrix`](https://docs.mfem.org/html/classmfem_1_1HypreParMatrix.html) and [`HypreParVector`](https://docs.mfem.org/html/classmfem_1_1HypreParVector.html)
-  + [`HypreSolver`](https://docs.mfem.org/html/classmfem_1_1HypreSolver.html) and other [hypre classes](https://docs.mfem.org/html/hypre_8hpp.html)
+  + [`ParMesh`](https://mfem.github.io/doxygen/html/solvers_8hpp.html)
+  + [`ParNCMesh`](https://mfem.github.io/doxygen/html/classmfem_1_1ParMesh.html)
+  + [`ParFiniteElementSpace`](https://mfem.github.io/doxygen/html/classmfem_1_1ParFiniteElementSpace.html)
+  + [`ParGridFunction`](https://mfem.github.io/doxygen/html/classmfem_1_1ParGridFunction.html)
+  + [`ParBilinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1ParBilinearForm.html) and [`ParLinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1ParLinearForm.html)
+  + [`HypreParMatrix`](https://mfem.github.io/doxygen/html/classmfem_1_1HypreParMatrix.html) and [`HypreParVector`](https://mfem.github.io/doxygen/html/classmfem_1_1HypreParVector.html)
+  + [`HypreSolver`](https://mfem.github.io/doxygen/html/classmfem_1_1HypreSolver.html) and other [hypre classes](https://mfem.github.io/doxygen/html/hypre_8hpp.html)
 
 #### GPU and general device support
 
@@ -211,10 +189,10 @@ backends (CUDA, OCCA, RAJA, OpenMP, etc.) and an internal lightweight
 device/host memory manager.
 
 - The main device-relevant classes and sources are:
-  + [`Device`](https://docs.mfem.org/html/device_8hpp.html)
-  + [`MemoryManager`](https://docs.mfem.org/html/mem_manager_8hpp.html)
-  + the [`mfem::forall`](https://docs.mfem.org/html/forall_8hpp.html) function
-  + the [`cuda.hpp`](https://docs.mfem.org/html/cuda_8hpp.html) and [`occa.hpp`](https://docs.mfem.org/html/occa_8hpp.html) files
+  + [`Device`](https://mfem.github.io/doxygen/html/device_8hpp.html)
+  + [`MemoryManager`](https://mfem.github.io/doxygen/html/mem_manager_8hpp.html)
+  + the [`MFEM_FORALL`](https://mfem.github.io/doxygen/html/forall_8hpp.html) macro
+  + the [`cuda.hpp`](https://mfem.github.io/doxygen/html/cuda_8hpp.html) and [`occa.hpp`](https://mfem.github.io/doxygen/html/occa_8hpp.html) files
 
 #### Utilities, building and documentation
 - The `general/` directory contains C++ classes that serve as utilities for
@@ -222,7 +200,8 @@ device/host memory manager.
 - The `config/` directory contains build-related files, both for the plain
   Makefile and the CMake build options.
 - The `doc/` directory contains configuration for the Doxygen code documentation
-  that can either be built locally or browsed online at https://docs.mfem.org.
+  that can either be built locally or browsed online at
+  https://mfem.github.io/doxygen/html/index.html.
 
 #### Examples and tests
 - `examples` and `miniapps` respectively gather simple and more fully-featured
@@ -344,25 +323,15 @@ Before you can start, you need a GitHub account, here are a few suggestions:
     change the code by default.
 
 - Code specifics
-  - All new public, protected, and private classes, methods, data members, and
-    functions have Doxygen-style documentation in source comments.
-  - Math formulas can be included in the Doxygen comments either with standard
-    LaTeX (`$..$` and `$$..$$`) for portions that need detailed explanation, or with
-    [Unicode](https://www.unicodeit.net/) or plain text when short or readable description is preferable.
-  - In addition to arguments and functionality, documentation should include the
-    current limitations of the code, any background information that is
-    implicitly assumed in the implementation, and the ownership and lifetime
-    of data.
+  - All significant new classes, methods and functions have Doxygen-style
+    documentation in source comments.
   - Consistent code styling is enforced with `make style` in the top-level
     directory. This requires [Artistic Style](http://astyle.sourceforge.net) (we
-    specifically use version 3.1). See also the file `config/mfem.astylerc`.
+    specifically use version 2.05.1). See also the file `config/mfem.astylerc`.
   - Use `mfem::out` and `mfem::err` instead of `std::cout` and `std::cerr` in
     internal library code. (You can use `std` in examples and miniapps.)
   - When manually resolving conflicts during a merge, make sure to mention the
     conflicted files in the commit message.
-  - All significant new features and changes should be documented in CHANGELOG.
-  - New examples and miniapps should have documentation on the MFEM webpage.
-
 
 ### Pull Requests
 
@@ -428,83 +397,6 @@ Before you can start, you need a GitHub account, here are a few suggestions:
 - If triggered, track the status of the LLNL GitLab tests. If failing, ask
   one of the _LLNL developers_ for details.
 
-
-### MFEM PR Rules
-
-The Pull Request (PR) approval process in MFEM is similar to the approval of papers in a peer-reviewed journal. In particular:
-
-1. There is an MFEM board of "editors" that evaluates new PRs and assigns "reviewers" for each PR.
-
-2. The assigned reviewers are responsible to carefully review and test the proposed PR.
-
-3. A PR can be (manually) merged in the *next* branch only if 2 of the assigned reviewers have approved it and it has passed internal testing. This merge can be performed by any of the assigned reviewers or by any of the editors.
-
-4. A PR can be merged in the *master* branch only if it has been tested successfully for a week in *next* and an editor has (optionally) taken a final look. This merge can be performed only by one of the editors.
-
-#### Responsibilities of Editors
-
-The current list of MFEM editors is:
-
-- [@v-dobrev](https://github.com/v-dobrev) (Veselin Dobrev)
-- [@tzanio](https://github.com/tzanio) (Tzanio Kolev)
-- [@pazner](https://github.com/pazner) (Will Pazner)
-- [@mlstowell](https://github.com/mlstowell) (Mark Stowell)
-
-**The responsibilities of the editors are:**
-
-1. To assign appropriate milestone and labels for new PRs, e.g. *bugfix*, *minor*, *api-change*, *high-impact*, etc.
-
-2. To assign at least 2 reviewers for new PRs. An editor can also be a reviewer. The editor, reviewers, and author should be listed as "Assignees" on the GitHub PR page. After assignment, the `in-review` label should be added.
-
-3. To complete the initial PR evaluation and assignments in a timely manner: 1 week from submission.
-
-4. To assist reviewers when they need help with their reviews (but also to stay out of the way when they don't).
-
-5. To remind the reviewers about timely completion of their review.
-
-6. To take a final look and complete the PR merge in *master*. The final look step is optional and shouldn't take more than 3 days.
-
-7. The assignment of bugfixes should be expedited proportional to their importance, e.g. in some cases the editor can assign much shorter review window.
-
-#### Responsibilities of Reviewers
-
-Everyone on the MFEM team can be asked to serve as a reviewer on a PR in their area of expertise.
-
-**The responsibilities of the reviewers are:**
-
-1. To let the editors know if the proposed assignment is not a good match for them.
-
-2. To communicate with the PR author, provide feedback and work with them to resolve issues.
-
-3. To ensure the quality of the PR by making sure that the code adheres to the [Developer Guidelines](#developer-guidelines), e.g. all methods, data members, and functions have documentation, including data ownership and lifetime, new examples/miniapps have a corresponding PR in mfem/web, major features have `CHANGELOG` entries, etc.
-
-3. To seek help from the editors in case of difficulties.
-
-4. To complete the review in a timely manner: 3 weeks from assignment.
-
-5. To test the PR thoroughly before merging in *next*. The PR author is also encouraged to perform testing and inform the reviewers about the results.
-
-6. To monitor the PR impact on the testing in the *next* branch and alert the editors that the PR is ready for merging in *master*.
-
-7. The review of bugfixes should be expedited proportional to their importance. The review window can be much less than three weeks in such cases.
-
-#### Responsibilities of Authors
-
-Authors should clearly indicate when a PR is ready for review (before that the PR should be marked as `Draft` or `[WIP]`).
-
-**The responsibilities of the authors are:**
-
-1. To follow the instructions and PR checklist in the `CONTRIBUTING.md` document in the MFEM repository.
-
-2. To respond to reviewer feedback in a timely manner.
-
-3. Authors are encouraged to perform testing and inform the reviewers about the results.
-
-4. Authors can use the "Reviewers" section of the GitHub PR page to suggest reviewers, but the "Assignees" section will show who the editor has assigned to do the reviews.
-
-5. To indicate when the PR is ready for review by adding the `ready-for-review` label.
-
-
 ### Pull Request Checklist
 
 Before a PR can be merged, it should satisfy the following:
@@ -558,9 +450,7 @@ Before a PR can be merged, it should satisfy the following:
      - [ ] The miniapps go at the end of the page, and are usually listed only under a specific "Application (PDE)" category.
      - [ ] Add a short description of the miniapp in the "Extensive Examples" section of `features.md`.
 - [ ] New capability:
-   - [ ] All new public, protected, and private classes, methods, data members, and functions have full Doxygen-style documentation in source comments. Documentation should include descriptions of member data, function arguments and return values, template parameters, and prerequisites for calling new functions.
-   - [ ] Pointer arguments and return values must specify whether ownership is being transferred or lent with the call.
-   - [ ] Any new functions should include descriptions of their intended use e.g. for internal use only, user-facing, etc., along with references to example code whenever possible/appropriate.
+   - [ ] All significant new classes, methods and functions have Doxygen-style documentation in source comments.
    - [ ] Consider adding new sample runs in existing examples to highlight the new capability.
    - [ ] Consider saving cool simulation pictures with the new capability in the Confluence gallery (LLNL only) or submitting them, via pull request, to the gallery section of the `mfem/web` repo.
    - [ ] If this is a major new feature, consider mentioning it in the short summary inside `README` *(rare)*.
@@ -571,7 +461,6 @@ Before a PR can be merged, it should satisfy the following:
 - [ ] (LLNL only) After merging:
    - [ ] Update internal tests to include the new features.
 
-
 ### Master/Next Workflow
 
 MFEM uses a `master`/`next`-branch workflow as described below:
@@ -654,7 +543,7 @@ MFEM uses a `master`/`next`-branch workflow as described below:
   ```
 - [ ] Create the release tarball and push to `mfem/releases`.
 - [ ] Recreate the `next` branch as described in previous section.
-- [ ] Update and push documentation to `mfem/doxygen`. Update the `README.md` file and the `html` link in the `mfem/doxygen` repo.
+- [ ] Update and push documentation  to `mfem/doxygen`.
 - [ ] Update URL shortlinks:
     - [ ] Create a shortlink at [http://bit.ly/](http://bit.ly/) for the release tarball, e.g. https://mfem.github.io/releases/mfem-3.1.tgz.
     - [ ] (LLNL only) Add and commit the new shortlink in the `links` and `links-mfem` files of the internal `mfem/downloads` repo.
@@ -663,10 +552,8 @@ MFEM uses a `master`/`next`-branch workflow as described below:
     - Update version and shortlinks in `src/index.md` and `src/download.md`.
     -  Use [cloc-1.62.pl](http://cloc.sourceforge.net/) and `ls -lh` to estimate the SLOC and the tarball size in `src/download.md`.
 
-
 ## LLNL Workflow
 
-
 ### Mirroring on Bitbucket
 
 - The GitHub `master` and `next` branches are mirrored to the LLNL institutional
@@ -686,17 +573,16 @@ MFEM uses a `master`/`next`-branch workflow as described below:
   - `mfem:gh-next` -- Bleeding-edge development version, may be broken, use at
      your own risk.
 
-
 ### Mirroring on GitLab
 
 - MFEM repository is also mirrored on the LLNL GitLab instance, in a
   semi-automated manner.
 
 - This instance is meant to complete CI testing with tests on Livermore
-  Computing systems. GitLab pipeline status is reported in the corresponding
+  Computing systems. Gitlab pipeline status is reported in the corresponding
   GitHub pull request.
 
-- In GitLab pipelines, TPLs (dependencies) are built using Spack, driven by Uberenv.
+- In Gitlab pipelines, TPLs (dependencies) are built using Spack, driven by Uberenv.
 
 - No change to the MFEM repo can be made on this instance.
 
@@ -709,7 +595,6 @@ In addition, developers can set local git hooks to run some quick checks on
 commit or push, see the [README](config/githooks/README.md) in the `config/githooks`
 directory.
 
-
 ### Linux and Mac smoke tests
 We use GitHub Actions to drive the default tests on the `master` and `next`
 branches. See the `.github/workflows` files and the logs at
@@ -721,7 +606,6 @@ constraint on jobs. Two virtual machines are configured - Mac (OS X) and Linux.
 - Tests on the `master` branch are triggered whenever a PR is issued on this branch.
 - Tests on the `next` branch are currently scheduled to run each night.
 
-
 ### Windows smoke test
 We use Appveyor to test building with the MS Visual C++ compiler in a Windows
 environment, as well as to test the CMake build. See the `.appveyor` file and the
@@ -731,7 +615,6 @@ build logs at
 CMake is used to generate the MSVC Project files and drive the build.  A release
 and debug build is performed with a simple run of `ex1` to verify the executable.
 
-
 ### Tests at LLNL
 
 - We mirror the `master` and `next` branches internally (to `gh-master` and

INSTALL

@@ -7,10 +7,6 @@
 
                                https://mfem.org
 
-This file provides a detailed description of how to build and install the MFEM
-library. For a simple build, see the step-by-step instructions on the website
-at https://mfem.org/building.
-
 The MFEM library has a serial and an MPI-based parallel version, which largely
 share the same code base. The only prerequisite for building the serial version
 of MFEM is a (modern) C++ compiler, such as g++. The parallel version of MFEM
@@ -20,17 +16,13 @@ requires an MPI C++ compiler, as well as the following external libraries:
   https://github.com/hypre-space/hypre
 
 - METIS (a family of multilevel partitioning algorithms)
-  https://github.com/mfem/tpls
-
-  Note: We recommend our mirror of metis-4.0.3/5.1.0 above because the METIS
-  webpage, http://glaros.dtc.umn.edu/gkhome/metis/metis/overview, is often down
-  and we don't support yet the new repo https://github.com/KarypisLab/METIS.
+  http://glaros.dtc.umn.edu/gkhome/metis/metis/overview
 
 The hypre dependency can be downloaded as a tarball from GitHub or from the
-project webpage https://www.llnl.gov/casc/hypre. For example, the 2.24.0 release
+project webpage https://www.llnl.gov/casc/hypre. For example, the 2.20.0 release
 of hypre is available at
 
-   https://github.com/hypre-space/hypre/archive/v2.24.0.tar.gz
+   https://github.com/hypre-space/hypre/archive/v2.20.0.tar.gz
 
 The METIS dependency can be disabled but that is not generally recommended, see
 the option MFEM_USE_METIS.
@@ -45,7 +37,7 @@ as CUDA, HIP, OCCA, OpenMP and RAJA.
   https://developer.nvidia.com/cuda-toolkit
 
 - HIP support requires an AMD GPU and an installation of the ROCm software stack
-  https://rocmdocs.amd.com
+  https://rocm.github.io/ROCmInstall.html#installing-from-amd-rocm-repositories
 
 - OCCA support requires the OCCA library
   https://libocca.org
@@ -66,8 +58,8 @@ following package managers:
 
 - Spack, https://github.com/spack/spack
 - OpenHPC, http://openhpc.community
-- Conda-forge, https://conda-forge.org (pre-built binaries linked with
-  OpenMPI/MPICH, hypre, and METIS)
+- Conda-forge, https://conda-forge.org (pre-built binaries linked with OpenMPI/MPICH, hypre, and METIS)
+- Homebrew/Science, https://github.com/Homebrew/homebrew-science (deprecated)
 
 We also recommend downloading and building the MFEM-based GLVis visualization
 tool which can be used to visualize the meshes and solution in MFEM's examples
@@ -86,7 +78,7 @@ Parallel build:
 
 CUDA build:
    make cuda -j 4
-   (build for a specific compute capability: 'make cuda -j 4 CUDA_ARCH=sm_70')
+   (build for a specific compute capability: 'make cuda -j 4 CUDA_ARCH=sm_30')
 
 HIP build:
    make hip -j 4
@@ -337,6 +329,10 @@ MFEM_USE_SUNDIALS = YES/NO
    library. When enabled, this option uses the SUNDIALS_* library options,
    see below.
 
+MFEM_USE_MESQUITE = YES/NO
+   Enable MFEM functionality based on the Mesquite library. When enabled, this
+   option uses the MESQUITE_* library options, see below.
+
 MFEM_USE_SUITESPARSE = YES/NO
    Enable MFEM functionality based on the SuiteSparse library. Currently, this
    option adds the classes UMFPackSolver and KLUSolver (both sparse serial
@@ -463,30 +459,10 @@ MFEM_USE_UMPIRE = YES/NO
    discovery, provision, and management of memory on machines with multiple
    memory devices like NUMA and GPUs.
 
-MFEM_USE_BENCHMARK = YES/NO
-   Enables support for Google Benchmark, a library to support the benchmarking
-   of functions, in the tests/benchmarks directory.
-
 MFEM_USE_HIOP = YES/NO
    Enable the usage of HiOp (https://github.com/LLNL/hiop) in MFEM. HiOp is an
    HPC solver for nonlinear optimization problems.
 
-MFEM_USE_CODIPACK = YES/NO
-  Enable automatic differentiation using the CoDiPack library.
-  www.scicomp.uni-kl.de/codi/
-
-MFEM_USE_ALGOIM = YES/NO
-  Enable the usage of Algoim - a collection of high-order accurate numerical
-  methods and C++ algorithms for working with implicitly-defined geometry and
-  level set methods. The Algoim library requires the Blitz++ library. The MFEM
-  provides interface to Algoim v1. Thus, to check out the specific state use:
-  git checkout 9c9ca0ef094d8ab0390ed36367a1151b459bbe0a
-  https://algoim.github.io
-
-MFEM_USE_ADFORWARD = YES/NO
-  Enable forward mode for AD packages. This option is valid
-  only if the AD package supports two modes (backward/forward).
-
 MFEM_USE_CUDA = YES/NO
    Enables support for CUDA devices in MFEM. CUDA is a parallel computing
    platform and programming model for general computing on graphical processing
@@ -532,16 +508,6 @@ MFEM_USE_MKL_CPARDISO = YES/NO
    MFEM_USE_LAPACK=YES, verify that the MKL LAPACK libraries are used. The
    OpenMP capabilities are disabled at link time.
 
-MFEM_USE_MOONOLITH = YES/NO
-   Enables the ParMoonolith interface for parallel non-conforming, non-matching,
-   variational, volumetric mesh information transfer. It requires the variable
-   MOONOLITH_DIR=<path to installation> to be defined in the environment in
-   order to be used with the Makefile. Makefile users are also required to
-   install moonolith using the command `make install_all`, see
-   https://bitbucket.org/zulianp/par_moonolith for details.
-   Although Moonolith is an MPI-based library, both serial (MFEM_USE_MPI=NO) and
-   parallel (MFEM_USE_MPI=YES) versions of MFEM are supported.
-
 MFEM_USE_CALIPER = YES/NO
    Enables the interface to Caliper. Caliper is a library to integrate
    performance profiling capabilities into applications. To use Caliper,
@@ -554,22 +520,9 @@ MFEM_USE_FMS = YES/NO
    Enables support for the FMS library which consists of the DataCollection
    sub-class mfem::FMSDataCollection for I/O in FMS formats, see the header file
    fem/fmsdatacollection.hpp. In addition, this option enables in-memory
-   conversion routines between FMS's FmsDataCollection structure and MFEM's
+   convetion routines between FMS's FmsDataCollection structure and MFEM's
    DataCollection class, see the header file fem/fmsconvert.hpp.
 
-MFEM_USE_PARELAG = YES/NO
-   Enables the miniapps that use the ParELAG library. MFEM does not currently
-   use ParELAG. In fact, ParELAG is dependent on MFEM. Therefore, this option
-   currently only concerns the miniapps.
-
-MFEM_USE_ENZYME = YES/NO
-   Enables automatic differentiation support through the LLVM plugin Enzyme.
-   This requires the compiler to be set to clang (>=14.0.0). We also advise to
-   use the link time optimization (LTO) plugin, to enable functions that you
-   define over multiple files (compilation units) and want to be differentiated
-   automatically, to work. This requires to also use LLVM/LLD for linking.
-   Recommended options are in config/defaults.mk.
-
 MFEM_BUILD_TAG = (any value)
    An optional tag to characterize the build. Exported to config/config.mk.
    Can be used to identify the MFEM build from other makefiles.
@@ -597,11 +550,10 @@ The specific libraries and their options are:
   Versions: HYPRE >= 2.10.0b  (HYPRE built without CUDA)
             HYPRE >= 2.20.0   (HYPRE built with '--enable-mixedint')
             HYPRE >= 2.22.1   (HYPRE built with CUDA)
-            HYPRE >= 2.23.0   (HYPRE built with HIP)
 
 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
   MFEM_USE_METIS_5 = YES (default is to use METIS 4).
-  URL: https://github.com/mfem/tpls (MFEM mirror, see above)
+  URL: http://glaros.dtc.umn.edu/gkhome/metis/metis/overview
   Options: METIS_OPT, METIS_LIB.
   Versions: METIS 4.0.3 or 5.1.0.
 
@@ -628,13 +580,14 @@ The specific libraries and their options are:
   both MPI and hypre.
   If MFEM_USE_CUDA is enabled, we expect that SUNDIALS is built with support
   for CUDA.
-  If MFEM_USE_HIP is enabled, we expect that SUNDIALS is built with support
-  for HIP.
-  URL: http://computing.llnl.gov/projects/sundials/sundials-software
+  URL: http://computation.llnl.gov/projects/sundials/sundials-software
   Options: SUNDIALS_OPT, SUNDIALS_LIB.
-  Versions: SUNDIALS >= 5.0.0,
-            SUNDIALS >= 5.4.0 for CUDA support, and
-            SUNDIALS >= 5.7.0 for HIP support.
+  Versions: SUNDIALS >= 5.0.0, SUNDIALS >= 5.4.0 for CUDA support.
+
+- Mesquite (optional), used when MFEM_USE_MESQUITE = YES.
+  URL: http://trilinos.org/oldsite/packages/mesquite
+  Options: MESQUITE_OPT, MESQUITE_LIB.
+  The Mesquite support is deprecated and will be removed in the future.
 
 - SuiteSparse (optional), used when MFEM_USE_SUITESPARSE = YES.
   URL: http://faculty.cse.tamu.edu/davis/suitesparse.html
@@ -659,7 +612,8 @@ The specific libraries and their options are:
   requires the PT-Scotch and Scalapack libraries as well as ParMETIS, which
   includes METIS 5 in its distribution. Starting with STRUMPACK v2.2.0, ParMETIS
   and PT-Scotch are optional dependencies.
-  The support for STRUMPACK was added in MFEM v3.3.2.
+  The support for STRUMPACK was added in MFEM v3.3.2 and it requires STRUMPACK
+  2.0.0 or later.
   URL: http://portal.nersc.gov/project/sparse/strumpack
   Options: STRUMPACK_OPT, STRUMPACK_LIB.
   Versions: STRUMPACK >= 3.0.0.
@@ -668,8 +622,7 @@ The specific libraries and their options are:
   C++ compiler that supports the C++-14 standard. For additional requirements
   and dependencies of specific modules, see the Ginkgo webpage below.
   URL: https://ginkgo-project.github.io
-  Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or
-           Debug).
+  Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or Debug).
   Versions: Ginkgo >= 1.4.0.
 
 - AmgX (optional), used when MFEM_USE_AMGX = YES.
@@ -698,15 +651,12 @@ The specific libraries and their options are:
   PETSc has been cloned on the same level as mfem and hypre:
     ./configure --download-fblaslapack=yes --download-scalapack=yes \
                 --download-mumps=yes --download-suitesparse=yes \
-                --with-hypre-dir=../hypre/src/hypre \
+                --with-hypre-dir=../hypre-2.10.0b/src/hypre \
                 --with-shared-libraries=0
-  When building PETSc with HIP, one may need to add a flag like -std=c2x to
-  CFLAGS to allow proper parsing of the hipsparse header under C.
   URL: https://www.mcs.anl.gov/petsc
   Options: PETSC_OPT, PETSC_LIB.
-  Versions: PETSc >= 3.8.0   (PETSc build without CUDA/HIP)
+  Versions: PETSc >= 3.8.0   (PETSc build without CUDA)
             PETSc >= 3.15.0  (PETSc built with CUDA)
-            PETSc >= 3.19.0  (PETSc built with HIP, older versions may work too)
 
 - SLEPc (optional), used when MFEM_USE_SLEPC = YES. SLEPc depends on PETSc and
   uses some of the PETSc options when compiled.
@@ -737,17 +687,12 @@ The specific libraries and their options are:
   URL: https://scorec.rpi.edu/pumi
        https://github.com/SCOREC/core
   Options: PUMI_OPT, PUMI_LIB.
-  Versions: PUMI >= 2.2.6.
+  Versions: PUMI == 2.2.3.
 
 - HiOp (optional), used when MFEM_USE_HIOP = YES.
   URL: https://github.com/LLNL/hiop
   Options: HIOP_OPT, HIOP_LIB.
-  Versions: HIOP >= 0.4.6.
-
-- CoDiPack (optional), used with MFEM_USE_CODIPACK = YES
-  URL: https://www.scicomp.uni-kl.de/codi/
-  Options: CODIPACK_OPT
-  Versions: 1.9.3
+  Versions: HIOP >= 0.4.
 
 - GSLIB (optional), used when MFEM_USE_GSLIB = YES. The gslib library must be
   built prior to the MFEM build, as follows: download gslib-1.0.7, untar it at
@@ -759,20 +704,6 @@ The specific libraries and their options are:
   Options: GSLIB_OPT, GSLIB_LIB.
   Versions: GSLIB >= 1.0.7.
 
-- ALGOIM (optional), used when MFEM_USE_ALGOIM=YES. The library provides only
-  headers so it just needs to be downloaded at the same level as MFEM. Download
-  the specific version we use as:
-  "git clone https://github.com/algoim/algoim.git;
-   git checkout 9c9ca0ef094d8ab0390ed36367a1151b459bbe0a"
-  ALGOIM depends on BLITZ and the library must be built prior to the MFEM build.
-  Download v1.0.2, untar it at the same level as MFEM and create a symbolic link:
-  "ln -s blitz-1.0.2 blitz".
-  Build Blitz using CMake as:
-  "cmake . -DCMAKE_INSTALL_PREFIX=.; make lib; make install"
-  URL: https://github.com/blitzpp/blitz/archive/refs/tags/1.0.2.tar.gz
-  Options: BLITZ_OPT, BLITZ_LIB
-  Versions: BLITZ = 1.0.2
-
 - MKL CPardiso (optional), used when MFEM_USE_MKL_CPARDISO = YES.
   URL: https://software.intel.com/content/www/us/en/develop/tools/math-kernel-library.html
   Options: MKL_CPARDISO_OPT, MKL_CPARDISO_LIB.
@@ -784,7 +715,7 @@ The specific libraries and their options are:
   Versions: CUDA >= 10.1.168.
 
 - HIP (optional), used when MFEM_USE_HIP = YES.
-  URL: https://rocmdocs.amd.com
+  URL: https://rocm.github.io/ROCmInstall.html
   Options: HIP_CXX, HIP_ARCH, HIP_OPT, HIP_LIB.
 
 - OCCA (optional), used when MFEM_USE_OCCA = YES.
@@ -796,18 +727,13 @@ The specific libraries and their options are:
   URL: https://github.com/CEED/libCEED
        https://ceed.exascaleproject.org/libceed
   Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED >= 0.12.
+  Versions: libCEED >= 0.8.
 
 - RAJA (optional), used when MFEM_USE_RAJA = YES.
-  Beginning with MFEM v4.5.1, only RAJA v2022.10.3+ is supported.
+  Beginning with MFEM v4.3, only RAJA v0.13.0+ is supported.
   URL: https://github.com/LLNL/RAJA
   Options: RAJA_DIR, RAJA_OPT, RAJA_LIB.
-  Versions: RAJA >= 2022.10.3.
-
-- Moonolith (optional), use when MFEM_USE_MOONOLITH = YES.
-  URL: https://bitbucket.org/zulianp/par_moonolith
-  Options: MOONOLITH_DIR
-  Versions: MOONOLITH >= 1.1.0.
+  Versions: RAJA >= 0.13.0.
 
 - Caliper (optional), used when MFEM_USE_CALIPER = YES.
   URL: https://github.com/LLNL/Caliper
@@ -818,12 +744,7 @@ The specific libraries and their options are:
   Umpire requires camp when the Umpire version is >= 3.0.0.
   URL: https://github.com/LLNL/Umpire
   Options: UMPIRE_DIR, UMPIRE_OPT, UMPIRE_LIB.
-  Versions: Umpire >= 3.0.0.
-
-- Benchmark, used when MFEM_USE_BENCHMARK = YES.
-  URL: https://github.com/google/benchmark
-  Options: BENCHMARK_DIR, BENCHMARK_LIB.
-  Versions: Benchmark >= 1.5.6.
+  Versions: Umpire >= 2.0.0.
 
 - MPFR (optional), used when MFEM_USE_MPFR = YES.
   URL: http://mpfr.org, it depends on the GMP library: https://gmplib.org
@@ -845,16 +766,6 @@ The specific libraries and their options are:
   Options: FMS_OPT, FMS_LIB.
   Versions: FMS >= 0.2.
 
-- ParELAG, used when MFEM_USE_PARELAG = YES.
-  URL: https://github.com/LLNL/parelag
-  Options: PARELAG_DIR, PARELAG_OPT, PARELAG_LIB.
-
-- Enzyme, used when MFEM_USE_ENZYME = YES. Requires LLVM/Clang >= 14.0.0.
-  URL: https://github.com/EnzymeAD/Enzyme
-  Options: ENZYME_DIR, ENZYME_OPT, ENZYME_LIB.
-  Versions: Enzyme >= v0.0.33.
-
-
 Building with CMake
 ===================
 The MFEM build system consists of two steps: configuration and compilation.
@@ -966,6 +877,7 @@ MFEM_USE_LEGACY_OPENMP
 MFEM_USE_OPENMP
 MFEM_USE_MEMALLOC
 MFEM_TIMER_TYPE - Set automatically, can be overwritten.
+MFEM_USE_MESQUITE
 MFEM_USE_SUITESPARSE
 MFEM_USE_SUPERLU
 MFEM_USE_MUMPS
@@ -978,21 +890,14 @@ MFEM_USE_MPFR
 MFEM_USE_ZLIB
 MFEM_USE_PUMI
 MFEM_USE_HIOP
-MFEM_USE_CODIPACK
-MFEM_USE_ADFORWARD
 MFEM_USE_CUDA
-MFEM_USE_HIP
 MFEM_USE_OCCA
 MFEM_USE_CEED
 MFEM_USE_RAJA
 MFEM_USE_UMPIRE
 MFEM_USE_SIDRE
-MFEM_USE_MOONOLITH
 MFEM_USE_CALIPER
 MFEM_USE_FMS
-MFEM_USE_BENCHMARK
-MFEM_USE_PARELAG
-MFEM_USE_ENZYME
 
 The following options are CMake specific:
 
@@ -1030,6 +935,7 @@ The CMake build system adds auto-detection for the following packages/libraries:
  - HYPRE
  - METIS - The option MFEM_USE_METIS_5 is auto-detected.
  - ParMETIS
+ - MESQUITE
  - SuiteSparse
  - SuperLUDist, STRUMPACK
  - Ginkgo
@@ -1041,17 +947,12 @@ The CMake build system adds auto-detection for the following packages/libraries:
  - POSIXCLOCKS
  - PUMI
  - HIOP
- - CoDiPack
  - OCCA
  - RAJA
  - UMPIRE
  - AXOM - Used when MFEM_USE_SIDRE is enabled
- - MOONOLITH
  - CALIPER
  - FMS
- - BENCHMARK
- - ParELAG
- - Enzyme
 
 The following built-in CMake packages are also used:
 

LICENSE

@@ -1,6 +1,6 @@
 BSD 3-Clause License
 
-Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC
+Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

README

@@ -7,23 +7,20 @@
 
                                https://mfem.org
 
-[MFEM](https://mfem.org) is a modular parallel C++ library for finite element
-methods. Its goal is to enable high-performance scalable finite element
-discretization research and application development on a wide variety of
-platforms, ranging from laptops to supercomputers.
+MFEM is a modular parallel C++ library for finite element methods. Its goal is
+to enable high-performance scalable finite element discretization research and
+application development on a wide variety of platforms, ranging from laptops to
+supercomputers.
 
-We welcome contributions and feedback from the community. Please see the file
-[CONTRIBUTING.md](CONTRIBUTING.md) for additional details about our development
-process.
+* For building instructions, see the file INSTALL, or type "make help".
 
-* For building instructions, see the file [INSTALL](INSTALL), or type "make help".
-
-* Copyright and licensing information can be found in files [LICENSE](LICENSE) and [NOTICE](NOTICE).
+* Copyright and licensing information can be found in files LICENSE and NOTICE.
 
 * The best starting point for new users interested in MFEM's features is to
   review the examples and miniapps at https://mfem.org/examples.
 
-* Instructions for learning with Docker are in [config/docker](config/docker).
+* Developers interested in contributing to the library, should read the
+  instructions and documentation in the CONTRIBUTING.md file.
 
 Conceptually, MFEM can be viewed as a finite element toolbox that provides the
 building blocks for developing finite element algorithms in a manner similar to
@@ -61,16 +58,12 @@ solvers from the hypre library. Comprehensive support for other external
 packages, e.g. PETSc, SUNDIALS and libCEED is also included, giving access to
 additional linear and nonlinear solvers, preconditioners, time integrators, etc.
 
-For examples of using MFEM, see the [examples/](examples) and [miniapps/](miniapps)
-directories, as well as the OpenGL visualization tool GLVis which is available
-at https://glvis.org.
-
-## License
+For examples of using MFEM, see the examples/ and miniapps/ directories, as well
+as the OpenGL visualization tool GLVis which is available at https://glvis.org.
 
 MFEM is distributed under the terms of the BSD-3 license. All new contributions
-must be made under this license. See [LICENSE](LICENSE) and [NOTICE](NOTICE) for
-details.
+must be made under this license. See LICENSE and NOTICE for details.
 
-SPDX-License-Identifier: BSD-3-Clause <br>
-LLNL Release Number: LLNL-CODE-806117 <br>
+SPDX-License-Identifier: BSD-3-Clause
+LLNL Release Number: LLNL-CODE-806117
 DOI: 10.11578/dc.20171025.1248

config/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/XSDKDefaults.cmake

@@ -212,6 +212,10 @@ IF (DEFINED TPL_ENABLE_SUNDIALS)
   SET(MFEM_USE_SUNDIALS ${TPL_ENABLE_SUNDIALS} CACHE BOOL "Enable SUNDIALS usage" FORCE)
 ENDIF()
 
+IF (DEFINED TPL_ENABLE_MESQUITE)
+  SET(MFEM_USE_MESQUITE ${TPL_ENABLE_MESQUITE} CACHE BOOL "Enable MESQUITE usage" FORCE)
+ENDIF()
+
 IF (DEFINED TPL_ENABLE_SUITESPARSE)
   SET(MFEM_USE_SUITESPARSE ${TPL_ENABLE_SUITESPARSE} CACHE BOOL "Enable SuiteSparse usage" FORCE)
 ENDIF()
@@ -279,11 +283,3 @@ ENDIF()
 IF (DEFINED TPL_ENABLE_UMPIRE)
   SET(MFEM_USE_UMPIRE ${TPL_ENABLE_UMPIRE} CACHE BOOL "Enable Umpire" FORCE)
 ENDIF()
-
-IF (DEFINED TPL_ENABLE_BENCHMARK)
-  SET(MFEM_USE_BENCHMARK ${TPL_ENABLE_BENCHMARK} CACHE BOOL "Enable Google-Benchmark" FORCE)
-ENDIF()
-
-IF (DEFINED TPL_ENABLE_PARELAG)
-  SET(MFEM_USE_PARELAG ${TPL_ENABLE_PARELAG} CACHE BOOL "Enable ParELAG" FORCE)
-ENDIF()

config/cmake/MFEMConfig.cmake.in

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -29,6 +29,7 @@ set(MFEM_USE_LEGACY_OPENMP @MFEM_USE_LEGACY_OPENMP@)
 set(MFEM_USE_MEMALLOC @MFEM_USE_MEMALLOC@)
 set(MFEM_TIMER_TYPE @MFEM_TIMER_TYPE@)
 set(MFEM_USE_SUNDIALS @MFEM_USE_SUNDIALS@)
+set(MFEM_USE_MESQUITE @MFEM_USE_MESQUITE@)
 set(MFEM_USE_SUITESPARSE @MFEM_USE_SUITESPARSE@)
 set(MFEM_USE_SUPERLU @MFEM_USE_SUPERLU@)
 set(MFEM_USE_MUMPS @MFEM_USE_MUMPS@)
@@ -53,16 +54,7 @@ set(MFEM_USE_CEED @MFEM_USE_CEED@)
 set(MFEM_USE_UMPIRE @MFEM_USE_UMPIRE@)
 set(MFEM_USE_SIMD @MFEM_USE_SIMD@)
 set(MFEM_USE_ADIOS2 @MFEM_USE_ADIOS2@)
-set(MFEM_USE_MOONOLITH @MFEM_USE_MOONOLITH@)
-set(MFEM_USE_CODIPACK @MFEM_USE_CODIPACK@)
-set(MFEM_USE_MKL_CPARDISO @MFEM_USE_MKL_CPARDISO@)
-set(MFEM_USE_MKL_PARDISO @MFEM_USE_MKL_PARDISO@)
-set(MFEM_USE_ADFORWARD @MFEM_USE_ADFORWARD@)
 set(MFEM_USE_CALIPER @MFEM_USE_CALIPER@)
-set(MFEM_USE_ALGOIM @MFEM_USE_ALGOIM@)
-set(MFEM_USE_BENCHMARK @MFEM_USE_BENCHMARK@)
-set(MFEM_USE_PARELAG @MFEM_USE_PARELAG@)
-set(MFEM_USE_ENZYME @MFEM_USE_ENZYME@)
 
 set(MFEM_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
 set(MFEM_CXX_FLAGS "@CMAKE_CXX_FLAGS@")

config/cmake/config.hpp.in

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -39,9 +39,6 @@
 // Description of the git commit used to build MFEM.
 #cmakedefine MFEM_GIT_STRING "@MFEM_GIT_STRING@"
 
-// Enable shared library build of MFEM.
-#cmakedefine MFEM_SHARED_BUILD
-
 // Build the parallel MFEM library.
 // Requires an MPI compiler, and the libraries HYPRE and METIS.
 #cmakedefine MFEM_USE_MPI
@@ -80,102 +77,94 @@
 // Internal MFEM option: enable group/batch allocation for some small objects.
 #cmakedefine MFEM_USE_MEMALLOC
 
-// Which library functions to use in class StopWatch for measuring time.
-// For a list of the available options, see INSTALL.
-// If not defined, an option is selected automatically.
-#cmakedefine MFEM_TIMER_TYPE @MFEM_TIMER_TYPE@
-
-// Enable MFEM functionality based on the SUNDIALS libraries.
-#cmakedefine MFEM_USE_SUNDIALS
+// Enable MFEM functionality based on the Mesquite library.
+#cmakedefine MFEM_USE_MESQUITE
 
 // Enable MFEM functionality based on the SuiteSparse library.
 #cmakedefine MFEM_USE_SUITESPARSE
 
 // Enable MFEM functionality based on the SuperLU_DIST library.
 #cmakedefine MFEM_USE_SUPERLU
-#cmakedefine MFEM_USE_SUPERLU5
 
 // Enable MFEM functionality based on the MUMPS library.
 #cmakedefine MFEM_USE_MUMPS
-#cmakedefine MFEM_MUMPS_VERSION @MFEM_MUMPS_VERSION@
 
 // Enable MFEM functionality based on the STRUMPACK library.
 #cmakedefine MFEM_USE_STRUMPACK
 
-// Enable functionality based on the Ginkgo library.
+// Enable functionality based on the Ginkgo library
 #cmakedefine MFEM_USE_GINKGO
 
-// Enable MFEM functionality based on the AmgX library.
+// Enable MFEM functionality based on the AmgX library
 #cmakedefine MFEM_USE_AMGX
 
-// Enable secure socket streams based on the GNUTLS library.
+// Enable MFEM functionality based on the GnuTLS library
 #cmakedefine MFEM_USE_GNUTLS
 
-// Enable Sidre support.
-#cmakedefine MFEM_USE_SIDRE
-
-// Enable the use of SIMD in the high performance templated classes.
-#cmakedefine MFEM_USE_SIMD
-
-// Enable FMS support.
-#cmakedefine MFEM_USE_FMS
-
-// Enable Conduit support.
-#cmakedefine MFEM_USE_CONDUIT
-
-// Enable functionality based on the NetCDF library (reading CUBIT files).
-#cmakedefine MFEM_USE_NETCDF
-
-// Enable functionality based on the PETSc library.
-#cmakedefine MFEM_USE_PETSC
-
-// Enable functionality based on the SLEPc library.
-#cmakedefine MFEM_USE_SLEPC
-
-// Enable functionality based on the MPFR library.
-#cmakedefine MFEM_USE_MPFR
-
-// Enable MFEM functionality based on the PUMI library.
-#cmakedefine MFEM_USE_PUMI
-
-// Enable Moonolith-based general interpolation between finite element spaces.
-#cmakedefine MFEM_USE_MOONOLITH
-
-// Enable MFEM functionality based on the HIOP library.
-#cmakedefine MFEM_USE_HIOP
-
-// Enable MFEM functionality based on the GSLIB library.
+// Enable MFEM functionality based on the GSLIB library
 #cmakedefine MFEM_USE_GSLIB
 
-// Build the NVIDIA GPU/CUDA-enabled version of the MFEM library.
+// Enable MFEM functionality based on the NetCDF library
+#cmakedefine MFEM_USE_NETCDF
+
+// Enable MFEM functionality based on the PETSc library
+#cmakedefine MFEM_USE_PETSC
+
+// Enable MFEM functionality based on the SLEPc library
+#cmakedefine MFEM_USE_SLEPC
+
+// Enable MFEM functionality based on the Sidre library
+#cmakedefine MFEM_USE_SIDRE
+
+// Enable the use of SIMD in the high performance templated classes
+#cmakedefine MFEM_USE_SIMD
+
+// Enable MFEM functionality based on the FMS library
+#cmakedefine MFEM_USE_FMS
+
+// Enable MFEM functionality based on Conduit
+#cmakedefine MFEM_USE_CONDUIT
+
+// Enable MFEM functionality based on the PUMI library
+#cmakedefine MFEM_USE_PUMI
+
+// Enable MFEM functionality based on the HiOp library
+#cmakedefine MFEM_USE_HIOP
+
+// Build the GPU/CUDA-enabled version of the MFEM library.
 // Requires a CUDA compiler (nvcc).
 #cmakedefine MFEM_USE_CUDA
 
-// Build the AMD GPU/HIP-enabled version of the MFEM library.
+// Build the HIP-enabled version of the MFEM library.
 // Requires a HIP compiler (hipcc).
 #cmakedefine MFEM_USE_HIP
 
-// Enable functionality based on the RAJA library.
+// Enable MFEM functionality based on the RAJA library
 #cmakedefine MFEM_USE_RAJA
 
-// Enable functionality based on the OCCA library.
+// Enable MFEM functionality based on the OCCA library
 #cmakedefine MFEM_USE_OCCA
 
-// Enable functionality based on the libCEED library.
+// Enable MFEM functionality based on the libCEED library
 #cmakedefine MFEM_USE_CEED
 
-// Enable functionality based on the Caliper library.
-#cmakedefine MFEM_USE_CALIPER
-
-// Enable functionality based on the Algoim library.
-#cmakedefine MFEM_USE_ALGOIM
-
-// Enable functionality based on the Umpire library.
+// Enable MFEM functionality based on the Umpire library
 #cmakedefine MFEM_USE_UMPIRE
 
-// Enable IO functionality based on the ADIOS2 library.
+// Enable MFEM functionality based on the ADIOS2 library
 #cmakedefine MFEM_USE_ADIOS2
 
+// Enable MFEM functionality based on the Caliper library
+#cmakedefine MFEM_USE_CALIPER
+
+// Which library functions to use in class StopWatch for measuring time.
+// For a list of the available options, see INSTALL.
+// If not defined, an option is selected automatically.
+#define MFEM_TIMER_TYPE @MFEM_TIMER_TYPE@
+
+// Enable MFEM functionality based on the SUNDIALS libraries.
+#cmakedefine MFEM_USE_SUNDIALS
+
 // Version of HYPRE used for building MFEM.
 #cmakedefine MFEM_HYPRE_VERSION @MFEM_HYPRE_VERSION@
 
@@ -186,19 +175,4 @@
 // Enable interface to the MKL CPardiso library.
 #cmakedefine MFEM_USE_MKL_CPARDISO
 
-// Enable interface to the MKL Pardiso library.
-#cmakedefine MFEM_USE_MKL_PARDISO
-
-// Use forward mode for automatic differentiation.
-#cmakedefine MFEM_USE_ADFORWARD
-
-// Enable the use of the CoDiPack library for AD.
-#cmakedefine MFEM_USE_CODIPACK
-
-// Enable functionality based on the Google Benchmark library.
-#cmakedefine MFEM_USE_BENCHMARK
-
-// Enable Enzyme for AD
-#cmakedefine MFEM_USE_ENZYME
-
 #endif // MFEM_CONFIG_HEADER

config/cmake/modules/FindAMGX.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindAlgoim.cmake

@@ -1,22 +0,0 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - ALGOIM_FOUND
-#   - ALGOIM_LIBRARIES
-#   - ALGOIM_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(Algoim ALGOIM ALGOIM_DIR
-        "include;src" "algoim_quad.hpp"
-       	"" ""
-        "Paths to headers required by Algoim."
-        "Libraries required by Algoim.")

config/cmake/modules/FindAxom.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindBenchmark.cmake

@@ -1,22 +0,0 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - BENCHMARK_FOUND
-#   - BENCHMARK_LIBRARIES
-#   - BENCHMARK_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(Benchmark BENCHMARK BENCHMARK_DIR
-        "include" "benchmark/benchmark.h"
-        "lib" "benchmark"
-        "Paths to headers required by Google Benchmark."
-        "Libraries required by Google Benchmark.")

config/cmake/modules/FindCODIPACK.cmake

@@ -1,24 +0,0 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Automatic differentiation using the CoDiPack library.
-#  www.scicomp.uni-kl.de/codi/
-# Sets the following variables:
-#   - CODIPACK_FOUND
-#   - CODIPACK_INCLUDE_DIRS
-#   - CODIPACK_LIBRARIES
-
-include(MfemCmakeUtilities)
-mfem_find_package(CODIPACK CODIPACK CODIPACK_DIR
-  "include" "codi.h"
-  "lib" ""
-  "Paths to headers required by CODIPACK."
-  "Libraries required by CODIPACK.")

config/cmake/modules/FindCXXABIDemangle.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindCaliper.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -16,22 +16,7 @@
 
 include(MfemCmakeUtilities)
 mfem_find_package(Caliper CALIPER CALIPER_DIR
-      "include" "caliper/cali.h"
-      "lib" "caliper"
-      "Paths to headers required by Caliper."
-      "Libraries required by Caliper.")
-
-# Append adiak path/lib if the user provided ADIAK_DIR
-if(ADIAK_DIR AND EXISTS ${ADIAK_DIR})
-    find_package(adiak NO_DEFAULT_PATH REQUIRED PATHS ${ADIAK_DIR}/lib/cmake/adiak ${ADIAK_DIR})
-    list(APPEND CALIPER_INCLUDE_DIRS ${adiak_INCLUDE_DIRS})
-    list(APPEND CALIPER_LIBRARIES ${adiak_LIBRARIES})
-endif()
-
-# Append gotcha path/lib if the user provided GOTCHA_DIR
-if(GOTCHA_DIR AND EXISTS ${GOTCHA_DIR})
-    find_package(gotcha NO_DEFAULT_PATH REQUIRED PATHS ${GOTCHA_DIR}/lib/cmake/gotcha ${GOTCHA_DIR})
-    list(APPEND CALIPER_INCLUDE_DIRS ${gotcha_INCLUDE_DIRS})
-    list(APPEND CALIPER_LIBRARIES ${gotcha_LIBRARIES})
-endif()
-
+       	"include" "caliper/cali.h"
+       	"lib" "caliper"
+  	"Paths to headers required by Caliper."
+       	"Libraries required by Caliper.")

config/cmake/modules/FindConduit.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -21,15 +21,8 @@ if(EXISTS ${CONDUIT_DIR}/include/conduit/conduit_relay_hdf5.hpp)
   # we only need  HDF5 if Conduit was built with HDF5 support
   set(Conduit_REQUIRED_PACKAGES "HDF5" CACHE STRING
       "Additional packages required by Conduit.")
-  # Suppress warning about HDF5_ROOT being set
-  if (POLICY CMP0074)
-    cmake_policy(SET CMP0074 NEW)
-  endif()
-  # HDF5_ROOT is needed in some cases, e.g. when HDF5_TARGET_NAMES is set and
-  # MFEM's FindHDF5.cmake is not used.
-  set(HDF5_ROOT ${HDF5_DIR} CACHE PATH "")
-else()
-  message(STATUS "Conduit Relay HDF5 Support is DISABLED")
+  else()
+    message(STATUS "Conduit Relay HDF5 Support is DISABLED")
 endif()
 
 include(MfemCmakeUtilities)
@@ -39,4 +32,4 @@ mfem_find_package(Conduit CONDUIT CONDUIT_DIR
   ADD_COMPONENT relay
     "include;include/conduit" conduit_relay.hpp "lib" conduit_relay
   ADD_COMPONENT blueprint
-    "include;include/conduit" conduit_blueprint.hpp "lib" conduit_blueprint)
+      "include;include/conduit" conduit_blueprint.hpp "lib" conduit_blueprint)

config/cmake/modules/FindENZYME.cmake

@@ -1,27 +0,0 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-message(STATUS "Looking for ENZYME ...")
-message(STATUS "   in ENZYME_DIR = ${ENZYME_DIR}")
-
-# Make sure the directory and version combination works. Do nothing otherwise.
-if(EXISTS "${ENZYME_DIR}/ClangEnzyme-${ENZYME_VERSION}.so")
-  message(STATUS "Found ENZYME: ${ENZYME_DIR}/ClangEnzyme-${ENZYME_VERSION}.so")
-
-  # Set ENZYME_FOUND
-  set(ENZYME_FOUND TRUE CACHE BOOL "ENZYME was found." FORCE)
-
-  # Set CXX flags to accommodate the Enzyme Clang plugin
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -load -Xclang ${ENZYME_DIR}/ClangEnzyme-${ENZYME_VERSION}.so -mllvm -enzyme-loose-types=1")
-  set(MFEM_USE_ENZYME YES)
-else()
-
-endif()

config/cmake/modules/FindFMS.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindGSLIB.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -15,5 +15,5 @@
 #   - GSLIB_INCLUDE_DIRS
 
 include(MfemCmakeUtilities)
-mfem_find_package(GSLIB GSLIB GSLIB_DIR "include" gslib.h "lib" gs
+mfem_find_package(gslib GSLIB GSLIB_DIR "include" gslib.h "lib" gs
   "Paths to headers required by GSLIB." "Libraries required by GSLIB.")

config/cmake/modules/FindHDF5.cmake

@@ -1,70 +0,0 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - HDF5_FOUND - If HDF5 was found
-#   - HDF5_LIBRARIES - The HDF5 libraries
-#   - HDF5_INCLUDE_DIRS - The HDF5 include directories
-
-# NOTE: Using this FindHDF5.cmake instead of the CMake provided version may lead
-#       to issues with some TPL libraries that depend (or may depend) on HDF5.
-#       For this reason, we should consider removing this file, or at least
-#       making it use the CMake provided version by default and apply the logic
-#       below only when specifically requested by a user.  -V. Dobrev
-
-# First Check for HDF5_DIR
-if(NOT HDF5_DIR)
-  message(FATAL_ERROR
-    "Could not find HDF5. HDF5 support needs explicit HDF5_DIR")
-endif()
-
-if (NOT HDF5_FIND_QUIETLY)
-  message(STATUS "Looking for HDF5 ...")
-  message(STATUS "   in HDF5_DIR = ${HDF5_DIR}")
-endif()
-
-# Find includes
-find_path( HDF5_INCLUDE_DIRS hdf5.h
-           PATHS  ${HDF5_DIR}/include/
-           NO_DEFAULT_PATH
-           NO_CMAKE_ENVIRONMENT_PATH
-           NO_CMAKE_PATH
-           NO_SYSTEM_ENVIRONMENT_PATH
-           NO_CMAKE_SYSTEM_PATH)
-
-find_library( __HDF5_LIBRARY NAMES hdf5 libhdf5 libhdf5_D libhdf5_debug
-              PATHS ${HDF5_DIR}/lib
-              NO_DEFAULT_PATH
-              NO_CMAKE_ENVIRONMENT_PATH
-              NO_CMAKE_PATH
-              NO_SYSTEM_ENVIRONMENT_PATH
-              NO_CMAKE_SYSTEM_PATH)
-
-find_library( __HDF5_HL_LIBRARY NAMES hdf5_hl libhdf5_hl libhdf5_hl_D libhdf5_hl_debug
-              PATHS ${HDF5_DIR}/lib
-              NO_DEFAULT_PATH
-              NO_CMAKE_ENVIRONMENT_PATH
-              NO_CMAKE_PATH
-              NO_SYSTEM_ENVIRONMENT_PATH
-              NO_CMAKE_SYSTEM_PATH)
-
-set(HDF5_LIBRARIES ${__HDF5_HL_LIBRARY} ${__HDF5_LIBRARY})
-
-include(FindPackageHandleStandardArgs)
-
-# Handle the QUIETLY and REQUIRED arguments and set HDF5_FOUND to TRUE if all
-# listed variables are TRUE
-find_package_handle_standard_args(HDF5
-                                  " *** HDF5 not found. Please set HDF5_DIR."
-                                  HDF5_LIBRARIES
-                                  HDF5_INCLUDE_DIRS
-                                  __HDF5_LIBRARY
-                                  __HDF5_HL_LIBRARY)

config/cmake/modules/FindHIOP.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindHIP.cmake

@@ -0,0 +1,692 @@
+###############################################################################
+# FindHIP.cmake
+###############################################################################
+include(CheckCXXCompilerFlag)
+###############################################################################
+# SET: Variable defaults
+###############################################################################
+# User defined flags
+set(HIP_HIPCC_FLAGS "" CACHE STRING "Semicolon delimited flags for HIPCC")
+set(HIP_HCC_FLAGS "" CACHE STRING "Semicolon delimited flags for HCC")
+set(HIP_CLANG_FLAGS "" CACHE STRING "Semicolon delimited flags for CLANG")
+set(HIP_NVCC_FLAGS "" CACHE STRING "Semicolon delimted flags for NVCC")
+mark_as_advanced(HIP_HIPCC_FLAGS HIP_HCC_FLAGS HIP_CLANG_FLAGS HIP_NVCC_FLAGS)
+
+set(_hip_configuration_types ${CMAKE_CONFIGURATION_TYPES} ${CMAKE_BUILD_TYPE} Debug MinSizeRel Release RelWithDebInfo)
+list(REMOVE_DUPLICATES _hip_configuration_types)
+foreach(config ${_hip_configuration_types})
+    string(TOUPPER ${config} config_upper)
+    set(HIP_HIPCC_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for HIPCC")
+    set(HIP_HCC_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for HCC")
+    set(HIP_CLANG_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for CLANG")
+    set(HIP_NVCC_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for NVCC")
+    mark_as_advanced(HIP_HIPCC_FLAGS_${config_upper} HIP_HCC_FLAGS_${config_upper} HIP_CLANG_FLAGS_${config_upper} HIP_NVCC_FLAGS_${config_upper})
+endforeach()
+option(HIP_HOST_COMPILATION_CPP "Host code compilation mode" ON)
+option(HIP_VERBOSE_BUILD "Print out the commands run while compiling the HIP source file.  With the Makefile generator this defaults to VERBOSE variable specified on the command line, but can be forced on with this option." OFF)
+mark_as_advanced(HIP_HOST_COMPILATION_CPP)
+
+###############################################################################
+# FIND: HIP and associated helper binaries
+###############################################################################
+
+get_filename_component(_IMPORT_PREFIX "${CMAKE_CURRENT_LIST_DIR}/../" REALPATH)
+
+# HIP is supported on Linux only
+if(UNIX AND NOT APPLE AND NOT CYGWIN)
+    # Search for HIP installation
+    if(NOT HIP_ROOT_DIR)
+        # Search in user specified path first
+        find_path(
+            HIP_ROOT_DIR
+            NAMES bin/hipconfig
+            PATHS
+            "$ENV{ROCM_PATH}/hip"
+            ENV HIP_PATH
+            ${_IMPORT_PREFIX}
+            /opt/rocm/hip
+            DOC "HIP installed location"
+            NO_DEFAULT_PATH
+            )
+        if(NOT EXISTS ${HIP_ROOT_DIR})
+            if(HIP_FIND_REQUIRED)
+                message(FATAL_ERROR "Specify HIP_ROOT_DIR")
+            elseif(NOT HIP_FIND_QUIETLY)
+                message("HIP_ROOT_DIR not found or specified")
+            endif()
+        endif()
+        # And push it back to the cache
+        set(HIP_ROOT_DIR ${HIP_ROOT_DIR} CACHE PATH "HIP installed location" FORCE)
+    endif()
+
+    # Find HIPCC executable
+    find_program(
+        HIP_HIPCC_EXECUTABLE
+        NAMES hipcc
+        PATHS
+        "${HIP_ROOT_DIR}"
+        ENV ROCM_PATH
+        ENV HIP_PATH
+        /opt/rocm
+        /opt/rocm/hip
+        PATH_SUFFIXES bin
+        NO_DEFAULT_PATH
+        )
+    if(NOT HIP_HIPCC_EXECUTABLE)
+        # Now search in default paths
+        find_program(HIP_HIPCC_EXECUTABLE hipcc)
+    endif()
+    mark_as_advanced(HIP_HIPCC_EXECUTABLE)
+
+    # Find HIPCONFIG executable
+    find_program(
+        HIP_HIPCONFIG_EXECUTABLE
+        NAMES hipconfig
+        PATHS
+        "${HIP_ROOT_DIR}"
+        ENV ROCM_PATH
+        ENV HIP_PATH
+        /opt/rocm
+        /opt/rocm/hip
+        PATH_SUFFIXES bin
+        NO_DEFAULT_PATH
+        )
+    if(NOT HIP_HIPCONFIG_EXECUTABLE)
+        # Now search in default paths
+        find_program(HIP_HIPCONFIG_EXECUTABLE hipconfig)
+    endif()
+    mark_as_advanced(HIP_HIPCONFIG_EXECUTABLE)
+
+    # Find HIPCC_CMAKE_LINKER_HELPER executable
+    find_program(
+        HIP_HIPCC_CMAKE_LINKER_HELPER
+        NAMES hipcc_cmake_linker_helper
+        PATHS
+        "${HIP_ROOT_DIR}"
+        ENV ROCM_PATH
+        ENV HIP_PATH
+        /opt/rocm
+        /opt/rocm/hip
+        PATH_SUFFIXES bin
+        NO_DEFAULT_PATH
+        )
+    if(NOT HIP_HIPCC_CMAKE_LINKER_HELPER)
+        # Now search in default paths
+        find_program(HIP_HIPCC_CMAKE_LINKER_HELPER hipcc_cmake_linker_helper)
+    endif()
+    mark_as_advanced(HIP_HIPCC_CMAKE_LINKER_HELPER)
+
+    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_VERSION)
+        # Compute the version
+        execute_process(
+            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --version
+            OUTPUT_VARIABLE _hip_version
+            ERROR_VARIABLE _hip_error
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+            ERROR_STRIP_TRAILING_WHITESPACE
+            )
+        if(NOT _hip_error)
+            set(HIP_VERSION ${_hip_version} CACHE STRING "Version of HIP as computed from hipcc")
+        else()
+            set(HIP_VERSION "0.0.0" CACHE STRING "Version of HIP as computed by FindHIP()")
+        endif()
+        mark_as_advanced(HIP_VERSION)
+    endif()
+    if(HIP_VERSION)
+        string(REPLACE "." ";" _hip_version_list "${HIP_VERSION}")
+        list(GET _hip_version_list 0 HIP_VERSION_MAJOR)
+        list(GET _hip_version_list 1 HIP_VERSION_MINOR)
+        list(GET _hip_version_list 2 HIP_VERSION_PATCH)
+        set(HIP_VERSION_STRING "${HIP_VERSION}")
+    endif()
+
+    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_PLATFORM)
+        # Compute the platform
+        execute_process(
+            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --platform
+            OUTPUT_VARIABLE _hip_platform
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+            )
+        set(HIP_PLATFORM ${_hip_platform} CACHE STRING "HIP platform as computed by hipconfig")
+        mark_as_advanced(HIP_PLATFORM)
+    endif()
+
+    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_COMPILER)
+        # Compute the compiler
+        execute_process(
+            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --compiler
+            OUTPUT_VARIABLE _hip_compiler
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+            )
+        set(HIP_COMPILER ${_hip_compiler} CACHE STRING "HIP compiler as computed by hipconfig")
+        mark_as_advanced(HIP_COMPILER)
+    endif()
+
+    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_RUNTIME)
+        # Compute the runtime
+        execute_process(
+            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --runtime
+            OUTPUT_VARIABLE _hip_runtime
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+            )
+        set(HIP_RUNTIME ${_hip_runtime} CACHE STRING "HIP runtime as computed by hipconfig")
+        mark_as_advanced(HIP_RUNTIME)
+    endif()
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(
+    HIP
+    REQUIRED_VARS
+    HIP_ROOT_DIR
+    HIP_HIPCC_EXECUTABLE
+    HIP_HIPCONFIG_EXECUTABLE
+    HIP_PLATFORM
+    HIP_COMPILER
+    HIP_RUNTIME
+    VERSION_VAR HIP_VERSION
+    )
+
+###############################################################################
+# Set HIP CMAKE Flags
+###############################################################################
+# Copy the invocation styles from CXX to HIP
+set(CMAKE_HIP_ARCHIVE_CREATE ${CMAKE_CXX_ARCHIVE_CREATE})
+set(CMAKE_HIP_ARCHIVE_APPEND ${CMAKE_CXX_ARCHIVE_APPEND})
+set(CMAKE_HIP_ARCHIVE_FINISH ${CMAKE_CXX_ARCHIVE_FINISH})
+set(CMAKE_SHARED_LIBRARY_SONAME_HIP_FLAG ${CMAKE_SHARED_LIBRARY_SONAME_CXX_FLAG})
+set(CMAKE_SHARED_LIBRARY_CREATE_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS})
+set(CMAKE_SHARED_LIBRARY_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_CXX_FLAGS})
+#set(CMAKE_SHARED_LIBRARY_LINK_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS})
+set(CMAKE_SHARED_LIBRARY_RUNTIME_HIP_FLAG ${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG})
+set(CMAKE_SHARED_LIBRARY_RUNTIME_HIP_FLAG_SEP ${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG_SEP})
+set(CMAKE_SHARED_LIBRARY_LINK_STATIC_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_STATIC_CXX_FLAGS})
+set(CMAKE_SHARED_LIBRARY_LINK_DYNAMIC_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_DYNAMIC_CXX_FLAGS})
+
+set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "")
+set(HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS "")
+
+if("${HIP_COMPILER}" STREQUAL "nvcc")
+    # Set the CMake Flags to use the nvcc Compiler.
+    set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
+    set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <LINK_LIBRARIES> -shared" )
+    set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
+elseif("${HIP_COMPILER}" STREQUAL "hcc")
+    # Set the CMake Flags to use the hcc Compiler.
+    set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
+    set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <LINK_LIBRARIES> -shared" )
+    set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
+elseif("${HIP_COMPILER}" STREQUAL "clang")
+    #Number of parallel jobs by default is 1
+    if(NOT DEFINED HIP_CLANG_NUM_PARALLEL_JOBS)
+      set(HIP_CLANG_NUM_PARALLEL_JOBS 1)
+    endif()
+    #Add support for parallel build and link
+    if(${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
+      check_cxx_compiler_flag("-parallel-jobs=1" HIP_CLANG_SUPPORTS_PARALLEL_JOBS)
+    endif()
+    if(HIP_CLANG_NUM_PARALLEL_JOBS GREATER 1)
+      if(${HIP_CLANG_SUPPORTS_PARALLEL_JOBS})
+        set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "-Wno-format-nonliteral -parallel-jobs=${HIP_CLANG_NUM_PARALLEL_JOBS}")
+        set(HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS "-parallel-jobs=${HIP_CLANG_NUM_PARALLEL_JOBS}")
+      else()
+        message("clang compiler doesn't support parallel jobs")
+      endif()
+    endif()
+
+    # Set the CMake Flags to use the HIP-Clang Compiler.
+    set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
+    set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <LINK_LIBRARIES> -shared" )
+    set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
+
+    if("${HIP_RUNTIME}" STREQUAL "rocclr")
+      if(TARGET host)
+        message(STATUS "host interface - found")
+        set(HIP_HOST_INTERFACE host)
+      endif()
+    endif()
+endif()
+
+###############################################################################
+# MACRO: Locate helper files
+###############################################################################
+macro(HIP_FIND_HELPER_FILE _name _extension)
+    set(_hip_full_name "${_name}.${_extension}")
+    get_filename_component(CMAKE_CURRENT_LIST_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH)
+    set(HIP_${_name} "${CMAKE_CURRENT_LIST_DIR}/FindHIP/${_hip_full_name}")
+    if(NOT EXISTS "${HIP_${_name}}")
+        set(error_message "${_hip_full_name} not found in ${CMAKE_CURRENT_LIST_DIR}/FindHIP")
+        if(HIP_FIND_REQUIRED)
+            message(FATAL_ERROR "${error_message}")
+        else()
+            if(NOT HIP_FIND_QUIETLY)
+                message(STATUS "${error_message}")
+            endif()
+        endif()
+    endif()
+    # Set this variable as internal, so the user isn't bugged with it.
+    set(HIP_${_name} ${HIP_${_name}} CACHE INTERNAL "Location of ${_full_name}" FORCE)
+endmacro()
+
+###############################################################################
+hip_find_helper_file(run_make2cmake cmake)
+hip_find_helper_file(run_hipcc cmake)
+###############################################################################
+
+###############################################################################
+# MACRO: Reset compiler flags
+###############################################################################
+macro(HIP_RESET_FLAGS)
+    unset(HIP_HIPCC_FLAGS)
+    unset(HIP_HCC_FLAGS)
+    unset(HIP_CLANG_FLAGS)
+    unset(HIP_NVCC_FLAGS)
+    foreach(config ${_hip_configuration_types})
+        string(TOUPPER ${config} config_upper)
+        unset(HIP_HIPCC_FLAGS_${config_upper})
+        unset(HIP_HCC_FLAGS_${config_upper})
+        unset(HIP_CLANG_FLAGS_${config_upper})
+        unset(HIP_NVCC_FLAGS_${config_upper})
+    endforeach()
+endmacro()
+
+###############################################################################
+# MACRO: Separate the options from the sources
+###############################################################################
+macro(HIP_GET_SOURCES_AND_OPTIONS _sources _cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options)
+    set(${_sources})
+    set(${_cmake_options})
+    set(${_hipcc_options})
+    set(${_hcc_options})
+    set(${_clang_options})
+    set(${_nvcc_options})
+    set(_hipcc_found_options FALSE)
+    set(_hcc_found_options FALSE)
+    set(_clang_found_options FALSE)
+    set(_nvcc_found_options FALSE)
+    foreach(arg ${ARGN})
+        if("x${arg}" STREQUAL "xHIPCC_OPTIONS")
+            set(_hipcc_found_options TRUE)
+            set(_hcc_found_options FALSE)
+            set(_clang_found_options FALSE)
+            set(_nvcc_found_options FALSE)
+        elseif("x${arg}" STREQUAL "xHCC_OPTIONS")
+            set(_hipcc_found_options FALSE)
+            set(_hcc_found_options TRUE)
+            set(_clang_found_options FALSE)
+            set(_nvcc_found_options FALSE)
+        elseif("x${arg}" STREQUAL "xCLANG_OPTIONS")
+            set(_hipcc_found_options FALSE)
+            set(_hcc_found_options FALSE)
+            set(_clang_found_options TRUE)
+            set(_nvcc_found_options FALSE)
+        elseif("x${arg}" STREQUAL "xNVCC_OPTIONS")
+            set(_hipcc_found_options FALSE)
+            set(_hcc_found_options FALSE)
+            set(_clang_found_options FALSE)
+            set(_nvcc_found_options TRUE)
+        elseif(
+                "x${arg}" STREQUAL "xEXCLUDE_FROM_ALL" OR
+                "x${arg}" STREQUAL "xSTATIC" OR
+                "x${arg}" STREQUAL "xSHARED" OR
+                "x${arg}" STREQUAL "xMODULE"
+                )
+            list(APPEND ${_cmake_options} ${arg})
+        else()
+            if(_hipcc_found_options)
+                list(APPEND ${_hipcc_options} ${arg})
+            elseif(_hcc_found_options)
+                list(APPEND ${_hcc_options} ${arg})
+            elseif(_clang_found_options)
+                list(APPEND ${_clang_options} ${arg})
+            elseif(_nvcc_found_options)
+                list(APPEND ${_nvcc_options} ${arg})
+            else()
+                # Assume this is a file
+                list(APPEND ${_sources} ${arg})
+            endif()
+        endif()
+    endforeach()
+endmacro()
+
+###############################################################################
+# MACRO: Add include directories to pass to the hipcc command
+###############################################################################
+set(HIP_HIPCC_INCLUDE_ARGS_USER "")
+macro(HIP_INCLUDE_DIRECTORIES)
+    foreach(dir ${ARGN})
+        list(APPEND HIP_HIPCC_INCLUDE_ARGS_USER $<$<BOOL:${dir}>:-I${dir}>)
+    endforeach()
+endmacro()
+
+###############################################################################
+# FUNCTION: Helper to avoid clashes of files with the same basename but different paths
+###############################################################################
+function(HIP_COMPUTE_BUILD_PATH path build_path)
+    # Convert to cmake style paths
+    file(TO_CMAKE_PATH "${path}" bpath)
+    if(IS_ABSOLUTE "${bpath}")
+        string(FIND "${bpath}" "${CMAKE_CURRENT_BINARY_DIR}" _binary_dir_pos)
+        if(_binary_dir_pos EQUAL 0)
+            file(RELATIVE_PATH bpath "${CMAKE_CURRENT_BINARY_DIR}" "${bpath}")
+        else()
+            file(RELATIVE_PATH bpath "${CMAKE_CURRENT_SOURCE_DIR}" "${bpath}")
+        endif()
+    endif()
+
+    # Remove leading /
+    string(REGEX REPLACE "^[/]+" "" bpath "${bpath}")
+    # Avoid absolute paths by removing ':'
+    string(REPLACE ":" "_" bpath "${bpath}")
+    # Avoid relative paths that go up the tree
+    string(REPLACE "../" "__/" bpath "${bpath}")
+    # Avoid spaces
+    string(REPLACE " " "_" bpath "${bpath}")
+    # Strip off the filename
+    get_filename_component(bpath "${bpath}" PATH)
+
+    set(${build_path} "${bpath}" PARENT_SCOPE)
+endfunction()
+
+###############################################################################
+# MACRO: Parse OPTIONS from ARGN & set variables prefixed by _option_prefix
+###############################################################################
+macro(HIP_PARSE_HIPCC_OPTIONS _option_prefix)
+    set(_hip_found_config)
+    foreach(arg ${ARGN})
+        # Determine if we are dealing with a per-configuration flag
+        foreach(config ${_hip_configuration_types})
+            string(TOUPPER ${config} config_upper)
+            if(arg STREQUAL "${config_upper}")
+                set(_hip_found_config _${arg})
+                # Clear arg to prevent it from being processed anymore
+                set(arg)
+            endif()
+        endforeach()
+        if(arg)
+            list(APPEND ${_option_prefix}${_hip_found_config} "${arg}")
+        endif()
+    endforeach()
+endmacro()
+
+###############################################################################
+# MACRO: Try and include dependency file if it exists
+###############################################################################
+macro(HIP_INCLUDE_HIPCC_DEPENDENCIES dependency_file)
+    set(HIP_HIPCC_DEPEND)
+    set(HIP_HIPCC_DEPEND_REGENERATE FALSE)
+
+    # Create the dependency file if it doesn't exist
+    if(NOT EXISTS ${dependency_file})
+        file(WRITE ${dependency_file} "# Generated by: FindHIP.cmake. Do not edit.\n")
+    endif()
+    # Include the dependency file
+    include(${dependency_file})
+
+    # Verify the existence of all the included files
+    if(HIP_HIPCC_DEPEND)
+        foreach(f ${HIP_HIPCC_DEPEND})
+            if(NOT EXISTS ${f})
+                # If they aren't there, regenerate the file again
+                set(HIP_HIPCC_DEPEND_REGENERATE TRUE)
+            endif()
+        endforeach()
+    else()
+        # No dependencies, so regenerate the file
+        set(HIP_HIPCC_DEPEND_REGENERATE TRUE)
+    endif()
+
+    # Regenerate the dependency file if needed
+    if(HIP_HIPCC_DEPEND_REGENERATE)
+        set(HIP_HIPCC_DEPEND ${dependency_file})
+        file(WRITE ${dependency_file} "# Generated by: FindHIP.cmake. Do not edit.\n")
+    endif()
+endmacro()
+
+###############################################################################
+# MACRO: Prepare cmake commands for the target
+###############################################################################
+macro(HIP_PREPARE_TARGET_COMMANDS _target _format _generated_files _source_files)
+    set(_hip_flags "")
+    string(TOUPPER "${CMAKE_BUILD_TYPE}" _hip_build_configuration)
+    if(HIP_HOST_COMPILATION_CPP)
+        set(HIP_C_OR_CXX CXX)
+    else()
+        set(HIP_C_OR_CXX C)
+    endif()
+    set(generated_extension ${CMAKE_${HIP_C_OR_CXX}_OUTPUT_EXTENSION})
+
+    # Initialize list of includes with those specified by the user. Append with
+    # ones specified to cmake directly.
+    set(HIP_HIPCC_INCLUDE_ARGS ${HIP_HIPCC_INCLUDE_ARGS_USER})
+
+    # Add the include directories
+    set(include_directories_generator "$<TARGET_PROPERTY:${_target},INCLUDE_DIRECTORIES>")
+    list(APPEND HIP_HIPCC_INCLUDE_ARGS "$<$<BOOL:${include_directories_generator}>:-I$<JOIN:${include_directories_generator}, -I>>")
+
+    get_directory_property(_hip_include_directories INCLUDE_DIRECTORIES)
+    list(REMOVE_DUPLICATES _hip_include_directories)
+    if(_hip_include_directories)
+        foreach(dir ${_hip_include_directories})
+            list(APPEND HIP_HIPCC_INCLUDE_ARGS $<$<BOOL:${dir}>:-I${dir}>)
+        endforeach()
+    endif()
+
+    HIP_GET_SOURCES_AND_OPTIONS(_hip_sources _hip_cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options ${ARGN})
+    HIP_PARSE_HIPCC_OPTIONS(HIP_HIPCC_FLAGS ${_hipcc_options})
+    HIP_PARSE_HIPCC_OPTIONS(HIP_HCC_FLAGS ${_hcc_options})
+    HIP_PARSE_HIPCC_OPTIONS(HIP_CLANG_FLAGS ${_clang_options})
+    HIP_PARSE_HIPCC_OPTIONS(HIP_NVCC_FLAGS ${_nvcc_options})
+
+    # Add the compile definitions
+    set(compile_definition_generator "$<TARGET_PROPERTY:${_target},COMPILE_DEFINITIONS>")
+    list(APPEND HIP_HIPCC_FLAGS "$<$<BOOL:${compile_definition_generator}>:-D$<JOIN:${compile_definition_generator}, -D>>")
+
+    # Check if we are building shared library.
+    set(_hip_build_shared_libs FALSE)
+    list(FIND _hip_cmake_options SHARED _hip_found_SHARED)
+    list(FIND _hip_cmake_options MODULE _hip_found_MODULE)
+    if(_hip_found_SHARED GREATER -1 OR _hip_found_MODULE GREATER -1)
+        set(_hip_build_shared_libs TRUE)
+    endif()
+    list(FIND _hip_cmake_options STATIC _hip_found_STATIC)
+    if(_hip_found_STATIC GREATER -1)
+        set(_hip_build_shared_libs FALSE)
+    endif()
+
+    # If we are building a shared library, add extra flags to HIP_HIPCC_FLAGS
+    if(_hip_build_shared_libs)
+        list(APPEND HIP_HCC_FLAGS "-fPIC")
+        list(APPEND HIP_CLANG_FLAGS "-fPIC")
+        list(APPEND HIP_NVCC_FLAGS "--shared -Xcompiler '-fPIC'")
+    endif()
+
+    # Set host compiler
+    set(HIP_HOST_COMPILER "${CMAKE_${HIP_C_OR_CXX}_COMPILER}")
+
+    # Set compiler flags
+    set(_HIP_HOST_FLAGS "set(CMAKE_HOST_FLAGS ${CMAKE_${HIP_C_OR_CXX}_FLAGS})")
+    set(_HIP_HIPCC_FLAGS "set(HIP_HIPCC_FLAGS ${HIP_HIPCC_FLAGS})")
+    set(_HIP_HCC_FLAGS "set(HIP_HCC_FLAGS ${HIP_HCC_FLAGS})")
+    set(_HIP_CLANG_FLAGS "set(HIP_CLANG_FLAGS ${HIP_CLANG_FLAGS})")
+    set(_HIP_NVCC_FLAGS "set(HIP_NVCC_FLAGS ${HIP_NVCC_FLAGS})")
+    foreach(config ${_hip_configuration_types})
+        string(TOUPPER ${config} config_upper)
+        set(_HIP_HOST_FLAGS "${_HIP_HOST_FLAGS}\nset(CMAKE_HOST_FLAGS_${config_upper} ${CMAKE_${HIP_C_OR_CXX}_FLAGS_${config_upper}})")
+        set(_HIP_HIPCC_FLAGS "${_HIP_HIPCC_FLAGS}\nset(HIP_HIPCC_FLAGS_${config_upper} ${HIP_HIPCC_FLAGS_${config_upper}})")
+        set(_HIP_HCC_FLAGS "${_HIP_HCC_FLAGS}\nset(HIP_HCC_FLAGS_${config_upper} ${HIP_HCC_FLAGS_${config_upper}})")
+        set(_HIP_CLANG_FLAGS "${_HIP_CLANG_FLAGS}\nset(HIP_CLANG_FLAGS_${config_upper} ${HIP_CLANG_FLAGS_${config_upper}})")
+        set(_HIP_NVCC_FLAGS "${_HIP_NVCC_FLAGS}\nset(HIP_NVCC_FLAGS_${config_upper} ${HIP_NVCC_FLAGS_${config_upper}})")
+    endforeach()
+
+    # Reset the output variable
+    set(_hip_generated_files "")
+    set(_hip_source_files "")
+
+    # Iterate over all arguments and create custom commands for all source files
+    foreach(file ${ARGN})
+        # Ignore any file marked as a HEADER_FILE_ONLY
+        get_source_file_property(_is_header ${file} HEADER_FILE_ONLY)
+        # Allow per source file overrides of the format. Also allows compiling non .cu files.
+        get_source_file_property(_hip_source_format ${file} HIP_SOURCE_PROPERTY_FORMAT)
+        if((${file} MATCHES "\\.cu$" OR _hip_source_format) AND NOT _is_header)
+            set(host_flag FALSE)
+        else()
+            set(host_flag TRUE)
+        endif()
+
+        if(NOT host_flag)
+            # Determine output directory
+            HIP_COMPUTE_BUILD_PATH("${file}" hip_build_path)
+            set(hip_compile_output_dir "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${_target}.dir/${hip_build_path}")
+
+            get_filename_component(basename ${file} NAME)
+            set(generated_file_path "${hip_compile_output_dir}/${CMAKE_CFG_INTDIR}")
+            set(generated_file_basename "${_target}_generated_${basename}${generated_extension}")
+
+            # Set file names
+            set(generated_file "${generated_file_path}/${generated_file_basename}")
+            set(cmake_dependency_file "${hip_compile_output_dir}/${generated_file_basename}.depend")
+            set(custom_target_script_pregen "${hip_compile_output_dir}/${generated_file_basename}.cmake.pre-gen")
+            set(custom_target_script "${hip_compile_output_dir}/${generated_file_basename}.cmake")
+
+            # Set properties for object files
+            set_source_files_properties("${generated_file}"
+                PROPERTIES
+                EXTERNAL_OBJECT true # This is an object file not to be compiled, but only be linked
+                )
+
+            # Don't add CMAKE_CURRENT_SOURCE_DIR if the path is already an absolute path
+            get_filename_component(file_path "${file}" PATH)
+            if(IS_ABSOLUTE "${file_path}")
+                set(source_file "${file}")
+            else()
+                set(source_file "${CMAKE_CURRENT_SOURCE_DIR}/${file}")
+            endif()
+
+            # Bring in the dependencies
+            HIP_INCLUDE_HIPCC_DEPENDENCIES(${cmake_dependency_file})
+
+            # Configure the build script
+            configure_file("${HIP_run_hipcc}" "${custom_target_script_pregen}" @ONLY)
+            file(GENERATE
+                OUTPUT "${custom_target_script}"
+                INPUT "${custom_target_script_pregen}"
+                )
+            set(main_dep DEPENDS ${source_file})
+            if(CMAKE_GENERATOR MATCHES "Makefiles")
+                set(verbose_output "$(VERBOSE)")
+            elseif(HIP_VERBOSE_BUILD)
+                set(verbose_output ON)
+            else()
+                set(verbose_output OFF)
+            endif()
+
+            # Create up the comment string
+            file(RELATIVE_PATH generated_file_relative_path "${CMAKE_BINARY_DIR}" "${generated_file}")
+            set(hip_build_comment_string "Building HIPCC object ${generated_file_relative_path}")
+
+            # Build the generated file and dependency file
+            add_custom_command(
+                OUTPUT ${generated_file}
+                # These output files depend on the source_file and the contents of cmake_dependency_file
+                ${main_dep}
+                DEPENDS ${HIP_HIPCC_DEPEND}
+                DEPENDS ${custom_target_script}
+                # Make sure the output directory exists before trying to write to it.
+                COMMAND ${CMAKE_COMMAND} -E make_directory "${generated_file_path}"
+                COMMAND ${CMAKE_COMMAND} ARGS
+                -D verbose:BOOL=${verbose_output}
+                -D build_configuration:STRING=${_hip_build_configuration}
+                -D "generated_file:STRING=${generated_file}"
+                -P "${custom_target_script}"
+                WORKING_DIRECTORY "${hip_compile_output_dir}"
+                COMMENT "${hip_build_comment_string}"
+                )
+
+            # Make sure the build system knows the file is generated
+            set_source_files_properties(${generated_file} PROPERTIES GENERATED TRUE)
+            list(APPEND _hip_generated_files ${generated_file})
+            list(APPEND _hip_source_files ${file})
+        endif()
+    endforeach()
+
+    # Set the return parameter
+    set(${_generated_files} ${_hip_generated_files})
+    set(${_source_files} ${_hip_source_files})
+endmacro()
+
+###############################################################################
+# HIP_ADD_EXECUTABLE
+###############################################################################
+macro(HIP_ADD_EXECUTABLE hip_target)
+    # Separate the sources from the options
+    HIP_GET_SOURCES_AND_OPTIONS(_sources _cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options ${ARGN})
+    HIP_PREPARE_TARGET_COMMANDS(${hip_target} OBJ _generated_files _source_files ${_sources} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} CLANG_OPTIONS ${_clang_options} NVCC_OPTIONS ${_nvcc_options})
+    if(_source_files)
+        list(REMOVE_ITEM _sources ${_source_files})
+    endif()
+    if("${HIP_COMPILER}" STREQUAL "hcc")
+        if("x${HCC_HOME}" STREQUAL "x")
+            if (DEFINED ENV{ROCM_PATH})
+                set(HCC_HOME "$ENV{ROCM_PATH}/hcc")
+            elseif(DEFINED ENV{HIP_PATH})
+                set(HCC_HOME "$ENV{HIP_PATH}/../hcc")
+            else()
+                set(HCC_HOME "/opt/rocm/hcc")
+            endif()
+        endif()
+        set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
+    elseif("${HIP_COMPILER}" STREQUAL "clang")
+        if("x${HIP_CLANG_PATH}" STREQUAL "x")
+            if(DEFINED ENV{HIP_CLANG_PATH})
+                set(HIP_CLANG_PATH $ENV{HIP_CLANG_PATH})
+            elseif(DEFINED ENV{ROCM_PATH})
+                set(HIP_CLANG_PATH "$ENV{ROCM_PATH}/llvm/bin")
+            elseif(DEFINED ENV{HIP_PATH})
+                set(HIP_CLANG_PATH "$ENV{HIP_PATH}/../llvm/bin")
+            else()
+                set(HIP_CLANG_PATH "/opt/rocm/llvm/bin")
+            endif()
+        endif()
+        set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
+    else()
+        set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
+    endif()
+    if ("${_sources}" STREQUAL "")
+        add_executable(${hip_target} ${_cmake_options} ${_generated_files} "")
+    else()
+        add_executable(${hip_target} ${_cmake_options} ${_generated_files} ${_sources})
+    endif()
+    set_target_properties(${hip_target} PROPERTIES LINKER_LANGUAGE HIP)
+    # Link with host
+    if (HIP_HOST_INTERFACE)
+        # hip rt should be rocclr, compiler should be clang
+        target_link_libraries(${hip_target} ${HIP_HOST_INTERFACE})
+    endif()
+endmacro()
+
+###############################################################################
+# HIP_ADD_LIBRARY
+###############################################################################
+macro(HIP_ADD_LIBRARY hip_target)
+    # Separate the sources from the options
+    HIP_GET_SOURCES_AND_OPTIONS(_sources _cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options ${ARGN})
+    HIP_PREPARE_TARGET_COMMANDS(${hip_target} OBJ _generated_files _source_files ${_sources} ${_cmake_options} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} CLANG_OPTIONS ${_clang_options} NVCC_OPTIONS ${_nvcc_options})
+    if(_source_files)
+        list(REMOVE_ITEM _sources ${_source_files})
+    endif()
+    if ("${_sources}" STREQUAL "")
+        add_library(${hip_target} ${_cmake_options} ${_generated_files} "")
+    else()
+        add_library(${hip_target} ${_cmake_options} ${_generated_files} ${_sources})
+    endif()
+    set_target_properties(${hip_target} PROPERTIES LINKER_LANGUAGE ${HIP_C_OR_CXX})
+    # Link with host
+    if (HIP_HOST_INTERFACE)
+        # hip rt should be rocclr, compiler should be clang
+        target_link_libraries(${hip_target} ${HIP_HOST_INTERFACE})
+    endif()
+endmacro()
+
+# vim: ts=4:sw=4:expandtab:smartindent

config/cmake/modules/FindHIP/run_hipcc.cmake

@@ -0,0 +1,182 @@
+###############################################################################
+# Runs commands using HIPCC
+###############################################################################
+
+###############################################################################
+# This file runs the hipcc commands to produce the desired output file
+# along with the dependency file needed by CMake to compute dependencies.
+#
+# Input variables:
+#
+# verbose:BOOL=<>               OFF: Be as quiet as possible (default)
+#                               ON : Describe each step
+# build_configuration:STRING=<> Build configuration. Defaults to Debug.
+# generated_file:STRING=<>      File to generate. Mandatory argument.
+
+if(NOT build_configuration)
+    set(build_configuration Debug)
+endif()
+if(NOT generated_file)
+    message(FATAL_ERROR "You must specify generated_file on the command line")
+endif()
+
+# Set these up as variables to make reading the generated file easier
+set(HIP_HIPCC_EXECUTABLE "@HIP_HIPCC_EXECUTABLE@") # path
+set(HIP_HIPCONFIG_EXECUTABLE "@HIP_HIPCONFIG_EXECUTABLE@") #path
+set(HIP_HOST_COMPILER "@HIP_HOST_COMPILER@") # path
+set(CMAKE_COMMAND "@CMAKE_COMMAND@") # path
+set(HIP_run_make2cmake "@HIP_run_make2cmake@") # path
+set(HCC_HOME "@HCC_HOME@") #path
+set(HIP_CLANG_PATH "@HIP_CLANG_PATH@") #path
+set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "@HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS@")
+
+@HIP_HOST_FLAGS@
+@_HIP_HIPCC_FLAGS@
+@_HIP_HCC_FLAGS@
+@_HIP_CLANG_FLAGS@
+@_HIP_NVCC_FLAGS@
+#Needed to bring the HIP_HIPCC_INCLUDE_ARGS variable in scope
+set(HIP_HIPCC_INCLUDE_ARGS @HIP_HIPCC_INCLUDE_ARGS@) # list
+
+set(cmake_dependency_file "@cmake_dependency_file@") # path
+set(source_file "@source_file@") # path
+set(host_flag "@host_flag@") # bool
+
+# Determine compiler and compiler flags
+execute_process(COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --platform OUTPUT_VARIABLE HIP_PLATFORM OUTPUT_STRIP_TRAILING_WHITESPACE)
+execute_process(COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --compiler OUTPUT_VARIABLE HIP_COMPILER OUTPUT_STRIP_TRAILING_WHITESPACE)
+execute_process(COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --runtime OUTPUT_VARIABLE HIP_RUNTIME OUTPUT_STRIP_TRAILING_WHITESPACE)
+if(NOT host_flag)
+    set(__CC ${HIP_HIPCC_EXECUTABLE})
+    if("${HIP_PLATFORM}" STREQUAL "amd")
+        if("${HIP_COMPILER}" STREQUAL "hcc")
+            if(NOT "x${HCC_HOME}" STREQUAL "x")
+                set(ENV{HCC_HOME} ${HCC_HOME})
+            endif()
+            set(__CC_FLAGS ${HIP_HIPCC_FLAGS} ${HIP_HCC_FLAGS} ${HIP_HIPCC_FLAGS_${build_configuration}} ${HIP_HCC_FLAGS_${build_configuration}})
+        elseif("${HIP_COMPILER}" STREQUAL "clang")
+            if(NOT "x${HIP_CLANG_PATH}" STREQUAL "x")
+                set(ENV{HIP_CLANG_PATH} ${HIP_CLANG_PATH})
+            endif()
+            # Temporarily include HIP_HCC_FLAGS for HIP-Clang for PyTorch builds
+            set(__CC_FLAGS ${HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS} ${HIP_HIPCC_FLAGS} ${HIP_HCC_FLAGS} ${HIP_CLANG_FLAGS} ${HIP_HIPCC_FLAGS_${build_configuration}} ${HIP_HCC_FLAGS_${build_configuration}} ${HIP_CLANG_FLAGS_${build_configuration}})
+        endif()
+    else()
+        set(__CC_FLAGS ${HIP_HIPCC_FLAGS} ${HIP_NVCC_FLAGS} ${HIP_HIPCC_FLAGS_${build_configuration}} ${HIP_NVCC_FLAGS_${build_configuration}})
+    endif()
+else()
+    set(__CC ${HIP_HOST_COMPILER})
+    set(__CC_FLAGS ${CMAKE_HOST_FLAGS} ${CMAKE_HOST_FLAGS_${build_configuration}})
+endif()
+set(__CC_INCLUDES ${HIP_HIPCC_INCLUDE_ARGS})
+
+# hip_execute_process - Executes a command with optional command echo and status message.
+#   status     - Status message to print if verbose is true
+#   command    - COMMAND argument from the usual execute_process argument structure
+#   ARGN       - Remaining arguments are the command with arguments
+#   HIP_result - Return value from running the command
+macro(hip_execute_process status command)
+    set(_command ${command})
+    if(NOT "x${_command}" STREQUAL "xCOMMAND")
+        message(FATAL_ERROR "Malformed call to hip_execute_process.  Missing COMMAND as second argument. (command = ${command})")
+    endif()
+    if(verbose)
+        execute_process(COMMAND "${CMAKE_COMMAND}" -E echo -- ${status})
+        # Build command string to print
+        set(hip_execute_process_string)
+        foreach(arg ${ARGN})
+            # Escape quotes if any
+            string(REPLACE "\"" "\\\"" arg ${arg})
+            # Surround args with spaces with quotes
+            if(arg MATCHES " ")
+                list(APPEND hip_execute_process_string "\"${arg}\"")
+            else()
+                list(APPEND hip_execute_process_string ${arg})
+            endif()
+        endforeach()
+        # Echo the command
+        execute_process(COMMAND ${CMAKE_COMMAND} -E echo ${hip_execute_process_string})
+    endif()
+    # Run the command
+    execute_process(COMMAND ${ARGN} RESULT_VARIABLE HIP_result)
+endmacro()
+
+# Delete the target file
+hip_execute_process(
+    "Removing ${generated_file}"
+    COMMAND "${CMAKE_COMMAND}" -E remove "${generated_file}"
+    )
+
+# Generate the dependency file
+hip_execute_process(
+    "Generating dependency file: ${cmake_dependency_file}.pre"
+    COMMAND "${__CC}"
+    -M
+    "${source_file}"
+    -o "${cmake_dependency_file}.pre"
+    ${__CC_FLAGS}
+    ${__CC_INCLUDES}
+    )
+
+if(HIP_result)
+    message(FATAL_ERROR "Error generating ${generated_file}")
+endif()
+
+# Generate the cmake readable dependency file to a temp file
+hip_execute_process(
+    "Generating temporary cmake readable file: ${cmake_dependency_file}.tmp"
+    COMMAND "${CMAKE_COMMAND}"
+    -D "input_file:FILEPATH=${cmake_dependency_file}.pre"
+    -D "output_file:FILEPATH=${cmake_dependency_file}.tmp"
+    -D "verbose=${verbose}"
+    -P "${HIP_run_make2cmake}"
+    )
+
+if(HIP_result)
+    message(FATAL_ERROR "Error generating ${generated_file}")
+endif()
+
+# Copy the file if it is different
+hip_execute_process(
+    "Copy if different ${cmake_dependency_file}.tmp to ${cmake_dependency_file}"
+    COMMAND "${CMAKE_COMMAND}" -E copy_if_different "${cmake_dependency_file}.tmp" "${cmake_dependency_file}"
+    )
+
+if(HIP_result)
+    message(FATAL_ERROR "Error generating ${generated_file}")
+endif()
+
+# Delete the temporary file
+hip_execute_process(
+    "Removing ${cmake_dependency_file}.tmp and ${cmake_dependency_file}.pre"
+    COMMAND "${CMAKE_COMMAND}" -E remove "${cmake_dependency_file}.tmp" "${cmake_dependency_file}.pre"
+    )
+
+if(HIP_result)
+    message(FATAL_ERROR "Error generating ${generated_file}")
+endif()
+
+# Generate the output file
+hip_execute_process(
+    "Generating ${generated_file}"
+    COMMAND "${__CC}"
+    -c
+    "${source_file}"
+    -o "${generated_file}"
+    ${__CC_FLAGS}
+    ${__CC_INCLUDES}
+    )
+
+if(HIP_result)
+    # Make sure that we delete the output file
+    hip_execute_process(
+        "Removing ${generated_file}"
+        COMMAND "${CMAKE_COMMAND}" -E remove "${generated_file}"
+        )
+    message(FATAL_ERROR "Error generating file ${generated_file}")
+else()
+    if(verbose)
+        message("Generated ${generated_file} successfully.")
+    endif()
+endif()
+# vim: ts=4:sw=4:expandtab:smartindent

config/cmake/modules/FindHIP/run_make2cmake.cmake

@@ -0,0 +1,50 @@
+###############################################################################
+# Computes dependencies using HIPCC
+###############################################################################
+
+###############################################################################
+# This file converts dependency files generated using hipcc to a format that
+# cmake can understand.
+
+# Input variables:
+#
+# input_file:STRING=<> Dependency file to parse. Required argument
+# output_file:STRING=<> Output file to generate. Required argument
+
+if(NOT input_file OR NOT output_file)
+    message(FATAL_ERROR "You must specify input_file and output_file on the command line")
+endif()
+
+file(READ ${input_file} depend_text)
+
+if (NOT "${depend_text}" STREQUAL "")
+    string(REPLACE " /" "\n/" depend_text ${depend_text})
+    string(REGEX REPLACE "^.*:" "" depend_text ${depend_text})
+    string(REGEX REPLACE "[ \\\\]*\n" ";" depend_text ${depend_text})
+
+    set(dependency_list "")
+
+    foreach(file ${depend_text})
+        string(REGEX REPLACE "^ +" "" file ${file})
+        if(NOT EXISTS "${file}")
+            message(WARNING " Removing non-existent dependency file: ${file}")
+            set(file "")
+        endif()
+
+        if(NOT IS_DIRECTORY "${file}")
+            get_filename_component(file_absolute "${file}" ABSOLUTE)
+            list(APPEND dependency_list "${file_absolute}")
+        endif()
+    endforeach()
+endif()
+
+# Remove the duplicate entries and sort them.
+list(REMOVE_DUPLICATES dependency_list)
+list(SORT dependency_list)
+
+foreach(file ${dependency_list})
+    set(hip_hipcc_depend "${hip_hipcc_depend} \"${file}\"\n")
+endforeach()
+
+file(WRITE ${output_file} "# Generated by: FindHIP.cmake. Do not edit.\nSET(HIP_HIPCC_DEPEND\n ${hip_hipcc_depend})\n\n")
+# vim: ts=4:sw=4:expandtab:smartindent

config/cmake/modules/FindHYPRE.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -14,51 +14,10 @@
 #   - HYPRE_LIBRARIES
 #   - HYPRE_INCLUDE_DIRS
 #   - HYPRE_VERSION
-#   - HYPRE_USING_CUDA (internal)
-#   - HYPRE_USING_HIP (internal)
-
-if (HYPRE_FOUND)
-  if (HYPRE_USING_CUDA)
-    find_package(CUDAToolkit REQUIRED)
-  endif()
-  if (HYPRE_USING_HIP)
-    find_package(rocsparse REQUIRED)
-    find_package(rocrand REQUIRED)
-  endif()
-  return()
-endif()
 
 include(MfemCmakeUtilities)
 mfem_find_package(HYPRE HYPRE HYPRE_DIR "include" "HYPRE.h" "lib" "HYPRE"
-  "Paths to headers required by HYPRE." "Libraries required by HYPRE."
-  CHECK_BUILD HYPRE_USING_CUDA FALSE
-  "
-#undef HYPRE_USING_CUDA
-#include <HYPRE_config.h>
-
-#ifndef HYPRE_USING_CUDA
-#error HYPRE is built without CUDA.
-#endif
-
-int main()
-{
-   return 0;
-}
-"
-  CHECK_BUILD HYPRE_USING_HIP FALSE
-  "
-#undef HYPRE_USING_HIP
-#include <HYPRE_config.h>
-
-#ifndef HYPRE_USING_HIP
-#error HYPRE is built without HIP.
-#endif
-
-int main()
-{
-   return 0;
-}
-")
+  "Paths to headers required by HYPRE." "Libraries required by HYPRE.")
 
 if (HYPRE_FOUND AND (NOT HYPRE_VERSION))
   try_run(HYPRE_VERSION_RUN_RESULT HYPRE_VERSION_COMPILE_RESULT
@@ -74,22 +33,3 @@ if (HYPRE_FOUND AND (NOT HYPRE_VERSION))
     message(FATAL_ERROR "Unable to determine HYPRE version.")
   endif()
 endif()
-
-if (HYPRE_FOUND AND HYPRE_USING_CUDA)
-  find_package(CUDAToolkit REQUIRED)
-  get_target_property(CUSPARSE_LIBRARIES CUDA::cusparse LOCATION)
-  get_target_property(CURAND_LIBRARIES CUDA::curand LOCATION)
-  list(APPEND HYPRE_LIBRARIES ${CUSPARSE_LIBRARIES} ${CURAND_LIBRARIES})
-  set(HYPRE_LIBRARIES ${HYPRE_LIBRARIES} CACHE STRING
-      "HYPRE libraries + dependencies." FORCE)
-  message(STATUS "Updated HYPRE_LIBRARIES: ${HYPRE_LIBRARIES}")
-endif()
-
-if (HYPRE_FOUND AND HYPRE_USING_HIP)
-  find_package(rocsparse REQUIRED)
-  find_package(rocrand REQUIRED)
-  list(APPEND HYPRE_LIBRARIES ${rocsparse_LIBRARIES} ${rocrand_LIBRARIES})
-  set(HYPRE_LIBRARIES ${HYPRE_LIBRARIES} CACHE STRING
-      "HYPRE libraries + dependencies." FORCE)
-  message(STATUS "Updated HYPRE_LIBRARIES: ${HYPRE_LIBRARIES}")
-endif()

config/cmake/modules/FindLIBDL.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindLibunwind.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindMETIS.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindMFEMBacktrace.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindMKL_CPARDISO.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindMKL_PARDISO.cmake

@@ -1,27 +0,0 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - MKL_PARDISO_FOUND
-#   - MKL_PARDISO_LIBRARIES
-#   - MKL_PARDISO_INCLUDE_DIRS
-
-if(NOT MKL_LIBRARY_DIR)
-  message(WARNING "Using default MKL library path. Double check the variable MKL_LIBRARY_DIR")
-  set(MKL_LIBRARY_DIR "lib/intel64")
-endif()
-
-include(MfemCmakeUtilities)
-mfem_find_package(MKL_PARDISO MKL_PARDISO
-    MKL_PARDISO_DIR "include" mkl_pardiso.h ${MKL_LIBRARY_DIR} mkl_core
-  "Paths to headers required by MKL Pardiso." "Libraries required by MKL PARDISO."
-  ADD_COMPONENT MKL_LP64 "include" "" ${MKL_LIBRARY_DIR} mkl_intel_lp64
-  ADD_COMPONENT MKL_SEQUENTIAL "include" "" ${MKL_LIBRARY_DIR} mkl_sequential)

config/cmake/modules/FindMPFR.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindMUMPS.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -11,9 +11,8 @@
 
 # Sets the following variables:
 #   - MUMPS_FOUND
-#   - MUMPS_LIBRARIES
 #   - MUMPS_INCLUDE_DIRS
-#   - MUMPS_VERSION
+#   - MUMPS_LIBRARIES
 
 include(MfemCmakeUtilities)
 mfem_find_package(MUMPS MUMPS MUMPS_DIR
@@ -22,18 +21,3 @@ mfem_find_package(MUMPS MUMPS MUMPS_DIR
   "Libraries required by MUMPS."
   ADD_COMPONENT mumps_common "include" dmumps_c.h "lib" mumps_common
   ADD_COMPONENT pord "include" dmumps_c.h "lib" pord)
-
-if (MUMPS_FOUND AND (NOT MUMPS_VERSION))
-  try_run(MUMPS_VERSION_RUN_RESULT MUMPS_VERSION_COMPILE_RESULT
-          ${CMAKE_CURRENT_BINARY_DIR}/config
-          ${CMAKE_CURRENT_SOURCE_DIR}/config/get_mumps_version.cpp
-          CMAKE_FLAGS -DINCLUDE_DIRECTORIES:STRING=${MUMPS_INCLUDE_DIRS}
-          RUN_OUTPUT_VARIABLE MUMPS_VERSION_OUTPUT)
-  if ((MUMPS_VERSION_RUN_RESULT EQUAL 0) AND MUMPS_VERSION_OUTPUT)
-    string(STRIP "${MUMPS_VERSION_OUTPUT}" MUMPS_VERSION)
-    set(MUMPS_VERSION ${MUMPS_VERSION} CACHE STRING "MUMPS version." FORCE)
-    message(STATUS "Found MUMPS version ${MUMPS_VERSION}")
-  else()
-    message(FATAL_ERROR "Unable to determine MUMPS version.")
-  endif()
-endif()

config/cmake/modules/FindMesquite.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -10,13 +10,11 @@
 # CONTRIBUTING.md for details.
 
 # Defines the following variables:
-#   - BLITZ_FOUND
-#   - BLITZ_LIBRARIES
-#   - BLITZ_INCLUDE_DIRS
+#   - MESQUITE_FOUND
+#   - MESQUITE_LIBRARIES
+#   - MESQUITE_INCLUDE_DIRS
 
 include(MfemCmakeUtilities)
-mfem_find_package(Blitz BLITZ BLITZ_DIR
-       	"include" "blitz/blitz.h"
-       	"lib" "blitz"
-  	"Paths to headers required by Blitz."
-       	"Libraries required by Blitz.")
+mfem_find_package(Mesquite MESQUITE MESQUITE_DIR
+  "include" "Mesquite_all_headers.hpp" "lib" "mesquite"
+  "Paths to headers required by Mesquite." "Libraries required by Mesquite.")

config/cmake/modules/FindNetCDF.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -14,38 +14,21 @@
 #   - NETCDF_LIBRARIES
 #   - NETCDF_INCLUDE_DIRS
 
-# Suppress warning about HDF5_ROOT being set
-if (POLICY CMP0074)
-  cmake_policy(SET CMP0074 NEW)
-endif()
-
 include(MfemCmakeUtilities)
 
 # FindHDF5.cmake uses HDF5_ROOT, so we "translate" from the MFEM convention
-# (Needed in some cases, e.g. when HDF5_TARGET_NAMES is set and MFEM's
-# FindHDF5.cmake is not used.)
 set(HDF5_ROOT ${HDF5_DIR} CACHE PATH "")
-
 # We need to guard against the case where HDF5 was already found but without
 # the HL extensions (in which case mfem_find_package will treat the package
 # as already having been found), so we reset the variable to force FindHDF5.cmake
 # to be called for a second time
 set(HDF5_FOUND OFF)
 enable_language(C) # FindHDF5.cmake uses the C compiler
-
 mfem_find_package(NetCDF NETCDF NETCDF_DIR "include" netcdf.h "lib" netcdf
   "Paths to headers required by NetCDF." "Libraries required by NetCDF.")
-
 # The HL extension libraries are in a separate variable and must precede
 # the "regular" hdf5 library, as hdf5_hl depends on hdf5
 # The netcdf library will always be the first element of NETCDF_LIBRARIES
 # and we need to insert after that library but before the hdf5 library, so
 # position 1 is used
-# (Needed in some cases, e.g. when HDF5_TARGET_NAMES is set and MFEM's
-# FindHDF5.cmake is not used.)
-if (HDF5_C_LIBRARY_hdf5_hl)
-  list(INSERT NETCDF_LIBRARIES 1 ${HDF5_C_LIBRARY_hdf5_hl})
-  if (NOT NetCDF_FIND_QUIETLY)
-    message(STATUS "Updated NetCDF libraries: ${NETCDF_LIBRARIES}")
-  endif()
-endif()
+list(INSERT NETCDF_LIBRARIES 1 ${HDF5_C_LIBRARY_hdf5_hl})

config/cmake/modules/FindOCCA.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindPARELAG.cmake

@@ -1,19 +0,0 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - PARELAG_FOUND
-#   - PARELAG_LIBRARIES
-#   - PARELAG_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(PARELAG PARELAG PARELAG_DIR "" "" "" ""
-  "Paths to headers required by ParELAG." "Libraries required by ParELAG.")

config/cmake/modules/FindPOSIXClocks.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindParMETIS.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindRAJA.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -14,27 +14,17 @@
 #   - RAJA_LIBRARIES
 #   - RAJA_INCLUDE_DIRS
 
-if (RAJA_FOUND)
-   return()
+include(MfemCmakeUtilities)
+mfem_find_package(RAJA RAJA RAJA_DIR "include" "RAJA/RAJA.hpp" "lib" "RAJA"
+  "Paths to headers required by RAJA." "Libraries required by RAJA.")
+
+if (NOT RAJA_CONFIG_CMAKE)
+   set(RAJA_CONFIG_CMAKE "${RAJA_DIR}/share/raja/cmake/raja-config.cmake")
 endif()
-message(STATUS "Looking for RAJA ...")
-if (RAJA_DIR)
-   message(STATUS "   in RAJA_DIR = ${RAJA_DIR}")
-   find_package(RAJA CONFIG NO_DEFAULT_PATH PATHS "${RAJA_DIR}")
-endif()
-if (NOT RAJA_FOUND)
-   message(STATUS "   in standard CMake locations")
-   find_package(RAJA CONFIG)
-endif()
-if (RAJA_FOUND)
-   set(RAJA_LIBRARIES "RAJA" CACHE STRING "RAJA imported target." FORCE)
-   set(RAJA_INCLUDE_DIRS "" CACHE STRING "RAJA include dirs (not used)" FORCE)
-   message(STATUS
-      "Found RAJA target: ${RAJA_LIBRARIES} (version: ${RAJA_VERSION})")
-else()
-   set(msg STATUS)
-   if (RAJA_FIND_REQUIRED)
-      set(msg FATAL_ERROR)
+if (EXISTS "${RAJA_CONFIG_CMAKE}")
+   include("${RAJA_CONFIG_CMAKE}")
+   if (ENABLE_CUDA AND NOT MFEM_USE_CUDA)
+      message(FATAL_ERROR
+         "RAJA is built with CUDA: MFEM_USE_CUDA=YES is required")
    endif()
-   message(${msg} "RAJA not found. Please set RAJA_DIR to the RAJA prefix.")
 endif()

config/cmake/modules/FindSLEPc.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindSTRUMPACK.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindSUNDIALS.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -22,8 +22,8 @@ mfem_find_package(SUNDIALS SUNDIALS SUNDIALS_DIR
     "include" nvector/nvector_serial.h "lib" sundials_nvecserial
   ADD_COMPONENT NVector_Cuda
     "include" nvector/nvector_cuda.h "lib" sundials_nveccuda
-  ADD_COMPONENT NVector_Hip
-    "include" nvector/nvector_hip.h "lib" sundials_nvechip
+  ADD_COMPONENT NVector_ParHyp
+    "include" nvector/nvector_parhyp.h "lib" sundials_nvecparhyp
   ADD_COMPONENT NVector_Parallel
     "include" nvector/nvector_parallel.h "lib" sundials_nvecparallel
   ADD_COMPONENT NVector_MPIPlusX

config/cmake/modules/FindScotch.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindSuiteSparse.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindSuperLUDist.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindUMPIRE.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -14,23 +14,6 @@
 #   - UMPIRE_LIBRARIES
 #   - UMPIRE_INCLUDE_DIRS
 
-if (NOT umpire_DIR AND UMPIRE_DIR)
-  set(umpire_DIR ${UMPIRE_DIR}/lib/cmake/umpire)
-endif()
-message(STATUS "Looking for UMPIRE ...")
-message(STATUS "   in UMPIRE_DIR = ${UMPIRE_DIR}")
-message(STATUS "      umpire_DIR = ${umpire_DIR}")
-find_package(umpire CONFIG)
-set(UMPIRE_FOUND ${umpire_FOUND})
-set(UMPIRE_LIBRARIES "umpire")
-if (UMPIRE_FOUND)
-  message(STATUS
-    "Found UMPIRE target: ${UMPIRE_LIBRARIES} (version: ${umpire_VERSION})")
-else()
-  set(msg STATUS)
-  if (UMPIRE_FIND_REQUIRED)
-    set(msg FATAL_ERROR)
-  endif()
-  message(${msg}
-    "UMPIRE not found. Please set UMPIRE_DIR to the install prefix.")
-endif()
+include(MfemCmakeUtilities)
+mfem_find_package(UMPIRE UMPIRE UMPIRE_DIR "include" "umpire/Umpire.hpp" "lib" "umpire"
+  "Paths to headers required by UMPIRE." "Libraries required by UMPIRE.")

config/cmake/modules/Find_GnuTLS.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindlibCEED.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -43,18 +43,23 @@ function(convert_filenames_to_full_paths NAMES)
   set(${NAMES} ${tmp_names} PARENT_SCOPE)
 endfunction()
 
-# Wrapper for add_executable
+# Wrapper for add_executable that calls the HIP wrapper if applicable
 macro(mfem_add_executable NAME)
-  add_executable(${NAME} ${ARGN})
-  if (MFEM_USE_CUDA)
-    set_target_properties(${NAME} PROPERTIES
-      CUDA_RESOLVE_DEVICE_SYMBOLS ON)
+  if (MFEM_USE_HIP)
+    hip_add_executable(${NAME} ${ARGN})
+    set_target_properties(${NAME} PROPERTIES LINKER_LANGUAGE CXX)
+  else()
+    add_executable(${NAME} ${ARGN})
   endif()
 endmacro()
 
-# Wrapper for add_library
+# Wrapper for add_library that calls the HIP wrapper if applicable
 macro(mfem_add_library NAME)
-  add_library(${NAME} ${ARGN})
+  if (MFEM_USE_HIP)
+    hip_add_library(${NAME} ${ARGN})
+  else()
+    add_library(${NAME} ${ARGN})
+  endif()
 endmacro()
 
 # Simple shortcut to add_custom_target() with option to add the target to the
@@ -87,14 +92,14 @@ macro(add_mfem_examples EXE_SRCS)
     # If CUDA is enabled, tag source files to be compiled with nvcc.
     if (MFEM_USE_CUDA)
       set_source_files_properties(${SRC_FILE} PROPERTIES LANGUAGE CUDA)
+    elseif(MFEM_USE_HIP)
+      set_source_files_properties(${SRC_FILE} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
     endif()
 
     get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
 
     string(REPLACE ".cpp" "" EXE_NAME "${EXE_PREFIX}${SRC_FILENAME}")
     mfem_add_executable(${EXE_NAME} ${SRC_FILE})
-    install(TARGETS ${EXE_NAME}
-            RUNTIME DESTINATION examples)
     add_dependencies(${MFEM_ALL_EXAMPLES_TARGET_NAME} ${EXE_NAME})
     if (EXE_NEEDED_BY)
       add_dependencies(${EXE_NEEDED_BY} ${EXE_NAME})
@@ -103,6 +108,25 @@ macro(add_mfem_examples EXE_SRCS)
       ${MFEM_EXEC_PREREQUISITES_TARGET_NAME} ${EXE_PREREQUISITE})
 
     target_link_libraries(${EXE_NAME} mfem)
+    if (MFEM_USE_MPI)
+      # Not needed: (mfem already links with MPI_CXX_LIBRARIES)
+      # target_link_libraries(${EXE_NAME} ${MPI_CXX_LIBRARIES})
+
+      # Language-specific include directories:
+      if (MPI_CXX_INCLUDE_PATH)
+        target_include_directories(${EXE_NAME} PRIVATE "${MPI_CXX_INCLUDE_PATH}")
+      endif()
+      if (MPI_CXX_COMPILE_FLAGS)
+        separate_arguments(MPI_CXX_COMPILE_ARGS UNIX_COMMAND
+          "${MPI_CXX_COMPILE_FLAGS}")
+        target_compile_options(${EXE_NAME} PRIVATE ${MPI_CXX_COMPILE_ARGS})
+      endif()
+
+      if (MPI_CXX_LINK_FLAGS)
+        set_target_properties(${EXE_NAME} PROPERTIES
+          LINK_FLAGS "${MPI_CXX_LINK_FLAGS}")
+      endif()
+    endif()
   endforeach(SRC_FILE)
 endmacro()
 
@@ -133,6 +157,8 @@ macro(add_mfem_miniapp MFEM_EXE_NAME)
       endforeach()
       set(EXTRA_OPTIONS_LIST ${LIST_})
     endif()
+  elseif(MFEM_USE_HIP)
+    set_source_files_properties(${MAIN_LIST} ${EXTRA_SOURCES_LIST} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
   endif()
 
   # Actually add the executable
@@ -143,16 +169,53 @@ macro(add_mfem_miniapp MFEM_EXE_NAME)
 
   # Append the additional libraries and options
   if (LIBRARIES_LIST)
-    target_link_libraries(${MFEM_EXE_NAME} PRIVATE ${LIBRARIES_LIST})
+    if(CMAKE_VERSION VERSION_GREATER 2.8.11)
+      target_link_libraries(${MFEM_EXE_NAME} PRIVATE ${LIBRARIES_LIST})
+    else()
+      target_link_libraries(${MFEM_EXE_NAME} ${LIBRARIES_LIST})
+    endif()
   endif()
   if (EXTRA_OPTIONS_LIST)
     string(REPLACE ";" " " EXTRA_OPTIONS_STRING "${EXTRA_OPTIONS_LIST}")
     message(STATUS "${MFEM_EXE_NAME}: add flags \"${EXTRA_OPTIONS_STRING}\"")
-    target_compile_options(${MFEM_EXE_NAME} PRIVATE ${EXTRA_OPTIONS_LIST})
+    if(CMAKE_VERSION VERSION_GREATER 2.8.11)
+      target_compile_options(${MFEM_EXE_NAME} PRIVATE ${EXTRA_OPTIONS_LIST})
+    else()
+      get_target_property(THIS_COMPILE_FLAGS ${MFEM_EXE_NAME} COMPILE_FLAGS)
+      if (THIS_COMPILE_FLAGS)
+        set(THIS_COMPILE_FLAGS "${THIS_COMPILE_FLAGS} ${EXTRA_OPTIONS_STRING}")
+      else()
+        set(THIS_COMPILE_FLAGS "${EXTRA_OPTIONS_STRING}")
+      endif()
+      set_target_properties(${MFEM_EXE_NAME}
+        PROPERTIES COMPILE_FLAGS ${THIS_COMPILE_FLAGS})
+    endif()
   endif()
   if (EXTRA_DEFINES_LIST)
     target_compile_definitions(${MFEM_EXE_NAME} PRIVATE ${EXTRA_DEFINES_LIST})
   endif()
+
+  # Handle the MPI separately
+  if (MFEM_USE_MPI)
+    # Add MPI_CXX_LIBRARIES, in case this target does not link with mfem.
+    if(CMAKE_VERSION VERSION_GREATER 2.8.11)
+      target_link_libraries(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_LIBRARIES})
+    else()
+      target_link_libraries(${MFEM_EXE_NAME} ${MPI_CXX_LIBRARIES})
+    endif()
+
+    if (MPI_CXX_INCLUDE_PATH)
+      target_include_directories(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_INCLUDE_PATH})
+    endif()
+    if (MPI_CXX_COMPILE_FLAGS)
+      target_compile_options(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_COMPILE_FLAGS})
+    endif()
+
+    if (MPI_CXX_LINK_FLAGS)
+      set_target_properties(${MFEM_EXE_NAME} PROPERTIES
+        LINK_FLAGS "${MPI_CXX_LINK_FLAGS}")
+    endif()
+  endif()
 endmacro()
 
 
@@ -397,9 +460,9 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
               break()
             endif()
           endforeach()
-          if (Required AND NOT ${ReqPack}_FOUND)
+          if (${Required} AND NOT ${ReqPack}_FOUND)
             message(FATAL_ERROR " *** Required package ${ReqPack} not found."
-              " Checked target names: ${ReqPack} ${${ReqPack}_TARGET_NAMES}")
+              "Checked target names: ${ReqPack} ${${ReqPack}_TARGET_NAMES}")
           endif()
         endif()
       endif()
@@ -465,23 +528,16 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
           endif()
           get_target_property(IsImported ${TargetName} IMPORTED)
           if (IsImported)
-            get_target_property(ImpConfigs ${TargetName} IMPORTED_CONFIGURATIONS)
-            # message(STATUS "${ReqPack} imported configs: ${ImpConfigs}")
             set(ImportConfig ${${ReqPack}_IMPORT_CONFIG})
             if (NOT ImportConfig)
               set(ImportConfig RELEASE)
-              list(FIND ImpConfigs ${ImportConfig} _Index)
-              if (_Index EQUAL -1)
-                list(GET ImpConfigs 0 ImportConfig)
-              endif()
             endif()
-            set(ImportConfigSuffix "_${ImportConfig}")
-            if ((NOT ImportConfig) OR ("${ImportConfig}" STREQUAL "NO_CONFIG"))
-              set(ImportConfig "NO_CONFIG")
-              set(ImportConfigSuffix "")
-              # message(FATAL_ERROR " *** ${ReqPack}: configuration "
-              #   "${ImportConfig} not found. Set ${ReqPack}_IMPORT_CONFIG "
-              #   "from the list: ${ImpConfigs}.")
+            get_target_property(ImpConfigs ${TargetName} IMPORTED_CONFIGURATIONS)
+            list(FIND ImpConfigs ${ImportConfig} _Index)
+            if (_Index EQUAL -1)
+              message(FATAL_ERROR " *** ${ReqPack}: configuration "
+                "${ImportConfig} not found. Set ${ReqPack}_IMPORT_CONFIG "
+                "from the list: ${ImpConfigs}.")
             endif()
           endif()
           # Set _Pack_LIBS
@@ -493,9 +549,8 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
             endif()
           else()
             # Set _Pack_LIBS from the target properties for ImportConfig
-            foreach (_prop IMPORTED_LOCATION${ImportConfigSuffix}
-                IMPORTED_LINK_INTERFACE_LIBRARIES${ImportConfigSuffix}
-                INTERFACE_LINK_LIBRARIES)
+            foreach (_prop IMPORTED_LOCATION_${ImportConfig}
+                IMPORTED_LINK_INTERFACE_LIBRARIES_${ImportConfig})
               get_target_property(_value ${TargetName} ${_prop})
               if (_value)
                 list(APPEND _Pack_LIBS ${_value})
@@ -507,7 +562,7 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
             endif()
           endif()
           # Set _Pack_INCS
-          foreach (_prop INCLUDE_DIRECTORIES INTERFACE_INCLUDE_DIRECTORIES)
+          foreach (_prop INCLUDE_DIRECTORIES)
             get_target_property(_value ${TargetName} ${_prop})
             if (_value)
               list(APPEND _Pack_INCS ${_value})
@@ -685,146 +740,6 @@ function(mfem_find_library Name Prefix Lib LibDoc CheckVar CheckSrc)
 endfunction(mfem_find_library)
 
 
-#
-# Extract compile and link options needed by the given target.
-#
-function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
-
-  if (NOT TARGET ${Target})
-    return()
-  endif()
-
-  # CMAKE_SHARED_LIBRARY_RUNTIME_C_FLAG -> '-Wl,-rpath,'
-  set(shared_link_flag ${CMAKE_SHARED_LIBRARY_RUNTIME_C_FLAG})
-  if (NOT shared_link_flag)
-    set(shared_link_flag "-Wl,-rpath,")
-  endif()
-
-  set(tgt "${Target}")
-  unset(CompileOpts)
-  unset(LinkOpts)
-  get_target_property(IsImported ${tgt} IMPORTED)
-  # message(STATUS "${tgt}[IMPORTED]: ${IsImported}")
-  # Generally, the possible target types are: STATIC_LIBRARY, MODULE_LIBRARY,
-  # SHARED_LIBRARY, INTERFACE_LIBRARY, EXECUTABLE.
-  get_target_property(type ${tgt} TYPE)
-  # message(STATUS "${tgt}[TYPE]: ${type}")
-  unset(ImportConfig)
-  get_target_property(ImportConfigs ${tgt} IMPORTED_CONFIGURATIONS)
-  if (ImportConfigs)
-    list(GET ImportConfigs 0 ImportConfig)
-  endif()
-  if (NOT ImportConfig)
-    set(ImportConfig RELEASE)
-  endif()
-  # message(STATUS "${tgt}[ImportConfig]: ${ImportConfig}")
-  # List all properties with: cmake --help-property-list
-  get_target_property(Defs ${tgt} INTERFACE_COMPILE_DEFINITIONS)
-  if (Defs)
-    list(REMOVE_DUPLICATES Defs)
-    foreach(Def ${Defs})
-      list(APPEND CompileOpts "-D${Def}")
-    endforeach()
-  endif()
-  get_target_property(Opts ${tgt} INTERFACE_COMPILE_OPTIONS)
-  if (Opts)
-    foreach(Opt ${Opts})
-      list(APPEND CompileOpts "${Opt}")
-    endforeach()
-  endif()
-  get_target_property(Dirs ${tgt} INTERFACE_INCLUDE_DIRECTORIES)
-  if (Dirs)
-    list(REMOVE_DUPLICATES Dirs)
-    foreach(Dir ${Dirs})
-      list(APPEND CompileOpts "-I\"${Dir}\"")
-    endforeach()
-  endif()
-  get_target_property(SysDirs ${tgt} INTERFACE_SYSTEM_INCLUDE_DIRECTORIES)
-  if (SysDirs)
-    list(REMOVE_DUPLICATES SysDirs)
-    foreach(SysDir ${SysDirs})
-      list(APPEND CompileOpts "-isystem \"${SysDir}\"")
-    endforeach()
-  endif()
-  set(AddInterfaceLinkLibs TRUE)
-  if ("${type}" STREQUAL "STATIC_LIBRARY")
-    get_target_property(Location ${tgt} LOCATION)
-    if (Location)
-      list(APPEND LinkOpts "\"${Location}\"")
-    else()
-      message(STATUS " *** Warning: [${tgt}] LOCATION not defined!")
-    endif()
-  elseif ("${type}" STREQUAL "SHARED_LIBRARY")
-    get_target_property(Location ${tgt} LOCATION)
-    if (Location)
-      get_filename_component(Dir ${Location} DIRECTORY)
-      get_filename_component(NameWE ${Location} NAME_WE)
-      string(REGEX REPLACE "^lib" "" LibName ${NameWE})
-      list(APPEND LinkOpts
-        "-L\"${Dir}\""
-        "${shared_link_flag}\"${Dir}\""
-        "-l${LibName}")
-    else()
-      message(STATUS " *** Warning: [${tgt}] LOCATION not defined!")
-    endif()
-  elseif ("${type}" STREQUAL "INTERFACE_LIBRARY")
-    # The INTERFACE_LINK_LIBRARIES property is handled below.
-    # Other properties we may need to handle:
-    # INTERFACE_LINK_DEPENDS
-    # INTERFACE_LINK_DIRECTORIES
-    # INTERFACE_LINK_OPTIONS
-  else()
-    message(STATUS " *** Warning: [${tgt}] uses target type '${type}'"
-      " which is not supported!")
-    set(AddInterfaceLinkLibs FALSE)
-  endif()
-  if (AddInterfaceLinkLibs)
-    get_target_property(Libs ${tgt} INTERFACE_LINK_LIBRARIES)
-    # message(STATUS "${tgt}[INTERFACE_LINK_LIBRARIES]: ${Libs}")
-    if (Libs)
-      foreach(Lib ${Libs})
-        if (NOT (TARGET ${Lib}))
-          # message(STATUS "Lib = ${Lib}")
-          # Filter-out generator expressions
-          if (NOT ("${Lib}" MATCHES "^\\$"))
-            list(APPEND LinkOpts "${Lib}")
-          endif()
-        else()
-          mfem_get_target_options(${Lib} COpts LOpts)
-          list(APPEND CompileOpts ${COpts})
-          list(APPEND LinkOpts ${LOpts})
-        endif()
-      endforeach()
-    endif()
-  endif()
-
-  # Other potentially relevant properties:
-  # - For all target types:
-  # IMPORTED_LIBNAME
-  # IMPORTED_LIBNAME_${ImportConfig}
-  # INTERFACE_AUTOUIC_OPTIONS
-  # INTERFACE_COMPILE_FEATURES
-  # INTERFACE_POSITION_INDEPENDENT_CODE
-  # INTERFACE_SOURCES
-  # INTERFACE_SYSTEM_INCLUDE_DIRECTORIES)
-  # - For non-"INTERFACE_LIBRARY" target types only:
-  # IMPORTED_LOCATION
-  # IMPORTED_LOCATION_${ImportConfig}
-  # IMPORTED_LINK_INTERFACE_LIBRARIES
-  # IMPORTED_LINK_INTERFACE_LIBRARIES_${ImportConfig}
-  # LINK_FLAGS
-  # LINK_FLAGS_${ImportConfig}
-  # LOCATION_${ImportConfig})
-
-  set(${CompileOptsVar} "${CompileOpts}" PARENT_SCOPE)
-  set(${LinkOptsVar} "${LinkOpts}" PARENT_SCOPE)
-
-  # message(STATUS "${tgt}[CompileOpts]: ${CompileOpts}")
-  # message(STATUS "${tgt}[LinkOpts]: ${LinkOpts}")
-
-endfunction(mfem_get_target_options)
-
-
 #
 #   Function that creates 'config.mk' from 'config.mk.in' for the both the
 #   build- and the install-locations and define install rules for 'config.mk'
@@ -843,16 +758,13 @@ function(mfem_export_mk_files)
   # Convert Boolean vars to YES/NO without writing the values to cache
   set(CONFIG_MK_BOOL_VARS MFEM_USE_MPI MFEM_USE_METIS MFEM_USE_METIS_5
       MFEM_DEBUG MFEM_USE_EXCEPTIONS MFEM_USE_ZLIB MFEM_USE_LIBUNWIND
-      MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_LEGACY_OPENMP MFEM_USE_OPENMP
-      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_SUITESPARSE
-      MFEM_USE_SUPERLU MFEM_USE_SUPERLU5 MFEM_USE_MUMPS MFEM_USE_STRUMPACK
-      MFEM_USE_GINKGO MFEM_USE_AMGX MFEM_USE_GNUTLS MFEM_USE_NETCDF
-      MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_FMS
-      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA
-      MFEM_USE_HIP MFEM_USE_RAJA MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_CALIPER
-      MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO
-      MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG
-      MFEM_USE_MOONOLITH MFEM_USE_ALGOIM MFEM_USE_ENZYME)
+      MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP MFEM_USE_LEGACY_OPENMP
+      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
+      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX
+      MFEM_USE_GNUTLS MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC
+      MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_CONDUIT MFEM_USE_PUMI
+      MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA MFEM_USE_UMPIRE MFEM_USE_SIMD
+      MFEM_USE_ADIOS2)
   foreach(var ${CONFIG_MK_BOOL_VARS})
     if (${var})
       set(${var} YES)
@@ -873,7 +785,7 @@ function(mfem_export_mk_files)
          "${cxx_std_flag} ${CMAKE_CXX_FLAGS_${BUILD_TYPE}} ${CMAKE_CXX_FLAGS}"
          MFEM_CXXFLAGS)
   set(MFEM_TPLFLAGS "")
-  foreach(dir ${TPL_INCLUDE_DIRS})
+  foreach(dir ${MFEM_TPL_INCLUDE_DIRS})
     set(MFEM_TPLFLAGS "${MFEM_TPLFLAGS} -I${dir}")
   endforeach()
   # TODO: MFEM_TPLFLAGS: add other TPL flags, in addition to the -I flags.
@@ -936,11 +848,9 @@ function(mfem_export_mk_files)
       string(REGEX REPLACE "^SCOREC::" "" libname ${pumilib})
       string(FIND "${pumilib}" ".a" staticlib)
       string(FIND "${pumilib}" ".so" sharedlib)
-      string(FIND "${pumilib}" ".dylib" dynamiclib)
       find_library(lib ${libname} PATHS ${PUMI_DIR}/lib NO_DEFAULT_PATH)
       if (NOT "${sharedlib}" MATCHES "-1" OR
-          NOT "${staticlib}" MATCHES "-1" OR
-          NOT "${dynamiclib}" MATCHES "-1"  )
+          NOT "${staticlib}" MATCHES "-1"   )
         set(MFEM_EXT_LIBS "${pumilib} ${MFEM_EXT_LIBS}")
       elseif (NOT "${lib}" MATCHES "lib-NOTFOUND")
         set(MFEM_EXT_LIBS "${lib} ${MFEM_EXT_LIBS}")
@@ -955,19 +865,9 @@ function(mfem_export_mk_files)
   foreach(lib ${TPL_LIBRARIES})
     get_filename_component(suffix ${lib} EXT)
     # handle interfaces (e.g., SCOREC::apf)
-    if ("${lib}" MATCHES "SCOREC::.*" OR "${lib}" MATCHES "Ginkgo::.*" OR "${lib}" MATCHES "ParMoonolith::.*")
-    elseif (TARGET "${lib}")
-      mfem_get_target_options(${lib} CompileOpts LinkOpts)
-      # Removing duplicates may lead to issues:
-      # list(REMOVE_DUPLICATES CompileOpts)
-      # list(REMOVE_DUPLICATES LinkOpts)
-      string(REPLACE ";" " " COpts "${CompileOpts}")
-      string(REPLACE ";" " " LOpts "${LinkOpts}")
-      # message(STATUS "${lib}[COpts]: '${COpts}'")
-      # message(STATUS "${lib}[LOpts]: '${LOpts}'")
-      set(MFEM_TPLFLAGS "${MFEM_TPLFLAGS} ${COpts}")
-      set(MFEM_EXT_LIBS "${MFEM_EXT_LIBS} ${LOpts}")
-      # message(FATAL_ERROR "***** interface lib found ... exiting *****")
+    if ("${lib}" MATCHES "SCOREC::.*" OR "${lib}" MATCHES "Ginkgo::.*")
+    elseif (NOT "${lib}" MATCHES "SCOREC::.*" AND "${lib}" MATCHES ".*::.*")
+      message(FATAL_ERROR "***** interface lib found ... exiting *****")
       # handle static and shared libs
     elseif ("${suffix}" STREQUAL "${CMAKE_SHARED_LIBRARY_SUFFIX}")
       get_filename_component(dir ${lib} DIRECTORY)

config/config.hpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -30,31 +30,10 @@
 #endif
 
 // Windows specific options
-#if defined(_WIN32) && !defined(_USE_MATH_DEFINES)
+#ifdef _WIN32
 // Macro needed to get defines like M_PI from <cmath>. (Visual Studio C++ only?)
 #define _USE_MATH_DEFINES
 #endif
-// Macro MFEM_EXPORT: this macro is used when declaring exported global
-// variables and static class variables in public header files, e.g.:
-//    extern MFEM_EXPORT Geometry Geometries;
-//    static MFEM_EXPORT Device device_singleton;
-// In cases where a class contains multiple static variables, instead of marking
-// all such variables with MFEM_EXPORT, one can mark the class with MFEM_EXPORT,
-// e.g.:
-//    class MFEM_EXPORT MemoryManager ...
-// Note: MFEM's GitHub CI includes a shared MSVC build that will fail if a
-// variable that needs MFEM_EXPORT does not have it. However, builds with
-// optional external libraries are not tested and may require separate checks to
-// determine the necessity of MFEM_EXPORT.
-#if defined(_MSC_VER) && defined(MFEM_SHARED_BUILD)
-#ifdef mfem_EXPORTS
-#define MFEM_EXPORT __declspec(dllexport)
-#else
-#define MFEM_EXPORT __declspec(dllimport)
-#endif
-#else
-#define MFEM_EXPORT
-#endif
 // On Cygwin the option -std=c++11 prevents the definition of M_PI. Defining
 // the following macro allows us to get M_PI and some needed functions, e.g.
 // posix_memalign(), strdup(), strerror_r().

config/config.hpp.in

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -30,18 +30,15 @@
 #define MFEM_VERSION_MINOR (((MFEM_VERSION)/100)%100)
 #define MFEM_VERSION_PATCH ((MFEM_VERSION)%100)
 
-// The absolute path of the MFEM source prefix.
+// The absolute path of the MFEM source prefix
 // #define MFEM_SOURCE_DIR "@MFEM_SOURCE_DIR@"
 
-// The absolute path of the MFEM installation prefix.
+// The absolute path of the MFEM installation prefix
 // #define MFEM_INSTALL_DIR "@MFEM_INSTALL_DIR@"
 
 // Description of the git commit used to build MFEM.
 // #define MFEM_GIT_STRING "@MFEM_GIT_STRING@"
 
-// Enable shared library build of MFEM.
-// #define MFEM_SHARED_BUILD
-
 // Build the parallel MFEM library.
 // Requires an MPI compiler, and the libraries HYPRE and METIS.
 // #define MFEM_USE_MPI
@@ -88,10 +85,13 @@
 // Enable MFEM functionality based on the SUNDIALS libraries.
 // #define MFEM_USE_SUNDIALS
 
+// Enable MFEM functionality based on the Mesquite library.
+// #define MFEM_USE_MESQUITE
+
 // Enable MFEM functionality based on the SuiteSparse library.
 // #define MFEM_USE_SUITESPARSE
 
-// Enable MFEM functionality based on the SuperLU_DIST library.
+// Enable MFEM functionality based on the SuperLU library.
 // #define MFEM_USE_SUPERLU
 // #define MFEM_USE_SUPERLU5
 
@@ -102,49 +102,46 @@
 // Enable MFEM functionality based on the STRUMPACK library.
 // #define MFEM_USE_STRUMPACK
 
-// Enable MFEM features based on the Ginkgo library.
+// Enable MFEM features based on the Ginkgo library
 // #define MFEM_USE_GINKGO
 
 // Enable MFEM functionality based on the AmgX library.
 // #define MFEM_USE_AMGX
 
-// Enable secure socket streams based on the GNUTLS library.
+// Enable secure socket streams based on the GNUTLS library
 // #define MFEM_USE_GNUTLS
 
-// Enable Sidre support.
+// Enable Sidre support
 // #define MFEM_USE_SIDRE
 
-// Enable the use of SIMD in the high performance templated classes.
+// Enable the use of SIMD in the high performance templated classes
 // #define MFEM_USE_SIMD
 
-// Enable FMS support.
+// Enable FMS support
 // #define MFEM_USE_FMS
 
-// Enable Conduit support.
+// Enable Conduit support
 // #define MFEM_USE_CONDUIT
 
-// Enable functionality based on the NetCDF library (reading CUBIT files).
+// Enable functionality based on the NetCDF library (reading CUBIT files)
 // #define MFEM_USE_NETCDF
 
-// Enable functionality based on the PETSc library.
+// Enable functionality based on the PETSc library
 // #define MFEM_USE_PETSC
 
-// Enable functionality based on the SLEPc library.
+// Enable functionality based on the SLEPc library
 // #define MFEM_USE_SLEPC
 
 // Enable functionality based on the MPFR library.
 // #define MFEM_USE_MPFR
 
-// Enable MFEM functionality based on the PUMI library.
+// Enable MFEM functionality based on the PUMI library
 // #define MFEM_USE_PUMI
 
-// Enable Moonolith-based general interpolation between finite element spaces.
-// #define MFEM_USE_MOONOLITH
-
 // Enable MFEM functionality based on the HIOP library.
 // #define MFEM_USE_HIOP
 
-// Enable MFEM functionality based on the GSLIB library.
+// Enable MFEM functionality based on the GSLIB library
 // #define MFEM_USE_GSLIB
 
 // Build the NVIDIA GPU/CUDA-enabled version of the MFEM library.
@@ -167,9 +164,6 @@
 // Enable functionality based on the Caliper library.
 // #define MFEM_USE_CALIPER
 
-// Enable functionality based on the Algoim library.
-// #define MFEM_USE_ALGOIM
-
 // Enable functionality based on the Umpire library.
 // #define MFEM_USE_UMPIRE
 
@@ -186,19 +180,4 @@
 // Enable interface to the MKL CPardiso library.
 // #define MFEM_USE_MKL_CPARDISO
 
-// Enable interface to the MKL Pardiso library.
-// #define MFEM_USE_MKL_PARDISO
-
-// Use forward mode for automatic differentiation.
-// #define MFEM_USE_ADFORWARD
-
-// Enable the use of the CoDiPack library for AD.
-// #define MFEM_USE_CODIPACK
-
-// Enable functionality based on the Google Benchmark library.
-// #define MFEM_USE_BENCHMARK
-
-// Enable the Enzyme LLVM plugin
-// #define MFEM_USE_ENZYME
-
 #endif // MFEM_CONFIG_HEADER

config/config.mk.in

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -29,6 +29,7 @@ MFEM_USE_OPENMP        = @MFEM_USE_OPENMP@
 MFEM_USE_MEMALLOC      = @MFEM_USE_MEMALLOC@
 MFEM_TIMER_TYPE        = @MFEM_TIMER_TYPE@
 MFEM_USE_SUNDIALS      = @MFEM_USE_SUNDIALS@
+MFEM_USE_MESQUITE      = @MFEM_USE_MESQUITE@
 MFEM_USE_SUITESPARSE   = @MFEM_USE_SUITESPARSE@
 MFEM_USE_SUPERLU       = @MFEM_USE_SUPERLU@
 MFEM_USE_SUPERLU5      = @MFEM_USE_SUPERLU5@
@@ -57,13 +58,6 @@ MFEM_USE_UMPIRE        = @MFEM_USE_UMPIRE@
 MFEM_USE_SIMD          = @MFEM_USE_SIMD@
 MFEM_USE_ADIOS2        = @MFEM_USE_ADIOS2@
 MFEM_USE_MKL_CPARDISO  = @MFEM_USE_MKL_CPARDISO@
-MFEM_USE_MKL_PARDISO   = @MFEM_USE_MKL_PARDISO@
-MFEM_USE_MOONOLITH     = @MFEM_USE_MOONOLITH@
-MFEM_USE_ADFORWARD     = @MFEM_USE_ADFORWARD@
-MFEM_USE_CODIPACK      = @MFEM_USE_CODIPACK@
-MFEM_USE_BENCHMARK     = @MFEM_USE_BENCHMARK@
-MFEM_USE_PARELAG       = @MFEM_USE_PARELAG@
-MFEM_USE_ENZYME        = @MFEM_USE_ENZYME@
 
 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@
@@ -93,7 +87,7 @@ MFEM_MPIEXEC_NP = @MFEM_MPIEXEC_NP@
 MFEM_MPI_NP     = @MFEM_MPI_NP@
 
 # The NVCC compiler cannot link with -x=cu
-MFEM_LINK_FLAGS := $(filter-out -x=cu -xhip, $(MFEM_FLAGS))
+MFEM_LINK_FLAGS := $(filter-out -x=cu, $(MFEM_FLAGS))
 
 # Optional extra configuration
 @MFEM_CONFIG_EXTRA@

config/defaults.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -19,7 +19,6 @@ if (NOT CMAKE_BUILD_TYPE)
 endif()
 
 # MFEM options. Set to mimic the default "defaults.mk" file.
-option(BUILD_SHARED_LIBS "Enable shared library build of MFEM" OFF)
 option(MFEM_USE_MPI "Enable MPI parallel build" OFF)
 option(MFEM_USE_METIS "Enable METIS usage" ${MFEM_USE_MPI})
 option(MFEM_USE_EXCEPTIONS "Enable the use of exceptions" OFF)
@@ -31,6 +30,7 @@ option(MFEM_USE_OPENMP "Enable the OpenMP backend" OFF)
 option(MFEM_USE_LEGACY_OPENMP "Enable legacy OpenMP usage" OFF)
 option(MFEM_USE_MEMALLOC "Enable the internal MEMALLOC option." ON)
 option(MFEM_USE_SUNDIALS "Enable SUNDIALS usage" OFF)
+option(MFEM_USE_MESQUITE "Enable MESQUITE usage" OFF)
 option(MFEM_USE_SUITESPARSE "Enable SuiteSparse usage" OFF)
 option(MFEM_USE_SUPERLU "Enable SuperLU_DIST usage" OFF)
 option(MFEM_USE_SUPERLU5 "Use the old SuperLU_DIST 5.1 version" OFF)
@@ -50,7 +50,6 @@ option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
 option(MFEM_USE_PUMI "Enable PUMI" OFF)
 option(MFEM_USE_HIOP "Enable HiOp" OFF)
 option(MFEM_USE_CUDA "Enable CUDA" OFF)
-option(MFEM_USE_HIP "Enable HIP" OFF)
 option(MFEM_USE_OCCA "Enable OCCA" OFF)
 option(MFEM_USE_RAJA "Enable RAJA" OFF)
 option(MFEM_USE_CEED "Enable CEED" OFF)
@@ -58,14 +57,7 @@ option(MFEM_USE_UMPIRE "Enable Umpire" OFF)
 option(MFEM_USE_SIMD "Enable use of SIMD intrinsics" OFF)
 option(MFEM_USE_ADIOS2 "Enable ADIOS2" OFF)
 option(MFEM_USE_CALIPER "Enable Caliper support" OFF)
-option(MFEM_USE_ALGOIM "Enable Algoim support" OFF)
 option(MFEM_USE_MKL_CPARDISO "Enable MKL CPardiso" OFF)
-option(MFEM_USE_MKL_PARDISO "Enable MKL Pardiso" OFF)
-option(MFEM_USE_ADFORWARD "Enable forward mode for AD" OFF)
-option(MFEM_USE_CODIPACK "Enable automatic differentiation (AD) using CoDiPack" OFF)
-option(MFEM_USE_BENCHMARK "Enable Google Benchmark" OFF)
-option(MFEM_USE_PARELAG "Enable ParELAG" OFF)
-option(MFEM_USE_ENZYME "Enable Enzyme" OFF)
 
 # Optional overrides for autodetected MPIEXEC and MPIEXEC_NUMPROC_FLAG
 # set(MFEM_MPIEXEC "mpirun" CACHE STRING "Command for running MPI tests")
@@ -82,7 +74,6 @@ set(MFEM_MPI_NP 4 CACHE STRING "Number of processes used for MPI tests")
 option(MFEM_ENABLE_TESTING "Enable the ctest framework for testing" ON)
 option(MFEM_ENABLE_EXAMPLES "Build all of the examples" OFF)
 option(MFEM_ENABLE_MINIAPPS "Build all of the miniapps" OFF)
-option(MFEM_ENABLE_GOOGLE_BENCHMARKS "Build all of the Google benchmarks" OFF)
 
 # Setting CXX/MPICXX on the command line or in user.cmake will overwrite the
 # autodetected C++ compiler.
@@ -106,7 +97,11 @@ set(HYPRE_DIR "${MFEM_DIR}/../hypre/src/hypre" CACHE PATH
 # If hypre was compiled to depend on BLAS and LAPACK:
 # set(HYPRE_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
 #     "Packages that HYPRE depends on.")
-# CUDA and HIP dependencies for HYPRE are handled in FindHYPRE.cmake.
+if (MFEM_USE_CUDA)
+   # This is only necessary when hypre is built with cuda:
+   set(HYPRE_REQUIRED_LIBRARIES "-lcusparse" "-lcurand" CACHE STRING
+       "Libraries that HYPRE depends on.")
+endif()
 
 set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library.")
 
@@ -120,6 +115,9 @@ set(SUNDIALS_DIR "${MFEM_DIR}/../sundials-5.0.0/instdir" CACHE PATH
 # set(SUNDIALS_REQUIRED_PACKAGES "SuiteSparse/KLU/AMD/BTF/COLAMD/config"
 #     CACHE STRING "Additional packages required by SUNDIALS.")
 
+set(MESQUITE_DIR "${MFEM_DIR}/../mesquite-2.99" CACHE PATH
+    "Path to the Mesquite library.")
+
 set(SuiteSparse_DIR "${MFEM_DIR}/../SuiteSparse" CACHE PATH
     "Path to the SuiteSparse library.")
 set(SuiteSparse_REQUIRED_PACKAGES "BLAS" "METIS"
@@ -130,18 +128,16 @@ set(ParMETIS_DIR "${MFEM_DIR}/../parmetis-4.0.3" CACHE PATH
 set(ParMETIS_REQUIRED_PACKAGES "METIS" CACHE STRING
     "Additional packages required by ParMETIS.")
 
-set(SuperLUDist_DIR "${MFEM_DIR}/../SuperLU_DIST_8.1.2" CACHE PATH
+set(SuperLUDist_DIR "${MFEM_DIR}/../SuperLU_DIST_6.3.1" CACHE PATH
     "Path to the SuperLU_DIST library.")
 # SuperLU_DIST may also depend on "OpenMP", depending on how it was compiled.
-set(SuperLUDist_REQUIRED_PACKAGES "MPI" "ParMETIS" "METIS"
-    "LAPACK" "BLAS" CACHE STRING
+set(SuperLUDist_REQUIRED_PACKAGES "MPI" "BLAS" "ParMETIS" CACHE STRING
     "Additional packages required by SuperLU_DIST.")
 
-set(MUMPS_DIR "${MFEM_DIR}/../MUMPS_5.5.0" CACHE PATH
+set(MUMPS_DIR "${MFEM_DIR}/../MUMPS_5.2.0" CACHE PATH
     "Path to the MUMPS library.")
-# MUMPS may also depend on "OpenMP", depending on how it was compiled.
-set(MUMPS_REQUIRED_PACKAGES "MPI" "MPI_Fortran" "ParMETIS" "METIS"
-    "ScaLAPACK" "LAPACK" "BLAS" CACHE STRING
+# Packages required by MUMPS, depending on how it was compiled.
+set(MUMPS_REQUIRED_PACKAGES "MPI" "BLAS" "METIS" "ScaLAPACK" CACHE STRING
     "Additional packages required by MUMPS.")
 # If the MPI package does not find all required Fortran libraries:
 # set(MUMPS_REQUIRED_LIBRARIES "gfortran" "mpi_mpifh" CACHE STRING
@@ -152,8 +148,7 @@ set(STRUMPACK_DIR "${MFEM_DIR}/../STRUMPACK-build" CACHE PATH
 # STRUMPACK may also depend on "OpenMP", depending on how it was compiled.
 # Starting with v2.2.0 of STRUMPACK, ParMETIS and Scotch are optional.
 set(STRUMPACK_REQUIRED_PACKAGES "MPI" "MPI_Fortran" "ParMETIS" "METIS"
-    "Scotch/ptscotch/ptscotcherr/scotch/scotcherr"
-    "ScaLAPACK" "LAPACK" "BLAS" CACHE STRING
+    "ScaLAPACK" "Scotch/ptscotch/ptscotcherr/scotch/scotcherr" CACHE STRING
     "Additional packages required by STRUMPACK.")
 # If the MPI package does not find all required Fortran libraries:
 # set(STRUMPACK_REQUIRED_LIBRARIES "gfortran" "mpi_mpifh" CACHE STRING
@@ -184,7 +179,6 @@ set(GNUTLS_DIR "" CACHE PATH "Path to the GnuTLS library.")
 
 set(GSLIB_DIR "" CACHE PATH "Path to the GSLIB library.")
 
-set(HDF5_DIR "/usr" CACHE PATH "Path to the HDF5 library.")
 set(NETCDF_DIR "" CACHE PATH "Path to the NetCDF library.")
 set(NetCDF_REQUIRED_PACKAGES "HDF5/C/HL" CACHE STRING
     "Additional packages required by NetCDF.")
@@ -225,38 +219,17 @@ set(MKL_CPARDISO_DIR "" CACHE STRING "MKL installation path.")
 set(MKL_MPI_WRAPPER_LIB "mkl_blacs_mpich_lp64" CACHE STRING "MKL MPI wrapper library")
 set(MKL_LIBRARY_DIR "" CACHE STRING "Custom library subdirectory")
 
-set(MKL_PARDISO_DIR "" CACHE STRING "MKL installation path.")
-
 set(OCCA_DIR "${MFEM_DIR}/../occa" CACHE PATH "Path to OCCA")
 set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")
 set(UMPIRE_DIR "${MFEM_DIR}/../umpire" CACHE PATH "Path to Umpire")
 set(CALIPER_DIR "${MFEM_DIR}/../caliper" CACHE PATH "Path to Caliper")
-set(BLITZ_DIR "${MFEM_DIR}/../blitz" CACHE PATH "Path to Blitz")
-set(ALGOIM_DIR "${MFEM_DIR}/../algoim" CACHE PATH "Path to Algoim")
-set(Algoim_REQUIRED_PACKAGES "Blitz" CACHE STRING
-    "Packages that ALGOIM depends on.")
-
-set(BENCHMARK_DIR "${MFEM_DIR}/../google-benchmark" CACHE PATH
-    "Path to Google Benchmark")
-
-# Provide paths, since ParELAG is dependent on MFEM and MFEM needs to be
-# compiled (or at least cmake needs to succeed) before compiling ParELAG.
-set(PARELAG_DIR "${MFEM_DIR}/../parelag" CACHE PATH "Path to ParELAG")
-set(PARELAG_INCLUDE_DIRS "${PARELAG_DIR}/src;${PARELAG_DIR}/build/src" CACHE
-    STRING "Path to ParELAG headers.")
-set(PARELAG_LIBRARIES "${PARELAG_DIR}/build/src/libParELAG.a" CACHE STRING
-    "The ParELAG library.")
 
 set(BLAS_INCLUDE_DIRS "" CACHE STRING "Path to BLAS headers.")
 set(BLAS_LIBRARIES "" CACHE STRING "The BLAS library.")
 set(LAPACK_INCLUDE_DIRS "" CACHE STRING "Path to LAPACK headers.")
 set(LAPACK_LIBRARIES "" CACHE STRING "The LAPACK library.")
 
-set(CODIPACK_INCLUDE_DIRS "${MFEM_DIR}/../CoDiPack/include" CACHE STRING
-    "Path to CoDiPack headers.")
-set(CODIPACK_LIBRARIES "")
-
 # Some useful variables:
 set(CMAKE_SKIP_PREPROCESSED_SOURCE_RULES ON) # Skip *.i rules
 set(CMAKE_SKIP_ASSEMBLY_SOURCE_RULES  ON)    # Skip *.s rules

config/defaults.mk

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -42,9 +42,6 @@ STATIC = YES
 SHARED = NO
 
 # CUDA configuration options
-#
-# If you set MFEM_USE_ENZYME=YES, CUDA_CXX has to be configured to use cuda with
-# clang as its host compiler.
 CUDA_CXX = nvcc
 CUDA_ARCH = sm_60
 CUDA_FLAGS = -x=cu --expt-extended-lambda -arch=$(CUDA_ARCH)
@@ -58,16 +55,13 @@ HIP_CXX = hipcc
 # example: gfx600 (tahiti), gfx700 (kaveri), gfx701 (hawaii), gfx801 (carrizo),
 # gfx900, gfx1010, etc.
 HIP_ARCH = gfx900
-HIP_FLAGS = --offload-arch=$(HIP_ARCH)
+HIP_FLAGS = --amdgpu-target=$(HIP_ARCH)
 HIP_XCOMPILER =
 HIP_XLINKER   = -Wl,
 
-# Flags for generating dependencies.
-DEP_FLAGS = -MM -MT
-
 ifneq ($(NOTMAC),)
    AR      = ar
-   ARFLAGS = crv
+   ARFLAGS = cruv
    RANLIB  = ranlib
    PICFLAG = $(XCOMPILER)-fPIC
    SO_EXT  = so
@@ -79,7 +73,7 @@ ifneq ($(NOTMAC),)
 else
    # Silence "has no symbols" warnings on Mac OS X
    AR      = ar
-   ARFLAGS = Scrv
+   ARFLAGS = Scruv
    RANLIB  = ranlib -no_warning_for_no_symbols
    PICFLAG = $(XCOMPILER)-fPIC
    SO_EXT  = dylib
@@ -92,9 +86,6 @@ else
    BUILD_RPATH = $(XLINKER)-undefined,dynamic_lookup
    INSTALL_SOFLAGS = $(subst $1 ,,$(call MAKE_SOFLAGS,$(MFEM_LIB_DIR)))
    INSTALL_RPATH = $(XLINKER)-undefined,dynamic_lookup
-   # Silence unused command line argument warnings when generating dependencies
-   # with mpicxx and clang
-   DEP_FLAGS := -Wno-unused-command-line-argument $(DEP_FLAGS)
 endif
 
 # Set CXXFLAGS to overwrite the default selection of DEBUG_FLAGS/OPTIM_FLAGS
@@ -131,6 +122,7 @@ MFEM_USE_LEGACY_OPENMP = NO
 MFEM_USE_MEMALLOC      = YES
 MFEM_TIMER_TYPE        = $(if $(NOTMAC),2,4)
 MFEM_USE_SUNDIALS      = NO
+MFEM_USE_MESQUITE      = NO
 MFEM_USE_SUITESPARSE   = NO
 MFEM_USE_SUPERLU       = NO
 MFEM_USE_SUPERLU5      = NO
@@ -155,18 +147,10 @@ MFEM_USE_RAJA          = NO
 MFEM_USE_OCCA          = NO
 MFEM_USE_CEED          = NO
 MFEM_USE_CALIPER       = NO
-MFEM_USE_ALGOIM        = NO
 MFEM_USE_UMPIRE        = NO
 MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
 MFEM_USE_MKL_CPARDISO  = NO
-MFEM_USE_MKL_PARDISO   = NO
-MFEM_USE_MOONOLITH     = NO
-MFEM_USE_ADFORWARD     = NO
-MFEM_USE_CODIPACK      = NO
-MFEM_USE_BENCHMARK     = NO
-MFEM_USE_PARELAG       = NO
-MFEM_USE_ENZYME        = NO
 
 # MPI library compile and link flags
 # These settings are used only when building MFEM with MPI + HIP
@@ -178,20 +162,6 @@ ifeq ($(MFEM_USE_MPI)$(MFEM_USE_HIP),YESYES)
    MPI_LIB = -L$(MPI_DIR)/lib $(XLINKER)-rpath,$(MPI_DIR)/lib -lmpi
 endif
 
-# ROCM/HIP directory such that ROCM/HIP libraries like rocsparse and rocrand are
-# found in $(HIP_DIR)/lib, usually as links. Typically, this directory is of
-# the form /opt/rocm-X.Y.Z which is called ROCM_PATH by hipconfig.
-ifeq ($(MFEM_USE_HIP),YES)
-   HIP_DIR := $(patsubst %/,%,$(dir $(shell which $(HIP_CXX))))
-   HIP_DIR := $(patsubst %/,%,$(dir $(HIP_DIR)))
-   ifeq (,$(wildcard $(HIP_DIR)/lib/librocsparse.*))
-      HIP_DIR := $(shell hipconfig --rocmpath 2> /dev/null)
-      ifeq (,$(wildcard $(HIP_DIR)/lib/librocsparse.*))
-         $(error Unable to determine HIP_DIR. Please set it manually.)
-      endif
-   endif
-endif
-
 # Compile and link options for zlib.
 ZLIB_DIR =
 ZLIB_OPT = $(if $(ZLIB_DIR),-I$(ZLIB_DIR)/include)
@@ -207,12 +177,7 @@ HYPRE_OPT = -I$(HYPRE_DIR)/include
 HYPRE_LIB = -L$(HYPRE_DIR)/lib -lHYPRE
 ifeq (YES,$(MFEM_USE_CUDA))
    # This is only necessary when hypre is built with cuda:
-   HYPRE_LIB += -lcusparse -lcurand -lcublas
-endif
-ifeq (YES,$(MFEM_USE_HIP))
-   # This is only necessary when hypre is built with hip:
-   HYPRE_LIB += -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib\
- -lrocsparse -lrocrand
+   HYPRE_LIB += -lcusparse -lcurand
 endif
 
 # METIS library configuration
@@ -251,28 +216,26 @@ POSIX_CLOCKS_LIB = -lrt
 # SUNDIALS library configuration
 # For sundials_nvecmpiplusx and nvecparallel remember to build with MPI_ENABLE=ON
 # and modify cmake variables for hypre for sundials
-SUNDIALS_DIR = @MFEM_DIR@/../sundials-5.0.0/instdir
-# SUNDIALS >= 6.4.0 requires C++14:
-ifeq ($(MFEM_USE_SUNDIALS),YES)
-   BASE_FLAGS = -std=c++14
-endif
-SUNDIALS_OPT = -I$(SUNDIALS_DIR)/include
-SUNDIALS_LIB = $(XLINKER)-rpath,$(SUNDIALS_DIR)/lib64\
- $(XLINKER)-rpath,$(SUNDIALS_DIR)/lib\
- -L$(SUNDIALS_DIR)/lib64 -L$(SUNDIALS_DIR)/lib\
+SUNDIALS_DIR    = @MFEM_DIR@/../sundials-5.0.0/instdir
+SUNDIALS_OPT    = -I$(SUNDIALS_DIR)/include
+SUNDIALS_LIBDIR = $(wildcard $(SUNDIALS_DIR)/lib*)
+SUNDIALS_LIB    = $(XLINKER)-rpath,$(SUNDIALS_LIBDIR) -L$(SUNDIALS_LIBDIR)\
  -lsundials_arkode -lsundials_cvodes -lsundials_nvecserial -lsundials_kinsol
+
 ifeq ($(MFEM_USE_MPI),YES)
    SUNDIALS_LIB += -lsundials_nvecparallel -lsundials_nvecmpiplusx
 endif
 ifeq ($(MFEM_USE_CUDA),YES)
    SUNDIALS_LIB += -lsundials_nveccuda
 endif
-ifeq ($(MFEM_USE_HIP),YES)
-   SUNDIALS_LIB += -lsundials_nvechip
-endif
 # If SUNDIALS was built with KLU:
 # MFEM_USE_SUITESPARSE = YES
 
+# MESQUITE library configuration
+MESQUITE_DIR = @MFEM_DIR@/../mesquite-2.99
+MESQUITE_OPT = -I$(MESQUITE_DIR)/include
+MESQUITE_LIB = -L$(MESQUITE_DIR)/lib -lmesquite
+
 # SuiteSparse library configuration
 LIB_RT = $(if $(NOTMAC),-lrt,)
 SUITESPARSE_DIR = @MFEM_DIR@/../SuiteSparse
@@ -288,10 +251,10 @@ ifeq ($(MFEM_USE_SUPERLU5),YES)
    SUPERLU_LIB = $(XLINKER)-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib\
       -lsuperlu_dist_5.1.0
 else
-   SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_8.1.2
+   SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_6.3.1
    SUPERLU_OPT = -I$(SUPERLU_DIR)/include
    SUPERLU_LIB = $(XLINKER)-rpath,$(SUPERLU_DIR)/lib64 -L$(SUPERLU_DIR)/lib64\
-      -lsuperlu_dist $(LAPACK_LIB)
+      -lsuperlu_dist -lblas
 endif
 
 # SCOTCH library configuration (required by STRUMPACK <= v2.1.0, optional in
@@ -311,50 +274,33 @@ SCALAPACK_LIB = -L$(SCALAPACK_DIR)/lib -lscalapack $(LAPACK_LIB)
 MPI_FORTRAN_LIB = -lmpifort
 # OpenMPI:
 # MPI_FORTRAN_LIB = -lmpi_mpifh
-# Additional Fortran library:
+# Additional Fortan library:
 # MPI_FORTRAN_LIB += -lgfortran
 
 # MUMPS library configuration
-MUMPS_DIR = @MFEM_DIR@/../MUMPS_5.5.0
+MUMPS_DIR = @MFEM_DIR@/../MUMPS_5.2.0
 MUMPS_OPT = -I$(MUMPS_DIR)/include
 MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib -ldmumps\
  -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB)
 
 # STRUMPACK library configuration
 STRUMPACK_DIR = @MFEM_DIR@/../STRUMPACK-build
-ifeq ($(MFEM_USE_STRUMPACK),YES)
-   BASE_FLAGS = -std=c++14
-endif
 STRUMPACK_OPT = -I$(STRUMPACK_DIR)/include $(SCOTCH_OPT)
 # If STRUMPACK was build with OpenMP support, the following may be need:
 # STRUMPACK_OPT += $(OPENMP_OPT)
 STRUMPACK_LIB = -L$(STRUMPACK_DIR)/lib -lstrumpack $(MPI_FORTRAN_LIB)\
  $(SCOTCH_LIB) $(SCALAPACK_LIB)
 
-# Ginkgo library configuration
+# Ginkgo library configuration (currently not needed)
 GINKGO_DIR = @MFEM_DIR@/../ginkgo/install
-GINKGO_SEARCH_DIR = $(subst @MFEM_DIR@,$(MFEM_DIR),$(GINKGO_DIR))
 GINKGO_BUILD_TYPE=Release
 ifeq ($(MFEM_USE_GINKGO),YES)
    BASE_FLAGS = -std=c++14
 endif
 GINKGO_OPT = -isystem $(GINKGO_DIR)/include
-GINKGO_LIB_DIR = $(sort $(dir $(wildcard\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.a\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.so\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.dylib\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.dll)))
-GINKGO_LINK_LIB_DIR = $(GINKGO_DIR)$(subst $(GINKGO_SEARCH_DIR),,$(GINKGO_LIB_DIR))
-ALL_GINKGO_LIBS_DEBUG = $(notdir $(basename $(wildcard\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*d.a\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*d.so\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*d.dylib\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*d.dll)))
-ALL_GINKGO_LIBS = $(notdir $(basename $(wildcard\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.a\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.so\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.dylib\
- $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.dll)))
+GINKGO_LIB_DIR = $(sort $(dir $(wildcard $(GINKGO_DIR)/lib*/libginkgo*.a  $(GINKGO_DIR)/lib*/libginkgo*.so $(GINKGO_DIR)/lib*/libginkgo*.dylib $(GINKGO_DIR)/lib*/libginkgo*.dll)))
+ALL_GINKGO_LIBS_DEBUG = $(notdir $(basename $(wildcard $(GINKGO_DIR)/lib*/libginkgo*d.a  $(GINKGO_DIR)/lib*/libginkgo*d.so $(GINKGO_DIR)/lib*/libginkgo*d.dylib $(GINKGO_DIR)/lib*/libginkgo*d.dll)))
+ALL_GINKGO_LIBS = $(notdir $(basename $(wildcard $(GINKGO_DIR)/lib*/libginkgo*.a  $(GINKGO_DIR)/lib*/libginkgo*.so $(GINKGO_DIR)/lib*/libginkgo*.dylib $(GINKGO_DIR)/lib*/libginkgo*.dll)))
 ALL_GINKGO_LIBS_RELEASE = $(filter-out $(ALL_GINKGO_LIBS_DEBUG),$(ALL_GINKGO_LIBS))
 GINKGO_LINK = $(subst libginkgo,-lginkgo,$(ALL_GINKGO_LIBS_RELEASE))
 ifeq ($(GINKGO_BUILD_TYPE),Debug)
@@ -363,8 +309,7 @@ ifeq ($(GINKGO_BUILD_TYPE),Debug)
   endif
 else
 endif
-GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_LINK_LIB_DIR) -L$(GINKGO_LINK_LIB_DIR)\
- $(GINKGO_LINK)
+GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_LIB_DIR) -L$(GINKGO_LIB_DIR) $(GINKGO_LINK)
 
 # AmgX library configuration
 AMGX_DIR = @MFEM_DIR@/../amgx
@@ -413,11 +358,6 @@ ifeq ($(SLEPC_FOUND),YES)
       $(subst $(CXX_XLINKER),$(XLINKER),$(SLEPC_DEP))
 endif
 
-ifeq ($(MFEM_USE_MOONOLITH),YES)
-  include $(MOONOLITH_DIR)/config/moonolith-config.makefile
-  MOONOLITH_LIB=$(MOONOLITH_LIBRARIES)
-endif
-
 # MPFR library configuration
 MPFR_OPT =
 MPFR_LIB = -lmpfr
@@ -466,11 +406,6 @@ HIOP_DIR = @MFEM_DIR@/../hiop/install
 HIOP_OPT = -I$(HIOP_DIR)/include
 HIOP_LIB = -L$(HIOP_DIR)/lib -lhiop $(LAPACK_LIB)
 
-# CoDiPack
-CODIPACK_DIR = @MFEM_DIR@/../CoDiPack
-CODIPACK_OPT = -I$(CODIPACK_DIR)
-CODIPACK_LIB =
-
 # GSLIB library
 GSLIB_DIR = @MFEM_DIR@/../gslib/build
 GSLIB_OPT = -I$(GSLIB_DIR)/include
@@ -480,9 +415,9 @@ GSLIB_LIB = -L$(GSLIB_DIR)/lib -lgs
 CUDA_OPT =
 CUDA_LIB = -lcusparse
 
-# HIP library configuration
+# HIP library configuration (currently not needed)
 HIP_OPT =
-HIP_LIB = -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib -lhipsparse
+HIP_LIB =
 
 # OCCA library configuration
 OCCA_DIR = @MFEM_DIR@/../occa
@@ -492,31 +427,7 @@ OCCA_LIB = $(XLINKER)-rpath,$(OCCA_DIR)/lib -L$(OCCA_DIR)/lib -locca
 # CALIPER library configuration
 CALIPER_DIR = @MFEM_DIR@/../caliper
 CALIPER_OPT = -I$(CALIPER_DIR)/include
-CALIPER_LIB = $(XLINKER)-rpath,$(CALIPER_DIR)/lib64 $(XLINKER)-rpath,$(CALIPER_DIR)/lib -L$(CALIPER_DIR)/lib64 -L$(CALIPER_DIR)/lib -lcaliper
-
-ifdef ADIAK_DIR
-   CALIPER_OPT += -I$(ADIAK_DIR)/include
-   CALIPER_LIB += $(XLINKER)-rpath,$(ADIAK_DIR)/lib64 $(XLINKER)-rpath,$(ADIAK_DIR)/lib -L$(ADIAK_DIR)/lib64 -L$(ADIAK_DIR)/lib -ladiak
-endif
-ifdef GOTCHA_DIR
-   CALIPER_OPT += -I$(GOTCHA_DIR)/include
-   CALIPER_LIB += $(XLINKER)-rpath,$(GOTCHA_DIR)/lib64 $(XLINKER)-rpath,$(GOTCHA_DIR)/lib -L$(GOTCHA_DIR)/lib64 -L$(GOTCHA_DIR)/lib -lgotcha
-endif
-
-# BLITZ library configuration
-BLITZ_DIR = @MFEM_DIR@/../blitz
-BLITZ_OPT = -I$(BLITZ_DIR)/include
-BLITZ_LIB = $(XLINKER)-rpath,$(BLITZ_DIR)/lib -L$(BLITZ_DIR)/lib -lblitz
-
-# ALGOIM library configuration
-ALGOIM_DIR = @MFEM_DIR@/../algoim
-ALGOIM_OPT = -I$(ALGOIM_DIR)/src $(BLITZ_OPT)
-ALGOIM_LIB = $(BLITZ_LIB)
-
-# BENCHMARK library configuration
-BENCHMARK_DIR = @MFEM_DIR@/../google-benchmark
-BENCHMARK_OPT = -I$(BENCHMARK_DIR)/include
-BENCHMARK_LIB = -L$(BENCHMARK_DIR)/lib -lbenchmark -lpthread
+CALIPER_LIB = $(XLINKER)-rpath,$(CALIPER_DIR)/lib64 -L$(CALIPER_DIR)/lib64 -lcaliper
 
 # libCEED library configuration
 CEED_DIR ?= @MFEM_DIR@/../libCEED
@@ -524,29 +435,20 @@ CEED_OPT = -I$(CEED_DIR)/include
 CEED_LIB = $(XLINKER)-rpath,$(CEED_DIR)/lib -L$(CEED_DIR)/lib -lceed
 
 # RAJA library configuration
-ifeq ($(MFEM_USE_RAJA),YES)
-   BASE_FLAGS = -std=c++14
-endif
 RAJA_DIR = @MFEM_DIR@/../raja
 RAJA_OPT = -I$(RAJA_DIR)/include
 ifdef CUB_DIR
    RAJA_OPT += -I$(CUB_DIR)
 endif
-
-CAMP_LIB = -lcamp
 ifdef CAMP_DIR
    RAJA_OPT += -I$(CAMP_DIR)/include
-   CAMP_LIB = $(XLINKER)-rpath,$(CAMP_DIR)/lib -L$(CAMP_DIR)/lib -lcamp
 endif
-RAJA_LIB = $(XLINKER)-rpath,$(RAJA_DIR)/lib -L$(RAJA_DIR)/lib -lRAJA $(CAMP_LIB)
+RAJA_LIB = $(XLINKER)-rpath,$(RAJA_DIR)/lib -L$(RAJA_DIR)/lib -lRAJA
 
 # UMPIRE library configuration
-ifeq ($(MFEM_USE_UMPIRE),YES)
-   BASE_FLAGS = -std=c++14
-endif
 UMPIRE_DIR = @MFEM_DIR@/../umpire
 UMPIRE_OPT = -I$(UMPIRE_DIR)/include $(if $(CAMP_DIR), -I$(CAMP_DIR)/include)
-UMPIRE_LIB = -L$(UMPIRE_DIR)/lib -lumpire $(CAMP_LIB)
+UMPIRE_LIB = -L$(UMPIRE_DIR)/lib -lumpire
 
 # MKL CPardiso library configuration
 MKL_CPARDISO_DIR ?=
@@ -557,35 +459,6 @@ MKL_CPARDISO_LIB = $(XLINKER)-rpath,$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
    -L$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR) -l$(MKL_MPI_WRAPPER)\
    -lmkl_intel_lp64 -lmkl_sequential -lmkl_core
 
-# MKL Pardiso library configuration
-MKL_PARDISO_DIR ?=
-MKL_LIBRARY_SUBDIR ?= lib
-MKL_PARDISO_OPT = -I$(MKL_PARDISO_DIR)/include
-MKL_PARDISO_LIB = $(XLINKER)-rpath,$(MKL_PARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
-   -L$(MKL_PARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
-   -lmkl_intel_lp64 -lmkl_sequential -lmkl_core
-
-# PARELAG library configuration
-PARELAG_DIR = @MFEM_DIR@/../parelag
-PARELAG_OPT = -I$(PARELAG_DIR)/src -I$(PARELAG_DIR)/build/src
-PARELAG_LIB = -L$(PARELAG_DIR)/build/src -lParELAG
-
-# Enzyme configuration
-
-# If you want to enable automatic differentiation at compile time, use the
-# options below, adapted to your configuration. To be more flexible, we
-# recommend using the Enzyme plugin during link time optimization. One option is
-# to add your options to the global compiler/linker flags like
-#
-# BASE_FLAGS += -flto
-# CXX_XLINKER += -fuse-ld=lld -Wl,--lto-legacy-pass-manager\
-#                -Wl,-mllvm=-load=$(ENZYME_DIR)/LLDEnzyme-$(ENZYME_VERSION).so -Wl,
-#
-ENZYME_DIR ?= @MFEM_DIR@/../enzyme
-ENZYME_VERSION ?= 14
-ENZYME_OPT = -fno-experimental-new-pass-manager -Xclang -load -Xclang $(ENZYME_DIR)/ClangEnzyme-$(ENZYME_VERSION).so
-ENZYME_LIB = ""
-
 # If YES, enable some informational messages
 VERBOSE = NO
 

config/docker/Dockerfile

@@ -1,25 +0,0 @@
-FROM ghcr.io/mfem/mfem-ubuntu-base:latest as builder
-
-# docker build -t ghcr.io/mfem/mfem-ubuntu .
-
-COPY ./config/docker/spack.yaml /opt/mfem-env/spack.yaml
-RUN apt-get install -y python3 && \
-    cd /opt/mfem-env && \
-    . /opt/spack/share/spack/setup-env.sh && \
-    spack env activate . && \
-    spack env view regenerate
-
-FROM ubuntu:22.04
-
-COPY --from=builder /opt/view /opt/view
-COPY --from=builder /opt/mfem-view /opt/mfem-view
-
-RUN apt-get update && \
-    apt-get install -y unzip gfortran && \
-    apt-get install -y libcurl4-openssl-dev libssl-dev
-
-ENV PATH=$PATH:/opt/mfem-view/bin
-ENV DEBIAN_FRONTEND=noninteractive
-
-# The user will see the view on shell into the container
-WORKDIR /opt/mfem-view

config/docker/Dockerfile.base

@@ -1,47 +0,0 @@
-FROM ghcr.io/rse-ops/cuda-ubuntu-20.04:cuda-11.0.3
-
-# docker build -f Dockerfile.base -t ghcr.io/mfem/mfem-ubuntu-base .
-
-RUN apt-get update && \
-    apt-get install -y unzip gfortran && \
-    spack compiler find && \
-    apt-get install -y libcurl4-openssl-dev libssl-dev
-
-# /code is the working directory for code
-WORKDIR /code
-COPY . /code
-
-# This is for a spack environment/view to install from there
-RUN mkdir -p /opt/mfem-env \
-&&  (echo "spack:" \
-&&   echo "  view:" \
-&&   echo "    mfem:" \
-&&   echo "      root: /opt/mfem-view" \
-&&   echo "      link_type: copy" \
-&&   echo "  packages:" \
-&&   echo "    all:" \
-&&   echo "      target:" \
-&&   echo "      - x86_64_v3" \
-&&   echo "  config:" \
-&&   echo "    concretizer: clingo" \
-&&   echo "    compiler:" \
-&&   echo "      target:" \
-&&   echo "      - x86_64_v3" \
-&&   echo "    install_missing_compilers: true" \
-&&   echo "  concretization: together") > /opt/mfem-env/spack.yaml
-
-RUN cd /opt/mfem-env && \
-    . /opt/spack/share/spack/setup-env.sh && \
-    spack env activate . && \
-    spack develop --path /code mfem@master+examples+miniapps && \
-    spack add mfem@master+examples+miniapps && \
-    spack install
-
-# ensure mfem always on various paths
-RUN cd /opt/mfem-env && \
-    spack env activate --sh -d . >> /etc/profile.d/z10_spack_environment.sh
-
-# Present the software install when we shell in
-# The view is at /opt/mfem-env/.spack-env/view
-WORKDIR /opt/software
-ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l", "-c"]

config/docker/README.md

@@ -1,209 +0,0 @@
-# mfem Docker
-
-We provide a [Dockerfile.base](Dockerfile.base) to build an ubuntu base image,
-and a [Dockerfile](Dockerfile) to build a smaller one with a multi-stage build.
-You can use this image for a demo of using mfem! 🎉️
-
-Updated containers are built and deployed on merges to the main branch and releases.
-If you want to request a build on demand, you can [manually run the workflow](https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow) thanks to the workflow dispatch event.
-
-## Usage
-
-We provide two containers, which you can either build or use directly from
-[GitHub packages](https://github.com/orgs/mfem/packages?repo_name=mfem).
-
- - `ghcr.io/mfem/mfem-ubuntu-base`: a "build from scratch" for mfem
- - `ghcr.io/mfem/mfem-ubuntu`: a quick build that uses the base container
-
-In the above, "ghcr.io" means "GitHub Container Registry" and
-is the [GitHub packages](https://github.com/features/packages) registry that supports
-Docker images and other OCI artifacts. 
-
-### Ubuntu
-
-> Use or build this container for a multi-stage, slimmer base to develop on top of mfem
-
-Note that this container is provided on GitHub packages [here](https://github.com/mfem/mfem/pkgs/container/mfem-ubuntu)
-so you don't need to build it. However, if you want to, you can do the following:
-
-```bash
-$ docker build -f config/docker/Dockerfile -t ghcr.io/mfem/mfem-ubuntu .
-```
-
-Note that this will pull the base image. If you want to rebuild it, see [ubuntu base](#ubuntu-base)
-below. Once you have built (or prefer to pull) you can shell into the container as follows:
-
-```bash
-$ docker run -it ghcr.io/mfem/mfem-ubuntu
-```
-
-This smaller image has a view where everything is installed.
-
-```bash
-$ ls
-bin  etc  include  lib  libexec  sbin  share  var
-```
-
- - Examples are in share/mfem/examples
- - Examples are in share/mfem/miniapps
-
-Using this container, if you want to develop a tool that _uses_ mfem, you can find the libraries / includes in:
-
-```bash
-$ ls include/ | grep mfem
-mfem
-mfem-performance.hpp
-mfem.hpp
-```
-
-And yes, this is the working directory when you shell into the container!
-You can find the examples here:
-
-
-```bash
-cd share/mfem/examples
-```
-
-Try quickly setting the `LD_LIBRARY_PATH` so we can see the shared libraries
-we need:
-
-```bash
-export LD_LIBRARY_PATH=/opt/mfem-view/lib:$LD_LIBRARY_PATH
-```
-
-And then run:
-
-```bash
-$ ./ex0
-Options used:
-   --mesh ../data/star.mesh
-   --order 1
-Number of unknowns: 101
-   Iteration :   0  (B r, r) = 0.184259
-   Iteration :   1  (B r, r) = 0.102754
-   Iteration :   2  (B r, r) = 0.00558141
-   Iteration :   3  (B r, r) = 1.5247e-05
-   Iteration :   4  (B r, r) = 1.13807e-07
-   Iteration :   5  (B r, r) = 6.27231e-09
-   Iteration :   6  (B r, r) = 3.76268e-11
-   Iteration :   7  (B r, r) = 6.07423e-13
-   Iteration :   8  (B r, r) = 4.10615e-15
-Average reduction factor = 0.140201
-```
-
-You can also explore the "mini apps," also in share/mfem, but under miniapps.
-
-```bash
-# This is run from the examples directory
-$ cd ../miniapps
-```
-```bash
-$ ls
-CMakeLists.txt  common            meshing  nurbs        shifted  toys
-adjoint         electromagnetics  mtop     parelag      solvers
-autodiff        gslib             navier   performance  tools
-```
-
-And an example in "toys"
-
-```bash
-cd toys
-```
-```bash
-$ ./automata -no-vis
-Options used:
-   --num-steps 16
-   --rule 90
-   --no-visualization
-
-Rule:
- 111 110 101 100 011 010 001 000
-  0   1   0   1   1   0   1   0
-
-Applying rule...done.
-```
-
-Have fun! As a reminder, this container is ideal for developing your own
-applications that might use mfem, or having a nice environment to test out
-examples.
-
-
-### Ubuntu Base
-
-> Use this build for a development environment with spack and mfem
-
-This container is also [provided on GitHub packages](https://github.com/mfem/mfem/pkgs/container/mfem-ubuntu-base), 
-however you can build it locally too:
-
-```bash
-$ docker build -f config/docker/Dockerfile.base -t ghcr.io/mfem/mfem-ubuntu-base .
-```
-
-To shell into the container:
-
-```bash
-$ docker run -it ghcr.io/mfem/mfem-ubuntu-base bash
-```
-
-Change directory to the mfem environment, setup spack, and activate the environment:
-
-```bash
-source /opt/spack/share/spack/setup-env.sh
-cd /opt/mfem-env/
-spack env activate .
-```
-
-Note that this environment is installing to the view at `/opt/view`. Since the environment
-knows to install mfem from `/code` this means that you could make changes in the container (or bind
-`/code` to your container) and then update spack:
-
-```bash
-# Note that concretization takes a hot minute!
-$ spack install
-```
-
-And if you want to load mfem:
-
-```bash
-$ spack load mfem
-$ env | grep mfem
-```
-
-In this development container, you can find the examples and miniapps alongside
-mfem under `/code`:
-
-```bash
-cd /code/examples
-```
-```bash
-$ ./ex0
-```
-```console
-Options used:
-   --mesh ../data/star.mesh
-   --order 1
-Number of unknowns: 101
-   Iteration :   0  (B r, r) = 0.184259
-   Iteration :   1  (B r, r) = 0.102754
-   Iteration :   2  (B r, r) = 0.00558141
-   Iteration :   3  (B r, r) = 1.5247e-05
-   Iteration :   4  (B r, r) = 1.13807e-07
-   Iteration :   5  (B r, r) = 6.27231e-09
-   Iteration :   6  (B r, r) = 3.76268e-11
-   Iteration :   7  (B r, r) = 6.07423e-13
-   Iteration :   8  (B r, r) = 4.10615e-15
-Average reduction factor = 0.140201
-```
-
-This container is likely ideal for someone that wants to develop mfem itself.
-For other use cases, we recommend using the slimmer image. As an example,
-if you want to develop with your own code base (and mfem as is in the container) 
-you can bind to somewhere else in the container (e.g., src)
-
-```bash
-$ docker run -it ghcr.io/mfem/mfem-ubuntu-base -v $PWD:/code bash
-```
-
-In the above, we can pretend your project is in the present working directory (PWD) and we are
-binding to source. You can then use the mfem in the container for development, and if you
-want to distribute your library or app in a container, you can use the mfem container as the base.

config/docker/spack.yaml

@@ -1,11 +0,0 @@
-spack:
-  specs: [mfem@master+examples+miniapps]
-  view: 
-    mfem:
-      root: /opt/mfem-view 
-      link_type: copy
-  concretization: together
-  develop:
-    mfem:
-      path: /code
-      spec: mfem@master+examples+miniapps

config/get_hypre_version.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //

config/get_mumps_version.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //

config/githooks/pre-push

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -21,7 +21,6 @@ if [[ "${option}" == "--help" ]]; then
   echo "  --copyright"
   echo "  --license"
   echo "  --release"
-  echo "  --math"
   echo "  --style"
   echo "  --history"
   echo ""
@@ -36,8 +35,8 @@ cd $(git rev-parse --show-toplevel)
 # copyright check
 copyright=true
 if [[ "${option}" == "--copyright" || "${option}" == "" ]]; then
-  if git grep -n "^\(#\|//\).*Copyright.*2010-20\(2[^4]\|[^2].\)" > matches.txt; then
-    echo "Please update the following files to Copyright (c) 2010-2024:"
+  if git grep -l "^\(#\|//\).*\(\-2020\|\ 2010,\)" > matches.txt; then
+    echo "Please update the following files to Copyright (c) 2010-2021:"
     cat matches.txt
     copyright=false
   fi
@@ -64,18 +63,6 @@ if [[ "${option}" == "--release" || "${option}" == "" ]]; then
   fi
 fi
 
-# math in doxygen check
-math=true
-if [[ "${option}" == "--math" || "${option}" == "" ]]; then
-  if grep '\\f' -R doc/CodeDocumentation.dox mfem.hpp config general linalg mesh fem examples miniapps | grep -v '\\frac' | grep -v fem/picojson.h | grep -v config/githooks/pre-push > matches.txt
-  then
-    echo "Please use $..$ and \$\$..\$\$ for LaTeX formulas in the following"
-    echo "comments instead of the Doxygen style \f$..\f$, \f[..\f], etc."
-    cat matches.txt
-    math=false
-  fi
-fi
-
 # wrap-up
 code=0
 if ! $copyright ; then
@@ -90,10 +77,6 @@ if ! $release ; then
   echo "release check failed, unroll log for details"
   code=1
 fi
-if ! $math ; then
-  echo "math in doxygen check failed, unroll log for details"
-  code=1
-fi
 
 # `code-style` is not just a check, it will actually reformat the code if
 # necessary. This means that if one pushes while the repo is in dirty state
@@ -104,12 +87,12 @@ fi
 ## style check
 #if [[ "${option}" == "--style" || "${option}" == "" ]]; then
 if [[ "${option}" == "--style" ]]; then
-  if which astyle && [[ "$(astyle --version)" == "Artistic Style Version 3.1" ]]; then
+  if which astyle && [[ "$(astyle --version)" == "Artistic Style Version 2.05.1" ]]; then
     cd tests/scripts
     if ! ./runtest code-style; then code=1; fi
     cd -
   else
-    echo "Warning: astyle not found or version is not 3.1"
+    echo "Warning: astyle not found or version is not 2.05.1"
   fi
 fi
 

config/makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -38,14 +38,14 @@ all: header config-mk
 MPI = $(MFEM_USE_MPI:NO=)
 GHV_CXX ?= $(MFEM_CXX)
 GHV = get_hypre_version
-GHV_FLAGS = $(MFEM_CXXFLAGS) $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(HYPRE_OPT))
+GHV_FLAGS = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(HYPRE_OPT))
 SMX = $(if $(MFEM_USE_PUMI:NO=),MFEM_USE_SIMMETRIX)
 SMX_PATH = $(PUMI_DIR)/include/gmi_sim.h
 SMX_FILE = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(SMX_PATH))
 MUMPS = $(MFEM_USE_MUMPS:NO=)
 GMV_CXX ?= $(MFEM_CXX)
 GMV = get_mumps_version
-GMV_FLAGS = $(MFEM_CXXFLAGS) $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(MUMPS_OPT))
+GMV_FLAGS = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(MUMPS_OPT))
 
 $(GHV): $(SRC)$(GHV).cpp
 	$(call mfem-info, Determining HYPRE version ...)
@@ -110,4 +110,3 @@ config-mk:
 
 clean:
 	rm -f $(CONFIG_HPP) $(CONFIG_MK) sample-runs-build.log
-	rm -f $(GHV) $(GHV).out $(GMV) $(GMV).out

config/sample-runs.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -32,7 +32,7 @@ groups_serial=(
 '"examples"
    "Examples:"
    "examples"
-   "ex{,1,2,3}[0-9].cpp"'
+   "ex{,1,2}[0-9].cpp"'
 #   "ex1.cpp"'
 '"sundials"
    "SUNDIALS examples:"
@@ -83,13 +83,11 @@ groups_serial=(
 '"tools"
    "Tools miniapps:"
    "miniapps/tools"
-   "convert-dc.cpp display-basis.cpp get-values.cpp load-dc.cpp
-    lor-transfer.cpp"'
+   "convert-dc.cpp display-basis.cpp get-values.cpp load-dc.cpp lor-transfer.cpp"'
 '"toys"
    "Toys miniapps:"
    "miniapps/toys"
-   "automata.cpp life.cpp lissajous.cpp mandel.cpp mondrian.cpp rubik.cpp
-    snake.cpp"'
+   "automata.cpp life.cpp lissajous.cpp mandel.cpp mondrian.cpp rubik.cpp snake.cpp"'
 '"convergence"
    "Convergence tests:"
    "tests/convergence"
@@ -100,7 +98,7 @@ groups_parallel=(
 '"examples"
    "Examples:"
    "examples"
-   "ex{,1,2,3}[0-9]p.cpp"'
+   "ex{,1,2}[0-9]p.cpp"'
 #   "ex1p.cpp"'
 '"sundials"
    "SUNDIALS examples:"
@@ -138,7 +136,7 @@ groups_parallel=(
 '"meshing"
    "Meshing miniapps:"
    "miniapps/meshing"
-   "pmesh-optimizer.cpp pmesh-fitting.cpp pminimal-surface.cpp"'
+   "pmesh-optimizer.cpp pminimal-surface.cpp"'
 '"electromagnetics"
    "Electromagnetics miniapps:"
    "miniapps/electromagnetics"
@@ -186,7 +184,7 @@ groups_all=(
 '"examples"
    "Examples:"
    "examples"
-   "ex\"{,1,2,3}[0-9]\"{,p}.cpp"'
+   "ex\"{,1,2}[0-9]\"{,p}.cpp"'
 '"sundials"
    "SUNDIALS examples:"
    "examples/sundials"
@@ -227,7 +225,7 @@ groups_all=(
    "Meshing miniapps:"
    "miniapps/meshing"
    "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp
-    {,p}mesh-optimizer.cpp pmesh-fitting.cpp {,p}minimal-surface.cpp"'
+    {,p}mesh-optimizer.cpp {,p}minimal-surface.cpp"'
 '"electromagnetics"
    "Electromagnetics miniapps:"
    "miniapps/electromagnetics"
@@ -240,8 +238,7 @@ groups_all=(
 '"gslib"
    "GSLIB miniapps:"
    "miniapps/gslib"
-   "field-diff.cpp field-interp.cpp findpts.cpp schwarz_ex1.cpp pfindpts.cpp
-    schwarz_ex1p.cpp"'
+   "field-diff.cpp field-interp.cpp findpts.cpp schwarz_ex1.cpp pfindpts.cpp schwarz_ex1p.cpp"'
 '"navier"
    "Navier miniapps:"
    "miniapps/navier"
@@ -261,13 +258,11 @@ groups_all=(
 '"tools"
    "Tools miniapps:"
    "miniapps/tools"
-   "convert-dc.cpp display-basis.cpp get-values.cpp load-dc.cpp
-    lor-transfer.cpp"'
+   "convert-dc.cpp display-basis.cpp get-values.cpp load-dc.cpp lor-transfer.cpp"'
 '"toys"
    "Toys miniapps:"
    "miniapps/toys"
-   "automata.cpp life.cpp lissajous.cpp mandel.cpp mondrian.cpp rubik.cpp
-    snake.cpp"'
+   "automata.cpp life.cpp lissajous.cpp mandel.cpp mondrian.cpp rubik.cpp snake.cpp"'
 '"convergence"
    "Convergence tests:"
    "tests/convergence"
@@ -315,7 +310,7 @@ function extract_sample_runs()
    sruns=`grep -v "^//.*  mpirun .* ${app}" "${src}" |
           grep "^//.*  ${app}" |
           sed -e "s/.*  ${app}/${vg_app}/g"`
-   runs="${sruns}"$'\n'"${pruns}"
+   runs="${sruns}${pruns}"
    if [ "$skip_gen_meshes" == "yes" ]; then
       runs=`printf "%s" "$runs" | grep -v ".* -m .*\.gen"`
    fi
@@ -554,14 +549,15 @@ function go()
    local cmd_line="${1##+( )}"
    cmd_line="${cmd_line%%+( )}"
    shopt -u extglob
+   eval local cmd=(${cmd_line})
    local res=""
    echo $sep
    echo "<${group}>" "${cmd_line}"
    echo $sep
    if [ "${timing}" == "yes" ]; then
-      timed_run eval "${cmd_line}"
+      timed_run "${cmd[@]}"
    else
-      eval "${cmd_line}"
+      "${cmd[@]}"
    fi
    if [ "$?" -eq 0 ]; then
       res="${green}  OK  ${none}"