Add GPU scan support for Array<T>::PartialSum

Fix comments in scan.hpp
Add more device support to Array<T>
2025-10-21 09:57:37 -07:00 · 2025-10-21 09:57:37 -07:00 · 2025-10-21 09:57:37 -07:00 · 2025-10-21 09:57:37 -07:00 · 2025-10-20 15:38:54 -07:00
879 changed files with 19530 additions and 144721 deletions
@@ -25,7 +25,7 @@ runs:
  steps:
    - uses: ./.github/actions/sanitize/config

-    - uses: actions/cache@v5
+    - uses: actions/cache@v4
      if: ${{env.DEBUG == 'true'}}
      id: debug
      with:
@@ -53,7 +53,7 @@ runs:
      run: echo CXXFLAGS=${{env.CXXFLAGS}} ${{env.UBSAN_CXXFLAGS}} >> $GITHUB_ENV
      shell: bash

-    - uses: mfem/github-actions/build-mfem@v2.6
+    - uses: mfem/github-actions/build-mfem@v2.5
      if: ${{steps.debug.outputs.cache-hit != 'true'}}
      env:
        CXXFLAGS: ${{env.CXXFLAGS}}
@@ -82,7 +82,7 @@ runs:
      run: find . -type f -name '*.o' -delete
      shell: bash

-    - uses: actions/upload-artifact@v7
+    - uses: actions/upload-artifact@v4
      with:
        name: build-${{inputs.par}}-${{inputs.sanitizer}}
        path: mfem/build
@@ -36,7 +36,7 @@ runs:
  steps:
    - uses: ./.github/actions/sanitize/config

-    - uses: actions/cache@v5
+    - uses: actions/cache@v4
      if: ${{env.DEBUG == 'true' && inputs.cache-skip != 'true'}}
      id: debug
      with:
@@ -49,7 +49,7 @@ runs:
        par: ${{inputs.par}}
        sanitizer: ${{inputs.sanitizer}}

-    - uses: actions/download-artifact@v8
+    - uses: actions/download-artifact@v4
      with:
        name: build-${{inputs.par}}-${{inputs.sanitizer}}
        path: mfem/build
@@ -23,7 +23,7 @@ inputs:
 runs:
  using: 'composite'
  steps:
-    - uses: actions/cache/restore@v5  # Cache for LLVM libcxx
+    - uses: actions/cache/restore@v4  # Cache for LLVM libcxx
      with:
        path: ${{env.LLVM_DIR}}
        fail-on-cache-miss: true
@@ -32,14 +32,14 @@ runs:
    - uses: ./.github/actions/sanitize/mpi
      if: ${{inputs.par == 'true'}}

-    - uses: actions/cache/restore@v5  # Cache for Hypre
+    - uses: actions/cache/restore@v4  # Cache for Hypre
      if: ${{inputs.par == 'true'}}
      with:
        path: ${{env.HYPRE_DIR}}
        fail-on-cache-miss: true
        key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-v2.5

-    - uses: actions/cache/restore@v5  # Cache for Metis
+    - uses: actions/cache/restore@v4  # Cache for Metis
      if: ${{inputs.par == 'true'}}
      with:
        path: ${{env.METIS_DIR}}
@@ -51,13 +51,13 @@ runs:
      run: ln -s -f ${{env.HYPRE_DIR}} hypre && ln -s -f ${{env.METIS_DIR}} metis-4.0
      shell: bash

-    - uses: actions/cache/restore@v5  # Cache for LSAN suppression file
+    - uses: actions/cache/restore@v4  # Cache for LSAN suppression file
      with:
        path: ${{env.LSAN_DIR}}
        fail-on-cache-miss: true
        key: build-lsan-suppression-file

-    - uses: actions/checkout@v6  # Checkout the repository
+    - uses: actions/checkout@v4  # Checkout the repository
      with:
        path: mfem
        # ref: ${{env.BRANCH}}
@@ -1,42 +0,0 @@
-# MFEM Pull Request Review Agent Guide
-
-## Purpose and scope
-Review MFEM PRs for correctness, maintainability, performance, portability, test coverage, and MFEM consistency. Use the diff and PR context; reference source files, tests, and CI results when available. Follow `CONTRIBUTING.md`, especially Developer Guidelines, PR rules, checklist, and testing.
-
-## Critical review pillars
- Correctness and numerical behavior
- API and user-facing impact
- Performance implications
- Maintainability and portability
-
-## Review workflow
-1. Read the PR description, linked issues, and intended behavior.
-2. Inspect the diff before commenting.
-3. Identify affected MFEM components, examples, tests, build or docs changes, and downstream APIs.
-4. Analyze the code against the critical review pillars.
-5. Compare the change against nearby code and MFEM patterns; flag unmotivated deviations.
-6. Check whether tests and documentation were updated appropriately.
-7. Review CI results and suggest actions.
-8. Produce a structured review with prioritized findings.
-9. Always limit conclusions to available evidence.
-
-## MFEM-specific review checklist
- Component-aware scope: identify the touched subsystem (FEM, solvers, preconditioners, linear algebra, mesh, examples, miniapps, build, or docs) and assess its impact against the review pillars.
- Numerical and algorithmic behavior: assess issues in convergence, stability, tolerances, precision, iteration limits, and failure handling. If clear opportunities exist to improve the algorithmic approach, call them out with expected impact.
- API and user-facing impact: assess backward compatibility, user-visible behavior and default changes, migration impact, deprecations, and whether documentation clearly explains user-facing API changes.
- Data structure and memory semantics: assess ownership, lifetime, aliasing, container behavior, and device-host synchronization.
- Parallel and serial behavior: assess whether the change preserves equivalent semantics in serial and parallel modes where applicable; if logic is currently mode-specific, check whether extension to the other mode is straightforward (clear abstractions, no hard-wired assumptions), document constraints, and call out expected behavior differences explicitly.
- Backend and portability impact: assess likely cross-backend risks in CPU, CUDA, HIP, OCCA, RAJA, partial assembly, fallback paths, compiler compatibility, and platform assumptions.
- Build, dependency, and configuration impact: assess CMake or make changes, optional dependency behavior, and feature-flag interactions.
- Tests and docs alignment: check available regression or unit coverage evidence for changed behavior, and ensure docs are updated for new flags, APIs, options, or behavior changes.
- MFEM developer-guideline fit: keep code lean, simple, general, logically separated, and portable; suggest C++17 improvements when they clearly improve safety, clarity, or maintainability.
- New source files, examples, or miniapps: if a PR adds source/header files, verify they are properly wired into the relevant `makefile` and `CMakeLists.txt`, referenced in docs where applicable (including `doc/CodeDocumentation.dox`), and added to top-level `.gitignore` only when generated artifacts require it.
- Changelog: verify `CHANGELOG` is updated if the PR introduces significant new features or user-facing changes.
- MFEM conventions: use `real_t`; use `mfem::out`/`mfem::err` instead of `std::cout`/`std::cerr` in library code; flag large/binary files; if AI assistance is apparent but undisclosed, suggest following `CONTRIBUTING.md`.
- Edge cases: if the PR touches complex or error-prone areas, suggest additional tests for edge cases, failure modes, and parallel behavior.
-
-## Commenting guidelines
- Keep comments concise, actionable, and grounded in the diff.
- Focus on correctness, behavior changes, and user impact over style nits.
- Be professional, concise, collaborative, technically precise, and avoid unsupported assumptions.
-
@@ -13,7 +13,7 @@ Note that some of these scripts use the shared MFEM GitHub Actions from the exte

  <https://github.com/mfem/github-actions>

-For a particular action, e.g. `mfem/github-actions/build-mfem@v2.5`, the `v2.5` suffix denotes the branch (or tag) in the above from which the action is taken.
+For a particular action, e.g. `mfem/github-actions/build-mfem@v2.5`, the `v2.5` suffix denotes the branch in the above from which the action is taken.

 The current CI workflows are:

@@ -43,7 +43,7 @@ jobs:
          remove-docker-images: 'true'

      - name: Checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4

      # It's easier to reference named variables than indexes of the matrix
      - name: Set Environment
@@ -153,7 +153,7 @@ jobs:
      # /home/runner/work/mfem/mfem/mfem
      # Note: Done now to access "install-hypre" and "install-metis" actions.
      - name: checkout mfem
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
        with:
          path: ${{ env.MFEM_TOP_DIR }}
          # Fetch the complete history for codecov to access commits ID
@@ -225,14 +225,14 @@ jobs:
      - name: cache hypre
        id: hypre-cache
        if: matrix.mpi == 'par'
-        uses: actions/cache@v5
+        uses: actions/cache@v4
        with:
          path: ${{ env.HYPRE_TOP_DIR }}
          key: ${{ runner.os }}-ompi-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-${{ matrix.precision }}-v2.5

      - name: get hypre
        if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
-        uses: mfem/github-actions/build-hypre@v2.6
+        uses: mfem/github-actions/build-hypre@v2.5
        with:
          archive: ${{ env.HYPRE_ARCHIVE }}
          dir: ${{ env.HYPRE_TOP_DIR }}
@@ -242,7 +242,7 @@ jobs:

      - name: get hypre (Windows)
        if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
-        uses: mfem/github-actions/build-hypre@v2.6
+        uses: mfem/github-actions/build-hypre@v2.5
        with:
          archive: ${{ env.HYPRE_ARCHIVE }}
          dir: ${{ env.HYPRE_TOP_DIR }}
@@ -255,14 +255,14 @@ jobs:
      - name: cache metis
        id: metis-cache
        if: matrix.mpi == 'par' && matrix.os != 'windows-latest'
-        uses: actions/cache@v5
+        uses: actions/cache@v4
        with:
          path: ${{ env.METIS_TOP_DIR }}
          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5

      - name: install metis
        if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.6
+        uses: mfem/github-actions/build-metis@v2.5
        with:
          archive: ${{ matrix.os != 'macos-latest' && env.METIS_ARCHIVE || env.METIS_ARCHIVE_MAC }}
          dir: ${{ env.METIS_TOP_DIR }}
@@ -270,7 +270,7 @@ jobs:
      - name: cache vcpkg (Windows)
        id: vcpkg-cache
        if: matrix.os == 'windows-latest'
-        uses: actions/cache@v5
+        uses: actions/cache@v4
        with:
          path: vcpkg_cache
          key: ${{ runner.os }}-${{ matrix.mpi }}-vcpkg-v1
@@ -295,8 +295,7 @@ jobs:
          export HOMEBREW_NO_INSTALL_CLEANUP=1
          brew update
          brew install enzyme
-          ENZYME_LLVM=$(brew info enzyme | sed -n 's/^Required.*:.*\(llvm[^ ]*\).*/\1/p')
-          echo "ENZYME_LLVM=$ENZYME_LLVM"
+          ENZYME_LLVM=$(brew info enzyme | sed -n 's/^Required:.*\(llvm[^ ]*\).*/\1/p')
          LLVM_PREFIX=$(brew --prefix $ENZYME_LLVM)
          echo "LLVM_PREFIX=$LLVM_PREFIX" >> $GITHUB_ENV
          echo "OMPI_CC=$LLVM_PREFIX/bin/clang" >> $GITHUB_ENV
@@ -304,7 +303,7 @@ jobs:

      # MFEM build and test
      - name: build
-        uses: mfem/github-actions/build-mfem@v2.6
+        uses: mfem/github-actions/build-mfem@v2.5
        env:
          VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
        with:
@@ -375,10 +374,8 @@ jobs:
      # Code coverage (process and upload reports)
      - name: codecov
        if: matrix.codecov == 'YES'
-        uses: mfem/github-actions/upload-coverage@v2.6
+        uses: mfem/github-actions/upload-coverage@v2.5
        with:
-          name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}
+          name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
          project_dir: ${{ env.MFEM_TOP_DIR }}
          directories: "fem general linalg mesh"
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
@@ -40,11 +40,11 @@ jobs:

    steps:
      - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4

      # Initializes the CodeQL tools for scanning.
      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v4
+        uses: github/codeql-action/init@v2
        with:
          languages: ${{ matrix.language }}
          # If you wish to specify custom queries, you can do so here or in a config file.
@@ -57,7 +57,7 @@ jobs:
      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
      # If this step fails, then you should remove it and run the build manually (see below)
      - name: Autobuild
-        uses: github/codeql-action/autobuild@v4
+        uses: github/codeql-action/autobuild@v2

      # ℹ️ Command-line programs to run using the OS shell.
      # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -70,4 +70,4 @@ jobs:
      #   ./location_of_script_within_repo/buildscript.sh

      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v4
+        uses: github/codeql-action/analyze@v2
@@ -39,7 +39,7 @@ jobs:

    steps:
      - name: checkout MFEM
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
        with:
          path: mfem

@@ -50,14 +50,14 @@ jobs:

      - name: Cache Hypre Install
        id: hypre-cache
-        uses: actions/cache@v5
+        uses: actions/cache@v4
        with:
          path: ${{ env.HYPRE_TOP_DIR }}
          key: ${{ runner.os }}-ompi-build-${{ env.HYPRE_TOP_DIR }}-v2.5

      - name: Get Hypre
        if: steps.hypre-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-hypre@v2.6
+        uses: mfem/github-actions/build-hypre@v2.5
        with:
          archive: ${{ env.HYPRE_ARCHIVE }}
          dir: ${{ env.HYPRE_TOP_DIR }}
@@ -65,21 +65,21 @@ jobs:

      - name: Cache Metis Install
        id: metis-cache
-        uses: actions/cache@v5
+        uses: actions/cache@v4
        with:
          path: ${{ env.METIS_TOP_DIR }}
          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5

      - name: Install Metis
        if: steps.metis-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.6
+        uses: mfem/github-actions/build-metis@v2.5
        with:
          archive: ${{ env.METIS_ARCHIVE }}
          dir: ${{ env.METIS_TOP_DIR }}

      # MFEM build and test
      - name: build-mfem
-        uses: mfem/github-actions/build-mfem@v2.6
+        uses: mfem/github-actions/build-mfem@v2.5
        with:
          os: ${{ runner.os }}
          target: opt
@@ -38,7 +38,7 @@ jobs:
       github.event.pull_request.head.repo.full_name != github.repository)
    steps:
      - name: checkout mfem
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4

      - name: copyright check
        id: copyright
@@ -93,7 +93,7 @@ jobs:
       github.event.pull_request.head.repo.full_name != github.repository)
    steps:
      - name: checkout mfem
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4

      - name: get astyle
        run: |
@@ -110,7 +110,7 @@ jobs:
       github.event.pull_request.head.repo.full_name != github.repository)
    steps:
      - name: checkout mfem
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4

      - name: get doxygen and graphviz
        run: |
@@ -135,7 +135,7 @@ jobs:
    runs-on: ubuntu-latest
    steps:
      - name: checkout mfem
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
        with:
          fetch-depth: 0

@@ -17,11 +17,11 @@ jobs:
    runs-on: ubuntu-latest
    name: 2.19.0
    steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
      - uses: ./.github/actions/sanitize/config
      - name: Cache
        id: cache
-        uses: actions/cache@v5
+        uses: actions/cache@v4
        with:
          path: ${{env.HYPRE_DIR}}
          key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-v2.5
@@ -30,7 +30,7 @@ jobs:
        uses: ./.github/actions/sanitize/mpi
      - name: Build
        if: steps.cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-hypre@v2.6
+        uses: mfem/github-actions/build-hypre@v2.5
        with:
          archive: ${{env.HYPRE_TGZ}}
          dir: ${{env.HYPRE_DIR}}
@@ -27,13 +27,13 @@ jobs:
            llvm_use_sanitizer: "Undefined"
    name: ${{matrix.sanitizer}}
    steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
      - uses: ./.github/actions/sanitize/config
        with:
          NO_FLAGS: true
      - name: Cache
        id: cache
-        uses: actions/cache@v5
+        uses: actions/cache@v4
        with:
          path: ${{env.LLVM_DIR}}
          key: build-libcxx-${{env.LLVM_VER}}-${{matrix.sanitizer}}
@@ -17,11 +17,11 @@ jobs:
    runs-on: ubuntu-latest
    name: lsan.supp
    steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
      - uses: ./.github/actions/sanitize/config
      - name: Cache
        id: cache
-        uses: actions/cache@v5
+        uses: actions/cache@v4
        with:
          path: ${{env.LSAN_DIR}}
          key: build-lsan-suppression-file
@@ -17,11 +17,11 @@ jobs:
    runs-on: ubuntu-latest
    name: 4.0.3
    steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
      - uses: ./.github/actions/sanitize/config
      - name: Cache
        id: cache
-        uses: actions/cache@v5
+        uses: actions/cache@v4
        with:
          path: ${{env.METIS_DIR}}
          key: ${{runner.os}}-build-${{env.METIS_DIR}}-v2.5
@@ -30,7 +30,7 @@ jobs:
        uses: ./.github/actions/sanitize/mpi
      - name: Build
        if: steps.cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.6
+        uses: mfem/github-actions/build-metis@v2.5
        with:
          archive: ${{env.METIS_TGZ}}
          dir: ${{env.METIS_DIR}}
@@ -28,7 +28,7 @@ jobs:
  build:
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
      - uses: ./.github/actions/sanitize/mfem
        with:
          par: ${{inputs.par}}
@@ -40,7 +40,7 @@ jobs:
    env:
      ex: ${{inputs.par && 'ex1p' || 'ex1'}}
    steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
      - uses: ./.github/actions/sanitize/restore
        id: restore
        with:
@@ -58,7 +58,7 @@ jobs:
    env:
      exclude: ${{inputs.par && '-E "_ser"' || ''}}
    steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
      - uses: ./.github/actions/sanitize/restore
        id: restore
        with:
@@ -82,7 +82,7 @@ jobs:
    env:
      exclude: ${{inputs.par && '-E "_ser"' || ''}}
    steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
      - uses: ./.github/actions/sanitize/restore
        id: restore
        with:
@@ -107,7 +107,7 @@ jobs:
      run: ${{inputs.par && '-R "_cpu_np"' || ''}}
      exclude: ${{inputs.par && '"unit_tests|debug"' || '"^unit_tests$|debug"'}}
    steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
      - uses: ./.github/actions/sanitize/restore
        id: restore
        with:
@@ -131,7 +131,7 @@ jobs:
    env:
      unit_tests: ${{inputs.par && 'punit_tests' || 'unit_tests'}}
    steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
      - uses: ./.github/actions/sanitize/restore
        id: restore
        with:
@@ -146,7 +146,7 @@ jobs:
        if: ${{steps.restore.outputs.cache-hit != 'true'}}
        working-directory: mfem/build/tests/unit
        run: find . -type f -name '*.o' -delete
-      - uses: actions/upload-artifact@v7
+      - uses: actions/upload-artifact@v4
        with:
          name: tests-${{inputs.par}}-${{inputs.sanitizer}}
          path: mfem/build/tests/unit/${{env.unit_tests}}
@@ -165,14 +165,14 @@ jobs:
      unit_tests: ${{inputs.par && 'punit_tests' || 'unit_tests'}}
      np: ${{inputs.par && '_np=2' || ''}}
    steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
      - uses: ./.github/actions/sanitize/restore
        id: restore
        with:
          par: ${{inputs.par}}
          sanitizer: ${{inputs.sanitizer}}
          cache-path: mfem/build/tests/unit/${{env.unit_tests}}
-      - uses: actions/download-artifact@v8
+      - uses: actions/download-artifact@v4
        if: ${{steps.restore.outputs.cache-hit != 'true'}}
        with:
          name: tests-${{inputs.par}}-${{inputs.sanitizer}}
@@ -19,15 +19,9 @@ CMakeFiles/
 # Clangd server cache
 *.cache*

-#vscode settings
-/.vscode/
-
 # Backup files
 *~

-# clangd index
-/.cache/
-
 # Default install location
 /mfem/

@@ -85,15 +79,12 @@ examples/sol_u.*
 examples/sol_p.*
 examples/sol_r.*
 examples/sol_i.*
-examples/sol_z.*
 examples/ex6p-checkpoint.*
 examples/order.*
 examples/ex9.mesh
 examples/ex9-mesh.*
 examples/ex9-init.*
 examples/ex9-final.*
-examples/Example41*
-examples/ex41-p-*.bp
 examples/deformed.*
 examples/velocity.*
 examples/elastic_energy.*
@@ -153,35 +144,6 @@ examples/ginkgo/ex1
 examples/ginkgo/refined.mesh
 examples/ginkgo/sol.gf

-examples/hdg/refined.mesh
-examples/hdg/displaced.mesh
-examples/hdg/mesh.*
-examples/hdg/ex5
-examples/hdg/ex5p
-examples/hdg/ex5.mesh
-examples/hdg/ex5-p-*.bp
-examples/hdg/ex6
-examples/hdg/ex6p
-examples/hdg/ex14
-examples/hdg/ex14p
-examples/hdg/ex17
-examples/hdg/ex17p
-examples/hdg/ex21
-examples/hdg/ex21p
-examples/hdg/sol.*
-examples/hdg/sol_p.*
-examples/hdg/sol_u.*
-examples/hdg/ex6p-checkpoint.*
-examples/hdg/Example5*
-examples/hdg/ParaView
-examples/hdg/ex21*.mesh
-examples/hdg/ex21*.sol
-examples/hdg/ex21p_*.*
-examples/hdg/ex41
-examples/hdg/ex41p
-examples/hdg/Example41*
-examples/hdg/ex41-p-*.bp
-
 examples/hiop/ex9
 examples/hiop/ex9p
 examples/hiop/ex9.mesh
@@ -254,9 +216,6 @@ miniapps/electromagnetics/Joule_[0-9]*
 miniapps/electromagnetics/Lorentz_[0-9]*
 miniapps/electromagnetics/Lorentz.dat

-miniapps/fluids/schrodinger-flow/schrodinger_flow
-miniapps/fluids/schrodinger-flow/pschrodinger_flow
-
 miniapps/gslib/field-diff
 miniapps/gslib/field-interp
 miniapps/gslib/findpts
@@ -264,7 +223,6 @@ miniapps/gslib/pfindpts
 miniapps/gslib/schwarz_ex1
 miniapps/gslib/schwarz_ex1p
 miniapps/gslib/interpolated.gf
-miniapps/gslib/particles_redist

 miniapps/meshing/mobius-strip
 miniapps/meshing/klein-bottle
@@ -311,8 +269,10 @@ miniapps/meshing/refined.mesh
 miniapps/meshing/bounding-box*
 miniapps/meshing/jacobian-determinant*

-miniapps/mtop/ParaView/
-miniapps/mtop/mtop_test_iso_elasticity
+miniapps/mtop/parheat
+miniapps/mtop/ParHeat/*
+miniapps/mtop/seqheat
+miniapps/mtop/SeqHeat/*

 miniapps/autodiff/paradiff
 miniapps/autodiff/seqadiff
@@ -322,26 +282,21 @@ miniapps/autodiff/seq_example
 miniapps/autodiff/seq_test
 miniapps/autodiff/Example/*

-miniapps/fluids/navier/navier_mms
-miniapps/fluids/navier/navier_kovasznay
-miniapps/fluids/navier/navier_kovasznay_vs
-miniapps/fluids/navier/navier_tgv
-miniapps/fluids/navier/navier_shear
-miniapps/fluids/navier/navier_3dfoc
-miniapps/fluids/navier/navier_turbchan
-miniapps/fluids/navier/navier_cht
-miniapps/fluids/navier/navier_bifurcation
-miniapps/fluids/navier/Navier_Bifurcation_[0-9]*
-miniapps/fluids/navier/ParaView
-miniapps/fluids/navier/tgv_out*.txt
-miniapps/fluids/navier/*_output
+miniapps/navier/navier_mms
+miniapps/navier/navier_kovasznay
+miniapps/navier/navier_kovasznay_vs
+miniapps/navier/navier_tgv
+miniapps/navier/navier_shear
+miniapps/navier/navier_3dfoc
+miniapps/navier/navier_turbchan
+miniapps/navier/navier_cht
+miniapps/navier/tgv_out*.txt
+miniapps/navier/*_output

 miniapps/nurbs/nurbs_ex1
 miniapps/nurbs/nurbs_ex1p
 miniapps/nurbs/nurbs_ex3
 miniapps/nurbs/nurbs_ex5
-miniapps/nurbs/nurbs_ex10
-miniapps/nurbs/nurbs_ex10p
 miniapps/nurbs/nurbs_ex11p
 miniapps/nurbs/nurbs_ex24
 miniapps/nurbs/nurbs_solenoidal
@@ -367,14 +322,7 @@ miniapps/nurbs/nurbs_naca_cmesh
 miniapps/nurbs/naca-cmesh.mesh
 miniapps/nurbs/glvis_naca-cmesh.mesh
 miniapps/nurbs/Naca_cmesh
-miniapps/nurbs/nurbs_mesh_info
-miniapps/nurbs/k*_*.dat
 miniapps/nurbs/*-Surface.mesh
-miniapps/nurbs/*.mesh
-miniapps/nurbs/*.sol
-miniapps/nurbs/deformed.*
-miniapps/nurbs/elastic_energy.*
-miniapps/nurbs/velocity.*

 miniapps/performance/ex1
 miniapps/performance/ex1p
@@ -396,7 +344,6 @@ miniapps/shifted/lsf_integral
 miniapps/tools/display-basis
 miniapps/tools/load-dc
 miniapps/tools/convert-dc
-miniapps/tools/compare-dc
 miniapps/tools/gridfunction-bounds
 miniapps/tools/lor-transfer
 miniapps/tools/plor-transfer
@@ -438,25 +385,6 @@ miniapps/solvers/ParaView
 miniapps/solvers/mesh.*
 miniapps/solvers/sol.*

-miniapps/hdg/anisodiff
-miniapps/hdg/anisodiff-hr
-miniapps/hdg/anisodiff*.mesh
-miniapps/hdg/sol_t*.gf
-miniapps/hdg/sol_q*.gf
-miniapps/hdg/Anisodiff*
-miniapps/hdg/mfem-logo
-miniapps/hdg/mfem-logo.mesh
-miniapps/hdg/Mfem-logo*
-miniapps/hdg/convdiff
-miniapps/hdg/convdiff*.mesh
-miniapps/hdg/Convdiff*
-miniapps/hdg/pconvdiff
-miniapps/hdg/panisodiff
-miniapps/hdg/panisodiff*.mesh
-miniapps/hdg/PAnisodiff*
-miniapps/hdg/pconvdiff*.mesh
-miniapps/hdg/PConvdiff*
-
 miniapps/hdiv-linear-solver/darcy
 miniapps/hdiv-linear-solver/grad_div

@@ -486,13 +414,6 @@ miniapps/tribol/contact-patch-test
 miniapps/diag-smoothers/abs-l1-jacobi
 miniapps/diag-smoothers/mg-abs-l1-jacobi

-miniapps/contact/contact
-miniapps/contact/ParaView
-
-miniapps/plasma/pic/electrostatic-*
-!miniapps/plasma/pic/electrostatic-*.cpp
-miniapps/plasma/pic/*.csv
-
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
@@ -9,550 +9,91 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

-# DESCRIPTION:
-###############################################################################
 # General GitLab pipelines configurations for supercomputers and Linux clusters
-# at Lawrence Livermore National Laboratory (LLNL).
-# This entire pipeline is LLNL-specific
-#
-# Important note: This file is a template provided by llnl/radiuss-shared-ci.
-# Remains to set variable values, change the reference to the radiuss-shared-ci
-# repo, opt-in and out optional features. The project can then extend it with
-# additional stages.
-#
-# In addition, each project should copy over and complete:
-# - .gitlab/custom-jobs-and-variables.yml
-# - .gitlab/subscribed-pipelines.yml
-#
-# The jobs should be specified in a file local to the project,
-# - .gitlab/jobs/${CI_MACHINE}.yml
-# or generated (see LLNL/Umpire for an example).
-###############################################################################
-#   MAP OF GITLAB CI
-#######################
-#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-# File dependencies: direct, through jobs, through variables
-#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-# .gitlab-ci.yml
-#  ├── .build-and-test  [job]
-#  │    ├── .gitlab/custom-jobs-and-variables.yml
-#  │    │    ├── .custom_job          [job]
-#  │    │    ├── .reproducer_vars     [job]
-#  │    │    ├── .report_job_success  [job]
-#  │    │    │    └── .gitlab/scripts/report_build_and_test    [script]
-#  │    │    │         ├── .gitlab/scripts/safe_create_rundir  [script]
-#  │    │    │         └── .gitlab/scripts/git_try_to_push     [script]
-#  │    │    ├── .report_job_failure  [job]
-#  │    │    │    └── .gitlab/scripts/report_build_and_test    [script]
-#  │    │    │         ├── .gitlab/scripts/safe_create_rundir  [script]
-#  │    │    │         └── .gitlab/scripts/git_try_to_push     [script]
-#  │    │    └── JOB_CMD              [var]
-#  │    │         └── tests/gitlab/build_and_test            [script]
-#  │    │              └── tests/gitlab/get_mfem_uberenv     [script]
-#  │    ├── <radiuss-shared-ci>/pipelines/matrix.yml  [conditional]
-#  │    │    ├── .on_matrix               [job]
-#  │    │    ├── .matrix_reproducer_init  [job]
-#  │    │    ├── .matrix_reproducer_vars  [job]
-#  │    │    ├── .matrix_reproducer_job   [job]
-#  │    │    ├── .matrix_job_command      [job]
-#  │    │    └── .job_on_matrix           [job]
-#  │    ├── <radiuss-shared-ci>/pipelines/dane.yml  [conditional]
-#  │    │    ├── .on_dane               [job]
-#  │    │    ├── .dane_reproducer_init  [job]
-#  │    │    ├── .dane_reproducer_vars  [job]
-#  │    │    ├── .dane_reproducer_job   [job]
-#  │    │    ├── .dane_job_command      [job]
-#  │    │    ├── .job_on_dane           [job]
-#  │    │    ├── allocate_resources     [job]
-#  │    │    └── release_resources      [job]
-#  │    ├── <radiuss-shared-ci>/pipelines/tioga.yml  [conditional]
-#  │    │    ├── .on_tioga               [job]
-#  │    │    ├── .tioga_reproducer_init  [job]
-#  │    │    ├── .tioga_reproducer_vars  [job]
-#  │    │    ├── .tioga_reproducer_job   [job]
-#  │    │    ├── .tioga_job_command      [job]
-#  │    │    ├── .job_on_tioga           [job]
-#  │    │    ├── allocate_resources      [job]
-#  │    │    └── release_resources       [job]
-#  │    ├── <artifact>/matrix-jobs.yml  [conditional, from 'generate-job-lists']
-#  │    │    ├── .gitlab/jobs/matrix.yml
-#  │    │    │    ├── .matrix_reproducer_vars      [job]
-#  │    │    │    ├── setup                        [job]
-#  │    │    │    │    └── ./tests/gitlab/build_and_test_setup  [script]
-#  │    │    │    ├── opt_mpi_cuda_gcc             [job]
-#  │    │    │    └── opt_mpi_cuda_hypre_cuda_gcc  [job]
-#  │    │    └── .gitlab/jobs/matrix-reports.yml  [used conditionally]
-#  │    │         ├── report_job_success
-#  │    │         └── report_job_failure
-#  │    ├── <artifact>/dane-jobs.yml    [conditional, from 'generate-job-lists']
-#  │    │    ├── .gitlab/jobs/dane.yml
-#  │    │    │    ├── .dane_reproducer_vars    [job]
-#  │    │    │    ├── setup                    [job]
-#  │    │    │    │    └── ./tests/gitlab/build_and_test_setup  [script]
-#  │    │    │    ├── debug_ser_gcc_10         [job]
-#  │    │    │    ├── debug_par_gcc_10         [job]
-#  │    │    │    ├── opt_ser_gcc_10           [job]
-#  │    │    │    ├── opt_par_gcc_10           [job]
-#  │    │    │    ├── opt_par_gcc_10_sundials  [job]
-#  │    │    │    ├── opt_par_gcc_10_petsc     [job]
-#  │    │    │    └── opt_par_gcc_10_pumi      [job]
-#  │    │    └── .gitlab/jobs/dane-reports.yml    [used conditionally]
-#  │    │         ├── report_job_success
-#  │    │         └── report_job_failure
-#  │    └── <artifact>/tioga-jobs.yml   [conditional, from 'generate-job-lists']
-#  │         ├── .gitlab/jobs/tioga.yml
-#  │         │    ├── .tioga_reproducer_vars  [job]
-#  │         │    ├── setup                   [job]
-#  │         │    │    └── ./tests/gitlab/build_and_test_setup  [script]
-#  │         │    └── cce_16_0_1              [job]
-#  │         └── .gitlab/jobs/tioga-reports.yml   [used conditionally]
-#  │              ├── report_job_success
-#  │              └── report_job_failure
-#  └── .gitlab/subscribed-pipelines.yml
-#       ├── .machine-check         [job]
-#       ├── generate-job-lists     [job]
-#       ├── dane-up-check          [job]
-#       ├── dane-build-and-test    [job]
-#       ├── dane-baseline          [job]
-#       │    └── .gitlab/dane-baseline.yml
-#       │         ├── .on_dane                       [job]
-#       │         ├── baselinecheck_mfem_intel_dane  [job]
-#       │         │    └── .gitlab/scripts/baseline            [script]
-#       │         ├── cleanup                        [job]
-#       │         ├── report_baseline                [job]
-#       │         │    ├── .gitlab/scripts/safe_create_rundir  [script]
-#       │         │    └── .gitlab/scripts/git_try_to_push     [script]
-#       │         ├── baselinepublish_mfem_dane      [job]
-#       │         │    └── .gitlab/scripts/rebaseline          [script]
-#       │         ├── .gitlab/custom-jobs-and-variables.yml
-#       │         │    └── <same as above: see .gitlab-ci.yml/.build-and-test>
-#       │         └── .gitlab/configs/setup-baseline.yml
-#       │              └── setup_baseline            [job]
-#       ├── tioga-up-check         [job]
-#       ├── tioga-build-and-test   [job]
-#       ├── matrix-up-check        [job]
-#       └── matrix-build-and-test  [job]
-#
-#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-# File tree hierarchy with file contents highlights
-#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-# In addition to the files in the MFEM repo, the Gitlab CI uses files from the
-# radiuss/radiuss-shared-ci project, see below, after the <mfem-root> tree.
-#
-# <mfem root>
-#  ├── .gitlab-ci.yml  [this file]
-#  │    ├── <jobs>
-#  │    │    └── .build-and-test
-#  │    ├── <included files>
-#  │    │    ├── .gitlab/subscribed-pipelines.yml
-#  │    │    ├── .gitlab/custom-jobs-and-variables.yml  [by ".build-and-test"]
-#  │    │    ├── <artifact>  [by ".build-and-test"]
-#  │    │    │    ├── artifact: '${CI_MACHINE}-jobs.yml'
-#  │    │    │    └── job:      'generate-job-lists'
-#  │    │    └── <external>  [by ".build-and-test"]
-#  │    │         ├── project: 'radiuss/radiuss-shared-ci'
-#  │    │         ├── ref:     'v2025.09.1'
-#  │    │         └── file:    'pipelines/${CI_MACHINE}.yml'
-#  │    └── <defined variables>
-#  │         ├── CUSTOM_CI_BUILDS_DIR
-#  │         ├── USER_CI_TOP_DIR
-#  │         ├── SHARED_REPOS_DIR
-#  │         ├── AUTOTEST_ROOT
-#  │         ├── MFEM_DATA_DIR
-#  │         ├── AUTOTEST
-#  │         ├── AUTOTEST_COMMIT
-#  │         ├── REBASELINE
-#  │         ├── GITHUB_PROJECT_NAME
-#  │         └── GITHUB_PROJECT_ORG
-#  ├── .gitlab
-#  │    ├── configs
-#  │    │    └── setup-baseline.yml
-#  │    │         ├── <jobs>
-#  │    │         │    └── setup_baseline
-#  │    │         └── <used variables>
-#  │    │              ├── MACHINE_NAME
-#  │    │              ├── REBASELINE
-#  │    │              ├── AUTOTEST
-#  │    │              ├── AUTOTEST_COMMIT
-#  │    │              ├── BUILD_ROOT
-#  │    │              ├── TPLS_REPO
-#  │    │              ├── TESTS_REPO
-#  │    │              ├── AUTOTEST_ROOT
-#  │    │              └── AUTOTEST_REPO
-#  │    ├── jobs
-#  │    │    ├── matrix-reports.yml
-#  │    │    │    ├── <jobs>
-#  │    │    │    │    ├── report_job_success
-#  │    │    │    │    └── report_job_failure
-#  │    │    │    └── <used jobs>
-#  │    │    │         ├── .on_matrix
-#  │    │    │         ├── .report_job_success
-#  │    │    │         └── .report_job_failure
-#  │    │    ├── matrix.yml
-#  │    │    │    ├── <jobs>
-#  │    │    │    │    ├── .matrix_reproducer_vars
-#  │    │    │    │    ├── setup
-#  │    │    │    │    ├── opt_mpi_cuda_gcc
-#  │    │    │    │    └── opt_mpi_cuda_hypre_cuda_gcc
-#  │    │    │    ├── <used jobs>
-#  │    │    │    │    ├── .reproducer_vars
-#  │    │    │    │    ├── .on_matrix
-#  │    │    │    │    └── .job_on_matrix
-#  │    │    │    ├── <included and used files>
-#  │    │    │    │    └── tests/gitlab/build_and_test_setup  [by "setup"]
-#  │    │    │    └── <defined variables>
-#  │    │    │         └── SPEC
-#  │    │    ├── dane-reports.yml
-#  │    │    │    ├── <jobs>
-#  │    │    │    │    ├── report_job_success
-#  │    │    │    │    └── report_job_failure
-#  │    │    │    └── <used jobs>
-#  │    │    │         ├── .on_dane
-#  │    │    │         ├── .report_job_success
-#  │    │    │         └── .report_job_failure
-#  │    │    ├── dane.yml
-#  │    │    │    ├── <jobs>
-#  │    │    │    │    ├── .dane_reproducer_vars
-#  │    │    │    │    ├── setup
-#  │    │    │    │    ├── debug_ser_gcc_10
-#  │    │    │    │    ├── debug_par_gcc_10
-#  │    │    │    │    ├── opt_ser_gcc_10
-#  │    │    │    │    ├── opt_par_gcc_10
-#  │    │    │    │    ├── opt_par_gcc_10_sundials
-#  │    │    │    │    ├── opt_par_gcc_10_petsc
-#  │    │    │    │    └── opt_par_gcc_10_pumi
-#  │    │    │    ├── <used jobs>
-#  │    │    │    │    ├── .reproducer_vars
-#  │    │    │    │    ├── .on_dane
-#  │    │    │    │    └── .job_on_dane
-#  │    │    │    ├── <included and used files>
-#  │    │    │    │    └── tests/gitlab/build_and_test_setup  [by "setup"]
-#  │    │    │    └── <defined variables>
-#  │    │    │         ├── SPEC
-#  │    │    │         └── THREADS
-#  │    │    ├── tioga-reports.yml
-#  │    │    │    ├── <jobs>
-#  │    │    │    │    ├── report_job_success
-#  │    │    │    │    └── report_job_failure
-#  │    │    │    └── <used jobs>
-#  │    │    │         ├── .on_tioga
-#  │    │    │         ├── .report_job_success
-#  │    │    │         └── .report_job_failure
-#  │    │    └── tioga.yml
-#  │    │         ├── <jobs>
-#  │    │         │    ├── .tioga_reproducer_vars
-#  │    │         │    ├── setup
-#  │    │         │    └── opt_mpi_rocm_hypre_rocm
-#  │    │         ├── <used jobs>
-#  │    │         │    ├── .reproducer_vars
-#  │    │         │    ├── .on_tioga
-#  │    │         │    └── .job_on_tioga
-#  │    │         ├── <included and used files>
-#  │    │         │    └── tests/gitlab/build_and_test_setup  [by "setup"]
-#  │    │         └── <defined variables>
-#  │    │              ├── SPEC
-#  │    │              └── THREADS
-#  │    ├── scripts
-#  │    │    ├── baseline
-#  │    │    │    └── <used variables>
-#  │    │    │         ├── BASELINE_TEST
-#  │    │    │         ├── SYS_TYPE
-#  │    │    │         ├── MACHINE_NAME
-#  │    │    │         ├── CI_PROJECT_DIR
-#  │    │    │         ├── ARTIFACTS_DIR
-#  │    │    │         ├── BUILD_ROOT
-#  │    │    │         └── TPLS_DIR
-#  │    │    ├── git_try_to_push
-#  │    │    ├── rebaseline
-#  │    │    │    └── <used variables>
-#  │    │    │         ├── CI_PROJECT_DIR
-#  │    │    │         ├── ARTIFACTS_DIR
-#  │    │    │         ├── SYS_TYPE
-#  │    │    │         ├── BUILD_ROOT
-#  │    │    │         ├── MACHINE_NAME
-#  │    │    │         └── CI_PIPELINE_ID
-#  │    │    ├── report_build_and_test
-#  │    │    │    ├── <used files>
-#  │    │    │    │    ├── .gitlab/scripts/safe_create_rundir
-#  │    │    │    │    └── .gitlab/scripts/git_try_to_push
-#  │    │    │    └── <used variables>
-#  │    │    │         ├── AUTOTEST_ROOT
-#  │    │    │         ├── CI_COMMIT_REF_SLUG
-#  │    │    │         ├── CI_PROJECT_DIR
-#  │    │    │         ├── CI_PIPELINE_URL
-#  │    │    │         ├── AUTOTEST_COMMIT
-#  │    │    │         └── CI_MACHINE
-#  │    │    └── safe_create_rundir
-#  │    ├── custom-jobs-and-variables.yml
-#  │    │    ├── <jobs>
-#  │    │    │    ├── .custom_job
-#  │    │    │    ├── .reproducer_vars
-#  │    │    │    ├── .report_job_success
-#  │    │    │    └── .report_job_failure
-#  │    │    ├── <used files>
-#  │    │    │    ├── tests/gitlab/build_and_test  [in JOB_CMD]
-#  │    │    │    └── .gitlab/scripts/report_build_and_test [by .report_job_*]
-#  │    │    ├── <defined variables>
-#  │    │    │    ├── JOB_CMD
-#  │    │    │    ├── BUILD_ROOT
-#  │    │    │    ├── ALLOC_NAME
-#  │    │    │    ├── TPLS_REPO
-#  │    │    │    ├── TESTS_REPO
-#  │    │    │    ├── AUTOTEST_REPO
-#  │    │    │    ├── MFEM_DATA_REPO
-#  │    │    │    ├── ARTIFACTS_DIR: artifacts
-#  │    │    │    ├── SLURM_OVERLAP: 1
-#  │    │    │    ├── DANE_SHARED_ALLOC
-#  │    │    │    ├── DANE_JOB_ALLOC
-#  │    │    │    ├── TIOGA_SHARED_ALLOC
-#  │    │    │    ├── TIOGA_JOB_ALLOC
-#  │    │    │    └── MATRIX_JOB_ALLOC
-#  │    │    └── <used variables>
-#  │    │         ├── SPEC
-#  │    │         ├── BUILD_ROOT
-#  │    │         └── ...
-#  │    ├── dane-baseline.yml
-#  │    │    ├── <jobs>
-#  │    │    │    ├── .on_dane
-#  │    │    │    ├── baselinecheck_mfem_intel_dane
-#  │    │    │    ├── cleanup
-#  │    │    │    ├── report_baseline
-#  │    │    │    └── baselinepublish_mfem_dane
-#  │    │    ├── <included and used files>
-#  │    │    │    ├── .gitlab/custom-jobs-and-variables.yml
-#  │    │    │    ├── .gitlab/configs/setup-baseline.yml
-#  │    │    │    ├── .gitlab/scripts/rebaseline
-#  │    │    │    ├── .gitlab/scripts/baseline
-#  │    │    │    └── .gitlab/scripts/git_try_to_push
-#  │    │    ├── <defined variables>
-#  │    │    │    ├── BASELINE_TEST: baseline
-#  │    │    │    ├── MACHINE_NAME: dane
-#  │    │    │    ├── TPLS_DIR
-#  │    │    │    └── export MFEM_TEST_NP
-#  │    │    └── <used variables>
-#  │    │         ├── ON_DANE
-#  │    │         ├── AUTOTEST       [defined by .gitlab-ci.yml]
-#  │    │         ├── BUILD_ROOT     [defined by custom-jobs-and-variables.yml]
-#  │    │         ├── TPLS_DIR       [defined by this file]
-#  │    │         ├── ARTIFACTS_DIR  [defined by custom-jobs-and-variables.yml]
-#  │    │         ├── MACHINE_NAME   [defined by this file]
-#  │    │         ├── AUTOTEST_COMMIT  [defined by .gitlab-ci.yml]
-#  │    │         ├── AUTOTEST_ROOT  [defined by .gitlab-ci.yml]
-#  │    │         ├── BASELINE_TEST  [defined by this file]
-#  │    │         └── REBASELINE     [defined by .gitlab-ci.yml]
-#  │    └── subscribed-pipelines.yml
-#  │         ├── <jobs>
-#  │         │    ├── .machine-check
-#  │         │    ├── generate-job-lists
-#  │         │    ├── dane-up-check
-#  │         │    ├── dane-build-and-test
-#  │         │    ├── dane-baseline
-#  │         │    ├── tioga-up-check
-#  │         │    ├── tioga-build-and-test
-#  │         │    ├── matrix-up-check
-#  │         │    └── matrix-build-and-test
-#  │         ├── <used jobs>
-#  │         │    └── .build-and-test  [from ".gitlab-ci.yml"]
-#  │         ├── <included files>
-#  │         │    └── .gitlab/dane-baseline.yml  [by "dane-baseline"]
-#  │         └── <used variables>
-#  │              ├── GITHUB_PROJECT_ORG
-#  │              ├── GITHUB_PROJECT_NAME
-#  │              ├── AUTOTEST
-#  │              ├── AUTOTEST_COMMIT
-#  │              └── REBASELINE
-#  └── tests
-#       ├── gitlab
-#       │    ├── build_and_test
-#       │    │    ├── <builds and tests a given MFEM spec with uberenv>
-#       │    │    ├── <used files>
-#       │    │    │    ├── tests/uberenv/uberenv.py        [deps mode, cloned]
-#       │    │    │    └── tests/gitlab/get_mfem_uberenv   [deps mode]
-#       │    │    └── <used variables>
-#       │    │         ├── SYS_TYPE
-#       │    │         ├── THREADS      [num. parallel jobs to build MFEM]
-#       │    │         ├── MODULE_LIST  [modules to load]
-#       │    │         ├── CI_JOB_ID
-#       │    │         ├── USE_DEV_SHM
-#       │    │         ├── SPACK_DEBUG
-#       │    │         ├── DEBUG_MODE
-#       │    │         ├── REGISTRY_TOKEN
-#       │    │         ├── CI_REGISTRY_USER  (defined by Gitlab)
-#       │    │         ├── USER
-#       │    │         ├── CI_REGISTRY_IMAGE (defined by Gitlab)
-#       │    │         └── CI_JOB_TOKEN      (defined by Gitlab)
-#       │    ├── build_and_test_setup
-#       │    │    ├── <updates MFEM_DATA_REPO and AUTOTEST_REPO using locks>
-#       │    │    └── <used variables>
-#       │    │         ├── MFEM_DATA_REPO
-#       │    │         ├── SHARED_REPOS_DIR
-#       │    │         ├── AUTOTEST_REPO
-#       │    │         └── AUTOTEST_ROOT
-#       │    └── get_mfem_uberenv
-#       │         ├── <github.com/mfem/mfem-uberenv.git -> tests/uberenv>
-#       │         └── <defines the uberenv hash to use>
-#       └── uberenv  [cloned by tests/gitlab/get_mfem_uberenv]
-#            └── uberenv.py
-#
-# <root of radiuss/radiuss-shared-ci, ref: 'v2025.09.1'>
-#  └── pipelines
-#       ├── matrix.yml
-#       │    ├── <jobs>
-#       │    │    ├── .on_matrix
-#       │    │    ├── .matrix_reproducer_init
-#       │    │    ├── .matrix_reproducer_vars
-#       │    │    ├── .matrix_reproducer_job
-#       │    │    ├── .matrix_job_command
-#       │    │    └── .job_on_matrix
-#       │    ├── <used jobs>
-#       │    │    └── .custom_job  [from .gitlab/custom-jobs-and-variables.yml]
-#       │    └── <used variables>
-#       │         ├── ON_MATRIX
-#       │         ├── ADVANCED_JOB
-#       │         ├── ALL_TARGETS
-#       │         ├── SYS_TYPE
-#       │         ├── LLNL_SERVICE_USER
-#       │         ├── USER
-#       │         ├── GITHUB_PROJECT_NAME
-#       │         ├── GITHUB_PROJECT_ORG
-#       │         ├── MATRIX_JOB_ALLOC
-#       │         └── JOB_CMD
-#       ├── dane.yml
-#       │    ├── <jobs>
-#       │    │    ├── .on_dane
-#       │    │    ├── .dane_reproducer_init
-#       │    │    ├── .dane_reproducer_vars
-#       │    │    ├── .dane_reproducer_job
-#       │    │    ├── .dane_job_command
-#       │    │    ├── .job_on_dane
-#       │    │    ├── allocate_resources
-#       │    │    └── release_resources
-#       │    ├── <used jobs>
-#       │    │    └── .custom_job  [from .gitlab/custom-jobs-and-variables.yml]
-#       │    ├── <defined variables>
-#       │    │    └── export JOBID
-#       │    └── <used variables>
-#       │         ├── ON_DANE
-#       │         ├── ADVANCED_JOB
-#       │         ├── ALL_TARGETS
-#       │         ├── SYS_TYPE
-#       │         ├── LLNL_SERVICE_USER
-#       │         ├── USER
-#       │         ├── GITHUB_PROJECT_NAME
-#       │         ├── GITHUB_PROJECT_ORG
-#       │         ├── DANE_JOB_ALLOC
-#       │         ├── JOB_CMD
-#       │         ├── JOBID
-#       │         ├── ALLOC_NAME
-#       │         └── DANE_SHARED_ALLOC
-#       └── tioga.yml
-#            ├── <jobs>
-#            │    ├── .on_tioga
-#            │    ├── .tioga_reproducer_init
-#            │    ├── .tioga_reproducer_vars
-#            │    ├── .tioga_reproducer_job
-#            │    ├── .tioga_job_command
-#            │    ├── .job_on_tioga
-#            │    ├── allocate_resources
-#            │    └── release_resources
-#            ├── <used jobs>
-#            │    └── .custom_job  [from .gitlab/custom-jobs-and-variables.yml]
-#            ├── <defined variables>
-#            │    └── PROXY
-#            └── <used variables>
-#                 ├── ON_TIOGA
-#                 ├── ADVANCED_JOB
-#                 ├── ALL_TARGETS
-#                 ├── SYS_TYPE
-#                 ├── LLNL_SERVICE_USER
-#                 ├── USER
-#                 ├── GITHUB_PROJECT_NAME
-#                 ├── GITHUB_PROJECT_ORG
-#                 ├── TIOGA_JOB_ALLOC
-#                 ├── JOB_CMD
-#                 ├── PROXY
-#                 ├── ALLOC_NAME
-#                 └── TIOGA_SHARED_ALLOC
+# at Lawrence Livermore National Laboratory (LLNL). This entire pipeline is
+# LLNL-specific!
+
+include:
+  - project: 'lc-templates/id_tokens'
+    file: 'id_tokens.yml'
+
+# The pipeline is divided into stages. Usually, jobs in a given stage wait for
+# the preceding stages to complete before to start. However, we sometimes use
+# the "needs" keyword and express the DAG of jobs for more efficiency.
+# - We use setup and setup_baseline phases to download content outside of mfem
+#   directory.
+# - Allocate/Release is where Dane resource are allocated/released once for all.
+# - Build and Test is where we build and MFEM for multiple toolchains.
+# - Baseline_checks gathers baseline-type test suites execution
+# - Baseline_publish, only available on master, allows to update baseline
+#   results
+stages:
+  - sub-pipelines

-###############################################################################
-# We define the following GitLab pipeline variables:
 variables:
-##### LC GITLAB CONFIGURATION
  CUSTOM_CI_BUILDS_DIR: "/usr/workspace/mfem/gitlab-runner"
-
-##### PROJECT VARIABLES
  USER_CI_TOP_DIR: "${CUSTOM_CI_BUILDS_DIR}/${GITLAB_USER_LOGIN}"
  SHARED_REPOS_DIR: "${USER_CI_TOP_DIR}/repos"
  AUTOTEST_ROOT: "${SHARED_REPOS_DIR}"
+  # MFEM_DATA_DIR is setup in '.gitlab/configs/setup-build-and-test.yml' and
+  # used in '.gitlab/configs/<machine>-config.yml':
  MFEM_DATA_DIR: "${SHARED_REPOS_DIR}/mfem-data"
-# AUTOTEST: enable (ON/YES) or disable (any other value) test reporting. See
-# also AUTOTEST_COMMIT.
-  AUTOTEST: "OFF"
-# AUTOTEST_COMMIT: used only when AUTOTEST is set to ON/YES.
-# * If AUTOTEST_COMMIT is set to ON/YES, reporting jobs will commit their
-#   files to the MFEM/autotest repo.
-# * If AUTOTEST_COMMIT is NOT set to ON/YES, reporting jobs will NOT commit
-#   their files to the MFEM/autotest repo. Instead they will just show the
-#   contents of the report files and remove them.
-  AUTOTEST_COMMIT: "ON"
-# REBASELINE:
+
 # Defines the default choice for updating the saved baseline results. By default
 # the baseline can only be updated from the master branch. This variable offers
 # the option to manually ask for rebaselining from another branch if necessary.
-  REBASELINE: "OFF"
+  REBASELINE: "NO"
+  AUTOTEST: "NO"
+  # AUTOTEST_COMMIT: used only when AUTOTEST is set to YES.
+  # * If AUTOTEST_COMMIT is NOT set to NO, reporting jobs will commit their
+  #   files to the MFEM/autotest repo.
+  # * If AUTOTEST_COMMIT is set to NO, reporting jobs will NOT commit their
+  #   files to the MFEM/autotest repo. Instead they will just show the contents
+  #   of the report files and remove them.
+  AUTOTEST_COMMIT: "YES"

-##### SHARED_CI CONFIGURATION
-# Required information about GitHub repository
-  GITHUB_PROJECT_NAME: "mfem"
-  GITHUB_PROJECT_ORG: "MFEM"
-# Override the pattern describing branches that will skip the "draft PR filter
-# test".  Add protected branches here. See default value in
-# preliminary-ignore-draft-pr.yml.
-#  ALWAYS_RUN_PATTERN: ""
-
-###############################################################################
-##### High level stages
-# We organize the test-pipelines stage with sub-pipelines. Each sub-pipeline
-# corresponds to a test batch on a given machine.
-stages:
-  - prerequisites
-  - test-pipelines
-
-###############################################################################
-# Template for jobs triggering a build-and-test sub-pipeline:
-.build-and-test:
-  stage: test-pipelines
+# Trigger subpipelines:
+dane-build-and-test:
+  stage: sub-pipelines
  variables:
-    # Explicitly pass down values that are not always propagated to child
-    # pipelines, e.g. when a variable is set in the "Settings -> CI" web
-    # interface (project variables).
-    # Note: in some cases, this does not work as expected, e.g. when the
-    # variable is not re-defined in the web interface; in such cases, the child
-    # pipeline gets a definition like '${AUTOTEST}', i.e. it behaves as if
-    # AUTOTEST is undefined, even though there is a default value in
-    # .gitlab-ci.yml.
+    # Explicitly pass down values that we want to be able to set when triggering
+    # pipelines manually or using scheduling
    AUTOTEST: "${AUTOTEST}"
    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
  trigger:
-    include:
-      - local: '.gitlab/custom-jobs-and-variables.yml'
-      - project: 'radiuss/radiuss-shared-ci'
-        ref: 'v2025.09.1'
-        file: 'pipelines/${CI_MACHINE}.yml'
-      - artifact: '${CI_MACHINE}-jobs.yml'
-        job: 'generate-job-lists'
+    include: .gitlab/dane-build-and-test.yml
    strategy: depend
-    forward:
-      pipeline_variables: true

-###############################################################################
-include:
-  # Sets ID tokens for every job using `default:`
-  - project: 'lc-templates/id_tokens'
-    file: 'id_tokens.yml'
-  # [Optional] checks preliminary to running the actual CI test
-  #- project: 'radiuss/radiuss-shared-ci'
-  #  ref: 'v2025.09.1'
-  #  file: 'preliminary-ignore-draft-pr.yml'
-  # pipelines subscribed by the project
-  - local: '.gitlab/subscribed-pipelines.yml'
+dane-baseline:
+  stage: sub-pipelines
+  variables:
+    # Explicitly pass down values that we want to be able to set when triggering
+    # pipelines manually or using scheduling
+    REBASELINE: "${REBASELINE}"
+    AUTOTEST: "${AUTOTEST}"
+    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
+  trigger:
+    include: .gitlab/dane-baseline.yml
+    strategy: depend
+
+lassen-build-and-test:
+  stage: sub-pipelines
+  variables:
+    # Explicitly pass down values that we want to be able to set when triggering
+    # pipelines manually or using scheduling
+    AUTOTEST: "${AUTOTEST}"
+    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
+  trigger:
+    include: .gitlab/lassen-build-and-test.yml
+    strategy: depend
+
+corona-build-and-test:
+  stage: sub-pipelines
+  variables:
+    # Explicitly pass down values that we want to be able to set when triggering
+    # pipelines manually or using scheduling
+    AUTOTEST: "${AUTOTEST}"
+    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
+  trigger:
+    include: .gitlab/corona-build-and-test.yml
+    strategy: depend
@@ -15,9 +15,9 @@ and nightly testing on GitLab.

 ## Top level

-The root configuration file is `.gitlab-ci.yml` at the root of MFEM repo. This
-file only defines three stages, a prerequisites one, and two main stages in
-which we trigger several sub-pipelines.
+The root configuration file is `.gitlab-ci.yml` at the root of MFEM repo.
+This file only defines one stage, in which we trigger several
+sub-pipelines.

 We use sub-pipelines to isolate the test for one combination of `machine`
 and `test type`.
@@ -25,8 +25,8 @@ and `test type`.
 Machines typically include:

 * Dane: Intel Sapphire Rapids
-* Matrix: Intel Sapphire Rapids + Nvidia H100 GPU
-* Tioga: AMD MI250X GPU
+* Lassen: Power9 + Nvidia GPU
+* Corona: AMD GPU

 Test types include:

@@ -39,31 +39,9 @@ altering the scheduling, execution and displaying of the others.

 ## Sub-pipelines

-### build-and-test
-
-The build-and-test sub-pipelines leverage RADIUSS Shared CI to share most of
-the CI implementation. RADIUSS Shared CI provides a shared CI infrastructure
-vetted on most LC systems of interest and efficiently leveraging each machine
-scheduler to increase CI throughput. The maintenance of RADIUSS Shared CI is
-shared among several RADIUSS projects.
-
-Jobs for the build-and-test sub-pipelines are defined in the jobs directory.
-Because build-and-test jobs leverage Uberenv and Spack to build the
-dependencies automatically, the jobs essentially consists in a `spack spec`
-defined in the jobs files, and some scheduling parameters defined in the
-`.gitlab/custom-jobs-and-variables.yml` file.
-
-Build-and-test jobs all run the `tests/gitlab/build_and_test` script.
-
-The build-and-test pipelines are controlled by the
-`.gitlab/subscribed-pipelines.yml` which defines which machines to run on and
-implements additional features like machine availability check, and job list
-generation.
-
-### baseline
-
-Baseline sub-pipelines are described by files with names reflecting the
-machine it runs on, e.g. `dane-baseline`.
+Each file is this directory is the root configuration file for one
+sub-pipeline. The naming reflects the corresponding couple (`machine`,
+`test_type`).

 Those files define the *stages* and the *jobs* for the sub-pipeline. They
 also contain any configuration that cannot be shared. For the most part
@@ -85,11 +63,11 @@ usage function. This should be improved.

 # More testing

-## Adding a new target to a build-and-test pipeline
+## Adding a new target to a build_and_test pipeline

-`build-and-test` pipelines rely on Spack to install dependencies. Spack is
+`build_and_test` pipelines rely on Spack to install dependencies. Spack is
 driven by Uberenv which helps freezing Spack configuration: the goal being to
-point to a specific commit in Spack and isolate its configuration so that it is
+point to specific commit in Spack and isolate its configuration so that it is
 not influenced by the user environment. More documentation about this can be
 found in `tests/gitlab`.

@@ -104,7 +82,7 @@ spack spec to use. Adding a job on Dane for example resumes to:
 <job_name>:
  variables:
    SPEC: "<spack_spec>"
-  extends: .job_on_dane
+  extends: .build_and_test_on_dane
 ```

 The remaining and non trivial work is to make sure this spec is working. To
@@ -0,0 +1,40 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+include:
+  - project: 'lc-templates/id_tokens'
+    file: 'id_tokens.yml'
+
+# We define the following GitLab pipeline variables:
+variables:
+
+# The path to the shared resource between all jobs. For example, external
+# repositories like 'tests' and 'tpls' are cloned here. Also, 'tpls' is built
+# once for all targets, so that build happen here. The BUILD_ROOT is unique to
+# the pipeline, preventing any form of concurrency with other pipelines. This
+# also means that the BUILD_ROOT directory will never be cleaned.
+# TODO: add a clean-up mechanism
+  BUILD_ROOT: ${USER_CI_TOP_DIR}/${CI_PROJECT_NAME}-${MACHINE_NAME}-pipeline-${CI_PIPELINE_ID}
+
+# On LLNL's Dane, there is only one allocation shared among jobs in order to
+# save time and resource. This allocation has to be uniquely named so that we
+# are sure to retrieve it.
+  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
+
+# Git repositories used in the pipeline
+  TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
+  TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
+  AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
+  MFEM_DATA_REPO: https://github.com/mfem/data.git
+
+# Directory used to place artifacts.
+  ARTIFACTS_DIR: artifacts
+  SLURM_OVERLAP: 1
@@ -0,0 +1,59 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# GitLab pipeline configuration for the Corona machine at LLNL
+variables:
+  MACHINE_NAME: corona
+
+.on_corona:
+  tags:
+    - shell
+    - corona
+  rules:
+    # Don't run corona jobs if...
+    # Note: This makes corona an "opt-in" machine. To activate builds on corona
+    # for a given GitLab clone of MFEM, go to Setting/CI-CD/variables, and set
+    # "ON_CORONA" to "ON". An LC account on for corona is required to trigger a
+    # pipeline there.
+    - if: '$CI_COMMIT_BRANCH =~ /_cnone/ || $ON_CORONA != "ON"'
+      when: never
+    # Don't run autotest update if...
+    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
+      when: never
+    # Report success on success status
+    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
+      when: on_success
+    # Report failure on failure status
+    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
+      when: on_failure
+    # Always release resource
+    - if: '$CI_JOB_NAME =~ /release_resource/'
+      when: always
+    # Always cleanup
+    - if: '$CI_JOB_NAME =~ /cleanup/'
+      when: always
+    # Default is to run if previous stage succeeded
+    - when: on_success
+
+# Spack helped builds
+# Generic corona build job, extending build script
+.build_and_test_on_corona:
+  extends: [.on_corona]
+  stage: build_and_test
+  script:
+    # THREADS is used by 'tests/gitlab/build_and_test', run below
+    - export THREADS=12
+    - echo ${ALLOC_NAME}
+    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
+    - echo ${JOBID}
+    - echo ${MFEM_DATA_DIR}
+    - echo ${SPEC}
+    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 15 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
@@ -0,0 +1,56 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# GitLab pipelines configurations for the Dane machine at LLNL
+variables:
+  MACHINE_NAME: dane
+
+.on_dane:
+  tags:
+    - shell
+    - dane
+  rules:
+    # Don't run dane jobs if...
+    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_DANE == "OFF"'
+      when: never
+    # Don't run autotest update if...
+    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
+      when: never
+    # Report success on success status
+    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
+      when: on_success
+    # Report failure on failure status
+    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
+      when: on_failure
+    # Always release resource
+    - if: '$CI_JOB_NAME =~ /release_resource/'
+      when: always
+    # Always cleanup
+    - if: '$CI_JOB_NAME =~ /cleanup/'
+      when: always
+    # Default is to run if previous stage succeeded
+    - when: on_success
+
+# Spack helped builds
+# Generic dane build job, extending build script
+.build_and_test_on_dane:
+  extends: [.on_dane]
+  stage: build_and_test
+  script:
+    # THREADS is used by 'tests/gitlab/build_and_test', run below
+    # Dane has 224 threads/node and we run 7 separate jobs: 224=7*32
+    - export THREADS=28
+    - echo ${ALLOC_NAME}
+    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
+    - echo ${JOBID}
+    - echo ${MFEM_DATA_DIR}
+    - echo ${SPEC}
+    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) --reservation=ci -t 60 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
@@ -0,0 +1,48 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# GitLab pipelines configurations for the Lassen machine at LLNL
+variables:
+  MACHINE_NAME: lassen
+
+.on_lassen:
+  tags:
+    - shell
+    - lassen
+  rules:
+    - if: '$CI_COMMIT_BRANCH =~ /_lnone/ || $ON_LASSEN == "OFF"' #run except if ...
+      when: never
+    # Don't run autotest update if...
+    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
+      when: never
+    # Report success on success status
+    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
+      when: on_success
+    # Report failure on failure status
+    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
+      when: on_failure
+    # Always cleanup
+    - if: '$CI_JOB_NAME =~ /cleanup/'
+      when: always
+    - when: on_success
+
+# Lassen uses a different job scheduler (spectrum lsf) that does not allow
+# pre-allocation the same way slurm does.  We use the pci queue on lassen
+# to speed-up the allocation.
+.build_and_test_on_lassen:
+  extends: [.on_lassen]
+  stage: build_and_test
+  script:
+    - echo ${MFEM_DATA_DIR}
+    - echo ${SPEC}
+    # Next script uses 'THREADS': leaving it empty --> it uses 'make all -j'
+    - lalloc 1 -W 45 -q pci --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
+  needs: [setup]
@@ -0,0 +1,77 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Jobs report
+.report_job_success:
+  script:
+    - echo ${MACHINE_NAME}
+    - echo ${AUTOTEST}
+    - echo ${AUTOTEST_COMMIT}
+    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
+    - cd ${AUTOTEST_ROOT}
+    - |
+      (
+        date
+        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
+        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
+        # every 5 seconds; we may want to add a counter for the number of
+        # retries to interrupt a potential infinite loop
+        while ! flock -n 9; do
+          sleep 5
+        done
+        echo "Acquired lock on '$PWD/autotest.lock'"
+        date
+        # Report SUCCESS while holding the file lock on 'autotest.lock'.
+        # The next script uses the following environment variables:
+        # - MACHINE_NAME, AUTOTEST_ROOT, AUTOTEST_COMMIT
+        # - CI_COMMIT_REF_SLUG, CI_PROJECT_DIR, CI_PIPELINE_URL
+        # It also calls the script '.gitlab/scripts/safe_create_rundir'.
+        ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test_success
+        err=$?
+        # sleep for a period to allow NFS to propagate the above changes;
+        # clearly, there is no guarantee that other NFS clients will see the
+        # changes even after the timeout
+        sleep 10
+        exit $err
+      ) 9> autotest.lock
+
+.report_job_failure:
+  script:
+    - echo ${MACHINE_NAME}
+    - echo ${AUTOTEST}
+    - echo ${AUTOTEST_COMMIT}
+    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
+    - cd ${AUTOTEST_ROOT}
+    - |
+      (
+        date
+        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
+        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
+        # every 5 seconds; we may want to add a counter for the number of
+        # retries to interrupt a potential infinite loop
+        while ! flock -n 9; do
+          sleep 5
+        done
+        echo "Acquired lock on '$PWD/autotest.lock'"
+        date
+        # Report FAILURE while holding the file lock on 'autotest.lock'.
+        # The next script uses the following environment variables:
+        # - MACHINE_NAME, AUTOTEST_ROOT, AUTOTEST_COMMIT
+        # - CI_COMMIT_REF_SLUG, CI_PROJECT_DIR, CI_PIPELINE_URL
+        # It also calls the script '.gitlab/scripts/safe_create_rundir'.
+        ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test_failure
+        err=$?
+        # sleep for a period to allow NFS to propagate the above changes;
+        # clearly, there is no guarantee that other NFS clients will see the
+        # changes even after the timeout
+        sleep 10
+        exit $err
+      ) 9> autotest.lock
@@ -0,0 +1,90 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test
+# script then symlinks the repo to the parent directory of the MFEM source
+# directory. Unit tests that depend on the mfem/data repo will then detect that
+# this directory is present and be enabled.
+setup:
+  tags:
+    - shell
+    - dane
+  stage: setup
+  variables:
+    GIT_STRATEGY: none
+  script:
+    #
+    # Setup MFEM_DATA_DIR=${SHARED_REPOS_DIR}/mfem-data, see '.gitlab-ci.yml'
+    # and '.gitlab/configs/<machine>-config.yml'
+    #
+    - echo "MACHINE_NAME = ${MACHINE_NAME}"
+    - echo "AUTOTEST = ${AUTOTEST}"
+    - echo "AUTOTEST_COMMIT = ${AUTOTEST_COMMIT}"
+    - echo "SHARED_REPOS_DIR ${SHARED_REPOS_DIR}"
+    - mkdir -p ${SHARED_REPOS_DIR} && cd ${SHARED_REPOS_DIR}
+    - command -v flock || echo "Required command 'flock' not found"
+    - |
+      (
+        date
+        echo "Waiting to acquire lock on '$PWD/mfem-data.lock' ..."
+        # try to get an exclusive lock on fd 9 (mfem-data.lock) repeating the
+        # try every 5 seconds; we may want to add a counter for the number of
+        # retries to interrupt a potential infinite loop
+        while ! flock -n 9; do
+          sleep 5
+        done
+        echo "Acquired lock on '$PWD/mfem-data.lock'"
+        date
+        # clone/update the mfem/data repo while holding the file lock on
+        # 'mfem-data.lock'
+        err=0
+        if [[ ! -d "mfem-data" ]]; then
+          git clone ${MFEM_DATA_REPO} "mfem-data"
+        else
+          cd "mfem-data" && git pull && cd ..
+        fi || err=1
+        # sleep for a period to allow NFS to propagate the above changes;
+        # clearly, there is no guarantee that other NFS clients will see the
+        # changes even after the timeout
+        sleep 10
+        exit $err
+      ) 9> mfem-data.lock
+    #
+    # Setup ${AUTOTEST_ROOT}/autotest:
+    #
+    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
+    - mkdir -p ${AUTOTEST_ROOT} && cd ${AUTOTEST_ROOT}
+    - |
+      (
+        date
+        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
+        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
+        # every 5 seconds; we may want to add a counter for the number of
+        # retries to interrupt a potential infinite loop
+        while ! flock -n 9; do
+          sleep 5
+        done
+        echo "Acquired lock on '$PWD/autotest.lock'"
+        date
+        # clone/update the autotest repo while holding the file lock on
+        # 'autotest.lock'
+        err=0
+        if [[ ! -d "autotest" ]]; then
+          git clone ${AUTOTEST_REPO}
+        else
+          cd autotest && git pull && cd ..
+        fi || err=1
+        # sleep for a period to allow NFS to propagate the above changes;
+        # clearly, there is no guarantee that other NFS clients will see the
+        # changes even after the timeout
+        sleep 10
+        exit $err
+      ) 9> autotest.lock
@@ -0,0 +1,67 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+stages:
+  - setup
+  - allocate_resource
+  - build_and_test
+  - release_resource_and_report
+
+# Slurm shared allocation
+allocate_resource:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_corona
+  stage: allocate_resource
+  script:
+    - echo ${ALLOC_NAME}
+    - salloc --exclusive --nodes=1 --partition=mi60 --time=45 --no-shell --job-name=${ALLOC_NAME}
+  timeout: 6h
+  needs: [setup]
+
+# Build and test jobs, simply provide a spec
+rocm_gcc_8.3.1:
+  variables:
+    SPEC: "@develop%gcc@8.3.1+rocm amdgpu_target=gfx906"
+  extends: .build_and_test_on_corona
+  needs: [allocate_resource]
+
+# Release slurm allocation
+release_resource:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_corona
+  stage: release_resource_and_report
+  script:
+    - echo ${ALLOC_NAME}
+    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
+    - echo ${JOBID}
+    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
+  needs: [rocm_gcc_8.3.1]
+
+# Jobs report
+report_job_success:
+  stage: release_resource_and_report
+  extends:
+    - .on_corona
+    - .report_job_success
+
+report_job_failure:
+  stage: release_resource_and_report
+  extends:
+    - .on_corona
+    - .report_job_failure
+
+include:
+  - local: .gitlab/configs/common.yml
+  - local: .gitlab/configs/corona-config.yml
+  - local: .gitlab/configs/setup-build-and-test.yml
+  - local: .gitlab/configs/report-build-and-test.yml
@@ -1,132 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-include:
-  - project: 'lc-templates/id_tokens'
-    file: 'id_tokens.yml'
-
-# We define the following GitLab pipeline variables:
-variables:
-# Set the build-and-test command.
-# Nested variables are allowed and useful to customize the job command. We
-# protect variables with quotes so that their value may remain a string even if
-# they contain whitespaces.
-  JOB_CMD:
-    value: tests/gitlab/build_and_test --spec \"${SPEC}\" --data-dir ${MFEM_DATA_DIR} --data
-# The path to the shared resource between all jobs in the 'dane-baseline'
-# pipeline. For example, external repositories like 'tests' and 'tpls' are
-# cloned here. Also, 'tpls' is built once for all targets, so that build happens
-# here. The BUILD_ROOT is unique to the pipeline, preventing any form of
-# concurrency with other pipelines. This directory is removed by the 'cleanup'
-# stage in the 'dane-baseline' pipeline.
-  BUILD_ROOT: ${USER_CI_TOP_DIR}/${CI_PROJECT_NAME}-${CI_MACHINE}-pipeline-${CI_PIPELINE_ID}
-
-# On LLNL's dane and tioga, the 'build-and-test' pipelines creates only one
-# allocation shared among jobs in the pipeline in order to save time and
-# resources. This allocation has to be uniquely named so that we are sure to
-# retrieve it and avoid collisions.
-  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
-
-# Git repositories used in the pipelines:
-# - TPLS_REPO and TESTS_REPO are used only by the 'dane-baseline' pipeline
-# - AUTOTEST_REPO is used by all pipelines
-# - MFEM_DATA_REPO is used only by the 'build-and-test' pipelines
-  TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
-  TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
-  AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
-  MFEM_DATA_REPO: https://github.com/mfem/data.git
-
-# Directory used to place artifacts:
-# - ARTIFACTS_DIR is only used by the 'dane-baseline' pipeline
-  ARTIFACTS_DIR: artifacts
-  SLURM_OVERLAP: 1
-
-# Dane
-# Arguments for top level allocation
-  DANE_SHARED_ALLOC: "--exclusive --reservation=ci --time=60 --nodes=1"
-# Arguments for job level allocation
-# Note: We repeat the reservation, necessary when jobs are manually re-triggered.
-  DANE_JOB_ALLOC: "--reservation=ci --overlap --nodes=1"
-
-# Tioga
-# Arguments for top level allocation
-  TIOGA_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=45m --nodes=1"
-# Arguments for job level allocation
-  TIOGA_JOB_ALLOC: "--nodes=1 --begin-time=+5s"
-
-# Matrix
-# Arguments for top level allocation
-  MATRIX_SHARED_ALLOC: "-p pdebug --exclusive --time=45 --nodes=1 -G 4"
-# Arguments for job level allocation
-# Note: We repeat the reservation, necessary when jobs are manually re-triggered.
-  MATRIX_JOB_ALLOC: "--overlap --nodes=1"
-
-# Configuration shared by build and test jobs specific to this project.
-# Not all configuration can be shared. Here projects can fine tune the
-# CI behavior.
-# See Umpire for an example (export junit test reports).
-.custom_job:
-  artifacts:
-    reports:
-
-# Note: this part is not used by the 'dane-baseline' pipeline.
-# FIXME: BUILD_ROOT, TPLS_REPO, TESTS_REPO are not needed here.
-#        Also, the definition of SHARED_REPOS_DIR is wrong.
-.reproducer_vars:
-  script:
-    - |
-      echo -e "
-      # Variables \n
-      export SPEC=\"${SPEC//\"/\\\"}\" \n
-      # Directories \n
-      export BUILD_ROOT=\"\${working_dir}\" \n
-      export SHARED_REPOS_DIR=\"\${BUILD_ROOT}/..\" \n
-      export MFEM_DATA_DIR=\"\${SHARED_REPOS_DIR}/mfem-data\" \n
-      # Repositories \n
-      export TPLS_REPO=\"${TPLS_REPO//\"/\\\"}\" \n
-      export TESTS_REPO=\"${TESTS_REPO//\"/\\\"}\" \n
-      export AUTOTEST_REPO=\"${AUTOTEST_REPO//\"/\\\"}\" \n
-      export MFEM_DATA_REPO=\"${MFEM_DATA_REPO//\"/\\\"}\" \n
-      # Setup directories \n
-      ./tests/gitlab/build_and_test_setup \n
-      # Using the CI build cache is optional and requires a token. Set it like so: \n
-      # export REGISTRY_TOKEN=\"<your token here>\" \n"
-      #
-
-# Jobs report
-.report_job_success:
-  script:
-    - ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test SUCCESS
-  rules:
-    - when: on_success
-
-.report_job_failure:
-  script:
-    - ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test FAILURE
-  rules:
-    - when: on_failure
-
-# Keep the following for debugging purposes: renaming this job from
-# '.show_variables' to 'show_variables' will insert this debug job at the
-# beginning of all child pipelines.
-.show_variables:
-  tags: [shell, oslic]
-  variables:
-    GIT_STRATEGY: none
-  stage: .pre
-  script:
-    - |
-      echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
-      echo "AUTOTEST=${AUTOTEST}"
-      echo "AUTOTEST_COMMIT=${AUTOTEST_COMMIT}"
-      echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
-      # Fail the job on purpose to prevent the rest of the pipeline from running
-      false
@@ -11,7 +11,6 @@

 variables:
  BASELINE_TEST: baseline
-  MACHINE_NAME: dane

 stages:
  - setup
@@ -20,25 +19,6 @@ stages:
  - cleanup
  - baseline_publish

-.on_dane:
-  tags:
-    - shell
-    - dane
-  rules:
-    # Don't run dane jobs if...
-    - if: '$ON_DANE == "OFF"'
-      when: never
-    # Don't run autotest update if...
-    # Note: in some cases, the content of AUTOTEST can be '${AUTOTEST}', so we
-    # need to treat that value as the default value of 'OFF'.
-    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "ON" && $AUTOTEST != "YES"'
-      when: never
-    # Always cleanup
-    - if: '$CI_JOB_NAME =~ /cleanup/'
-      when: always
-    # Default is to run if previous stage succeeded
-    - when: on_success
-
 baselinecheck_mfem_intel_dane:
  extends: [.on_dane]
  stage: baseline_check
@@ -49,9 +29,6 @@ baselinecheck_mfem_intel_dane:
    #       .gitlab/configs/setup-baseline.yml.
    TPLS_DIR: ${BUILD_ROOT}/tpls
  script:
-    - echo "AUTOTEST=$AUTOTEST"
-    - echo "AUTOTEST_COMMIT=$AUTOTEST_COMMIT"
-    - echo "AUTOTEST_ROOT=$AUTOTEST_ROOT"
    - echo ${BUILD_ROOT}
    - echo ${TPLS_DIR}
    # Used by the tests in MFEM/tests, dane has 224 threads/node:
@@ -112,13 +89,7 @@ report_baseline:
          cp ${rundir}/pipeline.txt ${rundir}/autotest-email.html
        fi
        msg="GitLab CI log for ${BASELINE_TEST} on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
-        # Note: in some cases, the content of AUTOTEST_COMMIT can be
-        # '${AUTOTEST_COMMIT}', so we need to treat that value as the default
-        # value of 'ON'.
-        if [[ "$AUTOTEST_COMMIT" == '${AUTOTEST_COMMIT}' ]]; then
-          AUTOTEST_COMMIT="ON"
-        fi
-        if [[ "$AUTOTEST_COMMIT" == "ON" || "$AUTOTEST_COMMIT" == "YES" ]]; then
+        if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
          git pull && \
          git add ${rundir} && \
          git commit -m "${msg}" && \
@@ -146,8 +117,8 @@ baselinepublish_mfem_dane:
  extends: [.on_dane]
  stage: baseline_publish
  rules:
-    # - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "ON"'
-    - if: '$REBASELINE == "ON"'
+    # - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "YES"'
+    - if: '$REBASELINE == "YES"'
      when: manual
  script:
    - echo ${BUILD_ROOT}
@@ -157,5 +128,6 @@ baselinepublish_mfem_dane:
    - .gitlab/scripts/rebaseline

 include:
-  - local: .gitlab/custom-jobs-and-variables.yml
+  - local: .gitlab/configs/common.yml
+  - local: .gitlab/configs/dane-config.yml
  - local: .gitlab/configs/setup-baseline.yml
@@ -0,0 +1,94 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+stages:
+  - setup
+  - allocate_resource
+  - build_and_test
+  - release_resource_and_report
+
+# Allocate
+allocate_resource:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_dane
+  stage: allocate_resource
+  script:
+    - echo ${ALLOC_NAME}
+    - salloc --exclusive --nodes=1 --reservation=ci --time=60 --no-shell --job-name=${ALLOC_NAME}
+  timeout: 6h
+
+# GitLab jobs for the Dane machine at LLNL
+debug_ser_gcc_10:
+  variables:
+    SPEC: "%gcc@10.3.1 +debug~mpi"
+  extends: .build_and_test_on_dane
+
+debug_par_gcc_10:
+  variables:
+    SPEC: "%gcc@10.3.1 +debug+mpi"
+  extends: .build_and_test_on_dane
+
+opt_ser_gcc_10:
+  variables:
+    SPEC: "%gcc@10.3.1 ~mpi"
+  extends: .build_and_test_on_dane
+
+opt_par_gcc_10:
+  variables:
+    SPEC: "%gcc@10.3.1"
+  extends: .build_and_test_on_dane
+
+opt_par_gcc_10_sundials:
+  variables:
+    SPEC: "%gcc@10.3.1 +sundials"
+  extends: .build_and_test_on_dane
+
+opt_par_gcc_10_petsc:
+  variables:
+    SPEC: "%gcc@10.3.1 +petsc ^petsc+mumps~superlu-dist"
+  extends: .build_and_test_on_dane
+
+opt_par_gcc_10_pumi:
+  variables:
+    SPEC: "%gcc@10.3.1 +pumi"
+  extends: .build_and_test_on_dane
+
+# Release
+release_resource:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_dane
+  stage: release_resource_and_report
+  script:
+    - echo ${ALLOC_NAME}
+    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
+    - echo ${JOBID}
+    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
+
+# Jobs report
+report_job_success:
+  stage: release_resource_and_report
+  extends:
+    - .on_dane
+    - .report_job_success
+
+report_job_failure:
+  stage: release_resource_and_report
+  extends:
+    - .on_dane
+    - .report_job_failure
+
+include:
+  - local: .gitlab/configs/common.yml
+  - local: .gitlab/configs/dane-config.yml
+  - local: .gitlab/configs/setup-build-and-test.yml
+  - local: .gitlab/configs/report-build-and-test.yml
@@ -1,19 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Jobs report
-report_job_success:
-  extends: [.on_dane, .report_job_success]
-  stage: jobs-stage-3
-
-report_job_failure:
-  extends: [.on_dane, .report_job_failure]
-  stage: jobs-stage-3
@@ -1,92 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Override reproducer section to define MFEM specific variables.
-.dane_reproducer_vars:
-  script:
-    - !reference [.reproducer_vars, script]
-
-# TODO: Setup script should be defined as a bash script (but then GIT_STRATEGY
-#       cannot be "none" anymore).
-
-# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test
-# script then symlinks the repo to the parent directory of the MFEM source
-# directory. Unit tests that depend on the mfem/data repo will then detect that
-# this directory is present and be enabled.
-setup:
-  extends: .on_dane
-  stage: jobs-stage-1
-  script:
-    - ./tests/gitlab/build_and_test_setup
-
-
-########################
-# Overridden shared jobs
-########################
-# When using shared jobs, we can duplicate them here to override description and
-# add necessary changes.
-# We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS} So that
-# the comparison with the original job is easier.
-
-
-############
-# Extra jobs
-############
-# We do not recommend using ${PROJECT_<MACHINE>_VARIANTS} and
-# ${PROJECT_<MACHINE>_DEPS} in the extra jobs. There is not reason not to fully
-# describe the spec here.
-
-.mfem_job_on_dane:
-  extends: .job_on_dane
-  stage: jobs-stage-2
-  variables:
-    # Dane has 224 threads/node and we run 7 separate jobs: 224=7*32
-    THREADS: 28
-
-debug_ser_gcc_10:
-  extends: .mfem_job_on_dane
-  variables:
-    SPEC: "%gcc@10.3.1 +debug~mpi"
-
-debug_par_gcc_10:
-  extends: .mfem_job_on_dane
-  variables:
-    SPEC: "%gcc@10.3.1 +debug+mpi"
-
-opt_ser_gcc_10:
-  extends: .mfem_job_on_dane
-  variables:
-    SPEC: "%gcc@10.3.1 ~mpi"
-
-opt_par_gcc_10:
-  extends: .mfem_job_on_dane
-  variables:
-    SPEC: "%gcc@10.3.1"
-
-opt_par_gcc_10_sundials:
-  extends: .mfem_job_on_dane
-  variables:
-    SPEC: "%gcc@10.3.1 +sundials"
-
-opt_par_gcc_10_petsc:
-  extends: .mfem_job_on_dane
-  variables:
-    SPEC: "%gcc@10.3.1 +petsc ^petsc+mumps~superlu-dist"
-
-opt_par_gcc_10_pumi:
-  extends: .mfem_job_on_dane
-  variables:
-    SPEC: "%gcc@10.3.1 +pumi"
-
-opt_par_gcc_10_gslib:
-  extends: .mfem_job_on_dane
-  variables:
-    SPEC: "%gcc@10.3.1 +gslib"
@@ -1,19 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Jobs report
-report_job_success:
-  extends: [.on_matrix, .report_job_success]
-  stage: jobs-stage-3
-
-report_job_failure:
-  extends: [.on_matrix, .report_job_failure]
-  stage: jobs-stage-3
@@ -1,70 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Override reproducer section to define UMPIRE specific variables.
-.matrix_reproducer_vars:
-  script:
-    - !reference [.reproducer_vars, script]
-
-#TODO: Setup script should be defined as a bash script (but then GIT_STRATEGY cannot be "none" anymore).
-
-# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test
-# script then symlinks the repo to the parent directory of the MFEM source
-# directory. Unit tests that depend on the mfem/data repo will then detect that
-# this directory is present and be enabled.
-setup:
-  extends: .on_matrix
-  stage: jobs-stage-1
-  script:
-    - ./tests/gitlab/build_and_test_setup
-
-
-########################
-# Overridden shared jobs
-########################
-# When using shared jobs , we can duplicate them here to override description and add necessary changes.
-# We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS} So that
-# the comparison with the original job is easier.
-
-
-############
-# Extra jobs
-############
-# We do not recommend using ${PROJECT_<MACHINE>_VARIANTS} and
-# ${PROJECT_<MACHINE>_DEPS} in the extra jobs. There is not reason not to fully
-# describe the spec here.
-
-.mfem_job_on_matrix:
-  extends: .job_on_matrix
-  stage: jobs-stage-2
-  variables:
-    # We run 2 jobs on 1 node that has 112 threads
-    THREADS: 48
-    # These modules need to be consistent with the uberenv configurations:
-    MODULE_LIST: "gcc/10.3.1-magic cuda/12.9.1"
-
-allocate_resources:
-  timeout: 4h
-
-opt_mpi_cuda_gcc:
-  extends: .mfem_job_on_matrix
-  variables:
-    SPEC: "%gcc@10.3.1 +mpi +cuda cuda_arch=90"
-
-opt_mpi_cuda_hypre_cuda_gcc:
-  extends: .mfem_job_on_matrix
-  variables:
-    SPEC: "%gcc@10.3.1 +mpi +cuda cuda_arch=90 ^hypre+cuda"
-
-opt_mpi_cuda_gcc_gslib:
-  extends: .mfem_job_on_matrix
-  variables:
-    SPEC: "%gcc@10.3.1 +mpi +cuda +gslib cuda_arch=90 ^hypre+cuda"
@@ -1,20 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Jobs report
-report_job_success:
-  extends: [.on_tioga, .report_job_success]
-  stage: jobs-stage-3
-
-report_job_failure:
-  extends: [.on_tioga, .report_job_failure]
-  stage: jobs-stage-3
-
@@ -1,70 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Override reproducer section to define UMPIRE specific variables.
-.tioga_reproducer_vars:
-  script:
-    - !reference [.reproducer_vars, script]
-
-#TODO: Setup script should be defined as a bash script (but then GIT_STRATEGY cannot be "none" anymore).
-
-# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test
-# script then symlinks the repo to the parent directory of the MFEM source
-# directory. Unit tests that depend on the mfem/data repo will then detect that
-# this directory is present and be enabled.
-setup:
-  extends: .on_tioga
-  stage: jobs-stage-1
-  script:
-    - ./tests/gitlab/build_and_test_setup
-
-########################
-# Overridden shared jobs
-########################
-# When using shared jobs , we can duplicate them here to override description and add necessary changes.
-# We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS} So that
-# the comparison with the original job is easier.
-
-
-############
-# Extra jobs
-############
-# We do not recommend using ${PROJECT_<MACHINE>_VARIANTS} and
-# ${PROJECT_<MACHINE>_DEPS} in the extra jobs. There is not reason not to fully
-# describe the spec here.
-
-# Build and test jobs, simply provide a spec
-
-#.tioga_job_command:
-#  script:
-#    - echo PROXY="${PROXY}"
-#    - echo TIOGA_JOB_ALLOC="${TIOGA_JOB_ALLOC}"
-#    - "printf '#!/bin/bash\n%s\n' \"${JOB_CMD}\" > flux_script.sh"
-#    - cat flux_script.sh
-#    - ${PROXY} flux watch $( ${PROXY} flux batch -o output.stdout.type=kvs ${TIOGA_JOB_ALLOC} flux_script.sh )
-#    - rm -f flux_script.sh
-
-.mfem_job_on_tioga:
-  extends: .job_on_tioga
-  stage: jobs-stage-2
-  variables:
-    # We run 1 job on 1 node that has 64 threads
-    THREADS: 64
-
-opt_mpi_rocm_hypre_rocm:
-  extends: .mfem_job_on_tioga
-  variables:
-    SPEC: "%rocmcc@=6.3.1 +rocm amdgpu_target=gfx90a ^hypre+rocm"
-
-# cce_16_0_1:
-#   extends: .mfem_job_on_tioga
-#   variables:
-#     SPEC: "%cce@=16.0.1"
@@ -0,0 +1,44 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+stages:
+  - setup
+  - build_and_test
+  - report
+
+opt_mpi_cuda_gcc:
+  variables:
+    SPEC: "%gcc@8.3.1 +mpi +cuda cuda_arch=70"
+  extends: .build_and_test_on_lassen
+
+opt_mpi_cuda_hypre_cuda_gcc:
+  variables:
+    SPEC: "%gcc@8.3.1 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
+  extends: .build_and_test_on_lassen
+
+# Jobs report
+report_job_success:
+  stage: report
+  extends:
+    - .on_lassen
+    - .report_job_success
+
+report_job_failure:
+  stage: report
+  extends:
+    - .on_lassen
+    - .report_job_failure
+
+include:
+  - local: .gitlab/configs/common.yml
+  - local: .gitlab/configs/lassen-config.yml
+  - local: .gitlab/configs/setup-build-and-test.yml
+  - local: .gitlab/configs/report-build-and-test.yml
@@ -32,9 +32,11 @@ mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}

 # run
 if [[ "${MACHINE_NAME}" == "dane" ]]; then
-  srun --nodes=1 -t 60 --exclusive --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  salloc --nodes=1 --exclusive --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "corona" ]]; then
-  srun --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+elif [[ ${MACHINE_NAME} == "lassen" ]]; then
+  lalloc 1 -q pci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 else
  echo "Unknown machine: MACHINE_NAME=$MACHINE_NAME"
  exit 1
@@ -1,118 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-function info_msg ()
-{
-  echo "[Information:] ${1}"
-}
-
-function error_msg ()
-{
-  echo "[Error:] ${1}"
-}
-
-# Perform a report while holding a lock file to prevent concurrency on
-# the destination.
-# Usage:
-# locked_clone <report_function> <lock_name>
-function locked_report ()
-{
-  if ! command -v flock
-  then
-    error_msg "Required command 'flock' not found"
-    exit 1
-  fi
-
-  info_msg "Will report ${1} while holding a lock in ${2}"
-
-  ( date; info_msg "Waiting to acquire lock on '${PWD}/${2}.lock' ..."
-    # try to get an exclusive lock on fd 9 (mfem-data.lock) repeating the
-    # try every 5 seconds; we may want to add a counter for the number of
-    # retries to interrupt a potential infinite loop
-    while ! flock -n 9; do sleep 5; done
-    date; info_msg "Acquired lock on '${PWD}/${2}.lock'"
-
-    report ${1}
-    err=$?
-
-    # sleep for a period to allow NFS to propagate the above changes;
-    # clearly, there is no guarantee that other NFS clients will see the
-    # changes even after the timeout
-    sleep 10
-    exit $err
-  ) 9> ${2}.lock
-}
-
-function report ()
-{
-  if [[ "${1}" == "SUCCESS" ]]
-  then
-    info_msg "All the ${MACHINE_NAME} jobs passed"
-    status_msg="The 'build-and-test' jobs on ${MACHINE_NAME} were SUCCESSFUL."
-  elif [[ "${1}" == "FAILURE" ]]
-  then
-    info_msg "At least one failure on ${MACHINE_NAME}"
-    status_msg="Some 'build-and-test' jobs on ${MACHINE_NAME} FAILED."
-  else
-    error_msg "Unknown status: ${1} ... aborting"
-    exit 1
-  fi
-
-  cd ${AUTOTEST_ROOT}/autotest || \
-    { error_msg "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
-  mkdir -p ${MACHINE_NAME}
-
-  rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
-  rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir)
-
-  printf "%s\n" "${status_msg}" \
-    "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.err
-
-  msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
-
-  if [[ "${1}" == "FAILURE" ]]
-  then
-    # Create 'autotest-email.html' to indicate failure:
-    cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
-  fi
-
-  # Note: in some cases, the content of AUTOTEST_COMMIT can be
-  # '${AUTOTEST_COMMIT}', so we need to treat that value as the default
-  # value of 'ON'.
-  if [[ "$AUTOTEST_COMMIT" == '${AUTOTEST_COMMIT}' ]]; then
-    AUTOTEST_COMMIT="ON"
-  fi
-  if [[ "$AUTOTEST_COMMIT" == "ON" || "$AUTOTEST_COMMIT" == "YES" ]]; then
-    git pull && \
-    git add ${rundir} && \
-    git commit -m "${msg}" && \
-    ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
-  else
-    for file in ${rundir}/*; do
-      echo "------------------------------"
-      echo "Content of '$file'"
-      echo "******************************"
-      cat $file
-      echo "******************************"
-    done
-    rm -rf ${rundir} || true
-  fi
-}
-
-export MACHINE_NAME=${CI_MACHINE}
-info_msg "MACHINE_NAME is ${MACHINE_NAME}"
-info_msg "AUTOTEST_ROOT is ${AUTOTEST_ROOT}"
-info_msg "AUTOTEST=$AUTOTEST"
-info_msg "AUTOTEST_COMMIT=$AUTOTEST_COMMIT"
-
-cd ${AUTOTEST_ROOT} && locked_report ${1} autotest
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+echo "Runs if there was at least one failure on ${MACHINE_NAME}"
+
+cd ${AUTOTEST_ROOT}/autotest || \
+  { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
+mkdir -p ${MACHINE_NAME}
+
+rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
+rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir)
+
+printf "%s\n" "Some 'build-and-test' jobs on ${MACHINE_NAME} FAILED." \
+  "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.err
+
+msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
+
+# Create 'autotest-email.html' to indicate failure:
+cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
+
+if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
+  git pull && \
+  git add ${rundir} && \
+  git commit -m "${msg}" && \
+  ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
+else
+  for file in ${rundir}/*; do
+    echo "------------------------------"
+    echo "Content of '$file'"
+    echo "******************************"
+    cat $file
+    echo "******************************"
+  done
+  rm -rf ${rundir} || true
+fi
@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+echo "Can only run if all the ${MACHINE_NAME} jobs passed"
+
+cd ${AUTOTEST_ROOT}/autotest || \
+  { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
+mkdir -p ${MACHINE_NAME}
+
+rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
+rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir)
+
+printf "%s\n" "The 'build-and-test' jobs on ${MACHINE_NAME} were SUCCESSFUL." \
+  "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.out
+
+msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
+
+if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
+  git pull && \
+  git add ${rundir} && \
+  git commit -m "${msg}" && \
+  ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
+else
+  for file in ${rundir}/*; do
+    echo "------------------------------"
+    echo "Content of '$file'"
+    echo "******************************"
+    cat $file
+    echo "******************************"
+  done
+  rm -rf ${rundir} || true
+fi
@@ -1,130 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# The template job to test whether a machine is up.
-# Expects CI_MACHINE defined to machine name.
-.machine-check:
-  stage: prerequisites
-  tags: [shell, oslic]
-  variables:
-    GIT_STRATEGY: none
-  script:
-    - |
-      if [[ $(jq '.[env.CI_MACHINE].total_nodes_up' /usr/global/tools/lorenz/data/loginnodeStatus) == 0 ]]
-      then
-        echo -e "\e[31mNo node available on ${CI_MACHINE}\e[0m"
-        false && \
-        curl --url "https://api.github.com/repos/${GITHUB_PROJECT_ORG}/${GITHUB_PROJECT_NAME}/statuses/${CI_COMMIT_SHA}" \
-             --header 'Content-Type: application/json' \
-             --header "authorization: Bearer ${GITHUB_TOKEN}" \
-             --data "{ \"state\": \"failure\", \"target_url\": \"${CI_PIPELINE_URL}\", \"description\": \"GitLab ${CI_MACHINE} down\", \"context\": \"ci/gitlab/${CI_MACHINE}\" }"
-        exit 1
-      fi
-
-###
-# Trigger a build-and-test pipeline for a machine.
-# Comment the jobs for machines you don’t need.
-###
-
-# One job to generate the job list for all the subpipelines
-generate-job-lists:
-  stage: prerequisites
-  tags: [shell, oslic]
-  variables:
-    LOCAL_JOBS_PATH: ".gitlab/jobs"
-  script:
-    - |
-      echo "AUTOTEST=$AUTOTEST"
-      echo "AUTOTEST_COMMIT=$AUTOTEST_COMMIT"
-      echo "AUTOTEST_ROOT=$AUTOTEST_ROOT"
-    - |
-      cat ${LOCAL_JOBS_PATH}/dane.yml > dane-jobs.yml
-      if [[ ${AUTOTEST} == "ON" || ${AUTOTEST} == "YES" ]]
-      then
-          cat ${LOCAL_JOBS_PATH}/dane-reports.yml >> dane-jobs.yml
-      fi
-    - |
-      cat ${LOCAL_JOBS_PATH}/matrix.yml > matrix-jobs.yml
-      if [[ ${AUTOTEST} == "ON" || ${AUTOTEST} == "YES" ]]
-      then
-          cat ${LOCAL_JOBS_PATH}/matrix-reports.yml >> matrix-jobs.yml
-      fi
-    - |
-      cat ${LOCAL_JOBS_PATH}/tioga.yml > tioga-jobs.yml
-      if [[ ${AUTOTEST} == "ON" || ${AUTOTEST} == "YES" ]]
-      then
-          cat ${LOCAL_JOBS_PATH}/tioga-reports.yml >> tioga-jobs.yml
-      fi
-  artifacts:
-    paths:
-      - dane-jobs.yml
-      - matrix-jobs.yml
-      - tioga-jobs.yml
-
-
-# DANE
-dane-up-check:
-  variables:
-    CI_MACHINE: "dane"
-  extends: [.machine-check]
-
-dane-build-and-test:
-  variables:
-    CI_MACHINE: "dane"
-  needs: [dane-up-check, generate-job-lists]
-  extends: [.build-and-test]
-
-# DANE, MFEM Specific
-dane-baseline:
-  stage: test-pipelines
-  variables:
-    # Explicitly pass down values that are not always propagated to child
-    # pipelines, e.g. when a variable is set in the "Settings -> CI" web
-    # interface (project variables).
-    # Note: in some cases, this does not work as expected, e.g. when the
-    # variable is not re-defined in the web interface; in such cases, the child
-    # pipeline gets a definition like '${AUTOTEST}', i.e. it behaves as if
-    # AUTOTEST is undefined, even though there is a default value in
-    # .gitlab-ci.yml.
-    AUTOTEST: "${AUTOTEST}"
-    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
-  trigger:
-    include: .gitlab/dane-baseline.yml
-    strategy: depend
-    forward:
-      pipeline_variables: true
-  needs: [dane-up-check]
-
-
-# TIOGA
-tioga-up-check:
-  variables:
-    CI_MACHINE: "tioga"
-  extends: [.machine-check]
-
-tioga-build-and-test:
-  variables:
-    CI_MACHINE: "tioga"
-  needs: [tioga-up-check, generate-job-lists]
-  extends: [.build-and-test]
-
-
-# Matrix
-matrix-up-check:
-  variables:
-    CI_MACHINE: "matrix"
-  extends: [.machine-check]
-
-matrix-build-and-test:
-  variables:
-    CI_MACHINE: "matrix"
-  needs: [matrix-up-check, generate-job-lists]
-  extends: [.build-and-test]
@@ -8,114 +8,9 @@
                               https://mfem.org


-Version 4.9.1 (development)
+Version 4.8.1 (development)
 ===========================

- Policy for AI-assisted contribution added to CONTRIBUTING.md
-
-Discretization improvements
---------------------------
- Added framework for mixed systems represented by classes (Par)DarcyForm, which
-  make construction of such systems easier and offer single-line algebraic
-  reduction of the system through DarcyReduction or total flux hybridization
-  through DarcyHybridization. This is supplemented by integrators for
-  convection-diffusion or linear elasticity supporting Local/Hybridizable
-  Discontinuous Galerkin (LDG/HDG) methods. Native trace-based error estimator
-  for AMR with HDG(-like) schemes is also offered by HDGErrorEstimator.
-
- Added broken Raviart-Thomas finite element collection (BrokenRT_FECollection).
-  The finite elements from this family have the same construction as classical
-  Raviart-Thomas elements, but do not share the facet DOFs with their neighbors,
-  i.e., they are discontinuous.
-
- Added NVIDIA cuDSS library interface. Implementation examples have been 
-  added to ex1 and ex1p. See https://developer.nvidia.com/cudss for more 
-  details. Supported versions >= 0.6.0.
-
- Extend FindPointsGSLIB to support surface meshes.
-
- Replaced legacy simplex quadrature rules with symmetric positive-weight
-  rules for triangles (orders 0-25) and tetrahedra (orders 0-20). These
-  rules guarantee all-positive weights and interior quadrature points,
-  improving numerical stability. Higher orders fall back to Grundmann-Moller.
-  Triangle rules: Witherden & Vincent, Comput. Math. Appl. 69(10):1232-1241,
-  2015.
-  Tet rules (d=1-13): Witherden & Vincent (ibid).
-  Tet rules (d=14-20): Chuluunbaatar et al., Comput. Math. Appl. 124:89-97,
-  2022.
-
- Improved the gridfunction projection routines. Projections work for Scalar,
-  Vector and VectorFE, also NURBS versions. Optionally different types of
-  projections can be selected, default behaviour has not changed.
- Added grid function projection methods for trace spaces, i.e., project
-  coefficients on the mesh skeleton.   
-
- GPU-enabled partial assembly (PA) for simplicial Bernstein (H1) basis based on
-  ragged tensor algorithms in (https://doi.org/10.1137/11082539X) for mass and
-  diffusion integrators.
-
- Added methods to estimate function extremum using piecewise linear bounds +
-  recursive subdivision.
-
-Meshing improvements
--------------------
- Added PA support for TMOP's adaptive limiting functionality. Multiple
-  GridFunctions and Coefficients can be combined to form a composite term.
-
- Improved support for 1D NURBS meshes with variable order, including using
-  the patches construct for 1D NURBS meshes.
-
- Added the option to include material interfaces (faces separating elements
-  with different element attributes) as additional boundary elements, for
-  parallel visualization, e.g. with GLVis. This is supported by both the Print
-  and PrintAsOne methods of ParMesh. See ParMesh::SetPrintInterfaces().
-
-New and updated examples and miniapps
-------------------------------------
- Modified example 5(p) to demonstrate usage of (Par)DarcyForm.
-
- Added directory 'examples/hdg' with modified versions examples 5, 6, 14, 17,
-  21 and 41 to demonstrate algebraically reduced or hybridized methods for
-  (convection-)diffusion, Darcy problem and linear elasticity.
- Added directory 'miniapps/hdg' with miniapps demonstrating capabilities of the
-  framework for mixed/reduced/hybridized systems:
-  * (p)anisodiff - Demonstrates the asymptotic or steady-state heat diffusion
-                   with an anisotropic heat conductivity tensor solved with
-                   adaptive mesh refinement.
-  * (p)convdiff - Shows examples of steady/time-evolving convection-diffusion
-                  problems.
-  * mfem-logo - Randomized single-step convection-diffusion of MFEM text mask as
-                the initial condition.
- Electromagnetics/lorentz miniapp has been updated to leverage the ParticleSet
-  capability.
- Added (Complex)PRefinementMultigrid solver option in DPG miniapps.
-
-Linear and nonlinear solvers
----------------------------
- Added support for trace spaces in PRefinementTransferOperator. This is used in
-  PRefinement multigrid methods for problems posed on trace spaces (see e.g. the
-  DPG miniapps).
-
-GPU computing
-------------
- Allow specifying GPU kernel launch bounds for native and RAJA GPU backends.
-
-Miscellaneous
-------------
- Added support for general 1D Gauss-Jacobi quadrature rules and Stroud conical
-  quadrature rules on triangles and tetrahedra.
-
- Fixed signed DOF handling in parallel grid-function reading (read constructor)
-  and saving via ParGridFunction::SaveAsOne(). Simplified the process of
-  applying the DOF signs by using the new method ApplyDofSigns() in class
-  ParFiniteElementSpace -- the method will return immediately if no sign flips
-  are needed.
-
- Implemented interior and trace face assembly in ParMixedBilinearForm.
-
-Version 4.9, released on Dec 11, 2025
-=====================================
-
 Starting with this version, MFEM requires a C++17 compiler.

 Discretization improvements
@@ -124,149 +19,70 @@ Discretization improvements
  nonlinear finite element operators, based on Enzyme or dual numbers AD at
  quadrature points. These features are part of the new mfem::future namespace
  and some of the API can change in the future. See the new dFEM minimal surface
-  miniapp in the miniapps/dfem/ directory for illustration of dFEM's use. Using
-  Enzyme for AD in MFEM is tested with clang v19 and requires clang/LLVM built
-  with plugin support. See INSTALL for more details.
+  miniapp in the miniapps/dfem/ directory for illustration of dFEM's use.

- Introduced initial support for particle methods in MFEM with new classes
-  Particle, ParticleSet and ParticleVector.
-  * Particle is a convenient interface for individual particle data.
-  * ParticleSet manages and stores particle data in a struct-of-arrays form,
-    carrying particle coordinates and IDs along with an arbitrary number of
-    Vector and integer data for each particle.
-  * ParticleVector is a Vector-derived container that stores vector data for an
-    arbitrary number of particles contiguously based on specified vdim/ordering.
-  See the new particle miniapps in miniapps/gslib/ and miniapps/fluids/navier/.
-
- Added a new miniapp and specialized AMG solver (AMGF) for optimization-based
-  contact mechanics. The miniapp solves large-scale frictionless contact using a
-  self-contained Interior Point (IP) solver, mortar-based contact constraints
-  provided by Tribol. The resulting linear systems are solved with the new AMGF
-  solver (see below). Benchmark examples include the two-block, ironing, and
-  beam-sphere problems. See the miniapps/contact/ directory.
-
- Added support for boundary integration to the hyperbolic framework. Two new
-  classes BdrHyperbolicDirichletIntegrator and BoundaryHyperbolicFlowIntegrator
-  have been introduced for implementation of weak Dirichlet boundary conditions
-  with a general flux or for the linear case respectively.
-
- Added a method to compute piecewise linear bounds on high-order functions on
-  tensor-product elements.
-
- Added support for interior face integration enabling DG methods in
-  ParMixedBilinearForm, ParNonlinearForm and ParBlockNonlinearForm.
+- Using Enzyme for AD in MFEM is tested with clang v19 and requires clang/LLVM
+  built with plugin support. See INSTALL for more details.

 - In the ParMoonolith integration, added support for variational resampling of
  H1 vector fields.

- Introduced IMEX ODE solvers based on a split-operator framework. Added
-  examples ex41 and ex41p demonstrating IMEX DG/CG discretizations of the
-  convection–diffusion equation, with ex41p using DG LOR preconditioning.
+- Added support for boundary integration to the hyperbolic framework. In this
+  regard, new classes `BdrHyperbolicDirichletIntegrator` and
+  `BoundaryHyperbolicFlowIntegrator` have been introduced for implementation
+  of weak Dirichlet boundary conditions with a general flux or for the linear
+  case respectively.
+
+- Added method to compute piecewise linear bounds on high-order functions on
+  tensor-product elements.
+
+- Parallel anisotropic refinement of hexahedral meshes is now supported,
+  provided that neighboring hexahedra are not refined in conflicting directions.
+  A new ParMesh method is added to check for such conflicts, before refinement.

 Meshing improvements
 --------------------
- The TMOP kernel hierarchy has been restructured to reduce compilation time.
-  Most large kernels have been split into smaller specific kernels for each
-  metric. The directory structure has been updated with assemble, metrics, mult
-  and tools subdirectories. New kernel dispatch and specialization system has
-  also been integrated. Unit tests have been revised to ensure --all tests pass.

 - Introduced NC-patch NURBS meshes, which are conforming element-wise but allow
  for nonconforming patch topology. This new mesh format supports element
  spacing formulas for refinement, as well as local refinement factors for a
  subset of knot vectors.

+- Added support for higher order meshes in Mesh::MakeSimplicial and
+  ParMesh::MakeSimplicial.
+
 - Added a new miniapp for interpolating a surface grid of points in 3D using a
  smooth NURBS surface, that can then be sampled at arbitrary resolution while
  staying close to the original geometry. See miniapps/nurbs/nurbs_surface.

- Parallel anisotropic refinement of hexahedral meshes is now supported,
-  provided that neighboring hexahedra are not refined in conflicting directions.
-  A new ParMesh method is added to check for such conflicts, before refinement.
-
- Added support for higher order meshes in (Par)Mesh::MakeSimplicial.
-
-Linear and nonlinear solvers
----------------------------
- Added FilteredSolver: a base class for solvers with filtering. It handles
-  cases where a solver performs well except in small subspaces, by adding a
-  filtering step formulated as a subspace correction.
-
- Added AMGFSolver: a derived class of FilteredSolver, specialized for AMG with
-  Filtering (AMGF), providing robust preconditioning for linear systems arising
-  in constrained optimization problems such as frictionless contact.
-
-  Added 'GetResiduals' and 'GetFinalAbsResidualNorm' to 'HyprePCG',
-  'HypreGMRES', and 'HypreFGMRES' to get 'r' and '|r|_p'. Note that the latter
-  computes '|r|_p' from 'r' instead of returning a cached value like the
-  relative 'GetFinalResidualNorm'. These require Hypre >= 2.15.0.
-
-  Changed the default solver parameters for 'HyprePCG' to 'tol=1e-6' and
-  'max_iter=1000'. This matches the default parameters in Hypre 3.0.
-
-  Added various helper functions for querying/modifying Hypre solvers:
-  'HypreSmoother::GetType', 'HypreSmoother::GetSOROptions',
-  'HypreSmoother::GetPolyOptions', 'HypreSmoother::GetWindowParameters',
-  'HypreSmoother::IsOperatorSymmetric', 'HyprePCG::GetTol',
-  'HyprePCG::GetAbsTol', 'HyprePCG::GetMaxIter', 'HyprePCG::SetUseTwoNorm',
-  'HypreGMRES::GetTol', 'HypreGMRES::GetAbsTol', 'HypreGMRES::GetMaxIter',
-  'HypreGMRES::GetKDim', 'HypreFGMRES::GetTol', 'HypreFGMRES::GetMaxIter',
-  'HypreFGMRES::GetKDim', and 'HypreBoomerAMG::GetMaxIter'.
-
 GPU computing
 -------------
- Added the 'gpu', 'raja-gpu', and 'ceed-gpu' backend aliases/shortcuts which
-  automatically select between CUDA or HIP.
-
- Added the option to enable GPU-aware MPI in MFEM using the environment
-  variable 'MFEM_GPU_AWARE_MPI' set to any value. Setting this environment
-  variable is an alternative to calling 'Device::SetGPUAwareMPI(true)'.
-
- Implemented a GPU-accelerated matrix-free AMR derefinement GridFunction update
-  operator. This supports mixed geometry meshes and variable order spaces, and
-  is the default derefinement operator constructed by FiniteElementSpace::Update
-  and ParFiniteElementSpace::Update. The operator requires the finite element
-  space to be nonconforming.
+- The function Vector::SetSubVector(const Array<int> &, const real_t) now
+  executes on device if either the vector or the array have the device flag
+  set. This is most often used for setting constant essential boundary
+  conditions. A new function Vector::SetSubVectorHost has been added in cases
+  where host execution is always needed (e.g. when the DOFs array is small).

 - Introduced MFEM_FOREACH_THREAD_DIRECT, which directly maps loop tasks to GPU
  threads, assigning one task per thread.

- The function Vector::SetSubVector(const Array<int> &, const real_t) now
-  executes on device if either the vector or the array have the device flag
-  set. This is most often used for setting constant essential BCs. A new method,
-  SetSubVectorHost, has been added for cases where host execution is always
-  needed (e.g. when the DOFs array is small).
-
- Added GPU support in GradientGridFunction and InnerProduct Coefficient classes
-  by implementing their Project methods.
-
+- Implemented a GPU-accelerated matrix-free AMR derefinement `GridFunction`
+  update operator. This supports mixed geometry meshes and variable order
+  spaces, and is the default derefinement operator constructed by
+  `FiniteElementSpace::Update` and `ParFiniteElementSpace::Update`.
+  The operator requires `FiniteElementSpace::Nonconforming() == true`.
 - Added new method: GridFunction::GetGradients, with GPU support, for computing
  the gradients of a GridFunction on all elements.
-
- The CUDA-specific names used by some of the unit tests like 'cunit_tests' and
-  'pcunit_tests' were replaced by names using 'gpu' instead of 'c' (short for
-  CUDA) or 'cuda'. These tests automatically run the CUDA/HIP tests based on the
-  MFEM build configuration.
+- Added GPU support in GradientGridFunctionCoefficient and
+  InnerProductCoefficient by implementing their Project methods.

 New and updated examples and miniapps
 -------------------------------------
- Added the miniapps/fluids directory and moved the previous Navier and the new
-  incompressible Schrödinger flow miniapps into it.
-
- Introduced the new Incompressible Schrödinger Flow (ISF) miniapp, which models
-  inviscid fluid dynamics by solving the linear Schrödinger equation, leveraging
-  the hydrodynamical analogy to quantum mechanics.
-
- New particle-related miniapps:
-  * New transient Navier-Stokes fluid-particles solver NavierParticles in
-    miniapps/fluids/navier/navier_particles, for modeling tracer particles in
-    fluid flow, demonstrating use of the new ParticleSet class.
-  * New Navier miniapp, miniapps/fluids/navier/navier_bifurcation, showing the
-    use of NavierParticles in a 2D bifurcating channel flow.
-  * New FindPointsGSLIB miniapp, miniapps/gslib/particles_redist, showing
-    parallel-redistribution of particle data between MPI ranks.
-  * Particle visualization features in common/particles_extras for viewing
-    particle locations and trajectories (ParticleTrajectories) using GLVis.
+- Added miniapps to demonstrate an implementation of the absolute-value
+  L(1)-Jacobi preconditioners in partially assembled operators. This includes
+  Multigrid wrapper to demonstrate the effectiveness of these Jacobi-type
+  operators as smoothers.
+  These miniapps can be found in `miniapps/diag-smoothers`.

 - Added a new miniapp (meshing/mesh-bounding-boxes) that computes the bounding
  boxes for each element of a given mesh, and the bounds on the determinant of
@@ -279,44 +95,36 @@ New and updated examples and miniapps
  of a charged particle, subject to Lorentz forces, in electrostatic and/or
  magnetostatic fields as computed by the volta or tesla miniapps.

- Added miniapps to demonstrate an implementation of the absolute-value
-  l1-Jacobi preconditioners in partially assembled operators. This includes
-  Multigrid wrapper to demonstrate the effectiveness of these Jacobi-type
-  operators as smoothers. See the miniapps/diag-smoothers/ directory.
-
- Updated the mtop miniapp with a GPU enabled forward and adjoint solver for
-  isotropic linear elasticity.
-
-Miscellaneous
-------------
- Introduced MFEM_FETCH_TPLS CMake option to enable downloading, configuring,
-  and building of TPLs alongside MFEM (currently supported TPLs are hypre,
-  METIS, and GSLIB).
-
- Added quadrature function support to the VisIt and Conduit data collections.
-
- Added access to the internal parallel matrix in Par(Mixed)BilinearForm and
-  related utility methods for elimination of BCs.
-
- FindPointsGSLIB has a new constructor that accepts the mesh object and
-  internally calls the Setup() method so users do not have to. The FreeData()
-  method has also been moved to the destructor so users do not need to manually
-  free-up the memory if the destructor is called before MPI_Finalize().
-
- Added parallel Address Sanitizer, serial and parallel Undefined Behavior
-  Sanitizer and serial Memory Sanitizer GitHub actions tests on Ubuntu.
-
-API changes
+API changes:
 -----------
 - mfem::internal::tensor and mfem::internal::dual have been moved to
  mfem::future::tensor and mfem::future::dual.

- API addition: in class Operator, added virtual functions: AbsMult, and
-  AbsMultTranspose; in class Vector, added Abs and Pow.
+- API addition: in class `Operator`, added virtual functions: `AbsMult`, and
+  `AbsMultTranspose`; in class `Vector`, added `Abs` and `Pow`.

- ParBilinearForm::EliminateEssentialVDofsInRhs() has been deprecated in favor
-  of ParallelEliminateEssentialTDofsInRhs().
+Miscellaneous
+-------------
+- Added the "gpu", "raja-gpu", and "ceed-gpu" backend aliases/shortcuts which
+  automatically select between CUDA or HIP.

+- The CUDA-specific names used by some of the unit tests like 'cunit_tests' and
+  'pcunit_tests' were replaced by names using 'gpu' instead of 'c' (short for
+  CUDA) or 'cuda'. These tests automatically run the CUDA/HIP tests based on the
+  MFEM build configuration.
+
+- Added the option to enable GPU-aware MPI in MFEM using the environment
+  variable 'MFEM_GPU_AWARE_MPI' set to any value. Setting this environment
+  variable is an alternative to calling 'Device::SetGPUAwareMPI(true)'.
+
+- Added parallel Address Sanitizer, serial and parallel Undefined Behavior
+  Sanitizer and serial Memory Sanitizer GitHub actions tests on Ubuntu.
+
+- FindPointsGSLIB has a new constructor that accepts the mesh object and
+  internally calls the Setup() method so that the user does not have to.
+  The FreeData() method has also been moved to the destructor so the user does
+  not need to manually free-up the memory if the destructor is called before
+  MPI_Finalize().

 Version 4.8, released on Apr 9, 2025
 ====================================
@@ -59,7 +59,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.9.1)
+set(${PROJECT_NAME}_VERSION 4.8.1)

 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -133,49 +133,33 @@ if (MFEM_USE_CUDA)
   if (NOT CMAKE_CUDA_HOST_COMPILER)
      set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
   endif()
-   if (NOT CMAKE_CUDA_ARCHITECTURES)
-      # make CUDA_ARCH resemble the same form as CMAKE_CUDA_ARCHITECTURES
-      string(REPLACE "sm_" "" CUDA_ARCH_TMP "${CUDA_ARCH}")
-      string(REPLACE "," ";" CUDA_ARCH "${CUDA_ARCH_TMP}")
-      set(CMAKE_CUDA_ARCHITECTURES "${CUDA_ARCH}")
+   if (CMAKE_VERSION VERSION_LESS 3.18.0)
+      set(CUDA_FLAGS "-arch=${CUDA_ARCH} ${CUDA_FLAGS}")
+   elseif (NOT CMAKE_CUDA_ARCHITECTURES)
+      string(REGEX REPLACE "^sm_" "" ARCH_NUMBER "${CUDA_ARCH}")
+      if ("${CUDA_ARCH}" STREQUAL "sm_${ARCH_NUMBER}")
+         set(CMAKE_CUDA_ARCHITECTURES "${ARCH_NUMBER}")
+      else()
+         message(FATAL_ERROR "Unknown CUDA_ARCH: ${CUDA_ARCH}")
+      endif()
   else()
      set(CUDA_ARCH "CMAKE_CUDA_ARCHITECTURES: ${CMAKE_CUDA_ARCHITECTURES}")
   endif()
   message(STATUS "Using CUDA architecture: ${CUDA_ARCH}")
   enable_language(CUDA)
   if (CMAKE_VERSION VERSION_LESS 3.18.0)
-      # backup try to detect if this is clang or nvcc
-      if(CMAKE_CUDA_COMPILER MATCHES "nvcc$")
-         # nvcc
-         set(MFEM_CUDA_COMPILER_IS_NVCC ON)
-         set(CUDA_FLAGS "${CUDA_FLAGS} --expt-extended-lambda --expt-relaxed-constexpr")
-         if ("all" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}"
-             OR "native" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}"
-             OR "all-major" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}")
-            set(CUDA_FLAGS "-arch=${CMAKE_CUDA_ARCHITECTURES} ${CUDA_FLAGS}")
-         else()
-            # build -gencode sequence for multiple architectures
-            foreach(ENTRY IN LISTS CMAKE_CUDA_ARCHITECTURES)
-               set(CUDA_FLAGS
-                   "-gencode arch=compute_${ENTRY},code=sm_${ENTRY} ${CUDA_FLAGS}")
-            endforeach()
-         endif()
-      else()
-         # build cuda-gpu-arch sequence for multiple architectures
-         # does not support all/all-major/native
-         foreach(ENTRY IN LISTS CMAKE_CUDA_ARCHITECTURES)
-            set(CUDA_FLAGS "-cuda-gpu-arch=sm_${ENTRY} ${CUDA_FLAGS}")
-         endforeach()
-      endif()
+     # backup try to detect if this is clang or nvcc
+     if(CMAKE_CUDA_COMPILER MATCHES "nvcc$")
+       # nvcc
+       set(MFEM_CUDA_COMPILER_IS_NVCC ON)
+       set(CUDA_FLAGS "${CUDA_FLAGS} --expt-extended-lambda --expt-relaxed-constexpr")
+     endif()
   else()
-      # TODO: all, native, all-major require CMake 3.24+
-      # backport support for CMake 3.18 to 3.24
-      if (CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA")
-         # nvcc
-         set(MFEM_CUDA_COMPILER_IS_NVCC ON)
-         set(CUDA_FLAGS
-             "${CUDA_FLAGS} --expt-extended-lambda --expt-relaxed-constexpr")
-      endif()
+     if (CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA")
+       # nvcc
+       set(MFEM_CUDA_COMPILER_IS_NVCC ON)
+       set(CUDA_FLAGS "${CUDA_FLAGS} --expt-extended-lambda --expt-relaxed-constexpr")
+     endif()
   endif()
   set(CMAKE_CUDA_STANDARD ${CMAKE_CXX_STANDARD} CACHE STRING
      "CUDA standard to use.")
@@ -258,16 +242,10 @@ endif()

 # AMD HIP
 if (MFEM_USE_HIP)
-  if (NOT CMAKE_HIP_ARCHITECTURES)
-    if (HIP_ARCH)
-      set(CMAKE_HIP_ARCHITECTURES CACHE STRING "HIP targets to compile for" "${HIP_ARCH}")
-      set(GPU_TARGETS "${HIP_ARCH}" CACHE STRING "HIP targets to compile for" FORCE)
-    endif()
-  else()
-    set(HIP_ARCH CACHE STRING "HIP targets to compile for" "${CMAKE_HIP_ARCHITECTURES}")
-    set(GPU_TARGETS "${CMAKE_HIP_ARCHITECTURES}" CACHE STRING "HIP targets to compile for" FORCE)
+  if (HIP_ARCH)
+    message(STATUS "Using HIP architecture: ${HIP_ARCH}")
+    set(GPU_TARGETS "${HIP_ARCH}" CACHE STRING "HIP targets to compile for")
  endif()
-  message(STATUS "Using HIP architecture: ${CMAKE_HIP_ARCHITECTURES}")
  if (ROCM_PATH)
    list(INSERT CMAKE_PREFIX_PATH 0 ${ROCM_PATH})
  endif()
@@ -300,19 +278,8 @@ if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
  endif()
 endif()

-# Warn user if deprecated FETCH_TPLS is provided
-if (DEFINED FETCH_TPLS)
-  message(STATUS "Setting MFEM_FETCH_TPLS to user-provided value of FETCH_TPLS (i.e., MFEM_FETCH_TPLS=${FETCH_TPLS})")
-  set (MFEM_FETCH_TPLS FETCH_TPLS)
-  message(DEPRECATION "The use of FETCH_TPLS is deprecated and will be removed in future verison. Please use MFEM_FETCH_TPLS instead.")
-endif()
-
-  # Umpire (must be included before hypre, so hypre can use it if needed)
+# Umpire (must be included before hypre, so hypre can use it if needed)
 if (MFEM_USE_UMPIRE)
-   # umpire uses FindCUDA, which needs CMP0146=OLD in CMake >= 3.27
-   if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.27.0)
-      cmake_policy(SET CMP0146 OLD)
-   endif()
   find_package(UMPIRE REQUIRED)
 endif()

@@ -433,15 +400,6 @@ if (MFEM_USE_STRUMPACK)
  endif()
 endif()

-# cuDSS can only be enabled in CUDA
-if (MFEM_USE_CUDSS)
-  if (MFEM_USE_CUDA)
-    find_package(CUDSS REQUIRED)
-  else()
-    message(FATAL_ERROR " *** cuDSS requires that CUDA be enabled.")
-  endif()
-endif()
-
 # GnuTLS
 if (MFEM_USE_GNUTLS)
  find_package(_GnuTLS REQUIRED)
@@ -640,7 +598,7 @@ find_package(Threads REQUIRED)
 set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
    SUNDIALS PETSC SLEPC MUMPS AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB HDF5
    NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
-    ADIOS2 MKL_CPARDISO MKL_PARDISO AMGX MAGMA CUSPARSE CUBLAS CUDSS CALIPER CODIPACK
+    ADIOS2 MKL_CPARDISO MKL_PARDISO AMGX MAGMA CUSPARSE CUBLAS CALIPER CODIPACK
    BENCHMARK PARELAG TRIBOL MPI_CXX HIP HIPBLAS HIPSPARSE MOONOLITH BLITZ
    ALGOIM ENZYME CUDA::cudart)

@@ -661,8 +619,6 @@ foreach(TPL IN LISTS MFEM_TPLS)
  endif()
 endforeach(TPL)

-# reverse to remove the first instance of entries in TPL_LIBRARIES
-# so later duplicates are kept (for dependency ordering)
 list(REVERSE TPL_LIBRARIES)
 list(REMOVE_DUPLICATES TPL_LIBRARIES)
 list(REVERSE TPL_LIBRARIES)
@@ -734,7 +690,6 @@ set(MFEM_INSTALL_DIR ${CMAKE_INSTALL_PREFIX})

 # Declaring the library
 mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
-target_compile_features(mfem PUBLIC cxx_std_${CMAKE_CXX_STANDARD})
 # message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
 target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES} ${TPL_TARGETS})
 if (TPL_TARGETS)
@@ -881,12 +836,11 @@ add_dependencies(exec
 #  - https://cmake.org/Bug/view.php?id=8438

 # Add a target to copy the mfem data directory to the build directory
-# Implementable as a single copy_directory_if_different command w/ CMake >= 3.26
-file(GLOB DATA_FILES CONFIGURE_DEPENDS ${PROJECT_SOURCE_DIR}/data/*)
-add_custom_target(copy_data
-  COMMAND ${CMAKE_COMMAND} -E make_directory data
-  COMMAND ${CMAKE_COMMAND} -E copy_if_different ${DATA_FILES} data
-  COMMENT "Syncing the data directory ...")
+add_custom_command(OUTPUT data_is_copied
+  COMMAND ${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/data data
+  COMMAND ${CMAKE_COMMAND} -E touch data_is_copied
+  COMMENT "Copying the data directory ...")
+add_custom_target(copy_data DEPENDS data_is_copied)
 # Add 'copy_data' as a prerequisite for all executables, if the source and the
 # build directories are not the same.
 if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
@@ -1018,15 +972,9 @@ install(FILES
 install(EXPORT ${PROJECT_NAME_UC}Targets
    DESTINATION ${INSTALL_CMAKE_DIR})

-# Install the data directory if present, i.e. if the copy_data target is built
-install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/data
-  DESTINATION ${MFEM_INSTALL_DIR} OPTIONAL)
-
 #-------------------------------------------------------------------------------
 # Create 'config.mk' from 'config.mk.in' for the build and install locations and
 # define install rules for 'config.mk' and 'test.mk'
 #-------------------------------------------------------------------------------
-if (MFEM_USE_CUDA OR MFEM_USE_HIP)
-  option(MFEM_EXPORT_GPU_CONFIG "Export config.mk for GPU-enabled downstream packages" ON)
-endif()
+
 mfem_export_mk_files()
@@ -3,13 +3,12 @@
 </p>

 <p align="center">
-<a href="https://github.com/mfem/mfem/blob/master/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-BSD-blue.svg"></a>
-<a href="https://github.com/mfem/mfem/releases/latest"><img alt="GitHub release" src="https://img.shields.io/github/v/release/mfem/mfem"></a>
-<a href="https://github.com/mfem/mfem/actions/workflows/repo-check.yml?query=branch%3Amaster"><img alt="Repo check" src="https://github.com/mfem/mfem/actions/workflows/repo-check.yml/badge.svg?branch=master"></a>
-<a href="https://github.com/mfem/mfem/actions/workflows/mfem-analysis.yml?query=branch%3Amaster"><img alt="Build Analysis" src="https://github.com/mfem/mfem/actions/workflows/mfem-analysis.yml/badge.svg?branch=master"></a>
-<a href="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml?query=branch%3Amaster"><img alt="Builds and Tests" src="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml/badge.svg?branch=master"></a>
+<a href="https://github.com/mfem/mfem/blob/master/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-BSD-brightgreen.svg"></a>
+<a href="https://github.com/mfem/mfem/actions?query=workflow%3Arepo-check+branch%3Amaster"><img alt="Repo check" src="https://github.com/mfem/mfem/actions/workflows/repo-check.yml/badge.svg?branch=master"></a>
+<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuild-analysis+branch%3Amaster"><img alt="Build Analysis" src="https://github.com/mfem/mfem/actions/workflows/mfem-analysis.yml/badge.svg?branch=master"></a>
+<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuilds-and-tests+branch%3Amaster"><img alt="Builds and Tests" src="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml/badge.svg?branch=master"></a>
 <a href="https://ci.appveyor.com/project/mfem/mfem"><img alt="Build Status" src="https://ci.appveyor.com/api/projects/status/19non9sqm6msi2wy?svg=true"></a>
-<a href="https://docs.mfem.org/html/index.html"><img alt="Documentation" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
+<a href="https://docs.mfem.org/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
 </p>


@@ -25,14 +24,6 @@ must be made under this license.
 Note also that MFEM has a [Code of Conduct](CODE_OF_CONDUCT.md). By participating
 in the MFEM community, you agree to abide by its rules.

-## AI Policy
- Use of AI code generation in MFEM is allowed but must be disclosed, e.g. by
-  selecting the `AI-assisted` label on the PR.
- By submitting a PR, the author acknowledges that they have reviewed and
-  understand the changes they are proposing.
- PR authors are still responsible for correctness, licensing, and attribution
-  of all changes.
-
 If you plan on contributing to MFEM, consider reviewing the
 [issue tracker](https://github.com/mfem/mfem/issues) first to check if a thread
 already exists for your desired feature or the bug you ran into. Use a pull
@@ -85,7 +76,7 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
  follow the [MFEM PR Rules](#mfem-pr-rules).
 - When your contribution is fully working and ready to be reviewed, add
  the `ready-for-review` label.
- PRs are treated similarly to journal submission, with an "editor" assigning two
+- PRs are treated similarly to journal submission with an "editor" assigning two
  reviewers to evaluate the changes.
 - The reviewers have 3 weeks to evaluate the PR and work with the author to
  fix issues and implement improvements.
@@ -120,17 +111,15 @@ The MFEM source code has the following structure:
  │   ├── amgx
  │   ├── caliper
  │   ├── ginkgo
-  │   ├── hdg
  │   ├── hiop
  │   ├── jupyter
  │   ├── moonolith
  │   ├── petsc
  │   ├── pumi
  │   ├── sundials
-  │   └── superlu
+  |   └── superlu
  ├── fem
  │   ├── ceed
-  │   ├── darcy
  │   ├── dfem
  │   ├── eltrans
  │   ├── fe
@@ -150,23 +139,19 @@ The MFEM source code has the following structure:
  │   ├── adjoint
  │   ├── autodiff
  │   ├── common
-  │   ├── contact
  │   ├── dfem
-  │   ├── diag-smoothers
  │   ├── dpg
  │   ├── electromagnetics
-  │   ├── fluids
  │   ├── gslib
-  │   ├── hdg
  │   ├── hdiv-linear-solver
  │   ├── hooke
  │   ├── meshing
  │   ├── mtop
  │   ├── multidomain
+  │   ├── navier
  │   ├── nurbs
  │   ├── parelag
  │   ├── performance
-  │   ├── plasma
  │   ├── shifted
  │   ├── solvers
  │   ├── spde
@@ -197,15 +182,15 @@ respectively.

 - The main finite element classes are:
  + [`FiniteElement`](https://docs.mfem.org/html/classmfem_1_1FiniteElement.html)
-  + [`FiniteElementCollection`](https://docs.mfem.org/html/classmfem_1_1FiniteElementCollection.html)
+  + [`FiniteElementCollection`](https://docs.mfem.org/html/classmfem_1_1FiniteElement.html)
  + [`FiniteElementSpace`](https://docs.mfem.org/html/classmfem_1_1FiniteElementSpace.html)
  + [`GridFunction`](https://docs.mfem.org/html/classmfem_1_1GridFunction.html)
  + [`BilinearFormIntegrator`](https://docs.mfem.org/html/classmfem_1_1BilinearFormIntegrator.html) and [`LinearFormIntegrator`](https://docs.mfem.org/html/classmfem_1_1LinearFormIntegrator.html)
-  + [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearForm.html), [`BilinearForm`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html) and [`MixedBilinearForm`](https://docs.mfem.org/html/classmfem_1_1MixedBilinearForm.html)
+  + [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearFormIntegrator.html), [`BilinearForm`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html) and [`MixedBilinearForm`](https://docs.mfem.org/html/classmfem_1_1MixedBilinearForm.html)

 - The main linear algebra classes and sources are
  + [`Operator`](https://docs.mfem.org/html/classmfem_1_1Operator.html) and [`BilinearForm`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html)
-  + [`Vector`](https://docs.mfem.org/html/classmfem_1_1Vector.html) and [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearForm.html)
+  + [`Vector`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html) and [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearForm.html)
  + [`DenseMatrix`](https://docs.mfem.org/html/classmfem_1_1DenseMatrix.html) and [`SparseMatrix`](https://docs.mfem.org/html/classmfem_1_1SparseMatrix.html)
  + Sparse [smoothers](https://docs.mfem.org/html/sparsesmoothers_8hpp.html) and linear [solvers](https://docs.mfem.org/html/solvers_8hpp.html)

@@ -217,8 +202,8 @@ shared geometric entities between different tasks. The parallel source files
 have a `p` prefix, e.g. `pmesh.cpp` vs. the serial `mesh.cpp`.

 - The main parallel classes are
-  + [`ParMesh`](https://docs.mfem.org/html/classmfem_1_1ParMesh.html)
-  + [`ParNCMesh`](https://docs.mfem.org/html/classmfem_1_1ParNCMesh.html)
+  + [`ParMesh`](https://docs.mfem.org/html/solvers_8hpp.html)
+  + [`ParNCMesh`](https://docs.mfem.org/html/classmfem_1_1ParMesh.html)
  + [`ParFiniteElementSpace`](https://docs.mfem.org/html/classmfem_1_1ParFiniteElementSpace.html)
  + [`ParGridFunction`](https://docs.mfem.org/html/classmfem_1_1ParGridFunction.html)
  + [`ParBilinearForm`](https://docs.mfem.org/html/classmfem_1_1ParBilinearForm.html) and [`ParLinearForm`](https://docs.mfem.org/html/classmfem_1_1ParLinearForm.html)
@@ -228,14 +213,14 @@ have a `p` prefix, e.g. `pmesh.cpp` vs. the serial `mesh.cpp`.
 #### GPU and general device support

 GPU and multi-core CPU support is based on device kernels supporting different
-backends (CUDA, HIP, OCCA, RAJA, OpenMP, etc.) and an internal lightweight
+backends (CUDA, OCCA, RAJA, OpenMP, etc.) and an internal lightweight
 device/host memory manager.

 - The main device-relevant classes and sources are:
  + [`Device`](https://docs.mfem.org/html/device_8hpp.html)
  + [`MemoryManager`](https://docs.mfem.org/html/mem_manager_8hpp.html)
  + the [`mfem::forall`](https://docs.mfem.org/html/forall_8hpp.html) function
-  + the [`cuda.hpp`](https://docs.mfem.org/html/cuda_8hpp.html), [`hip.hpp`](https://docs.mfem.org/html/hip_8hpp.html) and [`occa.hpp`](https://docs.mfem.org/html/occa_8hpp.html) files
+  + the [`cuda.hpp`](https://docs.mfem.org/html/cuda_8hpp.html) and [`occa.hpp`](https://docs.mfem.org/html/occa_8hpp.html) files

 #### Utilities, building and documentation
 - The `general/` directory contains C++ classes that serve as utilities for
@@ -249,8 +234,8 @@ device/host memory manager.
 - `examples` and `miniapps` respectively gather simple and more fully-featured
  demonstrations of the usage on MFEM. They both rely on `data/` for the
  collection of meshes.
- The `tests/` directory contains a unit test suite, additional tests, and
-  benchmarks.
+- The `tests/` directory contains a unit test suite and will later contain more
+  tests that run example codes.

 See also the [code overview](https://mfem.org/code-overview/) section on the MFEM
 website.
@@ -284,8 +269,8 @@ Before you can start, you need a GitHub account, here are a few suggestions:
  the top of https://github.com/mfem.
 - Consider making your membership public by going to https://github.com/orgs/mfem/people
  and clicking on the organization visibility drop box next to your name.
- Project discussions and announcements will be posted at https://github.com/orgs/mfem/discussions,
-  tagging the `@mfem/everyone` team when appropriate.
+- Project discussions and announcements will be posted at
+  https://github.com/orgs/mfem/teams/everyone.

 #### Structure
 - The MFEM source code is in the [mfem](https://github.com/mfem/mfem)
@@ -345,12 +330,11 @@ Before you can start, you need a GitHub account, here are a few suggestions:
  - Well-designed simple code is frequently more general and powerful.
  - Lean code base is easier to understand by new collaborators.
  - New features should be added only if they are necessary or generally useful.
-  - Introduction of language constructs not currently used in MFEM should be
+  - Introduction of language constructions not currently used in MFEM should be
    justified and generally avoided (to maintain portability to various systems
    and compilers, including early access hardware).
-  - We prefer basic C++. Use C++17 features judiciously, prioritizing readability,
-    consistency with existing MFEM code, and portability to different systems,
-    compilers and device backends.
+  - We prefer basic C++ and the C++03 standard, to keep the code readable by
+    a large audience and to make sure it compiles anywhere.

 - *Keep the code general and reasonably efficient*
  - The main goal is fast prototyping for research and application development.
@@ -393,7 +377,7 @@ Before you can start, you need a GitHub account, here are a few suggestions:
 - When your branch is ready for other developers to review / comment on
  the code, create a pull request towards `mfem:master`.

- Pull requests typically have titles like:
+- Pull request typically have titles like:

     `Description [new-feature-dev]`

@@ -414,12 +398,12 @@ Before you can start, you need a GitHub account, here are a few suggestions:
 - Add a description, appropriate labels and assign yourself to the PR. The MFEM
  team will add reviewers as appropriate.

- List outstanding TODO items in the description.
+- List outstanding TODO items in the description, see PR #222 for an example.

 - When your contribution is fully working and ready to be reviewed, add
-  or request the `ready-for-review` label.
+  the `ready-for-review` label.

- PRs are treated similarly to journal submission, with an "editor" assigning
+- PRs are treated similarly to journal submission with an "editor" assigning
  two reviewers to evaluate the changes. The reviewers have 3 weeks to evaluate
  the PR and work with the author to implement improvements and fix issues.

@@ -445,7 +429,7 @@ Before you can start, you need a GitHub account, here are a few suggestions:
  checks in GitHub Actions enforce MFEM-specific rules which are explained in
  the error messages and the `tests/scripts` directory.

- Also note that the tests `branch-history` and `repo-check` found in GitHub
+- Also note that the tests `branch-history` and `repos-checks` found in GitHub
  Actions can be triggered automatically before each push using git hooks. See
  the [git hooks README](config/githooks/README.md) for a detailed explanation.

@@ -502,15 +486,15 @@ Everyone on the MFEM team can be asked to serve as a reviewer on a PR in their a

 3. To ensure the quality of the PR by making sure that the code adheres to the [Developer Guidelines](#developer-guidelines), e.g. all methods, data members, and functions have documentation, including data ownership and lifetime, new examples/miniapps have a corresponding PR in mfem/web, major features have `CHANGELOG` entries, etc.

-4. To seek help from the editors in case of difficulties.
+3. To seek help from the editors in case of difficulties.

-5. To complete the review in a timely manner: 3 weeks from assignment.
+4. To complete the review in a timely manner: 3 weeks from assignment.

-6. To test the PR thoroughly before merging in *next*. The PR author is also encouraged to perform testing and inform the reviewers about the results.
+5. To test the PR thoroughly before merging in *next*. The PR author is also encouraged to perform testing and inform the reviewers about the results.

-7. To monitor the PR impact on the testing in the *next* branch and alert the editors that the PR is ready for merging in *master*.
+6. To monitor the PR impact on the testing in the *next* branch and alert the editors that the PR is ready for merging in *master*.

-8. The review of bugfixes should be expedited proportional to their importance. The review window can be much less than three weeks in such cases.
+7. The review of bugfixes should be expedited proportional to their importance. The review window can be much less than three weeks in such cases.

 #### Responsibilities of Authors

@@ -536,30 +520,30 @@ Before a PR can be merged, it should satisfy the following:
 - [ ] Code builds.
 - [ ] Code passes `make style`.
 - [ ] Update `CHANGELOG`:
-   - [ ] Is this a new feature users need to be aware of? New or updated example or miniapp?
-   - [ ] Does it make sense to create a new section in the `CHANGELOG` to group with other related features?
+    - [ ] Is this a new feature users need to be aware of? New or updated example or miniapp?
+    - [ ] Does it make sense to create a new section in the `CHANGELOG` to group with other related features?
 - [ ] Update `INSTALL`:
-   - [ ] Has a new optional library been added? If so, what range of versions of this library are required? (*Make sure the external library is compatible with our BSD license, e.g. it is not licensed under GPL!*)
-   - [ ] Have the version ranges for any required or optional libraries changed?
-   - [ ] Does `make` or `cmake` have a new target?
-   - [ ] Did the requirements or the installation process change? *(rare)*
+    - [ ] Had a new optional library been added? If so, what range of versions of this library are required? (*Make sure the external library is compatible with our BSD license, e.g. it is not licensed under GPL!*)
+    - [ ] Have the version ranges for any required or optional libraries changed?
+    - [ ] Does `make` or `cmake` have a new target?
+    - [ ] Did the requirements or the installation process change? *(rare)*
 - [ ] Update continuous integration server configurations if necessary (e.g. with new version requirements for each of MFEM's dependencies)
-   - [ ] `.github`
-   - [ ] `.appveyor.yml`
+    - [ ] `.github`
+    - [ ] `.appveyor.yml`
 - [ ] Update `.gitignore`:
-   - [ ] Check if `make distclean; git status` shows any files that were generated from the source by the project (not an IDE) but we don't want to track in the repository.
-   - [ ] Add new patterns (just for the new files above) and re-run the above test.
+    - [ ] Check if `make distclean; git status` shows any files that were generated from the source by the project (not an IDE) but we don't want to track in the repository.
+    - [ ] Add new patterns (just for the new files above) and re-run the above test.
 - [ ] New examples:
-   - [ ] All sample runs at the top of the example source file work.
-   - [ ] Update `examples/makefile`:
+    - [ ] All sample runs at the top of the example source file work.
+    - [ ] Update `examples/makefile`:
      - [ ] Add the example code to the appropriate `SEQ_EXAMPLES` and `PAR_EXAMPLES` variables.
      - [ ] Add any files generated by it to the `clean` target.
      - [ ] Add the example binary and any files generated by it to the top-level `.gitignore` file.
-   - [ ] Update `examples/CMakeLists.txt`:
+    - [ ] Update `examples/CMakeLists.txt`:
      - [ ] Add the example code to the `ALL_EXE_SRCS` variable.
      - [ ] Make sure `THIS_TEST_OPTIONS` is set correctly for the new example.
   - [ ] List the new example in `doc/CodeDocumentation.dox`.
-   - [ ] If new examples directory (e.g. `examples/pumi`), list it in `doc/CodeDocumentation.conf.in`
+   - [ ] If new examples directory (e.g.`examples/pumi`), list it in `doc/CodeDocumentation.conf.in`
   - [ ] Companion pull request for documentation in [mfem/web](https://github.com/mfem/web) repo:
      - [ ] Update or add example-specific documentation, see e.g. the `src/examples.md`.
      - [ ] Add the description, labels and screenshots in `src/examples.md` and `src/img`.
@@ -576,13 +560,13 @@ Before a PR can be merged, it should satisfy the following:
      - [ ] Add/update the `CMakeLists.txt` file in the new miniapp directory.
      - [ ] Consider adding a new test for the new miniapp.
   - [ ] List the new miniapp in `doc/CodeDocumentation.dox`
-   - [ ] If new miniapps directory (e.g. `miniapps/nurbs`), add it to `MINIAPP_SUBDIRS` in the `makefile`.
-   - [ ] If new miniapps directory (e.g. `miniapps/nurbs`), list it in `doc/CodeDocumentation.conf.in`
+   - [ ] If new miniapps directory (e.g.`miniapps/nurbs`), add it to `MINIAPP_SUBDIRS` in the `makefile`.
+   - [ ] If new miniapps directory (e.g.`miniapps/nurbs`), list it in `doc/CodeDocumentation.conf.in`
   - [ ] Companion pull request for documentation in [mfem/web](https://github.com/mfem/web) repo:
-      - [ ] Update or add miniapp-specific documentation, see e.g. the `src/meshing.md` and `src/electromagnetics.md` files.
-      - [ ] Add the description, labels and screenshots in `src/examples.md` and `src/img`.
-      - [ ] The miniapps go at the end of the page, and are usually listed only under a specific "Application (PDE)" category.
-      - [ ] Add a short description of the miniapp in the "Extensive Examples" section of `features.md`.
+     - [ ] Update or add miniapp-specific documentation, see e.g. the `src/meshing.md` and `src/electromagnetics.md` files.
+     - [ ] Add the description, labels and screenshots in `src/examples.md` and `src/img`.
+     - [ ] The miniapps go at the end of the page, and are usually listed only under a specific "Application (PDE)" category.
+     - [ ] Add a short description of the miniapp in the "Extensive Examples" section of `features.md`.
 - [ ] New capability:
   - [ ] All new public, protected, and private classes, methods, data members, and functions have full Doxygen-style documentation in source comments. Documentation should include descriptions of member data, function arguments and return values, template parameters, and prerequisites for calling new functions.
   - [ ] Pointer arguments and return values must specify whether ownership is being transferred or lent with the call.
@@ -684,7 +668,7 @@ MFEM uses a `master`/`next`-branch workflow as described below:
 - [ ] Update URL shortlinks:
    - [ ] Create a shortlink at [http://bit.ly/](http://bit.ly/) for the release tarball, e.g. https://mfem.github.io/releases/mfem-3.1.tgz.
    - [ ] (LLNL only) Add and commit the new shortlink in the `links` and `links-mfem` files of the internal `mfem/downloads` repo.
-    - [ ] Add the new shortlinks to the MFEM package in `spack`.
+    - [ ] Add the new shortlinks to the MFEM packages in `spack`, `homebrew/science`, `VisIt`, etc.
 - [ ] Update website in `mfem/web` repo:
    - Update version and shortlinks in `src/index.md` and `src/download.md`.
    -  Use [cloc-1.62.pl](http://cloc.sourceforge.net/) and `ls -lh` to estimate the SLOC and the tarball size in `src/download.md`.
@@ -736,24 +720,22 @@ commit or push, see the [README](config/githooks/README.md) in the `config/githo
 directory.


-### GitHub Actions smoke tests
-
+### Linux and Mac smoke tests
 We use GitHub Actions to drive the default tests on the `master` and `next`
 branches. See the `.github/workflows` files and the logs at
 [https://github.com/mfem/mfem/actions](https://github.com/mfem/mfem/actions).

-GitHub Actions testing should be kept lightweight, as there is a time
-constraint on jobs. The current workflows cover Linux, macOS, and Windows
-configurations.
+Testing using GitHub Actions should be kept lightweight, as there is a time
+constraint on jobs. Two virtual machines are configured - Mac (OS X) and Linux.

 - Tests on the `master` branch are triggered whenever a PR is issued on this branch.
 - Tests on the `next` branch are currently scheduled to run each night.

-### Additional Windows smoke test

-We also use Appveyor to test building with the MS Visual C++ compiler in a Windows
-environment, as well as to test the CMake build. See the `.appveyor.yml` file
-and the build logs at
+### Windows smoke test
+We use Appveyor to test building with the MS Visual C++ compiler in a Windows
+environment, as well as to test the CMake build. See the `.appveyor` file and the
+build logs at
 [https://ci.appveyor.com/project/mfem/mfem](https://ci.appveyor.com/project/mfem/mfem).

 CMake is used to generate the MSVC Project files and drive the build.  A release
@@ -38,13 +38,14 @@ the option MFEM_USE_METIS.
 MFEM also includes support for devices such as GPUs, and programming models such
 as CUDA, HIP, OCCA, OpenMP and RAJA.

- Starting with version 4.9, MFEM requires a C++17 compiler.
+- Starting with version 4.0, MFEM requires a C++11 compiler. We recommend using
+  a newer compiler, e.g. GCC version 4.9 or higher.

 - CUDA support requires an NVIDIA GPU and an installation of the CUDA Toolkit
  https://developer.nvidia.com/cuda-toolkit

 - HIP support requires an AMD GPU and an installation of the ROCm software stack
-  https://rocm.docs.amd.com
+  https://rocmdocs.amd.com

 - OCCA support requires the OCCA library
  https://libocca.org
@@ -82,9 +83,9 @@ Serial build:
 Parallel build:
   (download hypre and METIS 4 from above URLs)
   (build METIS 4 in ../metis-4.0 relative to mfem/)
-   (for METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)
   (build hypre in ../hypre relative to mfem/)
   make parallel -j 4
+(For METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)

 CUDA build:
   make cuda -j 4
@@ -114,15 +115,15 @@ Serial build:
 Parallel build:
   (download hypre and METIS 4 from above URLs)
   (build METIS 4 in ../metis-4.0 relative to mfem/)
-   (for METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)
   (build hypre in ../hypre relative to mfem/)
   mkdir <mfem-build-dir> ; cd <mfem-build-dir>
   cmake <mfem-source-dir> -DMFEM_USE_MPI=YES
   make -j 4
+(For METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)

 Parallel build with fetching of hypre and METIS:
-   mkdir <mfem-build-dir> ; cd <mfem-build-dir>
-   cmake <mfem-source-dir> -DMFEM_USE_MPI=YES -DMFEM_FETCH_TPLS=YES
+   mkdir <mfem-buil-dir> ; cd <mfem-build-dir>
+   cmake <mfem-source-dir> -DMFEM_USE_MPI=YES -DFETCH_TPLS=YES
   make -j 4

 CUDA build:
@@ -133,8 +134,7 @@ CUDA build:

 HIP build:
   mkdir <mfem-build-dir> ; cd <mfem-build-dir>
-   cmake <mfem-source-dir> -DMFEM_USE_HIP=YES -DHIP_ARCH=gfx942 \
-         -DCMAKE_CXX_COMPILER=amdclang++ -DCMAKE_HIP_COMPILER=amdclang++
+   cmake <mfem-source-dir> -DMFEM_USE_HIP=YES -DHIP_ARCH=gfx942 -DCMAKE_CXX_COMPILER=amdclang++ -DCMAKE_HIP_COMPILER=amdclang++
   make -j 4

 Example codes (serial/parallel, depending on the build):
@@ -269,7 +269,6 @@ Compilers:
   CXX      - C++ compiler, serial build
   MPICXX   - MPI C++ compiler, parallel build
   CUDA_CXX - The CUDA compiler, 'nvcc' or 'clang++'
-   HIP_CXX  - The HIP compiler, e.g. 'hipcc'

 Compiler options:
   OPTIM_FLAGS - Options for optimized build
@@ -396,11 +395,6 @@ MFEM_USE_STRUMPACK = YES/NO
   classes. When enabled, this option uses the STRUMPACK_* library options, see
   below.

-MFEM_USE_CUDSS = YES/NO
-   Enable MFEM functionality based on the cuDSS library. When using cuDSS, CUDA
-   support must be also enabled in MFEM, i.e. MFEM_USE_CUDA=YES must be set.
-   When enabled, this option uses the CUDSS_* library options, see below.
-
 MFEM_USE_GINKGO = YES/NO
   Enable MFEM functionality based on the Ginkgo library, which provides
   iterative linear solvers and preconditioners with OpenMP, CUDA backends, see
@@ -560,13 +554,13 @@ MFEM_USE_RAJA = YES/NO
 MFEM_USE_OCCA = YES/NO
   Enables support for the OCCA library in MFEM. OCCA is an open-source library
   which aims to make it easy to program different types of devices (e.g. CPU,
-   GPU, FPGA) by providing a unified API for interacting with JIT-compiled
+   GPU, FPGA) by providing an unified API for interacting with JIT-compiled
   backends. In order to use the OCCA CUDA backend, CUDA support must be enabled
   in MFEM as well, i.e. MFEM_USE_CUDA=YES must be set.

 MFEM_USE_GSLIB = YES/NO
   Enables MFEM functionality based on the GSLIB library, and specifically its
-   FindPoints component, which provides robust algorithms to evaluate finite
+   FindPoints component, which provides a robust algorithms to evaluate finite
   element functions in a collection of points in physical space. When enabled,
   the user can use the GSLIB-FindPoints methods as shown in miniapps/gslib.

@@ -725,24 +719,13 @@ The specific libraries and their options are:
  Options: STRUMPACK_OPT, STRUMPACK_LIB.
  Versions: STRUMPACK >= 3.0.0.

- CUDSS (optional), used when MFEM_USE_CUDSS = YES. Note that CUDSS requires 
-  CUDA 12.x toolkit and the cuDSS libraries. The supported communication backend
-  is OpenMPI 4.x (default), and OpenMPI 4.x or a later version must be pre-built.
-  The source files in the cuDSS tarball provide guidance for developing custom 
-  MPI implementations.
-  URL: https://developer.nvidia.com/cudss
-       https://docs.nvidia.com/cuda/cudss/advanced_features.html#communication-layer-library-in-cudss
-  Options: CUDSS_OPT, CUDSS_LIB.
-  Versions: cuDSS >= 0.6.0.
-
- Ginkgo (optional), used when MFEM_USE_GINKGO = YES. Ginkgo may have additional
-  requirements and module-specific dependencies; see the webpage below.
+- Ginkgo (optional), used when MFEM_USE_GINKGO = YES. Note that Ginkgo needs a
+  C++ compiler that supports the C++-17 standard. For additional requirements
+  and dependencies of specific modules, see the Ginkgo webpage below.
  URL: https://ginkgo-project.github.io
  Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or
           Debug).
-  Versions: Ginkgo >= 1.9.0. When building Ginkgo with distributed support, a
-            recent version of the "develop" branch is required (1.11 as defined
-            in include/ginkgo/config.hpp).
+  Versions: Ginkgo >= 1.9.0.

 - AmgX (optional), used when MFEM_USE_AMGX = YES.
  URL: https://github.com/NVIDIA/AMGX
@@ -808,7 +791,7 @@ The specific libraries and their options are:
  Options: CONDUIT_OPT, CONDUIT_LIB.
  Versions: Conduit >= 0.3.1.

- ADIOS2 (optional), used when MFEM_USE_ADIOS2 = YES.
+- ADIOS2 (optional) used when MFEM_USE_ADIOS2 = YES.
  URL: https://adios2.readthedocs.io/
  Versions: ADIOS >= 2.5.0.

@@ -884,7 +867,7 @@ The specific libraries and their options are:
  Options: RAJA_DIR, RAJA_OPT, RAJA_LIB.
  Versions: RAJA >= 2022.10.3.

- Moonolith (optional), used when MFEM_USE_MOONOLITH = YES.
+- Moonolith (optional), use when MFEM_USE_MOONOLITH = YES.
  URL: https://bitbucket.org/zulianp/par_moonolith
  Options: MOONOLITH_DIR
  Versions: MOONOLITH >= 1.1.0.
@@ -972,7 +955,7 @@ CMAKE_BUILD_TYPE which can be set to standard values like "Debug", and "Release"
 To use a specific generator use the "-G <generator>" option of cmake:

   cmake <mfem-source-dir> -G "Xcode"
-   cmake <mfem-source-dir> -G "Visual Studio 17 2022"
+   cmake <mfem-source-dir> -G "Visual Studio 12 2013"
   cmake <mfem-source-dir> -G "MinGW Makefiles"

 With CMake it is possible to build MFEM as a shared library using the standard
@@ -1098,10 +1081,9 @@ The following options are CMake specific:
 MFEM_ENABLE_TESTING  - Enable the ctest framework for testing.
 MFEM_ENABLE_EXAMPLES - Build all of the examples by default.
 MFEM_ENABLE_MINIAPPS - Build all of the miniapps by default.
-MFEM_FETCH_TPLS      - Enable fetching of all supported third-party libraries.
-MFEM_FETCH_GSLIB     - Enable fetching of gslib.
-MFEM_FETCH_HYPRE     - Enable fetching of hypre.
-MFEM_FETCH_METIS     - Enable fetching of metis.
+FETCH_TPLS           - Enable fetching of all supported third-party libraries.
+HYPRE_FETCH          - Enable fetching of hypre.
+METIS_FETCH          - Enable fetching of metis.

 External libraries (CMake):
 ---------------------------
@@ -1167,7 +1149,6 @@ The MFEM CMake build system also provides fetching (automated building) for the
 packages/libraries listed below. Note that when fetching is enabled, any related
 auto-detection functionality is disabled.

- - GSLIB
 - HYPRE
 - METIS

@@ -1217,7 +1198,7 @@ larger problems, there are two options:
 Specific options for HIP
 ========================
 MFEM expects the `ROCM_PATH` environment variable to be set to the path of the
-ROCm install, as well as having `$ROCM_PATH/bin` in `PATH`.
+ROCM install, as well as having `$ROCM_PATH/bin` in `PATH`.

 Specific options for RAJA+HIP+MPI
 =================================
@@ -28,7 +28,6 @@ license files. These software products and their licenses are as follows:
 * AmgXWrapper (linalg/amgxsolver.{hpp,cpp}) -- MIT license
 * Catch++ (tests/unit/catch.hpp) -- Boost 1.0 license
 * Gecko (general/gecko.{cpp,hpp}) -- BSD 3-clause license
-* gslib (fem/gslib.{cpp,hpp}, mesh/bb_grid_map.{cpp,hpp}) -- BSD 3-clause license
 * Picojson (fem/picojson.h) -- Custom 2-clause license
 * TinyXML2 (general/tinyxml2.{cpp,h}) -- zlib license
 * Zstr (general/zstr.hpp) -- MIT license
@@ -35,7 +35,6 @@ set(MFEM_USE_SUITESPARSE @MFEM_USE_SUITESPARSE@)
 set(MFEM_USE_SUPERLU @MFEM_USE_SUPERLU@)
 set(MFEM_USE_MUMPS @MFEM_USE_MUMPS@)
 set(MFEM_USE_STRUMPACK @MFEM_USE_STRUMPACK@)
-set(MFEM_USE_CUDSS @MFEM_USE_CUDSS@)
 set(MFEM_USE_GINKGO @MFEM_USE_GINKGO@)
 set(MFEM_USE_AMGX @MFEM_USE_AMGX@)
 set(MFEM_USE_MAGMA @MFEM_USE_MAGMA@)
@@ -110,14 +109,6 @@ if (MFEM_USE_RAJA)
   find_dependency(RAJA)
 endif()

-if (MFEM_USE_CUDSS)
-  find_dependency(cudss)
-endif (MFEM_USE_CUDSS)
-
-if (MFEM_USE_UMPIRE)
-   find_dependency(umpire)
-endif()
-
 if (NOT TARGET mfem)
  include(${CMAKE_CURRENT_LIST_DIR}/MFEMTargets.cmake)
 endif (NOT TARGET mfem)
@@ -108,15 +108,6 @@
 // Enable MFEM functionality based on the STRUMPACK library.
 #cmakedefine MFEM_USE_STRUMPACK

-// Enable MFEM functionality based on the cuDSS library.
-#cmakedefine MFEM_USE_CUDSS
-
-// CUDSS communication layer library path
-#cmakedefine MFEM_CUDSS_COMM_LIB "@MFEM_CUDSS_COMM_LIB@"
-
-// CUDSS threading layer library path
-#cmakedefine MFEM_CUDSS_THREADING_LIB "@MFEM_CUDSS_THREADING_LIB@"
-
 // Enable functionality based on the Ginkgo library.
 #cmakedefine MFEM_USE_GINKGO

@@ -1,68 +0,0 @@
-if (NOT cudss_DIR AND CUDSS_DIR)
-  set(cudss_DIR ${CUDSS_DIR}/lib/cmake/cudss)
-endif()
-message(STATUS "Looking for CUDSS ...")
-message(STATUS "   in CUDSS_DIR = ${CUDSS_DIR}")
-message(STATUS "      cudss_DIR = ${cudss_DIR}")
-find_package(cudss)
-set(CUDSS_FOUND ${cudss_FOUND})
-set(CUDSS_LIBRARIES "cudss")
-if (CUDSS_FOUND)
-  message(STATUS
-    "Found CUDSS target: ${CUDSS_LIBRARIES} (version: ${cudss_VERSION})")
-else()
-  set(msg STATUS)
-  if (CUDSS_FIND_REQUIRED)
-    set(msg FATAL_ERROR)
-  endif()
-  message(${msg}
-    "CUDSS not found. Please set CUDSS_DIR to the install prefix.")
-endif()
-
-if(CUDSS_FOUND AND TARGET cudss)
-  get_target_property(CUDSS_LIBRARY_LOCATION cudss IMPORTED_LOCATION)
-  if(NOT CUDSS_LIBRARY_LOCATION)
-    get_target_property(CUDSS_LIBRARY_LOCATION cudss IMPORTED_LOCATION_RELEASE)
-  endif()
-  if(CUDSS_LIBRARY_LOCATION)
-    get_filename_component(CUDSS_LIBRARY_DIR "${CUDSS_LIBRARY_LOCATION}" DIRECTORY)
-  else()
-    message(WARNING "Could not determine the location of the cuDSS library.")
-  endif()
-else()
-    message(WARNING "cuDSS target not available; cannot determine library directory.")
-endif()
-
-# Set the full name of the cuDSS threading library if OpenMP is enabled.
-# The threading layer library (libcudss_mtlayer_gomp.so) is located under the
-# cuDSS library directory by default.
-if (MFEM_USE_OPENMP)
-  find_file(
-    CUDSS_THREADING_LIB
-    NAMES libcudss_mtlayer_gomp.so
-    PATHS ${CUDSS_LIBRARY_DIR}
-    NO_DEFAULT_PATH
-  )
-  if (NOT DEFINED MFEM_CUDSS_THREADING_LIB AND CUDSS_THREADING_LIB)
-    set(MFEM_CUDSS_THREADING_LIB "${CUDSS_THREADING_LIB}")
-  endif()
-  message(STATUS "CUDSS threading layer library: ${MFEM_CUDSS_THREADING_LIB}")
-endif()
-
-# Set the full name of the cuDSS communication library if MFEM use OpenMPI.
-# The communication layer library (libcudss_commlayer_mpi.so) is located under the
-# cuDSS library directory by default.
-# The communication layer library is used pre-built communication layers for OpenMPI 
-# by default.
-if (MFEM_USE_MPI)
-  find_file(
-    CUDSS_COMM_LIB
-    NAMES libcudss_commlayer_openmpi.so
-    PATHS ${CUDSS_LIBRARY_DIR}
-    NO_DEFAULT_PATH
-  )
-  if (NOT DEFINED MFEM_CUDSS_COMM_LIB AND CUDSS_COMM_LIB)
-    set(MFEM_CUDSS_COMM_LIB "${CUDSS_COMM_LIB}")
-  endif()
-  message(STATUS "CUDSS communication layer library: ${MFEM_CUDSS_COMM_LIB}")
-endif()
@@ -9,47 +9,10 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

-# Defines the following variables if fetching of TPLs is disabled (default):
+# Defines the following variables:
 #   - GSLIB_FOUND
 #   - GSLIB_LIBRARIES
 #   - GSLIB_INCLUDE_DIRS
-# otherwise, the following are defined:
-#   - GSLIB (imported library target)
-
-if (MFEM_FETCH_GSLIB OR MFEM_FETCH_TPLS)
-  enable_language(C)
-  string(TOUPPER "${CMAKE_BUILD_TYPE}" BUILD_TYPE)
-  set(GSLIB_FETCH_VERSION 1.0.9)
-  set(GSLIB_C_FLAGS ${CMAKE_C_FLAGS_${BUILD_TYPE}})
-  if (CMAKE_C_FLAGS)
-    set(GSLIB_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BUILD_TYPE}}")
-  endif()
-  if (BUILD_SHARED_LIBS)
-    set(GSLIB_C_FLAGS "${GSLIB_C_FLAGS} -fPIC")
-  endif()
-  add_library(GSLIB STATIC IMPORTED)
-  # define external project and create future include directory so it is present
-  # to pass CMake checks at end of MFEM configuration step
-  message(STATUS "Will fetch GSLIB ${GSLIB_FETCH_VERSION} to be built with ${GSLIB_C_FLAGS}")
-  set(PREFIX ${CMAKE_BINARY_DIR}/fetch/gslib)
-  include(ExternalProject)
-  ExternalProject_Add(gslib
-    GIT_REPOSITORY https://github.com/Nek5000/gslib
-    GIT_TAG v${GSLIB_FETCH_VERSION}
-    GIT_SHALLOW TRUE
-    UPDATE_DISCONNECTED TRUE
-    PREFIX ${PREFIX}
-    CONFIGURE_COMMAND ""
-    BUILD_COMMAND cd ${PREFIX}/src/gslib && $(MAKE) clean && $(MAKE) DESTDIR=${PREFIX} MPI=$<BOOL:${MFEM_USE_MPI}> "CFLAGS= ${GSLIB_C_FLAGS}"
-    INSTALL_COMMAND "")
-  file(MAKE_DIRECTORY ${PREFIX}/include)
-  # set imported library target properties
-  add_dependencies(GSLIB gslib)
-  set_target_properties(GSLIB PROPERTIES
-    IMPORTED_LOCATION ${PREFIX}/lib/libgs.a
-    INTERFACE_INCLUDE_DIRECTORIES ${PREFIX}/include)
-  return()
-endif()

 include(MfemCmakeUtilities)
 mfem_find_package(GSLIB GSLIB GSLIB_DIR "include" gslib.h "lib" gs
@@ -37,21 +37,21 @@ if (HYPRE_FOUND OR TARGET HYPRE)
  endif()
 endif()

-if (MFEM_FETCH_HYPRE OR MFEM_FETCH_TPLS)
-  set(HYPRE_FETCH_VERSION 2.33.0)
-  set(HYPRE_FETCH_TAG "v${HYPRE_FETCH_VERSION}" CACHE STRING "Tag, branch, or commit for HYPRE")
-  add_library(HYPRE STATIC IMPORTED)
-  # set options and associated dependencies
+if (HYPRE_FETCH OR FETCH_TPLS)
+  # Collect all HYPRE_ENABLE variables and pass them to hypre, assuming they are BOOL.
  set(HYPRE_CMAKE_OPTIONS "")
-  list(APPEND HYPRE_CMAKE_OPTIONS -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE})
-  # collect all HYPRE_ENABLE variables and pass them to hypre, assuming they are BOOL.
  get_cmake_property(all_vars VARIABLES)
  foreach(var ${all_vars})
    if(var MATCHES "^HYPRE_ENABLE")
      list(APPEND HYPRE_CMAKE_OPTIONS "-D${var}:BOOL=${${var}}")
    endif()
  endforeach()
-  # process all MFEM_USE variables that impact hypre
+
+  set(HYPRE_FETCH_VERSION 2.33.0)
+  set(HYPRE_FETCH_TAG "v${HYPRE_FETCH_VERSION}" CACHE STRING "Tag, branch, or commit for HYPRE")
+  add_library(HYPRE STATIC IMPORTED)
+  # set options and associated dependencies
+  list(APPEND HYPRE_CMAKE_OPTIONS -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE})
  if (MFEM_USE_CUDA)
    list(APPEND HYPRE_CMAKE_OPTIONS -DHYPRE_ENABLE_CUDA:BOOL=ON -DCMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES})
    find_package(CUDAToolkit REQUIRED)
@@ -18,7 +18,7 @@
 #   - METIS (imported library target)
 #   - METIS_VERSION_5 (cache variable)

-if (MFEM_FETCH_METIS OR MFEM_FETCH_TPLS)
+if (METIS_FETCH OR FETCH_TPLS)
  set(METIS_FETCH_VERSION 4.0.3)
  add_library(METIS STATIC IMPORTED)
  # define external project
@@ -14,12 +14,12 @@
 #   - UMPIRE_LIBRARIES
 #   - UMPIRE_INCLUDE_DIRS

-if (NOT umpire_ROOT AND UMPIRE_DIR)
-  set(umpire_ROOT ${UMPIRE_DIR})
+if (NOT umpire_DIR AND UMPIRE_DIR)
+  set(umpire_DIR ${UMPIRE_DIR}/lib/cmake/umpire)
 endif()
 message(STATUS "Looking for UMPIRE ...")
 message(STATUS "   in UMPIRE_DIR = ${UMPIRE_DIR}")
-message(STATUS "      umpire_ROOT = ${umpire_ROOT}")
+message(STATUS "      umpire_DIR = ${umpire_DIR}")
 find_package(umpire CONFIG)
 set(UMPIRE_FOUND ${umpire_FOUND})
 set(UMPIRE_LIBRARIES "umpire")
@@ -701,6 +701,7 @@ endfunction(mfem_find_library)
 # Extract compile and link options needed by the given target.
 #
 function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
+
  if (NOT TARGET ${Target})
    return()
  endif()
@@ -798,12 +799,7 @@ function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
          # message(STATUS "Lib = ${Lib}")
          # Filter-out generator expressions
          if (NOT ("${Lib}" MATCHES "^\\$"))
-            if(NOT ("${Lib}" STREQUAL "dl"))
-              list(APPEND LinkOpts "${Lib}")
-            else()
-              # for some reason libdl doesn't include the "-l"
-              list(APPEND LinkOpts "-ldl")
-            endif()
+            list(APPEND LinkOpts "${Lib}")
          endif()
        else()
          mfem_get_target_options(${Lib} COpts LOpts)
@@ -892,18 +888,9 @@ function(mfem_export_mk_files)
      set(${var} NO)
    endif()
  endforeach()
-  if (MFEM_USE_CUDA AND MFEM_EXPORT_GPU_CONFIG)
-    set(MFEM_CXX ${CMAKE_CUDA_COMPILER})
-    if(MFEM_CUDA_COMPILER_IS_NVCC)
-      set(MFEM_HOST_CXX ${CMAKE_CUDA_HOST_COMPILER})
-    else()
-      set(MFEM_HOST_CXX ${CMAKE_CXX_COMPILER})
-    endif()
-  else()
-    # mfem doesn't use enable_language(HIP)
-    set(MFEM_CXX ${CMAKE_CXX_COMPILER})
-    set(MFEM_HOST_CXX ${CMAKE_CXX_COMPILER})
-  endif()
+  # TODO: Add support for MFEM_USE_CUDA=YES
+  set(MFEM_CXX ${CMAKE_CXX_COMPILER})
+  set(MFEM_HOST_CXX ${MFEM_CXX})
  set(MFEM_CPPFLAGS "")
  get_target_property(cxx_std mfem CXX_STANDARD)
  # For now, we ignore the setting of the CXX_EXTENSIONS property. If this
@@ -913,50 +900,6 @@ function(mfem_export_mk_files)
  string(STRIP
         "${cxx_std_flag} ${CMAKE_CXX_FLAGS_${BUILD_TYPE}} ${CMAKE_CXX_FLAGS}"
         MFEM_CXXFLAGS)
-  if(MFEM_EXPORT_GPU_CONFIG)
-    if (MFEM_USE_CUDA)
-      set(MFEM_CXXFLAGS "${MFEM_CXXFLAGS} ${CMAKE_CUDA_FLAGS}")
-      if (MFEM_CUDA_COMPILER_IS_NVCC)
-        set(MFEM_CXXFLAGS "-x=cu ${MFEM_CXXFLAGS} -ccbin ${CMAKE_CXX_COMPILER} --forward-unknown-to-host-compiler")
-        # The following intentionally hides CUDA deprecation warnings
-        foreach(ENTRY IN LISTS CUDAToolkit_INCLUDE_DIRS)
-          set(MFEM_CXXFLAGS "${MFEM_CXXFLAGS} -isystem ${ENTRY}")
-        endforeach()
-        if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.18.0)
-          # architecture flags not part of CMAKE_CUDA_FLAGS
-          if ("all" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}"
-              OR "native" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}"
-              OR "all-major" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}")
-            set(MFEM_CXXFLAGS "${MFEM_CXXFLAGS} -arch=${CMAKE_CUDA_ARCHITECTURES}")
-          else()
-            foreach (ENTRY IN LISTS CMAKE_CUDA_ARCHITECTURES)
-              set(MFEM_CXXFLAGS
-                "${MFEM_CXXFLAGS} -gencode arch=compute_${ENTRY},code=sm_${ENTRY}")
-            endforeach()
-          endif()
-        endif()
-      else()
-        set(MFEM_CXXFLAGS "${MFEM_CXXFLAGS} -xcuda --cuda-path=${CUDAToolkit_LIBRARY_ROOT}")
-        if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.18.0)
-          # architecture flags not part of CMAKE_CUDA_FLAGS
-          if ("all" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}"
-              OR "native" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}"
-              OR "all-major" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}")
-            # TODO: not supported
-          else()
-            foreach(ENTRY IN LISTS CMAKE_CUDA_ARCHITECTURES)
-              set(MFEM_CXXFLAGS "-cuda-gpu-arch=sm_${ENTRY} ${MFEM_CXXFLAGS}")
-            endforeach()
-          endif()
-        endif()
-      endif()
-    elseif (MFEM_USE_HIP)
-      set(MFEM_CXXFLAGS "${MFEM_CXXFLAGS} -xhip")
-      foreach(ENTRY IN LISTS CMAKE_HIP_ARCHITECTURES)
-        set(MFEM_CXXFLAGS "--offload-arch=${ENTRY} ${MFEM_CXXFLAGS}")
-      endforeach()
-    endif()
-  endif()
  set(MFEM_TPLFLAGS "")
  foreach(dir ${TPL_INCLUDE_DIRS})
    set(MFEM_TPLFLAGS "${MFEM_TPLFLAGS} -I${dir}")
@@ -987,9 +930,6 @@ function(mfem_export_mk_files)
    set(MFEM_SHARED NO)
    set(MFEM_STATIC YES)
  endif()
-  if (MFEM_USE_CUDA)
-    set(MFEM_EXT_LIBS "${MFEM_EXT_LIBS} -lcudart")
-  endif()
  set(MFEM_BUILD_TAG "${CMAKE_SYSTEM}")
  set(MFEM_PREFIX "${CMAKE_INSTALL_PREFIX}")
  # For the next 4 variables, these are the values for the build-tree version of
@@ -998,15 +938,8 @@ function(mfem_export_mk_files)
  set(MFEM_LIB_DIR "${PROJECT_BINARY_DIR}")
  set(MFEM_TEST_MK "${PROJECT_SOURCE_DIR}/config/test.mk")
  set(MFEM_CONFIG_EXTRA "MFEM_BUILD_DIR ?= ${PROJECT_BINARY_DIR}")
-  if (MFEM_USE_CUDA AND MFEM_EXPORT_GPU_CONFIG)
-    if (MFEM_CUDA_COMPILER_IS_NVCC)
-      set(MFEM_XLINKER "-Xlinker=")
-    else()
-      set(MFEM_XLINKER "${CMAKE_CUDA_LINKER_WRAPPER_FLAG}")
-    endif()
-  else()
-    set(MFEM_XLINKER "${CMAKE_CXX_LINKER_WRAPPER_FLAG}")
-  endif()
+  # TODO: CUDA/HIP support:
+  set(MFEM_XLINKER "${CMAKE_CXX_LINKER_WRAPPER_FLAG}")
  set(MFEM_MPIEXEC ${MPIEXEC})
  if (NOT MFEM_MPIEXEC)
    set(MFEM_MPIEXEC "mpirun")
@@ -1054,21 +987,16 @@ function(mfem_export_mk_files)
    # handle interfaces (e.g., SCOREC::apf)
    if ("${lib}" MATCHES "SCOREC::.*" OR "${lib}" MATCHES "Ginkgo::.*" OR "${lib}" MATCHES "ParMoonolith::.*")
    elseif (TARGET "${lib}")
-      mfem_get_target_options(${lib} CompileOpts2 LinkOpts2)
-      # remove generator expressions
-      string(GENEX_STRIP "${CompileOpts2}" CompileOpts)
-      string(GENEX_STRIP "${LinkOpts2}" LinkOpts)
+      mfem_get_target_options(${lib} CompileOpts LinkOpts)
      # Removing duplicates may lead to issues:
      # list(REMOVE_DUPLICATES CompileOpts)
      # list(REMOVE_DUPLICATES LinkOpts)
-      # message(WARNING "${lib}[LinkOpts]: ${LinkOpts}")      
-      # message(WARNING "${lib}[CompileOpts]: ${CompileOpts}")
-      foreach(LOpt IN LISTS LinkOpts)
-        set(MFEM_EXT_LIBS "${MFEM_EXT_LIBS} ${LOpt}")
-      endforeach()
-      foreach(COpt IN LISTS CompileOpts)
-        set(MFEM_TPLFLAGS "${MFEM_TPLFLAGS} ${COpt}")
-      endforeach()
+      string(REPLACE ";" " " COpts "${CompileOpts}")
+      string(REPLACE ";" " " LOpts "${LinkOpts}")
+      # message(STATUS "${lib}[COpts]: '${COpts}'")
+      # message(STATUS "${lib}[LOpts]: '${LOpts}'")
+      set(MFEM_TPLFLAGS "${MFEM_TPLFLAGS} ${COpts}")
+      set(MFEM_EXT_LIBS "${MFEM_EXT_LIBS} ${LOpts}")
      # message(FATAL_ERROR "***** interface lib found ... exiting *****")
      # handle static and shared libs
    elseif ("${suffix}" STREQUAL "${CMAKE_SHARED_LIBRARY_SUFFIX}")
@@ -1076,7 +1004,7 @@ function(mfem_export_mk_files)
      get_filename_component(fullLibName ${lib} NAME_WE)
      string(REGEX REPLACE "^lib" "" libname ${fullLibName})
      set(MFEM_EXT_LIBS
-        "${MFEM_EXT_LIBS} ${shared_link_flag}${dir} -L${dir} -l${libname}")
+          "${MFEM_EXT_LIBS} ${shared_link_flag}${dir} -L${dir} -l${libname}")
    else()
      set(MFEM_EXT_LIBS "${MFEM_EXT_LIBS} ${lib}")
    endif()
@@ -1085,7 +1013,7 @@ function(mfem_export_mk_files)
  # Create the build-tree version of 'config.mk'
  configure_file(
    "${PROJECT_SOURCE_DIR}/config/config.mk.in"
-    "${PROJECT_BINARY_DIR}/config/config.mk" @ONLY)
+    "${PROJECT_BINARY_DIR}/config/config.mk")
  # Copy 'test.mk' from the source-tree to the build-tree
  configure_file(
    "${PROJECT_SOURCE_DIR}/config/test.mk"
@@ -1103,7 +1031,7 @@ function(mfem_export_mk_files)
  # Create the install-tree version of 'config.mk'
  configure_file(
    "${PROJECT_SOURCE_DIR}/config/config.mk.in"
-    "${PROJECT_BINARY_DIR}/config/config-install.mk" @ONLY)
+    "${PROJECT_BINARY_DIR}/config/config-install.mk")

  # Install rules for 'config.mk' and 'test.mk'
  install(FILES ${PROJECT_SOURCE_DIR}/config/test.mk
@@ -157,10 +157,4 @@ constexpr real_t operator""_r(unsigned long long v)
 #endif
 #endif // MFEM_USE_MPI not defined

-#ifndef MFEM_USE_CUDA
-#ifdef MFEM_USE_CUDSS
-#error Building with cuDSS (MFEM_USE_CUDSS=YES) requires CUDA (MFEM_USE_CUDA=YES)
-#endif
-#endif // MFEM_USE_CUDSS not defined
-
 #endif // MFEM_CONFIG_HPP
@@ -108,15 +108,6 @@
 // Enable MFEM functionality based on the STRUMPACK library.
 // #define MFEM_USE_STRUMPACK

-// Enable MFEM functionality based on the cuDSS library.
-// #define MFEM_USE_CUDSS
-
-// CUDSS communication layer library path
-// #define MFEM_CUDSS_COMM_LIB "@MFEM_CUDSS_COMM_LIB@"
-
-// CUDSS threading layer library path
-// #define MFEM_CUDSS_THREADING_LIB "@MFEM_CUDSS_THREADING_LIB@"
-
 // Enable MFEM features based on the Ginkgo library.
 // #define MFEM_USE_GINKGO

@@ -36,9 +36,6 @@ MFEM_USE_SUPERLU       = @MFEM_USE_SUPERLU@
 MFEM_USE_SUPERLU5      = @MFEM_USE_SUPERLU5@
 MFEM_USE_MUMPS         = @MFEM_USE_MUMPS@
 MFEM_USE_STRUMPACK     = @MFEM_USE_STRUMPACK@
-MFEM_USE_CUDSS         = @MFEM_USE_CUDSS@
-MFEM_CUDSS_COMM_LIB    = @MFEM_CUDSS_COMM_LIB@
-MFEM_CUDSS_THREADING_LIB = @MFEM_CUDSS_THREADING_LIB@
 MFEM_USE_GINKGO        = @MFEM_USE_GINKGO@
 MFEM_USE_AMGX          = @MFEM_USE_AMGX@
 MFEM_USE_MAGMA         = @MFEM_USE_MAGMA@
@@ -38,7 +38,6 @@ option(MFEM_USE_SUPERLU "Enable SuperLU_DIST usage" OFF)
 option(MFEM_USE_SUPERLU5 "Use the old SuperLU_DIST 5.1 version" OFF)
 option(MFEM_USE_MUMPS "Enable MUMPS usage" OFF)
 option(MFEM_USE_STRUMPACK "Enable STRUMPACK usage" OFF)
-option(MFEM_USE_CUDSS "Enable cuDSS usage" OFF)
 option(MFEM_USE_GINKGO "Enable Ginkgo usage" OFF)
 option(MFEM_USE_AMGX "Enable AmgX usage" OFF)
 option(MFEM_USE_MAGMA "Enable MAGMA usage" OFF)
@@ -92,10 +91,9 @@ option(MFEM_ENABLE_BENCHMARKS "Build all of the benchmarks" OFF)

 # Allow a user to specify fetching of certain third-party libraries instead of
 # searching for existing installations.
-option(MFEM_FETCH_TPLS "Enable fetching of all supported third-party libraries" OFF)
-option(MFEM_FETCH_GSLIB "Enable fetching of GSLIB" OFF)
-option(MFEM_FETCH_HYPRE "Enable fetching of hypre" OFF)
-option(MFEM_FETCH_METIS "Enable fetching of METIS" OFF)
+option(FETCH_TPLS "Enable fetching of all supported third-party libraries" OFF)
+option(HYPRE_FETCH "Enable fetching of hypre" OFF)
+option(METIS_FETCH "Enable fetching of METIS" OFF)

 # Setting CXX/MPICXX on the command line or in user.cmake will overwrite the
 # autodetected C++ compiler.
@@ -18,7 +18,6 @@
 # Some choices below are based on the OS type:
 NOTMAC := $(subst Darwin,,$(shell uname -s))

-ASTYLE_BIN = astyle
 ETAGS_BIN = $(shell command -v etags 2> /dev/null)
 EGREP_BIN = $(shell command -v egrep 2> /dev/null)

@@ -153,7 +152,6 @@ MFEM_USE_SUPERLU       = NO
 MFEM_USE_SUPERLU5      = NO
 MFEM_USE_MUMPS         = NO
 MFEM_USE_STRUMPACK     = NO
-MFEM_USE_CUDSS         = NO
 MFEM_USE_GINKGO        = NO
 MFEM_USE_AMGX          = NO
 MFEM_USE_MAGMA         = NO
@@ -369,19 +367,6 @@ STRUMPACK_OPT = -I$(STRUMPACK_DIR)/include $(SCOTCH_OPT)
 STRUMPACK_LIB = -L$(STRUMPACK_DIR)/lib -lstrumpack $(MPI_FORTRAN_LIB)\
 $(SCOTCH_LIB) $(SCALAPACK_LIB)

-# CUDSS library configuration
-CUDSS_DIR         = @MFEM_DIR@/../cudss
-CUDSS_INCLUDE_DIR = $(CUDSS_DIR)/include
-CUDSS_LIBRARY_DIR = $(CUDSS_DIR)/lib
-CUDSS_OPT         = -I$(CUDSS_INCLUDE_DIR)
-CUDSS_LIB         = \
- $(XLINKER)-rpath,$(CUDSS_LIBRARY_DIR) -L$(CUDSS_LIBRARY_DIR) -lcudss
-# The cuDSS communication and threading libraries. 
-MFEM_CUDSS_COMM_LIB = $(abspath $(wildcard $(or $(CUDSS_COMM_LIB),\
-   $(subst @MFEM_DIR@,$(MFEM_DIR), $(CUDSS_LIBRARY_DIR)/libcudss_commlayer_openmpi.so))))
-MFEM_CUDSS_THREADING_LIB = $(abspath $(wildcard $(or $(CUDSS_THREADING_LIB),\
-   $(subst @MFEM_DIR@,$(MFEM_DIR),$(CUDSS_LIBRARY_DIR)/libcudss_mtlayer_gomp.so))))
-
 # Ginkgo library configuration
 GINKGO_DIR = @MFEM_DIR@/../ginkgo/install
 GINKGO_SEARCH_DIR = $(subst @MFEM_DIR@,$(MFEM_DIR),$(GINKGO_DIR))
@@ -422,7 +407,7 @@ AMGX_LIB = -L$(AMGX_DIR)/lib -lamgx -lcusparse -lcusolver -lcublas -lnvToolsExt
 # MAGMA library configuration
 MAGMA_DIR = @MFEM_DIR@/../magma
 MAGMA_OPT = -I$(MAGMA_DIR)/include
-MAGMA_LIB = -L$(MAGMA_DIR)/lib -l:libmagma.a $(LAPACK_LIB)
+MAGMA_LIB = -L$(MAGMA_DIR)/lib -l:libmagma.a -lcublas -lcusparse $(LAPACK_LIB)

 # GnuTLS library configuration
 GNUTLS_OPT =
@@ -635,7 +620,7 @@ PARELAG_LIB = -L$(PARELAG_DIR)/build/src -lParELAG
 AXOM_DIR = @MFEM_DIR@/../axom
 TRIBOL_DIR = @MFEM_DIR@/../tribol
 TRIBOL_OPT = -I$(TRIBOL_DIR)/include -I$(AXOM_DIR)/include
-TRIBOL_LIB = -L$(TRIBOL_DIR)/lib -ltribol -ltribol_shared -lredecomp -L$(AXOM_DIR)/lib -laxom_mint\
+TRIBOL_LIB = -L$(TRIBOL_DIR)/lib -ltribol -lredecomp -L$(AXOM_DIR)/lib -laxom_mint\
   -laxom_slam -laxom_slic -laxom_core

 # Enzyme configuration
@@ -101,7 +101,7 @@ $ cd ../miniapps
 $ ls
 CMakeLists.txt  common            meshing  nurbs        shifted  toys
 adjoint         electromagnetics  mtop     parelag      solvers
-autodiff        gslib             fluids   performance  tools
+autodiff        gslib             navier   performance  tools
 ```

 And an example in "toys"
@@ -85,10 +85,6 @@ groups_serial=(
   "DPG miniapps:"
   "miniapps/dpg"
   "{acoustics,convection-diffusion,diffusion,maxwell}.cpp"'
-'"isf"
-   "Schrodinger flow miniapps:"
-   "miniapps/fluids/schrodinger-flow"
-   "schrodinger_flow.cpp"'
 '"gslib"
   "GSLIB miniapps:"
   "miniapps/gslib"
@@ -170,10 +166,6 @@ groups_parallel=(
   "miniapps/electromagnetics"
   "joule.cpp"'
 #   "{joule,maxwell,tesla,volta}.cpp"' # todo: multiline sample runs
-'"isf"
-   "Schrodinger flow miniapps:"
-   "miniapps/fluids/schrodinger-flow"
-   "pschrodinger_flow.cpp"'
 '"adjoint"
   "Adjoint miniapps:"
   "miniapps/adjoint"
@@ -199,7 +191,7 @@ groups_parallel=(
 # todo: miniapps/multidomain
 '"navier"
   "Navier miniapps:"
-   "miniapps/fluids/navier"
+   "miniapps/navier"
   "navier_cht.cpp"'
 # todo: add other navier miniapps
 '"nurbs"
@@ -289,10 +281,6 @@ groups_all=(
   "miniapps/electromagnetics"
   "joule.cpp"'
 #   "{joule,maxwell,tesla,volta}.cpp"' # todo: multiline sample runs
-'"isf"
-   "Schrodinger flow miniapps:"
-   "miniapps/fluids/schrodinger-flow"
-   "{,p}schrodinger_flow.cpp"'
 '"adjoint"
   "Adjoint miniapps:"
   "miniapps/adjoint"
@@ -320,7 +308,7 @@ groups_all=(
 # todo: miniapps/multidomain
 '"navier"
   "Navier miniapps:"
-   "miniapps/fluids/navier"
+   "miniapps/navier"
   "navier_cht.cpp"'
 # todo: add other navier miniapps
 '"nurbs"
@@ -1,156 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see fem/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-# PYRAMID     = 7
-#
-
-dimension
-2
-
-elements
-25
-3 3 0 1 2 3
-3 3 1 4 5 2
-3 3 4 6 7 5
-3 3 6 8 9 7
-3 3 8 10 11 9
-3 3 10 12 13 11
-3 3 12 14 15 13
-3 3 14 16 17 15
-3 3 16 18 19 17
-3 3 18 20 21 19
-3 3 20 22 23 21
-3 3 22 24 25 23
-3 3 24 26 27 25
-3 3 26 28 29 27
-3 3 28 30 31 29
-3 3 30 32 33 31
-3 3 32 34 35 33
-3 3 17 19 36 37
-3 3 37 36 38 39
-3 3 39 38 40 41
-3 3 41 40 42 43
-3 3 43 42 44 45
-3 3 45 44 46 47
-3 3 47 46 48 49
-3 3 49 48 50 51
-
-boundary
-52
-2 1 0 1
-2 1 2 3
-1 1 3 0
-2 1 1 4
-2 1 5 2
-2 1 4 6
-2 1 7 5
-2 1 6 8
-2 1 9 7
-2 1 8 10
-2 1 11 9
-2 1 10 12
-2 1 13 11
-2 1 12 14
-2 1 15 13
-2 1 14 16
-2 1 17 15
-2 1 16 18
-2 1 18 20
-2 1 21 19
-2 1 20 22
-2 1 23 21
-2 1 22 24
-2 1 25 23
-2 1 24 26
-2 1 27 25
-2 1 26 28
-2 1 29 27
-2 1 28 30
-2 1 31 29
-2 1 30 32
-2 1 33 31
-2 1 32 34
-3 1 34 35
-2 1 35 33
-2 1 19 36
-2 1 37 17
-2 1 36 38
-2 1 39 37
-2 1 38 40
-2 1 41 39
-2 1 40 42
-2 1 43 41
-2 1 42 44
-2 1 45 43
-2 1 44 46
-2 1 47 45
-2 1 46 48
-2 1 49 47
-2 1 48 50
-4 1 50 51
-2 1 51 49
-
-vertices
-52
-2
-0 0
-1 0
-1 1
-0 1
-2 0
-2 1
-3 0
-3 1
-4 0
-4 1
-5 0
-5 1
-6 0
-6 1
-7 0
-7 1
-8 0
-8 1
-9 0
-9 1
-10 0
-10 1
-11 0
-11 1
-12 0
-12 1
-13 0
-13 1
-14 0
-14 1
-15 0
-15 1
-16 0
-16 1
-17 0
-17 1
-9 2
-8 2
-9 3
-8 3
-9 4
-8 4
-9 5
-8 5
-9 6
-8 6
-9 7
-8 7
-9 8
-8 8
-9 9
-8 9
@@ -1,86 +0,0 @@
-MFEM NURBS mesh v1.0
-
-dimension
-1
-
-# Four segments with different NURBS orders, described via patches.
-elements
-4
-1 1 0 1
-2 1 2 3
-3 1 4 5
-4 1 6 7
-
-boundary
-0
-
-edges
-4
-0 0 1
-1 2 3
-2 4 5
-3 6 7
-
-vertices
-8
-
-patches
-
-# Patch 0: linear (order 1, 3 spans)
-knotvectors
-1
-1 4  0 0  .4 .6 1 1
-
-dimension
-2
-
-controlpoints
-0.0  0.0  1.0
-0.6  0.4  1.0
-0.4  0.6  1.0
-1.0  1.0  1.0
-
-# Patch 1: quadratic (order 2, 2 spans)
-knotvectors
-1
-2 4  0 0 0  .5 1 1 1
-
-dimension
-2
-
-controlpoints
-1.0  0.0  1.0
-1.9  0.0  1.21
-2.0  0.9  1.22
-2.0  1.0  1.0
-
-# Patch 2: cubic (order 3, 3 spans)
-knotvectors
-1
-3 6  0 0 0 0 .33 .66 1 1 1 1
-
-dimension
-2
-
-controlpoints
-2.0   0.0  1.0
-2.1   0.2  1.31
-3.5   0.4  1.32
-2.5   0.6  1.33
-2.9   1.0  1.34
-3.0   1.0  1.0
-
-# Patch 3: quartic (order 4, 1 span)
-knotvectors
-1
-4 5  0 0 0 0 0 1 1 1 1 1
-
-dimension
-2
-
-controlpoints
-3.0    0.0  1.0
-3.45   0.5  1.41
-3.50   1.0  1.42
-3.75   0.8  1.43
-4.0    0.0  1.0
@@ -1,79 +0,0 @@
-MFEM NURBS mesh v1.0
-
-#
-# MFEM Geometry Types (see fem/geom.hpp):
-#
-# SEGMENT     = 1
-# SQUARE      = 3
-# CUBE        = 5
-#
-
-dimension
-1
-
-# Three segments with different NURBS orders, described via patches.
-elements
-3
-1 1 0 1
-2 1 2 3
-3 1 4 5
-
-boundary
-6
-1 0 0
-1 0 1
-1 0 2
-1 0 3
-1 0 4
-1 0 5
-
-edges
-3
-0 0 1
-1 2 3
-2 4 5
-
-vertices
-6
-
-patches
-
-# Patch 0: linear (order 1, 2 control points)
-knotvectors
-1
-1 2  0 0  1 1
-
-dimension
-2
-
-controlpoints
-0.0  0.0  1.0
-1.0  1.0  1.0
-
-# Patch 1: quadratic (order 2, 3 control points)
-knotvectors
-1
-2 3  0 0 0  1 1 1
-
-dimension
-2
-
-controlpoints
-1.0  0.0   1.0
-1.02 1.02  1.2
-2.0  1.0   1.0
-
-# Patch 2: cubic (order 3, 4 control points)
-knotvectors
-1
-3 4  0 0 0 0  1 1 1 1
-
-dimension
-2
-
-controlpoints
-2.0   0.0   1.0
-2.03  0.83  1.31
-2.33  1.03  1.32
-3.0   1.0   1.0
-
@@ -1,72 +0,0 @@
-MFEM NURBS mesh v1.0
-
-#
-# MFEM Geometry Types (see fem/geom.hpp):
-#
-# SEGMENT     = 1
-# SQUARE      = 3
-# CUBE        = 5
-#
-
-dimension
-1
-
-elements
-3
-1 1 0 1
-2 1 2 3
-3 1 4 5
-
-boundary
-6
-1 0 0
-1 0 1
-1 0 2
-1 0 3
-1 0 4
-1 0 5
-
-edges
-3
-0 0 1
-1 2 3
-2 4 5
-
-vertices
-6
-
-# Edge 0: linear (order 1, 2 control points)
-# Edge 1: quadratic (order 2, 3 control points)
-# Edge 2: cubic (order 3, 4 control points)
-knotvectors
-3
-1 2  0 0  1 1
-2 3  0 0 0  1 1 1
-3 4  0 0 0 0  1 1 1 1
-
-# One weight per control point, in the same order as the control points; (2 + 3 + 4) = 9 weights total
-weights
-1
-1
-1
-1
-1
-1
-1.2
-1.31
-1.32
-
-FiniteElementSpace
-FiniteElementCollection: NURBS
-VDim: 2
-Ordering: 1
-
-0.0  0.0
-1.0  1.0
-1.0  0.0
-2.0  1.0
-2.0  0.0
-3.0  1.0
-1.02 1.02
-2.03 0.83
-2.33 1.03
@@ -1,79 +0,0 @@
-MFEM NURBS mesh v1.0
-
-#
-# MFEM Geometry Types (see fem/geom.hpp):
-#
-# SEGMENT     = 1
-# SQUARE      = 3
-# CUBE        = 5
-#
-
-dimension
-1
-
-# Three segments with different NURBS orders, described via patches.
-elements
-3
-1 1 0 1
-2 1 2 3
-3 1 4 5
-
-boundary
-6
-1 0 0
-1 0 1
-1 0 2
-1 0 3
-1 0 4
-1 0 5
-
-edges
-3
-0 0 1
-1 2 3
-2 4 5
-
-vertices
-6
-
-patches
-
-# Patch 0: linear (order 1, 2 control points)
-knotvectors
-1
-1 2  0 0  1 1
-
-dimension
-3
-
-controlpoints
-0.0  0.0  0.01 1.0
-1.0  1.0  1.01 1.0
-
-# Patch 1: quadratic (order 2, 3 control points)
-knotvectors
-1
-2 3  0 0 0  1 1 1
-
-dimension
-3
-
-controlpoints
-1.0  0.0   0.02 1.0
-1.02 1.02  0.52 1.2
-2.0  1.0   1.02 1.0
-
-# Patch 2: cubic (order 3, 4 control points)
-knotvectors
-1
-3 4  0 0 0 0  1 1 1 1
-
-dimension
-3
-
-controlpoints
-2.0   0.0   0.03 1.0
-2.03  0.83  0.33 1.31
-2.33  1.03  0.63 1.32
-3.0   1.0   1.03 1.0
-
@@ -1,72 +0,0 @@
-MFEM NURBS mesh v1.0
-
-#
-# MFEM Geometry Types (see fem/geom.hpp):
-#
-# SEGMENT     = 1
-# SQUARE      = 3
-# CUBE        = 5
-#
-
-dimension
-1
-
-elements
-3
-1 1 0 1
-2 1 2 3
-3 1 4 5
-
-boundary
-6
-1 0 0
-1 0 1
-1 0 2
-1 0 3
-1 0 4
-1 0 5
-
-edges
-3
-0 0 1
-1 2 3
-2 4 5
-
-vertices
-6
-
-# Edge 0: linear (order 1, 2 control points)
-# Edge 1: quadratic (order 2, 3 control points)
-# Edge 2: cubic (order 3, 4 control points)
-knotvectors
-3
-1 2  0 0  1 1
-2 3  0 0 0  1 1 1
-3 4  0 0 0 0  1 1 1 1
-
-# One weight per control point, in the same order as the control points; (2 + 3 + 4) = 9 weights total
-weights
-1
-1
-1
-1
-1
-1
-1.2
-1.31
-1.32
-
-FiniteElementSpace
-FiniteElementCollection: NURBS
-VDim: 3
-Ordering: 1
-
-0.0  0.0   0.01
-1.0  1.0   1.01
-1.0  0.0   0.02
-2.0  1.0   1.02
-2.0  0.0   0.03
-3.0  1.0   1.03
-1.02 1.02  0.52
-2.03 0.83  0.33
-2.33 1.03  0.63
@@ -48,7 +48,7 @@ PROJECT_NAME           = MFEM
 # could be handy for archiving the generated documentation or if some version
 # control system is used.

-PROJECT_NUMBER         = v4.9.1
+PROJECT_NUMBER         = v4.8.1

 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -960,12 +960,10 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/fem/moonolith \
                         @MFEM_SOURCE_DIR@/fem/qinterp \
                         @MFEM_SOURCE_DIR@/fem/tmop \
-                         @MFEM_SOURCE_DIR@/fem/darcy \
                         @MFEM_SOURCE_DIR@/examples \
                         @MFEM_SOURCE_DIR@/examples/amgx \
                         @MFEM_SOURCE_DIR@/examples/caliper \
                         @MFEM_SOURCE_DIR@/examples/ginkgo \
-                         @MFEM_SOURCE_DIR@/examples/hdg \
                         @MFEM_SOURCE_DIR@/examples/hiop \
                         @MFEM_SOURCE_DIR@/examples/moonolith \
                         @MFEM_SOURCE_DIR@/examples/petsc \
@@ -975,15 +973,11 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/miniapps/adjoint \
                         @MFEM_SOURCE_DIR@/miniapps/autodiff \
                         @MFEM_SOURCE_DIR@/miniapps/common \
-                         @MFEM_SOURCE_DIR@/miniapps/contact \
                         @MFEM_SOURCE_DIR@/miniapps/dfem \
                         @MFEM_SOURCE_DIR@/miniapps/dpg \
                         @MFEM_SOURCE_DIR@/miniapps/dpg/util \
                         @MFEM_SOURCE_DIR@/miniapps/electromagnetics \
-                         @MFEM_SOURCE_DIR@/miniapps/fluids/navier \
-                         @MFEM_SOURCE_DIR@/miniapps/fluids/schrodinger-flow \
                         @MFEM_SOURCE_DIR@/miniapps/gslib \
-                         @MFEM_SOURCE_DIR@/miniapps/hdg \
                         @MFEM_SOURCE_DIR@/miniapps/hdiv-linear-solver \
                         @MFEM_SOURCE_DIR@/miniapps/hooke \
                         @MFEM_SOURCE_DIR@/miniapps/hooke/kernels \
@@ -993,6 +987,7 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/miniapps/meshing \
                         @MFEM_SOURCE_DIR@/miniapps/mtop \
                         @MFEM_SOURCE_DIR@/miniapps/multidomain \
+                         @MFEM_SOURCE_DIR@/miniapps/navier \
                         @MFEM_SOURCE_DIR@/miniapps/nurbs \
                         @MFEM_SOURCE_DIR@/miniapps/parelag \
                         @MFEM_SOURCE_DIR@/miniapps/performance \
@@ -117,8 +117,6 @@ namespace mfem {
 * - <a class="el" href="ex39p_8cpp_source.html">Example 39p</a>: parallel named mesh attributes
 * - <a class="el" href="ex40_8cpp_source.html">Example 40</a>: eikonal equation
 * - <a class="el" href="ex40p_8cpp_source.html">Example 40p</a>: parallel eikonal equation
- * - <a class="el" href="ex41_8cpp_source.html">Example 41</a>: DG/CG IMEX time-dependent advection-diffusion
- * - <a class="el" href="ex41p_8cpp_source.html">Example 41p</a>: parallel DG/CG IMEX time-dependent advection-diffusion
 *
 * <H4>AmgX Examples</H4>
 * - Variants of Examples
@@ -130,29 +128,13 @@ namespace mfem {
 * - Variants of Example
 *   <a class="el" href="examples_2caliper_2ex1_8cpp_source.html">1</a> and
 *   <a class="el" href="examples_2caliper_2ex1p_8cpp_source.html">1p</a>,
- *   demonstrating the use of MFEM's \link annotation.hpp Caliper integration\endlink.
+ *   demonstrating the use of MFEM's \link annotation.hpp Ginkgo integration\endlink.
 *
 * <H4>Ginkgo Examples</H4>
 * - Variants of Example
 *   <a class="el" href="examples_2ginkgo_2ex1_8cpp_source.html">1</a>,
 *   demonstrating the use of MFEM's \link ginkgo.hpp Ginkgo integration\endlink.
 *
- * <H4>HDG Examples</H4>
- * - Variants of Examples
- *   <a class="el" href="hdg_2ex5_8cpp_source.html">5</a>,
- *   <a class="el" href="hdg_2ex5p_8cpp_source.html">5p</a>,
- *   <a class="el" href="hdg_2ex6_8cpp_source.html">6</a>,
- *   <a class="el" href="hdg_2ex6p_8cpp_source.html">6p</a>,
- *   <a class="el" href="hdg_2ex14_8cpp_source.html">14</a>,
- *   <a class="el" href="hdg_2ex14p_8cpp_source.html">14p</a>,
- *   <a class="el" href="hdg_2ex17_8cpp_source.html">17</a>,
- *   <a class="el" href="hdg_2ex17p_8cpp_source.html">17p</a>,
- *   <a class="el" href="hdg_2ex21_8cpp_source.html">21</a>,
- *   <a class="el" href="hdg_2ex21p_8cpp_source.html">21p</a>,
- *   <a class="el" href="hdg_2ex41_8cpp_source.html">41</a> and
- *   <a class="el" href="hdg_2ex41p_8cpp_source.html">41p</a>,
- *   demonstrating the use of MFEM's \link darcyhybridization.hpp hybridization framework for mixed systems\endlink.
- *
 * <H4>HiOp Examples</H4>
 * - Variants of Examples
 *   <a class="el" href="hiop_2ex9_8cpp_source.html">9</a> and
@@ -206,8 +188,6 @@ namespace mfem {
 *  <a class="el" href="nurbs__ex1p_8cpp_source.html">1p</a>,
 *  <a class="el" href="nurbs__ex3_8cpp_source.html">3</a>,
 *  <a class="el" href="nurbs__ex5_8cpp_source.html">5</a>,
- *  <a class="el" href="nurbs__ex10_8cpp_source.html">10</a>,
- *  <a class="el" href="nurbs__ex10p_8cpp_source.html">10p</a>,
 *  <a class="el" href="nurbs__ex11p_8cpp_source.html">11p</a>, and
 *  <a class="el" href="nurbs__ex24_8cpp_source.html">24</a>,
 *   demonstrating howto perform NURBS-based Isogeometric Analysis.
@@ -216,7 +196,6 @@ namespace mfem {
 * - <a class="el" href="nurbs__curveint_8cpp_source.html">NURBS Interpolation</a>: NURBS interpolation of given geometry
 * - <a class="el" href="nurbs__naca__cmesh_8cpp_source.html">NURBS NACA Mesher</a>: generate NURBS based mesh around a NACA foil
 * - <a class="el" href="nurbs__printfunc_8cpp_source.html">NURBS Printer</a>: print the NURBS-basis
- * - <a class="el" href="nurbs__mesh_info_8cpp_source.html">NURBS Mesh info</a>: print the info of a NURBS mesh
 *
 * <H3>Miniapps</H3>
 * - <a class="el" href="volta_8cpp_source.html">Volta</a>: simple electrostatics simulation code
@@ -255,17 +234,11 @@ namespace mfem {
 * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Poisson problem
 * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Poisson problem
 * - <a class="el" href="generate__random__field_8cpp_source.html">SPDE Solvers</a>: SPDE solver random field generation
- * - <a class="el" href="contact-patch-test_8cpp_source.html">Tribol</a>: mortar contact patch test for elasticity
- * - <a class="el" href="contact_8cpp_source.html">Contact</a>: Frictionless contact examples using <a class="el" href="classmfem_1_1IPSolver.html#details">IP optimization</a> and the <a class="el" href="classmfem_1_1AMGFSolver.html#details">AMGF solver</a>
+ * - <a class="el" href="contact-patch-test_8cpp_source.html">Contact</a>: mortar contact patch test for elasticity
 * - <a class="el" href="multidomain_8cpp_source.html">Multidomain miniapp</a>: Multidomain and Submesh demonstration miniapp
 * - <a class="el" href="pdiffusion_8cpp_source.html">DPG Diffusion example</a>: DPG formulation for the diffusion problem
 * - <a class="el" href="pmaxwell_8cpp_source.html">DPG Maxwell example</a>: DPG formulation for the indefinite Maxwell problem
 * - <a class="el" href="lor__elast_8cpp_source.html">LOR Elasticity</a>: solve linear elasticity with LOR preconditioning on GPUs
- * - <a class="el" href="anisodiff_8cpp_source.html">HDG anisotropic diffusion</a>: (Hybridized) mixed solution of anisotropic diffusion
- * - <a class="el" href="panisodiff_8cpp_source.html">HDG anisotropic diffusion (parallel)</a>: Parallel (hybridized) mixed solution of anisotropic diffusion
- * - <a class="el" href="mfem-logo_8cpp_source.html">MFEM logo</a>: (Hybridized) mixed solution of randomized single-step convection-diffusion of MFEM text
- * - <a class="el" href="convdiff_8cpp_source.html">HDG convection-diffusion</a>: (Hybridized) mixed solution of convection-diffusion problems
- * - <a class="el" href="pconvdiff_8cpp_source.html">HDG convection-diffusion (parallel)</a>: Parallel (hybridized) mixed solution of convection-diffusion problems
 *
 * See also the <a class="el" href="https://mfem.org/examples/">examples documentation</a> online.
 */
@@ -46,7 +46,6 @@ list(APPEND ALL_EXE_SRCS
  ex38.cpp
  ex39.cpp
  ex40.cpp
-  ex41.cpp
  )

 if (MFEM_USE_MPI)
@@ -90,7 +89,6 @@ if (MFEM_USE_MPI)
    ex37p.cpp
    ex39p.cpp
    ex40p.cpp
-    ex41p.cpp
  )
 endif()

@@ -133,8 +131,6 @@ if (MFEM_ENABLE_TESTING)
      list(APPEND THIS_TEST_OPTIONS "-dg")
    elseif(${TEST_NAME} MATCHES "ex37p*")
      list(APPEND THIS_TEST_OPTIONS "-mi" "3")
-    elseif(${TEST_NAME} MATCHES "ex41p*")
-      list(APPEND THIS_TEST_OPTIONS "-tf" "1.0")
    endif()

    if (NOT (${TEST_NAME} MATCHES ".*p$"))
@@ -215,7 +211,7 @@ if (MFEM_ENABLE_TESTING)
    add_test(NAME ex1p_ceed_np=${MFEM_MPI_NP}
      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:ex1p> "-no-vis" "-d" "ceed-cpu" "-pa" "-a"
+      $<TARGET_FILE:ex1p> "-no-vis" "-d ceed-cpu" "-pa" "-a"
      ${MPIEXEC_POSTFLAGS})
  endif()
 endif()
@@ -230,9 +226,6 @@ if (MFEM_USE_GINKGO)
  add_subdirectory(ginkgo)
 endif()

-# Include the examples/hdg directory
-add_subdirectory(hdg)
-
 # Include the examples/hiop directory if HiOp is enabled
 if (MFEM_USE_HIOP)
  add_subdirectory(hiop)
@@ -64,7 +64,7 @@ PARALLEL_NAME := Parallel AMGX example
 $(MFEM_LIB_FILE):
 	$(error The MFEM library is not build)

-clean: clean-build clean-exec
+clean: clean-build

 clean-build:
 	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
@@ -64,12 +64,12 @@ ex1p-test-par: ex1p
 $(MFEM_LIB_FILE):
 	$(error The MFEM library is not built)

-clean: clean-build clean-exec
+clean: clean-build clean-exec $(SUBDIRS_CLEAN)

 clean-build:
 	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
 	rm -rf *.dSYM *.TVD.*breakpoints

 clean-exec:
-	@rm -f refined.mesh mesh.*
-	@rm -f sol.*
+	@rm -f refined.mesh displaced.mesh mesh.* ex5.mesh
+	@rm -f sphere_refined.* sol.* sol_u.* sol_p.* sol_r.* sol_i.*
@@ -1,767 +0,0 @@
-//                                MFEM Example 1
-//
-// Compile with: make ex1
-//
-// Sample runs:  ex1 -m ../data/square-disc.mesh
-//               ex1 -m ../data/star.mesh
-//               ex1 -m ../data/star-mixed.mesh
-//               ex1 -m ../data/escher.mesh
-//               ex1 -m ../data/fichera.mesh
-//               ex1 -m ../data/fichera-mixed.mesh
-//               ex1 -m ../data/toroid-wedge.mesh
-//               ex1 -m ../data/octahedron.mesh -o 1
-//               ex1 -m ../data/periodic-annulus-sector.msh
-//               ex1 -m ../data/periodic-torus-sector.msh
-//               ex1 -m ../data/square-disc-p2.vtk -o 2
-//               ex1 -m ../data/square-disc-p3.mesh -o 3
-//               ex1 -m ../data/square-disc-nurbs.mesh -o -1
-//               ex1 -m ../data/star-mixed-p2.mesh -o 2
-//               ex1 -m ../data/disc-nurbs.mesh -o -1
-//               ex1 -m ../data/pipe-nurbs.mesh -o -1
-//               ex1 -m ../data/fichera-mixed-p2.mesh -o 2
-//               ex1 -m ../data/star-surf.mesh
-//               ex1 -m ../data/square-disc-surf.mesh
-//               ex1 -m ../data/inline-segment.mesh
-//               ex1 -m ../data/amr-quad.mesh
-//               ex1 -m ../data/amr-hex.mesh
-//               ex1 -m ../data/fichera-amr.mesh
-//               ex1 -m ../data/mobius-strip.mesh
-//               ex1 -m ../data/mobius-strip.mesh -o -1 -sc
-//
-// Device sample runs:
-//               ex1 -pa -d cuda
-//               ex1 -fa -d cuda
-//               ex1 -pa -d raja-cuda
-//             * ex1 -pa -d raja-hip
-//               ex1 -pa -d occa-cuda
-//               ex1 -pa -d raja-omp
-//               ex1 -pa -d occa-omp
-//               ex1 -pa -d ceed-cpu
-//               ex1 -pa -d ceed-cpu -o 4 -a
-//               ex1 -pa -d ceed-cpu -m ../data/square-mixed.mesh
-//               ex1 -pa -d ceed-cpu -m ../data/fichera-mixed.mesh
-//             * ex1 -pa -d ceed-cuda
-//             * ex1 -pa -d ceed-hip
-//               ex1 -pa -d ceed-cuda:/gpu/cuda/shared
-//               ex1 -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/square-mixed.mesh
-//               ex1 -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/fichera-mixed.mesh
-//               ex1 -m ../data/beam-hex.mesh -pa -d cuda
-//               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cpu
-//               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cuda:/gpu/cuda/ref
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Poisson problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order, or if order < 1 using an isoparametric/isogeometric
-//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example highlights the use of mesh refinement, finite
-//               element grid functions, as well as linear and bilinear forms
-//               corresponding to the left-hand side and right-hand side of the
-//               discrete linear system. We also cover the explicit elimination
-//               of essential boundary conditions, static condensation, and the
-//               optional connection to the GLVis tool for visualization.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Define the analytical solution and forcing terms / boundary conditions
-typedef std::function<real_t(const Vector &, real_t)> TFunc;
-typedef std::function<void(const Vector &, Vector &)> VecFunc;
-typedef std::function<void(const Vector &, real_t, Vector &)> VecTFunc;
-typedef std::function<void(const Vector &, DenseMatrix &)> MatFunc;
-
-enum Problem
-{
-   SteadyDiffusion = 1,
-   MFEMLogo,
-   DiffusionRing,
-   DiffusionRingGauss,
-   DiffusionRingSine,
-   BoundaryLayer,
-   SteadyPeak,
-   SteadyVaryingAngle,
-   Sovinec,
-   Umansky,
-};
-
-struct ProblemParams
-{
-   Problem prob;
-   int nx, ny;
-   real_t x0, y0, sx, sy;
-   int order;
-   real_t k, ks, ka;
-   real_t t_0;
-   real_t a;
-   real_t c;
-};
-
-MatFunc GetKFun(const ProblemParams &params);
-TFunc GetTFun(const ProblemParams &params);
-TFunc GetFFun(const ProblemParams &params);
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../data/star.mesh";
-   int order = 1;
-   int iproblem = Problem::SteadyDiffusion;
-   ProblemParams pars;
-   pars.nx = 0;
-   pars.ny = 0;
-   pars.x0 = 0.;
-   pars.y0 = 0.;
-   pars.sx = 1.;
-   pars.sy = 1.;
-   pars.order = 1;
-   pars.t_0 = 1.;
-   pars.k = 1.;
-   pars.ks = 1.;
-   pars.ka = 0.;
-   pars.a = 0.;
-   pars.c = 1.;
-   bool static_cond = false;
-   bool pa = false;
-   bool fa = false;
-   const char *device_config = "cpu";
-   bool visualization = true;
-   bool algebraic_ceed = false;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&iproblem, "-p", "--problem",
-                  "Problem to solve:\n\t\t"
-                  "1=sine diffusion\n\t\t"
-                  "2=MFEM logo\n\t\t"
-                  "3=diffusion ring\n\t\t"
-                  "4=diffusion ring - Gauss source\n\t\t"
-                  "5=diffusion ring - sine source\n\t\t"
-                  "6=boundary layer\n\t\t"
-                  "7=steady peak\n\t\t"
-                  "8=steady varying angle\n\t\t"
-                  "9=Sovinec\n\t\t"
-                  "10=Umansky\n\t\t");
-   args.AddOption(&pars.k, "-k", "--kappa",
-                  "Heat conductivity");
-   args.AddOption(&pars.ks, "-ks", "--kappa_sym",
-                  "Symmetric anisotropy of the heat conductivity tensor");
-   args.AddOption(&pars.a, "-a", "--heat_capacity",
-                  "Heat capacity coefficient (0=indefinite problem)");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
-                  "--no-full-assembly", "Enable Full Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-#ifdef MFEM_USE_CEED
-   args.AddOption(&algebraic_ceed, "-a", "--algebraic", "-no-a", "--no-algebraic",
-                  "Use algebraic Ceed solver");
-#endif
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
-   //    the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   pars.prob = (Problem)iproblem;
-   pars.order = order;
-
-   Vector x_min(2), x_max(2);
-   mesh.GetBoundingBox(x_min, x_max);
-   pars.x0 = x_min(0);
-   pars.y0 = x_min(1);
-   pars.sx = x_max(0) - x_min(0);
-   pars.sy = x_max(1) - x_min(1);
-
-   // 4. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
-   //    largest number that gives a final mesh with no more than 50,000
-   //    elements.
-   {
-      int ref_levels =
-         (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh.UniformRefinement();
-      }
-   }
-
-   // 5. Define a finite element space on the mesh. Here we use continuous
-   //    Lagrange finite elements of the specified order. If order < 1, we
-   //    instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   bool delete_fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
-   }
-   else if (mesh.GetNodes())
-   {
-      fec = mesh.GetNodes()->OwnFEC();
-      delete_fec = false;
-      cout << "Using isoparametric FEs: " << fec->Name() << endl;
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
-   }
-   FiniteElementSpace fespace(&mesh, fec);
-   cout << "Number of finite element unknowns: "
-        << fespace.GetTrueVSize() << endl;
-
-   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
-   //    In this example, the boundary conditions are defined by marking all
-   //    the external boundary attributes from the mesh as essential (Dirichlet)
-   //    and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (mesh.bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(mesh.bdr_attributes.Max());
-      ess_bdr = 0;
-      // Apply boundary conditions on all external boundaries:
-      mesh.MarkExternalBoundaries(ess_bdr);
-      // Boundary conditions can also be applied based on named attributes:
-      // mesh.MarkNamedBoundaries(set_name, ess_bdr)
-
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
-   //    the basis functions in the finite element fespace.
-   auto fFun = GetFFun(pars);
-   FunctionCoefficient fcoeff(fFun); //temperature rhs
-
-   LinearForm b(&fespace);
-   //ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(fcoeff));
-   b.Assemble();
-
-   // 8. Define the solution vector x as a finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero,
-   //    which satisfies the boundary conditions.
-   GridFunction x(&fespace);
-   x = 0.0;
-
-   // 9. Set up the bilinear form a(.,.) on the finite element space
-   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //    domain integrator.
-
-   auto kFun = GetKFun(pars);
-   MatrixFunctionCoefficient kcoeff(dim, kFun); //tensor conductivity
-   ConstantCoefficient acoeff(pars.a); //heat capacity
-
-   BilinearForm a(&fespace);
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   if (fa)
-   {
-      a.SetAssemblyLevel(AssemblyLevel::FULL);
-      // Sort the matrix column indices when running on GPU or with OpenMP (i.e.
-      // when Device::IsEnabled() returns true). This makes the results
-      // bit-for-bit deterministic at the cost of somewhat longer run time.
-      a.EnableSparseMatrixSorting(Device::IsEnabled());
-   }
-   a.AddDomainIntegrator(new DiffusionIntegrator(kcoeff));
-
-   if (pars.a > 0)
-   {
-      a.AddDomainIntegrator(new MassIntegrator(acoeff));
-   }
-
-   // 10. Assemble the bilinear form and the corresponding linear system,
-   //     applying any necessary transformations such as: eliminating boundary
-   //     conditions, applying conforming constraints for non-conforming AMR,
-   //     static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
-   a.Assemble();
-
-   OperatorPtr A;
-   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-
-   cout << "Size of linear system: " << A->Height() << endl;
-
-   // 11. Solve the linear system A X = B.
-   if (!pa)
-   {
-#ifndef MFEM_USE_SUITESPARSE
-      // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
-      GSSmoother M((SparseMatrix&)(*A));
-      PCG(*A, M, B, X, 1, 1000, 1e-8, 0.0);
-#else
-      // If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-      UMFPackSolver umf_solver;
-      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      umf_solver.SetOperator(*A);
-      umf_solver.Mult(B, X);
-#endif
-   }
-   else
-   {
-      if (UsesTensorBasis(fespace))
-      {
-         if (algebraic_ceed)
-         {
-            ceed::AlgebraicSolver M(a, ess_tdof_list);
-            PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
-         }
-         else
-         {
-            OperatorJacobiSmoother M(a, ess_tdof_list);
-            PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
-         }
-      }
-      else
-      {
-         CG(*A, B, X, 1, 400, 1e-12, 0.0);
-      }
-   }
-
-   // 12. Recover the solution as a finite element grid function.
-   a.RecoverFEMSolution(X, b, x);
-
-   // 13. Save the refined mesh and the solution. This output can be viewed later
-   //     using GLVis: "glvis -m refined.mesh -g sol.gf".
-   /*ofstream mesh_ofs("refined.mesh");
-   mesh_ofs.precision(8);
-   mesh.Print(mesh_ofs);
-   ofstream sol_ofs("sol.gf");
-   sol_ofs.precision(8);
-   x.Save(sol_ofs);*/
-   /*auto tFun = GetTFun(pars);
-   FunctionCoefficient tcoeff(tFun); //temperature
-   x.ProjectCoefficient(tcoeff);*/
-
-   // 14. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << mesh << x << endl;
-      sol_sock << "window_title 'Solution'" << endl;
-      sol_sock << "keys Rljmmc" << endl;
-   }
-
-   // 15. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
-
-   return 0;
-}
-
-MatFunc GetKFun(const ProblemParams &params)
-{
-   const real_t &k = params.k;
-   const real_t &ks = params.ks;
-   const real_t &ka = params.ka;
-   const real_t &x0 = params.x0;
-   const real_t &y0 = params.y0;
-   const real_t &sx = params.sx;
-   const real_t &sy = params.sy;
-
-   switch (params.prob)
-   {
-      case Problem::SteadyDiffusion:
-      case Problem::BoundaryLayer:
-      case Problem::SteadyPeak:
-         return [=](const Vector &x, DenseMatrix &kappa)
-         {
-            const int ndim = x.Size();
-            kappa.Diag(k, ndim);
-            kappa(0,0) *= ks;
-            kappa(0,1) = +ka * k;
-            kappa(1,0) = -ka * k;
-            if (ndim > 2)
-            {
-               kappa(0,2) = +ka * k;
-               kappa(2,0) = -ka * k;
-            }
-         };
-      case Problem::MFEMLogo:
-         MFEM_ABORT("N/A");
-         break;
-      case Problem::DiffusionRing:
-      case Problem::DiffusionRingGauss:
-      case Problem::DiffusionRingSine:
-      case Problem::SteadyVaryingAngle:
-         return [=](const Vector &x, DenseMatrix &kappa)
-         {
-            const int ndim = x.Size();
-            Vector b(ndim);
-            b = 0.;
-
-            Vector dx(x);
-            dx(0) -= x0 + 0.5*sx;
-            dx(1) -= y0 + 0.5*sy;
-
-            const real_t r = hypot(dx(0), dx(1));
-            b(0) = (r>0.)?(-dx(1) / r):(1.);
-            b(1) = (r>0.)?(+dx(0) / r):(0.);
-
-            kappa.Diag(ks * k, ndim);
-            if (ks != 1.)
-            {
-               AddMult_a_VVt((1. - ks) * k, b, kappa);
-            }
-         };
-      case Problem::Sovinec:
-         return [=](const Vector &x, DenseMatrix &kappa)
-         {
-            const int ndim = x.Size();
-            Vector b(ndim);
-            b = 0.;
-
-            Vector dx(x);
-            dx(0) -= x0 + 0.5*sx;
-            dx(1) -= y0 + 0.5*sy;
-
-            //const real_t psi = cos(M_PI * dx(0)) * cos(M_PI * dx(1));
-            const real_t psi_x = M_PI * sin(M_PI * dx(0)) * cos(M_PI * dx(1));
-            const real_t psi_y = M_PI * cos(M_PI * dx(0)) * sin(M_PI * dx(1));
-            const real_t psi_norm = hypot(psi_x, psi_y);
-            if (psi_norm > 0.)
-            {
-               b(0) = -psi_y / psi_norm;
-               b(1) = +psi_x / psi_norm;
-            }
-            else
-            {
-               b = 0.;
-            }
-
-            kappa.Diag(ks * k, ndim);
-            if (ks != 1.)
-            {
-               AddMult_a_VVt((1. - ks) * k, b, kappa);
-            }
-         };
-      case Problem::Umansky:
-         return [=](const Vector &x, DenseMatrix &kappa)
-         {
-            const int ndim = x.Size();
-            Vector b(ndim);
-            const real_t s = hypot(sx, sy);
-            b(0) = +sx / s;
-            b(1) = +sy / s;
-
-            kappa.Diag(ks * k, ndim);
-            if (ks != 1.)
-            {
-               AddMult_a_VVt((1. - ks) * k, b, kappa);
-            }
-         };
-   }
-   return MatFunc();
-}
-
-TFunc GetTFun(const ProblemParams &params)
-{
-   const real_t &k = params.k;
-   const real_t &ks = params.ks;
-   //const real_t &ka = params.ka;
-   const real_t &t_0 = params.t_0;
-   const real_t &a = params.a;
-   const real_t &x0 = params.x0;
-   const real_t &y0 = params.y0;
-   const real_t &sx = params.sx;
-   const real_t &sy = params.sy;
-   const real_t hx = sx / params.nx;
-   const real_t hy = sy / params.ny;
-   const real_t &order = params.order;
-
-   auto kFun = GetKFun(params);
-
-   switch (params.prob)
-   {
-      case Problem::SteadyDiffusion:
-         return [=](const Vector &x, real_t t) -> real_t
-         {
-            const int ndim = x.Size();
-            real_t t0 = t_0 * sin(M_PI*x(0)) * sin(M_PI*x(1));
-            if (ndim > 2)
-            {
-               t0 *= sin(M_PI*x(2));
-            }
-
-            if (a <= 0.) { return t0; }
-
-            Vector ddT((ndim<=2)?(2):(4));
-            ddT(0) = -t_0 * M_PI*M_PI * sin(M_PI*x(0)) * sin(M_PI*x(1));//xx,yy
-            ddT(1) = +t_0 * M_PI*M_PI * cos(M_PI*x(0)) * cos(M_PI*x(1));//xy
-            if (ndim > 2)
-            {
-               ddT(0) *= sin(M_PI*x(2));//xx,yy,zz
-               ddT(1) *= sin(M_PI*x(2));//xy
-               //xz
-               ddT(2) = +t_0 * M_PI*M_PI * cos(M_PI*x(0)) * sin(M_PI*x(1)) * cos(M_PI*x(2));
-               //yz
-               ddT(3) = +t_0 * M_PI*M_PI * sin(M_PI*x(0)) * cos(M_PI*x(1)) * cos(M_PI*x(2));
-
-            }
-
-            DenseMatrix kappa;
-            kFun(x, kappa);
-
-            real_t div = -(kappa(0,0) + kappa(1,1)) * ddT(0) - (kappa(0,1) + kappa(1,0)) * ddT(1);
-            if (ndim > 2)
-            {
-               div += -kappa(2,2) * ddT(0) - (kappa(0,2) + kappa(2,0)) * ddT(2)
-               - (kappa(1,2) + kappa(2,1)) * ddT(3);
-            }
-            return t0 - div / a * t;
-         };
-      case Problem::MFEMLogo:
-         MFEM_ABORT("N/A");
-         break;
-      case Problem::DiffusionRing:
-         return [=](const Vector &x, real_t t) -> real_t
-         {
-            constexpr real_t r0 = 0.25;
-            constexpr real_t r1 = 0.35;
-            constexpr real_t dr01 = 0.025;
-            constexpr real_t theta0 = 11./12. * M_PI;
-            constexpr real_t dtheta0 = 1./48. * M_PI;
-
-            Vector dx(x);
-            dx(0) -= x0 + 0.5*sx;
-            dx(1) -= y0 + 0.5*sy;
-
-            const real_t r = hypot(dx(0), dx(1));
-            const real_t theta = fabs(atan2(dx(1), dx(0)));
-
-            if (r < r0 - dr01 || r > r1 + dr01 || theta < theta0 - dtheta0)
-            {
-               return 0.;
-            }
-
-            const real_t dr = min(r - r0 + dr01, r1 + dr01 - r) / dr01;
-            const real_t dth = (theta - theta0 + dtheta0) / dtheta0;
-            return min(1., dr) * min(1., dth) * t_0;
-         };
-      case Problem::DiffusionRingGauss:
-         return [=](const Vector &x, real_t t) -> real_t
-         {
-            constexpr real_t r0 = 0.025;
-            constexpr real_t x_c = 0.15;
-
-            const real_t dx_l = x(0) - (x0       + x_c  * sx);
-            const real_t dx_r = x(0) - (x0 + (1. - x_c) * sx);
-            const real_t dy = x(1) - (y0 + 0.5*sy);
-            const real_t r_l = hypot(dx_l, dy);
-            const real_t r_r = hypot(dx_r, dy);
-
-            return - exp(- r_l*r_l/(r0*r0)) + exp(- r_r*r_r/(r0*r0));
-         };
-      case Problem::DiffusionRingSine:
-         return [=](const Vector &x, real_t t) -> real_t
-         {
-            constexpr real_t r0 = 0.05;
-            constexpr real_t w0 = 16.;
-            Vector dx(x);
-            dx(0) -= x0 + 0.5*sx;
-            dx(1) -= y0 + 0.5*sy;
-
-            const real_t r = hypot(dx(0), dx(1));
-            if (r <= 0.) { return 0.; }
-            const real_t th = atan2(dx(1), dx(0));
-
-            const real_t C = w0 / r;
-            return 1. / (1. + t * k * C*C / a) * cos(w0*th) * sin(M_PI * r/r0);
-         };
-      case Problem::BoundaryLayer:
-         // C. Vogl, I. Joseph and M. Holec, Mesh refinement for anisotropic
-         // diffusion in magnetized plasmas, Computers and Mathematics with
-         // Applications, 145, pp. 159-174 (2023).
-         return [=](const Vector &x, real_t t) -> real_t
-         {
-            const real_t k_para = M_PI*M_PI * k * ks;
-            const real_t k_perp = k;
-            const real_t k_frac = sqrt(k_para/k_perp);
-            const real_t denom = 1. + exp(-k_frac);
-            const real_t e_down = exp(-k_frac * x(1));
-            const real_t e_up = exp(- k_frac * (1. - x(1)));
-            return - (e_down + e_up) / denom * sin(M_PI * x(0));
-         };
-      case Problem::SteadyPeak:
-         // B. van Es, B. Koern and Hugo de Blank, DISCRETIZATIONMETHODS
-         // FOR EXTREMELY ANISOTROPIC DIFFUSION. In 7th International
-         // Conference on Computational Fluid Dynamics (ICCFD 2012) (pp.
-         // ICCFD7-1401)
-         return [=](const Vector &x, real_t t) -> real_t
-         {
-            constexpr real_t s = 10.;
-            const real_t arg = sin(M_PI * x(0)) * sin(M_PI * x(1));
-            return x(0)*x(1) * pow(arg, s);
-         };
-      case Problem::SteadyVaryingAngle:
-         // B. van Es, B. Koern and Hugo de Blank, DISCRETIZATIONMETHODS
-         // FOR EXTREMELY ANISOTROPIC DIFFUSION. In 7th International
-         // Conference on Computational Fluid Dynamics (ICCFD 2012) (pp.
-         // ICCFD7-1401)
-         return [=](const Vector &x, real_t t) -> real_t
-         {
-            Vector dx(x);
-            dx(0) -= x0 + 0.5*sx;
-            dx(1) -= y0 + 0.5*sy;
-
-            const real_t r = hypot(dx(0), dx(1));
-            return 1. - r*r*r;
-         };
-      case Problem::Sovinec:
-         // C. R. Sovinec et al., Nonlinear magnetohydrodynamics simulation
-         // using high-order finite elements. Journal of Computational Physics,
-         // 195, pp. 355–386 (2004).
-         return [=](const Vector &x, real_t t) -> real_t
-         {
-            const real_t &kappa_perp = k * ks;
-            Vector dx(x);
-            dx(0) -= x0 + 0.5*sx;
-            dx(1) -= y0 + 0.5*sy;
-
-            const real_t psi = cos(M_PI * dx(0)) * cos(M_PI * dx(1));
-            return psi / kappa_perp;
-         };
-      case Problem::Umansky:
-         // M. V. Umansky, M. S. Day and T. D. Rognlien, On Numerical Solution
-         // of Strongly Anisotropic Diffusion Equation on Misaligned Grids,
-         // Numerical Heat Transfer, Part B: Fundamentals, 47(6), pp. 533-554
-         // (2005).
-         // Adopted from plasma-dev:miniapps/plasma/transport2d.cpp
-         return [=](const Vector &x, real_t t) -> real_t
-         {
-            if (x(0) < hx && x(1) < hy)
-            {
-               return 0.5 * (1.0 -
-                             std::pow(1.0 - x(0) / hx, order) +
-                             std::pow(1.0 - x(1) / hy, order));
-            }
-            else if (x(0) > sx - hx && x(1) > sy - hy)
-            {
-               return 0.5 * (1.0 +
-                             std::pow(1.0 - (sx - x(0)) / hx, order) -
-                             std::pow(1.0 - (sy - x(1)) / hy, order));
-            }
-            // else if (x_[0] > Lx_ - hx_ || x_[1] < hy_)
-            else if (hx * (x(1) + hy) < hy * x(0))
-            {
-               return 1.0;
-            }
-            // else if (x_[0] < hx_ || x_[1] > Ly_ - hy_)
-            else if (hx * x(1) > hy * (x(0) + hx))
-            {
-               return 0.0;
-            }
-            else
-            {
-               return 0.5 * (1.0 + std::tanh(M_LN10 * (x(0) / hx - x(1) / hy)));
-            }
-         };
-   }
-   return TFunc();
-}
-
-TFunc GetFFun(const ProblemParams &params)
-{
-   const real_t &k = params.k;
-   const real_t &ks = params.ks;
-   //const real_t &ka = params.ka;
-   const real_t &a = params.a;
-   const real_t &x0 = params.x0;
-   const real_t &y0 = params.y0;
-   const real_t &sx = params.sx;
-   const real_t &sy = params.sy;
-
-   auto TFun = GetTFun(params);
-   auto kFun = GetKFun(params);
-
-   switch (params.prob)
-   {
-      case Problem::SteadyDiffusion:
-      case Problem::MFEMLogo:
-      case Problem::DiffusionRing:
-      case Problem::DiffusionRingGauss:
-      case Problem::DiffusionRingSine:
-         return [=](const Vector &x, real_t) -> real_t
-         {
-            const real_t T = TFun(x, 0);
-            return ((a > 0.)?(a):(1.)) * T;
-         };
-      case Problem::BoundaryLayer:
-      case Problem::Umansky:
-         return [=](const Vector &x, real_t) -> real_t
-         {
-            return 0.;
-         };
-      case Problem::SteadyPeak:
-         return [=](const Vector &x, real_t) -> real_t
-         {
-            DenseMatrix kappa;
-            kFun(x, kappa);
-            constexpr real_t s = 10.;
-            const real_t arg = sin(M_PI * x(0)) * sin(M_PI * x(1));
-            const real_t arg_x = M_PI * cos(M_PI * x(0)) * sin(M_PI * x(1));
-            const real_t arg_y = M_PI * cos(M_PI * x(1)) * sin(M_PI * x(0));
-            const real_t T_xx = x(1) * pow(arg, s-2) * (2.*s * arg_x * arg + x(0) * s * ((s-1) * arg_x*arg_x - M_PI*M_PI * arg*arg));
-            const real_t T_yy = x(0) * pow(arg, s-2) * (2.*s * arg_y * arg + x(1) * s * ((s-1) * arg_y*arg_y - M_PI*M_PI * arg*arg));
-            return -kappa(0,0) * T_xx - kappa(1,1) * T_yy;
-         };
-      case Problem::SteadyVaryingAngle:
-         return [=](const Vector &x, real_t) -> real_t
-         {
-            const real_t kappa_r = ks * k;
-            Vector dx(x);
-            dx(0) -= x0 + 0.5*sx;
-            dx(1) -= y0 + 0.5*sy;
-
-            const real_t r = hypot(dx(0), dx(1));
-            const real_t T_rr = - 9. * r;
-            return -kappa_r * T_rr;
-         };
-      case Problem::Sovinec:
-         return [=](const Vector &x, real_t) -> real_t
-         {
-            Vector dx(x);
-            dx(0) -= x0 + 0.5*sx;
-            dx(1) -= y0 + 0.5*sy;
-
-            const real_t psi = cos(M_PI * dx(0)) * cos(M_PI * dx(1));
-            return 2.*M_PI*M_PI * psi;
-         };
-   }
-   return TFunc();
-}
@@ -50,10 +50,6 @@
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cuda:/gpu/cuda/ref
 //
-// Device simplices sample runs:
-//               ex1 -pa -d gpu -m ../data/inline-tet.mesh
-//               ex1 -pa -d gpu -m ../data/inline-tri.mesh
-//
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Poisson problem
 //               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
@@ -142,25 +138,25 @@ int main(int argc, char *argv[])
   }

   // 5. Define a finite element space on the mesh. Here we use continuous
-   //    Lagrange finite elements of the specified order.
-   //    - If order < 1, we instead use an isoparametric/isogeometric space.
-   //    - If the mesh is simplicial and partial assembly is requested,
-   //      we use the positive basis, which supports device execution.
+   //    Lagrange finite elements of the specified order. If order < 1, we
+   //    instead use an isoparametric/isogeometric space.
   FiniteElementCollection *fec;
-   auto basis_type = (pa && mesh.IsSimplexMesh()) ?
-                     BasisType::Positive : BasisType::GaussLobatto;
+   bool delete_fec;
   if (order > 0)
   {
-      fec = new H1_FECollection(order, dim, basis_type);
+      fec = new H1_FECollection(order, dim);
+      delete_fec = true;
   }
   else if (mesh.GetNodes())
   {
      fec = mesh.GetNodes()->OwnFEC();
+      delete_fec = false;
      cout << "Using isoparametric FEs: " << fec->Name() << endl;
   }
   else
   {
-      fec = new H1_FECollection(order = 1, dim, basis_type);
+      fec = new H1_FECollection(order = 1, dim);
+      delete_fec = true;
   }
   FiniteElementSpace fespace(&mesh, fec);
   cout << "Number of finite element unknowns: "
@@ -228,29 +224,17 @@ int main(int argc, char *argv[])
   // 11. Solve the linear system A X = B.
   if (!pa)
   {
-#ifdef MFEM_USE_CUDSS
-      if (Device::Allows(Backend::CUDA_MASK))
-      {
-         // Use cuDSS to solve the system.
-         CuDSSSolver cudss_solver;
-         cudss_solver.SetOperator(*A);
-         cudss_solver.Mult(B, X);
-      }
-      else
-#endif
-      {
 #ifndef MFEM_USE_SUITESPARSE
-         // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
-         GSSmoother M((SparseMatrix&)(*A));
-         PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
+      // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
+      GSSmoother M((SparseMatrix&)(*A));
+      PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
 #else
-         // If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-         UMFPackSolver umf_solver;
-         umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-         umf_solver.SetOperator(*A);
-         umf_solver.Mult(B, X);
+      // If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+      UMFPackSolver umf_solver;
+      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      umf_solver.SetOperator(*A);
+      umf_solver.Mult(B, X);
 #endif
-      }
   }
   else
   {
@@ -289,14 +273,17 @@ int main(int argc, char *argv[])
   if (visualization)
   {
      char vishost[] = "localhost";
-      int visport = 19916;
+      int  visport   = 19916;
      socketstream sol_sock(vishost, visport);
      sol_sock.precision(8);
      sol_sock << "solution\n" << mesh << x << flush;
   }

   // 15. Free the used memory.
-   if (order > 0) { delete fec; }
+   if (delete_fec)
+   {
+      delete fec;
+   }

   return 0;
 }
@@ -5,9 +5,9 @@
 // Sample runs:
 //    mpirun -np 4 ex12p -m ../data/beam-tri.mesh
 //    mpirun -np 4 ex12p -m ../data/beam-quad.mesh
-//    mpirun -np 4 ex12p -m ../data/beam-tet.mesh -s 464 -n 10 -o 2 -elast
+//    mpirun -np 4 ex12p -m ../data/beam-tet.mesh -s 462 -n 10 -o 2 -elast
 //    mpirun -np 4 ex12p -m ../data/beam-hex.mesh -s 3878
-//    mpirun -np 4 ex12p -m ../data/beam-wedge.mesh -s 82
+//    mpirun -np 4 ex12p -m ../data/beam-wedge.mesh -s 81
 //    mpirun -np 4 ex12p -m ../data/beam-tri.mesh -s 3877 -o 2 -sys
 //    mpirun -np 4 ex12p -m ../data/beam-quad.mesh -s 4544 -n 6 -o 3 -elast
 //    mpirun -np 4 ex12p -m ../data/beam-quad-nurbs.mesh
@@ -145,7 +145,7 @@ int main(int argc, char *argv[])
   // 4. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ser_ref_levels' of uniform refinement. By default,
   //    or if ser_ref_levels < 0, we choose it to be the largest number that
-   //    gives a final mesh with no more than 10,000 elements.
+   //    gives a final mesh with no more than 50,000 elements.
   {
      if (ser_ref_levels < 0)
      {
@@ -105,7 +105,6 @@ int main(int argc, char *argv[])
   bool visualization = true;
   bool visit = false;
   int vis_steps = 5;
-   bool solve_implicit_state = false;

   int precision = 8;
   cout.precision(precision);
@@ -127,9 +126,6 @@ int main(int argc, char *argv[])
                  "Alpha coefficient.");
   args.AddOption(&kappa, "-k", "--kappa",
                  "Kappa coefficient offset.");
-   args.AddOption(&solve_implicit_state, "-imp-state", "--implicit-state",
-                  "-imp-slope", "--implicit-slope",
-                  "Implicitly solve for stage state or slope.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -183,11 +179,6 @@ int main(int argc, char *argv[])

   // 7. Initialize the conduction operator and the visualization.
   ConductionOperator oper(fespace, alpha, kappa, u);
-   using ImplicitVariableType = ConductionOperator::ImplicitVariableType;
-   ImplicitVariableType imp_var = solve_implicit_state ?
-                                  ImplicitVariableType::STATE
-                                  : ImplicitVariableType::SLOPE;
-   oper.SetImplicitVariableType(imp_var);

   u_gf.SetFromTrueDofs(u);
   {
@@ -325,14 +316,11 @@ void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
 }

 void ConductionOperator::ImplicitSolve(const real_t dt,
-                                       const Vector &u, Vector &k)
+                                       const Vector &u, Vector &du_dt)
 {
   // Solve the equation:
-   //    M*k = -K(u + dt*k) for k = du/dt, if solving for stage-slope
-   // or
-   //    M*k = -dt*K(k) + M*u for k = u_s, if solving for stage-state
-   // where K is linearized by using u from the previous timestep, and
-   // the stage-state and slope relation: du/dt = (u_s - u)/dt.
+   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
+   // for du_dt, where K is linearized by using u from the previous timestep
   if (!T)
   {
      T = Add(1.0, Mmat, dt, Kmat);
@@ -340,20 +328,9 @@ void ConductionOperator::ImplicitSolve(const real_t dt,
      T_solver.SetOperator(*T);
   }
   MFEM_VERIFY(dt == current_dt, ""); // SDIRK methods use the same dt
-
-   // Construct current right-hand side for stage state vs. slope solve
-   if (ImplicitVarTypeIsState())
-   {
-      // k, on return, is the stage value u_s
-      Mmat.Mult(u, z);
-   }
-   else
-   {
-      // k, on return, is the stage slope du/dt
-      Kmat.Mult(u, z);
-      z.Neg();
-   }
-   T_solver.Mult(z, k);
+   Kmat.Mult(u, z);
+   z.Neg();
+   T_solver.Mult(z, du_dt);
 }

 void ConductionOperator::SetParameters(const Vector &u)
@@ -115,7 +115,6 @@ int main(int argc, char *argv[])
   bool visit = false;
   int vis_steps = 5;
   bool adios2 = false;
-   bool solve_implicit_state = false;

   int precision = 8;
   cout.precision(precision);
@@ -139,9 +138,6 @@ int main(int argc, char *argv[])
                  "Alpha coefficient.");
   args.AddOption(&kappa, "-k", "--kappa",
                  "Kappa coefficient offset.");
-   args.AddOption(&solve_implicit_state, "-imp-state", "--implicit-state",
-                  "-imp-slope", "--implicit-slope",
-                  "Implicitly solve for stage state or slope.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -216,11 +212,6 @@ int main(int argc, char *argv[])

   // 9. Initialize the conduction operator and the VisIt visualization.
   ConductionOperator oper(fespace, alpha, kappa, u);
-   using ImplicitVariableType = ConductionOperator::ImplicitVariableType;
-   ImplicitVariableType imp_var = solve_implicit_state ?
-                                  ImplicitVariableType::STATE
-                                  : ImplicitVariableType::SLOPE;
-   oper.SetImplicitVariableType(imp_var);

   u_gf.SetFromTrueDofs(u);
   {
@@ -416,14 +407,11 @@ void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
 }

 void ConductionOperator::ImplicitSolve(const real_t dt,
-                                       const Vector &u, Vector &k)
+                                       const Vector &u, Vector &du_dt)
 {
   // Solve the equation:
-   //    M*k = -K(u + dt*k) for k = du/dt, if solving for stage-slope
-   // or
-   //    M*k = -dt*K(k) + M*u for k = u_s, if solving for stage-state
-   // where K is linearized by using u from the previous timestep, and
-   // the stage-state and slope relation: du/dt = (u_s - u)/dt.
+   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
+   // for du_dt, where K is linearized by using u from the previous timestep
   if (!T)
   {
      T = Add(1.0, Mmat, dt, Kmat);
@@ -431,20 +419,9 @@ void ConductionOperator::ImplicitSolve(const real_t dt,
      T_solver.SetOperator(*T);
   }
   MFEM_VERIFY(dt == current_dt, ""); // SDIRK methods use the same dt
-
-   // Construct current right-hand side for stage state vs. slope solve
-   if (ImplicitVarTypeIsState())
-   {
-      // k, on return, is the stage value u
-      Mmat.Mult(u, z);
-   }
-   else
-   {
-      // k, on return, is the stage slope du/dt
-      Kmat.Mult(u, z);
-      z.Neg();
-   }
-   T_solver.Mult(z, k);
+   Kmat.Mult(u, z);
+   z.Neg();
+   T_solver.Mult(z, du_dt);
 }

 void ConductionOperator::SetParameters(const Vector &u)
@@ -18,10 +18,10 @@
 //               non-symmetric discontinuous Galerkin (DG) formulation.
 //
 //               Specifically, we approximate the weak form of -div(sigma(u))=0
-//               where sigma(u)=lambda*div(u)*I+mu*(grad(u)+grad(u)^T) is the
-//               stress tensor corresponding to displacement field u, and lambda
-//               and mu are the material Lame constants. The boundary conditions
-//               are Dirichlet, u=u_D on the fixed part of the boundary, namely
+//               where sigma(u)=lambda*div(u)*I+mu*(grad*u+u*grad) is the stress
+//               tensor corresponding to displacement field u, and lambda and mu
+//               are the material Lame constants. The boundary conditions are
+//               Dirichlet, u=u_D on the fixed part of the boundary, namely
 //               boundary attributes 1 and 2; on the rest of the boundary we use
 //               sigma(u).n=0 b.c. The geometry of the domain is assumed to be
 //               as follows:
@@ -18,10 +18,10 @@
 //               non-symmetric discontinuous Galerkin (DG) formulation.
 //
 //               Specifically, we approximate the weak form of -div(sigma(u))=0
-//               where sigma(u)=lambda*div(u)*I+mu*(grad(u)+grad(u)^T) is the
-//               stress tensor corresponding to displacement field u, and lambda
-//               and mu are the material Lame constants. The boundary conditions
-//               are Dirichlet, u=u_D on the fixed part of the boundary, namely
+//               where sigma(u)=lambda*div(u)*I+mu*(grad*u+u*grad) is the stress
+//               tensor corresponding to displacement field u, and lambda and mu
+//               are the material Lame constants. The boundary conditions are
+//               Dirichlet, u=u_D on the fixed part of the boundary, namely
 //               boundary attributes 1 and 2; on the rest of the boundary we use
 //               sigma(u).n=0 b.c. The geometry of the domain is assumed to be
 //               as follows:
@@ -42,11 +42,7 @@
 //               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
 //               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/square-mixed.mesh
 //               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/fichera-mixed.mesh
-//               mpirun -np 4 ex1p -pa -d ceed-cpu -m ../data/beam-tet.mesh
-//
-// Device simplices sample runs:
-//               mpirun -np 4 ex1p -pa -d gpu -m ../data/inline-tet.mesh
-//               mpirun -np 4 ex1p -pa -d gpu -m ../data/inline-tri.mesh
+//               mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Poisson problem
@@ -87,9 +83,6 @@ int main(int argc, char *argv[])
   const char *device_config = "cpu";
   bool visualization = true;
   bool algebraic_ceed = false;
-#ifdef MFEM_USE_CUDSS
-   bool cudss_solver = false;
-#endif

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -109,10 +102,6 @@ int main(int argc, char *argv[])
   args.AddOption(&algebraic_ceed, "-a", "--algebraic",
                  "-no-a", "--no-algebraic",
                  "Use algebraic Ceed solver");
-#endif
-#ifdef MFEM_USE_CUDSS
-   args.AddOption(&cudss_solver, "-cudss", "--cudss-solver", "-no-cudss",
-                  "--no-cudss-solver", "Use the cuDSS Solver.");
 #endif
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
@@ -169,20 +158,19 @@ int main(int argc, char *argv[])
   }

   // 7. Define a parallel finite element space on the parallel mesh. Here we
-   //    use continuous Lagrange finite elements of the specified order.
-   //    - If order < 1, we instead use an isoparametric/isogeometric space.
-   //    - If the mesh is simplicial and partial assembly is requested,
-   //      we use the positive basis, which supports device execution.
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
   FiniteElementCollection *fec;
-   auto basis_type = (pa && pmesh.IsSimplexMesh()) ?
-                     BasisType::Positive : BasisType::GaussLobatto;
+   bool delete_fec;
   if (order > 0)
   {
-      fec = new H1_FECollection(order, dim, basis_type);
+      fec = new H1_FECollection(order, dim);
+      delete_fec = true;
   }
   else if (pmesh.GetNodes())
   {
      fec = pmesh.GetNodes()->OwnFEC();
+      delete_fec = false;
      if (myid == 0)
      {
         cout << "Using isoparametric FEs: " << fec->Name() << endl;
@@ -190,7 +178,8 @@ int main(int argc, char *argv[])
   }
   else
   {
-      fec = new H1_FECollection(order = 1, dim, basis_type);
+      fec = new H1_FECollection(order = 1, dim);
+      delete_fec = true;
   }
   ParFiniteElementSpace fespace(&pmesh, fec);
   HYPRE_BigInt size = fespace.GlobalTrueVSize();
@@ -259,51 +248,33 @@ int main(int argc, char *argv[])
   // 13. Solve the linear system A X = B.
   //     * With full assembly, use the BoomerAMG preconditioner from hypre.
   //     * With partial assembly, use Jacobi smoothing, for now.
-#ifdef MFEM_USE_CUDSS
-   if (!pa && (Device::Allows(Backend::CUDA_MASK) && cudss_solver))
+   Solver *prec = NULL;
+   if (pa)
   {
-      // Solve using a direct solver with cuDSS
-      CuDSSSolver cudss_solver(MPI_COMM_WORLD);
-      cudss_solver.SetMatrixSymType(
-         CuDSSSolver::SYMMETRIC_POSITIVE_DEFINITE);
-      cudss_solver.SetMatrixViewType(CuDSSSolver::UPPER);
-      cudss_solver.SetOperator(*A);
-      cudss_solver.Mult(B, X);
-   }
-   else
-#endif
-   {
-      Solver *prec = NULL;
-      if (pa)
+      if (UsesTensorBasis(fespace))
      {
-         if (UsesTensorBasis(fespace))
+         if (algebraic_ceed)
         {
-            if (algebraic_ceed)
-            {
-               prec = new ceed::AlgebraicSolver(a, ess_tdof_list);
-            }
-            else
-            {
-               prec = new OperatorJacobiSmoother(a, ess_tdof_list);
-            }
+            prec = new ceed::AlgebraicSolver(a, ess_tdof_list);
+         }
+         else
+         {
+            prec = new OperatorJacobiSmoother(a, ess_tdof_list);
         }
      }
-      else
-      {
-         prec = new HypreBoomerAMG;
-      }
-      CGSolver cg(MPI_COMM_WORLD);
-      cg.SetRelTol(1e-12);
-      cg.SetMaxIter(2000);
-      cg.SetPrintLevel(1);
-      if (prec)
-      {
-         cg.SetPreconditioner(*prec);
-      }
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete prec;
   }
+   else
+   {
+      prec = new HypreBoomerAMG;
+   }
+   CGSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(1);
+   if (prec) { cg.SetPreconditioner(*prec); }
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+   delete prec;

   // 14. Recover the parallel grid function corresponding to X. This is the
   //     local finite element solution on each processor.
@@ -337,7 +308,10 @@ int main(int argc, char *argv[])
   }

   // 17. Free the used memory.
-   if (order > 0) { delete fec; }
+   if (delete_fec)
+   {
+      delete fec;
+   }

   return 0;
 }
@@ -119,7 +119,7 @@ int main(int argc, char *argv[])
   }

   LinearForm b(&fespace);
-   b.AddBoundaryIntegrator(new VectorBoundaryLFIntegrator(f));
+   b.AddDomainIntegrator(new VectorBoundaryLFIntegrator(f));

   // 6. Set up the bilinear form a(.,.) on the finite element space
   //    corresponding to the linear elasticity integrator with piece-wise
@@ -140,7 +140,7 @@ int main(int argc, char *argv[])
   }

   ParLinearForm b(&fespace);
-   b.AddBoundaryIntegrator(new VectorBoundaryLFIntegrator(f));
+   b.AddDomainIntegrator(new VectorBoundaryLFIntegrator(f));

   // 6. Set up the bilinear form a(.,.) on the finite element space
   //    corresponding to the linear elasticity integrator with piece-wise
@@ -471,13 +471,10 @@ int main(int argc, char *argv[])

      ofstream sol_r_ofs("sol_r.gf");
      ofstream sol_i_ofs("sol_i.gf");
-      ofstream sol_z_ofs("sol_z.gf");
      sol_r_ofs.precision(8);
      sol_i_ofs.precision(8);
-      sol_z_ofs.precision(8);
      u.real().Save(sol_r_ofs);
      u.imag().Save(sol_i_ofs);
-      u.Save(sol_z_ofs);
   }

   // 14. Send the solution by socket to a GLVis server.
@@ -302,21 +302,15 @@ int main(int argc, char *argv[])
      char vishost[] = "localhost";
      int  visport   = 19916;
      socketstream sol_sock_r(vishost, visport);
+      socketstream sol_sock_i(vishost, visport);
      sol_sock_r << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
      sol_sock_r.precision(8);
+      sol_sock_i.precision(8);
      sol_sock_r << "solution\n" << *pmesh << u_exact->real()
                 << "window_title 'Exact: Real Part'" << flush;
-      // Make sure all ranks have sent their real solution before initiating
-      // another set of GLVis connections (one from each rank):
-      MPI_Barrier(pmesh->GetComm());
-      socketstream sol_sock_i(vishost, visport);
-      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock_i.precision(8);
      sol_sock_i << "solution\n" << *pmesh << u_exact->imag()
                 << "window_title 'Exact: Imaginary Part'" << flush;
-      // Make sure all ranks have sent their imaginary solution before initiating
-      // another set of GLVis connections (one from each rank):
-      MPI_Barrier(pmesh->GetComm());
   }

   // 11. Set up the parallel sesquilinear form a(.,.) on the finite element
@@ -513,11 +507,10 @@ int main(int argc, char *argv[])
   // 15. Save the refined mesh and the solution in parallel. This output can be
   //     viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
   {
-      ostringstream mesh_name, sol_r_name, sol_i_name, sol_z_name;
+      ostringstream mesh_name, sol_r_name, sol_i_name;
      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
      sol_r_name << "sol_r." << setfill('0') << setw(6) << myid;
      sol_i_name << "sol_i." << setfill('0') << setw(6) << myid;
-      sol_z_name << "sol_z." << setfill('0') << setw(6) << myid;

      ofstream mesh_ofs(mesh_name.str().c_str());
      mesh_ofs.precision(8);
@@ -525,13 +518,10 @@ int main(int argc, char *argv[])

      ofstream sol_r_ofs(sol_r_name.str().c_str());
      ofstream sol_i_ofs(sol_i_name.str().c_str());
-      ofstream sol_z_ofs(sol_z_name.str().c_str());
      sol_r_ofs.precision(8);
      sol_i_ofs.precision(8);
-      sol_z_ofs.precision(8);
      u.real().Save(sol_r_ofs);
      u.imag().Save(sol_i_ofs);
-      u.Save(sol_z_ofs);
   }

   // 16. Send the solution by socket to a GLVis server.
@@ -540,21 +530,15 @@ int main(int argc, char *argv[])
      char vishost[] = "localhost";
      int  visport   = 19916;
      socketstream sol_sock_r(vishost, visport);
+      socketstream sol_sock_i(vishost, visport);
      sol_sock_r << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
      sol_sock_r.precision(8);
+      sol_sock_i.precision(8);
      sol_sock_r << "solution\n" << *pmesh << u.real()
                 << "window_title 'Solution: Real Part'" << flush;
-      // Make sure all ranks have sent their real solution before initiating
-      // another set of GLVis connections (one from each rank):
-      MPI_Barrier(pmesh->GetComm());
-      socketstream sol_sock_i(vishost, visport);
-      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock_i.precision(8);
      sol_sock_i << "solution\n" << *pmesh << u.imag()
                 << "window_title 'Solution: Imaginary Part'" << flush;
-      // Make sure all ranks have sent their imaginary solution before initiating
-      // another set of GLVis connections (one from each rank):
-      MPI_Barrier(pmesh->GetComm());
   }
   if (visualization && exact_sol)
   {
@@ -563,21 +547,15 @@ int main(int argc, char *argv[])
      char vishost[] = "localhost";
      int  visport   = 19916;
      socketstream sol_sock_r(vishost, visport);
+      socketstream sol_sock_i(vishost, visport);
      sol_sock_r << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
      sol_sock_r.precision(8);
+      sol_sock_i.precision(8);
      sol_sock_r << "solution\n" << *pmesh << u_exact->real()
                 << "window_title 'Error: Real Part'" << flush;
-      // Make sure all ranks have sent their real solution before initiating
-      // another set of GLVis connections (one from each rank):
-      MPI_Barrier(pmesh->GetComm());
-      socketstream sol_sock_i(vishost, visport);
-      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock_i.precision(8);
      sol_sock_i << "solution\n" << *pmesh << u_exact->imag()
                 << "window_title 'Error: Imaginary Part'" << flush;
-      // Make sure all ranks have sent their imaginary solution before initiating
-      // another set of GLVis connections (one from each rank):
-      MPI_Barrier(pmesh->GetComm());
   }
   if (visualization)
   {
@@ -95,15 +95,6 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   if (amg_elast && !static_cond && reorder_space)
-   {
-      if (myid == 0)
-         cerr << "\nThe AMG elasticity solver requires ordering byVDIM! "
-              << "Ignoring the specified option -nodes/--by-nodes.\n"
-              << endl;
-      reorder_space = false;
-   }
-
   // 3. Enable hardware devices such as GPUs, and programming models such as
   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
   Device device(device_config);
@@ -5,8 +5,8 @@
 // Sample runs:
 //     ex37 -alpha 10
 //     ex37 -alpha 10 -pv
-//     ex37 -lambda 0.1 -mu 0.1 -growth 1
-//     ex37 -o 2 -alpha 10.0 -mi 50 -vf 0.4 -ntol 1e-5 -growth 1.5
+//     ex37 -lambda 0.1 -mu 0.1
+//     ex37 -o 2 -alpha 5.0 -mi 50 -vf 0.4 -ntol 1e-5
 //     ex37 -r 6 -o 1 -alpha 25.0 -epsilon 0.02 -mi 50 -ntol 1e-5
 //
 // Description: This example code demonstrates the use of MFEM to solve a
@@ -55,6 +55,53 @@
 using namespace std;
 using namespace mfem;

+/**
+ * @brief Bregman projection of ρ = sigmoid(ψ) onto the subspace
+ *        ∫_Ω ρ dx = θ vol(Ω) as follows:
+ *
+ *        1. Compute the root of the R → R function
+ *            f(c) = ∫_Ω sigmoid(ψ + c) dx - θ vol(Ω)
+ *        2. Set ψ ← ψ + c.
+ *
+ * @param psi a GridFunction to be updated
+ * @param target_volume θ vol(Ω)
+ * @param tol Newton iteration tolerance
+ * @param max_its Newton maximum iteration number
+ * @return real_t Final volume, ∫_Ω sigmoid(ψ)
+ */
+real_t proj(GridFunction &psi, real_t target_volume, real_t tol=1e-12,
+            int max_its=10)
+{
+   MappedGridFunctionCoefficient sigmoid_psi(&psi, sigmoid);
+   MappedGridFunctionCoefficient der_sigmoid_psi(&psi, der_sigmoid);
+
+   LinearForm int_sigmoid_psi(psi.FESpace());
+   int_sigmoid_psi.AddDomainIntegrator(new DomainLFIntegrator(sigmoid_psi));
+   LinearForm int_der_sigmoid_psi(psi.FESpace());
+   int_der_sigmoid_psi.AddDomainIntegrator(new DomainLFIntegrator(
+                                              der_sigmoid_psi));
+   bool done = false;
+   for (int k=0; k<max_its; k++) // Newton iteration
+   {
+      int_sigmoid_psi.Assemble(); // Recompute f(c) with updated ψ
+      const real_t f = int_sigmoid_psi.Sum() - target_volume;
+
+      int_der_sigmoid_psi.Assemble(); // Recompute df(c) with updated ψ
+      const real_t df = int_der_sigmoid_psi.Sum();
+
+      const real_t dc = -f/df;
+      psi += dc;
+      if (abs(dc) < tol) { done = true; break; }
+   }
+   if (!done)
+   {
+      mfem_warning("Projection reached maximum iteration without converging. "
+                   "Result may not be accurate.");
+   }
+   int_sigmoid_psi.Assemble();
+   return int_sigmoid_psi.Sum();
+}
+
 /*
 * ---------------------------------------------------------------
 *                      ALGORITHM PREAMBLE
@@ -133,11 +180,10 @@ int main(int argc, char *argv[])
   int ref_levels = 5;
   int order = 2;
   real_t alpha = 1.0;
-   real_t growth = 2;
   real_t epsilon = 0.01;
   real_t vol_fraction = 0.5;
   int max_it = 1e3;
-   real_t itol = 1e-2;
+   real_t itol = 1e-1;
   real_t ntol = 1e-4;
   real_t rho_min = 1e-6;
   real_t lambda = 1.0;
@@ -152,8 +198,6 @@ int main(int argc, char *argv[])
                  "Order (degree) of the finite elements.");
   args.AddOption(&alpha, "-alpha", "--alpha-step-length",
                  "Step length for gradient descent.");
-   args.AddOption(&growth, "-growth", "--alpha-growth-rate",
-                  "Growth rate of step length for gradient descent.");
   args.AddOption(&epsilon, "-epsilon", "--epsilon-thickness",
                  "Length scale for ρ.");
   args.AddOption(&max_it, "-mi", "--max-it",
@@ -288,7 +332,6 @@ int main(int argc, char *argv[])
   }
   FilterSolver->SetEssentialBoundary(ess_bdr_filter);
   FilterSolver->SetupFEM();
-   FilterSolver->AssembleDiffusionBilinear();

   BilinearForm mass(&control_fes);
   mass.AddDomainIntegrator(new InverseIntegrator(new MassIntegrator(one)));
@@ -342,7 +385,7 @@ int main(int argc, char *argv[])
   // 11. Iterate:
   for (int k = 1; k <= max_it; k++)
   {
-      if (k > 1) { alpha = std::pow((real_t) k,growth); }
+      if (k > 1) { alpha *= ((real_t) k) / ((real_t) k-1); }

      mfem::out << "\nStep = " << k << std::endl;

@@ -379,9 +422,7 @@ int main(int argc, char *argv[])

      // Step 5 - Update design variable ψ ← proj(ψ - αG)
      psi.Add(-alpha, grad);
-      GridFunction alpha_grad(grad);
-      alpha_grad *= alpha;
-      const real_t material_volume = proj(psi, alpha_grad, target_volume);
+      const real_t material_volume = proj(psi, target_volume);

      // Compute ||ρ - ρ_old|| in control fes.
      real_t norm_increment = zerogf.ComputeL1Error(succ_diff_rho);
@@ -137,7 +137,7 @@ public:
        exponent(exponent_), rho_min(rho_min_)
   {
      MFEM_ASSERT(rho_min_ >= 0.0, "rho_min must be >= 0");
-      MFEM_ASSERT(rho_min_ < 1.0,  "rho_min must be < 1");
+      MFEM_ASSERT(rho_min_ < 1.0,  "rho_min must be > 1");
      MFEM_ASSERT(u, "displacement field is not set");
      MFEM_ASSERT(rho_filter, "density field is not set");
   }
@@ -231,12 +231,9 @@ private:
   FiniteElementCollection * fec = nullptr;
   FiniteElementSpace * fes = nullptr;
   Array<int> ess_bdr;
-   Array<int> ess_tdof_list;
   Array<int> neumann_bdr;
   GridFunction * u = nullptr;
   LinearForm * b = nullptr;
-   BilinearForm * a = nullptr;
-   OperatorPtr A;
   bool parallel;
 #ifdef MFEM_USE_MPI
   ParMesh * pmesh = nullptr;
@@ -270,8 +267,6 @@ public:
   void ResetFEM();
   void SetupFEM();

-   void UpdateEssentialTDofs();
-   void AssembleDiffusionBilinear(bool update_ess_tdofs=true);
   void Solve();
   GridFunction * GetFEMSolution();
   LinearForm * GetLinearForm() {return b;}
@@ -376,130 +371,6 @@ public:

 };

-/**
- * @brief Bregman projection of ρ = sigmoid(ψ) onto the subspace
- *        ∫_Ω ρ dx = θ vol(Ω) as follows:
- *
- *        1. Compute the root of the R → R function
- *            f(c) = ∫_Ω sigmoid(ψ + c) dx - θ vol(Ω)
- *           using the Illinois method
- *        2. Set ψ ← ψ + c.
- *
- * @param psi a GridFunction to be updated
- * @param alpha_grad alpha multiplied by gradient
- * @param target_volume θ vol(Ω)
- * @param tol Illinois iteration tolerance
- * @param max_its Illinois maximum iteration number
- * @return real_t Final volume (∫_Ω sigmoid(ψ) dx)
- */
-real_t proj(GridFunction &psi, GridFunction &alpha_grad, real_t target_volume,
-            real_t tol = 1e-12, int max_its = 100)
-{
-#ifdef MFEM_USE_MPI
-   FiniteElementSpace *fes = psi.FESpace();
-   ParFiniteElementSpace *pfes = dynamic_cast<ParFiniteElementSpace*>(fes);
-#endif
-   ConstantCoefficient zero_cf(0.0);
-   real_t a = -alpha_grad.ComputeMaxError(zero_cf);
-   real_t b = -a;
-   real_t y = 0.0;
-
-   MappedGridFunctionCoefficient sigmoid_psi(
-   &psi, [&y](const real_t x) { return sigmoid(x + y); });
-   std::unique_ptr<LinearForm> int_sigmoid_psi;
-#ifdef MFEM_USE_MPI
-   ParGridFunction *par_psi = dynamic_cast<ParGridFunction *>(&psi);
-   if (par_psi)
-   {
-      int_sigmoid_psi.reset(new ParLinearForm(par_psi->ParFESpace()));
-   }
-   else
-   {
-      int_sigmoid_psi.reset(new LinearForm(psi.FESpace()));
-   }
-#else
-   int_sigmoid_psi.reset(new LinearForm(psi.FESpace()));
-#endif
-   int_sigmoid_psi->AddDomainIntegrator(new DomainLFIntegrator(sigmoid_psi));
-
-   y = a;
-   int_sigmoid_psi->Assemble();
-   real_t f_a = int_sigmoid_psi->Sum();  // f_a := f(a) + θ vol(Ω)
-
-   y = b;
-   int_sigmoid_psi->Assemble();
-   real_t f_b = int_sigmoid_psi->Sum();  // f_b := f(b) + θ vol(Ω)
-#ifdef MFEM_USE_MPI
-   if (pfes)
-   {
-      MPI_Allreduce(MPI_IN_PLACE, &f_a, 1, MPITypeMap<real_t>::mpi_type,
-                    MPI_SUM, MPI_COMM_WORLD);
-      MPI_Allreduce(MPI_IN_PLACE, &f_b, 1, MPITypeMap<real_t>::mpi_type,
-                    MPI_SUM, MPI_COMM_WORLD);
-   }
-#endif
-   f_a -= target_volume; // f_a := f(a)
-   f_b -= target_volume; // f_b := f(b)
-   real_t c = 0.0;
-   real_t f_c = 0.0;
-   int side = 0;
-
-   bool done = false;
-   for (int k=0; k < max_its; k++)
-   {
-      c = (f_a * b - f_b * a) / (f_a - f_b);
-
-      if (abs(b - a) < tol * abs(b + a)) { done = true; break; }
-
-      y = c;
-      int_sigmoid_psi->Assemble();
-      f_c = int_sigmoid_psi->Sum(); // f_c := f(c) + θ vol(Ω)
-#ifdef MFEM_USE_MPI
-      if (pfes)
-      {
-         MPI_Allreduce(MPI_IN_PLACE, &f_c, 1, MPITypeMap<real_t>::mpi_type,
-                       MPI_SUM, MPI_COMM_WORLD);
-      }
-#endif
-      f_c -= target_volume; // f_c := f(c)
-
-      if (f_c * f_b > 0)
-      {
-         b = c;
-         f_b = f_c;
-         if (side == -1) { f_a /= 2.0; }
-         side = -1;
-      }
-      else if (f_c * f_a > 0)
-      {
-         a = c;
-         f_a = f_c;
-         if (side == 1) { f_b /= 2.0; }
-         side = 1;
-      }
-      else
-      {
-         done = true; break;
-      }
-   }
-   if (!done)
-   {
-      mfem_warning("Projection reached maximum iteration without converging. "
-                   "Result may not be accurate.");
-   }
-   y = 0.0;
-   psi += c;
-   int_sigmoid_psi->Assemble();
-   real_t material_volume = int_sigmoid_psi->Sum();
-#ifdef MFEM_USE_MPI
-   if (pfes)
-   {
-      MPI_Allreduce(MPI_IN_PLACE, &material_volume, 1,
-                    MPITypeMap<real_t>::mpi_type, MPI_SUM, MPI_COMM_WORLD);
-   }
-#endif
-   return material_volume;
-}

 // Poisson solver

@@ -551,8 +422,12 @@ void DiffusionSolver::SetupFEM()
   }
 }

-void DiffusionSolver::UpdateEssentialTDofs()
+void DiffusionSolver::Solve()
 {
+   OperatorPtr A;
+   Vector B, X;
+   Array<int> ess_tdof_list;
+
 #ifdef MFEM_USE_MPI
   if (parallel)
   {
@@ -565,39 +440,7 @@ void DiffusionSolver::UpdateEssentialTDofs()
 #else
   fes->GetEssentialTrueDofs(ess_bdr,ess_tdof_list);
 #endif
-}
-
-void DiffusionSolver::AssembleDiffusionBilinear(bool update_ess_tdofs)
-{
-   if (update_ess_tdofs)
-   {
-      UpdateEssentialTDofs();
-   }
-#ifdef MFEM_USE_MPI
-   if (parallel)
-   {
-      a = new ParBilinearForm(pfes);
-   }
-   else
-   {
-      a = new BilinearForm(fes);
-   }
-#else
-   a = new BilinearForm(fes);
-#endif
-   a->AddDomainIntegrator(new DiffusionIntegrator(*diffcf));
-   if (masscf)
-   {
-      a->AddDomainIntegrator(new MassIntegrator(*masscf));
-   }
-   a->Assemble();
-   a->FormSystemMatrix(ess_tdof_list, A);
-}
-
-void DiffusionSolver::Solve()
-{
-   Vector B, X;
-
+   *u=0.0;
   if (b)
   {
      delete b;
@@ -632,33 +475,31 @@ void DiffusionSolver::Solve()

   b->Assemble();

-   *u=0.0;
-   if (essbdr_cf)
-   {
-      u->ProjectBdrCoefficient(*essbdr_cf,ess_bdr);
-   }
+   BilinearForm * a = nullptr;

 #ifdef MFEM_USE_MPI
   if (parallel)
   {
-      X.SetSize(pfes->TrueVSize());
-      B.SetSize(pfes->TrueVSize());
-      dynamic_cast<ParGridFunction*>(u)->ParallelAssemble(X);
-      dynamic_cast<ParLinearForm*>(b)->ParallelAssemble(B);
-      dynamic_cast<ParBilinearForm*>(a)->ParallelEliminateTDofsInRHS(
-         ess_tdof_list, X, B);
+      a = new ParBilinearForm(pfes);
   }
   else
   {
-      X.NewDataAndSize(u->GetData(), u->Size());
-      B.NewDataAndSize(b->GetData(), b->Size());
-      a->EliminateVDofsInRHS(ess_tdof_list, X, B);
+      a = new BilinearForm(fes);
   }
 #else
-   X.NewDataAndSize(u->GetData(), u->Size());
-   B.NewDataAndSize(b->GetData(), b->Size());
-   a->EliminateVDofsInRHS(ess_tdof_list, X, B);
+   a = new BilinearForm(fes);
 #endif
+   a->AddDomainIntegrator(new DiffusionIntegrator(*diffcf));
+   if (masscf)
+   {
+      a->AddDomainIntegrator(new MassIntegrator(*masscf));
+   }
+   a->Assemble();
+   if (essbdr_cf)
+   {
+      u->ProjectBdrCoefficient(*essbdr_cf,ess_bdr);
+   }
+   a->FormLinearSystem(ess_tdof_list, *u, *b, A, X, B);

   CGSolver * cg = nullptr;
   Solver * M = nullptr;
@@ -687,6 +528,7 @@ void DiffusionSolver::Solve()
   delete M;
   delete cg;
   a->RecoverFEMSolution(X, *b, *u);
+   delete a;
 }

 GridFunction * DiffusionSolver::GetFEMSolution()
@@ -718,8 +560,6 @@ DiffusionSolver::~DiffusionSolver()
 #endif
   delete fec; fec = nullptr;
   delete b;
-   A.Clear();
-   delete a;
 }


@@ -4,8 +4,8 @@
 //
 // Sample runs:
 //    mpirun -np 4 ex37p -alpha 10 -pv
-//    mpirun -np 4 ex37p -lambda 0.1 -mu 0.1 -growth 1
-//    mpirun -np 4 ex37p -o 2 -alpha 10.0 -mi 50 -vf 0.4 -ntol 1e-5 -growth 1.5
+//    mpirun -np 4 ex37p -lambda 0.1 -mu 0.1
+//    mpirun -np 4 ex37p -o 2 -alpha 5.0 -mi 50 -vf 0.4 -ntol 1e-5
 //    mpirun -np 4 ex37p -r 6 -o 2 -alpha 10.0 -epsilon 0.02 -mi 50 -ntol 1e-5
 //
 // Description: This example code demonstrates the use of MFEM to solve a
@@ -54,6 +54,61 @@
 using namespace std;
 using namespace mfem;

+/**
+ * @brief Bregman projection of ρ = sigmoid(ψ) onto the subspace
+ *        ∫_Ω ρ dx = θ vol(Ω) as follows:
+ *
+ *        1. Compute the root of the R → R function
+ *            f(c) = ∫_Ω sigmoid(ψ + c) dx - θ vol(Ω)
+ *        2. Set ψ ← ψ + c.
+ *
+ * @param psi a GridFunction to be updated
+ * @param target_volume θ vol(Ω)
+ * @param tol Newton iteration tolerance
+ * @param max_its Newton maximum iteration number
+ * @return real_t Final volume, ∫_Ω sigmoid(ψ)
+ */
+real_t proj(ParGridFunction &psi, real_t target_volume, real_t tol=1e-12,
+            int max_its=10)
+{
+   MappedGridFunctionCoefficient sigmoid_psi(&psi, sigmoid);
+   MappedGridFunctionCoefficient der_sigmoid_psi(&psi, der_sigmoid);
+
+   ParLinearForm int_sigmoid_psi(psi.ParFESpace());
+   int_sigmoid_psi.AddDomainIntegrator(new DomainLFIntegrator(sigmoid_psi));
+   ParLinearForm int_der_sigmoid_psi(psi.ParFESpace());
+   int_der_sigmoid_psi.AddDomainIntegrator(new DomainLFIntegrator(
+                                              der_sigmoid_psi));
+   bool done = false;
+   for (int k=0; k<max_its; k++) // Newton iteration
+   {
+      int_sigmoid_psi.Assemble(); // Recompute f(c) with updated ψ
+      real_t f = int_sigmoid_psi.Sum();
+      MPI_Allreduce(MPI_IN_PLACE, &f, 1, MPITypeMap<real_t>::mpi_type,
+                    MPI_SUM, MPI_COMM_WORLD);
+      f -= target_volume;
+
+      int_der_sigmoid_psi.Assemble(); // Recompute df(c) with updated ψ
+      real_t df = int_der_sigmoid_psi.Sum();
+      MPI_Allreduce(MPI_IN_PLACE, &df, 1, MPITypeMap<real_t>::mpi_type,
+                    MPI_SUM, MPI_COMM_WORLD);
+
+      const real_t dc = -f/df;
+      psi += dc;
+      if (abs(dc) < tol) { done = true; break; }
+   }
+   if (!done)
+   {
+      mfem_warning("Projection reached maximum iteration without converging. "
+                   "Result may not be accurate.");
+   }
+   int_sigmoid_psi.Assemble();
+   real_t material_volume = int_sigmoid_psi.Sum();
+   MPI_Allreduce(MPI_IN_PLACE, &material_volume, 1,
+                 MPITypeMap<real_t>::mpi_type, MPI_SUM, MPI_COMM_WORLD);
+   return material_volume;
+}
+
 /*
 * ---------------------------------------------------------------
 *                      ALGORITHM PREAMBLE
@@ -138,11 +193,10 @@ int main(int argc, char *argv[])
   int ref_levels = 5;
   int order = 2;
   real_t alpha = 1.0;
-   real_t growth = 2;
   real_t epsilon = 0.01;
   real_t vol_fraction = 0.5;
   int max_it = 1e3;
-   real_t itol = 1e-2;
+   real_t itol = 1e-1;
   real_t ntol = 1e-4;
   real_t rho_min = 1e-6;
   real_t lambda = 1.0;
@@ -157,8 +211,6 @@ int main(int argc, char *argv[])
                  "Order (degree) of the finite elements.");
   args.AddOption(&alpha, "-alpha", "--alpha-step-length",
                  "Step length for gradient descent.");
-   args.AddOption(&growth, "-growth", "--alpha-growth-rate",
-                  "Growth rate of step length for gradient descent.");
   args.AddOption(&epsilon, "-epsilon", "--epsilon-thickness",
                  "Length scale for ρ.");
   args.AddOption(&max_it, "-mi", "--max-it",
@@ -307,7 +359,6 @@ int main(int argc, char *argv[])
   }
   FilterSolver->SetEssentialBoundary(ess_bdr_filter);
   FilterSolver->SetupFEM();
-   FilterSolver->AssembleDiffusionBilinear();

   ParBilinearForm mass(&control_fes);
   mass.AddDomainIntegrator(new InverseIntegrator(new MassIntegrator(one)));
@@ -361,7 +412,7 @@ int main(int argc, char *argv[])
   // 11. Iterate:
   for (int k = 1; k <= max_it; k++)
   {
-      if (k > 1) { alpha = std::pow((real_t) k,growth); }
+      if (k > 1) { alpha *= ((real_t) k) / ((real_t) k-1); }

      if (myid == 0)
      {
@@ -401,9 +452,7 @@ int main(int argc, char *argv[])

      // Step 5 - Update design variable ψ ← proj(ψ - αG)
      psi.Add(-alpha, grad);
-      ParGridFunction alpha_grad(grad);
-      alpha_grad *= alpha;
-      const real_t material_volume = proj(psi, alpha_grad, target_volume);
+      const real_t material_volume = proj(psi, target_volume);

      // Compute ||ρ - ρ_old|| in control fes.
      real_t norm_increment = zerogf.ComputeL1Error(succ_diff_rho);
@@ -9,7 +9,6 @@
 //               ex4 -m ../data/beam-hex.mesh -o 2 -pa
 //               ex4 -m ../data/escher.mesh
 //               ex4 -m ../data/fichera.mesh -o 2 -hb
-//               ex4 -m ../data/fichera.mesh -o 2 -hb -ea
 //               ex4 -m ../data/fichera-q2.vtk
 //               ex4 -m ../data/fichera-q3.mesh -o 2 -sc
 //               ex4 -m ../data/square-disc-nurbs.mesh
@@ -19,7 +18,6 @@
 //               ex4 -m ../data/amr-quad.mesh
 //               ex4 -m ../data/amr-hex.mesh
 //               ex4 -m ../data/amr-hex.mesh -o 2 -hb
-//               ex4 -m ../data/amr-hex.mesh -o 2 -hb -ea
 //               ex4 -m ../data/fichera-amr.mesh -o 2 -sc
 //               ex4 -m ../data/ref-prism.mesh -o 1
 //               ex4 -m ../data/octahedron.mesh -o 1
@@ -27,8 +25,6 @@
 //
 // Device sample runs:
 //               ex4 -m ../data/star.mesh -pa -d cuda
-//               ex4 -m ../data/star.mesh -hb -ea -d cuda
-//               ex4 -m ../data/amr-quad.mesh -hb -ea -d cuda
 //               ex4 -m ../data/star.mesh -pa -d raja-cuda
 //               ex4 -m ../data/star.mesh -pa -d raja-omp
 //               ex4 -m ../data/beam-hex.mesh -pa -d cuda
@@ -197,7 +193,7 @@ int main(int argc, char *argv[])
   cout << "Size of linear system: " << A->Height() << endl;

   // 11. Solve the linear system A X = B.
-   if (!pa && (!ea || hybridization))
+   if (!pa)
   {
 #ifndef MFEM_USE_SUITESPARSE
      // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
@@ -1,590 +0,0 @@
-//                                MFEM Example 41
-//
-// Compile with: make ex41
-//
-// Sample runs:
-//  ex41
-//  ex41 -cg
-//  ex41 -m ../data/periodic-hexagon.mesh -p 0 -r 2 -dt 0.005 -tf 10
-//  ex41 -m ../data/periodic-square.mesh -p 1 -r 2 -dt 0.005 -tf 9
-//  ex41 -m ../data/periodic-hexagon.mesh -p 1 -r 2 -dt 0.005 -tf 9
-//  ex41 -m ../data/amr-quad.mesh -p 1 -r 2 -dt 0.002 -tf 9
-//  ex41 -m ../data/star-q3.mesh -p 1 -r 2 -dt 0.001 -tf 9
-//  ex41 -m ../data/star-mixed.mesh -p 1 -r 2 -dt 0.005 -tf 9
-//  ex41 -m ../data/disc-nurbs.mesh -p 1 -r 3 -dt 0.005 -tf 9
-//  ex41 -m ../data/disc-nurbs.mesh -p 2 -r 3 -dt 0.005 -tf 9
-//  ex41 -m ../data/periodic-square.mesh -p 3 -r 4 -dt 0.0025 -tf 9 -vs 20
-//  ex41 -m ../data/periodic-cube.mesh -p 0 -r 2 -o 2 -dt 0.01 -tf 8
-//
-// Device sample runs:
-//
-// Description:  This example code solves the time-dependent advection-diffusion
-//               equation du/dt + v.grad(u) - a div(grad(u)) = 0, where v is a
-//               given fluid velocity, a is the diffusion coefficient, and
-//               u0(x)=u(0,x) is a given initial condition.
-//
-//               The example demonstrates the use of Discontinuous Galerkin (DG)
-//               bilinear forms in MFEM (face integrators), and the use of IMEX
-//               ODE time integrators.
-//
-//               The option to use continuous finite elements is available too.
-
-#include "mfem.hpp"
-
-using namespace std;
-using namespace mfem;
-
-// Mesh bounding box
-Vector bb_min, bb_max;
-
-// Velocity coefficient
-template<int problem=0>
-void velocity_function(const Vector &x, Vector &v)
-{
-   int dim = x.Size();
-
-   // map to the reference [-1,1] domain
-   Vector X(dim);
-   for (int i = 0; i < dim; i++)
-   {
-      real_t center = (bb_min[i] + bb_max[i]) * 0.5;
-      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
-   }
-   switch (problem)
-   {
-      case 0:
-      {
-         // Translations in 1D, 2D, and 3D
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = sqrt(2./3.); v(1) = sqrt(1./3.); break;
-            case 3: v(0) = sqrt(3./6.); v(1) = sqrt(2./6.); v(2) = sqrt(1./6.);
-               break;
-         }
-         break;
-      }
-      case 1:
-      case 2:
-      {
-         // Clockwise rotation in 2D around the origin
-         const real_t w = M_PI/2;
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = w*X(1); v(1) = -w*X(0); break;
-            case 3: v(0) = w*X(1); v(1) = -w*X(0); v(2) = 0.0; break;
-         }
-         break;
-      }
-      case 3:
-      {
-         // Clockwise twisting rotation in 2D around the origin
-         const real_t w = M_PI/2;
-         real_t d = max((X(0)+1.)*(1.-X(0)),0.) * max((X(1)+1.)*(1.-X(1)),0.);
-         d = d*d;
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = d*w*X(1); v(1) = -d*w*X(0); break;
-            case 3: v(0) = d*w*X(1); v(1) = -d*w*X(0); v(2) = 0.0; break;
-         }
-         break;
-      }
-   }
-}
-
-// Initial condition
-template<int problem=0>
-real_t u0_function(const Vector &x)
-{
-   int dim = x.Size();
-
-   // map to the reference [-1,1] domain
-   Vector X(dim);
-   for (int i = 0; i < dim; i++)
-   {
-      real_t center = (bb_min[i] + bb_max[i]) * 0.5;
-      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
-   }
-
-   switch (problem)
-   {
-      case 0:
-      case 1:
-      {
-         switch (dim)
-         {
-            case 1:
-               return exp(-40.*pow(X(0)-0.5,2));
-            case 2:
-            case 3:
-            {
-               real_t rx = 0.45, ry = 0.25, cx = 0., cy = -0.2, w = 10.;
-               if (dim == 3)
-               {
-                  const real_t s = (1. + 0.25*cos(2*M_PI*X(2)));
-                  rx *= s;
-                  ry *= s;
-               }
-               return ( std::erfc(w*(X(0)-cx-rx))*std::erfc(-w*(X(0)-cx+rx)) *
-                        std::erfc(w*(X(1)-cy-ry))*std::erfc(-w*(X(1)-cy+ry)) )/16;
-            }
-         }
-      }
-      case 2:
-      {
-         real_t x_ = X(0), y_ = X(1), rho, phi;
-         rho = std::hypot(x_, y_);
-         phi = atan2(y_, x_);
-         return pow(sin(M_PI*rho),2)*sin(3*phi);
-      }
-      case 3:
-      {
-         const real_t f = M_PI;
-         return sin(f*X(0))*sin(f*X(1));
-      }
-   }
-   return 0.0;
-}
-
-/// Solver for the implicit part of the ODE (the diffusion term).
-/// Solves systems of the form: (M + dt*S) k = rhs.
-class Implicit_Solver : public Solver
-{
-private:
-   SparseMatrix &M, &S, A;
-   CGSolver linear_solver;
-   BlockILU prec;
-   real_t dt;
-public:
-   Implicit_Solver(SparseMatrix &M_, SparseMatrix &S_,
-                   const FiniteElementSpace &fes)
-      : M(M_),
-        S(S_),
-        prec(fes.GetTypicalFE()->GetDof(),
-             BlockILU::Reordering::MINIMUM_DISCARDED_FILL),
-        dt(1.0)
-   {
-      linear_solver.iterative_mode = false;
-      linear_solver.SetRelTol(1e-9);
-      linear_solver.SetAbsTol(0.0);
-      linear_solver.SetMaxIter(100);
-      linear_solver.SetPrintLevel(0);
-      linear_solver.SetPreconditioner(prec);
-   }
-
-   void SetTimeStep(real_t dt_)
-   {
-      real_t ddt = dt-dt_;
-
-      // allow for some tolerance in the time stepping process
-      constexpr real_t epsilon = std::numeric_limits<real_t>::epsilon() * 10;
-
-      if (std::abs(ddt) > epsilon)
-      {
-         dt = dt_;
-         // Form operator A = M + dt*S
-         A = S;
-         A *= dt;
-         A += M;
-
-         // this will also call SetOperator on the preconditioner
-         linear_solver.SetOperator(A);
-      }
-   }
-
-   void SetOperator(const Operator &op) override
-   {
-      linear_solver.SetOperator(op);
-   }
-
-   void Mult(const Vector &x, Vector &y) const override
-   {
-      linear_solver.Mult(x, y);
-   }
-};
-
-/** A time-dependent operator for the right-hand side of the ODE. The weak
-    form of the advection-diffusion equation is M du/dt = K u - S u + b,
-    where M is the mass matrix, K and S are the advection and diffusion
-    matrices, and b describes the flow on the boundary. In the case of IMEX
-    evolution, the diffusion term is treated implicitly, and the advection
-    term is treated explicitly.  */
-class IMEX_Evolution : public TimeDependentOperator
-{
-private:
-   BilinearForm &M, &K, &S;
-   const Vector &b;
-   unique_ptr<Solver> M_prec;
-   CGSolver M_solver;
-   unique_ptr<Implicit_Solver> implicit_solver;
-
-   mutable Vector z;
-
-public:
-   IMEX_Evolution(BilinearForm &M_, BilinearForm &K_, BilinearForm &S_,
-                  const Vector &b_);
-
-   /// Evaluate k1=M^{-1}*G1(u,t); -> k1 = M^{-1}*(K*u + b)
-   void Mult1(const Vector &x, Vector &y) const;
-
-   /// Evaluate k2: M*k2 = G2(u+k2*dt,t); -> (M+S*dt)*k2=-S*u
-   void ImplicitSolve2(const real_t dt, const Vector &x, Vector &k);
-
-   void Mult(const Vector &x, Vector &y) const override
-   {
-      if (TimeDependentOperator::EvalMode::ADDITIVE_TERM_1 == GetEvalMode())
-      {
-         Mult1(x,y);
-      }
-      else
-      {
-         mfem_error("TimeDependentOperator::Mult() is not overridden!");
-      }
-   }
-
-   void ImplicitSolve(const real_t dt, const Vector &x, Vector &k) override
-   {
-      if (TimeDependentOperator::EvalMode::ADDITIVE_TERM_2 == GetEvalMode())
-      {
-         ImplicitSolve2(dt,x,k);
-      }
-      else
-      {
-         mfem_error("TimeDependentOperator::ImplicitSolve() is not overridden!");
-      }
-   }
-};
-
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   int problem = 0;
-   const char *mesh_file = "../data/periodic-square.mesh";
-   int ref_levels = 2;
-   int order = 3;
-   int ode_solver_type = 64; //IMEXRK3(3,4,3)
-   real_t t_final = 10.0;
-   real_t dt = 0.01;
-   bool paraview = false;
-   bool cg = false;
-   int vis_steps = 50;
-   real_t diffusion_term = 0.01;
-   real_t kappa = -1.0;
-   real_t sigma = -1.0;
-   bool visualization = true;
-   bool visit = false;
-   bool binary = false;
-   int precision = 8;
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
-   args.AddOption(&problem, "-p", "--problem",
-                  "Problem setup to use. See options in velocity_function().");
-   args.AddOption(&ref_levels, "-r", "--refine",
-                  "Number of times to refine the mesh uniformly.");
-   args.AddOption(&order, "-o", "--order", "Order of the finite elements.");
-   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  ODESolver::IMEXTypes.c_str());
-   args.AddOption(&t_final, "-tf", "--t-final", "Final time; start time is 0.");
-   args.AddOption(&dt, "-dt", "--time-step", "Time step.");
-   args.AddOption(&diffusion_term, "-dc", "--diffusion-coeff",
-                  "Diffusion coefficient in the PDE.");
-   args.AddOption(&paraview, "-paraview", "--paraview-datafiles", "-no-paraview",
-                  "--no-paraview-datafiles",
-                  "Save data files for ParaView (paraview.org) visualization.");
-   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
-                  "Visualize every n-th timestep.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&binary, "-binary", "--binary-datafiles", "-ascii",
-                  "--ascii-datafiles",
-                  "Use binary (Sidre) or ascii format for VisIt data files.");
-   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
-                  "--no-visit-datafiles",
-                  "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.AddOption(&cg, "-cg", "--continuous-galerkin", "-dg",
-                  "--discontinuous-galerkin",
-                  "Use Continuous-Galerkin Finite elements (Default is DG)");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   if (kappa < 0)
-   {
-      kappa = (order+1)*(order+1);
-   }
-   args.PrintOptions(cout);
-
-   // 2. Read the mesh from the given mesh file. We can handle geometrically
-   //    periodic meshes in this code.
-   Mesh mesh(mesh_file);
-   const int dim = mesh.Dimension();
-
-   // 3. Define the IMEX (Split) ODE solver used for time integration. The IMEX
-   // solvers currently available are: 61 - Forward Backward Euler,
-   // 62 - IMEXRK2(2,2,2), 63 - IMEXRK2(2,3,2), and  64 - IMEX_DIRK_RK3.
-   unique_ptr<ODESolver> ode_solver = ODESolver::SelectIMEX(ode_solver_type);
-
-   // 4. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement, where 'ref_levels' is a
-   //    command-line parameter.
-   for (int lev = 0; lev < ref_levels; lev++) {mesh.UniformRefinement();}
-   if (mesh.NURBSext) {mesh.SetCurvature(max(order, 1));}
-   mesh.GetBoundingBox(bb_min, bb_max, max(order, 1));
-
-   // 5. Define the discontinuous DG finite element space of the given
-   //    polynomial order on the refined mesh.
-   FiniteElementCollection *fec = NULL;
-   if (cg)
-   {
-      fec = new H1_FECollection(order, dim);
-   }
-   else
-   {
-      fec = new DG_FECollection(order, dim, BasisType::GaussLobatto);
-   }
-   FiniteElementSpace fes(&mesh, fec);
-
-   cout << "Number of unknowns: " << fes.GetVSize() << endl;
-
-   // 6. Set up the bilinear and linear forms corresponding to the DG
-   //    discretization. The DGTraceIntegrator involves integrals over mesh
-   //    interior faces.
-   std::unique_ptr<VectorFunctionCoefficient> velocity;
-   if (0==problem)
-   {
-      velocity.reset(new VectorFunctionCoefficient(dim, velocity_function<0>));
-   }
-   else if (1==problem)
-   {
-      velocity.reset(new VectorFunctionCoefficient(dim, velocity_function<1>));
-   }
-   else if (2==problem)
-   {
-      velocity.reset(new VectorFunctionCoefficient(dim, velocity_function<2>));
-   }
-   else if (3==problem)
-   {
-      velocity.reset(new VectorFunctionCoefficient(dim, velocity_function<3>));
-   }
-
-   ConstantCoefficient diff_coeff(diffusion_term);
-
-   BilinearForm m(&fes);
-   BilinearForm k(&fes);
-   BilinearForm s(&fes);
-
-   Vector b(fes.GetTrueVSize());
-   b = 0.0; //The inflow on the boundaries is set to zero.
-
-   m.AddDomainIntegrator(new MassIntegrator);
-
-   constexpr real_t alpha = -1.0;
-   k.AddDomainIntegrator(new ConvectionIntegrator(*velocity, alpha));
-
-   s.AddDomainIntegrator(new DiffusionIntegrator(diff_coeff));
-   if (!cg)
-   {
-      k.AddInteriorFaceIntegrator(new NonconservativeDGTraceIntegrator(*velocity,
-                                                                       alpha));
-      k.AddBdrFaceIntegrator(new NonconservativeDGTraceIntegrator(*velocity, alpha));
-      s.AddInteriorFaceIntegrator(new DGDiffusionIntegrator(diff_coeff, sigma,
-                                                            kappa));
-      s.AddBdrFaceIntegrator(new DGDiffusionIntegrator(diff_coeff, sigma, kappa));
-   }
-
-   // 7. Assemble the bilinear forms.
-
-   int skip_zeros = 0;
-   m.Assemble(skip_zeros);
-   k.Assemble(skip_zeros);
-   s.Assemble(skip_zeros);
-
-   m.Finalize(skip_zeros);
-   k.Finalize(skip_zeros);
-   s.Finalize(skip_zeros);
-
-   // 8. Define the initial conditions.
-   std::unique_ptr<FunctionCoefficient> u0;
-   if (0==problem)
-   {
-      u0.reset(new FunctionCoefficient(u0_function<0>));
-   }
-   else if (1==problem)
-   {
-      u0.reset(new FunctionCoefficient(u0_function<1>));
-   }
-   else if (2==problem)
-   {
-      u0.reset(new FunctionCoefficient(u0_function<2>));
-   }
-   else if (3==problem)
-   {
-      u0.reset(new FunctionCoefficient(u0_function<3>));
-   }
-
-   GridFunction u(&fes);
-   u.ProjectCoefficient(*u0);
-
-   // 9. Create data collection for solution output: either VisItDataCollection
-   //    for ascii data files, or SidreDataCollection for binary data files.
-   DataCollection *dc = NULL;
-   if (visit)
-   {
-      if (binary)
-      {
-#ifdef MFEM_USE_SIDRE
-         dc = new SidreDataCollection("Example41", &mesh);
-#else
-         MFEM_ABORT("Must build with MFEM_USE_SIDRE=YES for binary output.");
-#endif
-      }
-      else
-      {
-         dc = new VisItDataCollection("Example41", &mesh);
-         dc->SetPrecision(precision);
-      }
-      dc->RegisterField("solution", &u);
-      dc->SetCycle(0);
-      dc->SetTime(0.0);
-      dc->Save();
-   }
-
-   // 10. Set up paraview visualization, if desired.
-   unique_ptr<ParaViewDataCollection> pv;
-   if (paraview)
-   {
-      pv = make_unique<ParaViewDataCollection>("Example41", &mesh);
-      pv->SetPrefixPath("ParaView");
-      pv->RegisterField("solution", &u);
-      pv->SetLevelsOfDetail(order);
-      pv->SetDataFormat(VTKFormat::BINARY);
-      pv->SetHighOrderOutput(true);
-      pv->SetCycle(0);
-      pv->SetTime(0.0);
-      pv->Save();
-   }
-
-   socketstream sout;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      sout.open(vishost, visport);
-      if (!sout)
-      {
-         cout << "Unable to connect to GLVis server at "
-              << vishost << ':' << visport << endl;
-         visualization = false;
-         cout << "GLVis visualization disabled.\n";
-      }
-      else
-      {
-         sout.precision(precision);
-         sout << "solution\n" << mesh << u;
-         sout << "pause\n";
-         sout << flush;
-         cout << "GLVis visualization paused."
-              << " Press space (in the GLVis window) to resume it.\n";
-      }
-   }
-
-   // 11. Define the time-dependent evolution operator describing the ODE
-   //    right-hand side, and perform time-integration (looping over the time
-   //    iterations, ti, with a time-step dt).
-   IMEX_Evolution adv(m, k, s, b);
-
-   real_t t = 0.0;
-   adv.SetTime(t);
-   ode_solver->Init(adv);
-
-   bool done = false;
-   for (int ti = 0; !done; )
-   {
-      real_t dt_real = min(dt, t_final - t);
-      ode_solver->Step(u, t, dt_real);
-      ti++;
-
-      done = (t >= t_final - 1e-8*dt);
-
-      if (done || ti % vis_steps == 0)
-      {
-         cout << "time step: " << ti << ", time: " << t << endl;
-         if (paraview)
-         {
-            pv->SetCycle(ti);
-            pv->SetTime(t);
-            pv->Save();
-         }
-         if (visualization)
-         {
-            sout << "solution\n" << mesh << u << flush;
-         }
-         if (visit)
-         {
-            dc->SetCycle(ti);
-            dc->SetTime(t);
-            dc->Save();
-         }
-
-      }
-   }
-
-   // 12. Free the used memory.
-
-   delete fec;
-   return 0;
-}
-
-
-// Implementation of class IMEX_Evolution
-IMEX_Evolution::IMEX_Evolution(BilinearForm &M_, BilinearForm &K_,
-                               BilinearForm &S_, const Vector &b_)
-   : TimeDependentOperator(M_.FESpace()->GetTrueVSize()),
-     M(M_), K(K_), S(S_), b(b_), z(height)
-{
-   Array<int> ess_tdof_list;
-   if (M.GetAssemblyLevel() == AssemblyLevel::LEGACY)
-   {
-      M_prec = make_unique<DSmoother>(M.SpMat());
-      M_solver.SetOperator(M.SpMat());
-      implicit_solver = make_unique<Implicit_Solver>(M.SpMat(), S.SpMat(),
-                                                     *M.FESpace());
-   }
-   else
-   {
-      MFEM_ABORT("Implicit time integration is not supported with partial assembly");
-   }
-   M_solver.SetPreconditioner(*M_prec);
-   M_solver.iterative_mode = false;
-   M_solver.SetRelTol(1e-9);
-   M_solver.SetAbsTol(0.0);
-   M_solver.SetMaxIter(100);
-   M_solver.SetPrintLevel(0);
-}
-
-void IMEX_Evolution::Mult1(const Vector &x, Vector &y) const
-{
-   // Perform the explicit step
-   // y = M^{-1} (K x + b)
-   K.Mult(x, z);
-   z += b;
-   M_solver.Mult(z, y);
-}
-
-void IMEX_Evolution::ImplicitSolve2(const real_t dt, const Vector &x, Vector &k)
-{
-   // Perform the implicit step
-   // solve for k, k = -(M+dt S)^{-1} S x
-   MFEM_VERIFY(implicit_solver != NULL,
-               "Implicit time integration is not supported with partial assembly");
-   S.Mult(x, z);
-   z.Neg();
-   implicit_solver->SetTimeStep(dt);
-   implicit_solver->Mult(z, k);
-}
@@ -1,725 +0,0 @@
-//                                MFEM Example 41 - Parallel Version
-//
-// Compile with: make ex41p
-//
-// Sample runs:
-//  mpirun -np 4 ex41p
-//  mpirun -np 4 ex41p -cg
-//  mpirun -np 4 ex41p -m ../data/periodic-hexagon.mesh -p 0 -dt 0.005 -tf 10
-//  mpirun -np 4 ex41p -m ../data/periodic-square.mesh -p 1 -dt 0.005 -tf 9
-//  mpirun -np 4 ex41p -m ../data/periodic-hexagon.mesh -p 1  -dt 0.005 -tf 9
-//  mpirun -np 4 ex41p -m ../data/star-q3.mesh -p 1 -rp 1 -dt 0.001 -tf 9
-//  mpirun -np 4 ex41p -m ../data/disc-nurbs.mesh -p 1 -rp 1 -dt 0.005 -tf 9
-//  mpirun -np 4 ex41p -m ../data/disc-nurbs.mesh -p 2 -rp 1 -dt 0.005 -tf 9
-//  mpirun -np 4 ex41p -m ../data/periodic-square.mesh -rp 2 -dt 0.0025 -tf 9 -vs 20
-//  mpirun -np 4 ex41p -m ../data/periodic-cube.mesh -p 0 -rs 2 -o 2 -dt 0.01 -tf 8
-//
-// Device sample runs:
-//
-// Description:  This example code solves the time-dependent advection-diffusion
-//               equation du/dt + v.grad(u) - a div(grad(u)) = 0, where v is a
-//               given fluid velocity, a is the diffusion coefficient, and
-//               u0(x)=u(0,x) is a given initial condition.
-//
-//               The example demonstrates the use of Discontinuous Galerkin (DG)
-//               bilinear forms in MFEM (face integrators), DG-LOR Preconditioning
-//               and the use of IMEX ODE time integrators.
-//
-//               The Option to use Continuous Finite Elements is available too.
-
-#include "mfem.hpp"
-
-using namespace std;
-using namespace mfem;
-
-// Mesh bounding box
-Vector bb_min, bb_max;
-
-// Velocity coefficient
-template<int problem=0>
-void velocity_function(const Vector &x, Vector &v)
-{
-   int dim = x.Size();
-
-   // map to the reference [-1,1] domain
-   Vector X(dim);
-   for (int i = 0; i < dim; i++)
-   {
-      real_t center = (bb_min[i] + bb_max[i]) * 0.5;
-      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
-   }
-   switch (problem)
-   {
-      case 0:
-      {
-         // Translations in 1D, 2D, and 3D
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = sqrt(2./3.); v(1) = sqrt(1./3.); break;
-            case 3: v(0) = sqrt(3./6.); v(1) = sqrt(2./6.); v(2) = sqrt(1./6.);
-               break;
-         }
-         break;
-      }
-      case 1:
-      case 2:
-      {
-         // Clockwise rotation in 2D around the origin
-         const real_t w = M_PI/2;
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = w*X(1); v(1) = -w*X(0); break;
-            case 3: v(0) = w*X(1); v(1) = -w*X(0); v(2) = 0.0; break;
-         }
-         break;
-      }
-      case 3:
-      {
-         // Clockwise twisting rotation in 2D around the origin
-         const real_t w = M_PI/2;
-         real_t d = max((X(0)+1.)*(1.-X(0)),0.) * max((X(1)+1.)*(1.-X(1)),0.);
-         d = d*d;
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = d*w*X(1); v(1) = -d*w*X(0); break;
-            case 3: v(0) = d*w*X(1); v(1) = -d*w*X(0); v(2) = 0.0; break;
-         }
-         break;
-      }
-   }
-}
-
-// Initial condition
-template<int problem=0>
-real_t u0_function(const Vector &x)
-{
-   int dim = x.Size();
-
-   // map to the reference [-1,1] domain
-   Vector X(dim);
-   for (int i = 0; i < dim; i++)
-   {
-      real_t center = (bb_min[i] + bb_max[i]) * 0.5;
-      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
-   }
-
-   switch (problem)
-   {
-      case 0:
-      case 1:
-      {
-         switch (dim)
-         {
-            case 1:
-               return exp(-40.*pow(X(0)-0.5,2));
-            case 2:
-            case 3:
-            {
-               real_t rx = 0.45, ry = 0.25, cx = 0., cy = -0.2, w = 10.;
-               if (dim == 3)
-               {
-                  const real_t s = (1. + 0.25*cos(2*M_PI*X(2)));
-                  rx *= s;
-                  ry *= s;
-               }
-               return ( std::erfc(w*(X(0)-cx-rx))*std::erfc(-w*(X(0)-cx+rx)) *
-                        std::erfc(w*(X(1)-cy-ry))*std::erfc(-w*(X(1)-cy+ry)) )/16;
-            }
-         }
-      }
-      case 2:
-      {
-         real_t x_ = X(0), y_ = X(1), rho, phi;
-         rho = std::hypot(x_, y_);
-         phi = atan2(y_, x_);
-         return pow(sin(M_PI*rho),2)*sin(3*phi);
-      }
-      case 3:
-      {
-         const real_t f = M_PI;
-         return sin(f*X(0))*sin(f*X(1));
-      }
-   }
-   return 0.0;
-}
-
-/// Solver for the implicit part of the ODE (the diffusion term).
-/// Solves systems of the form: (M + dt*S) k = rhs.
-class Implicit_Solver : public Solver
-{
-private:
-   HypreParMatrix &M, &S;
-   unique_ptr<HypreParMatrix> A;
-   CGSolver linear_solver;
-   real_t dt;
-public:
-   Implicit_Solver(HypreParMatrix &M_, HypreParMatrix &S_,
-                   const FiniteElementSpace &fes)
-      : M(M_),
-        S(S_),
-        linear_solver(M.GetComm()),
-        dt(1.0)
-   {
-      linear_solver.iterative_mode = false;
-      linear_solver.SetRelTol(1e-9);
-      linear_solver.SetAbsTol(0.0);
-      linear_solver.SetMaxIter(100);
-      linear_solver.SetPrintLevel(0);
-   }
-
-   void SetTimeStep(real_t dt_)
-   {
-      real_t ddt = dt-dt_;
-
-      // syncronize ddt across all processes
-      MPI_Comm comm = M.GetComm();
-      int myrank;
-      MPI_Comm_rank(comm, &myrank);
-      MPI_Bcast(&ddt, 1, MFEM_MPI_REAL_T, 0, comm);
-
-      // allow for some tolerance in the time stepping process
-      constexpr real_t epsilon = std::numeric_limits<real_t>::epsilon() * 10;
-
-      if (fabs(ddt) > epsilon)
-      {
-         if (0==myrank)
-         {
-            cout << "Updating Implicit_Solver time step from " << dt
-                 << " to " << dt_ << endl;
-         }
-
-         dt = dt_;
-         // Form operator A = M + dt*S
-         A.reset(Add(dt, S, 1.0, M));
-         linear_solver.SetOperator(*A);
-      }
-   }
-
-   void SetOperator(const Operator &op) override
-   {
-      linear_solver.SetOperator(op);
-   }
-
-   void Mult(const Vector &x, Vector &y) const override
-   {
-      linear_solver.Mult(x, y);
-   }
-
-   void SetPreconditioner(Solver &precond)
-   {
-      linear_solver.SetPreconditioner(precond);
-   }
-};
-
-/** A time-dependent operator for the right-hand side of the ODE. The weak
-    form of the advection-diffusion equation is M du/dt = K u - S u + b,
-    where M is the mass matrix, K and S are the advection and diffusion
-    matrices, and b describes the flow on the boundary. In the case of IMEX
-    evolution, the diffusion term is treated implicitly, and the advection
-    term is treated explicitly.  */
-class IMEX_Evolution : public TimeDependentOperator
-{
-private:
-   OperatorHandle M, K, S, A;
-   const Vector &b;
-   unique_ptr<Solver> M_prec;
-   CGSolver M_solver;
-   unique_ptr<Implicit_Solver> implicit_solver;
-   unique_ptr<LORSolver<HypreBoomerAMG>> lor_solver;
-
-   mutable Vector z;
-
-public:
-   IMEX_Evolution(ParBilinearForm &M_, ParBilinearForm &K_, ParBilinearForm &S_,
-                  const Vector &b_, ParBilinearForm &A_);
-
-   /// Evaluate k1=M^{-1}*G1(u,t); -> k1 = M^{-1}*(K*u + b)
-   void Mult1(const Vector &x, Vector &y) const;
-
-   /// Evaluate k2: M*k2 = G2(u+k2*dt,t); -> (M+S*dt)*k2=-S*u
-   void ImplicitSolve2(const real_t dt, const Vector &x, Vector &k);
-
-   void Mult(const Vector &x, Vector &y) const override
-   {
-      if (TimeDependentOperator::EvalMode::ADDITIVE_TERM_1 == GetEvalMode())
-      {
-         Mult1(x,y);
-      }
-      else
-      {
-         mfem_error("TimeDependentOperator::Mult() is not overridden!");
-      }
-   }
-
-   void ImplicitSolve(const real_t dt, const Vector &x, Vector &k) override
-   {
-      if (TimeDependentOperator::EvalMode::ADDITIVE_TERM_2 == GetEvalMode())
-      {
-         ImplicitSolve2(dt,x,k);
-      }
-      else
-      {
-         mfem_error("TimeDependentOperator::ImplicitSolve() is not overridden!");
-      }
-   }
-};
-
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI and HYPRE.
-   Mpi::Init();
-   int num_procs = Mpi::WorldSize();
-   int myid = Mpi::WorldRank();
-   Hypre::Init();
-
-   // 2. Parse command-line options.
-   int problem = 0;
-   const char *mesh_file = "../data/periodic-square.mesh";
-   int ser_ref_levels = 2;
-   int par_ref_levels = 0;
-   int order = 3;
-   int ode_solver_type = 64; // 61 - Forward Backward Euler
-   // 62 - IMEXRK2(2,2,2)
-   // 63 - IMEXRK2(2,3,2)
-   // 64 - IMEXRK3(3,4,3)
-   real_t t_final = 10.0;
-   real_t dt = 0.01;
-   bool paraview = false;
-   bool cg = false;
-   int vis_steps = 50;
-   bool adios2 = false;
-   bool binary = false;
-   real_t diffusion_term = 0.01;
-   real_t kappa = -1.0;
-   real_t sigma = -1.0;
-   bool visualization = true;
-   bool visit = false;
-   int precision = 16;
-   cout.precision(precision);
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&problem, "-p", "--problem",
-                  "Problem setup to use. See options in velocity_function().");
-   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
-                  "Number of times to refine the mesh uniformly in parallel.");
-   args.AddOption(&order, "-o", "--order",
-                  "Order (degree) of the finite elements.");
-   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  ODESolver::IMEXTypes.c_str());
-   args.AddOption(&t_final, "-tf", "--t-final",
-                  "Final time; start time is 0.");
-   args.AddOption(&dt, "-dt", "--time-step",
-                  "Time step.");
-   args.AddOption(&diffusion_term, "-dc", "--diffusion-coeff",
-                  "Diffusion coefficient in the PDE.");
-   args.AddOption(&paraview, "-paraview", "--paraview-datafiles", "-no-paraview",
-                  "--no-paraview-datafiles",
-                  "Save data files for ParaView (paraview.org) visualization.");
-   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
-                  "--no-visit-datafiles",
-                  "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.AddOption(&adios2, "-adios2", "--adios2-streams", "-no-adios2",
-                  "--no-adios2-streams",
-                  "Save data using adios2 streams.");
-   args.AddOption(&binary, "-binary", "--binary-datafiles", "-ascii",
-                  "--ascii-datafiles",
-                  "Use binary (Sidre) or ascii format for VisIt data files.");
-   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
-                  "Visualize every n-th timestep.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&cg, "-cg", "--continuous-galerkin", "-dg",
-                  "--discontinuous-galerkin",
-                  "Use Continuous-Galerkin Finite elements (Default is DG)");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (Mpi::Root())
-      {
-         args.PrintUsage(cout);
-      }
-      return 1;
-   }
-   if (Mpi::Root())
-   {
-      args.PrintOptions(cout);
-   }
-   if (kappa < 0)
-   {
-      kappa = (order+1)*(order+1);
-   }
-
-   // 3. Read the mesh from the given mesh file. We can handle geometrically
-   //    periodic meshes in this code.
-   Mesh *mesh = new Mesh(mesh_file);
-   const int dim = mesh->Dimension();
-
-   // 4. Define the IMEX (Split) ODE solver used for time integration. The IMEX
-   // solvers currently available are: 55 - Forward Backward Euler,
-   // 56 - IMEXRK2(2,2,2), 57 - IMEXRK2(2,3,2), and
-   unique_ptr<ODESolver> ode_solver = ODESolver::SelectIMEX(ode_solver_type);
-
-   // 5. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement, where 'ref_levels' is a
-   //    command-line parameter.
-   for (int lev = 0; lev < ser_ref_levels; lev++) { mesh->UniformRefinement(); }
-   if (mesh->NURBSext)
-   {
-      mesh->SetCurvature(max(order, 1));
-   }
-   mesh->GetBoundingBox(bb_min, bb_max, max(order, 1));
-
-
-   // 6. Define the parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   for (int lev = 0; lev < par_ref_levels; lev++)
-   {
-      pmesh->UniformRefinement();
-   }
-
-   // 7. Define the discontinuous DG finite element space of the given
-   //    polynomial order on the refined mesh.
-   FiniteElementCollection *fec = NULL;
-   if (cg)
-   {
-      fec = new H1_FECollection(order, dim);
-   }
-   else
-   {
-      fec = new DG_FECollection(order, dim, BasisType::GaussLobatto);
-   }
-   ParFiniteElementSpace *fes = new ParFiniteElementSpace(pmesh, fec);
-
-   HYPRE_BigInt global_vSize = fes->GlobalTrueVSize();
-   if (Mpi::Root())
-   {
-      cout << "Number of unknowns: " << global_vSize << endl;
-   }
-
-   // 8. Set up the bilinear and linear forms corresponding to the DG
-   //    discretization. The DGTraceIntegrator involves integrals over mesh
-   //    interior faces.
-   std::unique_ptr<VectorFunctionCoefficient> velocity;
-   if (0==problem)
-   {
-      velocity.reset(new VectorFunctionCoefficient(dim, velocity_function<0>));
-   }
-   else if (1==problem)
-   {
-      velocity.reset(new VectorFunctionCoefficient(dim, velocity_function<1>));
-   }
-   else if (2==problem)
-   {
-      velocity.reset(new VectorFunctionCoefficient(dim, velocity_function<2>));
-   }
-   else if (3==problem)
-   {
-      velocity.reset(new VectorFunctionCoefficient(dim, velocity_function<3>));
-   }
-   ConstantCoefficient diff_coeff(diffusion_term);
-   ConstantCoefficient dt_diff_coeff(dt*diffusion_term);
-
-   ParBilinearForm *m = new ParBilinearForm(fes);
-   ParBilinearForm *k = new ParBilinearForm(fes);
-   ParBilinearForm *s = new ParBilinearForm(fes);
-
-   m->AddDomainIntegrator(new MassIntegrator());
-
-   constexpr real_t alpha = -1.0;
-   k->AddDomainIntegrator(new ConvectionIntegrator(*velocity, alpha));
-
-   s->AddDomainIntegrator(new DiffusionIntegrator(diff_coeff));
-
-   // For the preconditioner - create billinear form corresponding to
-   // operator (M + dt S)
-   ParBilinearForm *a = new ParBilinearForm(fes);
-   a->AddDomainIntegrator(new MassIntegrator);
-   a->AddDomainIntegrator(new DiffusionIntegrator(dt_diff_coeff));
-   if (!cg)
-   {
-      k->AddInteriorFaceIntegrator(new NonconservativeDGTraceIntegrator(*velocity,
-                                                                        alpha));
-      k->AddBdrFaceIntegrator(new NonconservativeDGTraceIntegrator(*velocity, alpha));
-      s->AddInteriorFaceIntegrator(new DGDiffusionIntegrator(diff_coeff, sigma,
-                                                             kappa));
-      s->AddBdrFaceIntegrator(new DGDiffusionIntegrator(diff_coeff, sigma, kappa));
-      a->AddInteriorFaceIntegrator(new DGDiffusionIntegrator(dt_diff_coeff, sigma,
-                                                             kappa));
-      a->AddBdrFaceIntegrator(new DGDiffusionIntegrator(dt_diff_coeff, sigma, kappa));
-   }
-
-   // 9. Assemble the bilinear forms.
-
-   int skip_zeros = 0;
-   m->Assemble(skip_zeros);
-   k->Assemble(skip_zeros);
-   s->Assemble(skip_zeros);
-   a->Assemble();
-
-   m->Finalize(skip_zeros);
-   k->Finalize(skip_zeros);
-   s->Finalize(skip_zeros);
-   a->Finalize(skip_zeros);
-   HypreParVector b(fes);
-   b = 0.0;
-
-   // 10. Define the initial conditions. Set up visualization (if desired).
-   std::unique_ptr<FunctionCoefficient> u0;
-   if (0==problem)
-   {
-      u0.reset(new FunctionCoefficient(u0_function<0>));
-   }
-   else if (1==problem)
-   {
-      u0.reset(new FunctionCoefficient(u0_function<1>));
-   }
-   else if (2==problem)
-   {
-      u0.reset(new FunctionCoefficient(u0_function<2>));
-   }
-   else if (3==problem)
-   {
-      u0.reset(new FunctionCoefficient(u0_function<3>));
-   }
-   ParGridFunction *u = new ParGridFunction(fes);
-   u->ProjectCoefficient(*u0);
-   HypreParVector *U = u->GetTrueDofs();
-
-   DataCollection *dc = NULL;
-   if (visit)
-   {
-      if (binary)
-      {
-#ifdef MFEM_USE_SIDRE
-         dc = new SidreDataCollection("Example41-Parallel", pmesh);
-#else
-         MFEM_ABORT("Must build with MFEM_USE_SIDRE=YES for binary output.");
-#endif
-      }
-      else
-      {
-         dc = new VisItDataCollection("Example41-Parallel", pmesh);
-         dc->SetPrecision(precision);
-         // To save the mesh using MFEM's parallel mesh format:
-         // dc->SetFormat(DataCollection::PARALLEL_FORMAT);
-      }
-      dc->RegisterField("solution", u);
-      dc->SetCycle(0);
-      dc->SetTime(0.0);
-      dc->Save();
-   }
-   ParaViewDataCollection *pd = NULL;
-   if (paraview)
-   {
-      pd = new ParaViewDataCollection("Example41P", pmesh);
-      pd->SetPrefixPath("ParaView");
-      pd->RegisterField("solution", u);
-      pd->SetLevelsOfDetail(order);
-      pd->SetDataFormat(VTKFormat::BINARY);
-      pd->SetHighOrderOutput(true);
-      pd->SetCycle(0);
-      pd->SetTime(0.0);
-      pd->Save();
-   }
-   socketstream sout;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      sout.open(vishost, visport);
-      if (!sout)
-      {
-         if (Mpi::Root())
-         {
-            cout << "Unable to connect to GLVis server at "
-                 << vishost << ':' << visport << endl;
-         }
-         visualization = false;
-         if (Mpi::Root())
-         {
-            cout << "GLVis visualization disabled.\n";
-         }
-      }
-      else
-      {
-         sout << "parallel " << num_procs << " " << myid << "\n";
-         sout.precision(precision);
-         sout << "solution\n" << *pmesh << *u;
-         sout << "pause\n";
-         sout << flush;
-         if (Mpi::Root())
-         {
-            cout << "GLVis visualization paused."
-                 << " Press space (in the GLVis window) to resume it.\n";
-         }
-      }
-   }
-#ifdef MFEM_USE_ADIOS2
-   ADIOS2DataCollection *adios2_dc = NULL;
-   if (adios2)
-   {
-      std::string postfix(mesh_file);
-      postfix.erase(0, std::string("../data/").size() );
-      postfix += "_o" + std::to_string(order);
-      const std::string collection_name = "ex41-p-" + postfix + ".bp";
-
-      adios2_dc = new ADIOS2DataCollection(MPI_COMM_WORLD, collection_name, pmesh);
-      // output data substreams are half the number of mpi processes
-      adios2_dc->SetParameter("SubStreams", std::to_string(num_procs/2) );
-      // adios2_dc->SetLevelsOfDetail(2);
-      adios2_dc->RegisterField("solution", u);
-      adios2_dc->SetCycle(0);
-      adios2_dc->SetTime(0.0);
-      adios2_dc->Save();
-   }
-#endif
-
-
-   // 11. Define the time-dependent evolution operator describing the ODE
-   //     right-hand side, and perform time-integration (looping over the time
-   //     iterations, ti, with a time-step dt).
-   IMEX_Evolution adv(*m, *k, *s, b, *a);
-
-   real_t t = 0.0;
-   adv.SetTime(t);
-   ode_solver->Init(adv);
-
-   bool done = false;
-   for (int ti = 0; !done; )
-   {
-      real_t dt_real = min(dt, t_final - t);
-      ode_solver->Step(*U, t, dt_real);
-      ti++;
-
-      done = (t >= t_final - 1e-8*dt);
-
-      if (done || ti % vis_steps == 0)
-      {
-         if (Mpi::Root())
-         {
-            cout << "time step: " << ti << ", time: " << t << endl;
-         }
-         *u = *U;
-         if (visualization)
-         {
-            sout << "parallel " << num_procs << " " << myid << "\n";
-            sout << "solution\n" << *pmesh << *u << flush;
-         }
-         if (visit)
-         {
-            dc->SetCycle(ti);
-            dc->SetTime(t);
-            dc->Save();
-         }
-         if (paraview)
-         {
-            pd->SetCycle(ti);
-            pd->SetTime(t);
-            pd->Save();
-         }
-#ifdef MFEM_USE_ADIOS2
-         // transient solutions can be visualized with ParaView
-         if (adios2)
-         {
-            adios2_dc->SetCycle(ti);
-            adios2_dc->SetTime(t);
-            adios2_dc->Save();
-         }
-#endif
-      }
-   }
-
-   // 12. Free the used memory.
-   delete pd;
-   delete U;
-   delete u;
-   delete a;
-   delete s;
-   delete k;
-   delete m;
-   delete fes;
-   delete pmesh;
-   delete dc;
-   delete fec;
-
-   return 0;
-}
-
-// Implementation of class IMEX_Evolution
-IMEX_Evolution::IMEX_Evolution(ParBilinearForm &M_, ParBilinearForm &K_,
-                               ParBilinearForm &S_, const Vector &b_, ParBilinearForm &A_)
-   : TimeDependentOperator(M_.ParFESpace()->GetTrueVSize()), b(b_),
-     M_solver(M_.ParFESpace()->GetComm()), z(height)
-{
-   if (M_.GetAssemblyLevel()==AssemblyLevel::LEGACY)
-   {
-      M.Reset(M_.ParallelAssemble(), true);
-      K.Reset(K_.ParallelAssemble(), true);
-      S.Reset(S_.ParallelAssemble(), true);
-   }
-   else
-   {
-      M.Reset(&M_, false);
-      K.Reset(&K_, false);
-      S.Reset(&S_, false);
-   }
-
-   M_solver.SetOperator(*M);
-
-   Array<int> ess_tdof_list;
-   if (M_.GetAssemblyLevel() == AssemblyLevel::LEGACY)
-   {
-      A.Reset(A_.ParallelAssemble(), true);
-      HypreParMatrix &M_mat = *M.As<HypreParMatrix>();
-      HypreParMatrix &S_mat = *S.As<HypreParMatrix>();
-      M_prec = make_unique<HypreSmoother>(M_mat, HypreSmoother::Jacobi);
-
-      implicit_solver = make_unique<Implicit_Solver>(M_mat, S_mat, *M_.FESpace());
-      lor_solver = make_unique<LORSolver<HypreBoomerAMG>>(A_, ess_tdof_list);
-      lor_solver->GetSolver().SetSystemsOptions(A_.ParFESpace()->GetVDim(), true);
-      implicit_solver -> SetPreconditioner(*lor_solver);
-   }
-   else
-   {
-      MFEM_ABORT("Implicit time integration is not supported with partial assembly");
-   }
-   M_solver.SetPreconditioner(*M_prec);
-   M_solver.iterative_mode = false;
-   M_solver.SetRelTol(1e-9);
-   M_solver.SetAbsTol(0.0);
-   M_solver.SetMaxIter(100);
-   M_solver.SetPrintLevel(0);
-}
-
-void IMEX_Evolution::Mult1(const Vector &x, Vector &y) const
-{
-   // Perform the explicit step
-   // y = M^{-1} (K x + b)
-   K->Mult(x, z);
-   z += b;
-   M_solver.Mult(z, y);
-}
-
-void IMEX_Evolution::ImplicitSolve2(const real_t dt, const Vector &x, Vector &k)
-{
-   // Perform the implicit step
-   // solve for k, k = -(M+dt S)^{-1} S x
-   MFEM_VERIFY(implicit_solver != NULL,
-               "Implicit time integration is not supported with partial assembly");
-   S->Mult(x, z);
-   z.Neg();
-   implicit_solver->SetTimeStep(dt);
-   implicit_solver->Mult(z, k);
-}
@@ -9,7 +9,6 @@
 //               mpirun -np 4 ex4p -m ../data/beam-hex.mesh -o 2 -pa
 //               mpirun -np 4 ex4p -m ../data/escher.mesh -o 2 -sc
 //               mpirun -np 4 ex4p -m ../data/fichera.mesh -o 2 -hb
-//               mpirun -np 4 ex4p -m ../data/fichera.mesh -o 2 -hb -ea
 //               mpirun -np 4 ex4p -m ../data/fichera-q2.vtk
 //               mpirun -np 4 ex4p -m ../data/fichera-q3.mesh -o 2 -sc
 //               mpirun -np 4 ex4p -m ../data/square-disc-nurbs.mesh -o 3
@@ -18,18 +17,14 @@
 //               mpirun -np 4 ex4p -m ../data/periodic-cube.mesh -no-bc
 //               mpirun -np 4 ex4p -m ../data/amr-quad.mesh
 //               mpirun -np 3 ex4p -m ../data/amr-quad.mesh -o 2 -hb
-//               mpirun -np 3 ex4p -m ../data/amr-quad.mesh -o 2 -hb -ea
 //               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -sc
 //               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -hb
-//               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -hb -ea
 //               mpirun -np 4 ex4p -m ../data/ref-prism.mesh -o 1
 //               mpirun -np 4 ex4p -m ../data/octahedron.mesh -o 1
 //               mpirun -np 4 ex4p -m ../data/star-surf.mesh -o 3 -hb
 //
 // Device sample runs:
 //               mpirun -np 4 ex4p -m ../data/star.mesh -pa -d cuda
-//               mpirun -np 4 ex4p -m ../data/star.mesh -ea -hb -d cuda
-//               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -ea -hb -d cuda
 //               mpirun -np 4 ex4p -m ../data/star.mesh -pa -d raja-cuda
 //               mpirun -np 4 ex4p -m ../data/star.mesh -pa -d raja-omp
 //               mpirun -np 4 ex4p -m ../data/beam-hex.mesh -pa -d cuda
@@ -235,7 +230,7 @@ int main(int argc, char *argv[])
   pcg->SetMaxIter(2000);
   pcg->SetPrintLevel(1);
   if (hybridization) { prec = new HypreBoomerAMG(*A.As<HypreParMatrix>()); }
-   else if (pa || ea) { prec = new OperatorJacobiSmoother(*a, ess_tdof_list); }
+   else if (pa) { prec = new OperatorJacobiSmoother(*a, ess_tdof_list); }
   else
   {
      ParFiniteElementSpace *prec_fespace =
@@ -1,543 +0,0 @@
-//                                MFEM Example 5
-//
-// Compile with: make ex5
-//
-// Sample runs:  ex5 -m ../data/square-disc.mesh
-//               ex5 -m ../data/star.mesh
-//               ex5 -m ../data/star.mesh -pa
-//               ex5 -m ../data/beam-tet.mesh
-//               ex5 -m ../data/beam-hex.mesh
-//               ex5 -m ../data/beam-hex.mesh -pa
-//               ex5 -m ../data/escher.mesh
-//               ex5 -m ../data/fichera.mesh
-//
-// Device sample runs:
-//               ex5 -m ../data/star.mesh -pa -d cuda
-//               ex5 -m ../data/star.mesh -pa -d raja-cuda
-//               ex5 -m ../data/star.mesh -pa -d raja-omp
-//               ex5 -m ../data/beam-hex.mesh -pa -d cuda
-//
-// Description:  This example code solves a simple 2D mixed electromagnetic
-//               diffusion problem corresponding to the mixed system
-//
-//                                 sigma E - curl B = f
-//                                  curl E +      B = g
-//
-//               with essential boundary condition E x n = <given tangential field>.
-//               Here, we use a given exact solution (E,B) and compute the
-//               corresponding r.h.s. (f,g).  We discretize with Nedelec
-//               finite elements (electric field E) and piecewise discontinuous
-//               integral polynomials (magnetic field B).
-//
-//               The example demonstrates the use of the DarcyForm class, as
-//               well as the collective saving of several grid functions in
-//               VisIt (visit.llnl.gov) and ParaView (paraview.org) formats.
-//
-//               We recommend viewing examples 1-4 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-#include <algorithm>
-
-using namespace std;
-using namespace mfem;
-
-// Define the analytical solution and forcing terms / boundary conditions
-void EFun_ex(const Vector & x, Vector & E);
-real_t BFun_ex(const Vector & x);
-void fFun(const Vector & x, Vector & f);
-real_t gFun(const Vector & x);
-real_t freq = 1.0, kappa;
-
-int main(int argc, char *argv[])
-{
-   StopWatch chrono;
-
-   // 1. Parse command-line options.
-   const char *mesh_file = "";
-   int nx = 0;
-   int ny = 0;
-   int order = 1;
-   bool hybridization = false;
-   bool pa = false;
-   const char *device_config = "cpu";
-   bool visualization = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&nx, "-nx", "--ncells-x",
-                  "Number of cells in x.");
-   args.AddOption(&ny, "-ny", "--ncells-y",
-                  "Number of cells in y.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
-                  " solution.");
-   args.AddOption(&hybridization, "-hb", "--hybridization", "-no-hb",
-                  "--no-hybridization", "Enable hybridization.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-   kappa = freq * M_PI;
-
-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
-   //    the same code.
-   if (ny <= 0)
-   {
-      ny = nx;
-   }
-
-   Mesh *mesh = NULL;
-   if (strlen(mesh_file) > 0)
-   {
-      mesh = new Mesh(mesh_file, 1, 1);
-   }
-   else
-   {
-      mesh = new Mesh(Mesh::MakeCartesian2D(nx, ny, Element::QUADRILATERAL));
-   }
-
-   int dim = mesh->Dimension();
-
-   // 4. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
-   //    largest number that gives a final mesh with no more than 10,000
-   //    elements.
-   if (strlen(mesh_file) > 0)
-   {
-      int ref_levels =
-         (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh->UniformRefinement();
-      }
-   }
-
-   // 5. Define a finite element space on the mesh. Here we use the
-   //    Raviart-Thomas finite elements of the specified order.
-   FiniteElementCollection *R_coll(new ND_FECollection(order+1, dim));
-   FiniteElementCollection *W_coll(new L2_FECollection(order, dim, 0,
-                                                       FiniteElement::INTEGRAL));
-
-   FiniteElementSpace *R_space = new FiniteElementSpace(mesh, R_coll);
-   FiniteElementSpace *W_space = new FiniteElementSpace(mesh, W_coll);
-
-   DarcyForm *darcy = new DarcyForm(R_space, W_space);
-
-   // 6. Define the BlockStructure of the problem, i.e. define the array of
-   //    offsets for each variable. The last component of the Array is the sum
-   //    of the dimensions of each block.
-   const Array<int> &block_offsets = darcy->GetOffsets();
-
-   std::cout << "***********************************************************\n";
-   std::cout << "dim(R) = " << block_offsets[1] - block_offsets[0] << "\n";
-   std::cout << "dim(W) = " << block_offsets[2] - block_offsets[1] << "\n";
-   std::cout << "dim(R+W) = " << block_offsets.Last() << "\n";
-   std::cout << "***********************************************************\n";
-
-   // 7. Define the coefficients, analytical solution, and rhs of the PDE.
-   ConstantCoefficient muinvsqrt(1.0);
-   ConstantCoefficient sigma(1.0);
-
-   VectorFunctionCoefficient fcoeff(dim, fFun);
-   FunctionCoefficient gcoeff(gFun);
-
-   VectorFunctionCoefficient Ecoeff(dim, EFun_ex);
-   FunctionCoefficient Bcoeff(BFun_ex);
-
-   // 8. Allocate memory (x, rhs) for the analytical solution and the right hand
-   //    side.  Define the GridFunction E,B for the finite element solution and
-   //    linear forms fform and gform for the right hand side.  The data
-   //    allocated by x and rhs are passed as a reference to the grid functions
-   //    (E,B) and the linear forms (fform, gform).
-   MemoryType mt = device.GetMemoryType();
-   BlockVector x(block_offsets, mt), rhs(block_offsets, mt);
-
-   LinearForm *fform(new LinearForm);
-   fform->Update(R_space, rhs.GetBlock(0), 0);
-   fform->AddDomainIntegrator(new VectorFEDomainLFIntegrator(fcoeff));
-   fform->Assemble();
-   fform->SyncAliasMemory(rhs);
-
-   LinearForm *gform(new LinearForm);
-   gform->Update(W_space, rhs.GetBlock(1), 0);
-   gform->AddDomainIntegrator(new DomainLFIntegrator(gcoeff));
-   gform->Assemble();
-   gform->SyncAliasMemory(rhs);
-
-   // 9. Assemble the finite element matrices for the Darcy operator
-   //
-   //                            D = [ M  C^T ]
-   //                                [ C   0  ]
-   //     where:
-   //
-   //     M = \int_\Omega \sigma E_h \cdot v_h d\Omega   E_h, v_h \in R_h
-   //     C   = \int_\Omega \curl E_h q_h d\Omega   E_h \in R_h, q_h \in W_h
-   //BilinearForm *mEVarf(new BilinearForm(R_space));
-   //MixedBilinearForm *cVarf(new MixedBilinearForm(R_space, W_space));
-   BilinearForm *mEVarf = darcy->GetFluxMassForm();
-   MixedBilinearForm *cVarf = darcy->GetFluxDivForm();
-   BilinearForm *mBVarf = darcy->GetPotentialMassForm();
-
-   //if (pa) { mEVarf->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   mEVarf->AddDomainIntegrator(new VectorFEMassIntegrator(sigma));
-   //mEVarf->Assemble();
-   //if (!pa) { mEVarf->Finalize(); }
-
-   //if (pa) { cVarf->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   cVarf->AddDomainIntegrator(new MixedScalarCurlIntegrator(muinvsqrt));
-   //cVarf->Assemble();
-   //if (!pa) { cVarf->Finalize(); }
-
-   mBVarf->AddDomainIntegrator(new MassIntegrator());
-
-   //set essential boundary condition
-
-   GridFunction E, B;
-   E.MakeRef(R_space, x.GetBlock(0), 0);
-   B.MakeRef(W_space, x.GetBlock(1), 0);
-
-   Array<int> ess_flux_tdofs_list;
-   if (mesh->bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(mesh->bdr_attributes.Max());
-      ess_bdr = 1;
-      R_space->GetEssentialTrueDofs(ess_bdr, ess_flux_tdofs_list);
-      E.ProjectBdrCoefficientTangent(Ecoeff, ess_bdr);
-   }
-
-   //set hybridization / assembly level
-
-   FiniteElementCollection *trace_coll = NULL;
-   FiniteElementSpace *trace_space = NULL;
-
-   chrono.Clear();
-   chrono.Start();
-
-   if (hybridization)
-   {
-      trace_coll = new ND_Trace_FECollection(order+1, dim);
-      trace_space = new FiniteElementSpace(mesh, trace_coll);
-      darcy->EnableHybridization(trace_space,
-                                 new TangentTraceJumpIntegrator(),
-                                 ess_flux_tdofs_list);
-   }
-
-   if (pa) { darcy->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-
-   darcy->Assemble();
-   //if (!pa) { darcy->Finalize(); }
-
-   OperatorHandle pDarcyOp;
-   Vector X, RHS;
-
-   //darcy->FormSystemMatrix(ess_flux_tdofs_list, pDarcyOp);
-   darcy->FormLinearSystem(ess_flux_tdofs_list, x, rhs,
-                           pDarcyOp, X, RHS);
-
-   chrono.Stop();
-   std::cout << "Assembly took " << chrono.RealTime() << "s.\n";
-
-
-   int maxIter(1000);
-   real_t rtol(1.e-6);
-   real_t atol(1.e-10);
-
-   if (hybridization)
-   {
-      // 10. Construct the preconditioner
-      GSSmoother prec(*pDarcyOp.As<SparseMatrix>());
-
-      // 11. Solve the linear system with GMRES.
-      //     Check the norm of the unpreconditioned residual.
-      chrono.Clear();
-      chrono.Start();
-      GMRESSolver solver;
-      solver.SetAbsTol(atol);
-      solver.SetRelTol(rtol);
-      solver.SetMaxIter(maxIter);
-      solver.SetOperator(*pDarcyOp);
-      solver.SetPreconditioner(prec);
-      solver.SetPrintLevel(1);
-
-      solver.Mult(RHS, X);
-      darcy->RecoverFEMSolution(X, rhs, x);
-
-      chrono.Stop();
-
-      if (solver.GetConverged())
-      {
-         std::cout << "GMRES converged in " << solver.GetNumIterations()
-                   << " iterations with a residual norm of "
-                   << solver.GetFinalNorm() << ".\n";
-      }
-      else
-      {
-         std::cout << "GMRES did not converge in " << solver.GetNumIterations()
-                   << " iterations. Residual norm is " << solver.GetFinalNorm()
-                   << ".\n";
-      }
-      std::cout << "GMRES solver took " << chrono.RealTime() << "s.\n";
-   }
-   else
-   {
-      // 10. Construct the operators for preconditioner
-      //
-      //                 P = [ diag(M)         0         ]
-      //                     [  0       C diag(M)^-1 C^T ]
-      //
-      //     Here we use Symmetric Gauss-Seidel to approximate the inverse of the
-      //     magnetic field Schur Complement
-      SparseMatrix *MinvBt = NULL;
-      Vector Md(mEVarf->Height());
-
-      BlockDiagonalPreconditioner darcyPrec(block_offsets);
-      Solver *invM, *invS;
-      SparseMatrix *S = NULL;
-
-      if (pa)
-      {
-         mEVarf->AssembleDiagonal(Md);
-         auto Md_host = Md.HostRead();
-         Vector invMd(mEVarf->Height());
-         for (int i=0; i<mEVarf->Height(); ++i)
-         {
-            invMd(i) = 1.0 / Md_host[i];
-         }
-
-         Vector BMBt_diag(cVarf->Height());
-         cVarf->AssembleDiagonal_ADAt(invMd, BMBt_diag);
-
-         Array<int> ess_tdof_list;  // empty
-
-         invM = new OperatorJacobiSmoother(Md, ess_tdof_list);
-         invS = new OperatorJacobiSmoother(BMBt_diag, ess_tdof_list);
-      }
-      else
-      {
-         SparseMatrix &M(mEVarf->SpMat());
-         M.GetDiag(Md);
-         Md.HostReadWrite();
-
-         SparseMatrix &C(cVarf->SpMat());
-         MinvBt = Transpose(C);
-
-         for (int i = 0; i < Md.Size(); i++)
-         {
-            MinvBt->ScaleRow(i, 1./Md(i));
-         }
-
-         S = Mult(C, *MinvBt);
-         if (mBVarf)
-         {
-            SparseMatrix &Mtm(mBVarf->SpMat());
-            SparseMatrix *Snew = Add(Mtm, *S);
-            delete S;
-            S = Snew;
-         }
-
-         invM = new DSmoother(M);
-
-#ifndef MFEM_USE_SUITESPARSE
-         invS = new GSSmoother(*S);
-#else
-         invS = new UMFPackSolver(*S);
-#endif
-      }
-
-      invM->iterative_mode = false;
-      invS->iterative_mode = false;
-
-      darcyPrec.SetDiagonalBlock(0, invM);
-      darcyPrec.SetDiagonalBlock(1, invS);
-
-      // 11. Solve the linear system with MINRES.
-      //     Check the norm of the unpreconditioned residual.
-
-      chrono.Clear();
-      chrono.Start();
-      FGMRESSolver solver;
-      solver.SetAbsTol(atol);
-      solver.SetRelTol(rtol);
-      solver.SetMaxIter(maxIter);
-      solver.SetOperator(*pDarcyOp);
-      solver.SetPreconditioner(darcyPrec);
-      solver.SetPrintLevel(1);
-
-      solver.Mult(RHS, X);
-      darcy->RecoverFEMSolution(X, rhs, x);
-
-      if (device.IsEnabled()) { x.HostRead(); }
-      chrono.Stop();
-
-      if (solver.GetConverged())
-      {
-         std::cout << "GMRES converged in " << solver.GetNumIterations()
-                   << " iterations with a residual norm of "
-                   << solver.GetFinalNorm() << ".\n";
-      }
-      else
-      {
-         std::cout << "GMRES did not converge in " << solver.GetNumIterations()
-                   << " iterations. Residual norm is " << solver.GetFinalNorm()
-                   << ".\n";
-      }
-      std::cout << "MINRES solver took " << chrono.RealTime() << "s.\n";
-
-      delete invM;
-      delete invS;
-      delete S;
-      //delete Bt;
-      delete MinvBt;
-   }
-
-   // 12. Create the grid functions E and B. Compute the L2 error norms.
-
-   int order_quad = max(2, 2*order+1);
-   const IntegrationRule *irs[Geometry::NumGeom];
-   for (int i=0; i < Geometry::NumGeom; ++i)
-   {
-      irs[i] = &(IntRules.Get(i, order_quad));
-   }
-
-   real_t err_E  = E.ComputeL2Error(Ecoeff, irs);
-   real_t norm_E = ComputeLpNorm(2., Ecoeff, *mesh, irs);
-   real_t err_B  = B.ComputeL2Error(Bcoeff, irs);
-   real_t norm_B = ComputeLpNorm(2., Bcoeff, *mesh, irs);
-
-   std::cout << "|| E_h - E_ex || / || E_ex || = " << err_E / norm_E << "\n";
-   std::cout << "|| B_h - B_ex || / || B_ex || = " << err_B / norm_B << "\n";
-
-   // 13. Save the mesh and the solution. This output can be viewed later using
-   //     GLVis: "glvis -m ex5.mesh -g sol_u.gf" or "glvis -m ex5.mesh -g
-   //     sol_p.gf".
-   {
-      ofstream mesh_ofs("ex5.mesh");
-      mesh_ofs.precision(8);
-      mesh->Print(mesh_ofs);
-
-      ofstream E_ofs("sol_E.gf");
-      E_ofs.precision(8);
-      E.Save(E_ofs);
-
-      ofstream B_ofs("sol_B.gf");
-      B_ofs.precision(8);
-      B.Save(B_ofs);
-   }
-
-   // 14. Save data in the VisIt format
-   VisItDataCollection visit_dc("Example5", mesh);
-   visit_dc.RegisterField("electric field", &E);
-   visit_dc.RegisterField("magnetic field", &B);
-   visit_dc.Save();
-
-   // 15. Save data in the ParaView format
-   ParaViewDataCollection paraview_dc("Example5", mesh);
-   paraview_dc.SetPrefixPath("ParaView");
-   paraview_dc.SetLevelsOfDetail(order);
-   paraview_dc.SetCycle(0);
-   paraview_dc.SetDataFormat(VTKFormat::BINARY);
-   paraview_dc.SetHighOrderOutput(true);
-   paraview_dc.SetTime(0.0); // set the time
-   paraview_dc.RegisterField("electric field",&E);
-   paraview_dc.RegisterField("magnetic field",&B);
-   paraview_dc.Save();
-
-
-   // 16. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream E_sock(vishost, visport);
-      E_sock.precision(8);
-      E_sock << "solution\n" << *mesh << E << "window_title 'Electric field'" << endl;
-      E_sock << "keys Rljvvvvvmmc" << endl;
-      socketstream B_sock(vishost, visport);
-      B_sock.precision(8);
-      B_sock << "solution\n" << *mesh << B << "window_title 'Magnetic field'" << endl;
-      B_sock << "keys Rljmmc" << endl;
-   }
-
-   // 17. Free the used memory.
-   delete fform;
-   delete gform;
-   //delete mEVarf;
-   //delete cVarf;
-   delete darcy;
-   delete W_space;
-   delete R_space;
-   delete trace_space;
-   delete W_coll;
-   delete R_coll;
-   delete trace_coll;
-   delete mesh;
-
-   return 0;
-}
-
-
-void EFun_ex(const Vector & x, Vector & E)
-{
-   const int dim = x.Size();
-
-   if (dim == 3)
-   {
-      E(0) = sin(kappa * x(1));
-      E(1) = sin(kappa * x(2));
-      E(2) = sin(kappa * x(0));
-   }
-   else
-   {
-      E(0) = sin(kappa * x(1));
-      E(1) = sin(kappa * x(0));
-      if (x.Size() == 3) { E(2) = 0.0; }
-   }
-}
-
-real_t BFun_ex(const Vector & x)
-{
-   return kappa * (-cos(kappa * x(0)) + cos(kappa * x(1)));
-}
-
-void fFun(const Vector & x, Vector & f)
-{
-   const int dim = x.Size();
-
-   if (dim == 3)
-   {
-      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
-      f(1) = (1. + kappa * kappa) * sin(kappa * x(2));
-      f(2) = (1. + kappa * kappa) * sin(kappa * x(0));
-   }
-   else
-   {
-      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
-      f(1) = (1. + kappa * kappa) * sin(kappa * x(0));
-      if (x.Size() == 3) { f(2) = 0.0; }
-   }
-}
-
-real_t gFun(const Vector & x)
-{
-   return 0.;
-}
@@ -3,17 +3,13 @@
 // Compile with: make ex5
 //
 // Sample runs:  ex5 -m ../data/square-disc.mesh
-//               ex5 -m ../data/square-disc.mesh -hb
 //               ex5 -m ../data/star.mesh
 //               ex5 -m ../data/star.mesh -pa
-//               ex5 -m ../data/star.mesh -hb
 //               ex5 -m ../data/beam-tet.mesh
 //               ex5 -m ../data/beam-hex.mesh
 //               ex5 -m ../data/beam-hex.mesh -pa
-//               ex5 -m ../data/beam-hex.mesh -hb
 //               ex5 -m ../data/escher.mesh
 //               ex5 -m ../data/fichera.mesh
-//               ex5 -m ../data/fichera.mesh -hb
 //
 // Device sample runs:
 //               ex5 -m ../data/star.mesh -pa -d cuda
@@ -33,16 +29,16 @@
 //               finite elements (velocity u) and piecewise discontinuous
 //               polynomials (pressure p).
 //
-//               The example demonstrates the use of the DarcyForm class, as
-//               well as hybridization of mixed systems and the collective
-//               saving of several grid functions in VisIt (visit.llnl.gov) and
-//               ParaView (paraview.org) formats.
+//               The example demonstrates the use of the BlockOperator class, as
+//               well as the collective saving of several grid functions in
+//               VisIt (visit.llnl.gov) and ParaView (paraview.org) formats.
 //
 //               We recommend viewing examples 1-4 before viewing this example.

 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
+#include <algorithm>

 using namespace std;
 using namespace mfem;
@@ -61,7 +57,6 @@ int main(int argc, char *argv[])
   // 1. Parse command-line options.
   const char *mesh_file = "../data/star.mesh";
   int order = 1;
-   bool hybridization = false;
   bool pa = false;
   const char *device_config = "cpu";
   bool visualization = 1;
@@ -71,8 +66,6 @@ int main(int argc, char *argv[])
                  "Mesh file to use.");
   args.AddOption(&order, "-o", "--order",
                  "Finite element order (polynomial degree).");
-   args.AddOption(&hybridization, "-hb", "--hybridization", "-no-hb",
-                  "--no-hybridization", "Enable hybridization.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
   args.AddOption(&device_config, "-d", "--device",
@@ -88,14 +81,6 @@ int main(int argc, char *argv[])
   }
   args.PrintOptions(cout);

-
-   if (pa && hybridization)
-   {
-      cerr << "Partial assembly is not supported for hybridization."
-           << endl;
-      return 1;
-   }
-
   // 2. Enable hardware devices such as GPUs, and programming models such as
   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
   Device device(device_config);
@@ -122,18 +107,20 @@ int main(int argc, char *argv[])

   // 5. Define a finite element space on the mesh. Here we use the
   //    Raviart-Thomas finite elements of the specified order.
-   FiniteElementCollection *R_coll(new RT_FECollection(order, dim));
-   FiniteElementCollection *W_coll(new L2_FECollection(order, dim));
+   FiniteElementCollection *hdiv_coll(new RT_FECollection(order, dim));
+   FiniteElementCollection *l2_coll(new L2_FECollection(order, dim));

-   FiniteElementSpace *R_space = new FiniteElementSpace(mesh, R_coll);
-   FiniteElementSpace *W_space = new FiniteElementSpace(mesh, W_coll);
+   FiniteElementSpace *R_space = new FiniteElementSpace(mesh, hdiv_coll);
+   FiniteElementSpace *W_space = new FiniteElementSpace(mesh, l2_coll);

-   // 6. Define the block structure of the problem, i.e. define the array of
+   // 6. Define the BlockStructure of the problem, i.e. define the array of
   //    offsets for each variable. The last component of the Array is the sum
   //    of the dimensions of each block.
-   DarcyForm *darcy = new DarcyForm(R_space, W_space, false);
-   const Array<int> &block_offsets = darcy->GetOffsets();
-   const Array<int> &block_trueOffsets = darcy->GetTrueOffsets();
+   Array<int> block_offsets(3); // number of variables + 1
+   block_offsets[0] = 0;
+   block_offsets[1] = R_space->GetVSize();
+   block_offsets[2] = W_space->GetVSize();
+   block_offsets.PartialSum();

   std::cout << "***********************************************************\n";
   std::cout << "dim(R) = " << block_offsets[1] - block_offsets[0] << "\n";
@@ -142,15 +129,14 @@ int main(int argc, char *argv[])
   std::cout << "***********************************************************\n";

   // 7. Define the coefficients, analytical solution, and rhs of the PDE.
-   const real_t k = 1.0;
-   ConstantCoefficient kcoeff(k); // Acoustic resistance
+   ConstantCoefficient k(1.0);

-   VectorFunctionCoefficient fcoeff(dim, fFun); // Velocity rhs
-   FunctionCoefficient fnatcoeff(f_natural); // Boundary velocity rhs
-   FunctionCoefficient gcoeff(gFun); // Pressure rhs
+   VectorFunctionCoefficient fcoeff(dim, fFun);
+   FunctionCoefficient fnatcoeff(f_natural);
+   FunctionCoefficient gcoeff(gFun);

-   VectorFunctionCoefficient ucoeff(dim, uFun_ex); // Velocity
-   FunctionCoefficient pcoeff(pFun_ex); // Pressure
+   VectorFunctionCoefficient ucoeff(dim, uFun_ex);
+   FunctionCoefficient pcoeff(pFun_ex);

   // 8. Allocate memory (x, rhs) for the analytical solution and the right hand
   //    side.  Define the GridFunction u,p for the finite element solution and
@@ -158,14 +144,20 @@ int main(int argc, char *argv[])
   //    allocated by x and rhs are passed as a reference to the grid functions
   //    (u,p) and the linear forms (fform, gform).
   MemoryType mt = device.GetMemoryType();
-   BlockVector x(block_offsets, mt);
+   BlockVector x(block_offsets, mt), rhs(block_offsets, mt);

-   LinearForm *fform = darcy->GetFluxRHS();
+   LinearForm *fform(new LinearForm);
+   fform->Update(R_space, rhs.GetBlock(0), 0);
   fform->AddDomainIntegrator(new VectorFEDomainLFIntegrator(fcoeff));
   fform->AddBoundaryIntegrator(new VectorFEBoundaryFluxLFIntegrator(fnatcoeff));
+   fform->Assemble();
+   fform->SyncAliasMemory(rhs);

-   LinearForm *gform = darcy->GetPotentialRHS();
+   LinearForm *gform(new LinearForm);
+   gform->Update(W_space, rhs.GetBlock(1), 0);
   gform->AddDomainIntegrator(new DomainLFIntegrator(gcoeff));
+   gform->Assemble();
+   gform->SyncAliasMemory(rhs);

   // 9. Assemble the finite element matrices for the Darcy operator
   //
@@ -175,190 +167,140 @@ int main(int argc, char *argv[])
   //
   //     M = \int_\Omega k u_h \cdot v_h d\Omega   u_h, v_h \in R_h
   //     B   = -\int_\Omega \div u_h q_h d\Omega   u_h \in R_h, q_h \in W_h
-   BilinearForm *mVarf = darcy->GetFluxMassForm();
-   MixedBilinearForm *bVarf = darcy->GetFluxDivForm();
+   BilinearForm *mVarf(new BilinearForm(R_space));
+   MixedBilinearForm *bVarf(new MixedBilinearForm(R_space, W_space));

-   mVarf->AddDomainIntegrator(new VectorFEMassIntegrator(kcoeff));
+   if (pa) { mVarf->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   mVarf->AddDomainIntegrator(new VectorFEMassIntegrator(k));
+   mVarf->Assemble();
+   if (!pa) { mVarf->Finalize(); }

-   ConstantCoefficient cdiv(-1.);
-   bVarf->AddDomainIntegrator(new VectorFEDivergenceIntegrator(cdiv));
+   if (pa) { bVarf->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   bVarf->AddDomainIntegrator(new VectorFEDivergenceIntegrator);
+   bVarf->Assemble();
+   if (!pa) { bVarf->Finalize(); }

-   // Set hybridization / assembly level
+   BlockOperator darcyOp(block_offsets);

-   Array<int> ess_flux_tdofs_list;
+   TransposeOperator *Bt = NULL;

-   FiniteElementCollection *trace_coll = NULL;
-   FiniteElementSpace *trace_space = NULL;
-
-   chrono.Clear();
-   chrono.Start();
-
-   if (hybridization)
+   if (pa)
   {
-      trace_coll = new DG_Interface_FECollection(order, dim);
-      trace_space = new FiniteElementSpace(mesh, trace_coll);
-      darcy->EnableHybridization(trace_space,
-                                 new NormalTraceJumpIntegrator(),
-                                 ess_flux_tdofs_list);
-   }
+      Bt = new TransposeOperator(bVarf);

-   if (pa) { darcy->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-
-   darcy->Assemble();
-
-   OperatorPtr A;
-   Vector X, B;
-   x = 0.;
-   darcy->FormLinearSystem(ess_flux_tdofs_list, x, A, X, B);
-
-   chrono.Stop();
-   std::cout << "Assembly took " << chrono.RealTime() << "s.\n";
-
-
-   constexpr int maxIter(1000);
-   constexpr real_t rtol(1.e-6);
-   constexpr real_t atol(1.e-10);
-
-   if (hybridization)
-   {
-      // 10. Construct the preconditioner
-      GSSmoother prec;
-
-      // 11. Solve the linear system with GMRES.
-      //     Check the norm of the unpreconditioned residual.
-      chrono.Clear();
-      chrono.Start();
-      GMRESSolver solver;
-      solver.SetAbsTol(atol);
-      solver.SetRelTol(rtol);
-      solver.SetMaxIter(maxIter);
-      solver.SetPreconditioner(prec);
-      solver.SetOperator(*A);
-      solver.SetPrintLevel(1);
-
-      solver.Mult(B, X);
-      darcy->RecoverFEMSolution(X, x);
-
-      chrono.Stop();
-
-      if (solver.GetConverged())
-      {
-         std::cout << "GMRES converged in " << solver.GetNumIterations()
-                   << " iterations with a residual norm of "
-                   << solver.GetFinalNorm() << ".\n";
-      }
-      else
-      {
-         std::cout << "GMRES did not converge in " << solver.GetNumIterations()
-                   << " iterations. Residual norm is " << solver.GetFinalNorm()
-                   << ".\n";
-      }
-      std::cout << "GMRES solver took " << chrono.RealTime() << "s.\n";
+      darcyOp.SetBlock(0,0, mVarf);
+      darcyOp.SetBlock(0,1, Bt, -1.0);
+      darcyOp.SetBlock(1,0, bVarf, -1.0);
   }
   else
   {
-      // 10. Construct the operators for preconditioner
-      //
-      //                 P = [ diag(M)         0         ]
-      //                     [  0       B diag(M)^-1 B^T ]
-      //
-      //     Here we use Symmetric Gauss-Seidel to approximate the inverse of the
-      //     pressure Schur Complement
-      SparseMatrix *MinvBt = NULL;
-      Vector Md(mVarf->Height());
+      SparseMatrix &M(mVarf->SpMat());
+      SparseMatrix &B(bVarf->SpMat());
+      B *= -1.;
+      Bt = new TransposeOperator(&B);

-      Solver *invM, *invS;
-      SparseMatrix *S = NULL;
+      darcyOp.SetBlock(0,0, &M);
+      darcyOp.SetBlock(0,1, Bt);
+      darcyOp.SetBlock(1,0, &B);
+   }

-      if (pa)
+   // 10. Construct the operators for preconditioner
+   //
+   //                 P = [ diag(M)         0         ]
+   //                     [  0       B diag(M)^-1 B^T ]
+   //
+   //     Here we use Symmetric Gauss-Seidel to approximate the inverse of the
+   //     pressure Schur Complement
+   SparseMatrix *MinvBt = NULL;
+   Vector Md(mVarf->Height());
+
+   BlockDiagonalPreconditioner darcyPrec(block_offsets);
+   Solver *invM, *invS;
+   SparseMatrix *S = NULL;
+
+   if (pa)
+   {
+      mVarf->AssembleDiagonal(Md);
+      auto Md_host = Md.HostRead();
+      Vector invMd(mVarf->Height());
+      for (int i=0; i<mVarf->Height(); ++i)
      {
-         mVarf->AssembleDiagonal(Md);
-         auto Md_host = Md.HostRead();
-         Vector invMd(mVarf->Height());
-         for (int i=0; i<mVarf->Height(); ++i)
-         {
-            invMd(i) = 1.0 / Md_host[i];
-         }
-
-         Vector BMBt_diag(bVarf->Height());
-         bVarf->AssembleDiagonal_ADAt(invMd, BMBt_diag);
-
-         Array<int> ess_tdof_list;  // empty
-
-         invM = new OperatorJacobiSmoother(Md, ess_tdof_list);
-         invS = new OperatorJacobiSmoother(BMBt_diag, ess_tdof_list);
+         invMd(i) = 1.0 / Md_host[i];
      }
-      else
+
+      Vector BMBt_diag(bVarf->Height());
+      bVarf->AssembleDiagonal_ADAt(invMd, BMBt_diag);
+
+      Array<int> ess_tdof_list;  // empty
+
+      invM = new OperatorJacobiSmoother(Md, ess_tdof_list);
+      invS = new OperatorJacobiSmoother(BMBt_diag, ess_tdof_list);
+   }
+   else
+   {
+      SparseMatrix &M(mVarf->SpMat());
+      M.GetDiag(Md);
+      Md.HostReadWrite();
+
+      SparseMatrix &B(bVarf->SpMat());
+      MinvBt = Transpose(B);
+
+      for (int i = 0; i < Md.Size(); i++)
      {
-         SparseMatrix &M(mVarf->SpMat());
-         M.GetDiag(Md);
-         Md.HostReadWrite();
+         MinvBt->ScaleRow(i, 1./Md(i));
+      }

-         SparseMatrix &Bm(bVarf->SpMat());
-         MinvBt = Transpose(Bm);
+      S = Mult(B, *MinvBt);

-         for (int i = 0; i < Md.Size(); i++)
-         {
-            MinvBt->ScaleRow(i, 1./Md(i));
-         }
-
-         S = Mult(Bm, *MinvBt);
-
-         invM = new DSmoother(M);
+      invM = new DSmoother(M);

 #ifndef MFEM_USE_SUITESPARSE
-         invS = new GSSmoother(*S);
+      invS = new GSSmoother(*S);
 #else
-         invS = new UMFPackSolver(*S);
+      invS = new UMFPackSolver(*S);
 #endif
-      }
-
-      invM->iterative_mode = false;
-      invS->iterative_mode = false;
-
-      BlockDiagonalPreconditioner darcyPrec(block_trueOffsets);
-      darcyPrec.SetDiagonalBlock(0, invM);
-      darcyPrec.SetDiagonalBlock(1, invS);
-
-      // 11. Solve the linear system with MINRES.
-      //     Check the norm of the unpreconditioned residual.
-
-      chrono.Clear();
-      chrono.Start();
-      MINRESSolver solver;
-      solver.SetAbsTol(atol);
-      solver.SetRelTol(rtol);
-      solver.SetMaxIter(maxIter);
-      solver.SetOperator(*A);
-      solver.SetPreconditioner(darcyPrec);
-      solver.SetPrintLevel(1);
-
-      solver.Mult(B, X);
-      darcy->RecoverFEMSolution(X, x);
-
-      if (device.IsEnabled()) { x.HostRead(); }
-      chrono.Stop();
-
-      if (solver.GetConverged())
-      {
-         std::cout << "MINRES converged in " << solver.GetNumIterations()
-                   << " iterations with a residual norm of "
-                   << solver.GetFinalNorm() << ".\n";
-      }
-      else
-      {
-         std::cout << "MINRES did not converge in " << solver.GetNumIterations()
-                   << " iterations. Residual norm is " << solver.GetFinalNorm()
-                   << ".\n";
-      }
-      std::cout << "MINRES solver took " << chrono.RealTime() << "s.\n";
-
-      delete invM;
-      delete invS;
-      delete S;
-      delete MinvBt;
   }

+   invM->iterative_mode = false;
+   invS->iterative_mode = false;
+
+   darcyPrec.SetDiagonalBlock(0, invM);
+   darcyPrec.SetDiagonalBlock(1, invS);
+
+   // 11. Solve the linear system with MINRES.
+   //     Check the norm of the unpreconditioned residual.
+   int maxIter(1000);
+   real_t rtol(1.e-6);
+   real_t atol(1.e-10);
+
+   chrono.Clear();
+   chrono.Start();
+   MINRESSolver solver;
+   solver.SetAbsTol(atol);
+   solver.SetRelTol(rtol);
+   solver.SetMaxIter(maxIter);
+   solver.SetOperator(darcyOp);
+   solver.SetPreconditioner(darcyPrec);
+   solver.SetPrintLevel(1);
+   x = 0.0;
+   solver.Mult(rhs, x);
+   if (device.IsEnabled()) { x.HostRead(); }
+   chrono.Stop();
+
+   if (solver.GetConverged())
+   {
+      std::cout << "MINRES converged in " << solver.GetNumIterations()
+                << " iterations with a residual norm of "
+                << solver.GetFinalNorm() << ".\n";
+   }
+   else
+   {
+      std::cout << "MINRES did not converge in " << solver.GetNumIterations()
+                << " iterations. Residual norm is " << solver.GetFinalNorm()
+                << ".\n";
+   }
+   std::cout << "MINRES solver took " << chrono.RealTime() << "s.\n";
+
   // 12. Create the grid functions u and p. Compute the L2 error norms.
   GridFunction u, p;
   u.MakeRef(R_space, x.GetBlock(0), 0);
@@ -428,13 +370,19 @@ int main(int argc, char *argv[])
   }

   // 17. Free the used memory.
-   delete darcy;
+   delete fform;
+   delete gform;
+   delete invM;
+   delete invS;
+   delete S;
+   delete Bt;
+   delete MinvBt;
+   delete mVarf;
+   delete bVarf;
   delete W_space;
   delete R_space;
-   delete trace_space;
-   delete W_coll;
-   delete R_coll;
-   delete trace_coll;
+   delete l2_coll;
+   delete hdiv_coll;
   delete mesh;

   return 0;
@@ -3,17 +3,13 @@
 // Compile with: make ex5p
 //
 // Sample runs:  mpirun -np 4 ex5p -m ../data/square-disc.mesh
-//               mpirun -np 4 ex5p -m ../data/square-disc.mesh -hb
 //               mpirun -np 4 ex5p -m ../data/star.mesh
 //               mpirun -np 4 ex5p -m ../data/star.mesh -r 2 -pa
-//               mpirun -np 4 ex5p -m ../data/star.mesh -hb
 //               mpirun -np 4 ex5p -m ../data/beam-tet.mesh
 //               mpirun -np 4 ex5p -m ../data/beam-hex.mesh
 //               mpirun -np 4 ex5p -m ../data/beam-hex.mesh -pa
-//               mpirun -np 4 ex5p -m ../data/beam-hex.mesh -hb
 //               mpirun -np 4 ex5p -m ../data/escher.mesh
 //               mpirun -np 4 ex5p -m ../data/fichera.mesh
-//               mpirun -np 4 ex5p -m ../data/fichera.mesh -hb
 //
 // Device sample runs:
 //               mpirun -np 4 ex5p -m ../data/star.mesh -r 2 -pa -d cuda
@@ -33,11 +29,11 @@
 //               finite elements (velocity u) and piecewise discontinuous
 //               polynomials (pressure p).
 //
-//               The example demonstrates the use of the ParDarcyForm class, as
-//               well as hybridization of mixed systems and the collective
-//               saving of several grid functions in VisIt (visit.llnl.gov) and
-//               ParaView (paraview.org) formats. Optional saving with ADIOS2
-//               (adios2.readthedocs.io) streams is also illustrated.
+//               The example demonstrates the use of the BlockOperator class, as
+//               well as the collective saving of several grid functions in
+//               VisIt (visit.llnl.gov) and ParaView (paraview.org) formats.
+//               Optional saving with ADIOS2 (adios2.readthedocs.io) streams is
+//               also illustrated.
 //
 //               We recommend viewing examples 1-4 before viewing this example.

@@ -70,11 +66,10 @@ int main(int argc, char *argv[])
   const char *mesh_file = "../data/star.mesh";
   int ref_levels = -1;
   int order = 1;
-   bool hybridization = false;
+   bool par_format = false;
   bool pa = false;
   const char *device_config = "cpu";
   bool visualization = 1;
-   bool par_format = false;
   bool adios2 = false;

   OptionsParser args(argc, argv);
@@ -84,8 +79,9 @@ int main(int argc, char *argv[])
                  "Number of times to refine the mesh uniformly.");
   args.AddOption(&order, "-o", "--order",
                  "Finite element order (polynomial degree).");
-   args.AddOption(&hybridization, "-hb", "--hybridization", "-no-hb",
-                  "--no-hybridization", "Enable hybridization.");
+   args.AddOption(&par_format, "-pf", "--parallel-format", "-sf",
+                  "--serial-format",
+                  "Format to use when saving the results for VisIt.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
   args.AddOption(&device_config, "-d", "--device",
@@ -93,9 +89,6 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
-   args.AddOption(&par_format, "-pf", "--parallel-format", "-sf",
-                  "--serial-format",
-                  "Format to use when saving the results for VisIt.");
   args.AddOption(&adios2, "-adios2", "--adios2-streams", "-no-adios2",
                  "--no-adios2-streams",
                  "Save data using adios2 streams.");
@@ -113,17 +106,6 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-
-   if (pa && hybridization)
-   {
-      if (verbose)
-      {
-         cerr << "Partial assembly is not supported for hybridization."
-              << endl;
-      }
-      return 1;
-   }
-
   // 3. Enable hardware devices such as GPUs, and programming models such as
   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
   Device device(device_config);
@@ -166,11 +148,11 @@ int main(int argc, char *argv[])

   // 7. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Raviart-Thomas finite elements of the specified order.
-   FiniteElementCollection *R_coll = new RT_FECollection(order, dim);
-   FiniteElementCollection *W_coll = new L2_FECollection(order, dim);
+   FiniteElementCollection *hdiv_coll(new RT_FECollection(order, dim));
+   FiniteElementCollection *l2_coll(new L2_FECollection(order, dim));

-   ParFiniteElementSpace *R_space = new ParFiniteElementSpace(pmesh, R_coll);
-   ParFiniteElementSpace *W_space = new ParFiniteElementSpace(pmesh, W_coll);
+   ParFiniteElementSpace *R_space = new ParFiniteElementSpace(pmesh, hdiv_coll);
+   ParFiniteElementSpace *W_space = new ParFiniteElementSpace(pmesh, l2_coll);

   HYPRE_BigInt dimR = R_space->GlobalTrueVSize();
   HYPRE_BigInt dimW = W_space->GlobalTrueVSize();
@@ -184,37 +166,55 @@ int main(int argc, char *argv[])
      std::cout << "***********************************************************\n";
   }

-   // 8. Define the block structure of the problem. block_offsets is used
+   // 8. Define the two BlockStructure of the problem.  block_offsets is used
   //    for Vector based on dof (like ParGridFunction or ParLinearForm),
-   //    block_trueOffsets is used for Vector based on true dof (HypreParVector
+   //    block_trueOffstes is used for Vector based on trueDof (HypreParVector
   //    for the rhs and solution of the linear system).  The offsets computed
   //    here are local to the processor.
-   ParDarcyForm *darcy = new ParDarcyForm(R_space, W_space, false);
-   const Array<int> &block_offsets = darcy->GetOffsets();
-   const Array<int> &block_trueOffsets = darcy->GetTrueOffsets();
+   Array<int> block_offsets(3); // number of variables + 1
+   block_offsets[0] = 0;
+   block_offsets[1] = R_space->GetVSize();
+   block_offsets[2] = W_space->GetVSize();
+   block_offsets.PartialSum();
+
+   Array<int> block_trueOffsets(3); // number of variables + 1
+   block_trueOffsets[0] = 0;
+   block_trueOffsets[1] = R_space->TrueVSize();
+   block_trueOffsets[2] = W_space->TrueVSize();
+   block_trueOffsets.PartialSum();

   // 9. Define the coefficients, analytical solution, and rhs of the PDE.
-   const real_t k = 1.0;
-   ConstantCoefficient kcoeff(k); // Acoustic resistance
+   ConstantCoefficient k(1.0);

-   VectorFunctionCoefficient fcoeff(dim, fFun); // Velocity rhs
-   FunctionCoefficient fnatcoeff(f_natural); // Boundary velocity rhs
-   FunctionCoefficient gcoeff(gFun); // Pressure rhs
+   VectorFunctionCoefficient fcoeff(dim, fFun);
+   FunctionCoefficient fnatcoeff(f_natural);
+   FunctionCoefficient gcoeff(gFun);

-   VectorFunctionCoefficient ucoeff(dim, uFun_ex); // Velocity
-   FunctionCoefficient pcoeff(pFun_ex); // Pressure
+   VectorFunctionCoefficient ucoeff(dim, uFun_ex);
+   FunctionCoefficient pcoeff(pFun_ex);

   // 10. Define the parallel grid function and parallel linear forms, solution
   //     vector and rhs.
   MemoryType mt = device.GetMemoryType();
-   BlockVector x(block_offsets, mt);
+   BlockVector x(block_offsets, mt), rhs(block_offsets, mt);
+   BlockVector trueX(block_trueOffsets, mt), trueRhs(block_trueOffsets, mt);

-   ParLinearForm *fform = darcy->GetParFluxRHS();
+   ParLinearForm *fform(new ParLinearForm);
+   fform->Update(R_space, rhs.GetBlock(0), 0);
   fform->AddDomainIntegrator(new VectorFEDomainLFIntegrator(fcoeff));
   fform->AddBoundaryIntegrator(new VectorFEBoundaryFluxLFIntegrator(fnatcoeff));
+   fform->Assemble();
+   fform->SyncAliasMemory(rhs);
+   fform->ParallelAssemble(trueRhs.GetBlock(0));
+   trueRhs.GetBlock(0).SyncAliasMemory(trueRhs);

-   ParLinearForm *gform = darcy->GetParPotentialRHS();
+   ParLinearForm *gform(new ParLinearForm);
+   gform->Update(W_space, rhs.GetBlock(1), 0);
   gform->AddDomainIntegrator(new DomainLFIntegrator(gcoeff));
+   gform->Assemble();
+   gform->SyncAliasMemory(rhs);
+   gform->ParallelAssemble(trueRhs.GetBlock(1));
+   trueRhs.GetBlock(1).SyncAliasMemory(trueRhs);

   // 11. Assemble the finite element matrices for the Darcy operator
   //
@@ -224,194 +224,145 @@ int main(int argc, char *argv[])
   //
   //     M = \int_\Omega k u_h \cdot v_h d\Omega   u_h, v_h \in R_h
   //     B   = -\int_\Omega \div u_h q_h d\Omega   u_h \in R_h, q_h \in W_h
-   ParBilinearForm *mVarf = darcy->GetParFluxMassForm();
-   ParMixedBilinearForm *bVarf = darcy->GetParFluxDivForm();
+   ParBilinearForm *mVarf(new ParBilinearForm(R_space));
+   ParMixedBilinearForm *bVarf(new ParMixedBilinearForm(R_space, W_space));

-   mVarf->AddDomainIntegrator(new VectorFEMassIntegrator(kcoeff));
+   HypreParMatrix *M = NULL;
+   HypreParMatrix *B = NULL;

-   ConstantCoefficient cdiv(-1.);
-   bVarf->AddDomainIntegrator(new VectorFEDivergenceIntegrator(cdiv));
+   if (pa) { mVarf->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   mVarf->AddDomainIntegrator(new VectorFEMassIntegrator(k));
+   mVarf->Assemble();
+   if (!pa) { mVarf->Finalize(); }

-   // Set hybridization / assembly level
+   if (pa) { bVarf->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   bVarf->AddDomainIntegrator(new VectorFEDivergenceIntegrator);
+   bVarf->Assemble();
+   if (!pa) { bVarf->Finalize(); }

-   Array<int> ess_flux_tdofs_list;
+   BlockOperator *darcyOp = new BlockOperator(block_trueOffsets);

-   FiniteElementCollection *trace_coll = NULL;
-   ParFiniteElementSpace *trace_space = NULL;
+   Array<int> empty_tdof_list;  // empty
+   OperatorPtr opM, opB;

-   chrono.Clear();
-   chrono.Start();
+   TransposeOperator *Bt = NULL;

-   if (hybridization)
+   if (pa)
   {
-      trace_coll = new DG_Interface_FECollection(order, dim);
-      trace_space = new ParFiniteElementSpace(pmesh, trace_coll);
-      darcy->EnableHybridization(trace_space,
-                                 new NormalTraceJumpIntegrator(),
-                                 ess_flux_tdofs_list);
-   }
+      mVarf->FormSystemMatrix(empty_tdof_list, opM);
+      bVarf->FormRectangularSystemMatrix(empty_tdof_list, empty_tdof_list, opB);
+      Bt = new TransposeOperator(opB.Ptr());

-   if (pa) { darcy->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-
-   darcy->Assemble();
-
-   OperatorPtr A;
-   Vector X, B;
-   x = 0.;
-   darcy->FormLinearSystem(ess_flux_tdofs_list, x, A, X, B);
-
-   chrono.Stop();
-   if (verbose)
-   {
-      std::cout << "Assembly took " << chrono.RealTime() << "s.\n";
-   }
-
-
-   const int maxIter(pa ? 1000 : 500);
-   constexpr real_t rtol(1.e-6);
-   constexpr real_t atol(1.e-10);
-
-   if (hybridization)
-   {
-      // 12. Construct the preconditioner
-      HypreBoomerAMG prec;
-
-      // 13. Solve the linear system with GMRES.
-      //     Check the norm of the unpreconditioned residual.
-      chrono.Clear();
-      chrono.Start();
-      GMRESSolver solver(MPI_COMM_WORLD);
-      solver.SetAbsTol(atol);
-      solver.SetRelTol(rtol);
-      solver.SetMaxIter(maxIter);
-      solver.SetPreconditioner(prec);
-      solver.SetOperator(*A);
-      solver.SetPrintLevel(1);
-
-      solver.Mult(B, X);
-      darcy->RecoverFEMSolution(X, x);
-
-      chrono.Stop();
-
-      if (verbose)
-      {
-         if (solver.GetConverged())
-         {
-            std::cout << "GMRES converged in " << solver.GetNumIterations()
-                      << " iterations with a residual norm of "
-                      << solver.GetFinalNorm() << ".\n";
-         }
-         else
-         {
-            std::cout << "GMRES did not converge in " << solver.GetNumIterations()
-                      << " iterations. Residual norm is " << solver.GetFinalNorm()
-                      << ".\n";
-         }
-         std::cout << "GMRES solver took " << chrono.RealTime() << "s.\n";
-      }
+      darcyOp->SetBlock(0,0, opM.Ptr());
+      darcyOp->SetBlock(0,1, Bt, -1.0);
+      darcyOp->SetBlock(1,0, opB.Ptr(), -1.0);
   }
   else
   {
-      // 12. Construct the operators for preconditioner
-      //
-      //                 P = [ diag(M)         0         ]
-      //                     [  0       B diag(M)^-1 B^T ]
-      //
-      //     Here we use Symmetric Gauss-Seidel to approximate the inverse of the
-      //     pressure Schur Complement
-      HypreParMatrix *MinvBt = NULL;
-      HypreParMatrix *S = NULL;
-      Solver *invM, *invS;
+      M = mVarf->ParallelAssemble();
+      B = bVarf->ParallelAssemble();
+      (*B) *= -1;
+      Bt = new TransposeOperator(B);

-      if (pa)
+      darcyOp->SetBlock(0,0, M);
+      darcyOp->SetBlock(0,1, Bt);
+      darcyOp->SetBlock(1,0, B);
+   }
+
+   // 12. Construct the operators for preconditioner
+   //
+   //                 P = [ diag(M)         0         ]
+   //                     [  0       B diag(M)^-1 B^T ]
+   //
+   //     Here we use Symmetric Gauss-Seidel to approximate the inverse of the
+   //     pressure Schur Complement.
+   HypreParMatrix *MinvBt = NULL;
+   HypreParVector *Md = NULL;
+   HypreParMatrix *S = NULL;
+   Vector Md_PA;
+   Solver *invM, *invS;
+
+   if (pa)
+   {
+      Md_PA.SetSize(R_space->GetTrueVSize());
+      mVarf->AssembleDiagonal(Md_PA);
+      auto Md_host = Md_PA.HostRead();
+      Vector invMd(Md_PA.Size());
+      for (int i=0; i<Md_PA.Size(); ++i)
      {
-         Vector Md(R_space->GetTrueVSize());
-         mVarf->AssembleDiagonal(Md);
-         auto Md_host = Md.HostRead();
-         Vector invMd(Md.Size());
-         for (int i=0; i<Md.Size(); ++i)
-         {
-            invMd(i) = 1.0 / Md_host[i];
-         }
-
-         Vector BMBt_diag(W_space->GetTrueVSize());
-         bVarf->AssembleDiagonal_ADAt(invMd, BMBt_diag);
-
-         Array<int> ess_tdof_list;  // empty
-
-         invM = new OperatorJacobiSmoother(Md, ess_tdof_list);
-         invS = new OperatorJacobiSmoother(BMBt_diag, ess_tdof_list);
+         invMd(i) = 1.0 / Md_host[i];
      }
+
+      Vector BMBt_diag(W_space->GetTrueVSize());
+      bVarf->AssembleDiagonal_ADAt(invMd, BMBt_diag);
+
+      Array<int> ess_tdof_list;  // empty
+
+      invM = new OperatorJacobiSmoother(Md_PA, ess_tdof_list);
+      invS = new OperatorJacobiSmoother(BMBt_diag, ess_tdof_list);
+   }
+   else
+   {
+      Md = new HypreParVector(MPI_COMM_WORLD, M->GetGlobalNumRows(),
+                              M->GetRowStarts());
+      M->GetDiag(*Md);
+
+      MinvBt = B->Transpose();
+      MinvBt->InvScaleRows(*Md);
+      S = ParMult(B, MinvBt);
+
+      invM = new HypreDiagScale(*M);
+      invS = new HypreBoomerAMG(*S);
+   }
+
+   invM->iterative_mode = false;
+   invS->iterative_mode = false;
+
+   BlockDiagonalPreconditioner *darcyPr = new BlockDiagonalPreconditioner(
+      block_trueOffsets);
+   darcyPr->SetDiagonalBlock(0, invM);
+   darcyPr->SetDiagonalBlock(1, invS);
+
+   // 13. Solve the linear system with MINRES.
+   //     Check the norm of the unpreconditioned residual.
+   int maxIter(pa ? 1000 : 500);
+   real_t rtol(1.e-6);
+   real_t atol(1.e-10);
+
+   chrono.Clear();
+   chrono.Start();
+   MINRESSolver solver(MPI_COMM_WORLD);
+   solver.SetAbsTol(atol);
+   solver.SetRelTol(rtol);
+   solver.SetMaxIter(maxIter);
+   solver.SetOperator(*darcyOp);
+   solver.SetPreconditioner(*darcyPr);
+   solver.SetPrintLevel(verbose);
+   trueX = 0.0;
+   solver.Mult(trueRhs, trueX);
+   if (device.IsEnabled()) { trueX.HostRead(); }
+   chrono.Stop();
+
+   if (verbose)
+   {
+      if (solver.GetConverged())
+         std::cout << "MINRES converged in " << solver.GetNumIterations()
+                   << " iterations with a residual norm of " << solver.GetFinalNorm() << ".\n";
      else
-      {
-         HypreParMatrix &M = *mVarf->ParallelAssembleInternalMatrix();
-         Vector Md;
-         M.GetDiag(Md);
-
-         HypreParMatrix &Bm = *bVarf->ParallelAssembleInternalMatrix();
-         MinvBt = Bm.Transpose();
-         MinvBt->InvScaleRows(Md);
-         S = ParMult(&Bm, MinvBt);
-
-         invM = new HypreDiagScale(M);
-         invS = new HypreBoomerAMG(*S);
-      }
-
-      invM->iterative_mode = false;
-      invS->iterative_mode = false;
-
-      BlockDiagonalPreconditioner darcyPrec(block_trueOffsets);
-      darcyPrec.SetDiagonalBlock(0, invM);
-      darcyPrec.SetDiagonalBlock(1, invS);
-
-      // 13. Solve the linear system with MINRES.
-      //     Check the norm of the unpreconditioned residual.
-
-      chrono.Clear();
-      chrono.Start();
-      MINRESSolver solver(MPI_COMM_WORLD);
-      solver.SetAbsTol(atol);
-      solver.SetRelTol(rtol);
-      solver.SetMaxIter(maxIter);
-      solver.SetOperator(*A);
-      solver.SetPreconditioner(darcyPrec);
-      solver.SetPrintLevel(1);
-
-      solver.Mult(B, X);
-      darcy->RecoverFEMSolution(X, x);
-
-      if (device.IsEnabled()) { x.HostRead(); }
-      chrono.Stop();
-
-      if (verbose)
-      {
-         if (solver.GetConverged())
-         {
-            std::cout << "MINRES converged in " << solver.GetNumIterations()
-                      << " iterations with a residual norm of "
-                      << solver.GetFinalNorm() << ".\n";
-         }
-         else
-         {
-            std::cout << "MINRES did not converge in " << solver.GetNumIterations()
-                      << " iterations. Residual norm is " << solver.GetFinalNorm()
-                      << ".\n";
-         }
-         std::cout << "MINRES solver took " << chrono.RealTime() << "s.\n";
-      }
-
-      delete invM;
-      delete invS;
-      delete S;
-      delete MinvBt;
+         std::cout << "MINRES did not converge in " << solver.GetNumIterations()
+                   << " iterations. Residual norm is " << solver.GetFinalNorm() << ".\n";
+      std::cout << "MINRES solver took " << chrono.RealTime() << "s. \n";
   }

   // 14. Extract the parallel grid function corresponding to the finite element
   //     approximation X. This is the local solution on each processor. Compute
   //     L2 error norms.
-   ParGridFunction u, p;
-   u.MakeRef(R_space, x.GetBlock(0), 0);
-   p.MakeRef(W_space, x.GetBlock(1), 0);
+   ParGridFunction *u(new ParGridFunction);
+   ParGridFunction *p(new ParGridFunction);
+   u->MakeRef(R_space, x.GetBlock(0), 0);
+   p->MakeRef(W_space, x.GetBlock(1), 0);
+   u->Distribute(&(trueX.GetBlock(0)));
+   p->Distribute(&(trueX.GetBlock(1)));

   int order_quad = max(2, 2*order+1);
   const IntegrationRule *irs[Geometry::NumGeom];
@@ -420,10 +371,10 @@ int main(int argc, char *argv[])
      irs[i] = &(IntRules.Get(i, order_quad));
   }

-   real_t err_u  = u.ComputeL2Error(ucoeff, irs);
-   real_t norm_u = ComputeGlobalLpNorm(2., ucoeff, *pmesh, irs);
-   real_t err_p  = p.ComputeL2Error(pcoeff, irs);
-   real_t norm_p = ComputeGlobalLpNorm(2., pcoeff, *pmesh, irs);
+   real_t err_u  = u->ComputeL2Error(ucoeff, irs);
+   real_t norm_u = ComputeGlobalLpNorm(2, ucoeff, *pmesh, irs);
+   real_t err_p  = p->ComputeL2Error(pcoeff, irs);
+   real_t norm_p = ComputeGlobalLpNorm(2, pcoeff, *pmesh, irs);

   if (verbose)
   {
@@ -445,17 +396,17 @@ int main(int argc, char *argv[])

      ofstream u_ofs(u_name.str().c_str());
      u_ofs.precision(8);
-      u.Save(u_ofs);
+      u->Save(u_ofs);

      ofstream p_ofs(p_name.str().c_str());
      p_ofs.precision(8);
-      p.Save(p_ofs);
+      p->Save(p_ofs);
   }

   // 16. Save data in the VisIt format
   VisItDataCollection visit_dc("Example5-Parallel", pmesh);
-   visit_dc.RegisterField("velocity", &u);
-   visit_dc.RegisterField("pressure", &p);
+   visit_dc.RegisterField("velocity", u);
+   visit_dc.RegisterField("pressure", p);
   visit_dc.SetFormat(!par_format ?
                      DataCollection::SERIAL_FORMAT :
                      DataCollection::PARALLEL_FORMAT);
@@ -468,9 +419,9 @@ int main(int argc, char *argv[])
   paraview_dc.SetDataFormat(VTKFormat::BINARY);
   paraview_dc.SetHighOrderOutput(true);
   paraview_dc.SetCycle(0);
-   paraview_dc.SetTime(0.0); // set the time
-   paraview_dc.RegisterField("velocity",&u);
-   paraview_dc.RegisterField("pressure",&p);
+   paraview_dc.SetTime(0.0);
+   paraview_dc.RegisterField("velocity",u);
+   paraview_dc.RegisterField("pressure",p);
   paraview_dc.Save();

   // 18. Optionally output a BP (binary pack) file using ADIOS2. This can be
@@ -481,14 +432,14 @@ int main(int argc, char *argv[])
      std::string postfix(mesh_file);
      postfix.erase(0, std::string("../data/").size() );
      postfix += "_o" + std::to_string(order);
-      const std::string collection_name = "ex5-p-" + postfix + ".bp";
+      const std::string collection_name = "ex5-p_" + postfix + ".bp";

      ADIOS2DataCollection adios2_dc(MPI_COMM_WORLD, collection_name, pmesh);
      adios2_dc.SetLevelsOfDetail(1);
      adios2_dc.SetCycle(1);
      adios2_dc.SetTime(0.0);
-      adios2_dc.RegisterField("velocity",&u);
-      adios2_dc.RegisterField("pressure",&p);
+      adios2_dc.RegisterField("velocity",u);
+      adios2_dc.RegisterField("pressure",p);
      adios2_dc.Save();
   }
 #endif
@@ -501,7 +452,7 @@ int main(int argc, char *argv[])
      socketstream u_sock(vishost, visport);
      u_sock << "parallel " << num_procs << " " << myid << "\n";
      u_sock.precision(8);
-      u_sock << "solution\n" << *pmesh << u << "window_title 'Velocity'"
+      u_sock << "solution\n" << *pmesh << *u << "window_title 'Velocity'"
             << endl;
      // Make sure all ranks have sent their 'u' solution before initiating
      // another set of GLVis connections (one from each rank):
@@ -509,18 +460,31 @@ int main(int argc, char *argv[])
      socketstream p_sock(vishost, visport);
      p_sock << "parallel " << num_procs << " " << myid << "\n";
      p_sock.precision(8);
-      p_sock << "solution\n" << *pmesh << p << "window_title 'Pressure'"
+      p_sock << "solution\n" << *pmesh << *p << "window_title 'Pressure'"
             << endl;
   }

   // 20. Free the used memory.
-   delete darcy;
+   delete fform;
+   delete gform;
+   delete u;
+   delete p;
+   delete darcyOp;
+   delete darcyPr;
+   delete invM;
+   delete invS;
+   delete S;
+   delete Md;
+   delete MinvBt;
+   delete Bt;
+   delete B;
+   delete M;
+   delete mVarf;
+   delete bVarf;
   delete W_space;
   delete R_space;
-   delete trace_space;
-   delete W_coll;
-   delete R_coll;
-   delete trace_coll;
+   delete l2_coll;
+   delete hdiv_coll;
   delete pmesh;

   return 0;
@@ -160,7 +160,6 @@ int main(int argc, char *argv[])
   bool paraview = false;
   bool binary = false;
   int vis_steps = 5;
-   bool solve_implicit_state = false;

   int precision = 8;
   cout.precision(precision);
@@ -188,9 +187,6 @@ int main(int argc, char *argv[])
                  "Final time; start time is 0.");
   args.AddOption(&dt, "-dt", "--time-step",
                  "Time step.");
-   args.AddOption(&solve_implicit_state, "-imp-state", "--implicit-state",
-                  "-imp-slope", "--implicit-slope",
-                  "Implicitly solve for stage state or slope.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -370,11 +366,6 @@ int main(int argc, char *argv[])
   //    right-hand side, and perform time-integration (looping over the time
   //    iterations, ti, with a time-step dt).
   FE_Evolution adv(m, k, b);
-   using ImplicitVariableType = FE_Evolution::ImplicitVariableType;
-   ImplicitVariableType imp_var = solve_implicit_state ?
-                                  ImplicitVariableType::STATE
-                                  : ImplicitVariableType::SLOPE;
-   adv.SetImplicitVariableType(imp_var);

   real_t t = 0.0;
   adv.SetTime(t);
@@ -468,17 +459,7 @@ void FE_Evolution::ImplicitSolve(const real_t dt, const Vector &x, Vector &k)
 {
   MFEM_VERIFY(dg_solver != NULL,
               "Implicit time integration is not supported with partial assembly");
-   // Construct current right-hand side for stage state vs. slope solve
-   if (ImplicitVarTypeIsState())
-   {
-      // k, on return, is the stage value u
-      M.Mult(x, z);
-   }
-   else
-   {
-      // k, on return, is the stage slope du/dt
-      K.Mult(x, z);
-   }
+   K.Mult(x, z);
   z += b;
   dg_solver->SetTimeStep(dt);
   dg_solver->Mult(z, k);
@@ -257,7 +257,6 @@ int main(int argc, char *argv[])
   bool adios2 = false;
   bool binary = false;
   int vis_steps = 5;
-   bool solve_implicit_state = false;
 #if MFEM_HYPRE_VERSION >= 21800
   PrecType prec_type = PrecType::AIR;
 #else
@@ -291,9 +290,6 @@ int main(int argc, char *argv[])
                  "Final time; start time is 0.");
   args.AddOption(&dt, "-dt", "--time-step",
                  "Time step.");
-   args.AddOption(&solve_implicit_state, "-imp-state", "--implicit-state",
-                  "-imp-slope", "--implicit-slope",
-                  "Implicitly solve for stage state or slope.");
   args.AddOption((int *)&prec_type, "-pt", "--prec-type", "Preconditioner for "
                  "implicit solves. 0 for ILU, 1 for pAIR-AMG.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -540,11 +536,6 @@ int main(int argc, char *argv[])
   //     right-hand side, and perform time-integration (looping over the time
   //     iterations, ti, with a time-step dt).
   FE_Evolution adv(*m, *k, *B, prec_type);
-   using ImplicitVariableType = FE_Evolution::ImplicitVariableType;
-   ImplicitVariableType imp_var = solve_implicit_state ?
-                                  ImplicitVariableType::STATE
-                                  : ImplicitVariableType::SLOPE;
-   adv.SetImplicitVariableType(imp_var);

   real_t t = 0.0;
   adv.SetTime(t);
@@ -685,17 +676,7 @@ FE_Evolution::FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_,
 //    (M - dt*K) d = K*u + b
 void FE_Evolution::ImplicitSolve(const real_t dt, const Vector &x, Vector &k)
 {
-   // Construct current right-hand side for stage state vs. slope solve
-   if (ImplicitVarTypeIsState())
-   {
-      // k, on return, is the stage value u
-      M->Mult(x, z);
-   }
-   else
-   {
-      // k, on return, is the stage slope du/dt
-      K->Mult(x, z);
-   }
+   K->Mult(x, z);
   z += b;
   dg_solver->SetTimeStep(dt);
   dg_solver->Mult(z, k);
@@ -14,12 +14,6 @@ list(APPEND GINKGO_EXAMPLES_SRCS
  ex1.cpp
 )

-if (MFEM_USE_MPI AND GINKGO_BUILD_MPI)
-    list(APPEND GINKGO_EXAMPLES_SRCS
-    ex1p.cpp
-)
-endif()
-
 # Include the source directory where mfem.hpp and mfem-performance.hpp are.
 include_directories(BEFORE ${PROJECT_BINARY_DIR})

@@ -207,7 +207,7 @@ int main(int argc, char *argv[])
            Ginkgo::IcPreconditioner ginkgo_precond(exec, "paric", 30);
            Ginkgo::CGSolver ginkgo_solver(exec, ginkgo_precond);
            ginkgo_solver.SetPrintLevel(print_lvl);
-            ginkgo_solver.SetRelTol(sqrt(1e-12));
+            ginkgo_solver.SetRelTol(1e-12);
            ginkgo_solver.SetAbsTol(0.0);
            ginkgo_solver.SetMaxIter(400);
            ginkgo_solver.SetOperator(*(A.Ptr()));
@@ -225,7 +225,7 @@ int main(int argc, char *argv[])
            Ginkgo::MFEMPreconditioner gko_M(exec, M);
            Ginkgo::CGSolver ginkgo_solver(exec, gko_M);
            ginkgo_solver.SetPrintLevel(print_lvl);
-            ginkgo_solver.SetRelTol(sqrt(1e-12));
+            ginkgo_solver.SetRelTol(1e-12);
            ginkgo_solver.SetAbsTol(0.0);
            ginkgo_solver.SetMaxIter(400);
            ginkgo_solver.SetOperator(*(A.Ptr()));
@@ -283,7 +283,7 @@ int main(int argc, char *argv[])
               Ginkgo::MFEMPreconditioner gko_M(exec, M);
               Ginkgo::CGSolver ginkgo_solver(exec, gko_M);
               ginkgo_solver.SetPrintLevel(print_lvl);
-               ginkgo_solver.SetRelTol(sqrt(1e-12));
+               ginkgo_solver.SetRelTol(1e-12);
               ginkgo_solver.SetAbsTol(0.0);
               ginkgo_solver.SetMaxIter(400);
               ginkgo_solver.SetOperator(*(A.Ptr()));
@@ -1,436 +0,0 @@
-//                       MFEM Example 1 - Parallel Version
-//                             GINKGO Modification
-//
-// Compile with: make ex1p
-//
-// Sample runs:  mpirun -np 4 ex1p -m ../data/square-disc.mesh
-//               mpirun -np 4 ex1p -m ../data/star.mesh
-//               mpirun -np 4 ex1p -m ../data/star-mixed.mesh
-//               mpirun -np 4 ex1p -m ../data/escher.mesh
-//               mpirun -np 4 ex1p -m ../data/fichera.mesh
-//               mpirun -np 4 ex1p -m ../data/fichera-mixed.mesh
-//               mpirun -np 4 ex1p -m ../data/toroid-wedge.mesh
-//               mpirun -np 4 ex1p -m ../data/octahedron.mesh -o 1
-//               mpirun -np 4 ex1p -m ../data/periodic-annulus-sector.msh
-//               mpirun -np 4 ex1p -m ../data/periodic-torus-sector.msh
-//               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
-//               mpirun -np 4 ex1p -m ../data/square-disc-p3.mesh -o 3
-//               mpirun -np 4 ex1p -m ../data/square-disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/star-mixed-p2.mesh -o 2
-//               mpirun -np 4 ex1p -m ../data/disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/pipe-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/ball-nurbs.mesh -o 2
-//               mpirun -np 4 ex1p -m ../data/fichera-mixed-p2.mesh -o 2
-//               mpirun -np 4 ex1p -m ../data/star-surf.mesh
-//               mpirun -np 4 ex1p -m ../data/square-disc-surf.mesh
-//               mpirun -np 4 ex1p -m ../data/inline-segment.mesh
-//               mpirun -np 4 ex1p -m ../data/amr-quad.mesh
-//               mpirun -np 4 ex1p -m ../data/amr-hex.mesh
-//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh
-//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh -o -1 -sc
-//
-// Device sample runs:
-//               mpirun -np 4 ex1p -pa -d cuda
-//               mpirun -np 4 ex1p -fa -d cuda
-//               mpirun -np 4 ex1p -pa -d occa-cuda
-//               mpirun -np 4 ex1p -pa -d raja-omp
-//               mpirun -np 4 ex1p -pa -d ceed-cpu
-//               mpirun -np 4 ex1p -pa -d ceed-cpu -o 4 -a
-//               mpirun -np 4 ex1p -pa -d ceed-cpu -m ../data/square-mixed.mesh
-//               mpirun -np 4 ex1p -pa -d ceed-cpu -m ../data/fichera-mixed.mesh
-//             * mpirun -np 4 ex1p -pa -d ceed-cuda
-//             * mpirun -np 4 ex1p -pa -d ceed-hip
-//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
-//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/square-mixed.mesh
-//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/fichera-mixed.mesh
-//               mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order, or if order < 1 using an isoparametric/isogeometric
-//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example highlights the use of mesh refinement, finite
-//               element grid functions, as well as linear and bilinear forms
-//               corresponding to the left-hand side and right-hand side of the
-//               discrete linear system. We also cover the explicit elimination
-//               of essential boundary conditions, static condensation, and the
-//               optional connection to the GLVis tool for visualization.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-#ifndef MFEM_USE_GINKGO
-#error This example requires that MFEM is built with MFEM_USE_GINKGO=YES
-#endif
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI and HYPRE.
-   Mpi::Init();
-   int num_procs = Mpi::WorldSize();
-   int myid = Mpi::WorldRank();
-   Hypre::Init();
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../../data/star.mesh";
-   int order = 1;
-   bool static_cond = false;
-   bool pa = false;
-   bool fa = false;
-   const char *device_config = "cpu";
-   bool visualization = true;
-   int solver_config = 0;
-   int print_lvl = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
-                  "--no-full-assembly", "Enable Full Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&solver_config, "-s", "--solver-config",
-                  "Solver and preconditioner combination: \n\t"
-                  "   0 - Ginkgo solver and Ginkgo preconditioner, \n\t"
-                  "   1 - Ginkgo solver and MFEM preconditioner, \n\t"
-                  "   2 - MFEM solver and Ginkgo preconditioner, \n\t"
-                  "   3 - MFEM solver and MFEM preconditioner.");
-   args.AddOption(&print_lvl, "-pl", "--print-level",
-                  "Print level for iterative solver (1 prints every iteration).");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.SetGPUAwareMPI(true);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   // 5. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 10,000 elements.
-   {
-      int ref_levels =
-         (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh.UniformRefinement();
-      }
-   }
-
-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
-   {
-      int par_ref_levels = 2;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         pmesh.UniformRefinement();
-      }
-   }
-
-   // 7. Define a parallel finite element space on the parallel mesh. Here we
-   //    use continuous Lagrange finite elements of the specified order. If
-   //    order < 1, we instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   bool delete_fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
-   }
-   else if (pmesh.GetNodes())
-   {
-      fec = pmesh.GetNodes()->OwnFEC();
-      delete_fec = false;
-      if (myid == 0)
-      {
-         cout << "Using isoparametric FEs: " << fec->Name() << endl;
-      }
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
-   }
-   ParFiniteElementSpace fespace(&pmesh, fec);
-   HYPRE_BigInt size = fespace.GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-   }
-
-   // 8. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet) and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (pmesh.bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 9. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (1,phi_i) where phi_i are the basis functions in fespace.
-   ParLinearForm b(&fespace);
-   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
-
-   // 10. Define the solution vector x as a parallel finite element grid
-   //     function corresponding to fespace. Initialize x with initial guess of
-   //     zero, which satisfies the boundary conditions.
-   ParGridFunction x(&fespace);
-   x = 0.0;
-
-   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the
-   //     Diffusion domain integrator.
-   ParBilinearForm a(&fespace);
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   if (fa)
-   {
-      a.SetAssemblyLevel(AssemblyLevel::FULL);
-      // Sort the matrix column indices when running on GPU or with OpenMP (i.e.
-      // when Device::IsEnabled() returns true). This makes the results
-      // bit-for-bit deterministic at the cost of somewhat longer run time.
-      a.EnableSparseMatrixSorting(Device::IsEnabled());
-   }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   // 12. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
-   a.Assemble();
-
-   OperatorPtr A;
-   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-
-   // 13. Solve the linear system A X = B.
-   if (!pa)
-   {
-      switch (solver_config)
-      {
-         // Solve the linear system with CG + Schwarz (with IC) from Ginkgo
-         case 0:
-         {
-            if (myid == 0) { cout << "Using Ginkgo solver + preconditioner...\n"; }
-            Ginkgo::GinkgoExecutor exec(device);
-            Ginkgo::IcPreconditioner local_solver(exec, "exact");
-            Ginkgo::SchwarzPreconditioner gko_M(exec, MPI_COMM_WORLD, local_solver);
-            Ginkgo::CGSolver ginkgo_solver(exec, MPI_COMM_WORLD, gko_M);
-            ginkgo_solver.SetPrintLevel(print_lvl);
-            ginkgo_solver.SetRelTol(sqrt(1e-12));
-            ginkgo_solver.SetAbsTol(0.0);
-            ginkgo_solver.SetMaxIter(400);
-            ginkgo_solver.SetOperator(*(A.Ptr()));
-            ginkgo_solver.Mult(B, X);
-            break;
-         }
-
-         // Solve the linear system with CG from Ginkgo + MFEM preconditioner
-         case 1:
-         {
-            if (myid == 0) { cout << "Using Ginkgo solver + MFEM preconditioner...\n"; }
-            Ginkgo::GinkgoExecutor exec(device);
-            //Create MFEM preconditioner and wrap it for Ginkgo's use.
-            HypreBoomerAMG M((HypreParMatrix&)(*A));
-            Ginkgo::MFEMPreconditioner gko_M(exec, M, MPI_COMM_WORLD);
-            Ginkgo::CGSolver ginkgo_solver(exec, MPI_COMM_WORLD, gko_M);
-            ginkgo_solver.SetPrintLevel(print_lvl);
-            ginkgo_solver.SetRelTol(sqrt(1e-12));
-            ginkgo_solver.SetAbsTol(0.0);
-            ginkgo_solver.SetMaxIter(400);
-            ginkgo_solver.SetOperator(*(A.Ptr()));
-            ginkgo_solver.Mult(B, X);
-            break;
-         }
-
-         // Ginkgo Schwarz preconditioner (local ParIC) +  MFEM CG solver
-         case 2:
-         {
-            if (myid == 0) { cout << "Using MFEM solver + Ginkgo preconditioner...\n"; }
-            Ginkgo::GinkgoExecutor exec(device);
-            Ginkgo::IcPreconditioner local_M(exec, "exact");
-            Ginkgo::SchwarzPreconditioner M(exec, MPI_COMM_WORLD, local_M);
-            M.SetOperator(*(A.Ptr()));  // Generate the preconditioner for the matrix A.
-            CGSolver cg(MPI_COMM_WORLD);
-            cg.SetRelTol(sqrt(1e-12));
-            cg.SetMaxIter(400);
-            cg.SetPrintLevel(1);
-            cg.SetPreconditioner(M);
-            cg.SetOperator(*A);
-            cg.Mult(B, X);
-            break;
-         }
-
-         // MFEM solver + MFEM preconditioner
-         case 3:
-         {
-            if (myid == 0) { cout << "Using MFEM solver + MFEM preconditioner...\n"; }
-            HypreBoomerAMG M((HypreParMatrix&)(*A));
-            CGSolver cg(MPI_COMM_WORLD);
-            cg.SetRelTol(sqrt(1e-12));
-            cg.SetMaxIter(400);
-            cg.SetPrintLevel(1);
-            cg.SetPreconditioner(M);
-            cg.SetOperator(*A);
-            cg.Mult(B, X);
-            break;
-         }
-      } // End switch on solver_config
-   }
-   // Partial assembly mode.  Cannot use Ginkgo preconditioners, but can use Ginkgo
-   // solvers.
-   else
-   {
-      if (UsesTensorBasis(fespace))
-      {
-         // Use Jacobi preconditioning in partial assembly mode.
-         OperatorJacobiSmoother M(a, ess_tdof_list);
-         switch (solver_config)
-         {
-            case 0:
-            {
-               if (myid == 0) { cout << "Using Ginkgo solver + preconditioner...\n"; }
-               MFEM_ABORT("Cannot use Ginkgo preconditioner in partial assembly mode.\n"
-                          "            Try -s 1 to test Ginkgo solver with an MFEM preconditioner.");
-               break;
-            }
-
-            // Use Ginkgo solver with MFEM preconditioner
-            case 1:
-            {
-               if (myid == 0) { cout << "Using Ginkgo solver + MFEM preconditioner...\n"; }
-               Ginkgo::GinkgoExecutor exec(device);
-               // Wrap MFEM preconditioner for Ginkgo's use.
-               Ginkgo::MFEMPreconditioner gko_M(exec, M, MPI_COMM_WORLD);
-               Ginkgo::CGSolver ginkgo_solver(exec, MPI_COMM_WORLD, gko_M);
-               ginkgo_solver.SetPrintLevel(print_lvl);
-               ginkgo_solver.SetRelTol(sqrt(1e-12));
-               ginkgo_solver.SetAbsTol(0.0);
-               ginkgo_solver.SetMaxIter(400);
-               ginkgo_solver.SetOperator(*(A.Ptr()));
-               ginkgo_solver.Mult(B, X);
-               break;
-            }
-
-            // No Ginkgo preconditioners work with matrix-free; error
-            case 2:
-            {
-               if (myid == 0) { cout << "Using Ginkgo solver + preconditioner...\n"; }
-               MFEM_ABORT("Cannot use Ginkgo preconditioner in partial assembly mode.\n"
-                          "            Try -s 1 to test Ginkgo solver with an MFEM preconditioner.");
-               break;
-            }
-
-            // Use MFEM solver and preconditioner
-            case 3:
-            {
-               if (myid == 0) { cout << "Using MFEM solver + MFEM preconditioner...\n"; }
-               CGSolver cg(MPI_COMM_WORLD);
-               cg.SetRelTol(sqrt(1e-12));
-               cg.SetMaxIter(400);
-               cg.SetPrintLevel(1);
-               cg.SetPreconditioner(M);
-               cg.SetOperator(*A);
-               cg.Mult(B, X);
-               break;
-            }
-         } // End switch on solver_config
-      }
-      else // CG with no preconditioning
-      {
-         if (myid == 0) { cout << "Using MFEM solver + no preconditioner...\n"; }
-         CGSolver cg(MPI_COMM_WORLD);
-         cg.SetRelTol(sqrt(1e-12));
-         cg.SetMaxIter(400);
-         cg.SetPrintLevel(1);
-         cg.SetOperator(*A);
-         cg.Mult(B, X);
-      }
-   }
-
-   // 14. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a.RecoverFEMSolution(X, b, x);
-
-   // 15. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh.Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 16. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << pmesh << x << flush;
-   }
-
-   // 17. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
-
-   return 0;
-}
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Will Pazner	552d6857cb	Add GPU scan support for Array<T>::PartialSum	2025-10-21 09:57:37 -07:00
Will Pazner	a37d46e917	Fix comments in scan.hpp	2025-10-21 09:57:37 -07:00
Will Pazner	4acdb072b6	Add more device support to Array<T>	2025-10-21 09:57:37 -07:00
Will Pazner	9dbb184537	Remove need to explicitly pass workspace array to reducers	2025-10-21 09:57:37 -07:00
Will Pazner	d67762a1c9	Move contents of array.cpp to array.hpp Remove explicit template instantiations	2025-10-20 15:38:54 -07:00