Remove white spaces

.github/actions/sanitize/mfem/action.yml

@@ -25,7 +25,7 @@ runs:
   steps:
     - uses: ./.github/actions/sanitize/config
 
-    - uses: actions/cache@v5
+    - uses: actions/cache@v4
       if: ${{env.DEBUG == 'true'}}
       id: debug
       with:

.github/actions/sanitize/restore/action.yml

@@ -36,7 +36,7 @@ runs:
   steps:
     - uses: ./.github/actions/sanitize/config
 
-    - uses: actions/cache@v5
+    - uses: actions/cache@v4
       if: ${{env.DEBUG == 'true' && inputs.cache-skip != 'true'}}
       id: debug
       with:

.github/actions/sanitize/setup/action.yml

@@ -23,7 +23,7 @@ inputs:
 runs:
   using: 'composite'
   steps:
-    - uses: actions/cache/restore@v5  # Cache for LLVM libcxx
+    - uses: actions/cache/restore@v4  # Cache for LLVM libcxx
       with:
         path: ${{env.LLVM_DIR}}
         fail-on-cache-miss: true
@@ -32,14 +32,14 @@ runs:
     - uses: ./.github/actions/sanitize/mpi
       if: ${{inputs.par == 'true'}}
 
-    - uses: actions/cache/restore@v5  # Cache for Hypre
+    - uses: actions/cache/restore@v4  # Cache for Hypre
       if: ${{inputs.par == 'true'}}
       with:
         path: ${{env.HYPRE_DIR}}
         fail-on-cache-miss: true
         key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-v2.5
 
-    - uses: actions/cache/restore@v5  # Cache for Metis
+    - uses: actions/cache/restore@v4  # Cache for Metis
       if: ${{inputs.par == 'true'}}
       with:
         path: ${{env.METIS_DIR}}
@@ -51,13 +51,13 @@ runs:
       run: ln -s -f ${{env.HYPRE_DIR}} hypre && ln -s -f ${{env.METIS_DIR}} metis-4.0
       shell: bash
 
-    - uses: actions/cache/restore@v5  # Cache for LSAN suppression file
+    - uses: actions/cache/restore@v4  # Cache for LSAN suppression file
       with:
         path: ${{env.LSAN_DIR}}
         fail-on-cache-miss: true
         key: build-lsan-suppression-file
 
-    - uses: actions/checkout@v6  # Checkout the repository
+    - uses: actions/checkout@v4  # Checkout the repository
       with:
         path: mfem
         # ref: ${{env.BRANCH}}

.github/workflows/build-container.yml

@@ -43,7 +43,7 @@ jobs:
           remove-docker-images: 'true'
 
       - name: Checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
 
       # It's easier to reference named variables than indexes of the matrix
       - name: Set Environment

.github/workflows/builds-and-tests.yml

@@ -153,7 +153,7 @@ jobs:
       # /home/runner/work/mfem/mfem/mfem
       # Note: Done now to access "install-hypre" and "install-metis" actions.
       - name: checkout mfem
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
         with:
           path: ${{ env.MFEM_TOP_DIR }}
           # Fetch the complete history for codecov to access commits ID
@@ -225,7 +225,7 @@ jobs:
       - name: cache hypre
         id: hypre-cache
         if: matrix.mpi == 'par'
-        uses: actions/cache@v5
+        uses: actions/cache@v4
         with:
           path: ${{ env.HYPRE_TOP_DIR }}
           key: ${{ runner.os }}-ompi-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-${{ matrix.precision }}-v2.5
@@ -255,7 +255,7 @@ jobs:
       - name: cache metis
         id: metis-cache
         if: matrix.mpi == 'par' && matrix.os != 'windows-latest'
-        uses: actions/cache@v5
+        uses: actions/cache@v4
         with:
           path: ${{ env.METIS_TOP_DIR }}
           key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5
@@ -270,7 +270,7 @@ jobs:
       - name: cache vcpkg (Windows)
         id: vcpkg-cache
         if: matrix.os == 'windows-latest'
-        uses: actions/cache@v5
+        uses: actions/cache@v4
         with:
           path: vcpkg_cache
           key: ${{ runner.os }}-${{ matrix.mpi }}-vcpkg-v1
@@ -295,8 +295,7 @@ jobs:
           export HOMEBREW_NO_INSTALL_CLEANUP=1
           brew update
           brew install enzyme
-          ENZYME_LLVM=$(brew info enzyme | sed -n 's/^Required.*:.*\(llvm[^ ]*\).*/\1/p')
-          echo "ENZYME_LLVM=$ENZYME_LLVM"
+          ENZYME_LLVM=$(brew info enzyme | sed -n 's/^Required:.*\(llvm[^ ]*\).*/\1/p')
           LLVM_PREFIX=$(brew --prefix $ENZYME_LLVM)
           echo "LLVM_PREFIX=$LLVM_PREFIX" >> $GITHUB_ENV
           echo "OMPI_CC=$LLVM_PREFIX/bin/clang" >> $GITHUB_ENV

.github/workflows/codeql-analysis.yml

@@ -40,11 +40,11 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v4
+        uses: github/codeql-action/init@v2
         with:
           languages: ${{ matrix.language }}
           # If you wish to specify custom queries, you can do so here or in a config file.
@@ -57,7 +57,7 @@ jobs:
       # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v4
+        uses: github/codeql-action/autobuild@v2
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -70,4 +70,4 @@ jobs:
       #   ./location_of_script_within_repo/buildscript.sh
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v4
+        uses: github/codeql-action/analyze@v2

.github/workflows/mfem-analysis.yml

@@ -39,7 +39,7 @@ jobs:
 
     steps:
       - name: checkout MFEM
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
         with:
           path: mfem
 
@@ -50,7 +50,7 @@ jobs:
 
       - name: Cache Hypre Install
         id: hypre-cache
-        uses: actions/cache@v5
+        uses: actions/cache@v4
         with:
           path: ${{ env.HYPRE_TOP_DIR }}
           key: ${{ runner.os }}-ompi-build-${{ env.HYPRE_TOP_DIR }}-v2.5
@@ -65,7 +65,7 @@ jobs:
 
       - name: Cache Metis Install
         id: metis-cache
-        uses: actions/cache@v5
+        uses: actions/cache@v4
         with:
           path: ${{ env.METIS_TOP_DIR }}
           key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5

.github/workflows/repo-check.yml

@@ -38,7 +38,7 @@ jobs:
        github.event.pull_request.head.repo.full_name != github.repository)
     steps:
       - name: checkout mfem
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
 
       - name: copyright check
         id: copyright
@@ -93,7 +93,7 @@ jobs:
        github.event.pull_request.head.repo.full_name != github.repository)
     steps:
       - name: checkout mfem
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
 
       - name: get astyle
         run: |
@@ -110,7 +110,7 @@ jobs:
        github.event.pull_request.head.repo.full_name != github.repository)
     steps:
       - name: checkout mfem
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
 
       - name: get doxygen and graphviz
         run: |
@@ -135,7 +135,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: checkout mfem
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 

.github/workflows/sanitize-build-hypre.yml

@@ -17,11 +17,11 @@ jobs:
     runs-on: ubuntu-latest
     name: 2.19.0
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/sanitize/config
       - name: Cache
         id: cache
-        uses: actions/cache@v5
+        uses: actions/cache@v4
         with:
           path: ${{env.HYPRE_DIR}}
           key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-v2.5

.github/workflows/sanitize-build-libcxx.yml

@@ -27,13 +27,13 @@ jobs:
             llvm_use_sanitizer: "Undefined"
     name: ${{matrix.sanitizer}}
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/sanitize/config
         with:
           NO_FLAGS: true
       - name: Cache
         id: cache
-        uses: actions/cache@v5
+        uses: actions/cache@v4
         with:
           path: ${{env.LLVM_DIR}}
           key: build-libcxx-${{env.LLVM_VER}}-${{matrix.sanitizer}}

.github/workflows/sanitize-build-lsan.yml

@@ -17,11 +17,11 @@ jobs:
     runs-on: ubuntu-latest
     name: lsan.supp
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/sanitize/config
       - name: Cache
         id: cache
-        uses: actions/cache@v5
+        uses: actions/cache@v4
         with:
           path: ${{env.LSAN_DIR}}
           key: build-lsan-suppression-file

.github/workflows/sanitize-build-metis.yml

@@ -17,11 +17,11 @@ jobs:
     runs-on: ubuntu-latest
     name: 4.0.3
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/sanitize/config
       - name: Cache
         id: cache
-        uses: actions/cache@v5
+        uses: actions/cache@v4
         with:
           path: ${{env.METIS_DIR}}
           key: ${{runner.os}}-build-${{env.METIS_DIR}}-v2.5

.github/workflows/sanitize-tests.yml

@@ -28,7 +28,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/sanitize/mfem
         with:
           par: ${{inputs.par}}
@@ -40,7 +40,7 @@ jobs:
     env:
       ex: ${{inputs.par && 'ex1p' || 'ex1'}}
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/sanitize/restore
         id: restore
         with:
@@ -58,7 +58,7 @@ jobs:
     env:
       exclude: ${{inputs.par && '-E "_ser"' || ''}}
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/sanitize/restore
         id: restore
         with:
@@ -82,7 +82,7 @@ jobs:
     env:
       exclude: ${{inputs.par && '-E "_ser"' || ''}}
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/sanitize/restore
         id: restore
         with:
@@ -107,7 +107,7 @@ jobs:
       run: ${{inputs.par && '-R "_cpu_np"' || ''}}
       exclude: ${{inputs.par && '"unit_tests|debug"' || '"^unit_tests$|debug"'}}
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/sanitize/restore
         id: restore
         with:
@@ -131,7 +131,7 @@ jobs:
     env:
       unit_tests: ${{inputs.par && 'punit_tests' || 'unit_tests'}}
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/sanitize/restore
         id: restore
         with:
@@ -165,7 +165,7 @@ jobs:
       unit_tests: ${{inputs.par && 'punit_tests' || 'unit_tests'}}
       np: ${{inputs.par && '_np=2' || ''}}
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/sanitize/restore
         id: restore
         with:

.gitignore

@@ -19,15 +19,9 @@ CMakeFiles/
 # Clangd server cache
 *.cache*
 
-#vscode settings
-/.vscode/
-
 # Backup files
 *~
 
-# clangd index
-/.cache/
-
 # Default install location
 /mfem/
 
@@ -85,17 +79,12 @@ examples/sol_u.*
 examples/sol_p.*
 examples/sol_r.*
 examples/sol_i.*
-examples/sol_z.*
 examples/ex6p-checkpoint.*
 examples/order.*
 examples/ex9.mesh
 examples/ex9-mesh.*
 examples/ex9-init.*
 examples/ex9-final.*
-examples/ex41.mesh
-examples/ex41-mesh.*
-examples/ex41-init.*
-examples/ex41-final.*
 examples/deformed.*
 examples/velocity.*
 examples/elastic_energy.*
@@ -227,9 +216,6 @@ miniapps/electromagnetics/Joule_[0-9]*
 miniapps/electromagnetics/Lorentz_[0-9]*
 miniapps/electromagnetics/Lorentz.dat
 
-miniapps/fluids/schrodinger-flow/schrodinger_flow
-miniapps/fluids/schrodinger-flow/pschrodinger_flow
-
 miniapps/gslib/field-diff
 miniapps/gslib/field-interp
 miniapps/gslib/findpts
@@ -237,7 +223,6 @@ miniapps/gslib/pfindpts
 miniapps/gslib/schwarz_ex1
 miniapps/gslib/schwarz_ex1p
 miniapps/gslib/interpolated.gf
-miniapps/gslib/particles_redist
 
 miniapps/meshing/mobius-strip
 miniapps/meshing/klein-bottle
@@ -284,8 +269,10 @@ miniapps/meshing/refined.mesh
 miniapps/meshing/bounding-box*
 miniapps/meshing/jacobian-determinant*
 
-miniapps/mtop/ParaView/
-miniapps/mtop/mtop_test_iso_elasticity
+miniapps/mtop/parheat
+miniapps/mtop/ParHeat/*
+miniapps/mtop/seqheat
+miniapps/mtop/SeqHeat/*
 
 miniapps/autodiff/paradiff
 miniapps/autodiff/seqadiff
@@ -295,26 +282,21 @@ miniapps/autodiff/seq_example
 miniapps/autodiff/seq_test
 miniapps/autodiff/Example/*
 
-miniapps/fluids/navier/navier_mms
-miniapps/fluids/navier/navier_kovasznay
-miniapps/fluids/navier/navier_kovasznay_vs
-miniapps/fluids/navier/navier_tgv
-miniapps/fluids/navier/navier_shear
-miniapps/fluids/navier/navier_3dfoc
-miniapps/fluids/navier/navier_turbchan
-miniapps/fluids/navier/navier_cht
-miniapps/fluids/navier/navier_bifurcation
-miniapps/fluids/navier/Navier_Bifurcation_[0-9]*
-miniapps/fluids/navier/ParaView
-miniapps/fluids/navier/tgv_out*.txt
-miniapps/fluids/navier/*_output
+miniapps/navier/navier_mms
+miniapps/navier/navier_kovasznay
+miniapps/navier/navier_kovasznay_vs
+miniapps/navier/navier_tgv
+miniapps/navier/navier_shear
+miniapps/navier/navier_3dfoc
+miniapps/navier/navier_turbchan
+miniapps/navier/navier_cht
+miniapps/navier/tgv_out*.txt
+miniapps/navier/*_output
 
 miniapps/nurbs/nurbs_ex1
 miniapps/nurbs/nurbs_ex1p
 miniapps/nurbs/nurbs_ex3
 miniapps/nurbs/nurbs_ex5
-miniapps/nurbs/nurbs_ex10
-miniapps/nurbs/nurbs_ex10p
 miniapps/nurbs/nurbs_ex11p
 miniapps/nurbs/nurbs_ex24
 miniapps/nurbs/nurbs_solenoidal
@@ -340,14 +322,7 @@ miniapps/nurbs/nurbs_naca_cmesh
 miniapps/nurbs/naca-cmesh.mesh
 miniapps/nurbs/glvis_naca-cmesh.mesh
 miniapps/nurbs/Naca_cmesh
-miniapps/nurbs/nurbs_mesh_info
-miniapps/nurbs/k*_*.dat
 miniapps/nurbs/*-Surface.mesh
-miniapps/nurbs/*.mesh
-miniapps/nurbs/*.sol
-miniapps/nurbs/deformed.*
-miniapps/nurbs/elastic_energy.*
-miniapps/nurbs/velocity.*
 
 miniapps/performance/ex1
 miniapps/performance/ex1p
@@ -369,7 +344,6 @@ miniapps/shifted/lsf_integral
 miniapps/tools/display-basis
 miniapps/tools/load-dc
 miniapps/tools/convert-dc
-miniapps/tools/compare-dc
 miniapps/tools/gridfunction-bounds
 miniapps/tools/lor-transfer
 miniapps/tools/plor-transfer
@@ -440,13 +414,6 @@ miniapps/tribol/contact-patch-test
 miniapps/diag-smoothers/abs-l1-jacobi
 miniapps/diag-smoothers/mg-abs-l1-jacobi
 
-miniapps/contact/contact
-miniapps/contact/ParaView
-
-miniapps/plasma/pic/electrostatic-*
-!miniapps/plasma/pic/electrostatic-*.cpp
-miniapps/plasma/pic/*.csv
-
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests

.gitlab-ci.yml

@@ -9,550 +9,91 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-# DESCRIPTION:
-###############################################################################
 # General GitLab pipelines configurations for supercomputers and Linux clusters
-# at Lawrence Livermore National Laboratory (LLNL).
-# This entire pipeline is LLNL-specific
-#
-# Important note: This file is a template provided by llnl/radiuss-shared-ci.
-# Remains to set variable values, change the reference to the radiuss-shared-ci
-# repo, opt-in and out optional features. The project can then extend it with
-# additional stages.
-#
-# In addition, each project should copy over and complete:
-# - .gitlab/custom-jobs-and-variables.yml
-# - .gitlab/subscribed-pipelines.yml
-#
-# The jobs should be specified in a file local to the project,
-# - .gitlab/jobs/${CI_MACHINE}.yml
-# or generated (see LLNL/Umpire for an example).
-###############################################################################
-#   MAP OF GITLAB CI
-#######################
-#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-# File dependencies: direct, through jobs, through variables
-#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-# .gitlab-ci.yml
-#  ├── .build-and-test  [job]
-#  │    ├── .gitlab/custom-jobs-and-variables.yml
-#  │    │    ├── .custom_job          [job]
-#  │    │    ├── .reproducer_vars     [job]
-#  │    │    ├── .report_job_success  [job]
-#  │    │    │    └── .gitlab/scripts/report_build_and_test    [script]
-#  │    │    │         ├── .gitlab/scripts/safe_create_rundir  [script]
-#  │    │    │         └── .gitlab/scripts/git_try_to_push     [script]
-#  │    │    ├── .report_job_failure  [job]
-#  │    │    │    └── .gitlab/scripts/report_build_and_test    [script]
-#  │    │    │         ├── .gitlab/scripts/safe_create_rundir  [script]
-#  │    │    │         └── .gitlab/scripts/git_try_to_push     [script]
-#  │    │    └── JOB_CMD              [var]
-#  │    │         └── tests/gitlab/build_and_test            [script]
-#  │    │              └── tests/gitlab/get_mfem_uberenv     [script]
-#  │    ├── <radiuss-shared-ci>/pipelines/matrix.yml  [conditional]
-#  │    │    ├── .on_matrix               [job]
-#  │    │    ├── .matrix_reproducer_init  [job]
-#  │    │    ├── .matrix_reproducer_vars  [job]
-#  │    │    ├── .matrix_reproducer_job   [job]
-#  │    │    ├── .matrix_job_command      [job]
-#  │    │    └── .job_on_matrix           [job]
-#  │    ├── <radiuss-shared-ci>/pipelines/dane.yml  [conditional]
-#  │    │    ├── .on_dane               [job]
-#  │    │    ├── .dane_reproducer_init  [job]
-#  │    │    ├── .dane_reproducer_vars  [job]
-#  │    │    ├── .dane_reproducer_job   [job]
-#  │    │    ├── .dane_job_command      [job]
-#  │    │    ├── .job_on_dane           [job]
-#  │    │    ├── allocate_resources     [job]
-#  │    │    └── release_resources      [job]
-#  │    ├── <radiuss-shared-ci>/pipelines/tioga.yml  [conditional]
-#  │    │    ├── .on_tioga               [job]
-#  │    │    ├── .tioga_reproducer_init  [job]
-#  │    │    ├── .tioga_reproducer_vars  [job]
-#  │    │    ├── .tioga_reproducer_job   [job]
-#  │    │    ├── .tioga_job_command      [job]
-#  │    │    ├── .job_on_tioga           [job]
-#  │    │    ├── allocate_resources      [job]
-#  │    │    └── release_resources       [job]
-#  │    ├── <artifact>/matrix-jobs.yml  [conditional, from 'generate-job-lists']
-#  │    │    ├── .gitlab/jobs/matrix.yml
-#  │    │    │    ├── .matrix_reproducer_vars      [job]
-#  │    │    │    ├── setup                        [job]
-#  │    │    │    │    └── ./tests/gitlab/build_and_test_setup  [script]
-#  │    │    │    ├── opt_mpi_cuda_gcc             [job]
-#  │    │    │    └── opt_mpi_cuda_hypre_cuda_gcc  [job]
-#  │    │    └── .gitlab/jobs/matrix-reports.yml  [used conditionally]
-#  │    │         ├── report_job_success
-#  │    │         └── report_job_failure
-#  │    ├── <artifact>/dane-jobs.yml    [conditional, from 'generate-job-lists']
-#  │    │    ├── .gitlab/jobs/dane.yml
-#  │    │    │    ├── .dane_reproducer_vars    [job]
-#  │    │    │    ├── setup                    [job]
-#  │    │    │    │    └── ./tests/gitlab/build_and_test_setup  [script]
-#  │    │    │    ├── debug_ser_gcc_10         [job]
-#  │    │    │    ├── debug_par_gcc_10         [job]
-#  │    │    │    ├── opt_ser_gcc_10           [job]
-#  │    │    │    ├── opt_par_gcc_10           [job]
-#  │    │    │    ├── opt_par_gcc_10_sundials  [job]
-#  │    │    │    ├── opt_par_gcc_10_petsc     [job]
-#  │    │    │    └── opt_par_gcc_10_pumi      [job]
-#  │    │    └── .gitlab/jobs/dane-reports.yml    [used conditionally]
-#  │    │         ├── report_job_success
-#  │    │         └── report_job_failure
-#  │    └── <artifact>/tioga-jobs.yml   [conditional, from 'generate-job-lists']
-#  │         ├── .gitlab/jobs/tioga.yml
-#  │         │    ├── .tioga_reproducer_vars  [job]
-#  │         │    ├── setup                   [job]
-#  │         │    │    └── ./tests/gitlab/build_and_test_setup  [script]
-#  │         │    └── cce_16_0_1              [job]
-#  │         └── .gitlab/jobs/tioga-reports.yml   [used conditionally]
-#  │              ├── report_job_success
-#  │              └── report_job_failure
-#  └── .gitlab/subscribed-pipelines.yml
-#       ├── .machine-check         [job]
-#       ├── generate-job-lists     [job]
-#       ├── dane-up-check          [job]
-#       ├── dane-build-and-test    [job]
-#       ├── dane-baseline          [job]
-#       │    └── .gitlab/dane-baseline.yml
-#       │         ├── .on_dane                       [job]
-#       │         ├── baselinecheck_mfem_intel_dane  [job]
-#       │         │    └── .gitlab/scripts/baseline            [script]
-#       │         ├── cleanup                        [job]
-#       │         ├── report_baseline                [job]
-#       │         │    ├── .gitlab/scripts/safe_create_rundir  [script]
-#       │         │    └── .gitlab/scripts/git_try_to_push     [script]
-#       │         ├── baselinepublish_mfem_dane      [job]
-#       │         │    └── .gitlab/scripts/rebaseline          [script]
-#       │         ├── .gitlab/custom-jobs-and-variables.yml
-#       │         │    └── <same as above: see .gitlab-ci.yml/.build-and-test>
-#       │         └── .gitlab/configs/setup-baseline.yml
-#       │              └── setup_baseline            [job]
-#       ├── tioga-up-check         [job]
-#       ├── tioga-build-and-test   [job]
-#       ├── matrix-up-check        [job]
-#       └── matrix-build-and-test  [job]
-#
-#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-# File tree hierarchy with file contents highlights
-#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-# In addition to the files in the MFEM repo, the Gitlab CI uses files from the
-# radiuss/radiuss-shared-ci project, see below, after the <mfem-root> tree.
-#
-# <mfem root>
-#  ├── .gitlab-ci.yml  [this file]
-#  │    ├── <jobs>
-#  │    │    └── .build-and-test
-#  │    ├── <included files>
-#  │    │    ├── .gitlab/subscribed-pipelines.yml
-#  │    │    ├── .gitlab/custom-jobs-and-variables.yml  [by ".build-and-test"]
-#  │    │    ├── <artifact>  [by ".build-and-test"]
-#  │    │    │    ├── artifact: '${CI_MACHINE}-jobs.yml'
-#  │    │    │    └── job:      'generate-job-lists'
-#  │    │    └── <external>  [by ".build-and-test"]
-#  │    │         ├── project: 'radiuss/radiuss-shared-ci'
-#  │    │         ├── ref:     'v2025.09.1'
-#  │    │         └── file:    'pipelines/${CI_MACHINE}.yml'
-#  │    └── <defined variables>
-#  │         ├── CUSTOM_CI_BUILDS_DIR
-#  │         ├── USER_CI_TOP_DIR
-#  │         ├── SHARED_REPOS_DIR
-#  │         ├── AUTOTEST_ROOT
-#  │         ├── MFEM_DATA_DIR
-#  │         ├── AUTOTEST
-#  │         ├── AUTOTEST_COMMIT
-#  │         ├── REBASELINE
-#  │         ├── GITHUB_PROJECT_NAME
-#  │         └── GITHUB_PROJECT_ORG
-#  ├── .gitlab
-#  │    ├── configs
-#  │    │    └── setup-baseline.yml
-#  │    │         ├── <jobs>
-#  │    │         │    └── setup_baseline
-#  │    │         └── <used variables>
-#  │    │              ├── MACHINE_NAME
-#  │    │              ├── REBASELINE
-#  │    │              ├── AUTOTEST
-#  │    │              ├── AUTOTEST_COMMIT
-#  │    │              ├── BUILD_ROOT
-#  │    │              ├── TPLS_REPO
-#  │    │              ├── TESTS_REPO
-#  │    │              ├── AUTOTEST_ROOT
-#  │    │              └── AUTOTEST_REPO
-#  │    ├── jobs
-#  │    │    ├── matrix-reports.yml
-#  │    │    │    ├── <jobs>
-#  │    │    │    │    ├── report_job_success
-#  │    │    │    │    └── report_job_failure
-#  │    │    │    └── <used jobs>
-#  │    │    │         ├── .on_matrix
-#  │    │    │         ├── .report_job_success
-#  │    │    │         └── .report_job_failure
-#  │    │    ├── matrix.yml
-#  │    │    │    ├── <jobs>
-#  │    │    │    │    ├── .matrix_reproducer_vars
-#  │    │    │    │    ├── setup
-#  │    │    │    │    ├── opt_mpi_cuda_gcc
-#  │    │    │    │    └── opt_mpi_cuda_hypre_cuda_gcc
-#  │    │    │    ├── <used jobs>
-#  │    │    │    │    ├── .reproducer_vars
-#  │    │    │    │    ├── .on_matrix
-#  │    │    │    │    └── .job_on_matrix
-#  │    │    │    ├── <included and used files>
-#  │    │    │    │    └── tests/gitlab/build_and_test_setup  [by "setup"]
-#  │    │    │    └── <defined variables>
-#  │    │    │         └── SPEC
-#  │    │    ├── dane-reports.yml
-#  │    │    │    ├── <jobs>
-#  │    │    │    │    ├── report_job_success
-#  │    │    │    │    └── report_job_failure
-#  │    │    │    └── <used jobs>
-#  │    │    │         ├── .on_dane
-#  │    │    │         ├── .report_job_success
-#  │    │    │         └── .report_job_failure
-#  │    │    ├── dane.yml
-#  │    │    │    ├── <jobs>
-#  │    │    │    │    ├── .dane_reproducer_vars
-#  │    │    │    │    ├── setup
-#  │    │    │    │    ├── debug_ser_gcc_10
-#  │    │    │    │    ├── debug_par_gcc_10
-#  │    │    │    │    ├── opt_ser_gcc_10
-#  │    │    │    │    ├── opt_par_gcc_10
-#  │    │    │    │    ├── opt_par_gcc_10_sundials
-#  │    │    │    │    ├── opt_par_gcc_10_petsc
-#  │    │    │    │    └── opt_par_gcc_10_pumi
-#  │    │    │    ├── <used jobs>
-#  │    │    │    │    ├── .reproducer_vars
-#  │    │    │    │    ├── .on_dane
-#  │    │    │    │    └── .job_on_dane
-#  │    │    │    ├── <included and used files>
-#  │    │    │    │    └── tests/gitlab/build_and_test_setup  [by "setup"]
-#  │    │    │    └── <defined variables>
-#  │    │    │         ├── SPEC
-#  │    │    │         └── THREADS
-#  │    │    ├── tioga-reports.yml
-#  │    │    │    ├── <jobs>
-#  │    │    │    │    ├── report_job_success
-#  │    │    │    │    └── report_job_failure
-#  │    │    │    └── <used jobs>
-#  │    │    │         ├── .on_tioga
-#  │    │    │         ├── .report_job_success
-#  │    │    │         └── .report_job_failure
-#  │    │    └── tioga.yml
-#  │    │         ├── <jobs>
-#  │    │         │    ├── .tioga_reproducer_vars
-#  │    │         │    ├── setup
-#  │    │         │    └── opt_mpi_rocm_hypre_rocm
-#  │    │         ├── <used jobs>
-#  │    │         │    ├── .reproducer_vars
-#  │    │         │    ├── .on_tioga
-#  │    │         │    └── .job_on_tioga
-#  │    │         ├── <included and used files>
-#  │    │         │    └── tests/gitlab/build_and_test_setup  [by "setup"]
-#  │    │         └── <defined variables>
-#  │    │              ├── SPEC
-#  │    │              └── THREADS
-#  │    ├── scripts
-#  │    │    ├── baseline
-#  │    │    │    └── <used variables>
-#  │    │    │         ├── BASELINE_TEST
-#  │    │    │         ├── SYS_TYPE
-#  │    │    │         ├── MACHINE_NAME
-#  │    │    │         ├── CI_PROJECT_DIR
-#  │    │    │         ├── ARTIFACTS_DIR
-#  │    │    │         ├── BUILD_ROOT
-#  │    │    │         └── TPLS_DIR
-#  │    │    ├── git_try_to_push
-#  │    │    ├── rebaseline
-#  │    │    │    └── <used variables>
-#  │    │    │         ├── CI_PROJECT_DIR
-#  │    │    │         ├── ARTIFACTS_DIR
-#  │    │    │         ├── SYS_TYPE
-#  │    │    │         ├── BUILD_ROOT
-#  │    │    │         ├── MACHINE_NAME
-#  │    │    │         └── CI_PIPELINE_ID
-#  │    │    ├── report_build_and_test
-#  │    │    │    ├── <used files>
-#  │    │    │    │    ├── .gitlab/scripts/safe_create_rundir
-#  │    │    │    │    └── .gitlab/scripts/git_try_to_push
-#  │    │    │    └── <used variables>
-#  │    │    │         ├── AUTOTEST_ROOT
-#  │    │    │         ├── CI_COMMIT_REF_SLUG
-#  │    │    │         ├── CI_PROJECT_DIR
-#  │    │    │         ├── CI_PIPELINE_URL
-#  │    │    │         ├── AUTOTEST_COMMIT
-#  │    │    │         └── CI_MACHINE
-#  │    │    └── safe_create_rundir
-#  │    ├── custom-jobs-and-variables.yml
-#  │    │    ├── <jobs>
-#  │    │    │    ├── .custom_job
-#  │    │    │    ├── .reproducer_vars
-#  │    │    │    ├── .report_job_success
-#  │    │    │    └── .report_job_failure
-#  │    │    ├── <used files>
-#  │    │    │    ├── tests/gitlab/build_and_test  [in JOB_CMD]
-#  │    │    │    └── .gitlab/scripts/report_build_and_test [by .report_job_*]
-#  │    │    ├── <defined variables>
-#  │    │    │    ├── JOB_CMD
-#  │    │    │    ├── BUILD_ROOT
-#  │    │    │    ├── ALLOC_NAME
-#  │    │    │    ├── TPLS_REPO
-#  │    │    │    ├── TESTS_REPO
-#  │    │    │    ├── AUTOTEST_REPO
-#  │    │    │    ├── MFEM_DATA_REPO
-#  │    │    │    ├── ARTIFACTS_DIR: artifacts
-#  │    │    │    ├── SLURM_OVERLAP: 1
-#  │    │    │    ├── DANE_SHARED_ALLOC
-#  │    │    │    ├── DANE_JOB_ALLOC
-#  │    │    │    ├── TIOGA_SHARED_ALLOC
-#  │    │    │    ├── TIOGA_JOB_ALLOC
-#  │    │    │    └── MATRIX_JOB_ALLOC
-#  │    │    └── <used variables>
-#  │    │         ├── SPEC
-#  │    │         ├── BUILD_ROOT
-#  │    │         └── ...
-#  │    ├── dane-baseline.yml
-#  │    │    ├── <jobs>
-#  │    │    │    ├── .on_dane
-#  │    │    │    ├── baselinecheck_mfem_intel_dane
-#  │    │    │    ├── cleanup
-#  │    │    │    ├── report_baseline
-#  │    │    │    └── baselinepublish_mfem_dane
-#  │    │    ├── <included and used files>
-#  │    │    │    ├── .gitlab/custom-jobs-and-variables.yml
-#  │    │    │    ├── .gitlab/configs/setup-baseline.yml
-#  │    │    │    ├── .gitlab/scripts/rebaseline
-#  │    │    │    ├── .gitlab/scripts/baseline
-#  │    │    │    └── .gitlab/scripts/git_try_to_push
-#  │    │    ├── <defined variables>
-#  │    │    │    ├── BASELINE_TEST: baseline
-#  │    │    │    ├── MACHINE_NAME: dane
-#  │    │    │    ├── TPLS_DIR
-#  │    │    │    └── export MFEM_TEST_NP
-#  │    │    └── <used variables>
-#  │    │         ├── ON_DANE
-#  │    │         ├── AUTOTEST       [defined by .gitlab-ci.yml]
-#  │    │         ├── BUILD_ROOT     [defined by custom-jobs-and-variables.yml]
-#  │    │         ├── TPLS_DIR       [defined by this file]
-#  │    │         ├── ARTIFACTS_DIR  [defined by custom-jobs-and-variables.yml]
-#  │    │         ├── MACHINE_NAME   [defined by this file]
-#  │    │         ├── AUTOTEST_COMMIT  [defined by .gitlab-ci.yml]
-#  │    │         ├── AUTOTEST_ROOT  [defined by .gitlab-ci.yml]
-#  │    │         ├── BASELINE_TEST  [defined by this file]
-#  │    │         └── REBASELINE     [defined by .gitlab-ci.yml]
-#  │    └── subscribed-pipelines.yml
-#  │         ├── <jobs>
-#  │         │    ├── .machine-check
-#  │         │    ├── generate-job-lists
-#  │         │    ├── dane-up-check
-#  │         │    ├── dane-build-and-test
-#  │         │    ├── dane-baseline
-#  │         │    ├── tioga-up-check
-#  │         │    ├── tioga-build-and-test
-#  │         │    ├── matrix-up-check
-#  │         │    └── matrix-build-and-test
-#  │         ├── <used jobs>
-#  │         │    └── .build-and-test  [from ".gitlab-ci.yml"]
-#  │         ├── <included files>
-#  │         │    └── .gitlab/dane-baseline.yml  [by "dane-baseline"]
-#  │         └── <used variables>
-#  │              ├── GITHUB_PROJECT_ORG
-#  │              ├── GITHUB_PROJECT_NAME
-#  │              ├── AUTOTEST
-#  │              ├── AUTOTEST_COMMIT
-#  │              └── REBASELINE
-#  └── tests
-#       ├── gitlab
-#       │    ├── build_and_test
-#       │    │    ├── <builds and tests a given MFEM spec with uberenv>
-#       │    │    ├── <used files>
-#       │    │    │    ├── tests/uberenv/uberenv.py        [deps mode, cloned]
-#       │    │    │    └── tests/gitlab/get_mfem_uberenv   [deps mode]
-#       │    │    └── <used variables>
-#       │    │         ├── SYS_TYPE
-#       │    │         ├── THREADS      [num. parallel jobs to build MFEM]
-#       │    │         ├── MODULE_LIST  [modules to load]
-#       │    │         ├── CI_JOB_ID
-#       │    │         ├── USE_DEV_SHM
-#       │    │         ├── SPACK_DEBUG
-#       │    │         ├── DEBUG_MODE
-#       │    │         ├── REGISTRY_TOKEN
-#       │    │         ├── CI_REGISTRY_USER  (defined by Gitlab)
-#       │    │         ├── USER
-#       │    │         ├── CI_REGISTRY_IMAGE (defined by Gitlab)
-#       │    │         └── CI_JOB_TOKEN      (defined by Gitlab)
-#       │    ├── build_and_test_setup
-#       │    │    ├── <updates MFEM_DATA_REPO and AUTOTEST_REPO using locks>
-#       │    │    └── <used variables>
-#       │    │         ├── MFEM_DATA_REPO
-#       │    │         ├── SHARED_REPOS_DIR
-#       │    │         ├── AUTOTEST_REPO
-#       │    │         └── AUTOTEST_ROOT
-#       │    └── get_mfem_uberenv
-#       │         ├── <github.com/mfem/mfem-uberenv.git -> tests/uberenv>
-#       │         └── <defines the uberenv hash to use>
-#       └── uberenv  [cloned by tests/gitlab/get_mfem_uberenv]
-#            └── uberenv.py
-#
-# <root of radiuss/radiuss-shared-ci, ref: 'v2025.09.1'>
-#  └── pipelines
-#       ├── matrix.yml
-#       │    ├── <jobs>
-#       │    │    ├── .on_matrix
-#       │    │    ├── .matrix_reproducer_init
-#       │    │    ├── .matrix_reproducer_vars
-#       │    │    ├── .matrix_reproducer_job
-#       │    │    ├── .matrix_job_command
-#       │    │    └── .job_on_matrix
-#       │    ├── <used jobs>
-#       │    │    └── .custom_job  [from .gitlab/custom-jobs-and-variables.yml]
-#       │    └── <used variables>
-#       │         ├── ON_MATRIX
-#       │         ├── ADVANCED_JOB
-#       │         ├── ALL_TARGETS
-#       │         ├── SYS_TYPE
-#       │         ├── LLNL_SERVICE_USER
-#       │         ├── USER
-#       │         ├── GITHUB_PROJECT_NAME
-#       │         ├── GITHUB_PROJECT_ORG
-#       │         ├── MATRIX_JOB_ALLOC
-#       │         └── JOB_CMD
-#       ├── dane.yml
-#       │    ├── <jobs>
-#       │    │    ├── .on_dane
-#       │    │    ├── .dane_reproducer_init
-#       │    │    ├── .dane_reproducer_vars
-#       │    │    ├── .dane_reproducer_job
-#       │    │    ├── .dane_job_command
-#       │    │    ├── .job_on_dane
-#       │    │    ├── allocate_resources
-#       │    │    └── release_resources
-#       │    ├── <used jobs>
-#       │    │    └── .custom_job  [from .gitlab/custom-jobs-and-variables.yml]
-#       │    ├── <defined variables>
-#       │    │    └── export JOBID
-#       │    └── <used variables>
-#       │         ├── ON_DANE
-#       │         ├── ADVANCED_JOB
-#       │         ├── ALL_TARGETS
-#       │         ├── SYS_TYPE
-#       │         ├── LLNL_SERVICE_USER
-#       │         ├── USER
-#       │         ├── GITHUB_PROJECT_NAME
-#       │         ├── GITHUB_PROJECT_ORG
-#       │         ├── DANE_JOB_ALLOC
-#       │         ├── JOB_CMD
-#       │         ├── JOBID
-#       │         ├── ALLOC_NAME
-#       │         └── DANE_SHARED_ALLOC
-#       └── tioga.yml
-#            ├── <jobs>
-#            │    ├── .on_tioga
-#            │    ├── .tioga_reproducer_init
-#            │    ├── .tioga_reproducer_vars
-#            │    ├── .tioga_reproducer_job
-#            │    ├── .tioga_job_command
-#            │    ├── .job_on_tioga
-#            │    ├── allocate_resources
-#            │    └── release_resources
-#            ├── <used jobs>
-#            │    └── .custom_job  [from .gitlab/custom-jobs-and-variables.yml]
-#            ├── <defined variables>
-#            │    └── PROXY
-#            └── <used variables>
-#                 ├── ON_TIOGA
-#                 ├── ADVANCED_JOB
-#                 ├── ALL_TARGETS
-#                 ├── SYS_TYPE
-#                 ├── LLNL_SERVICE_USER
-#                 ├── USER
-#                 ├── GITHUB_PROJECT_NAME
-#                 ├── GITHUB_PROJECT_ORG
-#                 ├── TIOGA_JOB_ALLOC
-#                 ├── JOB_CMD
-#                 ├── PROXY
-#                 ├── ALLOC_NAME
-#                 └── TIOGA_SHARED_ALLOC
+# at Lawrence Livermore National Laboratory (LLNL). This entire pipeline is
+# LLNL-specific!
+
+include:
+  - project: 'lc-templates/id_tokens'
+    file: 'id_tokens.yml'
+
+# The pipeline is divided into stages. Usually, jobs in a given stage wait for
+# the preceding stages to complete before to start. However, we sometimes use
+# the "needs" keyword and express the DAG of jobs for more efficiency.
+# - We use setup and setup_baseline phases to download content outside of mfem
+#   directory.
+# - Allocate/Release is where Dane resource are allocated/released once for all.
+# - Build and Test is where we build and MFEM for multiple toolchains.
+# - Baseline_checks gathers baseline-type test suites execution
+# - Baseline_publish, only available on master, allows to update baseline
+#   results
+stages:
+  - sub-pipelines
 
-###############################################################################
-# We define the following GitLab pipeline variables:
 variables:
-##### LC GITLAB CONFIGURATION
   CUSTOM_CI_BUILDS_DIR: "/usr/workspace/mfem/gitlab-runner"
-
-##### PROJECT VARIABLES
   USER_CI_TOP_DIR: "${CUSTOM_CI_BUILDS_DIR}/${GITLAB_USER_LOGIN}"
   SHARED_REPOS_DIR: "${USER_CI_TOP_DIR}/repos"
   AUTOTEST_ROOT: "${SHARED_REPOS_DIR}"
+  # MFEM_DATA_DIR is setup in '.gitlab/configs/setup-build-and-test.yml' and
+  # used in '.gitlab/configs/<machine>-config.yml':
   MFEM_DATA_DIR: "${SHARED_REPOS_DIR}/mfem-data"
-# AUTOTEST: enable (ON/YES) or disable (any other value) test reporting. See
-# also AUTOTEST_COMMIT.
-  AUTOTEST: "OFF"
-# AUTOTEST_COMMIT: used only when AUTOTEST is set to ON/YES.
-# * If AUTOTEST_COMMIT is set to ON/YES, reporting jobs will commit their
-#   files to the MFEM/autotest repo.
-# * If AUTOTEST_COMMIT is NOT set to ON/YES, reporting jobs will NOT commit
-#   their files to the MFEM/autotest repo. Instead they will just show the
-#   contents of the report files and remove them.
-  AUTOTEST_COMMIT: "ON"
-# REBASELINE:
+
 # Defines the default choice for updating the saved baseline results. By default
 # the baseline can only be updated from the master branch. This variable offers
 # the option to manually ask for rebaselining from another branch if necessary.
-  REBASELINE: "OFF"
+  REBASELINE: "NO"
+  AUTOTEST: "NO"
+  # AUTOTEST_COMMIT: used only when AUTOTEST is set to YES.
+  # * If AUTOTEST_COMMIT is NOT set to NO, reporting jobs will commit their
+  #   files to the MFEM/autotest repo.
+  # * If AUTOTEST_COMMIT is set to NO, reporting jobs will NOT commit their
+  #   files to the MFEM/autotest repo. Instead they will just show the contents
+  #   of the report files and remove them.
+  AUTOTEST_COMMIT: "YES"
 
-##### SHARED_CI CONFIGURATION
-# Required information about GitHub repository
-  GITHUB_PROJECT_NAME: "mfem"
-  GITHUB_PROJECT_ORG: "MFEM"
-# Override the pattern describing branches that will skip the "draft PR filter
-# test".  Add protected branches here. See default value in
-# preliminary-ignore-draft-pr.yml.
-#  ALWAYS_RUN_PATTERN: ""
-
-###############################################################################
-##### High level stages
-# We organize the test-pipelines stage with sub-pipelines. Each sub-pipeline
-# corresponds to a test batch on a given machine.
-stages:
-  - prerequisites
-  - test-pipelines
-
-###############################################################################
-# Template for jobs triggering a build-and-test sub-pipeline:
-.build-and-test:
-  stage: test-pipelines
+# Trigger subpipelines:
+dane-build-and-test:
+  stage: sub-pipelines
   variables:
-    # Explicitly pass down values that are not always propagated to child
-    # pipelines, e.g. when a variable is set in the "Settings -> CI" web
-    # interface (project variables).
-    # Note: in some cases, this does not work as expected, e.g. when the
-    # variable is not re-defined in the web interface; in such cases, the child
-    # pipeline gets a definition like '${AUTOTEST}', i.e. it behaves as if
-    # AUTOTEST is undefined, even though there is a default value in
-    # .gitlab-ci.yml.
+    # Explicitly pass down values that we want to be able to set when triggering
+    # pipelines manually or using scheduling
     AUTOTEST: "${AUTOTEST}"
     AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
   trigger:
-    include:
-      - local: '.gitlab/custom-jobs-and-variables.yml'
-      - project: 'radiuss/radiuss-shared-ci'
-        ref: 'v2025.09.1'
-        file: 'pipelines/${CI_MACHINE}.yml'
-      - artifact: '${CI_MACHINE}-jobs.yml'
-        job: 'generate-job-lists'
+    include: .gitlab/dane-build-and-test.yml
     strategy: depend
-    forward:
-      pipeline_variables: true
 
-###############################################################################
-include:
-  # Sets ID tokens for every job using `default:`
-  - project: 'lc-templates/id_tokens'
-    file: 'id_tokens.yml'
-  # [Optional] checks preliminary to running the actual CI test
-  #- project: 'radiuss/radiuss-shared-ci'
-  #  ref: 'v2025.09.1'
-  #  file: 'preliminary-ignore-draft-pr.yml'
-  # pipelines subscribed by the project
-  - local: '.gitlab/subscribed-pipelines.yml'
+dane-baseline:
+  stage: sub-pipelines
+  variables:
+    # Explicitly pass down values that we want to be able to set when triggering
+    # pipelines manually or using scheduling
+    REBASELINE: "${REBASELINE}"
+    AUTOTEST: "${AUTOTEST}"
+    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
+  trigger:
+    include: .gitlab/dane-baseline.yml
+    strategy: depend
+
+lassen-build-and-test:
+  stage: sub-pipelines
+  variables:
+    # Explicitly pass down values that we want to be able to set when triggering
+    # pipelines manually or using scheduling
+    AUTOTEST: "${AUTOTEST}"
+    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
+  trigger:
+    include: .gitlab/lassen-build-and-test.yml
+    strategy: depend
+
+corona-build-and-test:
+  stage: sub-pipelines
+  variables:
+    # Explicitly pass down values that we want to be able to set when triggering
+    # pipelines manually or using scheduling
+    AUTOTEST: "${AUTOTEST}"
+    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
+  trigger:
+    include: .gitlab/corona-build-and-test.yml
+    strategy: depend

.gitlab/README.md

@@ -15,9 +15,9 @@ and nightly testing on GitLab.
 
 ## Top level
 
-The root configuration file is `.gitlab-ci.yml` at the root of MFEM repo. This
-file only defines three stages, a prerequisites one, and two main stages in
-which we trigger several sub-pipelines.
+The root configuration file is `.gitlab-ci.yml` at the root of MFEM repo.
+This file only defines one stage, in which we trigger several
+sub-pipelines.
 
 We use sub-pipelines to isolate the test for one combination of `machine`
 and `test type`.
@@ -25,8 +25,8 @@ and `test type`.
 Machines typically include:
 
 * Dane: Intel Sapphire Rapids
-* Matrix: Intel Sapphire Rapids + Nvidia H100 GPU
-* Tioga: AMD MI250X GPU
+* Lassen: Power9 + Nvidia GPU
+* Corona: AMD GPU
 
 Test types include:
 
@@ -39,31 +39,9 @@ altering the scheduling, execution and displaying of the others.
 
 ## Sub-pipelines
 
-### build-and-test
-
-The build-and-test sub-pipelines leverage RADIUSS Shared CI to share most of
-the CI implementation. RADIUSS Shared CI provides a shared CI infrastructure
-vetted on most LC systems of interest and efficiently leveraging each machine
-scheduler to increase CI throughput. The maintenance of RADIUSS Shared CI is
-shared among several RADIUSS projects.
-
-Jobs for the build-and-test sub-pipelines are defined in the jobs directory.
-Because build-and-test jobs leverage Uberenv and Spack to build the
-dependencies automatically, the jobs essentially consists in a `spack spec`
-defined in the jobs files, and some scheduling parameters defined in the
-`.gitlab/custom-jobs-and-variables.yml` file.
-
-Build-and-test jobs all run the `tests/gitlab/build_and_test` script.
-
-The build-and-test pipelines are controlled by the
-`.gitlab/subscribed-pipelines.yml` which defines which machines to run on and
-implements additional features like machine availability check, and job list
-generation.
-
-### baseline
-
-Baseline sub-pipelines are described by files with names reflecting the
-machine it runs on, e.g. `dane-baseline`.
+Each file is this directory is the root configuration file for one
+sub-pipeline. The naming reflects the corresponding couple (`machine`,
+`test_type`).
 
 Those files define the *stages* and the *jobs* for the sub-pipeline. They
 also contain any configuration that cannot be shared. For the most part
@@ -85,11 +63,11 @@ usage function. This should be improved.
 
 # More testing
 
-## Adding a new target to a build-and-test pipeline
+## Adding a new target to a build_and_test pipeline
 
-`build-and-test` pipelines rely on Spack to install dependencies. Spack is
+`build_and_test` pipelines rely on Spack to install dependencies. Spack is
 driven by Uberenv which helps freezing Spack configuration: the goal being to
-point to a specific commit in Spack and isolate its configuration so that it is
+point to specific commit in Spack and isolate its configuration so that it is
 not influenced by the user environment. More documentation about this can be
 found in `tests/gitlab`.
 
@@ -104,7 +82,7 @@ spack spec to use. Adding a job on Dane for example resumes to:
 <job_name>:
   variables:
     SPEC: "<spack_spec>"
-  extends: .job_on_dane
+  extends: .build_and_test_on_dane
 ```
 
 The remaining and non trivial work is to make sure this spec is working. To

.gitlab/configs/common.yml

@@ -0,0 +1,40 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+include:
+  - project: 'lc-templates/id_tokens'
+    file: 'id_tokens.yml'
+
+# We define the following GitLab pipeline variables:
+variables:
+
+# The path to the shared resource between all jobs. For example, external
+# repositories like 'tests' and 'tpls' are cloned here. Also, 'tpls' is built
+# once for all targets, so that build happen here. The BUILD_ROOT is unique to
+# the pipeline, preventing any form of concurrency with other pipelines. This
+# also means that the BUILD_ROOT directory will never be cleaned.
+# TODO: add a clean-up mechanism
+  BUILD_ROOT: ${USER_CI_TOP_DIR}/${CI_PROJECT_NAME}-${MACHINE_NAME}-pipeline-${CI_PIPELINE_ID}
+
+# On LLNL's Dane, there is only one allocation shared among jobs in order to
+# save time and resource. This allocation has to be uniquely named so that we
+# are sure to retrieve it.
+  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
+
+# Git repositories used in the pipeline
+  TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
+  TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
+  AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
+  MFEM_DATA_REPO: https://github.com/mfem/data.git
+
+# Directory used to place artifacts.
+  ARTIFACTS_DIR: artifacts
+  SLURM_OVERLAP: 1

.gitlab/configs/corona-config.yml

@@ -0,0 +1,59 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# GitLab pipeline configuration for the Corona machine at LLNL
+variables:
+  MACHINE_NAME: corona
+
+.on_corona:
+  tags:
+    - shell
+    - corona
+  rules:
+    # Don't run corona jobs if...
+    # Note: This makes corona an "opt-in" machine. To activate builds on corona
+    # for a given GitLab clone of MFEM, go to Setting/CI-CD/variables, and set
+    # "ON_CORONA" to "ON". An LC account on for corona is required to trigger a
+    # pipeline there.
+    - if: '$CI_COMMIT_BRANCH =~ /_cnone/ || $ON_CORONA != "ON"'
+      when: never
+    # Don't run autotest update if...
+    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
+      when: never
+    # Report success on success status
+    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
+      when: on_success
+    # Report failure on failure status
+    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
+      when: on_failure
+    # Always release resource
+    - if: '$CI_JOB_NAME =~ /release_resource/'
+      when: always
+    # Always cleanup
+    - if: '$CI_JOB_NAME =~ /cleanup/'
+      when: always
+    # Default is to run if previous stage succeeded
+    - when: on_success
+
+# Spack helped builds
+# Generic corona build job, extending build script
+.build_and_test_on_corona:
+  extends: [.on_corona]
+  stage: build_and_test
+  script:
+    # THREADS is used by 'tests/gitlab/build_and_test', run below
+    - export THREADS=12
+    - echo ${ALLOC_NAME}
+    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
+    - echo ${JOBID}
+    - echo ${MFEM_DATA_DIR}
+    - echo ${SPEC}
+    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 15 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data

.gitlab/configs/dane-config.yml

@@ -0,0 +1,56 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# GitLab pipelines configurations for the Dane machine at LLNL
+variables:
+  MACHINE_NAME: dane
+
+.on_dane:
+  tags:
+    - shell
+    - dane
+  rules:
+    # Don't run dane jobs if...
+    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_DANE == "OFF"'
+      when: never
+    # Don't run autotest update if...
+    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
+      when: never
+    # Report success on success status
+    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
+      when: on_success
+    # Report failure on failure status
+    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
+      when: on_failure
+    # Always release resource
+    - if: '$CI_JOB_NAME =~ /release_resource/'
+      when: always
+    # Always cleanup
+    - if: '$CI_JOB_NAME =~ /cleanup/'
+      when: always
+    # Default is to run if previous stage succeeded
+    - when: on_success
+
+# Spack helped builds
+# Generic dane build job, extending build script
+.build_and_test_on_dane:
+  extends: [.on_dane]
+  stage: build_and_test
+  script:
+    # THREADS is used by 'tests/gitlab/build_and_test', run below
+    # Dane has 224 threads/node and we run 7 separate jobs: 224=7*32
+    - export THREADS=28
+    - echo ${ALLOC_NAME}
+    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
+    - echo ${JOBID}
+    - echo ${MFEM_DATA_DIR}
+    - echo ${SPEC}
+    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) --reservation=ci -t 60 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data

.gitlab/configs/lassen-config.yml

@@ -0,0 +1,48 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# GitLab pipelines configurations for the Lassen machine at LLNL
+variables:
+  MACHINE_NAME: lassen
+
+.on_lassen:
+  tags:
+    - shell
+    - lassen
+  rules:
+    - if: '$CI_COMMIT_BRANCH =~ /_lnone/ || $ON_LASSEN == "OFF"' #run except if ...
+      when: never
+    # Don't run autotest update if...
+    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
+      when: never
+    # Report success on success status
+    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
+      when: on_success
+    # Report failure on failure status
+    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
+      when: on_failure
+    # Always cleanup
+    - if: '$CI_JOB_NAME =~ /cleanup/'
+      when: always
+    - when: on_success
+
+# Lassen uses a different job scheduler (spectrum lsf) that does not allow
+# pre-allocation the same way slurm does.  We use the pci queue on lassen
+# to speed-up the allocation.
+.build_and_test_on_lassen:
+  extends: [.on_lassen]
+  stage: build_and_test
+  script:
+    - echo ${MFEM_DATA_DIR}
+    - echo ${SPEC}
+    # Next script uses 'THREADS': leaving it empty --> it uses 'make all -j'
+    - lalloc 1 -W 45 -q pci --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
+  needs: [setup]

.gitlab/configs/report-build-and-test.yml

@@ -0,0 +1,77 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Jobs report
+.report_job_success:
+  script:
+    - echo ${MACHINE_NAME}
+    - echo ${AUTOTEST}
+    - echo ${AUTOTEST_COMMIT}
+    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
+    - cd ${AUTOTEST_ROOT}
+    - |
+      (
+        date
+        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
+        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
+        # every 5 seconds; we may want to add a counter for the number of
+        # retries to interrupt a potential infinite loop
+        while ! flock -n 9; do
+          sleep 5
+        done
+        echo "Acquired lock on '$PWD/autotest.lock'"
+        date
+        # Report SUCCESS while holding the file lock on 'autotest.lock'.
+        # The next script uses the following environment variables:
+        # - MACHINE_NAME, AUTOTEST_ROOT, AUTOTEST_COMMIT
+        # - CI_COMMIT_REF_SLUG, CI_PROJECT_DIR, CI_PIPELINE_URL
+        # It also calls the script '.gitlab/scripts/safe_create_rundir'.
+        ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test_success
+        err=$?
+        # sleep for a period to allow NFS to propagate the above changes;
+        # clearly, there is no guarantee that other NFS clients will see the
+        # changes even after the timeout
+        sleep 10
+        exit $err
+      ) 9> autotest.lock
+
+.report_job_failure:
+  script:
+    - echo ${MACHINE_NAME}
+    - echo ${AUTOTEST}
+    - echo ${AUTOTEST_COMMIT}
+    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
+    - cd ${AUTOTEST_ROOT}
+    - |
+      (
+        date
+        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
+        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
+        # every 5 seconds; we may want to add a counter for the number of
+        # retries to interrupt a potential infinite loop
+        while ! flock -n 9; do
+          sleep 5
+        done
+        echo "Acquired lock on '$PWD/autotest.lock'"
+        date
+        # Report FAILURE while holding the file lock on 'autotest.lock'.
+        # The next script uses the following environment variables:
+        # - MACHINE_NAME, AUTOTEST_ROOT, AUTOTEST_COMMIT
+        # - CI_COMMIT_REF_SLUG, CI_PROJECT_DIR, CI_PIPELINE_URL
+        # It also calls the script '.gitlab/scripts/safe_create_rundir'.
+        ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test_failure
+        err=$?
+        # sleep for a period to allow NFS to propagate the above changes;
+        # clearly, there is no guarantee that other NFS clients will see the
+        # changes even after the timeout
+        sleep 10
+        exit $err
+      ) 9> autotest.lock

.gitlab/configs/setup-build-and-test.yml

@@ -0,0 +1,90 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test
+# script then symlinks the repo to the parent directory of the MFEM source
+# directory. Unit tests that depend on the mfem/data repo will then detect that
+# this directory is present and be enabled.
+setup:
+  tags:
+    - shell
+    - dane
+  stage: setup
+  variables:
+    GIT_STRATEGY: none
+  script:
+    #
+    # Setup MFEM_DATA_DIR=${SHARED_REPOS_DIR}/mfem-data, see '.gitlab-ci.yml'
+    # and '.gitlab/configs/<machine>-config.yml'
+    #
+    - echo "MACHINE_NAME = ${MACHINE_NAME}"
+    - echo "AUTOTEST = ${AUTOTEST}"
+    - echo "AUTOTEST_COMMIT = ${AUTOTEST_COMMIT}"
+    - echo "SHARED_REPOS_DIR ${SHARED_REPOS_DIR}"
+    - mkdir -p ${SHARED_REPOS_DIR} && cd ${SHARED_REPOS_DIR}
+    - command -v flock || echo "Required command 'flock' not found"
+    - |
+      (
+        date
+        echo "Waiting to acquire lock on '$PWD/mfem-data.lock' ..."
+        # try to get an exclusive lock on fd 9 (mfem-data.lock) repeating the
+        # try every 5 seconds; we may want to add a counter for the number of
+        # retries to interrupt a potential infinite loop
+        while ! flock -n 9; do
+          sleep 5
+        done
+        echo "Acquired lock on '$PWD/mfem-data.lock'"
+        date
+        # clone/update the mfem/data repo while holding the file lock on
+        # 'mfem-data.lock'
+        err=0
+        if [[ ! -d "mfem-data" ]]; then
+          git clone ${MFEM_DATA_REPO} "mfem-data"
+        else
+          cd "mfem-data" && git pull && cd ..
+        fi || err=1
+        # sleep for a period to allow NFS to propagate the above changes;
+        # clearly, there is no guarantee that other NFS clients will see the
+        # changes even after the timeout
+        sleep 10
+        exit $err
+      ) 9> mfem-data.lock
+    #
+    # Setup ${AUTOTEST_ROOT}/autotest:
+    #
+    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
+    - mkdir -p ${AUTOTEST_ROOT} && cd ${AUTOTEST_ROOT}
+    - |
+      (
+        date
+        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
+        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
+        # every 5 seconds; we may want to add a counter for the number of
+        # retries to interrupt a potential infinite loop
+        while ! flock -n 9; do
+          sleep 5
+        done
+        echo "Acquired lock on '$PWD/autotest.lock'"
+        date
+        # clone/update the autotest repo while holding the file lock on
+        # 'autotest.lock'
+        err=0
+        if [[ ! -d "autotest" ]]; then
+          git clone ${AUTOTEST_REPO}
+        else
+          cd autotest && git pull && cd ..
+        fi || err=1
+        # sleep for a period to allow NFS to propagate the above changes;
+        # clearly, there is no guarantee that other NFS clients will see the
+        # changes even after the timeout
+        sleep 10
+        exit $err
+      ) 9> autotest.lock

.gitlab/corona-build-and-test.yml

@@ -0,0 +1,67 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+stages:
+  - setup
+  - allocate_resource
+  - build_and_test
+  - release_resource_and_report
+
+# Slurm shared allocation
+allocate_resource:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_corona
+  stage: allocate_resource
+  script:
+    - echo ${ALLOC_NAME}
+    - salloc --exclusive --nodes=1 --partition=mi60 --time=45 --no-shell --job-name=${ALLOC_NAME}
+  timeout: 6h
+  needs: [setup]
+
+# Build and test jobs, simply provide a spec
+rocm_gcc_8.3.1:
+  variables:
+    SPEC: "@develop%gcc@8.3.1+rocm amdgpu_target=gfx906"
+  extends: .build_and_test_on_corona
+  needs: [allocate_resource]
+
+# Release slurm allocation
+release_resource:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_corona
+  stage: release_resource_and_report
+  script:
+    - echo ${ALLOC_NAME}
+    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
+    - echo ${JOBID}
+    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
+  needs: [rocm_gcc_8.3.1]
+
+# Jobs report
+report_job_success:
+  stage: release_resource_and_report
+  extends:
+    - .on_corona
+    - .report_job_success
+
+report_job_failure:
+  stage: release_resource_and_report
+  extends:
+    - .on_corona
+    - .report_job_failure
+
+include:
+  - local: .gitlab/configs/common.yml
+  - local: .gitlab/configs/corona-config.yml
+  - local: .gitlab/configs/setup-build-and-test.yml
+  - local: .gitlab/configs/report-build-and-test.yml

.gitlab/custom-jobs-and-variables.yml

@@ -1,132 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-include:
-  - project: 'lc-templates/id_tokens'
-    file: 'id_tokens.yml'
-
-# We define the following GitLab pipeline variables:
-variables:
-# Set the build-and-test command.
-# Nested variables are allowed and useful to customize the job command. We
-# protect variables with quotes so that their value may remain a string even if
-# they contain whitespaces.
-  JOB_CMD:
-    value: tests/gitlab/build_and_test --spec \"${SPEC}\" --data-dir ${MFEM_DATA_DIR} --data
-# The path to the shared resource between all jobs in the 'dane-baseline'
-# pipeline. For example, external repositories like 'tests' and 'tpls' are
-# cloned here. Also, 'tpls' is built once for all targets, so that build happens
-# here. The BUILD_ROOT is unique to the pipeline, preventing any form of
-# concurrency with other pipelines. This directory is removed by the 'cleanup'
-# stage in the 'dane-baseline' pipeline.
-  BUILD_ROOT: ${USER_CI_TOP_DIR}/${CI_PROJECT_NAME}-${CI_MACHINE}-pipeline-${CI_PIPELINE_ID}
-
-# On LLNL's dane and tioga, the 'build-and-test' pipelines creates only one
-# allocation shared among jobs in the pipeline in order to save time and
-# resources. This allocation has to be uniquely named so that we are sure to
-# retrieve it and avoid collisions.
-  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
-
-# Git repositories used in the pipelines:
-# - TPLS_REPO and TESTS_REPO are used only by the 'dane-baseline' pipeline
-# - AUTOTEST_REPO is used by all pipelines
-# - MFEM_DATA_REPO is used only by the 'build-and-test' pipelines
-  TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
-  TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
-  AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
-  MFEM_DATA_REPO: https://github.com/mfem/data.git
-
-# Directory used to place artifacts:
-# - ARTIFACTS_DIR is only used by the 'dane-baseline' pipeline
-  ARTIFACTS_DIR: artifacts
-  SLURM_OVERLAP: 1
-
-# Dane
-# Arguments for top level allocation
-  DANE_SHARED_ALLOC: "--exclusive --reservation=ci --time=60 --nodes=1"
-# Arguments for job level allocation
-# Note: We repeat the reservation, necessary when jobs are manually re-triggered.
-  DANE_JOB_ALLOC: "--reservation=ci --overlap --nodes=1"
-
-# Tioga
-# Arguments for top level allocation
-  TIOGA_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=45m --nodes=1"
-# Arguments for job level allocation
-  TIOGA_JOB_ALLOC: "--nodes=1 --begin-time=+5s"
-
-# Matrix
-# Arguments for top level allocation
-  MATRIX_SHARED_ALLOC: "-p pdebug --exclusive --time=45 --nodes=1 -G 4"
-# Arguments for job level allocation
-# Note: We repeat the reservation, necessary when jobs are manually re-triggered.
-  MATRIX_JOB_ALLOC: "--overlap --nodes=1"
-
-# Configuration shared by build and test jobs specific to this project.
-# Not all configuration can be shared. Here projects can fine tune the
-# CI behavior.
-# See Umpire for an example (export junit test reports).
-.custom_job:
-  artifacts:
-    reports:
-
-# Note: this part is not used by the 'dane-baseline' pipeline.
-# FIXME: BUILD_ROOT, TPLS_REPO, TESTS_REPO are not needed here.
-#        Also, the definition of SHARED_REPOS_DIR is wrong.
-.reproducer_vars:
-  script:
-    - |
-      echo -e "
-      # Variables \n
-      export SPEC=\"${SPEC//\"/\\\"}\" \n
-      # Directories \n
-      export BUILD_ROOT=\"\${working_dir}\" \n
-      export SHARED_REPOS_DIR=\"\${BUILD_ROOT}/..\" \n
-      export MFEM_DATA_DIR=\"\${SHARED_REPOS_DIR}/mfem-data\" \n
-      # Repositories \n
-      export TPLS_REPO=\"${TPLS_REPO//\"/\\\"}\" \n
-      export TESTS_REPO=\"${TESTS_REPO//\"/\\\"}\" \n
-      export AUTOTEST_REPO=\"${AUTOTEST_REPO//\"/\\\"}\" \n
-      export MFEM_DATA_REPO=\"${MFEM_DATA_REPO//\"/\\\"}\" \n
-      # Setup directories \n
-      ./tests/gitlab/build_and_test_setup \n
-      # Using the CI build cache is optional and requires a token. Set it like so: \n
-      # export REGISTRY_TOKEN=\"<your token here>\" \n"
-      #
-
-# Jobs report
-.report_job_success:
-  script:
-    - ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test SUCCESS
-  rules:
-    - when: on_success
-
-.report_job_failure:
-  script:
-    - ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test FAILURE
-  rules:
-    - when: on_failure
-
-# Keep the following for debugging purposes: renaming this job from
-# '.show_variables' to 'show_variables' will insert this debug job at the
-# beginning of all child pipelines.
-.show_variables:
-  tags: [shell, oslic]
-  variables:
-    GIT_STRATEGY: none
-  stage: .pre
-  script:
-    - |
-      echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
-      echo "AUTOTEST=${AUTOTEST}"
-      echo "AUTOTEST_COMMIT=${AUTOTEST_COMMIT}"
-      echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
-      # Fail the job on purpose to prevent the rest of the pipeline from running
-      false

.gitlab/dane-baseline.yml

@@ -11,7 +11,6 @@
 
 variables:
   BASELINE_TEST: baseline
-  MACHINE_NAME: dane
 
 stages:
   - setup
@@ -20,25 +19,6 @@ stages:
   - cleanup
   - baseline_publish
 
-.on_dane:
-  tags:
-    - shell
-    - dane
-  rules:
-    # Don't run dane jobs if...
-    - if: '$ON_DANE == "OFF"'
-      when: never
-    # Don't run autotest update if...
-    # Note: in some cases, the content of AUTOTEST can be '${AUTOTEST}', so we
-    # need to treat that value as the default value of 'OFF'.
-    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "ON" && $AUTOTEST != "YES"'
-      when: never
-    # Always cleanup
-    - if: '$CI_JOB_NAME =~ /cleanup/'
-      when: always
-    # Default is to run if previous stage succeeded
-    - when: on_success
-
 baselinecheck_mfem_intel_dane:
   extends: [.on_dane]
   stage: baseline_check
@@ -49,9 +29,6 @@ baselinecheck_mfem_intel_dane:
     #       .gitlab/configs/setup-baseline.yml.
     TPLS_DIR: ${BUILD_ROOT}/tpls
   script:
-    - echo "AUTOTEST=$AUTOTEST"
-    - echo "AUTOTEST_COMMIT=$AUTOTEST_COMMIT"
-    - echo "AUTOTEST_ROOT=$AUTOTEST_ROOT"
     - echo ${BUILD_ROOT}
     - echo ${TPLS_DIR}
     # Used by the tests in MFEM/tests, dane has 224 threads/node:
@@ -112,13 +89,7 @@ report_baseline:
           cp ${rundir}/pipeline.txt ${rundir}/autotest-email.html
         fi
         msg="GitLab CI log for ${BASELINE_TEST} on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
-        # Note: in some cases, the content of AUTOTEST_COMMIT can be
-        # '${AUTOTEST_COMMIT}', so we need to treat that value as the default
-        # value of 'ON'.
-        if [[ "$AUTOTEST_COMMIT" == '${AUTOTEST_COMMIT}' ]]; then
-          AUTOTEST_COMMIT="ON"
-        fi
-        if [[ "$AUTOTEST_COMMIT" == "ON" || "$AUTOTEST_COMMIT" == "YES" ]]; then
+        if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
           git pull && \
           git add ${rundir} && \
           git commit -m "${msg}" && \
@@ -146,8 +117,8 @@ baselinepublish_mfem_dane:
   extends: [.on_dane]
   stage: baseline_publish
   rules:
-    # - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "ON"'
-    - if: '$REBASELINE == "ON"'
+    # - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "YES"'
+    - if: '$REBASELINE == "YES"'
       when: manual
   script:
     - echo ${BUILD_ROOT}
@@ -157,5 +128,6 @@ baselinepublish_mfem_dane:
     - .gitlab/scripts/rebaseline
 
 include:
-  - local: .gitlab/custom-jobs-and-variables.yml
+  - local: .gitlab/configs/common.yml
+  - local: .gitlab/configs/dane-config.yml
   - local: .gitlab/configs/setup-baseline.yml

.gitlab/dane-build-and-test.yml

@@ -0,0 +1,94 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+stages:
+  - setup
+  - allocate_resource
+  - build_and_test
+  - release_resource_and_report
+
+# Allocate
+allocate_resource:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_dane
+  stage: allocate_resource
+  script:
+    - echo ${ALLOC_NAME}
+    - salloc --exclusive --nodes=1 --reservation=ci --time=60 --no-shell --job-name=${ALLOC_NAME}
+  timeout: 6h
+
+# GitLab jobs for the Dane machine at LLNL
+debug_ser_gcc_10:
+  variables:
+    SPEC: "%gcc@10.3.1 +debug~mpi"
+  extends: .build_and_test_on_dane
+
+debug_par_gcc_10:
+  variables:
+    SPEC: "%gcc@10.3.1 +debug+mpi"
+  extends: .build_and_test_on_dane
+
+opt_ser_gcc_10:
+  variables:
+    SPEC: "%gcc@10.3.1 ~mpi"
+  extends: .build_and_test_on_dane
+
+opt_par_gcc_10:
+  variables:
+    SPEC: "%gcc@10.3.1"
+  extends: .build_and_test_on_dane
+
+opt_par_gcc_10_sundials:
+  variables:
+    SPEC: "%gcc@10.3.1 +sundials"
+  extends: .build_and_test_on_dane
+
+opt_par_gcc_10_petsc:
+  variables:
+    SPEC: "%gcc@10.3.1 +petsc ^petsc+mumps~superlu-dist"
+  extends: .build_and_test_on_dane
+
+opt_par_gcc_10_pumi:
+  variables:
+    SPEC: "%gcc@10.3.1 +pumi"
+  extends: .build_and_test_on_dane
+
+# Release
+release_resource:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_dane
+  stage: release_resource_and_report
+  script:
+    - echo ${ALLOC_NAME}
+    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
+    - echo ${JOBID}
+    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
+
+# Jobs report
+report_job_success:
+  stage: release_resource_and_report
+  extends:
+    - .on_dane
+    - .report_job_success
+
+report_job_failure:
+  stage: release_resource_and_report
+  extends:
+    - .on_dane
+    - .report_job_failure
+
+include:
+  - local: .gitlab/configs/common.yml
+  - local: .gitlab/configs/dane-config.yml
+  - local: .gitlab/configs/setup-build-and-test.yml
+  - local: .gitlab/configs/report-build-and-test.yml

.gitlab/jobs/dane-reports.yml

@@ -1,19 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Jobs report
-report_job_success:
-  extends: [.on_dane, .report_job_success]
-  stage: jobs-stage-3
-
-report_job_failure:
-  extends: [.on_dane, .report_job_failure]
-  stage: jobs-stage-3

.gitlab/jobs/dane.yml

@@ -1,92 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Override reproducer section to define MFEM specific variables.
-.dane_reproducer_vars:
-  script:
-    - !reference [.reproducer_vars, script]
-
-# TODO: Setup script should be defined as a bash script (but then GIT_STRATEGY
-#       cannot be "none" anymore).
-
-# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test
-# script then symlinks the repo to the parent directory of the MFEM source
-# directory. Unit tests that depend on the mfem/data repo will then detect that
-# this directory is present and be enabled.
-setup:
-  extends: .on_dane
-  stage: jobs-stage-1
-  script:
-    - ./tests/gitlab/build_and_test_setup
-
-
-########################
-# Overridden shared jobs
-########################
-# When using shared jobs, we can duplicate them here to override description and
-# add necessary changes.
-# We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS} So that
-# the comparison with the original job is easier.
-
-
-############
-# Extra jobs
-############
-# We do not recommend using ${PROJECT_<MACHINE>_VARIANTS} and
-# ${PROJECT_<MACHINE>_DEPS} in the extra jobs. There is not reason not to fully
-# describe the spec here.
-
-.mfem_job_on_dane:
-  extends: .job_on_dane
-  stage: jobs-stage-2
-  variables:
-    # Dane has 224 threads/node and we run 7 separate jobs: 224=7*32
-    THREADS: 28
-
-debug_ser_gcc_10:
-  extends: .mfem_job_on_dane
-  variables:
-    SPEC: "%gcc@10.3.1 +debug~mpi"
-
-debug_par_gcc_10:
-  extends: .mfem_job_on_dane
-  variables:
-    SPEC: "%gcc@10.3.1 +debug+mpi"
-
-opt_ser_gcc_10:
-  extends: .mfem_job_on_dane
-  variables:
-    SPEC: "%gcc@10.3.1 ~mpi"
-
-opt_par_gcc_10:
-  extends: .mfem_job_on_dane
-  variables:
-    SPEC: "%gcc@10.3.1"
-
-opt_par_gcc_10_sundials:
-  extends: .mfem_job_on_dane
-  variables:
-    SPEC: "%gcc@10.3.1 +sundials"
-
-opt_par_gcc_10_petsc:
-  extends: .mfem_job_on_dane
-  variables:
-    SPEC: "%gcc@10.3.1 +petsc ^petsc+mumps~superlu-dist"
-
-opt_par_gcc_10_pumi:
-  extends: .mfem_job_on_dane
-  variables:
-    SPEC: "%gcc@10.3.1 +pumi"
-
-opt_par_gcc_10_gslib:
-  extends: .mfem_job_on_dane
-  variables:
-    SPEC: "%gcc@10.3.1 +gslib"

.gitlab/jobs/matrix-reports.yml

@@ -1,19 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Jobs report
-report_job_success:
-  extends: [.on_matrix, .report_job_success]
-  stage: jobs-stage-3
-
-report_job_failure:
-  extends: [.on_matrix, .report_job_failure]
-  stage: jobs-stage-3

.gitlab/jobs/matrix.yml

@@ -1,70 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Override reproducer section to define UMPIRE specific variables.
-.matrix_reproducer_vars:
-  script:
-    - !reference [.reproducer_vars, script]
-
-#TODO: Setup script should be defined as a bash script (but then GIT_STRATEGY cannot be "none" anymore).
-
-# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test
-# script then symlinks the repo to the parent directory of the MFEM source
-# directory. Unit tests that depend on the mfem/data repo will then detect that
-# this directory is present and be enabled.
-setup:
-  extends: .on_matrix
-  stage: jobs-stage-1
-  script:
-    - ./tests/gitlab/build_and_test_setup
-
-
-########################
-# Overridden shared jobs
-########################
-# When using shared jobs , we can duplicate them here to override description and add necessary changes.
-# We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS} So that
-# the comparison with the original job is easier.
-
-
-############
-# Extra jobs
-############
-# We do not recommend using ${PROJECT_<MACHINE>_VARIANTS} and
-# ${PROJECT_<MACHINE>_DEPS} in the extra jobs. There is not reason not to fully
-# describe the spec here.
-
-.mfem_job_on_matrix:
-  extends: .job_on_matrix
-  stage: jobs-stage-2
-  variables:
-    # We run 2 jobs on 1 node that has 112 threads
-    THREADS: 48
-    # These modules need to be consistent with the uberenv configurations:
-    MODULE_LIST: "gcc/10.3.1-magic cuda/12.9.1"
-
-allocate_resources:
-  timeout: 4h
-
-opt_mpi_cuda_gcc:
-  extends: .mfem_job_on_matrix
-  variables:
-    SPEC: "%gcc@10.3.1 +mpi +cuda cuda_arch=90"
-
-opt_mpi_cuda_hypre_cuda_gcc:
-  extends: .mfem_job_on_matrix
-  variables:
-    SPEC: "%gcc@10.3.1 +mpi +cuda cuda_arch=90 ^hypre+cuda"
-
-opt_mpi_cuda_gcc_gslib:
-  extends: .mfem_job_on_matrix
-  variables:
-    SPEC: "%gcc@10.3.1 +mpi +cuda +gslib cuda_arch=90 ^hypre+cuda"

.gitlab/jobs/tioga-reports.yml

@@ -1,20 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Jobs report
-report_job_success:
-  extends: [.on_tioga, .report_job_success]
-  stage: jobs-stage-3
-
-report_job_failure:
-  extends: [.on_tioga, .report_job_failure]
-  stage: jobs-stage-3
-

.gitlab/jobs/tioga.yml

@@ -1,70 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Override reproducer section to define UMPIRE specific variables.
-.tioga_reproducer_vars:
-  script:
-    - !reference [.reproducer_vars, script]
-
-#TODO: Setup script should be defined as a bash script (but then GIT_STRATEGY cannot be "none" anymore).
-
-# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test
-# script then symlinks the repo to the parent directory of the MFEM source
-# directory. Unit tests that depend on the mfem/data repo will then detect that
-# this directory is present and be enabled.
-setup:
-  extends: .on_tioga
-  stage: jobs-stage-1
-  script:
-    - ./tests/gitlab/build_and_test_setup
-
-########################
-# Overridden shared jobs
-########################
-# When using shared jobs , we can duplicate them here to override description and add necessary changes.
-# We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS} So that
-# the comparison with the original job is easier.
-
-
-############
-# Extra jobs
-############
-# We do not recommend using ${PROJECT_<MACHINE>_VARIANTS} and
-# ${PROJECT_<MACHINE>_DEPS} in the extra jobs. There is not reason not to fully
-# describe the spec here.
-
-# Build and test jobs, simply provide a spec
-
-#.tioga_job_command:
-#  script:
-#    - echo PROXY="${PROXY}"
-#    - echo TIOGA_JOB_ALLOC="${TIOGA_JOB_ALLOC}"
-#    - "printf '#!/bin/bash\n%s\n' \"${JOB_CMD}\" > flux_script.sh"
-#    - cat flux_script.sh
-#    - ${PROXY} flux watch $( ${PROXY} flux batch -o output.stdout.type=kvs ${TIOGA_JOB_ALLOC} flux_script.sh )
-#    - rm -f flux_script.sh
-
-.mfem_job_on_tioga:
-  extends: .job_on_tioga
-  stage: jobs-stage-2
-  variables:
-    # We run 1 job on 1 node that has 64 threads
-    THREADS: 64
-
-opt_mpi_rocm_hypre_rocm:
-  extends: .mfem_job_on_tioga
-  variables:
-    SPEC: "%rocmcc@=6.3.1 +rocm amdgpu_target=gfx90a ^hypre+rocm"
-
-# cce_16_0_1:
-#   extends: .mfem_job_on_tioga
-#   variables:
-#     SPEC: "%cce@=16.0.1"

.gitlab/lassen-build-and-test.yml

@@ -0,0 +1,44 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+stages:
+  - setup
+  - build_and_test
+  - report
+
+opt_mpi_cuda_gcc:
+  variables:
+    SPEC: "%gcc@8.3.1 +mpi +cuda cuda_arch=70"
+  extends: .build_and_test_on_lassen
+
+opt_mpi_cuda_hypre_cuda_gcc:
+  variables:
+    SPEC: "%gcc@8.3.1 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
+  extends: .build_and_test_on_lassen
+
+# Jobs report
+report_job_success:
+  stage: report
+  extends:
+    - .on_lassen
+    - .report_job_success
+
+report_job_failure:
+  stage: report
+  extends:
+    - .on_lassen
+    - .report_job_failure
+
+include:
+  - local: .gitlab/configs/common.yml
+  - local: .gitlab/configs/lassen-config.yml
+  - local: .gitlab/configs/setup-build-and-test.yml
+  - local: .gitlab/configs/report-build-and-test.yml

.gitlab/scripts/baseline

@@ -32,9 +32,11 @@ mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
 
 # run
 if [[ "${MACHINE_NAME}" == "dane" ]]; then
-  srun --nodes=1 -t 60 --exclusive --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  salloc --nodes=1 --exclusive --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "corona" ]]; then
-  srun --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+elif [[ ${MACHINE_NAME} == "lassen" ]]; then
+  lalloc 1 -q pci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 else
   echo "Unknown machine: MACHINE_NAME=$MACHINE_NAME"
   exit 1

.gitlab/scripts/report_build_and_test

@@ -1,118 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-function info_msg ()
-{
-  echo "[Information:] ${1}"
-}
-
-function error_msg ()
-{
-  echo "[Error:] ${1}"
-}
-
-# Perform a report while holding a lock file to prevent concurrency on
-# the destination.
-# Usage:
-# locked_clone <report_function> <lock_name>
-function locked_report ()
-{
-  if ! command -v flock
-  then
-    error_msg "Required command 'flock' not found"
-    exit 1
-  fi
-
-  info_msg "Will report ${1} while holding a lock in ${2}"
-
-  ( date; info_msg "Waiting to acquire lock on '${PWD}/${2}.lock' ..."
-    # try to get an exclusive lock on fd 9 (mfem-data.lock) repeating the
-    # try every 5 seconds; we may want to add a counter for the number of
-    # retries to interrupt a potential infinite loop
-    while ! flock -n 9; do sleep 5; done
-    date; info_msg "Acquired lock on '${PWD}/${2}.lock'"
-
-    report ${1}
-    err=$?
-
-    # sleep for a period to allow NFS to propagate the above changes;
-    # clearly, there is no guarantee that other NFS clients will see the
-    # changes even after the timeout
-    sleep 10
-    exit $err
-  ) 9> ${2}.lock
-}
-
-function report ()
-{
-  if [[ "${1}" == "SUCCESS" ]]
-  then
-    info_msg "All the ${MACHINE_NAME} jobs passed"
-    status_msg="The 'build-and-test' jobs on ${MACHINE_NAME} were SUCCESSFUL."
-  elif [[ "${1}" == "FAILURE" ]]
-  then
-    info_msg "At least one failure on ${MACHINE_NAME}"
-    status_msg="Some 'build-and-test' jobs on ${MACHINE_NAME} FAILED."
-  else
-    error_msg "Unknown status: ${1} ... aborting"
-    exit 1
-  fi
-
-  cd ${AUTOTEST_ROOT}/autotest || \
-    { error_msg "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
-  mkdir -p ${MACHINE_NAME}
-
-  rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
-  rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir)
-
-  printf "%s\n" "${status_msg}" \
-    "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.err
-
-  msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
-
-  if [[ "${1}" == "FAILURE" ]]
-  then
-    # Create 'autotest-email.html' to indicate failure:
-    cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
-  fi
-
-  # Note: in some cases, the content of AUTOTEST_COMMIT can be
-  # '${AUTOTEST_COMMIT}', so we need to treat that value as the default
-  # value of 'ON'.
-  if [[ "$AUTOTEST_COMMIT" == '${AUTOTEST_COMMIT}' ]]; then
-    AUTOTEST_COMMIT="ON"
-  fi
-  if [[ "$AUTOTEST_COMMIT" == "ON" || "$AUTOTEST_COMMIT" == "YES" ]]; then
-    git pull && \
-    git add ${rundir} && \
-    git commit -m "${msg}" && \
-    ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
-  else
-    for file in ${rundir}/*; do
-      echo "------------------------------"
-      echo "Content of '$file'"
-      echo "******************************"
-      cat $file
-      echo "******************************"
-    done
-    rm -rf ${rundir} || true
-  fi
-}
-
-export MACHINE_NAME=${CI_MACHINE}
-info_msg "MACHINE_NAME is ${MACHINE_NAME}"
-info_msg "AUTOTEST_ROOT is ${AUTOTEST_ROOT}"
-info_msg "AUTOTEST=$AUTOTEST"
-info_msg "AUTOTEST_COMMIT=$AUTOTEST_COMMIT"
-
-cd ${AUTOTEST_ROOT} && locked_report ${1} autotest

.gitlab/scripts/report_build_and_test_failure

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+echo "Runs if there was at least one failure on ${MACHINE_NAME}"
+
+cd ${AUTOTEST_ROOT}/autotest || \
+  { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
+mkdir -p ${MACHINE_NAME}
+
+rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
+rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir)
+
+printf "%s\n" "Some 'build-and-test' jobs on ${MACHINE_NAME} FAILED." \
+  "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.err
+
+msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
+
+# Create 'autotest-email.html' to indicate failure:
+cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
+
+if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
+  git pull && \
+  git add ${rundir} && \
+  git commit -m "${msg}" && \
+  ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
+else
+  for file in ${rundir}/*; do
+    echo "------------------------------"
+    echo "Content of '$file'"
+    echo "******************************"
+    cat $file
+    echo "******************************"
+  done
+  rm -rf ${rundir} || true
+fi

.gitlab/scripts/report_build_and_test_success

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+echo "Can only run if all the ${MACHINE_NAME} jobs passed"
+
+cd ${AUTOTEST_ROOT}/autotest || \
+  { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
+mkdir -p ${MACHINE_NAME}
+
+rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
+rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir)
+
+printf "%s\n" "The 'build-and-test' jobs on ${MACHINE_NAME} were SUCCESSFUL." \
+  "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.out
+
+msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
+
+if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
+  git pull && \
+  git add ${rundir} && \
+  git commit -m "${msg}" && \
+  ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
+else
+  for file in ${rundir}/*; do
+    echo "------------------------------"
+    echo "Content of '$file'"
+    echo "******************************"
+    cat $file
+    echo "******************************"
+  done
+  rm -rf ${rundir} || true
+fi

.gitlab/subscribed-pipelines.yml

@@ -1,130 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# The template job to test whether a machine is up.
-# Expects CI_MACHINE defined to machine name.
-.machine-check:
-  stage: prerequisites
-  tags: [shell, oslic]
-  variables:
-    GIT_STRATEGY: none
-  script:
-    - |
-      if [[ $(jq '.[env.CI_MACHINE].total_nodes_up' /usr/global/tools/lorenz/data/loginnodeStatus) == 0 ]]
-      then
-        echo -e "\e[31mNo node available on ${CI_MACHINE}\e[0m"
-        false && \
-        curl --url "https://api.github.com/repos/${GITHUB_PROJECT_ORG}/${GITHUB_PROJECT_NAME}/statuses/${CI_COMMIT_SHA}" \
-             --header 'Content-Type: application/json' \
-             --header "authorization: Bearer ${GITHUB_TOKEN}" \
-             --data "{ \"state\": \"failure\", \"target_url\": \"${CI_PIPELINE_URL}\", \"description\": \"GitLab ${CI_MACHINE} down\", \"context\": \"ci/gitlab/${CI_MACHINE}\" }"
-        exit 1
-      fi
-
-###
-# Trigger a build-and-test pipeline for a machine.
-# Comment the jobs for machines you don’t need.
-###
-
-# One job to generate the job list for all the subpipelines
-generate-job-lists:
-  stage: prerequisites
-  tags: [shell, oslic]
-  variables:
-    LOCAL_JOBS_PATH: ".gitlab/jobs"
-  script:
-    - |
-      echo "AUTOTEST=$AUTOTEST"
-      echo "AUTOTEST_COMMIT=$AUTOTEST_COMMIT"
-      echo "AUTOTEST_ROOT=$AUTOTEST_ROOT"
-    - |
-      cat ${LOCAL_JOBS_PATH}/dane.yml > dane-jobs.yml
-      if [[ ${AUTOTEST} == "ON" || ${AUTOTEST} == "YES" ]]
-      then
-          cat ${LOCAL_JOBS_PATH}/dane-reports.yml >> dane-jobs.yml
-      fi
-    - |
-      cat ${LOCAL_JOBS_PATH}/matrix.yml > matrix-jobs.yml
-      if [[ ${AUTOTEST} == "ON" || ${AUTOTEST} == "YES" ]]
-      then
-          cat ${LOCAL_JOBS_PATH}/matrix-reports.yml >> matrix-jobs.yml
-      fi
-    - |
-      cat ${LOCAL_JOBS_PATH}/tioga.yml > tioga-jobs.yml
-      if [[ ${AUTOTEST} == "ON" || ${AUTOTEST} == "YES" ]]
-      then
-          cat ${LOCAL_JOBS_PATH}/tioga-reports.yml >> tioga-jobs.yml
-      fi
-  artifacts:
-    paths:
-      - dane-jobs.yml
-      - matrix-jobs.yml
-      - tioga-jobs.yml
-
-
-# DANE
-dane-up-check:
-  variables:
-    CI_MACHINE: "dane"
-  extends: [.machine-check]
-
-dane-build-and-test:
-  variables:
-    CI_MACHINE: "dane"
-  needs: [dane-up-check, generate-job-lists]
-  extends: [.build-and-test]
-
-# DANE, MFEM Specific
-dane-baseline:
-  stage: test-pipelines
-  variables:
-    # Explicitly pass down values that are not always propagated to child
-    # pipelines, e.g. when a variable is set in the "Settings -> CI" web
-    # interface (project variables).
-    # Note: in some cases, this does not work as expected, e.g. when the
-    # variable is not re-defined in the web interface; in such cases, the child
-    # pipeline gets a definition like '${AUTOTEST}', i.e. it behaves as if
-    # AUTOTEST is undefined, even though there is a default value in
-    # .gitlab-ci.yml.
-    AUTOTEST: "${AUTOTEST}"
-    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
-  trigger:
-    include: .gitlab/dane-baseline.yml
-    strategy: depend
-    forward:
-      pipeline_variables: true
-  needs: [dane-up-check]
-
-
-# TIOGA
-tioga-up-check:
-  variables:
-    CI_MACHINE: "tioga"
-  extends: [.machine-check]
-
-tioga-build-and-test:
-  variables:
-    CI_MACHINE: "tioga"
-  needs: [tioga-up-check, generate-job-lists]
-  extends: [.build-and-test]
-
-
-# Matrix
-matrix-up-check:
-  variables:
-    CI_MACHINE: "matrix"
-  extends: [.machine-check]
-
-matrix-build-and-test:
-  variables:
-    CI_MACHINE: "matrix"
-  needs: [matrix-up-check, generate-job-lists]
-  extends: [.build-and-test]

CHANGELOG

@@ -8,44 +8,9 @@
                                https://mfem.org
 
 
-Version 4.9.1 (development)
+Version 4.8.1 (development)
 ===========================
 
-- Policy for AI-assisted contribution added to CONTRIBUTING.md
-
-Discretization improvements
----------------------------
-- Replaced legacy simplex quadrature rules with symmetric positive-weight
-  rules for triangles (orders 0-25) and tetrahedra (orders 0-20). These
-  rules guarantee all-positive weights and interior quadrature points,
-  improving numerical stability. Higher orders fall back to Grundmann-Moller.
-  Triangle rules: Witherden & Vincent, Comput. Math. Appl. 69(10):1232-1241,
-  2015.
-  Tet rules (d=1-13): Witherden & Vincent (ibid).
-  Tet rules (d=14-20): Chuluunbaatar et al., Comput. Math. Appl. 124:89-97,
-  2022.
-
-- Improved the gridfunction projection routines. Projections work for Scalar,
-  Vector and VectorFE, also NURBS versions. Optionally different types of
-  projections can be selected, default behaviour has not changed.
-
-- Added methods to estimate function extremum using piecewise linear bounds +
-  recursive subdivision.
-
-Meshing improvements
---------------------
-- Improved support for 1D NURBS meshes with variable order, including using
-  the patches construct for 1D NURBS meshes.
-
-New and updated examples and miniapps
--------------------------------------
-- Electromagnetics/lorentz miniapp has been updated to leverage the ParticleSet
-  capability.
-
-
-Version 4.9, released on Dec 11, 2025
-=====================================
-
 Starting with this version, MFEM requires a C++17 compiler.
 
 Discretization improvements
@@ -54,149 +19,70 @@ Discretization improvements
   nonlinear finite element operators, based on Enzyme or dual numbers AD at
   quadrature points. These features are part of the new mfem::future namespace
   and some of the API can change in the future. See the new dFEM minimal surface
-  miniapp in the miniapps/dfem/ directory for illustration of dFEM's use. Using
-  Enzyme for AD in MFEM is tested with clang v19 and requires clang/LLVM built
-  with plugin support. See INSTALL for more details.
+  miniapp in the miniapps/dfem/ directory for illustration of dFEM's use.
 
-- Introduced initial support for particle methods in MFEM with new classes
-  Particle, ParticleSet and ParticleVector.
-  * Particle is a convenient interface for individual particle data.
-  * ParticleSet manages and stores particle data in a struct-of-arrays form,
-    carrying particle coordinates and IDs along with an arbitrary number of
-    Vector and integer data for each particle.
-  * ParticleVector is a Vector-derived container that stores vector data for an
-    arbitrary number of particles contiguously based on specified vdim/ordering.
-  See the new particle miniapps in miniapps/gslib/ and miniapps/fluids/navier/.
-
-- Added a new miniapp and specialized AMG solver (AMGF) for optimization-based
-  contact mechanics. The miniapp solves large-scale frictionless contact using a
-  self-contained Interior Point (IP) solver, mortar-based contact constraints
-  provided by Tribol. The resulting linear systems are solved with the new AMGF
-  solver (see below). Benchmark examples include the two-block, ironing, and
-  beam-sphere problems. See the miniapps/contact/ directory.
-
-- Added support for boundary integration to the hyperbolic framework. Two new
-  classes BdrHyperbolicDirichletIntegrator and BoundaryHyperbolicFlowIntegrator
-  have been introduced for implementation of weak Dirichlet boundary conditions
-  with a general flux or for the linear case respectively.
-
-- Added a method to compute piecewise linear bounds on high-order functions on
-  tensor-product elements.
-
-- Added support for interior face integration enabling DG methods in
-  ParMixedBilinearForm, ParNonlinearForm and ParBlockNonlinearForm.
+- Using Enzyme for AD in MFEM is tested with clang v19 and requires clang/LLVM
+  built with plugin support. See INSTALL for more details.
 
 - In the ParMoonolith integration, added support for variational resampling of
   H1 vector fields.
 
-- Introduced IMEX ODE solvers based on a split-operator framework. Added
-  examples ex41 and ex41p demonstrating IMEX DG/CG discretizations of the
-  convection–diffusion equation, with ex41p using DG LOR preconditioning.
+- Added support for boundary integration to the hyperbolic framework. In this
+  regard, new classes `BdrHyperbolicDirichletIntegrator` and
+  `BoundaryHyperbolicFlowIntegrator` have been introduced for implementation
+  of weak Dirichlet boundary conditions with a general flux or for the linear
+  case respectively.
+
+- Added method to compute piecewise linear bounds on high-order functions on
+  tensor-product elements.
+
+- Parallel anisotropic refinement of hexahedral meshes is now supported,
+  provided that neighboring hexahedra are not refined in conflicting directions.
+  A new ParMesh method is added to check for such conflicts, before refinement.
 
 Meshing improvements
 --------------------
-- The TMOP kernel hierarchy has been restructured to reduce compilation time.
-  Most large kernels have been split into smaller specific kernels for each
-  metric. The directory structure has been updated with assemble, metrics, mult
-  and tools subdirectories. New kernel dispatch and specialization system has
-  also been integrated. Unit tests have been revised to ensure --all tests pass.
 
 - Introduced NC-patch NURBS meshes, which are conforming element-wise but allow
   for nonconforming patch topology. This new mesh format supports element
   spacing formulas for refinement, as well as local refinement factors for a
   subset of knot vectors.
 
+- Added support for higher order meshes in Mesh::MakeSimplicial and
+  ParMesh::MakeSimplicial.
+
 - Added a new miniapp for interpolating a surface grid of points in 3D using a
   smooth NURBS surface, that can then be sampled at arbitrary resolution while
   staying close to the original geometry. See miniapps/nurbs/nurbs_surface.
 
-- Parallel anisotropic refinement of hexahedral meshes is now supported,
-  provided that neighboring hexahedra are not refined in conflicting directions.
-  A new ParMesh method is added to check for such conflicts, before refinement.
-
-- Added support for higher order meshes in (Par)Mesh::MakeSimplicial.
-
-Linear and nonlinear solvers
-----------------------------
-- Added FilteredSolver: a base class for solvers with filtering. It handles
-  cases where a solver performs well except in small subspaces, by adding a
-  filtering step formulated as a subspace correction.
-
-- Added AMGFSolver: a derived class of FilteredSolver, specialized for AMG with
-  Filtering (AMGF), providing robust preconditioning for linear systems arising
-  in constrained optimization problems such as frictionless contact.
-
-  Added 'GetResiduals' and 'GetFinalAbsResidualNorm' to 'HyprePCG',
-  'HypreGMRES', and 'HypreFGMRES' to get 'r' and '|r|_p'. Note that the latter
-  computes '|r|_p' from 'r' instead of returning a cached value like the
-  relative 'GetFinalResidualNorm'. These require Hypre >= 2.15.0.
-
-  Changed the default solver parameters for 'HyprePCG' to 'tol=1e-6' and
-  'max_iter=1000'. This matches the default parameters in Hypre 3.0.
-
-  Added various helper functions for querying/modifying Hypre solvers:
-  'HypreSmoother::GetType', 'HypreSmoother::GetSOROptions',
-  'HypreSmoother::GetPolyOptions', 'HypreSmoother::GetWindowParameters',
-  'HypreSmoother::IsOperatorSymmetric', 'HyprePCG::GetTol',
-  'HyprePCG::GetAbsTol', 'HyprePCG::GetMaxIter', 'HyprePCG::SetUseTwoNorm',
-  'HypreGMRES::GetTol', 'HypreGMRES::GetAbsTol', 'HypreGMRES::GetMaxIter',
-  'HypreGMRES::GetKDim', 'HypreFGMRES::GetTol', 'HypreFGMRES::GetMaxIter',
-  'HypreFGMRES::GetKDim', and 'HypreBoomerAMG::GetMaxIter'.
-
 GPU computing
 -------------
-- Added the 'gpu', 'raja-gpu', and 'ceed-gpu' backend aliases/shortcuts which
-  automatically select between CUDA or HIP.
-
-- Added the option to enable GPU-aware MPI in MFEM using the environment
-  variable 'MFEM_GPU_AWARE_MPI' set to any value. Setting this environment
-  variable is an alternative to calling 'Device::SetGPUAwareMPI(true)'.
-
-- Implemented a GPU-accelerated matrix-free AMR derefinement GridFunction update
-  operator. This supports mixed geometry meshes and variable order spaces, and
-  is the default derefinement operator constructed by FiniteElementSpace::Update
-  and ParFiniteElementSpace::Update. The operator requires the finite element
-  space to be nonconforming.
+- The function Vector::SetSubVector(const Array<int> &, const real_t) now
+  executes on device if either the vector or the array have the device flag
+  set. This is most often used for setting constant essential boundary
+  conditions. A new function Vector::SetSubVectorHost has been added in cases
+  where host execution is always needed (e.g. when the DOFs array is small).
 
 - Introduced MFEM_FOREACH_THREAD_DIRECT, which directly maps loop tasks to GPU
   threads, assigning one task per thread.
 
-- The function Vector::SetSubVector(const Array<int> &, const real_t) now
-  executes on device if either the vector or the array have the device flag
-  set. This is most often used for setting constant essential BCs. A new method,
-  SetSubVectorHost, has been added for cases where host execution is always
-  needed (e.g. when the DOFs array is small).
-
-- Added GPU support in GradientGridFunction and InnerProduct Coefficient classes
-  by implementing their Project methods.
-
+- Implemented a GPU-accelerated matrix-free AMR derefinement `GridFunction`
+  update operator. This supports mixed geometry meshes and variable order
+  spaces, and is the default derefinement operator constructed by
+  `FiniteElementSpace::Update` and `ParFiniteElementSpace::Update`.
+  The operator requires `FiniteElementSpace::Nonconforming() == true`.
 - Added new method: GridFunction::GetGradients, with GPU support, for computing
   the gradients of a GridFunction on all elements.
-
-- The CUDA-specific names used by some of the unit tests like 'cunit_tests' and
-  'pcunit_tests' were replaced by names using 'gpu' instead of 'c' (short for
-  CUDA) or 'cuda'. These tests automatically run the CUDA/HIP tests based on the
-  MFEM build configuration.
+- Added GPU support in GradientGridFunctionCoefficient and
+  InnerProductCoefficient by implementing their Project methods.
 
 New and updated examples and miniapps
 -------------------------------------
-- Added the miniapps/fluids directory and moved the previous Navier and the new
-  incompressible Schrödinger flow miniapps into it.
-
-- Introduced the new Incompressible Schrödinger Flow (ISF) miniapp, which models
-  inviscid fluid dynamics by solving the linear Schrödinger equation, leveraging
-  the hydrodynamical analogy to quantum mechanics.
-
-- New particle-related miniapps:
-  * New transient Navier-Stokes fluid-particles solver NavierParticles in
-    miniapps/fluids/navier/navier_particles, for modeling tracer particles in
-    fluid flow, demonstrating use of the new ParticleSet class.
-  * New Navier miniapp, miniapps/fluids/navier/navier_bifurcation, showing the
-    use of NavierParticles in a 2D bifurcating channel flow.
-  * New FindPointsGSLIB miniapp, miniapps/gslib/particles_redist, showing
-    parallel-redistribution of particle data between MPI ranks.
-  * Particle visualization features in common/particles_extras for viewing
-    particle locations and trajectories (ParticleTrajectories) using GLVis.
+- Added miniapps to demonstrate an implementation of the absolute-value
+  L(1)-Jacobi preconditioners in partially assembled operators. This includes
+  Multigrid wrapper to demonstrate the effectiveness of these Jacobi-type
+  operators as smoothers.
+  These miniapps can be found in `miniapps/diag-smoothers`.
 
 - Added a new miniapp (meshing/mesh-bounding-boxes) that computes the bounding
   boxes for each element of a given mesh, and the bounds on the determinant of
@@ -209,44 +95,36 @@ New and updated examples and miniapps
   of a charged particle, subject to Lorentz forces, in electrostatic and/or
   magnetostatic fields as computed by the volta or tesla miniapps.
 
-- Added miniapps to demonstrate an implementation of the absolute-value
-  l1-Jacobi preconditioners in partially assembled operators. This includes
-  Multigrid wrapper to demonstrate the effectiveness of these Jacobi-type
-  operators as smoothers. See the miniapps/diag-smoothers/ directory.
-
-- Updated the mtop miniapp with a GPU enabled forward and adjoint solver for
-  isotropic linear elasticity.
-
-Miscellaneous
--------------
-- Introduced MFEM_FETCH_TPLS CMake option to enable downloading, configuring,
-  and building of TPLs alongside MFEM (currently supported TPLs are hypre,
-  METIS, and GSLIB).
-
-- Added quadrature function support to the VisIt and Conduit data collections.
-
-- Added access to the internal parallel matrix in Par(Mixed)BilinearForm and
-  related utility methods for elimination of BCs.
-
-- FindPointsGSLIB has a new constructor that accepts the mesh object and
-  internally calls the Setup() method so users do not have to. The FreeData()
-  method has also been moved to the destructor so users do not need to manually
-  free-up the memory if the destructor is called before MPI_Finalize().
-
-- Added parallel Address Sanitizer, serial and parallel Undefined Behavior
-  Sanitizer and serial Memory Sanitizer GitHub actions tests on Ubuntu.
-
-API changes
+API changes:
 -----------
 - mfem::internal::tensor and mfem::internal::dual have been moved to
   mfem::future::tensor and mfem::future::dual.
 
-- API addition: in class Operator, added virtual functions: AbsMult, and
-  AbsMultTranspose; in class Vector, added Abs and Pow.
+- API addition: in class `Operator`, added virtual functions: `AbsMult`, and
+  `AbsMultTranspose`; in class `Vector`, added `Abs` and `Pow`.
 
-- ParBilinearForm::EliminateEssentialVDofsInRhs() has been deprecated in favor
-  of ParallelEliminateEssentialTDofsInRhs().
+Miscellaneous
+-------------
+- Added the "gpu", "raja-gpu", and "ceed-gpu" backend aliases/shortcuts which
+  automatically select between CUDA or HIP.
 
+- The CUDA-specific names used by some of the unit tests like 'cunit_tests' and
+  'pcunit_tests' were replaced by names using 'gpu' instead of 'c' (short for
+  CUDA) or 'cuda'. These tests automatically run the CUDA/HIP tests based on the
+  MFEM build configuration.
+
+- Added the option to enable GPU-aware MPI in MFEM using the environment
+  variable 'MFEM_GPU_AWARE_MPI' set to any value. Setting this environment
+  variable is an alternative to calling 'Device::SetGPUAwareMPI(true)'.
+
+- Added parallel Address Sanitizer, serial and parallel Undefined Behavior
+  Sanitizer and serial Memory Sanitizer GitHub actions tests on Ubuntu.
+
+- FindPointsGSLIB has a new constructor that accepts the mesh object and
+  internally calls the Setup() method so that the user does not have to.
+  The FreeData() method has also been moved to the destructor so the user does
+  not need to manually free-up the memory if the destructor is called before
+  MPI_Finalize().
 
 Version 4.8, released on Apr 9, 2025
 ====================================

CMakeLists.txt

@@ -59,7 +59,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.9.1)
+set(${PROJECT_NAME}_VERSION 4.8.1)
 
 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -133,49 +133,33 @@ if (MFEM_USE_CUDA)
    if (NOT CMAKE_CUDA_HOST_COMPILER)
       set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
    endif()
-   if (NOT CMAKE_CUDA_ARCHITECTURES)
-      # make CUDA_ARCH resemble the same form as CMAKE_CUDA_ARCHITECTURES
-      string(REPLACE "sm_" "" CUDA_ARCH_TMP "${CUDA_ARCH}")
-      string(REPLACE "," ";" CUDA_ARCH "${CUDA_ARCH_TMP}")
-      set(CMAKE_CUDA_ARCHITECTURES "${CUDA_ARCH}")
+   if (CMAKE_VERSION VERSION_LESS 3.18.0)
+      set(CUDA_FLAGS "-arch=${CUDA_ARCH} ${CUDA_FLAGS}")
+   elseif (NOT CMAKE_CUDA_ARCHITECTURES)
+      string(REGEX REPLACE "^sm_" "" ARCH_NUMBER "${CUDA_ARCH}")
+      if ("${CUDA_ARCH}" STREQUAL "sm_${ARCH_NUMBER}")
+         set(CMAKE_CUDA_ARCHITECTURES "${ARCH_NUMBER}")
+      else()
+         message(FATAL_ERROR "Unknown CUDA_ARCH: ${CUDA_ARCH}")
+      endif()
    else()
       set(CUDA_ARCH "CMAKE_CUDA_ARCHITECTURES: ${CMAKE_CUDA_ARCHITECTURES}")
    endif()
    message(STATUS "Using CUDA architecture: ${CUDA_ARCH}")
    enable_language(CUDA)
    if (CMAKE_VERSION VERSION_LESS 3.18.0)
-      # backup try to detect if this is clang or nvcc
-      if(CMAKE_CUDA_COMPILER MATCHES "nvcc$")
-         # nvcc
-         set(MFEM_CUDA_COMPILER_IS_NVCC ON)
-         set(CUDA_FLAGS "${CUDA_FLAGS} --expt-extended-lambda --expt-relaxed-constexpr")
-         if ("all" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}"
-             OR "native" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}"
-             OR "all-major" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}")
-            set(CUDA_FLAGS "-arch=${CMAKE_CUDA_ARCHITECTURES} ${CUDA_FLAGS}")
-         else()
-            # build -gencode sequence for multiple architectures
-            foreach(ENTRY IN LISTS CMAKE_CUDA_ARCHITECTURES)
-               set(CUDA_FLAGS
-                   "-gencode arch=compute_${ENTRY},code=sm_${ENTRY} ${CUDA_FLAGS}")
-            endforeach()
-         endif()
-      else()
-         # build cuda-gpu-arch sequence for multiple architectures
-         # does not support all/all-major/native
-         foreach(ENTRY IN LISTS CMAKE_CUDA_ARCHITECTURES)
-            set(CUDA_FLAGS "-cuda-gpu-arch=sm_${ENTRY} ${CUDA_FLAGS}")
-         endforeach()
-      endif()
+     # backup try to detect if this is clang or nvcc
+     if(CMAKE_CUDA_COMPILER MATCHES "nvcc$")
+       # nvcc
+       set(MFEM_CUDA_COMPILER_IS_NVCC ON)
+       set(CUDA_FLAGS "${CUDA_FLAGS} --expt-extended-lambda --expt-relaxed-constexpr")
+     endif()
    else()
-      # TODO: all, native, all-major require CMake 3.24+
-      # backport support for CMake 3.18 to 3.24
-      if (CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA")
-         # nvcc
-         set(MFEM_CUDA_COMPILER_IS_NVCC ON)
-         set(CUDA_FLAGS
-             "${CUDA_FLAGS} --expt-extended-lambda --expt-relaxed-constexpr")
-      endif()
+     if (CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA")
+       # nvcc
+       set(MFEM_CUDA_COMPILER_IS_NVCC ON)
+       set(CUDA_FLAGS "${CUDA_FLAGS} --expt-extended-lambda --expt-relaxed-constexpr")
+     endif()
    endif()
    set(CMAKE_CUDA_STANDARD ${CMAKE_CXX_STANDARD} CACHE STRING
       "CUDA standard to use.")
@@ -258,16 +242,10 @@ endif()
 
 # AMD HIP
 if (MFEM_USE_HIP)
-  if (NOT CMAKE_HIP_ARCHITECTURES)
-    if (HIP_ARCH)
-      set(CMAKE_HIP_ARCHITECTURES CACHE STRING "HIP targets to compile for" "${HIP_ARCH}")
-      set(GPU_TARGETS "${HIP_ARCH}" CACHE STRING "HIP targets to compile for" FORCE)
-    endif()
-  else()
-    set(HIP_ARCH CACHE STRING "HIP targets to compile for" "${CMAKE_HIP_ARCHITECTURES}")
-    set(GPU_TARGETS "${CMAKE_HIP_ARCHITECTURES}" CACHE STRING "HIP targets to compile for" FORCE)
+  if (HIP_ARCH)
+    message(STATUS "Using HIP architecture: ${HIP_ARCH}")
+    set(GPU_TARGETS "${HIP_ARCH}" CACHE STRING "HIP targets to compile for")
   endif()
-  message(STATUS "Using HIP architecture: ${CMAKE_HIP_ARCHITECTURES}")
   if (ROCM_PATH)
     list(INSERT CMAKE_PREFIX_PATH 0 ${ROCM_PATH})
   endif()
@@ -300,19 +278,8 @@ if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
   endif()
 endif()
 
-# Warn user if deprecated FETCH_TPLS is provided
-if (DEFINED FETCH_TPLS)
-  message(STATUS "Setting MFEM_FETCH_TPLS to user-provided value of FETCH_TPLS (i.e., MFEM_FETCH_TPLS=${FETCH_TPLS})")
-  set (MFEM_FETCH_TPLS FETCH_TPLS)
-  message(DEPRECATION "The use of FETCH_TPLS is deprecated and will be removed in future verison. Please use MFEM_FETCH_TPLS instead.")
-endif()
-
-  # Umpire (must be included before hypre, so hypre can use it if needed)
+# Umpire (must be included before hypre, so hypre can use it if needed)
 if (MFEM_USE_UMPIRE)
-   # umpire uses FindCUDA, which needs CMP0146=OLD in CMake >= 3.27
-   if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.27.0)
-      cmake_policy(SET CMP0146 OLD)
-   endif()
    find_package(UMPIRE REQUIRED)
 endif()
 
@@ -652,8 +619,6 @@ foreach(TPL IN LISTS MFEM_TPLS)
   endif()
 endforeach(TPL)
 
-# reverse to remove the first instance of entries in TPL_LIBRARIES
-# so later duplicates are kept (for dependency ordering)
 list(REVERSE TPL_LIBRARIES)
 list(REMOVE_DUPLICATES TPL_LIBRARIES)
 list(REVERSE TPL_LIBRARIES)
@@ -725,7 +690,6 @@ set(MFEM_INSTALL_DIR ${CMAKE_INSTALL_PREFIX})
 
 # Declaring the library
 mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
-target_compile_features(mfem PUBLIC cxx_std_${CMAKE_CXX_STANDARD})
 # message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
 target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES} ${TPL_TARGETS})
 if (TPL_TARGETS)
@@ -872,12 +836,11 @@ add_dependencies(exec
 #  - https://cmake.org/Bug/view.php?id=8438
 
 # Add a target to copy the mfem data directory to the build directory
-# Implementable as a single copy_directory_if_different command w/ CMake >= 3.26
-file(GLOB DATA_FILES CONFIGURE_DEPENDS ${PROJECT_SOURCE_DIR}/data/*)
-add_custom_target(copy_data
-  COMMAND ${CMAKE_COMMAND} -E make_directory data
-  COMMAND ${CMAKE_COMMAND} -E copy_if_different ${DATA_FILES} data
-  COMMENT "Syncing the data directory ...")
+add_custom_command(OUTPUT data_is_copied
+  COMMAND ${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/data data
+  COMMAND ${CMAKE_COMMAND} -E touch data_is_copied
+  COMMENT "Copying the data directory ...")
+add_custom_target(copy_data DEPENDS data_is_copied)
 # Add 'copy_data' as a prerequisite for all executables, if the source and the
 # build directories are not the same.
 if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
@@ -1009,15 +972,9 @@ install(FILES
 install(EXPORT ${PROJECT_NAME_UC}Targets
     DESTINATION ${INSTALL_CMAKE_DIR})
 
-# Install the data directory if present, i.e. if the copy_data target is built
-install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/data
-  DESTINATION ${MFEM_INSTALL_DIR} OPTIONAL)
-
 #-------------------------------------------------------------------------------
 # Create 'config.mk' from 'config.mk.in' for the build and install locations and
 # define install rules for 'config.mk' and 'test.mk'
 #-------------------------------------------------------------------------------
-if (MFEM_USE_CUDA OR MFEM_USE_HIP)
-  option(MFEM_EXPORT_GPU_CONFIG "Export config.mk for GPU-enabled downstream packages" ON)
-endif()
+
 mfem_export_mk_files()

CONTRIBUTING.md

@@ -24,14 +24,6 @@ must be made under this license.
 Note also that MFEM has a [Code of Conduct](CODE_OF_CONDUCT.md). By participating
 in the MFEM community, you agree to abide by its rules.
 
-## AI Policy
-- Use of AI code generation in MFEM is allowed but must be disclosed, e.g. by
-  selecting the `AI-assisted` label on the PR.
-- By submitting a PR, the author acknowledges that they have reviewed and
-  understand the changes they are proposing.
-- PR authors are still responsible for correctness, licensing, and attribution
-  of all changes.
-
 If you plan on contributing to MFEM, consider reviewing the
 [issue tracker](https://github.com/mfem/mfem/issues) first to check if a thread
 already exists for your desired feature or the bug you ran into. Use a pull
@@ -137,10 +129,6 @@ The MFEM source code has the following structure:
   │   ├── moonolith
   │   ├── qinterp
   │   └── tmop
-  │   |   ├── assemble
-  │   |   ├── metrics
-  │   |   ├── mult
-  │   |   └── tools
   ├── general
   ├── linalg
   │   ├── batched
@@ -151,19 +139,16 @@ The MFEM source code has the following structure:
   │   ├── adjoint
   │   ├── autodiff
   │   ├── common
-  │   ├── contact
   │   ├── dfem
   │   ├── dpg
   │   ├── electromagnetics
-  │   ├── fluids
-  │   │   ├── navier
-  │   │   └── schrodinger-flow
   │   ├── gslib
   │   ├── hdiv-linear-solver
   │   ├── hooke
   │   ├── meshing
   │   ├── mtop
   │   ├── multidomain
+  │   ├── navier
   │   ├── nurbs
   │   ├── parelag
   │   ├── performance

INSTALL

@@ -123,7 +123,7 @@ Parallel build:
 
 Parallel build with fetching of hypre and METIS:
    mkdir <mfem-buil-dir> ; cd <mfem-build-dir>
-   cmake <mfem-source-dir> -DMFEM_USE_MPI=YES -DMFEM_FETCH_TPLS=YES
+   cmake <mfem-source-dir> -DMFEM_USE_MPI=YES -DFETCH_TPLS=YES
    make -j 4
 
 CUDA build:
@@ -725,9 +725,7 @@ The specific libraries and their options are:
   URL: https://ginkgo-project.github.io
   Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or
            Debug).
-  Versions: Ginkgo >= 1.9.0. When building Ginkgo with distributed support, a
-            recent version of the "develop" branch is required (1.11 as defined
-            in include/ginkgo/config.hpp).
+  Versions: Ginkgo >= 1.9.0.
 
 - AmgX (optional), used when MFEM_USE_AMGX = YES.
   URL: https://github.com/NVIDIA/AMGX
@@ -1083,10 +1081,9 @@ The following options are CMake specific:
 MFEM_ENABLE_TESTING  - Enable the ctest framework for testing.
 MFEM_ENABLE_EXAMPLES - Build all of the examples by default.
 MFEM_ENABLE_MINIAPPS - Build all of the miniapps by default.
-MFEM_FETCH_TPLS      - Enable fetching of all supported third-party libraries.
-MFEM_FETCH_GSLIB     - Enable fetching of gslib.
-MFEM_FETCH_HYPRE     - Enable fetching of hypre.
-MFEM_FETCH_METIS     - Enable fetching of metis.
+FETCH_TPLS           - Enable fetching of all supported third-party libraries.
+HYPRE_FETCH          - Enable fetching of hypre.
+METIS_FETCH          - Enable fetching of metis.
 
 External libraries (CMake):
 ---------------------------
@@ -1152,7 +1149,6 @@ The MFEM CMake build system also provides fetching (automated building) for the
 packages/libraries listed below. Note that when fetching is enabled, any related
 auto-detection functionality is disabled.
 
- - GSLIB
  - HYPRE
  - METIS
 

config/cmake/MFEMConfig.cmake.in

@@ -109,10 +109,6 @@ if (MFEM_USE_RAJA)
    find_dependency(RAJA)
 endif()
 
-if (MFEM_USE_UMPIRE)
-   find_dependency(umpire)
-endif()
-
 if (NOT TARGET mfem)
   include(${CMAKE_CURRENT_LIST_DIR}/MFEMTargets.cmake)
 endif (NOT TARGET mfem)

config/cmake/modules/FindGSLIB.cmake

@@ -9,47 +9,10 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-# Defines the following variables if fetching of TPLs is disabled (default):
+# Defines the following variables:
 #   - GSLIB_FOUND
 #   - GSLIB_LIBRARIES
 #   - GSLIB_INCLUDE_DIRS
-# otherwise, the following are defined:
-#   - GSLIB (imported library target)
-
-if (MFEM_FETCH_GSLIB OR MFEM_FETCH_TPLS)
-  enable_language(C)
-  string(TOUPPER "${CMAKE_BUILD_TYPE}" BUILD_TYPE)
-  set(GSLIB_FETCH_VERSION 1.0.9)
-  set(GSLIB_C_FLAGS ${CMAKE_C_FLAGS_${BUILD_TYPE}})
-  if (CMAKE_C_FLAGS)
-    set(GSLIB_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BUILD_TYPE}}")
-  endif()
-  if (BUILD_SHARED_LIBS)
-    set(GSLIB_C_FLAGS "${GSLIB_C_FLAGS} -fPIC")
-  endif()
-  add_library(GSLIB STATIC IMPORTED)
-  # define external project and create future include directory so it is present
-  # to pass CMake checks at end of MFEM configuration step
-  message(STATUS "Will fetch GSLIB ${GSLIB_FETCH_VERSION} to be built with ${GSLIB_C_FLAGS}")
-  set(PREFIX ${CMAKE_BINARY_DIR}/fetch/gslib)
-  include(ExternalProject)
-  ExternalProject_Add(gslib
-    GIT_REPOSITORY https://github.com/Nek5000/gslib
-    GIT_TAG v${GSLIB_FETCH_VERSION}
-    GIT_SHALLOW TRUE
-    UPDATE_DISCONNECTED TRUE
-    PREFIX ${PREFIX}
-    CONFIGURE_COMMAND ""
-    BUILD_COMMAND cd ${PREFIX}/src/gslib && $(MAKE) clean && $(MAKE) DESTDIR=${PREFIX} MPI=$<BOOL:${MFEM_USE_MPI}> "CFLAGS= ${GSLIB_C_FLAGS}"
-    INSTALL_COMMAND "")
-  file(MAKE_DIRECTORY ${PREFIX}/include)
-  # set imported library target properties
-  add_dependencies(GSLIB gslib)
-  set_target_properties(GSLIB PROPERTIES
-    IMPORTED_LOCATION ${PREFIX}/lib/libgs.a
-    INTERFACE_INCLUDE_DIRECTORIES ${PREFIX}/include)
-  return()
-endif()
 
 include(MfemCmakeUtilities)
 mfem_find_package(GSLIB GSLIB GSLIB_DIR "include" gslib.h "lib" gs

config/cmake/modules/FindHYPRE.cmake

@@ -37,21 +37,21 @@ if (HYPRE_FOUND OR TARGET HYPRE)
   endif()
 endif()
 
-if (MFEM_FETCH_HYPRE OR MFEM_FETCH_TPLS)
-  set(HYPRE_FETCH_VERSION 2.33.0)
-  set(HYPRE_FETCH_TAG "v${HYPRE_FETCH_VERSION}" CACHE STRING "Tag, branch, or commit for HYPRE")
-  add_library(HYPRE STATIC IMPORTED)
-  # set options and associated dependencies
+if (HYPRE_FETCH OR FETCH_TPLS)
+  # Collect all HYPRE_ENABLE variables and pass them to hypre, assuming they are BOOL.
   set(HYPRE_CMAKE_OPTIONS "")
-  list(APPEND HYPRE_CMAKE_OPTIONS -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE})
-  # collect all HYPRE_ENABLE variables and pass them to hypre, assuming they are BOOL.
   get_cmake_property(all_vars VARIABLES)
   foreach(var ${all_vars})
     if(var MATCHES "^HYPRE_ENABLE")
       list(APPEND HYPRE_CMAKE_OPTIONS "-D${var}:BOOL=${${var}}")
     endif()
   endforeach()
-  # process all MFEM_USE variables that impact hypre
+
+  set(HYPRE_FETCH_VERSION 2.33.0)
+  set(HYPRE_FETCH_TAG "v${HYPRE_FETCH_VERSION}" CACHE STRING "Tag, branch, or commit for HYPRE")
+  add_library(HYPRE STATIC IMPORTED)
+  # set options and associated dependencies
+  list(APPEND HYPRE_CMAKE_OPTIONS -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE})
   if (MFEM_USE_CUDA)
     list(APPEND HYPRE_CMAKE_OPTIONS -DHYPRE_ENABLE_CUDA:BOOL=ON -DCMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES})
     find_package(CUDAToolkit REQUIRED)

config/cmake/modules/FindMETIS.cmake

@@ -18,7 +18,7 @@
 #   - METIS (imported library target)
 #   - METIS_VERSION_5 (cache variable)
 
-if (MFEM_FETCH_METIS OR MFEM_FETCH_TPLS)
+if (METIS_FETCH OR FETCH_TPLS)
   set(METIS_FETCH_VERSION 4.0.3)
   add_library(METIS STATIC IMPORTED)
   # define external project

config/cmake/modules/FindUMPIRE.cmake

@@ -14,12 +14,12 @@
 #   - UMPIRE_LIBRARIES
 #   - UMPIRE_INCLUDE_DIRS
 
-if (NOT umpire_ROOT AND UMPIRE_DIR)
-  set(umpire_ROOT ${UMPIRE_DIR})
+if (NOT umpire_DIR AND UMPIRE_DIR)
+  set(umpire_DIR ${UMPIRE_DIR}/lib/cmake/umpire)
 endif()
 message(STATUS "Looking for UMPIRE ...")
 message(STATUS "   in UMPIRE_DIR = ${UMPIRE_DIR}")
-message(STATUS "      umpire_ROOT = ${umpire_ROOT}")
+message(STATUS "      umpire_DIR = ${umpire_DIR}")
 find_package(umpire CONFIG)
 set(UMPIRE_FOUND ${umpire_FOUND})
 set(UMPIRE_LIBRARIES "umpire")

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -701,6 +701,7 @@ endfunction(mfem_find_library)
 # Extract compile and link options needed by the given target.
 #
 function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
+
   if (NOT TARGET ${Target})
     return()
   endif()
@@ -798,12 +799,7 @@ function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
           # message(STATUS "Lib = ${Lib}")
           # Filter-out generator expressions
           if (NOT ("${Lib}" MATCHES "^\\$"))
-            if(NOT ("${Lib}" STREQUAL "dl"))
-              list(APPEND LinkOpts "${Lib}")
-            else()
-              # for some reason libdl doesn't include the "-l"
-              list(APPEND LinkOpts "-ldl")
-            endif()
+            list(APPEND LinkOpts "${Lib}")
           endif()
         else()
           mfem_get_target_options(${Lib} COpts LOpts)
@@ -892,18 +888,9 @@ function(mfem_export_mk_files)
       set(${var} NO)
     endif()
   endforeach()
-  if (MFEM_USE_CUDA AND MFEM_EXPORT_GPU_CONFIG)
-    set(MFEM_CXX ${CMAKE_CUDA_COMPILER})
-    if(MFEM_CUDA_COMPILER_IS_NVCC)
-      set(MFEM_HOST_CXX ${CMAKE_CUDA_HOST_COMPILER})
-    else()
-      set(MFEM_HOST_CXX ${CMAKE_CXX_COMPILER})
-    endif()
-  else()
-    # mfem doesn't use enable_language(HIP)
-    set(MFEM_CXX ${CMAKE_CXX_COMPILER})
-    set(MFEM_HOST_CXX ${CMAKE_CXX_COMPILER})
-  endif()
+  # TODO: Add support for MFEM_USE_CUDA=YES
+  set(MFEM_CXX ${CMAKE_CXX_COMPILER})
+  set(MFEM_HOST_CXX ${MFEM_CXX})
   set(MFEM_CPPFLAGS "")
   get_target_property(cxx_std mfem CXX_STANDARD)
   # For now, we ignore the setting of the CXX_EXTENSIONS property. If this
@@ -913,50 +900,6 @@ function(mfem_export_mk_files)
   string(STRIP
          "${cxx_std_flag} ${CMAKE_CXX_FLAGS_${BUILD_TYPE}} ${CMAKE_CXX_FLAGS}"
          MFEM_CXXFLAGS)
-  if(MFEM_EXPORT_GPU_CONFIG)
-    if (MFEM_USE_CUDA)
-      set(MFEM_CXXFLAGS "${MFEM_CXXFLAGS} ${CMAKE_CUDA_FLAGS}")
-      if (MFEM_CUDA_COMPILER_IS_NVCC)
-        set(MFEM_CXXFLAGS "-x=cu ${MFEM_CXXFLAGS} -ccbin ${CMAKE_CXX_COMPILER} --forward-unknown-to-host-compiler")
-        # The following intentionally hides CUDA deprecation warnings
-        foreach(ENTRY IN LISTS CUDAToolkit_INCLUDE_DIRS)
-          set(MFEM_CXXFLAGS "${MFEM_CXXFLAGS} -isystem ${ENTRY}")
-        endforeach()
-        if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.18.0)
-          # architecture flags not part of CMAKE_CUDA_FLAGS
-          if ("all" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}"
-              OR "native" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}"
-              OR "all-major" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}")
-            set(MFEM_CXXFLAGS "${MFEM_CXXFLAGS} -arch=${CMAKE_CUDA_ARCHITECTURES}")
-          else()
-            foreach (ENTRY IN LISTS CMAKE_CUDA_ARCHITECTURES)
-              set(MFEM_CXXFLAGS
-                "${MFEM_CXXFLAGS} -gencode arch=compute_${ENTRY},code=sm_${ENTRY}")
-            endforeach()
-          endif()
-        endif()
-      else()
-        set(MFEM_CXXFLAGS "${MFEM_CXXFLAGS} -xcuda --cuda-path=${CUDAToolkit_LIBRARY_ROOT}")
-        if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.18.0)
-          # architecture flags not part of CMAKE_CUDA_FLAGS
-          if ("all" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}"
-              OR "native" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}"
-              OR "all-major" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}")
-            # TODO: not supported
-          else()
-            foreach(ENTRY IN LISTS CMAKE_CUDA_ARCHITECTURES)
-              set(MFEM_CXXFLAGS "-cuda-gpu-arch=sm_${ENTRY} ${MFEM_CXXFLAGS}")
-            endforeach()
-          endif()
-        endif()
-      endif()
-    elseif (MFEM_USE_HIP)
-      set(MFEM_CXXFLAGS "${MFEM_CXXFLAGS} -xhip")
-      foreach(ENTRY IN LISTS CMAKE_HIP_ARCHITECTURES)
-        set(MFEM_CXXFLAGS "--offload-arch=${ENTRY} ${MFEM_CXXFLAGS}")
-      endforeach()
-    endif()
-  endif()
   set(MFEM_TPLFLAGS "")
   foreach(dir ${TPL_INCLUDE_DIRS})
     set(MFEM_TPLFLAGS "${MFEM_TPLFLAGS} -I${dir}")
@@ -987,9 +930,6 @@ function(mfem_export_mk_files)
     set(MFEM_SHARED NO)
     set(MFEM_STATIC YES)
   endif()
-  if (MFEM_USE_CUDA)
-    set(MFEM_EXT_LIBS "${MFEM_EXT_LIBS} -lcudart")
-  endif()
   set(MFEM_BUILD_TAG "${CMAKE_SYSTEM}")
   set(MFEM_PREFIX "${CMAKE_INSTALL_PREFIX}")
   # For the next 4 variables, these are the values for the build-tree version of
@@ -998,15 +938,8 @@ function(mfem_export_mk_files)
   set(MFEM_LIB_DIR "${PROJECT_BINARY_DIR}")
   set(MFEM_TEST_MK "${PROJECT_SOURCE_DIR}/config/test.mk")
   set(MFEM_CONFIG_EXTRA "MFEM_BUILD_DIR ?= ${PROJECT_BINARY_DIR}")
-  if (MFEM_USE_CUDA AND MFEM_EXPORT_GPU_CONFIG)
-    if (MFEM_CUDA_COMPILER_IS_NVCC)
-      set(MFEM_XLINKER "-Xlinker=")
-    else()
-      set(MFEM_XLINKER "${CMAKE_CUDA_LINKER_WRAPPER_FLAG}")
-    endif()
-  else()
-    set(MFEM_XLINKER "${CMAKE_CXX_LINKER_WRAPPER_FLAG}")
-  endif()
+  # TODO: CUDA/HIP support:
+  set(MFEM_XLINKER "${CMAKE_CXX_LINKER_WRAPPER_FLAG}")
   set(MFEM_MPIEXEC ${MPIEXEC})
   if (NOT MFEM_MPIEXEC)
     set(MFEM_MPIEXEC "mpirun")
@@ -1054,21 +987,16 @@ function(mfem_export_mk_files)
     # handle interfaces (e.g., SCOREC::apf)
     if ("${lib}" MATCHES "SCOREC::.*" OR "${lib}" MATCHES "Ginkgo::.*" OR "${lib}" MATCHES "ParMoonolith::.*")
     elseif (TARGET "${lib}")
-      mfem_get_target_options(${lib} CompileOpts2 LinkOpts2)
-      # remove generator expressions
-      string(GENEX_STRIP "${CompileOpts2}" CompileOpts)
-      string(GENEX_STRIP "${LinkOpts2}" LinkOpts)
+      mfem_get_target_options(${lib} CompileOpts LinkOpts)
       # Removing duplicates may lead to issues:
       # list(REMOVE_DUPLICATES CompileOpts)
       # list(REMOVE_DUPLICATES LinkOpts)
-      # message(WARNING "${lib}[LinkOpts]: ${LinkOpts}")      
-      # message(WARNING "${lib}[CompileOpts]: ${CompileOpts}")
-      foreach(LOpt IN LISTS LinkOpts)
-        set(MFEM_EXT_LIBS "${MFEM_EXT_LIBS} ${LOpt}")
-      endforeach()
-      foreach(COpt IN LISTS CompileOpts)
-        set(MFEM_TPLFLAGS "${MFEM_TPLFLAGS} ${COpt}")
-      endforeach()
+      string(REPLACE ";" " " COpts "${CompileOpts}")
+      string(REPLACE ";" " " LOpts "${LinkOpts}")
+      # message(STATUS "${lib}[COpts]: '${COpts}'")
+      # message(STATUS "${lib}[LOpts]: '${LOpts}'")
+      set(MFEM_TPLFLAGS "${MFEM_TPLFLAGS} ${COpts}")
+      set(MFEM_EXT_LIBS "${MFEM_EXT_LIBS} ${LOpts}")
       # message(FATAL_ERROR "***** interface lib found ... exiting *****")
       # handle static and shared libs
     elseif ("${suffix}" STREQUAL "${CMAKE_SHARED_LIBRARY_SUFFIX}")
@@ -1076,7 +1004,7 @@ function(mfem_export_mk_files)
       get_filename_component(fullLibName ${lib} NAME_WE)
       string(REGEX REPLACE "^lib" "" libname ${fullLibName})
       set(MFEM_EXT_LIBS
-        "${MFEM_EXT_LIBS} ${shared_link_flag}${dir} -L${dir} -l${libname}")
+          "${MFEM_EXT_LIBS} ${shared_link_flag}${dir} -L${dir} -l${libname}")
     else()
       set(MFEM_EXT_LIBS "${MFEM_EXT_LIBS} ${lib}")
     endif()
@@ -1085,7 +1013,7 @@ function(mfem_export_mk_files)
   # Create the build-tree version of 'config.mk'
   configure_file(
     "${PROJECT_SOURCE_DIR}/config/config.mk.in"
-    "${PROJECT_BINARY_DIR}/config/config.mk" @ONLY)
+    "${PROJECT_BINARY_DIR}/config/config.mk")
   # Copy 'test.mk' from the source-tree to the build-tree
   configure_file(
     "${PROJECT_SOURCE_DIR}/config/test.mk"
@@ -1103,7 +1031,7 @@ function(mfem_export_mk_files)
   # Create the install-tree version of 'config.mk'
   configure_file(
     "${PROJECT_SOURCE_DIR}/config/config.mk.in"
-    "${PROJECT_BINARY_DIR}/config/config-install.mk" @ONLY)
+    "${PROJECT_BINARY_DIR}/config/config-install.mk")
 
   # Install rules for 'config.mk' and 'test.mk'
   install(FILES ${PROJECT_SOURCE_DIR}/config/test.mk

config/defaults.cmake

@@ -91,10 +91,9 @@ option(MFEM_ENABLE_BENCHMARKS "Build all of the benchmarks" OFF)
 
 # Allow a user to specify fetching of certain third-party libraries instead of
 # searching for existing installations.
-option(MFEM_FETCH_TPLS "Enable fetching of all supported third-party libraries" OFF)
-option(MFEM_FETCH_GSLIB "Enable fetching of GSLIB" OFF)
-option(MFEM_FETCH_HYPRE "Enable fetching of hypre" OFF)
-option(MFEM_FETCH_METIS "Enable fetching of METIS" OFF)
+option(FETCH_TPLS "Enable fetching of all supported third-party libraries" OFF)
+option(HYPRE_FETCH "Enable fetching of hypre" OFF)
+option(METIS_FETCH "Enable fetching of METIS" OFF)
 
 # Setting CXX/MPICXX on the command line or in user.cmake will overwrite the
 # autodetected C++ compiler.

config/defaults.mk

@@ -18,7 +18,6 @@
 # Some choices below are based on the OS type:
 NOTMAC := $(subst Darwin,,$(shell uname -s))
 
-ASTYLE_BIN = astyle
 ETAGS_BIN = $(shell command -v etags 2> /dev/null)
 EGREP_BIN = $(shell command -v egrep 2> /dev/null)
 
@@ -408,7 +407,7 @@ AMGX_LIB = -L$(AMGX_DIR)/lib -lamgx -lcusparse -lcusolver -lcublas -lnvToolsExt
 # MAGMA library configuration
 MAGMA_DIR = @MFEM_DIR@/../magma
 MAGMA_OPT = -I$(MAGMA_DIR)/include
-MAGMA_LIB = -L$(MAGMA_DIR)/lib -l:libmagma.a $(LAPACK_LIB)
+MAGMA_LIB = -L$(MAGMA_DIR)/lib -l:libmagma.a -lcublas -lcusparse $(LAPACK_LIB)
 
 # GnuTLS library configuration
 GNUTLS_OPT =

config/docker/README.md

@@ -101,7 +101,7 @@ $ cd ../miniapps
 $ ls
 CMakeLists.txt  common            meshing  nurbs        shifted  toys
 adjoint         electromagnetics  mtop     parelag      solvers
-autodiff        gslib             fluids   performance  tools
+autodiff        gslib             navier   performance  tools
 ```
 
 And an example in "toys"

config/sample-runs.sh

@@ -85,10 +85,6 @@ groups_serial=(
    "DPG miniapps:"
    "miniapps/dpg"
    "{acoustics,convection-diffusion,diffusion,maxwell}.cpp"'
-'"isf"
-   "Schrodinger flow miniapps:"
-   "miniapps/fluids/schrodinger-flow"
-   "schrodinger_flow.cpp"'
 '"gslib"
    "GSLIB miniapps:"
    "miniapps/gslib"
@@ -170,10 +166,6 @@ groups_parallel=(
    "miniapps/electromagnetics"
    "joule.cpp"'
 #   "{joule,maxwell,tesla,volta}.cpp"' # todo: multiline sample runs
-'"isf"
-   "Schrodinger flow miniapps:"
-   "miniapps/fluids/schrodinger-flow"
-   "pschrodinger_flow.cpp"'
 '"adjoint"
    "Adjoint miniapps:"
    "miniapps/adjoint"
@@ -199,7 +191,7 @@ groups_parallel=(
 # todo: miniapps/multidomain
 '"navier"
    "Navier miniapps:"
-   "miniapps/fluids/navier"
+   "miniapps/navier"
    "navier_cht.cpp"'
 # todo: add other navier miniapps
 '"nurbs"
@@ -289,10 +281,6 @@ groups_all=(
    "miniapps/electromagnetics"
    "joule.cpp"'
 #   "{joule,maxwell,tesla,volta}.cpp"' # todo: multiline sample runs
-'"isf"
-   "Schrodinger flow miniapps:"
-   "miniapps/fluids/schrodinger-flow"
-   "{,p}schrodinger_flow.cpp"'
 '"adjoint"
    "Adjoint miniapps:"
    "miniapps/adjoint"
@@ -320,7 +308,7 @@ groups_all=(
 # todo: miniapps/multidomain
 '"navier"
    "Navier miniapps:"
-   "miniapps/fluids/navier"
+   "miniapps/navier"
    "navier_cht.cpp"'
 # todo: add other navier miniapps
 '"nurbs"

data/channel-bifurcation-2d.mesh

@@ -1,156 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see fem/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-# PYRAMID     = 7
-#
-
-dimension
-2
-
-elements
-25
-3 3 0 1 2 3
-3 3 1 4 5 2
-3 3 4 6 7 5
-3 3 6 8 9 7
-3 3 8 10 11 9
-3 3 10 12 13 11
-3 3 12 14 15 13
-3 3 14 16 17 15
-3 3 16 18 19 17
-3 3 18 20 21 19
-3 3 20 22 23 21
-3 3 22 24 25 23
-3 3 24 26 27 25
-3 3 26 28 29 27
-3 3 28 30 31 29
-3 3 30 32 33 31
-3 3 32 34 35 33
-3 3 17 19 36 37
-3 3 37 36 38 39
-3 3 39 38 40 41
-3 3 41 40 42 43
-3 3 43 42 44 45
-3 3 45 44 46 47
-3 3 47 46 48 49
-3 3 49 48 50 51
-
-boundary
-52
-2 1 0 1
-2 1 2 3
-1 1 3 0
-2 1 1 4
-2 1 5 2
-2 1 4 6
-2 1 7 5
-2 1 6 8
-2 1 9 7
-2 1 8 10
-2 1 11 9
-2 1 10 12
-2 1 13 11
-2 1 12 14
-2 1 15 13
-2 1 14 16
-2 1 17 15
-2 1 16 18
-2 1 18 20
-2 1 21 19
-2 1 20 22
-2 1 23 21
-2 1 22 24
-2 1 25 23
-2 1 24 26
-2 1 27 25
-2 1 26 28
-2 1 29 27
-2 1 28 30
-2 1 31 29
-2 1 30 32
-2 1 33 31
-2 1 32 34
-3 1 34 35
-2 1 35 33
-2 1 19 36
-2 1 37 17
-2 1 36 38
-2 1 39 37
-2 1 38 40
-2 1 41 39
-2 1 40 42
-2 1 43 41
-2 1 42 44
-2 1 45 43
-2 1 44 46
-2 1 47 45
-2 1 46 48
-2 1 49 47
-2 1 48 50
-4 1 50 51
-2 1 51 49
-
-vertices
-52
-2
-0 0
-1 0
-1 1
-0 1
-2 0
-2 1
-3 0
-3 1
-4 0
-4 1
-5 0
-5 1
-6 0
-6 1
-7 0
-7 1
-8 0
-8 1
-9 0
-9 1
-10 0
-10 1
-11 0
-11 1
-12 0
-12 1
-13 0
-13 1
-14 0
-14 1
-15 0
-15 1
-16 0
-16 1
-17 0
-17 1
-9 2
-8 2
-9 3
-8 3
-9 4
-8 4
-9 5
-8 5
-9 6
-8 6
-9 7
-8 7
-9 8
-8 8
-9 9
-8 9

data/nurbs-segments2d-patches-multispan.mesh

@@ -1,86 +0,0 @@
-MFEM NURBS mesh v1.0
-
-dimension
-1
-
-# Four segments with different NURBS orders, described via patches.
-elements
-4
-1 1 0 1
-2 1 2 3
-3 1 4 5
-4 1 6 7
-
-boundary
-0
-
-edges
-4
-0 0 1
-1 2 3
-2 4 5
-3 6 7
-
-vertices
-8
-
-patches
-
-# Patch 0: linear (order 1, 3 spans)
-knotvectors
-1
-1 4  0 0  .4 .6 1 1
-
-dimension
-2
-
-controlpoints
-0.0  0.0  1.0
-0.6  0.4  1.0
-0.4  0.6  1.0
-1.0  1.0  1.0
-
-# Patch 1: quadratic (order 2, 2 spans)
-knotvectors
-1
-2 4  0 0 0  .5 1 1 1
-
-dimension
-2
-
-controlpoints
-1.0  0.0  1.0
-1.9  0.0  1.21
-2.0  0.9  1.22
-2.0  1.0  1.0
-
-# Patch 2: cubic (order 3, 3 spans)
-knotvectors
-1
-3 6  0 0 0 0 .33 .66 1 1 1 1
-
-dimension
-2
-
-controlpoints
-2.0   0.0  1.0
-2.1   0.2  1.31
-3.5   0.4  1.32
-2.5   0.6  1.33
-2.9   1.0  1.34
-3.0   1.0  1.0
-
-# Patch 3: quartic (order 4, 1 span)
-knotvectors
-1
-4 5  0 0 0 0 0 1 1 1 1 1
-
-dimension
-2
-
-controlpoints
-3.0    0.0  1.0
-3.45   0.5  1.41
-3.50   1.0  1.42
-3.75   0.8  1.43
-4.0    0.0  1.0

data/nurbs-segments2d-patches.mesh

@@ -1,79 +0,0 @@
-MFEM NURBS mesh v1.0
-
-#
-# MFEM Geometry Types (see fem/geom.hpp):
-#
-# SEGMENT     = 1
-# SQUARE      = 3
-# CUBE        = 5
-#
-
-dimension
-1
-
-# Three segments with different NURBS orders, described via patches.
-elements
-3
-1 1 0 1
-2 1 2 3
-3 1 4 5
-
-boundary
-6
-1 0 0
-1 0 1
-1 0 2
-1 0 3
-1 0 4
-1 0 5
-
-edges
-3
-0 0 1
-1 2 3
-2 4 5
-
-vertices
-6
-
-patches
-
-# Patch 0: linear (order 1, 2 control points)
-knotvectors
-1
-1 2  0 0  1 1
-
-dimension
-2
-
-controlpoints
-0.0  0.0  1.0
-1.0  1.0  1.0
-
-# Patch 1: quadratic (order 2, 3 control points)
-knotvectors
-1
-2 3  0 0 0  1 1 1
-
-dimension
-2
-
-controlpoints
-1.0  0.0   1.0
-1.02 1.02  1.2
-2.0  1.0   1.0
-
-# Patch 2: cubic (order 3, 4 control points)
-knotvectors
-1
-3 4  0 0 0 0  1 1 1 1
-
-dimension
-2
-
-controlpoints
-2.0   0.0   1.0
-2.03  0.83  1.31
-2.33  1.03  1.32
-3.0   1.0   1.0
-

data/nurbs-segments2d.mesh

@@ -1,72 +0,0 @@
-MFEM NURBS mesh v1.0
-
-#
-# MFEM Geometry Types (see fem/geom.hpp):
-#
-# SEGMENT     = 1
-# SQUARE      = 3
-# CUBE        = 5
-#
-
-dimension
-1
-
-elements
-3
-1 1 0 1
-2 1 2 3
-3 1 4 5
-
-boundary
-6
-1 0 0
-1 0 1
-1 0 2
-1 0 3
-1 0 4
-1 0 5
-
-edges
-3
-0 0 1
-1 2 3
-2 4 5
-
-vertices
-6
-
-# Edge 0: linear (order 1, 2 control points)
-# Edge 1: quadratic (order 2, 3 control points)
-# Edge 2: cubic (order 3, 4 control points)
-knotvectors
-3
-1 2  0 0  1 1
-2 3  0 0 0  1 1 1
-3 4  0 0 0 0  1 1 1 1
-
-# One weight per control point, in the same order as the control points; (2 + 3 + 4) = 9 weights total
-weights
-1
-1
-1
-1
-1
-1
-1.2
-1.31
-1.32
-
-FiniteElementSpace
-FiniteElementCollection: NURBS
-VDim: 2
-Ordering: 1
-
-0.0  0.0
-1.0  1.0
-1.0  0.0
-2.0  1.0
-2.0  0.0
-3.0  1.0
-1.02 1.02
-2.03 0.83
-2.33 1.03

data/nurbs-segments3d-patches.mesh

@@ -1,79 +0,0 @@
-MFEM NURBS mesh v1.0
-
-#
-# MFEM Geometry Types (see fem/geom.hpp):
-#
-# SEGMENT     = 1
-# SQUARE      = 3
-# CUBE        = 5
-#
-
-dimension
-1
-
-# Three segments with different NURBS orders, described via patches.
-elements
-3
-1 1 0 1
-2 1 2 3
-3 1 4 5
-
-boundary
-6
-1 0 0
-1 0 1
-1 0 2
-1 0 3
-1 0 4
-1 0 5
-
-edges
-3
-0 0 1
-1 2 3
-2 4 5
-
-vertices
-6
-
-patches
-
-# Patch 0: linear (order 1, 2 control points)
-knotvectors
-1
-1 2  0 0  1 1
-
-dimension
-3
-
-controlpoints
-0.0  0.0  0.01 1.0
-1.0  1.0  1.01 1.0
-
-# Patch 1: quadratic (order 2, 3 control points)
-knotvectors
-1
-2 3  0 0 0  1 1 1
-
-dimension
-3
-
-controlpoints
-1.0  0.0   0.02 1.0
-1.02 1.02  0.52 1.2
-2.0  1.0   1.02 1.0
-
-# Patch 2: cubic (order 3, 4 control points)
-knotvectors
-1
-3 4  0 0 0 0  1 1 1 1
-
-dimension
-3
-
-controlpoints
-2.0   0.0   0.03 1.0
-2.03  0.83  0.33 1.31
-2.33  1.03  0.63 1.32
-3.0   1.0   1.03 1.0
-

data/nurbs-segments3d.mesh

@@ -1,72 +0,0 @@
-MFEM NURBS mesh v1.0
-
-#
-# MFEM Geometry Types (see fem/geom.hpp):
-#
-# SEGMENT     = 1
-# SQUARE      = 3
-# CUBE        = 5
-#
-
-dimension
-1
-
-elements
-3
-1 1 0 1
-2 1 2 3
-3 1 4 5
-
-boundary
-6
-1 0 0
-1 0 1
-1 0 2
-1 0 3
-1 0 4
-1 0 5
-
-edges
-3
-0 0 1
-1 2 3
-2 4 5
-
-vertices
-6
-
-# Edge 0: linear (order 1, 2 control points)
-# Edge 1: quadratic (order 2, 3 control points)
-# Edge 2: cubic (order 3, 4 control points)
-knotvectors
-3
-1 2  0 0  1 1
-2 3  0 0 0  1 1 1
-3 4  0 0 0 0  1 1 1 1
-
-# One weight per control point, in the same order as the control points; (2 + 3 + 4) = 9 weights total
-weights
-1
-1
-1
-1
-1
-1
-1.2
-1.31
-1.32
-
-FiniteElementSpace
-FiniteElementCollection: NURBS
-VDim: 3
-Ordering: 1
-
-0.0  0.0   0.01
-1.0  1.0   1.01
-1.0  0.0   0.02
-2.0  1.0   1.02
-2.0  0.0   0.03
-3.0  1.0   1.03
-1.02 1.02  0.52
-2.03 0.83  0.33
-2.33 1.03  0.63

doc/CodeDocumentation.conf.in

@@ -48,7 +48,7 @@ PROJECT_NAME           = MFEM
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = v4.9.1
+PROJECT_NUMBER         = v4.8.1
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -973,13 +973,10 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/miniapps/adjoint \
                          @MFEM_SOURCE_DIR@/miniapps/autodiff \
                          @MFEM_SOURCE_DIR@/miniapps/common \
-                         @MFEM_SOURCE_DIR@/miniapps/contact \
                          @MFEM_SOURCE_DIR@/miniapps/dfem \
                          @MFEM_SOURCE_DIR@/miniapps/dpg \
                          @MFEM_SOURCE_DIR@/miniapps/dpg/util \
                          @MFEM_SOURCE_DIR@/miniapps/electromagnetics \
-                         @MFEM_SOURCE_DIR@/miniapps/fluids/navier \
-                         @MFEM_SOURCE_DIR@/miniapps/fluids/schrodinger-flow \
                          @MFEM_SOURCE_DIR@/miniapps/gslib \
                          @MFEM_SOURCE_DIR@/miniapps/hdiv-linear-solver \
                          @MFEM_SOURCE_DIR@/miniapps/hooke \
@@ -990,6 +987,7 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/miniapps/meshing \
                          @MFEM_SOURCE_DIR@/miniapps/mtop \
                          @MFEM_SOURCE_DIR@/miniapps/multidomain \
+                         @MFEM_SOURCE_DIR@/miniapps/navier \
                          @MFEM_SOURCE_DIR@/miniapps/nurbs \
                          @MFEM_SOURCE_DIR@/miniapps/parelag \
                          @MFEM_SOURCE_DIR@/miniapps/performance \

doc/CodeDocumentation.dox

@@ -117,8 +117,6 @@ namespace mfem {
  * - <a class="el" href="ex39p_8cpp_source.html">Example 39p</a>: parallel named mesh attributes
  * - <a class="el" href="ex40_8cpp_source.html">Example 40</a>: eikonal equation
  * - <a class="el" href="ex40p_8cpp_source.html">Example 40p</a>: parallel eikonal equation
- * - <a class="el" href="ex41_8cpp_source.html">Example 41</a>: DG/CG IMEX time-dependent advection-diffusion
- * - <a class="el" href="ex41p_8cpp_source.html">Example 41p</a>: parallel DG/CG IMEX time-dependent advection-diffusion
  *
  * <H4>AmgX Examples</H4>
  * - Variants of Examples
@@ -190,8 +188,6 @@ namespace mfem {
  *  <a class="el" href="nurbs__ex1p_8cpp_source.html">1p</a>,
  *  <a class="el" href="nurbs__ex3_8cpp_source.html">3</a>,
  *  <a class="el" href="nurbs__ex5_8cpp_source.html">5</a>,
- *  <a class="el" href="nurbs__ex10_8cpp_source.html">10</a>,
- *  <a class="el" href="nurbs__ex10p_8cpp_source.html">10p</a>,
  *  <a class="el" href="nurbs__ex11p_8cpp_source.html">11p</a>, and
  *  <a class="el" href="nurbs__ex24_8cpp_source.html">24</a>,
  *   demonstrating howto perform NURBS-based Isogeometric Analysis.
@@ -200,7 +196,6 @@ namespace mfem {
  * - <a class="el" href="nurbs__curveint_8cpp_source.html">NURBS Interpolation</a>: NURBS interpolation of given geometry
  * - <a class="el" href="nurbs__naca__cmesh_8cpp_source.html">NURBS NACA Mesher</a>: generate NURBS based mesh around a NACA foil
  * - <a class="el" href="nurbs__printfunc_8cpp_source.html">NURBS Printer</a>: print the NURBS-basis
- * - <a class="el" href="nurbs__mesh_info_8cpp_source.html">NURBS Mesh info</a>: print the info of a NURBS mesh
  *
  * <H3>Miniapps</H3>
  * - <a class="el" href="volta_8cpp_source.html">Volta</a>: simple electrostatics simulation code
@@ -239,8 +234,7 @@ namespace mfem {
  * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Poisson problem
  * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Poisson problem
  * - <a class="el" href="generate__random__field_8cpp_source.html">SPDE Solvers</a>: SPDE solver random field generation
- * - <a class="el" href="contact-patch-test_8cpp_source.html">Tribol</a>: mortar contact patch test for elasticity
- * - <a class="el" href="contact_8cpp_source.html">Contact</a>: Frictionless contact examples using <a class="el" href="classmfem_1_1IPSolver.html#details">IP optimization</a> and the <a class="el" href="classmfem_1_1AMGFSolver.html#details">AMGF solver</a>
+ * - <a class="el" href="contact-patch-test_8cpp_source.html">Contact</a>: mortar contact patch test for elasticity
  * - <a class="el" href="multidomain_8cpp_source.html">Multidomain miniapp</a>: Multidomain and Submesh demonstration miniapp
  * - <a class="el" href="pdiffusion_8cpp_source.html">DPG Diffusion example</a>: DPG formulation for the diffusion problem
  * - <a class="el" href="pmaxwell_8cpp_source.html">DPG Maxwell example</a>: DPG formulation for the indefinite Maxwell problem

examples/CMakeLists.txt

@@ -46,7 +46,6 @@ list(APPEND ALL_EXE_SRCS
   ex38.cpp
   ex39.cpp
   ex40.cpp
-  ex41.cpp
   )
 
 if (MFEM_USE_MPI)
@@ -90,7 +89,6 @@ if (MFEM_USE_MPI)
     ex37p.cpp
     ex39p.cpp
     ex40p.cpp
-    ex41p.cpp
   )
 endif()
 
@@ -133,8 +131,6 @@ if (MFEM_ENABLE_TESTING)
       list(APPEND THIS_TEST_OPTIONS "-dg")
     elseif(${TEST_NAME} MATCHES "ex37p*")
       list(APPEND THIS_TEST_OPTIONS "-mi" "3")
-    elseif(${TEST_NAME} MATCHES "ex41p*")
-      list(APPEND THIS_TEST_OPTIONS "-tf" "1.0")
     endif()
 
     if (NOT (${TEST_NAME} MATCHES ".*p$"))
@@ -215,7 +211,7 @@ if (MFEM_ENABLE_TESTING)
     add_test(NAME ex1p_ceed_np=${MFEM_MPI_NP}
       COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
       ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:ex1p> "-no-vis" "-d" "ceed-cpu" "-pa" "-a"
+      $<TARGET_FILE:ex1p> "-no-vis" "-d ceed-cpu" "-pa" "-a"
       ${MPIEXEC_POSTFLAGS})
   endif()
 endif()

examples/amgx/makefile

@@ -64,7 +64,7 @@ PARALLEL_NAME := Parallel AMGX example
 $(MFEM_LIB_FILE):
 	$(error The MFEM library is not build)
 
-clean: clean-build clean-exec
+clean: clean-build
 
 clean-build:
 	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)

examples/caliper/makefile

@@ -64,12 +64,12 @@ ex1p-test-par: ex1p
 $(MFEM_LIB_FILE):
 	$(error The MFEM library is not built)
 
-clean: clean-build clean-exec
+clean: clean-build clean-exec $(SUBDIRS_CLEAN)
 
 clean-build:
 	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
 	rm -rf *.dSYM *.TVD.*breakpoints
 
 clean-exec:
-	@rm -f refined.mesh mesh.*
-	@rm -f sol.*
+	@rm -f refined.mesh displaced.mesh mesh.* ex5.mesh
+	@rm -f sphere_refined.* sol.* sol_u.* sol_p.* sol_r.* sol_i.*

examples/ex12p.cpp

@@ -5,9 +5,9 @@
 // Sample runs:
 //    mpirun -np 4 ex12p -m ../data/beam-tri.mesh
 //    mpirun -np 4 ex12p -m ../data/beam-quad.mesh
-//    mpirun -np 4 ex12p -m ../data/beam-tet.mesh -s 464 -n 10 -o 2 -elast
+//    mpirun -np 4 ex12p -m ../data/beam-tet.mesh -s 462 -n 10 -o 2 -elast
 //    mpirun -np 4 ex12p -m ../data/beam-hex.mesh -s 3878
-//    mpirun -np 4 ex12p -m ../data/beam-wedge.mesh -s 82
+//    mpirun -np 4 ex12p -m ../data/beam-wedge.mesh -s 81
 //    mpirun -np 4 ex12p -m ../data/beam-tri.mesh -s 3877 -o 2 -sys
 //    mpirun -np 4 ex12p -m ../data/beam-quad.mesh -s 4544 -n 6 -o 3 -elast
 //    mpirun -np 4 ex12p -m ../data/beam-quad-nurbs.mesh

examples/ex16.cpp

@@ -105,7 +105,6 @@ int main(int argc, char *argv[])
    bool visualization = true;
    bool visit = false;
    int vis_steps = 5;
-   bool solve_implicit_state = false;
 
    int precision = 8;
    cout.precision(precision);
@@ -127,9 +126,6 @@ int main(int argc, char *argv[])
                   "Alpha coefficient.");
    args.AddOption(&kappa, "-k", "--kappa",
                   "Kappa coefficient offset.");
-   args.AddOption(&solve_implicit_state, "-imp-state", "--implicit-state",
-                  "-imp-slope", "--implicit-slope",
-                  "Implicitly solve for stage state or slope.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -183,11 +179,6 @@ int main(int argc, char *argv[])
 
    // 7. Initialize the conduction operator and the visualization.
    ConductionOperator oper(fespace, alpha, kappa, u);
-   using ImplicitVariableType = ConductionOperator::ImplicitVariableType;
-   ImplicitVariableType imp_var = solve_implicit_state ?
-                                  ImplicitVariableType::STATE
-                                  : ImplicitVariableType::SLOPE;
-   oper.SetImplicitVariableType(imp_var);
 
    u_gf.SetFromTrueDofs(u);
    {
@@ -325,14 +316,11 @@ void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
 }
 
 void ConductionOperator::ImplicitSolve(const real_t dt,
-                                       const Vector &u, Vector &k)
+                                       const Vector &u, Vector &du_dt)
 {
    // Solve the equation:
-   //    M*k = -K(u + dt*k) for k = du/dt, if solving for stage-slope
-   // or
-   //    M*k = -dt*K(k) + M*u for k = u_s, if solving for stage-state
-   // where K is linearized by using u from the previous timestep, and
-   // the stage-state and slope relation: du/dt = (u_s - u)/dt.
+   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
+   // for du_dt, where K is linearized by using u from the previous timestep
    if (!T)
    {
       T = Add(1.0, Mmat, dt, Kmat);
@@ -340,20 +328,9 @@ void ConductionOperator::ImplicitSolve(const real_t dt,
       T_solver.SetOperator(*T);
    }
    MFEM_VERIFY(dt == current_dt, ""); // SDIRK methods use the same dt
-
-   // Construct current right-hand side for stage state vs. slope solve
-   if (ImplicitVarTypeIsState())
-   {
-      // k, on return, is the stage value u_s
-      Mmat.Mult(u, z);
-   }
-   else
-   {
-      // k, on return, is the stage slope du/dt
-      Kmat.Mult(u, z);
-      z.Neg();
-   }
-   T_solver.Mult(z, k);
+   Kmat.Mult(u, z);
+   z.Neg();
+   T_solver.Mult(z, du_dt);
 }
 
 void ConductionOperator::SetParameters(const Vector &u)

examples/ex16p.cpp

@@ -115,7 +115,6 @@ int main(int argc, char *argv[])
    bool visit = false;
    int vis_steps = 5;
    bool adios2 = false;
-   bool solve_implicit_state = false;
 
    int precision = 8;
    cout.precision(precision);
@@ -139,9 +138,6 @@ int main(int argc, char *argv[])
                   "Alpha coefficient.");
    args.AddOption(&kappa, "-k", "--kappa",
                   "Kappa coefficient offset.");
-   args.AddOption(&solve_implicit_state, "-imp-state", "--implicit-state",
-                  "-imp-slope", "--implicit-slope",
-                  "Implicitly solve for stage state or slope.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -216,11 +212,6 @@ int main(int argc, char *argv[])
 
    // 9. Initialize the conduction operator and the VisIt visualization.
    ConductionOperator oper(fespace, alpha, kappa, u);
-   using ImplicitVariableType = ConductionOperator::ImplicitVariableType;
-   ImplicitVariableType imp_var = solve_implicit_state ?
-                                  ImplicitVariableType::STATE
-                                  : ImplicitVariableType::SLOPE;
-   oper.SetImplicitVariableType(imp_var);
 
    u_gf.SetFromTrueDofs(u);
    {
@@ -416,14 +407,11 @@ void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
 }
 
 void ConductionOperator::ImplicitSolve(const real_t dt,
-                                       const Vector &u, Vector &k)
+                                       const Vector &u, Vector &du_dt)
 {
    // Solve the equation:
-   //    M*k = -K(u + dt*k) for k = du/dt, if solving for stage-slope
-   // or
-   //    M*k = -dt*K(k) + M*u for k = u_s, if solving for stage-state
-   // where K is linearized by using u from the previous timestep, and
-   // the stage-state and slope relation: du/dt = (u_s - u)/dt.
+   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
+   // for du_dt, where K is linearized by using u from the previous timestep
    if (!T)
    {
       T = Add(1.0, Mmat, dt, Kmat);
@@ -431,20 +419,9 @@ void ConductionOperator::ImplicitSolve(const real_t dt,
       T_solver.SetOperator(*T);
    }
    MFEM_VERIFY(dt == current_dt, ""); // SDIRK methods use the same dt
-
-   // Construct current right-hand side for stage state vs. slope solve
-   if (ImplicitVarTypeIsState())
-   {
-      // k, on return, is the stage value u
-      Mmat.Mult(u, z);
-   }
-   else
-   {
-      // k, on return, is the stage slope du/dt
-      Kmat.Mult(u, z);
-      z.Neg();
-   }
-   T_solver.Mult(z, k);
+   Kmat.Mult(u, z);
+   z.Neg();
+   T_solver.Mult(z, du_dt);
 }
 
 void ConductionOperator::SetParameters(const Vector &u)

examples/ex21.cpp

@@ -119,7 +119,7 @@ int main(int argc, char *argv[])
    }
 
    LinearForm b(&fespace);
-   b.AddBoundaryIntegrator(new VectorBoundaryLFIntegrator(f));
+   b.AddDomainIntegrator(new VectorBoundaryLFIntegrator(f));
 
    // 6. Set up the bilinear form a(.,.) on the finite element space
    //    corresponding to the linear elasticity integrator with piece-wise

examples/ex21p.cpp

@@ -140,7 +140,7 @@ int main(int argc, char *argv[])
    }
 
    ParLinearForm b(&fespace);
-   b.AddBoundaryIntegrator(new VectorBoundaryLFIntegrator(f));
+   b.AddDomainIntegrator(new VectorBoundaryLFIntegrator(f));
 
    // 6. Set up the bilinear form a(.,.) on the finite element space
    //    corresponding to the linear elasticity integrator with piece-wise

examples/ex22.cpp

@@ -471,13 +471,10 @@ int main(int argc, char *argv[])
 
       ofstream sol_r_ofs("sol_r.gf");
       ofstream sol_i_ofs("sol_i.gf");
-      ofstream sol_z_ofs("sol_z.gf");
       sol_r_ofs.precision(8);
       sol_i_ofs.precision(8);
-      sol_z_ofs.precision(8);
       u.real().Save(sol_r_ofs);
       u.imag().Save(sol_i_ofs);
-      u.Save(sol_z_ofs);
    }
 
    // 14. Send the solution by socket to a GLVis server.

examples/ex22p.cpp

@@ -302,21 +302,15 @@ int main(int argc, char *argv[])
       char vishost[] = "localhost";
       int  visport   = 19916;
       socketstream sol_sock_r(vishost, visport);
+      socketstream sol_sock_i(vishost, visport);
       sol_sock_r << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
       sol_sock_r.precision(8);
+      sol_sock_i.precision(8);
       sol_sock_r << "solution\n" << *pmesh << u_exact->real()
                  << "window_title 'Exact: Real Part'" << flush;
-      // Make sure all ranks have sent their real solution before initiating
-      // another set of GLVis connections (one from each rank):
-      MPI_Barrier(pmesh->GetComm());
-      socketstream sol_sock_i(vishost, visport);
-      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock_i.precision(8);
       sol_sock_i << "solution\n" << *pmesh << u_exact->imag()
                  << "window_title 'Exact: Imaginary Part'" << flush;
-      // Make sure all ranks have sent their imaginary solution before initiating
-      // another set of GLVis connections (one from each rank):
-      MPI_Barrier(pmesh->GetComm());
    }
 
    // 11. Set up the parallel sesquilinear form a(.,.) on the finite element
@@ -513,11 +507,10 @@ int main(int argc, char *argv[])
    // 15. Save the refined mesh and the solution in parallel. This output can be
    //     viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
    {
-      ostringstream mesh_name, sol_r_name, sol_i_name, sol_z_name;
+      ostringstream mesh_name, sol_r_name, sol_i_name;
       mesh_name << "mesh." << setfill('0') << setw(6) << myid;
       sol_r_name << "sol_r." << setfill('0') << setw(6) << myid;
       sol_i_name << "sol_i." << setfill('0') << setw(6) << myid;
-      sol_z_name << "sol_z." << setfill('0') << setw(6) << myid;
 
       ofstream mesh_ofs(mesh_name.str().c_str());
       mesh_ofs.precision(8);
@@ -525,13 +518,10 @@ int main(int argc, char *argv[])
 
       ofstream sol_r_ofs(sol_r_name.str().c_str());
       ofstream sol_i_ofs(sol_i_name.str().c_str());
-      ofstream sol_z_ofs(sol_z_name.str().c_str());
       sol_r_ofs.precision(8);
       sol_i_ofs.precision(8);
-      sol_z_ofs.precision(8);
       u.real().Save(sol_r_ofs);
       u.imag().Save(sol_i_ofs);
-      u.Save(sol_z_ofs);
    }
 
    // 16. Send the solution by socket to a GLVis server.
@@ -540,21 +530,15 @@ int main(int argc, char *argv[])
       char vishost[] = "localhost";
       int  visport   = 19916;
       socketstream sol_sock_r(vishost, visport);
+      socketstream sol_sock_i(vishost, visport);
       sol_sock_r << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
       sol_sock_r.precision(8);
+      sol_sock_i.precision(8);
       sol_sock_r << "solution\n" << *pmesh << u.real()
                  << "window_title 'Solution: Real Part'" << flush;
-      // Make sure all ranks have sent their real solution before initiating
-      // another set of GLVis connections (one from each rank):
-      MPI_Barrier(pmesh->GetComm());
-      socketstream sol_sock_i(vishost, visport);
-      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock_i.precision(8);
       sol_sock_i << "solution\n" << *pmesh << u.imag()
                  << "window_title 'Solution: Imaginary Part'" << flush;
-      // Make sure all ranks have sent their imaginary solution before initiating
-      // another set of GLVis connections (one from each rank):
-      MPI_Barrier(pmesh->GetComm());
    }
    if (visualization && exact_sol)
    {
@@ -563,21 +547,15 @@ int main(int argc, char *argv[])
       char vishost[] = "localhost";
       int  visport   = 19916;
       socketstream sol_sock_r(vishost, visport);
+      socketstream sol_sock_i(vishost, visport);
       sol_sock_r << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
       sol_sock_r.precision(8);
+      sol_sock_i.precision(8);
       sol_sock_r << "solution\n" << *pmesh << u_exact->real()
                  << "window_title 'Error: Real Part'" << flush;
-      // Make sure all ranks have sent their real solution before initiating
-      // another set of GLVis connections (one from each rank):
-      MPI_Barrier(pmesh->GetComm());
-      socketstream sol_sock_i(vishost, visport);
-      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock_i.precision(8);
       sol_sock_i << "solution\n" << *pmesh << u_exact->imag()
                  << "window_title 'Error: Imaginary Part'" << flush;
-      // Make sure all ranks have sent their imaginary solution before initiating
-      // another set of GLVis connections (one from each rank):
-      MPI_Barrier(pmesh->GetComm());
    }
    if (visualization)
    {

examples/ex37.cpp

@@ -5,8 +5,8 @@
 // Sample runs:
 //     ex37 -alpha 10
 //     ex37 -alpha 10 -pv
-//     ex37 -lambda 0.1 -mu 0.1 -growth 1
-//     ex37 -o 2 -alpha 10.0 -mi 50 -vf 0.4 -ntol 1e-5 -growth 1.5
+//     ex37 -lambda 0.1 -mu 0.1
+//     ex37 -o 2 -alpha 5.0 -mi 50 -vf 0.4 -ntol 1e-5
 //     ex37 -r 6 -o 1 -alpha 25.0 -epsilon 0.02 -mi 50 -ntol 1e-5
 //
 // Description: This example code demonstrates the use of MFEM to solve a
@@ -55,6 +55,53 @@
 using namespace std;
 using namespace mfem;
 
+/**
+ * @brief Bregman projection of ρ = sigmoid(ψ) onto the subspace
+ *        ∫_Ω ρ dx = θ vol(Ω) as follows:
+ *
+ *        1. Compute the root of the R → R function
+ *            f(c) = ∫_Ω sigmoid(ψ + c) dx - θ vol(Ω)
+ *        2. Set ψ ← ψ + c.
+ *
+ * @param psi a GridFunction to be updated
+ * @param target_volume θ vol(Ω)
+ * @param tol Newton iteration tolerance
+ * @param max_its Newton maximum iteration number
+ * @return real_t Final volume, ∫_Ω sigmoid(ψ)
+ */
+real_t proj(GridFunction &psi, real_t target_volume, real_t tol=1e-12,
+            int max_its=10)
+{
+   MappedGridFunctionCoefficient sigmoid_psi(&psi, sigmoid);
+   MappedGridFunctionCoefficient der_sigmoid_psi(&psi, der_sigmoid);
+
+   LinearForm int_sigmoid_psi(psi.FESpace());
+   int_sigmoid_psi.AddDomainIntegrator(new DomainLFIntegrator(sigmoid_psi));
+   LinearForm int_der_sigmoid_psi(psi.FESpace());
+   int_der_sigmoid_psi.AddDomainIntegrator(new DomainLFIntegrator(
+                                              der_sigmoid_psi));
+   bool done = false;
+   for (int k=0; k<max_its; k++) // Newton iteration
+   {
+      int_sigmoid_psi.Assemble(); // Recompute f(c) with updated ψ
+      const real_t f = int_sigmoid_psi.Sum() - target_volume;
+
+      int_der_sigmoid_psi.Assemble(); // Recompute df(c) with updated ψ
+      const real_t df = int_der_sigmoid_psi.Sum();
+
+      const real_t dc = -f/df;
+      psi += dc;
+      if (abs(dc) < tol) { done = true; break; }
+   }
+   if (!done)
+   {
+      mfem_warning("Projection reached maximum iteration without converging. "
+                   "Result may not be accurate.");
+   }
+   int_sigmoid_psi.Assemble();
+   return int_sigmoid_psi.Sum();
+}
+
 /*
  * ---------------------------------------------------------------
  *                      ALGORITHM PREAMBLE
@@ -133,11 +180,10 @@ int main(int argc, char *argv[])
    int ref_levels = 5;
    int order = 2;
    real_t alpha = 1.0;
-   real_t growth = 2;
    real_t epsilon = 0.01;
    real_t vol_fraction = 0.5;
    int max_it = 1e3;
-   real_t itol = 1e-2;
+   real_t itol = 1e-1;
    real_t ntol = 1e-4;
    real_t rho_min = 1e-6;
    real_t lambda = 1.0;
@@ -152,8 +198,6 @@ int main(int argc, char *argv[])
                   "Order (degree) of the finite elements.");
    args.AddOption(&alpha, "-alpha", "--alpha-step-length",
                   "Step length for gradient descent.");
-   args.AddOption(&growth, "-growth", "--alpha-growth-rate",
-                  "Growth rate of step length for gradient descent.");
    args.AddOption(&epsilon, "-epsilon", "--epsilon-thickness",
                   "Length scale for ρ.");
    args.AddOption(&max_it, "-mi", "--max-it",
@@ -288,7 +332,6 @@ int main(int argc, char *argv[])
    }
    FilterSolver->SetEssentialBoundary(ess_bdr_filter);
    FilterSolver->SetupFEM();
-   FilterSolver->AssembleDiffusionBilinear();
 
    BilinearForm mass(&control_fes);
    mass.AddDomainIntegrator(new InverseIntegrator(new MassIntegrator(one)));
@@ -342,7 +385,7 @@ int main(int argc, char *argv[])
    // 11. Iterate:
    for (int k = 1; k <= max_it; k++)
    {
-      if (k > 1) { alpha = std::pow((real_t) k,growth); }
+      if (k > 1) { alpha *= ((real_t) k) / ((real_t) k-1); }
 
       mfem::out << "\nStep = " << k << std::endl;
 
@@ -379,9 +422,7 @@ int main(int argc, char *argv[])
 
       // Step 5 - Update design variable ψ ← proj(ψ - αG)
       psi.Add(-alpha, grad);
-      GridFunction alpha_grad(grad);
-      alpha_grad *= alpha;
-      const real_t material_volume = proj(psi, alpha_grad, target_volume);
+      const real_t material_volume = proj(psi, target_volume);
 
       // Compute ||ρ - ρ_old|| in control fes.
       real_t norm_increment = zerogf.ComputeL1Error(succ_diff_rho);

examples/ex37.hpp

@@ -137,7 +137,7 @@ public:
         exponent(exponent_), rho_min(rho_min_)
    {
       MFEM_ASSERT(rho_min_ >= 0.0, "rho_min must be >= 0");
-      MFEM_ASSERT(rho_min_ < 1.0,  "rho_min must be < 1");
+      MFEM_ASSERT(rho_min_ < 1.0,  "rho_min must be > 1");
       MFEM_ASSERT(u, "displacement field is not set");
       MFEM_ASSERT(rho_filter, "density field is not set");
    }
@@ -231,12 +231,9 @@ private:
    FiniteElementCollection * fec = nullptr;
    FiniteElementSpace * fes = nullptr;
    Array<int> ess_bdr;
-   Array<int> ess_tdof_list;
    Array<int> neumann_bdr;
    GridFunction * u = nullptr;
    LinearForm * b = nullptr;
-   BilinearForm * a = nullptr;
-   OperatorPtr A;
    bool parallel;
 #ifdef MFEM_USE_MPI
    ParMesh * pmesh = nullptr;
@@ -270,8 +267,6 @@ public:
    void ResetFEM();
    void SetupFEM();
 
-   void UpdateEssentialTDofs();
-   void AssembleDiffusionBilinear(bool update_ess_tdofs=true);
    void Solve();
    GridFunction * GetFEMSolution();
    LinearForm * GetLinearForm() {return b;}
@@ -376,130 +371,6 @@ public:
 
 };
 
-/**
- * @brief Bregman projection of ρ = sigmoid(ψ) onto the subspace
- *        ∫_Ω ρ dx = θ vol(Ω) as follows:
- *
- *        1. Compute the root of the R → R function
- *            f(c) = ∫_Ω sigmoid(ψ + c) dx - θ vol(Ω)
- *           using the Illinois method
- *        2. Set ψ ← ψ + c.
- *
- * @param psi a GridFunction to be updated
- * @param alpha_grad alpha multiplied by gradient
- * @param target_volume θ vol(Ω)
- * @param tol Illinois iteration tolerance
- * @param max_its Illinois maximum iteration number
- * @return real_t Final volume (∫_Ω sigmoid(ψ) dx)
- */
-real_t proj(GridFunction &psi, GridFunction &alpha_grad, real_t target_volume,
-            real_t tol = 1e-12, int max_its = 100)
-{
-#ifdef MFEM_USE_MPI
-   FiniteElementSpace *fes = psi.FESpace();
-   ParFiniteElementSpace *pfes = dynamic_cast<ParFiniteElementSpace*>(fes);
-#endif
-   ConstantCoefficient zero_cf(0.0);
-   real_t a = -alpha_grad.ComputeMaxError(zero_cf);
-   real_t b = -a;
-   real_t y = 0.0;
-
-   MappedGridFunctionCoefficient sigmoid_psi(
-   &psi, [&y](const real_t x) { return sigmoid(x + y); });
-   std::unique_ptr<LinearForm> int_sigmoid_psi;
-#ifdef MFEM_USE_MPI
-   ParGridFunction *par_psi = dynamic_cast<ParGridFunction *>(&psi);
-   if (par_psi)
-   {
-      int_sigmoid_psi.reset(new ParLinearForm(par_psi->ParFESpace()));
-   }
-   else
-   {
-      int_sigmoid_psi.reset(new LinearForm(psi.FESpace()));
-   }
-#else
-   int_sigmoid_psi.reset(new LinearForm(psi.FESpace()));
-#endif
-   int_sigmoid_psi->AddDomainIntegrator(new DomainLFIntegrator(sigmoid_psi));
-
-   y = a;
-   int_sigmoid_psi->Assemble();
-   real_t f_a = int_sigmoid_psi->Sum();  // f_a := f(a) + θ vol(Ω)
-
-   y = b;
-   int_sigmoid_psi->Assemble();
-   real_t f_b = int_sigmoid_psi->Sum();  // f_b := f(b) + θ vol(Ω)
-#ifdef MFEM_USE_MPI
-   if (pfes)
-   {
-      MPI_Allreduce(MPI_IN_PLACE, &f_a, 1, MPITypeMap<real_t>::mpi_type,
-                    MPI_SUM, MPI_COMM_WORLD);
-      MPI_Allreduce(MPI_IN_PLACE, &f_b, 1, MPITypeMap<real_t>::mpi_type,
-                    MPI_SUM, MPI_COMM_WORLD);
-   }
-#endif
-   f_a -= target_volume; // f_a := f(a)
-   f_b -= target_volume; // f_b := f(b)
-   real_t c = 0.0;
-   real_t f_c = 0.0;
-   int side = 0;
-
-   bool done = false;
-   for (int k=0; k < max_its; k++)
-   {
-      c = (f_a * b - f_b * a) / (f_a - f_b);
-
-      if (abs(b - a) < tol * abs(b + a)) { done = true; break; }
-
-      y = c;
-      int_sigmoid_psi->Assemble();
-      f_c = int_sigmoid_psi->Sum(); // f_c := f(c) + θ vol(Ω)
-#ifdef MFEM_USE_MPI
-      if (pfes)
-      {
-         MPI_Allreduce(MPI_IN_PLACE, &f_c, 1, MPITypeMap<real_t>::mpi_type,
-                       MPI_SUM, MPI_COMM_WORLD);
-      }
-#endif
-      f_c -= target_volume; // f_c := f(c)
-
-      if (f_c * f_b > 0)
-      {
-         b = c;
-         f_b = f_c;
-         if (side == -1) { f_a /= 2.0; }
-         side = -1;
-      }
-      else if (f_c * f_a > 0)
-      {
-         a = c;
-         f_a = f_c;
-         if (side == 1) { f_b /= 2.0; }
-         side = 1;
-      }
-      else
-      {
-         done = true; break;
-      }
-   }
-   if (!done)
-   {
-      mfem_warning("Projection reached maximum iteration without converging. "
-                   "Result may not be accurate.");
-   }
-   y = 0.0;
-   psi += c;
-   int_sigmoid_psi->Assemble();
-   real_t material_volume = int_sigmoid_psi->Sum();
-#ifdef MFEM_USE_MPI
-   if (pfes)
-   {
-      MPI_Allreduce(MPI_IN_PLACE, &material_volume, 1,
-                    MPITypeMap<real_t>::mpi_type, MPI_SUM, MPI_COMM_WORLD);
-   }
-#endif
-   return material_volume;
-}
 
 // Poisson solver
 
@@ -551,8 +422,12 @@ void DiffusionSolver::SetupFEM()
    }
 }
 
-void DiffusionSolver::UpdateEssentialTDofs()
+void DiffusionSolver::Solve()
 {
+   OperatorPtr A;
+   Vector B, X;
+   Array<int> ess_tdof_list;
+
 #ifdef MFEM_USE_MPI
    if (parallel)
    {
@@ -565,39 +440,7 @@ void DiffusionSolver::UpdateEssentialTDofs()
 #else
    fes->GetEssentialTrueDofs(ess_bdr,ess_tdof_list);
 #endif
-}
-
-void DiffusionSolver::AssembleDiffusionBilinear(bool update_ess_tdofs)
-{
-   if (update_ess_tdofs)
-   {
-      UpdateEssentialTDofs();
-   }
-#ifdef MFEM_USE_MPI
-   if (parallel)
-   {
-      a = new ParBilinearForm(pfes);
-   }
-   else
-   {
-      a = new BilinearForm(fes);
-   }
-#else
-   a = new BilinearForm(fes);
-#endif
-   a->AddDomainIntegrator(new DiffusionIntegrator(*diffcf));
-   if (masscf)
-   {
-      a->AddDomainIntegrator(new MassIntegrator(*masscf));
-   }
-   a->Assemble();
-   a->FormSystemMatrix(ess_tdof_list, A);
-}
-
-void DiffusionSolver::Solve()
-{
-   Vector B, X;
-
+   *u=0.0;
    if (b)
    {
       delete b;
@@ -632,33 +475,31 @@ void DiffusionSolver::Solve()
 
    b->Assemble();
 
-   *u=0.0;
-   if (essbdr_cf)
-   {
-      u->ProjectBdrCoefficient(*essbdr_cf,ess_bdr);
-   }
+   BilinearForm * a = nullptr;
 
 #ifdef MFEM_USE_MPI
    if (parallel)
    {
-      X.SetSize(pfes->TrueVSize());
-      B.SetSize(pfes->TrueVSize());
-      dynamic_cast<ParGridFunction*>(u)->ParallelAssemble(X);
-      dynamic_cast<ParLinearForm*>(b)->ParallelAssemble(B);
-      dynamic_cast<ParBilinearForm*>(a)->ParallelEliminateTDofsInRHS(
-         ess_tdof_list, X, B);
+      a = new ParBilinearForm(pfes);
    }
    else
    {
-      X.NewDataAndSize(u->GetData(), u->Size());
-      B.NewDataAndSize(b->GetData(), b->Size());
-      a->EliminateVDofsInRHS(ess_tdof_list, X, B);
+      a = new BilinearForm(fes);
    }
 #else
-   X.NewDataAndSize(u->GetData(), u->Size());
-   B.NewDataAndSize(b->GetData(), b->Size());
-   a->EliminateVDofsInRHS(ess_tdof_list, X, B);
+   a = new BilinearForm(fes);
 #endif
+   a->AddDomainIntegrator(new DiffusionIntegrator(*diffcf));
+   if (masscf)
+   {
+      a->AddDomainIntegrator(new MassIntegrator(*masscf));
+   }
+   a->Assemble();
+   if (essbdr_cf)
+   {
+      u->ProjectBdrCoefficient(*essbdr_cf,ess_bdr);
+   }
+   a->FormLinearSystem(ess_tdof_list, *u, *b, A, X, B);
 
    CGSolver * cg = nullptr;
    Solver * M = nullptr;
@@ -687,6 +528,7 @@ void DiffusionSolver::Solve()
    delete M;
    delete cg;
    a->RecoverFEMSolution(X, *b, *u);
+   delete a;
 }
 
 GridFunction * DiffusionSolver::GetFEMSolution()
@@ -718,8 +560,6 @@ DiffusionSolver::~DiffusionSolver()
 #endif
    delete fec; fec = nullptr;
    delete b;
-   A.Clear();
-   delete a;
 }
 
 

examples/ex37p.cpp

@@ -4,8 +4,8 @@
 //
 // Sample runs:
 //    mpirun -np 4 ex37p -alpha 10 -pv
-//    mpirun -np 4 ex37p -lambda 0.1 -mu 0.1 -growth 1
-//    mpirun -np 4 ex37p -o 2 -alpha 10.0 -mi 50 -vf 0.4 -ntol 1e-5 -growth 1.5
+//    mpirun -np 4 ex37p -lambda 0.1 -mu 0.1
+//    mpirun -np 4 ex37p -o 2 -alpha 5.0 -mi 50 -vf 0.4 -ntol 1e-5
 //    mpirun -np 4 ex37p -r 6 -o 2 -alpha 10.0 -epsilon 0.02 -mi 50 -ntol 1e-5
 //
 // Description: This example code demonstrates the use of MFEM to solve a
@@ -54,6 +54,61 @@
 using namespace std;
 using namespace mfem;
 
+/**
+ * @brief Bregman projection of ρ = sigmoid(ψ) onto the subspace
+ *        ∫_Ω ρ dx = θ vol(Ω) as follows:
+ *
+ *        1. Compute the root of the R → R function
+ *            f(c) = ∫_Ω sigmoid(ψ + c) dx - θ vol(Ω)
+ *        2. Set ψ ← ψ + c.
+ *
+ * @param psi a GridFunction to be updated
+ * @param target_volume θ vol(Ω)
+ * @param tol Newton iteration tolerance
+ * @param max_its Newton maximum iteration number
+ * @return real_t Final volume, ∫_Ω sigmoid(ψ)
+ */
+real_t proj(ParGridFunction &psi, real_t target_volume, real_t tol=1e-12,
+            int max_its=10)
+{
+   MappedGridFunctionCoefficient sigmoid_psi(&psi, sigmoid);
+   MappedGridFunctionCoefficient der_sigmoid_psi(&psi, der_sigmoid);
+
+   ParLinearForm int_sigmoid_psi(psi.ParFESpace());
+   int_sigmoid_psi.AddDomainIntegrator(new DomainLFIntegrator(sigmoid_psi));
+   ParLinearForm int_der_sigmoid_psi(psi.ParFESpace());
+   int_der_sigmoid_psi.AddDomainIntegrator(new DomainLFIntegrator(
+                                              der_sigmoid_psi));
+   bool done = false;
+   for (int k=0; k<max_its; k++) // Newton iteration
+   {
+      int_sigmoid_psi.Assemble(); // Recompute f(c) with updated ψ
+      real_t f = int_sigmoid_psi.Sum();
+      MPI_Allreduce(MPI_IN_PLACE, &f, 1, MPITypeMap<real_t>::mpi_type,
+                    MPI_SUM, MPI_COMM_WORLD);
+      f -= target_volume;
+
+      int_der_sigmoid_psi.Assemble(); // Recompute df(c) with updated ψ
+      real_t df = int_der_sigmoid_psi.Sum();
+      MPI_Allreduce(MPI_IN_PLACE, &df, 1, MPITypeMap<real_t>::mpi_type,
+                    MPI_SUM, MPI_COMM_WORLD);
+
+      const real_t dc = -f/df;
+      psi += dc;
+      if (abs(dc) < tol) { done = true; break; }
+   }
+   if (!done)
+   {
+      mfem_warning("Projection reached maximum iteration without converging. "
+                   "Result may not be accurate.");
+   }
+   int_sigmoid_psi.Assemble();
+   real_t material_volume = int_sigmoid_psi.Sum();
+   MPI_Allreduce(MPI_IN_PLACE, &material_volume, 1,
+                 MPITypeMap<real_t>::mpi_type, MPI_SUM, MPI_COMM_WORLD);
+   return material_volume;
+}
+
 /*
  * ---------------------------------------------------------------
  *                      ALGORITHM PREAMBLE
@@ -138,11 +193,10 @@ int main(int argc, char *argv[])
    int ref_levels = 5;
    int order = 2;
    real_t alpha = 1.0;
-   real_t growth = 2;
    real_t epsilon = 0.01;
    real_t vol_fraction = 0.5;
    int max_it = 1e3;
-   real_t itol = 1e-2;
+   real_t itol = 1e-1;
    real_t ntol = 1e-4;
    real_t rho_min = 1e-6;
    real_t lambda = 1.0;
@@ -157,8 +211,6 @@ int main(int argc, char *argv[])
                   "Order (degree) of the finite elements.");
    args.AddOption(&alpha, "-alpha", "--alpha-step-length",
                   "Step length for gradient descent.");
-   args.AddOption(&growth, "-growth", "--alpha-growth-rate",
-                  "Growth rate of step length for gradient descent.");
    args.AddOption(&epsilon, "-epsilon", "--epsilon-thickness",
                   "Length scale for ρ.");
    args.AddOption(&max_it, "-mi", "--max-it",
@@ -307,7 +359,6 @@ int main(int argc, char *argv[])
    }
    FilterSolver->SetEssentialBoundary(ess_bdr_filter);
    FilterSolver->SetupFEM();
-   FilterSolver->AssembleDiffusionBilinear();
 
    ParBilinearForm mass(&control_fes);
    mass.AddDomainIntegrator(new InverseIntegrator(new MassIntegrator(one)));
@@ -361,7 +412,7 @@ int main(int argc, char *argv[])
    // 11. Iterate:
    for (int k = 1; k <= max_it; k++)
    {
-      if (k > 1) { alpha = std::pow((real_t) k,growth); }
+      if (k > 1) { alpha *= ((real_t) k) / ((real_t) k-1); }
 
       if (myid == 0)
       {
@@ -401,9 +452,7 @@ int main(int argc, char *argv[])
 
       // Step 5 - Update design variable ψ ← proj(ψ - αG)
       psi.Add(-alpha, grad);
-      ParGridFunction alpha_grad(grad);
-      alpha_grad *= alpha;
-      const real_t material_volume = proj(psi, alpha_grad, target_volume);
+      const real_t material_volume = proj(psi, target_volume);
 
       // Compute ||ρ - ρ_old|| in control fes.
       real_t norm_increment = zerogf.ComputeL1Error(succ_diff_rho);

examples/ex4.cpp

@@ -9,7 +9,6 @@
 //               ex4 -m ../data/beam-hex.mesh -o 2 -pa
 //               ex4 -m ../data/escher.mesh
 //               ex4 -m ../data/fichera.mesh -o 2 -hb
-//               ex4 -m ../data/fichera.mesh -o 2 -hb -ea
 //               ex4 -m ../data/fichera-q2.vtk
 //               ex4 -m ../data/fichera-q3.mesh -o 2 -sc
 //               ex4 -m ../data/square-disc-nurbs.mesh
@@ -19,7 +18,6 @@
 //               ex4 -m ../data/amr-quad.mesh
 //               ex4 -m ../data/amr-hex.mesh
 //               ex4 -m ../data/amr-hex.mesh -o 2 -hb
-//               ex4 -m ../data/amr-hex.mesh -o 2 -hb -ea
 //               ex4 -m ../data/fichera-amr.mesh -o 2 -sc
 //               ex4 -m ../data/ref-prism.mesh -o 1
 //               ex4 -m ../data/octahedron.mesh -o 1
@@ -27,8 +25,6 @@
 //
 // Device sample runs:
 //               ex4 -m ../data/star.mesh -pa -d cuda
-//               ex4 -m ../data/star.mesh -hb -ea -d cuda
-//               ex4 -m ../data/amr-quad.mesh -hb -ea -d cuda
 //               ex4 -m ../data/star.mesh -pa -d raja-cuda
 //               ex4 -m ../data/star.mesh -pa -d raja-omp
 //               ex4 -m ../data/beam-hex.mesh -pa -d cuda
@@ -197,7 +193,7 @@ int main(int argc, char *argv[])
    cout << "Size of linear system: " << A->Height() << endl;
 
    // 11. Solve the linear system A X = B.
-   if (!pa && (!ea || hybridization))
+   if (!pa)
    {
 #ifndef MFEM_USE_SUITESPARSE
       // Use a simple symmetric Gauss-Seidel preconditioner with PCG.

examples/ex41.cpp

@@ -1,589 +0,0 @@
-//                                MFEM Example 41
-//
-// Compile with: make ex41
-//
-// Sample runs:
-//  ex41
-//  ex41 -cg
-//  ex41 -m ../data/periodic-hexagon.mesh -p 0 -r 2 -dt 0.005 -tf 10
-//  ex41 -m ../data/periodic-square.mesh -p 1 -r 2 -dt 0.005 -tf 9
-//  ex41 -m ../data/periodic-hexagon.mesh -p 1 -r 2 -dt 0.005 -tf 9
-//  ex41 -m ../data/amr-quad.mesh -p 1 -r 2 -dt 0.002 -tf 9
-//  ex41 -m ../data/star-q3.mesh -p 1 -r 2 -dt 0.001 -tf 9
-//  ex41 -m ../data/star-mixed.mesh -p 1 -r 2 -dt 0.005 -tf 9
-//  ex41 -m ../data/disc-nurbs.mesh -p 1 -r 3 -dt 0.005 -tf 9
-//  ex41 -m ../data/disc-nurbs.mesh -p 2 -r 3 -dt 0.005 -tf 9
-//  ex41 -m ../data/periodic-square.mesh -p 3 -r 4 -dt 0.0025 -tf 9 -vs 20
-//  ex41 -m ../data/periodic-cube.mesh -p 0 -r 2 -o 2 -dt 0.01 -tf 8
-//
-// Device sample runs:
-//
-// Description:  This example code solves the time-dependent advection-diffusion
-//               equation du/dt + v.grad(u) - a div(grad(u)) = 0, where v is a
-//               given fluid velocity, a is the diffusion coefficient, and
-//               u0(x)=u(0,x) is a given initial condition.
-//
-//               The example demonstrates the use of Discontinuous Galerkin (DG)
-//               bilinear forms in MFEM (face integrators), and the use of IMEX
-//               ODE time integrators.
-//
-//               The option to use continuous finite elements is available too.
-
-#include "mfem.hpp"
-
-using namespace std;
-using namespace mfem;
-
-// Mesh bounding box
-Vector bb_min, bb_max;
-
-// Velocity coefficient
-template<int problem=0>
-void velocity_function(const Vector &x, Vector &v)
-{
-   int dim = x.Size();
-
-   // map to the reference [-1,1] domain
-   Vector X(dim);
-   for (int i = 0; i < dim; i++)
-   {
-      real_t center = (bb_min[i] + bb_max[i]) * 0.5;
-      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
-   }
-   switch (problem)
-   {
-      case 0:
-      {
-         // Translations in 1D, 2D, and 3D
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = sqrt(2./3.); v(1) = sqrt(1./3.); break;
-            case 3: v(0) = sqrt(3./6.); v(1) = sqrt(2./6.); v(2) = sqrt(1./6.);
-               break;
-         }
-         break;
-      }
-      case 1:
-      case 2:
-      {
-         // Clockwise rotation in 2D around the origin
-         const real_t w = M_PI/2;
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = w*X(1); v(1) = -w*X(0); break;
-            case 3: v(0) = w*X(1); v(1) = -w*X(0); v(2) = 0.0; break;
-         }
-         break;
-      }
-      case 3:
-      {
-         // Clockwise twisting rotation in 2D around the origin
-         const real_t w = M_PI/2;
-         real_t d = max((X(0)+1.)*(1.-X(0)),0.) * max((X(1)+1.)*(1.-X(1)),0.);
-         d = d*d;
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = d*w*X(1); v(1) = -d*w*X(0); break;
-            case 3: v(0) = d*w*X(1); v(1) = -d*w*X(0); v(2) = 0.0; break;
-         }
-         break;
-      }
-   }
-}
-
-// Initial condition
-template<int problem=0>
-real_t u0_function(const Vector &x)
-{
-   int dim = x.Size();
-
-   // map to the reference [-1,1] domain
-   Vector X(dim);
-   for (int i = 0; i < dim; i++)
-   {
-      real_t center = (bb_min[i] + bb_max[i]) * 0.5;
-      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
-   }
-
-   switch (problem)
-   {
-      case 0:
-      case 1:
-      {
-         switch (dim)
-         {
-            case 1:
-               return exp(-40.*pow(X(0)-0.5,2));
-            case 2:
-            case 3:
-            {
-               real_t rx = 0.45, ry = 0.25, cx = 0., cy = -0.2, w = 10.;
-               if (dim == 3)
-               {
-                  const real_t s = (1. + 0.25*cos(2*M_PI*X(2)));
-                  rx *= s;
-                  ry *= s;
-               }
-               return ( std::erfc(w*(X(0)-cx-rx))*std::erfc(-w*(X(0)-cx+rx)) *
-                        std::erfc(w*(X(1)-cy-ry))*std::erfc(-w*(X(1)-cy+ry)) )/16;
-            }
-         }
-      }
-      case 2:
-      {
-         real_t x_ = X(0), y_ = X(1), rho, phi;
-         rho = std::hypot(x_, y_);
-         phi = atan2(y_, x_);
-         return pow(sin(M_PI*rho),2)*sin(3*phi);
-      }
-      case 3:
-      {
-         const real_t f = M_PI;
-         return sin(f*X(0))*sin(f*X(1));
-      }
-   }
-   return 0.0;
-}
-
-/// Solver for the implicit part of the ODE (the diffusion term).
-/// Solves systems of the form: (M + dt*S) k = rhs.
-class Implicit_Solver : public Solver
-{
-private:
-   SparseMatrix &M, &S, A;
-   CGSolver linear_solver;
-   BlockILU prec;
-   real_t dt;
-public:
-   Implicit_Solver(SparseMatrix &M_, SparseMatrix &S_,
-                   const FiniteElementSpace &fes)
-      : M(M_),
-        S(S_),
-        prec(fes.GetTypicalFE()->GetDof(),
-             BlockILU::Reordering::MINIMUM_DISCARDED_FILL),
-        dt(1.0)
-   {
-      linear_solver.iterative_mode = false;
-      linear_solver.SetRelTol(1e-9);
-      linear_solver.SetAbsTol(0.0);
-      linear_solver.SetMaxIter(100);
-      linear_solver.SetPrintLevel(0);
-      linear_solver.SetPreconditioner(prec);
-   }
-
-   void SetTimeStep(real_t dt_)
-   {
-      real_t ddt = dt-dt_;
-
-      real_t epsilon;
-      epsilon = std::numeric_limits<real_t>::epsilon();
-      epsilon*=10;
-
-      if (std::abs(ddt) > epsilon)
-      {
-         dt = dt_;
-         // Form operator A = M + dt*S
-         A = S;
-         A *= dt;
-         A += M;
-
-         // this will also call SetOperator on the preconditioner
-         linear_solver.SetOperator(A);
-      }
-   }
-
-   void SetOperator(const Operator &op) override
-   {
-      linear_solver.SetOperator(op);
-   }
-
-   void Mult(const Vector &x, Vector &y) const override
-   {
-      linear_solver.Mult(x, y);
-   }
-};
-
-/** A time-dependent operator for the right-hand side of the ODE. The weak
-    form of the advection-diffusion equation is M du/dt = K u - S u + b,
-    where M is the mass matrix, K and S are the advection and diffusion
-    matrices, and b describes the flow on the boundary. In the case of IMEX
-    evolution, the diffusion term is treated implicitly, and the advection
-    term is treated explicitly.  */
-class IMEX_Evolution : public TimeDependentOperator
-{
-private:
-   BilinearForm &M, &K, &S;
-   const Vector &b;
-   unique_ptr<Solver> M_prec;
-   CGSolver M_solver;
-   unique_ptr<Implicit_Solver> implicit_solver;
-
-   mutable Vector z;
-
-public:
-   IMEX_Evolution(BilinearForm &M_, BilinearForm &K_, BilinearForm &S_,
-                  const Vector &b_);
-
-   /// Evaluate k1=M^{-1}*G1(u,t); -> k1 = M^{-1}*(K*u + b)
-   void Mult1(const Vector &x, Vector &y) const;
-
-   /// Evaluate k2: M*k2 = G2(u+k2*dt,t); -> (M+S*dt)*k2=-S*u
-   void ImplicitSolve2(const real_t dt, const Vector &x, Vector &k);
-
-   void Mult(const Vector &x, Vector &y) const override
-   {
-      if (TimeDependentOperator::EvalMode::ADDITIVE_TERM_1 == GetEvalMode())
-      {
-         Mult1(x,y);
-      }
-      else
-      {
-         mfem_error("TimeDependentOperator::Mult() is not overridden!");
-      }
-   }
-
-   void ImplicitSolve(const real_t dt, const Vector &x, Vector &k) override
-   {
-      if (TimeDependentOperator::EvalMode::ADDITIVE_TERM_2 == GetEvalMode())
-      {
-         ImplicitSolve2(dt,x,k);
-      }
-      else
-      {
-         mfem_error("TimeDependentOperator::ImplicitSolve() is not overridden!");
-      }
-   }
-
-};
-
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   int problem = 0;
-   const char *mesh_file = "../data/periodic-square.mesh";
-   int ref_levels = 2;
-   int order = 3;
-   int ode_solver_type = 64; //IMEXRK3(3,4,3)
-   real_t t_final = 10.0;
-   real_t dt = 0.01;
-   bool paraview = false;
-   bool cg = false;
-   int vis_steps = 50;
-   real_t diffusion_term = 0.01;
-   real_t kappa = -1.0;
-   real_t sigma = -1.0;
-   bool visualization = true;
-   bool visit = false;
-   bool binary = false;
-   int precision = 8;
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
-   args.AddOption(&problem, "-p", "--problem",
-                  "Problem setup to use. See options in velocity_function().");
-   args.AddOption(&ref_levels, "-r", "--refine",
-                  "Number of times to refine the mesh uniformly.");
-   args.AddOption(&order, "-o", "--order", "Order of the finite elements.");
-   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  ODESolver::IMEXTypes.c_str());
-   args.AddOption(&t_final, "-tf", "--t-final", "Final time; start time is 0.");
-   args.AddOption(&dt, "-dt", "--time-step", "Time step.");
-   args.AddOption(&diffusion_term, "-dc", "--diffusion-coeff",
-                  "Diffusion coefficient in the PDE.");
-   args.AddOption(&paraview, "-paraview", "--paraview-datafiles", "-no-paraview",
-                  "--no-paraview-datafiles",
-                  "Save data files for ParaView (paraview.org) visualization.");
-   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
-                  "Visualize every n-th timestep.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&binary, "-binary", "--binary-datafiles", "-ascii",
-                  "--ascii-datafiles",
-                  "Use binary (Sidre) or ascii format for VisIt data files.");
-   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
-                  "--no-visit-datafiles",
-                  "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.AddOption(&cg, "-cg", "--continuous-galerkin", "-dg",
-                  "--discontinuous-galerkin",
-                  "Use Continuous-Galerkin Finite elements (Default is DG)");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   if (kappa < 0)
-   {
-      kappa = (order+1)*(order+1);
-   }
-   args.PrintOptions(cout);
-
-   // 2. Read the mesh from the given mesh file. We can handle geometrically
-   //    periodic meshes in this code.
-   Mesh mesh(mesh_file);
-   const int dim = mesh.Dimension();
-
-   // 3. Define the IMEX (Split) ODE solver used for time integration. The IMEX
-   // solvers currently available are: 61 - Forward Backward Euler,
-   // 62 - IMEXRK2(2,2,2), 63 - IMEXRK2(2,3,2), and  64 - IMEX_DIRK_RK3.
-   unique_ptr<ODESolver> ode_solver = ODESolver::SelectIMEX(ode_solver_type);
-
-   // 4. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement, where 'ref_levels' is a
-   //    command-line parameter.
-   for (int lev = 0; lev < ref_levels; lev++) {mesh.UniformRefinement();}
-   if (mesh.NURBSext) {mesh.SetCurvature(max(order, 1));}
-   mesh.GetBoundingBox(bb_min, bb_max, max(order, 1));
-
-   // 5. Define the discontinuous DG finite element space of the given
-   //    polynomial order on the refined mesh.
-   FiniteElementCollection *fec = NULL;
-   if (cg)
-   {
-      fec = new H1_FECollection(order, dim);
-   }
-   else
-   {
-      fec = new DG_FECollection(order, dim, BasisType::GaussLobatto);
-   }
-   FiniteElementSpace fes(&mesh, fec);
-
-   cout << "Number of unknowns: " << fes.GetVSize() << endl;
-
-   // 6. Set up and assemble the bilinear and linear forms corresponding to the
-   //    DG discretization. The DGTraceIntegrator involves integrals over mesh
-   //    interior faces.
-   std::unique_ptr<VectorFunctionCoefficient> velocity;
-   if (0==problem)
-   {
-      velocity.reset(new VectorFunctionCoefficient(dim, velocity_function<0>));
-   }
-   else if (1==problem)
-   {
-      velocity.reset(new VectorFunctionCoefficient(dim, velocity_function<1>));
-   }
-   else if (2==problem)
-   {
-      velocity.reset(new VectorFunctionCoefficient(dim, velocity_function<2>));
-   }
-   else if (3==problem)
-   {
-      velocity.reset(new VectorFunctionCoefficient(dim, velocity_function<3>));
-   }
-
-   ConstantCoefficient diff_coeff(diffusion_term);
-
-   BilinearForm m(&fes);
-   BilinearForm k(&fes);
-   BilinearForm s(&fes);
-
-   Vector b(fes.GetTrueVSize());
-   b = 0.0; //The inflow on the boundaries is set to zero.
-
-   m.AddDomainIntegrator(new MassIntegrator);
-
-   constexpr real_t alpha = -1.0;
-   k.AddDomainIntegrator(new ConvectionIntegrator(*velocity, alpha));
-
-   s.AddDomainIntegrator(new DiffusionIntegrator(diff_coeff));
-   if (!cg)
-   {
-      k.AddInteriorFaceIntegrator(new NonconservativeDGTraceIntegrator(*velocity,
-                                                                       alpha));
-      k.AddBdrFaceIntegrator(new NonconservativeDGTraceIntegrator(*velocity, alpha));
-      s.AddInteriorFaceIntegrator(new DGDiffusionIntegrator(diff_coeff, sigma,
-                                                            kappa));
-      s.AddBdrFaceIntegrator(new DGDiffusionIntegrator(diff_coeff, sigma, kappa));
-   }
-
-
-   int skip_zeros = 0;
-   m.Assemble(skip_zeros);
-   k.Assemble(skip_zeros);
-   s.Assemble(skip_zeros);
-
-   m.Finalize(skip_zeros);
-   k.Finalize(skip_zeros);
-   s.Finalize(skip_zeros);
-
-   // 7. Define the initial conditions.
-   std::unique_ptr<FunctionCoefficient> u0;
-   if (0==problem)
-   {
-      u0.reset(new FunctionCoefficient(u0_function<0>));
-   }
-   else if (1==problem)
-   {
-      u0.reset(new FunctionCoefficient(u0_function<1>));
-   }
-   else if (2==problem)
-   {
-      u0.reset(new FunctionCoefficient(u0_function<2>));
-   }
-   else if (3==problem)
-   {
-      u0.reset(new FunctionCoefficient(u0_function<3>));
-   }
-
-   GridFunction u(&fes);
-   u.ProjectCoefficient(*u0);
-
-   // Create data collection for solution output: either VisItDataCollection for
-   // ascii data files, or SidreDataCollection for binary data files.
-   DataCollection *dc = NULL;
-   if (visit)
-   {
-      if (binary)
-      {
-#ifdef MFEM_USE_SIDRE
-         dc = new SidreDataCollection("Example41", &mesh);
-#else
-         MFEM_ABORT("Must build with MFEM_USE_SIDRE=YES for binary output.");
-#endif
-      }
-      else
-      {
-         dc = new VisItDataCollection("Example41", &mesh);
-         dc->SetPrecision(precision);
-      }
-      dc->RegisterField("solution", &u);
-      dc->SetCycle(0);
-      dc->SetTime(0.0);
-      dc->Save();
-   }
-
-   // 8. Set up paraview visualization, if desired.
-   unique_ptr<ParaViewDataCollection> pv;
-   if (paraview)
-   {
-      pv = make_unique<ParaViewDataCollection>("Example41", &mesh);
-      pv->SetPrefixPath("ParaView");
-      pv->RegisterField("solution", &u);
-      pv->SetLevelsOfDetail(order);
-      pv->SetDataFormat(VTKFormat::BINARY);
-      pv->SetHighOrderOutput(true);
-      pv->SetCycle(0);
-      pv->SetTime(0.0);
-      pv->Save();
-   }
-
-   socketstream sout;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      sout.open(vishost, visport);
-      if (!sout)
-      {
-         cout << "Unable to connect to GLVis server at "
-              << vishost << ':' << visport << endl;
-         visualization = false;
-         cout << "GLVis visualization disabled.\n";
-      }
-      else
-      {
-         sout.precision(precision);
-         sout << "solution\n" << mesh << u;
-         sout << "pause\n";
-         sout << flush;
-         cout << "GLVis visualization paused."
-              << " Press space (in the GLVis window) to resume it.\n";
-      }
-   }
-
-   // 9. Define the time-dependent evolution operator describing the ODE
-   //    right-hand side, and perform time-integration (looping over the time
-   //    iterations, ti, with a time-step dt).
-   IMEX_Evolution adv(m, k, s, b);
-
-   real_t t = 0.0;
-   adv.SetTime(t);
-   ode_solver->Init(adv);
-
-   bool done = false;
-   for (int ti = 0; !done; )
-   {
-      real_t dt_real = min(dt, t_final - t);
-      ode_solver->Step(u, t, dt_real);
-      ti++;
-
-      done = (t >= t_final - 1e-8*dt);
-
-      if (done || ti % vis_steps == 0)
-      {
-         cout << "time step: " << ti << ", time: " << t << endl;
-         if (paraview)
-         {
-            pv->SetCycle(ti);
-            pv->SetTime(t);
-            pv->Save();
-         }
-         if (visualization)
-         {
-            sout << "solution\n" << mesh << u << flush;
-         }
-         if (visit)
-         {
-            dc->SetCycle(ti);
-            dc->SetTime(t);
-            dc->Save();
-         }
-
-      }
-   }
-
-   delete fec;
-   return 0;
-}
-
-
-// Implementation of class IMEX_Evolution
-IMEX_Evolution::IMEX_Evolution(BilinearForm &M_, BilinearForm &K_,
-                               BilinearForm &S_, const Vector &b_)
-   : TimeDependentOperator(M_.FESpace()->GetTrueVSize()),
-     M(M_), K(K_), S(S_), b(b_), z(height)
-{
-   Array<int> ess_tdof_list;
-   if (M.GetAssemblyLevel() == AssemblyLevel::LEGACY)
-   {
-      M_prec = make_unique<DSmoother>(M.SpMat());
-      M_solver.SetOperator(M.SpMat());
-      implicit_solver = make_unique<Implicit_Solver>(M.SpMat(), S.SpMat(),
-                                                     *M.FESpace());
-   }
-   else
-   {
-      MFEM_ABORT("Implicit time integration is not supported with partial assembly");
-   }
-   M_solver.SetPreconditioner(*M_prec);
-   M_solver.iterative_mode = false;
-   M_solver.SetRelTol(1e-9);
-   M_solver.SetAbsTol(0.0);
-   M_solver.SetMaxIter(100);
-   M_solver.SetPrintLevel(0);
-}
-
-void IMEX_Evolution::Mult1(const Vector &x, Vector &y) const
-{
-   // Perform the explicit step
-   // y = M^{-1} (K x + b)
-   K.Mult(x, z);
-   z += b;
-   M_solver.Mult(z, y);
-}
-
-void IMEX_Evolution::ImplicitSolve2(const real_t dt, const Vector &x, Vector &k)
-{
-   // Perform the implicit step
-   // solve for k, k = -(M+dt S)^{-1} S x
-   MFEM_VERIFY(implicit_solver != NULL,
-               "Implicit time integration is not supported with partial assembly");
-   S.Mult(x, z);
-   z.Neg();
-   implicit_solver->SetTimeStep(dt);
-   implicit_solver->Mult(z, k);
-}

examples/ex41p.cpp

@@ -1,737 +0,0 @@
-//                                MFEM Example 41 - Parallel Version
-//
-// Compile with: make ex41p
-//
-// Sample runs:
-//  mpirun -np 4 ex41p
-//  mpirun -np 4 ex41p -cg
-//  mpirun -np 4 ex41p -m ../data/periodic-hexagon.mesh -p 0 -dt 0.005 -tf 10
-//  mpirun -np 4 ex41p -m ../data/periodic-square.mesh -p 1 -dt 0.005 -tf 9
-//  mpirun -np 4 ex41p -m ../data/periodic-hexagon.mesh -p 1  -dt 0.005 -tf 9
-//  mpirun -np 4 ex41p -m ../data/star-q3.mesh -p 1 -rp 1 -dt 0.001 -tf 9
-//  mpirun -np 4 ex41p -m ../data/disc-nurbs.mesh -p 1 -rp 1 -dt 0.005 -tf 9
-//  mpirun -np 4 ex41p -m ../data/disc-nurbs.mesh -p 2 -rp 1 -dt 0.005 -tf 9
-//  mpirun -np 4 ex41p -m ../data/periodic-square.mesh -rp 2 -dt 0.0025 -tf 9 -vs 20
-//  mpirun -np 4 ex41p -m ../data/periodic-cube.mesh -p 0 -rs 2 -o 2 -dt 0.01 -tf 8
-//
-// Device sample runs:
-//
-// Description:  This example code solves the time-dependent advection-diffusion
-//               equation du/dt + v.grad(u) - a div(grad(u)) = 0, where v is a
-//               given fluid velocity, a is the diffusion coefficient, and
-//               u0(x)=u(0,x) is a given initial condition.
-//
-//               The example demonstrates the use of Discontinuous Galerkin (DG)
-//               bilinear forms in MFEM (face integrators), DG-LOR Preconditioning
-//               and the use of IMEX ODE time integrators.
-//
-//               The Option to use Continuous Finite Elements is available too.
-
-#include "mfem.hpp"
-
-using namespace std;
-using namespace mfem;
-
-// Mesh bounding box
-Vector bb_min, bb_max;
-
-// Velocity coefficient
-template<int problem=0>
-void velocity_function(const Vector &x, Vector &v)
-{
-   int dim = x.Size();
-
-   // map to the reference [-1,1] domain
-   Vector X(dim);
-   for (int i = 0; i < dim; i++)
-   {
-      real_t center = (bb_min[i] + bb_max[i]) * 0.5;
-      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
-   }
-   switch (problem)
-   {
-      case 0:
-      {
-         // Translations in 1D, 2D, and 3D
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = sqrt(2./3.); v(1) = sqrt(1./3.); break;
-            case 3: v(0) = sqrt(3./6.); v(1) = sqrt(2./6.); v(2) = sqrt(1./6.);
-               break;
-         }
-         break;
-      }
-      case 1:
-      case 2:
-      {
-         // Clockwise rotation in 2D around the origin
-         const real_t w = M_PI/2;
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = w*X(1); v(1) = -w*X(0); break;
-            case 3: v(0) = w*X(1); v(1) = -w*X(0); v(2) = 0.0; break;
-         }
-         break;
-      }
-      case 3:
-      {
-         // Clockwise twisting rotation in 2D around the origin
-         const real_t w = M_PI/2;
-         real_t d = max((X(0)+1.)*(1.-X(0)),0.) * max((X(1)+1.)*(1.-X(1)),0.);
-         d = d*d;
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = d*w*X(1); v(1) = -d*w*X(0); break;
-            case 3: v(0) = d*w*X(1); v(1) = -d*w*X(0); v(2) = 0.0; break;
-         }
-         break;
-      }
-   }
-}
-
-
-// Initial condition
-template<int problem=0>
-real_t u0_function(const Vector &x)
-{
-   int dim = x.Size();
-
-   // map to the reference [-1,1] domain
-   Vector X(dim);
-   for (int i = 0; i < dim; i++)
-   {
-      real_t center = (bb_min[i] + bb_max[i]) * 0.5;
-      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
-   }
-
-   switch (problem)
-   {
-      case 0:
-      case 1:
-      {
-         switch (dim)
-         {
-            case 1:
-               return exp(-40.*pow(X(0)-0.5,2));
-            case 2:
-            case 3:
-            {
-               real_t rx = 0.45, ry = 0.25, cx = 0., cy = -0.2, w = 10.;
-               if (dim == 3)
-               {
-                  const real_t s = (1. + 0.25*cos(2*M_PI*X(2)));
-                  rx *= s;
-                  ry *= s;
-               }
-               return ( std::erfc(w*(X(0)-cx-rx))*std::erfc(-w*(X(0)-cx+rx)) *
-                        std::erfc(w*(X(1)-cy-ry))*std::erfc(-w*(X(1)-cy+ry)) )/16;
-            }
-         }
-      }
-      case 2:
-      {
-         real_t x_ = X(0), y_ = X(1), rho, phi;
-         rho = std::hypot(x_, y_);
-         phi = atan2(y_, x_);
-         return pow(sin(M_PI*rho),2)*sin(3*phi);
-      }
-      case 3:
-      {
-         const real_t f = M_PI;
-         return sin(f*X(0))*sin(f*X(1));
-      }
-   }
-   return 0.0;
-}
-
-
-class Implicit_Solver : public Solver
-{
-private:
-   HypreParMatrix &M, &S;
-   HypreParMatrix *A;
-   CGSolver linear_solver;
-   real_t dt;
-   SparseMatrix M_diag;
-public:
-   Implicit_Solver(HypreParMatrix &M_, HypreParMatrix &S_,
-                   const FiniteElementSpace &fes)
-      : M(M_),
-        S(S_),
-        A(nullptr),
-        linear_solver(M.GetComm()),
-        dt(1.0)
-   {
-      linear_solver.iterative_mode = false;
-      linear_solver.SetRelTol(1e-9);
-      linear_solver.SetAbsTol(0.0);
-      linear_solver.SetMaxIter(100);
-      linear_solver.SetPrintLevel(0);
-
-      M.GetDiag(M_diag);
-   }
-
-   void SetTimeStep(real_t dt_)
-   {
-      real_t ddt = dt-dt_;
-
-      // syncronize ddt across all processes
-      MPI_Comm comm = M.GetComm();
-      int myrank;
-      MPI_Comm_rank(comm, &myrank);
-      MPI_Bcast(&ddt, 1, MPI_DOUBLE, 0, comm);
-
-      real_t epsilon;
-      epsilon = std::numeric_limits<real_t>::epsilon();
-      // allow for some tolerance in the time stepping process
-      epsilon*=10;
-
-      if (fabs(ddt) > epsilon)
-      {
-         if (0==myrank)
-         {
-            cout << "Updating Implicit_Solver time step from " << dt
-                 << " to " << dt_ << endl;
-         }
-
-         delete A;
-         dt = dt_;
-         // Form operator A = M + dt*S
-         A = Add(dt, S, 1.0, M);
-         linear_solver.SetOperator(*A);
-      }
-   }
-
-   void SetOperator(const Operator &op) override
-   {
-      linear_solver.SetOperator(op);
-   }
-
-   void Mult(const Vector &x, Vector &y) const override
-   {
-      linear_solver.Mult(x, y);
-   }
-
-   void SetPreconditioner(Solver &precond)
-   {
-      linear_solver.SetPreconditioner(precond);
-   }
-
-   ~Implicit_Solver() override
-   {
-      delete A;
-   }
-};
-
-/** A time-dependent operator for the right-hand side of the ODE. The DG weak
-    form of the advection-diffusion equation is (M + dt S) du/dt = Su - K u + b
-    , where M and K are the mass and advection matrices, and b describes the
-    flow on the boundary. In the case of IMEX evolution, the diffusion term is
-    treated implicitly, and the advection term is treated explicitly.  */
-class IMEX_Evolution : public TimeDependentOperator
-{
-private:
-   OperatorHandle M, K, S, A;
-   const Vector &b;
-   Solver *M_prec;
-   CGSolver M_solver;
-   Implicit_Solver *implicit_solver;
-   LORSolver<HypreBoomerAMG>* lor_solver;
-
-   mutable Vector z;
-   mutable Vector w;
-
-public:
-   IMEX_Evolution(ParBilinearForm &M_, ParBilinearForm &K_, ParBilinearForm &S_,
-                  const Vector &b_, ParBilinearForm &A_);
-
-   virtual
-   ~IMEX_Evolution()
-   {
-      delete implicit_solver;
-      delete lor_solver;
-      delete M_prec;
-   }
-
-   void Mult1(const Vector &x, Vector &y) const;
-
-   void ImplicitSolve2(const real_t dt, const Vector &x, Vector &k);
-
-   void Mult(const Vector &x, Vector &y) const override
-   {
-      if (TimeDependentOperator::EvalMode::ADDITIVE_TERM_1 == GetEvalMode())
-      {
-         Mult1(x,y);
-      }
-      else
-      {
-         mfem_error("TimeDependentOperator::Mult() is not overridden!");
-      }
-   }
-
-   void ImplicitSolve(const real_t dt, const Vector &x, Vector &k) override
-   {
-      if (TimeDependentOperator::EvalMode::ADDITIVE_TERM_2 == GetEvalMode())
-      {
-         ImplicitSolve2(dt,x,k);
-      }
-      else
-      {
-         mfem_error("TimeDependentOperator::ImplicitSolve() is not overridden!");
-      }
-   }
-};
-
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI and HYPRE.
-   Mpi::Init();
-   int num_procs = Mpi::WorldSize();
-   int myid = Mpi::WorldRank();
-   Hypre::Init();
-
-   // 2. Parse command-line options.
-   int problem = 0;
-   const char *mesh_file = "../data/periodic-square.mesh";
-   int ser_ref_levels = 2;
-   int par_ref_levels = 0;
-   int order = 3;
-   int ode_solver_type = 64; // 61 - Forward Backward Euler
-   // 62 - IMEXRK2(2,2,2)
-   // 63 - IMEXRK2(2,3,2)
-   // 64 - IMEXRK3(3,4,3)
-   real_t t_final = 10.0;
-   real_t dt = 0.01;
-   bool paraview = false;
-   bool cg = false;
-   int vis_steps = 50;
-   bool adios2 = false;
-   bool binary = false;
-   real_t diffusion_term = 0.01;
-   real_t kappa = -1.0;
-   real_t sigma = -1.0;
-   bool visualization = true;
-   bool visit = false;
-   int precision = 16;
-   cout.precision(precision);
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&problem, "-p", "--problem",
-                  "Problem setup to use. See options in velocity_function().");
-   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
-                  "Number of times to refine the mesh uniformly in parallel.");
-   args.AddOption(&order, "-o", "--order",
-                  "Order (degree) of the finite elements.");
-   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  ODESolver::IMEXTypes.c_str());
-   args.AddOption(&t_final, "-tf", "--t-final",
-                  "Final time; start time is 0.");
-   args.AddOption(&dt, "-dt", "--time-step",
-                  "Time step.");
-   args.AddOption(&diffusion_term, "-dc", "--diffusion-coeff",
-                  "Diffusion coefficient in the PDE.");
-   args.AddOption(&paraview, "-paraview", "--paraview-datafiles", "-no-paraview",
-                  "--no-paraview-datafiles",
-                  "Save data files for ParaView (paraview.org) visualization.");
-   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
-                  "--no-visit-datafiles",
-                  "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.AddOption(&adios2, "-adios2", "--adios2-streams", "-no-adios2",
-                  "--no-adios2-streams",
-                  "Save data using adios2 streams.");
-   args.AddOption(&binary, "-binary", "--binary-datafiles", "-ascii",
-                  "--ascii-datafiles",
-                  "Use binary (Sidre) or ascii format for VisIt data files.");
-   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
-                  "Visualize every n-th timestep.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&cg, "-cg", "--continuous-galerkin", "-dg",
-                  "--discontinuous-galerkin",
-                  "Use Continuous-Galerkin Finite elements (Default is DG)");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (Mpi::Root())
-      {
-         args.PrintUsage(cout);
-      }
-      return 1;
-   }
-   if (Mpi::Root())
-   {
-      args.PrintOptions(cout);
-   }
-   if (kappa < 0)
-   {
-      kappa = (order+1)*(order+1);
-   }
-
-   // 3. Read the mesh from the given mesh file. We can handle geometrically
-   //    periodic meshes in this code.
-   Mesh *mesh = new Mesh(mesh_file);
-   const int dim = mesh->Dimension();
-
-   // 4. Define the IMEX (Split) ODE solver used for time integration. The IMEX
-   // solvers currently available are: 55 - Forward Backward Euler,
-   // 56 - IMEXRK2(2,2,2), 57 - IMEXRK2(2,3,2), and
-   unique_ptr<ODESolver> ode_solver = ODESolver::SelectIMEX(ode_solver_type);
-
-   // 5. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement, where 'ref_levels' is a
-   //    command-line parameter.
-   for (int lev = 0; lev < ser_ref_levels; lev++) { mesh->UniformRefinement(); }
-   if (mesh->NURBSext)
-   {
-      mesh->SetCurvature(max(order, 1));
-   }
-   mesh->GetBoundingBox(bb_min, bb_max, max(order, 1));
-
-
-   // 6. Define the parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   for (int lev = 0; lev < par_ref_levels; lev++)
-   {
-      pmesh->UniformRefinement();
-   }
-
-   // 7. Define the discontinuous DG finite element space of the given
-   //    polynomial order on the refined mesh.
-   FiniteElementCollection *fec = NULL;
-   if (cg)
-   {
-      fec = new H1_FECollection(order, dim);
-   }
-   else
-   {
-      fec = new DG_FECollection(order, dim, BasisType::GaussLobatto);
-   }
-   ParFiniteElementSpace *fes = new ParFiniteElementSpace(pmesh, fec);
-
-   HYPRE_BigInt global_vSize = fes->GlobalTrueVSize();
-   if (Mpi::Root())
-   {
-      cout << "Number of unknowns: " << global_vSize << endl;
-   }
-
-   // 8. Set up and assemble the bilinear and linear forms corresponding to the
-   //    DG discretization. The DGTraceIntegrator involves integrals over mesh
-   //    interior faces.
-   std::unique_ptr<VectorFunctionCoefficient> velocity;
-   if (0==problem)
-   {
-      velocity.reset(new VectorFunctionCoefficient(dim, velocity_function<0>));
-   }
-   else if (1==problem)
-   {
-      velocity.reset(new VectorFunctionCoefficient(dim, velocity_function<1>));
-   }
-   else if (2==problem)
-   {
-      velocity.reset(new VectorFunctionCoefficient(dim, velocity_function<2>));
-   }
-   else if (3==problem)
-   {
-      velocity.reset(new VectorFunctionCoefficient(dim, velocity_function<3>));
-   }
-   ConstantCoefficient diff_coeff(diffusion_term);
-   ConstantCoefficient dt_diff_coeff(dt*diffusion_term);
-
-   ParBilinearForm *m = new ParBilinearForm(fes);
-   ParBilinearForm *k = new ParBilinearForm(fes);
-   ParBilinearForm *s = new ParBilinearForm(fes);
-
-   m->AddDomainIntegrator(new MassIntegrator());
-
-   constexpr real_t alpha = -1.0;
-   k->AddDomainIntegrator(new ConvectionIntegrator(*velocity, alpha));
-
-   s->AddDomainIntegrator(new DiffusionIntegrator(diff_coeff));
-
-   // For the preconditioner - create billinear form corresponding to
-   // operator (M + dt S)
-   ParBilinearForm *a = new ParBilinearForm(fes);
-   a->AddDomainIntegrator(new MassIntegrator);
-   a->AddDomainIntegrator(new DiffusionIntegrator(dt_diff_coeff));
-   if (!cg)
-   {
-      k->AddInteriorFaceIntegrator(new NonconservativeDGTraceIntegrator(*velocity,
-                                                                        alpha));
-      k->AddBdrFaceIntegrator(new NonconservativeDGTraceIntegrator(*velocity, alpha));
-      s->AddInteriorFaceIntegrator(new DGDiffusionIntegrator(diff_coeff, sigma,
-                                                             kappa));
-      s->AddBdrFaceIntegrator(new DGDiffusionIntegrator(diff_coeff, sigma, kappa));
-      a->AddInteriorFaceIntegrator(new DGDiffusionIntegrator(dt_diff_coeff, sigma,
-                                                             kappa));
-      a->AddBdrFaceIntegrator(new DGDiffusionIntegrator(dt_diff_coeff, sigma, kappa));
-   }
-
-   int skip_zeros = 0;
-   m->Assemble(skip_zeros);
-   k->Assemble(skip_zeros);
-   s->Assemble(skip_zeros);
-   a->Assemble();
-
-   m->Finalize(skip_zeros);
-   k->Finalize(skip_zeros);
-   s->Finalize(skip_zeros);
-   a->Finalize(skip_zeros);
-   HypreParVector b(fes);
-   b = 0.0;
-
-   // 9. Define the initial conditions. Set up visualization (if desired).
-   std::unique_ptr<FunctionCoefficient> u0;
-   if (0==problem)
-   {
-      u0.reset(new FunctionCoefficient(u0_function<0>));
-   }
-   else if (1==problem)
-   {
-      u0.reset(new FunctionCoefficient(u0_function<1>));
-   }
-   else if (2==problem)
-   {
-      u0.reset(new FunctionCoefficient(u0_function<2>));
-   }
-   else if (3==problem)
-   {
-      u0.reset(new FunctionCoefficient(u0_function<3>));
-   }
-   ParGridFunction *u = new ParGridFunction(fes);
-   u->ProjectCoefficient(*u0);
-   HypreParVector *U = u->GetTrueDofs();
-
-   DataCollection *dc = NULL;
-   if (visit)
-   {
-      if (binary)
-      {
-#ifdef MFEM_USE_SIDRE
-         dc = new SidreDataCollection("Example41-Parallel", pmesh);
-#else
-         MFEM_ABORT("Must build with MFEM_USE_SIDRE=YES for binary output.");
-#endif
-      }
-      else
-      {
-         dc = new VisItDataCollection("Example41-Parallel", pmesh);
-         dc->SetPrecision(precision);
-         // To save the mesh using MFEM's parallel mesh format:
-         // dc->SetFormat(DataCollection::PARALLEL_FORMAT);
-      }
-      dc->RegisterField("solution", u);
-      dc->SetCycle(0);
-      dc->SetTime(0.0);
-      dc->Save();
-   }
-   ParaViewDataCollection *pd = NULL;
-   if (paraview)
-   {
-      pd = new ParaViewDataCollection("Example41P", pmesh);
-      pd->SetPrefixPath("ParaView");
-      pd->RegisterField("solution", u);
-      pd->SetLevelsOfDetail(order);
-      pd->SetDataFormat(VTKFormat::BINARY);
-      pd->SetHighOrderOutput(true);
-      pd->SetCycle(0);
-      pd->SetTime(0.0);
-      pd->Save();
-   }
-   socketstream sout;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      sout.open(vishost, visport);
-      if (!sout)
-      {
-         if (Mpi::Root())
-         {
-            cout << "Unable to connect to GLVis server at "
-                 << vishost << ':' << visport << endl;
-         }
-         visualization = false;
-         if (Mpi::Root())
-         {
-            cout << "GLVis visualization disabled.\n";
-         }
-      }
-      else
-      {
-         sout << "parallel " << num_procs << " " << myid << "\n";
-         sout.precision(precision);
-         sout << "solution\n" << *pmesh << *u;
-         sout << "pause\n";
-         sout << flush;
-         if (Mpi::Root())
-         {
-            cout << "GLVis visualization paused."
-                 << " Press space (in the GLVis window) to resume it.\n";
-         }
-      }
-   }
-#ifdef MFEM_USE_ADIOS2
-   ADIOS2DataCollection *adios2_dc = NULL;
-   if (adios2)
-   {
-      std::string postfix(mesh_file);
-      postfix.erase(0, std::string("../data/").size() );
-      postfix += "_o" + std::to_string(order);
-      const std::string collection_name = "ex41-p-" + postfix + ".bp";
-
-      adios2_dc = new ADIOS2DataCollection(MPI_COMM_WORLD, collection_name, pmesh);
-      // output data substreams are half the number of mpi processes
-      adios2_dc->SetParameter("SubStreams", std::to_string(num_procs/2) );
-      // adios2_dc->SetLevelsOfDetail(2);
-      adios2_dc->RegisterField("solution", u);
-      adios2_dc->SetCycle(0);
-      adios2_dc->SetTime(0.0);
-      adios2_dc->Save();
-   }
-#endif
-
-
-   // 10. Define the time-dependent evolution operator describing the
-   //     ODE right-hand side, and perform time-integration (looping
-   //     over the time iterations, ti, with a time-step dt).
-   IMEX_Evolution adv(*m, *k, *s, b, *a);
-
-   real_t t = 0.0;
-   adv.SetTime(t);
-   ode_solver->Init(adv);
-
-
-   bool done = false;
-   for (int ti = 0; !done; )
-   {
-      real_t dt_real = min(dt, t_final - t);
-      ode_solver->Step(*U, t, dt_real);
-      ti++;
-
-      done = (t >= t_final - 1e-8*dt);
-
-      if (done || ti % vis_steps == 0)
-      {
-         if (Mpi::Root())
-         {
-            cout << "time step: " << ti << ", time: " << t << endl;
-         }
-         *u = *U;
-         if (visualization)
-         {
-            sout << "parallel " << num_procs << " " << myid << "\n";
-            sout << "solution\n" << *pmesh << *u << flush;
-         }
-         if (paraview)
-         {
-            pd->SetCycle(ti);
-            pd->SetTime(t);
-            pd->Save();
-         }
-#ifdef MFEM_USE_ADIOS2
-         // transient solutions can be visualized with ParaView
-         if (adios2)
-         {
-            adios2_dc->SetCycle(ti);
-            adios2_dc->SetTime(t);
-            adios2_dc->Save();
-         }
-#endif
-      }
-   }
-
-   // 11. Free the used memory.
-   delete pd;
-   delete U;
-   delete u;
-   delete a;
-   delete s;
-   delete k;
-   delete m;
-   delete fes;
-   delete pmesh;
-   delete dc;
-   delete fec;
-
-   return 0;
-}
-
-// Implementation of class IMEX_Evolution
-IMEX_Evolution::IMEX_Evolution(ParBilinearForm &M_, ParBilinearForm &K_,
-                               ParBilinearForm &S_, const Vector &b_, ParBilinearForm &A_)
-   : TimeDependentOperator(M_.ParFESpace()->GetTrueVSize()), b(b_),
-     M_solver(M_.ParFESpace()->GetComm()), z(height), w(height)
-{
-   if (M_.GetAssemblyLevel()==AssemblyLevel::LEGACY)
-   {
-      M.Reset(M_.ParallelAssemble(), true);
-      K.Reset(K_.ParallelAssemble(), true);
-      S.Reset(S_.ParallelAssemble(), true);
-   }
-   else
-   {
-      M.Reset(&M_, false);
-      K.Reset(&K_, false);
-      S.Reset(&S_, false);
-   }
-
-   M_solver.SetOperator(*M);
-
-   Array<int> ess_tdof_list;
-   if (M_.GetAssemblyLevel() == AssemblyLevel::LEGACY)
-   {
-      A.Reset(A_.ParallelAssemble(), true);
-      HypreParMatrix &M_mat = *M.As<HypreParMatrix>();
-      HypreParMatrix &S_mat = *S.As<HypreParMatrix>();
-      HypreSmoother *hypre_prec = new HypreSmoother(M_mat, HypreSmoother::Jacobi);
-      M_prec = hypre_prec;
-
-      implicit_solver = new Implicit_Solver(M_mat, S_mat, *M_.FESpace());
-      lor_solver = new LORSolver<HypreBoomerAMG>(A_, ess_tdof_list);
-      lor_solver->GetSolver().SetSystemsOptions(A_.ParFESpace()->GetVDim(), true);
-      implicit_solver -> SetPreconditioner(*lor_solver);
-   }
-   else
-   {
-      MFEM_ABORT("Implicit time integration is not supported with partial assembly");
-   }
-   M_solver.SetPreconditioner(*M_prec);
-   M_solver.iterative_mode = false;
-   M_solver.SetRelTol(1e-9);
-   M_solver.SetAbsTol(0.0);
-   M_solver.SetMaxIter(100);
-   M_solver.SetPrintLevel(0);
-}
-
-void IMEX_Evolution::Mult1(const Vector &x, Vector &y) const
-{
-   // Perform the explicit step
-   // y = M^{-1} (K x + b)
-   K->Mult(x, z);
-   z += b;
-   M_solver.Mult(z, y);
-}
-
-void IMEX_Evolution::ImplicitSolve2(const real_t dt, const Vector &x, Vector &k)
-{
-   // Perform the implicit step
-   // solve for k, k = -(M+dt S)^{-1} S x
-   MFEM_VERIFY(implicit_solver != NULL,
-               "Implicit time integration is not supported with partial assembly");
-   S->Mult(x, z);
-   z*= -1.0;
-   implicit_solver->SetTimeStep(dt);
-   implicit_solver->Mult(z, k);
-}

examples/ex4p.cpp

@@ -9,7 +9,6 @@
 //               mpirun -np 4 ex4p -m ../data/beam-hex.mesh -o 2 -pa
 //               mpirun -np 4 ex4p -m ../data/escher.mesh -o 2 -sc
 //               mpirun -np 4 ex4p -m ../data/fichera.mesh -o 2 -hb
-//               mpirun -np 4 ex4p -m ../data/fichera.mesh -o 2 -hb -ea
 //               mpirun -np 4 ex4p -m ../data/fichera-q2.vtk
 //               mpirun -np 4 ex4p -m ../data/fichera-q3.mesh -o 2 -sc
 //               mpirun -np 4 ex4p -m ../data/square-disc-nurbs.mesh -o 3
@@ -18,18 +17,14 @@
 //               mpirun -np 4 ex4p -m ../data/periodic-cube.mesh -no-bc
 //               mpirun -np 4 ex4p -m ../data/amr-quad.mesh
 //               mpirun -np 3 ex4p -m ../data/amr-quad.mesh -o 2 -hb
-//               mpirun -np 3 ex4p -m ../data/amr-quad.mesh -o 2 -hb -ea
 //               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -sc
 //               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -hb
-//               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -hb -ea
 //               mpirun -np 4 ex4p -m ../data/ref-prism.mesh -o 1
 //               mpirun -np 4 ex4p -m ../data/octahedron.mesh -o 1
 //               mpirun -np 4 ex4p -m ../data/star-surf.mesh -o 3 -hb
 //
 // Device sample runs:
 //               mpirun -np 4 ex4p -m ../data/star.mesh -pa -d cuda
-//               mpirun -np 4 ex4p -m ../data/star.mesh -ea -hb -d cuda
-//               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -ea -hb -d cuda
 //               mpirun -np 4 ex4p -m ../data/star.mesh -pa -d raja-cuda
 //               mpirun -np 4 ex4p -m ../data/star.mesh -pa -d raja-omp
 //               mpirun -np 4 ex4p -m ../data/beam-hex.mesh -pa -d cuda
@@ -235,7 +230,7 @@ int main(int argc, char *argv[])
    pcg->SetMaxIter(2000);
    pcg->SetPrintLevel(1);
    if (hybridization) { prec = new HypreBoomerAMG(*A.As<HypreParMatrix>()); }
-   else if (pa || ea) { prec = new OperatorJacobiSmoother(*a, ess_tdof_list); }
+   else if (pa) { prec = new OperatorJacobiSmoother(*a, ess_tdof_list); }
    else
    {
       ParFiniteElementSpace *prec_fespace =

examples/ex9.cpp

@@ -160,7 +160,6 @@ int main(int argc, char *argv[])
    bool paraview = false;
    bool binary = false;
    int vis_steps = 5;
-   bool solve_implicit_state = false;
 
    int precision = 8;
    cout.precision(precision);
@@ -188,9 +187,6 @@ int main(int argc, char *argv[])
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
                   "Time step.");
-   args.AddOption(&solve_implicit_state, "-imp-state", "--implicit-state",
-                  "-imp-slope", "--implicit-slope",
-                  "Implicitly solve for stage state or slope.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -370,11 +366,6 @@ int main(int argc, char *argv[])
    //    right-hand side, and perform time-integration (looping over the time
    //    iterations, ti, with a time-step dt).
    FE_Evolution adv(m, k, b);
-   using ImplicitVariableType = FE_Evolution::ImplicitVariableType;
-   ImplicitVariableType imp_var = solve_implicit_state ?
-                                  ImplicitVariableType::STATE
-                                  : ImplicitVariableType::SLOPE;
-   adv.SetImplicitVariableType(imp_var);
 
    real_t t = 0.0;
    adv.SetTime(t);
@@ -468,17 +459,7 @@ void FE_Evolution::ImplicitSolve(const real_t dt, const Vector &x, Vector &k)
 {
    MFEM_VERIFY(dg_solver != NULL,
                "Implicit time integration is not supported with partial assembly");
-   // Construct current right-hand side for stage state vs. slope solve
-   if (ImplicitVarTypeIsState())
-   {
-      // k, on return, is the stage value u
-      M.Mult(x, z);
-   }
-   else
-   {
-      // k, on return, is the stage slope du/dt
-      K.Mult(x, z);
-   }
+   K.Mult(x, z);
    z += b;
    dg_solver->SetTimeStep(dt);
    dg_solver->Mult(z, k);

examples/ex9p.cpp

@@ -257,7 +257,6 @@ int main(int argc, char *argv[])
    bool adios2 = false;
    bool binary = false;
    int vis_steps = 5;
-   bool solve_implicit_state = false;
 #if MFEM_HYPRE_VERSION >= 21800
    PrecType prec_type = PrecType::AIR;
 #else
@@ -291,9 +290,6 @@ int main(int argc, char *argv[])
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
                   "Time step.");
-   args.AddOption(&solve_implicit_state, "-imp-state", "--implicit-state",
-                  "-imp-slope", "--implicit-slope",
-                  "Implicitly solve for stage state or slope.");
    args.AddOption((int *)&prec_type, "-pt", "--prec-type", "Preconditioner for "
                   "implicit solves. 0 for ILU, 1 for pAIR-AMG.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -540,11 +536,6 @@ int main(int argc, char *argv[])
    //     right-hand side, and perform time-integration (looping over the time
    //     iterations, ti, with a time-step dt).
    FE_Evolution adv(*m, *k, *B, prec_type);
-   using ImplicitVariableType = FE_Evolution::ImplicitVariableType;
-   ImplicitVariableType imp_var = solve_implicit_state ?
-                                  ImplicitVariableType::STATE
-                                  : ImplicitVariableType::SLOPE;
-   adv.SetImplicitVariableType(imp_var);
 
    real_t t = 0.0;
    adv.SetTime(t);
@@ -685,17 +676,7 @@ FE_Evolution::FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_,
 //    (M - dt*K) d = K*u + b
 void FE_Evolution::ImplicitSolve(const real_t dt, const Vector &x, Vector &k)
 {
-   // Construct current right-hand side for stage state vs. slope solve
-   if (ImplicitVarTypeIsState())
-   {
-      // k, on return, is the stage value u
-      M->Mult(x, z);
-   }
-   else
-   {
-      // k, on return, is the stage slope du/dt
-      K->Mult(x, z);
-   }
+   K->Mult(x, z);
    z += b;
    dg_solver->SetTimeStep(dt);
    dg_solver->Mult(z, k);

examples/ginkgo/CMakeLists.txt

@@ -14,12 +14,6 @@ list(APPEND GINKGO_EXAMPLES_SRCS
   ex1.cpp
 )
 
-if (MFEM_USE_MPI AND GINKGO_BUILD_MPI)
-    list(APPEND GINKGO_EXAMPLES_SRCS
-    ex1p.cpp
-)
-endif()
-
 # Include the source directory where mfem.hpp and mfem-performance.hpp are.
 include_directories(BEFORE ${PROJECT_BINARY_DIR})
 

examples/ginkgo/ex1.cpp

@@ -207,7 +207,7 @@ int main(int argc, char *argv[])
             Ginkgo::IcPreconditioner ginkgo_precond(exec, "paric", 30);
             Ginkgo::CGSolver ginkgo_solver(exec, ginkgo_precond);
             ginkgo_solver.SetPrintLevel(print_lvl);
-            ginkgo_solver.SetRelTol(sqrt(1e-12));
+            ginkgo_solver.SetRelTol(1e-12);
             ginkgo_solver.SetAbsTol(0.0);
             ginkgo_solver.SetMaxIter(400);
             ginkgo_solver.SetOperator(*(A.Ptr()));
@@ -225,7 +225,7 @@ int main(int argc, char *argv[])
             Ginkgo::MFEMPreconditioner gko_M(exec, M);
             Ginkgo::CGSolver ginkgo_solver(exec, gko_M);
             ginkgo_solver.SetPrintLevel(print_lvl);
-            ginkgo_solver.SetRelTol(sqrt(1e-12));
+            ginkgo_solver.SetRelTol(1e-12);
             ginkgo_solver.SetAbsTol(0.0);
             ginkgo_solver.SetMaxIter(400);
             ginkgo_solver.SetOperator(*(A.Ptr()));
@@ -283,7 +283,7 @@ int main(int argc, char *argv[])
                Ginkgo::MFEMPreconditioner gko_M(exec, M);
                Ginkgo::CGSolver ginkgo_solver(exec, gko_M);
                ginkgo_solver.SetPrintLevel(print_lvl);
-               ginkgo_solver.SetRelTol(sqrt(1e-12));
+               ginkgo_solver.SetRelTol(1e-12);
                ginkgo_solver.SetAbsTol(0.0);
                ginkgo_solver.SetMaxIter(400);
                ginkgo_solver.SetOperator(*(A.Ptr()));

examples/ginkgo/ex1p.cpp

@@ -1,436 +0,0 @@
-//                       MFEM Example 1 - Parallel Version
-//                             GINKGO Modification
-//
-// Compile with: make ex1p
-//
-// Sample runs:  mpirun -np 4 ex1p -m ../data/square-disc.mesh
-//               mpirun -np 4 ex1p -m ../data/star.mesh
-//               mpirun -np 4 ex1p -m ../data/star-mixed.mesh
-//               mpirun -np 4 ex1p -m ../data/escher.mesh
-//               mpirun -np 4 ex1p -m ../data/fichera.mesh
-//               mpirun -np 4 ex1p -m ../data/fichera-mixed.mesh
-//               mpirun -np 4 ex1p -m ../data/toroid-wedge.mesh
-//               mpirun -np 4 ex1p -m ../data/octahedron.mesh -o 1
-//               mpirun -np 4 ex1p -m ../data/periodic-annulus-sector.msh
-//               mpirun -np 4 ex1p -m ../data/periodic-torus-sector.msh
-//               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
-//               mpirun -np 4 ex1p -m ../data/square-disc-p3.mesh -o 3
-//               mpirun -np 4 ex1p -m ../data/square-disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/star-mixed-p2.mesh -o 2
-//               mpirun -np 4 ex1p -m ../data/disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/pipe-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/ball-nurbs.mesh -o 2
-//               mpirun -np 4 ex1p -m ../data/fichera-mixed-p2.mesh -o 2
-//               mpirun -np 4 ex1p -m ../data/star-surf.mesh
-//               mpirun -np 4 ex1p -m ../data/square-disc-surf.mesh
-//               mpirun -np 4 ex1p -m ../data/inline-segment.mesh
-//               mpirun -np 4 ex1p -m ../data/amr-quad.mesh
-//               mpirun -np 4 ex1p -m ../data/amr-hex.mesh
-//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh
-//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh -o -1 -sc
-//
-// Device sample runs:
-//               mpirun -np 4 ex1p -pa -d cuda
-//               mpirun -np 4 ex1p -fa -d cuda
-//               mpirun -np 4 ex1p -pa -d occa-cuda
-//               mpirun -np 4 ex1p -pa -d raja-omp
-//               mpirun -np 4 ex1p -pa -d ceed-cpu
-//               mpirun -np 4 ex1p -pa -d ceed-cpu -o 4 -a
-//               mpirun -np 4 ex1p -pa -d ceed-cpu -m ../data/square-mixed.mesh
-//               mpirun -np 4 ex1p -pa -d ceed-cpu -m ../data/fichera-mixed.mesh
-//             * mpirun -np 4 ex1p -pa -d ceed-cuda
-//             * mpirun -np 4 ex1p -pa -d ceed-hip
-//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
-//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/square-mixed.mesh
-//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/fichera-mixed.mesh
-//               mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order, or if order < 1 using an isoparametric/isogeometric
-//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example highlights the use of mesh refinement, finite
-//               element grid functions, as well as linear and bilinear forms
-//               corresponding to the left-hand side and right-hand side of the
-//               discrete linear system. We also cover the explicit elimination
-//               of essential boundary conditions, static condensation, and the
-//               optional connection to the GLVis tool for visualization.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-#ifndef MFEM_USE_GINKGO
-#error This example requires that MFEM is built with MFEM_USE_GINKGO=YES
-#endif
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI and HYPRE.
-   Mpi::Init();
-   int num_procs = Mpi::WorldSize();
-   int myid = Mpi::WorldRank();
-   Hypre::Init();
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../../data/star.mesh";
-   int order = 1;
-   bool static_cond = false;
-   bool pa = false;
-   bool fa = false;
-   const char *device_config = "cpu";
-   bool visualization = true;
-   int solver_config = 0;
-   int print_lvl = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
-                  "--no-full-assembly", "Enable Full Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&solver_config, "-s", "--solver-config",
-                  "Solver and preconditioner combination: \n\t"
-                  "   0 - Ginkgo solver and Ginkgo preconditioner, \n\t"
-                  "   1 - Ginkgo solver and MFEM preconditioner, \n\t"
-                  "   2 - MFEM solver and Ginkgo preconditioner, \n\t"
-                  "   3 - MFEM solver and MFEM preconditioner.");
-   args.AddOption(&print_lvl, "-pl", "--print-level",
-                  "Print level for iterative solver (1 prints every iteration).");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.SetGPUAwareMPI(true);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   // 5. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 10,000 elements.
-   {
-      int ref_levels =
-         (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh.UniformRefinement();
-      }
-   }
-
-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
-   {
-      int par_ref_levels = 2;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         pmesh.UniformRefinement();
-      }
-   }
-
-   // 7. Define a parallel finite element space on the parallel mesh. Here we
-   //    use continuous Lagrange finite elements of the specified order. If
-   //    order < 1, we instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   bool delete_fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
-   }
-   else if (pmesh.GetNodes())
-   {
-      fec = pmesh.GetNodes()->OwnFEC();
-      delete_fec = false;
-      if (myid == 0)
-      {
-         cout << "Using isoparametric FEs: " << fec->Name() << endl;
-      }
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
-   }
-   ParFiniteElementSpace fespace(&pmesh, fec);
-   HYPRE_BigInt size = fespace.GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-   }
-
-   // 8. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet) and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (pmesh.bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 9. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (1,phi_i) where phi_i are the basis functions in fespace.
-   ParLinearForm b(&fespace);
-   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
-
-   // 10. Define the solution vector x as a parallel finite element grid
-   //     function corresponding to fespace. Initialize x with initial guess of
-   //     zero, which satisfies the boundary conditions.
-   ParGridFunction x(&fespace);
-   x = 0.0;
-
-   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the
-   //     Diffusion domain integrator.
-   ParBilinearForm a(&fespace);
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   if (fa)
-   {
-      a.SetAssemblyLevel(AssemblyLevel::FULL);
-      // Sort the matrix column indices when running on GPU or with OpenMP (i.e.
-      // when Device::IsEnabled() returns true). This makes the results
-      // bit-for-bit deterministic at the cost of somewhat longer run time.
-      a.EnableSparseMatrixSorting(Device::IsEnabled());
-   }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   // 12. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
-   a.Assemble();
-
-   OperatorPtr A;
-   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-
-   // 13. Solve the linear system A X = B.
-   if (!pa)
-   {
-      switch (solver_config)
-      {
-         // Solve the linear system with CG + Schwarz (with IC) from Ginkgo
-         case 0:
-         {
-            if (myid == 0) { cout << "Using Ginkgo solver + preconditioner...\n"; }
-            Ginkgo::GinkgoExecutor exec(device);
-            Ginkgo::IcPreconditioner local_solver(exec, "exact");
-            Ginkgo::SchwarzPreconditioner gko_M(exec, MPI_COMM_WORLD, local_solver);
-            Ginkgo::CGSolver ginkgo_solver(exec, MPI_COMM_WORLD, gko_M);
-            ginkgo_solver.SetPrintLevel(print_lvl);
-            ginkgo_solver.SetRelTol(sqrt(1e-12));
-            ginkgo_solver.SetAbsTol(0.0);
-            ginkgo_solver.SetMaxIter(400);
-            ginkgo_solver.SetOperator(*(A.Ptr()));
-            ginkgo_solver.Mult(B, X);
-            break;
-         }
-
-         // Solve the linear system with CG from Ginkgo + MFEM preconditioner
-         case 1:
-         {
-            if (myid == 0) { cout << "Using Ginkgo solver + MFEM preconditioner...\n"; }
-            Ginkgo::GinkgoExecutor exec(device);
-            //Create MFEM preconditioner and wrap it for Ginkgo's use.
-            HypreBoomerAMG M((HypreParMatrix&)(*A));
-            Ginkgo::MFEMPreconditioner gko_M(exec, M, MPI_COMM_WORLD);
-            Ginkgo::CGSolver ginkgo_solver(exec, MPI_COMM_WORLD, gko_M);
-            ginkgo_solver.SetPrintLevel(print_lvl);
-            ginkgo_solver.SetRelTol(sqrt(1e-12));
-            ginkgo_solver.SetAbsTol(0.0);
-            ginkgo_solver.SetMaxIter(400);
-            ginkgo_solver.SetOperator(*(A.Ptr()));
-            ginkgo_solver.Mult(B, X);
-            break;
-         }
-
-         // Ginkgo Schwarz preconditioner (local ParIC) +  MFEM CG solver
-         case 2:
-         {
-            if (myid == 0) { cout << "Using MFEM solver + Ginkgo preconditioner...\n"; }
-            Ginkgo::GinkgoExecutor exec(device);
-            Ginkgo::IcPreconditioner local_M(exec, "exact");
-            Ginkgo::SchwarzPreconditioner M(exec, MPI_COMM_WORLD, local_M);
-            M.SetOperator(*(A.Ptr()));  // Generate the preconditioner for the matrix A.
-            CGSolver cg(MPI_COMM_WORLD);
-            cg.SetRelTol(sqrt(1e-12));
-            cg.SetMaxIter(400);
-            cg.SetPrintLevel(1);
-            cg.SetPreconditioner(M);
-            cg.SetOperator(*A);
-            cg.Mult(B, X);
-            break;
-         }
-
-         // MFEM solver + MFEM preconditioner
-         case 3:
-         {
-            if (myid == 0) { cout << "Using MFEM solver + MFEM preconditioner...\n"; }
-            HypreBoomerAMG M((HypreParMatrix&)(*A));
-            CGSolver cg(MPI_COMM_WORLD);
-            cg.SetRelTol(sqrt(1e-12));
-            cg.SetMaxIter(400);
-            cg.SetPrintLevel(1);
-            cg.SetPreconditioner(M);
-            cg.SetOperator(*A);
-            cg.Mult(B, X);
-            break;
-         }
-      } // End switch on solver_config
-   }
-   // Partial assembly mode.  Cannot use Ginkgo preconditioners, but can use Ginkgo
-   // solvers.
-   else
-   {
-      if (UsesTensorBasis(fespace))
-      {
-         // Use Jacobi preconditioning in partial assembly mode.
-         OperatorJacobiSmoother M(a, ess_tdof_list);
-         switch (solver_config)
-         {
-            case 0:
-            {
-               if (myid == 0) { cout << "Using Ginkgo solver + preconditioner...\n"; }
-               MFEM_ABORT("Cannot use Ginkgo preconditioner in partial assembly mode.\n"
-                          "            Try -s 1 to test Ginkgo solver with an MFEM preconditioner.");
-               break;
-            }
-
-            // Use Ginkgo solver with MFEM preconditioner
-            case 1:
-            {
-               if (myid == 0) { cout << "Using Ginkgo solver + MFEM preconditioner...\n"; }
-               Ginkgo::GinkgoExecutor exec(device);
-               // Wrap MFEM preconditioner for Ginkgo's use.
-               Ginkgo::MFEMPreconditioner gko_M(exec, M, MPI_COMM_WORLD);
-               Ginkgo::CGSolver ginkgo_solver(exec, MPI_COMM_WORLD, gko_M);
-               ginkgo_solver.SetPrintLevel(print_lvl);
-               ginkgo_solver.SetRelTol(sqrt(1e-12));
-               ginkgo_solver.SetAbsTol(0.0);
-               ginkgo_solver.SetMaxIter(400);
-               ginkgo_solver.SetOperator(*(A.Ptr()));
-               ginkgo_solver.Mult(B, X);
-               break;
-            }
-
-            // No Ginkgo preconditioners work with matrix-free; error
-            case 2:
-            {
-               if (myid == 0) { cout << "Using Ginkgo solver + preconditioner...\n"; }
-               MFEM_ABORT("Cannot use Ginkgo preconditioner in partial assembly mode.\n"
-                          "            Try -s 1 to test Ginkgo solver with an MFEM preconditioner.");
-               break;
-            }
-
-            // Use MFEM solver and preconditioner
-            case 3:
-            {
-               if (myid == 0) { cout << "Using MFEM solver + MFEM preconditioner...\n"; }
-               CGSolver cg(MPI_COMM_WORLD);
-               cg.SetRelTol(sqrt(1e-12));
-               cg.SetMaxIter(400);
-               cg.SetPrintLevel(1);
-               cg.SetPreconditioner(M);
-               cg.SetOperator(*A);
-               cg.Mult(B, X);
-               break;
-            }
-         } // End switch on solver_config
-      }
-      else // CG with no preconditioning
-      {
-         if (myid == 0) { cout << "Using MFEM solver + no preconditioner...\n"; }
-         CGSolver cg(MPI_COMM_WORLD);
-         cg.SetRelTol(sqrt(1e-12));
-         cg.SetMaxIter(400);
-         cg.SetPrintLevel(1);
-         cg.SetOperator(*A);
-         cg.Mult(B, X);
-      }
-   }
-
-   // 14. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a.RecoverFEMSolution(X, b, x);
-
-   // 15. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh.Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 16. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << pmesh << x << flush;
-   }
-
-   // 17. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
-
-   return 0;
-}

examples/ginkgo/makefile

@@ -20,8 +20,9 @@ CONFIG_MK = $(or $(wildcard $(MFEM_BUILD_DIR)/config/config.mk),\
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
 
+# Currently there are only serial Ginkgo examples
 SEQ_EXAMPLES = ex1
-PAR_EXAMPLES = ex1p
+PAR_EXAMPLES =
 ifeq ($(MFEM_USE_MPI),NO)
    EXAMPLES = $(SEQ_EXAMPLES)
 else
@@ -76,4 +77,4 @@ clean-build:
 	rm -rf *.dSYM *.TVD.*breakpoints
 
 clean-exec:
-	@rm -f refined.mesh sol.gf mesh.* sol.*
+	@rm -f refined.mesh sol.gf

examples/makefile

@@ -22,11 +22,11 @@ MFEM_LIB_FILE = mfem_is_not_built
 
 SEQ_EXAMPLES = ex0 ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 \
    ex17 ex18 ex19 ex20 ex21 ex22 ex23 ex24 ex25 ex26 ex27 ex28 ex29 ex30 \
-   ex31 ex33 ex34 ex36 ex37 ex38 ex39 ex40 ex41
+   ex31 ex33 ex34 ex36 ex37 ex38 ex39 ex40
 PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
    ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
    ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p ex33p ex34p ex35p ex36p \
-   ex37p ex39p ex40p ex41p
+   ex37p ex39p ex40p
 SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex14 ex22 ex24 ex25 ex26 ex34
 PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex14p \
    ex22p ex24p ex25p ex26p ex34p ex35p
@@ -71,7 +71,6 @@ endif
 
 SUBDIRS_ALL = $(addsuffix /all,$(SUBDIRS))
 SUBDIRS_TEST = $(addsuffix /test,$(SUBDIRS))
-SUBDIRS_TEST_NOCLEAN = $(addsuffix /test-noclean,$(SUBDIRS))
 SUBDIRS_CLEAN = $(addsuffix /clean,$(SUBDIRS))
 SUBDIRS_TPRINT = $(addsuffix /test-print,$(SUBDIRS))
 
@@ -88,9 +87,8 @@ SUBDIRS_TPRINT = $(addsuffix /test-print,$(SUBDIRS))
 
 all: $(EXAMPLES) $(SUBDIRS_ALL)
 
-.PHONY: $(SUBDIRS_ALL) $(SUBDIRS_TEST) $(SUBDIRS_TEST_NOCLEAN) \
-   $(SUBDIRS_CLEAN) $(SUBDIRS_TPRINT)
-$(SUBDIRS_ALL) $(SUBDIRS_TEST) $(SUBDIRS_TEST_NOCLEAN) $(SUBDIRS_CLEAN):
+.PHONY: $(SUBDIRS_ALL) $(SUBDIRS_TEST) $(SUBDIRS_CLEAN) $(SUBDIRS_TPRINT)
+$(SUBDIRS_ALL) $(SUBDIRS_TEST) $(SUBDIRS_CLEAN):
 	$(MAKE) -C $(@D) $(@F)
 $(SUBDIRS_TPRINT):
 	@$(MAKE) -C $(@D) $(@F)
@@ -109,7 +107,6 @@ endif
 MFEM_TESTS = EXAMPLES
 include $(MFEM_TEST_MK)
 test: $(SUBDIRS_TEST)
-test-noclean: $(SUBDIRS_TEST_NOCLEAN)
 test-print: $(SUBDIRS_TPRINT)
 
 # Testing: Parallel vs. serial runs
@@ -160,12 +157,6 @@ ex37-test-seq: ex37
 	@$(call mfem-test,$<,, Serial example,-mi 3)
 ex37p-test-par: ex37p
 	@$(call mfem-test,$<, $(RUN_MPI), Parallel example,-mi 3)
-ex39-test-seq: ex39
-	@$(call mfem-test,$<,, Serial example,-m ../data/compass.mesh)
-ex41-test-seq: ex41
-	@$(call mfem-test,$<,, Serial example,-tf 1.0)
-ex41p-test-par: ex41p
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel example,-tf 1.0)
 # Testing: optional tests
 ifeq ($(MFEM_USE_STRUMPACK),YES)
 ex11p-test-strumpack: ex11p
@@ -204,8 +195,8 @@ clean-build:
 clean-exec:
 	@rm -f refined.mesh displaced.mesh mesh.* ex5.mesh ex6p-checkpoint.*
 	@rm -rf Example5* Example9* Example15* Example16* Example23* ParaView
-	@rm -f sphere_refined.* sol.* sol_u.* sol_p.* sol_r.* sol_i.* sol_z.*
-	@rm -f order.* ex9.mesh ex9-mesh.* ex9-init.* ex9-final.*
+	@rm -f sphere_refined.* sol.* sol_u.* sol_p.* sol_r.* sol_i.* order.*
+	@rm -f ex9.mesh ex9-mesh.* ex9-init.* ex9-final.*
 	@rm -f deformed.* velocity.* elastic_energy.* mode_* mode_deriv_* flux.*
 	@rm -f ex5-p-*.bp ex9-p-*.bp ex12-p-*.bp ex16-p-*.bp
 	@rm -f ex16.mesh ex16-mesh.* ex16-init.* ex16-final.*

fem/CMakeLists.txt

@@ -128,46 +128,32 @@ set(SRCS
   normal_deriv_restriction.cpp
   staticcond.cpp
   tmop.cpp
-  tmop/pa.cpp
-  tmop/assemble/diag2_limit.cpp
-  tmop/assemble/diag2.cpp
-  tmop/assemble/grad2_limit.cpp
-  tmop/assemble/grad2.cpp
-  tmop/assemble/diag3_limit.cpp  
-  tmop/assemble/diag3.cpp
-  tmop/assemble/grad3_limit.cpp
-  tmop/assemble/grad3.cpp
-  tmop/metrics/001.cpp
-  tmop/metrics/002.cpp
-  tmop/metrics/007.cpp
-  tmop/metrics/056.cpp
-  tmop/metrics/077.cpp
-  tmop/metrics/080.cpp
-  tmop/metrics/094.cpp
-  tmop/metrics/302.cpp
-  tmop/metrics/303.cpp
-  tmop/metrics/315.cpp
-  tmop/metrics/318.cpp
-  tmop/metrics/321.cpp
-  tmop/metrics/332.cpp
-  tmop/metrics/338.cpp
-  tmop/mult/grad2_limit.cpp
-  tmop/mult/grad2.cpp
-  tmop/mult/mult2_limit.cpp
-  tmop/mult/mult2.cpp
-  tmop/mult/grad3_limit.cpp
-  tmop/mult/grad3.cpp
-  tmop/mult/mult3_limit.cpp
-  tmop/mult/mult3.cpp
-  tmop/tools/det2_jpr.cpp
-  tmop/tools/det3_jpr.cpp
-  tmop/tools/discrete.cpp
-  tmop/tools/energy2_limit.cpp
-  tmop/tools/energy2.cpp
-  tmop/tools/energy3_limit.cpp
-  tmop/tools/energy3.cpp
-  tmop/tools/target2.cpp
-  tmop/tools/target3.cpp
+  tmop/tmop_pa.cpp
+  tmop/tmop_pa_da3.cpp
+  tmop/tmop_pa_h2d.cpp
+  tmop/tmop_pa_h2d_c0.cpp
+  tmop/tmop_pa_h2m.cpp
+  tmop/tmop_pa_h2m_c0.cpp
+  tmop/tmop_pa_h2s.cpp
+  tmop/tmop_pa_h2s_c0.cpp
+  tmop/tmop_pa_h3d.cpp
+  tmop/tmop_pa_h3d_c0.cpp
+  tmop/tmop_pa_h3m.cpp
+  tmop/tmop_pa_h3m_c0.cpp
+  tmop/tmop_pa_h3s.cpp
+  tmop/tmop_pa_h3s_c0.cpp
+  tmop/tmop_pa_jp2.cpp
+  tmop/tmop_pa_jp3.cpp
+  tmop/tmop_pa_p2.cpp
+  tmop/tmop_pa_p2_c0.cpp
+  tmop/tmop_pa_p3.cpp
+  tmop/tmop_pa_p3_c0.cpp
+  tmop/tmop_pa_tc2.cpp
+  tmop/tmop_pa_tc3.cpp
+  tmop/tmop_pa_w2.cpp
+  tmop/tmop_pa_w2_c0.cpp
+  tmop/tmop_pa_w3.cpp
+  tmop/tmop_pa_w3_c0.cpp
   tmop_tools.cpp
   tmop_amr.cpp
   gslib.cpp
@@ -179,7 +165,6 @@ set(SRCS
   hyperbolic.cpp
   integrator.cpp
   bounds.cpp
-  particleset.cpp
   )
 
 set(HDRS
@@ -197,8 +182,6 @@ set(HDRS
   integ/bilininteg_hdiv_kernels.hpp
   integ/bilininteg_hcurlhdiv_kernels.hpp
   integ/bilininteg_mass_kernels.hpp
-  integ/bilininteg_vecdiffusion_pa.hpp
-  integ/bilininteg_vecmass_pa.hpp
   coefficient.hpp
   complex_fem.hpp
   convergence.hpp
@@ -296,12 +279,7 @@ set(HDRS
   tfespace.hpp
   tintrules.hpp
   tmop.hpp
-  tmop/pa.hpp
-  tmop/assemble/grad2.hpp
-  tmop/assemble/grad2.hpp
-  tmop/mult/mult2.hpp
-  tmop/mult/mult3.hpp
-  tmop/tools/energy2.hpp
+  tmop/tmop_pa.hpp
   tmop_tools.hpp
   tmop_amr.hpp
   gslib.hpp
@@ -309,7 +287,6 @@ set(HDRS
   hyperbolic.hpp
   integrator.hpp
   bounds.hpp
-  particleset.hpp
   )
 
 if (MFEM_USE_SIDRE)

fem/bilinearform.cpp

@@ -729,8 +729,7 @@ void BilinearForm::Assemble(int skip_zeros)
          tr = mesh -> GetBdrFaceTransformations (i);
          if (tr != NULL)
          {
-            mfem::DofTransformation doftrans;
-            fes -> GetElementVDofs (tr -> Elem1No, vdofs, doftrans);
+            fes -> GetElementVDofs (tr -> Elem1No, vdofs);
             fe1 = fes -> GetFE (tr -> Elem1No);
             // The fe2 object is really a dummy and not used on the boundaries,
             // but we can't dereference a NULL pointer, and we don't want to
@@ -744,7 +743,6 @@ void BilinearForm::Assemble(int skip_zeros)
 
                boundary_face_integs[k] -> AssembleFaceMatrix (*fe1, *fe2, *tr,
                                                               elemmat);
-               doftrans.TransformDual(elemmat);
                mat -> AddSubMatrix (vdofs, vdofs, elemmat, skip_zeros);
             }
          }
@@ -827,46 +825,14 @@ void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
                                     Vector &b, OperatorHandle &A, Vector &X,
                                     Vector &B, int copy_interior)
 {
-   const SparseMatrix *P = fes->GetConformingProlongation();
-   const SparseMatrix *R = fes->GetConformingRestriction();
    if (ext)
    {
       if (hybridization)
       {
          FormSystemMatrix(ess_tdof_list, A);
-
-         std::unique_ptr<ConstrainedOperator> A_constrained([&]()
-         {
-            Operator *op;
-            Operator::FormSystemOperator(ess_tdof_list, op);
-            return dynamic_cast<ConstrainedOperator*>(op);
-         }());
-         MFEM_ASSERT(A_constrained != nullptr, "");
-
-         Vector conf_b, conf_x;
-         if (P)
-         {
-            // Nonconforming
-            conf_b.SetSize(P->Width());
-            conf_x.SetSize(P->Width());
-            P->MultTranspose(b, conf_b);
-            R->Mult(x, conf_x);
-         }
-         else
-         {
-            // Conforming
-            conf_b.MakeRef(b, 0, b.Size());
-            conf_x.MakeRef(x, 0, x.Size());
-         }
-
-         A_constrained->EliminateRHS(conf_x, conf_b);
-
-         if (P)
-         {
-            R->MultTranspose(conf_b, b); // store eliminated rhs in b
-         }
-
-         hybridization->ReduceRHS(conf_b, B);
+         ConstrainedOperator A_constrained(this, ess_tdof_list);
+         A_constrained.EliminateRHS(x, b);
+         hybridization->ReduceRHS(b, B);
          X.SetSize(B.Size());
          X = 0.0;
       }
@@ -876,6 +842,7 @@ void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
       }
       return;
    }
+   const SparseMatrix *P = fes->GetConformingProlongation();
    FormSystemMatrix(ess_tdof_list, A);
 
    // Transform the system and perform the elimination in B, based on the
@@ -911,6 +878,7 @@ void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
       if (hybridization)
       {
          // Reduction to the Lagrange multipliers system
+         const SparseMatrix *R = fes->GetConformingRestriction();
          Vector conf_b(P->Width()), conf_x(P->Width());
          P->MultTranspose(b, conf_b);
          R->Mult(x, conf_x);
@@ -923,6 +891,7 @@ void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
       else
       {
          // Variational restriction with P
+         const SparseMatrix *R = fes->GetConformingRestriction();
          B.SetSize(P->Width());
          P->MultTranspose(b, B);
          X.SetSize(R->Height());
@@ -1725,7 +1694,6 @@ void MixedBilinearForm::Assemble(int skip_zeros)
          }
       }
 
-      DofTransformation dom_dof_trans, ran_dof_trans;
       for (int i = 0; i < trial_fes -> GetNBE(); i++)
       {
          const int bdr_attr = mesh->GetBdrAttribute(i);
@@ -1734,8 +1702,8 @@ void MixedBilinearForm::Assemble(int skip_zeros)
          ftr = mesh -> GetBdrFaceTransformations (i);
          if (ftr != NULL)
          {
-            trial_fes->GetElementVDofs(ftr->Elem1No, trial_vdofs, dom_dof_trans);
-            test_fes->GetElementVDofs(ftr->Elem1No, test_vdofs, ran_dof_trans);
+            trial_fes->GetElementVDofs(ftr->Elem1No, trial_vdofs);
+            test_fes->GetElementVDofs(ftr->Elem1No, test_vdofs);
             trial_fe1 = trial_fes->GetFE(ftr->Elem1No);
             test_fe1 = test_fes->GetFE(ftr->Elem1No);
             // The test_fe2 object is really a dummy and not used on the
@@ -1751,7 +1719,6 @@ void MixedBilinearForm::Assemble(int skip_zeros)
                boundary_face_integs[k]->AssembleFaceMatrix(*trial_fe1, *test_fe1, *trial_fe2,
                                                            *test_fe2,
                                                            *ftr, elemmat);
-               TransformDual(ran_dof_trans, dom_dof_trans, elemmat);
                mat->AddSubMatrix(test_vdofs, trial_vdofs, elemmat, skip_zeros);
             }
          }

fem/bilininteg.cpp

@@ -3066,6 +3066,7 @@ void VectorDiffusionIntegrator::AssembleElementMatrix(
 
    for (int i = 0; i < ir -> GetNPoints(); i++)
    {
+
       const IntegrationPoint &ip = ir->IntPoint(i);
       el.CalcDShape(ip, dshape);
 

fem/bilininteg.hpp

@@ -2596,40 +2596,41 @@ public:
     by scalar FE through standard transformation. */
 class VectorMassIntegrator: public BilinearFormIntegrator
 {
-   int vdim = -1, Q_order = 0;
+private:
+   int vdim;
    Vector shape, te_shape, vec;
    DenseMatrix partelmat;
    DenseMatrix mcoeff;
+   int Q_order;
 
 protected:
-   Coefficient *Q = nullptr;
-   VectorCoefficient *VQ = nullptr;
-   MatrixCoefficient *MQ = nullptr;
+   Coefficient *Q;
+   VectorCoefficient *VQ;
+   MatrixCoefficient *MQ;
    // PA extension
+   Vector pa_data;
    const DofToQuad *maps;         ///< Not owned
    const GeometricFactors *geom;  ///< Not owned
-   int ne, dim, dofs1D, quad1D, coeff_vdim;
-   Vector pa_data;
+   int dim, ne, nq, dofs1D, quad1D;
 
 public:
    /// Construct an integrator with coefficient 1.0
-   VectorMassIntegrator() = default;
-
+   VectorMassIntegrator()
+      : vdim(-1), Q_order(0), Q(NULL), VQ(NULL), MQ(NULL) { }
    /** Construct an integrator with scalar coefficient q.  If possible, save
        memory by using a scalar integrator since the resulting matrix is block
        diagonal with the same diagonal block repeated. */
-   VectorMassIntegrator(Coefficient &q, int qo = 0): Q_order(qo), Q(&q) { }
-
-   VectorMassIntegrator(Coefficient &q, const IntegrationRule *ir):
-      BilinearFormIntegrator(ir), Q(&q) { }
-
+   VectorMassIntegrator(Coefficient &q, int qo = 0)
+      : vdim(-1), Q_order(qo), Q(&q), VQ(NULL), MQ(NULL) { }
+   VectorMassIntegrator(Coefficient &q, const IntegrationRule *ir)
+      : BilinearFormIntegrator(ir), vdim(-1), Q_order(0), Q(&q), VQ(NULL),
+        MQ(NULL) { }
    /// Construct an integrator with diagonal coefficient q
-   VectorMassIntegrator(VectorCoefficient &q, int qo = 0):
-      vdim(q.GetVDim()), Q_order(qo), VQ(&q) { }
-
+   VectorMassIntegrator(VectorCoefficient &q, int qo = 0)
+      : vdim(q.GetVDim()), Q_order(qo), Q(NULL), VQ(&q), MQ(NULL) { }
    /// Construct an integrator with matrix coefficient q
-   VectorMassIntegrator(MatrixCoefficient &q, int qo = 0):
-      vdim(q.GetVDim()), Q_order(qo), MQ(&q) { }
+   VectorMassIntegrator(MatrixCoefficient &q, int qo = 0)
+      : vdim(q.GetVDim()), Q_order(qo), Q(NULL), VQ(NULL), MQ(&q) { }
 
    int GetVDim() const { return vdim; }
    void SetVDim(int vdim_) { vdim = vdim_; }
@@ -2641,7 +2642,6 @@ public:
                                const FiniteElement &test_fe,
                                ElementTransformation &Trans,
                                DenseMatrix &elmat) override;
-
    using BilinearFormIntegrator::AssemblePA;
    void AssemblePA(const FiniteElementSpace &fes) override;
    void AssembleMF(const FiniteElementSpace &fes) override;
@@ -2650,15 +2650,6 @@ public:
    void AddMultPA(const Vector &x, Vector &y) const override;
    void AddMultMF(const Vector &x, Vector &y) const override;
    bool SupportsCeed() const override { return DeviceCanUseCeed(); }
-
-   using VectorMassAddMultPAType =
-      void(*)(const int,  const int,
-              const Array<real_t>&, const Vector&,
-              const Vector&, Vector&, const int, const int);
-
-   MFEM_REGISTER_KERNELS(VectorMassAddMultPA,
-                         VectorMassAddMultPAType,
-                         (int, int, int));
 };
 
 
@@ -2710,7 +2701,7 @@ public:
 
 
 /** Integrator for $(-Q u, \nabla v)$ for Nedelec ($u$) and $H^1$ ($v$) elements.
-    This is equivalent to a weak divergence of the $H(curl)$ basis functions. */
+    This is equivalent to a weak divergence of the $H(curl$ basis functions. */
 class VectorFEWeakDivergenceIntegrator: public BilinearFormIntegrator
 {
 protected:
@@ -3129,21 +3120,23 @@ public:
     to be the spatial dimension (i.e. 2-dimension or 3-dimension). */
 class VectorDiffusionIntegrator : public BilinearFormIntegrator
 {
-   int vdim = -1;
-   DenseMatrix dshape, dshapedxt, pelmat;
-   DenseMatrix mcoeff;
-   Vector vcoeff;
-
 protected:
-   Coefficient *Q = nullptr;
-   VectorCoefficient *VQ = nullptr;
-   MatrixCoefficient *MQ = nullptr;
+   Coefficient *Q = NULL;
+   VectorCoefficient *VQ = NULL;
+   MatrixCoefficient *MQ = NULL;
+
    // PA extension
    const DofToQuad *maps;         ///< Not owned
    const GeometricFactors *geom;  ///< Not owned
-   int ne, dim, sdim, dofs1D, quad1D, coeff_vdim;
+   int dim, sdim, ne, dofs1D, quad1D;
    Vector pa_data;
 
+private:
+   DenseMatrix dshape, dshapedxt, pelmat;
+   int vdim = -1;
+   DenseMatrix mcoeff;
+   Vector vcoeff;
+
 public:
    VectorDiffusionIntegrator(const IntegrationRule *ir = nullptr);
 
@@ -3196,7 +3189,6 @@ public:
    void AssembleElementVector(const FiniteElement &el,
                               ElementTransformation &Tr,
                               const Vector &elfun, Vector &elvect) override;
-
    using BilinearFormIntegrator::AssemblePA;
    void AssemblePA(const FiniteElementSpace &fes) override;
    void AssembleMF(const FiniteElementSpace &fes) override;
@@ -3206,11 +3198,13 @@ public:
    void AddMultMF(const Vector &x, Vector &y) const override;
    bool SupportsCeed() const override { return DeviceCanUseCeed(); }
 
-   /// arguments: ne, coeff_vdim, B, G, pa_data, x, y, d1d, q1d, vdim
-   using ApplyKernelType = void (*)(const int, const int,
-                                    const Array<real_t> &, const Array<real_t> &,
-                                    const Vector &, const Vector &, Vector &,
-                                    const int, const int, const int);
+   /// arguments: ne, B, G, Bt, Gt, pa_data, x, y, d1d, q1d, vdim
+   using ApplyKernelType = void (*)(const int, const Array<real_t> &,
+                                    const Array<real_t> &,
+                                    const Array<real_t> &,
+                                    const Array<real_t> &, const Vector &,
+                                    const Vector &, Vector &, const int,
+                                    const int, const int);
 
    /// arguments: dim, vdim, d1d, q1d
    MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int, int));
@@ -3221,7 +3215,10 @@ public:
       ApplyPAKernels::Specialization<DIM, VDIM, D1D, Q1D>::Add();
    }
 
-   // struct Kernels { Kernels(); };
+   struct Kernels
+   {
+      Kernels();
+   };
 };
 
 /** Integrator for the linear elasticity form:

fem/bounds.cpp

@@ -39,16 +39,11 @@ void PLBound::Setup(const int nb_i, const int ncp_i,
    b_type = b_type_i;
    cp_type = cp_type_i;
    tol = tol_i;
-   lbound.SetSize(ncp, nb);
-   ubound.SetSize(ncp, nb);
-   lbound_t.SetSize(nb, ncp);
-   ubound_t.SetSize(nb, ncp);
+   lbound.SetSize(nb, ncp);
+   ubound.SetSize(nb, ncp);
    nodes.SetSize(nb);
    weights.SetSize(nb);
    control_points.SetSize(ncp);
-   xhat.SetSize(nb);
-   what.SetSize(nb);
-   cphat.SetSize(ncp);
 
    auto scalenodes = [](const Vector &in, const real_t a, const real_t b) -> Vector
    {
@@ -95,10 +90,6 @@ void PLBound::Setup(const int nb_i, const int ncp_i,
       MFEM_ABORT("Unsupported interval points. Use [0,1].\n");
    }
    control_points = scalenodes(control_points, 0.0, 1.0); // rescale to [0,1]
-   for (int i = 0; i < ncp; i++)
-   {
-      cphat(i) = 2.0*control_points(i) - 1.0;
-   }
 
    Poly_1D::Basis &basis1d(poly1d.GetBasis(nb-1, b_type));
 
@@ -134,28 +125,22 @@ void PLBound::Setup(const int nb_i, const int ncp_i,
       {
          if (j == 0)
          {
-            lbound(j,i) = bv(i);
-            ubound(j,i) = bv(i);
+            lbound(i, j) = bv(i);
+            ubound(i, j) = bv(i);
          }
          else if (j == ncp-1)
          {
-            lbound(j,i) = bv(i);
-            ubound(j,i) = bv(i);
+            lbound(i, j) = bv(i);
+            ubound(i, j) = bv(i);
          }
          else
          {
             vals(0)  = bv(i);
             vals(1) =  bmv(i) +  dm*bdmv(i);
             vals(2) =  bpv(i) +  dp*bdpv(i);
-            lbound(j,i) = vals.Min()-tol; // tolerance for good measure
-            ubound(j,i) = vals.Max()+tol; // tolerance for good measure
-            if (b_type == 2)
-            {
-               lbound(j,i) = std::max(lbound(j,i),0_r);
-            }
+            lbound(i, j) = vals.Min()-tol; // tolerance for good measure
+            ubound(i, j) = vals.Max()+tol; // tolerance for good measure
          }
-         lbound_t(i,j) = lbound(j,i);
-         ubound_t(i,j) = ubound(j,i);
       }
    }
 
@@ -187,11 +172,6 @@ void PLBound::Setup(const int nb_i, const int ncp_i,
          nodes(i) = irule.IntPoint(i).x;
       }
    }
-   for (int i = 0; i < nb; i++)
-   {
-      xhat(i) = 2.0*nodes(i) - 1.0;
-      what(i) = 2.0*weights(i);
-   }
 
    if (b_type == 2)
    {
@@ -227,8 +207,7 @@ void PLBound::Setup(const int nb_i, const int ncp_i,
    }
 }
 
-PLBound::PLBound(const FiniteElementSpace *fes, const int ncp_i,
-                 const int cp_type_i)
+PLBound::PLBound(FiniteElementSpace *fes, int ncp_i, int cp_type_i)
 {
    MFEM_VERIFY(!fes->IsVariableOrder(),
                "Variable order meshes not yet supported.");
@@ -285,15 +264,15 @@ PLBound::PLBound(const FiniteElementSpace *fes, const int ncp_i,
    Setup(nb, ncp, b_type, cp_type, tol);
 }
 
-void PLBound::Get1DBounds(const Vector &coeff, Vector &intmin,
-                          Vector &intmax) const
+void PLBound::Get1DBounds(Vector &coeff, Vector &intmin, Vector &intmax) const
 {
    real_t x,w;
    intmin.SetSize(ncp);
    intmax.SetSize(ncp);
    intmin = 0.0;
    intmax = 0.0;
-   Vector coeffm;
+   Vector coeffm(nb);
+   coeffm = 0.0;
 
    real_t a0 = 0.0;
    real_t a1 = 0.0;
@@ -321,8 +300,6 @@ void PLBound::Get1DBounds(const Vector &coeff, Vector &intmin,
    // compute L2 projection for linear bases: a0 + a1*x
    if (proj)
    {
-      coeffm.SetSize(nb);
-      coeffm = 0.0;
       for (int i = 0; i < nb; i++)
       {
          x = 2.0*nodes_int(i)-1;
@@ -363,14 +340,13 @@ void PLBound::Get1DBounds(const Vector &coeff, Vector &intmin,
       real_t c = coeffm(i);
       for (int j = 0; j < ncp; j++)
       {
-         intmin(j) += min(lbound(j,i)*c, ubound(j,i)*c);
-         intmax(j) += max(lbound(j,i)*c, ubound(j,i)*c);
+         intmin(j) += min(lbound(i,j)*c, ubound(i,j)*c);
+         intmax(j) += max(lbound(i,j)*c, ubound(i,j)*c);
       }
    }
 }
 
-void PLBound::Get2DBounds(const Vector &coeff, Vector &intmin,
-                          Vector &intmax) const
+void PLBound::Get2DBounds(Vector &coeff, Vector &intmin, Vector &intmax) const
 {
    intmin.SetSize(ncp*ncp);
    intmax.SetSize(ncp*ncp);
@@ -495,10 +471,10 @@ void PLBound::Get2DBounds(const Vector &coeff, Vector &intmin,
          real_t w1 = intmaxT(id2++);
          for (int k = 0; k < ncp; k++) // kth row
          {
-            vals(0) = w0*lbound(k,j);
-            vals(1) = w0*ubound(k,j);
-            vals(2) = w1*lbound(k,j);
-            vals(3) = w1*ubound(k,j);
+            vals(0) = w0*lbound(j,k);
+            vals(1) = w0*ubound(j,k);
+            vals(2) = w1*lbound(j,k);
+            vals(3) = w1*ubound(j,k);
             intmin(k*ncp+i) += vals.Min();
             intmax(k*ncp+i) += vals.Max();
          }
@@ -506,8 +482,7 @@ void PLBound::Get2DBounds(const Vector &coeff, Vector &intmin,
    }
 }
 
-void PLBound::Get3DBounds(const Vector &coeff, Vector &intmin,
-                          Vector &intmax) const
+void PLBound::Get3DBounds(Vector &coeff, Vector &intmin, Vector &intmax) const
 {
    int nb2 = nb*nb,
        ncp2 = ncp*ncp,
@@ -574,17 +549,17 @@ void PLBound::Get3DBounds(const Vector &coeff, Vector &intmin,
             for (int i = 0; i < nb; i++)
             {
                x = 2.0*nodes(i)-1; // x-coordinate
-               minNodalVals(i) -= a0V(j) + a1V(j)*x;
-               maxNodalVals(i) -= a0V(j) + a1V(j)*x;
+               minBounds(i) -= a0V(j) + a1V(j)*x;
+               maxBounds(i) -= a0V(j) + a1V(j)*x;
             }
             // Compute Bernstein coefficients
             LUFactors lu(basisMatLU.GetData(), lu_ip.GetData());
-            lu.Solve(nb, 1, minNodalVals.GetData());
-            lu.Solve(nb, 1, maxNodalVals.GetData());
+            lu.Solve(nb, 1, minBounds.GetData());
+            lu.Solve(nb, 1, maxBounds.GetData());
             for (int i = 0; i < nb; i++)
             {
-               intminT(i*ncp2+j) = minNodalVals(i);
-               intmaxT(i*ncp2+j) = maxNodalVals(i);
+               intminT(i*ncp2+j) = minBounds(i);
+               intmaxT(i*ncp2+j) = maxBounds(i);
             }
          }
       }
@@ -638,10 +613,10 @@ void PLBound::Get3DBounds(const Vector &coeff, Vector &intmin,
          real_t w1 = intmaxT(id2++);
          for (int k = 0; k < ncp; k++) // kth slice
          {
-            vals(0) = w0*lbound(k,j);
-            vals(1) = w0*ubound(k,j);
-            vals(2) = w1*lbound(k,j);
-            vals(3) = w1*ubound(k,j);
+            vals(0) = w0*lbound(j,k);
+            vals(1) = w0*ubound(j,k);
+            vals(2) = w1*lbound(j,k);
+            vals(3) = w1*ubound(j,k);
             intmin(k*ncp2+i) += vals.Min();
             intmax(k*ncp2+i) += vals.Max();
          }
@@ -649,7 +624,7 @@ void PLBound::Get3DBounds(const Vector &coeff, Vector &intmin,
    }
 }
 
-void PLBound::GetNDBounds(const int rdim, const Vector &coeff,
+void PLBound::GetNDBounds(int rdim, Vector &coeff,
                           Vector &intmin, Vector &intmax) const
 {
    if (rdim == 1)
@@ -674,8 +649,7 @@ void PLBound::SetupBernsteinBasisMat(DenseMatrix &basisMat,
                                      Vector &nodesBern) const
 {
    const int nbern = nodesBern.Size();
-   L2_SegmentElement el(nbern-1, 2);
-   // we use L2 to leverage lexicographic order
+   L2_SegmentElement el(nbern-1, 2); // we use L2 to leverage lexicographic order
    Array<int> ordering = el.GetLexicographicOrdering();
    basisMat.SetSize(nbern, nbern);
    Vector shape(nbern);
@@ -688,39 +662,6 @@ void PLBound::SetupBernsteinBasisMat(DenseMatrix &basisMat,
    }
 }
 
-DenseMatrix PLBound::GetBoundingMatrix(int dim, bool is_lower) const
-{
-   if (dim > 1)
-   {
-      const int ncpd = static_cast<int>(std::pow(ncp, dim));
-      const int nbd = static_cast<int>(std::pow(nb, dim));
-      DenseMatrix boundND(ncpd, nbd);
-      Vector phimin, phimax, col;
-      Vector coeffs(nbd);
-      coeffs = 0.0;
-      for (int j = 0; j < nbd; j++)
-      {
-         coeffs(j) = 1.0;
-         boundND.GetColumnReference(j, col);
-         GetNDBounds(dim, coeffs, phimin, phimax);
-         col = is_lower ? phimin : phimax;
-         coeffs(j) = 0.0;
-      }
-      return boundND;
-   }
-   return is_lower ? lbound : ubound;
-}
-
-DenseMatrix PLBound::GetLowerBoundMatrix(int dim) const
-{
-   return GetBoundingMatrix(dim, true);
-}
-
-DenseMatrix PLBound::GetUpperBoundMatrix(int dim) const
-{
-   return GetBoundingMatrix(dim, false);
-}
-
 constexpr int PLBound::min_ncp_gl_x[2][11];
 constexpr int PLBound::min_ncp_gll_x[2][11];
 constexpr int PLBound::min_ncp_pos_x[2][11];

fem/bounds.hpp

@@ -9,29 +9,24 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#ifndef MFEM_BOUNDS
-#define MFEM_BOUNDS
+#ifndef MFEM_BOUND
+#define MFEM_BOUND
 
 #include "../config/config.hpp"
-#include "../general/forall.hpp"
 #include "fespace.hpp"
 
 namespace mfem
 {
 
 /** @name Piecewise linear bounds of bases
-    \brief Piecewise linear bounds of bases can be used to compute bounds on
-    the grid function in each element. The bounds for the bases are constructed
-    based on the following parameters:
+    \brief Piecewise linear bounds of bases can be used to compute bounds on the grid function in each element. The bounds for the bases are constructed based on the following parameters:
 
     (i) @b nb: number of bases/nodes in 1D (i.e. polynomial order+1),
 
-    (ii) @b b_type: bases type, 0 - Lagrange interpolants on Gauss-Legendre
-    nodes, 1 - Lagrange interpolants on Gauss-Lobatto-Legendre nodes, and
+    (ii) @b b_type: bases type, 0 - Lagrange interpolants on Gauss-Legendre nodes, 1 - Lagrange interpolants on Gauss-Lobatto-Legendre nodes, and
     2 - Positive/Bernstein bases on uniformly distributed nodes,
 
-    (iii) @b ncp: number of control points used to construct the piecewise
-    linear bounds
+    (iii) @b ncp: number of control points used to construct the piecewise linear bounds
 
     (iv) @b cp_type: control point distribution. 0 - GL + end-points,
                     1 - Chebyshev.
@@ -40,9 +35,7 @@ namespace mfem
 
     If the user does not specify @b ncp and @b cp_type, the minimum value of
     @b ncp is used that would bound the bases for the @b cp_type. We default
-    to @b cp_type = 0 as it requires fewer number of points to bound the bases.
-    Typically, @b ncp = 2 @b nb is sufficient to get fairly compact bounds, and
-    increasing @b ncp results in tighter bounds.
+    to @b cp_type = 0 as it requires fewer number of points to bound the bases. Typically, @b ncp = 2 @b nb is sufficient to get fairly compact bounds, and increasing @b ncp results in tighter bounds.
 
     Finally, only tensor-product elements are currently supported.
 
@@ -61,9 +54,7 @@ private:
    bool proj = true; // Use linear projection to compute bounds.
    real_t tol = 0.0; // offset bounds to avoid round-off errors
    Vector nodes, weights, control_points;
-   Vector xhat, what, cphat;
-   DenseMatrix lbound, ubound; // ncp x nb matrices with bounds of all bases
-   DenseMatrix lbound_t, ubound_t; // nb x ncp transposes for device kernel
+   DenseMatrix lbound, ubound; // nb x ncp matrices with bounds of all bases
    // Some auxillary storage for computing the bounds with Bernstein
    DenseMatrix basisMatNodes; // Bernstein bases at equispaced nodes
    DenseMatrix basisMatInt;   // Bernstein bases at GLL nodes
@@ -89,9 +80,6 @@ private:
       {3,5,8,9,11,12,13,13,14,15,16}
    };
 
-   /// Helper function to extract lower or upper bounding matrix
-   DenseMatrix GetBoundingMatrix(int dim, bool is_lower) const;
-
 public:
    // Constructor
    PLBound(const int nb_i, const int ncp_i, const int b_type_i,
@@ -101,702 +89,48 @@ public:
    }
 
    // Constructor
-   PLBound(const FiniteElementSpace *fes,
-           const int ncp_i = -1, const int cp_type_i = 0);
+   PLBound(FiniteElementSpace *fes, int ncp_i = -1, int cp_type_i = 0);
 
-   /// Get minimum number of control points needed to bound the given bases
+   // Get minimum number of control points needed to bound the given bases
    int GetMinimumPointsForGivenBases(int nb_i, int b_type_i,
                                      int cp_type_i) const;
 
-   /// Print information about the bounds
+   // Print information about the bounds
    void Print(std::ostream &outp = mfem::out) const;
 
-   /** @brief Enable (default) or disable linear projection before bounding.
-    *
-    *  @details This projection increases the computational cost but results in
-    *  tighter bounds.
-    */
-   void SetProjectionFlagForBounding(bool proj_)
-   {
-      proj = proj_;
-   }
+   // Enable (default) or disable linear projection before bounding.
+   // This projection increases the computational cost but results in tighter
+   // bounds.
+   void SetProjectionFlagForBounding(bool proj_) { proj = proj_; }
 
-   /** @brief Compute piecewise linear bounds for the lexicographically-ordered
-    *  nodal coefficients in @a coeff in 1D/2D/3D.
-    *
-    *  @param[in] rdim    The spatial dimension of the element (1, 2, or 3).
-    *  @param[in] coeff   The vector of lexicographically-ordered coefficients.
-    *                     Should be of size nb^rdim, where nb is the number of
-    *                     bases/nodes in 1D. These coefficients must correspond
-    *                     to the bases type and number of bases, used in the
-    *                     constructor of PLBound.
-    *
-    *  @param[out] intmin The vector of minimum bound for all control points.
-    *  @param[out] intmax The vector of maximum bound for all control points.
-    *                     Both intmin and intmax are of size ncp^rdim, where
-    *                     ncp is the number of control points in 1D, and are
-    *                     ordered lexicographically.
-    */
-   void GetNDBounds(const int rdim, const Vector &coeff,
+   /// Compute piecewise linear bounds for the lexicographically-ordered
+   /// coefficients in @a coeff in 1D/2D/3D.
+   void GetNDBounds(int rdim, Vector &coeff,
                     Vector &intmin, Vector &intmax) const;
 
    /// Get number of control points used to compute the bounds.
    int GetNControlPoints() const { return ncp; }
-
-   /// Get the underlying 1D basis type.
-   int GetBasisType() const { return b_type; }
-
-   /// Get 1D control point locations (lexicographic order) in [0,1].
-   const Vector &GetControlPoints() const { return control_points; }
-
-   /** @brief Compute element-wise bounds from a lexicographic E-vector.
-    *
-    *  @details The expected layout of @a e_vec is `ND x VDIM x NE`, where
-    *  `ND = nb^rdim`, `VDIM = fes_vdim`, and `NE` is the number of elements.
-    *  The output layout matches GridFunction::GetElementBounds:
-    *  `NE x active_vdim`, with the element index varying fastest.
-    */
-   void GetElementBoundsKernel(const int rdim, const int fes_vdim,
-                               const Vector &e_vec, Vector &lower,
-                               Vector &upper, const int vdim = 0) const;
-
-   /** @brief Get lower and upper bounding matrix (ncp^dim x nb^dim)
-    *
-    *  @details The matrices can be used to compute the bounds at control points
-    *           by a simple matrix-vector product with the
-    *           lexicographically-ordered nodal coefficients.
-    *           The resulting output is also lexicographically-ordered.
-    *
-    *  @note These matrices do not account for the linear projection step that
-    *        is optionally done in GetNDBounds before bounding the function.
-    */
-   ///@{
-   DenseMatrix GetLowerBoundMatrix(int dim = 1) const;
-   DenseMatrix GetUpperBoundMatrix(int dim = 1) const;
-   ///@}
-
 private:
-   /** @brief Compute piecewise linear bounds for the lexicographically-ordered
-    *  nodal coefficients in @a coeff in 1D.
-    *  See GetNDBounds for details of the input and output parameters.
-    */
-   void Get1DBounds(const Vector &coeff, Vector &intmin, Vector &intmax) const;
+   /// Compute piecewise linear bounds for the lexicographically-ordered
+   /// coefficients in @a coeff in 1D.
+   void Get1DBounds(Vector &coeff, Vector &intmin, Vector &intmax) const;
 
-   /** @brief Compute piecewise linear bounds for the lexicographically-ordered
-    *  nodal coefficients in @a coeff in 2D.
-    *  See GetNDBounds for details of the input and output parameters.
-    */
-   void Get2DBounds(const Vector &coeff, Vector &intmin, Vector &intmax) const;
+   /// Compute piecewise linear bounds for the lexicographically-ordered
+   /// coefficients in @a coeff in 2D.
+   void Get2DBounds(Vector &coeff, Vector &intmin, Vector &intmax) const;
 
-   /** @brief Compute piecewise linear bounds for the lexicographically-ordered
-    *  nodal coefficients in @a coeff in 3D.
-    *  See GetNDBounds for details of the input and output parameters.
-    */
-   void Get3DBounds(const Vector &coeff, Vector &intmin, Vector &intmax) const;
+   /// Compute piecewise linear bounds for the lexicographically-ordered
+   /// coefficients in @a coeff in 3D.
+   void Get3DBounds(Vector &coeff, Vector &intmin, Vector &intmax) const;
 
-   /** @brief Setup matrix used to compute values at given 1D locations in [0,1]
-    *  for Bernstein bases.
-    */
+   /// Setup matrix used to compute values at given 1D locations in [0,1]
+   /// for Bernstein bases.
    void SetupBernsteinBasisMat(DenseMatrix &basisMat, Vector &nodesBern) const;
 
    void Setup(const int nb_i, const int ncp_i, const int b_type_i,
               const int cp_type_i, const real_t tol_i);
 };
 
-namespace internal
-{
-
-struct PLBoundDeviceData
-{
-   int nb;
-   int ncp;
-   const real_t *xhat;
-   const real_t *what;
-   const real_t *cphat;
-   const real_t *lbound;
-   const real_t *ubound;
-};
-
-template<int T_NB = 0, bool T_PROJ = true>
-inline void GetElementBoundsKernel1D(const PLBoundDeviceData &data,
-                                     const int fes_vdim,
-                                     const int ne,
-                                     const Vector &e_vec,
-                                     Vector &lower,
-                                     Vector &upper,
-                                     const int comp0,
-                                     const int ncomp)
-{
-   constexpr int GENERIC_MAX_ND = 32;
-   constexpr int MAX_ND = T_NB ? T_NB : GENERIC_MAX_ND;
-   constexpr int BLOCK_X = 2*MAX_ND;
-
-   const int nd = T_NB ? T_NB : data.nb;
-   MFEM_VERIFY(nd <= MAX_ND,
-               "Device element bounds kernel supports up to 32 "
-               "1D degrees of freedom.");
-
-   const auto E = Reshape(e_vec.Read(), nd, fes_vdim, ne);
-   auto L = Reshape(lower.Write(), ne, ncomp);
-   auto U = Reshape(upper.Write(), ne, ncomp);
-
-   mfem::forall_2D<BLOCK_X>(ne*ncomp, BLOCK_X, 1,
-                            [=] MFEM_HOST_DEVICE (int ec)
-   {
-      const int e = ec % ne;
-      const int c = ec / ne;
-      const int vc = comp0 + c;
-      const real_t *coeff = &E(0, vc, e);
-      const int tid = MFEM_THREAD_ID(x);
-
-      MFEM_SHARED real_t sproj[MAX_ND];
-      MFEM_SHARED real_t ssum0[MAX_ND];
-      MFEM_SHARED real_t ssum1[MAX_ND];
-      MFEM_SHARED real_t smin[BLOCK_X];
-      MFEM_SHARED real_t smax[BLOCK_X];
-      MFEM_SHARED real_t sa0;
-      MFEM_SHARED real_t sa1;
-
-      MFEM_FOREACH_THREAD(i, x, nd)
-      {
-         if constexpr (T_PROJ)
-         {
-            const real_t x = data.xhat[i];
-            const real_t w = data.what[i];
-            ssum0[i] = 0.5*coeff[i]*w;
-            ssum1[i] = 1.5*coeff[i]*w*x;
-         }
-         else
-         {
-            ssum0[i] = 0.0;
-            ssum1[i] = 0.0;
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(ii, x, 1)
-      {
-         sa0 = 0.0;
-         sa1 = 0.0;
-         for (int i = 0; i < nd; i++)
-         {
-            sa0 += ssum0[i];
-            sa1 += ssum1[i];
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(i, x, nd)
-      {
-         if constexpr (T_PROJ)
-         {
-            const real_t x = data.xhat[i];
-            sproj[i] = coeff[i] - sa0 - sa1*x;
-         }
-         else
-         {
-            sproj[i] = coeff[i];
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      real_t lower_local = HUGE_VAL;
-      real_t upper_local = -HUGE_VAL;
-      MFEM_FOREACH_THREAD(j, x, data.ncp)
-      {
-         real_t lo = 0.0;
-         real_t hi = 0.0;
-         if constexpr (T_PROJ)
-         {
-            const real_t xcp = data.cphat[j];
-            lo = sa0 + sa1*xcp;
-            hi = lo;
-         }
-
-         for (int i = 0; i < nd; i++)
-         {
-            const real_t val = sproj[i];
-            const real_t lv = data.lbound[j + i*data.ncp]*val;
-            const real_t uv = data.ubound[j + i*data.ncp]*val;
-            lo += lv < uv ? lv : uv;
-            hi += lv > uv ? lv : uv;
-         }
-         lower_local = lower_local < lo ? lower_local : lo;
-         upper_local = upper_local > hi ? upper_local : hi;
-      }
-
-      smin[tid] = lower_local;
-      smax[tid] = upper_local;
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(ii, x, 1)
-      {
-         real_t lower_ec = smin[0];
-         real_t upper_ec = smax[0];
-         const int nthreads = MFEM_THREAD_SIZE(x);
-         const int nactive = data.ncp < nthreads ? data.ncp : nthreads;
-         for (int t = 1; t < nactive; t++)
-         {
-            lower_ec = lower_ec < smin[t] ? lower_ec : smin[t];
-            upper_ec = upper_ec > smax[t] ? upper_ec : smax[t];
-         }
-         L(e, c) = lower_ec;
-         U(e, c) = upper_ec;
-      }
-   });
-}
-
-template<int T_NB = 0, int T_NCP = 0, bool T_PROJ = true>
-inline void GetElementBoundsKernel2D(const PLBoundDeviceData &data,
-                                     const int fes_vdim,
-                                     const int ne,
-                                     const Vector &e_vec,
-                                     Vector &lower,
-                                     Vector &upper,
-                                     const int comp0,
-                                     const int ncomp)
-{
-   constexpr int DEFAULT_MAX_NB = 8;
-   constexpr int DEFAULT_MAX_CP = 3*DEFAULT_MAX_NB;
-   constexpr int MAX_NB = T_NB ? T_NB : DEFAULT_MAX_NB;
-   constexpr int MAX_CP = T_NCP ? T_NCP : DEFAULT_MAX_CP;
-   constexpr int MAX_THREADS = MAX_CP*MAX_CP;
-
-   const int nb = data.nb;
-   const int ncp = data.ncp;
-   const int nd = nb*nb;
-   MFEM_VERIFY(nb <= MAX_NB,
-               "Device 2D element bounds kernel exceeds its compile-time "
-               "1D degree bound.");
-   MFEM_VERIFY(ncp <= MAX_CP,
-               "Device 2D element bounds kernel exceeds its compile-time "
-               "control-point bound.");
-   MFEM_VERIFY(ncp*ncp <= MAX_THREADS,
-               "Device 2D element bounds kernel exceeds its compile-time "
-               "thread-block bound.");
-
-   const auto E = Reshape(e_vec.Read(), nd, fes_vdim, ne);
-   auto L = Reshape(lower.Write(), ne, ncomp);
-   auto U = Reshape(upper.Write(), ne, ncomp);
-
-   mfem::forall_2D<MAX_THREADS>(ne*ncomp, ncp, ncp,
-                                [=] MFEM_HOST_DEVICE (int ec)
-   {
-      const int e = ec % ne;
-      const int c = ec / ne;
-      const int vc = comp0 + c;
-      const real_t *coeff = &E(0, vc, e);
-      const int tx = MFEM_THREAD_ID(x);
-      const int ty = MFEM_THREAD_ID(y);
-
-      MFEM_SHARED real_t sproj[MAX_NB*MAX_NB];
-      MFEM_SHARED real_t srow_min[MAX_NB*MAX_CP];
-      MFEM_SHARED real_t srow_max[MAX_NB*MAX_CP];
-      MFEM_SHARED real_t srow_a0[MAX_NB];
-      MFEM_SHARED real_t srow_a1[MAX_NB];
-      MFEM_SHARED real_t sa0[MAX_CP];
-      MFEM_SHARED real_t sa1[MAX_CP];
-      MFEM_SHARED real_t smin[MAX_THREADS];
-      MFEM_SHARED real_t smax[MAX_THREADS];
-
-      // Stage 1a: for each nodal row, form the per-node contributions to the
-      // row-wise linear fit used by the first 1D bounding solve.
-      MFEM_FOREACH_THREAD(jrow, y, nb)
-      {
-         const real_t *row_coeff = coeff + jrow*nb;
-         const int row_ncp_off = jrow*MAX_CP;
-         MFEM_FOREACH_THREAD(i, x, nb)
-         {
-            if constexpr (T_PROJ)
-            {
-               const real_t x = data.xhat[i];
-               const real_t w = data.what[i];
-               srow_min[row_ncp_off + i] = 0.5*row_coeff[i]*w;
-               srow_max[row_ncp_off + i] = 1.5*row_coeff[i]*w*x;
-            }
-            else
-            {
-               srow_min[row_ncp_off + i] = 0.0;
-               srow_max[row_ncp_off + i] = 0.0;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Stage 1b: reduce the row-wise projection coefficients a0/a1.
-      if constexpr (T_PROJ)
-      {
-         MFEM_FOREACH_THREAD(jrow, y, nb)
-         {
-            const int row_ncp_off = jrow*MAX_CP;
-            real_t a0 = 0.0;
-            real_t a1 = 0.0;
-            MFEM_FOREACH_THREAD(ii, x, 1)
-            {
-               for (int i = 0; i < nb; i++)
-               {
-                  a0 += srow_min[row_ncp_off + i];
-                  a1 += srow_max[row_ncp_off + i];
-               }
-               srow_a0[jrow] = a0;
-               srow_a1[jrow] = a1;
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-
-      // Stage 1c: subtract the row-wise linear fit once and cache the
-      // projected row coefficients for reuse across all x-control points.
-      MFEM_FOREACH_THREAD(jrow, y, nb)
-      {
-         const real_t *row_coeff = coeff + jrow*nb;
-         MFEM_FOREACH_THREAD(i, x, nb)
-         {
-            if constexpr (T_PROJ)
-            {
-               const real_t x = data.xhat[i];
-               sproj[jrow*MAX_NB + i] = row_coeff[i]
-                                        - srow_a0[jrow] - srow_a1[jrow]*x;
-            }
-            else
-            {
-               sproj[jrow*MAX_NB + i] = row_coeff[i];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Stage 1d: solve the first 1D bounding problem along each nodal row and
-      // store bounds at every x-direction control point.
-      MFEM_FOREACH_THREAD(icp, x, ncp)
-      {
-         MFEM_FOREACH_THREAD(jrow, y, nb)
-         {
-            const int row_cp_off = jrow*ncp;
-            real_t lo = 0.0;
-            real_t hi = 0.0;
-            if constexpr (T_PROJ)
-            {
-               const real_t xcp = data.cphat[icp];
-               lo = srow_a0[jrow] + srow_a1[jrow]*xcp;
-               hi = lo;
-            }
-            for (int i = 0; i < nb; i++)
-            {
-               const real_t val = sproj[jrow*MAX_NB + i];
-               const real_t lv = data.lbound[icp + i*data.ncp]*val;
-               const real_t uv = data.ubound[icp + i*data.ncp]*val;
-               lo += lv < uv ? lv : uv;
-               hi += lv > uv ? lv : uv;
-            }
-            srow_min[row_cp_off + icp] = lo;
-            srow_max[row_cp_off + icp] = hi;
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Stage 2a: from the row bounds, form the per-row contributions to the
-      // second 1D projection solve in the y-direction.
-      MFEM_FOREACH_THREAD(icp, x, ncp)
-      {
-         MFEM_FOREACH_THREAD(jrow, y, nb)
-         {
-            const int row_cp_off = jrow*ncp;
-            if constexpr (T_PROJ)
-            {
-               const real_t x = data.xhat[jrow];
-               const real_t w = data.what[jrow];
-               const real_t t = 0.5*(srow_min[row_cp_off + icp] +
-                                     srow_max[row_cp_off + icp]);
-               smin[row_cp_off + icp] = 0.5*t*w;
-               smax[row_cp_off + icp] = 1.5*t*w*x;
-            }
-            else
-            {
-               smin[row_cp_off + icp] = 0.0;
-               smax[row_cp_off + icp] = 0.0;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Stage 2b: reduce the y-direction projection coefficients for each
-      // x-control-point column.
-      MFEM_FOREACH_THREAD(jj, y, 1)
-      {
-         MFEM_FOREACH_THREAD(icp, x, ncp)
-         {
-            real_t a0 = 0.0;
-            real_t a1 = 0.0;
-            for (int jrow = 0; jrow < nb; jrow++)
-            {
-               a0 += smin[jrow*ncp + icp];
-               a1 += smax[jrow*ncp + icp];
-            }
-            sa0[icp] = a0;
-            sa1[icp] = a1;
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Stage 2c: subtract the y-direction linear fit from the intermediate
-      // row bounds so the final tensor-product bound uses the perturbation.
-      if constexpr (T_PROJ)
-      {
-         MFEM_FOREACH_THREAD(icp, x, ncp)
-         {
-            MFEM_FOREACH_THREAD(jrow, y, nb)
-            {
-               const int row_cp_off = jrow*ncp;
-               const real_t x = data.xhat[jrow];
-               const real_t t = sa0[icp] + sa1[icp]*x;
-               srow_min[row_cp_off + icp] -= t;
-               srow_max[row_cp_off + icp] -= t;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // Stage 3: each thread now owns one 2D control point (icp, kcp) and
-      // accumulates its final lower/upper bound from the row-bound data.
-      MFEM_FOREACH_THREAD(icp, x, ncp)
-      {
-         MFEM_FOREACH_THREAD(kcp, y, ncp)
-         {
-            real_t lo = 0.0;
-            real_t hi = 0.0;
-            if constexpr (T_PROJ)
-            {
-               const real_t xcp = data.cphat[kcp];
-               lo = sa0[icp] + sa1[icp]*xcp;
-               hi = lo;
-            }
-            for (int jrow = 0; jrow < nb; jrow++)
-            {
-               const real_t w0 = srow_min[jrow*ncp + icp];
-               const real_t w1 = srow_max[jrow*ncp + icp];
-               const real_t lb = data.lbound[kcp + jrow*data.ncp];
-               const real_t ub = data.ubound[kcp + jrow*data.ncp];
-               const real_t v0 = lb*w0;
-               const real_t v1 = ub*w0;
-               const real_t v2 = lb*w1;
-               const real_t v3 = ub*w1;
-               real_t vlo = v0 < v1 ? v0 : v1;
-               real_t vhi = v0 > v1 ? v0 : v1;
-               vlo = vlo < v2 ? vlo : v2;
-               vlo = vlo < v3 ? vlo : v3;
-               vhi = vhi > v2 ? vhi : v2;
-               vhi = vhi > v3 ? vhi : v3;
-               lo += vlo;
-               hi += vhi;
-            }
-            const int slot = kcp*ncp + icp;
-            smin[slot] = lo;
-            smax[slot] = hi;
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      const int lane = ty*ncp + tx;
-      const int nactive = ncp*ncp;
-      const int nthreads = MFEM_THREAD_SIZE(x)*MFEM_THREAD_SIZE(y);
-
-      // Reduce all 2D control-point bounds to one lower/upper pair per
-      // (element, component).
-      if (nthreads == 1)
-      {
-         if (tx == 0 && ty == 0)
-         {
-            real_t lower_ec = smin[0];
-            real_t upper_ec = smax[0];
-            for (int t = 1; t < nactive; t++)
-            {
-               lower_ec = lower_ec < smin[t] ? lower_ec : smin[t];
-               upper_ec = upper_ec > smax[t] ? upper_ec : smax[t];
-            }
-            L(e, c) = lower_ec;
-            U(e, c) = upper_ec;
-         }
-      }
-      else
-      {
-         for (int stride = (nactive + 1)/2; stride > 0;
-              stride = (stride + 1)/2)
-         {
-            if (lane < stride && lane + stride < nactive)
-            {
-               smin[lane] = smin[lane] < smin[lane + stride] ?
-                            smin[lane] : smin[lane + stride];
-               smax[lane] = smax[lane] > smax[lane + stride] ?
-                            smax[lane] : smax[lane + stride];
-            }
-            MFEM_SYNC_THREAD;
-            if (stride == 1) { break; }
-         }
-
-         if (lane == 0)
-         {
-            L(e, c) = smin[0];
-            U(e, c) = smax[0];
-         }
-      }
-   });
-}
-
-} // namespace internal
-
-inline void PLBound::GetElementBoundsKernel(const int rdim, const int fes_vdim,
-                                            const Vector &e_vec,
-                                            Vector &lower, Vector &upper,
-                                            const int vdim) const
-{
-   MFEM_VERIFY(b_type != BasisType::Positive,
-               "Bernstein device bounds are not implemented.");
-   if (rdim == 3)
-   {
-      MFEM_ABORT("Device element bounds kernel currently only supports 1D/2D.");
-   }
-   MFEM_VERIFY(rdim == 1 || rdim == 2, "Invalid element dimension.");
-   MFEM_VERIFY(vdim >= -1 && vdim <= fes_vdim, "Invalid vector component.");
-   const int nd = static_cast<int>(std::pow(nb, rdim));
-   const int ne = e_vec.Size()/(nd*fes_vdim);
-   const int ncomp = (vdim > 0) ? 1 : fes_vdim;
-
-   lower.SetSize(ne*ncomp, e_vec);
-   upper.SetSize(ne*ncomp, e_vec);
-   lower.UseDevice(true);
-   upper.UseDevice(true);
-
-   if (!proj)
-   {
-      MFEM_ABORT("Device element bounds kernel currently requires projection "
-                 "enabled.");
-   }
-
-   const real_t *dxhat = xhat.Read();
-   const real_t *dwhat = what.Read();
-   const real_t *dcphat = cphat.Read();
-   const real_t *dlbound = lbound.Read();
-   const real_t *dubound = ubound.Read();
-
-   internal::PLBoundDeviceData data
-   {
-      nb,
-      ncp,
-      dxhat,
-      dwhat,
-      dcphat,
-      dlbound,
-      dubound
-   };
-
-   const int comp0 = (vdim > 0) ? (vdim - 1) : 0;
-
-   if (rdim == 1)
-   {
-      switch (nb)
-      {
-         case 2: return internal::GetElementBoundsKernel1D<2, true>(data, fes_vdim, ne,
-                                                                     e_vec, lower, upper,
-                                                                     comp0, ncomp);
-         case 3: return internal::GetElementBoundsKernel1D<3, true>(data, fes_vdim, ne,
-                                                                     e_vec, lower, upper,
-                                                                     comp0, ncomp);
-         case 4: return internal::GetElementBoundsKernel1D<4, true>(data, fes_vdim, ne,
-                                                                     e_vec, lower, upper,
-                                                                     comp0, ncomp);
-         case 5: return internal::GetElementBoundsKernel1D<5, true>(data, fes_vdim, ne,
-                                                                     e_vec, lower, upper,
-                                                                     comp0, ncomp);
-         case 6: return internal::GetElementBoundsKernel1D<6, true>(data, fes_vdim, ne,
-                                                                     e_vec, lower, upper,
-                                                                     comp0, ncomp);
-         case 7: return internal::GetElementBoundsKernel1D<7, true>(data, fes_vdim, ne,
-                                                                     e_vec, lower, upper,
-                                                                     comp0, ncomp);
-         case 8: return internal::GetElementBoundsKernel1D<8, true>(data, fes_vdim, ne,
-                                                                     e_vec, lower, upper,
-                                                                     comp0, ncomp);
-         case 9: return internal::GetElementBoundsKernel1D<9, true>(data, fes_vdim, ne,
-                                                                     e_vec, lower, upper,
-                                                                     comp0, ncomp);
-         case 10: return internal::GetElementBoundsKernel1D<10, true>(data, fes_vdim, ne,
-                                                                       e_vec, lower, upper,
-                                                                       comp0, ncomp);
-         default: return internal::GetElementBoundsKernel1D<0, true>(data, fes_vdim, ne,
-                                                                      e_vec, lower, upper,
-                                                                      comp0, ncomp);
-      }
-   }
-#define MFEM_PLBOUND_2D_DISPATCH(NB, NCP) \
-   return internal::GetElementBoundsKernel2D<NB, NCP, true>(data, fes_vdim, ne, \
-                                                            e_vec, lower, upper, \
-                                                            comp0, ncomp)
-   switch (nb)
-   {
-      case 2:
-         switch (ncp)
-         {
-            case 4: MFEM_PLBOUND_2D_DISPATCH(2, 4);
-            case 6: MFEM_PLBOUND_2D_DISPATCH(2, 6);
-            case 8: MFEM_PLBOUND_2D_DISPATCH(2, 8);
-         }
-         break;
-      case 3:
-         switch (ncp)
-         {
-            case 6: MFEM_PLBOUND_2D_DISPATCH(3, 6);
-            case 9: MFEM_PLBOUND_2D_DISPATCH(3, 9);
-            case 12: MFEM_PLBOUND_2D_DISPATCH(3, 12);
-         }
-         break;
-      case 4:
-         switch (ncp)
-         {
-            case 8: MFEM_PLBOUND_2D_DISPATCH(4, 8);
-            case 12: MFEM_PLBOUND_2D_DISPATCH(4, 12);
-            case 16: MFEM_PLBOUND_2D_DISPATCH(4, 16);
-         }
-         break;
-      case 5:
-         switch (ncp)
-         {
-            case 10: MFEM_PLBOUND_2D_DISPATCH(5, 10);
-            case 15: MFEM_PLBOUND_2D_DISPATCH(5, 15);
-            case 20: MFEM_PLBOUND_2D_DISPATCH(5, 20);
-         }
-         break;
-      case 6:
-         switch (ncp)
-         {
-            case 12: MFEM_PLBOUND_2D_DISPATCH(6, 12);
-            case 18: MFEM_PLBOUND_2D_DISPATCH(6, 18);
-            case 24: MFEM_PLBOUND_2D_DISPATCH(6, 24);
-         }
-         break;
-      case 7:
-         switch (ncp)
-         {
-            case 14: MFEM_PLBOUND_2D_DISPATCH(7, 14);
-            case 21: MFEM_PLBOUND_2D_DISPATCH(7, 21);
-            case 28: MFEM_PLBOUND_2D_DISPATCH(7, 28);
-         }
-         break;
-      case 8:
-         switch (ncp)
-         {
-            case 16: MFEM_PLBOUND_2D_DISPATCH(8, 16);
-            case 24: MFEM_PLBOUND_2D_DISPATCH(8, 24);
-            case 32: MFEM_PLBOUND_2D_DISPATCH(8, 32);
-         }
-         break;
-   }
-#undef MFEM_PLBOUND_2D_DISPATCH
-   return internal::GetElementBoundsKernel2D<0, 0, true>(data, fes_vdim, ne,
-                                                         e_vec, lower, upper,
-                                                         comp0, ncomp);
-}
-
 } // namespace mfem
 
-#endif // MFEM_BOUNDS
+#endif // MFEM_BOUND

fem/coefficient.cpp

@@ -1085,29 +1085,6 @@ void SumCoefficient::SetTime(real_t t)
    this->Coefficient::SetTime(t);
 }
 
-void SumCoefficient::Project(QuadratureFunction &qf)
-{
-   if (a == nullptr)
-   {
-      // qf = alpha*aConst + beta * b
-      const real_t d_alpha_a = aConst*alpha;
-      const real_t d_beta = beta;
-      b->Project(qf);
-      auto d_qf = qf.ReadWrite();
-      mfem::forall(qf.Size(), [=] MFEM_HOST_DEVICE (int i)
-      {
-         d_qf[i] = d_alpha_a + d_beta*d_qf[i];
-      });
-   }
-   else
-   {
-      a->Project(qf);
-      QuadratureFunction qf_b(*qf.GetSpace());
-      b->Project(qf_b);
-      add(alpha, qf, beta, qf_b, qf);
-   }
-}
-
 void ProductCoefficient::SetTime(real_t t)
 {
    if (a) { a->SetTime(t); }
@@ -1115,23 +1092,6 @@ void ProductCoefficient::SetTime(real_t t)
    this->Coefficient::SetTime(t);
 }
 
-void ProductCoefficient::Project(QuadratureFunction &qf)
-{
-   if (a == nullptr)
-   {
-      // qf = aConst * b
-      b->Project(qf);
-      qf *= aConst;
-   }
-   else
-   {
-      a->Project(qf);
-      QuadratureFunction qf_b(qf.GetSpace());
-      b->Project(qf_b);
-      qf *= qf_b;
-   }
-}
-
 void RatioCoefficient::SetTime(real_t t)
 {
    if (a) { a->SetTime(t); }
@@ -1139,38 +1099,6 @@ void RatioCoefficient::SetTime(real_t t)
    this->Coefficient::SetTime(t);
 }
 
-void RatioCoefficient::Project(QuadratureFunction &qf)
-{
-   if (b == nullptr)
-   {
-      if (a == nullptr)
-      {
-         qf = aConst / bConst;
-      }
-      else
-      {
-         a->Project(qf);
-         qf *= 1.0/bConst;
-      }
-   }
-   else
-   {
-      if (a == nullptr)
-      {
-         b->Project(qf);
-         qf.Reciprocal();
-         qf *= aConst;
-      }
-      else
-      {
-         a->Project(qf);
-         QuadratureFunction qf_b(qf.GetSpace());
-         b->Project(qf_b);
-         qf /= qf_b;
-      }
-   }
-}
-
 void PowerCoefficient::SetTime(real_t t)
 {
    if (a) { a->SetTime(t); }
@@ -1302,73 +1230,6 @@ real_t TraceCoefficient::Eval(ElementTransformation &T,
    return ma.Trace();
 }
 
-VectorComponentCoefficient::VectorComponentCoefficient(VectorCoefficient &A,
-                                                       int c)
-   : a(&A), va(A.GetVDim())
-{
-   SetComponent(c);
-}
-
-void VectorComponentCoefficient::SetComponent(int c)
-{
-   MFEM_ASSERT(c < a->GetVDim() && c >= 0,
-               "VectorComponentCoefficient:  "
-               "Index not in range.");
-
-   component = c;
-}
-
-void VectorComponentCoefficient::SetTime(real_t t)
-{
-   if (a) { a->SetTime(t); }
-   this->Coefficient::SetTime(t);
-}
-
-real_t VectorComponentCoefficient::Eval(ElementTransformation &T,
-                                        const IntegrationPoint &ip)
-{
-   a->Eval(va, T, ip);
-   return va[component];
-}
-
-MatrixComponentCoefficient::MatrixComponentCoefficient(MatrixCoefficient &A,
-                                                       int ri, int ci)
-   : a(&A), ma(A.GetHeight(), A.GetWidth())
-{
-   SetRowIndex(ri);
-   SetColumnIndex(ci);
-}
-
-void MatrixComponentCoefficient::SetRowIndex(int ri)
-{
-   MFEM_ASSERT(ri < a->GetHeight() && ri >= 0,
-               "MatrixComponentCoefficient:  "
-               "Row index not in range.");
-
-   row_idx = ri;
-}
-
-void MatrixComponentCoefficient::SetColumnIndex(int ci)
-{
-   MFEM_ASSERT(ci < a->GetWidth() && ci >= 0,
-               "MatrixComponentCoefficient:  "
-               "Column index not in range.");
-   col_idx = ci;
-}
-
-void MatrixComponentCoefficient::SetTime(real_t t)
-{
-   if (a) { a->SetTime(t); }
-   this->Coefficient::SetTime(t);
-}
-
-real_t MatrixComponentCoefficient::Eval(ElementTransformation &T,
-                                        const IntegrationPoint &ip)
-{
-   a->Eval(ma, T, ip);
-   return ma(row_idx,col_idx);
-}
-
 VectorSumCoefficient::VectorSumCoefficient(int dim)
    : VectorCoefficient(dim),
      ACoef(NULL), BCoef(NULL),
@@ -2094,7 +1955,7 @@ void CoefficientVector::Project(MatrixCoefficient &coeff, bool transpose)
 {
    if (auto *const_coeff = dynamic_cast<MatrixConstantCoefficient*>(&coeff))
    {
-      SetConstant(const_coeff->GetMatrix(), transpose);
+      SetConstant(const_coeff->GetMatrix());
    }
    else if (auto *const_sym_coeff =
                dynamic_cast<SymmetricMatrixConstantCoefficient*>(&coeff))
@@ -2155,7 +2016,7 @@ void CoefficientVector::SetConstant(const Vector &constant)
    }
 }
 
-void CoefficientVector::SetConstant(const DenseMatrix &constant, bool transpose)
+void CoefficientVector::SetConstant(const DenseMatrix &constant)
 {
    const int nq = (storage & CoefficientStorage::CONSTANTS) ? 1 : qs.GetSize();
    const int width = constant.Width();
@@ -2168,8 +2029,7 @@ void CoefficientVector::SetConstant(const DenseMatrix &constant, bool transpose)
       {
          for (int i = 0; i < height; ++i)
          {
-            const real_t val = transpose ? constant(j,i) : constant(i,j);
-            (*this)[i + j*height + iq*vdim] = val;
+            (*this)[i + j*height + iq*vdim] = constant(i, j);
          }
       }
    }

fem/coefficient.hpp

@@ -52,9 +52,6 @@ public:
    /// Get the time for time dependent coefficients
    real_t GetTime() { return time; }
 
-   /// Returns dimension of the vector.
-   int GetVDim() { return 1; }
-
    /** @brief Evaluate the coefficient in the element described by @a T at the
        point @a ip. */
    /** @note When this method is called, the caller must make sure that the
@@ -117,10 +114,11 @@ public:
    /// Construct the constant coefficient using a vector of constants.
    /** @a c should be a vector defined by attributes, so for region with
        attribute @a i @a c[i-1] is the coefficient in that region */
-   PWConstCoefficient(const Vector &c) { UpdateConstants(c); }
+   PWConstCoefficient(Vector &c)
+   { constants.SetSize(c.Size()); constants=c; }
 
    /// Update the constants with vector @a c.
-   void UpdateConstants(const Vector &c) { constants = c; }
+   void UpdateConstants(Vector &c) { constants.SetSize(c.Size()); constants=c; }
 
    /// Return a reference to the i-th constant
    real_t &operator()(int i) { return constants(i-1); }
@@ -1334,8 +1332,8 @@ public:
    /// Get the coefficient located at (i,j) in the matrix.
    Coefficient* GetCoeff (int i, int j) { return Coeff[i*width+j]; }
 
-   /** @brief Set the coefficient located at (i,j) in the matrix.  By default
-       this will take ownership of the Coefficient passed in, but this
+   /** @brief Set the coefficient located at (i,j) in the matrix.  By default by
+       default this will take ownership of the Coefficient passed in, but this
        can be overridden with the @a own parameter. */
    void Set(int i, int j, Coefficient * c, bool own=true);
 
@@ -1458,9 +1456,6 @@ public:
    /// Set the time for internally stored coefficients
    void SetTime(real_t t) override;
 
-   /// @copydoc Coefficient::Project(QuadratureFunction &)
-   void Project(QuadratureFunction &qf) override;
-
    /// Reset the first term in the linear combination as a constant
    void SetAConst(real_t A) { a = NULL; aConst = A; }
    /// Return the first term in the linear combination
@@ -1642,9 +1637,6 @@ public:
    /// Set the time for internally stored coefficients
    void SetTime(real_t t) override;
 
-   /// @copydoc Coefficient::Project(QuadratureFunction &)
-   void Project(QuadratureFunction &qf) override;
-
    /// Reset the first term in the product as a constant
    void SetAConst(real_t A) { a = NULL; aConst = A; }
    /// Return the first term in the product
@@ -1693,9 +1685,6 @@ public:
    /// Set the time for internally stored coefficients
    void SetTime(real_t t) override;
 
-   /// @copydoc Coefficient::Project(QuadratureFunction &)
-   void Project(QuadratureFunction &qf) override;
-
    /// Reset the numerator in the ratio as a constant
    void SetAConst(real_t A) { a = NULL; aConst = A; }
    /// Return the numerator of the ratio
@@ -1875,85 +1864,6 @@ public:
                const IntegrationPoint &ip) override;
 };
 
-/// Scalar coefficient defined as component of a vector coefficient
-class VectorComponentCoefficient : public Coefficient
-{
-private:
-   VectorCoefficient *a = nullptr;
-
-   mutable Vector va;
-   int component;
-
-public:
-   /// Construct with a vector coefficient.
-   VectorComponentCoefficient(VectorCoefficient &A)
-      : a(&A), va(A.GetVDim()), component(0) {};
-
-   VectorComponentCoefficient(VectorCoefficient &A, int c);
-
-   /// Set the time for internally stored coefficients
-   void SetTime(real_t t) override;
-
-   /// Reset the vector coefficient
-   void SetACoef(VectorCoefficient &A) { a = &A; }
-
-   /// Return the vector coefficient
-   VectorCoefficient * GetACoef() const { return a; }
-
-   /// Set the component
-   void SetComponent(int c);
-
-   /// Return the component
-   int GetComponent() const { return component; }
-
-   /// Evaluate the trace coefficient at @a ip.
-   real_t Eval(ElementTransformation &T,
-               const IntegrationPoint &ip) override;
-};
-
-/// Scalar coefficient defined as component of a matrix coefficient
-class MatrixComponentCoefficient : public Coefficient
-{
-private:
-   MatrixCoefficient *a = nullptr;
-
-   mutable DenseMatrix ma;
-   int row_idx,col_idx;
-
-public:
-   MatrixComponentCoefficient(MatrixCoefficient &A)
-      : a(&A), ma(A.GetHeight(), A.GetWidth()), row_idx(0), col_idx(0) {};
-
-   /// Construct with the matrix coefficient.
-   MatrixComponentCoefficient(MatrixCoefficient &A, int ri, int ci);
-
-   /// Set the time for internally stored coefficients
-   void SetTime(real_t t) override;
-
-   /// Reset the matrix coefficient
-   void SetACoef(MatrixCoefficient &A) { a = &A; }
-
-   /// Return the matrix coefficient
-   MatrixCoefficient * GetACoef() const { return a; }
-
-   /// Reset the index
-   void SetRowIndex(int ri);
-
-   /// Return the index
-   int GetRowIndex() const { return row_idx; }
-
-   /// Reset the index
-   void SetColumnIndex(int ci);
-
-   /// Return the index
-   int GetColumnIndex() const { return col_idx; }
-
-
-   /// Evaluate the trace coefficient at @a ip.
-   real_t Eval(ElementTransformation &T,
-               const IntegrationPoint &ip) override;
-};
-
 /// Vector coefficient defined as the linear combination of two vectors
 class VectorSumCoefficient : public VectorCoefficient
 {
@@ -2601,7 +2511,7 @@ public:
    void SetConstant(const Vector &constant);
 
    /// Set this vector to the given constant matrix.
-   void SetConstant(const DenseMatrix &constant, bool transpose=false);
+   void SetConstant(const DenseMatrix &constant);
 
    /// Set this vector to the given constant symmetric matrix.
    void SetConstant(const DenseSymmetricMatrix &constant);

fem/complex_fem.cpp

@@ -11,15 +11,14 @@
 
 #include "complex_fem.hpp"
 #include "../general/forall.hpp"
-#include "../general/text.hpp"
 
 using namespace std;
 
 namespace mfem
 {
 
-ComplexGridFunction::ComplexGridFunction(FiniteElementSpace *f)
-   : Vector(2*(f->GetVSize())), fes(f), fec_owned(NULL)
+ComplexGridFunction::ComplexGridFunction(FiniteElementSpace *fes)
+   : Vector(2*(fes->GetVSize()))
 {
    UseDevice(true);
    this->Vector::operator=(0.0);
@@ -29,88 +28,12 @@ ComplexGridFunction::ComplexGridFunction(FiniteElementSpace *f)
 
    gfi = new GridFunction();
    gfi->MakeRef(fes, *this, fes->GetVSize());
-
-   fes_sequence = fes->GetSequence();
-}
-
-ComplexGridFunction::ComplexGridFunction(Mesh *m, std::istream &input)
-   : Vector(), fes(NULL), fec_owned(NULL)
-{
-   string buff;
-
-   // Grid functions are stored on the device
-   UseDevice(true);
-
-   input >> std::ws;
-   getline(input, buff);  // 'ComplexGridFunction'
-   filter_dos(buff);
-   if (buff != "ComplexGridFunction")
-   {
-      MFEM_ABORT("unrecognized file header: " << buff);
-   }
-
-   fes = new FiniteElementSpace;
-   fec_owned = fes->Load(m, input);
-
-   skip_comment_lines(input, '#');
-   istream::int_type next_char = input.peek();
-   if (next_char == 'N') // First letter of "NURBS_patches"
-   {
-      getline(input, buff);
-      filter_dos(buff);
-      if (buff == "NURBS_patches")
-      {
-         MFEM_ABORT("NURBS not yet supported with ComplexGridFunction objects");
-      }
-      else
-      {
-         MFEM_ABORT("unknown section: " << buff);
-      }
-   }
-   else
-   {
-      Vector::Load(input, 2*fes->GetVSize());
-
-      // if the mesh is a legacy (v1.1) NC mesh, it has old vertex ordering
-      if (fes->Nonconforming() &&
-          fes->GetMesh()->ncmesh->IsLegacyLoaded())
-      {
-         // LegacyNCReorder();
-         MFEM_ABORT("LegacyNCReorder not supported for "
-                    "ComplexGridFunction objects");
-      }
-   }
-
-   gfr = new GridFunction();
-   gfr->MakeRef(fes, *this, 0);
-
-   gfi = new GridFunction();
-   gfi->MakeRef(fes, *this, fes->GetVSize());
-
-   fes_sequence = fes->GetSequence();
-}
-
-void ComplexGridFunction::Destroy()
-{
-   delete gfr; delete gfi;
-
-   if (fec_owned)
-   {
-      delete fes;
-      delete fec_owned;
-      fec_owned = NULL;
-   }
 }
 
 void
 ComplexGridFunction::Update()
 {
-   if (fes->GetSequence() == fes_sequence)
-   {
-      return; // space and grid function are in sync, no-op
-   }
-   fes_sequence = fes->GetSequence();
-
+   FiniteElementSpace *fes = gfr->FESpace();
    const int vsize = fes->GetVSize();
 
    const Operator *T = fes->GetUpdateOperator();
@@ -161,17 +84,6 @@ ComplexGridFunction::Update()
    }
 }
 
-int ComplexGridFunction::VectorDim() const
-{
-   const FiniteElement *fe = fes->GetTypicalFE();
-   if (!fe || fe->GetRangeType() == FiniteElement::SCALAR)
-   {
-      return fes->GetVDim();
-   }
-   return fes->GetVDim()*std::max(fes->GetMesh()->SpaceDimension(),
-                                  fe->GetRangeDim());
-}
-
 void
 ComplexGridFunction::ProjectCoefficient(Coefficient &real_coeff,
                                         Coefficient &imag_coeff)
@@ -237,35 +149,6 @@ ComplexGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient
    gfi->SyncAliasMemory(*this);
 }
 
-void ComplexGridFunction::Save(std::ostream &os) const
-{
-   os << "ComplexGridFunction\n";
-   fes->Save(os);
-   os << '\n';
-   if (fes->GetOrdering() == Ordering::byNODES)
-   {
-      Vector::Print(os, 1);
-   }
-   else
-   {
-      Vector::Print(os, fes->GetVDim());
-   }
-   os.flush();
-}
-
-void ComplexGridFunction::Save(const char *fname, int precision) const
-{
-   ofstream ofs(fname);
-   ofs.precision(precision);
-   Save(ofs);
-}
-
-std::ostream &operator<<(std::ostream &os, const ComplexGridFunction &sol)
-{
-   sol.Save(os);
-   return os;
-}
-
 
 ComplexLinearForm::ComplexLinearForm(FiniteElementSpace *fes,
                                      ComplexOperator::Convention convention)
@@ -771,8 +654,8 @@ SesquilinearForm::Update(FiniteElementSpace *nfes)
 
 #ifdef MFEM_USE_MPI
 
-ParComplexGridFunction::ParComplexGridFunction(ParFiniteElementSpace *pf)
-   : Vector(2*(pf->GetVSize())), pfes(pf), fec_owned(NULL)
+ParComplexGridFunction::ParComplexGridFunction(ParFiniteElementSpace *pfes)
+   : Vector(2*(pfes->GetVSize()))
 {
    UseDevice(true);
    this->Vector::operator=(0.0);
@@ -782,105 +665,12 @@ ParComplexGridFunction::ParComplexGridFunction(ParFiniteElementSpace *pf)
 
    pgfi = new ParGridFunction();
    pgfi->MakeRef(pfes, *this, pfes->GetVSize());
-
-   fes_sequence = pfes->GetSequence();
-}
-
-ParComplexGridFunction::ParComplexGridFunction(ParMesh *m, std::istream &input)
-   : Vector(), pfes(NULL), fec_owned(NULL)
-{
-   string buff;
-
-   // Grid functions are stored on the device
-   UseDevice(true);
-
-   input >> std::ws;
-   getline(input, buff);  // 'ParComplexGridFunction'
-   filter_dos(buff);
-   if (buff != "ParComplexGridFunction")
-   {
-      MFEM_ABORT("unrecognized file header: " << buff);
-   }
-
-   FiniteElementSpace *fes = new FiniteElementSpace;
-   fec_owned = fes->Load(m, input);
-
-   pfes = new ParFiniteElementSpace(m, fec_owned, fes->GetVDim(),
-                                    fes->GetOrdering());
-
-   delete fes;
-
-   skip_comment_lines(input, '#');
-   istream::int_type next_char = input.peek();
-   if (next_char == 'N') // First letter of "NURBS_patches"
-   {
-      getline(input, buff);
-      filter_dos(buff);
-      if (buff == "NURBS_patches")
-      {
-         MFEM_ABORT("NURBS not yet supported with ComplexGridFunction objects");
-      }
-      else
-      {
-         MFEM_ABORT("unknown section: " << buff);
-      }
-   }
-   else
-   {
-      int vsize = pfes->GetVSize();
-      Vector::Load(input, 2*vsize);
-
-      real_t *data_  = const_cast<real_t*>(HostRead());
-      for (int i = 0; i < vsize; i++)
-      {
-         if (pfes->GetDofSign(i) < 0)
-         {
-            data_[i] = -data_[i];
-            data_[i+vsize] = -data_[i+vsize];
-         }
-      }
-
-
-      // if the mesh is a legacy (v1.1) NC mesh, it has old vertex ordering
-      if (pfes->Nonconforming() &&
-          pfes->GetMesh()->ncmesh->IsLegacyLoaded())
-      {
-         // LegacyNCReorder();
-         MFEM_ABORT("LegacyNCReorder not supported for "
-                    "ComplexGridFunction objects");
-      }
-   }
-
-   pgfr = new ParGridFunction();
-   pgfr->MakeRef(pfes, *this, 0);
-
-   pgfi = new ParGridFunction();
-   pgfi->MakeRef(pfes, *this, pfes->GetVSize());
-
-   fes_sequence = pfes->GetSequence();
-}
-
-void ParComplexGridFunction::Destroy()
-{
-   delete pgfr; delete pgfi;
-
-   if (fec_owned)
-   {
-      delete pfes;
-      delete fec_owned;
-      fec_owned = NULL;
-   }
 }
 
 void
 ParComplexGridFunction::Update()
 {
-   if (pfes->GetSequence() == fes_sequence)
-   {
-      return; // space and grid function are in sync, no-op
-   }
-   fes_sequence = pfes->GetSequence();
-
+   ParFiniteElementSpace *pfes = pgfr->ParFESpace();
    const int vsize = pfes->GetVSize();
 
    const Operator *T = pfes->GetUpdateOperator();
@@ -929,17 +719,6 @@ ParComplexGridFunction::Update()
    }
 }
 
-int ParComplexGridFunction::VectorDim() const
-{
-   const FiniteElement *fe = pfes->GetTypicalFE();
-   if (!fe || fe->GetRangeType() == FiniteElement::SCALAR)
-   {
-      return pfes->GetVDim();
-   }
-   return pfes->GetVDim()*std::max(pfes->GetMesh()->SpaceDimension(),
-                                   fe->GetRangeDim());
-}
-
 void
 ParComplexGridFunction::ProjectCoefficient(Coefficient &real_coeff,
                                            Coefficient &imag_coeff)
@@ -1010,6 +789,7 @@ ParComplexGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient
 void
 ParComplexGridFunction::Distribute(const Vector *tv)
 {
+   ParFiniteElementSpace *pfes = pgfr->ParFESpace();
    const int tvsize = pfes->GetTrueVSize();
 
    tv->Read();
@@ -1027,6 +807,7 @@ ParComplexGridFunction::Distribute(const Vector *tv)
 void
 ParComplexGridFunction::ParallelProject(Vector &tv) const
 {
+   ParFiniteElementSpace *pfes = pgfr->ParFESpace();
    const int tvsize = pfes->GetTrueVSize();
 
    tv.Write();
@@ -1044,60 +825,6 @@ ParComplexGridFunction::ParallelProject(Vector &tv) const
    tvi.SyncAliasMemory(tv);
 }
 
-void ParComplexGridFunction::Save(std::ostream &os) const
-{
-   os << "ParComplexGridFunction\n";
-   pfes->Save(os);
-   os << '\n';
-
-   int vsize = pfes->GetVSize();
-   real_t *data_  = const_cast<real_t*>(HostRead());
-   for (int i = 0; i < vsize; i++)
-   {
-      if (pfes->GetDofSign(i) < 0)
-      {
-         data_[i] = -data_[i];
-         data_[i+vsize] = -data_[i+vsize];
-      }
-   }
-
-   if (pfes->GetOrdering() == Ordering::byNODES)
-   {
-      Vector::Print(os, 1);
-   }
-   else
-   {
-      Vector::Print(os, pfes->GetVDim());
-   }
-
-   for (int i = 0; i < vsize; i++)
-   {
-      if (pfes->GetDofSign(i) < 0)
-      {
-         data_[i] = -data_[i];
-         data_[i+vsize] = -data_[i+vsize];
-      }
-   }
-
-   os.flush();
-}
-
-void ParComplexGridFunction::Save(const char *fname, int precision) const
-{
-   int rank = pfes->GetMyRank();
-   ostringstream fname_with_suffix;
-   fname_with_suffix << fname << "." << setfill('0') << setw(6) << rank;
-   ofstream ofs(fname_with_suffix.str().c_str());
-   ofs.precision(precision);
-   Save(ofs);
-}
-
-std::ostream &operator<<(std::ostream &os, const ParComplexGridFunction &sol)
-{
-   sol.Save(os);
-   return os;
-}
-
 
 ParComplexLinearForm::ParComplexLinearForm(ParFiniteElementSpace *pfes,
                                            ComplexOperator::Convention

fem/complex_fem.hpp

@@ -35,72 +35,15 @@ private:
    GridFunction * gfi;
 
 protected:
-   /// FE space on which the grid function lives. Owned if #fec_owned
-   /// is not NULL.
-   FiniteElementSpace *fes;
-
-   /** @brief Used when the grid function is read from a file. It can also be
-       set explicitly, see MakeOwner().
-
-       If not NULL, this pointer is owned by the ComplexGridFunction. */
-   FiniteElementCollection *fec_owned;
-
-   long fes_sequence; // see FiniteElementSpace::sequence, Mesh::sequence
-
-   void Destroy();
+   void Destroy() { delete gfr; delete gfi; }
 
 public:
    /** @brief Construct a ComplexGridFunction associated with the
        FiniteElementSpace @a *f. */
    ComplexGridFunction(FiniteElementSpace *f);
 
-   /** @brief Construct a ComplexGridFunction on the given Mesh, using the data
-       from @a input.
-
-       The content of @a input should be in the format created by the method
-       Save(). The reconstructed FiniteElementSpace and FiniteElementCollection
-       are owned by the ComplexGridFunction. */
-   ComplexGridFunction(Mesh *m, std::istream &input);
-
    void Update();
 
-   /** Return update counter, similar to Mesh::GetSequence(). Used to
-       check if it is up to date with the space. */
-   long GetSequence() const { return fes_sequence; }
-
-   /// Make the ComplexGridFunction the owner of #fec_owned and #fes.
-   /** If the new FiniteElementCollection, @a fec_, is NULL, ownership
-       of #fec_owned and #fes is taken away. */
-   void MakeOwner(FiniteElementCollection *fec_) { fec_owned = fec_; }
-
-   /// Returns a pointer to the FiniteElementCollection used to
-   /// construct this ComplexGridFunction if this class owns that
-   /// object. Otherwise this function will return NULL.
-   FiniteElementCollection *OwnFEC() { return fec_owned; }
-
-   /// Shortcut for calling FiniteElementSpace::GetVectorDim() on the
-   /// underlying #fes
-   int VectorDim() const;
-
-   /// Copy assignment. Only the data of the base class Vector is copied.
-   /** It is assumed that this object and @a rhs use FiniteElementSpace%s that
-       have the same size.
-
-       @note Defining this method overwrites the implicitly defined copy
-       assignment operator. */
-   ComplexGridFunction &operator=(const ComplexGridFunction &rhs)
-   { return operator=((const Vector &)rhs); }
-
-   /// Copy the data from @a v.
-   /** The size of @a v must be equal to double of the size of the associated
-       FiniteElementSpace #fes. */
-   ComplexGridFunction &operator=(const Vector &v)
-   {
-      MFEM_ASSERT(fes && v.Size() == 2*fes->GetVSize(), "");
-      Vector::operator=(v);
-      return *this;
-   }
-
    /// Assign constant values to the ComplexGridFunction data.
    ComplexGridFunction &operator=(const std::complex<real_t> & value)
    { *gfr = value.real(); *gfi = value.imag(); return *this; }
@@ -120,8 +63,8 @@ public:
                                              VectorCoefficient &imag_coeff,
                                              Array<int> &attr);
 
-   FiniteElementSpace *FESpace() { return fes; }
-   const FiniteElementSpace *FESpace() const { return fes; }
+   FiniteElementSpace *FESpace() { return gfr->FESpace(); }
+   const FiniteElementSpace *FESpace() const { return gfr->FESpace(); }
 
    GridFunction & real() { return *gfr; }
    GridFunction & imag() { return *gfi; }
@@ -136,52 +79,11 @@ public:
    /// @a gfr and @a gfi to match the ComplexGridFunction.
    void SyncAlias() { gfr->SyncAliasMemory(*this); gfi->SyncAliasMemory(*this); }
 
-   /// @brief Returns ||u_ex - u_h||_L2 for complex-valued scalar fields
-   ///
-   /// @see GridFunction::ComputeL2Error(Coefficient &exsol,
-   ///                                   const IntegrationRule *irs[],
-   ///                                   const Array<int> *elems) const
-   ///      for more detailed documentation.
-   virtual real_t ComputeL2Error(Coefficient &exsolr, Coefficient &exsoli,
-                                 const IntegrationRule *irs[] = NULL) const
-   {
-      real_t err_r = gfr->ComputeL2Error(exsolr, irs);
-      real_t err_i = gfi->ComputeL2Error(exsoli, irs);
-      return sqrt(err_r * err_r + err_i * err_i);
-   }
-
-   /// @brief Returns ||u_ex - u_h||_L2 for complex-valued vector fields
-   ///
-   /// @see GridFunction::ComputeL2Error(VectorCoefficient &exsol,
-   ///                                   const IntegrationRule *irs[],
-   ///                                   const Array<int> *elems) const
-   ///      for more detailed documentation.
-   virtual real_t ComputeL2Error(VectorCoefficient &exsolr,
-                                 VectorCoefficient &exsoli,
-                                 const IntegrationRule *irs[] = NULL,
-                                 Array<int> *elems = NULL) const
-   {
-      real_t err_r = gfr->ComputeL2Error(exsolr, irs, elems);
-      real_t err_i = gfi->ComputeL2Error(exsoli, irs, elems);
-      return sqrt(err_r * err_r + err_i * err_i);
-   }
-
-   /// Save the ComplexGridFunction to an output stream.
-   virtual void Save(std::ostream &out) const;
-
-   /// Save the ComplexGridFunction to a file
-   /** The given @a precision will be used for ASCII output. */
-   virtual void Save(const char *fname, int precision=16) const;
-
    /// Destroys the grid function.
    virtual ~ComplexGridFunction() { Destroy(); }
 
 };
 
-/** Overload operator<< for std::ostream and ComplexGridFunction; not valid
-    for the class ParComplexGridFunction */
-std::ostream &operator<<(std::ostream &out, const ComplexGridFunction &sol);
-
 /** Class for a complex-valued linear form
 
     The @a convention argument in the class's constructor is documented in the
@@ -443,23 +345,12 @@ public:
 class ParComplexGridFunction : public Vector
 {
 private:
+
    ParGridFunction * pgfr;
    ParGridFunction * pgfi;
 
 protected:
-   /// FE space on which the grid function lives. Owned if #fec_owned
-   /// is not NULL.
-   ParFiniteElementSpace *pfes;
-
-   /** @brief Used when the grid function is read from a file. It can also be
-       set explicitly, see MakeOwner().
-
-       If not NULL, this pointer is owned by the ParComplexGridFunction. */
-   FiniteElementCollection *fec_owned;
-
-   long fes_sequence; // see FiniteElementSpace::sequence, Mesh::sequence
-
-   void Destroy();
+   void Destroy() { delete pgfr; delete pgfi; }
 
 public:
 
@@ -467,33 +358,8 @@ public:
        ParFiniteElementSpace @a *pf. */
    ParComplexGridFunction(ParFiniteElementSpace *pf);
 
-   /** @brief Construct a ParComplexGridFunction on a given ParMesh,
-       @a pmesh, reading from an std::istream.
-
-       In the process, a ParFiniteElementSpace and a FiniteElementCollection are
-       constructed. The new ParComplexGridFunction assumes ownership of both. */
-   ParComplexGridFunction(ParMesh *pmesh, std::istream &input);
-
    void Update();
 
-   /** Return update counter, similar to Mesh::GetSequence(). Used to
-       check if it is up to date with the space. */
-   long GetSequence() const { return fes_sequence; }
-
-   /// Make the ParComplexGridFunction the owner of #fec_owned and #pfes.
-   /** If the new FiniteElementCollection, @a fec_, is NULL, ownership
-       of #fec_owned and #pfes is taken away. */
-   void MakeOwner(FiniteElementCollection *fec_) { fec_owned = fec_; }
-
-   /// Returns a pointer to the FiniteElementCollection used to
-   /// construct this ParComplexGridFunction if this class owns that
-   /// object. Otherwise this function will return NULL.
-   FiniteElementCollection *OwnFEC() { return fec_owned; }
-
-   /// Shortcut for calling FiniteElementSpace::GetVectorDim() on the
-   /// underlying #pfes
-   int VectorDim() const;
-
    /// Assign constant values to the ParComplexGridFunction data.
    ParComplexGridFunction &operator=(const std::complex<real_t> & value)
    { *pgfr = value.real(); *pgfi = value.imag(); return *this; }
@@ -519,11 +385,11 @@ public:
    /// Returns the vector restricted to the true dofs.
    void ParallelProject(Vector &tv) const;
 
-   FiniteElementSpace *FESpace() { return pfes; }
-   const FiniteElementSpace *FESpace() const { return pfes; }
+   FiniteElementSpace *FESpace() { return pgfr->FESpace(); }
+   const FiniteElementSpace *FESpace() const { return pgfr->FESpace(); }
 
-   ParFiniteElementSpace *ParFESpace() { return pfes; }
-   const ParFiniteElementSpace *ParFESpace() const { return pfes; }
+   ParFiniteElementSpace *ParFESpace() { return pgfr->ParFESpace(); }
+   const ParFiniteElementSpace *ParFESpace() const { return pgfr->ParFESpace(); }
 
    ParGridFunction & real() { return *pgfr; }
    ParGridFunction & imag() { return *pgfi; }
@@ -536,32 +402,17 @@ public:
 
    /// Update the alias memory location of the real and imaginary
    /// ParGridFunction @a pgfr and @a pgfi to match the ParComplexGridFunction.
-   void SyncAlias()
-   { pgfr->SyncAliasMemory(*this); pgfi->SyncAliasMemory(*this); }
+   void SyncAlias() { pgfr->SyncAliasMemory(*this); pgfi->SyncAliasMemory(*this); }
+
 
-   /// @brief Returns ||u_ex - u_h||_L2 in parallel for complex-valued
-   ///        scalar fields
-   ///
-   /// @see GridFunction::ComputeL2Error(Coefficient &exsol,
-   ///                                   const IntegrationRule *irs[],
-   ///                                   const Array<int> *elems) const
-   ///      for more detailed documentation.
    virtual real_t ComputeL2Error(Coefficient &exsolr, Coefficient &exsoli,
-                                 const IntegrationRule *irs[] = NULL,
-                                 Array<int> *elems = NULL) const
+                                 const IntegrationRule *irs[] = NULL) const
    {
-      real_t err_r = pgfr->ComputeL2Error(exsolr, irs, elems);
-      real_t err_i = pgfi->ComputeL2Error(exsoli, irs, elems);
-      return hypot(err_r, err_i);
+      real_t err_r = pgfr->ComputeL2Error(exsolr, irs);
+      real_t err_i = pgfi->ComputeL2Error(exsoli, irs);
+      return sqrt(err_r * err_r + err_i * err_i);
    }
 
-   /// @brief Returns ||u_ex - u_h||_L2 in parallel for complex-valued
-   ///        vector fields
-   ///
-   /// @see GridFunction::ComputeL2Error(VectorCoefficient &exsol,
-   ///                                   const IntegrationRule *irs[],
-   ///                                   const Array<int> *elems) const
-   ///      for more detailed documentation.
    virtual real_t ComputeL2Error(VectorCoefficient &exsolr,
                                  VectorCoefficient &exsoli,
                                  const IntegrationRule *irs[] = NULL,
@@ -569,28 +420,15 @@ public:
    {
       real_t err_r = pgfr->ComputeL2Error(exsolr, irs, elems);
       real_t err_i = pgfi->ComputeL2Error(exsoli, irs, elems);
-      return hypot(err_r, err_i);
+      return sqrt(err_r * err_r + err_i * err_i);
    }
 
-   /// Save the local portion of the ParComplexGridFunction
-   /** This differs from the serial ComplexGridFunction::Save in that it
-       takes into account the signs of the local dofs. */
-   void Save(std::ostream &out) const;
-
-   /// Save the ParComplexGridFunction to files
-   /** Saves one file for each MPI rank. The files will be given suffixes
-       according to the MPI rank. The given @a precision will be used for ASCII
-       output. */
-   void Save(const char *fname, int precision=16) const;
 
    /// Destroys grid function.
    virtual ~ParComplexGridFunction() { Destroy(); }
 
 };
 
-/** Overload operator<< for std::ostream and ParComplexGridFunction */
-std::ostream &operator<<(std::ostream &out, const ParComplexGridFunction &sol);
-
 /** Class for a complex-valued, parallel linear form
 
     The @a convention argument in the class's constructor is documented in the

fem/conduitdatacollection.cpp

@@ -70,8 +70,8 @@ ConduitDataCollection::~ConduitDataCollection()
 void ConduitDataCollection::Save()
 {
    std::string dir_name = MeshDirectoryName();
-   int err_ = create_directory(dir_name, mesh, myid);
-   if (err_)
+   int err = create_directory(dir_name, mesh, myid);
+   if (err)
    {
       MFEM_ABORT("Error creating directory: " << dir_name);
    }
@@ -88,7 +88,6 @@ void ConduitDataCollection::Save()
                  << verify_info.to_json());
    }
 
-   // wrap all grid functions
    FieldMapConstIterator itr;
    for ( itr = field_map.begin(); itr != field_map.end(); itr++)
    {
@@ -104,16 +103,6 @@ void ConduitDataCollection::Save()
       }
    }
 
-   // wrap all quadrature functions
-   QFieldMapConstIterator qf_itr;
-   for ( qf_itr = q_field_map.begin(); qf_itr != q_field_map.end(); qf_itr++)
-   {
-      std::string name = qf_itr->first;
-      QuadratureFunction *qf = qf_itr->second;
-      QuadratureFunctionToBlueprintField(qf,
-                                         n_mesh["fields"][name]);
-   }
-
    // save mesh data
    SaveMeshAndFields(myid,
                      n_mesh,
@@ -168,16 +157,6 @@ ConduitDataCollection::SetProtocol(const std::string &protocol)
    relay_protocol = protocol;
 }
 
-// Conduit data type id for the MFEM precision
-constexpr conduit::index_t mfem_precision_conduit_id =
-#if defined(MFEM_USE_DOUBLE)
-   CONDUIT_NATIVE_DOUBLE_ID;
-#elif defined(MFEM_USE_SINGLE)
-   CONDUIT_NATIVE_FLOAT_ID;
-#else
-#error Unknown MFEM precision
-#endif
-
 //------------------------------
 // begin static public methods
 //------------------------------
@@ -227,41 +206,42 @@ ConduitDataCollection::BlueprintMeshToMesh(const Node &n_mesh,
    // get the number of points
    int num_verts = n_coordset_vals[0].dtype().number_of_elements();
    // get vals for points
-   const real_t *verts_ptr = NULL;
+   const double *verts_ptr = NULL;
 
    // the mfem mesh constructor needs coords with interleaved (aos) type
-   // ordering, even for 1d + 2d we always need 3 real_t (double/float) b/c it
-   // uses Array<Vertex> and Vertex is a pod of 3 real_t. we check for this
+   // ordering, even for 1d + 2d we always need 3 doubles b/c it uses
+   // Array<Vertex> and Vertex is a pod of 3 doubles. we check for this
    // case, if we don't have it we convert the data
 
    if (ndims == 3 &&
-       n_coordset_vals[0].dtype().id() == mfem_precision_conduit_id &&
+       n_coordset_vals[0].dtype().is_double() &&
        blueprint::mcarray::is_interleaved(n_coordset_vals) )
    {
-      // already interleaved mcarray of 3 real_t (double/float),
+      // already interleaved mcarray of 3 doubles,
       // return ptr to beginning
       verts_ptr = n_coordset_vals[0].value();
    }
    else
    {
       Node n_tmp;
-      // check all vals, if we don't have real_t (double/float) convert
-      // to real_t
+      // check all vals, if we don't have doubles convert
+      // to doubles
       NodeConstIterator itr = n_coordset_vals.children();
       while (itr.has_next())
       {
          const Node &c_vals = itr.next();
          std::string c_name = itr.name();
 
-         if ( c_vals.dtype().id() == mfem_precision_conduit_id )
+         if ( c_vals.dtype().is_double() )
          {
             // zero copy current coords
             n_tmp[c_name].set_external(c_vals);
+
          }
          else
          {
             // convert
-            c_vals.to_data_type(mfem_precision_conduit_id, n_tmp[c_name]);
+            c_vals.to_double_array(n_tmp[c_name]);
          }
       }
 
@@ -270,13 +250,13 @@ ConduitDataCollection::BlueprintMeshToMesh(const Node &n_mesh,
       if (ndims < 3)
       {
          // add dummy z
-         n_tmp["z"].set(DataType(mfem_precision_conduit_id, num_verts));
+         n_tmp["z"].set(DataType::c_double(num_verts));
       }
 
       if (ndims < 2)
       {
          // add dummy y
-         n_tmp["y"].set(DataType(mfem_precision_conduit_id, num_verts));
+         n_tmp["y"].set(DataType::c_double(num_verts));
       }
 
       Node &n_conv_coords_vals = n_conv["coordsets"][coords_name]["values"];
@@ -472,7 +452,7 @@ ConduitDataCollection::BlueprintMeshToMesh(const Node &n_mesh,
    //       if nodes gf is attached later, it resets the space dim based
    //       on the gf's fes.
    Mesh *mesh = new Mesh(// from coordset
-      const_cast<real_t*>(verts_ptr),
+      const_cast<double*>(verts_ptr),
       num_verts,
       // from topology
       const_cast<int*>(elem_indices),
@@ -539,7 +519,7 @@ ConduitDataCollection::BlueprintFieldToGridFunction(Mesh *mesh,
    // can't return a gf that zero copies the conduit data
    Node n_conv;
 
-   const real_t *vals_ptr = NULL;
+   const double *vals_ptr = NULL;
 
    int vdim = 1;
 
@@ -549,10 +529,10 @@ ConduitDataCollection::BlueprintFieldToGridFunction(Mesh *mesh,
    {
       vdim = n_field["values"].number_of_children();
 
-      // need to check that we have real_t (double/float) and
+      // need to check that we have doubles and
       // cover supported layouts
 
-      if ( n_field["values"][0].dtype().id() == mfem_precision_conduit_id )
+      if ( n_field["values"][0].dtype().is_double() )
       {
          // check for contig
          if (n_field["values"].is_contiguous())
@@ -576,26 +556,27 @@ ConduitDataCollection::BlueprintFieldToGridFunction(Mesh *mesh,
             vals_ptr = n_conv["values"].child(0).value();
          }
       }
-      else // convert to real_t (double/float) and use contig
+      else // convert to doubles and use contig
       {
          Node n_tmp;
-         // check all vals, if we don't have real_t (double/float) convert
-         // to real_t
+         // check all vals, if we don't have doubles convert
+         // to doubles
          NodeConstIterator itr = n_field["values"].children();
          while (itr.has_next())
          {
             const Node &c_vals = itr.next();
             std::string c_name = itr.name();
 
-            if ( c_vals.dtype().id() == mfem_precision_conduit_id )
+            if ( c_vals.dtype().is_double() )
             {
                // zero copy current coords
                n_tmp[c_name].set_external(c_vals);
+
             }
             else
             {
                // convert
-               c_vals.to_data_type(mfem_precision_conduit_id, n_tmp[c_name]);
+               c_vals.to_double_array(n_tmp[c_name]);
             }
          }
 
@@ -608,15 +589,14 @@ ConduitDataCollection::BlueprintFieldToGridFunction(Mesh *mesh,
    }
    else
    {
-      if (n_field["values"].dtype().id() == mfem_precision_conduit_id &&
+      if (n_field["values"].dtype().is_double() &&
           n_field["values"].is_compact())
       {
          vals_ptr = n_field["values"].value();
       }
       else
       {
-         n_field["values"].to_data_type(mfem_precision_conduit_id,
-                                        n_conv["values"]);
+         n_field["values"].to_double_array(n_conv["values"]);
          vals_ptr = n_conv["values"].value();
       }
    }
@@ -640,14 +620,14 @@ ConduitDataCollection::BlueprintFieldToGridFunction(Mesh *mesh,
 
    if (zero_copy)
    {
-      res = new GridFunction(fes,const_cast<real_t*>(vals_ptr));
+      res = new GridFunction(fes,const_cast<double*>(vals_ptr));
    }
    else
    {
       // copy case, this constructor will alloc the space for the GF data
       res = new GridFunction(fes);
       // create an mfem vector that wraps the conduit data
-      Vector vals_vec(const_cast<real_t*>(vals_ptr),fes->GetVSize());
+      Vector vals_vec(const_cast<double*>(vals_ptr),fes->GetVSize());
       // copy values into the result
       (*res) = vals_vec;
    }
@@ -659,155 +639,6 @@ ConduitDataCollection::BlueprintFieldToGridFunction(Mesh *mesh,
    return res;
 }
 
-//---------------------------------------------------------------------------//
-mfem::QuadratureFunction *
-ConduitDataCollection::BlueprintFieldToQuadratureFunction(Mesh *mesh,
-                                                          const Node &n_field,
-                                                          bool zero_copy)
-{
-   // n_conv holds converted data (when necessary for mfem api)
-   // if n_conv is used ( !n_conv.dtype().empty() ) we
-   // know that some data allocation was necessary, so we
-   // can't return a qf that zero copies the conduit data
-   Node n_conv;
-
-   const real_t *vals_ptr = NULL;
-   int vdim = 1;
-
-   if (n_field["values"].dtype().is_object())
-   {
-      vdim = n_field["values"].number_of_children();
-
-      // need to check that we have real_t (double/float) and
-      // cover supported layouts
-      if ( n_field["values"][0].dtype().id() == mfem_precision_conduit_id )
-      {
-         // quad funcs use what mfem calls byVDIM
-         // and what conduit calls interleaved
-         // check for interleaved
-         if (blueprint::mcarray::is_interleaved(n_field["values"]))
-         {
-            // conduit mcarray interleaved == mfem byVDIM
-            vals_ptr = n_field["values"].child(0).value();
-         }
-         else
-         {
-            // for mcarray generic case --  default to byVDIM
-            // aka interleaved
-            blueprint::mcarray::to_interleaved(n_field["values"],
-                                               n_conv["values"]);
-            vals_ptr = n_conv["values"].child(0).value();
-         }
-      }
-      else // convert to real_t (double/float) and use interleaved
-      {
-         Node n_tmp;
-         // check all vals, if we don't have real_t (double/float) convert
-         // to real_t
-         NodeConstIterator itr = n_field["values"].children();
-         while (itr.has_next())
-         {
-            const Node &c_vals = itr.next();
-            std::string c_name = itr.name();
-
-            if ( c_vals.dtype().id() == mfem_precision_conduit_id )
-            {
-               // zero copy current coords
-               n_tmp[c_name].set_external(c_vals);
-            }
-            else
-            {
-               // convert
-               c_vals.to_data_type(mfem_precision_conduit_id, n_tmp[c_name]);
-            }
-         }
-
-         // for mcarray generic case --  default to byVDIM
-         // aka interleaved
-         blueprint::mcarray::to_interleaved(n_tmp,
-                                            n_conv["values"]);
-         vals_ptr = n_conv["values"].child(0).value();
-      }
-   }
-   else // scalar case
-   {
-      if (n_field["values"].dtype().id() == mfem_precision_conduit_id &&
-          n_field["values"].is_compact())
-      {
-         vals_ptr = n_field["values"].value();
-      }
-      else
-      {
-         n_field["values"].to_data_type(mfem_precision_conduit_id,
-                                        n_conv["values"]);
-         vals_ptr = n_conv["values"].value();
-      }
-   }
-
-   if (zero_copy && !n_conv.dtype().is_empty())
-   {
-      //Info: "Cannot zero-copy since data conversions were necessary"
-      zero_copy = false;
-   }
-
-   // we need basis name to create the proper mfem quad space and quad func
-   // the pattern used to encode the quad space params is:
-   // QF_{ORDER}_{VDIM}
-   // ORDER is the degree of the polynomials for the quad rule
-   // VDIM  is the number of components at each quad point (scalar, vector, etc)
-
-   int qf_order = 0;
-   int qf_vdim  = 0;
-   std::string qf_name = n_field["basis"].as_string();
-   const char *qf_name_cstr = qf_name.c_str();
-   if (!strncmp(qf_name_cstr, "QF_", 3))
-   {
-      // parse {ORDER}
-      qf_order = atoi(qf_name_cstr + 3);
-      // find second `_`
-      const char *qf_vdim_cstr = strstr(qf_name_cstr+3,"_");
-      if (qf_vdim_cstr == NULL)
-      {
-         MFEM_ABORT("Error parsing quadrature function description string: "
-                    << qf_name << std::endl
-                    << "Expected: QF_{ORDER}_{VDIM}");
-      }
-      // parse {VDIM}
-      qf_vdim  = atoi(qf_vdim_cstr+1);
-   }
-   else
-   {
-      MFEM_ABORT("Error parsing quadrature function description string: "
-                 << qf_name << std::endl
-                 << "Expected: QF_{ORDER}_{VDIM}");
-   }
-   MFEM_VERIFY(qf_vdim == vdim, "vector dimension mismatch: vdim = " << vdim
-               << ", qf_vdim = " << qf_vdim);
-
-   mfem::QuadratureSpace *quad_space = new mfem::QuadratureSpace(mesh, qf_order);
-   mfem::QuadratureFunction *res = new mfem::QuadratureFunction();
-
-   if (zero_copy)
-   {
-      res->SetSpace(quad_space, const_cast<real_t*>(vals_ptr), vdim);
-      res->SetOwnsSpace(true);
-   }
-   else
-   {
-      res->SetSpace(quad_space, vdim);
-      res->SetOwnsSpace(true);
-      // copy case, this constructor will alloc the space for the quad data
-      // create an mfem vector that wraps the conduit data
-      Vector vals_vec(const_cast<real_t*>(vals_ptr),res->Size());
-      // copy values into the result
-      (*res) = vals_vec;
-   }
-
-   return res;
-}
-
-
-
 //---------------------------------------------------------------------------//
 void
 ConduitDataCollection::MeshToBlueprintMesh(Mesh *mesh,
@@ -825,20 +656,20 @@ ConduitDataCollection::MeshToBlueprintMesh(Mesh *mesh,
    // Setup main coordset
    ////////////////////////////////////////////
 
-   // Assumes  mfem::Vertex has the layout of a real_t (double/float) array.
+   // Assumes  mfem::Vertex has the layout of a double array.
 
-   // this logic assumes an mfem vertex is always 3 real_t (double/float) wide
+   // this logic assumes an mfem vertex is always 3 doubles wide
    int stride = sizeof(mfem::Vertex);
    int num_vertices = mesh->GetNV();
 
-   MFEM_ASSERT( ( stride == 3 * sizeof(real_t) ),
+   MFEM_ASSERT( ( stride == 3 * sizeof(double) ),
                 "Unexpected stride for Vertex");
 
    Node &n_mesh_coords = n_mesh["coordsets"][coordset_name];
    n_mesh_coords["type"] =  "explicit";
 
 
-   real_t *coords_ptr = mesh->GetVertex(0);
+   double *coords_ptr = mesh->GetVertex(0);
 
    n_mesh_coords["values/x"].set_external(coords_ptr,
                                           num_vertices,
@@ -849,14 +680,14 @@ ConduitDataCollection::MeshToBlueprintMesh(Mesh *mesh,
    {
       n_mesh_coords["values/y"].set_external(coords_ptr,
                                              num_vertices,
-                                             sizeof(real_t),
+                                             sizeof(double),
                                              stride);
    }
    if (dim >= 3)
    {
       n_mesh_coords["values/z"].set_external(coords_ptr,
                                              num_vertices,
-                                             sizeof(real_t) * 2,
+                                             sizeof(double) * 2,
                                              stride);
    }
 
@@ -1111,59 +942,6 @@ ConduitDataCollection::GridFunctionToBlueprintField(mfem::GridFunction *gf,
 
 }
 
-//---------------------------------------------------------------------------//
-void
-ConduitDataCollection::QuadratureFunctionToBlueprintField(
-   mfem::QuadratureFunction *qf,
-   Node &n_field,
-   const std::string &main_topology_name)
-{
-   // For quadrature functions, use basis pattern:
-   //   QF_{ORDER}_{VDIM}
-
-   int qf_vdim  = qf->GetVDim();
-   int qf_order = qf->GetSpace()->GetOrder();
-   int qf_size  = qf->GetSpace()->GetSize();
-
-   {
-      std::ostringstream oss;
-      oss << "QF_" << qf_order << "_" << qf_vdim;
-
-      n_field["basis"] = oss.str();
-      n_field["topology"] = main_topology_name;
-   }
-
-   if (qf_vdim == 1) // scalar case
-   {
-      n_field["values"].set_external(const_cast<real_t *>(qf->HostRead()),
-                                     qf_size);
-   }
-   else // vector case
-   {
-      // deal with striding of all components
-      // quadrature functions are always byVDIM
-      // or what conduit calls interleaved
-
-      index_t offset = 0;
-      index_t stride = sizeof(real_t) * qf_vdim;
-
-      for (int d = 0;  d < qf_vdim; d++)
-      {
-         std::ostringstream oss;
-         oss << "v" << d;
-         std::string comp_name = oss.str();
-         n_field["values"][comp_name].set_external(const_cast<real_t *>(qf->HostRead()),
-                                                   qf_size,
-                                                   offset,
-                                                   stride);
-         offset += sizeof(real_t);
-      }
-   }
-
-}
-
-
-
 //------------------------------
 // end static public methods
 //------------------------------
@@ -1189,7 +967,7 @@ ConduitDataCollection::RootFileName()
 //---------------------------------------------------------------------------//
 std::string
 ConduitDataCollection::MeshFileName(int domain_id,
-                                    const std::string &relay_protocol_)
+                                    const std::string &relay_protocol)
 {
    std::string res = prefix_path +
                      name  +
@@ -1198,7 +976,7 @@ ConduitDataCollection::MeshFileName(int domain_id,
                      "/domain_" +
                      to_padded_string(domain_id, pad_digits_rank) +
                      "." +
-                     relay_protocol_;
+                     relay_protocol;
 
    return res;
 }
@@ -1216,7 +994,7 @@ ConduitDataCollection::MeshDirectoryName()
 
 //---------------------------------------------------------------------------//
 std::string
-ConduitDataCollection::MeshFilePattern(const std::string &relay_protocol_)
+ConduitDataCollection::MeshFilePattern(const std::string &relay_protocol)
 {
    std::ostringstream oss;
    oss << name
@@ -1225,7 +1003,7 @@ ConduitDataCollection::MeshFilePattern(const std::string &relay_protocol_)
        << "/domain_%0"
        << pad_digits_rank
        << "d."
-       << relay_protocol_;
+       << relay_protocol;
 
    return oss.str();
 }
@@ -1235,14 +1013,14 @@ ConduitDataCollection::MeshFilePattern(const std::string &relay_protocol_)
 void
 ConduitDataCollection::SaveRootFile(int num_domains,
                                     const Node &n_mesh,
-                                    const std::string &relay_protocol_)
+                                    const std::string &relay_protocol)
 {
    // default to json root file, except for hdf5 case
    std::string root_proto = "json";
 
-   if (relay_protocol_ == "hdf5")
+   if (relay_protocol == "hdf5")
    {
-      root_proto = relay_protocol_;
+      root_proto = relay_protocol;
    }
 
    Node n_root;
@@ -1273,14 +1051,14 @@ ConduitDataCollection::SaveRootFile(int num_domains,
       }
    }
    // add extra header info
-   n_root["protocol/name"]    =  relay_protocol_;
+   n_root["protocol/name"]    =  relay_protocol;
    n_root["protocol/version"] = "0.3.1";
 
 
    // we will save one file per domain, so trees == files
    n_root["number_of_files"]  = num_domains;
    n_root["number_of_trees"]  = num_domains;
-   n_root["file_pattern"]     = MeshFilePattern(relay_protocol_);
+   n_root["file_pattern"]     = MeshFilePattern(relay_protocol);
    n_root["tree_pattern"]     = "";
 
    // Add the time, time step, and cycle
@@ -1295,9 +1073,9 @@ ConduitDataCollection::SaveRootFile(int num_domains,
 void
 ConduitDataCollection::SaveMeshAndFields(int domain_id,
                                          const Node &n_mesh,
-                                         const std::string &relay_protocol_)
+                                         const std::string &relay_protocol)
 {
-   relay::io::save(n_mesh, MeshFileName(domain_id, relay_protocol_));
+   relay::io::save(n_mesh, MeshFileName(domain_id, relay_protocol));
 }
 
 //---------------------------------------------------------------------------//
@@ -1394,13 +1172,13 @@ ConduitDataCollection::LoadRootFile(Node &root_out)
 //---------------------------------------------------------------------------//
 void
 ConduitDataCollection::LoadMeshAndFields(int domain_id,
-                                         const std::string &relay_protocol_)
+                                         const std::string &relay_protocol)
 {
    // Note: This path doesn't use any info from the root file
    // it uses the implicit mfem ConduitDataCollection layout
 
    Node n_mesh;
-   relay::io::load( MeshFileName(domain_id, relay_protocol_), n_mesh);
+   relay::io::load( MeshFileName(domain_id, relay_protocol), n_mesh);
 
 
    Node verify_info;

fem/conduitdatacollection.hpp

@@ -50,11 +50,11 @@ namespace mfem
     Those that construct MFEM objects from Conduit Nodes (Conduit Blueprint to
     MFEM) provide a zero-copy option. Zero-copy is only possible if the
     blueprint data matches the data types provided by the MFEM API, for example:
-    ints for connectivity arrays, real_t (double/float) for field value arrays,
-    allocations that match MFEM's striding options, etc. If these constraints
-    are not met, MFEM objects that own the data are created and returned. In
-    either case pointers to new MFEM object instances are returned, the
-    zero-copy only applies to data backing the MFEM object instances.
+    ints for connectivity arrays, doubles for field value arrays, allocations
+    that match MFEM's striding options, etc. If these constraints are not met,
+    MFEM objects that own the data are created and returned. In either case
+    pointers to new MFEM object instances are returned, the zero-copy only
+    applies to data backing the MFEM object instances.
 
     @note QuadratureFunction%s (q-fields) are not supported.
 
@@ -183,21 +183,6 @@ public:
                                             conduit::Node &out,
                                             const std::string &main_topology_name = "main");
 
-   /// Describes a MFEM quadrature function using the mesh blueprint
-   /** Sets up passed conduit::Node out to describe the given quadrature function
-       using the mesh field blueprint.
-
-       Zero-copies as much data as possible.
-
-       @a main_toplogy_name is used to set the associated topology name.
-       With the default setting, the resulting field is associated with the
-       topology `main`.
-   */
-   static void QuadratureFunctionToBlueprintField(QuadratureFunction *qf,
-                                                  conduit::Node &out,
-                                                  const std::string &main_topology_name = "main");
-
-
    /// Constructs and MFEM mesh from a Conduit Blueprint Description
    /** @a main_topology_name is used to select which topology to use, when
        empty ("") the first topology entry will be used.
@@ -205,7 +190,7 @@ public:
        If zero_copy == true, tries to construct a mesh that points to the data
        described by the conduit node. This is only possible if the data in the
        node matches the data types needed for the MFEM API (ints for
-       connectivity, real_t for field values, etc). If these constraints are
+       connectivity, doubles for field values, etc). If these constraints are
        not met, a mesh that owns the data is created and returned.
    */
    static Mesh *BlueprintMeshToMesh(const conduit::Node &n_mesh,
@@ -215,7 +200,7 @@ public:
    /// Constructs and MFEM Grid Function from a Conduit Blueprint Description
    /** If zero_copy == true, tries to construct a grid function that points to
        the data described by the conduit node. This is only possible if the data
-       in the node matches the data types needed for the MFEM API (real_t for
+       in the node matches the data types needed for the MFEM API (doubles for
        field values, allocated in soa or aos ordering, etc).  If these
        constraints are not met, a grid function that owns the data is created
        and returned.
@@ -223,17 +208,6 @@ public:
    static GridFunction *BlueprintFieldToGridFunction(Mesh *mesh,
                                                      const conduit::Node &n_field,
                                                      bool zero_copy = false);
-   /// Constructs and MFEM Quadrature Function from a Conduit Blueprint Description
-   /** If zero_copy == true, tries to construct a quadrature function that points to
-       the data described by the conduit node. This is only possible if the data
-       in the node matches the data types needed for the MFEM API (real_t for
-       field values, allocated in an interleavred/byVDIM order, etc). If these
-       constraints are not met, a grid function that owns the data is created
-       and returned.
-   */
-   static QuadratureFunction *BlueprintFieldToQuadratureFunction(Mesh *mesh,
-                                                                 const conduit::Node &n_field,
-                                                                 bool zero_copy = false);
 
 private:
    /// Converts from MFEM element type enum to mesh bp shape name

fem/datacollection.cpp

@@ -310,9 +310,9 @@ void DataCollection::SaveField(const std::string &field_name)
    }
 }
 
-void DataCollection::SaveQField(const std::string &field_name)
+void DataCollection::SaveQField(const std::string &q_field_name)
 {
-   QFieldMapIterator it = q_field_map.find(field_name);
+   QFieldMapIterator it = q_field_map.find(q_field_name);
    if (it != q_field_map.end())
    {
       SaveOneQField(it);
@@ -430,9 +430,7 @@ void VisItDataCollection::RegisterField(const std::string& name,
    }
 
    DataCollection::RegisterField(name, gf);
-   field_info_map[name] = VisItFieldInfo("nodes", gf->VectorDim(), LOD,
-                                         gf->FESpace()->FEColl()->Name(),
-                                         gf->FESpace()->FEColl()->GetOrder());
+   field_info_map[name] = VisItFieldInfo("nodes", gf->VectorDim(), LOD);
    visit_levels_of_detail = std::max(visit_levels_of_detail, LOD);
 }
 
@@ -451,14 +449,7 @@ void VisItDataCollection::RegisterQField(const std::string& name,
    }
 
    DataCollection::RegisterQField(name, qf);
-   // For quadrature functions, use basis pattern:
-   //   QF_{ORDER}_{VDIM}
-   int qf_vdim  = qf->GetVDim();
-   int qf_order = qf->GetSpace()->GetOrder();
-   std::ostringstream oss;
-   oss << "QF_" << qf_order << "_" << qf_vdim;
-   field_info_map[name] = VisItFieldInfo("quadrature", qf->GetVDim(), LOD,
-                                         oss.str(), qf_order);
+   field_info_map[name] = VisItFieldInfo("elements", 1, LOD);
    visit_levels_of_detail = std::max(visit_levels_of_detail, LOD);
 }
 
@@ -492,8 +483,6 @@ void VisItDataCollection::SaveRootFile()
                            to_padded_string(cycle, pad_digits_cycle) +
                            ".mfem_root";
    std::ofstream root_file(root_name);
-   MFEM_VERIFY(root_file.is_open(),
-               "Failed to open ofstream " << root_name);
    root_file << GetVisItRootString();
    if (!root_file)
    {
@@ -634,8 +623,7 @@ void VisItDataCollection::LoadFields()
          {
             field_map.Register(it->first, new GridFunction(mesh, file), own_data);
          }
-         else if ((it->second).association == "elements" || // old style
-                  (it->second).association == "quadrature") // new style
+         else if ((it->second).association == "elements")
          {
             q_field_map.Register(it->first, new QuadratureFunction(mesh, file), own_data);
          }
@@ -649,8 +637,7 @@ void VisItDataCollection::LoadFields()
                it->first,
                new ParGridFunction(dynamic_cast<ParMesh*>(mesh), file), own_data);
          }
-         else if ((it->second).association == "elements" || // old style
-                  (it->second).association == "quadrature") // new style
+         else if ((it->second).association == "elements")
          {
             q_field_map.Register(it->first, new QuadratureFunction(mesh, file), own_data);
          }
@@ -689,8 +676,6 @@ std::string VisItDataCollection::GetVisItRootString()
       ftags["assoc"] = picojson::value((it->second).association);
       ftags["comps"] = picojson::value(to_string((it->second).num_components));
       ftags["lod"] = picojson::value(to_string((it->second).lod));
-      ftags["basis"] = picojson::value((it->second).basis);
-      ftags["order"] = picojson::value(to_string((it->second).order));
       field["path"] = picojson::value(path_str + it->first + file_ext_format);
       field["tags"] = picojson::value(ftags);
       fields[it->first] = picojson::value(field);
@@ -767,31 +752,9 @@ void VisItDataCollection::ParseVisItRootString(const std::string& json)
            it != fields_obj.end(); ++it)
       {
          picojson::value tags = it->second.get("tags");
-
-         // defaults that allow us to parse older mfem_root files
-         int lod = 1;
-         std::string basis = "";
-         int order = -1;
-
-         if (tags.contains("lod"))
-         {
-            lod = to_int(tags.get("lod").get<std::string>());
-         }
-
-         if (tags.contains("basis"))
-         {
-            basis = tags.get("comps").get<std::string>();
-         }
-
-         if (tags.contains("order"))
-         {
-            order = to_int(tags.get("comps").get<std::string>());
-         }
-
          field_info_map[it->first] =
             VisItFieldInfo(tags.get("assoc").get<std::string>(),
-                           to_int(tags.get("comps").get<std::string>()),
-                           lod, basis, order);
+                           to_int(tags.get("comps").get<std::string>()));
       }
    }
 }
@@ -817,11 +780,6 @@ void ParaViewDataCollectionBase::SetHighOrderOutput(bool high_order_output_)
    high_order_output = high_order_output_;
 }
 
-void ParaViewDataCollectionBase::SetBoundaryOutput(bool bdr_output_)
-{
-   bdr_output = bdr_output_;
-}
-
 void ParaViewDataCollectionBase::SetCompressionLevel(int compression_level_)
 {
    MFEM_ASSERT(compression_level_ >= -1 && compression_level_ <= 9,
@@ -977,27 +935,18 @@ void ParaViewDataCollection::Save()
    std::string vtu_prefix = col_path + "/" + GenerateVTUPath() + "/";
 
    // Save the local part of the mesh and grid functions fields to the local
-   // VTU file. Also save coefficient fields.
+   // VTU file
    {
-      std::string os_str = vtu_prefix + GenerateVTUFileName("proc", myid);
-      std::ofstream os(os_str);
-      MFEM_VERIFY(os.is_open(),
-                  "Failed to open ofstream " << os_str);
+      std::ofstream os(vtu_prefix + GenerateVTUFileName("proc", myid));
       os.precision(precision);
       SaveDataVTU(os, levels_of_detail);
    }
 
-   // Save the local part of the quadrature function fields.
+   // Save the local part of the quadrature function fields
    for (const auto &qfield : q_field_map)
    {
-      MFEM_VERIFY(!bdr_output,
-                  "QuadratureFunction output is not supported for "
-                  "ParaViewDataCollection on domain boundary!");
       const std::string &field_name = qfield.first;
-      std::string os_str = vtu_prefix + GenerateVTUFileName(field_name, myid);
-      std::ofstream os(os_str);
-      MFEM_VERIFY(os.is_open(),
-                  "Failed to open ofstream " << os_str);
+      std::ofstream os(vtu_prefix + GenerateVTUFileName(field_name, myid));
       qfield.second->SaveVTU(os, pv_data_format, GetCompressionLevel(), field_name);
    }
 
@@ -1008,13 +957,10 @@ void ParaViewDataCollection::Save()
    {
       // Create the main PVTU file
       {
-         std::string os_str = vtu_prefix + GeneratePVTUFileName("data");
-         std::ofstream pvtu_out(os_str);
-         MFEM_VERIFY(pvtu_out.is_open(),
-                     "Failed to open ofstream " << os_str);
+         std::ofstream pvtu_out(vtu_prefix + GeneratePVTUFileName("data"));
          WritePVTUHeader(pvtu_out);
 
-         // Grid function fields and coefficient fields
+         // Grid function fields
          pvtu_out << "<PPointData>\n";
          for (auto &field_it : field_map)
          {
@@ -1025,24 +971,7 @@ void ParaViewDataCollection::Save()
                      << VTKComponentLabels(vec_dim) << " "
                      << "format=\"" << GetDataFormatString() << "\" />\n";
          }
-         for (auto &field_it : coeff_field_map)
-         {
-            int vec_dim = 1;
-            pvtu_out << "<PDataArray type=\"" << GetDataTypeString()
-                     << "\" Name=\"" << field_it.first
-                     << "\" NumberOfComponents=\"" << vec_dim << "\" "
-                     << "format=\"" << GetDataFormatString() << "\" />\n";
-         }
-         for (auto &field_it : vcoeff_field_map)
-         {
-            int vec_dim = field_it.second->GetVDim();
-            pvtu_out << "<PDataArray type=\"" << GetDataTypeString()
-                     << "\" Name=\"" << field_it.first
-                     << "\" NumberOfComponents=\"" << vec_dim << "\" "
-                     << "format=\"" << GetDataFormatString() << "\" />\n";
-         }
          pvtu_out << "</PPointData>\n";
-
          // Element attributes
          pvtu_out << "<PCellData>\n";
          pvtu_out << "\t<PDataArray type=\"Int32\" Name=\"" << "attribute"
@@ -1066,10 +995,8 @@ void ParaViewDataCollection::Save()
          const std::string &q_field_name = q_field.first;
          std::string q_fname = GeneratePVTUPath() + "/"
                                + GeneratePVTUFileName(q_field_name);
-         std::string os_str = col_path + "/" + q_fname;
-         std::ofstream pvtu_out(os_str);
-         MFEM_VERIFY(pvtu_out.is_open(),
-                     "Failed to open ofstream " << os_str);
+
+         std::ofstream pvtu_out(col_path + "/" + q_fname);
          WritePVTUHeader(pvtu_out);
          int vec_dim = q_field.second->GetVDim();
          pvtu_out << "<PPointData>\n";
@@ -1142,8 +1069,7 @@ void ParaViewDataCollection::SaveDataVTU(std::ostream &os, int ref)
    }
    os << " version=\"2.2\" byte_order=\"" << VTKByteOrder() << "\">\n";
    os << "<UnstructuredGrid>\n";
-   mesh->PrintVTU(os,ref,pv_data_format,high_order_output,GetCompressionLevel(),
-                  bdr_output);
+   mesh->PrintVTU(os,ref,pv_data_format,high_order_output,GetCompressionLevel());
 
    // dump out the grid functions as point data
    os << "<PointData >\n";
@@ -1151,21 +1077,8 @@ void ParaViewDataCollection::SaveDataVTU(std::ostream &os, int ref)
    // iterate over all grid functions
    for (FieldMapIterator it=field_map.begin(); it!=field_map.end(); ++it)
    {
-      MFEM_VERIFY(!bdr_output,
-                  "GridFunction output is not supported for "
-                  "ParaViewDataCollection on domain boundary!");
       SaveGFieldVTU(os,ref,it);
    }
-   // save the coefficient functions
-   // iterate over all Coefficient and VectorCoefficient functions
-   for (const auto &kv : coeff_field_map)
-   {
-      SaveCoeffFieldVTU(os, ref, kv.first, *kv.second);
-   }
-   for (const auto &kv : vcoeff_field_map)
-   {
-      SaveVCoeffFieldVTU(os, ref, kv.first, *kv.second);
-   }
    os << "</PointData>\n";
    // close the mesh
    os << "</Piece>\n"; // close the piece open in the PrintVTU method
@@ -1188,6 +1101,7 @@ void ParaViewDataCollection::SaveGFieldVTU(std::ostream &os, int ref_,
       << "format=\"" << GetDataFormatString() << "\" >" << '\n';
    if (vec_dim == 1)
    {
+      // scalar data
       for (int i = 0; i < mesh->GetNE(); i++)
       {
          RefG = GlobGeometryRefiner.Refine(
@@ -1217,131 +1131,11 @@ void ParaViewDataCollection::SaveGFieldVTU(std::ostream &os, int ref_,
          }
       }
    }
-   if (pv_data_format != VTKFormat::ASCII)
+
+   if (IsBinaryFormat())
    {
-      WriteBase64WithSizeAndClear(os, buf, GetCompressionLevel());
-   }
-   os << "</DataArray>" << std::endl;
-}
-
-void ParaViewDataCollection::SaveCoeffFieldVTU(std::ostream &os, int ref_,
-                                               const std::string &name, Coefficient &coeff)
-{
-   RefinedGeometry *RefG;
-   real_t val;
-   std::vector<char> buf;
-   int vec_dim = 1;
-   os << "<DataArray type=\"" << GetDataTypeString()
-      << "\" Name=\"" << name
-      << "\" NumberOfComponents=\"" << vec_dim << "\""
-      << " format=\"" << GetDataFormatString() << "\" >" << '\n';
-   {
-      // scalar data
-      if (!bdr_output)
-      {
-         for (int i = 0; i < mesh->GetNE(); i++)
-         {
-            RefG = GlobGeometryRefiner.Refine(
-                      mesh->GetElementBaseGeometry(i), ref_, 1);
-
-            ElementTransformation *eltrans = mesh->GetElementTransformation(i);
-            const IntegrationRule *ir = &RefG->RefPts;
-            for (int j = 0; j < ir->GetNPoints(); j++)
-            {
-               const IntegrationPoint &ip = ir->IntPoint(j);
-               eltrans->SetIntPoint(&ip);
-               val = coeff.Eval(*eltrans, ip);
-               WriteBinaryOrASCII(os, buf, val, "\n", pv_data_format);
-            }
-         }
-      }
-      else
-      {
-         for (int i = 0; i < mesh->GetNBE(); i++)
-         {
-            RefG = GlobGeometryRefiner.Refine(
-                      mesh->GetBdrElementBaseGeometry(i), ref_, 1);
-
-            ElementTransformation *eltrans = mesh->GetBdrElementTransformation(i);
-            const IntegrationRule *ir = &RefG->RefPts;
-            for (int j = 0; j < ir->GetNPoints(); j++)
-            {
-               const IntegrationPoint &ip = ir->IntPoint(j);
-               eltrans->SetIntPoint(&ip);
-               val = coeff.Eval(*eltrans, ip);
-               WriteBinaryOrASCII(os, buf, val, "\n", pv_data_format);
-            }
-         }
-      }
-   }
-   if (pv_data_format != VTKFormat::ASCII)
-   {
-      WriteBase64WithSizeAndClear(os, buf, GetCompressionLevel());
-   }
-   os << "</DataArray>" << std::endl;
-}
-
-void ParaViewDataCollection::SaveVCoeffFieldVTU(std::ostream &os, int ref_,
-                                                const std::string &name, VectorCoefficient &coeff)
-{
-   RefinedGeometry *RefG;
-   Vector val;
-   std::vector<char> buf;
-   int vec_dim = coeff.GetVDim();
-   os << "<DataArray type=\"" << GetDataTypeString()
-      << "\" Name=\"" << name
-      << "\" NumberOfComponents=\"" << vec_dim << "\""
-      << " format=\"" << GetDataFormatString() << "\" >" << '\n';
-   {
-      // vector data
-      if (!bdr_output)
-      {
-         for (int i = 0; i < mesh->GetNE(); i++)
-         {
-            RefG = GlobGeometryRefiner.Refine(
-                      mesh->GetElementBaseGeometry(i), ref_, 1);
-
-            ElementTransformation *eltrans = mesh->GetElementTransformation(i);
-            const IntegrationRule *ir = &RefG->RefPts;
-            for (int j = 0; j < ir->GetNPoints(); j++)
-            {
-               const IntegrationPoint &ip = ir->IntPoint(j);
-               eltrans->SetIntPoint(&ip);
-               coeff.Eval(val, *eltrans, ip);
-               for (int jj = 0; jj < val.Size(); jj++)
-               {
-                  WriteBinaryOrASCII(os, buf, val(jj), " ", pv_data_format);
-               }
-               if (pv_data_format == VTKFormat::ASCII) { os << '\n'; }
-            }
-         }
-      }
-      else
-      {
-         for (int i = 0; i < mesh->GetNBE(); i++)
-         {
-            RefG = GlobGeometryRefiner.Refine(
-                      mesh->GetBdrElementBaseGeometry(i), ref_, 1);
-
-            ElementTransformation *eltrans = mesh->GetBdrElementTransformation(i);
-            const IntegrationRule *ir = &RefG->RefPts;
-            for (int j = 0; j < ir->GetNPoints(); j++)
-            {
-               const IntegrationPoint &ip = ir->IntPoint(j);
-               eltrans->SetIntPoint(&ip);
-               coeff.Eval(val, *eltrans, ip);
-               for (int jj = 0; jj < val.Size(); jj++)
-               {
-                  WriteBinaryOrASCII(os, buf, val(jj), " ", pv_data_format);
-               }
-               if (pv_data_format == VTKFormat::ASCII) { os << '\n'; }
-            }
-         }
-      }
-   }
-   if (pv_data_format != VTKFormat::ASCII)
-   {
-      WriteBase64WithSizeAndClear(os, buf, GetCompressionLevel());
+      WriteVTKEncodedCompressed(os,buf.data(),buf.size(),GetCompressionLevel());
+      os << '\n';
    }
    os << "</DataArray>" << std::endl;
 }

fem/datacollection.hpp

@@ -133,7 +133,6 @@ private:
 
    /// A collection of named QuadratureFunctions
    typedef NamedFieldsMap<QuadratureFunction> QFieldMap;
-
 public:
    typedef GFieldMap::MapType FieldMapType;
    typedef GFieldMap::iterator FieldMapIterator;
@@ -250,9 +249,10 @@ public:
    { field_map.Deregister(field_name, own_data); }
 
    /// Add a QuadratureFunction to the collection.
-   virtual void RegisterQField(const std::string& field_name,
+   virtual void RegisterQField(const std::string& q_field_name,
                                QuadratureFunction *qf)
-   { q_field_map.Register(field_name, qf, own_data); }
+   { q_field_map.Register(q_field_name, qf, own_data); }
+
 
    /// Remove a QuadratureFunction from the collection
    virtual void DeregisterQField(const std::string& field_name)
@@ -280,13 +280,13 @@ public:
 #endif
 
    /// Check if a QuadratureFunction with the given name is in the collection.
-   bool HasQField(const std::string& field_name) const
-   { return q_field_map.Has(field_name); }
+   bool HasQField(const std::string& q_field_name) const
+   { return q_field_map.Has(q_field_name); }
 
    /// Get a pointer to a QuadratureFunction in the collection.
    /** Returns NULL if @a field_name is not in the collection. */
-   QuadratureFunction *GetQField(const std::string& field_name)
-   { return q_field_map.Get(field_name); }
+   QuadratureFunction *GetQField(const std::string& q_field_name)
+   { return q_field_map.Get(q_field_name); }
 
    /// Get a const reference to the internal field map.
    /** The keys in the map are the field names and the values are pointers to
@@ -302,13 +302,11 @@ public:
 
    /// Get a pointer to the mesh in the collection
    Mesh *GetMesh() { return mesh; }
-
    /// Set/change the mesh associated with the collection
    /** When passed a Mesh, assumes the serial case: MPI rank id is set to 0 and
        MPI num_procs is set to 1.  When passed a ParMesh, MPI info from the
        ParMesh is used to set the DataCollection's MPI rank and num_procs. */
    virtual void SetMesh(Mesh *new_mesh);
-
 #ifdef MFEM_USE_MPI
    /// Set/change the mesh associated with the collection.
    /** For this case, @a comm is used to set the DataCollection's MPI rank id
@@ -371,7 +369,8 @@ public:
    /// Save one field, assuming the collection directory already exists.
    virtual void SaveField(const std::string &field_name);
    /// Save one q-field, assuming the collection directory already exists.
-   virtual void SaveQField(const std::string &field_name);
+   virtual void SaveQField(const std::string &q_field_name);
+
    /// Load the collection. Not implemented in the base class DataCollection.
    virtual void Load(int cycle_ = 0);
 
@@ -408,18 +407,12 @@ public:
 class VisItFieldInfo
 {
 public:
-   std::string association = "";
-   int num_components = 0;
-   int lod = 1;
-   std::string basis = "";
-   int order = -1;
-   VisItFieldInfo() = default;
-   VisItFieldInfo(std::string association_, int num_components_, int lod_ = 1,
-                  std::string basis_ = "", int order_ = -1)
-   {
-      association = association_; num_components = num_components_; lod =lod_;
-      basis = basis_; order = order_;
-   }
+   std::string association;
+   int num_components;
+   int lod;
+   VisItFieldInfo() { association = ""; num_components = 0; lod = 1;}
+   VisItFieldInfo(std::string association_, int num_components_, int lod_ = 1)
+   { association = association_; num_components = num_components_; lod =lod_;}
 };
 
 /// Data collection with VisIt I/O routines
@@ -517,9 +510,7 @@ protected:
    int compression_level = -1;
    bool high_order_output = false;
    bool restart_mode = false;
-   bool bdr_output = false;
    VTKFormat pv_data_format = VTKFormat::BINARY;
-
 public:
    ParaViewDataCollectionBase(const std::string &name, Mesh *mesh);
 
@@ -552,10 +543,6 @@ public:
    /// Reading high-order data requires ParaView 5.5 or later.
    void SetHighOrderOutput(bool high_order_output_);
 
-   /// @brief Configures collection to save only fields evaluated on boundaries of
-   /// the mesh.
-   void SetBoundaryOutput(bool bdr_output_);
-
    /// If compression is enabled, return the compression level, else return 0.
    int GetCompressionLevel() const;
 
@@ -577,6 +564,8 @@ public:
    ///
    /// If restart is enabled, new writes will preserve timestep metadata for any
    /// solutions prior to the currently defined time.
+   ///
+   /// Initially, restart mode is disabled.
    void UseRestartMode(bool restart_mode_);
 };
 
@@ -586,23 +575,11 @@ class ParaViewDataCollection : public ParaViewDataCollectionBase
 private:
    std::fstream pvd_stream;
 
-   /// A collection of named Coefficients and VectorCoefficients
-   using CoeffFieldMap = NamedFieldsMap<Coefficient>;
-   using VCoeffFieldMap = NamedFieldsMap<VectorCoefficient>;
-
-   /** A FieldMap mapping registered names to Coefficient and VectorCoefficient
-       pointers. */
-   CoeffFieldMap coeff_field_map;
-   VCoeffFieldMap vcoeff_field_map;
 protected:
    void WritePVTUHeader(std::ostream &out);
    void WritePVTUFooter(std::ostream &out, const std::string &vtu_prefix);
    void SaveDataVTU(std::ostream &out, int ref);
    void SaveGFieldVTU(std::ostream& out, int ref_, const FieldMapIterator& it);
-   void SaveCoeffFieldVTU(std::ostream& out, int ref_, const std::string &name,
-                          Coefficient &coeff);
-   void SaveVCoeffFieldVTU(std::ostream& out, int ref_, const std::string &name,
-                           VectorCoefficient& coeff);
    const char *GetDataFormatString() const;
    const char *GetDataTypeString() const;
 
@@ -621,25 +598,6 @@ public:
    ParaViewDataCollection(const std::string& collection_name,
                           Mesh *mesh_ = nullptr);
 
-   /// Get a const reference to the internal coefficient-field map.
-   const typename CoeffFieldMap::MapType &GetCoeffFieldMap() const
-   { return coeff_field_map.GetMap(); }
-   const typename VCoeffFieldMap::MapType &GetVCoeffFieldMap() const
-   { return vcoeff_field_map.GetMap(); }
-
-   /// Add a Coefficient or VectorCoefficient to the collection.
-   void RegisterCoeffField(const std::string& field_name, Coefficient *coeff)
-   { coeff_field_map.Register(field_name, coeff, own_data); }
-   void RegisterVCoeffField(const std::string& field_name,
-                            VectorCoefficient *vcoeff)
-   { vcoeff_field_map.Register(field_name, vcoeff, own_data); }
-
-   /// Remove a Coefficient or VectorCoefficient from the collection
-   void DeregisterCoeffField(const std::string& field_name)
-   { coeff_field_map.Deregister(field_name, own_data); }
-   void DeregisterVCoeffField(const std::string& field_name)
-   { vcoeff_field_map.Deregister(field_name, own_data); }
-
    /// Save the collection - the directory name is constructed based on the
    /// cycle value
    void Save() override;

fem/dfem/assemble.hpp

@@ -1,403 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-#pragma once
-
-#include "util.hpp"
-
-namespace mfem::future
-{
-
-/// @brief Assemble element matrix for three dimensional data.
-///
-/// Note: In the below layouts, total_trial_op_dim is > 1 if
-/// there are more than one inputs dependent on the derivative variable.
-///
-/// @param A Memory for one element matrix with layout
-/// [test_ndof, test_vdim, trial_ndof, trial_vdim].
-/// @param fhat Memory to hold the residual computation with layout
-/// [test_vdim, test_op_dim, nqp].
-/// @param qpdc The quadrature point data cache with data layout
-/// [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, nqp].
-/// @param itod Input Trial Operator Dimension array. If the trial
-/// operator is not dependent, the dimension is 0 to indicate that.
-/// @param inputs The input field operator types.
-/// @param output The output field operator types.
-/// @param input_dtqmaps The input DofToQuad maps.
-/// @param output_dtqmap The output DofToQuad maps.
-/// @param scratch_shmem Scratch shared memory for computations.
-/// @param q1d The number of quadrature points in one dimension.
-/// @param td1d The number of trial dofs in one dimension.
-template <typename input_fop_ts, size_t num_inputs, typename output_fop_t>
-MFEM_HOST_DEVICE void assemble_element_mat_t3d(
-   const DeviceTensor<4, real_t>& A,
-   const DeviceTensor<3, real_t>& fhat,
-   const DeviceTensor<5, const real_t>& qpdc,
-   const DeviceTensor<1, const real_t>& itod,
-   const input_fop_ts& inputs,
-   const output_fop_t& output,
-   const std::array<DofToQuadMap, num_inputs>& input_dtqmaps,
-   const DofToQuadMap& output_dtqmap,
-   std::array<DeviceTensor<1>, 6>& scratch_shmem,
-   const int& q1d,
-   const int& td1d)
-{
-   constexpr int dimension = 3;
-
-   // [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, num_qp]
-   const int test_vdim = qpdc.GetShape()[0];
-   const int test_op_dim = qpdc.GetShape()[1];
-   const int trial_vdim = qpdc.GetShape()[2];
-
-   // [num_test_dof, ...]
-   const auto num_test_dof = A.GetShape()[0];
-
-   for (int Jx = 0; Jx < td1d; Jx++)
-   {
-      for (int Jy = 0; Jy < td1d; Jy++)
-      {
-         for (int Jz = 0; Jz < td1d; Jz++)
-         {
-            const int J = Jx + td1d * (Jy + td1d * Jz);
-
-            for (int j = 0; j < trial_vdim; j++)
-            {
-               for (int tv = 0; tv < test_vdim; tv++)
-               {
-                  for (int tod = 0; tod < test_op_dim; tod++)
-                  {
-                     MFEM_FOREACH_THREAD(qx, x, q1d)
-                     {
-                        MFEM_FOREACH_THREAD(qy, y, q1d)
-                        {
-                           MFEM_FOREACH_THREAD(qz, z, q1d)
-                           {
-                              const int q = qx + q1d * (qy + q1d * qz);
-                              fhat(tv, tod, q) = 0.0;
-                           }
-                        }
-                     }
-                  }
-               }
-
-               // MSVC lambda capture workaround
-               [[maybe_unused]] const auto& inputs_ref = inputs;
-
-               int m_offset = 0;
-               for_constexpr<num_inputs>([&](auto s)
-               {
-                  using fop_t = std::decay_t<decltype(get<s>(inputs_ref))>;
-
-                  const int trial_op_dim = static_cast<int>(itod(static_cast<int>(s)));
-                  if (trial_op_dim == 0)
-                  {
-                     // This is inside a lambda so we have to return
-                     // instead of idiomatic 'continue'.
-                     return;
-                  }
-
-                  auto& B = input_dtqmaps[s].B;
-                  auto& G = input_dtqmaps[s].G;
-
-                  if constexpr (is_value_fop<fop_t>::value)
-                  {
-                     MFEM_FOREACH_THREAD(qx, x, q1d)
-                     {
-                        MFEM_FOREACH_THREAD(qy, y, q1d)
-                        {
-                           MFEM_FOREACH_THREAD(qz, z, q1d)
-                           {
-                              const int q = qx + q1d * (qy + q1d * qz);
-
-                              for (int m = 0; m < trial_op_dim; m++)
-                              {
-                                 for (int i = 0; i < test_vdim; i++)
-                                 {
-                                    for (int k = 0; k < test_op_dim; k++)
-                                    {
-                                       const real_t f = qpdc(i, k, j, m + m_offset, q);
-                                       fhat(i, k, q) += f * B(qx, 0, Jx) * B(qy, 0, Jy) * B(qz, 0, Jz);
-                                    }
-                                 }
-                              }
-                           }
-                        }
-                     }
-                  }
-                  else if constexpr (is_gradient_fop<fop_t>::value)
-                  {
-                     MFEM_FOREACH_THREAD(qx, x, q1d)
-                     {
-                        MFEM_FOREACH_THREAD(qy, y, q1d)
-                        {
-                           MFEM_FOREACH_THREAD(qz, z, q1d)
-                           {
-                              const int q = qx + q1d * (qy + q1d * qz);
-                              for (int m = 0; m < trial_op_dim; m++)
-                              {
-                                 for (int i = 0; i < test_vdim; i++)
-                                 {
-                                    for (int k = 0; k < test_op_dim; k++)
-                                    {
-                                       const real_t f = qpdc(i, k, j, m + m_offset, q);
-                                       if (m == 0)
-                                       {
-                                          fhat(i, k, q) += f * G(qx, 0, Jx) * B(qy, 0, Jy) * B(qz, 0, Jz);
-                                       }
-                                       else if (m == 1)
-                                       {
-                                          fhat(i, k, q) += f * B(qx, 0, Jx) * G(qy, 0, Jy) * B(qz, 0, Jz);
-                                       }
-                                       else if (m == 2)
-                                       {
-                                          fhat(i, k, q) += f * B(qx, 0, Jx) * B(qy, 0, Jy) * G(qz, 0, Jz);
-                                       }
-                                    }
-                                 }
-                              }
-                           }
-                        }
-                     }
-                  }
-                  else
-                  {
-#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
-                     MFEM_ABORT("sum factorized sparse matrix assemble routine "
-                                "not implemented for field operator");
-#endif
-                  }
-                  MFEM_SYNC_THREAD;
-                  m_offset += trial_op_dim;
-               });
-
-               auto bvtfhat = Reshape(&A(0, 0, J, j), num_test_dof, test_vdim);
-               map_quadrature_data_to_fields(bvtfhat, fhat, output, output_dtqmap,
-                                             scratch_shmem, dimension, true);
-            }
-         }
-      }
-   }
-}
-
-/// @brief Assemble element matrix for two dimensional data.
-///
-/// Note: In the below layouts, total_trial_op_dim is > 1 if
-/// there are more than one inputs dependent on the derivative variable.
-///
-/// @param A Memory for one element matrix with layout
-/// [test_ndof, test_vdim, trial_ndof, trial_vdim].
-/// @param fhat Memory to hold the residual computation with layout
-/// [test_vdim, test_op_dim, nqp].
-/// @param qpdc The quadrature point data cache with data layout
-/// [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, nqp].
-/// @param itod Input Trial Operator Dimension array. If the trial
-/// operator is not dependent, the dimension is 0 to indicate that.
-/// @param inputs The input field operator types.
-/// @param output The output field operator types.
-/// @param input_dtqmaps The input DofToQuad maps.
-/// @param output_dtqmap The output DofToQuad maps.
-/// @param scratch_shmem Scratch shared memory for computations.
-/// @param q1d The number of quadrature points in one dimension.
-/// @param td1d The number of trial dofs in one dimension.
-template <typename input_fop_ts, size_t num_inputs, typename output_fop_t>
-MFEM_HOST_DEVICE void assemble_element_mat_t2d(
-   const DeviceTensor<4, real_t>& A,
-   const DeviceTensor<3, real_t>& fhat,
-   const DeviceTensor<5, const real_t>& qpdc,
-   const DeviceTensor<1, const real_t>& itod,
-   const input_fop_ts& inputs,
-   const output_fop_t& output,
-   const std::array<DofToQuadMap, num_inputs>& input_dtqmaps,
-   const DofToQuadMap& output_dtqmap,
-   std::array<DeviceTensor<1>, 6>& scratch_shmem,
-   const int& q1d,
-   const int& td1d)
-{
-   constexpr int dimension = 2;
-
-   // [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, num_qp]
-   const int test_vdim = qpdc.GetShape()[0];
-   const int test_op_dim = qpdc.GetShape()[1];
-   const int trial_vdim = qpdc.GetShape()[2];
-
-   // [num_test_dof, ...]
-   const auto num_test_dof = A.GetShape()[0];
-
-   for (int Jx = 0; Jx < td1d; Jx++)
-   {
-      for (int Jy = 0; Jy < td1d; Jy++)
-      {
-         const int J = Jy + Jx * td1d;
-
-         for (int j = 0; j < trial_vdim; j++)
-         {
-            for (int tv = 0; tv < test_vdim; tv++)
-            {
-               for (int tod = 0; tod < test_op_dim; tod++)
-               {
-                  MFEM_FOREACH_THREAD(qx, x, q1d)
-                  {
-                     MFEM_FOREACH_THREAD(qy, y, q1d)
-                     {
-                        const int q = qy + qx * q1d;
-                        fhat(tv, tod, q) = 0.0;
-                     }
-                  }
-               }
-            }
-
-            // MSVC lambda capture workaround
-            [[maybe_unused]] const auto& inputs_ref = inputs;
-
-            int m_offset = 0;
-            for_constexpr<num_inputs>([&](auto s)
-            {
-               using fop_t = std::decay_t<decltype(get<s>(inputs_ref))>;
-
-               const int trial_op_dim = static_cast<int>(itod(static_cast<int>(s)));
-               if (trial_op_dim == 0)
-               {
-                  // This is inside a lambda so we have to return
-                  // instead of idiomatic 'continue'.
-                  return;
-               }
-
-               auto& B = input_dtqmaps[s].B;
-               auto& G = input_dtqmaps[s].G;
-
-               if constexpr (is_value_fop<fop_t>::value)
-               {
-                  MFEM_FOREACH_THREAD(qx, x, q1d)
-                  {
-                     MFEM_FOREACH_THREAD(qy, y, q1d)
-                     {
-                        const int q = qy + qx * q1d;
-                        for (int m = 0; m < trial_op_dim; m++)
-                        {
-                           for (int i = 0; i < test_vdim; i++)
-                           {
-                              for (int k = 0; k < test_op_dim; k++)
-                              {
-                                 const real_t f = qpdc(i, k, j, m + m_offset, q);
-                                 fhat(i, k, q) += f * B(qx, 0, Jx) * B(qy, 0, Jy);
-                              }
-                           }
-                        }
-                     }
-                  }
-               }
-               else if constexpr (is_gradient_fop<fop_t>::value)
-               {
-                  MFEM_FOREACH_THREAD(qx, x, q1d)
-                  {
-                     MFEM_FOREACH_THREAD(qy, y, q1d)
-                     {
-                        const int q = qy + qx * q1d;
-                        for (int m = 0; m < trial_op_dim; m++)
-                        {
-                           for (int i = 0; i < test_vdim; i++)
-                           {
-                              for (int k = 0; k < test_op_dim; k++)
-                              {
-                                 const real_t f = qpdc(i, k, j, m + m_offset, q);
-                                 if (m == 0)
-                                 {
-                                    fhat(i, k, q) += f * B(qx, 0, Jx) * G(qy, 0, Jy);
-                                 }
-                                 else
-                                 {
-                                    fhat(i, k, q) += f * G(qx, 0, Jx) * B(qy, 0, Jy);
-                                 }
-                              }
-                           }
-                        }
-                     }
-                  }
-               }
-               else
-               {
-#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
-                  MFEM_ABORT("sum factorized sparse matrix assemble routine "
-                             "not implemented for field operator");
-#endif
-               }
-               MFEM_SYNC_THREAD;
-               m_offset += trial_op_dim;
-            });
-
-            auto bvtfhat = Reshape(&A(0, 0, J, j), num_test_dof, test_vdim);
-            map_quadrature_data_to_fields(bvtfhat, fhat, output, output_dtqmap,
-                                          scratch_shmem, dimension, true);
-         }
-      }
-   }
-}
-
-/// @brief Assemble element matrix for two or three dimensional data.
-///
-/// Note: In the below layouts, total_trial_op_dim is > 1 if
-/// there are more than one inputs dependent on the derivative variable.
-///
-/// @param A Memory for one element matrix with layout
-/// [test_ndof, test_vdim, trial_ndof, trial_vdim].
-/// @param fhat Memory to hold the residual computation with layout
-/// [test_vdim, test_op_dim, nqp].
-/// @param qpdc The quadrature point data cache with data layout
-/// [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, nqp].
-/// @param itod Input Trial Operator Dimension array. If the trial
-/// operator is not dependent, the dimension is 0 to indicate that.
-/// @param inputs The input field operator types.
-/// @param output The output field operator types.
-/// @param input_dtqmaps The input DofToQuad maps.
-/// @param output_dtqmap The output DofToQuad maps.
-/// @param scratch_shmem Scratch shared memory for computations.
-/// @param dimension The spatial dimension.
-/// @param q1d The number of quadrature points in one dimension.
-/// @param td1d The number of trial dofs in one dimension.
-/// @param use_sum_factorization Indicator if sum factorization is used.
-template <typename input_fop_ts, size_t num_inputs, typename output_fop_t>
-MFEM_HOST_DEVICE void assemble_element_mat_naive(
-   const DeviceTensor<4, real_t>& A,
-   const DeviceTensor<3, real_t>& fhat,
-   const DeviceTensor<5, const real_t>& qpdc,
-   const DeviceTensor<1, const real_t>& itod,
-   const input_fop_ts& inputs,
-   const output_fop_t& output,
-   const std::array<DofToQuadMap, num_inputs>& input_dtqmaps,
-   const DofToQuadMap& output_dtqmap,
-   std::array<DeviceTensor<1>, 6>& scratch_shmem,
-   const int& dimension,
-   const int& q1d,
-   const int& td1d,
-   const bool& use_sum_factorization)
-{
-   if (use_sum_factorization)
-   {
-      if (dimension == 2)
-      {
-         assemble_element_mat_t2d(A, fhat, qpdc, itod, inputs, output,
-                                  input_dtqmaps, output_dtqmap, scratch_shmem, q1d, td1d);
-      }
-      else if (dimension == 3)
-      {
-         assemble_element_mat_t3d(A, fhat, qpdc, itod, inputs, output,
-                                  input_dtqmaps, output_dtqmap, scratch_shmem, q1d, td1d);
-      }
-   }
-   else
-   {
-#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
-      MFEM_ABORT("element matrix assemble not implemented for non tensor "
-                 "product basis");
-#endif
-   }
-}
-
-} // namespace mfem::future

fem/dfem/doperator.hpp

@@ -22,7 +22,6 @@
 #include "interpolate.hpp"
 #include "integrate.hpp"
 #include "qfunction_apply.hpp"
-#include "assemble.hpp"
 
 namespace mfem::future
 {
@@ -31,23 +30,14 @@ namespace mfem::future
 using action_t =
    std::function<void(std::vector<Vector> &, const std::vector<Vector> &, Vector &)>;
 
-/// @brief Type alias for a function that computes the cache for the action of a derivative
-using derivative_setup_t =
-   std::function<void(std::vector<Vector> &, const Vector &)>;
-
 /// @brief Type alias for a function that computes the action of a derivative
 using derivative_action_t =
    std::function<void(std::vector<Vector> &, const Vector &, Vector &)>;
 
-/// @brief Type alias for a function that assembles the SparseMatrix of a
-/// derivative operator
-using assemble_derivative_sparsematrix_callback_t =
-   std::function<void(std::vector<Vector> &, SparseMatrix *&)>;
-
-/// @brief Type alias for a function that assembles the HypreParMatrix of a
+/// @brief Type alias for a function that assembles the sparse matrix of a
 /// derivative operator
 using assemble_derivative_hypreparmatrix_callback_t =
-   std::function<void(std::vector<Vector> &, HypreParMatrix *&)>;
+   std::function<void(std::vector<Vector> &, HypreParMatrix &)>;
 
 /// @brief Type alias for a function that applies the appropriate restriction to
 /// the solution and parameters
@@ -91,8 +81,6 @@ public:
       const std::vector<Vector *> &parameters_l,
       const restriction_callback_t &restriction_callback,
       const std::function<void(Vector &, Vector &)> &prolongation_transpose,
-      const std::vector<assemble_derivative_sparsematrix_callback_t>
-      &assemble_derivative_sparsematrix_callbacks,
       const std::vector<assemble_derivative_hypreparmatrix_callback_t>
       &assemble_derivative_hypreparmatrix_callbacks) :
       Operator(height, width),
@@ -103,8 +91,6 @@ public:
       derivative_actions_transpose(derivative_actions_transpose),
       transpose_direction(transpose_direction),
       prolongation_transpose(prolongation_transpose),
-      assemble_derivative_sparsematrix_callbacks(
-         assemble_derivative_sparsematrix_callbacks),
       assemble_derivative_hypreparmatrix_callbacks(
          assemble_derivative_hypreparmatrix_callbacks)
    {
@@ -170,29 +156,14 @@ public:
       prolongation_transpose(daction_l, result_t);
    };
 
-   /// @brief Assemble the derivative operator into a SparseMatrix.
-   ///
-   /// @param A The SparseMatrix to assemble the derivative operator into. Can
-   /// be an uninitialized object.
-   void Assemble(SparseMatrix *&A)
-   {
-      MFEM_ASSERT(!assemble_derivative_sparsematrix_callbacks.empty(),
-                  "derivative can't be assembled into a SparseMatrix");
-
-      for (const auto &f : assemble_derivative_sparsematrix_callbacks)
-      {
-         f(fields_e, A);
-      }
-   }
-
    /// @brief Assemble the derivative operator into a HypreParMatrix.
    ///
    /// @param A The HypreParMatrix to assemble the derivative operator into. Can
    /// be an uninitialized object.
-   void Assemble(HypreParMatrix *&A)
+   void Assemble(HypreParMatrix &A)
    {
       MFEM_ASSERT(!assemble_derivative_hypreparmatrix_callbacks.empty(),
-                  "derivative can't be assembled into a HypreParMatrix");
+                  "derivative can't be assembled into a matrix");
 
       for (const auto &f : assemble_derivative_hypreparmatrix_callbacks)
       {
@@ -225,10 +196,6 @@ private:
 
    std::function<void(Vector &, Vector &)> prolongation_transpose;
 
-   /// Callbacks that assemble derivatives into a SparseMatrix.
-   std::vector<assemble_derivative_sparsematrix_callback_t>
-   assemble_derivative_sparsematrix_callbacks;
-
    /// Callbacks that assemble derivatives into a HypreParMatrix.
    std::vector<assemble_derivative_hypreparmatrix_callback_t>
    assemble_derivative_hypreparmatrix_callbacks;
@@ -244,8 +211,8 @@ private:
 ///
 /// The operator is constructed with solution fields that it will act on and
 /// parameter fields that define coefficients. Quadrature functions are added by
-/// e.g. using AddDomainIntegrator() which specify how the operator evaluates
-/// those functions and parameters at quadrature points.
+/// e.g. using AddDomainIntegrator() which specify how the operator evaluates f
+/// those functionas and parameters at quadrature points.
 ///
 /// Derivatives can be computed by obtaining a DerivativeOperator using
 /// GetDerivative().
@@ -313,22 +280,6 @@ public:
       }
    }
 
-   /// @brief Add an integrator to the operator.
-   /// Called only from AddDomainIntegrator() and AddBoundaryIntegrator().
-   template <
-      typename entity_t,
-      typename qfunc_t,
-      typename input_t,
-      typename output_t,
-      typename derivative_ids_t>
-   void AddIntegrator(
-      qfunc_t &qfunc,
-      input_t inputs,
-      output_t outputs,
-      const IntegrationRule &integration_rule,
-      const Array<int> &attributes,
-      derivative_ids_t derivative_ids);
-
    /// @brief Add a domain integrator to the operator.
    ///
    /// @param qfunc The quadrature function to be added.
@@ -354,31 +305,6 @@ public:
       const Array<int> &domain_attributes,
       derivative_ids_t derivative_ids = std::make_index_sequence<0> {});
 
-   /// @brief Add a boundary integrator to the operator.
-   ///
-   /// @param qfunc The quadrature function to be added.
-   /// @param inputs Tuple of FieldOperators for the inputs of the quadrature
-   /// function.
-   /// @param outputs Tuple of FieldOperators for the outputs of the quadrature
-   /// function.
-   /// @param integration_rule IntegrationRule to use with this integrator.
-   /// @param boundary_attributes Boundary attributes marker array indicating over
-   /// which attributes this integrator will integrate over.
-   /// @param derivative_ids Derivatives to be made available for this
-   /// integrator.
-   template <
-      typename qfunc_t,
-      typename input_t,
-      typename output_t,
-      typename derivative_ids_t = decltype(std::make_index_sequence<0> {})>
-   void AddBoundaryIntegrator(
-      qfunc_t &qfunc,
-      input_t inputs,
-      output_t outputs,
-      const IntegrationRule &integration_rule,
-      const Array<int> &boundary_attributes,
-      derivative_ids_t derivative_ids = std::make_index_sequence<0> {});
-
    /// @brief Set the parameters for the operator.
    ///
    /// This has to be called before using Mult() or MultTranspose().
@@ -431,34 +357,6 @@ public:
 
       const size_t derivative_idx = FindIdx(derivative_id, fields);
 
-      std::vector<Vector> s_l(solutions_l.size());
-      for (size_t i = 0; i < s_l.size(); i++)
-      {
-         s_l[i] = *sol_l[i];
-      }
-
-      std::vector<Vector> p_l(parameters_l.size());
-      for (size_t i = 0; i < p_l.size(); i++)
-      {
-         p_l[i] = *par_l[i];
-      }
-
-      fields_e.resize(solutions_l.size() + parameters_l.size());
-      restriction_callback(s_l, p_l, fields_e);
-
-      // Dummy
-      Vector dir_l;
-      if (derivative_idx > s_l.size())
-      {
-         dir_l = p_l[derivative_idx - s_l.size()];
-      }
-      else
-      {
-         dir_l = s_l[derivative_idx];
-      }
-
-      derivative_setup_callbacks[derivative_id][0](fields_e, dir_l);
-
       return std::make_shared<DerivativeOperator>(
                 height,
                 GetTrueVSize(fields[derivative_idx]),
@@ -472,7 +370,6 @@ public:
                 par_l,
                 restriction_callback,
                 prolongation_transpose,
-                assemble_derivative_sparsematrix_callbacks[derivative_id],
                 assemble_derivative_hypreparmatrix_callbacks[derivative_id]);
    }
 
@@ -482,14 +379,10 @@ private:
    MultLevel mult_level = TVECTOR;
 
    std::vector<action_t> action_callbacks;
-   std::map<size_t, std::vector<derivative_setup_t>> derivative_setup_callbacks;
    std::map<size_t,
        std::vector<derivative_action_t>> derivative_action_callbacks;
    std::map<size_t,
        std::vector<derivative_action_t>> daction_transpose_callbacks;
-   std::map<size_t,
-       std::vector<assemble_derivative_sparsematrix_callback_t>>
-       assemble_derivative_sparsematrix_callbacks;
    std::map<size_t,
        std::vector<assemble_derivative_hypreparmatrix_callback_t>>
        assemble_derivative_hypreparmatrix_callbacks;
@@ -510,8 +403,6 @@ private:
    std::function<void(Vector &, Vector &)> output_restriction_transpose;
    restriction_callback_t restriction_callback;
 
-   std::map<size_t, Vector> derivative_qp_caches;
-
    std::map<size_t, size_t> assembled_vector_sizes;
 
    bool use_tensor_product_structure = true;
@@ -532,52 +423,7 @@ void DifferentiableOperator::AddDomainIntegrator(
    const Array<int> &domain_attributes,
    derivative_ids_t derivative_ids)
 {
-   AddIntegrator<Entity::Element>(
-      qfunc, inputs, outputs, integration_rule, domain_attributes, derivative_ids);
-}
-
-template <
-   typename qfunc_t,
-   typename input_t,
-   typename output_t,
-   typename derivative_ids_t>
-void DifferentiableOperator::AddBoundaryIntegrator(
-   qfunc_t &qfunc,
-   input_t inputs,
-   output_t outputs,
-   const IntegrationRule &integration_rule,
-   const Array<int> &boundary_attributes,
-   derivative_ids_t derivative_ids)
-{
-
-   if (mesh.GetNFbyType(FaceType::Boundary) != mesh.GetNBE())
-   {
-      MFEM_ABORT("AddBoundaryIntegrator on meshes with interior boundaries is not supported.");
-   }
-   AddIntegrator<Entity::BoundaryElement>(
-      qfunc, inputs, outputs, integration_rule, boundary_attributes, derivative_ids);
-}
-
-template <
-   typename entity_t,
-   typename qfunc_t,
-   typename input_t,
-   typename output_t,
-   typename derivative_ids_t>
-void DifferentiableOperator::AddIntegrator(
-   qfunc_t &qfunc,
-   input_t inputs,
-   output_t outputs,
-   const IntegrationRule &integration_rule,
-   const Array<int> &attributes,
-   derivative_ids_t derivative_ids)
-{
-   if constexpr (!(std::is_same_v<entity_t, Entity::Element> ||
-                   std::is_same_v<entity_t, Entity::BoundaryElement>))
-   {
-      static_assert(dfem::always_false<entity_t>,
-                    "entity type not supported in AddIntegrator");
-   }
+   using entity_t = Entity::Element;
 
    static constexpr size_t num_inputs =
       tuple_size<decltype(inputs)>::value;
@@ -638,44 +484,25 @@ void DifferentiableOperator::AddIntegrator(
       inputs_vdim[i] = get<i>(inputs).vdim;
    });
 
-   const Array<int> *elem_attributes = nullptr;
-   if constexpr (std::is_same_v<entity_t, Entity::Element>)
+
+   Array<int> elem_attributes;
+   elem_attributes.SetSize(mesh.GetNE());
+   for (int i = 0; i < mesh.GetNE(); ++i)
    {
-      elem_attributes = &mesh.GetElementAttributes();
-   }
-   else if constexpr (std::is_same_v<entity_t, Entity::BoundaryElement>)
-   {
-      elem_attributes = &mesh.GetBdrFaceAttributes();
+      elem_attributes[i] = mesh.GetAttribute(i);
    }
 
    const auto output_fop = get<0>(outputs);
    test_space_field_idx = FindIdx(output_fop.GetFieldId(), fields);
 
    bool use_sum_factorization = false;
-   Element::Type entity_element_type;
-   if constexpr (std::is_same_v<entity_t, Entity::Element>)
+   auto entity_element_type =
+      Element::TypeFromGeometry(mesh.GetTypicalElementGeometry());
+   if ((entity_element_type == Element::QUADRILATERAL ||
+        entity_element_type == Element::HEXAHEDRON) &&
+       use_tensor_product_structure == true)
    {
-      entity_element_type =
-         Element::TypeFromGeometry(mesh.GetTypicalElementGeometry());
-
-      if ((entity_element_type == Element::QUADRILATERAL ||
-           entity_element_type == Element::HEXAHEDRON) &&
-          use_tensor_product_structure == true)
-      {
-         use_sum_factorization = true;
-      }
-   }
-   else if constexpr (std::is_same_v<entity_t, Entity::BoundaryElement>)
-   {
-      entity_element_type =
-         Element::TypeFromGeometry(mesh.GetTypicalFaceGeometry());
-
-      if ((entity_element_type == Element::SEGMENT ||
-           entity_element_type == Element::QUADRILATERAL) &&
-          use_tensor_product_structure == true)
-      {
-         use_sum_factorization = true;
-      }
+      use_sum_factorization = true;
    }
 
    ElementDofOrdering element_dof_ordering = ElementDofOrdering::NATIVE;
@@ -713,17 +540,8 @@ void DifferentiableOperator::AddIntegrator(
    prolongation_transpose = get_prolongation_transpose(
                                fields[test_space_field_idx], output_fop, mesh.GetComm());
 
-   int dimension;
-   if constexpr (std::is_same_v<entity_t, Entity::Element>)
-   {
-      dimension = mesh.Dimension();
-   }
-   else if constexpr (std::is_same_v<entity_t, Entity::BoundaryElement>)
-   {
-      dimension = mesh.Dimension() - 1;
-   }
-
-   [[maybe_unused]] const int num_elements = GetNumEntities<entity_t>(mesh);
+   const int dimension = mesh.Dimension();
+   [[maybe_unused]] const int num_elements = GetNumEntities<Entity::Element>(mesh);
    const int num_entities = GetNumEntities<entity_t>(mesh);
    const int num_qp = integration_rule.GetNPoints();
 
@@ -797,12 +615,6 @@ void DifferentiableOperator::AddIntegrator(
          thread_blocks.z = 1;
       }
    }
-   else if (dimension == 1)
-   {
-      thread_blocks.x = q1d;
-      thread_blocks.y = 1;
-      thread_blocks.z = 1;
-   }
 
    action_callbacks.push_back(
       // Explicitly capture everything we need, so we can make explicit choice
@@ -818,7 +630,7 @@ void DifferentiableOperator::AddIntegrator(
          test_vdim,             // int (= output_fop.vdim)
          test_op_dim,           // int (derived from output_fop)
          inputs,                // mfem::future::tuple
-         attributes,            // Array<int>
+         domain_attributes,     // Array<int>
          ir_weights,            // DeviceTensor
          use_sum_factorization, // bool
          input_dtq_maps,        // std::array<DofToQuadMap, num_fields>
@@ -851,13 +663,13 @@ void DifferentiableOperator::AddIntegrator(
                                           action_shmem_info.field_sizes,
                                           num_entities);
 
-      const bool has_attr = attributes.Size() > 0;
-      const auto d_attr = attributes.Read();
-      const auto d_elem_attr = elem_attributes->Read();
+      const bool has_attr = domain_attributes.Size() > 0;
+      const auto d_domain_attr = domain_attributes.Read();
+      const auto d_elem_attr = elem_attributes.Read();
 
       forall([=] MFEM_HOST_DEVICE (int e, void *shmem)
       {
-         if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
+         if (has_attr && !d_domain_attr[d_elem_attr[e] - 1]) { return; }
 
          auto [input_dtq_shmem, output_dtq_shmem, fields_shmem, input_shmem,
                                 residual_shmem, scratch_shmem] =
@@ -904,8 +716,7 @@ void DifferentiableOperator::AddIntegrator(
 
          // print_shared_memory_info(shmem_info);
 
-         Vector direction_e(get_restriction<entity_t>(fields[d_field_idx],
-                                                      element_dof_ordering)->Height());
+         Vector direction_e;
          Vector derivative_action_e(output_e_size);
          derivative_action_e = 0.0;
 
@@ -917,84 +728,24 @@ void DifferentiableOperator::AddIntegrator(
          }
          const auto input_is_dependent = it->second;
 
-         // Trial operator dimension for each input.
-         // The trial operator dimension is set for each input that is
-         // dependent and if it is independent the dimension is 0.
-         Vector inputs_trial_op_dim(num_inputs);
-         int total_trial_op_dim = 0;
-         {
-            auto itod = Reshape(inputs_trial_op_dim.HostReadWrite(), num_inputs);
-            int idx = 0;
-            for_constexpr<num_inputs>([&](auto s)
-            {
-               if (!input_is_dependent[s])
-               {
-                  itod(idx) = 0;
-               }
-               else
-               {
-                  // TODO: BUG! Make this a general function that works for all kinds of inputs.
-                  itod(idx) = input_size_on_qp[s] / get<s>(inputs).vdim;
-               }
-               total_trial_op_dim += static_cast<int>(itod(idx));
-               idx++;
-            });
-         }
-
-         // First Input index of the derivative
-         const size_t d_input_idx = [d_field_idx, &input_to_field]
-         {
-            for (size_t i = 0; i < input_to_field.size(); i++)
-            {
-               if (input_to_field[i] == d_field_idx)
-               {
-                  return i;
-               }
-            }
-            return size_t(SIZE_MAX);
-         }();
-
-         const int trial_vdim = GetVDim(fields[d_field_idx]);
-         const int num_trial_dof =
-            get_restriction<entity_t>(fields[d_field_idx], element_dof_ordering)->Height() /
-            inputs_vdim[d_input_idx] / num_entities;
-         const int num_trial_dof_1d =
-            input_dtq_maps[d_input_idx].B.GetShape()[DofToQuadMap::Index::DOF];
-
-         Vector Ae_mem(num_test_dof * test_vdim * num_trial_dof * trial_vdim *
-                       num_entities);
-         Ae_mem = 0.0;
-
-         // Quadrature point local derivative cache for each element, with data
-         // layout:
-         // [test_vdim, test_op_dim, trial_vdim, trial_op_dim, qp, num_entities].
-         derivative_qp_caches[derivative_id] = Vector(test_vdim * test_op_dim *
-                                                      trial_vdim *
-                                                      total_trial_op_dim * num_qp * num_entities);
-         // Create local references for MSVC lambda capture compatibility
-         auto& fields_ref = this->fields;
-         auto& derivative_qp_caches_ref = this->derivative_qp_caches[derivative_id];
-
-         // In each of the callbacks we're saving the derivatives in the quadrature point
-         // caches. This trades memory with computational effort but also minimizes
-         // data movement on each multiplication of the gradient with a directional
-         // vector.
-         derivative_setup_callbacks[derivative_id].push_back(
+         derivative_action_callbacks[derivative_id].push_back(
             [
                // capture by copy:
                dimension,             // int
                num_entities,          // int
+               num_test_dof,          // int
                num_qp,                // int
                q1d,                   // int
                test_vdim,             // int (= output_fop.vdim)
                test_op_dim,           // int (derived from output_fop)
                inputs,                // mfem::future::tuple
-               attributes,     // Array<int>
+               domain_attributes,     // Array<int>
                ir_weights,            // DeviceTensor
                use_sum_factorization, // bool
                input_dtq_maps,        // std::array<DofToQuadMap, num_fields>
                output_dtq_maps,       // std::array<DofToQuadMap, num_fields>
                input_to_field,        // std::array<int, s>
+               output_fop,            // class derived from FieldOperator
                qfunc,                 // qfunc_t
                thread_blocks,         // ThreadBlocks
                shmem_cache,           // Vector (local)
@@ -1002,37 +753,35 @@ void DifferentiableOperator::AddIntegrator(
                // TODO: make this Array<int> a member of the DifferentiableOperator
                //       and capture it by ref.
                elem_attributes,       // Array<int>
-               element_dof_ordering,  // ElementDofOrdering
 
+               input_is_dependent,    // std::array<bool, num_inputs>
                direction,             // FieldDescriptor
                direction_e,           // Vector
+               derivative_action_e,   // Vector
+               element_dof_ordering,  // ElementDofOrdering
                da_size_on_qp,         // int
 
-               total_trial_op_dim,
-               trial_vdim,
-               inputs_trial_op_dim,
-
                // capture by ref:
-               &qpdc_mem = derivative_qp_caches_ref
-            ](std::vector<Vector> &f_e, const Vector &dir_l) mutable
+               &or_transpose
+            ](
+               std::vector<Vector> &f_e, const Vector &dir_l,
+               Vector &der_action_l) mutable
          {
             restriction<entity_t>(direction, dir_l, direction_e,
                                   element_dof_ordering);
+            auto ye = Reshape(derivative_action_e.ReadWrite(), num_test_dof,
+                              test_vdim, num_entities);
             auto wrapped_fields_e = wrap_fields(f_e, shmem_info.field_sizes,
                                                 num_entities);
             auto wrapped_direction_e = Reshape(direction_e.ReadWrite(),
                                                shmem_info.direction_size,
                                                num_entities);
 
-            auto qpdc = Reshape(qpdc_mem.ReadWrite(), test_vdim, test_op_dim,
-                                trial_vdim, total_trial_op_dim, num_qp, num_entities);
-
-            auto itod = Reshape(inputs_trial_op_dim.Read(), num_inputs);
-
-            const auto d_elem_attr = elem_attributes->Read();
-            const bool has_attr = attributes.Size() > 0;
-            const auto d_domain_attr = attributes.Read();
+            const auto d_elem_attr = elem_attributes.Read();
+            const bool has_attr = domain_attributes.Size() > 0;
+            const auto d_domain_attr = domain_attributes.Read();
 
+            derivative_action_e = 0.0;
             forall([=] MFEM_HOST_DEVICE (int e, real_t *shmem)
             {
                if (has_attr && !d_domain_attr[d_elem_attr[e] - 1]) { return; }
@@ -1050,104 +799,20 @@ void DifferentiableOperator::AddIntegrator(
                   inputs, ir_weights, scratch_shmem, dimension,
                   use_sum_factorization);
 
+               // TODO: Probably redundant
                set_zero(shadow_shmem);
 
-               auto qpdc_e = Reshape(&qpdc(0, 0, 0, 0, 0, e), test_vdim, test_op_dim,
-                                     trial_vdim, total_trial_op_dim, num_qp);
-               call_qfunction_derivative<qf_param_ts>(
-                  qfunc, input_shmem, shadow_shmem, residual_shmem, qpdc_e, itod, da_size_on_qp,
-                  q1d, dimension, use_sum_factorization);
-            }, num_entities, thread_blocks, shmem_info.total_size,
-            shmem_cache.ReadWrite());
-         });
-
-         // The derivative action only uses the quadrature point caches and applies
-         // them to an input vector before integrating with the desired trial operator.
-         derivative_action_callbacks[derivative_id].push_back(
-            [
-               // capture by copy:
-               dimension,             // int
-               num_entities,          // int
-               num_test_dof,          // int
-               num_qp,                // int
-               q1d,                   // int
-               test_vdim,             // int (= output_fop.vdim)
-               test_op_dim,           // int (derived from output_fop)
-               inputs,                // mfem::future::tuple
-               attributes,            // Array<int>
-               ir_weights,            // DeviceTensor
-               use_sum_factorization, // bool
-               input_dtq_maps,        // std::array<DofToQuadMap, num_fields>
-               output_dtq_maps,       // std::array<DofToQuadMap, num_fields>
-               output_fop,            // class derived from FieldOperator
-               thread_blocks,         // ThreadBlocks
-               shmem_cache,           // Vector (local)
-               shmem_info,            // SharedMemoryInfo
-               // TODO: make this Array<int> a member of the DifferentiableOperator
-               //       and capture it by ref.
-               elem_attributes,       // Array<int>
-
-               input_is_dependent,    // std::array<bool, num_inputs>
-               direction,             // FieldDescriptor
-               direction_e,           // Vector
-               derivative_action_e,   // Vector
-               element_dof_ordering,  // ElementDofOrdering
-               inputs_trial_op_dim,
-               total_trial_op_dim,
-               trial_vdim,
-               // capture by ref:
-               &qpdc_mem = derivative_qp_caches_ref,
-               &or_transpose
-            ](
-               std::vector<Vector> &f_e, const Vector &dir_l,
-               Vector &der_action_l) mutable
-         {
-            restriction<entity_t>(direction, dir_l, direction_e,
-                                  element_dof_ordering);
-            auto ye = Reshape(derivative_action_e.ReadWrite(), num_test_dof,
-                              test_vdim, num_entities);
-            auto wrapped_fields_e = wrap_fields(f_e, shmem_info.field_sizes,
-                                                num_entities);
-            auto wrapped_direction_e = Reshape(direction_e.ReadWrite(),
-                                               shmem_info.direction_size,
-                                               num_entities);
-
-            auto qpdc = Reshape(qpdc_mem.Read(), test_vdim, test_op_dim,
-                                trial_vdim, total_trial_op_dim, num_qp, num_entities);
-
-            auto itod = Reshape(inputs_trial_op_dim.Read(), num_inputs);
-
-            const bool has_attr = attributes.Size() > 0;
-            const auto d_attr = attributes.Read();
-            const auto d_elem_attr = elem_attributes->Read();
-
-            derivative_action_e = 0.0;
-            forall([=] MFEM_HOST_DEVICE (int e, real_t *shmem)
-            {
-               if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
-
-               auto [input_dtq_shmem, output_dtq_shmem, fields_shmem,
-                                      direction_shmem, input_shmem,
-                                      shadow_shmem_, residual_shmem,
-                                      scratch_shmem] =
-                        unpack_shmem(shmem, shmem_info, input_dtq_maps, output_dtq_maps,
-                                     wrapped_fields_e, wrapped_direction_e, num_qp, e);
-               auto &shadow_shmem = shadow_shmem_;
-
                map_direction_to_quadrature_data_conditional(
                   shadow_shmem, direction_shmem, input_dtq_shmem, inputs,
                   ir_weights, scratch_shmem, input_is_dependent, dimension,
                   use_sum_factorization);
 
+               call_qfunction_derivative_action<qf_param_ts>(
+                  qfunc, input_shmem, shadow_shmem, residual_shmem,
+                  da_size_on_qp, num_qp, q1d, dimension, use_sum_factorization);
+
                auto fhat = Reshape(&residual_shmem(0, 0), test_vdim,
                                    test_op_dim, num_qp);
-
-               auto qpdce = Reshape(&qpdc(0, 0, 0, 0, 0, e), test_vdim, test_op_dim,
-                                    trial_vdim, total_trial_op_dim, num_qp);
-
-               apply_qpdc(fhat, shadow_shmem, qpdce, itod, q1d, dimension,
-                          use_sum_factorization);
-
                auto y = Reshape(&ye(0, 0, e), num_test_dof, test_vdim);
                map_quadrature_data_to_fields(
                   y, fhat, output_fop, output_dtq_shmem[0],
@@ -1156,246 +821,6 @@ void DifferentiableOperator::AddIntegrator(
             shmem_cache.ReadWrite());
             or_transpose(derivative_action_e, der_action_l);
          });
-
-         assemble_derivative_sparsematrix_callbacks[derivative_id].push_back(
-            [
-               // capture by copy:
-               dimension,             // int
-               num_entities,          // int
-               num_test_dof,          // int
-               num_qp,                // int
-               q1d,                   // int
-               test_vdim,             // int (= output_fop.vdim)
-               test_op_dim,           // int (derived from output_fop)
-               inputs,                // mfem::future::tuple
-               attributes,     // Array<int>
-               use_sum_factorization, // bool
-               input_dtq_maps,        // std::array<DofToQuadMap, num_fields>
-               output_dtq_maps,       // std::array<DofToQuadMap, num_fields>
-               input_to_field,        // std::array<int, s>
-               output_fop,            // class derived from FieldOperator
-               thread_blocks,         // ThreadBlocks
-               shmem_cache,           // Vector (local)
-               shmem_info,            // SharedMemoryInfo
-               // TODO: make this Array<int> a member of the DifferentiableOperator
-               //       and capture it by ref.
-               elem_attributes,       // Array<int>
-
-               input_is_dependent,    // std::array<bool, num_inputs>
-               direction_e,           // Vector
-               total_trial_op_dim,
-               trial_vdim,
-               num_trial_dof,
-               num_trial_dof_1d,
-               inputs_trial_op_dim,
-               Ae_mem,
-               output_to_field,
-
-               // capture by ref:
-               &qpdc_mem = derivative_qp_caches_ref,
-               &fields = fields_ref
-            ](std::vector<Vector> &f_e, SparseMatrix *&A) mutable
-         {
-            auto wrapped_fields_e = wrap_fields(f_e, shmem_info.field_sizes,
-                                                num_entities);
-            auto wrapped_direction_e = Reshape(direction_e.ReadWrite(),
-                                               shmem_info.direction_size,
-                                               num_entities);
-
-            auto qpdc = Reshape(qpdc_mem.Read(), test_vdim, test_op_dim,
-                                trial_vdim, total_trial_op_dim, num_qp, num_entities);
-
-            auto itod = Reshape(inputs_trial_op_dim.Read(), num_inputs);
-
-            auto Ae = Reshape(Ae_mem.ReadWrite(), num_test_dof, test_vdim, num_trial_dof,
-                              trial_vdim, num_entities);
-
-            const auto d_elem_attr = elem_attributes->Read();
-            const bool has_attr = attributes.Size() > 0;
-            const auto d_domain_attr = attributes.Read();
-
-            forall([=] MFEM_HOST_DEVICE (int e, real_t *shmem)
-            {
-               if (has_attr && !d_domain_attr[d_elem_attr[e] - 1]) { return; }
-
-               auto [input_dtq_shmem, output_dtq_shmem, fields_shmem,
-                                      direction_shmem, input_shmem,
-                                      shadow_shmem_, residual_shmem,
-                                      scratch_shmem] =
-                        unpack_shmem(shmem, shmem_info, input_dtq_maps, output_dtq_maps,
-                                     wrapped_fields_e, wrapped_direction_e, num_qp, e);
-
-               auto fhat = Reshape(&residual_shmem(0, 0), test_vdim, test_op_dim, num_qp);
-               auto Aee = Reshape(&Ae(0, 0, 0, 0, e), num_test_dof, test_vdim, num_trial_dof,
-                                  trial_vdim);
-               auto qpdce = Reshape(&qpdc(0, 0, 0, 0, 0, e), test_vdim, test_op_dim,
-                                    trial_vdim, total_trial_op_dim, num_qp);
-               assemble_element_mat_naive(Aee, fhat, qpdce, itod, inputs, output_fop,
-                                          input_dtq_shmem, output_dtq_shmem[0], scratch_shmem, dimension, q1d,
-                                          num_trial_dof_1d, use_sum_factorization);
-            }, num_entities, thread_blocks, shmem_info.total_size,
-            shmem_cache.ReadWrite());
-
-            FieldDescriptor *trial_field = nullptr;
-            for (size_t s = 0; s < num_inputs; s++)
-            {
-               if (input_is_dependent[s])
-               {
-                  trial_field = &fields[input_to_field[s]];
-               }
-            }
-
-            auto trial_fes = *std::get_if<const ParFiniteElementSpace *>
-                             (&trial_field->data);
-            auto test_fes = *std::get_if<const ParFiniteElementSpace *>
-                            (&fields[output_to_field[0]].data);
-
-            A = new SparseMatrix(test_fes->GetVSize(), trial_fes->GetVSize());
-
-            auto tmp = Reshape(Ae_mem.HostReadWrite(), num_test_dof * test_vdim,
-                               num_trial_dof * trial_vdim, num_entities);
-            for (int e = 0; e < num_entities; e++)
-            {
-               DenseMatrix Aee(&tmp(0, 0, e), num_test_dof * test_vdim,
-                               num_trial_dof * trial_vdim);
-
-               Array<int> test_vdofs, trial_vdofs;
-               test_fes->GetElementVDofs(e, test_vdofs);
-               trial_fes->GetElementVDofs(e, trial_vdofs);
-
-               if (use_sum_factorization)
-               {
-                  Array<int> test_vdofs_mapped(test_vdofs.Size());
-
-                  const Array<int> &test_dofmap =
-                     dynamic_cast<const TensorBasisElement&>(*test_fes->GetFE(0)).GetDofMap();
-
-                  if (test_dofmap.Size() == 0)
-                  {
-                     test_vdofs_mapped = test_vdofs;
-                  }
-                  else
-                  {
-                     MFEM_ASSERT(test_dofmap.Size() == num_test_dof,
-                                 "internal error: dof map of the test space does not "
-                                 "match previously determined number of test space dofs");
-
-                     for (int vd = 0; vd < test_vdim; vd++)
-                     {
-                        for (int i = 0; i < num_test_dof; i++)
-                        {
-                           test_vdofs_mapped[i + vd * num_test_dof] =
-                              test_vdofs[test_dofmap[i] + vd * num_test_dof];
-                        }
-                     }
-                  }
-
-                  Array<int> trial_vdofs_mapped(trial_vdofs.Size());
-                  const Array<int> &trial_dofmap =
-                     dynamic_cast<const TensorBasisElement&>(*trial_fes->GetFE(0)).GetDofMap();
-
-                  if (trial_dofmap.Size() == 0)
-                  {
-                     trial_vdofs_mapped = trial_vdofs;
-                  }
-                  else
-                  {
-                     MFEM_ASSERT(trial_dofmap.Size() == num_trial_dof,
-                                 "internal error: dof map of the test space does not "
-                                 "match previously determined number of test space dofs");
-
-                     for (int vd = 0; vd < trial_vdim; vd++)
-                     {
-                        for (int i = 0; i < num_trial_dof; i++)
-                        {
-                           trial_vdofs_mapped[i + vd * num_trial_dof] =
-                              trial_vdofs[trial_dofmap[i] + vd * num_trial_dof];
-                        }
-                     }
-                  }
-
-                  A->AddSubMatrix(test_vdofs_mapped, trial_vdofs_mapped, Aee, 1);
-               }
-               else
-               {
-                  A->AddSubMatrix(test_vdofs, trial_vdofs, Aee, 1);
-               }
-            }
-            A->Finalize();
-         });
-
-         // Create local references for MSVC lambda capture compatibility
-         auto& assemble_derivative_sparsematrix_callbacks_ref =
-            this->assemble_derivative_sparsematrix_callbacks[derivative_id];
-
-         assemble_derivative_hypreparmatrix_callbacks[derivative_id].push_back(
-            [
-               input_is_dependent,
-               input_to_field,
-               output_to_field,
-               &spmatcb = assemble_derivative_sparsematrix_callbacks_ref,
-               &fields = fields_ref
-            ](std::vector<Vector> &f_e, HypreParMatrix *&A) mutable
-         {
-            SparseMatrix *spmat = nullptr;
-            for (const auto &f : spmatcb)
-            {
-               f(f_e, spmat);
-            }
-
-            if (spmat == nullptr)
-            {
-               MFEM_ABORT("internal error");
-            }
-
-            bool same_test_and_trial = false;
-            for (size_t s = 0; s < num_inputs; s++)
-            {
-               if (input_is_dependent[s])
-               {
-                  if (output_to_field[0] == input_to_field[s])
-                  {
-                     same_test_and_trial = true;
-                     break;
-                  }
-               }
-            }
-
-            FieldDescriptor *trial_field = nullptr;
-            for (size_t s = 0; s < num_inputs; s++)
-            {
-               if (input_is_dependent[s])
-               {
-                  trial_field = &fields[input_to_field[s]];
-               }
-            }
-
-            auto trial_fes = *std::get_if<const ParFiniteElementSpace *>
-                             (&trial_field->data);
-            auto test_fes = *std::get_if<const ParFiniteElementSpace *>
-                            (&fields[output_to_field[0]].data);
-
-            if (same_test_and_trial)
-            {
-               HypreParMatrix tmp(test_fes->GetComm(),
-                                  test_fes->GlobalVSize(),
-                                  test_fes->GetDofOffsets(),
-                                  spmat);
-               A = RAP(&tmp, test_fes->Dof_TrueDof_Matrix());
-            }
-            else
-            {
-               HypreParMatrix tmp(test_fes->GetComm(),
-                                  test_fes->GlobalVSize(),
-                                  trial_fes->GlobalVSize(),
-                                  test_fes->GetDofOffsets(),
-                                  trial_fes->GetDofOffsets(),
-                                  spmat);
-               A = RAP(test_fes->Dof_TrueDof_Matrix(), &tmp,
-                       trial_fes->Dof_TrueDof_Matrix());
-            }
-            delete spmat;
-         });
       }, derivative_ids);
    }
 }