Merge remote-tracking branch 'origin/master' into bravais-dev

# Conflicts:
#	fem/fe_coll.cpp
#	fem/geom.hpp
#	fem/intrules.cpp
#	mesh/mesh.cpp
#	mesh/mesh.hpp
#	mesh/pyramid.cpp
#	mesh/pyramid.hpp

.gitlab-ci.yml

@@ -9,91 +9,550 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
+# DESCRIPTION:
+###############################################################################
 # General GitLab pipelines configurations for supercomputers and Linux clusters
-# at Lawrence Livermore National Laboratory (LLNL). This entire pipeline is
-# LLNL-specific!
-
-include:
-  - project: 'lc-templates/id_tokens'
-    file: 'id_tokens.yml'
-
-# The pipeline is divided into stages. Usually, jobs in a given stage wait for
-# the preceding stages to complete before to start. However, we sometimes use
-# the "needs" keyword and express the DAG of jobs for more efficiency.
-# - We use setup and setup_baseline phases to download content outside of mfem
-#   directory.
-# - Allocate/Release is where Dane resource are allocated/released once for all.
-# - Build and Test is where we build and MFEM for multiple toolchains.
-# - Baseline_checks gathers baseline-type test suites execution
-# - Baseline_publish, only available on master, allows to update baseline
-#   results
-stages:
-  - sub-pipelines
+# at Lawrence Livermore National Laboratory (LLNL).
+# This entire pipeline is LLNL-specific
+#
+# Important note: This file is a template provided by llnl/radiuss-shared-ci.
+# Remains to set variable values, change the reference to the radiuss-shared-ci
+# repo, opt-in and out optional features. The project can then extend it with
+# additional stages.
+#
+# In addition, each project should copy over and complete:
+# - .gitlab/custom-jobs-and-variables.yml
+# - .gitlab/subscribed-pipelines.yml
+#
+# The jobs should be specified in a file local to the project,
+# - .gitlab/jobs/${CI_MACHINE}.yml
+# or generated (see LLNL/Umpire for an example).
+###############################################################################
+#   MAP OF GITLAB CI
+#######################
+#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+# File dependencies: direct, through jobs, through variables
+#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+# .gitlab-ci.yml
+#  ├── .build-and-test  [job]
+#  │    ├── .gitlab/custom-jobs-and-variables.yml
+#  │    │    ├── .custom_job          [job]
+#  │    │    ├── .reproducer_vars     [job]
+#  │    │    ├── .report_job_success  [job]
+#  │    │    │    └── .gitlab/scripts/report_build_and_test    [script]
+#  │    │    │         ├── .gitlab/scripts/safe_create_rundir  [script]
+#  │    │    │         └── .gitlab/scripts/git_try_to_push     [script]
+#  │    │    ├── .report_job_failure  [job]
+#  │    │    │    └── .gitlab/scripts/report_build_and_test    [script]
+#  │    │    │         ├── .gitlab/scripts/safe_create_rundir  [script]
+#  │    │    │         └── .gitlab/scripts/git_try_to_push     [script]
+#  │    │    └── JOB_CMD              [var]
+#  │    │         └── tests/gitlab/build_and_test            [script]
+#  │    │              └── tests/gitlab/get_mfem_uberenv     [script]
+#  │    ├── <radiuss-shared-ci>/pipelines/matrix.yml  [conditional]
+#  │    │    ├── .on_matrix               [job]
+#  │    │    ├── .matrix_reproducer_init  [job]
+#  │    │    ├── .matrix_reproducer_vars  [job]
+#  │    │    ├── .matrix_reproducer_job   [job]
+#  │    │    ├── .matrix_job_command      [job]
+#  │    │    └── .job_on_matrix           [job]
+#  │    ├── <radiuss-shared-ci>/pipelines/dane.yml  [conditional]
+#  │    │    ├── .on_dane               [job]
+#  │    │    ├── .dane_reproducer_init  [job]
+#  │    │    ├── .dane_reproducer_vars  [job]
+#  │    │    ├── .dane_reproducer_job   [job]
+#  │    │    ├── .dane_job_command      [job]
+#  │    │    ├── .job_on_dane           [job]
+#  │    │    ├── allocate_resources     [job]
+#  │    │    └── release_resources      [job]
+#  │    ├── <radiuss-shared-ci>/pipelines/tioga.yml  [conditional]
+#  │    │    ├── .on_tioga               [job]
+#  │    │    ├── .tioga_reproducer_init  [job]
+#  │    │    ├── .tioga_reproducer_vars  [job]
+#  │    │    ├── .tioga_reproducer_job   [job]
+#  │    │    ├── .tioga_job_command      [job]
+#  │    │    ├── .job_on_tioga           [job]
+#  │    │    ├── allocate_resources      [job]
+#  │    │    └── release_resources       [job]
+#  │    ├── <artifact>/matrix-jobs.yml  [conditional, from 'generate-job-lists']
+#  │    │    ├── .gitlab/jobs/matrix.yml
+#  │    │    │    ├── .matrix_reproducer_vars      [job]
+#  │    │    │    ├── setup                        [job]
+#  │    │    │    │    └── ./tests/gitlab/build_and_test_setup  [script]
+#  │    │    │    ├── opt_mpi_cuda_gcc             [job]
+#  │    │    │    └── opt_mpi_cuda_hypre_cuda_gcc  [job]
+#  │    │    └── .gitlab/jobs/matrix-reports.yml  [used conditionally]
+#  │    │         ├── report_job_success
+#  │    │         └── report_job_failure
+#  │    ├── <artifact>/dane-jobs.yml    [conditional, from 'generate-job-lists']
+#  │    │    ├── .gitlab/jobs/dane.yml
+#  │    │    │    ├── .dane_reproducer_vars    [job]
+#  │    │    │    ├── setup                    [job]
+#  │    │    │    │    └── ./tests/gitlab/build_and_test_setup  [script]
+#  │    │    │    ├── debug_ser_gcc_10         [job]
+#  │    │    │    ├── debug_par_gcc_10         [job]
+#  │    │    │    ├── opt_ser_gcc_10           [job]
+#  │    │    │    ├── opt_par_gcc_10           [job]
+#  │    │    │    ├── opt_par_gcc_10_sundials  [job]
+#  │    │    │    ├── opt_par_gcc_10_petsc     [job]
+#  │    │    │    └── opt_par_gcc_10_pumi      [job]
+#  │    │    └── .gitlab/jobs/dane-reports.yml    [used conditionally]
+#  │    │         ├── report_job_success
+#  │    │         └── report_job_failure
+#  │    └── <artifact>/tioga-jobs.yml   [conditional, from 'generate-job-lists']
+#  │         ├── .gitlab/jobs/tioga.yml
+#  │         │    ├── .tioga_reproducer_vars  [job]
+#  │         │    ├── setup                   [job]
+#  │         │    │    └── ./tests/gitlab/build_and_test_setup  [script]
+#  │         │    └── cce_16_0_1              [job]
+#  │         └── .gitlab/jobs/tioga-reports.yml   [used conditionally]
+#  │              ├── report_job_success
+#  │              └── report_job_failure
+#  └── .gitlab/subscribed-pipelines.yml
+#       ├── .machine-check         [job]
+#       ├── generate-job-lists     [job]
+#       ├── dane-up-check          [job]
+#       ├── dane-build-and-test    [job]
+#       ├── dane-baseline          [job]
+#       │    └── .gitlab/dane-baseline.yml
+#       │         ├── .on_dane                       [job]
+#       │         ├── baselinecheck_mfem_intel_dane  [job]
+#       │         │    └── .gitlab/scripts/baseline            [script]
+#       │         ├── cleanup                        [job]
+#       │         ├── report_baseline                [job]
+#       │         │    ├── .gitlab/scripts/safe_create_rundir  [script]
+#       │         │    └── .gitlab/scripts/git_try_to_push     [script]
+#       │         ├── baselinepublish_mfem_dane      [job]
+#       │         │    └── .gitlab/scripts/rebaseline          [script]
+#       │         ├── .gitlab/custom-jobs-and-variables.yml
+#       │         │    └── <same as above: see .gitlab-ci.yml/.build-and-test>
+#       │         └── .gitlab/configs/setup-baseline.yml
+#       │              └── setup_baseline            [job]
+#       ├── tioga-up-check         [job]
+#       ├── tioga-build-and-test   [job]
+#       ├── matrix-up-check        [job]
+#       └── matrix-build-and-test  [job]
+#
+#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+# File tree hierarchy with file contents highlights
+#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+# In addition to the files in the MFEM repo, the Gitlab CI uses files from the
+# radiuss/radiuss-shared-ci project, see below, after the <mfem-root> tree.
+#
+# <mfem root>
+#  ├── .gitlab-ci.yml  [this file]
+#  │    ├── <jobs>
+#  │    │    └── .build-and-test
+#  │    ├── <included files>
+#  │    │    ├── .gitlab/subscribed-pipelines.yml
+#  │    │    ├── .gitlab/custom-jobs-and-variables.yml  [by ".build-and-test"]
+#  │    │    ├── <artifact>  [by ".build-and-test"]
+#  │    │    │    ├── artifact: '${CI_MACHINE}-jobs.yml'
+#  │    │    │    └── job:      'generate-job-lists'
+#  │    │    └── <external>  [by ".build-and-test"]
+#  │    │         ├── project: 'radiuss/radiuss-shared-ci'
+#  │    │         ├── ref:     'v2025.09.1'
+#  │    │         └── file:    'pipelines/${CI_MACHINE}.yml'
+#  │    └── <defined variables>
+#  │         ├── CUSTOM_CI_BUILDS_DIR
+#  │         ├── USER_CI_TOP_DIR
+#  │         ├── SHARED_REPOS_DIR
+#  │         ├── AUTOTEST_ROOT
+#  │         ├── MFEM_DATA_DIR
+#  │         ├── AUTOTEST
+#  │         ├── AUTOTEST_COMMIT
+#  │         ├── REBASELINE
+#  │         ├── GITHUB_PROJECT_NAME
+#  │         └── GITHUB_PROJECT_ORG
+#  ├── .gitlab
+#  │    ├── configs
+#  │    │    └── setup-baseline.yml
+#  │    │         ├── <jobs>
+#  │    │         │    └── setup_baseline
+#  │    │         └── <used variables>
+#  │    │              ├── MACHINE_NAME
+#  │    │              ├── REBASELINE
+#  │    │              ├── AUTOTEST
+#  │    │              ├── AUTOTEST_COMMIT
+#  │    │              ├── BUILD_ROOT
+#  │    │              ├── TPLS_REPO
+#  │    │              ├── TESTS_REPO
+#  │    │              ├── AUTOTEST_ROOT
+#  │    │              └── AUTOTEST_REPO
+#  │    ├── jobs
+#  │    │    ├── matrix-reports.yml
+#  │    │    │    ├── <jobs>
+#  │    │    │    │    ├── report_job_success
+#  │    │    │    │    └── report_job_failure
+#  │    │    │    └── <used jobs>
+#  │    │    │         ├── .on_matrix
+#  │    │    │         ├── .report_job_success
+#  │    │    │         └── .report_job_failure
+#  │    │    ├── matrix.yml
+#  │    │    │    ├── <jobs>
+#  │    │    │    │    ├── .matrix_reproducer_vars
+#  │    │    │    │    ├── setup
+#  │    │    │    │    ├── opt_mpi_cuda_gcc
+#  │    │    │    │    └── opt_mpi_cuda_hypre_cuda_gcc
+#  │    │    │    ├── <used jobs>
+#  │    │    │    │    ├── .reproducer_vars
+#  │    │    │    │    ├── .on_matrix
+#  │    │    │    │    └── .job_on_matrix
+#  │    │    │    ├── <included and used files>
+#  │    │    │    │    └── tests/gitlab/build_and_test_setup  [by "setup"]
+#  │    │    │    └── <defined variables>
+#  │    │    │         └── SPEC
+#  │    │    ├── dane-reports.yml
+#  │    │    │    ├── <jobs>
+#  │    │    │    │    ├── report_job_success
+#  │    │    │    │    └── report_job_failure
+#  │    │    │    └── <used jobs>
+#  │    │    │         ├── .on_dane
+#  │    │    │         ├── .report_job_success
+#  │    │    │         └── .report_job_failure
+#  │    │    ├── dane.yml
+#  │    │    │    ├── <jobs>
+#  │    │    │    │    ├── .dane_reproducer_vars
+#  │    │    │    │    ├── setup
+#  │    │    │    │    ├── debug_ser_gcc_10
+#  │    │    │    │    ├── debug_par_gcc_10
+#  │    │    │    │    ├── opt_ser_gcc_10
+#  │    │    │    │    ├── opt_par_gcc_10
+#  │    │    │    │    ├── opt_par_gcc_10_sundials
+#  │    │    │    │    ├── opt_par_gcc_10_petsc
+#  │    │    │    │    └── opt_par_gcc_10_pumi
+#  │    │    │    ├── <used jobs>
+#  │    │    │    │    ├── .reproducer_vars
+#  │    │    │    │    ├── .on_dane
+#  │    │    │    │    └── .job_on_dane
+#  │    │    │    ├── <included and used files>
+#  │    │    │    │    └── tests/gitlab/build_and_test_setup  [by "setup"]
+#  │    │    │    └── <defined variables>
+#  │    │    │         ├── SPEC
+#  │    │    │         └── THREADS
+#  │    │    ├── tioga-reports.yml
+#  │    │    │    ├── <jobs>
+#  │    │    │    │    ├── report_job_success
+#  │    │    │    │    └── report_job_failure
+#  │    │    │    └── <used jobs>
+#  │    │    │         ├── .on_tioga
+#  │    │    │         ├── .report_job_success
+#  │    │    │         └── .report_job_failure
+#  │    │    └── tioga.yml
+#  │    │         ├── <jobs>
+#  │    │         │    ├── .tioga_reproducer_vars
+#  │    │         │    ├── setup
+#  │    │         │    └── opt_mpi_rocm_hypre_rocm
+#  │    │         ├── <used jobs>
+#  │    │         │    ├── .reproducer_vars
+#  │    │         │    ├── .on_tioga
+#  │    │         │    └── .job_on_tioga
+#  │    │         ├── <included and used files>
+#  │    │         │    └── tests/gitlab/build_and_test_setup  [by "setup"]
+#  │    │         └── <defined variables>
+#  │    │              ├── SPEC
+#  │    │              └── THREADS
+#  │    ├── scripts
+#  │    │    ├── baseline
+#  │    │    │    └── <used variables>
+#  │    │    │         ├── BASELINE_TEST
+#  │    │    │         ├── SYS_TYPE
+#  │    │    │         ├── MACHINE_NAME
+#  │    │    │         ├── CI_PROJECT_DIR
+#  │    │    │         ├── ARTIFACTS_DIR
+#  │    │    │         ├── BUILD_ROOT
+#  │    │    │         └── TPLS_DIR
+#  │    │    ├── git_try_to_push
+#  │    │    ├── rebaseline
+#  │    │    │    └── <used variables>
+#  │    │    │         ├── CI_PROJECT_DIR
+#  │    │    │         ├── ARTIFACTS_DIR
+#  │    │    │         ├── SYS_TYPE
+#  │    │    │         ├── BUILD_ROOT
+#  │    │    │         ├── MACHINE_NAME
+#  │    │    │         └── CI_PIPELINE_ID
+#  │    │    ├── report_build_and_test
+#  │    │    │    ├── <used files>
+#  │    │    │    │    ├── .gitlab/scripts/safe_create_rundir
+#  │    │    │    │    └── .gitlab/scripts/git_try_to_push
+#  │    │    │    └── <used variables>
+#  │    │    │         ├── AUTOTEST_ROOT
+#  │    │    │         ├── CI_COMMIT_REF_SLUG
+#  │    │    │         ├── CI_PROJECT_DIR
+#  │    │    │         ├── CI_PIPELINE_URL
+#  │    │    │         ├── AUTOTEST_COMMIT
+#  │    │    │         └── CI_MACHINE
+#  │    │    └── safe_create_rundir
+#  │    ├── custom-jobs-and-variables.yml
+#  │    │    ├── <jobs>
+#  │    │    │    ├── .custom_job
+#  │    │    │    ├── .reproducer_vars
+#  │    │    │    ├── .report_job_success
+#  │    │    │    └── .report_job_failure
+#  │    │    ├── <used files>
+#  │    │    │    ├── tests/gitlab/build_and_test  [in JOB_CMD]
+#  │    │    │    └── .gitlab/scripts/report_build_and_test [by .report_job_*]
+#  │    │    ├── <defined variables>
+#  │    │    │    ├── JOB_CMD
+#  │    │    │    ├── BUILD_ROOT
+#  │    │    │    ├── ALLOC_NAME
+#  │    │    │    ├── TPLS_REPO
+#  │    │    │    ├── TESTS_REPO
+#  │    │    │    ├── AUTOTEST_REPO
+#  │    │    │    ├── MFEM_DATA_REPO
+#  │    │    │    ├── ARTIFACTS_DIR: artifacts
+#  │    │    │    ├── SLURM_OVERLAP: 1
+#  │    │    │    ├── DANE_SHARED_ALLOC
+#  │    │    │    ├── DANE_JOB_ALLOC
+#  │    │    │    ├── TIOGA_SHARED_ALLOC
+#  │    │    │    ├── TIOGA_JOB_ALLOC
+#  │    │    │    └── MATRIX_JOB_ALLOC
+#  │    │    └── <used variables>
+#  │    │         ├── SPEC
+#  │    │         ├── BUILD_ROOT
+#  │    │         └── ...
+#  │    ├── dane-baseline.yml
+#  │    │    ├── <jobs>
+#  │    │    │    ├── .on_dane
+#  │    │    │    ├── baselinecheck_mfem_intel_dane
+#  │    │    │    ├── cleanup
+#  │    │    │    ├── report_baseline
+#  │    │    │    └── baselinepublish_mfem_dane
+#  │    │    ├── <included and used files>
+#  │    │    │    ├── .gitlab/custom-jobs-and-variables.yml
+#  │    │    │    ├── .gitlab/configs/setup-baseline.yml
+#  │    │    │    ├── .gitlab/scripts/rebaseline
+#  │    │    │    ├── .gitlab/scripts/baseline
+#  │    │    │    └── .gitlab/scripts/git_try_to_push
+#  │    │    ├── <defined variables>
+#  │    │    │    ├── BASELINE_TEST: baseline
+#  │    │    │    ├── MACHINE_NAME: dane
+#  │    │    │    ├── TPLS_DIR
+#  │    │    │    └── export MFEM_TEST_NP
+#  │    │    └── <used variables>
+#  │    │         ├── ON_DANE
+#  │    │         ├── AUTOTEST       [defined by .gitlab-ci.yml]
+#  │    │         ├── BUILD_ROOT     [defined by custom-jobs-and-variables.yml]
+#  │    │         ├── TPLS_DIR       [defined by this file]
+#  │    │         ├── ARTIFACTS_DIR  [defined by custom-jobs-and-variables.yml]
+#  │    │         ├── MACHINE_NAME   [defined by this file]
+#  │    │         ├── AUTOTEST_COMMIT  [defined by .gitlab-ci.yml]
+#  │    │         ├── AUTOTEST_ROOT  [defined by .gitlab-ci.yml]
+#  │    │         ├── BASELINE_TEST  [defined by this file]
+#  │    │         └── REBASELINE     [defined by .gitlab-ci.yml]
+#  │    └── subscribed-pipelines.yml
+#  │         ├── <jobs>
+#  │         │    ├── .machine-check
+#  │         │    ├── generate-job-lists
+#  │         │    ├── dane-up-check
+#  │         │    ├── dane-build-and-test
+#  │         │    ├── dane-baseline
+#  │         │    ├── tioga-up-check
+#  │         │    ├── tioga-build-and-test
+#  │         │    ├── matrix-up-check
+#  │         │    └── matrix-build-and-test
+#  │         ├── <used jobs>
+#  │         │    └── .build-and-test  [from ".gitlab-ci.yml"]
+#  │         ├── <included files>
+#  │         │    └── .gitlab/dane-baseline.yml  [by "dane-baseline"]
+#  │         └── <used variables>
+#  │              ├── GITHUB_PROJECT_ORG
+#  │              ├── GITHUB_PROJECT_NAME
+#  │              ├── AUTOTEST
+#  │              ├── AUTOTEST_COMMIT
+#  │              └── REBASELINE
+#  └── tests
+#       ├── gitlab
+#       │    ├── build_and_test
+#       │    │    ├── <builds and tests a given MFEM spec with uberenv>
+#       │    │    ├── <used files>
+#       │    │    │    ├── tests/uberenv/uberenv.py        [deps mode, cloned]
+#       │    │    │    └── tests/gitlab/get_mfem_uberenv   [deps mode]
+#       │    │    └── <used variables>
+#       │    │         ├── SYS_TYPE
+#       │    │         ├── THREADS      [num. parallel jobs to build MFEM]
+#       │    │         ├── MODULE_LIST  [modules to load]
+#       │    │         ├── CI_JOB_ID
+#       │    │         ├── USE_DEV_SHM
+#       │    │         ├── SPACK_DEBUG
+#       │    │         ├── DEBUG_MODE
+#       │    │         ├── REGISTRY_TOKEN
+#       │    │         ├── CI_REGISTRY_USER  (defined by Gitlab)
+#       │    │         ├── USER
+#       │    │         ├── CI_REGISTRY_IMAGE (defined by Gitlab)
+#       │    │         └── CI_JOB_TOKEN      (defined by Gitlab)
+#       │    ├── build_and_test_setup
+#       │    │    ├── <updates MFEM_DATA_REPO and AUTOTEST_REPO using locks>
+#       │    │    └── <used variables>
+#       │    │         ├── MFEM_DATA_REPO
+#       │    │         ├── SHARED_REPOS_DIR
+#       │    │         ├── AUTOTEST_REPO
+#       │    │         └── AUTOTEST_ROOT
+#       │    └── get_mfem_uberenv
+#       │         ├── <github.com/mfem/mfem-uberenv.git -> tests/uberenv>
+#       │         └── <defines the uberenv hash to use>
+#       └── uberenv  [cloned by tests/gitlab/get_mfem_uberenv]
+#            └── uberenv.py
+#
+# <root of radiuss/radiuss-shared-ci, ref: 'v2025.09.1'>
+#  └── pipelines
+#       ├── matrix.yml
+#       │    ├── <jobs>
+#       │    │    ├── .on_matrix
+#       │    │    ├── .matrix_reproducer_init
+#       │    │    ├── .matrix_reproducer_vars
+#       │    │    ├── .matrix_reproducer_job
+#       │    │    ├── .matrix_job_command
+#       │    │    └── .job_on_matrix
+#       │    ├── <used jobs>
+#       │    │    └── .custom_job  [from .gitlab/custom-jobs-and-variables.yml]
+#       │    └── <used variables>
+#       │         ├── ON_MATRIX
+#       │         ├── ADVANCED_JOB
+#       │         ├── ALL_TARGETS
+#       │         ├── SYS_TYPE
+#       │         ├── LLNL_SERVICE_USER
+#       │         ├── USER
+#       │         ├── GITHUB_PROJECT_NAME
+#       │         ├── GITHUB_PROJECT_ORG
+#       │         ├── MATRIX_JOB_ALLOC
+#       │         └── JOB_CMD
+#       ├── dane.yml
+#       │    ├── <jobs>
+#       │    │    ├── .on_dane
+#       │    │    ├── .dane_reproducer_init
+#       │    │    ├── .dane_reproducer_vars
+#       │    │    ├── .dane_reproducer_job
+#       │    │    ├── .dane_job_command
+#       │    │    ├── .job_on_dane
+#       │    │    ├── allocate_resources
+#       │    │    └── release_resources
+#       │    ├── <used jobs>
+#       │    │    └── .custom_job  [from .gitlab/custom-jobs-and-variables.yml]
+#       │    ├── <defined variables>
+#       │    │    └── export JOBID
+#       │    └── <used variables>
+#       │         ├── ON_DANE
+#       │         ├── ADVANCED_JOB
+#       │         ├── ALL_TARGETS
+#       │         ├── SYS_TYPE
+#       │         ├── LLNL_SERVICE_USER
+#       │         ├── USER
+#       │         ├── GITHUB_PROJECT_NAME
+#       │         ├── GITHUB_PROJECT_ORG
+#       │         ├── DANE_JOB_ALLOC
+#       │         ├── JOB_CMD
+#       │         ├── JOBID
+#       │         ├── ALLOC_NAME
+#       │         └── DANE_SHARED_ALLOC
+#       └── tioga.yml
+#            ├── <jobs>
+#            │    ├── .on_tioga
+#            │    ├── .tioga_reproducer_init
+#            │    ├── .tioga_reproducer_vars
+#            │    ├── .tioga_reproducer_job
+#            │    ├── .tioga_job_command
+#            │    ├── .job_on_tioga
+#            │    ├── allocate_resources
+#            │    └── release_resources
+#            ├── <used jobs>
+#            │    └── .custom_job  [from .gitlab/custom-jobs-and-variables.yml]
+#            ├── <defined variables>
+#            │    └── PROXY
+#            └── <used variables>
+#                 ├── ON_TIOGA
+#                 ├── ADVANCED_JOB
+#                 ├── ALL_TARGETS
+#                 ├── SYS_TYPE
+#                 ├── LLNL_SERVICE_USER
+#                 ├── USER
+#                 ├── GITHUB_PROJECT_NAME
+#                 ├── GITHUB_PROJECT_ORG
+#                 ├── TIOGA_JOB_ALLOC
+#                 ├── JOB_CMD
+#                 ├── PROXY
+#                 ├── ALLOC_NAME
+#                 └── TIOGA_SHARED_ALLOC
 
+###############################################################################
+# We define the following GitLab pipeline variables:
 variables:
+##### LC GITLAB CONFIGURATION
   CUSTOM_CI_BUILDS_DIR: "/usr/workspace/mfem/gitlab-runner"
+
+##### PROJECT VARIABLES
   USER_CI_TOP_DIR: "${CUSTOM_CI_BUILDS_DIR}/${GITLAB_USER_LOGIN}"
   SHARED_REPOS_DIR: "${USER_CI_TOP_DIR}/repos"
   AUTOTEST_ROOT: "${SHARED_REPOS_DIR}"
-  # MFEM_DATA_DIR is setup in '.gitlab/configs/setup-build-and-test.yml' and
-  # used in '.gitlab/configs/<machine>-config.yml':
   MFEM_DATA_DIR: "${SHARED_REPOS_DIR}/mfem-data"
-
+# AUTOTEST: enable (ON/YES) or disable (any other value) test reporting. See
+# also AUTOTEST_COMMIT.
+  AUTOTEST: "OFF"
+# AUTOTEST_COMMIT: used only when AUTOTEST is set to ON/YES.
+# * If AUTOTEST_COMMIT is set to ON/YES, reporting jobs will commit their
+#   files to the MFEM/autotest repo.
+# * If AUTOTEST_COMMIT is NOT set to ON/YES, reporting jobs will NOT commit
+#   their files to the MFEM/autotest repo. Instead they will just show the
+#   contents of the report files and remove them.
+  AUTOTEST_COMMIT: "ON"
+# REBASELINE:
 # Defines the default choice for updating the saved baseline results. By default
 # the baseline can only be updated from the master branch. This variable offers
 # the option to manually ask for rebaselining from another branch if necessary.
-  REBASELINE: "NO"
-  AUTOTEST: "NO"
-  # AUTOTEST_COMMIT: used only when AUTOTEST is set to YES.
-  # * If AUTOTEST_COMMIT is NOT set to NO, reporting jobs will commit their
-  #   files to the MFEM/autotest repo.
-  # * If AUTOTEST_COMMIT is set to NO, reporting jobs will NOT commit their
-  #   files to the MFEM/autotest repo. Instead they will just show the contents
-  #   of the report files and remove them.
-  AUTOTEST_COMMIT: "YES"
+  REBASELINE: "OFF"
 
-# Trigger subpipelines:
-dane-build-and-test:
-  stage: sub-pipelines
+##### SHARED_CI CONFIGURATION
+# Required information about GitHub repository
+  GITHUB_PROJECT_NAME: "mfem"
+  GITHUB_PROJECT_ORG: "MFEM"
+# Override the pattern describing branches that will skip the "draft PR filter
+# test".  Add protected branches here. See default value in
+# preliminary-ignore-draft-pr.yml.
+#  ALWAYS_RUN_PATTERN: ""
+
+###############################################################################
+##### High level stages
+# We organize the test-pipelines stage with sub-pipelines. Each sub-pipeline
+# corresponds to a test batch on a given machine.
+stages:
+  - prerequisites
+  - test-pipelines
+
+###############################################################################
+# Template for jobs triggering a build-and-test sub-pipeline:
+.build-and-test:
+  stage: test-pipelines
   variables:
-    # Explicitly pass down values that we want to be able to set when triggering
-    # pipelines manually or using scheduling
+    # Explicitly pass down values that are not always propagated to child
+    # pipelines, e.g. when a variable is set in the "Settings -> CI" web
+    # interface (project variables).
+    # Note: in some cases, this does not work as expected, e.g. when the
+    # variable is not re-defined in the web interface; in such cases, the child
+    # pipeline gets a definition like '${AUTOTEST}', i.e. it behaves as if
+    # AUTOTEST is undefined, even though there is a default value in
+    # .gitlab-ci.yml.
     AUTOTEST: "${AUTOTEST}"
     AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
   trigger:
-    include: .gitlab/dane-build-and-test.yml
+    include:
+      - local: '.gitlab/custom-jobs-and-variables.yml'
+      - project: 'radiuss/radiuss-shared-ci'
+        ref: 'v2025.09.1'
+        file: 'pipelines/${CI_MACHINE}.yml'
+      - artifact: '${CI_MACHINE}-jobs.yml'
+        job: 'generate-job-lists'
     strategy: depend
+    forward:
+      pipeline_variables: true
 
-dane-baseline:
-  stage: sub-pipelines
-  variables:
-    # Explicitly pass down values that we want to be able to set when triggering
-    # pipelines manually or using scheduling
-    REBASELINE: "${REBASELINE}"
-    AUTOTEST: "${AUTOTEST}"
-    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
-  trigger:
-    include: .gitlab/dane-baseline.yml
-    strategy: depend
-
-lassen-build-and-test:
-  stage: sub-pipelines
-  variables:
-    # Explicitly pass down values that we want to be able to set when triggering
-    # pipelines manually or using scheduling
-    AUTOTEST: "${AUTOTEST}"
-    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
-  trigger:
-    include: .gitlab/lassen-build-and-test.yml
-    strategy: depend
-
-corona-build-and-test:
-  stage: sub-pipelines
-  variables:
-    # Explicitly pass down values that we want to be able to set when triggering
-    # pipelines manually or using scheduling
-    AUTOTEST: "${AUTOTEST}"
-    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
-  trigger:
-    include: .gitlab/corona-build-and-test.yml
-    strategy: depend
+###############################################################################
+include:
+  # Sets ID tokens for every job using `default:`
+  - project: 'lc-templates/id_tokens'
+    file: 'id_tokens.yml'
+  # [Optional] checks preliminary to running the actual CI test
+  #- project: 'radiuss/radiuss-shared-ci'
+  #  ref: 'v2025.09.1'
+  #  file: 'preliminary-ignore-draft-pr.yml'
+  # pipelines subscribed by the project
+  - local: '.gitlab/subscribed-pipelines.yml'

.gitlab/README.md

@@ -8,6 +8,8 @@
                                https://mfem.org
 
 
+FIXME: this file needs to be updated
+
 This directory contains most of the GitLab CI configuration. MFEM runs both PR
 and nightly testing on GitLab.
 
@@ -15,9 +17,9 @@ and nightly testing on GitLab.
 
 ## Top level
 
-The root configuration file is `.gitlab-ci.yml` at the root of MFEM repo.
-This file only defines one stage, in which we trigger several
-sub-pipelines.
+The root configuration file is `.gitlab-ci.yml` at the root of MFEM repo. This
+file only defines three stages, a prerequisites one, and two main stages in
+which we trigger several sub-pipelines.
 
 We use sub-pipelines to isolate the test for one combination of `machine`
 and `test type`.
@@ -25,8 +27,8 @@ and `test type`.
 Machines typically include:
 
 * Dane: Intel Sapphire Rapids
-* Lassen: Power9 + Nvidia GPU
-* Corona: AMD GPU
+* Matrix: Intel Sapphire Rapids + Nvidia H100 GPU
+* Tioga: AMD MI250X GPU
 
 Test types include:
 
@@ -39,9 +41,31 @@ altering the scheduling, execution and displaying of the others.
 
 ## Sub-pipelines
 
-Each file is this directory is the root configuration file for one
-sub-pipeline. The naming reflects the corresponding couple (`machine`,
-`test_type`).
+### build-and-test
+
+The build-and-test sub-pipelines leverage RADIUSS Shared CI to share most of
+the CI implementation. RADIUSS Shared CI provides a shared CI infrastructure
+vetted on most LC systems of interest and efficiently leveraging each machine
+scheduler to increase CI throughput. The maintenance of RADIUSS Shared CI is
+shared among several RADIUSS projects.
+
+Jobs for the build-and-test sub-pipelines are defined in the jobs directory.
+Because build-and-test jobs leverage Uberenv and Spack to build the
+dependencies automatically, the jobs essentially consists in a `spack spec`
+defined in the jobs files, and some scheduling parameters defined in the
+`.gitlab/custom-jobs-and-variables.yml` file.
+
+Build-and-test jobs all run the `tests/gitlab/build_and_test` script.
+
+The build-and-test pipelines are controlled by the
+`.gitlab/subscribed-pipelines.yml` which defines which machines to run on and
+implements additional features like machine availability check, and job list
+generation.
+
+### baseline
+
+Baseline sub-pipelines are described by files with names reflecting the
+machine it runs on, e.g. `dane-baseline`.
 
 Those files define the *stages* and the *jobs* for the sub-pipeline. They
 also contain any configuration that cannot be shared. For the most part
@@ -63,11 +87,11 @@ usage function. This should be improved.
 
 # More testing
 
-## Adding a new target to a build_and_test pipeline
+## Adding a new target to a build-and-test pipeline
 
-`build_and_test` pipelines rely on Spack to install dependencies. Spack is
+`build-and-test` pipelines rely on Spack to install dependencies. Spack is
 driven by Uberenv which helps freezing Spack configuration: the goal being to
-point to specific commit in Spack and isolate its configuration so that it is
+point to a specific commit in Spack and isolate its configuration so that it is
 not influenced by the user environment. More documentation about this can be
 found in `tests/gitlab`.
 
@@ -82,7 +106,7 @@ spack spec to use. Adding a job on Dane for example resumes to:
 <job_name>:
   variables:
     SPEC: "<spack_spec>"
-  extends: .build_and_test_on_dane
+  extends: .job_on_dane
 ```
 
 The remaining and non trivial work is to make sure this spec is working. To

.gitlab/configs/common.yml

@@ -1,40 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-include:
-  - project: 'lc-templates/id_tokens'
-    file: 'id_tokens.yml'
-
-# We define the following GitLab pipeline variables:
-variables:
-
-# The path to the shared resource between all jobs. For example, external
-# repositories like 'tests' and 'tpls' are cloned here. Also, 'tpls' is built
-# once for all targets, so that build happen here. The BUILD_ROOT is unique to
-# the pipeline, preventing any form of concurrency with other pipelines. This
-# also means that the BUILD_ROOT directory will never be cleaned.
-# TODO: add a clean-up mechanism
-  BUILD_ROOT: ${USER_CI_TOP_DIR}/${CI_PROJECT_NAME}-${MACHINE_NAME}-pipeline-${CI_PIPELINE_ID}
-
-# On LLNL's Dane, there is only one allocation shared among jobs in order to
-# save time and resource. This allocation has to be uniquely named so that we
-# are sure to retrieve it.
-  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
-
-# Git repositories used in the pipeline
-  TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
-  TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
-  AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
-  MFEM_DATA_REPO: https://github.com/mfem/data.git
-
-# Directory used to place artifacts.
-  ARTIFACTS_DIR: artifacts
-  SLURM_OVERLAP: 1

.gitlab/configs/corona-config.yml

@@ -1,59 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# GitLab pipeline configuration for the Corona machine at LLNL
-variables:
-  MACHINE_NAME: corona
-
-.on_corona:
-  tags:
-    - shell
-    - corona
-  rules:
-    # Don't run corona jobs if...
-    # Note: This makes corona an "opt-in" machine. To activate builds on corona
-    # for a given GitLab clone of MFEM, go to Setting/CI-CD/variables, and set
-    # "ON_CORONA" to "ON". An LC account on for corona is required to trigger a
-    # pipeline there.
-    - if: '$CI_COMMIT_BRANCH =~ /_cnone/ || $ON_CORONA != "ON"'
-      when: never
-    # Don't run autotest update if...
-    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
-      when: never
-    # Report success on success status
-    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
-      when: on_success
-    # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
-      when: on_failure
-    # Always release resource
-    - if: '$CI_JOB_NAME =~ /release_resource/'
-      when: always
-    # Always cleanup
-    - if: '$CI_JOB_NAME =~ /cleanup/'
-      when: always
-    # Default is to run if previous stage succeeded
-    - when: on_success
-
-# Spack helped builds
-# Generic corona build job, extending build script
-.build_and_test_on_corona:
-  extends: [.on_corona]
-  stage: build_and_test
-  script:
-    # THREADS is used by 'tests/gitlab/build_and_test', run below
-    - export THREADS=12
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - echo ${MFEM_DATA_DIR}
-    - echo ${SPEC}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 15 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data

.gitlab/configs/dane-config.yml

@@ -1,56 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# GitLab pipelines configurations for the Dane machine at LLNL
-variables:
-  MACHINE_NAME: dane
-
-.on_dane:
-  tags:
-    - shell
-    - dane
-  rules:
-    # Don't run dane jobs if...
-    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_DANE == "OFF"'
-      when: never
-    # Don't run autotest update if...
-    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
-      when: never
-    # Report success on success status
-    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
-      when: on_success
-    # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
-      when: on_failure
-    # Always release resource
-    - if: '$CI_JOB_NAME =~ /release_resource/'
-      when: always
-    # Always cleanup
-    - if: '$CI_JOB_NAME =~ /cleanup/'
-      when: always
-    # Default is to run if previous stage succeeded
-    - when: on_success
-
-# Spack helped builds
-# Generic dane build job, extending build script
-.build_and_test_on_dane:
-  extends: [.on_dane]
-  stage: build_and_test
-  script:
-    # THREADS is used by 'tests/gitlab/build_and_test', run below
-    # Dane has 224 threads/node and we run 7 separate jobs: 224=7*32
-    - export THREADS=28
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - echo ${MFEM_DATA_DIR}
-    - echo ${SPEC}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) --reservation=ci -t 60 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data

.gitlab/configs/lassen-config.yml

@@ -1,48 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# GitLab pipelines configurations for the Lassen machine at LLNL
-variables:
-  MACHINE_NAME: lassen
-
-.on_lassen:
-  tags:
-    - shell
-    - lassen
-  rules:
-    - if: '$CI_COMMIT_BRANCH =~ /_lnone/ || $ON_LASSEN == "OFF"' #run except if ...
-      when: never
-    # Don't run autotest update if...
-    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
-      when: never
-    # Report success on success status
-    - if: '$CI_JOB_NAME =~ /report_job_success/ && $AUTOTEST == "YES"'
-      when: on_success
-    # Report failure on failure status
-    - if: '$CI_JOB_NAME =~ /report_job_failure/ && $AUTOTEST == "YES"'
-      when: on_failure
-    # Always cleanup
-    - if: '$CI_JOB_NAME =~ /cleanup/'
-      when: always
-    - when: on_success
-
-# Lassen uses a different job scheduler (spectrum lsf) that does not allow
-# pre-allocation the same way slurm does.  We use the pci queue on lassen
-# to speed-up the allocation.
-.build_and_test_on_lassen:
-  extends: [.on_lassen]
-  stage: build_and_test
-  script:
-    - echo ${MFEM_DATA_DIR}
-    - echo ${SPEC}
-    # Next script uses 'THREADS': leaving it empty --> it uses 'make all -j'
-    - lalloc 1 -W 45 -q pci --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
-  needs: [setup]

.gitlab/configs/report-build-and-test.yml

@@ -1,77 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Jobs report
-.report_job_success:
-  script:
-    - echo ${MACHINE_NAME}
-    - echo ${AUTOTEST}
-    - echo ${AUTOTEST_COMMIT}
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - cd ${AUTOTEST_ROOT}
-    - |
-      (
-        date
-        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
-        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -n 9; do
-          sleep 5
-        done
-        echo "Acquired lock on '$PWD/autotest.lock'"
-        date
-        # Report SUCCESS while holding the file lock on 'autotest.lock'.
-        # The next script uses the following environment variables:
-        # - MACHINE_NAME, AUTOTEST_ROOT, AUTOTEST_COMMIT
-        # - CI_COMMIT_REF_SLUG, CI_PROJECT_DIR, CI_PIPELINE_URL
-        # It also calls the script '.gitlab/scripts/safe_create_rundir'.
-        ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test_success
-        err=$?
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock
-
-.report_job_failure:
-  script:
-    - echo ${MACHINE_NAME}
-    - echo ${AUTOTEST}
-    - echo ${AUTOTEST_COMMIT}
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - cd ${AUTOTEST_ROOT}
-    - |
-      (
-        date
-        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
-        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -n 9; do
-          sleep 5
-        done
-        echo "Acquired lock on '$PWD/autotest.lock'"
-        date
-        # Report FAILURE while holding the file lock on 'autotest.lock'.
-        # The next script uses the following environment variables:
-        # - MACHINE_NAME, AUTOTEST_ROOT, AUTOTEST_COMMIT
-        # - CI_COMMIT_REF_SLUG, CI_PROJECT_DIR, CI_PIPELINE_URL
-        # It also calls the script '.gitlab/scripts/safe_create_rundir'.
-        ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test_failure
-        err=$?
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock

.gitlab/configs/setup-build-and-test.yml

@@ -1,90 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test
-# script then symlinks the repo to the parent directory of the MFEM source
-# directory. Unit tests that depend on the mfem/data repo will then detect that
-# this directory is present and be enabled.
-setup:
-  tags:
-    - shell
-    - dane
-  stage: setup
-  variables:
-    GIT_STRATEGY: none
-  script:
-    #
-    # Setup MFEM_DATA_DIR=${SHARED_REPOS_DIR}/mfem-data, see '.gitlab-ci.yml'
-    # and '.gitlab/configs/<machine>-config.yml'
-    #
-    - echo "MACHINE_NAME = ${MACHINE_NAME}"
-    - echo "AUTOTEST = ${AUTOTEST}"
-    - echo "AUTOTEST_COMMIT = ${AUTOTEST_COMMIT}"
-    - echo "SHARED_REPOS_DIR ${SHARED_REPOS_DIR}"
-    - mkdir -p ${SHARED_REPOS_DIR} && cd ${SHARED_REPOS_DIR}
-    - command -v flock || echo "Required command 'flock' not found"
-    - |
-      (
-        date
-        echo "Waiting to acquire lock on '$PWD/mfem-data.lock' ..."
-        # try to get an exclusive lock on fd 9 (mfem-data.lock) repeating the
-        # try every 5 seconds; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -n 9; do
-          sleep 5
-        done
-        echo "Acquired lock on '$PWD/mfem-data.lock'"
-        date
-        # clone/update the mfem/data repo while holding the file lock on
-        # 'mfem-data.lock'
-        err=0
-        if [[ ! -d "mfem-data" ]]; then
-          git clone ${MFEM_DATA_REPO} "mfem-data"
-        else
-          cd "mfem-data" && git pull && cd ..
-        fi || err=1
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> mfem-data.lock
-    #
-    # Setup ${AUTOTEST_ROOT}/autotest:
-    #
-    - echo "AUTOTEST_ROOT ${AUTOTEST_ROOT}"
-    - mkdir -p ${AUTOTEST_ROOT} && cd ${AUTOTEST_ROOT}
-    - |
-      (
-        date
-        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
-        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; we may want to add a counter for the number of
-        # retries to interrupt a potential infinite loop
-        while ! flock -n 9; do
-          sleep 5
-        done
-        echo "Acquired lock on '$PWD/autotest.lock'"
-        date
-        # clone/update the autotest repo while holding the file lock on
-        # 'autotest.lock'
-        err=0
-        if [[ ! -d "autotest" ]]; then
-          git clone ${AUTOTEST_REPO}
-        else
-          cd autotest && git pull && cd ..
-        fi || err=1
-        # sleep for a period to allow NFS to propagate the above changes;
-        # clearly, there is no guarantee that other NFS clients will see the
-        # changes even after the timeout
-        sleep 10
-        exit $err
-      ) 9> autotest.lock

.gitlab/corona-build-and-test.yml

@@ -1,67 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-stages:
-  - setup
-  - allocate_resource
-  - build_and_test
-  - release_resource_and_report
-
-# Slurm shared allocation
-allocate_resource:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_corona
-  stage: allocate_resource
-  script:
-    - echo ${ALLOC_NAME}
-    - salloc --exclusive --nodes=1 --partition=mi60 --time=45 --no-shell --job-name=${ALLOC_NAME}
-  timeout: 6h
-  needs: [setup]
-
-# Build and test jobs, simply provide a spec
-rocm_gcc_8.3.1:
-  variables:
-    SPEC: "@develop%gcc@8.3.1+rocm amdgpu_target=gfx906"
-  extends: .build_and_test_on_corona
-  needs: [allocate_resource]
-
-# Release slurm allocation
-release_resource:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_corona
-  stage: release_resource_and_report
-  script:
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
-  needs: [rocm_gcc_8.3.1]
-
-# Jobs report
-report_job_success:
-  stage: release_resource_and_report
-  extends:
-    - .on_corona
-    - .report_job_success
-
-report_job_failure:
-  stage: release_resource_and_report
-  extends:
-    - .on_corona
-    - .report_job_failure
-
-include:
-  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/corona-config.yml
-  - local: .gitlab/configs/setup-build-and-test.yml
-  - local: .gitlab/configs/report-build-and-test.yml

.gitlab/custom-jobs-and-variables.yml

@@ -0,0 +1,132 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+include:
+  - project: 'lc-templates/id_tokens'
+    file: 'id_tokens.yml'
+
+# We define the following GitLab pipeline variables:
+variables:
+# Set the build-and-test command.
+# Nested variables are allowed and useful to customize the job command. We
+# protect variables with quotes so that their value may remain a string even if
+# they contain whitespaces.
+  JOB_CMD:
+    value: tests/gitlab/build_and_test --spec \"${SPEC}\" --data-dir ${MFEM_DATA_DIR} --data
+# The path to the shared resource between all jobs in the 'dane-baseline'
+# pipeline. For example, external repositories like 'tests' and 'tpls' are
+# cloned here. Also, 'tpls' is built once for all targets, so that build happens
+# here. The BUILD_ROOT is unique to the pipeline, preventing any form of
+# concurrency with other pipelines. This directory is removed by the 'cleanup'
+# stage in the 'dane-baseline' pipeline.
+  BUILD_ROOT: ${USER_CI_TOP_DIR}/${CI_PROJECT_NAME}-${CI_MACHINE}-pipeline-${CI_PIPELINE_ID}
+
+# On LLNL's dane and tioga, the 'build-and-test' pipelines creates only one
+# allocation shared among jobs in the pipeline in order to save time and
+# resources. This allocation has to be uniquely named so that we are sure to
+# retrieve it and avoid collisions.
+  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
+
+# Git repositories used in the pipelines:
+# - TPLS_REPO and TESTS_REPO are used only by the 'dane-baseline' pipeline
+# - AUTOTEST_REPO is used by all pipelines
+# - MFEM_DATA_REPO is used only by the 'build-and-test' pipelines
+  TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
+  TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
+  AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
+  MFEM_DATA_REPO: https://github.com/mfem/data.git
+
+# Directory used to place artifacts:
+# - ARTIFACTS_DIR is only used by the 'dane-baseline' pipeline
+  ARTIFACTS_DIR: artifacts
+  SLURM_OVERLAP: 1
+
+# Dane
+# Arguments for top level allocation
+  DANE_SHARED_ALLOC: "--exclusive --reservation=ci --time=60 --nodes=1"
+# Arguments for job level allocation
+# Note: We repeat the reservation, necessary when jobs are manually re-triggered.
+  DANE_JOB_ALLOC: "--reservation=ci --overlap --nodes=1"
+
+# Tioga
+# Arguments for top level allocation
+  TIOGA_SHARED_ALLOC: "--queue=pci --exclusive --time-limit=45m --nodes=1"
+# Arguments for job level allocation
+  TIOGA_JOB_ALLOC: "--nodes=1 --begin-time=+5s"
+
+# Matrix
+# Arguments for top level allocation
+  MATRIX_SHARED_ALLOC: "-p pdebug --exclusive --time=45 --nodes=1 -G 4"
+# Arguments for job level allocation
+# Note: We repeat the reservation, necessary when jobs are manually re-triggered.
+  MATRIX_JOB_ALLOC: "--overlap --nodes=1"
+
+# Configuration shared by build and test jobs specific to this project.
+# Not all configuration can be shared. Here projects can fine tune the
+# CI behavior.
+# See Umpire for an example (export junit test reports).
+.custom_job:
+  artifacts:
+    reports:
+
+# Note: this part is not used by the 'dane-baseline' pipeline.
+# FIXME: BUILD_ROOT, TPLS_REPO, TESTS_REPO are not needed here.
+#        Also, the definition of SHARED_REPOS_DIR is wrong.
+.reproducer_vars:
+  script:
+    - |
+      echo -e "
+      # Variables \n
+      export SPEC=\"${SPEC//\"/\\\"}\" \n
+      # Directories \n
+      export BUILD_ROOT=\"\${working_dir}\" \n
+      export SHARED_REPOS_DIR=\"\${BUILD_ROOT}/..\" \n
+      export MFEM_DATA_DIR=\"\${SHARED_REPOS_DIR}/mfem-data\" \n
+      # Repositories \n
+      export TPLS_REPO=\"${TPLS_REPO//\"/\\\"}\" \n
+      export TESTS_REPO=\"${TESTS_REPO//\"/\\\"}\" \n
+      export AUTOTEST_REPO=\"${AUTOTEST_REPO//\"/\\\"}\" \n
+      export MFEM_DATA_REPO=\"${MFEM_DATA_REPO//\"/\\\"}\" \n
+      # Setup directories \n
+      ./tests/gitlab/build_and_test_setup \n
+      # Using the CI build cache is optional and requires a token. Set it like so: \n
+      # export REGISTRY_TOKEN=\"<your token here>\" \n"
+      #
+
+# Jobs report
+.report_job_success:
+  script:
+    - ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test SUCCESS
+  rules:
+    - when: on_success
+
+.report_job_failure:
+  script:
+    - ${CI_PROJECT_DIR}/.gitlab/scripts/report_build_and_test FAILURE
+  rules:
+    - when: on_failure
+
+# Keep the following for debugging purposes: renaming this job from
+# '.show_variables' to 'show_variables' will insert this debug job at the
+# beginning of all child pipelines.
+.show_variables:
+  tags: [shell, oslic]
+  variables:
+    GIT_STRATEGY: none
+  stage: .pre
+  script:
+    - |
+      echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+      echo "AUTOTEST=${AUTOTEST}"
+      echo "AUTOTEST_COMMIT=${AUTOTEST_COMMIT}"
+      echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+      # Fail the job on purpose to prevent the rest of the pipeline from running
+      false

.gitlab/dane-baseline.yml

@@ -11,6 +11,7 @@
 
 variables:
   BASELINE_TEST: baseline
+  MACHINE_NAME: dane
 
 stages:
   - setup
@@ -19,6 +20,25 @@ stages:
   - cleanup
   - baseline_publish
 
+.on_dane:
+  tags:
+    - shell
+    - dane
+  rules:
+    # Don't run dane jobs if...
+    - if: '$ON_DANE == "OFF"'
+      when: never
+    # Don't run autotest update if...
+    # Note: in some cases, the content of AUTOTEST can be '${AUTOTEST}', so we
+    # need to treat that value as the default value of 'OFF'.
+    - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "ON" && $AUTOTEST != "YES"'
+      when: never
+    # Always cleanup
+    - if: '$CI_JOB_NAME =~ /cleanup/'
+      when: always
+    # Default is to run if previous stage succeeded
+    - when: on_success
+
 baselinecheck_mfem_intel_dane:
   extends: [.on_dane]
   stage: baseline_check
@@ -29,6 +49,9 @@ baselinecheck_mfem_intel_dane:
     #       .gitlab/configs/setup-baseline.yml.
     TPLS_DIR: ${BUILD_ROOT}/tpls
   script:
+    - echo "AUTOTEST=$AUTOTEST"
+    - echo "AUTOTEST_COMMIT=$AUTOTEST_COMMIT"
+    - echo "AUTOTEST_ROOT=$AUTOTEST_ROOT"
     - echo ${BUILD_ROOT}
     - echo ${TPLS_DIR}
     # Used by the tests in MFEM/tests, dane has 224 threads/node:
@@ -89,7 +112,13 @@ report_baseline:
           cp ${rundir}/pipeline.txt ${rundir}/autotest-email.html
         fi
         msg="GitLab CI log for ${BASELINE_TEST} on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
-        if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
+        # Note: in some cases, the content of AUTOTEST_COMMIT can be
+        # '${AUTOTEST_COMMIT}', so we need to treat that value as the default
+        # value of 'ON'.
+        if [[ "$AUTOTEST_COMMIT" == '${AUTOTEST_COMMIT}' ]]; then
+          AUTOTEST_COMMIT="ON"
+        fi
+        if [[ "$AUTOTEST_COMMIT" == "ON" || "$AUTOTEST_COMMIT" == "YES" ]]; then
           git pull && \
           git add ${rundir} && \
           git commit -m "${msg}" && \
@@ -117,8 +146,8 @@ baselinepublish_mfem_dane:
   extends: [.on_dane]
   stage: baseline_publish
   rules:
-    # - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "YES"'
-    - if: '$REBASELINE == "YES"'
+    # - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "ON"'
+    - if: '$REBASELINE == "ON"'
       when: manual
   script:
     - echo ${BUILD_ROOT}
@@ -128,6 +157,5 @@ baselinepublish_mfem_dane:
     - .gitlab/scripts/rebaseline
 
 include:
-  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/dane-config.yml
+  - local: .gitlab/custom-jobs-and-variables.yml
   - local: .gitlab/configs/setup-baseline.yml

.gitlab/dane-build-and-test.yml

@@ -1,94 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-stages:
-  - setup
-  - allocate_resource
-  - build_and_test
-  - release_resource_and_report
-
-# Allocate
-allocate_resource:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_dane
-  stage: allocate_resource
-  script:
-    - echo ${ALLOC_NAME}
-    - salloc --exclusive --nodes=1 --reservation=ci --time=60 --no-shell --job-name=${ALLOC_NAME}
-  timeout: 6h
-
-# GitLab jobs for the Dane machine at LLNL
-debug_ser_gcc_10:
-  variables:
-    SPEC: "%gcc@10.3.1 +debug~mpi"
-  extends: .build_and_test_on_dane
-
-debug_par_gcc_10:
-  variables:
-    SPEC: "%gcc@10.3.1 +debug+mpi"
-  extends: .build_and_test_on_dane
-
-opt_ser_gcc_10:
-  variables:
-    SPEC: "%gcc@10.3.1 ~mpi"
-  extends: .build_and_test_on_dane
-
-opt_par_gcc_10:
-  variables:
-    SPEC: "%gcc@10.3.1"
-  extends: .build_and_test_on_dane
-
-opt_par_gcc_10_sundials:
-  variables:
-    SPEC: "%gcc@10.3.1 +sundials"
-  extends: .build_and_test_on_dane
-
-opt_par_gcc_10_petsc:
-  variables:
-    SPEC: "%gcc@10.3.1 +petsc ^petsc+mumps~superlu-dist"
-  extends: .build_and_test_on_dane
-
-opt_par_gcc_10_pumi:
-  variables:
-    SPEC: "%gcc@10.3.1 +pumi"
-  extends: .build_and_test_on_dane
-
-# Release
-release_resource:
-  variables:
-    GIT_STRATEGY: none
-  extends: .on_dane
-  stage: release_resource_and_report
-  script:
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
-
-# Jobs report
-report_job_success:
-  stage: release_resource_and_report
-  extends:
-    - .on_dane
-    - .report_job_success
-
-report_job_failure:
-  stage: release_resource_and_report
-  extends:
-    - .on_dane
-    - .report_job_failure
-
-include:
-  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/dane-config.yml
-  - local: .gitlab/configs/setup-build-and-test.yml
-  - local: .gitlab/configs/report-build-and-test.yml

.gitlab/jobs/dane-reports.yml

@@ -0,0 +1,19 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Jobs report
+report_job_success:
+  extends: [.on_dane, .report_job_success]
+  stage: jobs-stage-3
+
+report_job_failure:
+  extends: [.on_dane, .report_job_failure]
+  stage: jobs-stage-3

.gitlab/jobs/dane.yml

@@ -0,0 +1,87 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Override reproducer section to define MFEM specific variables.
+.dane_reproducer_vars:
+  script:
+    - !reference [.reproducer_vars, script]
+
+# TODO: Setup script should be defined as a bash script (but then GIT_STRATEGY
+#       cannot be "none" anymore).
+
+# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test
+# script then symlinks the repo to the parent directory of the MFEM source
+# directory. Unit tests that depend on the mfem/data repo will then detect that
+# this directory is present and be enabled.
+setup:
+  extends: .on_dane
+  stage: jobs-stage-1
+  script:
+    - ./tests/gitlab/build_and_test_setup
+
+
+########################
+# Overridden shared jobs
+########################
+# When using shared jobs, we can duplicate them here to override description and
+# add necessary changes.
+# We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS} So that
+# the comparison with the original job is easier.
+
+
+############
+# Extra jobs
+############
+# We do not recommend using ${PROJECT_<MACHINE>_VARIANTS} and
+# ${PROJECT_<MACHINE>_DEPS} in the extra jobs. There is not reason not to fully
+# describe the spec here.
+
+.mfem_job_on_dane:
+  extends: .job_on_dane
+  stage: jobs-stage-2
+  variables:
+    # Dane has 224 threads/node and we run 7 separate jobs: 224=7*32
+    THREADS: 28
+
+debug_ser_gcc_10:
+  extends: .mfem_job_on_dane
+  variables:
+    SPEC: "%gcc@10.3.1 +debug~mpi"
+
+debug_par_gcc_10:
+  extends: .mfem_job_on_dane
+  variables:
+    SPEC: "%gcc@10.3.1 +debug+mpi"
+
+opt_ser_gcc_10:
+  extends: .mfem_job_on_dane
+  variables:
+    SPEC: "%gcc@10.3.1 ~mpi"
+
+opt_par_gcc_10:
+  extends: .mfem_job_on_dane
+  variables:
+    SPEC: "%gcc@10.3.1"
+
+opt_par_gcc_10_sundials:
+  extends: .mfem_job_on_dane
+  variables:
+    SPEC: "%gcc@10.3.1 +sundials"
+
+opt_par_gcc_10_petsc:
+  extends: .mfem_job_on_dane
+  variables:
+    SPEC: "%gcc@10.3.1 +petsc ^petsc+mumps~superlu-dist"
+
+opt_par_gcc_10_pumi:
+  extends: .mfem_job_on_dane
+  variables:
+    SPEC: "%gcc@10.3.1 +pumi"

.gitlab/jobs/matrix-reports.yml

@@ -0,0 +1,19 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Jobs report
+report_job_success:
+  extends: [.on_matrix, .report_job_success]
+  stage: jobs-stage-3
+
+report_job_failure:
+  extends: [.on_matrix, .report_job_failure]
+  stage: jobs-stage-3

.gitlab/jobs/matrix.yml

@@ -0,0 +1,65 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Override reproducer section to define UMPIRE specific variables.
+.matrix_reproducer_vars:
+  script:
+    - !reference [.reproducer_vars, script]
+
+#TODO: Setup script should be defined as a bash script (but then GIT_STRATEGY cannot be "none" anymore).
+
+# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test
+# script then symlinks the repo to the parent directory of the MFEM source
+# directory. Unit tests that depend on the mfem/data repo will then detect that
+# this directory is present and be enabled.
+setup:
+  extends: .on_matrix
+  stage: jobs-stage-1
+  script:
+    - ./tests/gitlab/build_and_test_setup
+
+
+########################
+# Overridden shared jobs
+########################
+# When using shared jobs , we can duplicate them here to override description and add necessary changes.
+# We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS} So that
+# the comparison with the original job is easier.
+
+
+############
+# Extra jobs
+############
+# We do not recommend using ${PROJECT_<MACHINE>_VARIANTS} and
+# ${PROJECT_<MACHINE>_DEPS} in the extra jobs. There is not reason not to fully
+# describe the spec here.
+
+.mfem_job_on_matrix:
+  extends: .job_on_matrix
+  stage: jobs-stage-2
+  variables:
+    # We run 2 jobs on 1 node that has 112 threads
+    THREADS: 48
+    # These modules need to be consistent with the uberenv configurations:
+    MODULE_LIST: "gcc/10.3.1-magic cuda/12.9.1"
+
+allocate_resources:
+  timeout: 4h
+
+opt_mpi_cuda_gcc:
+  extends: .mfem_job_on_matrix
+  variables:
+    SPEC: "%gcc@10.3.1 +mpi +cuda cuda_arch=90"
+
+opt_mpi_cuda_hypre_cuda_gcc:
+  extends: .mfem_job_on_matrix
+  variables:
+    SPEC: "%gcc@10.3.1 +mpi +cuda cuda_arch=90 ^hypre+cuda"

.gitlab/jobs/tioga-reports.yml

@@ -0,0 +1,20 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Jobs report
+report_job_success:
+  extends: [.on_tioga, .report_job_success]
+  stage: jobs-stage-3
+
+report_job_failure:
+  extends: [.on_tioga, .report_job_failure]
+  stage: jobs-stage-3
+

.gitlab/jobs/tioga.yml

@@ -0,0 +1,70 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Override reproducer section to define UMPIRE specific variables.
+.tioga_reproducer_vars:
+  script:
+    - !reference [.reproducer_vars, script]
+
+#TODO: Setup script should be defined as a bash script (but then GIT_STRATEGY cannot be "none" anymore).
+
+# Setup clones the mfem/data repo in ${SHARED_REPOS_DIR}. The build_and_test
+# script then symlinks the repo to the parent directory of the MFEM source
+# directory. Unit tests that depend on the mfem/data repo will then detect that
+# this directory is present and be enabled.
+setup:
+  extends: .on_tioga
+  stage: jobs-stage-1
+  script:
+    - ./tests/gitlab/build_and_test_setup
+
+########################
+# Overridden shared jobs
+########################
+# When using shared jobs , we can duplicate them here to override description and add necessary changes.
+# We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS} So that
+# the comparison with the original job is easier.
+
+
+############
+# Extra jobs
+############
+# We do not recommend using ${PROJECT_<MACHINE>_VARIANTS} and
+# ${PROJECT_<MACHINE>_DEPS} in the extra jobs. There is not reason not to fully
+# describe the spec here.
+
+# Build and test jobs, simply provide a spec
+
+#.tioga_job_command:
+#  script:
+#    - echo PROXY="${PROXY}"
+#    - echo TIOGA_JOB_ALLOC="${TIOGA_JOB_ALLOC}"
+#    - "printf '#!/bin/bash\n%s\n' \"${JOB_CMD}\" > flux_script.sh"
+#    - cat flux_script.sh
+#    - ${PROXY} flux watch $( ${PROXY} flux batch -o output.stdout.type=kvs ${TIOGA_JOB_ALLOC} flux_script.sh )
+#    - rm -f flux_script.sh
+
+.mfem_job_on_tioga:
+  extends: .job_on_tioga
+  stage: jobs-stage-2
+  variables:
+    # We run 1 job on 1 node that has 64 threads
+    THREADS: 64
+
+opt_mpi_rocm_hypre_rocm:
+  extends: .mfem_job_on_tioga
+  variables:
+    SPEC: "%rocmcc@=6.3.1 +rocm amdgpu_target=gfx90a ^hypre+rocm"
+
+# cce_16_0_1:
+#   extends: .mfem_job_on_tioga
+#   variables:
+#     SPEC: "%cce@=16.0.1"

.gitlab/lassen-build-and-test.yml

@@ -1,44 +0,0 @@
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-stages:
-  - setup
-  - build_and_test
-  - report
-
-opt_mpi_cuda_gcc:
-  variables:
-    SPEC: "%gcc@8.3.1 +mpi +cuda cuda_arch=70"
-  extends: .build_and_test_on_lassen
-
-opt_mpi_cuda_hypre_cuda_gcc:
-  variables:
-    SPEC: "%gcc@8.3.1 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
-  extends: .build_and_test_on_lassen
-
-# Jobs report
-report_job_success:
-  stage: report
-  extends:
-    - .on_lassen
-    - .report_job_success
-
-report_job_failure:
-  stage: report
-  extends:
-    - .on_lassen
-    - .report_job_failure
-
-include:
-  - local: .gitlab/configs/common.yml
-  - local: .gitlab/configs/lassen-config.yml
-  - local: .gitlab/configs/setup-build-and-test.yml
-  - local: .gitlab/configs/report-build-and-test.yml

.gitlab/scripts/baseline

@@ -35,8 +35,6 @@ if [[ "${MACHINE_NAME}" == "dane" ]]; then
   salloc --nodes=1 --exclusive --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "corona" ]]; then
   salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
-elif [[ ${MACHINE_NAME} == "lassen" ]]; then
-  lalloc 1 -q pci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 else
   echo "Unknown machine: MACHINE_NAME=$MACHINE_NAME"
   exit 1

.gitlab/scripts/report_build_and_test

@@ -0,0 +1,118 @@
+#!/bin/bash
+
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+function info_msg ()
+{
+  echo "[Information:] ${1}"
+}
+
+function error_msg ()
+{
+  echo "[Error:] ${1}"
+}
+
+# Perform a report while holding a lock file to prevent concurrency on
+# the destination.
+# Usage:
+# locked_clone <report_function> <lock_name>
+function locked_report ()
+{
+  if ! command -v flock
+  then
+    error_msg "Required command 'flock' not found"
+    exit 1
+  fi
+
+  info_msg "Will report ${1} while holding a lock in ${2}"
+
+  ( date; info_msg "Waiting to acquire lock on '${PWD}/${2}.lock' ..."
+    # try to get an exclusive lock on fd 9 (mfem-data.lock) repeating the
+    # try every 5 seconds; we may want to add a counter for the number of
+    # retries to interrupt a potential infinite loop
+    while ! flock -n 9; do sleep 5; done
+    date; info_msg "Acquired lock on '${PWD}/${2}.lock'"
+
+    report ${1}
+    err=$?
+
+    # sleep for a period to allow NFS to propagate the above changes;
+    # clearly, there is no guarantee that other NFS clients will see the
+    # changes even after the timeout
+    sleep 10
+    exit $err
+  ) 9> ${2}.lock
+}
+
+function report ()
+{
+  if [[ "${1}" == "SUCCESS" ]]
+  then
+    info_msg "All the ${MACHINE_NAME} jobs passed"
+    status_msg="The 'build-and-test' jobs on ${MACHINE_NAME} were SUCCESSFUL."
+  elif [[ "${1}" == "FAILURE" ]]
+  then
+    info_msg "At least one failure on ${MACHINE_NAME}"
+    status_msg="Some 'build-and-test' jobs on ${MACHINE_NAME} FAILED."
+  else
+    error_msg "Unknown status: ${1} ... aborting"
+    exit 1
+  fi
+
+  cd ${AUTOTEST_ROOT}/autotest || \
+    { error_msg "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
+  mkdir -p ${MACHINE_NAME}
+
+  rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
+  rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir)
+
+  printf "%s\n" "${status_msg}" \
+    "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.err
+
+  msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
+
+  if [[ "${1}" == "FAILURE" ]]
+  then
+    # Create 'autotest-email.html' to indicate failure:
+    cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
+  fi
+
+  # Note: in some cases, the content of AUTOTEST_COMMIT can be
+  # '${AUTOTEST_COMMIT}', so we need to treat that value as the default
+  # value of 'ON'.
+  if [[ "$AUTOTEST_COMMIT" == '${AUTOTEST_COMMIT}' ]]; then
+    AUTOTEST_COMMIT="ON"
+  fi
+  if [[ "$AUTOTEST_COMMIT" == "ON" || "$AUTOTEST_COMMIT" == "YES" ]]; then
+    git pull && \
+    git add ${rundir} && \
+    git commit -m "${msg}" && \
+    ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
+  else
+    for file in ${rundir}/*; do
+      echo "------------------------------"
+      echo "Content of '$file'"
+      echo "******************************"
+      cat $file
+      echo "******************************"
+    done
+    rm -rf ${rundir} || true
+  fi
+}
+
+export MACHINE_NAME=${CI_MACHINE}
+info_msg "MACHINE_NAME is ${MACHINE_NAME}"
+info_msg "AUTOTEST_ROOT is ${AUTOTEST_ROOT}"
+info_msg "AUTOTEST=$AUTOTEST"
+info_msg "AUTOTEST_COMMIT=$AUTOTEST_COMMIT"
+
+cd ${AUTOTEST_ROOT} && locked_report ${1} autotest

.gitlab/scripts/report_build_and_test_failure

@@ -1,45 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-echo "Runs if there was at least one failure on ${MACHINE_NAME}"
-
-cd ${AUTOTEST_ROOT}/autotest || \
-  { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
-mkdir -p ${MACHINE_NAME}
-
-rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
-rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir)
-
-printf "%s\n" "Some 'build-and-test' jobs on ${MACHINE_NAME} FAILED." \
-  "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.err
-
-msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
-
-# Create 'autotest-email.html' to indicate failure:
-cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
-
-if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
-  git pull && \
-  git add ${rundir} && \
-  git commit -m "${msg}" && \
-  ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
-else
-  for file in ${rundir}/*; do
-    echo "------------------------------"
-    echo "Content of '$file'"
-    echo "******************************"
-    cat $file
-    echo "******************************"
-  done
-  rm -rf ${rundir} || true
-fi

.gitlab/scripts/report_build_and_test_success

@@ -1,42 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-echo "Can only run if all the ${MACHINE_NAME} jobs passed"
-
-cd ${AUTOTEST_ROOT}/autotest || \
-  { echo "Invalid 'autotest' dir: ${AUTOTEST_ROOT}/autotest"; exit 1; }
-mkdir -p ${MACHINE_NAME}
-
-rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
-rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir $rundir)
-
-printf "%s\n" "The 'build-and-test' jobs on ${MACHINE_NAME} were SUCCESSFUL." \
-  "Pipeline URL:" "$CI_PIPELINE_URL" > ${rundir}/gitlab.out
-
-msg="GitLab CI log for build-and-test on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
-
-if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
-  git pull && \
-  git add ${rundir} && \
-  git commit -m "${msg}" && \
-  ${CI_PROJECT_DIR}/.gitlab/scripts/git_try_to_push
-else
-  for file in ${rundir}/*; do
-    echo "------------------------------"
-    echo "Content of '$file'"
-    echo "******************************"
-    cat $file
-    echo "******************************"
-  done
-  rm -rf ${rundir} || true
-fi

.gitlab/subscribed-pipelines.yml

@@ -0,0 +1,130 @@
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# The template job to test whether a machine is up.
+# Expects CI_MACHINE defined to machine name.
+.machine-check:
+  stage: prerequisites
+  tags: [shell, oslic]
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - |
+      if [[ $(jq '.[env.CI_MACHINE].total_nodes_up' /usr/global/tools/lorenz/data/loginnodeStatus) == 0 ]]
+      then
+        echo -e "\e[31mNo node available on ${CI_MACHINE}\e[0m"
+        false && \
+        curl --url "https://api.github.com/repos/${GITHUB_PROJECT_ORG}/${GITHUB_PROJECT_NAME}/statuses/${CI_COMMIT_SHA}" \
+             --header 'Content-Type: application/json' \
+             --header "authorization: Bearer ${GITHUB_TOKEN}" \
+             --data "{ \"state\": \"failure\", \"target_url\": \"${CI_PIPELINE_URL}\", \"description\": \"GitLab ${CI_MACHINE} down\", \"context\": \"ci/gitlab/${CI_MACHINE}\" }"
+        exit 1
+      fi
+
+###
+# Trigger a build-and-test pipeline for a machine.
+# Comment the jobs for machines you don’t need.
+###
+
+# One job to generate the job list for all the subpipelines
+generate-job-lists:
+  stage: prerequisites
+  tags: [shell, oslic]
+  variables:
+    LOCAL_JOBS_PATH: ".gitlab/jobs"
+  script:
+    - |
+      echo "AUTOTEST=$AUTOTEST"
+      echo "AUTOTEST_COMMIT=$AUTOTEST_COMMIT"
+      echo "AUTOTEST_ROOT=$AUTOTEST_ROOT"
+    - |
+      cat ${LOCAL_JOBS_PATH}/dane.yml > dane-jobs.yml
+      if [[ ${AUTOTEST} == "ON" || ${AUTOTEST} == "YES" ]]
+      then
+          cat ${LOCAL_JOBS_PATH}/dane-reports.yml >> dane-jobs.yml
+      fi
+    - |
+      cat ${LOCAL_JOBS_PATH}/matrix.yml > matrix-jobs.yml
+      if [[ ${AUTOTEST} == "ON" || ${AUTOTEST} == "YES" ]]
+      then
+          cat ${LOCAL_JOBS_PATH}/matrix-reports.yml >> matrix-jobs.yml
+      fi
+    - |
+      cat ${LOCAL_JOBS_PATH}/tioga.yml > tioga-jobs.yml
+      if [[ ${AUTOTEST} == "ON" || ${AUTOTEST} == "YES" ]]
+      then
+          cat ${LOCAL_JOBS_PATH}/tioga-reports.yml >> tioga-jobs.yml
+      fi
+  artifacts:
+    paths:
+      - dane-jobs.yml
+      - matrix-jobs.yml
+      - tioga-jobs.yml
+
+
+# DANE
+dane-up-check:
+  variables:
+    CI_MACHINE: "dane"
+  extends: [.machine-check]
+
+dane-build-and-test:
+  variables:
+    CI_MACHINE: "dane"
+  needs: [dane-up-check, generate-job-lists]
+  extends: [.build-and-test]
+
+# DANE, MFEM Specific
+dane-baseline:
+  stage: test-pipelines
+  variables:
+    # Explicitly pass down values that are not always propagated to child
+    # pipelines, e.g. when a variable is set in the "Settings -> CI" web
+    # interface (project variables).
+    # Note: in some cases, this does not work as expected, e.g. when the
+    # variable is not re-defined in the web interface; in such cases, the child
+    # pipeline gets a definition like '${AUTOTEST}', i.e. it behaves as if
+    # AUTOTEST is undefined, even though there is a default value in
+    # .gitlab-ci.yml.
+    AUTOTEST: "${AUTOTEST}"
+    AUTOTEST_COMMIT: "${AUTOTEST_COMMIT}"
+  trigger:
+    include: .gitlab/dane-baseline.yml
+    strategy: depend
+    forward:
+      pipeline_variables: true
+  needs: [dane-up-check]
+
+
+# TIOGA
+tioga-up-check:
+  variables:
+    CI_MACHINE: "tioga"
+  extends: [.machine-check]
+
+tioga-build-and-test:
+  variables:
+    CI_MACHINE: "tioga"
+  needs: [tioga-up-check, generate-job-lists]
+  extends: [.build-and-test]
+
+
+# Matrix
+matrix-up-check:
+  variables:
+    CI_MACHINE: "matrix"
+  extends: [.machine-check]
+
+matrix-build-and-test:
+  variables:
+    CI_MACHINE: "matrix"
+  needs: [matrix-up-check, generate-job-lists]
+  extends: [.build-and-test]

mesh/ncnurbs.cpp

@@ -1014,7 +1014,7 @@ void NCNURBSExtension::GetMasterFaceDofs(bool dof, int mf,
       std::array<int, 4> orderedVertices;
       std::array<bool, 4> edgeBdry;
       std::set<int> vbdry;
-      int nf1, nf2;
+      int nf1 = 0, nf2 = 0;
       auto SetEdgeEntries = [&](int eidx, int edge, const Array<int> &evert,
                                 int &vstart)
       {

miniapps/common/makefile

@@ -41,7 +41,7 @@ ifneq (clean,$(MAKECMDGOALS))
    override shared = $(if $(MFEM_SHARED:YES=),,YES)
 endif
 
-SEQ_MINIOBJS = mesh_extras.o fem_extras.o
+SEQ_MINIOBJS = mesh_extras.o fem_extras.o bravais.o
 ifeq ($(MFEM_USE_MPI),NO)
    MINIOBJS = $(SEQ_MINIOBJS)
 else

miniapps/meta-material/makefile

@@ -0,0 +1,81 @@
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability see http://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../..
+MFEM_BUILD_DIR ?= ../..
+SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/miniapps/meta-material/,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_MINIAPPS = 
+PAR_MINIAPPS = meta_material
+ifeq ($(MFEM_USE_MPI),NO)
+   MINIAPPS = $(SEQ_MINIAPPS)
+else
+   MINIAPPS = $(PAR_MINIAPPS) $(SEQ_MINIAPPS)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+COMMON_O=../common/bravais.o ../common/pfem_extras.o
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK) $(COMMON_O)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ \
+	$(COMMON_O) $(MFEM_LIBS)
+
+all: $(MINIAPPS)
+
+# Rules for building the miniapps
+$(MINIAPPS): \
+%: $(SRC)%.cpp %_solver.o $(COMMON_O) $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $@_solver.o $(COMMON_O) $(MFEM_LIBS)
+
+# Rules for compiling miniapp dependencies
+$(COMMON_O) $(addsuffix _solver.o,$(MINIAPPS)): \
+%.o: $(SRC)%.cpp $(SRC)%.hpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $(<) -o $(@)
+
+MFEM_TESTS = MINIAPPS
+include $(MFEM_TEST_MK)
+
+# Testing: Parallel vs. serial runs
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) 4
+volta-test-par: meta_material
+	@$(call mfem-test,$<, $(RUN_MPI), Meta-Material miniapp,\
+	-bl 1 -rs 0 -rp 0)
+
+# Testing: "test" target and mfem-test* variables are defined in config/test.mk
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean: clean-build clean-exec
+
+clean-build:
+	rm -f *.o *~ test_meta_mat meta_material
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:
+	@rm -f mobius-strip.mesh klein-bottle.mesh mesh-explorer.mesh
+	@rm -f partitioning.txt shaper.mesh

miniapps/meta-material/meta_material.cpp

@@ -0,0 +1,692 @@
+#include "mfem.hpp"
+#include "../common/bravais.hpp"
+#include <fstream>
+#include <iostream>
+#include <cerrno>      // errno
+
+#include "meta_material_solver.hpp"
+
+#ifndef _WIN32
+#include <sys/stat.h>  // mkdir
+#else
+#include <direct.h>    // _mkdir
+#define mkdir(dir, mode) _mkdir(dir)
+#endif
+
+using namespace std;
+using namespace mfem;
+using namespace mfem::miniapps;
+using namespace mfem::bravais;
+
+// Volume Fraction Coefficient
+static int prob_ = -1;
+double vol_frac_coef(const Vector &);
+
+int CreateDirectory(const string &dir_name, MPI_Comm & comm, int myid);
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Comm comm = MPI_COMM_WORLD;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(comm, &num_procs);
+   MPI_Comm_rank(comm, &myid);
+
+   // 2. Parse command-line options.
+   int bl_type = 1;
+   string lattice_label = "";
+   int order = 1;
+   // int sr = 0, pr = 2;
+   int logging = 0;
+
+   bool visualization = 1;
+   bool visit = true;
+   bool densityCalc = true;
+   bool stiffnessCalc = false;
+   bool bandGapCalc = false;
+   bool band_gap_mid_pts = false;
+
+   double a = -1.0, b = -1.0, c = -1.0;
+   double alpha = -1.0, beta = -1.0, gamma = -1.0;
+   double alpha_deg = -1.0, beta_deg = -1.0, gamma_deg = -1.0;
+   double lcf = 0.3;
+   double density_tol = 0.05;
+   double stiffness_tol = 0.05;
+   double band_gap_tol = 0.05;
+   int    band_gap_max_ref = 2;
+   int    band_gap_samp_pow = 2;
+   // double lambda = 2.07748e+9;
+   // double mu = 0.729927e+9;
+   // Gallium Arsenide at T=300K
+   // double lambda = 5.34e+11;
+   // double mu = 3.285e+11;
+   // double rho0 = 0.0;
+
+   // Acrylonitrile Butadiene Styrene (ABS)
+   // double    rho = 1110.0; // Mass density 1.075 kg/m^3
+   double    rho = 1000.0; // Mass density 1.0 kg/m^3
+   double      E = 2.0e9;  // Young's Modulus 2GPa
+   double     nu = 0.4064; // Poisson's Ratio
+   double epsRel = 10.0;   // Relative Dielectric Permittivity
+   double  muRel = 1.0;    // Relative Magnetic Permeability
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&bl_type, "-bl", "--bravais-lattice",
+                  "Bravais Lattice Type: \n"
+                  "  1 - Primitive Cubic (a),\n"
+                  "  2 - Face-Centered Cubic (a),\n"
+                  "  3 - Body-Centered Cubic (a),\n"
+                  "  4 - Tetragonal (a, c),\n"
+                  "  5 - Body-Centered Tetragonal (a, c),\n"
+                  "  6 - Orthorhombic (a < b < c),\n"
+                  "  7 - Face-Centered Orthorhombic (a < b < c),\n"
+                  "  8 - Body-Centered Orthorhombic (a < b < c),\n"
+                  "  9 - C-Centered Orthorhombic (a < b, c),\n"
+                  " 10 - Hexagonal Prism (a, c),\n"
+                  " 11 - Rhombohedral (a, 0 < alpha < pi),\n"
+                  " 12 - Monoclinic (a, b <= c, 0 < alpha < pi/2),\n"
+                  " 13 - C-Centered Monoclinic (a, b <= c, 0 < alpha < pi/2),\n"
+                  " 14 - Triclinic (0 < alpha, beta, gamma < pi)\n"
+                 );
+   args.AddOption(&a, "-a", "--lattice-a",
+                  "Lattice spacing a");
+   args.AddOption(&b, "-b", "--lattice-b",
+                  "Lattice spacing b");
+   args.AddOption(&c, "-c", "--lattice-c",
+                  "Lattice spacing c");
+   args.AddOption(&alpha, "-alpha", "--lattice-alpha",
+                  "Lattice angle alpha");
+   args.AddOption(&beta, "-beta", "--lattice-beta",
+                  "Lattice angle beta");
+   args.AddOption(&gamma, "-gamma", "--lattice-gamma",
+                  "Lattice angle gamma");
+   args.AddOption(&alpha_deg, "-alpha-deg", "--lattice-alpha-degrees",
+                  "Lattice angle alpha in degrees");
+   args.AddOption(&beta_deg, "-beta-deg", "--lattice-beta-degrees",
+                  "Lattice angle beta in degrees");
+   args.AddOption(&gamma_deg, "-gamma-deg", "--lattice-gamma-degrees",
+                  "Lattice angle gamma in degrees");
+   args.AddOption(&lcf, "-lcf", "--lattice-coef-frac",
+                  "Fraction of inscribed circle radius for rods");
+   args.AddOption(&density_tol, "-rtol", "--density-tolerance",
+                  "Stopping tolerance specified as a relative difference "
+                  "in computed density");
+   args.AddOption(&stiffness_tol, "-ctol", "--stiffness-tolerance",
+                  "Stopping tolerance specified as a relative difference "
+                  "in the 2-norm of the computed stiffness tensor");
+   args.AddOption(&band_gap_tol, "-bgtol", "--band-gap-tolerance",
+                  "Stopping tolerance specified as a relative difference "
+                  "in the computed band gap");
+   args.AddOption(&band_gap_max_ref, "-bgmr", "--band-gap-max-ref",
+                  "Maximum number of uniform mesh refinements to perform "
+                  "when computing the band gap");
+   args.AddOption(&band_gap_samp_pow, "-bgsp", "--band-gap-samp-pow",
+                  "Number of sampling points per segment is 2^samp_pow "
+                  "when computing the band gap");
+   args.AddOption(&band_gap_mid_pts, "-bgmp", "--band-gap-mid-pts",
+                  "-no-bgmp", "--no-band-gap-mid-pts",
+                  "Whether to include mid points in the band gap computation");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   // args.AddOption(&sr, "-sr", "--serial-refinement",
+   //               "Number of serial refinement levels.");
+   // args.AddOption(&pr, "-pr", "--parallel-refinement",
+   //               "Number of parallel refinement levels.");
+   // args.AddOption(&prob_, "-p", "--problem-type",
+   //                "Problem Geometry.");
+   // args.AddOption(&lambda, "-l", "--lambda",
+   //               "Lambda");
+   // args.AddOption(&mu, "-m", "--mu",
+   //               "Mu");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&visit, "-visit", "--visit", "-no-visit",
+                  "--no-visualization",
+                  "Enable or disable VisIt visualization.");
+   args.AddOption(&densityCalc, "-rho", "--density", "-no-rho", "--no-density",
+                  "Enable or disable density calculation.");
+   args.AddOption(&stiffnessCalc, "-C", "--stiffness",
+                  "-no-C", "--no-stiffness",
+                  "Enable or disable stiffness tensor calculation.");
+   // args.AddOption(&dispersionPlot, "-disp", "--dispersion",
+   //             "-no-disp", "--no-dispersion",
+   //             "Enable or disable dispersion plot calculation.");
+   args.AddOption(&bandGapCalc, "-bg", "--band-gap",
+                  "-no-bg", "--no-band-gap",
+                  "Enable or disable band gap calculation.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   if ( alpha_deg > 0.0 ) { alpha = alpha_deg * M_PI / 180.0; }
+   if (  beta_deg > 0.0 ) { beta =  beta_deg * M_PI / 180.0; }
+   if ( gamma_deg > 0.0 ) { gamma = gamma_deg * M_PI / 180.0; }
+
+   BRAVAIS_LATTICE_TYPE lattice_type = (BRAVAIS_LATTICE_TYPE)(bl_type + 5);
+   BravaisLattice * bravais = BravaisLatticeFactory(lattice_type,
+                                                    a, b, c,
+                                                    alpha, beta, gamma,
+                                                    logging);
+   BravaisLattice3D * bravais3d = dynamic_cast<BravaisLattice3D*>(bravais);
+
+   lattice_label = bravais->GetLatticeTypeLabel();
+
+   if ( myid == 0 )
+   {
+      cout << "Lattice Type: " << lattice_label << endl;
+   }
+
+   bravais3d->GetAxialLengths(a, b, c);
+   bravais3d->GetInteraxialAngles(alpha, beta, gamma);
+
+   ostringstream oss_prefix;
+   oss_prefix << "Meta-Material-" << lattice_label
+              << "-" << (int)round(100.0*a)
+              << "-" << (int)round(100.0*b)
+              << "-" << (int)round(100.0*c)
+              << "-" << (int)round(180.0*alpha / M_PI)
+              << "-" << (int)round(180.0*beta  / M_PI)
+              << "-" << (int)round(180.0*gamma / M_PI)
+              << "-" << (int)round(1000.0*lcf);
+   // << "-r" << sr + pr;
+
+   CreateDirectory(oss_prefix.str(),comm,myid);
+
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   /*
+   Mesh * mesh = bravais->GetPeriodicWignerSeitzMesh();
+
+   int euler = mesh->EulerNumber();
+   if ( myid == 0 ) { cout << "Initial Euler Number:  " << euler << endl; }
+   mesh->CheckElementOrientation(false);
+   mesh->CheckBdrElementOrientation(false);
+
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement.
+   {
+      int ref_levels = sr;
+      for (int l = 0; l < ref_levels; l++)
+      {
+         mesh->UniformRefinement();
+         int euler = mesh->EulerNumber();
+         if ( myid == 0 )
+         {
+            cout << l+1 << ", Refined Euler Number:  " << euler << endl;
+         }
+         mesh->CheckElementOrientation(false);
+         mesh->CheckBdrElementOrientation(false);
+      }
+   }
+   */
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   // ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   /*
+   delete mesh;
+   {
+      int par_ref_levels = pr;
+      for (int l = 0; l < par_ref_levels; l++)
+      {
+         pmesh->UniformRefinement();
+      }
+   }
+   */
+   // L2_ParFESpace * L2FESpace    = new L2_ParFESpace(pmesh, 0,
+   //                                               pmesh->Dimension());
+   /*
+   // int nElems = L2FESpace->GetVSize();
+   int nElems = mesh->GetNE();
+   cout << myid << ": nElems = " << nElems << endl;
+   delete mesh;
+   */
+   // LatticeCoefficient latCoef(*bravais, lcf);
+
+   // ParGridFunction * vf0 = new ParGridFunction(L2FESpace);
+   // ParGridFunction * vf1 = new ParGridFunction(L2FESpace);
+
+   // FunctionCoefficient vfFunc(vol_frac_coef);
+   // vf0->ProjectCoefficient(latCoef);
+   /*
+   vf1->ProjectCoefficient(vfFunc);
+
+   double vf13 = (*vf0)[13];
+   double dvf = 0.01 * (0.5 - vf13);
+   (*vf1)[13] += dvf;
+   */
+   VisData vd("localhost", 19916, 1440, 900, 238, 238, 10, 45);
+   /*
+   if (visualization)
+   {
+      socketstream vf_sock;
+      VisualizeField(vf_sock, *vf0, "Volume Fraction 0", vd);
+      vd.IncrementWindow();
+      // socketstream vf1_sock;
+      // VisualizeField(vf1_sock, *vf1, "Volume Fraction 1", vd);
+      // vd.IncrementWindow();
+   }
+   */
+   /*
+   meta_material::Density density(*pmesh, bravais->GetUnitCellVolume(),
+                                  0.0, rho);
+
+   density.SetVolumeFraction(*vf0);
+   */
+   if ( myid == 0 )
+   {
+      cout << "Volume of the unit cell: "
+           << bravais->GetUnitCellVolume() << endl;
+   }
+
+   if (densityCalc)
+   {
+      LatticeCoefficient rhoCoef(*bravais, lcf, 0.0, rho);
+
+      // The density computation does not require a periodic mesh
+      Mesh * mesh_rho = bravais->GetWignerSeitzMesh();
+      mesh_rho->EnsureNCMesh();
+      mesh_rho->UniformRefinement();
+      mesh_rho->UniformRefinement();
+      ParMesh *pmesh_rho = new ParMesh(MPI_COMM_WORLD, *mesh_rho);
+      delete mesh_rho;
+
+      meta_material::Density density(*pmesh_rho, rho,
+                                     bravais->GetUnitCellVolume(),
+                                     rhoCoef, density_tol);
+
+      vector<double> effective_rho;
+      density.GetHomogenizedProperties(effective_rho);
+      if ( myid == 0 )
+      {
+         ostringstream oss;
+         oss << oss_prefix.str() << "/density.dat";
+         ofstream ofs(oss.str().c_str());
+
+         cout << "Effective Density:  ";
+         for (unsigned int i=0; i<effective_rho.size(); i++)
+         {
+            cout << effective_rho[i]; ofs << effective_rho[i];
+            if ( i < effective_rho.size()-1 ) { cout << ", "; ofs << "\t"; }
+         }
+         cout << endl; ofs << endl;
+         ofs.close();
+      }
+
+      if (visualization)
+      {
+         density.InitializeGLVis(vd);
+         density.DisplayToGLVis();
+      }
+      if ( visit )
+      {
+         density.WriteVisItFields(oss_prefix.str(), "Density");
+      }
+      delete pmesh_rho;
+   }
+
+   if (stiffnessCalc)
+   {
+      double lambda = E * nu / ( (1.0 + nu) * (1.0 - 2.0 * nu) );
+      double mu = 0.5 * E / (1.0 + nu);
+      double mat_scale = 1.0e-6;
+
+      LatticeCoefficient lambdaCoef(*bravais, lcf, lambda * mat_scale, lambda);
+      LatticeCoefficient muCoef(*bravais, lcf, mu * mat_scale, mu);
+      /*
+      meta_material::StiffnessTensor elasticity(*pmesh,
+                                                bravais->GetUnitCellVolume(),
+                                                lambda * mat_scale, mu * mat_scale,
+                                                lambda, mu);
+
+      elasticity.SetVolumeFraction(*vf0);
+      */
+      Mesh * mesh_C = bravais->GetPeriodicWignerSeitzMesh();
+
+      if ( mesh_C->EulerNumber() != 0 )
+      {
+         MFEM_ABORT("Euler number equal to " << mesh_C->EulerNumber()
+                    << ". Periodic Bravais Lattice meshes "
+                    "should have Euler number 0!");
+      }
+
+      // mesh_C->UniformRefinement();
+      mesh_C->UniformRefinement();
+      mesh_C->EnsureNCMesh();
+
+      ParMesh *pmesh_C = new ParMesh(MPI_COMM_WORLD, *mesh_C);
+      delete mesh_C;
+
+      meta_material::StiffnessTensor elasticity(*pmesh_C,
+                                                bravais->GetUnitCellVolume(),
+                                                lambdaCoef, muCoef,
+                                                stiffness_tol);
+
+      vector<double> elas;
+      elasticity.GetHomogenizedProperties(elas);
+      if ( myid == 0 )
+      {
+         ostringstream oss;
+         oss << oss_prefix.str() << "/stiffness_tensor.dat";
+         ofstream ofs(oss.str().c_str());
+
+         cout << "Effective Elasticity Tensor:  " << endl;
+         int k = 0;
+         for (unsigned int i=0; i<6; i++)
+         {
+            for (unsigned int j=0; j<i; j++)
+            {
+               cout << " -----------";
+               ofs << elas[(11 - j) * j / 2 + i] << "\t";
+            }
+            for (unsigned int j=i; j<6; j++)
+            {
+               cout << " " << elas[k]; ofs << elas[k];
+               if ( k < 20 ) { ofs << "\t"; }
+               k++;
+            }
+            cout << endl; ofs << endl;
+         }
+         cout << endl;
+         ofs.close();
+      }
+      if (visualization)
+      {
+         elasticity.InitializeGLVis(vd);
+         elasticity.DisplayToGLVis();
+      }
+      if ( visit )
+      {
+         elasticity.WriteVisItFields(oss_prefix.str(), "StiffnessTensor");
+      }
+      delete pmesh_C;
+   }
+
+   if (bandGapCalc)
+   {
+      LatticeCoefficient epsCoef(*bravais, lcf, 1.0, epsRel);
+      LatticeCoefficient  muCoef(*bravais, lcf, 1.0,  muRel);
+
+      Mesh * mesh_bg = bravais->GetPeriodicWignerSeitzMesh();
+
+      if ( mesh_bg->EulerNumber() != 0 )
+      {
+         MFEM_ABORT("Euler number equal to " << mesh_bg->EulerNumber()
+                    << ". Periodic Bravais Lattice meshes "
+                    "should have Euler number 0!");
+      }
+      // mesh_bg->UniformRefinement();
+
+      ParMesh *pmesh_bg = new ParMesh(MPI_COMM_WORLD, *mesh_bg);
+      delete mesh_bg;
+
+      meta_material::MaxwellBandGap maxwell_bg(*pmesh_bg, *bravais,
+                                               band_gap_samp_pow,
+                                               epsCoef, muCoef,
+                                               band_gap_mid_pts,
+                                               band_gap_max_ref,
+                                               band_gap_tol);
+
+      vector<double> bg;
+      maxwell_bg.GetHomogenizedProperties(bg);
+
+      if ( myid == 0 )
+      {
+         ostringstream oss;
+         oss << oss_prefix.str() << "/disp.dat";
+         ofstream ofs(oss.str().c_str());
+
+         maxwell_bg.PrintDispersionPlot(ofs);
+         ofs.close();
+      }
+      if (visualization)
+      {
+         maxwell_bg.InitializeGLVis(vd);
+         maxwell_bg.DisplayToGLVis();
+      }
+      if ( visit )
+      {
+         maxwell_bg.WriteVisItFields(oss_prefix.str(), "MaxwellBandGap");
+      }
+      delete pmesh_bg;
+   }
+
+   // delete vf0;
+   // delete vf1;
+   // delete L2FESpace;
+   // delete pmesh;
+   delete bravais;
+
+   MPI_Finalize();
+
+   if ( myid == 0 )
+   {
+      cout << "Exiting Main" << endl;
+   }
+
+   return 0;
+}
+
+int CreateDirectory(const string &dir_name, MPI_Comm & comm, int myid)
+{
+   int err;
+#ifndef MFEM_USE_MPI
+   err = mkdir(dir_name.c_str(), 0775);
+   err = (err && (errno != EEXIST)) ? 1 : 0;
+#else
+   if (myid == 0)
+   {
+      err = mkdir(dir_name.c_str(), 0775);
+      err = (err && (errno != EEXIST)) ? 1 : 0;
+      MPI_Bcast(&err, 1, MPI_INT, 0, comm);
+   }
+   else
+   {
+      // Wait for rank 0 to create the directory
+      MPI_Bcast(&err, 1, MPI_INT, 0, comm);
+   }
+#endif
+   return err;
+}
+
+double
+distToLine(double ox, double oy, double oz,
+           double tx, double ty, double tz, const Vector & x)
+{
+   double xo_data[3];
+   double xt_data[3];
+   Vector xo(xo_data, 3);
+   Vector xt(xt_data, 3);
+
+   // xo = x - {ox,oy,oz}
+   xo[0] = x[0] - ox;
+   xo[1] = x[1] - oy;
+   xo[2] = x[2] - oz;
+
+   // xt = cross_product({tx,ty,tz}, xo)
+   xt[0] = ty * xo[2] - tz * xo[1];
+   xt[1] = tz * xo[0] - tx * xo[2];
+   xt[2] = tx * xo[1] - ty * xo[0];
+
+   return xt.Norml2();
+}
+double
+vol_frac_coef(const Vector & x)
+{
+   switch ( prob_ )
+   {
+      case -1:
+         // Uniform
+         return 1.0;
+         break;
+      case 0:
+         // Slab
+         if ( fabs(x(0)) <= 0.25 ) { return 1.0; }
+         break;
+      case 1:
+         // Cylinder
+         if ( sqrt(x(0)*x(0)+x(1)*x(1)) <= 0.5 ) { return 1.0; }
+         break;
+      case 2:
+         // Sphere
+         if ( x.Norml2() <= 0.5 ) { return 1.0; }
+         break;
+      case 3:
+         // Sphere and 3 Rods
+      {
+         double r1 = 0.14, r2 = 0.36, r3 = 0.105;
+         if ( x.Norml2() <= r1 ) { return 0.0; }
+         if ( x.Norml2() <= r2 ) { return 1.0; }
+         if ( sqrt(x(1)*x(1)+x(2)*x(2)) <= r3 ) { return 1.0; }
+         if ( sqrt(x(2)*x(2)+x(0)*x(0)) <= r3 ) { return 1.0; }
+         if ( sqrt(x(0)*x(0)+x(1)*x(1)) <= r3 ) { return 1.0; }
+      }
+      break;
+      case 4:
+         // Sphere and 4 Rods
+      {
+         double r1 = 0.14, r2 = 0.28, r3 = 0.1;
+         if ( x.Norml2() <= r1 ) { return 0.0; }
+         if ( x.Norml2() <= r2 ) { return 1.0; }
+
+         Vector y = x;
+         y[0] -= 0.5; y[1] -= 0.5; y[2] -= 0.5;
+         if ( y.Norml2() <= r1 ) { return 0.0; }
+         if ( y.Norml2() <= r2 ) { return 1.0; }
+         y = x; y[0] -= 0.5; y[1] -= 0.5; y[2] += 0.5;
+         if ( y.Norml2() <= r1 ) { return 0.0; }
+         if ( y.Norml2() <= r2 ) { return 1.0; }
+         y = x; y[0] -= 0.5; y[1] += 0.5; y[2] -= 0.5;
+         if ( y.Norml2() <= r1 ) { return 0.0; }
+         if ( y.Norml2() <= r2 ) { return 1.0; }
+         y = x; y[0] -= 0.5; y[1] += 0.5; y[2] += 0.5;
+         if ( y.Norml2() <= r1 ) { return 0.0; }
+         if ( y.Norml2() <= r2 ) { return 1.0; }
+         y = x; y[0] += 0.5; y[1] -= 0.5; y[2] -= 0.5;
+         if ( y.Norml2() <= r1 ) { return 0.0; }
+         if ( y.Norml2() <= r2 ) { return 1.0; }
+         y = x; y[0] += 0.5; y[1] -= 0.5; y[2] += 0.5;
+         if ( y.Norml2() <= r1 ) { return 0.0; }
+         if ( y.Norml2() <= r2 ) { return 1.0; }
+         y = x; y[0] += 0.5; y[1] += 0.5; y[2] -= 0.5;
+         if ( y.Norml2() <= r1 ) { return 0.0; }
+         if ( y.Norml2() <= r2 ) { return 1.0; }
+         y = x; y[0] += 0.5; y[1] += 0.5; y[2] += 0.5;
+         if ( y.Norml2() <= r1 ) { return 0.0; }
+         if ( y.Norml2() <= r2 ) { return 1.0; }
+
+         double a = r3;
+         double b = 1.0/sqrt(3.0);
+         if ( distToLine(0.0, 0.0, 0.0, b, b, b, x) <= a ) { return 1.0; }
+         if ( distToLine(0.0, 0.0, 0.0,-b, b, b, x) <= a ) { return 1.0; }
+         if ( distToLine(0.0, 0.0, 0.0,-b,-b, b, x) <= a ) { return 1.0; }
+         if ( distToLine(0.0, 0.0, 0.0, b,-b, b, x) <= a ) { return 1.0; }
+      }
+      break;
+      case 5:
+         // Two spheres in a BCC configuration
+         if ( x.Norml2() <= 0.3 )
+         {
+            return 1.0;
+         }
+         else
+         {
+            for (int i=0; i<8; i++)
+            {
+               int i1 = i%2;
+               int i2 = (i/2)%2;
+               int i4 = i/4;
+
+               Vector u = x;
+               u(0) -= i1?-0.5:0.5;
+               u(1) -= i2?-0.5:0.5;
+               u(2) -= i4?-0.5:0.5;
+
+               if ( u.Norml2() <= 0.2 ) { return 1.0; }
+            }
+         }
+         break;
+      case 6:
+         // Sphere and 6 Rods
+      {
+         double r1 = 0.12, r2 = 0.19, r3 = 0.08;
+         if ( x.Norml2() <= r1 ) { return 0.0; }
+         if ( x.Norml2() <= r2 ) { return 1.0; }
+
+         Vector y = x;
+         y[0] -= 0.5; y[1] -= 0.5;
+         if ( y.Norml2() <= r1 ) { return 0.0; }
+         if ( y.Norml2() <= r2 ) { return 1.0; }
+         y = x; y[0] -= 0.5; y[1] += 0.5;
+         if ( y.Norml2() <= r1 ) { return 0.0; }
+         if ( y.Norml2() <= r2 ) { return 1.0; }
+         y = x; y[0] += 0.5; y[1] -= 0.5;
+         if ( y.Norml2() <= r1 ) { return 0.0; }
+         if ( y.Norml2() <= r2 ) { return 1.0; }
+         y = x; y[0] += 0.5; y[1] += 0.5;
+         if ( y.Norml2() <= r1 ) { return 0.0; }
+         if ( y.Norml2() <= r2 ) { return 1.0; }
+
+         y = x; y[1] -= 0.5; y[2] -= 0.5;
+         if ( y.Norml2() <= r1 ) { return 0.0; }
+         if ( y.Norml2() <= r2 ) { return 1.0; }
+         y = x; y[1] -= 0.5; y[2] += 0.5;
+         if ( y.Norml2() <= r1 ) { return 0.0; }
+         if ( y.Norml2() <= r2 ) { return 1.0; }
+         y = x; y[1] += 0.5; y[2] -= 0.5;
+         if ( y.Norml2() <= r1 ) { return 0.0; }
+         if ( y.Norml2() <= r2 ) { return 1.0; }
+         y = x; y[1] += 0.5; y[2] += 0.5;
+         if ( y.Norml2() <= r1 ) { return 0.0; }
+         if ( y.Norml2() <= r2 ) { return 1.0; }
+
+         y = x; y[2] -= 0.5; y[0] -= 0.5;
+         if ( y.Norml2() <= r1 ) { return 0.0; }
+         if ( y.Norml2() <= r2 ) { return 1.0; }
+         y = x; y[2] -= 0.5; y[0] += 0.5;
+         if ( y.Norml2() <= r1 ) { return 0.0; }
+         if ( y.Norml2() <= r2 ) { return 1.0; }
+         y = x; y[2] += 0.5; y[0] -= 0.5;
+         if ( y.Norml2() <= r1 ) { return 0.0; }
+         if ( y.Norml2() <= r2 ) { return 1.0; }
+         y = x; y[2] += 0.5; y[0] += 0.5;
+         if ( y.Norml2() <= r1 ) { return 0.0; }
+         if ( y.Norml2() <= r2 ) { return 1.0; }
+
+         double a = r3;
+         double b = 1.0/sqrt(2.0);
+         if ( distToLine(0.0, 0.0, 0.0, b, b, 0, x) <= a ) { return 1.0; }
+         if ( distToLine(0.0, 0.0, 0.0, b,-b, 0, x) <= a ) { return 1.0; }
+         if ( distToLine(0.0, 0.0, 0.0, b, 0, b, x) <= a ) { return 1.0; }
+         if ( distToLine(0.0, 0.0, 0.0, b, 0,-b, x) <= a ) { return 1.0; }
+         if ( distToLine(0.0, 0.0, 0.0, 0, b, b, x) <= a ) { return 1.0; }
+         if ( distToLine(0.0, 0.0, 0.0, 0, b,-b, x) <= a ) { return 1.0; }
+
+      }
+      break;
+      case 7:
+         if ( fabs(x(1)) + fabs(x(2)) < 0.25 ||
+              fabs(x(0)) + fabs(x(2)) < 0.25 ||
+              fabs(x(0)) + fabs(x(1) - 0.5) < 0.25 )
+         {
+            return 1.0;
+         }
+         break;
+   }
+   return 0.0;
+}

miniapps/meta-material/meta_material_solver.hpp

@@ -0,0 +1,731 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_META_MATERIAL_SOLVER
+#define MFEM_META_MATERIAL_SOLVER
+
+#include "../../config/config.hpp"
+
+#ifdef MFEM_USE_MPI
+
+#include "../common/pfem_extras.hpp"
+#include "../common/bravais.hpp"
+
+extern "C"
+{
+   void dsygv_(int *ITYPE, char *JOBZ, char *UPLO, int *N,
+               double *A, int * LDA, double *B, int *LDB, double *W,
+               double *WORK, int *LWORK, int *INFO);
+}
+
+namespace mfem
+{
+
+using common::H1_ParFESpace;
+using common::ND_ParFESpace;
+using common::RT_ParFESpace;
+using common::L2_ParFESpace;
+using common::ParDiscreteGradOperator;
+using common::ParDiscreteCurlOperator;
+using common::ParDiscreteInterpolationOperator;
+using common::VisData;
+using bravais::BravaisLattice;
+
+namespace meta_material
+{
+
+class LinearCoefficient : public GridFunctionCoefficient
+{
+public:
+   LinearCoefficient(GridFunction * gf, double a0, double a1);
+
+   virtual double Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip);
+
+   double GetSensitivity(ElementTransformation &T,
+                         const IntegrationPoint &ip)
+   { return (c1_ - c0_); }
+
+private:
+   double c0_;
+   double c1_;
+};
+/*
+class PenaltyCoefficient : public GridFunctionCoefficient
+{
+public:
+   PenaltyCoefficient(GridFunction * gf, int penalty, double a0, double a1);
+
+   virtual double Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip);
+
+   double GetSensitivity(ElementTransformation &T,
+                         const IntegrationPoint &ip);
+
+private:
+   int penalty_;
+   double c0_;
+   double c1_;
+};
+*/
+class Homogenization
+{
+public:
+   Homogenization(MPI_Comm comm);
+   virtual ~Homogenization() {}
+
+   void SetVolumeFraction(ParGridFunction & vf) { vf_ = &vf; newVF_ = true; }
+
+   virtual void GetHomogenizedProperties(std::vector<double> & p) = 0;
+   virtual void GetPropertySensitivities(std::vector<ParGridFunction> & dp)
+      = 0;
+
+   void InitializeGLVis(VisData & vd) {}
+
+   void DisplayToGLVis() {}
+
+   void WriteVisItFields(const std::string & prefix,
+                         const std::string & label) {}
+
+protected:
+   MPI_Comm comm_;
+   int      myid_;
+   int      numProcs_;
+   bool     newVF_;
+
+   ParGridFunction * vf_;
+};
+
+class Density : public Homogenization
+{
+public:
+   // Density(ParMesh & pmesh, double vol,
+   //       double rho0, double rho1);
+   Density(ParMesh & pmesh, double refDensity, double vol,
+           Coefficient &rhoCoef, double tol = 0.05);
+   ~Density();
+
+   // void SetVolumeFraction(ParGridFunction & vf);
+
+   void GetHomogenizedProperties(std::vector<double> & p);
+   void GetPropertySensitivities(std::vector<ParGridFunction> & dp) {}
+
+   void InitializeGLVis(VisData & vd);
+
+   void DisplayToGLVis();
+
+   void WriteVisItFields(const std::string & prefix,
+                         const std::string & label);
+
+private:
+   void updateRho();
+
+   ParMesh         * pmesh_;
+   L2_ParFESpace   * L2FESpace_;
+   ParGridFunction * rho_;
+   ParGridFunction * divGradRho_;
+   ParLinearForm   * cellVol_;
+
+   // LinearCoefficient   rhoCoef_;
+   Coefficient * rhoCoef_;
+   ConstantCoefficient one_;
+
+   double refDensity_;
+   double vol_;
+   double tol_;
+
+   VisData      * vd_;
+   socketstream * sock_;
+   socketstream * sock2_;
+};
+
+class StiffnessTensor : public Homogenization
+{
+public:
+   /*
+   StiffnessTensor(ParMesh & pmesh, double vol,
+                    double lambda0, double mu0,
+                    double lambda1, double mu1);
+   */
+   StiffnessTensor(ParMesh & pmesh, double vol,
+                   Coefficient &lambdaCoef, Coefficient &muCoef,
+                   double tol = 0.05);
+   ~StiffnessTensor();
+
+   // void SetVolumeFraction(ParGridFunction & vf);
+
+   void GetHomogenizedProperties(std::vector<double> & p);
+   void GetPropertySensitivities(std::vector<ParGridFunction> & dp) {}
+
+   void InitializeGLVis(VisData & vd);
+
+   void DisplayToGLVis();
+
+   void WriteVisItFields(const std::string & prefix,
+                         const std::string & label);
+
+private:
+
+   void TensorGradient(const Vector & x, Vector & y);
+   void TensorGradientTranspose(const Vector & x, Vector & y);
+   void TensorMassMatrix(const Vector & x, Vector & y);
+   void RestrictedTensorMassMatrix(int r, const Vector & x, Vector & y);
+   void RestrictedVectorAdd(int r, const Vector & x, Vector & y);
+
+   // Produces a 2-tensor coefficient for the diagonal portion of an
+   // elasticity operator.  The coefficient is defined by:
+   //     / lambda + 2 * mu, i == j == axis
+   // K = |              mu, i == j != axis
+   //     \               0, i != j
+   class DiagElasticityCoef : public MatrixCoefficient
+   {
+   public:
+      DiagElasticityCoef(Coefficient & lambda,
+                         Coefficient & mu,
+                         int axis);
+
+      virtual void Eval(DenseMatrix &K, ElementTransformation &T,
+                        const IntegrationPoint &ip);
+
+   private:
+      int axis_;
+      Coefficient * lambda_;
+      Coefficient * mu_;
+   };
+
+   // Produces a 2-tensor coefficient for the off-diagonal portion of an
+   // elasticity operator.  The coefficient is defined by:
+   //     / lambda, i == axis0, j == axis1
+   // K = |     mu, i == axis1, j == axis0
+   //     \      0, otherwise
+   class OffDiagElasticityCoef : public MatrixCoefficient
+   {
+   public:
+      OffDiagElasticityCoef(Coefficient & lambda,
+                            Coefficient & mu,
+                            int axis0, int axis1);
+
+      virtual void Eval(DenseMatrix &K, ElementTransformation &T,
+                        const IntegrationPoint &ip);
+
+   private:
+      int axis0_;
+      int axis1_;
+      Coefficient * lambda_;
+      Coefficient * mu_;
+   };
+
+   void solve(const Vector & E, Vector & Chi);
+
+   int dim_;
+   int irOrder_;
+   int geom_;
+   bool amg_elast_;
+
+   ParMesh * pmesh_;
+
+   L2_ParFESpace * L2FESpace_;
+   L2_ParFESpace * L2VFESpace_;
+   H1_ParFESpace * H1FESpace_;
+   H1_ParFESpace * H1VFESpace_;
+   ND_ParFESpace * HCurlFESpace_;
+   ND_ParFESpace * HCurlVFESpace_;
+   RT_ParFESpace * HDivFESpace_;
+
+   // LinearCoefficient lambdaCoef_;
+   // LinearCoefficient muCoef_;
+   Coefficient * lambdaCoef_;
+   Coefficient * muCoef_;
+
+   DiagElasticityCoef    xxCoef_;
+   DiagElasticityCoef    yyCoef_;
+   DiagElasticityCoef    zzCoef_;
+   OffDiagElasticityCoef yzCoef_;
+   OffDiagElasticityCoef xzCoef_;
+   OffDiagElasticityCoef xyCoef_;
+
+   ParGridFunction * lambda_;
+   ParGridFunction * mu_;
+
+   ParBilinearForm * a_;
+   ParBilinearForm * m_[6];
+   ParGridFunction * Chi_[6];
+   ParGridFunction * E_[3];
+   ParGridFunction * F_[6];
+   ParLinearForm   * MF_[6];
+
+   ParDiscreteGradOperator * grad_;
+
+   BilinearFormIntegrator * diffInteg_[3];
+   ParGridFunction * errSol_;
+   ErrorEstimator  * errEst_[3];
+   ParGridFunction * errors_[4];
+
+   ParGridFunction * b_;
+   // HypreParVector  * tmp1_;
+
+   double vol_;
+   double tol_;
+
+   VisData      * vd_;
+   socketstream   socks_[8];
+   socketstream   err_socks_[4];
+
+   int seqVF_;
+};
+
+class ParDiscreteVectorProductOperator
+   : public ParDiscreteInterpolationOperator
+{
+public:
+   ParDiscreteVectorProductOperator(ParFiniteElementSpace *dfes,
+                                    ParFiniteElementSpace *rfes,
+                                    const Vector & v);
+
+private:
+   VectorConstantCoefficient vCoef_;
+};
+
+class ParDiscreteVectorCrossProductOperator
+   : public ParDiscreteInterpolationOperator
+{
+public:
+   ParDiscreteVectorCrossProductOperator(ParFiniteElementSpace *dfes,
+                                         ParFiniteElementSpace *rfes,
+                                         const Vector & v);
+private:
+   VectorConstantCoefficient vCoef_;
+};
+
+//class MaxwellBlochWaveEquation;
+//class MaxwellBlochWaveEquation::MaxwellBlochWaveProjector;
+
+class MaxwellBlochWaveEquation
+{
+public:
+   MaxwellBlochWaveEquation(ParMesh & pmesh,
+                            int order);
+   ~MaxwellBlochWaveEquation();
+
+   // Where kappa is the phase shift vector
+   void SetKappa(const Vector & kappa);
+
+   // Where beta*zeta = kappa
+   void SetBeta(double beta);
+   void SetZeta(const Vector & zeta);
+
+   // void SetAzimuth(double alpha_a);
+   // void SetInclination(double alpha_i);
+
+   // void SetOmega(double omega);
+   void SetAbsoluteTolerance(double atol);
+   void SetNumEigs(int nev);
+   void SetMassCoef(Coefficient & m);
+   void SetStiffnessCoef(Coefficient & k);
+
+   void Setup();
+
+   void SetInitialVectors(int num_vecs, HypreParVector ** vecs);
+
+   // void SetBravaisLattice(BravaisLattice & bravais) { bravais_ = &bravais; }
+
+   void Update();
+
+   /// Solve the eigenproblem
+   void Solve();
+
+   /// Collect the converged eigenvalues
+   void GetEigenvalues(std::vector<double> & eigenvalues);
+
+   /// A convenience method which combines six methods into one
+   void GetEigenvalues(int nev, const Vector & kappa,
+                       std::vector<HypreParVector*> & init_vecs,
+                       std::vector<double> & eigenvalues);
+
+   /// Extract a single eigenvector
+   HypreParVector * ReturnEigenvector(unsigned int i);
+
+   void CopyEigenvector(unsigned int i,
+                        HypreParVector & V);
+
+   void GetEigenvector(unsigned int i,
+                       HypreParVector & Er,
+                       HypreParVector & Ei,
+                       HypreParVector & Br,
+                       HypreParVector & Bi);
+   void GetEigenvectorE(unsigned int i,
+                        HypreParVector & Er,
+                        HypreParVector & Ei);
+   void GetEigenvectorB(unsigned int i,
+                        HypreParVector & Br,
+                        HypreParVector & Bi);
+
+   BlockOperator * GetAOperator() { return A_; }
+   BlockOperator * GetMOperator() { return M_; }
+
+   Solver   * GetPreconditioner() { return Precond_; }
+   Operator * GetSubSpaceProjector() { return SubSpaceProj_; }
+
+   ParFiniteElementSpace * GetH1FESpace() { return H1FESpace_; }
+   ParFiniteElementSpace * GetHCurlFESpace() { return HCurlFESpace_; }
+   ParFiniteElementSpace * GetHDivFESpace()  { return HDivFESpace_; }
+
+   // void TestVector(const HypreParVector & v);
+
+   ParGridFunction * GetEigenvectorEnergy(unsigned int i) { return energy_[i]; }
+
+   void GetFourierCoefficients(HypreParVector & Vr,
+                               HypreParVector & Vi,
+                               Array2D<double> &f);
+
+   void IdentifyDegeneracies(double zero_tol, double rel_tol,
+                             std::vector<std::set<int> > & degen);
+
+   void GetFieldAverages(unsigned int i,
+                         Vector & Er, Vector & Ei,
+                         Vector & Br, Vector & Bi,
+                         Vector & Dr, Vector & Di,
+                         Vector & Hr, Vector & Hi);
+
+   void ComputeHomogenizedCoefs();
+
+   void DetermineBasis(const Vector & v1, std::vector<Vector> & e);
+
+   void WriteVisitFields(const std::string & prefix,
+                         const std::string & label);
+
+   void GetSolverStats(double &meanTime, double &stdDevTime,
+                       double &meanIter, double &stdDevIter,
+                       int &nSolves);
+
+   void TestProjector() const;
+
+private:
+
+   MPI_Comm comm_;
+   int myid_;
+   int hcurl_loc_size_;
+   int hdiv_loc_size_;
+   int nev_;
+
+   bool newBeta_;
+   bool newZeta_;
+   bool newOmega_;
+   bool newMCoef_;
+   bool newKCoef_;
+
+   ParMesh        * pmesh_;
+   H1_ParFESpace  * H1FESpace_;
+   ND_ParFESpace  * HCurlFESpace_;
+   RT_ParFESpace  * HDivFESpace_;
+   // L2_ParFESpace  * L2FESpace_;
+
+   // BravaisLattice     * bravais_;
+   // HCurlFourierSeries * fourierHCurl_;
+
+   double           atol_;
+   double           beta_;
+
+   Vector           zeta_;
+   Vector           kappa_;
+
+   Coefficient    * mCoef_;
+   Coefficient    * kCoef_;
+
+   Array<int>       block_offsets_;
+   Array<int>       block_trueOffsets_;
+   Array<int>       block_trueOffsets2_;
+   Array<HYPRE_Int> tdof_offsets_;
+
+   BlockOperator  * A_;
+   BlockOperator  * M_;
+   BlockOperator  * C_;
+
+   BlockVector    * blkHCurl_;
+   BlockVector    * blkHDiv_;
+
+   HypreParMatrix * M1_;
+   HypreParMatrix * M2_;
+   HypreParMatrix * S1_;
+   HypreParMatrix * T1_;
+   HypreParMatrix * T12_;
+   HypreParMatrix * Z12_;
+
+   HypreParMatrix * DKZ_;
+   // HypreParMatrix * DKZT_;
+
+   HypreAMS       * T1Inv_;
+
+   ParDiscreteCurlOperator * Curl_;
+   ParDiscreteVectorCrossProductOperator * Zeta_;
+
+   BlockDiagonalPreconditioner * BDP_;
+
+   Solver   * Precond_;
+   //MaxwellBlochWaveProjector * SubSpaceProj_;
+   Operator * SubSpaceProj_;
+
+   HypreParVector ** vecs_;
+   HypreParVector * vec0_;
+
+   HypreLOBPCG * lobpcg_;
+   HypreAME    * ame_;
+
+   ParGridFunction ** energy_;
+   /*
+    HypreParVector * AvgHCurl_coskx_[3];
+    HypreParVector * AvgHCurl_sinkx_[3];
+    HypreParVector * AvgHDiv_coskx_[3];
+    HypreParVector * AvgHDiv_sinkx_[3];
+
+    HypreParVector * AvgHCurl_eps_coskx_[3];
+    HypreParVector * AvgHCurl_eps_sinkx_[3];
+    HypreParVector * AvgHDiv_muInv_coskx_[3];
+    HypreParVector * AvgHDiv_muInv_sinkx_[3];
+   */
+   std::vector<double> solve_times_;
+   std::vector<int>    solve_iters_;
+
+   class MaxwellBlochWavePrecond : public Solver
+   {
+   public:
+      MaxwellBlochWavePrecond(ParFiniteElementSpace & HCurlFESpace,
+                              BlockDiagonalPreconditioner & BDP,
+                              Operator & subSpaceProj,
+                              double w);
+
+      ~MaxwellBlochWavePrecond();
+
+      void Mult(const Vector & x, Vector & y) const;
+
+      void SetOperator(const Operator & A);
+
+   private:
+      int myid_;
+
+      // ParFiniteElementSpace * HCurlFESpace_;
+      BlockDiagonalPreconditioner * BDP_;
+      const Operator * A_;
+      Operator * subSpaceProj_;
+      // mutable HypreParVector *r_, *u_, *v_;
+      mutable HypreParVector *u_;
+      // double w_;
+   };
+
+   class MaxwellBlochWaveProjector : public Operator
+   {
+   public:
+      MaxwellBlochWaveProjector(ParFiniteElementSpace & HCurlFESpace,
+                                ParFiniteElementSpace & H1FESpace,
+                                BlockOperator & M,
+                                double beta, const Vector & zeta);
+      ~MaxwellBlochWaveProjector();
+
+      void SetBeta(double beta);
+      void SetZeta(const Vector & zeta);
+
+      void Setup();
+      virtual void Mult(const Vector &x, Vector &y) const;
+
+   private:
+      int myid_;
+      int locSize_;
+
+      bool newBeta_;
+      bool newZeta_;
+
+      ParFiniteElementSpace * HCurlFESpace_;
+      ParFiniteElementSpace * H1FESpace_;
+
+      double beta_;
+      Vector zeta_;
+
+      HypreParMatrix * T01_;
+      HypreParMatrix * Z01_;
+      HypreParMatrix * A0_;
+      HypreParMatrix * DKZ_;
+
+      MINRESSolver   * minres_;
+
+      Array<int>       block_offsets0_;
+      Array<int>       block_offsets1_;
+      Array<int>       block_trueOffsets0_;
+      Array<int>       block_trueOffsets1_;
+
+      BlockOperator * S0_;
+      BlockOperator * M_;
+      BlockOperator * G_;
+
+      mutable HypreParVector * urDummy_;
+      mutable HypreParVector * uiDummy_;
+      mutable HypreParVector * vrDummy_;
+      mutable HypreParVector * viDummy_;
+
+      mutable BlockVector * u0_;
+      mutable BlockVector * v0_;
+      mutable BlockVector * u1_;
+      mutable BlockVector * v1_;
+   };
+};
+
+class InverseCoefficient : public TransformedCoefficient
+{
+public:
+   InverseCoefficient(Coefficient * q) : TransformedCoefficient(q, inv_) {}
+private:
+   static double inv_(double v) { return 1.0/v; }
+};
+
+class MaxwellBlochWaveSolver
+{
+public:
+   MaxwellBlochWaveSolver(ParMesh & pmesh, BravaisLattice & bravais,
+                          Coefficient & epsCoef, Coefficient & muCoef,
+                          int max_ref = 2, int nev = 24, double tol = 0.05);
+   ~MaxwellBlochWaveSolver();
+
+   // Where kappa is the phase shift vector
+   void SetKappa(const Vector & kappa);
+
+   // Where beta*zeta = kappa and |zeta| = 1
+   void SetBeta(double beta);
+   void SetZeta(const Vector & zeta);
+
+   void GetEigenfrequencies(std::vector<double> & omega);
+
+   MaxwellBlochWaveEquation * GetFineSolver()
+   { return mbwe_[mbwe_.size()-1]; }
+
+   HypreParVector * ReturnFineEigenvector(int i);
+
+   void InitializeGLVis(VisData & vd);
+
+   void DisplayToGLVis();
+
+   void WriteVisItFields(const std::string & prefix,
+                         const std::string & label);
+
+private:
+
+   void createPartitioning(ParFiniteElementSpace & pfes, HYPRE_Int *& part);
+
+   int max_lvl_;
+   int nev_;
+   double tol_;
+
+   std::vector<ParMesh*> pmesh_;
+   std::vector<MaxwellBlochWaveEquation*> mbwe_;
+   std::vector<const Operator*> refineOp_;
+   std::vector<std::pair<HypreParVector*, HypreParVector*> > EField_;
+   //std::vector<std::vector<std::pair<ParGridFunction,
+   //                ParGridFunction> > > efield_;
+   std::vector<std::pair<ParGridFunction*, ParGridFunction*> > efield_;
+   std::vector<std::vector<HypreParVector*> > initialVecs_;
+   std::vector<int> locSize_;
+   std::vector<HYPRE_Int*> part_;
+
+   Vector kappa_;
+   Coefficient * epsCoef_;
+   InverseCoefficient muInvCoef_;
+
+   // Coefficient * muCoef_;
+
+};
+
+class MaxwellDispersion
+{
+public:
+   MaxwellDispersion(ParMesh & pmesh, BravaisLattice & bravais,
+                     int sample_power,
+                     Coefficient & epsCoef, Coefficient & muCoef,
+                     bool midPts = false, int max_ref = 2,
+                     int nev = 24, double tol = 0.05);
+   ~MaxwellDispersion();
+
+   const std::vector<std::vector<std::map<int,std::vector<double> > > > &
+   GetDispersionData();
+
+   void PrintDispersionPlot(std::ostream & os);
+
+   void InitializeGLVis(VisData & vd);
+
+   void DisplayToGLVis();
+
+   void WriteVisItFields(const std::string & prefix,
+                         const std::string & label);
+
+private:
+
+   void buildRawBasis();
+
+   void approxEigenfrequencies(std::vector<double> & omega);
+
+   void traverseBrillouinZone();
+
+   std::string modLabel(const std::string & label) const;
+
+   void findAndReplace(const std::string & f, const std::string & r,
+                       std::string & str) const;
+
+   BravaisLattice         * bravais_;
+   MaxwellBlochWaveSolver * mbws_;
+
+   HypreParVector * Ax_;
+   HypreParVector * Mx_;
+
+   DenseMatrix A_;
+   DenseMatrix M_;
+
+   std::vector<HypreParVector*> rawBasis_;
+   std::vector<HypreParVector*> projBasis_;
+
+   std::map<std::string,std::vector<double> > sp_eigs_;
+   std::vector<std::vector<std::map<int,std::vector<double> > > > seg_eigs_;
+
+   int n_pow_;
+   int n_div_;
+   int samp_pow_;
+   int nev_;
+
+   bool midPts_;
+};
+
+class MaxwellBandGap : public Homogenization
+{
+public:
+   MaxwellBandGap(ParMesh & pmesh, BravaisLattice & bravais,
+                  int samp_pow,
+                  Coefficient & epsCoef, Coefficient & muCoef,
+                  bool midPts = false, int max_ref = 2, double tol = 0.05);
+   ~MaxwellBandGap();
+
+   void GetHomogenizedProperties(std::vector<double> & p);
+   void GetPropertySensitivities(std::vector<ParGridFunction> & dp) {}
+
+   void PrintDispersionPlot(std::ostream & os);
+
+   void InitializeGLVis(VisData & vd);
+
+   void DisplayToGLVis();
+
+   void WriteVisItFields(const std::string & prefix,
+                         const std::string & label);
+private:
+
+   MaxwellDispersion * disp_;
+};
+
+} // namespace meta_material
+} // namespace mfem
+
+#endif // MFEM_USE_MPI
+
+#endif // MFEM_META_MATERIAL_SOLVER

tests/gitlab/README.md

@@ -7,8 +7,18 @@
 
                                https://mfem.org
 
+FIXME: this file needs to be updated
+
 This directory contains utility scripts related to GitLab testing at LLNL.
 
+* `build_and_test_setup` is used in CI to setup certain directories before
+  running the tests.
+
+   This script was designed to be used in CI context. It is easier for the sake
+   of reproducibility to clone MFEM `data` and `autotest` repository manually
+   next to your MFEM repository, _unless you are using the inline reproducer
+   provided in CI, in which case the required variables will be set for you._
+
 * `build_and_test` is used in CI to build TPLs (dependencies) and MFEM and to
   perform testing.
 

tests/gitlab/build_and_test

@@ -11,6 +11,12 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
+# Initialize modules for users not using bash as a default shell
+if test -e /usr/share/lmod/lmod/init/bash
+then
+  . /usr/share/lmod/lmod/init/bash
+fi
+
 set -o errexit
 set -o nounset
 
@@ -60,15 +66,40 @@ function usage()
 }
 
 project_dir="$(pwd)"
+hostname="$(hostname)"
 
-mode=""
 spec=""
+mode=""
 data_dir=""
 with_data=false
 
 sys_type=${SYS_TYPE:-""}
-
 threads=${THREADS:-""}
+module_list=${MODULE_LIST:-""}
+job_unique_id=${CI_JOB_ID:-""}
+use_dev_shm=${USE_DEV_SHM:-true}
+spack_debug=${SPACK_DEBUG:-false}
+debug_mode=${DEBUG_MODE:-false}
+
+# CI_REGISTRY_USER, CI_REGISTRY_IMAGE, and CI_JOB_TOKEN are automatically set by
+# Gitlab, see https://docs.gitlab.com/ee/ci/variables/predefined_variables.html.
+# The Spack build cache files can be viewed from the Gitlab web interface under
+# "Deploy" -> "Container Registry".
+#
+# Since the CI_<VARIABLES> are only set in CI, we provide sensible defaults for
+# usage outside CI context (e.g. to reproduce a CI scenario locally).
+#
+# REGISTRY_TOKEN when set allows to provide our own personal access token to
+# the CI registry. Be sure to set the token with at least read access to the
+# registry. It is optional outside CI but will speed up the build if many
+# dependencies are needed.
+#
+registry_token=${REGISTRY_TOKEN:-""}
+ci_registry_user=${CI_REGISTRY_USER:-"${USER}"}
+ci_registry_image=${CI_REGISTRY_IMAGE:-"czregistry.llnl.gov:5050/mfem/mfem-autotest"}
+ci_registry_token=${CI_JOB_TOKEN:-"${registry_token}"}
+
+spec_tab=()
 
 # Options
 while [[ $# -gt 0 ]]
@@ -77,9 +108,19 @@ do
 
     case $key in
         --spec)
-            spec="$2"
-            shift # past argument
-            shift # past value
+            shift
+            arg="${1}"
+            if [[ "${arg}" =~ ^\" ]]; then
+              arg="${arg#\"}"
+              while [[ ! "${arg}" =~ [^\\]\"$ ]]; do
+                spec_tab+=("${arg}")
+                shift
+                arg="${1}"
+              done
+              arg="${arg%\"}"
+            fi
+            spec_tab+=("${arg}")
+            shift
             ;;
         --deps-only|--build-only|--test-only)
             mode="$key"
@@ -99,48 +140,133 @@ do
             exit 0;
             ;;
         *)  # unknown option
-            echo "ERROR: option $key is unknown"
+            echo "[Error]: option $key is unknown"
             exit 1;
             ;;
     esac
 done
 
+spec="${spec_tab[@]}"
+
+timed_message ()
+{
+    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    echo "~ $(date --rfc-3339=seconds) ~ ${1}"
+    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+}
+
+at_exit()
+{
+  if [[ "${debug_mode}" != "true" ]]; then
+    if [[ "${#exit_rm_list[@]}" != 0 ]]; then
+      echo ${script_name}:at_exit: rm -rf "${exit_rm_list[@]}"
+      rm -rf "${exit_rm_list[@]}"
+    fi
+    if [[ "${exit_distclean}" == 1 ]]; then
+      echo ${script_name}:at_exit: \
+        make -C "${project_dir}" distclean "> /dev/null 2>&1"
+      make -C "${project_dir}" distclean > /dev/null 2>&1
+    fi
+  fi
+}
+
+if [[ ${debug_mode} == true ]]
+then
+    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    echo "~~~~~ Debug mode:"
+    echo "~~~~~ - Spack debug mode."
+    echo "~~~~~ - Deactivated shared memory."
+    echo "~~~~~ - Do not push to buildcache."
+    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    use_dev_shm=false
+    spack_debug=true
+fi
+
+if [[ -n ${module_list} ]]
+then
+    timed_message "Modules to load: ${module_list}"
+    module load ${module_list}
+fi
+
+prefix=""
+
+if [[ -d /dev/shm && ${use_dev_shm} == true ]]
+then
+    prefix="/dev/shm/${hostname}"
+    if [[ -z ${job_unique_id} ]]; then
+      job_unique_id=manual_job_$(date +%s)
+      while [[ -d ${prefix}-${job_unique_id} ]] ; do
+          sleep 1
+          job_unique_id=manual_job_$(date +%s)
+      done
+    fi
+
+    prefix="${prefix}-${job_unique_id}"
+else
+    # We set the prefix in the parent directory so that spack dependencies are
+    # not installed inside the source tree.
+    prefix="${project_dir}/../spack-and-build-root"
+fi
+
+echo "Creating directory ${prefix}"
+echo "project_dir: ${project_dir}"
+
+mkdir -p ${prefix}
+
+# List of files/directories to remove at exit; more entries are added below.
+# This list will be removed even if there's an error. In debug mode, this
+# cleanup is not performed.
+exit_rm_list=("${prefix}")
+exit_distclean=
+trap 'at_exit' EXIT HUP INT QUIT ABRT KILL PIPE TERM XCPU
+
+# Prefix tree:
+#   <prefix>
+#    ├── spack
+#    ├── spack_env
+#    └── spack-user-cache
+
+spack_cmd="${prefix}/spack/bin/spack"
+spack_env_path="${prefix}/spack_env"
+command -v python3 > /dev/null && python_cmd=python3 || python_cmd=python
+uberenv_cmd="${python_cmd} tests/uberenv/uberenv.py"
+if [[ ${spack_debug} == true ]]
+then
+    spack_cmd="${spack_cmd} --debug --stacktrace"
+    uberenv_cmd="${uberenv_cmd} --spack-debug"
+fi
+
 # Dependencies
 if [[ "${mode}" != "--build-only" && "${mode}" != "--test-only" ]]
 then
-    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
-    echo "~~~~~ Building Dependencies"
-    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    timed_message "Building dependencies"
 
     if [[ -z ${spec} ]]
     then
-        echo "--spec is mandatory to build dependencies, aborting..."
+        echo "[Error]: SPEC is undefined, aborting..."
         exit 1
     fi
 
-    upstream_opt=""
-    if [[ ${sys_type} != "" ]]
-    then
-        upstream_opt="--upstream=/usr/workspace/mfem/mfem-dependencies/install"
-    fi
+    prefix_opt="--prefix=${prefix}"
 
-    prefix_opt=""
-
-    # Do not run on /dev/shm if only installing dependencies: they will be lost
-    # otherwise.
-    if [[ -d "/dev/shm" && "${mode}" != "--deps-only" ]]
-    then
-        prefix="/dev/shm/${USER}_${CI_PIPELINE_ID:-"NONE"}_${RANDOM}"
-        mkdir -p ${prefix}
-        # clean up ${prefix} at exit:
-        trap 'rm -rf "${prefix}"' EXIT
-        prefix_opt="--prefix=${prefix}"
-    fi
     echo ${spec} > spec.txt
 
-    echo "Fetching uberenv."
+    # We force Spack to put all generated files (cache and configuration of
+    # all sorts) in a unique location so that there can be no collision
+    # with existing or concurrent Spack.
+    spack_user_cache="${prefix}/spack-user-cache"
+    export SPACK_DISABLE_LOCAL_CONFIG=""
+    export SPACK_USER_CACHE_PATH="${spack_user_cache}"
+    mkdir -p ${spack_user_cache}
+
+    echo "Fetching uberenv ..."
+    # FIXME: the messages from the next clone appear out-of-place, at the end of
+    #        the logs on Gitlab. (On Lassen the messages are where expected.)
     tests/gitlab/get_mfem_uberenv || { echo "Error fetching Uberenv"; exit 1; }
 
+    # Remove "${project_dir}/tests/uberenv" at exit (even if there's an error)
+    exit_rm_list=("${exit_rm_list[@]}" "${project_dir}/tests/uberenv")
+
     echo "Removing existing configuration"
     make distclean
 
@@ -149,10 +275,38 @@ then
         config/defaults.mk.new
     mv -f config/defaults.mk.new config/defaults.mk
 
-    echo "Installing dependencies."
-    command -v python > /dev/null && python_cmd=python || python_cmd=python3
-    $python_cmd tests/uberenv/uberenv.py --spec="${spec}" "${upstream_opt}" \
-        "${prefix_opt}"
+    # generate cmake cache file with uberenv and radiuss spack package
+    timed_message "Spack setup and environment"
+    ${uberenv_cmd} --setup-and-env-only --spec="${spec}" ${prefix_opt}
+
+    if [[ -n ${ci_registry_token} ]]
+    then
+        timed_message "GitLab registry as Spack Buildcache"
+        ${spack_cmd} -D ${spack_env_path} mirror add --unsigned \
+          --oci-username ${ci_registry_user} \
+          --oci-password ${ci_registry_token} \
+          gitlab_ci oci://${ci_registry_image}
+    fi
+
+    timed_message "Spack build of dependencies"
+    ${uberenv_cmd} --skip-setup-and-env --spec="${spec}" ${prefix_opt} || \
+    {
+      # Save Spack logs
+      cd ${SPACK_USER_CACHE_PATH}
+      log_archive="${project_dir}/spack-logs-${hostname}-${CI_JOB_ID}.tgz"
+      tar zvcf "${log_archive}" stage/*/*/spack*.txt
+      timed_message "Spack build failed! See log archive: ${log_archive}"
+      exit 1
+    }
+
+    if [[ -n ${ci_registry_token} && ${debug_mode} == false ]]
+    then
+        timed_message "Push dependencies to buildcache"
+        ${spack_cmd} -D ${spack_env_path} buildcache push \
+          --only dependencies gitlab_ci
+    fi
+
+    timed_message "Dependencies built"
 
     # Make sure that a configuration was generated by spack (part 1).
     cp config/config.mk config/spack-config.mk
@@ -170,10 +324,10 @@ then
         cp config/spack-config.mk config/config.mk
         cp config/spack_config.hpp config/_config.hpp
     else
-        echo "No result for at least one of"
-        echo "    ${project_dir}/config/spack-config.mk"
-        echo "    ${project_dir}/config/spack_config.hpp"
-        echo "ERROR: Spack generated configuration not found."
+        echo "[Error]: No result for at least one of"
+        echo "[Error]:     ${project_dir}/config/spack-config.mk"
+        echo "[Error]:     ${project_dir}/config/spack_config.hpp"
+        echo "[Error]: Spack generated configuration not found."
         exit 1
     fi
 
@@ -188,10 +342,10 @@ then
         # project parent dir.
         if [[ -e "../data" ]]; then
             if [[ -L "../data" ]]; then
-                echo "'../data' link already exists. Deleting."
+                echo "[Information]: '../data' link already exists. Deleting."
                 rm "../data"
             else
-                echo "Error: '../data' already exists and it's NOT a link"
+                echo "[Error]: '../data' already exists and it's NOT a link"
                 exit 1
             fi
         fi
@@ -204,7 +358,7 @@ then
 
     if [[ "$with_data" == "true" && ! -d "../data" ]]
     then
-        echo "ERROR: '$data_dir' is not a directory while asking for --data"
+        echo "[Error]: '$data_dir' is not a directory while asking for --data"
         exit 1
     fi
 fi
@@ -216,42 +370,45 @@ then
     echo "~ Project Dir: ${project_dir}"
     echo "~ Data Dir: ${data_dir}"
     echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    echo ""
 
-    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
-    echo "~~~~~ MFEM configuration"
-    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    timed_message "MFEM Configuration"
 
     make info
 
-    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
-    echo "~~~~~ Building MFEM"
-    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    timed_message "Building MFEM"
+
+    # If the build is terminated, e.g. job timeout, run 'make distclean'
+    exit_distclean=1
 
     make all -j ${threads}
+
+    exit_distclean=
 fi
 
 # Test
 if [[ "${mode}" != "--deps-only" && "${mode}" != "--build-only" ]]
 then
-    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
-    echo "~~~~~ Testing MFEM"
-    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    timed_message "Testing MFEM"
+
+    # If the build is terminated, e.g. job timeout, run 'make distclean'
+    exit_distclean=1
 
     make test
     test_status=$?
 
-    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
-    echo "~~~~~ Cleaning MFEM"
-    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    timed_message "Cleaning MFEM"
 
     if make distclean > distclean.log 2>&1; then
-      echo "[make distclean]: OK"
+      echo "[Information]: make distclean OK"
     else
       cat distclean.log
       echo
-      echo "[make distclean]: FAILED (see above)"
+      echo "[Information]: make distclean FAILED (see above)"
     fi
     rm -f distclean.log
 
+    exit_distclean=
+
     exit $test_status
 fi

tests/gitlab/build_and_test_setup

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+function info_msg ()
+{
+  echo "[Information:] ${1}"
+}
+
+function error_msg ()
+{
+  echo "[Error:] ${1}"
+}
+
+# Perform a clone while holding a lock file to prevent concurrency on
+# the destination.
+# Usage:
+# locked_clone <repository_url> <destination_dir>
+function locked_clone ()
+{
+  if ! command -v flock
+  then
+    error_msg "Required command 'flock' not found"
+    exit 1
+  fi
+
+  info_msg "Will clone ${1} into ${2}"
+
+  ( date; info_msg "Waiting to acquire lock on '${PWD}/${2}.lock' ..."
+    # try to get an exclusive lock on fd 9 (mfem-data.lock) repeating the
+    # try every 5 seconds; we may want to add a counter for the number of
+    # retries to interrupt a potential infinite loop
+    while ! flock -n 9; do sleep 5; done
+    date; info_msg "Acquired lock on '${PWD}/${2}.lock'"
+    # clone/update the autotest repo while holding the file lock on
+    # 'autotest.lock'
+    err=0
+    if [[ ! -d "${2}" ]]; then
+      git clone ${1} ${2}
+    else
+      cd ${2} && git pull && cd ..
+    fi || err=1
+    # sleep for a period to allow NFS to propagate the above changes;
+    # clearly, there is no guarantee that other NFS clients will see the
+    # changes even after the timeout
+    sleep 10
+    exit $err
+  ) 9> ${2}.lock
+}
+
+# Setup MFEM_DATA_DIR=${SHARED_REPOS_DIR}/mfem-data, see '.gitlab-ci.yml'
+info_msg "MFEM_DATA_REPO is ${MFEM_DATA_REPO}"
+info_msg "SHARED_REPOS_DIR is ${SHARED_REPOS_DIR}"
+
+mkdir -p ${SHARED_REPOS_DIR} && cd ${SHARED_REPOS_DIR}
+locked_clone ${MFEM_DATA_REPO} mfem-data
+
+# Setup ${AUTOTEST_ROOT}/autotest:
+info_msg "AUTOTEST_REPO is ${AUTOTEST_REPO}"
+info_msg "AUTOTEST_ROOT is ${AUTOTEST_ROOT}"
+
+mkdir -p ${AUTOTEST_ROOT} && cd ${AUTOTEST_ROOT}
+locked_clone ${AUTOTEST_REPO} autotest

tests/gitlab/generate_spack_upstream

@@ -1,38 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# This script is meant be launched in the root directory of MFEM to compile all
-# the dependencies of MFEM for all the specs in CI. By doing so, we obtain a
-# Spack instance that can be used as an upstream.
-# Note: needs mfem-uberenv in MFEM repo (run tests/gitlab/get_mfem_uberenv)
-
-# This patch applies shared permissions to Spack installations.
-cd tests/uberenv
-git apply upstream-permission.patch
-cd -
-
-# Get the hostname
-hostid=${HOSTNAME//[[:digit:]]/}
-
-# call uberenv for all specs in CI
-git grep -e "^[^#]" .gitlab/$hostid* | grep "SPEC" \
-                                     | cut -d' ' -f6- \
-                                     | sed 's/"//g' \
-                                     | while read -r line; do
-  python ./tests/uberenv/uberenv.py --spec="$line"
-done
-
-# We revert the patch to leave the repo as found.
-cd tests/uberenv
-git apply -R upstream-permission.patch
-cd -

tests/gitlab/get_mfem_uberenv

@@ -18,7 +18,7 @@ set -o errexit
 set -o nounset
 
 uberenv_url="https://github.com/mfem/mfem-uberenv.git"
-uberenv_ref="fe5fa88876b29ff03177d44a5bd3e09c84ccdcbf"
+uberenv_ref="a9bd720f83cf1f8afa26b4e69737fcd4a6d462f9"
 
 [[ ! -d tests/uberenv ]] && git clone ${uberenv_url} tests/uberenv
 cd tests/uberenv

tests/gitlab/reproduce-ci-jobs-interactively.md

@@ -1,3 +1,6 @@
+
+FIXME: this file needs to be updated
+
 # HowTo: Reproduce CI jobs interactively.
 
 We rely on Spack, driven by uberenv, to build MFEM dependencies automatically
@@ -26,6 +29,48 @@ Those modes are essentially the same, but we emphasize building the
 dependencies as a first isolated step because it makes it clear what is
 happening and how to use this workflow.
 
+## Shortcut:
+
+To help developers reproduce jobs from the CI, a reproducer script is printed
+in each job that will allow for fast and accurate reproduction of the same
+scenario as in the CI job. Below is an example extracted for a CI job log:
+
+```bash
+working_dir="/usr/workspace/${USER}/mfem/2405156-$(date +%s)"
+mkdir -p ${working_dir} && cd ${working_dir}
+git clone https://github.com/MFEM/mfem.git --single-branch --depth=1
+cd mfem
+git fetch origin --depth=1 4868222660f03e15ebf7a6daca90800b64baf69d
+git checkout 4868222660f03e15ebf7a6daca90800b64baf69d
+git submodule update --init --recursive
+
+# Variables
+export SPEC="%gcc@8.3.1 +mpi +cuda cuda_arch=70"
+
+# Directories
+export BUILD_ROOT="${working_dir}"
+export SHARED_REPOS_DIR="${BUILD_ROOT}/.."
+export MFEM_DATA_DIR="${SHARED_REPOS_DIR}/mfem-data"
+
+# Repositories
+export TPLS_REPO="ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git"
+export TESTS_REPO="ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git"
+export AUTOTEST_REPO="ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git"
+export MFEM_DATA_REPO="https://github.com/mfem/data.git"
+
+# Setup directories
+./tests/gitlab/build_and_test_setup
+
+# Using the CI build cache is optional and requires a token. Set it like so:
+# export REGISTRY_TOKEN="<your token here>"
+
+lalloc 1 -W 45 -q pci --atsdisable tests/gitlab/build_and_test --spec "%gcc@8.3.1 +mpi +cuda cuda_arch=70" --data-dir "/usr/workspace/mfem/gitlab-runner/bernede1/repos/mfem-data" --data
+```
+
+**NOTE**
+
+The REGISTRY_TOKEN can be set using a GitLab Personal Access Token (PAT) with read access to gitlab container registry. This allows to speed up the local builds by fetching the dependencies library instead of building them.
+
 ## Prerequisite: Retrieve Uberenv
 
 ```bash
@@ -88,6 +133,11 @@ but still ready to use.
 
 ## Build and test MFEM
 
+**NOTE**
+
+If you need mfem data and/or the autotest repo, you may clone them next to your
+mfem repository.
+
 ### Option 1: Without using scripts
 
 ```bash