comment out monitor methods

Fixes compilation errors when compiling with gcc-13 on Fedora 38

.appveyor.yml

@@ -31,7 +31,7 @@ install:
 - ps: ( get-content "GKlib\gk_arch.h") | % { If ($_.ReadCount -ge 52) {$_ -replace "#ifdef __MSC__","#ifdef DISABLE_THIS_ANCIENT_MSC_CHECK"} Else {$_} } | set-content "GKlib\gk_arch.h"
 - cmake -H. -Bbuild
 # -DCMAKE_BUILD_TYPE=Release
-- cmake --build build
+- cmake --build build --target metis -j 4
 - set METIS_PATH=%cd%
 - cd ..
 
@@ -40,7 +40,7 @@ install:
 - 7z x v2.19.0.tar.gz -so | 7z x -si -ttar > nul
 - cd hypre-2.19.0/src
 - cmake -H. -Bbuild
-- cmake --build build
+- cmake --build build -j 4
 - cmake --build build --target install
 - cd ../..
 
@@ -49,14 +49,20 @@ install:
 
 # MFEM
 before_build:
-- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DHYPRE_DIR=%cd%\..\hypre-2.19.0\src\hypre -DMETIS_LIBRARIES=%METIS_PATH%\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%METIS_PATH%\include
-- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_serial -DMFEM_USE_MPI=FALSE
+- cmake . -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DHYPRE_DIR=%cd%\..\hypre-2.19.0\src\hypre -DMETIS_LIBRARIES=%METIS_PATH%\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%METIS_PATH%\include
+# - cmake . -Bbuild_serial -DMFEM_USE_MPI=FALSE
+- cmake . -Bbuild_shared_serial -DMFEM_USE_MPI=FALSE -DBUILD_SHARED_LIBS=ON
 
 build_script:
-- cmake --build build_parallel --config Release -j 4
-- cmake --build build_serial --config Release -j 4
-- cmake --build build_serial --target exec --config Release -j 4
+- cmake --build build_parallel --config Debug -j 4
+# - cmake --build build_serial --config Release -j 4
+# - cmake --build build_serial --target exec --config Release -j 4
+- cmake --build build_shared_serial --config Release -j 4
+- cmake --build build_shared_serial --target exec --config Release -j 4
 
 after_build:
-- cd build_serial
+# - cd build_serial
+# - ctest -C Release --output-on-failure
+# - cd ..
+- cd build_shared_serial
 - ctest -C Release --output-on-failure

.github/workflows/builds-and-tests.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -74,12 +74,19 @@ jobs:
             codecov: YES
           - os: windows-latest
             codecov: NO
+          - os: windows-latest
+            target: opt
+            mpi: par
+            config-opts: '-DBUILD_SHARED_LIBS=ON'
           - os: ubuntu-latest
             target: opt
             codecov: NO
             mpi: par
             build-system: cmake
             hypre-target: int32
+            # This option can be set to pass additional configuration options to
+            # the MFEM configuration command.
+            # config-opts: '-DCMAKE_VERBOSE_MAKEFILE=ON'
           - os: ubuntu-latest
             target: opt
             codecov: NO
@@ -155,7 +162,7 @@ jobs:
 
     - name: get hypre
       if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
-      uses: mfem/github-actions/build-hypre@v2.2
+      uses: mfem/github-actions/build-hypre@v2.4
       with:
         archive: ${{ env.HYPRE_ARCHIVE }}
         dir: ${{ env.HYPRE_TOP_DIR }}
@@ -164,7 +171,7 @@ jobs:
 
     - name: get hypre (Windows)
       if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
-      uses: mfem/github-actions/build-hypre@v2.2
+      uses: mfem/github-actions/build-hypre@v2.4
       with:
         archive: ${{ env.HYPRE_ARCHIVE }}
         dir: ${{ env.HYPRE_TOP_DIR }}
@@ -183,7 +190,7 @@ jobs:
 
     - name: install metis
       if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@v2.2
+      uses: mfem/github-actions/build-metis@v2.4
       with:
         archive: ${{ env.METIS_ARCHIVE }}
         dir: ${{ env.METIS_TOP_DIR }}
@@ -210,7 +217,7 @@ jobs:
 
     # MFEM build and test
     - name: build
-      uses: mfem/github-actions/build-mfem@v2.2
+      uses: mfem/github-actions/build-mfem@v2.4
       env:
         VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
       with:
@@ -222,7 +229,7 @@ jobs:
         hypre-dir: ${{ env.HYPRE_TOP_DIR }}
         metis-dir: ${{ env.METIS_TOP_DIR }}
         mfem-dir: ${{ env.MFEM_TOP_DIR }}
-        config-options: ${{ env.MFEM_EXTRA_CONFIG }}
+        config-options: ${{ matrix.config-opts }}
         library-only: ${{ matrix.target == 'dbg' }}
 
     # Run checks (and only checks) on debug targets
@@ -256,13 +263,15 @@ jobs:
       if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os != 'ubuntu-latest'
       run: |
         CTEST_CONFIG="Release"
-        cd ${{ env.MFEM_TOP_DIR }}/build && ctest --output-on-failure -C ${CTEST_CONFIG}
+        cd ${{ env.MFEM_TOP_DIR }}/build && \
+        ctest --output-on-failure -C ${CTEST_CONFIG} || \
+        ctest --rerun-failed --output-on-failure -C ${CTEST_CONFIG}
       shell: bash
 
     # Code coverage (process and upload reports)
     - name: codecov
       if: matrix.codecov == 'YES'
-      uses: mfem/github-actions/upload-coverage@v2.2
+      uses: mfem/github-actions/upload-coverage@v2.4
       with:
         name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
         project_dir: ${{ env.MFEM_TOP_DIR }}

.github/workflows/mfem-analysis.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -57,7 +57,7 @@ jobs:
 
     - name: Get Hypre
       if: steps.hypre-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-hypre@v2.2
+      uses: mfem/github-actions/build-hypre@v2.4
       with:
         archive: ${{ env.HYPRE_ARCHIVE }}
         dir: ${{ env.HYPRE_TOP_DIR }}
@@ -72,14 +72,14 @@ jobs:
 
     - name: Install Metis
       if: steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@v2.2
+      uses: mfem/github-actions/build-metis@v2.4
       with:
         archive: ${{ env.METIS_ARCHIVE }}
         dir: ${{ env.METIS_TOP_DIR }}
 
     # MFEM build and test
     - name: build-mfem
-      uses: mfem/github-actions/build-mfem@v2.2
+      uses: mfem/github-actions/build-mfem@v2.4
       with:
         os: ${{ runner.os }}
         target: opt

.github/workflows/mfem-sanitizer.yml

@@ -0,0 +1,70 @@
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+name: "Sanitizer"
+
+permissions:
+  actions: write
+
+on:
+  push:
+    branches:
+      - master
+      - next
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  Serial:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Cancel Previous Runs
+      uses: styfle/cancel-workflow-action@0.11.0
+      with:
+        access_token: ${{ github.token }}
+
+    - name: MFEM Checkout
+      uses: actions/checkout@v3
+      with:
+        path: mfem
+
+    - name: MFEM Build
+      uses: mfem/github-actions/build-mfem@v2.4
+      with:
+        os: ${{ runner.os }}
+        target: opt
+        mpi: seq
+        hypre-dir: unused-hypre-dir
+        metis-dir: unused-metis-dir
+        mfem-dir: mfem
+        build-system: make
+        library-only: false
+        config-options:
+          CXX="clang++-14"
+          CXXFLAGS="-g -O1 -std=c++11
+                    -fsanitize=address
+                    -fno-omit-frame-pointer
+                    -fsanitize-address-use-after-scope"
+
+    - name: MFEM Info
+      working-directory: mfem
+      run: make info
+
+    - name: MFEM Sanitize
+      working-directory: mfem
+      run:
+        ASAN_OPTIONS="detect_leaks=1,
+                      strict_init_order=1,
+                      strict_string_checks=1,
+                      check_initialization_order=1,
+                      detect_stack_use_after_return=1"
+        make test

.github/workflows/repo-check.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

.gitignore

@@ -21,6 +21,9 @@ CMakeFiles/
 # Default install location
 /mfem/
 
+# Typical build directory
+/build/
+
 # Generated files in main directory, config/ and docs/
 /deps.mk
 config/_config.hpp
@@ -118,11 +121,9 @@ examples/amgx/sol.gf
 examples/amgx/mesh.*
 examples/amgx/sol.*
 
-examples/gingko/ex1
-examples/gingko/refined.mesh
-examples/gingko/sol.gf
-examples/gingko/mesh.*
-examples/gingko/sol.*
+examples/ginkgo/ex1
+examples/ginkgo/refined.mesh
+examples/ginkgo/sol.gf
 
 examples/hiop/ex9
 examples/hiop/ex9p
@@ -148,8 +149,8 @@ examples/petsc/velocity.*
 examples/petsc/elastic_energy.*
 examples/petsc/mode_*
 
-examples/pumi/ex1
-examples/pumi/ex[126]p
+examples/pumi/ex[12]
+examples/pumi/ex[16]p
 examples/pumi/refined.mesh
 examples/pumi/sol.gf
 examples/pumi/mesh.*
@@ -194,6 +195,9 @@ miniapps/gslib/field-diff
 miniapps/gslib/field-interp
 miniapps/gslib/findpts
 miniapps/gslib/pfindpts
+miniapps/gslib/schwarz_ex1
+miniapps/gslib/schwarz_ex1p
+miniapps/gslib/interpolated.gf
 
 miniapps/meshing/mobius-strip
 miniapps/meshing/klein-bottle
@@ -206,6 +210,7 @@ miniapps/meshing/trimmer
 miniapps/meshing/reflector
 miniapps/meshing/mesh-optimizer
 miniapps/meshing/pmesh-optimizer
+miniapps/meshing/pmesh-fitting
 miniapps/meshing/minimal-surface
 miniapps/meshing/pminimal-surface
 miniapps/meshing/polar-nc
@@ -245,6 +250,7 @@ miniapps/navier/navier_tgv
 miniapps/navier/navier_shear
 miniapps/navier/navier_3dfoc
 miniapps/navier/navier_turbchan
+miniapps/navier/navier_cht
 miniapps/navier/tgv_out*.txt
 miniapps/navier/*_output
 
@@ -316,6 +322,16 @@ miniapps/parelag/*.mesh
 miniapps/multidomain/multidomain
 miniapps/hooke/hooke
 
+miniapps/dpg/diffusion
+miniapps/dpg/pdiffusion
+miniapps/dpg/convection-diffusion
+miniapps/dpg/pconvection-diffusion
+miniapps/dpg/acoustics
+miniapps/dpg/pacoustics
+miniapps/dpg/maxwell
+miniapps/dpg/pmaxwell
+miniapps/dpg/ParaView
+
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests

.gitlab-ci.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

.gitlab/configs/common.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

.gitlab/configs/corona-config.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

.gitlab/configs/lassen-config.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

.gitlab/configs/quartz-config.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

.gitlab/configs/report-build-and-test.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -22,12 +22,10 @@
         date
         echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
         # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
+        # every 5 seconds; we may want to add a counter for the number of
         # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
+        while ! flock -n 9; do
+          sleep 5
         done
         echo "Acquired lock on '$PWD/autotest.lock'"
         date
@@ -57,12 +55,10 @@
         date
         echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
         # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
+        # every 5 seconds; we may want to add a counter for the number of
         # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
+        while ! flock -n 9; do
+          sleep 5
         done
         echo "Acquired lock on '$PWD/autotest.lock'"
         date

.gitlab/configs/setup-baseline.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -47,12 +47,10 @@ setup_baseline:
         date
         echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
         # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
+        # every 5 seconds; we may want to add a counter for the number of
         # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
+        while ! flock -n 9; do
+          sleep 5
         done
         echo "Acquired lock on '$PWD/autotest.lock'"
         date

.gitlab/configs/setup-build-and-test.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -35,13 +35,11 @@ setup:
       (
         date
         echo "Waiting to acquire lock on '$PWD/mfem-data.lock' ..."
-        # try to get an exclusive lock on fd 9 (mfem-data.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
+        # try to get an exclusive lock on fd 9 (mfem-data.lock) repeating the
+        # try every 5 seconds; we may want to add a counter for the number of
         # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
+        while ! flock -n 9; do
+          sleep 5
         done
         echo "Acquired lock on '$PWD/mfem-data.lock'"
         date
@@ -69,12 +67,10 @@ setup:
         date
         echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
         # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
+        # every 5 seconds; we may want to add a counter for the number of
         # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
+        while ! flock -n 9; do
+          sleep 5
         done
         echo "Acquired lock on '$PWD/autotest.lock'"
         date

.gitlab/corona-build-and-test.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

.gitlab/lassen-build-and-test.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -14,14 +14,14 @@ stages:
   - build_and_test
   - report
 
-opt_mpi_cuda_xl_16_1_1_8:
+opt_mpi_cuda_xl_16_1_1_12:
   variables:
-    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=70"
+    SPEC: "%xl@16.1.1.12 +mpi +cuda cuda_arch=70"
   extends: .build_and_test_on_lassen
 
 opt_mpi_cuda_hypre_cuda_xl:
   variables:
-    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
+    SPEC: "%xl@16.1.1.12 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
   extends: .build_and_test_on_lassen
 
 # Jobs report

.gitlab/quartz-baseline.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -51,6 +51,8 @@ cleanup:
   script:
     - echo "BUILD_ROOT=${BUILD_ROOT}"
     - rm -rf "${BUILD_ROOT}" || true
+    - echo "CI_PROJECT_DIR=${CI_PROJECT_DIR}"
+    - make -C "${CI_PROJECT_DIR}" distclean
 
 report_baseline:
   extends: [.on_quartz]
@@ -66,12 +68,10 @@ report_baseline:
         date
         echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
         # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
+        # every 5 seconds; we may want to add a counter for the number of
         # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
+        while ! flock -n 9; do
+          sleep 5
         done
         echo "Acquired lock on '$PWD/autotest.lock'"
         date
@@ -82,12 +82,14 @@ report_baseline:
         rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-${BASELINE_TEST}-${CI_COMMIT_REF_SLUG}"
         rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir ${rundir})
         cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
-        # We create an autotest-email.html file, because that's how we signal that there was a diff (temporary).
-        if [[ -f ${rundir}/${BASELINE_TEST}.err ]]; then
-          cp ${rundir}/${BASELINE_TEST}.err ${rundir}/autotest-email.html
-        fi
         printf "%s\n" "" "Pipeline URL:" "$CI_PIPELINE_URL" \
           >> ${rundir}/pipeline.txt
+        # We create an autotest-email.html file, because that's how we signal
+        # that there was an error / diff (temporary).
+        if [[ -f ${rundir}/${BASELINE_TEST}.err ]] || \
+           [[ -f ${rundir}/${BASELINE_TEST}-${SYS_TYPE}.diff ]]; then
+          cp ${rundir}/pipeline.txt ${rundir}/autotest-email.html
+        fi
         msg="GitLab CI log for ${BASELINE_TEST} on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
         if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
           git pull && \

.gitlab/quartz-build-and-test.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -27,44 +27,39 @@ allocate_resource:
   timeout: 6h
 
 # GitLab jobs for the Quartz machine at LLNL
-debug_ser_gcc_4_9_3:
+debug_ser_gcc_10:
   variables:
-    SPEC: "%gcc@4.9.3 +debug~mpi"
+    SPEC: "%gcc@10.3.1 +debug~mpi"
   extends: .build_and_test_on_quartz
 
-debug_ser_gcc_6_1_0:
+debug_par_gcc_10:
   variables:
-    SPEC: "%gcc@6.1.0 +debug~mpi"
+    SPEC: "%gcc@10.3.1 +debug+mpi"
   extends: .build_and_test_on_quartz
 
-debug_par_gcc_6_1_0:
+opt_ser_gcc_10:
   variables:
-    SPEC: "%gcc@6.1.0 +debug+mpi"
+    SPEC: "%gcc@10.3.1 ~mpi"
   extends: .build_and_test_on_quartz
 
-opt_ser_gcc_6_1_0:
+opt_par_gcc_10:
   variables:
-    SPEC: "%gcc@6.1.0 ~mpi"
+    SPEC: "%gcc@10.3.1"
   extends: .build_and_test_on_quartz
 
-opt_par_gcc_6_1_0:
+opt_par_gcc_10_sundials:
   variables:
-    SPEC: "%gcc@6.1.0"
+    SPEC: "%gcc@10.3.1 +sundials"
   extends: .build_and_test_on_quartz
 
-opt_par_gcc_6_1_0_sundials:
+opt_par_gcc_10_petsc:
   variables:
-    SPEC: "%gcc@6.1.0 +sundials"
+    SPEC: "%gcc@10.3.1 +petsc ^petsc+mumps~superlu-dist"
   extends: .build_and_test_on_quartz
 
-opt_par_gcc_6_1_0_petsc:
+opt_par_gcc_10_pumi:
   variables:
-    SPEC: "%gcc@6.1.0 +petsc ^petsc+mumps~superlu-dist"
-  extends: .build_and_test_on_quartz
-
-opt_par_gcc_6_1_0_pumi:
-  variables:
-    SPEC: "%gcc@6.1.0 +pumi"
+    SPEC: "%gcc@10.3.1 +pumi"
   extends: .build_and_test_on_quartz
 
 # Release

.gitlab/scripts/baseline

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -42,47 +42,34 @@ fi
 # post
 mkdir ${artifacts_path}
 
-if [[ -s ${glob_err} ]]
-then
-  echo "ERROR during ${BASELINE_TEST} execution";
-  echo "Here is the ${glob_err} file content";
-  cat ${glob_err}
-  cp ${glob_err} ${artifacts_path}/${glob_err}
-  exit 1;
-elif [[ ! -f ${base_patch} && ! -f ${base_out} ]]
-then
-  echo "Something went WRONG in ${BASELINE_TEST}:";
-  echo "Either ${base_patch} or ${base_out} should exists";
-  exit 1;
-elif [[ -f ${base_patch} ]]
-then
-  echo "${BASELINE_TEST}: Differences found, patch generated"
-  cp ${base_patch} ${artifacts_path}/${base_patch}
-elif [[ -f ${base_out} ]]
-then
-  echo "${BASELINE_TEST}: Differences found, replacement file generated"
-  cp ${base_out} ${artifacts_path}/${base_out}
-fi
-
+status=0
 if [[ -f ${BASELINE_TEST}.out ]]; then
   cp ${BASELINE_TEST}.out ${artifacts_path}
 fi
-
+if [[ -s ${glob_err} ]]; then
+  echo "ERROR during ${BASELINE_TEST} execution"
+  echo "Here is the ${glob_err} file content"
+  cat ${glob_err}
+  cp ${glob_err} ${artifacts_path}/${glob_err}
+  status=1
+fi
+if [[ -f ${base_patch} ]]; then
+  echo "${BASELINE_TEST}: Differences found, patch generated"
+  cp ${base_patch} ${artifacts_path}/${base_patch}
+elif [[ -f ${base_out} ]]; then
+  echo "${BASELINE_TEST}: Differences found, replacement file generated"
+  cp ${base_out} ${artifacts_path}/${base_out}
+fi
 # base_diff won't even exist if there is no difference.
-if [[ -f ${base_diff} ]]
-then
+if [[ -f ${base_diff} ]]; then
   echo "${BASELINE_TEST}: Relevant differences (filtered diff) ..."
   cat ${base_diff}
   cp ${base_diff} ${artifacts_path}/${base_diff}
-  # We create a .err file, because that's how we signal that there was a diff.
-  cp ${base_diff} ${artifacts_path}/gitlab-${BASELINE_TEST}-${MACHINE_NAME}.err
+  status=1
 fi
-
-if [[ ! -s ${base_diff} ]]
-then
+if [[ $status -eq 0 ]]; then
   echo "${BASELINE_TEST}: PASSED"
-  true
 else
   echo "${BASELINE_TEST}: FAILED"
-  false
 fi
+exit $status

.gitlab/scripts/git_try_to_push

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

.gitlab/scripts/rebaseline

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

.gitlab/scripts/report_build_and_test_failure

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

.gitlab/scripts/report_build_and_test_success

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

.gitlab/scripts/safe_create_rundir

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

CHANGELOG

@@ -8,61 +8,94 @@
                                https://mfem.org
 
 
-Version 4.5.1 (development)
+Version 4.5.3 (development)
 ===========================
-- When using discontinuous (L2) spaces, use local (element-wise) L2 projection
-  as the coarsening operator for non-conforming AMR meshes.
 
-Meshing improvements
---------------------
+- Added new methods in the Mesh class to set and get attributes on NURBS patches
+  and patch boundaries.
+
+- Improved lambda body debugging with the addition of mfem::forall functions.
+  These functions can take the place of the MFEM_FORALL macros, which have been
+  preserved for backwards compatibility.
+
+- TMOP improvement: added asymptotically-balanced compound metrics 90, 94, 328,
+  338. Added the tmop-metric-magnitude tool for tracking how metrics change
+  under geometric perturbations.
+
+New and updated examples and miniapps
+-------------------------------------
+- Added a miniapp pmesh-fitting in miniapps/meshing for interface and boundary
+  fitting to implicit domains defined using level-set functions.
+
+- Moved the distance solver methods from miniapps/shifted to miniapps/common.
+
+Discretization improvements
+---------------------------
+- Face restriction operators for Nedelec and Raviart-Thomas finite element
+  spaces are now supported through the ConformingFaceRestriction class.
+
+- VectorFEBoundaryFluxLFIntegrator is now supported on device/GPU.
+
+- Added support for p-refined meshes in FindPointsGSLIB.
+
+New and updated examples and miniapps
+-------------------------------------
+
+- Added new Discontinuous Petrov-Galerkin (DPG) miniapp which includes
+  serial and parallel examples for diffusion, convection-diffusion, acoustics
+  and Maxwell equations. The miniapp also includes new classes such as 
+  (Par)DPGWeakForm, (Par)ComplexDPGWeakForm, BlockStaticCondensation and 
+  ComplexBlockStaticCondensation. Three new integrators are added in support of 
+  DPG systems: TraceIntegrator, NormalTraceIntegrator and TangentTraceIntegrator
+
+Integrations, testing and documentation
+---------------------------------------
+- Added an address sanitizer GitHub action for a serial build/test on Ubuntu,
+  based on Clang/LLVM (https://clang.llvm.org/docs/AddressSanitizer.html).
+
+Version 4.5.2, released on March 23, 2023
+=========================================
+
 - Added support for pyramids in non-conforming meshes. Currently only isotropic
   refinement is supported in this case.
 
+- Removed the support for the Mesquite toolkit. We recommend using MFEM's TMOP
+  functionality instead for mesh optimization. See the mesh-optimizer miniapp.
+
+- Added a fast normalization-based distance solver, see the Distance miniapp
+  in the miniapps/shifted/ directory.
+
+- Added a new meshing miniapp, Reflector, which reflects a high-order or NURBS
+  hexahedral mesh about a plane.
+
 - Updated logic in FindPointsGSLIB to ignore points found near (but outside) the
   domain boundary.
 
-- Added support for pyramids in Gmsh meshes.
+- Added an option to auto-balance compound TMOP metrics.
 
 - Fixed a bug in TMOP metric 301.
 
-- Added an option to auto-balance compound TMOP metrics.
+- When using discontinuous (L2) spaces, use local (element-wise) L2 projection
+  as the coarsening operator for non-conforming AMR meshes.
 
-Discretization improvements
----------------------------
-- TBD
+- Added support for GridFunction::GetGradients() and GetVectorGradient() on
+  face-neighbor elements.
 
-Linear and nonlinear solvers
-----------------------------
-- Added a fast normalization-based distance solver, see the Distance miniapp
-  in the miniapps/shifted/ directory.
+- Added support for pyramids in Gmsh meshes.
 
-New and updated examples and miniapps
--------------------------------------
-- Added a new meshing miniapp, reflector, which reflects a high-order or NURBS
-  hexahedral mesh about a plane.
+- The Mesh Explorer miniapp can now save mesh files in the VisIt or ParaView
+  formats using the corresponding DataCollection. See option 'D' in the menu.
 
-- The mesh-explorer miniapp can now save mesh files in the VisIt or ParaView
-  formats using the corresponding DataCollection objects. See option 'D' in the
-  main menu.
+- VisItDataCollection now correctly handles data collection names containing
+  underscores.
 
-Integrations, testing and documentation
----------------------------------------
-- Removed the support for the Mesquite toolkit. We recommend using MFEM's TMOP
-  functionality instead for mesh optimization. See the mesh-optimizer miniapp.
+- Added support for shared Windows builds with MSVC through CMake.
+  Developers note: this enhancement is facilitated by the addition of the macro
+  MFEM_EXPORT, see config.hpp for more details on its usage.
 
 - The following integrations have updated minimum version requirements:
   * RAJA >= 2022.10.3
 
-Miscellaneous
--------------
-- VisItDataCollection now correctly handles collection names containing
-  underscores.
-
-- VisItDataCollection::SetPadDigits() no longer alters the number of digits
-  used to represent the MPI rank because VisIt seems to require 6 digits.
-  This parameter can still be explicitly overridden with
-  VisItDataCollection::SetPadDigitsRank().
-
 API changes
 -----------
 - The implicit cast methods of class Vector to 'double *' and 'const double *'
@@ -73,6 +106,11 @@ API changes
   deprecated, and Mesh::GetFaceGeometry (which provides identical functionality)
   should be used instead.
 
+- VisItDataCollection::SetPadDigits() no longer alters the number of digits
+  used to represent the MPI rank because VisIt seems to require 6 digits.
+  This parameter can still be explicitly overridden with SetPadDigitsRank().
+
+
 Version 4.5, released on October 22, 2022
 =========================================
 
@@ -116,9 +154,6 @@ Discretization improvements
 - Added a class CoefficientVector for efficient access of variable coefficient
   values at quadrature points (in particular for GPU/device kernels).
 
-- Added support for GridFunction::GetGradients() and
-  GriFunction::GetVectorGradient() on face-neighbor elements.
-
 - Added WhiteGaussianNoiseDomainLFIntegrator: a LinearFormIntegrator class for
   spatial Gaussian white noise.
 

CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -13,6 +13,7 @@
 # Version 3.8 fixes the handling of CMAKE_CXX_STANDARD for try_compile.
 # Version 3.8 or newer is required for direct CUDA support.
 cmake_minimum_required(VERSION 3.8)
+message(STATUS "CMake version: ${CMAKE_VERSION}")
 set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH
   "Path to optional user configuration file.")
 
@@ -24,8 +25,11 @@ set(CMAKE_CXX_EXTENSIONS OFF)
 # Load user settings before the defaults - this way the defaults will not
 # overwrite the user set options. If the user has not set all options, we still
 # have the defaults.
-message(STATUS "(optional) USER_CONFIG = ${USER_CONFIG}")
-include("${USER_CONFIG}" OPTIONAL)
+include("${USER_CONFIG}" OPTIONAL RESULT_VARIABLE USER_CONFIG_LOADED)
+if (USER_CONFIG_LOADED)
+  set(USER_CONFIG_LOADED "LOADED")
+endif()
+message(STATUS "Loading USER_CONFIG = ${USER_CONFIG} (${USER_CONFIG_LOADED})")
 include("${CMAKE_CURRENT_SOURCE_DIR}/config/defaults.cmake")
 
 # Allow overwriting of the compiler by setting CXX/MPICXX on the command line or
@@ -53,7 +57,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.5.1)
+set(${PROJECT_NAME}_VERSION 4.5.3)
 
 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -183,6 +187,23 @@ else()
   set(MFEM_DEBUG OFF)
 endif()
 
+# Shared build on Windows
+if (WIN32 AND BUILD_SHARED_LIBS)
+  # CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS works only with MSVC?
+  set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON CACHE BOOL
+      "Automatically export symbols for shared Windows DLL build")
+  # Place all .dll and .exe files in the same sub-directory, 'bin/<config>', of
+  # the build directory, so that the .exe files can find the .dll files at
+  # runtime:
+  set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin")
+  # Note: CMake tests are run from the build-tree sub-directory where the test
+  # is defined, e.g. <build-dir>/tests/unit, so hard coded paths to meshes still
+  # work fine.
+  # Not strictly necessary:
+  # set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
+  # set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
+endif()
+
 # AMD HIP
 if (MFEM_USE_HIP)
   if (HIP_ARCH)
@@ -519,6 +540,7 @@ list(REVERSE TPL_LIBRARIES)
 list(REMOVE_DUPLICATES TPL_INCLUDE_DIRS)
 # message(STATUS "TPL_INCLUDE_DIRS = ${TPL_INCLUDE_DIRS}")
 
+message(STATUS "MFEM shared library: BUILD_SHARED_LIBS = ${BUILD_SHARED_LIBS}")
 message(STATUS "MFEM build type: CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}")
 message(STATUS "MFEM version: v${MFEM_VERSION_STRING}")
 message(STATUS "MFEM git string: ${MFEM_GIT_STRING}")
@@ -574,7 +596,19 @@ target_include_directories(mfem
   PUBLIC
     $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+    # The INSTALL_INTERFACE is handled below
     ${TPL_INCLUDE_DIRS})
+if (MFEM_USE_MPI)
+  if (MPI_CXX_COMPILE_FLAGS)
+    separate_arguments(MPI_CXX_COMPILE_ARGS UNIX_COMMAND
+      "${MPI_CXX_COMPILE_FLAGS}")
+    target_compile_options(mfem PUBLIC ${MPI_CXX_COMPILE_ARGS})
+  endif()
+  if (MPI_CXX_LINK_FLAGS)
+    target_link_libraries(mfem PUBLIC ${MPI_CXX_LINK_FLAGS})
+  endif()
+endif()
+
 set_target_properties(mfem PROPERTIES VERSION "${mfem_VERSION}")
 set_target_properties(mfem PROPERTIES SOVERSION "${mfem_VERSION}")
 
@@ -586,6 +620,7 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
 endif()
 
 # Generate configuration file in the build directory: config/_config.hpp.
+set(MFEM_SHARED_BUILD ${BUILD_SHARED_LIBS})
 configure_file(
   "${PROJECT_SOURCE_DIR}/config/cmake/config.hpp.in"
   "${PROJECT_BINARY_DIR}/config/_config.hpp")
@@ -705,7 +740,7 @@ set(INSTALL_LIB_DIR lib
 set(INSTALL_CMAKE_DIR lib/cmake/mfem
   CACHE PATH "Relative path for installing cmake config files.")
 
-target_include_directories(mfem
+target_include_directories(mfem BEFORE
   PUBLIC
     $<INSTALL_INTERFACE:${INSTALL_INCLUDE_DIR}>)
 
@@ -757,10 +792,8 @@ endif()
 
 # Install the configuration header files
 install(FILES ${PROJECT_BINARY_DIR}/config/_config.hpp
-  DESTINATION ${INSTALL_INCLUDE_DIR}/mfem/config
-  RENAME config.hpp)
-
-install(FILES ${PROJECT_SOURCE_DIR}/config/tconfig.hpp
+  ${PROJECT_SOURCE_DIR}/config/config.hpp
+  ${PROJECT_SOURCE_DIR}/config/tconfig.hpp
   DESTINATION ${INSTALL_INCLUDE_DIR}/mfem/config)
 
 # Package the whole thing up nicely

CONTRIBUTING.md

@@ -209,7 +209,7 @@ device/host memory manager.
 - The main device-relevant classes and sources are:
   + [`Device`](https://docs.mfem.org/html/device_8hpp.html)
   + [`MemoryManager`](https://docs.mfem.org/html/mem_manager_8hpp.html)
-  + the [`MFEM_FORALL`](https://docs.mfem.org/html/forall_8hpp.html) macro
+  + the [`mfem::forall`](https://docs.mfem.org/html/forall_8hpp.html) function
   + the [`cuda.hpp`](https://docs.mfem.org/html/cuda_8hpp.html) and [`occa.hpp`](https://docs.mfem.org/html/occa_8hpp.html) files
 
 #### Utilities, building and documentation

LICENSE

@@ -1,6 +1,6 @@
 BSD 3-Clause License
 
-Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC
+Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

config/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/MFEMConfig.cmake.in

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/config.hpp.in

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -39,6 +39,9 @@
 // Description of the git commit used to build MFEM.
 #cmakedefine MFEM_GIT_STRING "@MFEM_GIT_STRING@"
 
+// Enable shared library build of MFEM.
+#cmakedefine MFEM_SHARED_BUILD
+
 // Build the parallel MFEM library.
 // Requires an MPI compiler, and the libraries HYPRE and METIS.
 #cmakedefine MFEM_USE_MPI

config/cmake/modules/FindAMGX.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindAlgoim.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindAxom.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindBenchmark.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindBlitz.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindCODIPACK.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindCXXABIDemangle.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindCaliper.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindConduit.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -21,8 +21,15 @@ if(EXISTS ${CONDUIT_DIR}/include/conduit/conduit_relay_hdf5.hpp)
   # we only need  HDF5 if Conduit was built with HDF5 support
   set(Conduit_REQUIRED_PACKAGES "HDF5" CACHE STRING
       "Additional packages required by Conduit.")
-  else()
-    message(STATUS "Conduit Relay HDF5 Support is DISABLED")
+  # Suppress warning about HDF5_ROOT being set
+  if (POLICY CMP0074)
+    cmake_policy(SET CMP0074 NEW)
+  endif()
+  # HDF5_ROOT is needed in some cases, e.g. when HDF5_TARGET_NAMES is set and
+  # MFEM's FindHDF5.cmake is not used.
+  set(HDF5_ROOT ${HDF5_DIR} CACHE PATH "")
+else()
+  message(STATUS "Conduit Relay HDF5 Support is DISABLED")
 endif()
 
 include(MfemCmakeUtilities)
@@ -32,4 +39,4 @@ mfem_find_package(Conduit CONDUIT CONDUIT_DIR
   ADD_COMPONENT relay
     "include;include/conduit" conduit_relay.hpp "lib" conduit_relay
   ADD_COMPONENT blueprint
-      "include;include/conduit" conduit_blueprint.hpp "lib" conduit_blueprint)
+    "include;include/conduit" conduit_blueprint.hpp "lib" conduit_blueprint)

config/cmake/modules/FindENZYME.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindFMS.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindGSLIB.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindHDF5.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindHIOP.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindHYPRE.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindLIBDL.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindLibunwind.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindMETIS.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindMFEMBacktrace.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindMKL_CPARDISO.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindMPFR.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindMUMPS.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindNetCDF.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -14,11 +14,17 @@
 #   - NETCDF_LIBRARIES
 #   - NETCDF_INCLUDE_DIRS
 
+# Suppress warning about HDF5_ROOT being set
+if (POLICY CMP0074)
+  cmake_policy(SET CMP0074 NEW)
+endif()
+
 include(MfemCmakeUtilities)
 
 # FindHDF5.cmake uses HDF5_ROOT, so we "translate" from the MFEM convention
-# (MFEM's FindHDF5.cmake does not need HDF5_ROOT)
-# set(HDF5_ROOT ${HDF5_DIR} CACHE PATH "")
+# (Needed in some cases, e.g. when HDF5_TARGET_NAMES is set and MFEM's
+# FindHDF5.cmake is not used.)
+set(HDF5_ROOT ${HDF5_DIR} CACHE PATH "")
 
 # We need to guard against the case where HDF5 was already found but without
 # the HL extensions (in which case mfem_find_package will treat the package
@@ -35,6 +41,11 @@ mfem_find_package(NetCDF NETCDF NETCDF_DIR "include" netcdf.h "lib" netcdf
 # The netcdf library will always be the first element of NETCDF_LIBRARIES
 # and we need to insert after that library but before the hdf5 library, so
 # position 1 is used
-# (MFEM's FindHDF5.cmake does not set HDF5_C_LIBRARY_hdf5_hl and the HL library
-# is already added to NETCDF_LIBRARIES)
-# list(INSERT NETCDF_LIBRARIES 1 ${HDF5_C_LIBRARY_hdf5_hl})
+# (Needed in some cases, e.g. when HDF5_TARGET_NAMES is set and MFEM's
+# FindHDF5.cmake is not used.)
+if (HDF5_C_LIBRARY_hdf5_hl)
+  list(INSERT NETCDF_LIBRARIES 1 ${HDF5_C_LIBRARY_hdf5_hl})
+  if (NOT NetCDF_FIND_QUIETLY)
+    message(STATUS "Updated NetCDF libraries: ${NETCDF_LIBRARIES}")
+  endif()
+endif()

config/cmake/modules/FindOCCA.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindPARELAG.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindPOSIXClocks.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindParMETIS.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindRAJA.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindSLEPc.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindSTRUMPACK.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindSUNDIALS.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindScotch.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindSuiteSparse.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindSuperLUDist.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindUMPIRE.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/Find_GnuTLS.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindlibCEED.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -103,25 +103,6 @@ macro(add_mfem_examples EXE_SRCS)
       ${MFEM_EXEC_PREREQUISITES_TARGET_NAME} ${EXE_PREREQUISITE})
 
     target_link_libraries(${EXE_NAME} mfem)
-    if (MFEM_USE_MPI)
-      # Not needed: (mfem already links with MPI_CXX_LIBRARIES)
-      # target_link_libraries(${EXE_NAME} ${MPI_CXX_LIBRARIES})
-
-      # Language-specific include directories:
-      if (MPI_CXX_INCLUDE_PATH)
-        target_include_directories(${EXE_NAME} PRIVATE "${MPI_CXX_INCLUDE_PATH}")
-      endif()
-      if (MPI_CXX_COMPILE_FLAGS)
-        separate_arguments(MPI_CXX_COMPILE_ARGS UNIX_COMMAND
-          "${MPI_CXX_COMPILE_FLAGS}")
-        target_compile_options(${EXE_NAME} PRIVATE ${MPI_CXX_COMPILE_ARGS})
-      endif()
-
-      if (MPI_CXX_LINK_FLAGS)
-        set_target_properties(${EXE_NAME} PROPERTIES
-          LINK_FLAGS "${MPI_CXX_LINK_FLAGS}")
-      endif()
-    endif()
   endforeach(SRC_FILE)
 endmacro()
 
@@ -172,26 +153,6 @@ macro(add_mfem_miniapp MFEM_EXE_NAME)
   if (EXTRA_DEFINES_LIST)
     target_compile_definitions(${MFEM_EXE_NAME} PRIVATE ${EXTRA_DEFINES_LIST})
   endif()
-
-  # Handle the MPI separately
-  if (MFEM_USE_MPI)
-    # Add MPI_CXX_LIBRARIES, in case this target does not link with mfem.
-    target_link_libraries(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_LIBRARIES})
-
-    if (MPI_CXX_INCLUDE_PATH)
-      target_include_directories(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_INCLUDE_PATH})
-    endif()
-    if (MPI_CXX_COMPILE_FLAGS)
-      separate_arguments(MPI_CXX_COMPILE_ARGS UNIX_COMMAND
-        "${MPI_CXX_COMPILE_FLAGS}")
-      target_compile_options(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_COMPILE_ARGS})
-    endif()
-
-    if (MPI_CXX_LINK_FLAGS)
-      set_target_properties(${MFEM_EXE_NAME} PROPERTIES
-        LINK_FLAGS "${MPI_CXX_LINK_FLAGS}")
-    endif()
-  endif()
 endmacro()
 
 
@@ -436,9 +397,9 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
               break()
             endif()
           endforeach()
-          if (${Required} AND NOT ${ReqPack}_FOUND)
+          if (Required AND NOT ${ReqPack}_FOUND)
             message(FATAL_ERROR " *** Required package ${ReqPack} not found."
-              "Checked target names: ${ReqPack} ${${ReqPack}_TARGET_NAMES}")
+              " Checked target names: ${ReqPack} ${${ReqPack}_TARGET_NAMES}")
           endif()
         endif()
       endif()
@@ -504,14 +465,18 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
           endif()
           get_target_property(IsImported ${TargetName} IMPORTED)
           if (IsImported)
+            get_target_property(ImpConfigs ${TargetName} IMPORTED_CONFIGURATIONS)
+            # message(STATUS "${ReqPack} imported configs: ${ImpConfigs}")
             set(ImportConfig ${${ReqPack}_IMPORT_CONFIG})
             if (NOT ImportConfig)
               set(ImportConfig RELEASE)
+              list(FIND ImpConfigs ${ImportConfig} _Index)
+              if (_Index EQUAL -1)
+                list(GET ImpConfigs 0 ImportConfig)
+              endif()
             endif()
             set(ImportConfigSuffix "_${ImportConfig}")
-            get_target_property(ImpConfigs ${TargetName} IMPORTED_CONFIGURATIONS)
-            list(FIND ImpConfigs ${ImportConfig} _Index)
-            if ((_Index EQUAL -1) OR ("${ImportConfig}" STREQUAL "NO_CONFIG"))
+            if ((NOT ImportConfig) OR ("${ImportConfig}" STREQUAL "NO_CONFIG"))
               set(ImportConfig "NO_CONFIG")
               set(ImportConfigSuffix "")
               # message(FATAL_ERROR " *** ${ReqPack}: configuration "
@@ -529,7 +494,8 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
           else()
             # Set _Pack_LIBS from the target properties for ImportConfig
             foreach (_prop IMPORTED_LOCATION${ImportConfigSuffix}
-                IMPORTED_LINK_INTERFACE_LIBRARIES${ImportConfigSuffix})
+                IMPORTED_LINK_INTERFACE_LIBRARIES${ImportConfigSuffix}
+                INTERFACE_LINK_LIBRARIES)
               get_target_property(_value ${TargetName} ${_prop})
               if (_value)
                 list(APPEND _Pack_LIBS ${_value})
@@ -780,6 +746,7 @@ function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
       list(APPEND CompileOpts "-isystem \"${SysDir}\"")
     endforeach()
   endif()
+  set(AddInterfaceLinkLibs TRUE)
   if ("${type}" STREQUAL "STATIC_LIBRARY")
     get_target_property(Location ${tgt} LOCATION)
     if (Location)
@@ -801,18 +768,7 @@ function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
       message(STATUS " *** Warning: [${tgt}] LOCATION not defined!")
     endif()
   elseif ("${type}" STREQUAL "INTERFACE_LIBRARY")
-    get_target_property(Libs ${tgt} INTERFACE_LINK_LIBRARIES)
-    if (Libs)
-      foreach(Lib ${Libs})
-        if (NOT (TARGET ${Lib}))
-          list(APPEND LinkOpts "${Lib}")
-        else()
-          mfem_get_target_options(${Lib} COpts LOpts)
-          list(APPEND CompileOpts ${COpts})
-          list(APPEND LinkOpts ${LOpts})
-        endif()
-      endforeach()
-    endif()
+    # The INTERFACE_LINK_LIBRARIES property is handled below.
     # Other properties we may need to handle:
     # INTERFACE_LINK_DEPENDS
     # INTERFACE_LINK_DIRECTORIES
@@ -820,6 +776,26 @@ function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
   else()
     message(STATUS " *** Warning: [${tgt}] uses target type '${type}'"
       " which is not supported!")
+    set(AddInterfaceLinkLibs FALSE)
+  endif()
+  if (AddInterfaceLinkLibs)
+    get_target_property(Libs ${tgt} INTERFACE_LINK_LIBRARIES)
+    # message(STATUS "${tgt}[INTERFACE_LINK_LIBRARIES]: ${Libs}")
+    if (Libs)
+      foreach(Lib ${Libs})
+        if (NOT (TARGET ${Lib}))
+          # message(STATUS "Lib = ${Lib}")
+          # Filter-out generator expressions
+          if (NOT ("${Lib}" MATCHES "^\\$"))
+            list(APPEND LinkOpts "${Lib}")
+          endif()
+        else()
+          mfem_get_target_options(${Lib} COpts LOpts)
+          list(APPEND CompileOpts ${COpts})
+          list(APPEND LinkOpts ${LOpts})
+        endif()
+      endforeach()
+    endif()
   endif()
 
   # Other potentially relevant properties:
@@ -843,6 +819,9 @@ function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
   set(${CompileOptsVar} "${CompileOpts}" PARENT_SCOPE)
   set(${LinkOptsVar} "${LinkOpts}" PARENT_SCOPE)
 
+  # message(STATUS "${tgt}[CompileOpts]: ${CompileOpts}")
+  # message(STATUS "${tgt}[LinkOpts]: ${LinkOpts}")
+
 endfunction(mfem_get_target_options)
 
 
@@ -894,7 +873,7 @@ function(mfem_export_mk_files)
          "${cxx_std_flag} ${CMAKE_CXX_FLAGS_${BUILD_TYPE}} ${CMAKE_CXX_FLAGS}"
          MFEM_CXXFLAGS)
   set(MFEM_TPLFLAGS "")
-  foreach(dir ${MFEM_TPL_INCLUDE_DIRS})
+  foreach(dir ${TPL_INCLUDE_DIRS})
     set(MFEM_TPLFLAGS "${MFEM_TPLFLAGS} -I${dir}")
   endforeach()
   # TODO: MFEM_TPLFLAGS: add other TPL flags, in addition to the -I flags.

config/config.hpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -30,11 +30,30 @@
 #endif
 
 // Windows specific options
-#ifdef _WIN32
-#ifndef _USE_MATH_DEFINES
+#if defined(_WIN32) && !defined(_USE_MATH_DEFINES)
 // Macro needed to get defines like M_PI from <cmath>. (Visual Studio C++ only?)
 #define _USE_MATH_DEFINES
 #endif
+// Macro MFEM_EXPORT: this macro is used when declaring exported global
+// variables and static class variables in public header files, e.g.:
+//    extern MFEM_EXPORT Geometry Geometries;
+//    static MFEM_EXPORT Device device_singleton;
+// In cases where a class contains multiple static variables, instead of marking
+// all such variables with MFEM_EXPORT, one can mark the class with MFEM_EXPORT,
+// e.g.:
+//    class MFEM_EXPORT MemoryManager ...
+// Note: MFEM's GitHub CI includes a shared MSVC build that will fail if a
+// variable that needs MFEM_EXPORT does not have it. However, builds with
+// optional external libraries are not tested and may require separate checks to
+// determine the necessity of MFEM_EXPORT.
+#if defined(_MSC_VER) && defined(MFEM_SHARED_BUILD)
+#ifdef mfem_EXPORTS
+#define MFEM_EXPORT __declspec(dllexport)
+#else
+#define MFEM_EXPORT __declspec(dllimport)
+#endif
+#else
+#define MFEM_EXPORT
 #endif
 // On Cygwin the option -std=c++11 prevents the definition of M_PI. Defining
 // the following macro allows us to get M_PI and some needed functions, e.g.

config/config.hpp.in

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -39,6 +39,9 @@
 // Description of the git commit used to build MFEM.
 // #define MFEM_GIT_STRING "@MFEM_GIT_STRING@"
 
+// Enable shared library build of MFEM.
+// #define MFEM_SHARED_BUILD
+
 // Build the parallel MFEM library.
 // Requires an MPI compiler, and the libraries HYPRE and METIS.
 // #define MFEM_USE_MPI

config/config.mk.in

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/defaults.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -19,6 +19,7 @@ if (NOT CMAKE_BUILD_TYPE)
 endif()
 
 # MFEM options. Set to mimic the default "defaults.mk" file.
+option(BUILD_SHARED_LIBS "Enable shared library build of MFEM" OFF)
 option(MFEM_USE_MPI "Enable MPI parallel build" OFF)
 option(MFEM_USE_METIS "Enable METIS usage" ${MFEM_USE_MPI})
 option(MFEM_USE_EXCEPTIONS "Enable the use of exceptions" OFF)

config/defaults.mk

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -58,7 +58,7 @@ HIP_CXX = hipcc
 # example: gfx600 (tahiti), gfx700 (kaveri), gfx701 (hawaii), gfx801 (carrizo),
 # gfx900, gfx1010, etc.
 HIP_ARCH = gfx900
-HIP_FLAGS = --amdgpu-target=$(HIP_ARCH)
+HIP_FLAGS = --offload-arch=$(HIP_ARCH)
 HIP_XCOMPILER =
 HIP_XLINKER   = -Wl,
 

config/get_hypre_version.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //

config/get_mumps_version.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //

config/githooks/pre-push

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -35,8 +35,8 @@ cd $(git rev-parse --show-toplevel)
 # copyright check
 copyright=true
 if [[ "${option}" == "--copyright" || "${option}" == "" ]]; then
-  if git grep -l "^\(#\|//\).*\(\-2021\|\ 2010,\)" > matches.txt; then
-    echo "Please update the following files to Copyright (c) 2010-2022:"
+  if git grep -n "^\(#\|//\).*Copyright.*2010-20\(2[^3]\|[^2].\)" > matches.txt; then
+    echo "Please update the following files to Copyright (c) 2010-2023:"
     cat matches.txt
     copyright=false
   fi

config/makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/sample-runs.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -138,7 +138,7 @@ groups_parallel=(
 '"meshing"
    "Meshing miniapps:"
    "miniapps/meshing"
-   "pmesh-optimizer.cpp pminimal-surface.cpp"'
+   "pmesh-optimizer.cpp pmesh-fitting.cpp pminimal-surface.cpp"'
 '"electromagnetics"
    "Electromagnetics miniapps:"
    "miniapps/electromagnetics"
@@ -227,7 +227,7 @@ groups_all=(
    "Meshing miniapps:"
    "miniapps/meshing"
    "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp
-    {,p}mesh-optimizer.cpp {,p}minimal-surface.cpp"'
+    {,p}mesh-optimizer.cpp pmesh-fitting.cpp {,p}minimal-surface.cpp"'
 '"electromagnetics"
    "Electromagnetics miniapps:"
    "miniapps/electromagnetics"

config/tconfig.hpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //

config/test.mk

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

data/segment-nurbs.mesh

@@ -0,0 +1,46 @@
+MFEM NURBS mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# SEGMENT     = 1
+# SQUARE      = 3
+# CUBE        = 5
+#
+
+dimension
+1
+
+elements
+1
+1 1 0 1
+
+boundary
+2
+1 0 0
+2 0 1
+
+edges
+1
+0 0 1
+
+vertices
+2
+
+knotvectors
+1
+1 2 0 0 1 1
+
+weights
+1
+1
+
+FiniteElementSpace
+FiniteElementCollection: NURBS1
+VDim: 1
+Ordering: 1
+
+0
+1
+
+

doc/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

doc/CodeDocumentation.conf.in

@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = v4.5.1
+PROJECT_NUMBER         = v4.5.3
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

doc/CodeDocumentation.dox

@@ -39,7 +39,7 @@ namespace mfem {
  * - Device
  * - Memory
  * - MemoryManager
- * - MFEM_FORALL macro in forall.hpp
+ * - mfem::forall functions in forall.hpp
  *
  * <H3>Example codes</H3>
  * - <a class="el" href="ex0_8cpp_source.html">Example 0</a>: simplest example, nodal H1 FEM for the Laplace problem

doc/makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

examples/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

examples/amgx/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

examples/amgx/makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

examples/caliper/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

examples/caliper/makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

examples/ex20p.cpp

@@ -246,17 +246,23 @@ int main(int argc, char *argv[])
    e_var /= (nsteps + 1);
    double e_sd = sqrt(e_var);
 
+   double e_loc_stats[2];
+   double *e_stats = (myid == 0) ? new double[2 * num_procs] : (double*)NULL;
+
+   e_loc_stats[0] = e_mean;
+   e_loc_stats[1] = e_sd;
+   MPI_Gather(e_loc_stats, 2, MPI_DOUBLE, e_stats, 2, MPI_DOUBLE, 0, comm);
+
    if (myid == 0)
    {
-      cout << endl << "Mean and standard deviation of the energy" << endl;
-   }
-   for (int i = 0; i < num_procs; i++)
-   {
-      if (myid == i)
+      cout << endl << "Mean and standard deviation of the energy "
+           << "for different initial conditions" << endl;
+      for (int i = 0; i < num_procs; i++)
       {
-         cout << myid << ": " << e_mean << "\t" << e_sd << endl;
+         cout << i << ": " << e_stats[2 * i + 0]
+              << "\t" << e_stats[2 * i + 1] << endl;
       }
-      MPI_Barrier(comm);
+      delete [] e_stats;
    }
 
    // 9. Finalize the GnuPlot output

examples/ex25.cpp

@@ -32,6 +32,7 @@
 //               We recommend viewing Example 22 before viewing this example.
 
 #include "mfem.hpp"
+#include <memory>
 #include <fstream>
 #include <iostream>
 
@@ -467,16 +468,14 @@ int main(int argc, char *argv[])
       offsets[2] = fespace->GetTrueVSize();
       offsets.PartialSum();
 
-      Operator *pc_r = nullptr;
-      Operator *pc_i = nullptr;
-      int s = (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0;
+      std::unique_ptr<Operator> pc_r;
+      std::unique_ptr<Operator> pc_i;
+      double s = (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0;
       if (pa)
       {
          // Jacobi Smoother
-         OperatorJacobiSmoother *d00 = new OperatorJacobiSmoother(prec, ess_tdof_list);
-         ScaledOperator *d11 = new ScaledOperator(d00, s);
-         pc_r = d00;
-         pc_i = d11;
+         pc_r.reset(new OperatorJacobiSmoother(prec, ess_tdof_list));
+         pc_i.reset(new ScaledOperator(pc_r.get(), s));
       }
       else
       {
@@ -485,15 +484,13 @@ int main(int argc, char *argv[])
          prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
 
          // Gauss-Seidel Smoother
-         GSSmoother *gs00 = new GSSmoother(*PCOpAh.As<SparseMatrix>());
-         ScaledOperator *gs11 = new ScaledOperator(gs00, s);
-         pc_r = gs00;
-         pc_i = gs11;
+         pc_r.reset(new GSSmoother(*PCOpAh.As<SparseMatrix>()));
+         pc_i.reset(new ScaledOperator(pc_r.get(), s));
       }
 
       BlockDiagonalPreconditioner BlockDP(offsets);
-      BlockDP.SetDiagonalBlock(0, pc_r);
-      BlockDP.SetDiagonalBlock(1, pc_i);
+      BlockDP.SetDiagonalBlock(0, pc_r.get());
+      BlockDP.SetDiagonalBlock(1, pc_i.get());
 
       GMRESSolver gmres;
       gmres.SetPrintLevel(1);

examples/ex25p.cpp

@@ -524,16 +524,14 @@ int main(int argc, char *argv[])
       offsets[2] = fespace->GetTrueVSize();
       offsets.PartialSum();
 
-      Operator *pc_r = nullptr;
-      Operator *pc_i = nullptr;
+      std::unique_ptr<Operator> pc_r;
+      std::unique_ptr<Operator> pc_i;
       int s = (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0;
       if (pa)
       {
          // Jacobi Smoother
-         OperatorJacobiSmoother *d00 = new OperatorJacobiSmoother(prec, ess_tdof_list);
-         ScaledOperator *d11 = new ScaledOperator(d00, s);
-         pc_r = d00;
-         pc_i = d11;
+         pc_r.reset(new OperatorJacobiSmoother(prec, ess_tdof_list));
+         pc_i.reset(new ScaledOperator(pc_r.get(), s));
       }
       else
       {
@@ -541,15 +539,13 @@ int main(int argc, char *argv[])
          prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
 
          // Hypre AMS
-         HypreAMS *ams00 = new HypreAMS(*PCOpAh.As<HypreParMatrix>(), fespace);
-         ScaledOperator *ams11 = new ScaledOperator(ams00, s);
-         pc_r = ams00;
-         pc_i = ams11;
+         pc_r.reset(new HypreAMS(*PCOpAh.As<HypreParMatrix>(), fespace));
+         pc_i.reset(new ScaledOperator(pc_r.get(), s));
       }
 
       BlockDiagonalPreconditioner BlockDP(offsets);
-      BlockDP.SetDiagonalBlock(0, pc_r);
-      BlockDP.SetDiagonalBlock(1, pc_i);
+      BlockDP.SetDiagonalBlock(0, pc_r.get());
+      BlockDP.SetDiagonalBlock(1, pc_i.get());
 
       GMRESSolver gmres(MPI_COMM_WORLD);
       gmres.SetPrintLevel(1);

examples/ex30.cpp

@@ -90,7 +90,7 @@ int main(int argc, char *argv[])
    const char *mesh_file = "../data/star.mesh";
    int order = 1;
    int nc_limit = 1;
-   int max_elems = 1e5;
+   int max_elems = 100*1000;
    double double_max_elems = double(max_elems);
    bool visualization = true;
    double osc_threshold = 1e-3;

examples/ex33.cpp

@@ -123,7 +123,7 @@ int main(int argc, char *argv[])
 
    // 2. Compute the rational expansion coefficients that define the
    //    integer-order PDEs.
-   const int power_of_laplace = floor(alpha);
+   const int power_of_laplace = (int)floor(alpha);
    double exponent_to_approximate = alpha - power_of_laplace;
    bool integer_order = false;
    // Check if alpha is an integer or not.

examples/ginkgo/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

examples/ginkgo/makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -78,4 +78,4 @@ clean-build:
 	rm -rf *.dSYM *.TVD.*breakpoints
 
 clean-exec:
-	@rm -f refined.mesh sol.gf mesh.* sol.*
+	@rm -f refined.mesh sol.gf

examples/hiop/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

examples/hiop/makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #