address pedantic self-assign warning in macos-latest-make-opt-seq-int32 CI test

remove __nullptr header
add two examples demonstrating use of internal boundary interface integrators to solve simple polynomial MMS problems
2023-07-12 15:11:50 -06:00 · 2023-07-12 14:55:06 -06:00 · 2023-07-12 14:39:30 -06:00 · 2023-07-11 11:26:30 -06:00 · 2023-07-10 11:36:39 -07:00 · 2023-07-10 11:34:54 -07:00
912 changed files with 44543 additions and 16756 deletions
@@ -31,7 +31,7 @@ install:
 - ps: ( get-content "GKlib\gk_arch.h") | % { If ($_.ReadCount -ge 52) {$_ -replace "#ifdef __MSC__","#ifdef DISABLE_THIS_ANCIENT_MSC_CHECK"} Else {$_} } | set-content "GKlib\gk_arch.h"
 - cmake -H. -Bbuild
 # -DCMAKE_BUILD_TYPE=Release
- cmake --build build
+- cmake --build build --target metis -j 4
 - set METIS_PATH=%cd%
 - cd ..

@@ -40,7 +40,7 @@ install:
 - 7z x v2.19.0.tar.gz -so | 7z x -si -ttar > nul
 - cd hypre-2.19.0/src
 - cmake -H. -Bbuild
- cmake --build build
+- cmake --build build -j 4
 - cmake --build build --target install
 - cd ../..

@@ -49,14 +49,20 @@ install:

 # MFEM
 before_build:
- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DHYPRE_DIR=%cd%\..\hypre-2.19.0\src\hypre -DMETIS_LIBRARIES=%METIS_PATH%\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%METIS_PATH%\include
- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_serial -DMFEM_USE_MPI=FALSE
+- cmake . -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DHYPRE_DIR=%cd%\..\hypre-2.19.0\src\hypre -DMETIS_LIBRARIES=%METIS_PATH%\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%METIS_PATH%\include
+# - cmake . -Bbuild_serial -DMFEM_USE_MPI=FALSE
+- cmake . -Bbuild_shared_serial -DMFEM_USE_MPI=FALSE -DBUILD_SHARED_LIBS=ON

 build_script:
- cmake --build build_parallel --config Release -j 4
- cmake --build build_serial --config Release -j 4
- cmake --build build_serial --target exec --config Release -j 4
+- cmake --build build_parallel --config Debug -j 4
+# - cmake --build build_serial --config Release -j 4
+# - cmake --build build_serial --target exec --config Release -j 4
+- cmake --build build_shared_serial --config Release -j 4
+- cmake --build build_shared_serial --target exec --config Release -j 4

 after_build:
- cd build_serial
+# - cd build_serial
+# - ctest -C Release --output-on-failure
+# - cd ..
+- cd build_shared_serial
 - ctest -C Release --output-on-failure
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -74,12 +74,19 @@ jobs:
            codecov: YES
          - os: windows-latest
            codecov: NO
+          - os: windows-latest
+            target: opt
+            mpi: par
+            config-opts: '-DBUILD_SHARED_LIBS=ON'
          - os: ubuntu-latest
            target: opt
            codecov: NO
            mpi: par
            build-system: cmake
            hypre-target: int32
+            # This option can be set to pass additional configuration options to
+            # the MFEM configuration command.
+            # config-opts: '-DCMAKE_VERBOSE_MAKEFILE=ON'
          - os: ubuntu-latest
            target: opt
            codecov: NO
@@ -155,7 +162,7 @@ jobs:

    - name: get hypre
      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
-      uses: mfem/github-actions/build-hypre@v2.2
+      uses: mfem/github-actions/build-hypre@v2.4
      with:
        archive: ${{ env.HYPRE_ARCHIVE }}
        dir: ${{ env.HYPRE_TOP_DIR }}
@@ -164,7 +171,7 @@ jobs:

    - name: get hypre (Windows)
      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
-      uses: mfem/github-actions/build-hypre@v2.2
+      uses: mfem/github-actions/build-hypre@v2.4
      with:
        archive: ${{ env.HYPRE_ARCHIVE }}
        dir: ${{ env.HYPRE_TOP_DIR }}
@@ -183,7 +190,7 @@ jobs:

    - name: install metis
      if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@v2.2
+      uses: mfem/github-actions/build-metis@v2.4
      with:
        archive: ${{ env.METIS_ARCHIVE }}
        dir: ${{ env.METIS_TOP_DIR }}
@@ -210,7 +217,7 @@ jobs:

    # MFEM build and test
    - name: build
-      uses: mfem/github-actions/build-mfem@v2.2
+      uses: mfem/github-actions/build-mfem@v2.4
      env:
        VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
      with:
@@ -222,7 +229,7 @@ jobs:
        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
        metis-dir: ${{ env.METIS_TOP_DIR }}
        mfem-dir: ${{ env.MFEM_TOP_DIR }}
-        config-options: ${{ env.MFEM_EXTRA_CONFIG }}
+        config-options: ${{ matrix.config-opts }}
        library-only: ${{ matrix.target == 'dbg' }}

    # Run checks (and only checks) on debug targets
@@ -256,13 +263,15 @@ jobs:
      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os != 'ubuntu-latest'
      run: |
        CTEST_CONFIG="Release"
-        cd ${{ env.MFEM_TOP_DIR }}/build && ctest --output-on-failure -C ${CTEST_CONFIG}
+        cd ${{ env.MFEM_TOP_DIR }}/build && \
+        ctest --output-on-failure -C ${CTEST_CONFIG} || \
+        ctest --rerun-failed --output-on-failure -C ${CTEST_CONFIG}
      shell: bash

    # Code coverage (process and upload reports)
    - name: codecov
      if: matrix.codecov == 'YES'
-      uses: mfem/github-actions/upload-coverage@v2.2
+      uses: mfem/github-actions/upload-coverage@v2.4
      with:
        name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
        project_dir: ${{ env.MFEM_TOP_DIR }}
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -57,7 +57,7 @@ jobs:

    - name: Get Hypre
      if: steps.hypre-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-hypre@v2.2
+      uses: mfem/github-actions/build-hypre@v2.4
      with:
        archive: ${{ env.HYPRE_ARCHIVE }}
        dir: ${{ env.HYPRE_TOP_DIR }}
@@ -72,14 +72,14 @@ jobs:

    - name: Install Metis
      if: steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@v2.2
+      uses: mfem/github-actions/build-metis@v2.4
      with:
        archive: ${{ env.METIS_ARCHIVE }}
        dir: ${{ env.METIS_TOP_DIR }}

    # MFEM build and test
    - name: build-mfem
-      uses: mfem/github-actions/build-mfem@v2.2
+      uses: mfem/github-actions/build-mfem@v2.4
      with:
        os: ${{ runner.os }}
        target: opt
@@ -0,0 +1,70 @@
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+name: "Sanitizer"
+
+permissions:
+  actions: write
+
+on:
+  push:
+    branches:
+      - master
+      - next
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  Serial:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Cancel Previous Runs
+      uses: styfle/cancel-workflow-action@0.11.0
+      with:
+        access_token: ${{ github.token }}
+
+    - name: MFEM Checkout
+      uses: actions/checkout@v3
+      with:
+        path: mfem
+
+    - name: MFEM Build
+      uses: mfem/github-actions/build-mfem@v2.4
+      with:
+        os: ${{ runner.os }}
+        target: opt
+        mpi: seq
+        hypre-dir: unused-hypre-dir
+        metis-dir: unused-metis-dir
+        mfem-dir: mfem
+        build-system: make
+        library-only: false
+        config-options:
+          CXX="clang++-14"
+          CXXFLAGS="-g -O1 -std=c++11
+                    -fsanitize=address
+                    -fno-omit-frame-pointer
+                    -fsanitize-address-use-after-scope"
+
+    - name: MFEM Info
+      working-directory: mfem
+      run: make info
+
+    - name: MFEM Sanitize
+      working-directory: mfem
+      run:
+        ASAN_OPTIONS="detect_leaks=1,
+                      strict_init_order=1,
+                      strict_string_checks=1,
+                      check_initialization_order=1,
+                      detect_stack_use_after_return=1"
+        make test
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -21,11 +21,15 @@ CMakeFiles/
 # Default install location
 /mfem/

+# Typical build directory
+/build/
+
 # Generated files in main directory, config/ and docs/
 /deps.mk
 config/_config.hpp
 config/config.mk
 config/sample-runs-build.log
+config/user.cmake
 config/user.mk
 doc/CodeDocumentation.conf
 doc/CodeDocumentation.html
@@ -118,11 +122,9 @@ examples/amgx/sol.gf
 examples/amgx/mesh.*
 examples/amgx/sol.*

-examples/gingko/ex1
-examples/gingko/refined.mesh
-examples/gingko/sol.gf
-examples/gingko/mesh.*
-examples/gingko/sol.*
+examples/ginkgo/ex1
+examples/ginkgo/refined.mesh
+examples/ginkgo/sol.gf

 examples/hiop/ex9
 examples/hiop/ex9p
@@ -148,8 +150,8 @@ examples/petsc/velocity.*
 examples/petsc/elastic_energy.*
 examples/petsc/mode_*

-examples/pumi/ex1
-examples/pumi/ex[126]p
+examples/pumi/ex[12]
+examples/pumi/ex[16]p
 examples/pumi/refined.mesh
 examples/pumi/sol.gf
 examples/pumi/mesh.*
@@ -194,6 +196,9 @@ miniapps/gslib/field-diff
 miniapps/gslib/field-interp
 miniapps/gslib/findpts
 miniapps/gslib/pfindpts
+miniapps/gslib/schwarz_ex1
+miniapps/gslib/schwarz_ex1p
+miniapps/gslib/interpolated.gf

 miniapps/meshing/mobius-strip
 miniapps/meshing/klein-bottle
@@ -206,9 +211,11 @@ miniapps/meshing/trimmer
 miniapps/meshing/reflector
 miniapps/meshing/mesh-optimizer
 miniapps/meshing/pmesh-optimizer
+miniapps/meshing/pmesh-fitting
 miniapps/meshing/minimal-surface
 miniapps/meshing/pminimal-surface
 miniapps/meshing/polar-nc
+miniapps/meshing/mesh-quality
 miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
 miniapps/meshing/toroid-*.mesh
@@ -245,6 +252,7 @@ miniapps/navier/navier_tgv
 miniapps/navier/navier_shear
 miniapps/navier/navier_3dfoc
 miniapps/navier/navier_turbchan
+miniapps/navier/navier_cht
 miniapps/navier/tgv_out*.txt
 miniapps/navier/*_output

@@ -310,12 +318,28 @@ miniapps/solvers/ParaView
 miniapps/solvers/mesh.*
 miniapps/solvers/sol.*

+miniapps/hdiv-linear-solver/darcy
+miniapps/hdiv-linear-solver/grad_div
+
 miniapps/parelag/MultilevelHcurlHdivSolver
 miniapps/parelag/*.mesh

 miniapps/multidomain/multidomain
 miniapps/hooke/hooke

+miniapps/dpg/diffusion
+miniapps/dpg/pdiffusion
+miniapps/dpg/convection-diffusion
+miniapps/dpg/pconvection-diffusion
+miniapps/dpg/acoustics
+miniapps/dpg/pacoustics
+miniapps/dpg/maxwell
+miniapps/dpg/pmaxwell
+miniapps/dpg/ParaView
+
+miniapps/spde/generate_random_field
+miniapps/spde/ParaView
+
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
@@ -328,6 +352,8 @@ tests/unit/tmop_pa_tests_*
 tests/unit/ptmop_pa_tests_*
 tests/unit/ceed_tests
 tests/unit/debug_device_tests
+tests/unit/parallel_in_serial.mesh
+tests/unit/parallel_in_serial.gf

 # Benchmark binaries
 tests/benchmarks/bench_ceed
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -22,12 +22,10 @@
        date
        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
+        # every 5 seconds; we may want to add a counter for the number of
        # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
+        while ! flock -n 9; do
+          sleep 5
        done
        echo "Acquired lock on '$PWD/autotest.lock'"
        date
@@ -57,12 +55,10 @@
        date
        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
+        # every 5 seconds; we may want to add a counter for the number of
        # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
+        while ! flock -n 9; do
+          sleep 5
        done
        echo "Acquired lock on '$PWD/autotest.lock'"
        date
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -47,12 +47,10 @@ setup_baseline:
        date
        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
+        # every 5 seconds; we may want to add a counter for the number of
        # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
+        while ! flock -n 9; do
+          sleep 5
        done
        echo "Acquired lock on '$PWD/autotest.lock'"
        date
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -35,13 +35,11 @@ setup:
      (
        date
        echo "Waiting to acquire lock on '$PWD/mfem-data.lock' ..."
-        # try to get an exclusive lock on fd 9 (mfem-data.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
+        # try to get an exclusive lock on fd 9 (mfem-data.lock) repeating the
+        # try every 5 seconds; we may want to add a counter for the number of
        # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
+        while ! flock -n 9; do
+          sleep 5
        done
        echo "Acquired lock on '$PWD/mfem-data.lock'"
        date
@@ -69,12 +67,10 @@ setup:
        date
        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
+        # every 5 seconds; we may want to add a counter for the number of
        # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
+        while ! flock -n 9; do
+          sleep 5
        done
        echo "Acquired lock on '$PWD/autotest.lock'"
        date
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -14,14 +14,14 @@ stages:
  - build_and_test
  - report

-opt_mpi_cuda_xl_16_1_1_8:
+opt_mpi_cuda_xl_16_1_1_12:
  variables:
-    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=70"
+    SPEC: "%xl@16.1.1.12 +mpi +cuda cuda_arch=70"
  extends: .build_and_test_on_lassen

 opt_mpi_cuda_hypre_cuda_xl:
  variables:
-    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
+    SPEC: "%xl@16.1.1.12 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
  extends: .build_and_test_on_lassen

 # Jobs report
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -51,6 +51,8 @@ cleanup:
  script:
    - echo "BUILD_ROOT=${BUILD_ROOT}"
    - rm -rf "${BUILD_ROOT}" || true
+    - echo "CI_PROJECT_DIR=${CI_PROJECT_DIR}"
+    - make -C "${CI_PROJECT_DIR}" distclean

 report_baseline:
  extends: [.on_quartz]
@@ -66,12 +68,10 @@ report_baseline:
        date
        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
-        # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
-        # command to hang indefinitely sometimes, so we use the timeout & retry
-        # as a workaround; we may want to add a counter for the number of
+        # every 5 seconds; we may want to add a counter for the number of
        # retries to interrupt a potential infinite loop
-        while ! flock -w 5 9; do
-          true
+        while ! flock -n 9; do
+          sleep 5
        done
        echo "Acquired lock on '$PWD/autotest.lock'"
        date
@@ -82,12 +82,14 @@ report_baseline:
        rundir="${MACHINE_NAME}/$(date +%Y-%m-%d)-gitlab-${BASELINE_TEST}-${CI_COMMIT_REF_SLUG}"
        rundir=$(${CI_PROJECT_DIR}/.gitlab/scripts/safe_create_rundir ${rundir})
        cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
-        # We create an autotest-email.html file, because that's how we signal that there was a diff (temporary).
-        if [[ -f ${rundir}/${BASELINE_TEST}.err ]]; then
-          cp ${rundir}/${BASELINE_TEST}.err ${rundir}/autotest-email.html
-        fi
        printf "%s\n" "" "Pipeline URL:" "$CI_PIPELINE_URL" \
          >> ${rundir}/pipeline.txt
+        # We create an autotest-email.html file, because that's how we signal
+        # that there was an error / diff (temporary).
+        if [[ -f ${rundir}/${BASELINE_TEST}.err ]] || \
+           [[ -f ${rundir}/${BASELINE_TEST}-${SYS_TYPE}.diff ]]; then
+          cp ${rundir}/pipeline.txt ${rundir}/autotest-email.html
+        fi
        msg="GitLab CI log for ${BASELINE_TEST} on ${MACHINE_NAME} ($(date +%Y-%m-%d))"
        if [[ "$AUTOTEST_COMMIT" != "NO" ]]; then
          git pull && \
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -27,44 +27,39 @@ allocate_resource:
  timeout: 6h

 # GitLab jobs for the Quartz machine at LLNL
-debug_ser_gcc_4_9_3:
+debug_ser_gcc_10:
  variables:
-    SPEC: "%gcc@4.9.3 +debug~mpi"
+    SPEC: "%gcc@10.3.1 +debug~mpi"
  extends: .build_and_test_on_quartz

-debug_ser_gcc_6_1_0:
+debug_par_gcc_10:
  variables:
-    SPEC: "%gcc@6.1.0 +debug~mpi"
+    SPEC: "%gcc@10.3.1 +debug+mpi"
  extends: .build_and_test_on_quartz

-debug_par_gcc_6_1_0:
+opt_ser_gcc_10:
  variables:
-    SPEC: "%gcc@6.1.0 +debug+mpi"
+    SPEC: "%gcc@10.3.1 ~mpi"
  extends: .build_and_test_on_quartz

-opt_ser_gcc_6_1_0:
+opt_par_gcc_10:
  variables:
-    SPEC: "%gcc@6.1.0 ~mpi"
+    SPEC: "%gcc@10.3.1"
  extends: .build_and_test_on_quartz

-opt_par_gcc_6_1_0:
+opt_par_gcc_10_sundials:
  variables:
-    SPEC: "%gcc@6.1.0"
+    SPEC: "%gcc@10.3.1 +sundials"
  extends: .build_and_test_on_quartz

-opt_par_gcc_6_1_0_sundials:
+opt_par_gcc_10_petsc:
  variables:
-    SPEC: "%gcc@6.1.0 +sundials"
+    SPEC: "%gcc@10.3.1 +petsc ^petsc+mumps~superlu-dist"
  extends: .build_and_test_on_quartz

-opt_par_gcc_6_1_0_petsc:
+opt_par_gcc_10_pumi:
  variables:
-    SPEC: "%gcc@6.1.0 +petsc ^petsc+mumps~superlu-dist"
-  extends: .build_and_test_on_quartz
-
-opt_par_gcc_6_1_0_pumi:
-  variables:
-    SPEC: "%gcc@6.1.0 +pumi"
+    SPEC: "%gcc@10.3.1 +pumi"
  extends: .build_and_test_on_quartz

 # Release
@@ -1,6 +1,6 @@
 #!/bin/bash

-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -42,47 +42,34 @@ fi
 # post
 mkdir ${artifacts_path}

-if [[ -s ${glob_err} ]]
-then
-  echo "ERROR during ${BASELINE_TEST} execution";
-  echo "Here is the ${glob_err} file content";
-  cat ${glob_err}
-  cp ${glob_err} ${artifacts_path}/${glob_err}
-  exit 1;
-elif [[ ! -f ${base_patch} && ! -f ${base_out} ]]
-then
-  echo "Something went WRONG in ${BASELINE_TEST}:";
-  echo "Either ${base_patch} or ${base_out} should exists";
-  exit 1;
-elif [[ -f ${base_patch} ]]
-then
-  echo "${BASELINE_TEST}: Differences found, patch generated"
-  cp ${base_patch} ${artifacts_path}/${base_patch}
-elif [[ -f ${base_out} ]]
-then
-  echo "${BASELINE_TEST}: Differences found, replacement file generated"
-  cp ${base_out} ${artifacts_path}/${base_out}
-fi
-
+status=0
 if [[ -f ${BASELINE_TEST}.out ]]; then
  cp ${BASELINE_TEST}.out ${artifacts_path}
 fi
-
+if [[ -s ${glob_err} ]]; then
+  echo "ERROR during ${BASELINE_TEST} execution"
+  echo "Here is the ${glob_err} file content"
+  cat ${glob_err}
+  cp ${glob_err} ${artifacts_path}/${glob_err}
+  status=1
+fi
+if [[ -f ${base_patch} ]]; then
+  echo "${BASELINE_TEST}: Differences found, patch generated"
+  cp ${base_patch} ${artifacts_path}/${base_patch}
+elif [[ -f ${base_out} ]]; then
+  echo "${BASELINE_TEST}: Differences found, replacement file generated"
+  cp ${base_out} ${artifacts_path}/${base_out}
+fi
 # base_diff won't even exist if there is no difference.
-if [[ -f ${base_diff} ]]
-then
+if [[ -f ${base_diff} ]]; then
  echo "${BASELINE_TEST}: Relevant differences (filtered diff) ..."
  cat ${base_diff}
  cp ${base_diff} ${artifacts_path}/${base_diff}
-  # We create a .err file, because that's how we signal that there was a diff.
-  cp ${base_diff} ${artifacts_path}/gitlab-${BASELINE_TEST}-${MACHINE_NAME}.err
+  status=1
 fi
-
-if [[ ! -s ${base_diff} ]]
-then
+if [[ $status -eq 0 ]]; then
  echo "${BASELINE_TEST}: PASSED"
-  true
 else
  echo "${BASELINE_TEST}: FAILED"
-  false
 fi
+exit $status
@@ -1,6 +1,6 @@
 #!/bin/bash

-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,6 +1,6 @@
 #!/bin/bash

-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,6 +1,6 @@
 #!/bin/bash

-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,6 +1,6 @@
 #!/bin/bash

-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,6 +1,6 @@
 #!/bin/bash

-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -8,61 +8,121 @@
                               https://mfem.org


-Version 4.5.1 (development)
+Version 4.5.3 (development)
 ===========================
- When using discontinuous (L2) spaces, use local (element-wise) L2 projection
-  as the coarsening operator for non-conforming AMR meshes.
+
+New and updated examples and miniapps
+-------------------------------------
+- Added a new miniapp, Mesh Quality, for evaluating mesh quality using size,
+  skewness, and aspect-ratio computed from the Jacobian of the transformation.
+
+- Added a new miniapp for interface and boundary fitting to implicit domains
+  defined using level-set functions. See miniapps/meshing/pmesh-fitting.cpp
+
+- Added new Discontinuous Petrov-Galerkin (DPG) miniapp which includes serial
+  and parallel examples for diffusion, convection-diffusion, acoustics and
+  Maxwell equations. The miniapp includes new classes such as (Par)DPGWeakForm,
+  (Par)ComplexDPGWeakForm and (Complex)BlockStaticCondensation. Three new
+  integrators are added in support of DPG systems: TraceIntegrator,
+  NormalTraceIntegrator and TangentTraceIntegrator.
+
+- Added a new H(div) solvers miniapp in miniapps/hdiv-linear-solver,
+  demonstrating the use of a matrix-free saddle-point solver methodology,
+  suitable for high-order discretizations and for GPU acceleration. Examples
+  illustrating the solution of Darcy and grad-div problems are included.
+
+- Added a random refinement option to the mesh-explorer miniapp to assist users
+  in experimenting with nonconforming meshes.
+
+- Moved the distance solver methods from miniapps/shifted to miniapps/common.

 Meshing improvements
 --------------------
+- Added new methods in the Mesh class to set and get attributes on NURBS patches
+  and patch boundaries.
+
+- TMOP improvement: added asymptotically-balanced compound metrics 90, 94, 328,
+  338. Added the tmop-metric-magnitude tool for tracking how metrics change
+  under geometric perturbations.
+
+Discretization improvements
+---------------------------
+- Face restriction operators for Nedelec and Raviart-Thomas finite element
+  spaces are now supported through the ConformingFaceRestriction class.
+
+- VectorFEBoundaryFluxLFIntegrator is now supported on device/GPU.
+
+- Added support for p-refined meshes in FindPointsGSLIB.
+
+Linear and nonlinear solvers
+----------------------------
+- Updated interface to MUMPS direct solver to support multiple right-hand
+  sides, block low-rank compression, builds using 64-bit integers, and other
+  improvements.
+
+- Added an interface to the MKL Pardiso sparse direct solver developed by Intel.
+  This interface provides a serial (OpenMP shared memory) version of Pardiso for
+  use with SparseMatrix. This complements the existing parallel (MPI distributed
+  memory) version already available through the CPardiso MFEM integration.
+
+Integrations, testing and documentation
+---------------------------------------
+- Added an address sanitizer GitHub action for a serial build/test on Ubuntu,
+  based on Clang/LLVM (https://clang.llvm.org/docs/AddressSanitizer.html).
+
+Miscellaneous
+-------------
+- Improved lambda body debugging with the addition of mfem::forall functions.
+  These functions can take the place of the MFEM_FORALL macros, which have been
+  preserved for backwards compatibility.
+
+- Reorganized files for bilinear form, linear form, and nonlinear form integrators
+  in the fem/integ/ subdirectory.
+
+
+Version 4.5.2, released on March 23, 2023
+=========================================
+
 - Added support for pyramids in non-conforming meshes. Currently only isotropic
  refinement is supported in this case.

+- Removed the support for the Mesquite toolkit. We recommend using MFEM's TMOP
+  functionality instead for mesh optimization. See the mesh-optimizer miniapp.
+
+- Added a fast normalization-based distance solver, see the Distance miniapp
+  in the miniapps/shifted/ directory.
+
+- Added a new meshing miniapp, Reflector, which reflects a high-order or NURBS
+  hexahedral mesh about a plane.
+
 - Updated logic in FindPointsGSLIB to ignore points found near (but outside) the
  domain boundary.

- Added support for pyramids in Gmsh meshes.
+- Added an option to auto-balance compound TMOP metrics.

 - Fixed a bug in TMOP metric 301.

- Added an option to auto-balance compound TMOP metrics.
+- When using discontinuous (L2) spaces, use local (element-wise) L2 projection
+  as the coarsening operator for non-conforming AMR meshes.

-Discretization improvements
---------------------------
- TBD
+- Added support for GridFunction::GetGradients() and GetVectorGradient() on
+  face-neighbor elements.

-Linear and nonlinear solvers
----------------------------
- Added a fast normalization-based distance solver, see the Distance miniapp
-  in the miniapps/shifted/ directory.
+- Added support for pyramids in Gmsh meshes.

-New and updated examples and miniapps
-------------------------------------
- Added a new meshing miniapp, reflector, which reflects a high-order or NURBS
-  hexahedral mesh about a plane.
+- The Mesh Explorer miniapp can now save mesh files in the VisIt or ParaView
+  formats using the corresponding DataCollection. See option 'D' in the menu.

- The mesh-explorer miniapp can now save mesh files in the VisIt or ParaView
-  formats using the corresponding DataCollection objects. See option 'D' in the
-  main menu.
+- VisItDataCollection now correctly handles data collection names containing
+  underscores.

-Integrations, testing and documentation
---------------------------------------
- Removed the support for the Mesquite toolkit. We recommend using MFEM's TMOP
-  functionality instead for mesh optimization. See the mesh-optimizer miniapp.
+- Added support for shared Windows builds with MSVC through CMake.
+  Developers note: this enhancement is facilitated by the addition of the macro
+  MFEM_EXPORT, see config.hpp for more details on its usage.

 - The following integrations have updated minimum version requirements:
  * RAJA >= 2022.10.3

-Miscellaneous
-------------
- VisItDataCollection now correctly handles collection names containing
-  underscores.
-
- VisItDataCollection::SetPadDigits() no longer alters the number of digits
-  used to represent the MPI rank because VisIt seems to require 6 digits.
-  This parameter can still be explicitly overridden with
-  VisItDataCollection::SetPadDigitsRank().
-
 API changes
 -----------
 - The implicit cast methods of class Vector to 'double *' and 'const double *'
@@ -73,6 +133,11 @@ API changes
  deprecated, and Mesh::GetFaceGeometry (which provides identical functionality)
  should be used instead.

+- VisItDataCollection::SetPadDigits() no longer alters the number of digits
+  used to represent the MPI rank because VisIt seems to require 6 digits.
+  This parameter can still be explicitly overridden with SetPadDigitsRank().
+
+
 Version 4.5, released on October 22, 2022
 =========================================

@@ -116,9 +181,6 @@ Discretization improvements
 - Added a class CoefficientVector for efficient access of variable coefficient
  values at quadrature points (in particular for GPU/device kernels).

- Added support for GridFunction::GetGradients() and
-  GriFunction::GetVectorGradient() on face-neighbor elements.
-
 - Added WhiteGaussianNoiseDomainLFIntegrator: a LinearFormIntegrator class for
  spatial Gaussian white noise.

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -13,6 +13,7 @@
 # Version 3.8 fixes the handling of CMAKE_CXX_STANDARD for try_compile.
 # Version 3.8 or newer is required for direct CUDA support.
 cmake_minimum_required(VERSION 3.8)
+message(STATUS "CMake version: ${CMAKE_VERSION}")
 set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH
  "Path to optional user configuration file.")

@@ -24,8 +25,11 @@ set(CMAKE_CXX_EXTENSIONS OFF)
 # Load user settings before the defaults - this way the defaults will not
 # overwrite the user set options. If the user has not set all options, we still
 # have the defaults.
-message(STATUS "(optional) USER_CONFIG = ${USER_CONFIG}")
-include("${USER_CONFIG}" OPTIONAL)
+include("${USER_CONFIG}" OPTIONAL RESULT_VARIABLE USER_CONFIG_LOADED)
+if (USER_CONFIG_LOADED)
+  set(USER_CONFIG_LOADED "LOADED")
+endif()
+message(STATUS "Loading USER_CONFIG = ${USER_CONFIG} (${USER_CONFIG_LOADED})")
 include("${CMAKE_CURRENT_SOURCE_DIR}/config/defaults.cmake")

 # Allow overwriting of the compiler by setting CXX/MPICXX on the command line or
@@ -53,7 +57,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.5.1)
+set(${PROJECT_NAME}_VERSION 4.5.3)

 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -78,7 +82,7 @@ if (MFEM_USE_CONDUIT OR
   #  * find_package(PETSc REQUIRED)
   set(XSDK_ENABLE_C ON)
 endif()
-if (MFEM_USE_STRUMPACK)
+if (MFEM_USE_STRUMPACK OR MFEM_USE_MUMPS)
  # Just needed to find the MPI_Fortran libraries to link with
  set(XSDK_ENABLE_Fortran ON)
 endif()
@@ -183,6 +187,23 @@ else()
  set(MFEM_DEBUG OFF)
 endif()

+# Shared build on Windows
+if (WIN32 AND BUILD_SHARED_LIBS)
+  # CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS works only with MSVC?
+  set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON CACHE BOOL
+      "Automatically export symbols for shared Windows DLL build")
+  # Place all .dll and .exe files in the same sub-directory, 'bin/<config>', of
+  # the build directory, so that the .exe files can find the .dll files at
+  # runtime:
+  set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin")
+  # Note: CMake tests are run from the build-tree sub-directory where the test
+  # is defined, e.g. <build-dir>/tests/unit, so hard coded paths to meshes still
+  # work fine.
+  # Not strictly necessary:
+  # set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
+  # set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
+endif()
+
 # AMD HIP
 if (MFEM_USE_HIP)
  if (HIP_ARCH)
@@ -312,6 +333,7 @@ endif()
 if (MFEM_USE_MUMPS)
  if (MFEM_USE_MPI)
    find_package(MUMPS REQUIRED mumps_common pord)
+    set(MFEM_MUMPS_VERSION ${MUMPS_VERSION})
  else()
    message(FATAL_ERROR " *** MUMPS requires that MPI be enabled.")
  endif()
@@ -445,12 +467,18 @@ if (MFEM_USE_ADIOS2)
   find_package(ADIOS2 REQUIRED)
 endif()

+# MKL CPardiso
 if (MFEM_USE_MKL_CPARDISO)
  if (MFEM_USE_MPI)
   find_package(MKL_CPARDISO REQUIRED MKL_SEQUENTIAL MKL_LP64 MKL_MPI_WRAPPER)
  endif()
 endif()

+# MKL Pardiso
+if (MFEM_USE_MKL_PARDISO)
+   find_package(MKL_PARDISO REQUIRED MKL_SEQUENTIAL MKL_LP64)
+endif()
+
 # PARELAG
 if (MFEM_USE_PARELAG)
   find_package(PARELAG REQUIRED)
@@ -500,8 +528,8 @@ find_package(Threads REQUIRED)
 set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
    SUNDIALS PETSC SLEPC MUMPS AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
    NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
-    ADIOS2 CUBLAS CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG
-    MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)
+    ADIOS2 CUBLAS CUSPARSE MKL_CPARDISO MKL_PARDISO AMGX CALIPER CODIPACK
+    BENCHMARK PARELAG MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)

 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
@@ -519,6 +547,7 @@ list(REVERSE TPL_LIBRARIES)
 list(REMOVE_DUPLICATES TPL_INCLUDE_DIRS)
 # message(STATUS "TPL_INCLUDE_DIRS = ${TPL_INCLUDE_DIRS}")

+message(STATUS "MFEM shared library: BUILD_SHARED_LIBS = ${BUILD_SHARED_LIBS}")
 message(STATUS "MFEM build type: CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}")
 message(STATUS "MFEM version: v${MFEM_VERSION_STRING}")
 message(STATUS "MFEM git string: ${MFEM_GIT_STRING}")
@@ -574,7 +603,19 @@ target_include_directories(mfem
  PUBLIC
    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+    # The INSTALL_INTERFACE is handled below
    ${TPL_INCLUDE_DIRS})
+if (MFEM_USE_MPI)
+  if (MPI_CXX_COMPILE_FLAGS)
+    separate_arguments(MPI_CXX_COMPILE_ARGS UNIX_COMMAND
+      "${MPI_CXX_COMPILE_FLAGS}")
+    target_compile_options(mfem PUBLIC ${MPI_CXX_COMPILE_ARGS})
+  endif()
+  if (MPI_CXX_LINK_FLAGS)
+    target_link_libraries(mfem PUBLIC ${MPI_CXX_LINK_FLAGS})
+  endif()
+endif()
+
 set_target_properties(mfem PROPERTIES VERSION "${mfem_VERSION}")
 set_target_properties(mfem PROPERTIES SOVERSION "${mfem_VERSION}")

@@ -586,6 +627,7 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
 endif()

 # Generate configuration file in the build directory: config/_config.hpp.
+set(MFEM_SHARED_BUILD ${BUILD_SHARED_LIBS})
 configure_file(
  "${PROJECT_SOURCE_DIR}/config/cmake/config.hpp.in"
  "${PROJECT_BINARY_DIR}/config/_config.hpp")
@@ -705,7 +747,7 @@ set(INSTALL_LIB_DIR lib
 set(INSTALL_CMAKE_DIR lib/cmake/mfem
  CACHE PATH "Relative path for installing cmake config files.")

-target_include_directories(mfem
+target_include_directories(mfem BEFORE
  PUBLIC
    $<INSTALL_INTERFACE:${INSTALL_INCLUDE_DIR}>)

@@ -757,10 +799,8 @@ endif()

 # Install the configuration header files
 install(FILES ${PROJECT_BINARY_DIR}/config/_config.hpp
-  DESTINATION ${INSTALL_INCLUDE_DIR}/mfem/config
-  RENAME config.hpp)
-
-install(FILES ${PROJECT_SOURCE_DIR}/config/tconfig.hpp
+  ${PROJECT_SOURCE_DIR}/config/config.hpp
+  ${PROJECT_SOURCE_DIR}/config/tconfig.hpp
  DESTINATION ${INSTALL_INCLUDE_DIR}/mfem/config)

 # Package the whole thing up nicely
@@ -121,6 +121,7 @@ The MFEM source code has the following structure:
  ├── fem
  │   ├── ceed
  │   ├── fe
+  │   ├── integ
  │   ├── lor
  │   ├── moonolith
  │   ├── qinterp
@@ -136,6 +137,7 @@ The MFEM source code has the following structure:
  │   ├── common
  │   ├── electromagnetics
  │   ├── gslib
+  │   ├── hdiv-linear-solver
  │   ├── hooke
  │   ├── meshing
  │   ├── mtop
@@ -209,7 +211,7 @@ device/host memory manager.
 - The main device-relevant classes and sources are:
  + [`Device`](https://docs.mfem.org/html/device_8hpp.html)
  + [`MemoryManager`](https://docs.mfem.org/html/mem_manager_8hpp.html)
-  + the [`MFEM_FORALL`](https://docs.mfem.org/html/forall_8hpp.html) macro
+  + the [`mfem::forall`](https://docs.mfem.org/html/forall_8hpp.html) function
  + the [`cuda.hpp`](https://docs.mfem.org/html/cuda_8hpp.html) and [`occa.hpp`](https://docs.mfem.org/html/occa_8hpp.html) files

 #### Utilities, building and documentation
@@ -1,6 +1,6 @@
 BSD 3-Clause License

-Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC
+Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC
 All rights reserved.

 Redistribution and use in source and binary forms, with or without
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -39,6 +39,9 @@
 // Description of the git commit used to build MFEM.
 #cmakedefine MFEM_GIT_STRING "@MFEM_GIT_STRING@"

+// Enable shared library build of MFEM.
+#cmakedefine MFEM_SHARED_BUILD
+
 // Build the parallel MFEM library.
 // Requires an MPI compiler, and the libraries HYPRE and METIS.
 #cmakedefine MFEM_USE_MPI
@@ -77,96 +80,101 @@
 // Internal MFEM option: enable group/batch allocation for some small objects.
 #cmakedefine MFEM_USE_MEMALLOC

+// Which library functions to use in class StopWatch for measuring time.
+// For a list of the available options, see INSTALL.
+// If not defined, an option is selected automatically.
+#cmakedefine MFEM_TIMER_TYPE @MFEM_TIMER_TYPE@
+
+// Enable MFEM functionality based on the SUNDIALS libraries.
+#cmakedefine MFEM_USE_SUNDIALS
+
 // Enable MFEM functionality based on the SuiteSparse library.
 #cmakedefine MFEM_USE_SUITESPARSE

 // Enable MFEM functionality based on the SuperLU_DIST library.
 #cmakedefine MFEM_USE_SUPERLU
+#cmakedefine MFEM_USE_SUPERLU5

 // Enable MFEM functionality based on the MUMPS library.
 #cmakedefine MFEM_USE_MUMPS
+#cmakedefine MFEM_MUMPS_VERSION @MFEM_MUMPS_VERSION@

 // Enable MFEM functionality based on the STRUMPACK library.
 #cmakedefine MFEM_USE_STRUMPACK

-// Enable functionality based on the Ginkgo library
+// Enable functionality based on the Ginkgo library.
 #cmakedefine MFEM_USE_GINKGO

-// Enable MFEM functionality based on the AmgX library
+// Enable MFEM functionality based on the AmgX library.
 #cmakedefine MFEM_USE_AMGX

-// Enable MFEM functionality based on the GnuTLS library
+// Enable secure socket streams based on the GNUTLS library.
 #cmakedefine MFEM_USE_GNUTLS

-// Enable MFEM functionality based on the GSLIB library
-#cmakedefine MFEM_USE_GSLIB
-
-// Enable MFEM functionality based on the NetCDF library
-#cmakedefine MFEM_USE_NETCDF
-
-// Enable MFEM functionality based on the PETSc library
-#cmakedefine MFEM_USE_PETSC
-
-// Enable MFEM functionality based on the SLEPc library
-#cmakedefine MFEM_USE_SLEPC
-
-// Enable MFEM functionality based on the Sidre library
+// Enable Sidre support.
 #cmakedefine MFEM_USE_SIDRE

-// Enable the use of SIMD in the high performance templated classes
+// Enable the use of SIMD in the high performance templated classes.
 #cmakedefine MFEM_USE_SIMD

-// Enable MFEM functionality based on the FMS library
+// Enable FMS support.
 #cmakedefine MFEM_USE_FMS

-// Enable MFEM functionality based on Conduit
+// Enable Conduit support.
 #cmakedefine MFEM_USE_CONDUIT

-// Enable MFEM functionality based on the PUMI library
+// Enable functionality based on the NetCDF library (reading CUBIT files).
+#cmakedefine MFEM_USE_NETCDF
+
+// Enable functionality based on the PETSc library.
+#cmakedefine MFEM_USE_PETSC
+
+// Enable functionality based on the SLEPc library.
+#cmakedefine MFEM_USE_SLEPC
+
+// Enable functionality based on the MPFR library.
+#cmakedefine MFEM_USE_MPFR
+
+// Enable MFEM functionality based on the PUMI library.
 #cmakedefine MFEM_USE_PUMI

-// Enable MFEM functionality based on the Moonolith library
+// Enable Moonolith-based general interpolation between finite element spaces.
 #cmakedefine MFEM_USE_MOONOLITH

-// Enable MFEM functionality based on the HiOp library
+// Enable MFEM functionality based on the HIOP library.
 #cmakedefine MFEM_USE_HIOP

-// Build the GPU/CUDA-enabled version of the MFEM library.
+// Enable MFEM functionality based on the GSLIB library.
+#cmakedefine MFEM_USE_GSLIB
+
+// Build the NVIDIA GPU/CUDA-enabled version of the MFEM library.
 // Requires a CUDA compiler (nvcc).
 #cmakedefine MFEM_USE_CUDA

-// Build the HIP-enabled version of the MFEM library.
+// Build the AMD GPU/HIP-enabled version of the MFEM library.
 // Requires a HIP compiler (hipcc).
 #cmakedefine MFEM_USE_HIP

-// Enable MFEM functionality based on the RAJA library
+// Enable functionality based on the RAJA library.
 #cmakedefine MFEM_USE_RAJA

-// Enable MFEM functionality based on the OCCA library
+// Enable functionality based on the OCCA library.
 #cmakedefine MFEM_USE_OCCA

-// Enable MFEM functionality based on the libCEED library
+// Enable functionality based on the libCEED library.
 #cmakedefine MFEM_USE_CEED

-// Enable MFEM functionality based on the Umpire library
-#cmakedefine MFEM_USE_UMPIRE
-
-// Enable MFEM functionality based on the ADIOS2 library
-#cmakedefine MFEM_USE_ADIOS2
-
-// Enable MFEM functionality based on the Caliper library
+// Enable functionality based on the Caliper library.
 #cmakedefine MFEM_USE_CALIPER

-// Enable MFEM functionality based on the Algoim library
+// Enable functionality based on the Algoim library.
 #cmakedefine MFEM_USE_ALGOIM

-// Which library functions to use in class StopWatch for measuring time.
-// For a list of the available options, see INSTALL.
-// If not defined, an option is selected automatically.
-#define MFEM_TIMER_TYPE @MFEM_TIMER_TYPE@
+// Enable functionality based on the Umpire library.
+#cmakedefine MFEM_USE_UMPIRE

-// Enable MFEM functionality based on the SUNDIALS libraries.
-#cmakedefine MFEM_USE_SUNDIALS
+// Enable IO functionality based on the ADIOS2 library.
+#cmakedefine MFEM_USE_ADIOS2

 // Version of HYPRE used for building MFEM.
 #cmakedefine MFEM_HYPRE_VERSION @MFEM_HYPRE_VERSION@
@@ -178,13 +186,13 @@
 // Enable interface to the MKL CPardiso library.
 #cmakedefine MFEM_USE_MKL_CPARDISO

-// Use forward mode for automatic differentiation
+// Use forward mode for automatic differentiation.
 #cmakedefine MFEM_USE_ADFORWARD

-// Enable the use of the CoDiPack library for AD
+// Enable the use of the CoDiPack library for AD.
 #cmakedefine MFEM_USE_CODIPACK

-// Enable MFEM functionality based on the Google Benchmark library.
+// Enable functionality based on the Google Benchmark library.
 #cmakedefine MFEM_USE_BENCHMARK

 // Enable Enzyme for AD
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -21,8 +21,15 @@ if(EXISTS ${CONDUIT_DIR}/include/conduit/conduit_relay_hdf5.hpp)
  # we only need  HDF5 if Conduit was built with HDF5 support
  set(Conduit_REQUIRED_PACKAGES "HDF5" CACHE STRING
      "Additional packages required by Conduit.")
-  else()
-    message(STATUS "Conduit Relay HDF5 Support is DISABLED")
+  # Suppress warning about HDF5_ROOT being set
+  if (POLICY CMP0074)
+    cmake_policy(SET CMP0074 NEW)
+  endif()
+  # HDF5_ROOT is needed in some cases, e.g. when HDF5_TARGET_NAMES is set and
+  # MFEM's FindHDF5.cmake is not used.
+  set(HDF5_ROOT ${HDF5_DIR} CACHE PATH "")
+else()
+  message(STATUS "Conduit Relay HDF5 Support is DISABLED")
 endif()

 include(MfemCmakeUtilities)
@@ -32,4 +39,4 @@ mfem_find_package(Conduit CONDUIT CONDUIT_DIR
  ADD_COMPONENT relay
    "include;include/conduit" conduit_relay.hpp "lib" conduit_relay
  ADD_COMPONENT blueprint
-      "include;include/conduit" conduit_blueprint.hpp "lib" conduit_blueprint)
+    "include;include/conduit" conduit_blueprint.hpp "lib" conduit_blueprint)
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -0,0 +1,27 @@
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - MKL_PARDISO_FOUND
+#   - MKL_PARDISO_LIBRARIES
+#   - MKL_PARDISO_INCLUDE_DIRS
+
+if(NOT MKL_LIBRARY_DIR)
+  message(WARNING "Using default MKL library path. Double check the variable MKL_LIBRARY_DIR")
+  set(MKL_LIBRARY_DIR "lib/intel64")
+endif()
+
+include(MfemCmakeUtilities)
+mfem_find_package(MKL_PARDISO MKL_PARDISO
+    MKL_PARDISO_DIR "include" mkl_pardiso.h ${MKL_LIBRARY_DIR} mkl_core
+  "Paths to headers required by MKL Pardiso." "Libraries required by MKL PARDISO."
+  ADD_COMPONENT MKL_LP64 "include" "" ${MKL_LIBRARY_DIR} mkl_intel_lp64
+  ADD_COMPONENT MKL_SEQUENTIAL "include" "" ${MKL_LIBRARY_DIR} mkl_sequential)
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -11,8 +11,9 @@

 # Sets the following variables:
 #   - MUMPS_FOUND
-#   - MUMPS_INCLUDE_DIRS
 #   - MUMPS_LIBRARIES
+#   - MUMPS_INCLUDE_DIRS
+#   - MUMPS_VERSION

 include(MfemCmakeUtilities)
 mfem_find_package(MUMPS MUMPS MUMPS_DIR
@@ -21,3 +22,18 @@ mfem_find_package(MUMPS MUMPS MUMPS_DIR
  "Libraries required by MUMPS."
  ADD_COMPONENT mumps_common "include" dmumps_c.h "lib" mumps_common
  ADD_COMPONENT pord "include" dmumps_c.h "lib" pord)
+
+if (MUMPS_FOUND AND (NOT MUMPS_VERSION))
+  try_run(MUMPS_VERSION_RUN_RESULT MUMPS_VERSION_COMPILE_RESULT
+          ${CMAKE_CURRENT_BINARY_DIR}/config
+          ${CMAKE_CURRENT_SOURCE_DIR}/config/get_mumps_version.cpp
+          CMAKE_FLAGS -DINCLUDE_DIRECTORIES:STRING=${MUMPS_INCLUDE_DIRS}
+          RUN_OUTPUT_VARIABLE MUMPS_VERSION_OUTPUT)
+  if ((MUMPS_VERSION_RUN_RESULT EQUAL 0) AND MUMPS_VERSION_OUTPUT)
+    string(STRIP "${MUMPS_VERSION_OUTPUT}" MUMPS_VERSION)
+    set(MUMPS_VERSION ${MUMPS_VERSION} CACHE STRING "MUMPS version." FORCE)
+    message(STATUS "Found MUMPS version ${MUMPS_VERSION}")
+  else()
+    message(FATAL_ERROR "Unable to determine MUMPS version.")
+  endif()
+endif()
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -14,11 +14,17 @@
 #   - NETCDF_LIBRARIES
 #   - NETCDF_INCLUDE_DIRS

+# Suppress warning about HDF5_ROOT being set
+if (POLICY CMP0074)
+  cmake_policy(SET CMP0074 NEW)
+endif()
+
 include(MfemCmakeUtilities)

 # FindHDF5.cmake uses HDF5_ROOT, so we "translate" from the MFEM convention
-# (MFEM's FindHDF5.cmake does not need HDF5_ROOT)
-# set(HDF5_ROOT ${HDF5_DIR} CACHE PATH "")
+# (Needed in some cases, e.g. when HDF5_TARGET_NAMES is set and MFEM's
+# FindHDF5.cmake is not used.)
+set(HDF5_ROOT ${HDF5_DIR} CACHE PATH "")

 # We need to guard against the case where HDF5 was already found but without
 # the HL extensions (in which case mfem_find_package will treat the package
@@ -35,6 +41,11 @@ mfem_find_package(NetCDF NETCDF NETCDF_DIR "include" netcdf.h "lib" netcdf
 # The netcdf library will always be the first element of NETCDF_LIBRARIES
 # and we need to insert after that library but before the hdf5 library, so
 # position 1 is used
-# (MFEM's FindHDF5.cmake does not set HDF5_C_LIBRARY_hdf5_hl and the HL library
-# is already added to NETCDF_LIBRARIES)
-# list(INSERT NETCDF_LIBRARIES 1 ${HDF5_C_LIBRARY_hdf5_hl})
+# (Needed in some cases, e.g. when HDF5_TARGET_NAMES is set and MFEM's
+# FindHDF5.cmake is not used.)
+if (HDF5_C_LIBRARY_hdf5_hl)
+  list(INSERT NETCDF_LIBRARIES 1 ${HDF5_C_LIBRARY_hdf5_hl})
+  if (NOT NetCDF_FIND_QUIETLY)
+    message(STATUS "Updated NetCDF libraries: ${NETCDF_LIBRARIES}")
+  endif()
+endif()
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -103,25 +103,6 @@ macro(add_mfem_examples EXE_SRCS)
      ${MFEM_EXEC_PREREQUISITES_TARGET_NAME} ${EXE_PREREQUISITE})

    target_link_libraries(${EXE_NAME} mfem)
-    if (MFEM_USE_MPI)
-      # Not needed: (mfem already links with MPI_CXX_LIBRARIES)
-      # target_link_libraries(${EXE_NAME} ${MPI_CXX_LIBRARIES})
-
-      # Language-specific include directories:
-      if (MPI_CXX_INCLUDE_PATH)
-        target_include_directories(${EXE_NAME} PRIVATE "${MPI_CXX_INCLUDE_PATH}")
-      endif()
-      if (MPI_CXX_COMPILE_FLAGS)
-        separate_arguments(MPI_CXX_COMPILE_ARGS UNIX_COMMAND
-          "${MPI_CXX_COMPILE_FLAGS}")
-        target_compile_options(${EXE_NAME} PRIVATE ${MPI_CXX_COMPILE_ARGS})
-      endif()
-
-      if (MPI_CXX_LINK_FLAGS)
-        set_target_properties(${EXE_NAME} PROPERTIES
-          LINK_FLAGS "${MPI_CXX_LINK_FLAGS}")
-      endif()
-    endif()
  endforeach(SRC_FILE)
 endmacro()

@@ -172,26 +153,6 @@ macro(add_mfem_miniapp MFEM_EXE_NAME)
  if (EXTRA_DEFINES_LIST)
    target_compile_definitions(${MFEM_EXE_NAME} PRIVATE ${EXTRA_DEFINES_LIST})
  endif()
-
-  # Handle the MPI separately
-  if (MFEM_USE_MPI)
-    # Add MPI_CXX_LIBRARIES, in case this target does not link with mfem.
-    target_link_libraries(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_LIBRARIES})
-
-    if (MPI_CXX_INCLUDE_PATH)
-      target_include_directories(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_INCLUDE_PATH})
-    endif()
-    if (MPI_CXX_COMPILE_FLAGS)
-      separate_arguments(MPI_CXX_COMPILE_ARGS UNIX_COMMAND
-        "${MPI_CXX_COMPILE_FLAGS}")
-      target_compile_options(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_COMPILE_ARGS})
-    endif()
-
-    if (MPI_CXX_LINK_FLAGS)
-      set_target_properties(${MFEM_EXE_NAME} PROPERTIES
-        LINK_FLAGS "${MPI_CXX_LINK_FLAGS}")
-    endif()
-  endif()
 endmacro()


@@ -436,9 +397,9 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
              break()
            endif()
          endforeach()
-          if (${Required} AND NOT ${ReqPack}_FOUND)
+          if (Required AND NOT ${ReqPack}_FOUND)
            message(FATAL_ERROR " *** Required package ${ReqPack} not found."
-              "Checked target names: ${ReqPack} ${${ReqPack}_TARGET_NAMES}")
+              " Checked target names: ${ReqPack} ${${ReqPack}_TARGET_NAMES}")
          endif()
        endif()
      endif()
@@ -504,14 +465,18 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
          endif()
          get_target_property(IsImported ${TargetName} IMPORTED)
          if (IsImported)
+            get_target_property(ImpConfigs ${TargetName} IMPORTED_CONFIGURATIONS)
+            # message(STATUS "${ReqPack} imported configs: ${ImpConfigs}")
            set(ImportConfig ${${ReqPack}_IMPORT_CONFIG})
            if (NOT ImportConfig)
              set(ImportConfig RELEASE)
+              list(FIND ImpConfigs ${ImportConfig} _Index)
+              if (_Index EQUAL -1)
+                list(GET ImpConfigs 0 ImportConfig)
+              endif()
            endif()
            set(ImportConfigSuffix "_${ImportConfig}")
-            get_target_property(ImpConfigs ${TargetName} IMPORTED_CONFIGURATIONS)
-            list(FIND ImpConfigs ${ImportConfig} _Index)
-            if ((_Index EQUAL -1) OR ("${ImportConfig}" STREQUAL "NO_CONFIG"))
+            if ((NOT ImportConfig) OR ("${ImportConfig}" STREQUAL "NO_CONFIG"))
              set(ImportConfig "NO_CONFIG")
              set(ImportConfigSuffix "")
              # message(FATAL_ERROR " *** ${ReqPack}: configuration "
@@ -529,7 +494,8 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
          else()
            # Set _Pack_LIBS from the target properties for ImportConfig
            foreach (_prop IMPORTED_LOCATION${ImportConfigSuffix}
-                IMPORTED_LINK_INTERFACE_LIBRARIES${ImportConfigSuffix})
+                IMPORTED_LINK_INTERFACE_LIBRARIES${ImportConfigSuffix}
+                INTERFACE_LINK_LIBRARIES)
              get_target_property(_value ${TargetName} ${_prop})
              if (_value)
                list(APPEND _Pack_LIBS ${_value})
@@ -780,6 +746,7 @@ function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
      list(APPEND CompileOpts "-isystem \"${SysDir}\"")
    endforeach()
  endif()
+  set(AddInterfaceLinkLibs TRUE)
  if ("${type}" STREQUAL "STATIC_LIBRARY")
    get_target_property(Location ${tgt} LOCATION)
    if (Location)
@@ -801,18 +768,7 @@ function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
      message(STATUS " *** Warning: [${tgt}] LOCATION not defined!")
    endif()
  elseif ("${type}" STREQUAL "INTERFACE_LIBRARY")
-    get_target_property(Libs ${tgt} INTERFACE_LINK_LIBRARIES)
-    if (Libs)
-      foreach(Lib ${Libs})
-        if (NOT (TARGET ${Lib}))
-          list(APPEND LinkOpts "${Lib}")
-        else()
-          mfem_get_target_options(${Lib} COpts LOpts)
-          list(APPEND CompileOpts ${COpts})
-          list(APPEND LinkOpts ${LOpts})
-        endif()
-      endforeach()
-    endif()
+    # The INTERFACE_LINK_LIBRARIES property is handled below.
    # Other properties we may need to handle:
    # INTERFACE_LINK_DEPENDS
    # INTERFACE_LINK_DIRECTORIES
@@ -820,6 +776,26 @@ function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
  else()
    message(STATUS " *** Warning: [${tgt}] uses target type '${type}'"
      " which is not supported!")
+    set(AddInterfaceLinkLibs FALSE)
+  endif()
+  if (AddInterfaceLinkLibs)
+    get_target_property(Libs ${tgt} INTERFACE_LINK_LIBRARIES)
+    # message(STATUS "${tgt}[INTERFACE_LINK_LIBRARIES]: ${Libs}")
+    if (Libs)
+      foreach(Lib ${Libs})
+        if (NOT (TARGET ${Lib}))
+          # message(STATUS "Lib = ${Lib}")
+          # Filter-out generator expressions
+          if (NOT ("${Lib}" MATCHES "^\\$"))
+            list(APPEND LinkOpts "${Lib}")
+          endif()
+        else()
+          mfem_get_target_options(${Lib} COpts LOpts)
+          list(APPEND CompileOpts ${COpts})
+          list(APPEND LinkOpts ${LOpts})
+        endif()
+      endforeach()
+    endif()
  endif()

  # Other potentially relevant properties:
@@ -843,6 +819,9 @@ function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
  set(${CompileOptsVar} "${CompileOpts}" PARENT_SCOPE)
  set(${LinkOptsVar} "${LinkOpts}" PARENT_SCOPE)

+  # message(STATUS "${tgt}[CompileOpts]: ${CompileOpts}")
+  # message(STATUS "${tgt}[LinkOpts]: ${LinkOpts}")
+
 endfunction(mfem_get_target_options)


@@ -894,7 +873,7 @@ function(mfem_export_mk_files)
         "${cxx_std_flag} ${CMAKE_CXX_FLAGS_${BUILD_TYPE}} ${CMAKE_CXX_FLAGS}"
         MFEM_CXXFLAGS)
  set(MFEM_TPLFLAGS "")
-  foreach(dir ${MFEM_TPL_INCLUDE_DIRS})
+  foreach(dir ${TPL_INCLUDE_DIRS})
    set(MFEM_TPLFLAGS "${MFEM_TPLFLAGS} -I${dir}")
  endforeach()
  # TODO: MFEM_TPLFLAGS: add other TPL flags, in addition to the -I flags.
@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -30,11 +30,30 @@
 #endif

 // Windows specific options
-#ifdef _WIN32
-#ifndef _USE_MATH_DEFINES
+#if defined(_WIN32) && !defined(_USE_MATH_DEFINES)
 // Macro needed to get defines like M_PI from <cmath>. (Visual Studio C++ only?)
 #define _USE_MATH_DEFINES
 #endif
+// Macro MFEM_EXPORT: this macro is used when declaring exported global
+// variables and static class variables in public header files, e.g.:
+//    extern MFEM_EXPORT Geometry Geometries;
+//    static MFEM_EXPORT Device device_singleton;
+// In cases where a class contains multiple static variables, instead of marking
+// all such variables with MFEM_EXPORT, one can mark the class with MFEM_EXPORT,
+// e.g.:
+//    class MFEM_EXPORT MemoryManager ...
+// Note: MFEM's GitHub CI includes a shared MSVC build that will fail if a
+// variable that needs MFEM_EXPORT does not have it. However, builds with
+// optional external libraries are not tested and may require separate checks to
+// determine the necessity of MFEM_EXPORT.
+#if defined(_MSC_VER) && defined(MFEM_SHARED_BUILD)
+#ifdef mfem_EXPORTS
+#define MFEM_EXPORT __declspec(dllexport)
+#else
+#define MFEM_EXPORT __declspec(dllimport)
+#endif
+#else
+#define MFEM_EXPORT
 #endif
 // On Cygwin the option -std=c++11 prevents the definition of M_PI. Defining
 // the following macro allows us to get M_PI and some needed functions, e.g.
@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -30,15 +30,18 @@
 #define MFEM_VERSION_MINOR (((MFEM_VERSION)/100)%100)
 #define MFEM_VERSION_PATCH ((MFEM_VERSION)%100)

-// The absolute path of the MFEM source prefix
+// The absolute path of the MFEM source prefix.
 // #define MFEM_SOURCE_DIR "@MFEM_SOURCE_DIR@"

-// The absolute path of the MFEM installation prefix
+// The absolute path of the MFEM installation prefix.
 // #define MFEM_INSTALL_DIR "@MFEM_INSTALL_DIR@"

 // Description of the git commit used to build MFEM.
 // #define MFEM_GIT_STRING "@MFEM_GIT_STRING@"

+// Enable shared library build of MFEM.
+// #define MFEM_SHARED_BUILD
+
 // Build the parallel MFEM library.
 // Requires an MPI compiler, and the libraries HYPRE and METIS.
 // #define MFEM_USE_MPI
@@ -88,7 +91,7 @@
 // Enable MFEM functionality based on the SuiteSparse library.
 // #define MFEM_USE_SUITESPARSE

-// Enable MFEM functionality based on the SuperLU library.
+// Enable MFEM functionality based on the SuperLU_DIST library.
 // #define MFEM_USE_SUPERLU
 // #define MFEM_USE_SUPERLU5

@@ -99,40 +102,40 @@
 // Enable MFEM functionality based on the STRUMPACK library.
 // #define MFEM_USE_STRUMPACK

-// Enable MFEM features based on the Ginkgo library
+// Enable MFEM features based on the Ginkgo library.
 // #define MFEM_USE_GINKGO

 // Enable MFEM functionality based on the AmgX library.
 // #define MFEM_USE_AMGX

-// Enable secure socket streams based on the GNUTLS library
+// Enable secure socket streams based on the GNUTLS library.
 // #define MFEM_USE_GNUTLS

-// Enable Sidre support
+// Enable Sidre support.
 // #define MFEM_USE_SIDRE

-// Enable the use of SIMD in the high performance templated classes
+// Enable the use of SIMD in the high performance templated classes.
 // #define MFEM_USE_SIMD

-// Enable FMS support
+// Enable FMS support.
 // #define MFEM_USE_FMS

-// Enable Conduit support
+// Enable Conduit support.
 // #define MFEM_USE_CONDUIT

-// Enable functionality based on the NetCDF library (reading CUBIT files)
+// Enable functionality based on the NetCDF library (reading CUBIT files).
 // #define MFEM_USE_NETCDF

-// Enable functionality based on the PETSc library
+// Enable functionality based on the PETSc library.
 // #define MFEM_USE_PETSC

-// Enable functionality based on the SLEPc library
+// Enable functionality based on the SLEPc library.
 // #define MFEM_USE_SLEPC

 // Enable functionality based on the MPFR library.
 // #define MFEM_USE_MPFR

-// Enable MFEM functionality based on the PUMI library
+// Enable MFEM functionality based on the PUMI library.
 // #define MFEM_USE_PUMI

 // Enable Moonolith-based general interpolation between finite element spaces.
@@ -141,7 +144,7 @@
 // Enable MFEM functionality based on the HIOP library.
 // #define MFEM_USE_HIOP

-// Enable MFEM functionality based on the GSLIB library
+// Enable MFEM functionality based on the GSLIB library.
 // #define MFEM_USE_GSLIB

 // Build the NVIDIA GPU/CUDA-enabled version of the MFEM library.
@@ -183,10 +186,13 @@
 // Enable interface to the MKL CPardiso library.
 // #define MFEM_USE_MKL_CPARDISO

-// Use forward mode for automatic differentiation
+// Enable interface to the MKL Pardiso library.
+// #define MFEM_USE_MKL_PARDISO
+
+// Use forward mode for automatic differentiation.
 // #define MFEM_USE_ADFORWARD

-// Enable the use of the CoDiPack library for AD
+// Enable the use of the CoDiPack library for AD.
 // #define MFEM_USE_CODIPACK

 // Enable functionality based on the Google Benchmark library.
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -57,6 +57,7 @@ MFEM_USE_UMPIRE        = @MFEM_USE_UMPIRE@
 MFEM_USE_SIMD          = @MFEM_USE_SIMD@
 MFEM_USE_ADIOS2        = @MFEM_USE_ADIOS2@
 MFEM_USE_MKL_CPARDISO  = @MFEM_USE_MKL_CPARDISO@
+MFEM_USE_MKL_PARDISO   = @MFEM_USE_MKL_PARDISO@
 MFEM_USE_MOONOLITH     = @MFEM_USE_MOONOLITH@
 MFEM_USE_ADFORWARD     = @MFEM_USE_ADFORWARD@
 MFEM_USE_CODIPACK      = @MFEM_USE_CODIPACK@
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -19,6 +19,7 @@ if (NOT CMAKE_BUILD_TYPE)
 endif()

 # MFEM options. Set to mimic the default "defaults.mk" file.
+option(BUILD_SHARED_LIBS "Enable shared library build of MFEM" OFF)
 option(MFEM_USE_MPI "Enable MPI parallel build" OFF)
 option(MFEM_USE_METIS "Enable METIS usage" ${MFEM_USE_MPI})
 option(MFEM_USE_EXCEPTIONS "Enable the use of exceptions" OFF)
@@ -59,6 +60,7 @@ option(MFEM_USE_ADIOS2 "Enable ADIOS2" OFF)
 option(MFEM_USE_CALIPER "Enable Caliper support" OFF)
 option(MFEM_USE_ALGOIM "Enable Algoim support" OFF)
 option(MFEM_USE_MKL_CPARDISO "Enable MKL CPardiso" OFF)
+option(MFEM_USE_MKL_PARDISO "Enable MKL Pardiso" OFF)
 option(MFEM_USE_ADFORWARD "Enable forward mode for AD" OFF)
 option(MFEM_USE_CODIPACK "Enable automatic differentiation (AD) using CoDiPack" OFF)
 option(MFEM_USE_BENCHMARK "Enable Google Benchmark" OFF)
@@ -133,16 +135,18 @@ set(ParMETIS_DIR "${MFEM_DIR}/../parmetis-4.0.3" CACHE PATH
 set(ParMETIS_REQUIRED_PACKAGES "METIS" CACHE STRING
    "Additional packages required by ParMETIS.")

-set(SuperLUDist_DIR "${MFEM_DIR}/../SuperLU_DIST_6.3.1" CACHE PATH
+set(SuperLUDist_DIR "${MFEM_DIR}/../SuperLU_DIST_8.1.2" CACHE PATH
    "Path to the SuperLU_DIST library.")
 # SuperLU_DIST may also depend on "OpenMP", depending on how it was compiled.
-set(SuperLUDist_REQUIRED_PACKAGES "MPI" "BLAS" "ParMETIS" CACHE STRING
+set(SuperLUDist_REQUIRED_PACKAGES "MPI" "ParMETIS" "METIS"
+    "LAPACK" "BLAS" CACHE STRING
    "Additional packages required by SuperLU_DIST.")

-set(MUMPS_DIR "${MFEM_DIR}/../MUMPS_5.2.0" CACHE PATH
+set(MUMPS_DIR "${MFEM_DIR}/../MUMPS_5.5.0" CACHE PATH
    "Path to the MUMPS library.")
-# Packages required by MUMPS, depending on how it was compiled.
-set(MUMPS_REQUIRED_PACKAGES "MPI" "BLAS" "METIS" "ScaLAPACK" CACHE STRING
+# MUMPS may also depend on "OpenMP", depending on how it was compiled.
+set(MUMPS_REQUIRED_PACKAGES "MPI" "MPI_Fortran" "ParMETIS" "METIS"
+    "ScaLAPACK" "LAPACK" "BLAS" CACHE STRING
    "Additional packages required by MUMPS.")
 # If the MPI package does not find all required Fortran libraries:
 # set(MUMPS_REQUIRED_LIBRARIES "gfortran" "mpi_mpifh" CACHE STRING
@@ -225,6 +229,8 @@ set(MKL_CPARDISO_DIR "" CACHE STRING "MKL installation path.")
 set(MKL_MPI_WRAPPER_LIB "mkl_blacs_mpich_lp64" CACHE STRING "MKL MPI wrapper library")
 set(MKL_LIBRARY_DIR "" CACHE STRING "Custom library subdirectory")

+set(MKL_PARDISO_DIR "" CACHE STRING "MKL installation path.")
+
 set(OCCA_DIR "${MFEM_DIR}/../occa" CACHE PATH "Path to OCCA")
 set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -58,7 +58,7 @@ HIP_CXX = hipcc
 # example: gfx600 (tahiti), gfx700 (kaveri), gfx701 (hawaii), gfx801 (carrizo),
 # gfx900, gfx1010, etc.
 HIP_ARCH = gfx900
-HIP_FLAGS = --amdgpu-target=$(HIP_ARCH)
+HIP_FLAGS = --offload-arch=$(HIP_ARCH)
 HIP_XCOMPILER =
 HIP_XLINKER   = -Wl,

@@ -160,6 +160,7 @@ MFEM_USE_UMPIRE        = NO
 MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
 MFEM_USE_MKL_CPARDISO  = NO
+MFEM_USE_MKL_PARDISO   = NO
 MFEM_USE_MOONOLITH     = NO
 MFEM_USE_ADFORWARD     = NO
 MFEM_USE_CODIPACK      = NO
@@ -284,10 +285,10 @@ ifeq ($(MFEM_USE_SUPERLU5),YES)
   SUPERLU_LIB = $(XLINKER)-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib\
      -lsuperlu_dist_5.1.0
 else
-   SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_6.3.1
+   SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_8.1.2
   SUPERLU_OPT = -I$(SUPERLU_DIR)/include
   SUPERLU_LIB = $(XLINKER)-rpath,$(SUPERLU_DIR)/lib64 -L$(SUPERLU_DIR)/lib64\
-      -lsuperlu_dist -lblas
+      -lsuperlu_dist $(LAPACK_LIB)
 endif

 # SCOTCH library configuration (required by STRUMPACK <= v2.1.0, optional in
@@ -311,7 +312,7 @@ MPI_FORTRAN_LIB = -lmpifort
 # MPI_FORTRAN_LIB += -lgfortran

 # MUMPS library configuration
-MUMPS_DIR = @MFEM_DIR@/../MUMPS_5.2.0
+MUMPS_DIR = @MFEM_DIR@/../MUMPS_5.5.0
 MUMPS_OPT = -I$(MUMPS_DIR)/include
 MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib -ldmumps\
 -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB)
@@ -484,7 +485,6 @@ ifdef GOTCHA_DIR
   CALIPER_LIB += $(XLINKER)-rpath,$(GOTCHA_DIR)/lib64 $(XLINKER)-rpath,$(GOTCHA_DIR)/lib -L$(GOTCHA_DIR)/lib64 -L$(GOTCHA_DIR)/lib -lgotcha
 endif

-
 # BLITZ library configuration
 BLITZ_DIR = @MFEM_DIR@/../blitz
 BLITZ_OPT = -I$(BLITZ_DIR)/include
@@ -539,6 +539,14 @@ MKL_CPARDISO_LIB = $(XLINKER)-rpath,$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
   -L$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR) -l$(MKL_MPI_WRAPPER)\
   -lmkl_intel_lp64 -lmkl_sequential -lmkl_core

+# MKL Pardiso library configuration
+MKL_PARDISO_DIR ?=
+MKL_LIBRARY_SUBDIR ?= lib
+MKL_PARDISO_OPT = -I$(MKL_PARDISO_DIR)/include
+MKL_PARDISO_LIB = $(XLINKER)-rpath,$(MKL_PARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
+   -L$(MKL_PARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
+   -lmkl_intel_lp64 -lmkl_sequential -lmkl_core
+
 # PARELAG library configuration
 PARELAG_DIR = @MFEM_DIR@/../parelag
 PARELAG_OPT = -I$(PARELAG_DIR)/src -I$(PARELAG_DIR)/build/src
@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -1,6 +1,6 @@
 #!/bin/bash

-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -35,8 +35,8 @@ cd $(git rev-parse --show-toplevel)
 # copyright check
 copyright=true
 if [[ "${option}" == "--copyright" || "${option}" == "" ]]; then
-  if git grep -l "^\(#\|//\).*\(\-2021\|\ 2010,\)" > matches.txt; then
-    echo "Please update the following files to Copyright (c) 2010-2022:"
+  if git grep -n "^\(#\|//\).*Copyright.*2010-20\(2[^3]\|[^2].\)" > matches.txt; then
+    echo "Please update the following files to Copyright (c) 2010-2023:"
    cat matches.txt
    copyright=false
  fi
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,6 +1,6 @@
 #!/bin/bash

-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -138,7 +138,7 @@ groups_parallel=(
 '"meshing"
   "Meshing miniapps:"
   "miniapps/meshing"
-   "pmesh-optimizer.cpp pminimal-surface.cpp"'
+   "pmesh-optimizer.cpp pmesh-fitting.cpp pminimal-surface.cpp"'
 '"electromagnetics"
   "Electromagnetics miniapps:"
   "miniapps/electromagnetics"
@@ -227,7 +227,7 @@ groups_all=(
   "Meshing miniapps:"
   "miniapps/meshing"
   "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp
-    {,p}mesh-optimizer.cpp {,p}minimal-surface.cpp"'
+    {,p}mesh-optimizer.cpp pmesh-fitting.cpp {,p}minimal-surface.cpp"'
 '"electromagnetics"
   "Electromagnetics miniapps:"
   "miniapps/electromagnetics"
@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -0,0 +1,46 @@
+MFEM NURBS mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# SEGMENT     = 1
+# SQUARE      = 3
+# CUBE        = 5
+#
+
+dimension
+1
+
+elements
+1
+1 1 0 1
+
+boundary
+2
+1 0 0
+2 0 1
+
+edges
+1
+0 0 1
+
+vertices
+2
+
+knotvectors
+1
+1 2 0 0 1 1
+
+weights
+1
+1
+
+FiniteElementSpace
+FiniteElementCollection: NURBS1
+VDim: 1
+Ordering: 1
+
+0
+1
+
+
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.

-PROJECT_NUMBER         = v4.5.1
+PROJECT_NUMBER         = v4.5.3

 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -795,6 +795,7 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/miniapps/common \
                         @MFEM_SOURCE_DIR@/miniapps/electromagnetics \
                         @MFEM_SOURCE_DIR@/miniapps/gslib \
+                         @MFEM_SOURCE_DIR@/miniapps/hdiv-linear-solver \
                         @MFEM_SOURCE_DIR@/miniapps/hooke \
                         @MFEM_SOURCE_DIR@/miniapps/hooke/kernels \
                         @MFEM_SOURCE_DIR@/miniapps/hooke/materials \
@@ -810,7 +811,10 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/miniapps/shifted \
                         @MFEM_SOURCE_DIR@/miniapps/solvers \
                         @MFEM_SOURCE_DIR@/miniapps/tools \
-                         @MFEM_SOURCE_DIR@/miniapps/toys
+                         @MFEM_SOURCE_DIR@/miniapps/toys \
+                         @MFEM_SOURCE_DIR@/miniapps/spde \
+                         @MFEM_SOURCE_DIR@/miniapps/dpg \
+                         @MFEM_SOURCE_DIR@/miniapps/dpg/util

 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
@@ -39,7 +39,7 @@ namespace mfem {
 * - Device
 * - Memory
 * - MemoryManager
- * - MFEM_FORALL macro in forall.hpp
+ * - mfem::forall functions in forall.hpp
 *
 * <H3>Example codes</H3>
 * - <a class="el" href="ex0_8cpp_source.html">Example 0</a>: simplest example, nodal H1 FEM for the Laplace problem
@@ -186,6 +186,7 @@ namespace mfem {
 * - <a class="el" href="extruder_8cpp_source.html">Extruder</a>: extrude a low-dimensional mesh into a higher dimension
 * - <a class="el" href="mesh-explorer_8cpp_source.html">Mesh Explorer</a>: visualize and manipulate meshes
 * - <a class="el" href="mesh-optimizer_8cpp_source.html">Mesh Optimizer</a>: optimize high-order meshes, <a class="el" href="mesh-optimizer_8cpp_source.html">serial</a> and <a class="el" href="pmesh-optimizer_8cpp_source.html">parallel</a> versions
+ * - <a class="el" href="mesh-quality_8cpp_source.html">Mesh Quality</a>: visualize and check mesh quality
 * - <a class="el" href="trimmer_8cpp_source.html">Trimmer</a>: trim elements from existing meshes
 * - <a class="el" href="display-basis_8cpp_source.html">Display Basis</a>: visualize finite element basis functions
 * - <a class="el" href="get-values_8cpp_source.html">Get Values</a>: extract field values via DataCollection classes
@@ -198,12 +199,15 @@ namespace mfem {
 * - <a class="el" href="distance_8cpp_source.html">Distance</a>: finite element distance function solver
 * - <a class="el" href="diffusion_8cpp_source.html">Shifted Diffusion</a>: shifted boundary diffusion solver
 * - <a class="el" href="extrapolate_8cpp_source.html">Extrapolation</a>: PDE-based extrapolation of finite element functions
- * - <a class="el" href="distance_8cpp_source.html">Block Solvers</a>: comparison of saddle point system solvers
+ * - <a class="el" href="block-solvers_8cpp_source.html">Block Solvers</a>: comparison of saddle point system solvers
 * - <a class="el" href="parheat_8cpp_source.html">Optimization gradients</a>: Gradients of PDE-constrained function
 * - <a class="el" href="par__example_8cpp_source.html">Parallel AD</a>: Parallel p-Laplacian example
 * - <a class="el" href="seq__example_8cpp_source.html">Serial AD</a>: Serial p-Laplacian example
 * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
 * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
+ * - <a class="el" href="generate__random__field_8cpp_source.html">SPDE Solvers</a>: SPDE solver random field generation
+ * - <a class="el" href="pdiffusion_8cpp_source.html">DPG Diffusion example</a>: DPG formulation for the diffusion problem
+ * - <a class="el" href="pmaxwell_8cpp_source.html">DPG Maxwell example</a>: DPG formulation for the indefinite Maxwell problem
 *
 * See also the <a class="el" href="https://mfem.org/examples/">examples documentation</a> online.
 */
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -40,6 +40,8 @@ list(APPEND ALL_EXE_SRCS
  ex30.cpp
  ex31.cpp
  ex33.cpp
+  ex34.cpp
+  ex35.cpp
  )

 if (MFEM_USE_MPI)
@@ -161,6 +163,15 @@ if (MFEM_ENABLE_TESTING)
      $<TARGET_FILE:ex11p> "-no-vis" "--superlu"
      ${MPIEXEC_POSTFLAGS})
  endif()
+
+  # If MUMPS is enabled, add a test run that uses it.
+  if (MFEM_USE_MUMPS)
+    add_test(NAME ex25p_mumps_np=${MFEM_MPI_NP}
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:ex25p> "-no-vis" "--mumps-solver"
+      ${MPIEXEC_POSTFLAGS})
+  endif()
 endif()

 # Include the examples/amgx directory if AmgX is enabled
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -246,17 +246,23 @@ int main(int argc, char *argv[])
   e_var /= (nsteps + 1);
   double e_sd = sqrt(e_var);

+   double e_loc_stats[2];
+   double *e_stats = (myid == 0) ? new double[2 * num_procs] : (double*)NULL;
+
+   e_loc_stats[0] = e_mean;
+   e_loc_stats[1] = e_sd;
+   MPI_Gather(e_loc_stats, 2, MPI_DOUBLE, e_stats, 2, MPI_DOUBLE, 0, comm);
+
   if (myid == 0)
   {
-      cout << endl << "Mean and standard deviation of the energy" << endl;
-   }
-   for (int i = 0; i < num_procs; i++)
-   {
-      if (myid == i)
+      cout << endl << "Mean and standard deviation of the energy "
+           << "for different initial conditions" << endl;
+      for (int i = 0; i < num_procs; i++)
      {
-         cout << myid << ": " << e_mean << "\t" << e_sd << endl;
+         cout << i << ": " << e_stats[2 * i + 0]
+              << "\t" << e_stats[2 * i + 1] << endl;
      }
-      MPI_Barrier(comm);
+      delete [] e_stats;
   }

   // 9. Finalize the GnuPlot output
@@ -32,6 +32,7 @@
 //               We recommend viewing Example 22 before viewing this example.

 #include "mfem.hpp"
+#include <memory>
 #include <fstream>
 #include <iostream>

@@ -44,7 +45,7 @@ using namespace std;
 using namespace mfem;

 // Class for setting up a simple Cartesian PML region
-class CartesianPML
+class PML
 {
 private:
   Mesh *mesh;
@@ -69,7 +70,7 @@ private:

 public:
   // Constructor
-   CartesianPML(Mesh *mesh_,Array2D<double> length_);
+   PML(Mesh *mesh_,Array2D<double> length_);

   // Return Computational Domain Boundary
   Array2D<double> GetCompDomainBdr() {return comp_dom_bdr;}
@@ -91,12 +92,12 @@ public:
 class PMLDiagMatrixCoefficient : public VectorCoefficient
 {
 private:
-   CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML *, Vector &);
+   PML * pml = nullptr;
+   void (*Function)(const Vector &, PML *, Vector &);
 public:
-   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, PML *,
                                              Vector &),
-                            CartesianPML * pml_)
+                            PML * pml_)
      : VectorCoefficient(dim), pml(pml_), Function(F)
   {}

@@ -125,13 +126,13 @@ void source(const Vector &x, Vector & f);

 // Functions for computing the necessary coefficients after PML stretching.
 // J is the Jacobian matrix of the stretching function
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_JT_J_inv_Re(const Vector &x, PML * pml, Vector &D);
+void detJ_JT_J_inv_Im(const Vector &x, PML * pml, Vector &D);
+void detJ_JT_J_inv_abs(const Vector &x, PML * pml, Vector &D);

-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_inv_JT_J_Re(const Vector &x, PML * pml, Vector &D);
+void detJ_inv_JT_J_Im(const Vector &x, PML * pml, Vector &D);
+void detJ_inv_JT_J_abs(const Vector &x, PML * pml, Vector &D);

 Array2D<double> comp_domain_bdr;
 Array2D<double> domain_bdr;
@@ -267,7 +268,7 @@ int main(int argc, char *argv[])
         length = 0.25;
         break;
   }
-   CartesianPML * pml = new CartesianPML(mesh,length);
+   PML * pml = new PML(mesh,length);
   comp_domain_bdr = pml->GetCompDomainBdr();
   domain_bdr = pml->GetDomainBdr();

@@ -467,16 +468,14 @@ int main(int argc, char *argv[])
      offsets[2] = fespace->GetTrueVSize();
      offsets.PartialSum();

-      Operator *pc_r = nullptr;
-      Operator *pc_i = nullptr;
-      int s = (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0;
+      std::unique_ptr<Operator> pc_r;
+      std::unique_ptr<Operator> pc_i;
+      double s = (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0;
      if (pa)
      {
         // Jacobi Smoother
-         OperatorJacobiSmoother *d00 = new OperatorJacobiSmoother(prec, ess_tdof_list);
-         ScaledOperator *d11 = new ScaledOperator(d00, s);
-         pc_r = d00;
-         pc_i = d11;
+         pc_r.reset(new OperatorJacobiSmoother(prec, ess_tdof_list));
+         pc_i.reset(new ScaledOperator(pc_r.get(), s));
      }
      else
      {
@@ -485,15 +484,13 @@ int main(int argc, char *argv[])
         prec.FormSystemMatrix(ess_tdof_list, PCOpAh);

         // Gauss-Seidel Smoother
-         GSSmoother *gs00 = new GSSmoother(*PCOpAh.As<SparseMatrix>());
-         ScaledOperator *gs11 = new ScaledOperator(gs00, s);
-         pc_r = gs00;
-         pc_i = gs11;
+         pc_r.reset(new GSSmoother(*PCOpAh.As<SparseMatrix>()));
+         pc_i.reset(new ScaledOperator(pc_r.get(), s));
      }

      BlockDiagonalPreconditioner BlockDP(offsets);
-      BlockDP.SetDiagonalBlock(0, pc_r);
-      BlockDP.SetDiagonalBlock(1, pc_i);
+      BlockDP.SetDiagonalBlock(0, pc_r.get());
+      BlockDP.SetDiagonalBlock(1, pc_i.get());

      GMRESSolver gmres;
      gmres.SetPrintLevel(1);
@@ -807,7 +804,7 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
   }
 }

-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_JT_J_inv_Re(const Vector &x, PML * pml, Vector &D)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det(1.0, 0.0);
@@ -824,7 +821,7 @@ void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D)
   }
 }

-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_JT_J_inv_Im(const Vector &x, PML * pml, Vector &D)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -841,7 +838,7 @@ void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D)
   }
 }

-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_JT_J_inv_abs(const Vector &x, PML * pml, Vector &D)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -858,7 +855,7 @@ void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D)
   }
 }

-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_inv_JT_J_Re(const Vector &x, PML * pml, Vector &D)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det(1.0, 0.0);
@@ -883,7 +880,7 @@ void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D)
   }
 }

-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_inv_JT_J_Im(const Vector &x, PML * pml, Vector &D)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -907,7 +904,7 @@ void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D)
   }
 }

-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_inv_JT_J_abs(const Vector &x, PML * pml, Vector &D)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -931,14 +928,14 @@ void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D)
   }
 }

-CartesianPML::CartesianPML(Mesh *mesh_, Array2D<double> length_)
+PML::PML(Mesh *mesh_, Array2D<double> length_)
   : mesh(mesh_), length(length_)
 {
   dim = mesh->Dimension();
   SetBoundaries();
 }

-void CartesianPML::SetBoundaries()
+void PML::SetBoundaries()
 {
   comp_dom_bdr.SetSize(dim, 2);
   dom_bdr.SetSize(dim, 2);
@@ -953,7 +950,7 @@ void CartesianPML::SetBoundaries()
   }
 }

-void CartesianPML::SetAttributes(Mesh *mesh_)
+void PML::SetAttributes(Mesh *mesh_)
 {
   // Initialize bdr attributes
   for (int i = 0; i < mesh_->GetNBE(); ++i)
@@ -1002,8 +999,8 @@ void CartesianPML::SetAttributes(Mesh *mesh_)
   mesh_->SetAttributes();
 }

-void CartesianPML::StretchFunction(const Vector &x,
-                                   vector<complex<double>> &dxs)
+void PML::StretchFunction(const Vector &x,
+                          vector<complex<double>> &dxs)
 {
   complex<double> zi = complex<double>(0., 1.);

@@ -44,7 +44,7 @@ using namespace std;
 using namespace mfem;

 // Class for setting up a simple Cartesian PML region
-class CartesianPML
+class PML
 {
 private:
   Mesh *mesh;
@@ -69,7 +69,7 @@ private:

 public:
   // Constructor
-   CartesianPML(Mesh *mesh_,Array2D<double> length_);
+   PML(Mesh *mesh_,Array2D<double> length_);

   // Return Computational Domain Boundary
   Array2D<double> GetCompDomainBdr() {return comp_dom_bdr;}
@@ -91,12 +91,12 @@ public:
 class PMLDiagMatrixCoefficient : public VectorCoefficient
 {
 private:
-   CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML *, Vector &);
+   PML * pml = nullptr;
+   void (*Function)(const Vector &, PML *, Vector &);
 public:
-   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, PML *,
                                              Vector &),
-                            CartesianPML * pml_)
+                            PML * pml_)
      : VectorCoefficient(dim), pml(pml_), Function(F)
   {}

@@ -125,13 +125,13 @@ void source(const Vector &x, Vector & f);

 // Functions for computing the necessary coefficients after PML stretching.
 // J is the Jacobian matrix of the stretching function
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_JT_J_inv_Re(const Vector &x, PML * pml, Vector & D);
+void detJ_JT_J_inv_Im(const Vector &x, PML * pml, Vector & D);
+void detJ_JT_J_inv_abs(const Vector &x, PML * pml, Vector & D);

-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_inv_JT_J_Re(const Vector &x, PML * pml, Vector & D);
+void detJ_inv_JT_J_Im(const Vector &x, PML * pml, Vector & D);
+void detJ_inv_JT_J_abs(const Vector &x, PML * pml, Vector & D);

 Array2D<double> comp_domain_bdr;
 Array2D<double> domain_bdr;
@@ -295,7 +295,7 @@ int main(int argc, char *argv[])
         length = 0.25;
         break;
   }
-   CartesianPML * pml = new CartesianPML(mesh,length);
+   PML * pml = new PML(mesh,length);
   comp_domain_bdr = pml->GetCompDomainBdr();
   domain_bdr = pml->GetDomainBdr();

@@ -478,11 +478,11 @@ int main(int argc, char *argv[])
   if (!pa && mumps_solver)
   {
      HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
-      MUMPSSolver mumps;
+      MUMPSSolver mumps(A->GetComm());
      mumps.SetPrintLevel(0);
      mumps.SetMatrixSymType(MUMPSSolver::MatType::UNSYMMETRIC);
      mumps.SetOperator(*A);
-      mumps.Mult(B,X);
+      mumps.Mult(B, X);
      delete A;
   }
 #endif
@@ -524,16 +524,14 @@ int main(int argc, char *argv[])
      offsets[2] = fespace->GetTrueVSize();
      offsets.PartialSum();

-      Operator *pc_r = nullptr;
-      Operator *pc_i = nullptr;
+      std::unique_ptr<Operator> pc_r;
+      std::unique_ptr<Operator> pc_i;
      int s = (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0;
      if (pa)
      {
         // Jacobi Smoother
-         OperatorJacobiSmoother *d00 = new OperatorJacobiSmoother(prec, ess_tdof_list);
-         ScaledOperator *d11 = new ScaledOperator(d00, s);
-         pc_r = d00;
-         pc_i = d11;
+         pc_r.reset(new OperatorJacobiSmoother(prec, ess_tdof_list));
+         pc_i.reset(new ScaledOperator(pc_r.get(), s));
      }
      else
      {
@@ -541,15 +539,13 @@ int main(int argc, char *argv[])
         prec.FormSystemMatrix(ess_tdof_list, PCOpAh);

         // Hypre AMS
-         HypreAMS *ams00 = new HypreAMS(*PCOpAh.As<HypreParMatrix>(), fespace);
-         ScaledOperator *ams11 = new ScaledOperator(ams00, s);
-         pc_r = ams00;
-         pc_i = ams11;
+         pc_r.reset(new HypreAMS(*PCOpAh.As<HypreParMatrix>(), fespace));
+         pc_i.reset(new ScaledOperator(pc_r.get(), s));
      }

      BlockDiagonalPreconditioner BlockDP(offsets);
-      BlockDP.SetDiagonalBlock(0, pc_r);
-      BlockDP.SetDiagonalBlock(1, pc_i);
+      BlockDP.SetDiagonalBlock(0, pc_r.get());
+      BlockDP.SetDiagonalBlock(1, pc_i.get());

      GMRESSolver gmres(MPI_COMM_WORLD);
      gmres.SetPrintLevel(1);
@@ -884,7 +880,7 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
   }
 }

-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_JT_J_inv_Re(const Vector &x, PML * pml, Vector & D)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det(1.0, 0.0);
@@ -901,7 +897,7 @@ void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D)
   }
 }

-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_JT_J_inv_Im(const Vector &x, PML * pml, Vector & D)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -918,7 +914,7 @@ void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D)
   }
 }

-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_JT_J_inv_abs(const Vector &x, PML * pml, Vector & D)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -935,7 +931,7 @@ void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D)
   }
 }

-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_inv_JT_J_Re(const Vector &x, PML * pml, Vector & D)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det(1.0, 0.0);
@@ -960,7 +956,7 @@ void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D)
   }
 }

-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_inv_JT_J_Im(const Vector &x, PML * pml, Vector & D)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -984,7 +980,7 @@ void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D)
   }
 }

-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_inv_JT_J_abs(const Vector &x, PML * pml, Vector & D)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -1008,14 +1004,14 @@ void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D)
   }
 }

-CartesianPML::CartesianPML(Mesh *mesh_, Array2D<double> length_)
+PML::PML(Mesh *mesh_, Array2D<double> length_)
   : mesh(mesh_), length(length_)
 {
   dim = mesh->Dimension();
   SetBoundaries();
 }

-void CartesianPML::SetBoundaries()
+void PML::SetBoundaries()
 {
   comp_dom_bdr.SetSize(dim, 2);
   dom_bdr.SetSize(dim, 2);
@@ -1030,7 +1026,7 @@ void CartesianPML::SetBoundaries()
   }
 }

-void CartesianPML::SetAttributes(ParMesh *pmesh)
+void PML::SetAttributes(ParMesh *pmesh)
 {
   // Initialize bdr attributes
   for (int i = 0; i < pmesh->GetNBE(); ++i)
@@ -1080,8 +1076,8 @@ void CartesianPML::SetAttributes(ParMesh *pmesh)
   pmesh->SetAttributes();
 }

-void CartesianPML::StretchFunction(const Vector &x,
-                                   vector<complex<double>> &dxs)
+void PML::StretchFunction(const Vector &x,
+                          vector<complex<double>> &dxs)
 {
   complex<double> zi = complex<double>(0., 1.);

@@ -90,7 +90,7 @@ int main(int argc, char *argv[])
   const char *mesh_file = "../data/star.mesh";
   int order = 1;
   int nc_limit = 1;
-   int max_elems = 1e5;
+   int max_elems = 100*1000;
   double double_max_elems = double(max_elems);
   bool visualization = true;
   double osc_threshold = 1e-3;
@@ -123,7 +123,7 @@ int main(int argc, char *argv[])

   // 2. Compute the rational expansion coefficients that define the
   //    integer-order PDEs.
-   const int power_of_laplace = floor(alpha);
+   const int power_of_laplace = (int)floor(alpha);
   double exponent_to_approximate = alpha - power_of_laplace;
   bool integer_order = false;
   // Check if alpha is an integer or not.
@@ -0,0 +1,412 @@
+//                                MFEM Example 34
+//
+// Compile with: make ex34
+//
+// Sample runs:  ex34
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               discontinuous Galerkin (DG) finite element discretization of
+//               the Laplace problem -Delta u = f with Dirichlet boundary
+//               conditions. Finite element spaces of any order, including zero
+//               on regular grids, are supported. The example highlights the
+//               use of coupling solution domains though custom physics defined
+//               on internal boundaries.
+//
+//               We recommend viewing examples 1 and 14 before viewing this
+//               example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+class InteriorLFIntegrator : public LinearFormIntegrator
+{
+public:
+   InteriorLFIntegrator(Coefficient &Q)
+      : Q(Q)
+   {}
+
+   void AssembleRHSElementVect(const FiniteElement &el1,
+                               const FiniteElement &el2,
+                               FaceElementTransformations &trans,
+                               Vector &mesh_coords_bar) override;
+
+   void AssembleRHSElementVect(const FiniteElement &el,
+                               ElementTransformation &trans,
+                               Vector &elvect) override
+   {
+      mfem_error("AssembleRHSElementVect(...)");
+   }
+
+private:
+   Coefficient &Q;
+
+#ifndef MFEM_THREAD_SAFE
+   Vector shape1;
+   Vector shape2;
+#endif
+};
+
+Mesh generate_mesh(int ref, int internal_bdr_attr = 5);
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   int ref_levels = 0;
+   int order = 1;
+   int sol_order = 3;
+   double jump = -2;
+   double sigma = -1.0;
+   double kappa = -1.0;
+   double eta = 0.0;
+   bool visualization = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&ref_levels, "-r", "--refine",
+                  "Number of times to refine the mesh uniformly, -1 for auto.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) >= 0.");
+   args.AddOption(&sigma, "-s", "--sigma",
+                  "One of the three DG penalty parameters, typically +1/-1."
+                  " See the documentation of class DGDiffusionIntegrator.");
+   args.AddOption(&kappa, "-k", "--kappa",
+                  "One of the three DG penalty parameters, should be positive."
+                  " Negative values are replaced with (order+1)^2.");
+   args.AddOption(&eta, "-e", "--eta", "BR2 penalty parameter.");
+   args.AddOption(&sol_order, "-so", "--solution_order",
+                  "Polynomial order of the exact solution >= 0.");
+   args.AddOption(&jump, "-j", "--jump",
+                  "Value of the discontinuity between the material regions.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   if (kappa < 0)
+   {
+      kappa = (order+1)*(order+1);
+   }
+   if (sol_order < 0)
+   {
+      sol_order = 1;
+   }
+   args.PrintOptions(cout);
+
+   // 2. Construct the (serial) mesh and refine it if requested.
+   auto mesh = generate_mesh(ref_levels);
+
+   int dim = mesh.Dimension();
+
+   if (mesh.NURBSext)
+   {
+      mesh.SetCurvature(max(order, 1));
+   }
+
+   // 3. Define a finite element space on the mesh. Here we use discontinuous
+   //    finite elements of the specified order >= 0.
+   DG_FECollection fec(order, dim);
+   FiniteElementSpace fespace(&mesh, &fec);
+   cout << "Number of unknowns: " << fespace.GetVSize() << endl;
+
+   // 4. Set up the linear form b(.) which corresponds to the right-hand side of
+   //    the FEM linear system.
+   LinearForm b(&fespace);
+
+   Array<int> p1_attr_marker(mesh.attributes.Max());
+   p1_attr_marker = 0;
+   p1_attr_marker[0] = 1;
+
+   FunctionCoefficient p1_source([sol_order](const Vector &p)
+   {
+      const double x = p(0);
+      const double val = -(sol_order - 1)*sol_order*pow(x, sol_order-2);
+      return val;
+   });
+   b.AddDomainIntegrator(new DomainLFIntegrator(p1_source), p1_attr_marker);
+
+   Array<int> p2_attr_marker(mesh.attributes.Max());
+   p2_attr_marker = 0;
+   p2_attr_marker[1] = 1;
+
+   FunctionCoefficient p2_source([sol_order](const Vector &p)
+   {
+      const double x = p(0);
+      double val = -(sol_order - 1)*sol_order*pow(x - 2, sol_order-2);
+      if (sol_order % 2 == 0)
+      {
+         val *= -1.0;
+      }
+      return val;
+   });
+   b.AddDomainIntegrator(new DomainLFIntegrator(p2_source), p2_attr_marker);
+
+   ConstantCoefficient one(1.0);
+
+   Array<int> p1_bdr_attr_marker(mesh.bdr_attributes.Max());
+   p1_bdr_attr_marker = 0;
+   p1_bdr_attr_marker[0] = 1;
+
+   ConstantCoefficient left_bc_val(0.0);
+   b.AddBdrFaceIntegrator(
+      new DGDirichletLFIntegrator(left_bc_val, one, sigma, kappa),
+      p1_bdr_attr_marker);
+
+   Array<int> p2_bdr_attr_marker(mesh.bdr_attributes.Max());
+   p2_bdr_attr_marker = 0;
+   p2_bdr_attr_marker[1] = 1;
+
+   ConstantCoefficient right_bc_val(2.0 + jump);
+   b.AddBdrFaceIntegrator(
+      new DGDirichletLFIntegrator(right_bc_val, one, sigma, kappa),
+      p2_bdr_attr_marker);
+
+   Array<int> internal_bdr_attr_marker(mesh.bdr_attributes.Max());
+   internal_bdr_attr_marker = 0;
+   internal_bdr_attr_marker[4] = 1;
+
+   ConstantCoefficient interface_flux(sol_order);
+   b.AddInternalBoundaryFaceIntegrator(
+      new InteriorLFIntegrator(interface_flux),
+      internal_bdr_attr_marker);
+
+   b.Assemble();
+
+   // 5. Define the solution vector x as a finite element grid function
+   //    corresponding to fespace. Initialize x with initial guess of zero.
+   GridFunction x(&fespace);
+   x = 0.0;
+
+   // 6. Set up the bilinear form a(.,.) on the finite element space
+   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
+   //    domain integrator and the interior and boundary DG face integrators.
+   //    Note that boundary conditions are imposed weakly in the form, so there
+   //    is no need for dof elimination. After assembly and finalizing we
+   //    extract the corresponding sparse matrix A.
+   BilinearForm a(&fespace);
+   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   a.AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+   a.AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa),
+                          p1_bdr_attr_marker);
+   a.AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa),
+                          p2_bdr_attr_marker);
+   if (eta > 0)
+   {
+      a.AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
+      a.AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
+   }
+
+   // 7. Negate the DG interface terms along the internal boundary so that the
+   //    only coupling between domains is from the chosen model (constant flux
+   //    in this case).
+   ProductCoefficient neg_one(-1.0, one);
+   a.AddInternalBoundaryFaceIntegrator(new DGDiffusionIntegrator(neg_one, sigma,
+                                                                 kappa),
+                                       internal_bdr_attr_marker);
+   if (eta > 0)
+   {
+      a.AddInternalBoundaryFaceIntegrator(new DGDiffusionBR2Integrator(fespace,
+                                                                       neg_one, eta),
+                                          internal_bdr_attr_marker);
+   }
+
+   a.Assemble();
+   a.Finalize();
+   const SparseMatrix &A = a.SpMat();
+
+#ifndef MFEM_USE_SUITESPARSE
+   // 8. Define a simple symmetric Gauss-Seidel preconditioner and use it to
+   //    solve the system Ax=b with PCG in the symmetric case, and GMRES in the
+   //    non-symmetric one.
+   GSSmoother M(A);
+   if (sigma == -1.0)
+   {
+      PCG(A, M, b, x, 1, 500, 1e-12, 0.0);
+   }
+   else
+   {
+      GMRES(A, M, b, x, 1, 500, 500, 1e-24, 0.0);
+   }
+#else
+   // 8. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+   UMFPackSolver umf_solver;
+   umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+   umf_solver.SetOperator(A);
+   umf_solver.Mult(b, x);
+#endif
+
+   // 9. Save the refined mesh and the solution. This output can be viewed later
+   //    using GLVis: "glvis -m refined.mesh -g sol.gf".
+   ofstream mesh_ofs("refined.mesh");
+   mesh_ofs.precision(8);
+   mesh.Print(mesh_ofs);
+   ofstream sol_ofs("sol.gf");
+   sol_ofs.precision(8);
+   x.Save(sol_ofs);
+
+   // 10. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << mesh << x << flush;
+   }
+
+   return 0;
+}
+
+void InteriorLFIntegrator::AssembleRHSElementVect(
+   const FiniteElement &el1,
+   const FiniteElement &el2,
+   FaceElementTransformations &trans,
+   Vector &elvect)
+{
+   int ndof1 = el1.GetDof();
+   int ndof2 = el2.GetDof();
+   int ndof = ndof1 + ndof2;
+
+#ifdef MFEM_THREAD_SAFE
+   Vector shape1;
+   Vector shape2;
+#endif
+   shape1.SetSize(ndof1);
+   shape2.SetSize(ndof2);
+
+   const auto *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order = 2 * max(el1.GetOrder(), el2.GetOrder());
+      ir = &IntRules.Get(trans.GetGeometryType(), order);
+   }
+
+   elvect.SetSize(ndof);
+   Vector elvect1(elvect.GetData(), ndof1);
+   Vector elvect2(elvect.GetData() + ndof1, ndof2);
+   elvect = 0.0;
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const auto &ip = ir->IntPoint(i);
+
+      // Set the integration point in the face and the neighboring element
+      trans.SetAllIntPoints(&ip);
+
+      const double w = ip.weight * trans.Weight();
+
+      // Access the neighboring element's integration point
+      const auto &eip1 = trans.GetElement1IntPoint();
+      const auto &eip2 = trans.GetElement2IntPoint();
+
+      double Q_val = Q.Eval(trans, ip);
+
+      el1.CalcShape(eip1, shape1);
+      el2.CalcShape(eip2, shape2);
+
+      elvect1.Add(Q_val * w, shape1);
+      elvect2.Add(-Q_val * w, shape2);
+   }
+}
+
+Mesh generate_mesh(int ref, int internal_bdr_attr)
+{
+   int nxy = 4 * (ref+1);
+   auto mesh = Mesh::MakeCartesian2D(nxy, nxy, Element::TRIANGLE, true, 2.0, 1.0);
+   // auto mesh = Mesh::MakeCartesian2D(nxy, nxy, Element::QUADRILATERAL, true, 2.0, 1.0);
+
+   // assign element attributes to left and right sides
+   for (int i = 0; i < mesh.GetNE(); ++i)
+   {
+      auto *elem = mesh.GetElement(i);
+
+      Array<int> verts;
+      elem->GetVertices(verts);
+
+      bool left = true;
+      for (int j = 0; j < verts.Size(); ++j)
+      {
+         auto *vtx = mesh.GetVertex(verts[j]);
+         if (vtx[0] <= 1.0)
+         {
+            continue;
+         }
+         else
+         {
+            left = false;
+         }
+      }
+      if (left)
+      {
+         elem->SetAttribute(1);
+      }
+      else
+      {
+         elem->SetAttribute(2);
+      }
+   }
+
+   // assign boundary element attributes to left and right sides
+   for (int i = 0; i < mesh.GetNBE(); ++i)
+   {
+      auto *elem = mesh.GetBdrElement(i);
+
+      Array<int> verts;
+      elem->GetVertices(verts);
+
+      bool left = true;
+      bool right = true;
+      bool top = true;
+      bool bottom = true;
+      for (int j = 0; j < verts.Size(); ++j)
+      {
+         auto *vtx = mesh.GetVertex(verts[j]);
+         left = left && abs(vtx[0] - 0.0) < 1e-12;
+         right = right && abs(vtx[0] - 2.0) < 1e-12;
+         top = top && abs(vtx[1] - 1.0) < 1e-12;
+         bottom = bottom && abs(vtx[1] - 0.0) < 1e-12;
+      }
+      if (left)
+      {
+         elem->SetAttribute(1);
+      }
+      else if (right)
+      {
+         elem->SetAttribute(2);
+      }
+      else if (top)
+      {
+         elem->SetAttribute(3);
+      }
+      else if (bottom)
+      {
+         elem->SetAttribute(4);
+      }
+   }
+
+   // add internal boundary elements
+   for (int i = 0; i < mesh.GetNumFaces(); ++i)
+   {
+      int e1, e2;
+      mesh.GetFaceElements(i, &e1, &e2);
+      if (e1 >= 0 && e2 >= 0 && mesh.GetAttribute(e1) != mesh.GetAttribute(e2))
+      {
+         // This is the internal face between attributes.
+         auto *new_elem = mesh.GetFace(i)->Duplicate(&mesh);
+         new_elem->SetAttribute(internal_bdr_attr);
+         mesh.AddBdrElement(new_elem);
+      }
+   }
+
+   mesh.FinalizeTopology(); // Finalize to build relevant tables
+   mesh.Finalize();
+   mesh.SetAttributes();
+
+   return mesh;
+}
@@ -0,0 +1,451 @@
+//                                MFEM Example 36
+//
+// Compile with: make ex36
+//
+// Sample runs:  ex36
+//
+// Description:  This example code demonstrates the use of MFEM to define a
+//               discontinuous Galerkin (DG) finite element discretization of
+//               the Laplace problem -Delta u = f with Dirichlet boundary
+//               conditions. Finite element spaces of any order, including zero
+//               on regular grids, are supported. The example highlights the
+//               use of coupling solution domains though custom physics defined
+//               on internal boundaries.
+//
+//               We recommend viewing examples 1, 14, and 34 before viewing this
+//               example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+class InteriorMassIntegrator : public BilinearFormIntegrator
+{
+public:
+   InteriorMassIntegrator(Coefficient &Q)
+      : Q(Q)
+   {}
+
+   void AssembleFaceMatrix(const FiniteElement &el1,
+                           const FiniteElement &el2,
+                           FaceElementTransformations &trans,
+                           DenseMatrix &elmat) override;
+
+   using BilinearFormIntegrator::AssembleFaceMatrix;
+
+private:
+   Coefficient &Q;
+
+#ifndef MFEM_THREAD_SAFE
+   Vector shape1;
+   Vector shape2;
+   DenseMatrix elmat11;
+   DenseMatrix elmat12;
+   DenseMatrix elmat21;
+   DenseMatrix elmat22;
+#endif
+};
+
+Mesh generate_mesh(int ref, int internal_bdr_attr = 5);
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   int ref_levels = 0;
+   int order = 1;
+   int sol_order = 3;
+   double jump = -2;
+   double sigma = -1.0;
+   double kappa = -1.0;
+   double eta = 0.0;
+   bool visualization = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&ref_levels, "-r", "--refine",
+                  "Number of times to refine the mesh uniformly, -1 for auto.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) >= 0.");
+   args.AddOption(&sigma, "-s", "--sigma",
+                  "One of the three DG penalty parameters, typically +1/-1."
+                  " See the documentation of class DGDiffusionIntegrator.");
+   args.AddOption(&kappa, "-k", "--kappa",
+                  "One of the three DG penalty parameters, should be positive."
+                  " Negative values are replaced with (order+1)^2.");
+   args.AddOption(&eta, "-e", "--eta", "BR2 penalty parameter.");
+   args.AddOption(&sol_order, "-so", "--solution_order",
+                  "Polynomial order of the exact solution >= 0.");
+   args.AddOption(&jump, "-j", "--jump",
+                  "Value of the discontinuity between the material regions.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   if (kappa < 0)
+   {
+      kappa = (order+1)*(order+1);
+   }
+   if (sol_order < 0)
+   {
+      sol_order = 1;
+   }
+   args.PrintOptions(cout);
+
+   // 2. Construct the (serial) mesh and refine it if requested.
+   auto mesh = generate_mesh(ref_levels);
+
+   int dim = mesh.Dimension();
+
+   if (mesh.NURBSext)
+   {
+      mesh.SetCurvature(max(order, 1));
+   }
+
+   // 3. Define a finite element space on the mesh. Here we use discontinuous
+   //    finite elements of the specified order >= 0.
+   DG_FECollection fec(order, dim);
+   FiniteElementSpace fespace(&mesh, &fec);
+   cout << "Number of unknowns: " << fespace.GetVSize() << endl;
+
+   // 4. Set up the linear form b(.) which corresponds to the right-hand side of
+   //    the FEM linear system.
+   LinearForm b(&fespace);
+
+   Array<int> p1_attr_marker(mesh.attributes.Max());
+   p1_attr_marker = 0;
+   p1_attr_marker[0] = 1;
+
+   FunctionCoefficient p1_source([sol_order](const Vector &p)
+   {
+      const double x = p(0);
+      const double val = -(sol_order - 1)*sol_order*pow(x, sol_order-2);
+      return val;
+   });
+   b.AddDomainIntegrator(new DomainLFIntegrator(p1_source), p1_attr_marker);
+
+   Array<int> p2_attr_marker(mesh.attributes.Max());
+   p2_attr_marker = 0;
+   p2_attr_marker[1] = 1;
+
+   FunctionCoefficient p2_source([sol_order](const Vector &p)
+   {
+      const double x = p(0);
+      double val = -(sol_order - 1)*sol_order*pow(x - 2, sol_order-2);
+      if (sol_order % 2 == 0)
+      {
+         val *= -1.0;
+      }
+      return val;
+   });
+   b.AddDomainIntegrator(new DomainLFIntegrator(p2_source), p2_attr_marker);
+
+   ConstantCoefficient one(1.0);
+
+   Array<int> p1_bdr_attr_marker(mesh.bdr_attributes.Max());
+   p1_bdr_attr_marker = 0;
+   p1_bdr_attr_marker[0] = 1;
+
+   ConstantCoefficient left_bc_val(0.0);
+   b.AddBdrFaceIntegrator(
+      new DGDirichletLFIntegrator(left_bc_val, one, sigma, kappa),
+      p1_bdr_attr_marker);
+
+   Array<int> p2_bdr_attr_marker(mesh.bdr_attributes.Max());
+   p2_bdr_attr_marker = 0;
+   p2_bdr_attr_marker[1] = 1;
+
+   ConstantCoefficient right_bc_val(2.0 + jump);
+   b.AddBdrFaceIntegrator(
+      new DGDirichletLFIntegrator(right_bc_val, one, sigma, kappa),
+      p2_bdr_attr_marker);
+
+   b.Assemble();
+
+   // 5. Define the solution vector x as a finite element grid function
+   //    corresponding to fespace. Initialize x with initial guess of zero.
+   GridFunction x(&fespace);
+   x = 0.0;
+
+   // 6. Set up the bilinear form a(.,.) on the finite element space
+   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
+   //    domain integrator and the interior and boundary DG face integrators.
+   //    Note that boundary conditions are imposed weakly in the form, so there
+   //    is no need for dof elimination. After assembly and finalizing we
+   //    extract the corresponding sparse matrix A.
+   BilinearForm a(&fespace);
+   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   a.AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
+   a.AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa),
+                          p1_bdr_attr_marker);
+   a.AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa),
+                          p2_bdr_attr_marker);
+   if (eta > 0)
+   {
+      a.AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
+      a.AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
+   }
+
+   // 7. Negate the DG interface terms along the internal boundary so that the
+   //    only coupling between domains is from the chosen model (constant flux
+   //    in this case).
+   Array<int> internal_bdr_attr_marker(mesh.bdr_attributes.Max());
+   internal_bdr_attr_marker = 0;
+   internal_bdr_attr_marker[4] = 1;
+
+   ProductCoefficient neg_one(-1.0, one);
+   a.AddInternalBoundaryFaceIntegrator(new DGDiffusionIntegrator(neg_one, sigma,
+                                                                 kappa),
+                                       internal_bdr_attr_marker);
+   if (eta > 0)
+   {
+      a.AddInternalBoundaryFaceIntegrator(new DGDiffusionBR2Integrator(fespace,
+                                                                       neg_one, eta),
+                                          internal_bdr_attr_marker);
+   }
+
+   ConstantCoefficient mass_coeff(sol_order / jump);
+   a.AddInternalBoundaryFaceIntegrator(new InteriorMassIntegrator(mass_coeff),
+                                       internal_bdr_attr_marker);
+
+   a.Assemble();
+   a.Finalize();
+   const SparseMatrix &A = a.SpMat();
+
+#ifndef MFEM_USE_SUITESPARSE
+   // 8. Define a simple symmetric Gauss-Seidel preconditioner and use it to
+   //    solve the system Ax=b with PCG in the symmetric case, and GMRES in the
+   //    non-symmetric one.
+   GSSmoother M(A);
+   if (sigma == -1.0 && !(jump < 0))
+   {
+      PCG(A, M, b, x, 1, 500, 1e-12, 0.0);
+   }
+   else
+   {
+      GMRES(A, M, b, x, 1, 500, 500, 1e-24, 0.0);
+   }
+#else
+   // 8. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+   UMFPackSolver umf_solver;
+   umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+   umf_solver.SetOperator(A);
+   umf_solver.Mult(b, x);
+#endif
+
+   // 9. Save the refined mesh and the solution. This output can be viewed later
+   //    using GLVis: "glvis -m refined.mesh -g sol.gf".
+   ofstream mesh_ofs("refined.mesh");
+   mesh_ofs.precision(8);
+   mesh.Print(mesh_ofs);
+   ofstream sol_ofs("sol.gf");
+   sol_ofs.precision(8);
+   x.Save(sol_ofs);
+
+   // 10. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << mesh << x << flush;
+   }
+
+   return 0;
+}
+
+void InteriorMassIntegrator::AssembleFaceMatrix(
+   const FiniteElement &el1,
+   const FiniteElement &el2,
+   FaceElementTransformations &trans,
+   DenseMatrix &elmat)
+{
+   int ndof1 = el1.GetDof();
+   int ndof2 = el2.GetDof();
+   int ndof = ndof1 + ndof2;
+
+#ifdef MFEM_THREAD_SAFE
+   Vector shape1;
+   Vector shape2;
+   DenseMatrix elmat11;
+   DenseMatrix elmat12;
+   DenseMatrix elmat21;
+   DenseMatrix elmat22;
+#endif
+   shape1.SetSize(ndof1);
+   shape2.SetSize(ndof2);
+
+   elmat11.SetSize(ndof1);
+   elmat12.SetSize(ndof1, ndof2);
+   elmat21.SetSize(ndof2, ndof1);
+   elmat22.SetSize(ndof2);
+
+   const auto *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order = 2 * max(el1.GetOrder(), el2.GetOrder());
+      ir = &IntRules.Get(trans.GetGeometryType(), order);
+   }
+
+   elmat.SetSize(ndof);
+   elmat = 0.0;
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const auto &ip = ir->IntPoint(i);
+
+      // Set the integration point in the face and the neighboring element
+      trans.SetAllIntPoints(&ip);
+
+      const double w = ip.weight * trans.Weight();
+
+      // Access the neighboring element's integration point
+      const auto &eip1 = trans.GetElement1IntPoint();
+      const auto &eip2 = trans.GetElement2IntPoint();
+
+      el1.CalcShape(eip1, shape1);
+      el2.CalcShape(eip2, shape2);
+
+      const double Q_val = Q.Eval(trans, ip);
+
+      elmat11 = 0.0;
+      AddMult_a_VVt(Q_val * w, shape1, elmat11);
+
+      elmat12 = 0.0;
+      AddMult_a_VWt(-Q_val * w, shape2, shape1, elmat12);
+
+      elmat21 = 0.0;
+      AddMult_a_VWt(-Q_val * w, shape1, shape2, elmat21);
+
+      elmat22 = 0.0;
+      AddMult_a_VVt(Q_val * w, shape2, elmat22);
+
+      for (int j = 0; j < ndof1; ++j)
+      {
+         for (int k = 0; k < ndof1; ++k)
+         {
+            elmat(j, k) += elmat11(j, k);
+         }
+      }
+
+      for (int j = 0; j < ndof1; ++j)
+      {
+         for (int k = 0; k < ndof2; ++k)
+         {
+            elmat(j, k + ndof1) += elmat12(j, k);
+            elmat(k + ndof1, j) += elmat21(k, j);
+         }
+      }
+
+      for (int j = 0; j < ndof2; ++j)
+      {
+         for (int k = 0; k < ndof2; ++k)
+         {
+            elmat(j + ndof1, k + ndof1) += elmat22(j, k);
+         }
+      }
+   }
+}
+
+Mesh generate_mesh(int ref, int internal_bdr_attr)
+{
+   int nxy = 4 * (ref+1);
+   auto mesh = Mesh::MakeCartesian2D(nxy, nxy, Element::TRIANGLE, true, 2.0, 1.0);
+   // auto mesh = Mesh::MakeCartesian2D(nxy, nxy, Element::QUADRILATERAL, true, 2.0, 1.0);
+
+   // assign element attributes to left and right sides
+   for (int i = 0; i < mesh.GetNE(); ++i)
+   {
+      auto *elem = mesh.GetElement(i);
+
+      Array<int> verts;
+      elem->GetVertices(verts);
+
+      bool left = true;
+      for (int j = 0; j < verts.Size(); ++j)
+      {
+         auto *vtx = mesh.GetVertex(verts[j]);
+         if (vtx[0] <= 1.0)
+         {
+            continue;
+         }
+         else
+         {
+            left = false;
+         }
+      }
+      if (left)
+      {
+         elem->SetAttribute(1);
+      }
+      else
+      {
+         elem->SetAttribute(2);
+      }
+   }
+
+   // assign boundary element attributes to left and right sides
+   for (int i = 0; i < mesh.GetNBE(); ++i)
+   {
+      auto *elem = mesh.GetBdrElement(i);
+
+      Array<int> verts;
+      elem->GetVertices(verts);
+
+      bool left = true;
+      bool right = true;
+      bool top = true;
+      bool bottom = true;
+      for (int j = 0; j < verts.Size(); ++j)
+      {
+         auto *vtx = mesh.GetVertex(verts[j]);
+         left = left && abs(vtx[0] - 0.0) < 1e-12;
+         right = right && abs(vtx[0] - 2.0) < 1e-12;
+         top = top && abs(vtx[1] - 1.0) < 1e-12;
+         bottom = bottom && abs(vtx[1] - 0.0) < 1e-12;
+      }
+      if (left)
+      {
+         elem->SetAttribute(1);
+      }
+      else if (right)
+      {
+         elem->SetAttribute(2);
+      }
+      else if (top)
+      {
+         elem->SetAttribute(3);
+      }
+      else if (bottom)
+      {
+         elem->SetAttribute(4);
+      }
+   }
+
+   // add internal boundary elements
+   for (int i = 0; i < mesh.GetNumFaces(); ++i)
+   {
+      int e1, e2;
+      mesh.GetFaceElements(i, &e1, &e2);
+      if (e1 >= 0 && e2 >= 0 && mesh.GetAttribute(e1) != mesh.GetAttribute(e2))
+      {
+         // This is the internal face between attributes.
+         auto *new_elem = mesh.GetFace(i)->Duplicate(&mesh);
+         new_elem->SetAttribute(internal_bdr_attr);
+         mesh.AddBdrElement(new_elem);
+      }
+   }
+
+   mesh.FinalizeTopology(); // Finalize to build relevant tables
+   mesh.Finalize();
+   mesh.SetAttributes();
+
+   return mesh;
+}
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
--- a/Show More
+++ b/Show More