Add some comments

Big speedup, but can we do this in a portable way?

.binder/environment.yml

@@ -5,7 +5,7 @@ channels:
 dependencies:
 - xeus-cling=0.13.0
 - xwidgets=0.26.0
-# NOTE: it is possible that these are not needed for the lab frontend
+# NOTE: it's possible these aren't needed for the lab frontend
 - widgetsnbextension=3.5.1
 - pip
 - pip:

.github/workflows/README.md

@@ -1,48 +0,0 @@
-                    Finite Element Discretization Library
-                                   __
-                       _ __ ___   / _|  ___  _ __ ___
-                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
-                      | | | | | ||  _||  __/| | | | | |
-                      |_| |_| |_||_|   \___||_| |_| |_|
-
-                               https://mfem.org
-
-
-This directory contains the GitHub CI scripts for MFEM.
-
-Note that some of these scripts use the shared MFEM GitHub Actions from the external mfem/github-actions repository:
-
-  https://github.com/mfem/github-actions
-
-For a particular action, e.g. `mfem/github-actions/build-mfem@v2.1`, the `v2.1` suffix denotes the branch in the above from which the action is taken.
-
-The current CI workflows are:
-
-### `repo-check.yml`
-
-Runs a number of static repository-level sanity checks.
-
-- `file-headers-check` checks copyright date, license, etc. using the `--copyright`, `--license` and `--release` options of the `config/githooks/pre-push` script.
-
-- `code-style` checks the code style using the `--style` option of the `config/githooks/pre-push` script.
-
-- `documentation` checks the documentation build using the `tests/scripts/documentation` script.
-
-- `branch-history` guards against accidental commits of large files using the `--history` option of the `config/githooks/pre-push` script.
-
-### `mfem-analysis.yml` (`build-analysis`)
-
-Checks if the code builds and satisfies minimal requirements.
-
-- `gitignore` builds hypre, METIS, and MFEM using `mfem/github-actions/build-hypre`, `mfem/github-actions/build-metis`, and `mfem/github-actions/build-mfem` and checks for correct `.gitignore` settings by running the `tests/scripts/gitignore` script.
-
-### `builds-and-tests.yml`
-
-Runs a matrix of builds and tests runs with different compilers, OS, mfem/hypre settings, etc. Also processes and upload Codecov reports.
-
-Uses the following GitHub Actions from https://github.com/mfem/github-actions:
-
-- `mfem/github-actions/build-hypre`
-- `mfem/github-actions/build-metis`
-- `mfem/github-actions/build-mfem`
-- `mfem/github-actions/upload-coverage`

.github/workflows/builds-and-tests.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -166,7 +166,7 @@ jobs:
 
     # MFEM build and test
     - name: build
-      uses: mfem/github-actions/build-mfem@v2.1
+      uses: mfem/github-actions/build-mfem@v2.0
       with:
         os: ${{ matrix.os }}
         target: ${{ matrix.target }}

.github/workflows/mfem-analysis.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

.github/workflows/repo-check.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

.gitignore

@@ -15,6 +15,19 @@
 CMakeCache.txt
 CMakeFiles/
 
+# NSYS files
+*.nsys-rep
+
+# Benchmark files
+*.tex
+*.org
+*.aux
+*.log
+*.gnuplot
+*.table
+*.pdf
+*.png
+
 # Backup files
 *~
 
@@ -51,8 +64,8 @@ examples/ex1[04-9]
 examples/ex1[0-9]p
 examples/ex2[0-9]
 examples/ex2[0-9]p
-examples/ex3[0-9]
-examples/ex3[0-9]p
+examples/ex30
+examples/ex30p
 
 examples/refined.mesh
 examples/displaced.mesh
@@ -78,7 +91,6 @@ examples/deformed.*
 examples/velocity.*
 examples/elastic_energy.*
 examples/mode_*
-examples/mode_deriv_*
 examples/ex5-p-*.bp
 examples/ex9-p-*.bp
 examples/ex12-p-*.bp
@@ -260,8 +272,6 @@ miniapps/performance/sol.*
 
 miniapps/shifted/distance
 miniapps/shifted/ParaViewDistance
-miniapps/shifted/extrapolate
-miniapps/shifted/ParaViewExtrapolate
 miniapps/shifted/diffusion
 miniapps/shifted/diffusion.mesh
 miniapps/shifted/diffusion.gf
@@ -319,6 +329,9 @@ tests/unit/debug_device_tests
 
 # Benchmark binaries
 tests/benchmarks/bench_ceed
+tests/benchmarks/bench_solvers
+tests/benchmarks/bench_linext
+tests/benchmarks/bench_lor_assembly
 tests/benchmarks/bench_tmop
 tests/benchmarks/bench_vector
 tests/benchmarks/bench_virtuals

.gitlab-ci.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

.gitlab/configs/common.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

.gitlab/configs/corona-config.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -18,14 +18,14 @@ variables:
     - shell
     - corona
   rules:
-    # Don't run corona jobs if...
+    # Don’t run corona jobs if...
     # Note: This makes corona an "opt-in" machine. To activate builds on corona
     # for a given GitLab clone of MFEM, go to Setting/CI-CD/variables, and set
     # "ON_CORONA" to "ON". An LC account on for corona is required to trigger a
     # pipeline there.
     - if: '$CI_COMMIT_BRANCH =~ /_cnone/ || $ON_CORONA != "ON"'
       when: never
-    # Don't run autotest update if...
+    # Don’t run autotest update if...
     - if: '$CI_JOB_NAME =~ /report/ && $AUTOTEST != "YES"'
       when: never
     # Report success on success status

.gitlab/configs/lassen-config.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

.gitlab/configs/quartz-config.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

.gitlab/configs/report-build-and-test.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -20,8 +20,8 @@
     - |
       (
         date
-        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
-        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
+        echo "Waiting to aquire lock on '$PWD/autotest.lock' ..."
+        # try to get an excusive lock on fd 9 (autotest.lock) repeating the try
         # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
         # command to hang indefinitely sometimes, so we use the timeout & retry
         # as a workaround; we may want to add a counter for the number of
@@ -29,7 +29,7 @@
         while ! flock -w 5 9; do
           true
         done
-        echo "Acquired lock on '$PWD/autotest.lock'"
+        echo "Aquired lock on '$PWD/autotest.lock'"
         date
         # Report SUCCESS while holding the file lock on 'autotest.lock'.
         # The next script uses the following environment variables:
@@ -55,8 +55,8 @@
     - |
       (
         date
-        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
-        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
+        echo "Waiting to aquire lock on '$PWD/autotest.lock' ..."
+        # try to get an excusive lock on fd 9 (autotest.lock) repeating the try
         # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
         # command to hang indefinitely sometimes, so we use the timeout & retry
         # as a workaround; we may want to add a counter for the number of
@@ -64,7 +64,7 @@
         while ! flock -w 5 9; do
           true
         done
-        echo "Acquired lock on '$PWD/autotest.lock'"
+        echo "Aquired lock on '$PWD/autotest.lock'"
         date
         # Report FAILURE while holding the file lock on 'autotest.lock'.
         # The next script uses the following environment variables:

.gitlab/configs/setup-baseline.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -45,8 +45,8 @@ setup_baseline:
     - |
       (
         date
-        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
-        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
+        echo "Waiting to aquire lock on '$PWD/autotest.lock' ..."
+        # try to get an excusive lock on fd 9 (autotest.lock) repeating the try
         # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
         # command to hang indefinitely sometimes, so we use the timeout & retry
         # as a workaround; we may want to add a counter for the number of
@@ -54,7 +54,7 @@ setup_baseline:
         while ! flock -w 5 9; do
           true
         done
-        echo "Acquired lock on '$PWD/autotest.lock'"
+        echo "Aquired lock on '$PWD/autotest.lock'"
         date
         # clone/update the autotest repo while holding the file lock on
         # 'autotest.lock'

.gitlab/configs/setup-build-and-test.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -34,8 +34,8 @@ setup:
     - |
       (
         date
-        echo "Waiting to acquire lock on '$PWD/mfem-data.lock' ..."
-        # try to get an exclusive lock on fd 9 (mfem-data.lock) repeating the try
+        echo "Waiting to aquire lock on '$PWD/mfem-data.lock' ..."
+        # try to get an excusive lock on fd 9 (mfem-data.lock) repeating the try
         # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
         # command to hang indefinitely sometimes, so we use the timeout & retry
         # as a workaround; we may want to add a counter for the number of
@@ -43,7 +43,7 @@ setup:
         while ! flock -w 5 9; do
           true
         done
-        echo "Acquired lock on '$PWD/mfem-data.lock'"
+        echo "Aquired lock on '$PWD/mfem-data.lock'"
         date
         # clone/update the mfem/data repo while holding the file lock on
         # 'mfem-data.lock'
@@ -67,8 +67,8 @@ setup:
     - |
       (
         date
-        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
-        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
+        echo "Waiting to aquire lock on '$PWD/autotest.lock' ..."
+        # try to get an excusive lock on fd 9 (autotest.lock) repeating the try
         # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
         # command to hang indefinitely sometimes, so we use the timeout & retry
         # as a workaround; we may want to add a counter for the number of
@@ -76,7 +76,7 @@ setup:
         while ! flock -w 5 9; do
           true
         done
-        echo "Acquired lock on '$PWD/autotest.lock'"
+        echo "Aquired lock on '$PWD/autotest.lock'"
         date
         # clone/update the autotest repo while holding the file lock on
         # 'autotest.lock'

.gitlab/corona-build-and-test.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

.gitlab/lassen-build-and-test.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -19,11 +19,6 @@ opt_mpi_cuda_xl_16_1_1_8:
     SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=70"
   extends: .build_and_test_on_lassen
 
-opt_mpi_cuda_hypre_cuda_xl:
-  variables:
-    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=70 ^hypre+cuda~shared cuda_arch=70"
-  extends: .build_and_test_on_lassen
-
 # Jobs report
 report_job_success:
   stage: report

.gitlab/quartz-baseline.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -64,8 +64,8 @@ report_baseline:
     - |
       (
         date
-        echo "Waiting to acquire lock on '$PWD/autotest.lock' ..."
-        # try to get an exclusive lock on fd 9 (autotest.lock) repeating the try
+        echo "Waiting to aquire lock on '$PWD/autotest.lock' ..."
+        # try to get an excusive lock on fd 9 (autotest.lock) repeating the try
         # every 5 seconds; simply using no timeout, i.e. 'flock 9', causes the
         # command to hang indefinitely sometimes, so we use the timeout & retry
         # as a workaround; we may want to add a counter for the number of
@@ -73,7 +73,7 @@ report_baseline:
         while ! flock -w 5 9; do
           true
         done
-        echo "Acquired lock on '$PWD/autotest.lock'"
+        echo "Aquired lock on '$PWD/autotest.lock'"
         date
         # ----------------------
         cd ${AUTOTEST_ROOT}/autotest || \

.gitlab/quartz-build-and-test.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -59,7 +59,7 @@ opt_par_gcc_6_1_0_sundials:
 
 opt_par_gcc_6_1_0_petsc:
   variables:
-    SPEC: "%gcc@6.1.0 +petsc ^petsc+mumps~superlu-dist"
+    SPEC: "%gcc@6.1.0 +petsc ^petsc+mumps"
   extends: .build_and_test_on_quartz
 
 opt_par_gcc_6_1_0_pumi:

.gitlab/scripts/baseline

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -29,9 +29,9 @@ mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
 
 # run
 if [[ "${MACHINE_NAME}" == "quartz" || "${MACHINE_NAME}" == "ruby" ]]; then
-  salloc --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  srun --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "corona" ]]; then
-  salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
+  srun --nodes=1 -t 60 -p mi60 ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 elif [[ ${MACHINE_NAME} == "lassen" ]]; then
   lalloc 1 -q pdebug ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
 else

.gitlab/scripts/rebaseline

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

.gitlab/scripts/report_build_and_test_failure

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

.gitlab/scripts/report_build_and_test_success

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

.gitlab/scripts/safe_create_rundir

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

CHANGELOG

@@ -8,146 +8,92 @@
                                https://mfem.org
 
 
-Version 4.4, released on March 21, 2022
-=======================================
-
-Linear and nonlinear solvers
-----------------------------
-- Added support for using the hypre library built with HIP support. Similar to
-  the existing hypre + CUDA support, most of the MFEM examples and miniapps work
-  transparently with hypre + HIP builds. This includes the BoomerAMG, AMS, and
-  ADS solvers.
-
-- Added a simple singleton class, Hypre, to automatically set hypre's global
-  parameters, particularly GPU-relevant options. Updated parallel example codes
-  and miniapps to call Hypre::Init() where appropriate.
-
-- Added hipSPARSE support for sparse matrix-vector multiplications.
-
-- More explicit and consistent formatting of the output of iterative solvers
+Version 4.3.1 (development)
+===========================
+- More explicit and consistent formating of the output of iterative solvers
   with the new IterativeSolver::PrintLevel options. See linalg/solvers.hpp.
 
-Meshing improvements
---------------------
-- New TMOP-based methods for hr-adaptivity, interface fitting, and tangential
-  relaxation of high-order meshes.
+- Added support for automatic differentiation. Users can select between native
+  implementation and external library implementation during configuration. One
+  parallel and two serial examples are implemented in the miniapps/autodiff/
+  directory.
+
+- GridFunctionCoefficient (and the related vector, gradient, divergence, and
+  curl classes) now work properly with LORDiscretization and LORSolver.
+
+- Added support for mesh preprocessing to resolve fine scale problem data
+  before simulation. This feature uses adaptive mesh refinement to control the
+  associated data oscillation error. See the new Example 30/30p.
+
+- Switched from Artistic Style (astyle) version 2.05.1 to version 3.1 for code
+  formatting. See the "make style" target.
+
+- Split the fem/fe.?pp files into separate files in the new fem/fe/ directory
+  to simplify and clarify the organization of FiniteElement classes.
+
+- Added support for hr-adaptivity using TMOP-based error estimator.
+
+- Coefficient::SetTime now propagates the new time into internally stored
+  Coefficient objects.
+
+- Added initial support for google-benchmarks in the tests/benchmarks directory.
+  It can be enabled with MFEM_USE_BENCHMARK=YES.
+
+- Added Binder (mybinder.org) configuration files for C++ MFEM Jupyter Notebooks
+  with inline GLVis visualization as well as a new examples/jupyter/ directory
+  with a sample notebook based on Example 1. Implementation based on xeus-cling,
+  github.com/jupyter-xeus/xeus-cling + xeus-glvis, github.com/GLVis/xeus-glvis.
+
+- Added 'double' atomicAdd implementation for previous versions of CUDA.
+
+- Adding lowest order Nedelec and Raviart-Thomas basis functions on wedge
+  shaped elements.
 
 - Added initial support for meshes with pyramidal elements, including several
   pyramidal meshes in the data/ directory and support for the lowest order H1,
   Nedelec, Raviart-Thomas, and L2 basis functions on pyramids.
 
-- Added a simpler interface to access mesh face information, see FaceInformation
-  and GetFaceInformation in the Mesh class.
+- Updated the hypre interface according to changes in hypre-2.22.1. The ADS
+  solver is now fully working on GPUs.
 
-- Gmsh meshes where all elements have zero physical tag (the default Gmsh output
-  format if no physical groups are defined) are now successfully loaded, and
-  elements are reassigned attribute number 1.
+- Tetrahedral meshes no longer need to be reordered to support high order
+  Nedelec basis functions. This will allow future support for Nedelec basis
+  functions on wedges and pyramids which are not amenable to reordering. The
+  ReorientTetMesh method of the Mesh and ParMesh classes has been deprecated.
 
-- Added ParMesh adjacency set (adjset) creation support to the Conduit Mesh
-  Blueprint MFEM wrapper functions in ConduitDataCollection.
+- Gmsh meshes where all elements have zero physical tag (the default Gmsh
+  output format if no physical groups are defined) are now successfully loaded,
+  and elements are reassigned attribute number 1.
 
-Discretization improvements
----------------------------
-- Added general dof transformation to support high order Nedelec basis functions
-  on tetrahedral meshes without reordering. The ReorientTetMesh method of the
-  Mesh and ParMesh classes has been deprecated. See the new DofTransformation
-  class in fem/doftrans.hpp.
-
-- GPU-enabled partial (PA) and element (EA) assembly for discontinuous Galerkin
-  methods on nonconforming AMR meshes.
-
-- Support for arbitrary order Nedelec and Raviart-Thomas elements on wedges.
-
-- Added special Nedelec and Raviart-Thomas basis functions for modeling three
-  dimensional vector fields in 1D and 2D domains, see the new Example 31/31p.
-
-- GridFunctionCoefficient (and the related vector, gradient, divergence, and
-  curl classes) now work properly with LORDiscretization and LORSolver.
-
-- Added PA support for the action of MixedScalarCurlIntegrator in 2D and
-  MixedVectorGradientIntegrator in 2D and 3D, as well as their transposes.
-
-- Coefficient::SetTime now propagates the new time into internally stored
-  Coefficient objects.
-
-- Split the fem/fe.?pp files into separate files in the new fem/fe/ directory to
-  simplify and clarify the organization of FiniteElement classes.
-
-New and updated examples and miniapps
--------------------------------------
-- Added two new miniapps with initial support for automatic differentiation (AD)
-  in the miniapps/autodiff/ directory. Users can select between external library
-  and native implementation during configuration. The support for AD will be
-  extended in future releases of MFEM.
-
-- Added Binder (mybinder.org) configuration files for C++ MFEM Jupyter Notebooks
-  with inline GLVis visualization in the new examples/jupyter/ directory with a
-  sample notebook based on Example 1. The implementation is based on xeus-cling,
-  see github.com/jupyter-xeus/xeus-cling and github.com/GLVis/xeus-glvis.
-
-- Added a new miniapp (Extrapolation) for PDE-based extrapolation of finite
-  element functions from known values in a set of elements to the rest of the
-  computational domain. See miniapps/shifted/extrapolate.cpp.
-
-- Added new miniapp that uses the ParELAG library, its hybrid smoothers, and the
+- Added new miniapps that use the ParELAG library, its hybrid smoothers, and the
   hierarchy of spaces created by the element-based AMG (AMGe) methodology in
   ParELAG to build multigrid solvers for H(curl) and H(div) forms. See the
   miniapps/parelag directory for more details.
 
-- Added a new Example 30/30p demonstrating support for mesh preprocessing to
-  resolve fine scale problem data before simulation. This feature uses adaptive
-  mesh refinement to control the associated data oscillation error.
-
-- Added new Examples 31, 31p and 32p showing anisotropic definite Maxwell
-  serial/parallel solver and parallel eigensolver 1D, 2D, or 3D.
-
-- Updated the mesh-optimizer and pmesh-optimizer miniapps to demonstrate the
-  hr-adaptivity and interface fitting capability.
-
-- The HPC versions of ex1 and ex1p (in miniapps/performance) now support runtime
-  selection of either 2D or 3D meshes.
-
-Integrations, testing and documentation
----------------------------------------
-- Doxygen documentation for all releases is now available at docs.mfem.org.
-
-- The following integrations have updated minimum version requirements:
-  * HIOP >= 0.4.6
-  * HYPRE >= 2.23.0 for HIP support
-  * libCEED >= 0.10
-  * PUMI >= 2.2.6
-  * RAJA >= 0.14.0
-  * Umpire >= 3.0.0
-  see INSTALL for more details.
-
-- Added new optional integrations with ParELAG and CoDiPack (version >= 1.9.3+).
-
-- Added initial support for Google Benchmark (version >= 1.5.6) in the
-  tests/benchmarks directory. It can be enabled with MFEM_USE_BENCHMARK=YES.
-
-- Switched from Artistic Style (astyle) version 2.05.1 to version 3.1 for code
-  formatting. See the "make style" target.
-
-Miscellaneous
--------------
-- Added a simple singleton class, Mpi, as a replacement for MPI_Session. New
-  code should use Mpi::Init() and other Mpi methods instead of MPI_Session.
-
-- Added ParaView visualization of QuadratureFunction fields, through both
-  QuadratureFunction::SaveVTU and ParaViewDataCollection::RegisterQField.
-
 - Fixed several MinGW build issues on Windows.
 
-- Added 'double' atomicAdd implementation for previous versions of CUDA.
-
-- HypreParVector and Vector now support C++ move semantics, and the copy
-  constructor for HypreParVector now copies the local vector data.
-
-- Removed the 'u' flag in the ar command, to update all files in the archive,
+- Remove the 'u' flag in the ar command, to update all files in the archive,
   avoiding file name collisions from different subdirectories.
 
-- Various other simplifications, extensions, and bugfixes in the code.
+- Added initial TMOP-based capabilities for surface fitting and tangential
+  relaxation in the mesh-optimizer and pmesh-optimizer miniapps.
 
+- Added ParMesh Adjaceny Set (adjset) creation support to the Conduit Mesh
+  Blueprint MFEM wrapper functions in ConduitDataCollection.
+
+- Added full assembly / device support for the LinearForm integrators:
+  * DomainLF: L(v) := (f, v)
+  * VectorDomainLF: L(v) := ((f1,...,fn), (v1,...,vn))
+  * DomainLFGrad: L(v) := (f,grad v)
+  * VectorDomainLFGrad:  L(v) := ((f1x,f1y,f1z,...,fnx,fny,fnz),grad(v1,...,vn))
+  It can be enabled by setting the linear assembly level to
+  'LinearAssemblyLevel::FULL' with the 'LinearForm::SetAssemblyLevel' method.
+
+- `HypreParVector` and `Vector` now support move semantics, and the copy
+  constructor for `HypreParVector` now copies the local vector data.
+
+- The HPC versions of ex1 and ex1p (in miniapps/performance) now support
+  runtime selection of either 2D or 3D meshes.
 
 Version 4.3, released on July 29, 2021
 ======================================

CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -16,6 +16,9 @@ set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH
 
 # Require C++11 and disable compiler-specific extensions
 set(CMAKE_CXX_STANDARD 11)
+if (MFEM_USE_GINKGO)
+  set(CMAKE_CXX_STANDARD 14)
+endif()
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 
@@ -51,7 +54,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.4.0)
+set(${PROJECT_NAME}_VERSION 4.3.1)
 
 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -81,9 +84,6 @@ if (MFEM_USE_STRUMPACK)
   # Just needed to find the MPI_Fortran libraries to link with
   set(XSDK_ENABLE_Fortran ON)
 endif()
-if (MFEM_USE_GINKGO AND ("${CMAKE_CXX_STANDARD}" LESS "14"))
-  set(CMAKE_CXX_STANDARD 14)
-endif()
 
 # Include xSDK default CMake file.
 include("${CMAKE_CURRENT_SOURCE_DIR}/config/XSDKDefaults.cmake")
@@ -180,19 +180,6 @@ else()
   set(MFEM_DEBUG OFF)
 endif()
 
-# AMD HIP
-if (MFEM_USE_HIP)
-  if (HIP_ARCH)
-    message(STATUS "Using HIP architecture: ${HIP_ARCH}")
-    set(GPU_TARGETS "${HIP_ARCH}" CACHE STRING "HIP targets to compile for")
-  endif()
-  if (ROCM_PATH)
-    list(INSERT CMAKE_PREFIX_PATH 0 ${ROCM_PATH})
-  endif()
-  find_package(HIP REQUIRED)
-  find_package(HIPSPARSE REQUIRED)
-endif()
-
 # MPI -> hypre; PETSc (optional)
 if (MFEM_USE_MPI)
   find_package(MPI REQUIRED)
@@ -367,7 +354,7 @@ endif()
 if (MFEM_USE_PUMI)
   # If PUMI_DIR was specified, only link to that directory,
   # i.e. don't link to another installation in /usr/lib by mistake
-  find_package(SCOREC 2.2.6 REQUIRED OPTIONAL_COMPONENTS gmi_sim
+  find_package(SCOREC 2.1.0 REQUIRED OPTIONAL_COMPONENTS gmi_sim
     CONFIG PATHS ${PUMI_DIR} NO_DEFAULT_PATH)
   if (SCOREC_FOUND)
     # Define a header file with the MFEM_USE_SIMMETRIX preprocessor variable
@@ -416,6 +403,18 @@ if (MFEM_USE_CALIPER)
   find_package(Caliper REQUIRED)
 endif()
 
+# AMD HIP
+if (MFEM_USE_HIP)
+  find_package(HIP REQUIRED)
+  if (HIP_ARCH)
+    message(STATUS "Using HIP architecture: ${HIP_ARCH}")
+    list(APPEND HIP_HIPCC_FLAGS "--amdgpu-target=${HIP_ARCH}")
+    if (MFEM_USE_GINKGO)
+      list(APPEND HIP_HIPCC_FLAGS "-std=c++14")
+    endif()
+  endif()
+endif()
+
 # ADIOS2 for parallel I/O
 if (MFEM_USE_ADIOS2)
    find_package(ADIOS2 REQUIRED)
@@ -455,11 +454,10 @@ endif()
 #    With newer versions of SuiteSparse which include METIS header using 64-bit
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
-set(MFEM_TPLS OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS
-    PETSC SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
-    NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
-    ADIOS2 CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG
-    MPI_CXX HIP HIPSPARSE)
+set(MFEM_TPLS OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS PETSC
+    SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB NETCDF
+    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
+    CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG MPI_CXX)
 
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
@@ -502,6 +500,8 @@ endforeach()
 
 if (MFEM_USE_CUDA)
   set_source_files_properties(${SOURCES} PROPERTIES LANGUAGE CUDA)
+elseif(MFEM_USE_HIP)
+  set_source_files_properties(${SOURCES} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
 endif()
 
 add_subdirectory(config)

CONTRIBUTING.md

@@ -8,7 +8,7 @@
 <a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuild-analysis+branch%3Amaster"><img alt="Build Analysis" src="https://github.com/mfem/mfem/actions/workflows/mfem-analysis.yml/badge.svg?branch=master"></a>
 <a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuilds-and-tests+branch%3Amaster"><img alt="Builds and Tests" src="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml/badge.svg?branch=master"></a>
 <a href="https://ci.appveyor.com/project/mfem/mfem"><img alt="Build Status" src="https://ci.appveyor.com/api/projects/status/19non9sqm6msi2wy?svg=true"></a>
-<a href="https://docs.mfem.org/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
+<a href="https://mfem.github.io/doxygen/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
 </p>
 
 
@@ -157,24 +157,24 @@ classes implementing the finite element, mesh and linear algebra concepts
 respectively.
 
 - The main mesh classes are:
-  + [`Mesh`](https://docs.mfem.org/html/classmfem_1_1Mesh.html)
-  + [`NCMesh`](https://docs.mfem.org/html/classmfem_1_1NCMesh.html)
-  + [`Element`](https://docs.mfem.org/html/classmfem_1_1Element.html)
-  + [`ElementTransformation`](https://docs.mfem.org/html/classmfem_1_1ElementTransformation.html)
+  + [`Mesh`](https://mfem.github.io/doxygen/html/classmfem_1_1Mesh.html)
+  + [`NCMesh`](https://mfem.github.io/doxygen/html/classmfem_1_1NCMesh.html)
+  + [`Element`](https://mfem.github.io/doxygen/html/classmfem_1_1Element.html)
+  + [`ElementTransformation`](https://mfem.github.io/doxygen/html/classmfem_1_1ElementTransformation.html)
 
 - The main finite element classes are:
-  + [`FiniteElement`](https://docs.mfem.org/html/classmfem_1_1FiniteElement.html)
-  + [`FiniteElementCollection`](https://docs.mfem.org/html/classmfem_1_1FiniteElement.html)
-  + [`FiniteElementSpace`](https://docs.mfem.org/html/classmfem_1_1FiniteElementSpace.html)
-  + [`GridFunction`](https://docs.mfem.org/html/classmfem_1_1GridFunction.html)
-  + [`BilinearFormIntegrator`](https://docs.mfem.org/html/classmfem_1_1BilinearFormIntegrator.html) and [`LinearFormIntegrator`](https://docs.mfem.org/html/classmfem_1_1LinearFormIntegrator.html)
-  + [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearFormIntegrator.html), [`BilinearForm`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html) and [`MixedBilinearForm`](https://docs.mfem.org/html/classmfem_1_1MixedBilinearForm.html)
+  + [`FiniteElement`](https://mfem.github.io/doxygen/html/classmfem_1_1FiniteElement.html)
+  + [`FiniteElementCollection`](https://mfem.github.io/doxygen/html/classmfem_1_1FiniteElement.html)
+  + [`FiniteElementSpace`](https://mfem.github.io/doxygen/html/classmfem_1_1FiniteElementSpace.html)
+  + [`GridFunction`](https://mfem.github.io/doxygen/html/classmfem_1_1GridFunction.html)
+  + [`BilinearFormIntegrator`](https://mfem.github.io/doxygen/html/classmfem_1_1BilinearFormIntegrator.html) and [`LinearFormIntegrator`](https://mfem.github.io/doxygen/html/classmfem_1_1LinearFormIntegrator.html)
+  + [`LinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1LinearFormIntegrator.html), [`BilinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html) and [`MixedBilinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1MixedBilinearForm.html)
 
 - The main linear algebra classes and sources are
-  + [`Operator`](https://docs.mfem.org/html/classmfem_1_1Operator.html) and [`BilinearForm`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html)
-  + [`Vector`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html) and [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearForm.html)
-  + [`DenseMatrix`](https://docs.mfem.org/html/classmfem_1_1DenseMatrix.html) and [`SparseMatrix`](https://docs.mfem.org/html/classmfem_1_1SparseMatrix.html)
-  + Sparse [smoothers](https://docs.mfem.org/html/sparsesmoothers_8hpp.html) and linear [solvers](https://docs.mfem.org/html/solvers_8hpp.html)
+  + [`Operator`](https://mfem.github.io/doxygen/html/classmfem_1_1Operator.html) and [`BilinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html)
+  + [`Vector`](https://mfem.github.io/doxygen/html/classmfem_1_1BilinearForm.html) and [`LinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1LinearForm.html)
+  + [`DenseMatrix`](https://mfem.github.io/doxygen/html/classmfem_1_1DenseMatrix.html) and [`SparseMatrix`](https://mfem.github.io/doxygen/html/classmfem_1_1SparseMatrix.html)
+  + Sparse [smoothers](https://mfem.github.io/doxygen/html/sparsesmoothers_8hpp.html) and linear [solvers](https://mfem.github.io/doxygen/html/solvers_8hpp.html)
 
 #### Parallel implementation
 
@@ -184,13 +184,13 @@ shared geometric entities between different tasks. The parallel source files
 have a `p` prefix, e.g. `pmesh.cpp` vs. the serial `mesh.cpp`.
 
 - The main parallel classes are
-  + [`ParMesh`](https://docs.mfem.org/html/solvers_8hpp.html)
-  + [`ParNCMesh`](https://docs.mfem.org/html/classmfem_1_1ParMesh.html)
-  + [`ParFiniteElementSpace`](https://docs.mfem.org/html/classmfem_1_1ParFiniteElementSpace.html)
-  + [`ParGridFunction`](https://docs.mfem.org/html/classmfem_1_1ParGridFunction.html)
-  + [`ParBilinearForm`](https://docs.mfem.org/html/classmfem_1_1ParBilinearForm.html) and [`ParLinearForm`](https://docs.mfem.org/html/classmfem_1_1ParLinearForm.html)
-  + [`HypreParMatrix`](https://docs.mfem.org/html/classmfem_1_1HypreParMatrix.html) and [`HypreParVector`](https://docs.mfem.org/html/classmfem_1_1HypreParVector.html)
-  + [`HypreSolver`](https://docs.mfem.org/html/classmfem_1_1HypreSolver.html) and other [hypre classes](https://docs.mfem.org/html/hypre_8hpp.html)
+  + [`ParMesh`](https://mfem.github.io/doxygen/html/solvers_8hpp.html)
+  + [`ParNCMesh`](https://mfem.github.io/doxygen/html/classmfem_1_1ParMesh.html)
+  + [`ParFiniteElementSpace`](https://mfem.github.io/doxygen/html/classmfem_1_1ParFiniteElementSpace.html)
+  + [`ParGridFunction`](https://mfem.github.io/doxygen/html/classmfem_1_1ParGridFunction.html)
+  + [`ParBilinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1ParBilinearForm.html) and [`ParLinearForm`](https://mfem.github.io/doxygen/html/classmfem_1_1ParLinearForm.html)
+  + [`HypreParMatrix`](https://mfem.github.io/doxygen/html/classmfem_1_1HypreParMatrix.html) and [`HypreParVector`](https://mfem.github.io/doxygen/html/classmfem_1_1HypreParVector.html)
+  + [`HypreSolver`](https://mfem.github.io/doxygen/html/classmfem_1_1HypreSolver.html) and other [hypre classes](https://mfem.github.io/doxygen/html/hypre_8hpp.html)
 
 #### GPU and general device support
 
@@ -199,10 +199,10 @@ backends (CUDA, OCCA, RAJA, OpenMP, etc.) and an internal lightweight
 device/host memory manager.
 
 - The main device-relevant classes and sources are:
-  + [`Device`](https://docs.mfem.org/html/device_8hpp.html)
-  + [`MemoryManager`](https://docs.mfem.org/html/mem_manager_8hpp.html)
-  + the [`MFEM_FORALL`](https://docs.mfem.org/html/forall_8hpp.html) macro
-  + the [`cuda.hpp`](https://docs.mfem.org/html/cuda_8hpp.html) and [`occa.hpp`](https://docs.mfem.org/html/occa_8hpp.html) files
+  + [`Device`](https://mfem.github.io/doxygen/html/device_8hpp.html)
+  + [`MemoryManager`](https://mfem.github.io/doxygen/html/mem_manager_8hpp.html)
+  + the [`MFEM_FORALL`](https://mfem.github.io/doxygen/html/forall_8hpp.html) macro
+  + the [`cuda.hpp`](https://mfem.github.io/doxygen/html/cuda_8hpp.html) and [`occa.hpp`](https://mfem.github.io/doxygen/html/occa_8hpp.html) files
 
 #### Utilities, building and documentation
 - The `general/` directory contains C++ classes that serve as utilities for
@@ -210,7 +210,8 @@ device/host memory manager.
 - The `config/` directory contains build-related files, both for the plain
   Makefile and the CMake build options.
 - The `doc/` directory contains configuration for the Doxygen code documentation
-  that can either be built locally or browsed online at https://docs.mfem.org.
+  that can either be built locally or browsed online at
+  https://mfem.github.io/doxygen/html/index.html.
 
 #### Examples and tests
 - `examples` and `miniapps` respectively gather simple and more fully-featured
@@ -430,10 +431,10 @@ The Pull Request (PR) approval process in MFEM is similar to the approval of pap
 
 The current list of MFEM editors is:
 
-- [@v-dobrev](https://github.com/v-dobrev) (Veselin Dobrev)
-- [@tzanio](https://github.com/tzanio) (Tzanio Kolev)
-- [@pazner](https://github.com/pazner) (Will Pazner)
-- [@mlstowell](https://github.com/mlstowell) (Mark Stowell)
+- @v-dobrev (Veselin Dobrev)
+- @tzanio (Tzanio Kolev)
+- @pazner (Will Pazner)
+- @mlstowell (Mark Stowell)
 
 **The responsibilities of the editors are:**
 
@@ -639,7 +640,7 @@ MFEM uses a `master`/`next`-branch workflow as described below:
   ```
 - [ ] Create the release tarball and push to `mfem/releases`.
 - [ ] Recreate the `next` branch as described in previous section.
-- [ ] Update and push documentation to `mfem/doxygen`. Update the `README.md` file and the `html` link in the `mfem/doxygen` repo.
+- [ ] Update and push documentation  to `mfem/doxygen`.
 - [ ] Update URL shortlinks:
     - [ ] Create a shortlink at [http://bit.ly/](http://bit.ly/) for the release tarball, e.g. https://mfem.github.io/releases/mfem-3.1.tgz.
     - [ ] (LLNL only) Add and commit the new shortlink in the `links` and `links-mfem` files of the internal `mfem/downloads` repo.

INSTALL

@@ -19,10 +19,10 @@ requires an MPI C++ compiler, as well as the following external libraries:
   http://glaros.dtc.umn.edu/gkhome/metis/metis/overview
 
 The hypre dependency can be downloaded as a tarball from GitHub or from the
-project webpage https://www.llnl.gov/casc/hypre. For example, the 2.24.0 release
+project webpage https://www.llnl.gov/casc/hypre. For example, the 2.20.0 release
 of hypre is available at
 
-   https://github.com/hypre-space/hypre/archive/v2.24.0.tar.gz
+   https://github.com/hypre-space/hypre/archive/v2.20.0.tar.gz
 
 The METIS dependency can be disabled but that is not generally recommended, see
 the option MFEM_USE_METIS.
@@ -37,7 +37,7 @@ as CUDA, HIP, OCCA, OpenMP and RAJA.
   https://developer.nvidia.com/cuda-toolkit
 
 - HIP support requires an AMD GPU and an installation of the ROCm software stack
-  https://rocmdocs.amd.com
+  https://rocm.github.io/ROCmInstall.html#installing-from-amd-rocm-repositories
 
 - OCCA support requires the OCCA library
   https://libocca.org
@@ -58,8 +58,8 @@ following package managers:
 
 - Spack, https://github.com/spack/spack
 - OpenHPC, http://openhpc.community
-- Conda-forge, https://conda-forge.org (pre-built binaries linked with
-  OpenMPI/MPICH, hypre, and METIS)
+- Conda-forge, https://conda-forge.org (pre-built binaries linked with OpenMPI/MPICH, hypre, and METIS)
+- Homebrew/Science, https://github.com/Homebrew/homebrew-science (deprecated)
 
 We also recommend downloading and building the MFEM-based GLVis visualization
 tool which can be used to visualize the meshes and solution in MFEM's examples
@@ -78,7 +78,7 @@ Parallel build:
 
 CUDA build:
    make cuda -j 4
-   (build for a specific compute capability: 'make cuda -j 4 CUDA_ARCH=sm_70')
+   (build for a specific compute capability: 'make cuda -j 4 CUDA_ARCH=sm_30')
 
 HIP build:
    make hip -j 4
@@ -567,7 +567,6 @@ The specific libraries and their options are:
   Versions: HYPRE >= 2.10.0b  (HYPRE built without CUDA)
             HYPRE >= 2.20.0   (HYPRE built with '--enable-mixedint')
             HYPRE >= 2.22.1   (HYPRE built with CUDA)
-            HYPRE >= 2.23.0   (HYPRE built with HIP)
 
 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
   MFEM_USE_METIS_5 = YES (default is to use METIS 4).
@@ -640,8 +639,7 @@ The specific libraries and their options are:
   C++ compiler that supports the C++-14 standard. For additional requirements
   and dependencies of specific modules, see the Ginkgo webpage below.
   URL: https://ginkgo-project.github.io
-  Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or
-           Debug).
+  Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or Debug).
   Versions: Ginkgo >= 1.4.0.
 
 - AmgX (optional), used when MFEM_USE_AMGX = YES.
@@ -706,14 +704,14 @@ The specific libraries and their options are:
   URL: https://scorec.rpi.edu/pumi
        https://github.com/SCOREC/core
   Options: PUMI_OPT, PUMI_LIB.
-  Versions: PUMI >= 2.2.6.
+  Versions: PUMI == 2.2.3.
 
 - HiOp (optional), used when MFEM_USE_HIOP = YES.
   URL: https://github.com/LLNL/hiop
   Options: HIOP_OPT, HIOP_LIB.
   Versions: HIOP >= 0.4.6.
 
-- CoDiPack (optional), used with MFEM_USE_CODIPACK = YES
+- CoDiPack (optiobal), used with MFEM_USE_CODIPACK = YES
   URL: https://www.scicomp.uni-kl.de/codi/
   Options: CODIPACK_OPT
   Versions: 1.9.3
@@ -739,7 +737,7 @@ The specific libraries and their options are:
   Versions: CUDA >= 10.1.168.
 
 - HIP (optional), used when MFEM_USE_HIP = YES.
-  URL: https://rocmdocs.amd.com
+  URL: https://rocm.github.io/ROCmInstall.html
   Options: HIP_CXX, HIP_ARCH, HIP_OPT, HIP_LIB.
 
 - OCCA (optional), used when MFEM_USE_OCCA = YES.
@@ -751,7 +749,7 @@ The specific libraries and their options are:
   URL: https://github.com/CEED/libCEED
        https://ceed.exascaleproject.org/libceed
   Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED >= 0.10.
+  Versions: libCEED >= 0.8.
 
 - RAJA (optional), used when MFEM_USE_RAJA = YES.
   Beginning with MFEM v4.3, only RAJA v0.14.0+ is supported.
@@ -926,7 +924,6 @@ MFEM_USE_HIOP
 MFEM_USE_CODIPACK
 MFEM_USE_ADFORWARD
 MFEM_USE_CUDA
-MFEM_USE_HIP
 MFEM_USE_OCCA
 MFEM_USE_CEED
 MFEM_USE_RAJA

LICENSE

@@ -1,6 +1,6 @@
 BSD 3-Clause License
 
-Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC
+Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

config/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/MFEMConfig.cmake.in

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/config.hpp.in

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //

config/cmake/modules/FindAMGX.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindAxom.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindBenchmark.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindCODIPACK.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindCXXABIDemangle.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindCaliper.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindConduit.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindFMS.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindGSLIB.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindHIOP.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindHIP.cmake

@@ -0,0 +1,692 @@
+###############################################################################
+# FindHIP.cmake
+###############################################################################
+include(CheckCXXCompilerFlag)
+###############################################################################
+# SET: Variable defaults
+###############################################################################
+# User defined flags
+set(HIP_HIPCC_FLAGS "" CACHE STRING "Semicolon delimited flags for HIPCC")
+set(HIP_HCC_FLAGS "" CACHE STRING "Semicolon delimited flags for HCC")
+set(HIP_CLANG_FLAGS "" CACHE STRING "Semicolon delimited flags for CLANG")
+set(HIP_NVCC_FLAGS "" CACHE STRING "Semicolon delimted flags for NVCC")
+mark_as_advanced(HIP_HIPCC_FLAGS HIP_HCC_FLAGS HIP_CLANG_FLAGS HIP_NVCC_FLAGS)
+
+set(_hip_configuration_types ${CMAKE_CONFIGURATION_TYPES} ${CMAKE_BUILD_TYPE} Debug MinSizeRel Release RelWithDebInfo)
+list(REMOVE_DUPLICATES _hip_configuration_types)
+foreach(config ${_hip_configuration_types})
+    string(TOUPPER ${config} config_upper)
+    set(HIP_HIPCC_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for HIPCC")
+    set(HIP_HCC_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for HCC")
+    set(HIP_CLANG_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for CLANG")
+    set(HIP_NVCC_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for NVCC")
+    mark_as_advanced(HIP_HIPCC_FLAGS_${config_upper} HIP_HCC_FLAGS_${config_upper} HIP_CLANG_FLAGS_${config_upper} HIP_NVCC_FLAGS_${config_upper})
+endforeach()
+option(HIP_HOST_COMPILATION_CPP "Host code compilation mode" ON)
+option(HIP_VERBOSE_BUILD "Print out the commands run while compiling the HIP source file.  With the Makefile generator this defaults to VERBOSE variable specified on the command line, but can be forced on with this option." OFF)
+mark_as_advanced(HIP_HOST_COMPILATION_CPP)
+
+###############################################################################
+# FIND: HIP and associated helper binaries
+###############################################################################
+
+get_filename_component(_IMPORT_PREFIX "${CMAKE_CURRENT_LIST_DIR}/../" REALPATH)
+
+# HIP is supported on Linux only
+if(UNIX AND NOT APPLE AND NOT CYGWIN)
+    # Search for HIP installation
+    if(NOT HIP_ROOT_DIR)
+        # Search in user specified path first
+        find_path(
+            HIP_ROOT_DIR
+            NAMES bin/hipconfig
+            PATHS
+            "$ENV{ROCM_PATH}/hip"
+            ENV HIP_PATH
+            ${_IMPORT_PREFIX}
+            /opt/rocm/hip
+            DOC "HIP installed location"
+            NO_DEFAULT_PATH
+            )
+        if(NOT EXISTS ${HIP_ROOT_DIR})
+            if(HIP_FIND_REQUIRED)
+                message(FATAL_ERROR "Specify HIP_ROOT_DIR")
+            elseif(NOT HIP_FIND_QUIETLY)
+                message("HIP_ROOT_DIR not found or specified")
+            endif()
+        endif()
+        # And push it back to the cache
+        set(HIP_ROOT_DIR ${HIP_ROOT_DIR} CACHE PATH "HIP installed location" FORCE)
+    endif()
+
+    # Find HIPCC executable
+    find_program(
+        HIP_HIPCC_EXECUTABLE
+        NAMES hipcc
+        PATHS
+        "${HIP_ROOT_DIR}"
+        ENV ROCM_PATH
+        ENV HIP_PATH
+        /opt/rocm
+        /opt/rocm/hip
+        PATH_SUFFIXES bin
+        NO_DEFAULT_PATH
+        )
+    if(NOT HIP_HIPCC_EXECUTABLE)
+        # Now search in default paths
+        find_program(HIP_HIPCC_EXECUTABLE hipcc)
+    endif()
+    mark_as_advanced(HIP_HIPCC_EXECUTABLE)
+
+    # Find HIPCONFIG executable
+    find_program(
+        HIP_HIPCONFIG_EXECUTABLE
+        NAMES hipconfig
+        PATHS
+        "${HIP_ROOT_DIR}"
+        ENV ROCM_PATH
+        ENV HIP_PATH
+        /opt/rocm
+        /opt/rocm/hip
+        PATH_SUFFIXES bin
+        NO_DEFAULT_PATH
+        )
+    if(NOT HIP_HIPCONFIG_EXECUTABLE)
+        # Now search in default paths
+        find_program(HIP_HIPCONFIG_EXECUTABLE hipconfig)
+    endif()
+    mark_as_advanced(HIP_HIPCONFIG_EXECUTABLE)
+
+    # Find HIPCC_CMAKE_LINKER_HELPER executable
+    find_program(
+        HIP_HIPCC_CMAKE_LINKER_HELPER
+        NAMES hipcc_cmake_linker_helper
+        PATHS
+        "${HIP_ROOT_DIR}"
+        ENV ROCM_PATH
+        ENV HIP_PATH
+        /opt/rocm
+        /opt/rocm/hip
+        PATH_SUFFIXES bin
+        NO_DEFAULT_PATH
+        )
+    if(NOT HIP_HIPCC_CMAKE_LINKER_HELPER)
+        # Now search in default paths
+        find_program(HIP_HIPCC_CMAKE_LINKER_HELPER hipcc_cmake_linker_helper)
+    endif()
+    mark_as_advanced(HIP_HIPCC_CMAKE_LINKER_HELPER)
+
+    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_VERSION)
+        # Compute the version
+        execute_process(
+            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --version
+            OUTPUT_VARIABLE _hip_version
+            ERROR_VARIABLE _hip_error
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+            ERROR_STRIP_TRAILING_WHITESPACE
+            )
+        if(NOT _hip_error)
+            set(HIP_VERSION ${_hip_version} CACHE STRING "Version of HIP as computed from hipcc")
+        else()
+            set(HIP_VERSION "0.0.0" CACHE STRING "Version of HIP as computed by FindHIP()")
+        endif()
+        mark_as_advanced(HIP_VERSION)
+    endif()
+    if(HIP_VERSION)
+        string(REPLACE "." ";" _hip_version_list "${HIP_VERSION}")
+        list(GET _hip_version_list 0 HIP_VERSION_MAJOR)
+        list(GET _hip_version_list 1 HIP_VERSION_MINOR)
+        list(GET _hip_version_list 2 HIP_VERSION_PATCH)
+        set(HIP_VERSION_STRING "${HIP_VERSION}")
+    endif()
+
+    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_PLATFORM)
+        # Compute the platform
+        execute_process(
+            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --platform
+            OUTPUT_VARIABLE _hip_platform
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+            )
+        set(HIP_PLATFORM ${_hip_platform} CACHE STRING "HIP platform as computed by hipconfig")
+        mark_as_advanced(HIP_PLATFORM)
+    endif()
+
+    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_COMPILER)
+        # Compute the compiler
+        execute_process(
+            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --compiler
+            OUTPUT_VARIABLE _hip_compiler
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+            )
+        set(HIP_COMPILER ${_hip_compiler} CACHE STRING "HIP compiler as computed by hipconfig")
+        mark_as_advanced(HIP_COMPILER)
+    endif()
+
+    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_RUNTIME)
+        # Compute the runtime
+        execute_process(
+            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --runtime
+            OUTPUT_VARIABLE _hip_runtime
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+            )
+        set(HIP_RUNTIME ${_hip_runtime} CACHE STRING "HIP runtime as computed by hipconfig")
+        mark_as_advanced(HIP_RUNTIME)
+    endif()
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(
+    HIP
+    REQUIRED_VARS
+    HIP_ROOT_DIR
+    HIP_HIPCC_EXECUTABLE
+    HIP_HIPCONFIG_EXECUTABLE
+    HIP_PLATFORM
+    HIP_COMPILER
+    HIP_RUNTIME
+    VERSION_VAR HIP_VERSION
+    )
+
+###############################################################################
+# Set HIP CMAKE Flags
+###############################################################################
+# Copy the invocation styles from CXX to HIP
+set(CMAKE_HIP_ARCHIVE_CREATE ${CMAKE_CXX_ARCHIVE_CREATE})
+set(CMAKE_HIP_ARCHIVE_APPEND ${CMAKE_CXX_ARCHIVE_APPEND})
+set(CMAKE_HIP_ARCHIVE_FINISH ${CMAKE_CXX_ARCHIVE_FINISH})
+set(CMAKE_SHARED_LIBRARY_SONAME_HIP_FLAG ${CMAKE_SHARED_LIBRARY_SONAME_CXX_FLAG})
+set(CMAKE_SHARED_LIBRARY_CREATE_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS})
+set(CMAKE_SHARED_LIBRARY_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_CXX_FLAGS})
+#set(CMAKE_SHARED_LIBRARY_LINK_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS})
+set(CMAKE_SHARED_LIBRARY_RUNTIME_HIP_FLAG ${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG})
+set(CMAKE_SHARED_LIBRARY_RUNTIME_HIP_FLAG_SEP ${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG_SEP})
+set(CMAKE_SHARED_LIBRARY_LINK_STATIC_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_STATIC_CXX_FLAGS})
+set(CMAKE_SHARED_LIBRARY_LINK_DYNAMIC_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_DYNAMIC_CXX_FLAGS})
+
+set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "")
+set(HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS "")
+
+if("${HIP_COMPILER}" STREQUAL "nvcc")
+    # Set the CMake Flags to use the nvcc Compiler.
+    set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
+    set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <LINK_LIBRARIES> -shared" )
+    set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
+elseif("${HIP_COMPILER}" STREQUAL "hcc")
+    # Set the CMake Flags to use the hcc Compiler.
+    set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
+    set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <LINK_LIBRARIES> -shared" )
+    set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
+elseif("${HIP_COMPILER}" STREQUAL "clang")
+    #Number of parallel jobs by default is 1
+    if(NOT DEFINED HIP_CLANG_NUM_PARALLEL_JOBS)
+      set(HIP_CLANG_NUM_PARALLEL_JOBS 1)
+    endif()
+    #Add support for parallel build and link
+    if(${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
+      check_cxx_compiler_flag("-parallel-jobs=1" HIP_CLANG_SUPPORTS_PARALLEL_JOBS)
+    endif()
+    if(HIP_CLANG_NUM_PARALLEL_JOBS GREATER 1)
+      if(${HIP_CLANG_SUPPORTS_PARALLEL_JOBS})
+        set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "-Wno-format-nonliteral -parallel-jobs=${HIP_CLANG_NUM_PARALLEL_JOBS}")
+        set(HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS "-parallel-jobs=${HIP_CLANG_NUM_PARALLEL_JOBS}")
+      else()
+        message("clang compiler doesn't support parallel jobs")
+      endif()
+    endif()
+
+    # Set the CMake Flags to use the HIP-Clang Compiler.
+    set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
+    set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <LINK_LIBRARIES> -shared" )
+    set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
+
+    if("${HIP_RUNTIME}" STREQUAL "rocclr")
+      if(TARGET host)
+        message(STATUS "host interface - found")
+        set(HIP_HOST_INTERFACE host)
+      endif()
+    endif()
+endif()
+
+###############################################################################
+# MACRO: Locate helper files
+###############################################################################
+macro(HIP_FIND_HELPER_FILE _name _extension)
+    set(_hip_full_name "${_name}.${_extension}")
+    get_filename_component(CMAKE_CURRENT_LIST_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH)
+    set(HIP_${_name} "${CMAKE_CURRENT_LIST_DIR}/FindHIP/${_hip_full_name}")
+    if(NOT EXISTS "${HIP_${_name}}")
+        set(error_message "${_hip_full_name} not found in ${CMAKE_CURRENT_LIST_DIR}/FindHIP")
+        if(HIP_FIND_REQUIRED)
+            message(FATAL_ERROR "${error_message}")
+        else()
+            if(NOT HIP_FIND_QUIETLY)
+                message(STATUS "${error_message}")
+            endif()
+        endif()
+    endif()
+    # Set this variable as internal, so the user isn't bugged with it.
+    set(HIP_${_name} ${HIP_${_name}} CACHE INTERNAL "Location of ${_full_name}" FORCE)
+endmacro()
+
+###############################################################################
+hip_find_helper_file(run_make2cmake cmake)
+hip_find_helper_file(run_hipcc cmake)
+###############################################################################
+
+###############################################################################
+# MACRO: Reset compiler flags
+###############################################################################
+macro(HIP_RESET_FLAGS)
+    unset(HIP_HIPCC_FLAGS)
+    unset(HIP_HCC_FLAGS)
+    unset(HIP_CLANG_FLAGS)
+    unset(HIP_NVCC_FLAGS)
+    foreach(config ${_hip_configuration_types})
+        string(TOUPPER ${config} config_upper)
+        unset(HIP_HIPCC_FLAGS_${config_upper})
+        unset(HIP_HCC_FLAGS_${config_upper})
+        unset(HIP_CLANG_FLAGS_${config_upper})
+        unset(HIP_NVCC_FLAGS_${config_upper})
+    endforeach()
+endmacro()
+
+###############################################################################
+# MACRO: Separate the options from the sources
+###############################################################################
+macro(HIP_GET_SOURCES_AND_OPTIONS _sources _cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options)
+    set(${_sources})
+    set(${_cmake_options})
+    set(${_hipcc_options})
+    set(${_hcc_options})
+    set(${_clang_options})
+    set(${_nvcc_options})
+    set(_hipcc_found_options FALSE)
+    set(_hcc_found_options FALSE)
+    set(_clang_found_options FALSE)
+    set(_nvcc_found_options FALSE)
+    foreach(arg ${ARGN})
+        if("x${arg}" STREQUAL "xHIPCC_OPTIONS")
+            set(_hipcc_found_options TRUE)
+            set(_hcc_found_options FALSE)
+            set(_clang_found_options FALSE)
+            set(_nvcc_found_options FALSE)
+        elseif("x${arg}" STREQUAL "xHCC_OPTIONS")
+            set(_hipcc_found_options FALSE)
+            set(_hcc_found_options TRUE)
+            set(_clang_found_options FALSE)
+            set(_nvcc_found_options FALSE)
+        elseif("x${arg}" STREQUAL "xCLANG_OPTIONS")
+            set(_hipcc_found_options FALSE)
+            set(_hcc_found_options FALSE)
+            set(_clang_found_options TRUE)
+            set(_nvcc_found_options FALSE)
+        elseif("x${arg}" STREQUAL "xNVCC_OPTIONS")
+            set(_hipcc_found_options FALSE)
+            set(_hcc_found_options FALSE)
+            set(_clang_found_options FALSE)
+            set(_nvcc_found_options TRUE)
+        elseif(
+                "x${arg}" STREQUAL "xEXCLUDE_FROM_ALL" OR
+                "x${arg}" STREQUAL "xSTATIC" OR
+                "x${arg}" STREQUAL "xSHARED" OR
+                "x${arg}" STREQUAL "xMODULE"
+                )
+            list(APPEND ${_cmake_options} ${arg})
+        else()
+            if(_hipcc_found_options)
+                list(APPEND ${_hipcc_options} ${arg})
+            elseif(_hcc_found_options)
+                list(APPEND ${_hcc_options} ${arg})
+            elseif(_clang_found_options)
+                list(APPEND ${_clang_options} ${arg})
+            elseif(_nvcc_found_options)
+                list(APPEND ${_nvcc_options} ${arg})
+            else()
+                # Assume this is a file
+                list(APPEND ${_sources} ${arg})
+            endif()
+        endif()
+    endforeach()
+endmacro()
+
+###############################################################################
+# MACRO: Add include directories to pass to the hipcc command
+###############################################################################
+set(HIP_HIPCC_INCLUDE_ARGS_USER "")
+macro(HIP_INCLUDE_DIRECTORIES)
+    foreach(dir ${ARGN})
+        list(APPEND HIP_HIPCC_INCLUDE_ARGS_USER $<$<BOOL:${dir}>:-I${dir}>)
+    endforeach()
+endmacro()
+
+###############################################################################
+# FUNCTION: Helper to avoid clashes of files with the same basename but different paths
+###############################################################################
+function(HIP_COMPUTE_BUILD_PATH path build_path)
+    # Convert to cmake style paths
+    file(TO_CMAKE_PATH "${path}" bpath)
+    if(IS_ABSOLUTE "${bpath}")
+        string(FIND "${bpath}" "${CMAKE_CURRENT_BINARY_DIR}" _binary_dir_pos)
+        if(_binary_dir_pos EQUAL 0)
+            file(RELATIVE_PATH bpath "${CMAKE_CURRENT_BINARY_DIR}" "${bpath}")
+        else()
+            file(RELATIVE_PATH bpath "${CMAKE_CURRENT_SOURCE_DIR}" "${bpath}")
+        endif()
+    endif()
+
+    # Remove leading /
+    string(REGEX REPLACE "^[/]+" "" bpath "${bpath}")
+    # Avoid absolute paths by removing ':'
+    string(REPLACE ":" "_" bpath "${bpath}")
+    # Avoid relative paths that go up the tree
+    string(REPLACE "../" "__/" bpath "${bpath}")
+    # Avoid spaces
+    string(REPLACE " " "_" bpath "${bpath}")
+    # Strip off the filename
+    get_filename_component(bpath "${bpath}" PATH)
+
+    set(${build_path} "${bpath}" PARENT_SCOPE)
+endfunction()
+
+###############################################################################
+# MACRO: Parse OPTIONS from ARGN & set variables prefixed by _option_prefix
+###############################################################################
+macro(HIP_PARSE_HIPCC_OPTIONS _option_prefix)
+    set(_hip_found_config)
+    foreach(arg ${ARGN})
+        # Determine if we are dealing with a per-configuration flag
+        foreach(config ${_hip_configuration_types})
+            string(TOUPPER ${config} config_upper)
+            if(arg STREQUAL "${config_upper}")
+                set(_hip_found_config _${arg})
+                # Clear arg to prevent it from being processed anymore
+                set(arg)
+            endif()
+        endforeach()
+        if(arg)
+            list(APPEND ${_option_prefix}${_hip_found_config} "${arg}")
+        endif()
+    endforeach()
+endmacro()
+
+###############################################################################
+# MACRO: Try and include dependency file if it exists
+###############################################################################
+macro(HIP_INCLUDE_HIPCC_DEPENDENCIES dependency_file)
+    set(HIP_HIPCC_DEPEND)
+    set(HIP_HIPCC_DEPEND_REGENERATE FALSE)
+
+    # Create the dependency file if it doesn't exist
+    if(NOT EXISTS ${dependency_file})
+        file(WRITE ${dependency_file} "# Generated by: FindHIP.cmake. Do not edit.\n")
+    endif()
+    # Include the dependency file
+    include(${dependency_file})
+
+    # Verify the existence of all the included files
+    if(HIP_HIPCC_DEPEND)
+        foreach(f ${HIP_HIPCC_DEPEND})
+            if(NOT EXISTS ${f})
+                # If they aren't there, regenerate the file again
+                set(HIP_HIPCC_DEPEND_REGENERATE TRUE)
+            endif()
+        endforeach()
+    else()
+        # No dependencies, so regenerate the file
+        set(HIP_HIPCC_DEPEND_REGENERATE TRUE)
+    endif()
+
+    # Regenerate the dependency file if needed
+    if(HIP_HIPCC_DEPEND_REGENERATE)
+        set(HIP_HIPCC_DEPEND ${dependency_file})
+        file(WRITE ${dependency_file} "# Generated by: FindHIP.cmake. Do not edit.\n")
+    endif()
+endmacro()
+
+###############################################################################
+# MACRO: Prepare cmake commands for the target
+###############################################################################
+macro(HIP_PREPARE_TARGET_COMMANDS _target _format _generated_files _source_files)
+    set(_hip_flags "")
+    string(TOUPPER "${CMAKE_BUILD_TYPE}" _hip_build_configuration)
+    if(HIP_HOST_COMPILATION_CPP)
+        set(HIP_C_OR_CXX CXX)
+    else()
+        set(HIP_C_OR_CXX C)
+    endif()
+    set(generated_extension ${CMAKE_${HIP_C_OR_CXX}_OUTPUT_EXTENSION})
+
+    # Initialize list of includes with those specified by the user. Append with
+    # ones specified to cmake directly.
+    set(HIP_HIPCC_INCLUDE_ARGS ${HIP_HIPCC_INCLUDE_ARGS_USER})
+
+    # Add the include directories
+    set(include_directories_generator "$<TARGET_PROPERTY:${_target},INCLUDE_DIRECTORIES>")
+    list(APPEND HIP_HIPCC_INCLUDE_ARGS "$<$<BOOL:${include_directories_generator}>:-I$<JOIN:${include_directories_generator}, -I>>")
+
+    get_directory_property(_hip_include_directories INCLUDE_DIRECTORIES)
+    list(REMOVE_DUPLICATES _hip_include_directories)
+    if(_hip_include_directories)
+        foreach(dir ${_hip_include_directories})
+            list(APPEND HIP_HIPCC_INCLUDE_ARGS $<$<BOOL:${dir}>:-I${dir}>)
+        endforeach()
+    endif()
+
+    HIP_GET_SOURCES_AND_OPTIONS(_hip_sources _hip_cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options ${ARGN})
+    HIP_PARSE_HIPCC_OPTIONS(HIP_HIPCC_FLAGS ${_hipcc_options})
+    HIP_PARSE_HIPCC_OPTIONS(HIP_HCC_FLAGS ${_hcc_options})
+    HIP_PARSE_HIPCC_OPTIONS(HIP_CLANG_FLAGS ${_clang_options})
+    HIP_PARSE_HIPCC_OPTIONS(HIP_NVCC_FLAGS ${_nvcc_options})
+
+    # Add the compile definitions
+    set(compile_definition_generator "$<TARGET_PROPERTY:${_target},COMPILE_DEFINITIONS>")
+    list(APPEND HIP_HIPCC_FLAGS "$<$<BOOL:${compile_definition_generator}>:-D$<JOIN:${compile_definition_generator}, -D>>")
+
+    # Check if we are building shared library.
+    set(_hip_build_shared_libs FALSE)
+    list(FIND _hip_cmake_options SHARED _hip_found_SHARED)
+    list(FIND _hip_cmake_options MODULE _hip_found_MODULE)
+    if(_hip_found_SHARED GREATER -1 OR _hip_found_MODULE GREATER -1)
+        set(_hip_build_shared_libs TRUE)
+    endif()
+    list(FIND _hip_cmake_options STATIC _hip_found_STATIC)
+    if(_hip_found_STATIC GREATER -1)
+        set(_hip_build_shared_libs FALSE)
+    endif()
+
+    # If we are building a shared library, add extra flags to HIP_HIPCC_FLAGS
+    if(_hip_build_shared_libs)
+        list(APPEND HIP_HCC_FLAGS "-fPIC")
+        list(APPEND HIP_CLANG_FLAGS "-fPIC")
+        list(APPEND HIP_NVCC_FLAGS "--shared -Xcompiler '-fPIC'")
+    endif()
+
+    # Set host compiler
+    set(HIP_HOST_COMPILER "${CMAKE_${HIP_C_OR_CXX}_COMPILER}")
+
+    # Set compiler flags
+    set(_HIP_HOST_FLAGS "set(CMAKE_HOST_FLAGS ${CMAKE_${HIP_C_OR_CXX}_FLAGS})")
+    set(_HIP_HIPCC_FLAGS "set(HIP_HIPCC_FLAGS ${HIP_HIPCC_FLAGS})")
+    set(_HIP_HCC_FLAGS "set(HIP_HCC_FLAGS ${HIP_HCC_FLAGS})")
+    set(_HIP_CLANG_FLAGS "set(HIP_CLANG_FLAGS ${HIP_CLANG_FLAGS})")
+    set(_HIP_NVCC_FLAGS "set(HIP_NVCC_FLAGS ${HIP_NVCC_FLAGS})")
+    foreach(config ${_hip_configuration_types})
+        string(TOUPPER ${config} config_upper)
+        set(_HIP_HOST_FLAGS "${_HIP_HOST_FLAGS}\nset(CMAKE_HOST_FLAGS_${config_upper} ${CMAKE_${HIP_C_OR_CXX}_FLAGS_${config_upper}})")
+        set(_HIP_HIPCC_FLAGS "${_HIP_HIPCC_FLAGS}\nset(HIP_HIPCC_FLAGS_${config_upper} ${HIP_HIPCC_FLAGS_${config_upper}})")
+        set(_HIP_HCC_FLAGS "${_HIP_HCC_FLAGS}\nset(HIP_HCC_FLAGS_${config_upper} ${HIP_HCC_FLAGS_${config_upper}})")
+        set(_HIP_CLANG_FLAGS "${_HIP_CLANG_FLAGS}\nset(HIP_CLANG_FLAGS_${config_upper} ${HIP_CLANG_FLAGS_${config_upper}})")
+        set(_HIP_NVCC_FLAGS "${_HIP_NVCC_FLAGS}\nset(HIP_NVCC_FLAGS_${config_upper} ${HIP_NVCC_FLAGS_${config_upper}})")
+    endforeach()
+
+    # Reset the output variable
+    set(_hip_generated_files "")
+    set(_hip_source_files "")
+
+    # Iterate over all arguments and create custom commands for all source files
+    foreach(file ${ARGN})
+        # Ignore any file marked as a HEADER_FILE_ONLY
+        get_source_file_property(_is_header ${file} HEADER_FILE_ONLY)
+        # Allow per source file overrides of the format. Also allows compiling non .cu files.
+        get_source_file_property(_hip_source_format ${file} HIP_SOURCE_PROPERTY_FORMAT)
+        if((${file} MATCHES "\\.cu$" OR _hip_source_format) AND NOT _is_header)
+            set(host_flag FALSE)
+        else()
+            set(host_flag TRUE)
+        endif()
+
+        if(NOT host_flag)
+            # Determine output directory
+            HIP_COMPUTE_BUILD_PATH("${file}" hip_build_path)
+            set(hip_compile_output_dir "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${_target}.dir/${hip_build_path}")
+
+            get_filename_component(basename ${file} NAME)
+            set(generated_file_path "${hip_compile_output_dir}/${CMAKE_CFG_INTDIR}")
+            set(generated_file_basename "${_target}_generated_${basename}${generated_extension}")
+
+            # Set file names
+            set(generated_file "${generated_file_path}/${generated_file_basename}")
+            set(cmake_dependency_file "${hip_compile_output_dir}/${generated_file_basename}.depend")
+            set(custom_target_script_pregen "${hip_compile_output_dir}/${generated_file_basename}.cmake.pre-gen")
+            set(custom_target_script "${hip_compile_output_dir}/${generated_file_basename}.cmake")
+
+            # Set properties for object files
+            set_source_files_properties("${generated_file}"
+                PROPERTIES
+                EXTERNAL_OBJECT true # This is an object file not to be compiled, but only be linked
+                )
+
+            # Don't add CMAKE_CURRENT_SOURCE_DIR if the path is already an absolute path
+            get_filename_component(file_path "${file}" PATH)
+            if(IS_ABSOLUTE "${file_path}")
+                set(source_file "${file}")
+            else()
+                set(source_file "${CMAKE_CURRENT_SOURCE_DIR}/${file}")
+            endif()
+
+            # Bring in the dependencies
+            HIP_INCLUDE_HIPCC_DEPENDENCIES(${cmake_dependency_file})
+
+            # Configure the build script
+            configure_file("${HIP_run_hipcc}" "${custom_target_script_pregen}" @ONLY)
+            file(GENERATE
+                OUTPUT "${custom_target_script}"
+                INPUT "${custom_target_script_pregen}"
+                )
+            set(main_dep DEPENDS ${source_file})
+            if(CMAKE_GENERATOR MATCHES "Makefiles")
+                set(verbose_output "$(VERBOSE)")
+            elseif(HIP_VERBOSE_BUILD)
+                set(verbose_output ON)
+            else()
+                set(verbose_output OFF)
+            endif()
+
+            # Create up the comment string
+            file(RELATIVE_PATH generated_file_relative_path "${CMAKE_BINARY_DIR}" "${generated_file}")
+            set(hip_build_comment_string "Building HIPCC object ${generated_file_relative_path}")
+
+            # Build the generated file and dependency file
+            add_custom_command(
+                OUTPUT ${generated_file}
+                # These output files depend on the source_file and the contents of cmake_dependency_file
+                ${main_dep}
+                DEPENDS ${HIP_HIPCC_DEPEND}
+                DEPENDS ${custom_target_script}
+                # Make sure the output directory exists before trying to write to it.
+                COMMAND ${CMAKE_COMMAND} -E make_directory "${generated_file_path}"
+                COMMAND ${CMAKE_COMMAND} ARGS
+                -D verbose:BOOL=${verbose_output}
+                -D build_configuration:STRING=${_hip_build_configuration}
+                -D "generated_file:STRING=${generated_file}"
+                -P "${custom_target_script}"
+                WORKING_DIRECTORY "${hip_compile_output_dir}"
+                COMMENT "${hip_build_comment_string}"
+                )
+
+            # Make sure the build system knows the file is generated
+            set_source_files_properties(${generated_file} PROPERTIES GENERATED TRUE)
+            list(APPEND _hip_generated_files ${generated_file})
+            list(APPEND _hip_source_files ${file})
+        endif()
+    endforeach()
+
+    # Set the return parameter
+    set(${_generated_files} ${_hip_generated_files})
+    set(${_source_files} ${_hip_source_files})
+endmacro()
+
+###############################################################################
+# HIP_ADD_EXECUTABLE
+###############################################################################
+macro(HIP_ADD_EXECUTABLE hip_target)
+    # Separate the sources from the options
+    HIP_GET_SOURCES_AND_OPTIONS(_sources _cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options ${ARGN})
+    HIP_PREPARE_TARGET_COMMANDS(${hip_target} OBJ _generated_files _source_files ${_sources} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} CLANG_OPTIONS ${_clang_options} NVCC_OPTIONS ${_nvcc_options})
+    if(_source_files)
+        list(REMOVE_ITEM _sources ${_source_files})
+    endif()
+    if("${HIP_COMPILER}" STREQUAL "hcc")
+        if("x${HCC_HOME}" STREQUAL "x")
+            if (DEFINED ENV{ROCM_PATH})
+                set(HCC_HOME "$ENV{ROCM_PATH}/hcc")
+            elseif(DEFINED ENV{HIP_PATH})
+                set(HCC_HOME "$ENV{HIP_PATH}/../hcc")
+            else()
+                set(HCC_HOME "/opt/rocm/hcc")
+            endif()
+        endif()
+        set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
+    elseif("${HIP_COMPILER}" STREQUAL "clang")
+        if("x${HIP_CLANG_PATH}" STREQUAL "x")
+            if(DEFINED ENV{HIP_CLANG_PATH})
+                set(HIP_CLANG_PATH $ENV{HIP_CLANG_PATH})
+            elseif(DEFINED ENV{ROCM_PATH})
+                set(HIP_CLANG_PATH "$ENV{ROCM_PATH}/llvm/bin")
+            elseif(DEFINED ENV{HIP_PATH})
+                set(HIP_CLANG_PATH "$ENV{HIP_PATH}/../llvm/bin")
+            else()
+                set(HIP_CLANG_PATH "/opt/rocm/llvm/bin")
+            endif()
+        endif()
+        set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
+    else()
+        set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
+    endif()
+    if ("${_sources}" STREQUAL "")
+        add_executable(${hip_target} ${_cmake_options} ${_generated_files} "")
+    else()
+        add_executable(${hip_target} ${_cmake_options} ${_generated_files} ${_sources})
+    endif()
+    set_target_properties(${hip_target} PROPERTIES LINKER_LANGUAGE HIP)
+    # Link with host
+    if (HIP_HOST_INTERFACE)
+        # hip rt should be rocclr, compiler should be clang
+        target_link_libraries(${hip_target} ${HIP_HOST_INTERFACE})
+    endif()
+endmacro()
+
+###############################################################################
+# HIP_ADD_LIBRARY
+###############################################################################
+macro(HIP_ADD_LIBRARY hip_target)
+    # Separate the sources from the options
+    HIP_GET_SOURCES_AND_OPTIONS(_sources _cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options ${ARGN})
+    HIP_PREPARE_TARGET_COMMANDS(${hip_target} OBJ _generated_files _source_files ${_sources} ${_cmake_options} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} CLANG_OPTIONS ${_clang_options} NVCC_OPTIONS ${_nvcc_options})
+    if(_source_files)
+        list(REMOVE_ITEM _sources ${_source_files})
+    endif()
+    if ("${_sources}" STREQUAL "")
+        add_library(${hip_target} ${_cmake_options} ${_generated_files} "")
+    else()
+        add_library(${hip_target} ${_cmake_options} ${_generated_files} ${_sources})
+    endif()
+    set_target_properties(${hip_target} PROPERTIES LINKER_LANGUAGE ${HIP_C_OR_CXX})
+    # Link with host
+    if (HIP_HOST_INTERFACE)
+        # hip rt should be rocclr, compiler should be clang
+        target_link_libraries(${hip_target} ${HIP_HOST_INTERFACE})
+    endif()
+endmacro()
+
+# vim: ts=4:sw=4:expandtab:smartindent

config/cmake/modules/FindHIP/run_hipcc.cmake

@@ -0,0 +1,182 @@
+###############################################################################
+# Runs commands using HIPCC
+###############################################################################
+
+###############################################################################
+# This file runs the hipcc commands to produce the desired output file
+# along with the dependency file needed by CMake to compute dependencies.
+#
+# Input variables:
+#
+# verbose:BOOL=<>               OFF: Be as quiet as possible (default)
+#                               ON : Describe each step
+# build_configuration:STRING=<> Build configuration. Defaults to Debug.
+# generated_file:STRING=<>      File to generate. Mandatory argument.
+
+if(NOT build_configuration)
+    set(build_configuration Debug)
+endif()
+if(NOT generated_file)
+    message(FATAL_ERROR "You must specify generated_file on the command line")
+endif()
+
+# Set these up as variables to make reading the generated file easier
+set(HIP_HIPCC_EXECUTABLE "@HIP_HIPCC_EXECUTABLE@") # path
+set(HIP_HIPCONFIG_EXECUTABLE "@HIP_HIPCONFIG_EXECUTABLE@") #path
+set(HIP_HOST_COMPILER "@HIP_HOST_COMPILER@") # path
+set(CMAKE_COMMAND "@CMAKE_COMMAND@") # path
+set(HIP_run_make2cmake "@HIP_run_make2cmake@") # path
+set(HCC_HOME "@HCC_HOME@") #path
+set(HIP_CLANG_PATH "@HIP_CLANG_PATH@") #path
+set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "@HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS@")
+
+@HIP_HOST_FLAGS@
+@_HIP_HIPCC_FLAGS@
+@_HIP_HCC_FLAGS@
+@_HIP_CLANG_FLAGS@
+@_HIP_NVCC_FLAGS@
+#Needed to bring the HIP_HIPCC_INCLUDE_ARGS variable in scope
+set(HIP_HIPCC_INCLUDE_ARGS @HIP_HIPCC_INCLUDE_ARGS@) # list
+
+set(cmake_dependency_file "@cmake_dependency_file@") # path
+set(source_file "@source_file@") # path
+set(host_flag "@host_flag@") # bool
+
+# Determine compiler and compiler flags
+execute_process(COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --platform OUTPUT_VARIABLE HIP_PLATFORM OUTPUT_STRIP_TRAILING_WHITESPACE)
+execute_process(COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --compiler OUTPUT_VARIABLE HIP_COMPILER OUTPUT_STRIP_TRAILING_WHITESPACE)
+execute_process(COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --runtime OUTPUT_VARIABLE HIP_RUNTIME OUTPUT_STRIP_TRAILING_WHITESPACE)
+if(NOT host_flag)
+    set(__CC ${HIP_HIPCC_EXECUTABLE})
+    if("${HIP_PLATFORM}" STREQUAL "amd")
+        if("${HIP_COMPILER}" STREQUAL "hcc")
+            if(NOT "x${HCC_HOME}" STREQUAL "x")
+                set(ENV{HCC_HOME} ${HCC_HOME})
+            endif()
+            set(__CC_FLAGS ${HIP_HIPCC_FLAGS} ${HIP_HCC_FLAGS} ${HIP_HIPCC_FLAGS_${build_configuration}} ${HIP_HCC_FLAGS_${build_configuration}})
+        elseif("${HIP_COMPILER}" STREQUAL "clang")
+            if(NOT "x${HIP_CLANG_PATH}" STREQUAL "x")
+                set(ENV{HIP_CLANG_PATH} ${HIP_CLANG_PATH})
+            endif()
+            # Temporarily include HIP_HCC_FLAGS for HIP-Clang for PyTorch builds
+            set(__CC_FLAGS ${HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS} ${HIP_HIPCC_FLAGS} ${HIP_HCC_FLAGS} ${HIP_CLANG_FLAGS} ${HIP_HIPCC_FLAGS_${build_configuration}} ${HIP_HCC_FLAGS_${build_configuration}} ${HIP_CLANG_FLAGS_${build_configuration}})
+        endif()
+    else()
+        set(__CC_FLAGS ${HIP_HIPCC_FLAGS} ${HIP_NVCC_FLAGS} ${HIP_HIPCC_FLAGS_${build_configuration}} ${HIP_NVCC_FLAGS_${build_configuration}})
+    endif()
+else()
+    set(__CC ${HIP_HOST_COMPILER})
+    set(__CC_FLAGS ${CMAKE_HOST_FLAGS} ${CMAKE_HOST_FLAGS_${build_configuration}})
+endif()
+set(__CC_INCLUDES ${HIP_HIPCC_INCLUDE_ARGS})
+
+# hip_execute_process - Executes a command with optional command echo and status message.
+#   status     - Status message to print if verbose is true
+#   command    - COMMAND argument from the usual execute_process argument structure
+#   ARGN       - Remaining arguments are the command with arguments
+#   HIP_result - Return value from running the command
+macro(hip_execute_process status command)
+    set(_command ${command})
+    if(NOT "x${_command}" STREQUAL "xCOMMAND")
+        message(FATAL_ERROR "Malformed call to hip_execute_process.  Missing COMMAND as second argument. (command = ${command})")
+    endif()
+    if(verbose)
+        execute_process(COMMAND "${CMAKE_COMMAND}" -E echo -- ${status})
+        # Build command string to print
+        set(hip_execute_process_string)
+        foreach(arg ${ARGN})
+            # Escape quotes if any
+            string(REPLACE "\"" "\\\"" arg ${arg})
+            # Surround args with spaces with quotes
+            if(arg MATCHES " ")
+                list(APPEND hip_execute_process_string "\"${arg}\"")
+            else()
+                list(APPEND hip_execute_process_string ${arg})
+            endif()
+        endforeach()
+        # Echo the command
+        execute_process(COMMAND ${CMAKE_COMMAND} -E echo ${hip_execute_process_string})
+    endif()
+    # Run the command
+    execute_process(COMMAND ${ARGN} RESULT_VARIABLE HIP_result)
+endmacro()
+
+# Delete the target file
+hip_execute_process(
+    "Removing ${generated_file}"
+    COMMAND "${CMAKE_COMMAND}" -E remove "${generated_file}"
+    )
+
+# Generate the dependency file
+hip_execute_process(
+    "Generating dependency file: ${cmake_dependency_file}.pre"
+    COMMAND "${__CC}"
+    -M
+    "${source_file}"
+    -o "${cmake_dependency_file}.pre"
+    ${__CC_FLAGS}
+    ${__CC_INCLUDES}
+    )
+
+if(HIP_result)
+    message(FATAL_ERROR "Error generating ${generated_file}")
+endif()
+
+# Generate the cmake readable dependency file to a temp file
+hip_execute_process(
+    "Generating temporary cmake readable file: ${cmake_dependency_file}.tmp"
+    COMMAND "${CMAKE_COMMAND}"
+    -D "input_file:FILEPATH=${cmake_dependency_file}.pre"
+    -D "output_file:FILEPATH=${cmake_dependency_file}.tmp"
+    -D "verbose=${verbose}"
+    -P "${HIP_run_make2cmake}"
+    )
+
+if(HIP_result)
+    message(FATAL_ERROR "Error generating ${generated_file}")
+endif()
+
+# Copy the file if it is different
+hip_execute_process(
+    "Copy if different ${cmake_dependency_file}.tmp to ${cmake_dependency_file}"
+    COMMAND "${CMAKE_COMMAND}" -E copy_if_different "${cmake_dependency_file}.tmp" "${cmake_dependency_file}"
+    )
+
+if(HIP_result)
+    message(FATAL_ERROR "Error generating ${generated_file}")
+endif()
+
+# Delete the temporary file
+hip_execute_process(
+    "Removing ${cmake_dependency_file}.tmp and ${cmake_dependency_file}.pre"
+    COMMAND "${CMAKE_COMMAND}" -E remove "${cmake_dependency_file}.tmp" "${cmake_dependency_file}.pre"
+    )
+
+if(HIP_result)
+    message(FATAL_ERROR "Error generating ${generated_file}")
+endif()
+
+# Generate the output file
+hip_execute_process(
+    "Generating ${generated_file}"
+    COMMAND "${__CC}"
+    -c
+    "${source_file}"
+    -o "${generated_file}"
+    ${__CC_FLAGS}
+    ${__CC_INCLUDES}
+    )
+
+if(HIP_result)
+    # Make sure that we delete the output file
+    hip_execute_process(
+        "Removing ${generated_file}"
+        COMMAND "${CMAKE_COMMAND}" -E remove "${generated_file}"
+        )
+    message(FATAL_ERROR "Error generating file ${generated_file}")
+else()
+    if(verbose)
+        message("Generated ${generated_file} successfully.")
+    endif()
+endif()
+# vim: ts=4:sw=4:expandtab:smartindent

config/cmake/modules/FindHIP/run_make2cmake.cmake

@@ -0,0 +1,50 @@
+###############################################################################
+# Computes dependencies using HIPCC
+###############################################################################
+
+###############################################################################
+# This file converts dependency files generated using hipcc to a format that
+# cmake can understand.
+
+# Input variables:
+#
+# input_file:STRING=<> Dependency file to parse. Required argument
+# output_file:STRING=<> Output file to generate. Required argument
+
+if(NOT input_file OR NOT output_file)
+    message(FATAL_ERROR "You must specify input_file and output_file on the command line")
+endif()
+
+file(READ ${input_file} depend_text)
+
+if (NOT "${depend_text}" STREQUAL "")
+    string(REPLACE " /" "\n/" depend_text ${depend_text})
+    string(REGEX REPLACE "^.*:" "" depend_text ${depend_text})
+    string(REGEX REPLACE "[ \\\\]*\n" ";" depend_text ${depend_text})
+
+    set(dependency_list "")
+
+    foreach(file ${depend_text})
+        string(REGEX REPLACE "^ +" "" file ${file})
+        if(NOT EXISTS "${file}")
+            message(WARNING " Removing non-existent dependency file: ${file}")
+            set(file "")
+        endif()
+
+        if(NOT IS_DIRECTORY "${file}")
+            get_filename_component(file_absolute "${file}" ABSOLUTE)
+            list(APPEND dependency_list "${file_absolute}")
+        endif()
+    endforeach()
+endif()
+
+# Remove the duplicate entries and sort them.
+list(REMOVE_DUPLICATES dependency_list)
+list(SORT dependency_list)
+
+foreach(file ${dependency_list})
+    set(hip_hipcc_depend "${hip_hipcc_depend} \"${file}\"\n")
+endforeach()
+
+file(WRITE ${output_file} "# Generated by: FindHIP.cmake. Do not edit.\nSET(HIP_HIPCC_DEPEND\n ${hip_hipcc_depend})\n\n")
+# vim: ts=4:sw=4:expandtab:smartindent

config/cmake/modules/FindHYPRE.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -14,33 +14,10 @@
 #   - HYPRE_LIBRARIES
 #   - HYPRE_INCLUDE_DIRS
 #   - HYPRE_VERSION
-#   - HYPRE_USING_HIP (internal)
-
-if (HYPRE_FOUND)
-  if (HYPRE_USING_HIP)
-    find_package(rocsparse REQUIRED)
-    find_package(rocrand REQUIRED)
-  endif()
-  return()
-endif()
 
 include(MfemCmakeUtilities)
 mfem_find_package(HYPRE HYPRE HYPRE_DIR "include" "HYPRE.h" "lib" "HYPRE"
-  "Paths to headers required by HYPRE." "Libraries required by HYPRE."
-  CHECK_BUILD HYPRE_USING_HIP FALSE
-  "
-#undef HYPRE_USING_HIP
-#include <HYPRE_config.h>
-
-#ifndef HYPRE_USING_HIP
-#error HYPRE is built without HIP.
-#endif
-
-int main()
-{
-   return 0;
-}
-")
+  "Paths to headers required by HYPRE." "Libraries required by HYPRE.")
 
 if (HYPRE_FOUND AND (NOT HYPRE_VERSION))
   try_run(HYPRE_VERSION_RUN_RESULT HYPRE_VERSION_COMPILE_RESULT
@@ -56,12 +33,3 @@ if (HYPRE_FOUND AND (NOT HYPRE_VERSION))
     message(FATAL_ERROR "Unable to determine HYPRE version.")
   endif()
 endif()
-
-if (HYPRE_FOUND AND HYPRE_USING_HIP)
-  find_package(rocsparse REQUIRED)
-  find_package(rocrand REQUIRED)
-  list(APPEND HYPRE_LIBRARIES ${rocsparse_LIBRARIES} ${rocrand_LIBRARIES})
-  set(HYPRE_LIBRARIES ${HYPRE_LIBRARIES} CACHE STRING
-      "HYPRE libraries + dependencies." FORCE)
-  message(STATUS "Updated HYPRE_LIBRARIES: ${HYPRE_LIBRARIES}")
-endif()

config/cmake/modules/FindLIBDL.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindLibunwind.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindMETIS.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindMFEMBacktrace.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindMKL_CPARDISO.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindMPFR.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindMUMPS.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindMesquite.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindNetCDF.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindOCCA.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindPARELAG.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindPOSIXClocks.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindParMETIS.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindRAJA.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindSLEPc.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindSTRUMPACK.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindSUNDIALS.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindScotch.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindSuiteSparse.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindSuperLUDist.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindUMPIRE.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/Find_GnuTLS.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindlibCEED.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -46,7 +46,8 @@ endfunction()
 # Wrapper for add_executable that calls the HIP wrapper if applicable
 macro(mfem_add_executable NAME)
   if (MFEM_USE_HIP)
-    add_executable(${NAME} ${ARGN})
+    hip_add_executable(${NAME} ${ARGN})
+    set_target_properties(${NAME} PROPERTIES LINKER_LANGUAGE CXX)
   else()
     add_executable(${NAME} ${ARGN})
   endif()
@@ -55,7 +56,7 @@ endmacro()
 # Wrapper for add_library that calls the HIP wrapper if applicable
 macro(mfem_add_library NAME)
   if (MFEM_USE_HIP)
-    add_library(${NAME} ${ARGN})
+    hip_add_library(${NAME} ${ARGN})
   else()
     add_library(${NAME} ${ARGN})
   endif()
@@ -91,6 +92,8 @@ macro(add_mfem_examples EXE_SRCS)
     # If CUDA is enabled, tag source files to be compiled with nvcc.
     if (MFEM_USE_CUDA)
       set_source_files_properties(${SRC_FILE} PROPERTIES LANGUAGE CUDA)
+    elseif(MFEM_USE_HIP)
+      set_source_files_properties(${SRC_FILE} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
     endif()
 
     get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
@@ -156,6 +159,8 @@ macro(add_mfem_miniapp MFEM_EXE_NAME)
       endforeach()
       set(EXTRA_OPTIONS_LIST ${LIST_})
     endif()
+  elseif(MFEM_USE_HIP)
+    set_source_files_properties(${MAIN_LIST} ${EXTRA_SOURCES_LIST} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
   endif()
 
   # Actually add the executable
@@ -529,15 +534,12 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
             if (NOT ImportConfig)
               set(ImportConfig RELEASE)
             endif()
-            set(ImportConfigSuffix "_${ImportConfig}")
             get_target_property(ImpConfigs ${TargetName} IMPORTED_CONFIGURATIONS)
             list(FIND ImpConfigs ${ImportConfig} _Index)
-            if ((_Index EQUAL -1) OR ("${ImportConfig}" STREQUAL "NO_CONFIG"))
-              set(ImportConfig "NO_CONFIG")
-              set(ImportConfigSuffix "")
-              # message(FATAL_ERROR " *** ${ReqPack}: configuration "
-              #   "${ImportConfig} not found. Set ${ReqPack}_IMPORT_CONFIG "
-              #   "from the list: ${ImpConfigs}.")
+            if (_Index EQUAL -1)
+              message(FATAL_ERROR " *** ${ReqPack}: configuration "
+                "${ImportConfig} not found. Set ${ReqPack}_IMPORT_CONFIG "
+                "from the list: ${ImpConfigs}.")
             endif()
           endif()
           # Set _Pack_LIBS
@@ -549,8 +551,8 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
             endif()
           else()
             # Set _Pack_LIBS from the target properties for ImportConfig
-            foreach (_prop IMPORTED_LOCATION${ImportConfigSuffix}
-                IMPORTED_LINK_INTERFACE_LIBRARIES${ImportConfigSuffix})
+            foreach (_prop IMPORTED_LOCATION_${ImportConfig}
+                IMPORTED_LINK_INTERFACE_LIBRARIES_${ImportConfig})
               get_target_property(_value ${TargetName} ${_prop})
               if (_value)
                 list(APPEND _Pack_LIBS ${_value})
@@ -562,7 +564,7 @@ function(mfem_find_package Name Prefix DirVar IncSuffixes Header LibSuffixes
             endif()
           endif()
           # Set _Pack_INCS
-          foreach (_prop INCLUDE_DIRECTORIES INTERFACE_INCLUDE_DIRECTORIES)
+          foreach (_prop INCLUDE_DIRECTORIES)
             get_target_property(_value ${TargetName} ${_prop})
             if (_value)
               list(APPEND _Pack_INCS ${_value})
@@ -740,133 +742,6 @@ function(mfem_find_library Name Prefix Lib LibDoc CheckVar CheckSrc)
 endfunction(mfem_find_library)
 
 
-#
-# Extract compile and link options needed by the given target.
-#
-function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
-
-  if (NOT TARGET ${Target})
-    return()
-  endif()
-
-  # CMAKE_SHARED_LIBRARY_RUNTIME_C_FLAG -> '-Wl,-rpath,'
-  set(shared_link_flag ${CMAKE_SHARED_LIBRARY_RUNTIME_C_FLAG})
-  if (NOT shared_link_flag)
-    set(shared_link_flag "-Wl,-rpath,")
-  endif()
-
-  set(tgt "${Target}")
-  unset(CompileOpts)
-  unset(LinkOpts)
-  get_target_property(IsImported ${tgt} IMPORTED)
-  # message(STATUS "${tgt}[IMPORTED]: ${IsImported}")
-  # Generally, the possible target types are: STATIC_LIBRARY, MODULE_LIBRARY,
-  # SHARED_LIBRARY, INTERFACE_LIBRARY, EXECUTABLE.
-  get_target_property(type ${tgt} TYPE)
-  # message(STATUS "${tgt}[TYPE]: ${type}")
-  unset(ImportConfig)
-  get_target_property(ImportConfigs ${tgt} IMPORTED_CONFIGURATIONS)
-  if (ImportConfigs)
-    list(GET ImportConfigs 0 ImportConfig)
-  endif()
-  if (NOT ImportConfig)
-    set(ImportConfig RELEASE)
-  endif()
-  # message(STATUS "${tgt}[ImportConfig]: ${ImportConfig}")
-  # List all properties with: cmake --help-property-list
-  get_target_property(Defs ${tgt} INTERFACE_COMPILE_DEFINITIONS)
-  if (Defs)
-    list(REMOVE_DUPLICATES Defs)
-    foreach(Def ${Defs})
-      list(APPEND CompileOpts "-D${Def}")
-    endforeach()
-  endif()
-  get_target_property(Opts ${tgt} INTERFACE_COMPILE_OPTIONS)
-  if (Opts)
-    foreach(Opt ${Opts})
-      list(APPEND CompileOpts "${Opt}")
-    endforeach()
-  endif()
-  get_target_property(Dirs ${tgt} INTERFACE_INCLUDE_DIRECTORIES)
-  if (Dirs)
-    list(REMOVE_DUPLICATES Dirs)
-    foreach(Dir ${Dirs})
-      list(APPEND CompileOpts "-I\"${Dir}\"")
-    endforeach()
-  endif()
-  get_target_property(SysDirs ${tgt} INTERFACE_SYSTEM_INCLUDE_DIRECTORIES)
-  if (SysDirs)
-    list(REMOVE_DUPLICATES SysDirs)
-    foreach(SysDir ${SysDirs})
-      list(APPEND CompileOpts "-isystem \"${SysDir}\"")
-    endforeach()
-  endif()
-  if ("${type}" STREQUAL "STATIC_LIBRARY")
-    get_target_property(Location ${tgt} LOCATION)
-    if (Location)
-      list(APPEND LinkOpts "\"${Location}\"")
-    else()
-      message(STATUS " *** Warning: [${tgt}] LOCATION not defined!")
-    endif()
-  elseif ("${type}" STREQUAL "SHARED_LIBRARY")
-    get_target_property(Location ${tgt} LOCATION)
-    if (Location)
-      get_filename_component(Dir ${Location} DIRECTORY)
-      get_filename_component(NameWE ${Location} NAME_WE)
-      string(REGEX REPLACE "^lib" "" LibName ${NameWE})
-      list(APPEND LinkOpts
-        "-L\"${Dir}\""
-        "${shared_link_flag}\"${Dir}\""
-        "-l${LibName}")
-    else()
-      message(STATUS " *** Warning: [${tgt}] LOCATION not defined!")
-    endif()
-  elseif ("${type}" STREQUAL "INTERFACE_LIBRARY")
-    get_target_property(Libs ${tgt} INTERFACE_LINK_LIBRARIES)
-    if (Libs)
-      foreach(Lib ${Libs})
-        if (NOT (TARGET ${Lib}))
-          list(APPEND LinkOpts "${Lib}")
-        else()
-          mfem_get_target_options(${Lib} COpts LOpts)
-          list(APPEND CompileOpts ${COpts})
-          list(APPEND LinkOpts ${LOpts})
-        endif()
-      endforeach()
-    endif()
-    # Other properties we may need to handle:
-    # INTERFACE_LINK_DEPENDS
-    # INTERFACE_LINK_DIRECTORIES
-    # INTERFACE_LINK_OPTIONS
-  else()
-    message(STATUS " *** Warning: [${tgt}] uses target type '${type}'"
-      " which is not supported!")
-  endif()
-
-  # Other potentially relevant properties:
-  # - For all target types:
-  # IMPORTED_LIBNAME
-  # IMPORTED_LIBNAME_${ImportConfig}
-  # INTERFACE_AUTOUIC_OPTIONS
-  # INTERFACE_COMPILE_FEATURES
-  # INTERFACE_POSITION_INDEPENDENT_CODE
-  # INTERFACE_SOURCES
-  # INTERFACE_SYSTEM_INCLUDE_DIRECTORIES)
-  # - For non-"INTERFACE_LIBRARY" target types only:
-  # IMPORTED_LOCATION
-  # IMPORTED_LOCATION_${ImportConfig}
-  # IMPORTED_LINK_INTERFACE_LIBRARIES
-  # IMPORTED_LINK_INTERFACE_LIBRARIES_${ImportConfig}
-  # LINK_FLAGS
-  # LINK_FLAGS_${ImportConfig}
-  # LOCATION_${ImportConfig})
-
-  set(${CompileOptsVar} "${CompileOpts}" PARENT_SCOPE)
-  set(${LinkOptsVar} "${LinkOpts}" PARENT_SCOPE)
-
-endfunction(mfem_get_target_options)
-
-
 #
 #   Function that creates 'config.mk' from 'config.mk.in' for the both the
 #   build- and the install-locations and define install rules for 'config.mk'
@@ -885,15 +760,13 @@ function(mfem_export_mk_files)
   # Convert Boolean vars to YES/NO without writing the values to cache
   set(CONFIG_MK_BOOL_VARS MFEM_USE_MPI MFEM_USE_METIS MFEM_USE_METIS_5
       MFEM_DEBUG MFEM_USE_EXCEPTIONS MFEM_USE_ZLIB MFEM_USE_LIBUNWIND
-      MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_LEGACY_OPENMP MFEM_USE_OPENMP
+      MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP MFEM_USE_LEGACY_OPENMP
       MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
-      MFEM_USE_SUPERLU MFEM_USE_SUPERLU5 MFEM_USE_MUMPS MFEM_USE_STRUMPACK
-      MFEM_USE_GINKGO MFEM_USE_AMGX MFEM_USE_GNUTLS MFEM_USE_NETCDF
-      MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_FMS
-      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA
-      MFEM_USE_HIP MFEM_USE_RAJA MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_CALIPER
-      MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO
-      MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG)
+      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX
+      MFEM_USE_GNUTLS MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC
+      MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_CONDUIT MFEM_USE_PUMI
+      MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA MFEM_USE_UMPIRE MFEM_USE_SIMD
+      MFEM_USE_ADIOS2 MFEM_USE_BENCHMARK MFEM_USE_PARELAG)
   foreach(var ${CONFIG_MK_BOOL_VARS})
     if (${var})
       set(${var} YES)
@@ -995,18 +868,8 @@ function(mfem_export_mk_files)
     get_filename_component(suffix ${lib} EXT)
     # handle interfaces (e.g., SCOREC::apf)
     if ("${lib}" MATCHES "SCOREC::.*" OR "${lib}" MATCHES "Ginkgo::.*")
-    elseif (TARGET "${lib}")
-      mfem_get_target_options(${lib} CompileOpts LinkOpts)
-      # Removing duplicates may lead to issues:
-      # list(REMOVE_DUPLICATES CompileOpts)
-      # list(REMOVE_DUPLICATES LinkOpts)
-      string(REPLACE ";" " " COpts "${CompileOpts}")
-      string(REPLACE ";" " " LOpts "${LinkOpts}")
-      # message(STATUS "${lib}[COpts]: '${COpts}'")
-      # message(STATUS "${lib}[LOpts]: '${LOpts}'")
-      set(MFEM_TPLFLAGS "${MFEM_TPLFLAGS} ${COpts}")
-      set(MFEM_EXT_LIBS "${MFEM_EXT_LIBS} ${LOpts}")
-      # message(FATAL_ERROR "***** interface lib found ... exiting *****")
+    elseif (NOT "${lib}" MATCHES "SCOREC::.*" AND "${lib}" MATCHES ".*::.*")
+      message(FATAL_ERROR "***** interface lib found ... exiting *****")
       # handle static and shared libs
     elseif ("${suffix}" STREQUAL "${CMAKE_SHARED_LIBRARY_SUFFIX}")
       get_filename_component(dir ${lib} DIRECTORY)

config/config.hpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //

config/config.hpp.in

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //

config/config.mk.in

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -91,7 +91,7 @@ MFEM_MPIEXEC_NP = @MFEM_MPIEXEC_NP@
 MFEM_MPI_NP     = @MFEM_MPI_NP@
 
 # The NVCC compiler cannot link with -x=cu
-MFEM_LINK_FLAGS := $(filter-out -x=cu -xhip, $(MFEM_FLAGS))
+MFEM_LINK_FLAGS := $(filter-out -x=cu, $(MFEM_FLAGS))
 
 # Optional extra configuration
 @MFEM_CONFIG_EXTRA@

config/defaults.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -50,7 +50,6 @@ option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
 option(MFEM_USE_PUMI "Enable PUMI" OFF)
 option(MFEM_USE_HIOP "Enable HiOp" OFF)
 option(MFEM_USE_CUDA "Enable CUDA" OFF)
-option(MFEM_USE_HIP "Enable HIP" OFF)
 option(MFEM_USE_OCCA "Enable OCCA" OFF)
 option(MFEM_USE_RAJA "Enable RAJA" OFF)
 option(MFEM_USE_CEED "Enable CEED" OFF)
@@ -108,7 +107,6 @@ if (MFEM_USE_CUDA)
    set(HYPRE_REQUIRED_LIBRARIES "-lcusparse" "-lcurand" CACHE STRING
        "Libraries that HYPRE depends on.")
 endif()
-# HIP dependency for HYPRE is handled in FindHYPRE.cmake.
 
 set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library.")
 
@@ -228,11 +226,6 @@ set(MKL_LIBRARY_DIR "" CACHE STRING "Custom library subdirectory")
 
 set(OCCA_DIR "${MFEM_DIR}/../occa" CACHE PATH "Path to OCCA")
 set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
-# If RAJA is built with external CAMP:
-# set(RAJA_REQUIRED_PACKAGES "camp"
-#     CACHE STRING "Packages that RAJA depends on.")
-# set(camp_DIR "${MFEM_DIR}/../camp/lib/cmake/camp"
-#     CACHE PATH "Path to CAMP CMake files.")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")
 set(UMPIRE_DIR "${MFEM_DIR}/../umpire" CACHE PATH "Path to Umpire")
 set(CALIPER_DIR "${MFEM_DIR}/../caliper" CACHE PATH "Path to Caliper")
@@ -252,8 +245,7 @@ set(BLAS_LIBRARIES "" CACHE STRING "The BLAS library.")
 set(LAPACK_INCLUDE_DIRS "" CACHE STRING "Path to LAPACK headers.")
 set(LAPACK_LIBRARIES "" CACHE STRING "The LAPACK library.")
 
-set(CODIPACK_INCLUDE_DIRS "${MFEM_DIR}/../CoDiPack/include" CACHE STRING
-    "Path to CoDiPack headers.")
+set(CODIPACK_INCLUDE_DIRS "${MFEM_DIR}/../CoDiPack/inlude" CACHE STRING "Path to CoDiPack headers.")
 set(CODIPACK_LIBRARIES "")
 
 # Some useful variables:

config/defaults.mk

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -172,20 +172,6 @@ ifeq ($(MFEM_USE_MPI)$(MFEM_USE_HIP),YESYES)
    MPI_LIB = -L$(MPI_DIR)/lib $(XLINKER)-rpath,$(MPI_DIR)/lib -lmpi
 endif
 
-# ROCM/HIP directory such that ROCM/HIP libraries like rocsparse and rocrand are
-# found in $(HIP_DIR)/lib, usually as links. Typically, this directoory is of
-# the form /opt/rocm-X.Y.Z which is called ROCM_PATH by hipconfig.
-ifeq ($(MFEM_USE_HIP),YES)
-   HIP_DIR := $(patsubst %/,%,$(dir $(shell which $(HIP_CXX))))
-   HIP_DIR := $(patsubst %/,%,$(dir $(HIP_DIR)))
-   ifeq (,$(wildcard $(HIP_DIR)/lib/librocsparse.*))
-      HIP_DIR := $(shell hipconfig --rocmpath 2> /dev/null)
-      ifeq (,$(wildcard $(HIP_DIR)/lib/librocsparse.*))
-         $(error Unable to determine HIP_DIR. Please set it manually.)
-      endif
-   endif
-endif
-
 # Compile and link options for zlib.
 ZLIB_DIR =
 ZLIB_OPT = $(if $(ZLIB_DIR),-I$(ZLIB_DIR)/include)
@@ -203,11 +189,6 @@ ifeq (YES,$(MFEM_USE_CUDA))
    # This is only necessary when hypre is built with cuda:
    HYPRE_LIB += -lcusparse -lcurand
 endif
-ifeq (YES,$(MFEM_USE_HIP))
-   # This is only necessary when hypre is built with hip:
-   HYPRE_LIB += -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib\
- -lrocsparse -lrocrand
-endif
 
 # METIS library configuration
 ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK)$(MFEM_USE_MUMPS),NONONO)
@@ -447,11 +428,11 @@ GSLIB_LIB = -L$(GSLIB_DIR)/lib -lgs
 
 # CUDA library configuration
 CUDA_OPT =
-CUDA_LIB = -lcusparse
+CUDA_LIB = -lcusparse -lnvToolsExt
 
-# HIP library configuration
+# HIP library configuration (currently not needed)
 HIP_OPT =
-HIP_LIB = -L$(HIP_DIR)/lib $(XLINKER)-rpath,$(HIP_DIR)/lib -lhipsparse
+HIP_LIB =
 
 # OCCA library configuration
 OCCA_DIR = @MFEM_DIR@/../occa

config/get_hypre_version.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //

config/get_mumps_version.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //

config/githooks/pre-push

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -35,8 +35,8 @@ cd $(git rev-parse --show-toplevel)
 # copyright check
 copyright=true
 if [[ "${option}" == "--copyright" || "${option}" == "" ]]; then
-  if git grep -l "^\(#\|//\).*\(\-2021\|\ 2010,\)" > matches.txt; then
-    echo "Please update the following files to Copyright (c) 2010-2022:"
+  if git grep -l "^\(#\|//\).*\(\-2020\|\ 2010,\)" > matches.txt; then
+    echo "Please update the following files to Copyright (c) 2010-2021:"
     cat matches.txt
     copyright=false
   fi

config/makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/sample-runs.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -32,7 +32,7 @@ groups_serial=(
 '"examples"
    "Examples:"
    "examples"
-   "ex{,1,2,3}[0-9].cpp"'
+   "ex{,1,2}[0-9].cpp"'
 #   "ex1.cpp"'
 '"sundials"
    "SUNDIALS examples:"
@@ -83,13 +83,11 @@ groups_serial=(
 '"tools"
    "Tools miniapps:"
    "miniapps/tools"
-   "convert-dc.cpp display-basis.cpp get-values.cpp load-dc.cpp
-    lor-transfer.cpp"'
+   "convert-dc.cpp display-basis.cpp get-values.cpp load-dc.cpp lor-transfer.cpp"'
 '"toys"
    "Toys miniapps:"
    "miniapps/toys"
-   "automata.cpp life.cpp lissajous.cpp mandel.cpp mondrian.cpp rubik.cpp
-    snake.cpp"'
+   "automata.cpp life.cpp lissajous.cpp mandel.cpp mondrian.cpp rubik.cpp snake.cpp"'
 '"convergence"
    "Convergence tests:"
    "tests/convergence"
@@ -100,7 +98,7 @@ groups_parallel=(
 '"examples"
    "Examples:"
    "examples"
-   "ex{,1,2,3}[0-9]p.cpp"'
+   "ex{,1,2}[0-9]p.cpp"'
 #   "ex1p.cpp"'
 '"sundials"
    "SUNDIALS examples:"
@@ -186,7 +184,7 @@ groups_all=(
 '"examples"
    "Examples:"
    "examples"
-   "ex\"{,1,2,3}[0-9]\"{,p}.cpp"'
+   "ex\"{,1,2}[0-9]\"{,p}.cpp"'
 '"sundials"
    "SUNDIALS examples:"
    "examples/sundials"
@@ -240,8 +238,7 @@ groups_all=(
 '"gslib"
    "GSLIB miniapps:"
    "miniapps/gslib"
-   "field-diff.cpp field-interp.cpp findpts.cpp schwarz_ex1.cpp pfindpts.cpp
-    schwarz_ex1p.cpp"'
+   "field-diff.cpp field-interp.cpp findpts.cpp schwarz_ex1.cpp pfindpts.cpp schwarz_ex1p.cpp"'
 '"navier"
    "Navier miniapps:"
    "miniapps/navier"
@@ -261,13 +258,11 @@ groups_all=(
 '"tools"
    "Tools miniapps:"
    "miniapps/tools"
-   "convert-dc.cpp display-basis.cpp get-values.cpp load-dc.cpp
-    lor-transfer.cpp"'
+   "convert-dc.cpp display-basis.cpp get-values.cpp load-dc.cpp lor-transfer.cpp"'
 '"toys"
    "Toys miniapps:"
    "miniapps/toys"
-   "automata.cpp life.cpp lissajous.cpp mandel.cpp mondrian.cpp rubik.cpp
-    snake.cpp"'
+   "automata.cpp life.cpp lissajous.cpp mandel.cpp mondrian.cpp rubik.cpp snake.cpp"'
 '"convergence"
    "Convergence tests:"
    "tests/convergence"

config/tconfig.hpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //

config/test.mk

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -58,14 +58,14 @@ ifneq (,$(filter test%,$(MAKECMDGOALS)))
    MAKEFLAGS += -k
 endif
 # Test runs of the examples/miniapps with parameters - check exit code:
-# 0 means success, 242 means the test was skipped, anything else means error
+# 0 means success, 255 means the test was skipped, anything else means error
 mfem-test = \
    printf "   $(3) [$(2) $(1) ... ]: "; \
    $(call $(TIMEFUN),$(TIMECMD),$(2) ./$(1) $(if $(5),,-no-vis )$(4) \
      > $(1).stderr 2>&1); \
    err="$$3"; \
    if [ "$$3" = 0 ]; then $(PRINT_OK); \
-   else if [ "$$3" = 242 ]; then $(PRINT_SKIP); err=0; \
+   else if [ "$$3" = 255 ]; then $(PRINT_SKIP); err=0; \
    else $(PRINT_FAILED); cat $(1).stderr; fi; fi; \
    rm -f $(1).stderr; exit $$err
 
@@ -76,7 +76,7 @@ mfem-test-file = \
    $(call $(TIMEFUN),$(TIMECMD),$(2) ./$(1) -no-vis > $(1).stderr 2>&1); \
    err="$$3"; \
    if [ "$$3" = 0 ] && [ -e $(4) ]; then $(PRINT_OK); \
-   else if [ "$$3" = 242 ] && [ -e $(4) ]; then $(PRINT_SKIP); err=0; \
+   else if [ "$$3" = 255 ] && [ -e $(4) ]; then $(PRINT_SKIP); err=0; \
    else $(PRINT_FAILED); cat $(1).stderr; err=64; fi; fi; \
    rm -f $(1).stderr; exit $$err
 

data/hexagon.mesh

@@ -1,43 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-#
-
-dimension
-2
-
-# format: <attribute> <geometry type> <vertex 0> <vertex 1> ...
-elements
-3
-
-1 3 0 1 2 3
-2 3 0 3 4 5
-3 3 0 5 6 1
-
-boundary
-6
-1 1 1 2
-2 1 2 3
-3 1 3 4
-4 1 4 5
-5 1 5 6
-6 1 6 1
-
-vertices
-7
-2
-0.0 0.0
--0.50 -0.8660254037844386
-0.50 -0.8660254037844386
-1.0 0.0
-0.50 0.8660254037844386
--0.50 0.8660254037844386
--1.0 0.0

data/star-q3.fms

@@ -15,146 +15,146 @@ DataCollection/Fields/0/NumberOfVectorComponents: 2
 DataCollection/Fields/0/FieldDescriptorName: CoordsDescriptor
 DataCollection/Fields/0/Data/Size: 422
 DataCollection/Fields/0/Data/Type: FMS_DOUBLE
-DataCollection/Fields/0/Data/Values: [-0.016886, 1.000000, 0.309017,
-1.309020, -0.809017, -0.500000,
--0.809017, -1.618030, 0.309017,
--0.500000, 1.309020, 0.519420,
-1.154510, 0.809019, 0.147680,
--0.095492, -0.654508, -0.415586,
--1.213520, -1.213520, -0.392210,
--0.654508, -0.095492, 0.139949,
-0.809019, 1.154510, 0.660184,
--0.264063, -0.800064, -0.231060,
-0.663691, 0.183114, 0.317639,
-0.543082, 0.598483, 0.345112,
-0.478298, 0.027703, 0.095229,
-0.012368, -0.092534, -0.334412,
--0.313767, -0.140526, -0.293881,
--0.534056, -0.660290, -0.537646,
--0.655590, -0.121396, -0.274504,
--0.346497, -0.296570, 0.004737,
--0.098835, 0.069287, 0.082675,
-0.318799, 0.467183, 0.564505,
-0.595190, 0.846237, 0.671735,
-1.051500, 1.103010, 0.964008,
-0.821603, 1.257520, 1.206010,
-1.142350, 0.975686, 0.781273,
-0.717257, 0.475684, 0.642352,
-0.268930, 0.211049, 0.174181,
-0.039345, -0.147746, -0.177481,
--0.365164, -0.230328, -0.551503,
--0.603005, -0.497587, -0.389864,
--0.757514, -0.706011, -0.675487,
--0.528946, -0.943851, -1.078690,
--1.087600, -0.955467, -1.483190,
--1.348360, -1.483190, -1.348360,
--1.085930, -0.938010, -0.943851,
--1.078690, -0.681476, -0.540944,
--0.757514, -0.706011, -0.540614,
--0.367058, -0.551503, -0.603005,
--0.365164, -0.230328, -0.138552,
--0.206896, 0.174181, 0.039345,
-0.268468, 0.222269, 0.475684,
-0.642352, 0.759791, 0.719381,
-1.142350, 0.975686, 1.257520,
-1.206010, 0.972837, 0.836119,
-1.051500, 1.103010, 0.214572,
-0.407449, 0.288323, 0.449827,
--0.086700, -0.027358, -0.200560,
--0.166595, -0.271802, -0.418426,
--0.426131, -0.551441, -0.096117,
--0.206969, -0.027946, -0.184969,
-0.211136, 0.260131, 0.407172,
-0.430781, 0.718277, 0.885068,
-0.753103, 0.957692, 0.866273,
-1.024530, 0.934099, 1.093820,
-0.348422, 0.524463, 0.404903,
-0.587376, 0.054525, 0.146431,
--0.078026, -0.007795, -0.329488,
--0.302967, -0.488115, -0.439332,
--0.498515, -0.453527, -0.633059,
--0.570251, -0.655787, -0.791132,
--0.802013, -0.956872, -1.094720,
--1.208560, -1.207720, -1.339910,
--0.693371, -0.795404, -0.803594,
--0.945068, -0.464668, -0.631721,
--0.458968, -0.547876, -0.332878,
--0.485696, -0.286385, -0.408481,
-0.053021, -0.058200, 0.110846,
--0.002086, 0.381384, 0.416784,
-0.551133, 0.613261, 0.872474,
-0.901208, 1.038300, 1.084660,
-0.737459, 0.751250, 0.890002,
-0.915210, 0.010915, 0.000000,
-0.951057, 0.951057, 0.587785,
-1.538840, -0.587785, 0.000000,
--0.951057, -1.538840, -0.951057,
--0.015847, 0.475529, 0.951057,
-0.492248, 1.244950, 1.063310,
-0.274399, 0.293893, -0.293892,
--0.296404, -1.063310, -1.244950,
--0.453865, -0.951057, -0.475529,
-0.466620, 0.792932, -0.013913,
--0.748783, -0.497528, 0.021382,
--0.017158, 0.172591, 0.330125,
-0.458568, 0.457971, 0.137740,
-0.299049, 0.588394, 0.667324,
-0.432341, 0.634346, 0.117322,
-0.193603, 0.211702, 0.098278,
--0.199438, -0.077304, -0.082243,
--0.216296, -0.458634, -0.592374,
--0.563926, -0.680404, -0.135751,
--0.302942, -0.469005, -0.453640,
--0.182727, -0.314240, 0.024270,
-0.021546, 0.158510, 0.317019,
-0.485799, 0.492951, 0.792548,
-0.634038, 0.951057, 0.951057,
-0.777915, 0.613430, 0.951057,
-0.951057, 0.793994, 0.635800,
-1.049020, 1.146990, 1.084480,
-0.924310, 1.440880, 1.342910,
-1.380330, 1.221820, 0.948209,
-0.856297, 0.746293, 0.904802,
-0.476242, 0.393234, 0.489821,
-0.391857, 0.194471, 0.075751,
-0.097964, 0.195929, -0.097964,
--0.195928, -0.173234, -0.078922,
--0.489821, -0.391856, -0.467007,
--0.397859, -0.746293, -0.904802,
--0.945206, -0.849559, -1.380330,
--1.221820, -1.440880, -1.342910,
--1.100830, -0.923191, -1.049020,
--1.146990, -0.774515, -0.621542,
--0.951057, -0.951057, -0.803055,
--0.635255, -0.951057, -0.951057,
--0.792548, -0.634038, -0.454301,
--0.479369, -0.158510, -0.317019,
-0.149331, 0.178643, 0.295860,
-0.306275, 0.246225, 0.406610,
-0.367954, 0.498458, -0.014929,
-0.084917, -0.094272, -0.000726,
--0.245374, -0.340755, -0.435351,
--0.490564, -0.176355, -0.294974,
--0.148010, -0.328246, 0.136248,
-0.182883, 0.328957, 0.309903,
-0.646446, 0.622546, 0.800859,
-0.801803, 0.643087, 0.654473,
-0.796963, 0.816799, 0.748189,
-0.898148, 0.819092, 0.970033,
-1.049540, 1.162660, 1.115310,
-1.266700, 0.539960, 0.728396,
-0.656318, 0.783152, 0.302716,
-0.403858, 0.219006, 0.309252,
-0.009667, 0.107161, -0.118211,
-0.011786, -0.284704, -0.179858,
--0.411567, -0.296676, -0.570168,
--0.635348, -0.726784, -0.793211,
--1.016860, -1.112970, -1.170910,
--1.281530, -0.737860, -0.851723,
--0.883732, -0.995070, -0.653153,
--0.769939, -0.631918, -0.797688,
--0.616289, -0.806819, -0.638485,
--0.790356, -0.136399, -0.322769,
+DataCollection/Fields/0/Data/Values: [-0.016886, 1.000000, 0.309017, 
+1.309020, -0.809017, -0.500000, 
+-0.809017, -1.618030, 0.309017, 
+-0.500000, 1.309020, 0.519420, 
+1.154510, 0.809019, 0.147680, 
+-0.095492, -0.654508, -0.415586, 
+-1.213520, -1.213520, -0.392210, 
+-0.654508, -0.095492, 0.139949, 
+0.809019, 1.154510, 0.660184, 
+-0.264063, -0.800064, -0.231060, 
+0.663691, 0.183114, 0.317639, 
+0.543082, 0.598483, 0.345112, 
+0.478298, 0.027703, 0.095229, 
+0.012368, -0.092534, -0.334412, 
+-0.313767, -0.140526, -0.293881, 
+-0.534056, -0.660290, -0.537646, 
+-0.655590, -0.121396, -0.274504, 
+-0.346497, -0.296570, 0.004737, 
+-0.098835, 0.069287, 0.082675, 
+0.318799, 0.467183, 0.564505, 
+0.595190, 0.846237, 0.671735, 
+1.051500, 1.103010, 0.964008, 
+0.821603, 1.257520, 1.206010, 
+1.142350, 0.975686, 0.781273, 
+0.717257, 0.475684, 0.642352, 
+0.268930, 0.211049, 0.174181, 
+0.039345, -0.147746, -0.177481, 
+-0.365164, -0.230328, -0.551503, 
+-0.603005, -0.497587, -0.389864, 
+-0.757514, -0.706011, -0.675487, 
+-0.528946, -0.943851, -1.078690, 
+-1.087600, -0.955467, -1.483190, 
+-1.348360, -1.483190, -1.348360, 
+-1.085930, -0.938010, -0.943851, 
+-1.078690, -0.681476, -0.540944, 
+-0.757514, -0.706011, -0.540614, 
+-0.367058, -0.551503, -0.603005, 
+-0.365164, -0.230328, -0.138552, 
+-0.206896, 0.174181, 0.039345, 
+0.268468, 0.222269, 0.475684, 
+0.642352, 0.759791, 0.719381, 
+1.142350, 0.975686, 1.257520, 
+1.206010, 0.972837, 0.836119, 
+1.051500, 1.103010, 0.214572, 
+0.407449, 0.288323, 0.449827, 
+-0.086700, -0.027358, -0.200560, 
+-0.166595, -0.271802, -0.418426, 
+-0.426131, -0.551441, -0.096117, 
+-0.206969, -0.027946, -0.184969, 
+0.211136, 0.260131, 0.407172, 
+0.430781, 0.718277, 0.885068, 
+0.753103, 0.957692, 0.866273, 
+1.024530, 0.934099, 1.093820, 
+0.348422, 0.524463, 0.404903, 
+0.587376, 0.054525, 0.146431, 
+-0.078026, -0.007795, -0.329488, 
+-0.302967, -0.488115, -0.439332, 
+-0.498515, -0.453527, -0.633059, 
+-0.570251, -0.655787, -0.791132, 
+-0.802013, -0.956872, -1.094720, 
+-1.208560, -1.207720, -1.339910, 
+-0.693371, -0.795404, -0.803594, 
+-0.945068, -0.464668, -0.631721, 
+-0.458968, -0.547876, -0.332878, 
+-0.485696, -0.286385, -0.408481, 
+0.053021, -0.058200, 0.110846, 
+-0.002086, 0.381384, 0.416784, 
+0.551133, 0.613261, 0.872474, 
+0.901208, 1.038300, 1.084660, 
+0.737459, 0.751250, 0.890002, 
+0.915210, 0.010915, 0.000000, 
+0.951057, 0.951057, 0.587785, 
+1.538840, -0.587785, 0.000000, 
+-0.951057, -1.538840, -0.951057, 
+-0.015847, 0.475529, 0.951057, 
+0.492248, 1.244950, 1.063310, 
+0.274399, 0.293893, -0.293892, 
+-0.296404, -1.063310, -1.244950, 
+-0.453865, -0.951057, -0.475529, 
+0.466620, 0.792932, -0.013913, 
+-0.748783, -0.497528, 0.021382, 
+-0.017158, 0.172591, 0.330125, 
+0.458568, 0.457971, 0.137740, 
+0.299049, 0.588394, 0.667324, 
+0.432341, 0.634346, 0.117322, 
+0.193603, 0.211702, 0.098278, 
+-0.199438, -0.077304, -0.082243, 
+-0.216296, -0.458634, -0.592374, 
+-0.563926, -0.680404, -0.135751, 
+-0.302942, -0.469005, -0.453640, 
+-0.182727, -0.314240, 0.024270, 
+0.021546, 0.158510, 0.317019, 
+0.485799, 0.492951, 0.792548, 
+0.634038, 0.951057, 0.951057, 
+0.777915, 0.613430, 0.951057, 
+0.951057, 0.793994, 0.635800, 
+1.049020, 1.146990, 1.084480, 
+0.924310, 1.440880, 1.342910, 
+1.380330, 1.221820, 0.948209, 
+0.856297, 0.746293, 0.904802, 
+0.476242, 0.393234, 0.489821, 
+0.391857, 0.194471, 0.075751, 
+0.097964, 0.195929, -0.097964, 
+-0.195928, -0.173234, -0.078922, 
+-0.489821, -0.391856, -0.467007, 
+-0.397859, -0.746293, -0.904802, 
+-0.945206, -0.849559, -1.380330, 
+-1.221820, -1.440880, -1.342910, 
+-1.100830, -0.923191, -1.049020, 
+-1.146990, -0.774515, -0.621542, 
+-0.951057, -0.951057, -0.803055, 
+-0.635255, -0.951057, -0.951057, 
+-0.792548, -0.634038, -0.454301, 
+-0.479369, -0.158510, -0.317019, 
+0.149331, 0.178643, 0.295860, 
+0.306275, 0.246225, 0.406610, 
+0.367954, 0.498458, -0.014929, 
+0.084917, -0.094272, -0.000726, 
+-0.245374, -0.340755, -0.435351, 
+-0.490564, -0.176355, -0.294974, 
+-0.148010, -0.328246, 0.136248, 
+0.182883, 0.328957, 0.309903, 
+0.646446, 0.622546, 0.800859, 
+0.801803, 0.643087, 0.654473, 
+0.796963, 0.816799, 0.748189, 
+0.898148, 0.819092, 0.970033, 
+1.049540, 1.162660, 1.115310, 
+1.266700, 0.539960, 0.728396, 
+0.656318, 0.783152, 0.302716, 
+0.403858, 0.219006, 0.309252, 
+0.009667, 0.107161, -0.118211, 
+0.011786, -0.284704, -0.179858, 
+-0.411567, -0.296676, -0.570168, 
+-0.635348, -0.726784, -0.793211, 
+-1.016860, -1.112970, -1.170910, 
+-1.281530, -0.737860, -0.851723, 
+-0.883732, -0.995070, -0.653153, 
+-0.769939, -0.631918, -0.797688, 
+-0.616289, -0.806819, -0.638485, 
+-0.790356, -0.136399, -0.322769, 
 -0.165339, -0.309622]
 DataCollection/Mesh/PartitionInfo/Size: 2
 DataCollection/Mesh/PartitionInfo/Type: FMS_UINT64
@@ -170,69 +170,69 @@ DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/EntityType: FMS_EDGE
 DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/NumEntities: 50
 DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Size: 100
 DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Type: FMS_INT32
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Values: [11, 0, 26,
-11, 26, 14,
-14, 0, 27,
-14, 27, 17,
-17, 0, 28,
-17, 28, 20,
-20, 0, 29,
-20, 29, 23,
-23, 0, 30,
-23, 30, 11,
-11, 1, 12,
-1, 26, 12,
-12, 3, 13,
-3, 26, 13,
-13, 2, 14,
-2, 15, 2,
-27, 15, 15,
-5, 16, 5,
-27, 16, 16,
-4, 17, 4,
-18, 4, 28,
-18, 18, 7,
-19, 7, 28,
-19, 19, 6,
-20, 6, 21,
-6, 29, 21,
-21, 9, 22,
-9, 29, 22,
-22, 8, 23,
-8, 24, 8,
-30, 24, 24,
-10, 25, 10,
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Values: [11, 0, 26, 
+11, 26, 14, 
+14, 0, 27, 
+14, 27, 17, 
+17, 0, 28, 
+17, 28, 20, 
+20, 0, 29, 
+20, 29, 23, 
+23, 0, 30, 
+23, 30, 11, 
+11, 1, 12, 
+1, 26, 12, 
+12, 3, 13, 
+3, 26, 13, 
+13, 2, 14, 
+2, 15, 2, 
+27, 15, 15, 
+5, 16, 5, 
+27, 16, 16, 
+4, 17, 4, 
+18, 4, 28, 
+18, 18, 7, 
+19, 7, 28, 
+19, 19, 6, 
+20, 6, 21, 
+6, 29, 21, 
+21, 9, 22, 
+9, 29, 22, 
+22, 8, 23, 
+8, 24, 8, 
+30, 24, 24, 
+10, 25, 10, 
 30, 25, 25, 1]
 DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/EntityType: FMS_QUADRILATERAL
 DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/NumEntities: 20
 DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Size: 80
 DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Type: FMS_INT32
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Values: [0, 1, 2,
-3, 3, 4,
-5, 6, 6,
-7, 8, 9,
-9, 10, 11,
-12, 12, 13,
-14, 0, 15,
-16, 17, 1,
-17, 18, 19,
-20, 2, 20,
-21, 22, 22,
-23, 24, 4,
-24, 25, 26,
-27, 5, 27,
-28, 29, 29,
-30, 31, 7,
-31, 32, 33,
-34, 8, 34,
-35, 36, 36,
-37, 38, 10,
-38, 39, 40,
-41, 11, 41,
-42, 43, 43,
-44, 45, 13,
-45, 46, 47,
-48, 14, 48,
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Values: [0, 1, 2, 
+3, 3, 4, 
+5, 6, 6, 
+7, 8, 9, 
+9, 10, 11, 
+12, 12, 13, 
+14, 0, 15, 
+16, 17, 1, 
+17, 18, 19, 
+20, 2, 20, 
+21, 22, 22, 
+23, 24, 4, 
+24, 25, 26, 
+27, 5, 27, 
+28, 29, 29, 
+30, 31, 7, 
+31, 32, 33, 
+34, 8, 34, 
+35, 36, 36, 
+37, 38, 10, 
+38, 39, 40, 
+41, 11, 41, 
+42, 43, 43, 
+44, 45, 13, 
+45, 46, 47, 
+48, 14, 48, 
 49, 15]
 DataCollection/Mesh/Components/0/Name: volume
 DataCollection/Mesh/Components/0/Dimension: 2

doc/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

doc/CodeDocumentation.conf.in

@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = v4.4.0
+PROJECT_NUMBER         = v4.3.1
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

doc/CodeDocumentation.dox

@@ -98,11 +98,6 @@ namespace mfem {
  * - <a class="el" href="ex28p_8cpp_source.html">Example 28p</a>: parallel sliding contact in elasticity
  * - <a class="el" href="ex29_8cpp_source.html">Example 29</a>: Laplace solve on a 3D-embedded surface
  * - <a class="el" href="ex29p_8cpp_source.html">Example 29p</a>: parallel Laplace solve on a 3D-embedded surface
- * - <a class="el" href="ex30_8cpp_source.html">Example 30</a>: mesh preprocessing to resolve problem data
- * - <a class="el" href="ex30p_8cpp_source.html">Example 30p</a>: parallel mesh preprocessing to resolve problem data
- * - <a class="el" href="ex31_8cpp_source.html">Example 31</a>: Nedelec H(curl) FEM for the definite anisotropic Maxwell problem
- * - <a class="el" href="ex31p_8cpp_source.html">Example 31p</a>: parallel Nedelec H(curl) FEM for the definite anisotropic Maxwell problem
- * - <a class="el" href="ex32p_8cpp_source.html">Example 32p</a>: parallel anisotropic Maxwell eigensolver
  *
  * <H4>AmgX Examples</H4>
  * - Variants of Examples
@@ -195,13 +190,12 @@ namespace mfem {
  * - <a class="el" href="field-interp_8cpp_source.html">Field Interp</a>: transfer a grid functions between meshes
  * - <a class="el" href="distance_8cpp_source.html">Distance</a>: finite element distance function solver
  * - <a class="el" href="diffusion_8cpp_source.html">Shifted Diffusion</a>: shifted boundary diffusion solver
- * - <a class="el" href="extrapolate_8cpp_source.html">Extrapolation</a>: PDE-based extrapolation of finite element functions
  * - <a class="el" href="distance_8cpp_source.html">Block Solvers</a>: comparison of saddle point system solvers
  * - <a class="el" href="parheat_8cpp_source.html">Optimization gradients</a>: Gradients of PDE-constrained function
- * - <a class="el" href="par__example_8cpp_source.html">Parallel AD</a>: Parallel p-Laplacian example
- * - <a class="el" href="seq__example_8cpp_source.html">Serial AD</a>: Serial p-Laplacian example
  * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
  * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
+ * - <a class="el" href="par__example_8cpp_source.html">Parallel pLaplacian example using AD</a>: Parallel pLaplacian example
+ * - <a class="el" href="seq__example_8cpp_source.html">Serial pLaplacian example using AD</a>: Serial pLaplacian example
  *
  * See also the <a class="el" href="https://mfem.org/examples/">examples documentation</a> online.
  */

doc/README

@@ -21,7 +21,8 @@ create the file CodeDocumentation.html, which can be viewed in any web browser.
 
 Alternatively, the (pre-build) documentation can be browsed directly online at
 
-   https://docs.mfem.org
+   https://mfem.github.io/doxygen/html/index.html
+
 
 Some building considerations:
 

doc/makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -16,7 +16,7 @@ DOXYGEN_CONF = CodeDocumentation.conf
 # doxygen uses: graphviz, latex
 html: $(DOXYGEN_CONF)
 	@# Generate the html documentation
-	@( cat $(DOXYGEN_CONF) ; echo "$(MFEM_DOXYGEN_FLAGS)" ) | doxygen -
+	@doxygen $(DOXYGEN_CONF)
 	@echo "<meta http-equiv=\"REFRESH\" content=\"0;URL=CodeDocumentation/html/index.html\">" > CodeDocumentation.html
 	@cat warnings.log 1>&2
 	@# Generate the log of undocumented methods
@@ -29,3 +29,4 @@ clean:
 $(DOXYGEN_CONF): $(MFEM_DIR)/doc/$(DOXYGEN_CONF).in
 	@sed -e 's%@MFEM_SOURCE_DIR@%$(MFEM_DIR)%g' $(<) \
 	  > $(DOXYGEN_CONF)
+

examples/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -38,7 +38,6 @@ list(APPEND ALL_EXE_SRCS
   ex28.cpp
   ex29.cpp
   ex30.cpp
-  ex31.cpp
   )
 
 if (MFEM_USE_MPI)
@@ -73,8 +72,6 @@ if (MFEM_USE_MPI)
     ex28p.cpp
     ex29p.cpp
     ex30p.cpp
-    ex31p.cpp
-    ex32p.cpp
     )
 endif()
 

examples/amgx/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

examples/amgx/ex1p.cpp

@@ -37,11 +37,11 @@ using namespace mfem;
 
 int main(int argc, char *argv[])
 {
-   // 1. Initialize MPI and HYPRE.
-   Mpi::Init(argc, argv);
-   int num_procs = Mpi::WorldSize();
-   int myid = Mpi::WorldRank();
-   Hypre::Init();
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
 
    // 2. Parse command-line options.
    const char *mesh_file = "../../data/star.mesh";
@@ -87,6 +87,7 @@ int main(int argc, char *argv[])
       {
          args.PrintUsage(cout);
       }
+      MPI_Finalize();
       return 1;
    }
    if (myid == 0)
@@ -319,6 +320,7 @@ int main(int argc, char *argv[])
    {
       delete fec;
    }
+   MPI_Finalize();
 
    return 0;
 }

examples/amgx/makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

examples/caliper/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

examples/caliper/ex1p.cpp

@@ -58,11 +58,11 @@ using namespace mfem;
 
 int main(int argc, char *argv[])
 {
-   // 1. Initialize MPI and HYPRE.
-   Mpi::Init(argc, argv);
-   int num_procs = Mpi::WorldSize();
-   int myid = Mpi::WorldRank();
-   Hypre::Init();
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
    // Define Caliper ConfigManager
    cali::ConfigManager mgr;
    // Caliper instrumentation
@@ -101,6 +101,7 @@ int main(int argc, char *argv[])
       {
          args.PrintUsage(cout);
       }
+      MPI_Finalize();
       return 1;
    }
    if (myid == 0)
@@ -292,6 +293,7 @@ int main(int argc, char *argv[])
    }
    // Flush output before MPI_finalize
    mgr.flush();
+   MPI_Finalize();
 
    return 0;
 }

examples/caliper/makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

examples/dpg_tests/acoustics/fosls.cpp

@@ -1,294 +0,0 @@
-//                 MFEM FOSLS acoustics Example
-//
-// Compile with: make fosls
-//
-// Definite/Indefinite Helmholtz
-
-//     - Δ p ± ω^2 p = f̃ ,   in Ω
-//                 p = p_0, on ∂Ω
-
-// First Order System
-
-//   ∇ p - ω u = 0, in Ω
-// - ∇⋅u ± ω p = f, in Ω
-//           p = p_0, in ∂Ω
-// where f:=f̃/ω 
-
-// FOSLS:
-//       minimize  1/2(||∇p - ω u||^2 + ||-∇⋅u ± ω p - f||^2)
-
-// (p,u) ∈ H^1(Ω) × H(div,Ω)
-// -------------------------------------------------------------------
-// |   |            p           |            u             |   RHS    | 
-// -------------------------------------------------------------------
-// | q |  (∇ p,∇ q) + ω^2(p,q)  | ∓ ω (∇⋅u,q) - ω (u, ∇ q) | ± ω(f,q) |
-// |   |                        |                          |          |
-// | v | ∓ ω (p,∇⋅v) - ω (∇ p,v)|  (∇⋅u,∇⋅v) + ω^2 (u,v)   | -(f,∇⋅v) |
-
-// where (q,v) ∈ H^1(Ω) × H(div,Ω)
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// #define DEFINITE
-
-double p_exact(const Vector &x);
-void u_exact(const Vector &x, Vector & u);
-double rhs_func(const Vector &x);
-void gradp_exact(const Vector &x, Vector &gradu);
-double divu_exact(const Vector &x);
-double d2_exact(const Vector &x);
-
-int dim;
-double omega;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../../data/inline-quad.mesh";
-   int order = 1;
-   bool visualization = true;
-   double rnum=1.0;
-   int sr = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree)");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
-                  "Number of wavelengths");      
-   args.AddOption(&sr, "-sr", "--serial_ref",
-                  "Number of serial refinements.");                               
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   omega = 2.0 * M_PI * rnum;
-
-
-   Mesh mesh(mesh_file, 1, 1);
-   dim = mesh.Dimension();
-
-   for (int i = 0; i < sr; i++ )
-   {
-      mesh.UniformRefinement();
-   }
-
-   FiniteElementCollection *H1fec = new H1_FECollection(order, dim);
-   FiniteElementCollection *RTfec = new RT_FECollection(order-1, dim);
-   FiniteElementSpace * H1fes = new FiniteElementSpace(&mesh, H1fec);
-   FiniteElementSpace * RTfes = new FiniteElementSpace(&mesh, RTfec);
-
-   Array<FiniteElementSpace *> fespaces(2);
-   fespaces[0] = H1fes;
-   fespaces[1] = RTfes;
-
-   Array<int> ess_bdr;
-   Array<int> ess_tdof_list;
-   if (mesh.bdr_attributes.Size())
-   {
-      ess_bdr.SetSize(mesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      fespaces[0]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   BlockBilinearForm a(fespaces);
-   a.SetDiagonalPolicy(mfem::Operator::DIAG_KEEP);
-   cout << "H1 fespace = " << H1fes->GetTrueVSize() << endl;
-   cout << "RT fespace = " << RTfes->GetTrueVSize() << endl;
-
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient negone(-1.0);
-   ConstantCoefficient omeg(omega);
-   ConstantCoefficient negomeg(-omega);
-   ConstantCoefficient omeg2(omega*omega);
-
-
-   Array2D<BilinearFormIntegrator * > blfi(2,2);
-
-   // blfi(0,0) = (∇ p,∇ q) + ω^2(p,q)
-   SumIntegrator * integ00 = new SumIntegrator();
-   integ00->AddIntegrator(new DiffusionIntegrator(one));
-   integ00->AddIntegrator(new MassIntegrator(omeg2));
-   blfi(0,0) = integ00;
-
-   // blfi(0,1) = ∓ ω (∇⋅u,q) - ω (u, ∇ q)
-   SumIntegrator * integ01 = new SumIntegrator();
-#ifdef DEFINITE
-   // -ω (∇⋅u,q)
-   integ01->AddIntegrator(new MixedScalarDivergenceIntegrator(negomeg));
-#else   
-   // ω (∇⋅u,q)
-   integ01->AddIntegrator(new MixedScalarDivergenceIntegrator(omeg));
-#endif
-   // - ω (u, ∇ q)
-   integ01->AddIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg));
-   blfi(0,1) = integ01;
-
-   // blfi(1,0) = ∓ ω (p,∇⋅v) - ω (∇ p,v)
-   SumIntegrator * integ10 = new SumIntegrator();
-#ifdef DEFINITE
-   // - ω (p,∇⋅v)
-   integ10->AddIntegrator(new MixedScalarWeakGradientIntegrator(omeg));
-#else
-   // ω (p,∇⋅v)
-   integ10->AddIntegrator(new MixedScalarWeakGradientIntegrator(negomeg));
-#endif
-   // - ω (∇ p,v)
-   integ10->AddIntegrator(new MixedVectorGradientIntegrator(negomeg));
-   blfi(1,0) = integ10;
-
-   // blfi(1,1) = (∇⋅u,∇⋅v) + ω^2 (u,v)
-   SumIntegrator * integ11 = new SumIntegrator();
-   integ11->AddIntegrator(new DivDivIntegrator(one));
-   integ11->AddIntegrator(new VectorFEMassIntegrator(omeg2));
-   blfi(1,1) = integ11;
-
-
-   BlockLinearForm b(fespaces);
-   Array<LinearFormIntegrator * > lfi(2);
-   // ± ω (f,q)
-   FunctionCoefficient f_rhs(rhs_func);
-#ifdef DEFINITE  
-   ProductCoefficient w_f(omeg,f_rhs);
-#else   
-   ProductCoefficient w_f(negomeg,f_rhs);
-#endif   
-   // lfi[0] = new DomainLFIntegrator(w_f);
-   lfi[0] = new DomainLFIntegrator(w_f);
-
-
-   //  -(f,∇⋅v)
-   ProductCoefficient neg_f(negone,f_rhs);
-   // lfi[1] = new VectorFEDomainLFDivIntegrator(f_rhs);
-   lfi[1] = new VectorFEDomainLFDivIntegrator(neg_f);
-
-   TestBlockBilinearFormIntegrator * integ = new TestBlockBilinearFormIntegrator();
-   integ->SetIntegrators(blfi);
-   a.AddDomainIntegrator(integ);
-   a.Assemble();
-
-   TestBlockLinearFormIntegrator * lininteg = new TestBlockLinearFormIntegrator();
-   lininteg->SetIntegrators(lfi);
-   b.AddDomainIntegrator(lininteg);
-   b.Assemble();
-
-   int size = 0;
-   for (int i = 0; i<fespaces.Size(); i++)
-   {
-      size += fespaces[i]->GetVSize();
-   }
-
-   Vector x(size);
-   x = 0.0;
-   FunctionCoefficient p_ex(p_exact);
-   VectorFunctionCoefficient gradp_ex(dim,gradp_exact);
-   VectorFunctionCoefficient u_ex(dim,u_exact);
-   FunctionCoefficient divu_ex(divu_exact);
-   GridFunction p_gf, u_gf;   
-   GridFunction pex_gf(H1fes);   
-
-   p_gf.MakeRef(H1fes,x,0);
-   // p_gf.ProjectBdrCoefficient(p_ex,ess_bdr);
-   p_gf.ProjectCoefficient(p_ex);
-   pex_gf.ProjectCoefficient(p_ex);
-
-   u_gf.MakeRef(RTfes,x,H1fes->GetVSize());
-   u_gf = 0.;
-
-
-   OperatorPtr A;
-   Vector X,B;
-   a.FormLinearSystem(ess_tdof_list,x,b,A,X,B);
-
-   GSSmoother M((SparseMatrix&)(*A));
-   CGSolver cg;
-   cg.SetRelTol(1e-10);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(1);
-   cg.SetPreconditioner(M);
-   cg.SetOperator(*A);
-   cg.Mult(B, X);
-
-   a.RecoverFEMSolution(X,b,x);
-
-   p_gf.MakeRef(H1fes,x,0);
-   u_gf.MakeRef(RTfes,x,H1fes->GetVSize());
-
-
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream solu_sock(vishost, visport);
-      solu_sock.precision(8);
-      solu_sock << "solution\n" << mesh << p_gf <<
-                "window_title 'Numerical p' "
-                << flush;
-      // socketstream sols_sock(vishost, visport);
-      // sols_sock.precision(8);
-      // sols_sock << "solution\n" << mesh << u_gf <<
-               //  "window_title 'Numerical sigma' "
-               //  << flush;
-
-      socketstream solex_sock(vishost, visport);
-      solex_sock.precision(8);
-      solex_sock << "solution\n" << mesh << pex_gf <<
-                    "window_title 'Exact p' "
-                    << flush;          
-   }
-
-   return 0;
-}
-
-
-double rhs_func(const Vector &x)
-{
-   double p = p_exact(x);
-   double divu = divu_exact(x);
-   // f = - ∇⋅u ± ω p, 
-#ifdef DEFINITE   
-   return -divu + omega * p;
-#else
-   return -divu - omega * p;
-#endif   
-}
-
-double p_exact(const Vector &x)
-{
-   return sin(omega*x.Sum());
-}
-
-void gradp_exact(const Vector &x, Vector &grad)
-{
-   grad.SetSize(x.Size());
-   grad = omega * cos(omega * x.Sum());
-}
-
-void u_exact(const Vector &x, Vector & u)
-{
-   gradp_exact(x,u);
-   u *= 1./omega;
-}
-
-double divu_exact(const Vector &x)
-{
-   return d2_exact(x)/omega;
-}
-
-double d2_exact(const Vector &x)
-{
-   return -dim * omega * omega * sin(omega*x.Sum());
-}

examples/dpg_tests/acoustics/makefile

@@ -1,59 +0,0 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Use the MFEM build directory
-MFEM_DIR ?= ../../..
-MFEM_BUILD_DIR ?= ../../..
-SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/acoustics,)
-CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
-
-MFEM_LIB_FILE = mfem_is_not_built
--include $(CONFIG_MK)
-
-SEQ_EXAMPLES = fosls uw_dpg  strong_dpg
-PAR_EXAMPLES = uw_dpgp
-
-ifeq ($(MFEM_USE_MPI),NO)
-   EXAMPLES = $(SEQ_EXAMPLES)
-else
-   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
-endif
-
-.SUFFIXES:
-.SUFFIXES: .o .cpp .mk
-.PHONY: all clean clean-build clean-exec
-
-# Remove built-in rule
-%: %.cpp
-
-# Replace the default implicit rule for *.cpp files
-%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
-
-all: $(EXAMPLES) 
-
-MFEM_TESTS = EXAMPLES
-include $(MFEM_TEST_MK)
-
-# Testing: Parallel vs. serial runs
-RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
-%-test-par: %
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
-%-test-seq: %
-	@$(call mfem-test,$<,, Serial example)
-
-clean: clean-build clean-exec 
-
-clean-build:
-	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
-	rm -rf *.dSYM *.TVD.*breakpoints
-
-clean-exec:

examples/dpg_tests/acoustics/strong_dpg.cpp

@@ -1,271 +0,0 @@
-//                 MFEM DPG_strong acoustics Example
-//
-// Compile with: make strong_dpg
-//
-// Definite/Indefinite Helmholtz
-
-//     - Δ p ± ω^2 p = f̃ ,   in Ω
-//                 p = p_0, on ∂Ω
-
-// First Order System
-
-//   ∇ p - ω u = 0, in Ω
-// - ∇⋅u ± ω p = f, in Ω
-//           p = p_0, in ∂Ω
-// where f:=f̃/ω 
-
-// Strong DPG formulation
-// (p,u) ∈ H^1(Ω) × H(div,Ω)
-// 
-//   (∇ p, v) - ω (u,v) = 0,     in Ω, ∀ v ∈ (L^2)^dim
-//  -(∇⋅u, q) ± ω (p,q) = (f,q), in Ω, ∀ q ∈  L^2 
-//                    p = p_0, in ∂Ω
-// 
-// ------------------------------------
-// |   |      p    |     u    |  RHS  | 
-// ------------------------------------
-// | q | ± ω (p,q) | -(∇⋅u,q) | (f,q) |
-// |   |           |          |       |
-// | v | (∇ p, v)  | -ω (u,v) |       |
-
-// where (q,v) ∈ L^2  × (L^2)^dim
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// #define DEFINITE
-
-double p_exact(const Vector &x);
-void u_exact(const Vector &x, Vector & u);
-double rhs_func(const Vector &x);
-void gradp_exact(const Vector &x, Vector &gradu);
-double divu_exact(const Vector &x);
-double d2_exact(const Vector &x);
-
-int dim;
-double omega;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../../data/inline-quad.mesh";
-   int order = 1;
-   int delta_order = 1;
-   bool visualization = true;
-   double rnum=1.0;
-   int ref = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree)");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
-                  "Number of wavelengths");      
-   args.AddOption(&delta_order, "-do", "--delta_order",
-                  "Order enrichment for DPG test space.");                  
-   args.AddOption(&ref, "-ref", "--serial_ref",
-                  "Number of serial refinements.");                               
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   omega = 2.0 * M_PI * rnum;
-
-
-   Mesh mesh(mesh_file, 1, 1);
-   dim = mesh.Dimension();
-
-
-   for (int i = 0; i < ref; i++ )
-   {
-      mesh.UniformRefinement();
-   }
-
-   // Define spaces
-   // H1 space for p
-   FiniteElementCollection *p_fec = new H1_FECollection(order, dim);
-   FiniteElementSpace * p_fes = new FiniteElementSpace(&mesh, p_fec);
-
-   // H(div) for u
-   FiniteElementCollection *u_fec = new RT_FECollection(order-1, dim);
-   FiniteElementSpace * u_fes = new FiniteElementSpace(&mesh, u_fec);
-
-   // testspace fe collections
-   int test_order = order+delta_order;
-   FiniteElementCollection * q_fec = new L2_FECollection(test_order-1, dim);
-   FiniteElementCollection * v_fec = new L2_FECollection(test_order-1, dim);
-
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient negone(-1.0);
-   ConstantCoefficient omeg(omega);
-   ConstantCoefficient negomeg(-omega);
-
-   // Normal equation weak formulation
-   Array<FiniteElementSpace * > trial_fes; 
-   Array<FiniteElementCollection * > test_fec; 
-
-   trial_fes.Append(p_fes);
-   trial_fes.Append(u_fes);
-   test_fec.Append(q_fec);
-   test_fec.Append(v_fec);
-
-   NormalEquations * a = new NormalEquations(trial_fes,test_fec);
-   a->SetTestFECollVdim(1,dim);
-   
-   a->StoreMatrices(true);
-
-//  ± ω (p, q)
-#ifdef DEFINITE
-//  ω (p, q)
-   a->AddTrialIntegrator(new MassIntegrator(omeg),0,0);
-#else
-// -ω (p, q)
-   a->AddTrialIntegrator(new MassIntegrator(negomeg),0,0);
-#endif
-
-// -(∇⋅u, q) 
-   a->AddTrialIntegrator(new MixedScalarDivergenceIntegrator(negone),1,0);
-
-// -ω (u,v)
-   a->AddTrialIntegrator(new VectorFEMassIntegrator(negomeg),1,1);
-
-// (∇ p, v)
-   a->AddTrialIntegrator(new GradientIntegrator(one),0,1);
-
-// (v,δv)
-   a->AddTestIntegrator(new VectorMassIntegrator(one),1,1);
-    
-// (q,δq)
-   a->AddTestIntegrator(new MassIntegrator(one),0,0);
-
-   FunctionCoefficient f_rhs(rhs_func);
-   a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs),0);
-
-   Array<int> ess_tdof_list;
-   Array<int> ess_bdr;
-   if (mesh.bdr_attributes.Size())
-   {
-      ess_bdr.SetSize(mesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      p_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   FunctionCoefficient p_ex(p_exact);
-   VectorFunctionCoefficient gradp_ex(dim,gradp_exact);
-   VectorFunctionCoefficient u_ex(dim,u_exact);
-   FunctionCoefficient divu_ex(divu_exact);
-   GridFunction p_gf, u_gf;   
-   GridFunction pex_gf(p_fes);   
-
-   Array<int> offsets(3);
-   offsets[0] = 0;
-   offsets[1] = p_fes->GetVSize();
-   offsets[2] = u_fes->GetVSize();
-   offsets.PartialSum();
-   BlockVector x(offsets);
-   x = 0.0;
-
-   p_gf.MakeRef(p_fes,x.GetBlock(0));
-   p_gf.ProjectBdrCoefficient(p_ex,ess_bdr);
-
-   u_gf.MakeRef(u_fes,x.GetBlock(1));
-
-   a->Assemble();
-
-   OperatorPtr Ah;
-   Vector X,B;
-   a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
-
-   BlockMatrix * A = Ah.As<BlockMatrix>();
-
-   BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
-   M->owns_blocks = 1;
-   for (int i=0; i<A->NumRowBlocks(); i++)
-   {
-      M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
-   }
-
-   CGSolver cg;
-   cg.SetRelTol(1e-12);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(3);
-   cg.SetPreconditioner(*M);
-   cg.SetOperator(*A);
-   cg.Mult(B, X);
-   delete M;
-
-   a->RecoverFEMSolution(X,x);
-
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream p_out;
-      socketstream u_out;
-      p_out.open(vishost, visport);
-      u_out.open(vishost, visport);
-      p_out.precision(8);
-      p_out << "solution\n" << mesh << p_gf <<
-               "window_title 'Numerical p' "
-               << flush;
-
-      u_out.precision(8);
-      u_out << "solution\n" << mesh << u_gf <<
-            "window_title 'Numerical flux' "
-            << flush;
-   }
-
-   return 0;
-}
-
-
-double rhs_func(const Vector &x)
-{
-   double p = p_exact(x);
-   double divu = divu_exact(x);
-   // f = - ∇⋅u ± ω p, 
-#ifdef DEFINITE   
-   return -divu + omega * p;
-#else
-   return -divu - omega * p;
-#endif   
-}
-
-double p_exact(const Vector &x)
-{
-   return sin(omega*x.Sum());
-}
-
-void gradp_exact(const Vector &x, Vector &grad)
-{
-   grad.SetSize(x.Size());
-   grad = omega * cos(omega * x.Sum());
-}
-
-void u_exact(const Vector &x, Vector & u)
-{
-   gradp_exact(x,u);
-   u *= 1./omega;
-}
-
-double divu_exact(const Vector &x)
-{
-   return d2_exact(x)/omega;
-}
-
-double d2_exact(const Vector &x)
-{
-   return -dim * omega * omega * sin(omega*x.Sum());
-}

examples/dpg_tests/acoustics/uw_dpg.cpp

@@ -1,546 +0,0 @@
-//                   MFEM Ultraweak DPG acoustics example
-//
-// Compile with: make uw_dpg
-//
-// ./uw_dpg -m ../../../data/inline-quad.mesh -rnum 40 -theta 0.7 -prob 1 -graph-norm -ref 40 -o 3
-
-//     - Δ p ± ω^2 p = f̃ ,   in Ω
-//                 p = p_0, on ∂Ω
-
-// First Order System
-
-//   ∇ p - ω u = 0, in Ω
-// - ∇⋅u ± ω p = f, in Ω
-//           p = p_0, in ∂Ω
-// where f:=f̃/ω 
-
-// UW-DPG:
-// 
-// p ∈ L^2(Ω), u ∈ (L^2(Ω))^dim 
-// p̂ ∈ H^1/2(Ω), û ∈ H^-1/2(Ω)  
-// -(p,  ∇⋅v) - ω (u , v) + < p̂, v⋅n> = 0,      ∀ v ∈ H(div,Ω)      
-//  (u , ∇ q) ± ω (p , q) + < û, q >  = (f,q)   ∀ q ∈ H^1(Ω)
-//                                  p̂ = p_0     on ∂Ω 
-
-// Note: 
-// p̂ := p on Γ_h (skeleton)
-// û := -u on Γ_h  
-
-// -------------------------------------------------------------
-// |   |     p     |     u     |    p̂      |    û    |  RHS    |
-// -------------------------------------------------------------
-// | v | -(p, ∇⋅v) | - ω (u,v) | < p̂, v⋅n> |         |         |
-// |   |           |           |           |         |         |
-// | q | ± ω (p,q) | (u , ∇ q) |           | < û,q > |  (f,q)  |  
-
-// where (q,v) ∈  H^1(Ω) × H(div,Ω) 
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// #define DEFINITE
-void acoustics_solution(const Vector & X, double & p, Vector & dp, double & d2p);
-double p_exact(const Vector &x);
-void u_exact(const Vector &x, Vector & u);
-double rhs_func(const Vector &x);
-double divu_exact(const Vector &x);
-double hatp_exact(const Vector & X);
-void hatu_exact(const Vector & X, Vector & hatu);
-
-int dim;
-double omega;
-
-enum prob_type
-{
-   plane_wave,
-   gaussian_beam  
-};
-
-prob_type prob;
-
-int main(int argc, char *argv[])
-{
-   const char *mesh_file = "../../../data/inline-quad.mesh";
-   int order = 1;
-   int delta_order = 1;
-   bool visualization = true;
-   double rnum=1.0;
-   int ref = 1;
-   double theta = 0.0;
-   bool adjoint_graph_norm = false;
-   int iprob = 0;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree)");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
-                  "Number of wavelengths");      
-   args.AddOption(&delta_order, "-do", "--delta_order",
-                  "Order enrichment for DPG test space.");     
-   args.AddOption(&theta, "-theta", "--theta",
-                  "Theta parameter for AMR");    
-   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
-                  " 0: plane wave, 1: Gaussian beam");                                    
-   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
-                  "-no-graph-norm", "--no-adjoint-graph-norm",
-                  "Enable or disable Adjoint Graph Norm on the test space");                                
-   args.AddOption(&ref, "-ref", "--serial_ref",
-                  "Number of serial refinements.");                               
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   if (iprob > 1) { iprob = 0; }
-   prob = (prob_type)iprob;
-
-
-   omega = 2.0 * M_PI * rnum;
-
-
-   Mesh mesh(mesh_file, 1, 1);
-   dim = mesh.Dimension();
-
-
-   // Define spaces
-   // L2 space for p
-   FiniteElementCollection *p_fec = new L2_FECollection(order-1,dim);
-   FiniteElementSpace *p_fes = new FiniteElementSpace(&mesh,p_fec);
-
-   // Vector L2 space for u 
-   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
-   FiniteElementSpace *u_fes = new FiniteElementSpace(&mesh,u_fec, dim); 
-
-   // H^1/2 space for p̂  
-   FiniteElementCollection * hatp_fec = new H1_Trace_FECollection(order,dim);
-   FiniteElementSpace *hatp_fes = new FiniteElementSpace(&mesh,hatp_fec);
-
-   // H^-1/2 space for û  
-   FiniteElementCollection * hatu_fec = new RT_Trace_FECollection(order-1,dim);   
-   FiniteElementSpace *hatu_fes = new FiniteElementSpace(&mesh,hatu_fec);
-
-   // testspace fe collections
-   int test_order = order+delta_order;
-   FiniteElementCollection * q_fec = new H1_FECollection(test_order, dim);
-   FiniteElementCollection * v_fec = new RT_FECollection(test_order-1, dim);
-
-
-   // Coefficients
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient zero(0.0);
-   Vector vec0(dim); vec0 = 0.;
-   VectorConstantCoefficient vzero(vec0);
-   ConstantCoefficient negone(-1.0);
-   ConstantCoefficient omeg(omega);
-   ConstantCoefficient omeg2(omega*omega);
-   ConstantCoefficient negomeg(-omega);
-
-   // Normal equation weak formulation
-   Array<FiniteElementSpace * > trial_fes; 
-   Array<FiniteElementCollection * > test_fec; 
-
-   trial_fes.Append(p_fes);
-   trial_fes.Append(u_fes);
-   trial_fes.Append(hatp_fes);
-   trial_fes.Append(hatu_fes);
-
-   test_fec.Append(q_fec);
-   test_fec.Append(v_fec);
-
-   NormalEquations * a = new NormalEquations(trial_fes,test_fec);
-   a->StoreMatrices(true);
-
-
-   // ± ω (p,q)
-#ifdef DEFINITE
-   a->AddTrialIntegrator(new MixedScalarMassIntegrator(omeg),0,0);
-#else
-   a->AddTrialIntegrator(new MixedScalarMassIntegrator(negomeg),0,0);
-#endif   
-
-// (u , ∇ q)
-   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,0);
-
-// -(p, ∇⋅v)
-   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),0,1);
-
-// - ω (u,v)
-   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(negomeg)),1,1);
-
-// < p̂, v⋅n>
-   a->AddTrialIntegrator(new NormalTraceIntegrator,2,1);
-
-// < û,q >
-   a->AddTrialIntegrator(new TraceIntegrator,3,0);
-
-
-// test integrators 
-
-   //space-induced norm for H(div) × H1
-   // (∇q,∇δq)
-   a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
-   // (q,δq)
-   a->AddTestIntegrator(new MassIntegrator(one),0,0);
-   // (∇⋅v,∇⋅δv)
-   a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
-   // (v,δv)
-   a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
-
-   // additional integrators for the adjoint graph norm
-   if (adjoint_graph_norm)
-   {   
-      // -ω (∇q,δv)
-      a->AddTestIntegrator(new MixedVectorGradientIntegrator(negomeg),0,1);
-      // -ω (v,δq)
-      a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg),1,0);
-      // ω^2 (v,δv)
-      a->AddTestIntegrator(new VectorFEMassIntegrator(omeg2),1,1);
-
-#ifdef DEFINITE
-      // - ω (∇⋅v,δq)   
-      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(negomeg),1,0);
-      // - ω (q,∇⋅v)   
-      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(omeg),0,1);
-#else
-      // ω (∇⋅v,δq)   
-      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(omeg),1,0);
-      // ω (q,∇⋅v)   
-      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(negomeg),0,1);
-#endif
-      // ω^2 (q,δq)
-      a->AddTestIntegrator(new MassIntegrator(omeg2),0,0);
-   }
-
-   // RHS
-   FunctionCoefficient f_rhs(rhs_func);
-   a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs),0);
-
-
-   FunctionCoefficient hatpex(hatp_exact);
-   FunctionCoefficient pex(p_exact);
-   VectorFunctionCoefficient uex(dim,u_exact);
-   Array<int> elements_to_refine;
-   GridFunction hatp_gf;
-
-
-   socketstream p_out;
-   // socketstream u_out;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      p_out.open(vishost, visport);
-      // u_out.open(vishost, visport);
-   }
-
-   double res0 = 0.;
-   double err0 = 0.;
-   int dof0;
-   mfem::out << " Refinement |" 
-             << "    Dofs    |" 
-             << "  L2 Error  |" 
-             << " Relative % |" 
-             << "  Rate  |" 
-             << "  Residual  |" 
-             << "  Rate  |" << endl;
-   mfem::out << " --------------------"      
-             <<  "-------------------"    
-             <<  "-------------------"    
-             <<  "-------------------" << endl;   
-
-
-   for (int i = 0; i<ref; i++)
-   {
-      a->Assemble();
-
-      Array<int> ess_tdof_list;
-      Array<int> ess_bdr;
-      if (mesh.bdr_attributes.Size())
-      {
-         ess_bdr.SetSize(mesh.bdr_attributes.Max());
-         ess_bdr = 1;
-         hatp_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-
-   // shift the ess_tdofs
-      for (int i = 0; i < ess_tdof_list.Size(); i++)
-      {
-         ess_tdof_list[i] += p_fes->GetTrueVSize() + u_fes->GetTrueVSize();
-      }
-
-      Array<int> offsets(5);
-      offsets[0] = 0;
-      offsets[1] = p_fes->GetVSize();
-      offsets[2] = u_fes->GetVSize();
-      offsets[3] = hatp_fes->GetVSize();
-      offsets[4] = hatu_fes->GetVSize();
-      offsets.PartialSum();
-      BlockVector x(offsets);
-      x = 0.0;
-      hatp_gf.MakeRef(hatp_fes,x.GetBlock(2));
-      hatp_gf.ProjectBdrCoefficient(hatpex,ess_bdr);
-
-      OperatorPtr Ah;
-      Vector X,B;
-      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
-
-      BlockMatrix * A = Ah.As<BlockMatrix>();
-
-      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
-      M->owns_blocks = 1;
-      for (int i=0; i<A->NumRowBlocks(); i++)
-      {
-         M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
-      }
-
-      CGSolver cg;
-      cg.SetRelTol(1e-8);
-      cg.SetMaxIter(20000);
-      cg.SetPrintLevel(3);
-      cg.SetPreconditioner(*M);
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete M;
-
-      a->RecoverFEMSolution(X,x);
-      Vector & residuals = a->ComputeResidual(x);
-
-      double residual = residuals.Norml2();
-
-      elements_to_refine.SetSize(0);
-      double max_resid = residuals.Max();
-      for (int iel = 0; iel<mesh.GetNE(); iel++)
-      {
-         if (residuals[iel] > theta * max_resid)
-         {
-            elements_to_refine.Append(iel);
-         }
-      }
-      
-      GridFunction p_gf;
-      p_gf.MakeRef(p_fes,x.GetBlock(0));
-
-      GridFunction u_gf;
-      u_gf.MakeRef(u_fes,x.GetBlock(1));
-
-      GridFunction pex_gf(p_fes);
-      GridFunction uex_gf(u_fes);
-      pex_gf.ProjectCoefficient(pex);
-      uex_gf.ProjectCoefficient(uex);
-
-
-      // Error
-      int dofs = X.Size();
-      double p_err = p_gf.ComputeL2Error(pex);
-      double p_norm = uex_gf.ComputeL2Error(zero);
-      double u_err = u_gf.ComputeL2Error(uex);
-      double u_norm = u_gf.ComputeL2Error(vzero);
-
-      double L2Error = sqrt(p_err*p_err + u_err*u_err);
-      double L2norm = sqrt(p_norm * p_norm + u_norm * u_norm);
-
-      double rel_error = L2Error/L2norm;
-
-      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
-      double rate_res = (i) ? dim*log(res0/residual)/log((double)dof0/dofs) : 0.0;
-
-      err0 = L2Error;
-      res0 = residual;
-      dof0 = dofs;
-      mfem::out << std::right << std::setw(11) << i << " | " 
-                << std::setw(10) <<  dof0 << " | " 
-                << std::setprecision(3) 
-                << std::setw(10) << std::scientific <<  err0 << " | " 
-                << std::setprecision(3) 
-                << std::setw(10) << std::fixed <<  rel_error * 100. << " | " 
-                << std::setprecision(2) 
-                << std::setw(6) << std::fixed << rate_err << " | " 
-                << std::setprecision(3) 
-                << std::setw(10) << std::scientific <<  res0 << " | " 
-                << std::setprecision(2) 
-                << std::setw(6) << std::fixed << rate_res << " | " 
-                << std::resetiosflags(std::ios::showbase)
-                << std::endl;
-
-      if (visualization)
-      {
-         p_out.precision(8);
-         p_out << "solution\n" << mesh << p_gf <<
-                  "window_title 'Numerical presure' "
-                  << flush;
-
-         // u_out.precision(8);
-         // u_out << "solution\n" << mesh << u_gf <<
-         //       "window_title 'Numerical velocity' "
-         //       << flush;
-      }
-
-      if (i == ref)
-         break;
-
-      mesh.GeneralRefinement(elements_to_refine,1,1);
-      for (int i =0; i<trial_fes.Size(); i++)
-      {
-         trial_fes[i]->Update(false);
-      }
-      a->Update();
-   }
-
-   delete a;
-   delete q_fec;
-   delete v_fec;
-   delete hatp_fes;
-   delete hatp_fec;
-   delete hatu_fes;
-   delete hatu_fec;
-   delete u_fec;
-   delete p_fec;
-   delete u_fes;
-   delete p_fes;
-
-   return 0;
-}
-
-double rhs_func(const Vector &x)
-{
-   double p = p_exact(x);
-   double divu = divu_exact(x);
-   // f = - ∇⋅u ± ω p, 
-#ifdef DEFINITE   
-   return -divu + omega * p;
-#else
-   return -divu - omega * p;
-#endif   
-}
-
-double p_exact(const Vector &x)
-{
-   double p, d2p;
-   Vector dp;
-   acoustics_solution(x,p,dp,d2p);
-   return p;
-}
-
-void u_exact(const Vector &x, Vector & u)
-{
-   double p, d2p;
-   acoustics_solution(x,p,u,d2p);
-   u *= 1./omega;
-}
-
-double divu_exact(const Vector &x)
-{
-   double p, d2p;
-   Vector dp;
-   acoustics_solution(x,p,dp,d2p);
-   return d2p/omega;
-}
-
-double hatp_exact(const Vector & X)
-{
-   return p_exact(X);
-}
-
-void hatu_exact(const Vector & X, Vector & hatu)
-{
-   u_exact(X,hatu);
-   hatu *= -1.;
-}
-
-void acoustics_solution(const Vector & X, double & p, Vector & dp, double & d2p)
-{
-   dp.SetSize(X.Size());
-   switch (prob)
-   {
-   case plane_wave:
-   {
-      p = sin(omega*X.Sum());
-      dp = omega * cos(omega * X.Sum());
-      d2p = -dim * omega * omega * sin(omega*X.Sum());
-   }
-      break;
-   default:
-   {
-      double rk = omega;
-      double alpha = 45 * M_PI/180.;
-      double sina = sin(alpha); 
-      double cosa = cos(alpha);
-      // shift the origin
-      double xprim=X(0) + 0.1; 
-      double yprim=X(1) + 0.1;
-
-      double  x = xprim*sina - yprim*cosa;
-      double  y = xprim*cosa + yprim*sina;
-      double  dxdxprim = sina, dxdyprim = -cosa;
-      double  dydxprim = cosa, dydyprim =  sina;
-      //wavelength
-      double rl = 2.*M_PI/rk;
-
-      // beam waist radius
-      double w0 = 0.05;
-
-      // function w
-      double fact = rl/M_PI/(w0*w0);
-      double aux = 1. + (fact*y)*(fact*y);
-
-      double w = w0*sqrt(aux);
-      double dwdy = w0*fact*fact*y/sqrt(aux);
-      double d2wdydy = w0*fact*fact*(1. - (fact*y)*(fact*y)/aux)/sqrt(aux);
-
-      double phi0 = atan(fact*y);
-      double dphi0dy = cos(phi0)*cos(phi0)*fact;
-      double d2phi0dydy = -2.*cos(phi0)*sin(phi0)*fact*dphi0dy;
-
-      double r = y + 1./y/(fact*fact);
-      double drdy = 1. - 1./(y*y)/(fact*fact);
-      double d2rdydy = 2./(y*y*y)/(fact*fact);
-
-      // pressure
-      complex<double> zi = complex<double>(0., 1.);
-      complex<double> ze = - x*x/(w*w) - zi*rk*y - zi * M_PI * x * x/rl/r + zi*phi0/2.;
-
-      complex<double> zdedx = -2.*x/(w*w) - 2.*zi*M_PI*x/rl/r;
-      complex<double> zdedy = 2.*x*x/(w*w*w)*dwdy - zi*rk + zi*M_PI*x*x/rl/(r*r)*drdy + zi*dphi0dy/2.;
-      complex<double> zd2edxdx = -2./(w*w) - 2.*zi*M_PI/rl/r;
-      complex<double> zd2edxdy = 4.*x/(w*w*w)*dwdy + 2.*zi*M_PI*x/rl/(r*r)*drdy;
-      complex<double> zd2edydx = zd2edxdy;
-      complex<double> zd2edydy = -6.*x*x/(w*w*w*w)*dwdy*dwdy + 2.*x*x/(w*w*w)*d2wdydy - 2.*zi*M_PI*x*x/rl/(r*r*r)*drdy*drdy
-                              + zi*M_PI*x*x/rl/(r*r)*d2rdydy + zi/2.*d2phi0dydy;
-
-      double pf = pow(2.0/M_PI/(w*w),0.25);
-      double dpfdy = -pow(2./M_PI/(w*w),-0.75)/M_PI/(w*w*w)*dwdy;
-      double d2pfdydy = -1./M_PI*pow(2./M_PI,-0.75)*(-1.5*pow(w,-2.5)
-                        *dwdy*dwdy + pow(w,-1.5)*d2wdydy);
-
-
-      complex<double> zp = pf*exp(ze);
-      complex<double> zdpdx = zp*zdedx;
-      complex<double> zdpdy = dpfdy*exp(ze)+zp*zdedy;
-      complex<double> zd2pdxdx = zdpdx*zdedx + zp*zd2edxdx;
-      complex<double> zd2pdxdy = zdpdy*zdedx + zp*zd2edxdy;
-      complex<double> zd2pdydx = dpfdy*exp(ze)*zdedx + zdpdx*zdedy + zp*zd2edydx;
-      complex<double> zd2pdydy = d2pfdydy*exp(ze) + dpfdy*exp(ze)*zdedy + zdpdy*zdedy + zp*zd2edydy;
-
-      p = zp.real();
-      dp[0] = (zdpdx*dxdxprim + zdpdy*dydxprim).real();
-      dp[1] = (zdpdx*dxdyprim + zdpdy*dydyprim).real();
-
-      d2p = (  (zd2pdxdx*dxdxprim + zd2pdydx*dydxprim)*dxdxprim + (zd2pdxdy*dxdxprim + zd2pdydy*dydxprim)*dydxprim
-            + (zd2pdxdx*dxdyprim + zd2pdydx*dydyprim)*dxdyprim + (zd2pdxdy*dxdyprim + zd2pdydy*dydyprim)*dydyprim ).real();
-   }
-   break;
-   }
-
-}

examples/dpg_tests/acoustics/uw_dpgp.cpp

@@ -1,507 +0,0 @@
-//                   MFEM Ultraweak DPG MPI acoustics (Helmholtz) example
-//
-// Compile with: make uw_dpgp
-//
-//     - Δ p ± ω^2 p = f̃ ,   in Ω
-//                 p = p_0, on ∂Ω
-//
-// First Order System
-
-//   ∇ p - ω u = 0, in Ω
-// - ∇⋅u ± ω p = f, in Ω
-//           p = p_0, in ∂Ω
-// where f:=f̃/ω 
-// 
-// UW-DPG:
-// 
-// p ∈ L^2(Ω), u ∈ (L^2(Ω))^dim 
-// p̂ ∈ H^1/2(Ω), û ∈ H^-1/2(Ω)  
-// -(p,  ∇⋅v) - ω (u , v) + < p̂, v⋅n> = 0,      ∀ v ∈ H(div,Ω)      
-//  (u , ∇ q) ± ω (p , q) + < û, q >  = (f,q)   ∀ q ∈ H^1(Ω)
-//                                  p̂ = p_0     on ∂Ω 
-
-// Note: 
-// p̂ := p on Γ_h (skeleton)
-// û := -u on Γ_h  
-
-// -------------------------------------------------------------
-// |   |     p     |     u     |    p̂      |    û    |  RHS    |
-// -------------------------------------------------------------
-// | v | -(p, ∇⋅v) | - ω (u,v) | < p̂, v⋅n> |         |         |
-// |   |           |           |           |         |         |
-// | q | ± ω (p,q) | (u , ∇ q) |           | < û,q > |  (f,q)  |  
-
-// where (q,v) ∈  H^1(Ω) × H(div,Ω) 
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// #define DEFINITE
-
-double p_exact(const Vector &x);
-void u_exact(const Vector &x, Vector & u);
-double rhs_func(const Vector &x);
-void gradp_exact(const Vector &x, Vector &gradu);
-double divu_exact(const Vector &x);
-double d2_exact(const Vector &x);
-double hatp_exact(const Vector & X);
-void hatu_exact(const Vector & X, Vector & hatu);
-
-int dim;
-double omega;
-
-int main(int argc, char *argv[])
-{
-   MPI_Session mpi;
-   int num_procs = mpi.WorldSize();
-   int myid = mpi.WorldRank();
-
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../../data/inline-quad.mesh";
-   int order = 1;
-   int delta_order = 1;
-   bool visualization = true;
-   double rnum=1.0;
-   int ref = 1;
-   double theta = 0.0;
-   bool adjoint_graph_norm = false;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree)");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
-                  "Number of wavelengths");      
-   args.AddOption(&delta_order, "-do", "--delta_order",
-                  "Order enrichment for DPG test space.");     
-   args.AddOption(&theta, "-theta", "--theta",
-                  "Theta parameter for AMR");                    
-   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
-                  "-no-graph-norm", "--no-adjoint-graph-norm",
-                  "Enable or disable Adjoint Graph Norm on the test space");                                
-   args.AddOption(&ref, "-ref", "--serial_ref",
-                  "Number of serial refinements.");  
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   omega = 2.0 * M_PI * rnum;
-
-   Mesh mesh(mesh_file, 1, 1);
-   dim = mesh.Dimension();
-
-   mesh.EnsureNCMesh();
-
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
-
-   // Define spaces
-   // L2 space for p
-   FiniteElementCollection *p_fec = new L2_FECollection(order-1,dim);
-   ParFiniteElementSpace *p_fes = new ParFiniteElementSpace(&pmesh,p_fec);
-
-   // Vector L2 space for u 
-   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
-   ParFiniteElementSpace *u_fes = new ParFiniteElementSpace(&pmesh,u_fec, dim); 
-
-   // H^1/2 space for p̂  
-   FiniteElementCollection * hatp_fec = new H1_Trace_FECollection(order,dim);
-   ParFiniteElementSpace *hatp_fes = new ParFiniteElementSpace(&pmesh,hatp_fec);
-
-   // H^-1/2 space for û  
-   FiniteElementCollection * hatu_fec = new RT_Trace_FECollection(order-1,dim);   
-   ParFiniteElementSpace *hatu_fes = new ParFiniteElementSpace(&pmesh,hatu_fec);
-
-   // testspace fe collections
-   int test_order = order+delta_order;
-   FiniteElementCollection * q_fec = new H1_FECollection(test_order, dim);
-   FiniteElementCollection * v_fec = new RT_FECollection(test_order-1, dim);
-
-
-   Array<ParFiniteElementSpace * > trial_fes; 
-   trial_fes.Append(p_fes);
-   trial_fes.Append(u_fes);
-   trial_fes.Append(hatp_fes);
-   trial_fes.Append(hatu_fes);
-
-   Array<FiniteElementCollection * > test_fec; 
-   test_fec.Append(q_fec);
-   test_fec.Append(v_fec);
-
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient zero(0.0);
-   Vector vec0(dim); vec0 = 0.;
-   VectorConstantCoefficient vzero(vec0);
-   ConstantCoefficient negone(-1.0);
-   ConstantCoefficient omeg(omega);
-   ConstantCoefficient omeg2(omega*omega);
-   ConstantCoefficient negomeg(-omega);
-
-   ParNormalEquations * a = new ParNormalEquations(trial_fes,test_fec);
-   a->StoreMatrices(true);
-
-
-   // Integrators
-
-   // ± ω (p,q)
-#ifdef DEFINITE
-   a->AddTrialIntegrator(new MixedScalarMassIntegrator(omeg),0,0);
-#else
-   a->AddTrialIntegrator(new MixedScalarMassIntegrator(negomeg),0,0);
-#endif   
-
-// (u , ∇ q)
-   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,0);
-
-// -(p, ∇⋅v)
-   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),0,1);
-
-// - ω (u,v)
-   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(negomeg)),1,1);
-
-// < p̂, v⋅n>
-   a->AddTrialIntegrator(new NormalTraceIntegrator,2,1);
-
-// < û,q >
-   a->AddTrialIntegrator(new TraceIntegrator,3,0);
-
-
-   // test integrators 
-
-   //space-induced norm for H(div) × H1
-   // (∇q,∇δq)
-   a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
-   // (q,δq)
-   a->AddTestIntegrator(new MassIntegrator(one),0,0);
-   // (∇⋅v,∇⋅δv)
-   a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
-   // (v,δv)
-   a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
-
-   // additional integrators for the adjoint graph norm
-   if (adjoint_graph_norm)
-   {   
-      // -ω (∇q,δv)
-      a->AddTestIntegrator(new MixedVectorGradientIntegrator(negomeg),0,1);
-      // -ω (v,δq)
-      a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg),1,0);
-      // ω^2 (v,δv)
-      a->AddTestIntegrator(new VectorFEMassIntegrator(omeg2),1,1);
-
-#ifdef DEFINITE
-      // - ω (∇⋅v,δq)   
-      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(negomeg),1,0);
-      // - ω (q,∇⋅v)   
-      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(omeg),0,1);
-#else
-      // ω (∇⋅v,δq)   
-      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(omeg),1,0);
-      // ω (q,∇⋅v)   
-      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(negomeg),0,1);
-#endif
-      // ω^2 (q,δq)
-      a->AddTestIntegrator(new MassIntegrator(omeg2),0,0);
-   }   
-
-   // RHS
-   FunctionCoefficient f_rhs(rhs_func);
-   a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs),0);
-
-
-   FunctionCoefficient hatpex(hatp_exact);
-   FunctionCoefficient pex(p_exact);
-   VectorFunctionCoefficient uex(dim,u_exact);
-   Array<int> elements_to_refine;
-   ParGridFunction hatp_gf;
-
-
-
-
-   socketstream p_out;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      p_out.open(vishost, visport);
-   }
-   double res0 = 0.;
-   double err0 = 0.;
-   int dof0;
-   if (myid == 0)
-   {
-      mfem::out << " Refinement |" 
-               << "    Dofs    |" 
-               << "  L2 Error  |" 
-               << " Relative % |" 
-               << "  Rate  |" 
-               << "  Residual  |" 
-               << "  Rate  |" << endl;
-      mfem::out << " --------------------"      
-               <<  "-------------------"    
-               <<  "-------------------"    
-               <<  "-------------------" << endl;   
-   }
-
-
-   for (int i = 0; i<ref; i++)
-   {
-      a->Assemble();
-
-      Array<int> ess_tdof_list;
-      Array<int> ess_bdr;
-      if (pmesh.bdr_attributes.Size())
-      {
-         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
-         ess_bdr = 1;
-         hatp_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-
-      // shift the ess_tdofs
-      for (int i = 0; i < ess_tdof_list.Size(); i++)
-      {
-         ess_tdof_list[i] += p_fes->GetTrueVSize() + u_fes->GetTrueVSize();
-      }
-
-      Array<int> offsets(5);
-      offsets[0] = 0;
-      offsets[1] = p_fes->GetVSize();
-      offsets[2] = u_fes->GetVSize();
-      offsets[3] = hatp_fes->GetVSize();
-      offsets[4] = hatu_fes->GetVSize();
-      offsets.PartialSum();
-      BlockVector x(offsets);
-      x = 0.0;
-      hatp_gf.MakeRef(hatp_fes,x.GetBlock(2));
-      hatp_gf.ProjectBdrCoefficient(hatpex,ess_bdr);
-
-      Vector X,B;
-      OperatorPtr Ah;
-      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
-
-      BlockOperator * A = Ah.As<BlockOperator>();
-
-      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
-      M->owns_blocks = 1;
-
-      HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
-      HypreBoomerAMG * amg1 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(1,1));
-      HypreBoomerAMG * amg2 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(2,2));
-      amg0->SetPrintLevel(0);
-      amg1->SetPrintLevel(0);
-      amg2->SetPrintLevel(0);
-      amg0->SetRelaxType(16);
-      amg1->SetRelaxType(16);
-      amg2->SetRelaxType(16);
-
-      M->SetDiagonalBlock(0,amg0);
-      M->SetDiagonalBlock(1,amg1);
-      M->SetDiagonalBlock(2,amg2);
-      // for (int i = 0; i < 3; i++)
-      // {
-      //    MUMPSSolver * mumps = new MUMPSSolver;
-      //    mumps->SetOperator(A->GetBlock(i,i));
-      //    M->SetDiagonalBlock(i,mumps);
-      // }
-
-      HypreSolver * prec;
-      if (dim == 2)
-      {
-         prec = new HypreAMS((HypreParMatrix &)A->GetBlock(3,3), hatu_fes);
-      }
-      else
-      {
-         prec = new HypreADS((HypreParMatrix &)A->GetBlock(3,3), hatu_fes);
-      }
-      M->SetDiagonalBlock(3,prec);
-
-      CGSolver cg(MPI_COMM_WORLD);
-      cg.SetRelTol(1e-10);
-      cg.SetMaxIter(20000);
-      cg.SetPrintLevel(-1);
-      cg.SetPreconditioner(*M);
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete M;
-
-      a->RecoverFEMSolution(X,x);
-
-      Vector & residuals = a->ComputeResidual(x);
-
-      double residual = residuals.Norml2();
-      
-      double maxresidual = residuals.Max(); 
-      double globalresidual = residual * residual; 
-
-      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
-      MPI_Allreduce(MPI_IN_PLACE,&globalresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
-
-      globalresidual = sqrt(globalresidual);
-
-
-      elements_to_refine.SetSize(0);
-      for (int iel = 0; iel<pmesh.GetNE(); iel++)
-      {
-         if (residuals[iel] > theta * maxresidual)
-         {
-            elements_to_refine.Append(iel);
-         }
-      }
-
-
-      ParGridFunction p_gf;
-      p_gf.MakeRef(p_fes,x.GetBlock(0));
-
-      ParGridFunction u_gf;
-      u_gf.MakeRef(u_fes,x.GetBlock(1));
-
-
-      ParGridFunction pex_gf(p_fes);
-      ParGridFunction uex_gf(u_fes);
-      pex_gf.ProjectCoefficient(pex);
-      uex_gf.ProjectCoefficient(uex);
-
-      int dofs = p_fes->GlobalTrueVSize()
-               + u_fes->GlobalTrueVSize()
-               + hatp_fes->GlobalTrueVSize()
-               + hatu_fes->GlobalTrueVSize();
-
-      double p_err = p_gf.ComputeL2Error(pex);
-      double p_norm = pex_gf.ComputeL2Error(zero);
-      double u_err = u_gf.ComputeL2Error(uex);
-      double u_norm = uex_gf.ComputeL2Error(vzero);
-
-      double L2Error = sqrt(p_err*p_err + u_err*u_err);
-      double L2norm = sqrt(p_norm * p_norm + u_norm * u_norm);
-
-      double rel_error = L2Error/L2norm;
-
-      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
-      double rate_res = (i) ? dim*log(res0/globalresidual)/log((double)dof0/dofs) : 0.0;
-
-      err0 = L2Error;
-      res0 = globalresidual;
-      dof0 = dofs;
-
-         std::ios oldState(nullptr);
-      if (myid == 0)
-      {
-         mfem::out << std::right << std::setw(11) << i << " | " 
-                   << std::setw(10) <<  dof0 << " | " 
-                   << std::setprecision(3) 
-                   << std::setw(10) << std::scientific <<  err0 << " | " 
-                   << std::setprecision(3) 
-                   << std::setw(10) << std::fixed <<  rel_error * 100. << " | " 
-                   << std::setprecision(2) 
-                   << std::setw(6) << std::fixed << rate_err << " | " 
-                   << std::setprecision(3) 
-                   << std::setw(10) << std::scientific <<  res0 << " | " 
-                   << std::setprecision(2) 
-                   << std::setw(6) << std::fixed << rate_res << " | " 
-                   << std::setprecision(5) 
-                   << std::scientific 
-                   << std::endl;
-      }   
-
-      if (visualization)
-      {
-         p_out << "parallel " << num_procs << " " << myid << "\n";
-         p_out.precision(8);
-         p_out << "solution\n" << pmesh << p_gf <<
-                  "window_title 'Numerical pressure' "
-                  << flush;
-      }
-
-      if (i == ref)
-         break;
-
-      pmesh.GeneralRefinement(elements_to_refine,1,1);
-      for (int i =0; i<trial_fes.Size(); i++)
-      {
-         trial_fes[i]->Update(false);
-      }
-      a->Update();
-
-   }
-
-   delete a;
-   delete q_fec;
-   delete v_fec;
-   delete hatp_fes;
-   delete hatp_fec;
-   delete hatu_fes;
-   delete hatu_fec;
-   delete u_fec;
-   delete p_fec;
-   delete u_fes;
-   delete p_fes;
-
-   return 0;
-}
-
-double rhs_func(const Vector &x)
-{
-   double p = p_exact(x);
-   double divu = divu_exact(x);
-   // f = - ∇⋅u ± ω p, 
-#ifdef DEFINITE   
-   return -divu + omega * p;
-#else
-   return -divu - omega * p;
-#endif   
-}
-
-double p_exact(const Vector &x)
-{
-   return sin(omega*x.Sum());
-}
-
-void gradp_exact(const Vector &x, Vector &grad)
-{
-   grad.SetSize(x.Size());
-   grad = omega * cos(omega * x.Sum());
-}
-
-void u_exact(const Vector &x, Vector & u)
-{
-   gradp_exact(x,u);
-   u *= 1./omega;
-}
-
-double divu_exact(const Vector &x)
-{
-   return d2_exact(x)/omega;
-}
-
-double d2_exact(const Vector &x)
-{
-   return -dim * omega * omega * sin(omega*x.Sum());
-}
-
-double hatp_exact(const Vector & X)
-{
-   return p_exact(X);
-}
-
-void hatu_exact(const Vector & X, Vector & hatu)
-{
-   u_exact(X,hatu);
-   hatu *= -1.;
-}