renaming and alternative interface

.github/workflows/repo-check.yml

@@ -1,42 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-name: repo-check
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
-
-jobs:
-  copyright-check:
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: checkout mfem
-      uses: actions/checkout@v2
-      with:
-        path: mfem
-
-    - name: copyright check
-      run: |
-        cd mfem
-        if git grep -l "^#.*\-2020" > matches.txt
-        then
-          echo "Please update the following files to Copyright (c) 2010-2021:"
-          cat matches.txt
-          exit 1
-        else
-          echo "No outdated copyright found."
-        fi

.gitignore

@@ -66,7 +66,6 @@ examples/sol_u.*
 examples/sol_p.*
 examples/sol_r.*
 examples/sol_i.*
-examples/ex6p-checkpoint.*
 examples/ex9.mesh
 examples/ex9-mesh.*
 examples/ex9-init.*
@@ -216,7 +215,6 @@ miniapps/meshing/polar-nc.mesh
 
 miniapps/navier/navier_mms
 miniapps/navier/navier_kovasznay
-miniapps/navier/navier_kovasznay_vs
 miniapps/navier/navier_tgv
 miniapps/navier/navier_shear
 miniapps/navier/navier_3dfoc
@@ -238,9 +236,6 @@ miniapps/performance/refined.mesh
 miniapps/performance/mesh.*
 miniapps/performance/sol.*
 
-miniapps/shifted/distance
-miniapps/shifted/ParaViewDistance
-
 miniapps/tools/display-basis
 miniapps/tools/load-dc
 miniapps/tools/convert-dc
@@ -268,8 +263,6 @@ miniapps/toys/lissajous.mesh
 miniapps/toys/lissajous.gf
 miniapps/toys/mondrian.mesh
 
-miniapps/solvers/block-solvers
-
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests

.gitlab-ci.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -23,6 +23,10 @@
 # also means that the BUILD_ROOT directory will never be cleaned.
 # TODO: add a clean-up mechanism
 #
+# BUILD_PATH:
+# In BUILD_ROOT, we want to separate builds depending on the machine used,
+# typically because we build on a set of dependencies per machine.
+#
 # REBASELINE:
 # Defines the default choice for updating the saved baseline results. By default
 # the baseline can only be updated from the master branch. This variable offers
@@ -41,8 +45,9 @@
 
 variables:
   BUILD_ROOT: ${CI_BUILDS_DIR}/${CI_PROJECT_NAME}_${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID}
+  BUILD_PATH: ${CI_BUILDS_DIR}/${CI_PROJECT_NAME}_${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID}/${PLAT}/${TOOLCHAIN}
   REBASELINE: "NO"
-  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
+  MFEM_ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
   TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
   TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
   ARTIFACTS_DIR: artifacts
@@ -51,18 +56,23 @@ variables:
 # points, however, we use "needs" keyword to express the DAG of jobs for more
 # efficiency.
 # - We use setup phase to download content outside of mfem directory.
-# - Allocate/Release is where quartz resources are allocated/released once for all.
-# - Build and Test is where we build and MFEM for multiple toolchains.
+# - Allocate is where quartz resources are allocated once for all.
+# - Libs is where dependencies are built.
+# - Build is where we build MFEM for multiple toolchains.
+# - Test is where we perform a first set of tests on MFEM build.
 # - Baseline_checks gathers baseline-type test suites execution
 # - Baseline_publish, only available on master, allows to update baseline
 #   results
+# - Deallocate releases quartz resources
 stages:
-  - q_allocate_resources
-  - q_build_and_test
-  - q_release_resources
-  - l_build_and_test
-  - c_build_and_test
   - setup
+  - allocate
+  - libs
+  - build
+  - test
+  - deallocate
+  - lassen_build
+  - lassen_test
   - baseline_check
   - baseline_publish
 
@@ -85,25 +95,92 @@ setup:
     - if [ ! -d "tests" ]; then git clone ${TESTS_REPO}; fi
     - cd tpls && git pull && cd ..
     - cd tests && git pull && cd ..
-  needs: []
 
-.build_toss_3_x86_64_ib_script:
-  script:
-    - export THREADS=12
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 30 -N 1 tests/gitlab/build_and_test
+# Share configurations (templates) for toolchain configuration.
+# Note: Any job using this template shouldn't redefine "before_script" key.
+.with_gcc_6_1_0:
+  variables:
+    TOOLCHAIN: gcc_6_1_0
+    CXX: g++
+    CC: gcc
+  before_script:
+    - module load gcc/6.1.0
 
-.build_toss_3_x86_64_ib_corona_script:
-  script:
-    - srun -p mi60 -t 15 -N 1 tests/gitlab/build_and_test
+# On lassen
+.with_gcc_8_3_1:
+  variables:
+    TOOLCHAIN: gcc_8_3_1
+    CXX: g++
+    CC: gcc
+  before_script:
+    - module load gcc/8.3.1
 
-# Lassen and Butte use a different job scheduler (spectrum lsf) that does not
-# allow pre-allocation the same way slurm does.
-.build_blueos_3_ppc64le_ib_script:
+.with_gcc_4_9_3:
+  variables:
+    TOOLCHAIN: gcc_4_9_3
+    CXX: g++
+    CC: gcc
+  before_script:
+    - module load gcc/4.9.3
+
+.if_srun_alloc_for_make: &if_srun_alloc_for_make |
+  if [[ "${MPIEXEC}" == "srun" ]]
+  then
+    export JOB_ID=$(squeue -h --name=${MFEM_ALLOC_NAME} --format=%A)
+    [[ -n "JOB_ID" ]] && export JOB_ID="--jobid=${JOB_ID}"
+    RESOURCES="--extra-node-info=1:${MAKE_PAR}:1 --cpus-per-task=${MAKE_PAR} --ntasks=1"
+    # Allocation prefix:
+    export EXEC_PREFIX="srun ${JOB_ID} ${RESOURCES}"
+  fi
+
+# Build of dependencies and mfem:
+# if on quartz, uses a part of preallocated resources
+# Note: Those jobs don't rely on MFEM repository, hence the use of
+# "GIT_STRATEGY: none" preventing its clone.
+.build_hypre:
+  variables:
+    EXEC_PREFIX: ""
+    GIT_STRATEGY: none
+  stage: libs
   script:
-    - lalloc 1 -W 15 tests/gitlab/build_and_test
+    - cd ${BUILD_ROOT}/tpls
+    - mkdir -p ${BUILD_PATH}
+    - make clean
+    - *if_srun_alloc_for_make
+    - ${EXEC_PREFIX} make -j ${MAKE_PAR} hypre PREFIX=${BUILD_PATH}/hypre/src/ BUILD_DIR=${BUILD_PATH} LOG_DIR=${BUILD_PATH}
+
+.build_metis:
+  variables:
+    EXEC_PREFIX: ""
+    GIT_STRATEGY: none
+  stage: libs
+  script:
+    - cd ${BUILD_ROOT}/tpls
+    - mkdir -p ${BUILD_PATH}
+    - *if_srun_alloc_for_make
+    - ${EXEC_PREFIX} make -j ${MAKE_PAR} ${METIS} PREFIX=${BUILD_PATH}/ BUILD_DIR=${BUILD_PATH} LOG_DIR=${BUILD_PATH} CC=${CC} CXX=${CXX}
+
+# Building MFEM
+.build_mfem:
+  variables:
+    EXEC_PREFIX: ""
+  stage: build
+  script:
+    - mkdir -p ${BUILD_PATH}
+    - cp -r ${CI_PROJECT_DIR} ${BUILD_PATH}/${CI_PROJECT_NAME}_${MFEM_USE_MPI}_${MFEM_DEBUG}
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_${MFEM_USE_MPI}_${MFEM_DEBUG}
+    - *if_srun_alloc_for_make
+    - make config CXX=${CXX} MPICXX=mpicxx MFEM_USE_MPI=${MFEM_USE_MPI} MFEM_DEBUG=${MFEM_DEBUG} MFEM_MPIEXEC="${EXEC_PREFIX}" MFEM_MPIEXEC_NP=${MPIEXEC_NP}
+    - ${EXEC_PREFIX} make -j ${MAKE_PAR} all
+
+# TODO: Could this phase use a parallel run?
+.sanitycheck_mfem:
+  stage: test
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_${MFEM_USE_MPI}_${MFEM_DEBUG}
+    - make test
 
 # Shared script for baseline and sample-run-baseline, the value of BASELINE_TEST
 # differentiates between the two tests.
@@ -125,7 +202,7 @@ setup:
   cd tests
   mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
   # run
-  srun --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${ADDITIONAL_DIR}"
+  salloc --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${ADDITIONAL_DIR}"
   # post
   mkdir ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}
   if [[ -s ${_glob_err} ]]
@@ -228,4 +305,4 @@ setup:
 # The list on jobs is defined in machine-specific files.
 include:
   - local: .gitlab/quartz.yml
-    #  - local: .gitlab/lassen.yml
+  - local: .gitlab/lassen.yml

.gitlab/lassen.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

.gitlab/quartz.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -15,85 +15,131 @@
   tags:
     - shell
     - quartz
-  rules:
-    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"' #run except if ...
-      when: never
-    - if: '$CI_JOB_NAME =~ /release_resources/'
-      when: always
-    - when: on_success
+  variables:
+    PLAT: quartz
+    MPIEXEC: srun
+    MPIEXEC_NP: "-n"
+    MAKE_PAR: 6
+    BASELINE_PAR: 18
 
 # Allocate
-q_allocate_resources:
+allocate_quartz:
   variables:
     GIT_STRATEGY: none
   extends: .on_quartz
-  stage: q_allocate_resources
+  stage: allocate
   script:
-    - salloc --exclusive --nodes=1 --partition=pdebug --time=30 --no-shell --job-name=${ALLOC_NAME}
+    - |
+      if [[ "${MPIEXEC}" == "srun" ]]
+      then
+      salloc --nodes=1 --extra-node-info=2:18:1 --partition=pdebug --time=30 --no-shell --job-name=${MFEM_ALLOC_NAME}
+      fi
   timeout: 6h
 
 # Release
-q_release_resources:
+deallocate_quartz:
   variables:
     GIT_STRATEGY: none
   extends: .on_quartz
-  stage: q_release_resources
+  stage: deallocate
   script:
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
+    - |
+      if [[ "${MPIEXEC}" == "srun" ]]
+      then
+      export JOB_ID=$(squeue -h --name=${MFEM_ALLOC_NAME} --format=%A)
+      ([[ -n "${JOB_ID}" ]] && scancel ${JOB_ID})
+      fi
+  when: always
 
-# Spack helped builds
-# Generic quartz build job, extending build script
-.build_and_test_on_quartz:
-  extends: [.build_toss_3_x86_64_ib_script, .on_quartz]
-  stage: q_build_and_test
+# Build external libraries (tpls)
+build_hypre_gcc_6_1_0_quartz:
+  extends: [.build_hypre, .with_gcc_6_1_0, .on_quartz]
+
+build_parmetis_gcc_6_1_0_quartz:
+  extends: [.build_metis, .with_gcc_6_1_0, .on_quartz]
+  variables:
+    METIS: parmetis
+
+build_metis4_gcc_6_1_0_quartz:
+  extends: [.build_metis, .with_gcc_6_1_0, .on_quartz]
+  variables:
+    METIS: metis4
+
+build_metis5_gcc_6_1_0_quartz:
+  extends: [.build_metis, .with_gcc_6_1_0, .on_quartz]
+  variables:
+    METIS: metis5
+  allow_failure: true
 
 # Build MFEM
-debug_ser_gcc_4_9_3:
+build_mfem_debug_ser_gcc_4_9_3_quartz:
+  extends: [.build_mfem, .with_gcc_4_9_3, .on_quartz]
   variables:
-    SPEC: "%gcc@4.9.3 +debug~mpi"
-  extends: .build_and_test_on_quartz
+    MFEM_DEBUG: "YES"
+    MFEM_USE_MPI: "NO"
 
-debug_ser_gcc_6_1_0:
+build_mfem_debug_ser_gcc_6_1_0_quartz:
+  extends: [.build_mfem, .with_gcc_6_1_0, .on_quartz]
   variables:
-    SPEC: "%gcc@6.1.0 +debug~mpi"
-  extends: .build_and_test_on_quartz
+    MFEM_DEBUG: "YES"
+    MFEM_USE_MPI: "NO"
 
-debug_par_gcc_6_1_0:
+build_mfem_debug_par_gcc_6_1_0_quartz:
+  extends: [.build_mfem, .with_gcc_6_1_0, .on_quartz]
   variables:
-    SPEC: "%gcc@6.1.0 +debug+mpi"
-  extends: .build_and_test_on_quartz
+    MFEM_DEBUG: "YES"
+    MFEM_USE_MPI: "YES"
+  needs: [build_hypre_gcc_6_1_0_quartz, build_parmetis_gcc_6_1_0_quartz]
 
-opt_ser_gcc_6_1_0:
+build_mfem_opt_ser_gcc_6_1_0_quartz:
+  extends: [.build_mfem, .with_gcc_6_1_0, .on_quartz]
   variables:
-    SPEC: "%gcc@6.1.0 ~mpi"
-  extends: .build_and_test_on_quartz
+    MFEM_DEBUG: "NO"
+    MFEM_USE_MPI: "NO"
 
-opt_par_gcc_6_1_0:
+build_mfem_opt_par_gcc_6_1_0_quartz:
+  extends: [.build_mfem, .with_gcc_6_1_0, .on_quartz]
   variables:
-    SPEC: "%gcc@6.1.0"
-  extends: .build_and_test_on_quartz
+    MFEM_DEBUG: "NO"
+    MFEM_USE_MPI: "YES"
+  needs: [build_hypre_gcc_6_1_0_quartz, build_parmetis_gcc_6_1_0_quartz]
 
-opt_par_gcc_6_1_0_sundials:
-  variables:
-    SPEC: "%gcc@6.1.0 +sundials"
-  extends: .build_and_test_on_quartz
+### Tests
 
-opt_par_gcc_6_1_0_petsc:
+# Sanity check
+sanitycheck_mfem_debug_ser_gcc_4_9_3_quartz:
+  extends: [.sanitycheck_mfem, .with_gcc_4_9_3, .on_quartz]
   variables:
-    SPEC: "%gcc@6.1.0 +petsc ^petsc+mumps"
-  extends: .build_and_test_on_quartz
+    MFEM_DEBUG: "YES"
+    MFEM_USE_MPI: "NO"
+  needs: [build_mfem_debug_ser_gcc_4_9_3_quartz, allocate_quartz]
 
-opt_par_gcc_6_1_0_pumi:
+sanitycheck_mfem_debug_ser_gcc_6_1_0_quartz:
+  extends: [.sanitycheck_mfem, .with_gcc_6_1_0, .on_quartz]
   variables:
-    SPEC: "%gcc@6.1.0 +pumi"
-  extends: .build_and_test_on_quartz
+    MFEM_DEBUG: "YES"
+    MFEM_USE_MPI: "NO"
+  needs: [build_mfem_debug_ser_gcc_6_1_0_quartz, allocate_quartz]
+
+sanitycheck_opt_ser_mfem_gcc_6_1_0_quartz:
+  extends: [.sanitycheck_mfem, .with_gcc_6_1_0, .on_quartz]
+  variables:
+    MFEM_DEBUG: "NO"
+    MFEM_USE_MPI: "NO"
+  needs: [build_mfem_opt_ser_gcc_6_1_0_quartz, allocate_quartz]
+
+sanitycheck_opt_par_mfem_gcc_6_1_0_quartz:
+  extends: [.sanitycheck_mfem, .with_gcc_6_1_0, .on_quartz]
+  variables:
+    MFEM_DEBUG: "NO"
+    MFEM_USE_MPI: "YES"
+  needs: [build_mfem_opt_par_gcc_6_1_0_quartz, allocate_quartz]
 
 # Baseline
 baselinecheck_mfem_intel_quartz:
   extends: [.baselinecheck_mfem, .on_quartz]
-  needs: [setup]
+  needs: [allocate_quartz]
 
 baselinepublish_mfem_quartz:
-  extends: [.on_quartz, .rebaseline_mfem]
+  extends: [.rebaseline_mfem, .on_quartz]
   needs: [baselinecheck_mfem_intel_quartz]

.travis.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

CHANGELOG

@@ -11,117 +11,20 @@
 Version 4.2.1 (development)
 ===========================
 
-- Added high-order matrix-free auxiliary Maxwell solver for H(curl) problems,
-  as described in Barker and Kolev 2020 (https://doi.org/10.1002/nla.2348). See
-  Example 3p and linalg/auxiliary.?pp.
-
-- Added a new miniapp block-solvers that compares the performance of various
-  solvers for mixed finite element discretization of the second order scalar
-  elliptic equations. Currently available solvers in the miniapp include a
-  block-diagonal preconditioner that is based on approximate Schur complement
-  (implemented in ex5p), and a newly implemented solver DivFreeSolver, which
-  exploits a multilevel decomposition of the Raviart-Thomas space and its
-  divergence-free subspace. See the miniapps/solvers directory for more details.
-
-- Added a new miniapp for computing (signed) distance functions to a point
-  source or zero level set. See miniapps/shifted/distance.cpp.
-
-- Added matrix-free GPU-enabled implementations of GradientInterpolator and
-  IdentityInterpolator.
-
 - Added interface to MUMPS direct solver. Its usage is demonstrated in ex25p.
   See http://mumps.enseeiht.fr/ for more details. Supported versions >= 5.1.1.
 
 - Added three ESDIRK time integrators: implicit trapezoid rule, L-stable
   ESDIRK-32, and A-stable ESDIRK-33.
 
-- Introduced a new non-conforming mesh format that fixes known inconsistencies
-  of legacy "MFEM mesh v1.1" NC format and works consistently in both serial and
-  parallel. ParMesh::ParPrint can now print non-conforming AMR meshes that can
-  be used to restart a parallel AMR computation. Example 6p has been extended to
-  demonstrate restarting from a previously saved checkpoint. Note that parallel
-  NC data files are compatible with serial code, e.g., can be viewed with serial
-  GLVis. Loading of legacy NC mesh files is still supported.
-
 - Added a "scaled Jacobian" visualization option in the Mesh Explorer miniapp to
   help identify elements with poor mesh quality.
 
-- Added support for the "BR2" discontinuous Galerkin discretization for
-  diffusion via DGDiffusionBR2Integrator (see Example 14/14p).
-
-- Generalized the Multigrid class to support non-geometric multigrid. The
-  previous functionality, based on FiniteElementSpaceHierarchy, is now available
-  in the derived class GeometricMultigrid.
-
 - Upgraded the Catch unit test framework from version 2.13.0 to version 2.13.2.
 
 - Implemented a filter method for the Navier miniapp to stabilize highly
   turbulent flows in direct numerical simulation.
 
-- Added HIP support to the CMake build system.
-
-- Added support for reading high-order Lagrange meshes in VTK format. Arbitrary-
-  orders and all element types are supported. See the VTK blog for more info:
-  https://blog.kitware.com/wp-content/uploads/2018/09/Source_Issue_43.pdf.
-
-- Added support for reading VTK meshes in XML format.
-
-- Added partial assembly and device support to Example 25/25p, with diagonal
-  preconditioning.
-
-- Implemented a variable step-size IMEX (VSSIMEX) method for the Navier miniapp.
-
-- Added new mesh quality metrics and improved the untangling capabilities of the
-  TMOP-based mesh optimization algorithms.
-
-- Added convective and skew-symmetric integrators for the nonlinear term in the
-  Navier-Stokes equations.
-
-- Changed the interface for the error estimator.
-
-- Implemented the parallel Kelly error indicator for scalar-valued problems.
-
-- Added new classes DenseSymmetricMatrix and SymmetricMatrixCoefficient for
-  efficient evaluation of symmetric matrix coefficients. This replaces the now
-  deprecated EvalSymmetric in MatrixCoefficient. Added DiagonalMatrixCoefficient
-  for clarity, which is a typedef of VectorCoefficient.
-
-- Added support for AMG preconditioners for non-symmetric systems (e.g.
-  advection-dominated problems) using hypre's approximate ideal restriction
-  (AIR) AMG. Requires hypre version 2.14.0 or newer. Usage is illustrated in
-  example 9/9p.
-
-- Implemented an adaptive linear solver tolerance option for NewtonSolver based
-  on the algorithm of Eisenstat and Walker.
-
-- Added support for nonscalar coefficient with VectorDiffusionIntegrator.
-
-- Extending support for L2 basis functions using MapTypes VALUE and INTEGRAL in
-  linear interpolators and GridFunction "GetValue" methods.
-
-- Variable order spaces, p- and hp-refinement. This is the initial (serial)
-  support for variable-order FiniteElementCollection and FiniteElementSpace.
-  The new method FiniteElementSpace::SetElementOrder can be called to set an
-  arbitrary order for each mesh element. The conforming interpolation matrix
-  will now automatically constrain p- and hp- interfaces, enabling general
-  hp-refinement in both 2D and 3D, on uniform or mixed NC meshes. Support for
-  parallel variable-order spaces will follow shortly.
-
-- Added support for creating refined meshes for all element types (e.g. by
-  splitting high-order elements into low-order refined elements), including
-  mixed meshes. The LOR Transfer miniapp (miniapps/tools/lor-transfer.cpp) now
-  supports meshes with any element geometry.
-
-- Gitlab CI: use Spack (and Uberenv) to automate the build of TPLs.
-
-libCEED integration improvements
---------------------------------
-- Refactor the libCEED integration
-
-- Add support for VectorCoefficient with libCEED backends.
-
-- Add support for ConvectionIntegrator, and VectorConvectionNLFIntegrator with libCEED backends.
-
 
 Version 4.2, released on October 30, 2020
 =========================================
@@ -994,7 +897,7 @@ New and updated examples and miniapps
 
 - Added a new meshing miniapp, Shaper, that can be used to resolve complicated
   material interfaces by mesh refinement, e.g. as a tool for initial mesh
-  generation from prescribed "material()" function. Both conforming and
+  generation from prescribed "material()" function.  Both conforming and
   non-conforming (isotropic and anisotropic) refinements are supported.
 
 - Added a new meshing miniapp, Mesh Optimizer, that demonstrates the use of TMOP
@@ -1006,7 +909,7 @@ Discretization improvements
 ---------------------------
 - Added a FindPoints method of the Mesh and ParMesh classes that returns the
   elements that contain a given set of points, together with the coordinates of
-  the points in the reference space of the corresponding element. In parallel,
+  the points in the reference space of the corresponding element.  In parallel,
   if a point is shared by multiple processors, only one of them will mark that
   point as found. Note that the current implementation of this method is not
   optimal and/or 100% reliable. See the mesh-explorer miniapp for an example.
@@ -1333,7 +1236,7 @@ New and improved linear solvers
   which is a sparse direct solver for distributed memory architectures. As such
   it can only be enabled along with MFEM_USE_MPI. When MFEM is configured with
   MFEM_USE_SUPERLU, one also needs to alter the version of METIS, since SuperLU
-  requires ParMETIS (which comes packaged with a serial version of METIS). See
+  requires ParMETIS (which comes packaged with a serial version of METIS).  See
   http://crd-legacy.lbl.gov/~xiaoye/SuperLU for SuperLU_DIST details.
 
 - Added a wrapper for the KLU solver in SuiteSparse see
@@ -1744,7 +1647,7 @@ New and updated examples
   (ADS) in hypre.
 
 - Modified Example 1 to use isoparametric discretization (use the FE space from
-  the mesh) including NURBS meshes and spaces. Updated Example 2 to support
+  the mesh) including NURBS meshes and spaces.  Updated Example 2 to support
   arbitrary order spaces. Updated all examples to work with NURBS meshes and
   spaces, as well as to not use projection onto discontinuous polynomial spaces
   for visualization (this is now handled directly in GLVis when necessary).
@@ -1836,7 +1739,7 @@ Version 1.1, released on Sep 13, 2010
 New MFEM format for general meshes
 ----------------------------------
 - New MFEM mesh v1.0 format with uniform structure for any dimension and support
-  for curved meshes including in 3D. Class Mesh will recognize and read the new
+  for curved meshes including in 3D.  Class Mesh will recognize and read the new
   format (in addition to all previously used formats) and Mesh::Print uses the
   new format by default. The old print function was renamed to Mesh::PrintXG.
 

CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -9,13 +9,12 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-# The variable CMAKE_CXX_STANDARD and related were introduced in CMake v3.1
-cmake_minimum_required(VERSION 3.1)
+cmake_minimum_required(VERSION 2.8.11)
 set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH
   "Path to optional user configuration file.")
 
 # Require C++11 and disable compiler-specific extensions
-set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 
@@ -88,9 +87,6 @@ include("${CMAKE_CURRENT_SOURCE_DIR}/config/XSDKDefaults.cmake")
 # Enable languages.
 enable_language(CXX)
 if (MFEM_USE_CUDA)
-   if (MFEM_USE_HIP)
-      message(FATAL_ERROR " *** MFEM_USE_HIP cannot be combined with MFEM_USE_CUDA.")
-   endif()
    # MFEM_USE_CUDA requires CMake 3.8 or newer (for direct CUDA support)
    cmake_minimum_required(VERSION 3.8 FATAL_ERROR)
    # Use ${CMAKE_CXX_COMPILER} as the cuda host compiler.
@@ -360,15 +356,6 @@ if (MFEM_USE_UMPIRE)
    find_package(UMPIRE REQUIRED)
 endif()
 
-# AMD HIP
-if (MFEM_USE_HIP)
-  find_package(HIP REQUIRED)
-  if (HIP_ARCH)
-    message(STATUS "Using HIP architecture: ${HIP_ARCH}")
-    list(APPEND HIP_HIPCC_FLAGS "--amdgpu-target=${HIP_ARCH}")
-  endif()
-endif()
-
 # ADIOS2 for parallel I/O
 if (MFEM_USE_ADIOS2)
    find_package(ADIOS2 REQUIRED)
@@ -444,9 +431,9 @@ foreach(DIR IN LISTS MFEM_SOURCE_DIRS)
 endforeach()
 
 if (MFEM_USE_CUDA)
-  set_source_files_properties(${SOURCES} PROPERTIES LANGUAGE CUDA)
-elseif(MFEM_USE_HIP)
-  set_source_files_properties(${SOURCES} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
+   foreach(file IN LISTS SOURCES)
+      set_property(SOURCE ${file} PROPERTY LANGUAGE CUDA)
+   endforeach()
 endif()
 
 add_subdirectory(config)
@@ -465,7 +452,7 @@ set(MFEM_INSTALL_DIR ${CMAKE_INSTALL_PREFIX} CACHE PATH
     "The MFEM install directory" FORCE)
 
 # Declaring the library
-mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
+add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
 # message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
 if (CMAKE_VERSION VERSION_GREATER 2.8.11)
   target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES})
@@ -622,7 +609,7 @@ endif()
 if (MFEM_USE_CEED)
   install(DIRECTORY ${MFEM_SOURCE_DIRS}
     DESTINATION ${INSTALL_INCLUDE_DIR}/mfem
-    FILES_MATCHING PATTERN "fem/ceed/*.h")
+    FILES_MATCHING PATTERN "fem/libceed/*.h")
 endif()
 
 # Install ${HEADERS}

CONTRIBUTING.md

@@ -104,7 +104,7 @@ The MFEM source code has the following structure:
   │   ├── sundials
   |   └── superlu
   ├── fem
-  │   └── ceed
+  │   └── libceed
   ├── general
   ├── linalg
   │   └── simd
@@ -118,13 +118,10 @@ The MFEM source code has the following structure:
   │   ├── navier
   │   ├── nurbs
   │   ├── performance
-  │   ├── shifted
-  │   ├── solvers
   │   ├── tools
   │   └── toys
   └── tests
       ├── convergence
-      ├── gitlab
       ├── par-mesh-format
       ├── scripts
       └── unit
@@ -564,9 +561,7 @@ MFEM uses a `master`/`next`-branch workflow as described below:
   Computing systems. Gitlab pipeline status is reported in the corresponding
   GitHub pull request.
 
-- In Gitlab pipelines, TPLs (dependencies) are built using Spack, driven by Uberenv.
-
-- No change to the MFEM repo can be made on this instance.
+- No change can be made on this instance.
 
 ## Automated Testing
 

INSTALL

@@ -692,7 +692,7 @@ The specific libraries and their options are:
 - CUDA (optional), used when MFEM_USE_CUDA = YES.
   URL: https://developer.nvidia.com/cuda-toolkit
   Options: CUDA_CXX, CUDA_ARCH, CUDA_OPT, CUDA_LIB.
-  Versions: CUDA >= 10.1.168.
+  Versions: CUDA >= 9.1, older versions may work too.
 
 - HIP (optional), used when MFEM_USE_HIP = YES.
   URL: https://rocm.github.io/ROCmInstall.html
@@ -710,10 +710,10 @@ The specific libraries and their options are:
   Versions: libCEED >= 0.7.
 
 - RAJA (optional), used when MFEM_USE_RAJA = YES.
-  Beginning with MFEM v4.3, only RAJA v0.13.0+ is supported.
+  Beginning with MFEM v4.1, only RAJA v0.10.0+ is supported.
   URL: https://github.com/LLNL/RAJA
   Options: RAJA_DIR, RAJA_OPT, RAJA_LIB.
-  Versions: RAJA >= 0.13.0.
+  Versions: RAJA >= 0.10.0.
 
 - Umpire, used when MFEM_USE_UMPIRE = YES.
   URL: https://github.com/LLNL/Umpire

LICENSE

@@ -1,6 +1,6 @@
 BSD 3-Clause License
 
-Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC
+Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

NOTICE

@@ -29,5 +29,4 @@ license files. These software products and their licenses are as follows:
 * Catch++ (tests/unit/catch.hpp) -- Boost 1.0 license
 * Gecko (general/gecko.{cpp,hpp}) -- BSD 3-clause license
 * Picojson (fem/picojson.h) -- Custom 2-clause license
-* TinyXML2 (general/tinyxml2.{cpp,h}) -- zlib license
 * Zstr (general/zstr.hpp) -- MIT license

config/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/MFEMConfig.cmake.in

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/config.hpp.in

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -132,10 +132,6 @@
 // Requires a CUDA compiler (nvcc).
 #cmakedefine MFEM_USE_CUDA
 
-// Build the HIP-enabled version of the MFEM library.
-// Requires a HIP compiler (hipcc).
-#cmakedefine MFEM_USE_HIP
-
 // Enable MFEM functionality based on the RAJA library
 #cmakedefine MFEM_USE_RAJA
 

config/cmake/modules/FindAMGX.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindAxom.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindCXXABIDemangle.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindConduit.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindGSLIB.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindHIOP.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindHIP.cmake

@@ -1,692 +0,0 @@
-###############################################################################
-# FindHIP.cmake
-###############################################################################
-include(CheckCXXCompilerFlag)
-###############################################################################
-# SET: Variable defaults
-###############################################################################
-# User defined flags
-set(HIP_HIPCC_FLAGS "" CACHE STRING "Semicolon delimited flags for HIPCC")
-set(HIP_HCC_FLAGS "" CACHE STRING "Semicolon delimited flags for HCC")
-set(HIP_CLANG_FLAGS "" CACHE STRING "Semicolon delimited flags for CLANG")
-set(HIP_NVCC_FLAGS "" CACHE STRING "Semicolon delimted flags for NVCC")
-mark_as_advanced(HIP_HIPCC_FLAGS HIP_HCC_FLAGS HIP_CLANG_FLAGS HIP_NVCC_FLAGS)
-
-set(_hip_configuration_types ${CMAKE_CONFIGURATION_TYPES} ${CMAKE_BUILD_TYPE} Debug MinSizeRel Release RelWithDebInfo)
-list(REMOVE_DUPLICATES _hip_configuration_types)
-foreach(config ${_hip_configuration_types})
-    string(TOUPPER ${config} config_upper)
-    set(HIP_HIPCC_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for HIPCC")
-    set(HIP_HCC_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for HCC")
-    set(HIP_CLANG_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for CLANG")
-    set(HIP_NVCC_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for NVCC")
-    mark_as_advanced(HIP_HIPCC_FLAGS_${config_upper} HIP_HCC_FLAGS_${config_upper} HIP_CLANG_FLAGS_${config_upper} HIP_NVCC_FLAGS_${config_upper})
-endforeach()
-option(HIP_HOST_COMPILATION_CPP "Host code compilation mode" ON)
-option(HIP_VERBOSE_BUILD "Print out the commands run while compiling the HIP source file.  With the Makefile generator this defaults to VERBOSE variable specified on the command line, but can be forced on with this option." OFF)
-mark_as_advanced(HIP_HOST_COMPILATION_CPP)
-
-###############################################################################
-# FIND: HIP and associated helper binaries
-###############################################################################
-
-get_filename_component(_IMPORT_PREFIX "${CMAKE_CURRENT_LIST_DIR}/../" REALPATH)
-
-# HIP is supported on Linux only
-if(UNIX AND NOT APPLE AND NOT CYGWIN)
-    # Search for HIP installation
-    if(NOT HIP_ROOT_DIR)
-        # Search in user specified path first
-        find_path(
-            HIP_ROOT_DIR
-            NAMES bin/hipconfig
-            PATHS
-            "$ENV{ROCM_PATH}/hip"
-            ENV HIP_PATH
-            ${_IMPORT_PREFIX}
-            /opt/rocm/hip
-            DOC "HIP installed location"
-            NO_DEFAULT_PATH
-            )
-        if(NOT EXISTS ${HIP_ROOT_DIR})
-            if(HIP_FIND_REQUIRED)
-                message(FATAL_ERROR "Specify HIP_ROOT_DIR")
-            elseif(NOT HIP_FIND_QUIETLY)
-                message("HIP_ROOT_DIR not found or specified")
-            endif()
-        endif()
-        # And push it back to the cache
-        set(HIP_ROOT_DIR ${HIP_ROOT_DIR} CACHE PATH "HIP installed location" FORCE)
-    endif()
-
-    # Find HIPCC executable
-    find_program(
-        HIP_HIPCC_EXECUTABLE
-        NAMES hipcc
-        PATHS
-        "${HIP_ROOT_DIR}"
-        ENV ROCM_PATH
-        ENV HIP_PATH
-        /opt/rocm
-        /opt/rocm/hip
-        PATH_SUFFIXES bin
-        NO_DEFAULT_PATH
-        )
-    if(NOT HIP_HIPCC_EXECUTABLE)
-        # Now search in default paths
-        find_program(HIP_HIPCC_EXECUTABLE hipcc)
-    endif()
-    mark_as_advanced(HIP_HIPCC_EXECUTABLE)
-
-    # Find HIPCONFIG executable
-    find_program(
-        HIP_HIPCONFIG_EXECUTABLE
-        NAMES hipconfig
-        PATHS
-        "${HIP_ROOT_DIR}"
-        ENV ROCM_PATH
-        ENV HIP_PATH
-        /opt/rocm
-        /opt/rocm/hip
-        PATH_SUFFIXES bin
-        NO_DEFAULT_PATH
-        )
-    if(NOT HIP_HIPCONFIG_EXECUTABLE)
-        # Now search in default paths
-        find_program(HIP_HIPCONFIG_EXECUTABLE hipconfig)
-    endif()
-    mark_as_advanced(HIP_HIPCONFIG_EXECUTABLE)
-
-    # Find HIPCC_CMAKE_LINKER_HELPER executable
-    find_program(
-        HIP_HIPCC_CMAKE_LINKER_HELPER
-        NAMES hipcc_cmake_linker_helper
-        PATHS
-        "${HIP_ROOT_DIR}"
-        ENV ROCM_PATH
-        ENV HIP_PATH
-        /opt/rocm
-        /opt/rocm/hip
-        PATH_SUFFIXES bin
-        NO_DEFAULT_PATH
-        )
-    if(NOT HIP_HIPCC_CMAKE_LINKER_HELPER)
-        # Now search in default paths
-        find_program(HIP_HIPCC_CMAKE_LINKER_HELPER hipcc_cmake_linker_helper)
-    endif()
-    mark_as_advanced(HIP_HIPCC_CMAKE_LINKER_HELPER)
-
-    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_VERSION)
-        # Compute the version
-        execute_process(
-            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --version
-            OUTPUT_VARIABLE _hip_version
-            ERROR_VARIABLE _hip_error
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            ERROR_STRIP_TRAILING_WHITESPACE
-            )
-        if(NOT _hip_error)
-            set(HIP_VERSION ${_hip_version} CACHE STRING "Version of HIP as computed from hipcc")
-        else()
-            set(HIP_VERSION "0.0.0" CACHE STRING "Version of HIP as computed by FindHIP()")
-        endif()
-        mark_as_advanced(HIP_VERSION)
-    endif()
-    if(HIP_VERSION)
-        string(REPLACE "." ";" _hip_version_list "${HIP_VERSION}")
-        list(GET _hip_version_list 0 HIP_VERSION_MAJOR)
-        list(GET _hip_version_list 1 HIP_VERSION_MINOR)
-        list(GET _hip_version_list 2 HIP_VERSION_PATCH)
-        set(HIP_VERSION_STRING "${HIP_VERSION}")
-    endif()
-
-    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_PLATFORM)
-        # Compute the platform
-        execute_process(
-            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --platform
-            OUTPUT_VARIABLE _hip_platform
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            )
-        set(HIP_PLATFORM ${_hip_platform} CACHE STRING "HIP platform as computed by hipconfig")
-        mark_as_advanced(HIP_PLATFORM)
-    endif()
-
-    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_COMPILER)
-        # Compute the compiler
-        execute_process(
-            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --compiler
-            OUTPUT_VARIABLE _hip_compiler
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            )
-        set(HIP_COMPILER ${_hip_compiler} CACHE STRING "HIP compiler as computed by hipconfig")
-        mark_as_advanced(HIP_COMPILER)
-    endif()
-
-    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_RUNTIME)
-        # Compute the runtime
-        execute_process(
-            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --runtime
-            OUTPUT_VARIABLE _hip_runtime
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            )
-        set(HIP_RUNTIME ${_hip_runtime} CACHE STRING "HIP runtime as computed by hipconfig")
-        mark_as_advanced(HIP_RUNTIME)
-    endif()
-endif()
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(
-    HIP
-    REQUIRED_VARS
-    HIP_ROOT_DIR
-    HIP_HIPCC_EXECUTABLE
-    HIP_HIPCONFIG_EXECUTABLE
-    HIP_PLATFORM
-    HIP_COMPILER
-    HIP_RUNTIME
-    VERSION_VAR HIP_VERSION
-    )
-
-###############################################################################
-# Set HIP CMAKE Flags
-###############################################################################
-# Copy the invocation styles from CXX to HIP
-set(CMAKE_HIP_ARCHIVE_CREATE ${CMAKE_CXX_ARCHIVE_CREATE})
-set(CMAKE_HIP_ARCHIVE_APPEND ${CMAKE_CXX_ARCHIVE_APPEND})
-set(CMAKE_HIP_ARCHIVE_FINISH ${CMAKE_CXX_ARCHIVE_FINISH})
-set(CMAKE_SHARED_LIBRARY_SONAME_HIP_FLAG ${CMAKE_SHARED_LIBRARY_SONAME_CXX_FLAG})
-set(CMAKE_SHARED_LIBRARY_CREATE_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS})
-set(CMAKE_SHARED_LIBRARY_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_CXX_FLAGS})
-#set(CMAKE_SHARED_LIBRARY_LINK_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS})
-set(CMAKE_SHARED_LIBRARY_RUNTIME_HIP_FLAG ${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG})
-set(CMAKE_SHARED_LIBRARY_RUNTIME_HIP_FLAG_SEP ${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG_SEP})
-set(CMAKE_SHARED_LIBRARY_LINK_STATIC_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_STATIC_CXX_FLAGS})
-set(CMAKE_SHARED_LIBRARY_LINK_DYNAMIC_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_DYNAMIC_CXX_FLAGS})
-
-set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "")
-set(HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS "")
-
-if("${HIP_COMPILER}" STREQUAL "nvcc")
-    # Set the CMake Flags to use the nvcc Compiler.
-    set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
-    set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <LINK_LIBRARIES> -shared" )
-    set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-elseif("${HIP_COMPILER}" STREQUAL "hcc")
-    # Set the CMake Flags to use the hcc Compiler.
-    set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
-    set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <LINK_LIBRARIES> -shared" )
-    set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-elseif("${HIP_COMPILER}" STREQUAL "clang")
-    #Number of parallel jobs by default is 1
-    if(NOT DEFINED HIP_CLANG_NUM_PARALLEL_JOBS)
-      set(HIP_CLANG_NUM_PARALLEL_JOBS 1)
-    endif()
-    #Add support for parallel build and link
-    if(${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
-      check_cxx_compiler_flag("-parallel-jobs=1" HIP_CLANG_SUPPORTS_PARALLEL_JOBS)
-    endif()
-    if(HIP_CLANG_NUM_PARALLEL_JOBS GREATER 1)
-      if(${HIP_CLANG_SUPPORTS_PARALLEL_JOBS})
-        set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "-Wno-format-nonliteral -parallel-jobs=${HIP_CLANG_NUM_PARALLEL_JOBS}")
-        set(HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS "-parallel-jobs=${HIP_CLANG_NUM_PARALLEL_JOBS}")
-      else()
-        message("clang compiler doesn't support parallel jobs")
-      endif()
-    endif()
-
-    # Set the CMake Flags to use the HIP-Clang Compiler.
-    set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
-    set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <LINK_LIBRARIES> -shared" )
-    set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-
-    if("${HIP_RUNTIME}" STREQUAL "rocclr")
-      if(TARGET host)
-        message(STATUS "host interface - found")
-        set(HIP_HOST_INTERFACE host)
-      endif()
-    endif()
-endif()
-
-###############################################################################
-# MACRO: Locate helper files
-###############################################################################
-macro(HIP_FIND_HELPER_FILE _name _extension)
-    set(_hip_full_name "${_name}.${_extension}")
-    get_filename_component(CMAKE_CURRENT_LIST_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH)
-    set(HIP_${_name} "${CMAKE_CURRENT_LIST_DIR}/FindHIP/${_hip_full_name}")
-    if(NOT EXISTS "${HIP_${_name}}")
-        set(error_message "${_hip_full_name} not found in ${CMAKE_CURRENT_LIST_DIR}/FindHIP")
-        if(HIP_FIND_REQUIRED)
-            message(FATAL_ERROR "${error_message}")
-        else()
-            if(NOT HIP_FIND_QUIETLY)
-                message(STATUS "${error_message}")
-            endif()
-        endif()
-    endif()
-    # Set this variable as internal, so the user isn't bugged with it.
-    set(HIP_${_name} ${HIP_${_name}} CACHE INTERNAL "Location of ${_full_name}" FORCE)
-endmacro()
-
-###############################################################################
-hip_find_helper_file(run_make2cmake cmake)
-hip_find_helper_file(run_hipcc cmake)
-###############################################################################
-
-###############################################################################
-# MACRO: Reset compiler flags
-###############################################################################
-macro(HIP_RESET_FLAGS)
-    unset(HIP_HIPCC_FLAGS)
-    unset(HIP_HCC_FLAGS)
-    unset(HIP_CLANG_FLAGS)
-    unset(HIP_NVCC_FLAGS)
-    foreach(config ${_hip_configuration_types})
-        string(TOUPPER ${config} config_upper)
-        unset(HIP_HIPCC_FLAGS_${config_upper})
-        unset(HIP_HCC_FLAGS_${config_upper})
-        unset(HIP_CLANG_FLAGS_${config_upper})
-        unset(HIP_NVCC_FLAGS_${config_upper})
-    endforeach()
-endmacro()
-
-###############################################################################
-# MACRO: Separate the options from the sources
-###############################################################################
-macro(HIP_GET_SOURCES_AND_OPTIONS _sources _cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options)
-    set(${_sources})
-    set(${_cmake_options})
-    set(${_hipcc_options})
-    set(${_hcc_options})
-    set(${_clang_options})
-    set(${_nvcc_options})
-    set(_hipcc_found_options FALSE)
-    set(_hcc_found_options FALSE)
-    set(_clang_found_options FALSE)
-    set(_nvcc_found_options FALSE)
-    foreach(arg ${ARGN})
-        if("x${arg}" STREQUAL "xHIPCC_OPTIONS")
-            set(_hipcc_found_options TRUE)
-            set(_hcc_found_options FALSE)
-            set(_clang_found_options FALSE)
-            set(_nvcc_found_options FALSE)
-        elseif("x${arg}" STREQUAL "xHCC_OPTIONS")
-            set(_hipcc_found_options FALSE)
-            set(_hcc_found_options TRUE)
-            set(_clang_found_options FALSE)
-            set(_nvcc_found_options FALSE)
-        elseif("x${arg}" STREQUAL "xCLANG_OPTIONS")
-            set(_hipcc_found_options FALSE)
-            set(_hcc_found_options FALSE)
-            set(_clang_found_options TRUE)
-            set(_nvcc_found_options FALSE)
-        elseif("x${arg}" STREQUAL "xNVCC_OPTIONS")
-            set(_hipcc_found_options FALSE)
-            set(_hcc_found_options FALSE)
-            set(_clang_found_options FALSE)
-            set(_nvcc_found_options TRUE)
-        elseif(
-                "x${arg}" STREQUAL "xEXCLUDE_FROM_ALL" OR
-                "x${arg}" STREQUAL "xSTATIC" OR
-                "x${arg}" STREQUAL "xSHARED" OR
-                "x${arg}" STREQUAL "xMODULE"
-                )
-            list(APPEND ${_cmake_options} ${arg})
-        else()
-            if(_hipcc_found_options)
-                list(APPEND ${_hipcc_options} ${arg})
-            elseif(_hcc_found_options)
-                list(APPEND ${_hcc_options} ${arg})
-            elseif(_clang_found_options)
-                list(APPEND ${_clang_options} ${arg})
-            elseif(_nvcc_found_options)
-                list(APPEND ${_nvcc_options} ${arg})
-            else()
-                # Assume this is a file
-                list(APPEND ${_sources} ${arg})
-            endif()
-        endif()
-    endforeach()
-endmacro()
-
-###############################################################################
-# MACRO: Add include directories to pass to the hipcc command
-###############################################################################
-set(HIP_HIPCC_INCLUDE_ARGS_USER "")
-macro(HIP_INCLUDE_DIRECTORIES)
-    foreach(dir ${ARGN})
-        list(APPEND HIP_HIPCC_INCLUDE_ARGS_USER $<$<BOOL:${dir}>:-I${dir}>)
-    endforeach()
-endmacro()
-
-###############################################################################
-# FUNCTION: Helper to avoid clashes of files with the same basename but different paths
-###############################################################################
-function(HIP_COMPUTE_BUILD_PATH path build_path)
-    # Convert to cmake style paths
-    file(TO_CMAKE_PATH "${path}" bpath)
-    if(IS_ABSOLUTE "${bpath}")
-        string(FIND "${bpath}" "${CMAKE_CURRENT_BINARY_DIR}" _binary_dir_pos)
-        if(_binary_dir_pos EQUAL 0)
-            file(RELATIVE_PATH bpath "${CMAKE_CURRENT_BINARY_DIR}" "${bpath}")
-        else()
-            file(RELATIVE_PATH bpath "${CMAKE_CURRENT_SOURCE_DIR}" "${bpath}")
-        endif()
-    endif()
-
-    # Remove leading /
-    string(REGEX REPLACE "^[/]+" "" bpath "${bpath}")
-    # Avoid absolute paths by removing ':'
-    string(REPLACE ":" "_" bpath "${bpath}")
-    # Avoid relative paths that go up the tree
-    string(REPLACE "../" "__/" bpath "${bpath}")
-    # Avoid spaces
-    string(REPLACE " " "_" bpath "${bpath}")
-    # Strip off the filename
-    get_filename_component(bpath "${bpath}" PATH)
-
-    set(${build_path} "${bpath}" PARENT_SCOPE)
-endfunction()
-
-###############################################################################
-# MACRO: Parse OPTIONS from ARGN & set variables prefixed by _option_prefix
-###############################################################################
-macro(HIP_PARSE_HIPCC_OPTIONS _option_prefix)
-    set(_hip_found_config)
-    foreach(arg ${ARGN})
-        # Determine if we are dealing with a per-configuration flag
-        foreach(config ${_hip_configuration_types})
-            string(TOUPPER ${config} config_upper)
-            if(arg STREQUAL "${config_upper}")
-                set(_hip_found_config _${arg})
-                # Clear arg to prevent it from being processed anymore
-                set(arg)
-            endif()
-        endforeach()
-        if(arg)
-            list(APPEND ${_option_prefix}${_hip_found_config} "${arg}")
-        endif()
-    endforeach()
-endmacro()
-
-###############################################################################
-# MACRO: Try and include dependency file if it exists
-###############################################################################
-macro(HIP_INCLUDE_HIPCC_DEPENDENCIES dependency_file)
-    set(HIP_HIPCC_DEPEND)
-    set(HIP_HIPCC_DEPEND_REGENERATE FALSE)
-
-    # Create the dependency file if it doesn't exist
-    if(NOT EXISTS ${dependency_file})
-        file(WRITE ${dependency_file} "# Generated by: FindHIP.cmake. Do not edit.\n")
-    endif()
-    # Include the dependency file
-    include(${dependency_file})
-
-    # Verify the existence of all the included files
-    if(HIP_HIPCC_DEPEND)
-        foreach(f ${HIP_HIPCC_DEPEND})
-            if(NOT EXISTS ${f})
-                # If they aren't there, regenerate the file again
-                set(HIP_HIPCC_DEPEND_REGENERATE TRUE)
-            endif()
-        endforeach()
-    else()
-        # No dependencies, so regenerate the file
-        set(HIP_HIPCC_DEPEND_REGENERATE TRUE)
-    endif()
-
-    # Regenerate the dependency file if needed
-    if(HIP_HIPCC_DEPEND_REGENERATE)
-        set(HIP_HIPCC_DEPEND ${dependency_file})
-        file(WRITE ${dependency_file} "# Generated by: FindHIP.cmake. Do not edit.\n")
-    endif()
-endmacro()
-
-###############################################################################
-# MACRO: Prepare cmake commands for the target
-###############################################################################
-macro(HIP_PREPARE_TARGET_COMMANDS _target _format _generated_files _source_files)
-    set(_hip_flags "")
-    string(TOUPPER "${CMAKE_BUILD_TYPE}" _hip_build_configuration)
-    if(HIP_HOST_COMPILATION_CPP)
-        set(HIP_C_OR_CXX CXX)
-    else()
-        set(HIP_C_OR_CXX C)
-    endif()
-    set(generated_extension ${CMAKE_${HIP_C_OR_CXX}_OUTPUT_EXTENSION})
-
-    # Initialize list of includes with those specified by the user. Append with
-    # ones specified to cmake directly.
-    set(HIP_HIPCC_INCLUDE_ARGS ${HIP_HIPCC_INCLUDE_ARGS_USER})
-
-    # Add the include directories
-    set(include_directories_generator "$<TARGET_PROPERTY:${_target},INCLUDE_DIRECTORIES>")
-    list(APPEND HIP_HIPCC_INCLUDE_ARGS "$<$<BOOL:${include_directories_generator}>:-I$<JOIN:${include_directories_generator}, -I>>")
-
-    get_directory_property(_hip_include_directories INCLUDE_DIRECTORIES)
-    list(REMOVE_DUPLICATES _hip_include_directories)
-    if(_hip_include_directories)
-        foreach(dir ${_hip_include_directories})
-            list(APPEND HIP_HIPCC_INCLUDE_ARGS $<$<BOOL:${dir}>:-I${dir}>)
-        endforeach()
-    endif()
-
-    HIP_GET_SOURCES_AND_OPTIONS(_hip_sources _hip_cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options ${ARGN})
-    HIP_PARSE_HIPCC_OPTIONS(HIP_HIPCC_FLAGS ${_hipcc_options})
-    HIP_PARSE_HIPCC_OPTIONS(HIP_HCC_FLAGS ${_hcc_options})
-    HIP_PARSE_HIPCC_OPTIONS(HIP_CLANG_FLAGS ${_clang_options})
-    HIP_PARSE_HIPCC_OPTIONS(HIP_NVCC_FLAGS ${_nvcc_options})
-
-    # Add the compile definitions
-    set(compile_definition_generator "$<TARGET_PROPERTY:${_target},COMPILE_DEFINITIONS>")
-    list(APPEND HIP_HIPCC_FLAGS "$<$<BOOL:${compile_definition_generator}>:-D$<JOIN:${compile_definition_generator}, -D>>")
-
-    # Check if we are building shared library.
-    set(_hip_build_shared_libs FALSE)
-    list(FIND _hip_cmake_options SHARED _hip_found_SHARED)
-    list(FIND _hip_cmake_options MODULE _hip_found_MODULE)
-    if(_hip_found_SHARED GREATER -1 OR _hip_found_MODULE GREATER -1)
-        set(_hip_build_shared_libs TRUE)
-    endif()
-    list(FIND _hip_cmake_options STATIC _hip_found_STATIC)
-    if(_hip_found_STATIC GREATER -1)
-        set(_hip_build_shared_libs FALSE)
-    endif()
-
-    # If we are building a shared library, add extra flags to HIP_HIPCC_FLAGS
-    if(_hip_build_shared_libs)
-        list(APPEND HIP_HCC_FLAGS "-fPIC")
-        list(APPEND HIP_CLANG_FLAGS "-fPIC")
-        list(APPEND HIP_NVCC_FLAGS "--shared -Xcompiler '-fPIC'")
-    endif()
-
-    # Set host compiler
-    set(HIP_HOST_COMPILER "${CMAKE_${HIP_C_OR_CXX}_COMPILER}")
-
-    # Set compiler flags
-    set(_HIP_HOST_FLAGS "set(CMAKE_HOST_FLAGS ${CMAKE_${HIP_C_OR_CXX}_FLAGS})")
-    set(_HIP_HIPCC_FLAGS "set(HIP_HIPCC_FLAGS ${HIP_HIPCC_FLAGS})")
-    set(_HIP_HCC_FLAGS "set(HIP_HCC_FLAGS ${HIP_HCC_FLAGS})")
-    set(_HIP_CLANG_FLAGS "set(HIP_CLANG_FLAGS ${HIP_CLANG_FLAGS})")
-    set(_HIP_NVCC_FLAGS "set(HIP_NVCC_FLAGS ${HIP_NVCC_FLAGS})")
-    foreach(config ${_hip_configuration_types})
-        string(TOUPPER ${config} config_upper)
-        set(_HIP_HOST_FLAGS "${_HIP_HOST_FLAGS}\nset(CMAKE_HOST_FLAGS_${config_upper} ${CMAKE_${HIP_C_OR_CXX}_FLAGS_${config_upper}})")
-        set(_HIP_HIPCC_FLAGS "${_HIP_HIPCC_FLAGS}\nset(HIP_HIPCC_FLAGS_${config_upper} ${HIP_HIPCC_FLAGS_${config_upper}})")
-        set(_HIP_HCC_FLAGS "${_HIP_HCC_FLAGS}\nset(HIP_HCC_FLAGS_${config_upper} ${HIP_HCC_FLAGS_${config_upper}})")
-        set(_HIP_CLANG_FLAGS "${_HIP_CLANG_FLAGS}\nset(HIP_CLANG_FLAGS_${config_upper} ${HIP_CLANG_FLAGS_${config_upper}})")
-        set(_HIP_NVCC_FLAGS "${_HIP_NVCC_FLAGS}\nset(HIP_NVCC_FLAGS_${config_upper} ${HIP_NVCC_FLAGS_${config_upper}})")
-    endforeach()
-
-    # Reset the output variable
-    set(_hip_generated_files "")
-    set(_hip_source_files "")
-
-    # Iterate over all arguments and create custom commands for all source files
-    foreach(file ${ARGN})
-        # Ignore any file marked as a HEADER_FILE_ONLY
-        get_source_file_property(_is_header ${file} HEADER_FILE_ONLY)
-        # Allow per source file overrides of the format. Also allows compiling non .cu files.
-        get_source_file_property(_hip_source_format ${file} HIP_SOURCE_PROPERTY_FORMAT)
-        if((${file} MATCHES "\\.cu$" OR _hip_source_format) AND NOT _is_header)
-            set(host_flag FALSE)
-        else()
-            set(host_flag TRUE)
-        endif()
-
-        if(NOT host_flag)
-            # Determine output directory
-            HIP_COMPUTE_BUILD_PATH("${file}" hip_build_path)
-            set(hip_compile_output_dir "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${_target}.dir/${hip_build_path}")
-
-            get_filename_component(basename ${file} NAME)
-            set(generated_file_path "${hip_compile_output_dir}/${CMAKE_CFG_INTDIR}")
-            set(generated_file_basename "${_target}_generated_${basename}${generated_extension}")
-
-            # Set file names
-            set(generated_file "${generated_file_path}/${generated_file_basename}")
-            set(cmake_dependency_file "${hip_compile_output_dir}/${generated_file_basename}.depend")
-            set(custom_target_script_pregen "${hip_compile_output_dir}/${generated_file_basename}.cmake.pre-gen")
-            set(custom_target_script "${hip_compile_output_dir}/${generated_file_basename}.cmake")
-
-            # Set properties for object files
-            set_source_files_properties("${generated_file}"
-                PROPERTIES
-                EXTERNAL_OBJECT true # This is an object file not to be compiled, but only be linked
-                )
-
-            # Don't add CMAKE_CURRENT_SOURCE_DIR if the path is already an absolute path
-            get_filename_component(file_path "${file}" PATH)
-            if(IS_ABSOLUTE "${file_path}")
-                set(source_file "${file}")
-            else()
-                set(source_file "${CMAKE_CURRENT_SOURCE_DIR}/${file}")
-            endif()
-
-            # Bring in the dependencies
-            HIP_INCLUDE_HIPCC_DEPENDENCIES(${cmake_dependency_file})
-
-            # Configure the build script
-            configure_file("${HIP_run_hipcc}" "${custom_target_script_pregen}" @ONLY)
-            file(GENERATE
-                OUTPUT "${custom_target_script}"
-                INPUT "${custom_target_script_pregen}"
-                )
-            set(main_dep DEPENDS ${source_file})
-            if(CMAKE_GENERATOR MATCHES "Makefiles")
-                set(verbose_output "$(VERBOSE)")
-            elseif(HIP_VERBOSE_BUILD)
-                set(verbose_output ON)
-            else()
-                set(verbose_output OFF)
-            endif()
-
-            # Create up the comment string
-            file(RELATIVE_PATH generated_file_relative_path "${CMAKE_BINARY_DIR}" "${generated_file}")
-            set(hip_build_comment_string "Building HIPCC object ${generated_file_relative_path}")
-
-            # Build the generated file and dependency file
-            add_custom_command(
-                OUTPUT ${generated_file}
-                # These output files depend on the source_file and the contents of cmake_dependency_file
-                ${main_dep}
-                DEPENDS ${HIP_HIPCC_DEPEND}
-                DEPENDS ${custom_target_script}
-                # Make sure the output directory exists before trying to write to it.
-                COMMAND ${CMAKE_COMMAND} -E make_directory "${generated_file_path}"
-                COMMAND ${CMAKE_COMMAND} ARGS
-                -D verbose:BOOL=${verbose_output}
-                -D build_configuration:STRING=${_hip_build_configuration}
-                -D "generated_file:STRING=${generated_file}"
-                -P "${custom_target_script}"
-                WORKING_DIRECTORY "${hip_compile_output_dir}"
-                COMMENT "${hip_build_comment_string}"
-                )
-
-            # Make sure the build system knows the file is generated
-            set_source_files_properties(${generated_file} PROPERTIES GENERATED TRUE)
-            list(APPEND _hip_generated_files ${generated_file})
-            list(APPEND _hip_source_files ${file})
-        endif()
-    endforeach()
-
-    # Set the return parameter
-    set(${_generated_files} ${_hip_generated_files})
-    set(${_source_files} ${_hip_source_files})
-endmacro()
-
-###############################################################################
-# HIP_ADD_EXECUTABLE
-###############################################################################
-macro(HIP_ADD_EXECUTABLE hip_target)
-    # Separate the sources from the options
-    HIP_GET_SOURCES_AND_OPTIONS(_sources _cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options ${ARGN})
-    HIP_PREPARE_TARGET_COMMANDS(${hip_target} OBJ _generated_files _source_files ${_sources} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} CLANG_OPTIONS ${_clang_options} NVCC_OPTIONS ${_nvcc_options})
-    if(_source_files)
-        list(REMOVE_ITEM _sources ${_source_files})
-    endif()
-    if("${HIP_COMPILER}" STREQUAL "hcc")
-        if("x${HCC_HOME}" STREQUAL "x")
-            if (DEFINED ENV{ROCM_PATH})
-                set(HCC_HOME "$ENV{ROCM_PATH}/hcc")
-            elseif(DEFINED ENV{HIP_PATH})
-                set(HCC_HOME "$ENV{HIP_PATH}/../hcc")
-            else()
-                set(HCC_HOME "/opt/rocm/hcc")
-            endif()
-        endif()
-        set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-    elseif("${HIP_COMPILER}" STREQUAL "clang")
-        if("x${HIP_CLANG_PATH}" STREQUAL "x")
-            if(DEFINED ENV{HIP_CLANG_PATH})
-                set(HIP_CLANG_PATH $ENV{HIP_CLANG_PATH})
-            elseif(DEFINED ENV{ROCM_PATH})
-                set(HIP_CLANG_PATH "$ENV{ROCM_PATH}/llvm/bin")
-            elseif(DEFINED ENV{HIP_PATH})
-                set(HIP_CLANG_PATH "$ENV{HIP_PATH}/../llvm/bin")
-            else()
-                set(HIP_CLANG_PATH "/opt/rocm/llvm/bin")
-            endif()
-        endif()
-        set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-    else()
-        set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-    endif()
-    if ("${_sources}" STREQUAL "")
-        add_executable(${hip_target} ${_cmake_options} ${_generated_files} "")
-    else()
-        add_executable(${hip_target} ${_cmake_options} ${_generated_files} ${_sources})
-    endif()
-    set_target_properties(${hip_target} PROPERTIES LINKER_LANGUAGE HIP)
-    # Link with host
-    if (HIP_HOST_INTERFACE)
-        # hip rt should be rocclr, compiler should be clang
-        target_link_libraries(${hip_target} ${HIP_HOST_INTERFACE})
-    endif()
-endmacro()
-
-###############################################################################
-# HIP_ADD_LIBRARY
-###############################################################################
-macro(HIP_ADD_LIBRARY hip_target)
-    # Separate the sources from the options
-    HIP_GET_SOURCES_AND_OPTIONS(_sources _cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options ${ARGN})
-    HIP_PREPARE_TARGET_COMMANDS(${hip_target} OBJ _generated_files _source_files ${_sources} ${_cmake_options} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} CLANG_OPTIONS ${_clang_options} NVCC_OPTIONS ${_nvcc_options})
-    if(_source_files)
-        list(REMOVE_ITEM _sources ${_source_files})
-    endif()
-    if ("${_sources}" STREQUAL "")
-        add_library(${hip_target} ${_cmake_options} ${_generated_files} "")
-    else()
-        add_library(${hip_target} ${_cmake_options} ${_generated_files} ${_sources})
-    endif()
-    set_target_properties(${hip_target} PROPERTIES LINKER_LANGUAGE ${HIP_C_OR_CXX})
-    # Link with host
-    if (HIP_HOST_INTERFACE)
-        # hip rt should be rocclr, compiler should be clang
-        target_link_libraries(${hip_target} ${HIP_HOST_INTERFACE})
-    endif()
-endmacro()
-
-# vim: ts=4:sw=4:expandtab:smartindent

config/cmake/modules/FindHIP/run_hipcc.cmake

@@ -1,182 +0,0 @@
-###############################################################################
-# Runs commands using HIPCC
-###############################################################################
-
-###############################################################################
-# This file runs the hipcc commands to produce the desired output file
-# along with the dependency file needed by CMake to compute dependencies.
-#
-# Input variables:
-#
-# verbose:BOOL=<>               OFF: Be as quiet as possible (default)
-#                               ON : Describe each step
-# build_configuration:STRING=<> Build configuration. Defaults to Debug.
-# generated_file:STRING=<>      File to generate. Mandatory argument.
-
-if(NOT build_configuration)
-    set(build_configuration Debug)
-endif()
-if(NOT generated_file)
-    message(FATAL_ERROR "You must specify generated_file on the command line")
-endif()
-
-# Set these up as variables to make reading the generated file easier
-set(HIP_HIPCC_EXECUTABLE "@HIP_HIPCC_EXECUTABLE@") # path
-set(HIP_HIPCONFIG_EXECUTABLE "@HIP_HIPCONFIG_EXECUTABLE@") #path
-set(HIP_HOST_COMPILER "@HIP_HOST_COMPILER@") # path
-set(CMAKE_COMMAND "@CMAKE_COMMAND@") # path
-set(HIP_run_make2cmake "@HIP_run_make2cmake@") # path
-set(HCC_HOME "@HCC_HOME@") #path
-set(HIP_CLANG_PATH "@HIP_CLANG_PATH@") #path
-set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "@HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS@")
-
-@HIP_HOST_FLAGS@
-@_HIP_HIPCC_FLAGS@
-@_HIP_HCC_FLAGS@
-@_HIP_CLANG_FLAGS@
-@_HIP_NVCC_FLAGS@
-#Needed to bring the HIP_HIPCC_INCLUDE_ARGS variable in scope
-set(HIP_HIPCC_INCLUDE_ARGS @HIP_HIPCC_INCLUDE_ARGS@) # list
-
-set(cmake_dependency_file "@cmake_dependency_file@") # path
-set(source_file "@source_file@") # path
-set(host_flag "@host_flag@") # bool
-
-# Determine compiler and compiler flags
-execute_process(COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --platform OUTPUT_VARIABLE HIP_PLATFORM OUTPUT_STRIP_TRAILING_WHITESPACE)
-execute_process(COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --compiler OUTPUT_VARIABLE HIP_COMPILER OUTPUT_STRIP_TRAILING_WHITESPACE)
-execute_process(COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --runtime OUTPUT_VARIABLE HIP_RUNTIME OUTPUT_STRIP_TRAILING_WHITESPACE)
-if(NOT host_flag)
-    set(__CC ${HIP_HIPCC_EXECUTABLE})
-    if("${HIP_PLATFORM}" STREQUAL "amd")
-        if("${HIP_COMPILER}" STREQUAL "hcc")
-            if(NOT "x${HCC_HOME}" STREQUAL "x")
-                set(ENV{HCC_HOME} ${HCC_HOME})
-            endif()
-            set(__CC_FLAGS ${HIP_HIPCC_FLAGS} ${HIP_HCC_FLAGS} ${HIP_HIPCC_FLAGS_${build_configuration}} ${HIP_HCC_FLAGS_${build_configuration}})
-        elseif("${HIP_COMPILER}" STREQUAL "clang")
-            if(NOT "x${HIP_CLANG_PATH}" STREQUAL "x")
-                set(ENV{HIP_CLANG_PATH} ${HIP_CLANG_PATH})
-            endif()
-            # Temporarily include HIP_HCC_FLAGS for HIP-Clang for PyTorch builds
-            set(__CC_FLAGS ${HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS} ${HIP_HIPCC_FLAGS} ${HIP_HCC_FLAGS} ${HIP_CLANG_FLAGS} ${HIP_HIPCC_FLAGS_${build_configuration}} ${HIP_HCC_FLAGS_${build_configuration}} ${HIP_CLANG_FLAGS_${build_configuration}})
-        endif()
-    else()
-        set(__CC_FLAGS ${HIP_HIPCC_FLAGS} ${HIP_NVCC_FLAGS} ${HIP_HIPCC_FLAGS_${build_configuration}} ${HIP_NVCC_FLAGS_${build_configuration}})
-    endif()
-else()
-    set(__CC ${HIP_HOST_COMPILER})
-    set(__CC_FLAGS ${CMAKE_HOST_FLAGS} ${CMAKE_HOST_FLAGS_${build_configuration}})
-endif()
-set(__CC_INCLUDES ${HIP_HIPCC_INCLUDE_ARGS})
-
-# hip_execute_process - Executes a command with optional command echo and status message.
-#   status     - Status message to print if verbose is true
-#   command    - COMMAND argument from the usual execute_process argument structure
-#   ARGN       - Remaining arguments are the command with arguments
-#   HIP_result - Return value from running the command
-macro(hip_execute_process status command)
-    set(_command ${command})
-    if(NOT "x${_command}" STREQUAL "xCOMMAND")
-        message(FATAL_ERROR "Malformed call to hip_execute_process.  Missing COMMAND as second argument. (command = ${command})")
-    endif()
-    if(verbose)
-        execute_process(COMMAND "${CMAKE_COMMAND}" -E echo -- ${status})
-        # Build command string to print
-        set(hip_execute_process_string)
-        foreach(arg ${ARGN})
-            # Escape quotes if any
-            string(REPLACE "\"" "\\\"" arg ${arg})
-            # Surround args with spaces with quotes
-            if(arg MATCHES " ")
-                list(APPEND hip_execute_process_string "\"${arg}\"")
-            else()
-                list(APPEND hip_execute_process_string ${arg})
-            endif()
-        endforeach()
-        # Echo the command
-        execute_process(COMMAND ${CMAKE_COMMAND} -E echo ${hip_execute_process_string})
-    endif()
-    # Run the command
-    execute_process(COMMAND ${ARGN} RESULT_VARIABLE HIP_result)
-endmacro()
-
-# Delete the target file
-hip_execute_process(
-    "Removing ${generated_file}"
-    COMMAND "${CMAKE_COMMAND}" -E remove "${generated_file}"
-    )
-
-# Generate the dependency file
-hip_execute_process(
-    "Generating dependency file: ${cmake_dependency_file}.pre"
-    COMMAND "${__CC}"
-    -M
-    "${source_file}"
-    -o "${cmake_dependency_file}.pre"
-    ${__CC_FLAGS}
-    ${__CC_INCLUDES}
-    )
-
-if(HIP_result)
-    message(FATAL_ERROR "Error generating ${generated_file}")
-endif()
-
-# Generate the cmake readable dependency file to a temp file
-hip_execute_process(
-    "Generating temporary cmake readable file: ${cmake_dependency_file}.tmp"
-    COMMAND "${CMAKE_COMMAND}"
-    -D "input_file:FILEPATH=${cmake_dependency_file}.pre"
-    -D "output_file:FILEPATH=${cmake_dependency_file}.tmp"
-    -D "verbose=${verbose}"
-    -P "${HIP_run_make2cmake}"
-    )
-
-if(HIP_result)
-    message(FATAL_ERROR "Error generating ${generated_file}")
-endif()
-
-# Copy the file if it is different
-hip_execute_process(
-    "Copy if different ${cmake_dependency_file}.tmp to ${cmake_dependency_file}"
-    COMMAND "${CMAKE_COMMAND}" -E copy_if_different "${cmake_dependency_file}.tmp" "${cmake_dependency_file}"
-    )
-
-if(HIP_result)
-    message(FATAL_ERROR "Error generating ${generated_file}")
-endif()
-
-# Delete the temporary file
-hip_execute_process(
-    "Removing ${cmake_dependency_file}.tmp and ${cmake_dependency_file}.pre"
-    COMMAND "${CMAKE_COMMAND}" -E remove "${cmake_dependency_file}.tmp" "${cmake_dependency_file}.pre"
-    )
-
-if(HIP_result)
-    message(FATAL_ERROR "Error generating ${generated_file}")
-endif()
-
-# Generate the output file
-hip_execute_process(
-    "Generating ${generated_file}"
-    COMMAND "${__CC}"
-    -c
-    "${source_file}"
-    -o "${generated_file}"
-    ${__CC_FLAGS}
-    ${__CC_INCLUDES}
-    )
-
-if(HIP_result)
-    # Make sure that we delete the output file
-    hip_execute_process(
-        "Removing ${generated_file}"
-        COMMAND "${CMAKE_COMMAND}" -E remove "${generated_file}"
-        )
-    message(FATAL_ERROR "Error generating file ${generated_file}")
-else()
-    if(verbose)
-        message("Generated ${generated_file} successfully.")
-    endif()
-endif()
-# vim: ts=4:sw=4:expandtab:smartindent

config/cmake/modules/FindHIP/run_make2cmake.cmake

@@ -1,50 +0,0 @@
-###############################################################################
-# Computes dependencies using HIPCC
-###############################################################################
-
-###############################################################################
-# This file converts dependency files generated using hipcc to a format that
-# cmake can understand.
-
-# Input variables:
-#
-# input_file:STRING=<> Dependency file to parse. Required argument
-# output_file:STRING=<> Output file to generate. Required argument
-
-if(NOT input_file OR NOT output_file)
-    message(FATAL_ERROR "You must specify input_file and output_file on the command line")
-endif()
-
-file(READ ${input_file} depend_text)
-
-if (NOT "${depend_text}" STREQUAL "")
-    string(REPLACE " /" "\n/" depend_text ${depend_text})
-    string(REGEX REPLACE "^.*:" "" depend_text ${depend_text})
-    string(REGEX REPLACE "[ \\\\]*\n" ";" depend_text ${depend_text})
-
-    set(dependency_list "")
-
-    foreach(file ${depend_text})
-        string(REGEX REPLACE "^ +" "" file ${file})
-        if(NOT EXISTS "${file}")
-            message(WARNING " Removing non-existent dependency file: ${file}")
-            set(file "")
-        endif()
-
-        if(NOT IS_DIRECTORY "${file}")
-            get_filename_component(file_absolute "${file}" ABSOLUTE)
-            list(APPEND dependency_list "${file_absolute}")
-        endif()
-    endforeach()
-endif()
-
-# Remove the duplicate entries and sort them.
-list(REMOVE_DUPLICATES dependency_list)
-list(SORT dependency_list)
-
-foreach(file ${dependency_list})
-    set(hip_hipcc_depend "${hip_hipcc_depend} \"${file}\"\n")
-endforeach()
-
-file(WRITE ${output_file} "# Generated by: FindHIP.cmake. Do not edit.\nSET(HIP_HIPCC_DEPEND\n ${hip_hipcc_depend})\n\n")
-# vim: ts=4:sw=4:expandtab:smartindent

config/cmake/modules/FindHYPRE.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindLIBDL.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindLibunwind.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindMETIS.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -25,16 +25,16 @@ mfem_find_package(METIS METIS METIS_DIR "include;Lib" "metis.h"
 
 int main()
 {
-    idx_t n = 10;
-    idx_t nparts = 5;
-    idx_t edgecut;
-    idx_t* partitioning = new idx_t[10];
-    idx_t* I = partitioning,
+    int n = 10;
+    int nparts = 5;
+    int edgecut;
+    int* partitioning = new int[10];
+    int* I = partitioning,
        * J = partitioning;
 
-    idx_t ncon = 1;
+    int ncon = 1;
     int err;
-    idx_t options[40];
+    int options[40];
 
     METIS_SetDefaultOptions(options);
     options[10] = 1; // set METIS_OPTION_CONTIG

config/cmake/modules/FindMFEMBacktrace.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindMKL_CPARDISO.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindMPFR.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindMUMPS.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindMesquite.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindNetCDF.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindOCCA.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindPOSIXClocks.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindParMETIS.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindRAJA.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindSLEPc.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindSTRUMPACK.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindSUNDIALS.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindScotch.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindSuiteSparse.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindSuperLUDist.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindUMPIRE.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/Find_GnuTLS.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/FindlibCEED.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -43,24 +43,6 @@ function(convert_filenames_to_full_paths NAMES)
   set(${NAMES} ${tmp_names} PARENT_SCOPE)
 endfunction()
 
-# Wrapper for add_executable that calls the HIP wrapper if applicable
-macro(mfem_add_executable NAME)
-  if (MFEM_USE_HIP)
-    hip_add_executable(${NAME} ${ARGN})
-  else()
-    add_executable(${NAME} ${ARGN})
-  endif()
-endmacro()
-
-# Wrapper for add_library that calls the HIP wrapper if applicable
-macro(mfem_add_library NAME)
-  if (MFEM_USE_HIP)
-    hip_add_library(${NAME} ${ARGN})
-  else()
-    add_library(${NAME} ${ARGN})
-  endif()
-endmacro()
-
 # Simple shortcut to add_custom_target() with option to add the target to the
 # main target.
 function(add_mfem_target TARGET_NAME ADD_TO_ALL)
@@ -74,7 +56,7 @@ function(add_mfem_target TARGET_NAME ADD_TO_ALL)
 endfunction()
 
 # Add mfem examples
-macro(add_mfem_examples EXE_SRCS)
+function(add_mfem_examples EXE_SRCS)
   set(EXE_PREFIX "")
   set(EXE_PREREQUISITE "")
   set(EXE_NEEDED_BY "")
@@ -90,15 +72,13 @@ macro(add_mfem_examples EXE_SRCS)
   foreach(SRC_FILE IN LISTS ${EXE_SRCS})
     # If CUDA is enabled, tag source files to be compiled with nvcc.
     if (MFEM_USE_CUDA)
-      set_source_files_properties(${SRC_FILE} PROPERTIES LANGUAGE CUDA)
-    elseif(MFEM_USE_HIP)
-      set_source_files_properties(${SRC_FILE} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
+      set_property(SOURCE ${SRC_FILE} PROPERTY LANGUAGE CUDA)
     endif()
 
     get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
 
     string(REPLACE ".cpp" "" EXE_NAME "${EXE_PREFIX}${SRC_FILENAME}")
-    mfem_add_executable(${EXE_NAME} ${SRC_FILE})
+    add_executable(${EXE_NAME} ${SRC_FILE})
     add_dependencies(${MFEM_ALL_EXAMPLES_TARGET_NAME} ${EXE_NAME})
     if (EXE_NEEDED_BY)
       add_dependencies(${EXE_NEEDED_BY} ${EXE_NAME})
@@ -127,10 +107,10 @@ macro(add_mfem_examples EXE_SRCS)
       endif()
     endif()
   endforeach(SRC_FILE)
-endmacro()
+endfunction()
 
 # A slightly more versatile function for adding miniapps to MFEM
-macro(add_mfem_miniapp MFEM_EXE_NAME)
+function(add_mfem_miniapp MFEM_EXE_NAME)
   # Parse the input arguments looking for the things we need
   set(POSSIBLE_ARGS "MAIN" "EXTRA_SOURCES" "EXTRA_HEADERS" "EXTRA_OPTIONS" "EXTRA_DEFINES" "LIBRARIES")
   set(CURRENT_ARG)
@@ -146,7 +126,8 @@ macro(add_mfem_miniapp MFEM_EXE_NAME)
 
   # If CUDA is enabled, tag source files to be compiled with nvcc.
   if (MFEM_USE_CUDA)
-    set_source_files_properties(${MAIN_LIST} ${EXTRA_SOURCES_LIST} PROPERTIES LANGUAGE CUDA)
+    set_property(SOURCE ${MAIN_LIST} ${EXTRA_SOURCES_LIST}
+      PROPERTY LANGUAGE CUDA)
     if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.12.0)
       list(TRANSFORM EXTRA_OPTIONS_LIST PREPEND "-Xcompiler=")
     else()
@@ -156,13 +137,11 @@ macro(add_mfem_miniapp MFEM_EXE_NAME)
       endforeach()
       set(EXTRA_OPTIONS_LIST ${LIST_})
     endif()
-  elseif(MFEM_USE_HIP)
-    set_source_files_properties(${MAIN_LIST} ${EXTRA_SOURCES_LIST} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
   endif()
 
   # Actually add the executable
-  mfem_add_executable(${MFEM_EXE_NAME} ${MAIN_LIST}
-      ${EXTRA_SOURCES_LIST} ${EXTRA_HEADERS_LIST})
+  add_executable(${MFEM_EXE_NAME} ${MAIN_LIST}
+    ${EXTRA_SOURCES_LIST} ${EXTRA_HEADERS_LIST})
   add_dependencies(${MFEM_ALL_MINIAPPS_TARGET_NAME} ${MFEM_EXE_NAME})
   add_dependencies(${MFEM_EXE_NAME} ${MFEM_EXEC_PREREQUISITES_TARGET_NAME})
 
@@ -215,7 +194,7 @@ macro(add_mfem_miniapp MFEM_EXE_NAME)
         LINK_FLAGS "${MPI_CXX_LINK_FLAGS}")
     endif()
   endif()
-endmacro()
+endfunction()
 
 
 # Auxiliary function, used in mfem_find_package().
@@ -775,13 +754,7 @@ function(mfem_export_mk_files)
   set(MFEM_CXX ${CMAKE_CXX_COMPILER})
   set(MFEM_HOST_CXX ${MFEM_CXX})
   set(MFEM_CPPFLAGS "")
-  get_target_property(cxx_std mfem CXX_STANDARD)
-  # For now, we ignore the setting of the CXX_EXTENSIONS property. If this
-  # property is set, then we need to use a variable like:
-  #    CMAKE_CXX11_EXTENSION_COMPILE_OPTION
-  set(cxx_std_flag ${CMAKE_CXX${cxx_std}_STANDARD_COMPILE_OPTION})
-  string(STRIP
-         "${cxx_std_flag} ${CMAKE_CXX_FLAGS_${BUILD_TYPE}} ${CMAKE_CXX_FLAGS}"
+  string(STRIP "${CMAKE_CXX_FLAGS_${BUILD_TYPE}} ${CMAKE_CXX_FLAGS}"
          MFEM_CXXFLAGS)
   set(MFEM_TPLFLAGS "")
   foreach(dir ${MFEM_TPL_INCLUDE_DIRS})

config/config.hpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -34,12 +34,6 @@
 // Macro needed to get defines like M_PI from <cmath>. (Visual Studio C++ only?)
 #define _USE_MATH_DEFINES
 #endif
-// On Cygwin the option -std=c++11 prevents the definition of M_PI. Defining
-// the following macro allows us to get M_PI and some needed functions, e.g.
-// posix_memalign(), strdup(), strerror_r().
-#ifdef __CYGWIN__
-#define _XOPEN_SOURCE 600
-#endif
 
 // Check dependencies:
 

config/config.hpp.in

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //

config/config.mk.in

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/defaults.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -76,9 +76,6 @@ option(MFEM_ENABLE_MINIAPPS "Build all of the miniapps" OFF)
 # Set the target CUDA architecture
 set(CUDA_ARCH "sm_60" CACHE STRING "Target CUDA architecture.")
 
-# Set the target HIP architecture
-set(HIP_ARCH "gfx900" CACHE STRING "Target HIP architecture.")
-
 set(MFEM_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 
 # The *_DIR paths below will be the first place searched for the corresponding

config/defaults.mk

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -161,7 +161,7 @@ endif
 ZLIB_DIR =
 ZLIB_OPT = $(if $(ZLIB_DIR),-I$(ZLIB_DIR)/include)
 ZLIB_LIB = $(if $(ZLIB_DIR),$(ZLIB_RPATH) -L$(ZLIB_DIR)/lib ,)-lz
-ZLIB_RPATH = $(XLINKER)-rpath,$(ZLIB_DIR)/lib
+ZLIB_RPATH = -Wl,-rpath,$(ZLIB_DIR)/lib
 
 LIBUNWIND_OPT = -g
 LIBUNWIND_LIB = $(if $(NOTMAC),-lunwind -ldl,)
@@ -231,21 +231,19 @@ MESQUITE_LIB = -L$(MESQUITE_DIR)/lib -lmesquite
 LIB_RT = $(if $(NOTMAC),-lrt,)
 SUITESPARSE_DIR = @MFEM_DIR@/../SuiteSparse
 SUITESPARSE_OPT = -I$(SUITESPARSE_DIR)/include
-SUITESPARSE_LIB = $(XLINKER)-rpath,$(SUITESPARSE_DIR)/lib\
- -L$(SUITESPARSE_DIR)/lib -lklu -lbtf -lumfpack -lcholmod -lcolamd -lamd -lcamd\
- -lccolamd -lsuitesparseconfig $(LIB_RT) $(METIS_LIB) $(LAPACK_LIB)
+SUITESPARSE_LIB = -Wl,-rpath,$(SUITESPARSE_DIR)/lib -L$(SUITESPARSE_DIR)/lib\
+ -lklu -lbtf -lumfpack -lcholmod -lcolamd -lamd -lcamd -lccolamd\
+ -lsuitesparseconfig $(LIB_RT) $(METIS_LIB) $(LAPACK_LIB)
 
 # SuperLU library configuration
 ifeq ($(MFEM_USE_SUPERLU5),YES)
    SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_5.1.0
    SUPERLU_OPT = -I$(SUPERLU_DIR)/include
-   SUPERLU_LIB = $(XLINKER)-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib\
-      -lsuperlu_dist_5.1.0
+   SUPERLU_LIB = -Wl,-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib -lsuperlu_dist_5.1.0
 else
    SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_6.3.1
    SUPERLU_OPT = -I$(SUPERLU_DIR)/include
-   SUPERLU_LIB = $(XLINKER)-rpath,$(SUPERLU_DIR)/lib64 -L$(SUPERLU_DIR)/lib64\
-      -lsuperlu_dist -lblas
+   SUPERLU_LIB = -Wl,-rpath,$(SUPERLU_DIR)/lib64 -L$(SUPERLU_DIR)/lib64 -lsuperlu_dist -lblas
 endif
 
 # SCOTCH library configuration (required by STRUMPACK <= v2.1.0, optional in
@@ -271,7 +269,7 @@ MPI_FORTRAN_LIB = -lmpifort
 # MUMPS library configuration
 MUMPS_DIR = @MFEM_DIR@/../MUMPS_5.2.0
 MUMPS_OPT = -I$(MUMPS_DIR)/include
-MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib -ldmumps\
+MUMPS_LIB = -Wl,-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib -ldmumps\
  -lmumps_common -lpord $(SCALAPACK_LIB) $(LAPACK_LIB) $(MPI_FORTRAN_LIB)
 
 # STRUMPACK library configuration
@@ -301,8 +299,8 @@ GNUTLS_LIB = -lgnutls
 NETCDF_DIR = $(HOME)/local
 HDF5_DIR   = $(HOME)/local
 NETCDF_OPT = -I$(NETCDF_DIR)/include -I$(HDF5_DIR)/include $(ZLIB_OPT)
-NETCDF_LIB = $(XLINKER)-rpath,$(NETCDF_DIR)/lib -L$(NETCDF_DIR)/lib\
- $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib\
+NETCDF_LIB = -Wl,-rpath,$(NETCDF_DIR)/lib -L$(NETCDF_DIR)/lib\
+ -Wl,-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib\
  -lnetcdf -lhdf5_hl -lhdf5 $(ZLIB_LIB)
 
 # PETSc library configuration (version greater or equal to 3.8 or the dev branch)
@@ -314,10 +312,9 @@ PETSC_INC_VAR = PETSC_CC_INCLUDES
 PETSC_LIB_VAR = PETSC_EXTERNAL_LIB_BASIC
 ifeq ($(PETSC_FOUND),YES)
    PETSC_OPT := $(shell sed -n "s/$(PETSC_INC_VAR) = *//p" $(PETSC_VARS))
-   PETSC_DEP := $(shell sed -n "s/$(PETSC_LIB_VAR) = *//p" $(PETSC_VARS))
-   PETSC_LIB = $(XLINKER)-rpath,$(abspath $(PETSC_DIR))/lib\
-      -L$(abspath $(PETSC_DIR))/lib -lpetsc\
-      $(subst $(CXX_XLINKER),$(XLINKER),$(PETSC_DEP))
+   PETSC_LIB := $(shell sed -n "s/$(PETSC_LIB_VAR) = *//p" $(PETSC_VARS))
+   PETSC_LIB := -Wl,-rpath,$(abspath $(PETSC_DIR))/lib\
+      -L$(abspath $(PETSC_DIR))/lib -lpetsc $(PETSC_LIB)
 endif
 
 SLEPC_DIR := $(MFEM_DIR)/../slepc
@@ -329,10 +326,9 @@ ifeq ($(SLEPC_FOUND),YES)
    SLEPC_OPT := $(shell sed -n "s/$(SLEPC_INC_VAR) *= *//p" $(SLEPC_VARS))
    # Some additional external libraries might be defined in this file
    -include ${SLEPC_DIR}/${PETSC_ARCH}/lib/slepc/conf/slepcvariables
-   SLEPC_DEP := $(shell sed -n "s/$(SLEPC_LIB_VAR) *= *//p" $(SLEPC_VARS))
-   SLEPC_LIB = $(XLINKER)-rpath,$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib\
-      -L$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib -lslepc\
-      $(subst $(CXX_XLINKER),$(XLINKER),$(SLEPC_DEP))
+   SLEPC_LIB := $(shell sed -n "s/$(SLEPC_LIB_VAR) *= *//p" $(SLEPC_VARS))
+   SLEPC_LIB := -Wl,-rpath,$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib\
+      -L$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib -lslepc $(SLEPC_LIB)
 endif
 
 # MPFR library configuration
@@ -343,7 +339,7 @@ MPFR_LIB = -lmpfr
 CONDUIT_DIR = @MFEM_DIR@/../conduit
 CONDUIT_OPT = -I$(CONDUIT_DIR)/include/conduit
 CONDUIT_LIB = \
-   $(XLINKER)-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \
+   -Wl,-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \
    -lconduit -lconduit_relay -lconduit_blueprint  -ldl
 
 # Check if Conduit was built with hdf5 support, by looking
@@ -351,7 +347,7 @@ CONDUIT_LIB = \
 CONDUIT_HDF5_HEADER=$(CONDUIT_DIR)/include/conduit/conduit_relay_hdf5.hpp
 ifneq (,$(wildcard $(CONDUIT_HDF5_HEADER)))
    CONDUIT_OPT += -I$(HDF5_DIR)/include
-   CONDUIT_LIB += $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \
+   CONDUIT_LIB += -Wl,-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \
                   -lhdf5 $(ZLIB_LIB)
 endif
 
@@ -361,9 +357,9 @@ SIDRE_DIR = @MFEM_DIR@/../axom
 SIDRE_OPT = -I$(SIDRE_DIR)/include -I$(CONDUIT_DIR)/include/conduit\
  -I$(HDF5_DIR)/include
 SIDRE_LIB = \
-   $(XLINKER)-rpath,$(SIDRE_DIR)/lib -L$(SIDRE_DIR)/lib \
-   $(XLINKER)-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \
-   $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \
+   -Wl,-rpath,$(SIDRE_DIR)/lib -L$(SIDRE_DIR)/lib \
+   -Wl,-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \
+   -Wl,-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \
    -laxom -lconduit -lconduit_relay -lconduit_blueprint -lhdf5 $(ZLIB_LIB) -ldl
 
 # PUMI
@@ -419,9 +415,9 @@ MKL_CPARDISO_DIR ?=
 MKL_MPI_WRAPPER ?= mkl_blacs_mpich_lp64
 MKL_LIBRARY_SUBDIR ?= lib
 MKL_CPARDISO_OPT = -I$(MKL_CPARDISO_DIR)/include
-MKL_CPARDISO_LIB = $(XLINKER)-rpath,$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
-   -L$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR) -l$(MKL_MPI_WRAPPER)\
-   -lmkl_intel_lp64 -lmkl_sequential -lmkl_core
+MKL_CPARDISO_LIB = -Wl,-rpath,$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
+                   -L$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR) -l$(MKL_MPI_WRAPPER)\
+                   -lmkl_intel_lp64 -lmkl_sequential -lmkl_core
 
 # If YES, enable some informational messages
 VERBOSE = NO

config/get_hypre_version.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //

config/get_mumps_version.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //

config/makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/sample-runs.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

config/tconfig.hpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //

config/test.mk

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

data/amr-hex.mesh

@@ -1,257 +1,241 @@
-MFEM NC mesh v1.0
+MFEM mesh v1.1
 
-# NCMesh supported geometry types:
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
 # TRIANGLE    = 2
 # SQUARE      = 3
 # TETRAHEDRON = 4
 # CUBE        = 5
-# PRISM       = 6
+#
 
 dimension
 3
 
-# rank attr geom ref_type nodes/children
 elements
-137
--1 1 5 7 1 18 35 52 69 86 103 120
--1 1 5 7 2 3 4 5 6 7 8 17
-0 1 5 0 0 27 81 30 35 82 87 85
-0 1 5 0 27 8 28 81 82 36 83 87
-0 1 5 0 81 28 20 29 87 83 37 84
-0 1 5 0 30 81 29 11 85 87 84 38
-0 1 5 0 35 82 87 85 16 31 86 34
-0 1 5 0 82 36 83 87 31 21 32 86
--1 1 5 7 9 10 11 12 13 14 15 16
-0 1 5 0 87 88 179 91 92 180 185 183
-0 1 5 0 88 83 89 179 180 94 181 185
-0 1 5 0 179 89 37 90 185 181 98 182
-0 1 5 0 91 179 90 84 183 185 182 99
-0 1 5 0 92 180 185 183 86 93 184 97
-0 1 5 0 180 94 181 185 93 32 95 184
-0 1 5 0 185 181 98 182 184 95 26 96
-0 1 5 0 183 185 182 99 97 184 96 33
-0 1 5 0 85 87 84 38 34 86 33 24
--1 1 5 7 19 20 21 22 23 24 25 26
-0 1 5 0 8 39 100 28 36 101 105 83
-0 1 5 0 39 1 40 100 101 45 102 105
-0 1 5 0 100 40 9 41 105 102 46 103
-0 1 5 0 28 100 41 20 83 105 103 37
-0 1 5 0 36 101 105 83 21 42 104 32
-0 1 5 0 101 45 102 105 42 17 43 104
-0 1 5 0 105 102 46 103 104 43 22 44
--1 1 5 7 27 28 29 30 31 32 33 34
-0 1 5 0 83 106 186 89 94 187 191 181
-0 1 5 0 106 105 107 186 187 110 188 191
-0 1 5 0 186 107 103 108 191 188 112 189
-0 1 5 0 89 186 108 37 181 191 189 98
-0 1 5 0 94 187 191 181 32 109 190 95
-0 1 5 0 187 110 188 191 109 104 111 190
-0 1 5 0 191 188 112 189 190 111 44 113
-0 1 5 0 181 191 189 98 95 190 113 26
--1 1 5 7 36 37 38 39 40 49 50 51
-0 1 5 0 20 41 114 49 37 103 119 117
-0 1 5 0 41 9 47 114 103 46 115 119
-0 1 5 0 114 47 3 48 119 115 53 116
-0 1 5 0 49 114 48 10 117 119 116 54
--1 1 5 7 41 42 43 44 45 46 47 48
-0 1 5 0 37 108 192 122 98 189 197 195
-0 1 5 0 108 103 120 192 189 112 193 197
-0 1 5 0 192 120 119 121 197 193 126 194
-0 1 5 0 122 192 121 117 195 197 194 127
-0 1 5 0 98 189 197 195 26 113 196 125
-0 1 5 0 189 112 193 197 113 44 123 196
-0 1 5 0 197 193 126 194 196 123 118 124
-0 1 5 0 195 197 194 127 125 196 124 52
-0 1 5 0 103 46 115 119 44 22 50 118
-0 1 5 0 119 115 53 116 118 50 18 51
-0 1 5 0 117 119 116 54 52 118 51 23
--1 1 5 7 53 54 55 56 57 58 67 68
-0 1 5 0 11 29 128 56 38 84 132 130
-0 1 5 0 29 20 49 128 84 37 117 132
-0 1 5 0 128 49 10 55 132 117 54 129
-0 1 5 0 56 128 55 2 130 132 129 59
-0 1 5 0 38 84 132 130 24 33 131 58
--1 1 5 7 59 60 61 62 63 64 65 66
-0 1 5 0 84 90 198 133 99 182 202 200
-0 1 5 0 90 37 122 198 182 98 195 202
-0 1 5 0 198 122 117 134 202 195 127 199
-0 1 5 0 133 198 134 132 200 202 199 136
-0 1 5 0 99 182 202 200 33 96 201 135
-0 1 5 0 182 98 195 202 96 26 125 201
-0 1 5 0 202 195 127 199 201 125 52 137
-0 1 5 0 200 202 199 136 135 201 137 131
-0 1 5 0 132 117 54 129 131 52 23 57
-0 1 5 0 130 132 129 59 58 131 57 19
--1 1 5 7 70 71 72 81 82 83 84 85
-0 1 5 0 16 31 86 34 64 172 175 173
-0 1 5 0 31 21 32 86 172 65 164 175
--1 1 5 7 73 74 75 76 77 78 79 80
-0 1 5 0 86 93 184 97 176 219 222 220
-0 1 5 0 93 32 95 184 219 169 216 222
-0 1 5 0 184 95 26 96 222 216 150 203
-0 1 5 0 97 184 96 33 220 222 203 145
-0 1 5 0 176 219 222 220 175 177 221 178
-0 1 5 0 219 169 216 222 177 164 171 221
-0 1 5 0 222 216 150 203 221 171 66 147
-0 1 5 0 220 222 203 145 178 221 147 138
-0 1 5 0 34 86 33 24 173 175 138 67
-0 1 5 0 64 172 175 173 4 60 174 63
-0 1 5 0 172 65 164 175 60 12 61 174
-0 1 5 0 175 164 66 138 174 61 25 62
-0 1 5 0 173 175 138 67 63 174 62 15
--1 1 5 7 87 88 89 90 99 100 101 102
-0 1 5 0 21 42 104 32 65 162 166 164
-0 1 5 0 42 17 43 104 162 71 163 166
-0 1 5 0 104 43 22 44 166 163 72 152
--1 1 5 7 91 92 93 94 95 96 97 98
-0 1 5 0 32 109 190 95 169 214 218 216
-0 1 5 0 109 104 111 190 214 168 215 218
-0 1 5 0 190 111 44 113 218 215 160 209
-0 1 5 0 95 190 113 26 216 218 209 150
-0 1 5 0 169 214 218 216 164 167 217 171
-0 1 5 0 214 168 215 218 167 166 170 217
-0 1 5 0 218 215 160 209 217 170 152 157
-0 1 5 0 216 218 209 150 171 217 157 66
-0 1 5 0 65 162 166 164 12 68 165 61
-0 1 5 0 162 71 163 166 68 5 69 165
-0 1 5 0 166 163 72 152 165 69 13 70
-0 1 5 0 164 166 152 66 61 165 70 25
--1 1 5 7 104 113 114 115 116 117 118 119
--1 1 5 7 105 106 107 108 109 110 111 112
-0 1 5 0 26 113 196 125 150 209 213 204
-0 1 5 0 113 44 123 196 209 160 210 213
-0 1 5 0 196 123 118 124 213 210 161 211
-0 1 5 0 125 196 124 52 204 213 211 151
-0 1 5 0 150 209 213 204 66 157 212 148
-0 1 5 0 209 160 210 213 157 152 158 212
-0 1 5 0 213 210 161 211 212 158 156 159
-0 1 5 0 204 213 211 151 148 212 159 139
-0 1 5 0 44 22 50 118 152 72 153 156
-0 1 5 0 118 50 18 51 156 153 76 154
-0 1 5 0 52 118 51 23 139 156 154 77
-0 1 5 0 66 152 156 139 25 70 155 75
-0 1 5 0 152 72 153 156 70 13 73 155
-0 1 5 0 156 153 76 154 155 73 7 74
-0 1 5 0 139 156 154 77 75 155 74 14
--1 1 5 7 121 122 131 132 133 134 135 136
-0 1 5 0 24 33 131 58 67 138 143 141
--1 1 5 7 123 124 125 126 127 128 129 130
-0 1 5 0 33 96 201 135 145 203 208 206
-0 1 5 0 96 26 125 201 203 150 204 208
-0 1 5 0 201 125 52 137 208 204 151 205
-0 1 5 0 135 201 137 131 206 208 205 146
-0 1 5 0 145 203 208 206 138 147 207 144
-0 1 5 0 203 150 204 208 147 66 148 207
-0 1 5 0 208 204 151 205 207 148 139 149
-0 1 5 0 206 208 205 146 144 207 149 143
-0 1 5 0 131 52 23 57 143 139 77 140
-0 1 5 0 58 131 57 19 141 143 140 80
-0 1 5 0 67 138 143 141 15 62 142 79
-0 1 5 0 138 66 139 143 62 25 75 142
-0 1 5 0 143 139 77 140 142 75 14 78
-0 1 5 0 141 143 140 80 79 142 78 6
+120
+1 5 0 27 81 30 35 82 87 85
+1 5 30 81 29 11 85 87 84 38
+1 5 85 87 84 38 34 86 33 24
+1 5 35 82 87 85 16 31 86 34
+1 5 82 36 83 87 31 21 32 86
+1 5 180 94 181 185 93 32 95 184
+1 5 92 180 185 183 86 93 184 97
+1 5 183 185 182 99 97 184 96 33
+1 5 185 181 98 182 184 95 26 96
+1 5 179 89 37 90 185 181 98 182
+1 5 91 179 90 84 183 185 182 99
+1 5 87 88 179 91 92 180 185 183
+1 5 88 83 89 179 180 94 181 185
+1 5 81 28 20 29 87 83 37 84
+1 5 27 8 28 81 82 36 83 87
+1 5 8 39 100 28 36 101 105 83
+1 5 36 101 105 83 21 42 104 32
+1 5 101 45 102 105 42 17 43 104
+1 5 39 1 40 100 101 45 102 105
+1 5 100 40 9 41 105 102 46 103
+1 5 105 102 46 103 104 43 22 44
+1 5 186 107 103 108 191 188 112 189
+1 5 106 105 107 186 187 110 188 191
+1 5 187 110 188 191 109 104 111 190
+1 5 191 188 112 189 190 111 44 113
+1 5 181 191 189 98 95 190 113 26
+1 5 94 187 191 181 32 109 190 95
+1 5 83 106 186 89 94 187 191 181
+1 5 89 186 108 37 181 191 189 98
+1 5 28 100 41 20 83 105 103 37
+1 5 20 41 114 49 37 103 119 117
+1 5 37 108 192 122 98 189 197 195
+1 5 122 192 121 117 195 197 194 127
+1 5 195 197 194 127 125 196 124 52
+1 5 98 189 197 195 26 113 196 125
+1 5 189 112 193 197 113 44 123 196
+1 5 197 193 126 194 196 123 118 124
+1 5 192 120 119 121 197 193 126 194
+1 5 108 103 120 192 189 112 193 197
+1 5 103 46 115 119 44 22 50 118
+1 5 41 9 47 114 103 46 115 119
+1 5 114 47 3 48 119 115 53 116
+1 5 119 115 53 116 118 50 18 51
+1 5 117 119 116 54 52 118 51 23
+1 5 49 114 48 10 117 119 116 54
+1 5 128 49 10 55 132 117 54 129
+1 5 56 128 55 2 130 132 129 59
+1 5 11 29 128 56 38 84 132 130
+1 5 29 20 49 128 84 37 117 132
+1 5 84 90 198 133 99 182 202 200
+1 5 90 37 122 198 182 98 195 202
+1 5 198 122 117 134 202 195 127 199
+1 5 133 198 134 132 200 202 199 136
+1 5 200 202 199 136 135 201 137 131
+1 5 202 195 127 199 201 125 52 137
+1 5 182 98 195 202 96 26 125 201
+1 5 99 182 202 200 33 96 201 135
+1 5 38 84 132 130 24 33 131 58
+1 5 130 132 129 59 58 131 57 19
+1 5 132 117 54 129 131 52 23 57
+1 5 131 52 23 57 143 139 77 140
+1 5 58 131 57 19 141 143 140 80
+1 5 24 33 131 58 67 138 143 141
+1 5 33 96 201 135 145 203 208 206
+1 5 96 26 125 201 203 150 204 208
+1 5 201 125 52 137 208 204 151 205
+1 5 135 201 137 131 206 208 205 146
+1 5 206 208 205 146 144 207 149 143
+1 5 208 204 151 205 207 148 139 149
+1 5 203 150 204 208 147 66 148 207
+1 5 145 203 208 206 138 147 207 144
+1 5 138 66 139 143 62 25 75 142
+1 5 67 138 143 141 15 62 142 79
+1 5 141 143 140 80 79 142 78 6
+1 5 143 139 77 140 142 75 14 78
+1 5 139 156 154 77 75 155 74 14
+1 5 52 118 51 23 139 156 154 77
+1 5 118 50 18 51 156 153 76 154
+1 5 156 153 76 154 155 73 7 74
+1 5 152 72 153 156 70 13 73 155
+1 5 44 22 50 118 152 72 153 156
+1 5 209 160 210 213 157 152 158 212
+1 5 213 210 161 211 212 158 156 159
+1 5 196 123 118 124 213 210 161 211
+1 5 113 44 123 196 209 160 210 213
+1 5 26 113 196 125 150 209 213 204
+1 5 125 196 124 52 204 213 211 151
+1 5 204 213 211 151 148 212 159 139
+1 5 150 209 213 204 66 157 212 148
+1 5 66 152 156 139 25 70 155 75
+1 5 164 166 152 66 61 165 70 25
+1 5 216 218 209 150 171 217 157 66
+1 5 169 214 218 216 164 167 217 171
+1 5 32 109 190 95 169 214 218 216
+1 5 95 190 113 26 216 218 209 150
+1 5 190 111 44 113 218 215 160 209
+1 5 109 104 111 190 214 168 215 218
+1 5 214 168 215 218 167 166 170 217
+1 5 218 215 160 209 217 170 152 157
+1 5 104 43 22 44 166 163 72 152
+1 5 166 163 72 152 165 69 13 70
+1 5 162 71 163 166 68 5 69 165
+1 5 42 17 43 104 162 71 163 166
+1 5 21 42 104 32 65 162 166 164
+1 5 65 162 166 164 12 68 165 61
+1 5 172 65 164 175 60 12 61 174
+1 5 175 164 66 138 174 61 25 62
+1 5 219 169 216 222 177 164 171 221
+1 5 176 219 222 220 175 177 221 178
+1 5 220 222 203 145 178 221 147 138
+1 5 222 216 150 203 221 171 66 147
+1 5 184 95 26 96 222 216 150 203
+1 5 97 184 96 33 220 222 203 145
+1 5 86 93 184 97 176 219 222 220
+1 5 93 32 95 184 219 169 216 222
+1 5 31 21 32 86 172 65 164 175
+1 5 16 31 86 34 64 172 175 173
+1 5 34 86 33 24 173 175 138 67
+1 5 173 175 138 67 63 174 62 15
+1 5 64 172 175 173 4 60 174 63
 
-# attr geom nodes
 boundary
 96
 1 3 30 81 27 0
 2 3 0 27 82 35
 5 3 30 0 35 85
-1 3 81 28 8 27
-2 3 27 8 36 82
-1 3 29 20 28 81
 1 3 11 29 81 30
 5 3 11 30 85 38
+5 3 38 85 34 24
 2 3 35 82 31 16
 5 3 85 35 16 34
 2 3 82 36 21 31
-5 3 38 85 34 24
+1 3 29 20 28 81
+1 3 81 28 8 27
+2 3 27 8 36 82
 1 3 28 100 39 8
 2 3 8 39 101 36
+2 3 36 101 42 21
+2 3 101 45 17 42
+3 3 45 102 43 17
 1 3 100 40 1 39
 2 3 39 1 45 101
 3 3 1 40 102 45
 1 3 41 9 40 100
 3 3 40 9 46 102
-1 3 20 41 100 28
-2 3 36 101 42 21
-2 3 101 45 17 42
-3 3 45 102 43 17
 3 3 102 46 22 43
+1 3 20 41 100 28
 1 3 49 114 41 20
+3 3 46 115 50 22
 1 3 114 47 9 41
 3 3 9 47 115 46
 1 3 48 3 47 114
 3 3 47 3 53 115
 4 3 3 48 116 53
-1 3 10 48 114 49
-4 3 48 10 54 116
-3 3 46 115 50 22
 3 3 115 53 18 50
 4 3 53 116 51 18
 4 3 116 54 23 51
-1 3 56 128 29 11
-5 3 56 11 38 130
-1 3 128 49 20 29
+1 3 10 48 114 49
+4 3 48 10 54 116
 1 3 55 10 49 128
 4 3 10 55 129 54
 1 3 2 55 128 56
 4 3 55 2 59 129
 5 3 2 56 130 59
+1 3 56 128 29 11
+5 3 56 11 38 130
+1 3 128 49 20 29
 5 3 130 38 24 58
-4 3 54 129 57 23
 4 3 129 59 19 57
 5 3 59 130 58 19
-2 3 16 31 172 64
-5 3 34 16 64 173
-2 3 31 21 65 172
-5 3 24 34 173 67
-2 3 64 172 60 4
-5 3 173 64 4 63
-6 3 4 60 174 63
-2 3 172 65 12 60
-6 3 60 12 61 174
-6 3 174 61 25 62
-5 3 67 173 63 15
-6 3 63 174 62 15
-2 3 21 42 162 65
-2 3 42 17 71 162
-3 3 17 43 163 71
-3 3 43 22 72 163
-2 3 65 162 68 12
-6 3 12 68 165 61
-2 3 162 71 5 68
-3 3 71 163 69 5
-6 3 68 5 69 165
-3 3 163 72 13 69
-6 3 165 69 13 70
-6 3 61 165 70 25
-3 3 22 50 153 72
-3 3 50 18 76 153
-4 3 18 51 154 76
-4 3 51 23 77 154
-6 3 25 70 155 75
-3 3 72 153 73 13
-6 3 70 13 73 155
-3 3 153 76 7 73
-4 3 76 154 74 7
-6 3 155 73 7 74
-4 3 154 77 14 74
-6 3 75 155 74 14
-5 3 58 24 67 141
+4 3 54 129 57 23
 4 3 23 57 140 77
 4 3 57 19 80 140
 5 3 19 58 141 80
+5 3 58 24 67 141
+6 3 62 25 75 142
 5 3 141 67 15 79
 6 3 15 62 142 79
-6 3 62 25 75 142
-4 3 77 140 78 14
-6 3 142 75 14 78
 4 3 140 80 6 78
 5 3 80 141 79 6
 6 3 79 142 78 6
+4 3 77 140 78 14
+6 3 142 75 14 78
+4 3 154 77 14 74
+6 3 75 155 74 14
+4 3 51 23 77 154
+3 3 50 18 76 153
+4 3 18 51 154 76
+3 3 153 76 7 73
+4 3 76 154 74 7
+6 3 155 73 7 74
+3 3 72 153 73 13
+6 3 70 13 73 155
+3 3 22 50 153 72
+6 3 25 70 155 75
+6 3 61 165 70 25
+3 3 43 22 72 163
+3 3 163 72 13 69
+6 3 165 69 13 70
+2 3 162 71 5 68
+3 3 71 163 69 5
+6 3 68 5 69 165
+2 3 42 17 71 162
+3 3 17 43 163 71
+2 3 21 42 162 65
+2 3 65 162 68 12
+6 3 12 68 165 61
+2 3 172 65 12 60
+6 3 60 12 61 174
+6 3 174 61 25 62
+2 3 31 21 65 172
+2 3 16 31 172 64
+5 3 34 16 64 173
+5 3 24 34 173 67
+5 3 67 173 63 15
+6 3 63 174 62 15
+2 3 64 172 60 4
+5 3 173 64 4 63
+6 3 4 60 174 63
 
-# vert_id p1 p2
 vertex_parents
 215
 8 0 1
@@ -470,9 +454,28 @@ vertex_parents
 221 171 178
 222 203 219
 
-# top-level node coordinates
-coordinates
-8
+coarse_elements
+17
+7 11 12 9 10 6 5 8 7
+7 0 14 13 1 3 4 120 2
+7 27 22 21 28 26 23 24 25
+7 15 18 19 29 16 17 20 122
+7 31 38 37 32 34 35 36 33
+7 30 40 41 44 124 39 42 43
+7 49 50 51 52 56 55 54 53
+7 47 48 45 46 57 126 59 58
+7 113 114 111 112 108 107 110 109
+7 116 115 128 117 119 105 106 118
+7 93 96 95 94 92 97 98 91
+7 103 102 99 130 104 101 100 90
+7 85 84 83 86 88 81 82 87
+7 132 80 77 76 89 79 78 75
+7 63 64 65 66 70 69 68 67
+7 62 134 60 61 72 71 74 73
+7 121 123 125 127 129 131 133 135
+
+vertices
+223
 3
 0 0 0
 1 0 0
@@ -482,5 +485,218 @@ coordinates
 1 0 1
 0 1 1
 1 1 1
-
-mfem_mesh_end
+0.5 0 0
+1 0.5 0
+0.5 1 0
+0 0.5 0
+0.5 0 1
+1 0.5 1
+0.5 1 1
+0 0.5 1
+0 0 0.5
+1 0 0.5
+1 1 0.5
+0 1 0.5
+0.5 0.5 0
+0.5 0 0.5
+1 0.5 0.5
+0.5 1 0.5
+0 0.5 0.5
+0.5 0.5 1
+0.5 0.5 0.5
+0.25 0 0
+0.5 0.25 0
+0.25 0.5 0
+0 0.25 0
+0.25 0 0.5
+0.5 0.25 0.5
+0.25 0.5 0.5
+0 0.25 0.5
+0 0 0.25
+0.5 0 0.25
+0.5 0.5 0.25
+0 0.5 0.25
+0.75 0 0
+1 0.25 0
+0.75 0.5 0
+0.75 0 0.5
+1 0.25 0.5
+0.75 0.5 0.5
+1 0 0.25
+1 0.5 0.25
+1 0.75 0
+0.75 1 0
+0.5 0.75 0
+1 0.75 0.5
+0.75 1 0.5
+0.5 0.75 0.5
+1 1 0.25
+0.5 1 0.25
+0.25 1 0
+0 0.75 0
+0.25 1 0.5
+0 0.75 0.5
+0 1 0.25
+0.25 0 1
+0.5 0.25 1
+0.25 0.5 1
+0 0.25 1
+0 0 0.75
+0.5 0 0.75
+0.5 0.5 0.75
+0 0.5 0.75
+0.75 0 1
+1 0.25 1
+0.75 0.5 1
+1 0 0.75
+1 0.5 0.75
+1 0.75 1
+0.75 1 1
+0.5 0.75 1
+1 1 0.75
+0.5 1 0.75
+0.25 1 1
+0 0.75 1
+0 1 0.75
+0.25 0.25 0
+0.25 0 0.25
+0.5 0.25 0.25
+0.25 0.5 0.25
+0 0.25 0.25
+0.25 0.25 0.5
+0.25 0.25 0.25
+0.375 0.25 0.25
+0.5 0.375 0.25
+0.375 0.5 0.25
+0.25 0.375 0.25
+0.25 0.25 0.375
+0.375 0.25 0.5
+0.5 0.25 0.375
+0.5 0.375 0.5
+0.375 0.5 0.5
+0.25 0.375 0.5
+0.5 0.5 0.375
+0.25 0.5 0.375
+0.75 0.25 0
+0.75 0 0.25
+1 0.25 0.25
+0.75 0.5 0.25
+0.75 0.25 0.5
+0.75 0.25 0.25
+0.625 0.25 0.25
+0.75 0.375 0.25
+0.625 0.5 0.25
+0.625 0.25 0.5
+0.75 0.25 0.375
+0.75 0.375 0.5
+0.75 0.5 0.375
+0.625 0.5 0.5
+0.75 0.75 0
+1 0.75 0.25
+0.75 1 0.25
+0.5 0.75 0.25
+0.75 0.75 0.5
+0.75 0.75 0.25
+0.75 0.625 0.25
+0.625 0.75 0.25
+0.5 0.625 0.25
+0.75 0.625 0.5
+0.625 0.75 0.5
+0.5 0.625 0.5
+0.75 0.75 0.375
+0.5 0.75 0.375
+0.25 0.75 0
+0.25 1 0.25
+0 0.75 0.25
+0.25 0.75 0.5
+0.25 0.75 0.25
+0.25 0.625 0.25
+0.375 0.75 0.25
+0.25 0.625 0.5
+0.25 0.75 0.375
+0.375 0.75 0.5
+0.25 0.5 0.75
+0.5 0.75 0.75
+0.25 1 0.75
+0 0.75 0.75
+0.25 0.75 1
+0.25 0.75 0.75
+0.25 0.625 0.75
+0.25 0.5 0.625
+0.25 0.75 0.625
+0.375 0.5 0.75
+0.5 0.625 0.75
+0.375 0.75 0.75
+0.5 0.5 0.625
+0.5 0.75 0.625
+0.75 0.5 0.75
+1 0.75 0.75
+0.75 1 0.75
+0.75 0.75 1
+0.75 0.75 0.75
+0.625 0.5 0.75
+0.75 0.625 0.75
+0.625 0.75 0.75
+0.75 0.5 0.625
+0.75 0.75 0.625
+0.75 0 0.75
+1 0.25 0.75
+0.5 0.25 0.75
+0.75 0.25 1
+0.75 0.25 0.75
+0.625 0.25 0.75
+0.75 0.25 0.625
+0.5 0.25 0.625
+0.75 0.375 0.75
+0.5 0.375 0.75
+0.25 0 0.75
+0 0.25 0.75
+0.25 0.25 1
+0.25 0.25 0.75
+0.25 0.25 0.625
+0.375 0.25 0.75
+0.25 0.375 0.75
+0.375 0.375 0.25
+0.375 0.25 0.375
+0.5 0.375 0.375
+0.375 0.5 0.375
+0.25 0.375 0.375
+0.375 0.375 0.5
+0.375 0.375 0.375
+0.625 0.375 0.25
+0.625 0.25 0.375
+0.75 0.375 0.375
+0.625 0.5 0.375
+0.625 0.375 0.5
+0.625 0.375 0.375
+0.625 0.625 0.25
+0.75 0.625 0.375
+0.625 0.75 0.375
+0.5 0.625 0.375
+0.625 0.625 0.5
+0.625 0.625 0.375
+0.375 0.625 0.25
+0.375 0.75 0.375
+0.25 0.625 0.375
+0.375 0.625 0.5
+0.375 0.625 0.375
+0.375 0.5 0.625
+0.5 0.625 0.625
+0.375 0.75 0.625
+0.25 0.625 0.625
+0.375 0.625 0.75
+0.375 0.625 0.625
+0.625 0.5 0.625
+0.75 0.625 0.625
+0.625 0.75 0.625
+0.625 0.625 0.75
+0.625 0.625 0.625
+0.625 0.25 0.625
+0.75 0.375 0.625
+0.5 0.375 0.625
+0.625 0.375 0.75
+0.625 0.375 0.625
+0.375 0.25 0.625
+0.25 0.375 0.625
+0.375 0.375 0.75
+0.375 0.375 0.625

data/amr-quad.mesh

@@ -1,63 +1,56 @@
-MFEM NC mesh v1.0
+MFEM mesh v1.1
 
-# NCMesh supported geometry types:
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
 # TRIANGLE    = 2
 # SQUARE      = 3
 # TETRAHEDRON = 4
 # CUBE        = 5
-# PRISM       = 6
+#
 
 dimension
 2
 
-# rank attr geom ref_type nodes/children
 elements
-37
--1 1 3 3 1 10 19 28
--1 1 3 3 2 3 4 9
-0 1 3 0 0 9 21 12
-0 1 3 0 9 4 10 21
--1 1 3 3 5 6 7 8
-0 1 3 0 21 22 37 25
-0 1 3 0 22 10 23 37
-0 1 3 0 37 23 8 24
-0 1 3 0 25 37 24 11
-0 1 3 0 12 21 11 7
--1 1 3 3 11 12 13 14
-0 1 3 0 4 13 26 10
-0 1 3 0 13 1 14 26
-0 1 3 0 26 14 5 15
--1 1 3 3 15 16 17 18
-0 1 3 0 10 27 38 23
-0 1 3 0 27 26 28 38
-0 1 3 0 38 28 15 29
-0 1 3 0 23 38 29 8
--1 1 3 3 20 25 26 27
--1 1 3 3 21 22 23 24
-0 1 3 0 8 29 39 33
-0 1 3 0 29 15 31 39
-0 1 3 0 39 31 30 32
-0 1 3 0 33 39 32 18
-0 1 3 0 15 5 16 30
-0 1 3 0 30 16 3 17
-0 1 3 0 18 30 17 6
--1 1 3 3 29 30 35 36
-0 1 3 0 7 11 34 20
--1 1 3 3 31 32 33 34
-0 1 3 0 11 24 40 35
-0 1 3 0 24 8 33 40
-0 1 3 0 40 33 18 36
-0 1 3 0 35 40 36 34
-0 1 3 0 34 18 6 19
-0 1 3 0 20 34 19 2
+28
+1 3 0 9 21 12
+1 3 12 21 11 7
+1 3 21 22 37 25
+1 3 25 37 24 11
+1 3 37 23 8 24
+1 3 22 10 23 37
+1 3 9 4 10 21
+1 3 4 13 26 10
+1 3 13 1 14 26
+1 3 26 14 5 15
+1 3 38 28 15 29
+1 3 27 26 28 38
+1 3 10 27 38 23
+1 3 23 38 29 8
+1 3 8 29 39 33
+1 3 33 39 32 18
+1 3 39 31 30 32
+1 3 29 15 31 39
+1 3 15 5 16 30
+1 3 30 16 3 17
+1 3 18 30 17 6
+1 3 34 18 6 19
+1 3 40 33 18 36
+1 3 24 8 33 40
+1 3 11 24 40 35
+1 3 35 40 36 34
+1 3 7 11 34 20
+1 3 20 34 19 2
 
-# attr geom nodes
 boundary
 16
 1 1 0 9
 4 1 12 0
-1 1 9 4
 4 1 7 12
+1 1 9 4
 1 1 4 13
 1 1 13 1
 2 1 1 14
@@ -66,12 +59,11 @@ boundary
 2 1 16 3
 3 1 3 17
 3 1 17 6
-4 1 20 7
 3 1 6 19
+4 1 20 7
 3 1 19 2
 4 1 2 20
 
-# vert_id p1 p2
 vertex_parents
 37
 4 0 1
@@ -112,13 +104,59 @@ vertex_parents
 39 29 32
 40 24 36
 
-# top-level node coordinates
-coordinates
-4
+coarse_elements
+9
+3 2 5 4 3
+3 0 6 28 1
+3 12 11 10 13
+3 7 8 9 30
+3 14 17 16 15
+3 32 18 19 20
+3 24 23 22 25
+3 26 34 21 27
+3 29 31 33 35
+
+vertices
+41
 2
 0 0
 1 0
 0 1
 1 1
-
-mfem_mesh_end
+0.5 0
+1 0.5
+0.5 1
+0 0.5
+0.5 0.5
+0.25 0
+0.5 0.25
+0.25 0.5
+0 0.25
+0.75 0
+1 0.25
+0.75 0.5
+1 0.75
+0.75 1
+0.5 0.75
+0.25 1
+0 0.75
+0.25 0.25
+0.375 0.25
+0.5 0.375
+0.375 0.5
+0.25 0.375
+0.75 0.25
+0.625 0.25
+0.75 0.375
+0.625 0.5
+0.75 0.75
+0.75 0.625
+0.625 0.75
+0.5 0.625
+0.25 0.75
+0.25 0.625
+0.375 0.75
+0.375 0.375
+0.625 0.375
+0.625 0.625
+0.375 0.625

data/beam-quad-amr.mesh

@@ -1,42 +1,51 @@
-MFEM NC mesh v1.0
+MFEM mesh v1.1
 
-# NCMesh supported geometry types:
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
 # TRIANGLE    = 2
 # SQUARE      = 3
 # TETRAHEDRON = 4
 # CUBE        = 5
-# PRISM       = 6
+#
 
 dimension
 2
 
-# rank attr geom ref_type nodes/children
 elements
-20
-0 1 3 0 1 2 11 10
-0 1 3 0 2 3 12 11
-0 1 3 0 3 4 13 12
-0 2 3 0 4 5 14 13
-0 2 3 0 5 6 15 14
-0 2 3 0 6 7 16 15
-0 2 3 0 7 8 17 16
--1 1 3 3 8 13 14 15
--1 1 3 3 9 10 11 12
-0 1 3 0 0 23 30 26
-0 1 3 0 23 18 24 30
-0 1 3 0 30 24 22 25
-0 1 3 0 26 30 25 21
-0 1 3 0 18 1 19 22
-0 1 3 0 22 19 10 20
--1 1 3 3 16 17 18 19
-0 1 3 0 21 25 31 29
-0 1 3 0 25 22 27 31
-0 1 3 0 31 27 20 28
-0 1 3 0 29 31 28 9
+17
+1 3 0 23 30 26
+1 3 26 30 25 21
+1 3 30 24 22 25
+1 3 23 18 24 30
+1 3 18 1 19 22
+1 3 22 19 10 20
+1 3 31 27 20 28
+1 3 25 22 27 31
+1 3 21 25 31 29
+1 3 29 31 28 9
+1 3 1 2 11 10
+1 3 2 3 12 11
+1 3 3 4 13 12
+2 3 4 5 14 13
+2 3 5 6 15 14
+2 3 6 7 16 15
+2 3 7 8 17 16
 
-# attr geom nodes
 boundary
 25
+3 1 0 23
+1 1 26 0
+1 1 21 26
+3 1 23 18
+3 1 18 1
+3 1 10 20
+3 1 20 28
+1 1 29 21
+3 1 28 9
+1 1 9 29
 3 1 1 2
 3 1 11 10
 3 1 2 3
@@ -52,18 +61,7 @@ boundary
 3 1 7 8
 2 1 8 17
 3 1 17 16
-3 1 0 23
-1 1 26 0
-3 1 23 18
-1 1 21 26
-3 1 18 1
-3 1 10 20
-1 1 29 21
-3 1 20 28
-3 1 28 9
-1 1 9 29
 
-# vert_id p1 p2
 vertex_parents
 14
 18 0 1
@@ -81,21 +79,14 @@ vertex_parents
 30 23 25
 31 25 28
 
-# root element orientation
-root_state
-8
-1
-1
-1
-1
-1
-1
-0
-0
+coarse_elements
+3
+3 0 3 2 1
+3 8 7 6 9
+3 17 4 5 18
 
-# top-level node coordinates
-coordinates
-18
+vertices
+32
 2
 0 0
 1 0
@@ -115,5 +106,17 @@ coordinates
 6 1
 7 1
 8 1
-
-mfem_mesh_end
+0.5 0
+1 0.5
+0.5 1
+0 0.5
+0.5 0.5
+0.25 0
+0.5 0.25
+0.25 0.5
+0 0.25
+0.5 0.75
+0.25 1
+0 0.75
+0.25 0.25
+0.25 0.75

data/beam-wedge.vtk

@@ -31,14 +31,14 @@ POINTS 27 double
 7 0.5 1
 8 0.5 1
 CELLS 8 56
-6 0 18 9 1 19 10
-6 1 19 10 2 20 11
-6 2 20 11 3 21 12
-6 3 21 12 4 22 13
-6 4 22 13 5 23 14
-6 5 23 14 6 24 15
-6 6 24 15 7 25 16
-6 7 25 16 8 26 17
+6 0 9 18 1 10 19
+6 1 10 19 2 11 20
+6 2 11 20 3 12 21
+6 3 12 21 4 13 22
+6 4 13 22 5 14 23
+6 5 14 23 6 15 24
+6 6 15 24 7 16 25
+6 7 16 25 8 17 26
 CELL_TYPES 8
 13
 13

data/inline-oneTri.mesh

@@ -1,7 +0,0 @@
-MFEM INLINE mesh v1.0
-
-type = tri
-nx = 1
-ny = 1
-sx = 3.14
-sy = 3.14

doc/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

doc/CodeDocumentation.conf.in

@@ -781,8 +781,6 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/miniapps/navier \
                          @MFEM_SOURCE_DIR@/miniapps/nurbs \
                          @MFEM_SOURCE_DIR@/miniapps/performance \
-                         @MFEM_SOURCE_DIR@/miniapps/shifted \
-                         @MFEM_SOURCE_DIR@/miniapps/solvers \
                          @MFEM_SOURCE_DIR@/miniapps/tools \
                          @MFEM_SOURCE_DIR@/miniapps/toys
 
@@ -821,9 +819,7 @@ RECURSIVE              = NO
 # run.
 
 EXCLUDE                = @MFEM_SOURCE_DIR@/config/_config.hpp \
-                         @MFEM_SOURCE_DIR@/config/get_hypre_version.cpp \
-                         @MFEM_SOURCE_DIR@/general/tinyxml2.h \
-                         @MFEM_SOURCE_DIR@/general/tinyxml2.cpp
+                         @MFEM_SOURCE_DIR@/config/get_hypre_version.cpp
 
 # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
 # directories that are symbolic links (a Unix file system feature) are excluded

doc/CodeDocumentation.dox

@@ -176,8 +176,6 @@ namespace mfem {
  * - <a class="el" href="findpts_8cpp_source.html">Find Points</a>: evaluate grid function in physical space, <a class="el" href="findpts_8cpp_source.html">serial</a> and <a class="el" href="pfindpts_8cpp_source.html">parallel</a> versions
  * - <a class="el" href="field-diff_8cpp_source.html">Field Diff</a>: compare grid functions on different meshes
  * - <a class="el" href="field-interp_8cpp_source.html">Field Interp</a>: transfer a grid functions between meshes
- * - <a class="el" href="distance_8cpp_source.html">Distance</a>: finite element distance function solver
- * - <a class="el" href="distance_8cpp_source.html">Block Solvers</a>: comparison of saddle point system solvers
  * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
  * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
  *

doc/makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

examples/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

examples/amgx/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

examples/amgx/ex1.cpp

@@ -205,14 +205,10 @@ int main(int argc, char *argv[])
 
       if (amgx_solver)
       {
-         amgx.SetConvergenceCheck(true);
          amgx.Mult(B,X);
       }
       else
       {
-         // Omit convergence check at the AmgX level when using as a
-         // preconditioner.
-         amgx.SetConvergenceCheck(false);
          PCG(*A.As<SparseMatrix>(), amgx, B, X, 3, 40, 1e-12, 0.0);
       }
    }

examples/amgx/ex1p.cpp

@@ -264,7 +264,6 @@ int main(int argc, char *argv[])
       }
 
       amgx.SetOperator(*A.As<HypreParMatrix>());
-      amgx.SetConvergenceCheck(true);
       amgx.Mult(B, X);
 
       // Release MPI communicators and resources created by AmgX

examples/amgx/makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

examples/ex1.cpp

@@ -30,7 +30,6 @@
 // Device sample runs:
 //               ex1 -pa -d cuda
 //               ex1 -pa -d raja-cuda
-//             * ex1 -pa -d raja-hip
 //               ex1 -pa -d occa-cuda
 //               ex1 -pa -d raja-omp
 //               ex1 -pa -d occa-omp

examples/ex10p.cpp

@@ -178,7 +178,6 @@ int main(int argc, char *argv[])
    double visc = 1e-2;
    double mu = 0.25;
    double K = 5.0;
-   bool adaptive_lin_rtol = true;
    bool visualization = true;
    int vis_steps = 1;
 
@@ -207,9 +206,6 @@ int main(int argc, char *argv[])
                   "Shear modulus in the Neo-Hookean hyperelastic model.");
    args.AddOption(&K, "-K", "--bulk-modulus",
                   "Bulk modulus in the Neo-Hookean hyperelastic model.");
-   args.AddOption(&adaptive_lin_rtol, "-alrtol", "--adaptive-lin-rtol",
-                  "-no-alrtol", "--no-adaptive-lin-rtol",
-                  "Enable or disable adaptive linear solver rtol.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -578,7 +574,6 @@ HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace &f,
    newton_solver.SetPrintLevel(1); // print Newton iterations
    newton_solver.SetRelTol(rel_tol);
    newton_solver.SetAbsTol(0.0);
-   newton_solver.SetAdaptiveLinRtol(2, 0.5, 0.9);
    newton_solver.SetMaxIter(10);
 }
 

examples/ex14.cpp

@@ -5,7 +5,6 @@
 // Sample runs:  ex14 -m ../data/inline-quad.mesh -o 0
 //               ex14 -m ../data/star.mesh -r 4 -o 2
 //               ex14 -m ../data/star-mixed.mesh -r 4 -o 2
-//               ex14 -m ../data/star-mixed.mesh -r 2 -o 2 -k 0 -e 1
 //               ex14 -m ../data/escher.mesh -s 1
 //               ex14 -m ../data/fichera.mesh -s 1 -k 1
 //               ex14 -m ../data/fichera-mixed.mesh -s 1 -k 1
@@ -45,7 +44,6 @@ int main(int argc, char *argv[])
    int order = 1;
    double sigma = -1.0;
    double kappa = -1.0;
-   double eta = 0.0;
    bool visualization = 1;
 
    OptionsParser args(argc, argv);
@@ -56,12 +54,11 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Finite element order (polynomial degree) >= 0.");
    args.AddOption(&sigma, "-s", "--sigma",
-                  "One of the three DG penalty parameters, typically +1/-1."
+                  "One of the two DG penalty parameters, typically +1/-1."
                   " See the documentation of class DGDiffusionIntegrator.");
    args.AddOption(&kappa, "-k", "--kappa",
-                  "One of the three DG penalty parameters, should be positive."
+                  "One of the two DG penalty parameters, should be positive."
                   " Negative values are replaced with (order+1)^2.");
-   args.AddOption(&eta, "-e", "--eta", "BR2 penalty parameter.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -133,11 +130,6 @@ int main(int argc, char *argv[])
    a->AddDomainIntegrator(new DiffusionIntegrator(one));
    a->AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
    a->AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
-   if (eta > 0)
-   {
-      a->AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
-      a->AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
-   }
    a->Assemble();
    a->Finalize();
    const SparseMatrix &A = a->SpMat();

examples/ex14p.cpp

@@ -5,7 +5,6 @@
 // Sample runs:  mpirun -np 4 ex14p -m ../data/inline-quad.mesh -o 0
 //               mpirun -np 4 ex14p -m ../data/star.mesh -o 2
 //               mpirun -np 4 ex14p -m ../data/star-mixed.mesh -o 2
-//               mpirun -np 4 ex14p -m ../data/star-mixed.mesh -o 2 -k 0 -e 1
 //               mpirun -np 4 ex14p -m ../data/escher.mesh -s 1
 //               mpirun -np 4 ex14p -m ../data/fichera.mesh -s 1 -k 1
 //               mpirun -np 4 ex14p -m ../data/fichera-mixed.mesh -s 1 -k 1
@@ -83,7 +82,6 @@ int main(int argc, char *argv[])
    int order = 1;
    double sigma = -1.0;
    double kappa = -1.0;
-   double eta = 0.0;
    bool visualization = 1;
 
    OptionsParser args(argc, argv);
@@ -97,12 +95,11 @@ int main(int argc, char *argv[])
    args.AddOption(&order, "-o", "--order",
                   "Finite element order (polynomial degree) >= 0.");
    args.AddOption(&sigma, "-s", "--sigma",
-                  "One of the three DG penalty parameters, typically +1/-1."
+                  "One of the two DG penalty parameters, typically +1/-1."
                   " See the documentation of class DGDiffusionIntegrator.");
    args.AddOption(&kappa, "-k", "--kappa",
-                  "One of the three DG penalty parameters, should be positive."
+                  "One of the two DG penalty parameters, should be positive."
                   " Negative values are replaced with (order+1)^2.");
-   args.AddOption(&eta, "-e", "--eta", "BR2 penalty parameter.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -197,11 +194,6 @@ int main(int argc, char *argv[])
    a->AddDomainIntegrator(new DiffusionIntegrator(one));
    a->AddInteriorFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
    a->AddBdrFaceIntegrator(new DGDiffusionIntegrator(one, sigma, kappa));
-   if (eta > 0)
-   {
-      a->AddInteriorFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
-      a->AddBdrFaceIntegrator(new DGDiffusionBR2Integrator(fespace, eta));
-   }
    a->Assemble();
    a->Finalize();
 

examples/ex15p.cpp

@@ -19,13 +19,6 @@
 //               mpirun -np 4 ex15p -m ../data/square-disc.mesh
 //               mpirun -np 4 ex15p -m ../data/escher.mesh -r 2 -tf 0.3
 //
-//               Different estimators:
-//
-//               mpirun -np 4 ex15p -est 0 -e 1e-4
-//               mpirun -np 4 ex15p -est 1 -e 1e-6
-//               mpirun -np 4 ex15p -est 1 -o 3 -tf 0.3
-//               mpirun -np 4 ex15p -est 2 -o 2
-//
 // Description:  Building on Example 6, this example demonstrates dynamic AMR.
 //               The mesh is adapted to a time-dependent solution by refinement
 //               as well as by derefinement. For simplicity, the solution is
@@ -35,11 +28,8 @@
 //               At each outer iteration the right hand side function is changed
 //               to mimic a time dependent problem.  Within each inner iteration
 //               the problem is solved on a sequence of meshes which are locally
-//               refined according to a chosen error estimator. Currently there
-//               are three error estimators supported: A L2 formulation of the
-//               Zienkiewicz-Zhu error estimator (0), a Kelly error indicator (1)
-//               and a traditional Zienkiewicz-Zhu error estimator (2). At the
-//               end of the inner iteration the error estimates are also used to
+//               refined according to a simple ZZ error estimator.  At the end
+//               of the inner iteration the error estimates are also used to
 //               identify any elements which may be over-refined and a single
 //               derefinement step is performed.  After each refinement or
 //               derefinement step a rebalance operation is performed to keep
@@ -97,7 +87,6 @@ int main(int argc, char *argv[])
    int nc_limit = 3;         // maximum level of hanging nodes
    bool visualization = true;
    bool visit = false;
-   int which_estimator = 0;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -118,9 +107,6 @@ int main(int argc, char *argv[])
                   "Maximum level of hanging nodes.");
    args.AddOption(&t_final, "-tf", "--t-final",
                   "Final time; start time is 0.");
-   args.AddOption(&which_estimator, "-est", "--estimator",
-                  "Which estimator to use: "
-                  "0 = L2ZZ, 1 = Kelly, 2 = ZZ. Defaults to L2ZZ.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -228,43 +214,17 @@ int main(int argc, char *argv[])
    //     provide the method ComputeElementFlux. We supply an L2 space for the
    //     discontinuous flux and an H(div) space for the smoothed flux.
    L2_FECollection flux_fec(order, dim);
+   ParFiniteElementSpace flux_fes(&pmesh, &flux_fec, sdim);
    RT_FECollection smooth_flux_fec(order-1, dim);
-   ErrorEstimator* estimator;
-   switch (which_estimator)
-   {
-      case 1:
-      {
-         auto flux_fes = new ParFiniteElementSpace(&pmesh, &flux_fec, sdim);
-         estimator = new KellyErrorEstimator(*integ, x, flux_fes);
-         break;
-      }
-      case 2:
-      {
-         auto flux_fes = new ParFiniteElementSpace(&pmesh, &fec, sdim);
-         estimator = new ZienkiewiczZhuEstimator(*integ, x, flux_fes);
-         break;
-      }
-
-      default:
-         if (myid == 0)
-         {
-            std::cout << "Unkown estimator. Falling back to L2ZZ." << std::endl;
-         }
-      case 0:
-      {
-         auto flux_fes = new ParFiniteElementSpace(&pmesh, &flux_fec, sdim);
-         auto smooth_flux_fes = new ParFiniteElementSpace(&pmesh, &smooth_flux_fec);
-         estimator = new L2ZienkiewiczZhuEstimator(*integ, x, flux_fes, smooth_flux_fes);
-         break;
-      }
-   }
+   ParFiniteElementSpace smooth_flux_fes(&pmesh, &smooth_flux_fec);
+   L2ZienkiewiczZhuEstimator estimator(*integ, x, flux_fes, smooth_flux_fes);
 
    // 11. As in Example 6p, we also need a refiner. This time the refinement
    //     strategy is based on a fixed threshold that is applied locally to each
    //     element. The global threshold is turned off by setting the total error
    //     fraction to zero. We also enforce a maximum refinement ratio between
    //     adjacent elements.
-   ThresholdRefiner refiner(*estimator);
+   ThresholdRefiner refiner(estimator);
    refiner.SetTotalErrorFraction(0.0); // use purely local threshold
    refiner.SetLocalErrorGoal(max_elem_error);
    refiner.PreferConformingRefinement();
@@ -273,7 +233,7 @@ int main(int argc, char *argv[])
    // 12. A derefiner selects groups of elements that can be coarsened to form
    //     a larger element. A conservative enough threshold needs to be set to
    //     prevent derefining elements that would immediately be refined again.
-   ThresholdDerefiner derefiner(*estimator);
+   ThresholdDerefiner derefiner(estimator);
    derefiner.SetThreshold(hysteresis * max_elem_error);
    derefiner.SetNCLimit(nc_limit);
 
@@ -356,7 +316,7 @@ int main(int argc, char *argv[])
          refiner.Apply(pmesh);
          if (myid == 0)
          {
-            cout << ", total error: " << estimator->GetTotalError() << endl;
+            cout << ", total error: " << estimator.GetTotalError() << endl;
          }
 
          // 21. Quit the AMR loop if the termination criterion has been met
@@ -386,8 +346,6 @@ int main(int argc, char *argv[])
       }
    }
 
-   delete estimator;
-
    // 25. Exit
    MPI_Finalize();
    return 0;

examples/ex25.cpp

@@ -10,10 +10,6 @@
 //               ex25 -o 2 -f 8.0 -ref 3 -prob 4 -m ../data/inline-quad.mesh
 //               ex25 -o 2 -f 2.0 -ref 1 -prob 4 -m ../data/inline-hex.mesh
 //
-// Device sample runs:
-//               ex25 -o 2 -f 8.0 -ref 3 -prob 4 -m ../data/inline-quad.mesh -pa -d cuda
-//               ex25 -o 2 -f 2.0 -ref 1 -prob 4 -m ../data/inline-hex.mesh -pa -d cuda
-//
 // Description:  This example code solves a simple electromagnetic wave
 //               propagation problem corresponding to the second order
 //               indefinite Maxwell equation
@@ -161,10 +157,7 @@ int main(int argc, char *argv[])
    int iprob = 4;
    double freq = 5.0;
    bool herm_conv = true;
-   bool umf_solver = false;
    bool visualization = 1;
-   bool pa = false;
-   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -183,28 +176,15 @@ int main(int argc, char *argv[])
                   "Frequency (in Hz).");
    args.AddOption(&herm_conv, "-herm", "--hermitian", "-no-herm",
                   "--no-hermitian", "Use convention for Hermitian operators.");
-#ifdef MFEM_USE_SUITESPARSE
-   args.AddOption(&umf_solver, "-umf", "--umfpack", "-no-umf",
-                  "--no-umfpack", "Use the UMFPack Solver.");
-#endif
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
    args.Parse();
 
    if (iprob > 4) { iprob = 4; }
    prob = (prob_type)iprob;
 
-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Setup the mesh
+   // 2. Setup the mesh
    if (!mesh_file)
    {
       exact_known = true;
@@ -245,7 +225,7 @@ int main(int argc, char *argv[])
    // Setup PML length
    Array2D<double> length(dim, 2); length = 0.0;
 
-   // 4. Setup the Cartesian PML region.
+   // 3. Setup the Cartesian PML region.
    switch (prob)
    {
       case disc:
@@ -271,19 +251,19 @@ int main(int argc, char *argv[])
    comp_domain_bdr = pml->GetCompDomainBdr();
    domain_bdr = pml->GetDomainBdr();
 
-   // 5. Refine the mesh to increase the resolution.
+   // 4. Refine the mesh to increase the resolution.
    for (int l = 0; l < ref_levels; l++)
    {
       mesh->UniformRefinement();
    }
 
-   // 6. Reorient mesh in case of a tet mesh
+   // 5. Reorient mesh in case of a tet mesh
    mesh->ReorientTetMesh();
 
    // Set element attributes in order to distinguish elements in the PML region
    pml->SetAttributes(mesh);
 
-   // 7. Define a finite element space on the mesh. Here we use the Nedelec
+   // 6. Define a finite element space on the mesh. Here we use the Nedelec
    //    finite elements of the specified order.
    FiniteElementCollection *fec = new ND_FECollection(order, dim);
    FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
@@ -291,7 +271,7 @@ int main(int argc, char *argv[])
 
    cout << "Number of finite element unknowns: " << size << endl;
 
-   // 8. Determine the list of true essential boundary dofs. In this example,
+   // 7. Determine the list of true essential boundary dofs. In this example,
    //    the boundary conditions are defined based on the specific mesh and the
    //    problem type.
    Array<int> ess_tdof_list;
@@ -333,12 +313,12 @@ int main(int argc, char *argv[])
    }
    fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
 
-   // 9. Setup Complex Operator convention
+   // 8. Setup Complex Operator convention
    ComplexOperator::Convention conv =
       herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
 
-   // 10. Set up the linear form b(.) which corresponds to the right-hand side of
-   //     the FEM linear system.
+   // 9. Set up the linear form b(.) which corresponds to the right-hand side of
+   //    the FEM linear system.
    VectorFunctionCoefficient f(dim, source);
    ComplexLinearForm b(fespace, conv);
    if (prob == load_src)
@@ -348,7 +328,7 @@ int main(int argc, char *argv[])
    b.Vector::operator=(0.0);
    b.Assemble();
 
-   // 11. Define the solution vector x as a complex finite element grid function
+   // 10. Define the solution vector x as a complex finite element grid function
    //     corresponding to fespace.
    ComplexGridFunction x(fespace);
    x = 0.0;
@@ -356,7 +336,7 @@ int main(int argc, char *argv[])
    VectorFunctionCoefficient E_Im(dim, E_bdr_data_Im);
    x.ProjectBdrCoefficientTangent(E_Re, E_Im, ess_bdr);
 
-   // 12. Set up the sesquilinear form a(.,.)
+   // 11. Set up the sesquilinear form a(.,.)
    //
    //     In Comp
    //     Domain:   1/mu (Curl E, Curl F) - omega^2 * epsilon (E,F)
@@ -410,35 +390,32 @@ int main(int argc, char *argv[])
    a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_Re),
                          new VectorFEMassIntegrator(restr_c2_Im));
 
-   // 13. Assemble the bilinear form and the corresponding linear system,
+   // 12. Assemble the bilinear form and the corresponding linear system,
    //     applying any necessary transformations such as: assembly, eliminating
    //     boundary conditions, applying conforming constraints for
    //     non-conforming AMR, etc.
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
    a.Assemble(0);
 
    OperatorPtr A;
    Vector B, X;
    a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
 
-   // 14. Solve using a direct or an iterative solver
+   // 13. Solve using a direct or an iterative solver
 #ifdef MFEM_USE_SUITESPARSE
-   if (!pa && umf_solver)
    {
       ComplexUMFPackSolver csolver(*A.As<ComplexSparseMatrix>());
       csolver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
       csolver.SetPrintLevel(1);
       csolver.Mult(B, X);
    }
-#endif
-   // 14a. Set up the Bilinear form a(.,.) for the preconditioner
+#else
+   // 13a. Set up the Bilinear form a(.,.) for the preconditioner
    //
    //    In Comp
    //    Domain:   1/mu (Curl E, Curl F) + omega^2 * epsilon (E,F)
    //
    //    In PML:   1/mu (abs(1/det(J) J^T J) Curl E, Curl F)
    //              + omega^2 * epsilon (abs(det(J) * (J^T J)^-1) * E, F)
-   if (pa || !umf_solver)
    {
       ConstantCoefficient absomeg(pow(omega, 2) * epsilon);
       RestrictedCoefficient restr_absomeg(absomeg,attr);
@@ -458,57 +435,39 @@ int main(int argc, char *argv[])
       prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_abs));
       prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_abs));
 
-      if (pa) { prec.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
       prec.Assemble();
 
-      // 14b. Define and apply a GMRES solver for AU=B with a block diagonal
-      //      preconditioner based on the Gauss-Seidel or Jacobi sparse smoother.
+      OperatorPtr PCOpAh;
+      prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
+
+      // 13b. Define and apply a GMRES solver for AU=B with a block diagonal
+      //      preconditioner based on the Gauss-Seidel sparse smoother.
       Array<int> offsets(3);
       offsets[0] = 0;
       offsets[1] = fespace->GetTrueVSize();
       offsets[2] = fespace->GetTrueVSize();
       offsets.PartialSum();
 
-      Operator *pc_r = nullptr;
-      Operator *pc_i = nullptr;
-      int s = (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0;
-      if (pa)
-      {
-         // Jacobi Smoother
-         OperatorJacobiSmoother *d00 = new OperatorJacobiSmoother(prec, ess_tdof_list);
-         ScaledOperator *d11 = new ScaledOperator(d00, s);
-         pc_r = d00;
-         pc_i = d11;
-      }
-      else
-      {
-         OperatorPtr PCOpAh;
-         prec.SetDiagonalPolicy(mfem::Operator::DIAG_ONE);
-         prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
-
-         // Gauss-Seidel Smoother
-         GSSmoother *gs00 = new GSSmoother(*PCOpAh.As<SparseMatrix>());
-         ScaledOperator *gs11 = new ScaledOperator(gs00, s);
-         pc_r = gs00;
-         pc_i = gs11;
-      }
-
-      BlockDiagonalPreconditioner BlockDP(offsets);
-      BlockDP.SetDiagonalBlock(0, pc_r);
-      BlockDP.SetDiagonalBlock(1, pc_i);
+      GSSmoother gs00(*PCOpAh.As<SparseMatrix>());
+      BlockDiagonalPreconditioner BlockGS(offsets);
+      ScaledOperator gs11(&gs00,
+                          (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0);
+      BlockGS.SetDiagonalBlock(0,&gs00);
+      BlockGS.SetDiagonalBlock(1,&gs11);
 
       GMRESSolver gmres;
       gmres.SetPrintLevel(1);
       gmres.SetKDim(200);
-      gmres.SetMaxIter(pa ? 5000 : 2000);
+      gmres.SetMaxIter(2000);
       gmres.SetRelTol(1e-5);
       gmres.SetAbsTol(0.0);
       gmres.SetOperator(*A);
-      gmres.SetPreconditioner(BlockDP);
+      gmres.SetPreconditioner(BlockGS);
       gmres.Mult(B, X);
    }
+#endif
 
-   // 15. Recover the solution as a finite element grid function and compute the
+   // 14. Recover the solution as a finite element grid function and compute the
    //     errors if the exact solution is known.
    a.RecoverFEMSolution(X, b, x);
 
@@ -545,7 +504,7 @@ int main(int argc, char *argv[])
            << sqrt(L2Error_Re*L2Error_Re + L2Error_Im*L2Error_Im) << "\n\n";
    }
 
-   // 16. Save the refined mesh and the solution. This output can be viewed
+   // 15. Save the refined mesh and the solution. This output can be viewed
    //     later using GLVis: "glvis -m mesh -g sol".
    {
       ofstream mesh_ofs("ex25.mesh");
@@ -560,7 +519,7 @@ int main(int argc, char *argv[])
       x.imag().Save(sol_i_ofs);
    }
 
-   // 17. Send the solution by socket to a GLVis server.
+   // 16. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       // Define visualization keys for GLVis (see GLVis documentation)
@@ -611,7 +570,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 18. Free the used memory.
+   // 17. Free the used memory.
    delete pml;
    delete fespace;
    delete fec;
@@ -957,14 +916,7 @@ void CartesianPML::SetBoundaries()
 
 void CartesianPML::SetAttributes(Mesh *mesh_)
 {
-   // Initialize bdr attributes
-   for (int i = 0; i < mesh_->GetNBE(); ++i)
-   {
-      mesh_->GetBdrElement(i)->SetAttribute(i+1);
-   }
-
    int nrelem = mesh_->GetNE();
-
    elems.SetSize(nrelem);
 
    // Loop through the elements and identify which of them are in the PML

examples/ex25p.cpp

@@ -10,10 +10,6 @@
 //               mpirun -np 4 ex25p -o 2 -f 8.0 -rs 2 -rp 2 -prob 4 -m ../data/inline-quad.mesh
 //               mpirun -np 4 ex25p -o 2 -f 2.0 -rs 1 -rp 1 -prob 4 -m ../data/inline-hex.mesh
 //
-// Device sample runs:
-//               mpirun -np 4 ex25p -o 1 -f 3.0 -rs 3 -rp 1 -prob 2 -pa -d cuda
-//               mpirun -np 4 ex25p -o 2 -f 1.0 -rs 1 -rp 1 -prob 3 -pa -d cuda
-//
 // Description:  This example code solves a simple electromagnetic wave
 //               propagation problem corresponding to the second order
 //               indefinite Maxwell equation
@@ -171,8 +167,6 @@ int main(int argc, char *argv[])
    bool slu_solver  = false;
    bool mumps_solver = false;
    bool visualization = 1;
-   bool pa = false;
-   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -204,10 +198,6 @@ int main(int argc, char *argv[])
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (slu_solver && mumps_solver)
    {
@@ -221,12 +211,7 @@ int main(int argc, char *argv[])
    if (iprob > 4) { iprob = 4; }
    prob = (prob_type)iprob;
 
-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Setup the (serial) mesh on all processors.
+   // 3. Setup the (serial) mesh on all processors.
    if (!mesh_file)
    {
       exact_known = true;
@@ -274,7 +259,7 @@ int main(int argc, char *argv[])
    // Setup PML length
    Array2D<double> length(dim, 2); length = 0.0;
 
-   // 5. Setup the Cartesian PML region.
+   // 4. Setup the Cartesian PML region.
    switch (prob)
    {
       case disc:
@@ -300,13 +285,13 @@ int main(int argc, char *argv[])
    comp_domain_bdr = pml->GetCompDomainBdr();
    domain_bdr = pml->GetDomainBdr();
 
-   // 6. Refine the serial mesh on all processors to increase the resolution.
+   // 5. Refine the serial mesh on all processors to increase the resolution.
    for (int l = 0; l < ref_levels; l++)
    {
       mesh->UniformRefinement();
    }
 
-   // 7. Define a parallel mesh by a partitioning of the serial mesh.
+   // 6. Define a parallel mesh by a partitioning of the serial mesh.
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
    delete mesh;
    {
@@ -316,13 +301,13 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 7a. Reorient mesh in case of a tet mesh
+   // 6a. Reorient mesh in case of a tet mesh
    pmesh->ReorientTetMesh();
 
-   // 8. Set element attributes in order to distinguish elements in the PML
+   // 7. Set element attributes in order to distinguish elements in the PML
    pml->SetAttributes(pmesh);
 
-   // 9. Define a parallel finite element space on the parallel mesh. Here we
+   // 8. Define a parallel finite element space on the parallel mesh. Here we
    //    use the Nedelec finite elements of the specified order.
    FiniteElementCollection *fec = new ND_FECollection(order, dim);
    ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
@@ -332,9 +317,9 @@ int main(int argc, char *argv[])
       cout << "Number of finite element unknowns: " << size << endl;
    }
 
-   // 10. Determine the list of true (i.e. parallel conforming) essential
-   //     boundary dofs. In this example, the boundary conditions are defined
-   //     based on the specific mesh and the problem type.
+   // 9. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    based on the specific mesh and the problem type.
    Array<int> ess_tdof_list;
    Array<int> ess_bdr;
    if (pmesh->bdr_attributes.Size())
@@ -374,11 +359,11 @@ int main(int argc, char *argv[])
    }
    fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
 
-   // 11. Setup Complex Operator convention
+   // 10. Setup Complex Operator convention
    ComplexOperator::Convention conv =
       herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
 
-   // 12. Set up the parallel linear form b(.) which corresponds to the
+   // 11. Set up the parallel linear form b(.) which corresponds to the
    //     right-hand side of the FEM linear system.
    VectorFunctionCoefficient f(dim, source);
    ParComplexLinearForm b(fespace, conv);
@@ -389,7 +374,7 @@ int main(int argc, char *argv[])
    b.Vector::operator=(0.0);
    b.Assemble();
 
-   // 13. Define the solution vector x as a parallel complex finite element grid
+   // 12. Define the solution vector x as a parallel complex finite element grid
    //     function corresponding to fespace.
    ParComplexGridFunction x(fespace);
    x = 0.0;
@@ -397,7 +382,7 @@ int main(int argc, char *argv[])
    VectorFunctionCoefficient E_Im(dim, E_bdr_data_Im);
    x.ProjectBdrCoefficientTangent(E_Re, E_Im, ess_bdr);
 
-   // 14. Set up the parallel sesquilinear form a(.,.)
+   // 13. Set up the parallel sesquilinear form a(.,.)
    //
    //     In Comp
    //     Domain:   1/mu (Curl E, Curl F) - omega^2 * epsilon (E,F)
@@ -451,20 +436,19 @@ int main(int argc, char *argv[])
    a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_Re),
                          new VectorFEMassIntegrator(restr_c2_Im));
 
-   // 15. Assemble the parallel bilinear form and the corresponding linear
+   // 14. Assemble the parallel bilinear form and the corresponding linear
    //     system, applying any necessary transformations such as: parallel
    //     assembly, eliminating boundary conditions, applying conforming
    //     constraints for non-conforming AMR, etc.
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
    a.Assemble();
 
    OperatorPtr Ah;
    Vector B, X;
    a.FormLinearSystem(ess_tdof_list, x, b, Ah, X, B);
 
-   // 16. Solve using a direct or an iterative solver
+   // 15. Solve using a direct or an iterative solver
 #ifdef MFEM_USE_SUPERLU
-   if (!pa && slu_solver)
+   if (slu_solver)
    {
       // Transform to monolithic HypreParMatrix
       HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
@@ -479,7 +463,7 @@ int main(int argc, char *argv[])
    }
 #endif
 #ifdef MFEM_USE_MUMPS
-   if (!pa && mumps_solver)
+   if (mumps_solver)
    {
       HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
       MUMPSSolver mumps;
@@ -497,7 +481,7 @@ int main(int argc, char *argv[])
    //
    //    In PML:   1/mu (abs(1/det(J) J^T J) Curl E, Curl F)
    //              + omega^2 * epsilon (abs(det(J) * (J^T J)^-1) * E, F)
-   if (pa || (!slu_solver && !mumps_solver))
+   if (!slu_solver && !mumps_solver)
    {
       ConstantCoefficient absomeg(pow(omega, 2) * epsilon);
       RestrictedCoefficient restr_absomeg(absomeg,attr);
@@ -517,9 +501,11 @@ int main(int argc, char *argv[])
       prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_abs));
       prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_abs));
 
-      if (pa) { prec.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
       prec.Assemble();
 
+      OperatorPtr PCOpAh;
+      prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
+
       // 16b. Define and apply a parallel GMRES solver for AU=B with a block
       //      diagonal preconditioner based on hypre's AMS preconditioner.
       Array<int> offsets(3);
@@ -528,41 +514,21 @@ int main(int argc, char *argv[])
       offsets[2] = fespace->GetTrueVSize();
       offsets.PartialSum();
 
-      Operator *pc_r = nullptr;
-      Operator *pc_i = nullptr;
-      int s = (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0;
-      if (pa)
-      {
-         // Jacobi Smoother
-         OperatorJacobiSmoother *d00 = new OperatorJacobiSmoother(prec, ess_tdof_list);
-         ScaledOperator *d11 = new ScaledOperator(d00, s);
-         pc_r = d00;
-         pc_i = d11;
-      }
-      else
-      {
-         OperatorPtr PCOpAh;
-         prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
-
-         // Hypre AMS
-         HypreAMS *ams00 = new HypreAMS(*PCOpAh.As<HypreParMatrix>(), fespace);
-         ScaledOperator *ams11 = new ScaledOperator(ams00, s);
-         pc_r = ams00;
-         pc_i = ams11;
-      }
-
-      BlockDiagonalPreconditioner BlockDP(offsets);
-      BlockDP.SetDiagonalBlock(0, pc_r);
-      BlockDP.SetDiagonalBlock(1, pc_i);
+      HypreAMS ams00(*PCOpAh.As<HypreParMatrix>(),fespace);
+      BlockDiagonalPreconditioner BlockAMS(offsets);
+      ScaledOperator ams11(&ams00,
+                           (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0);
+      BlockAMS.SetDiagonalBlock(0,&ams00);
+      BlockAMS.SetDiagonalBlock(1,&ams11);
 
       GMRESSolver gmres(MPI_COMM_WORLD);
       gmres.SetPrintLevel(1);
       gmres.SetKDim(200);
-      gmres.SetMaxIter(pa ? 5000 : 2000);
+      gmres.SetMaxIter(2000);
       gmres.SetRelTol(1e-5);
       gmres.SetAbsTol(0.0);
       gmres.SetOperator(*Ah);
-      gmres.SetPreconditioner(BlockDP);
+      gmres.SetPreconditioner(BlockAMS);
       gmres.Mult(B, X);
    }
 
@@ -1037,12 +1003,8 @@ void CartesianPML::SetBoundaries()
 
 void CartesianPML::SetAttributes(ParMesh *pmesh)
 {
-   // Initialize bdr attributes
-   for (int i = 0; i < pmesh->GetNBE(); ++i)
-   {
-      pmesh->GetBdrElement(i)->SetAttribute(i+1);
-   }
-
+   int myid;
+   MPI_Comm_rank(MPI_COMM_WORLD,&myid);
    int nrelem = pmesh->GetNE();
 
    // Initialize list with 1

examples/ex26.cpp

@@ -40,7 +40,7 @@ using namespace mfem;
 // in the FiniteElementSpaceHierarchy. The preconditioner uses a CG solver on
 // the coarsest level and second order Chebyshev accelerated smoothers on the
 // other levels.
-class DiffusionMultigrid : public GeometricMultigrid
+class DiffusionMultigrid : public Multigrid
 {
 private:
    ConstantCoefficient one;
@@ -49,7 +49,7 @@ public:
    // Constructs a diffusion multigrid for the given FiniteElementSpaceHierarchy
    // and the array of essential boundaries
    DiffusionMultigrid(FiniteElementSpaceHierarchy& fespaces, Array<int>& ess_bdr)
-      : GeometricMultigrid(fespaces), one(1.0)
+      : Multigrid(fespaces), one(1.0)
    {
       ConstructCoarseOperatorAndSolver(fespaces.GetFESpaceAtLevel(0), ess_bdr);
 

examples/ex26p.cpp

@@ -37,7 +37,7 @@ using namespace mfem;
 // all spaces except the coarsest one in the ParFiniteElementSpaceHierarchy.
 // The multigrid uses a PCG solver preconditioned with AMG on the coarsest level
 // and second order Chebyshev accelerated smoothers on the other levels.
-class DiffusionMultigrid : public GeometricMultigrid
+class DiffusionMultigrid : public Multigrid
 {
 private:
    ConstantCoefficient one;
@@ -48,7 +48,7 @@ public:
    // and the array of essential boundaries
    DiffusionMultigrid(ParFiniteElementSpaceHierarchy& fespaces,
                       Array<int>& ess_bdr)
-      : GeometricMultigrid(fespaces), one(1.0)
+      : Multigrid(fespaces), one(1.0)
    {
       ConstructCoarseOperatorAndSolver(fespaces.GetFESpaceAtLevel(0), ess_bdr);
 

examples/ex3p.cpp

@@ -22,7 +22,6 @@
 //
 // Device sample runs:
 //               mpirun -np 4 ex3p -m ../data/star.mesh -pa -d cuda
-//               mpirun -np 4 ex3p -m ../data/star.mesh -no-pa -d cuda
 //               mpirun -np 4 ex3p -m ../data/star.mesh -pa -d raja-cuda
 //               mpirun -np 4 ex3p -m ../data/star.mesh -pa -d raja-omp
 //               mpirun -np 4 ex3p -m ../data/beam-hex.mesh -pa -d cuda
@@ -69,10 +68,7 @@ int main(int argc, char *argv[])
    bool static_cond = false;
    bool pa = false;
    const char *device_config = "cpu";
-   bool visualization = true;
-#ifdef MFEM_USE_AMGX
-   bool useAmgX = false;
-#endif
+   bool visualization = 1;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -90,11 +86,6 @@ int main(int argc, char *argv[])
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
-#ifdef MFEM_USE_AMGX
-   args.AddOption(&useAmgX, "-amgx", "--useAmgX", "-no-amgx",
-                  "--no-useAmgX",
-                  "Enable or disable AmgX in MatrixFreeAMS.");
-#endif
 
    args.Parse();
    if (!args.Good())
@@ -167,10 +158,9 @@ int main(int argc, char *argv[])
    //    by marking all the boundary attributes from the mesh as essential
    //    (Dirichlet) and converting them to a list of true dofs.
    Array<int> ess_tdof_list;
-   Array<int> ess_bdr;
    if (pmesh->bdr_attributes.Size())
    {
-      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
       ess_bdr = 1;
       fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
    }
@@ -214,20 +204,20 @@ int main(int argc, char *argv[])
    Vector B, X;
    a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
 
-   // 13. Solve the system AX=B using PCG with an AMS preconditioner.
-   if (pa)
+   // 13. Solve the system AX=B using PCG with the AMS preconditioner from hypre
+   //     (in the full assembly case) or CG with Jacobi preconditioner (in the
+   //     partial assembly case).
+
+   if (pa) // Jacobi preconditioning in partial assembly mode
    {
-#ifdef MFEM_USE_AMGX
-      MatrixFreeAMS ams(*a, *A, *fespace, muinv, sigma, NULL, ess_bdr, useAmgX);
-#else
-      MatrixFreeAMS ams(*a, *A, *fespace, muinv, sigma, NULL, ess_bdr);
-#endif
+      OperatorJacobiSmoother Jacobi(*a, ess_tdof_list);
+
       CGSolver cg(MPI_COMM_WORLD);
       cg.SetRelTol(1e-12);
       cg.SetMaxIter(1000);
       cg.SetPrintLevel(1);
       cg.SetOperator(*A);
-      cg.SetPreconditioner(ams);
+      cg.SetPreconditioner(Jacobi);
       cg.Mult(B, X);
    }
    else

examples/ex5.cpp

@@ -29,7 +29,7 @@
 //               finite elements (velocity u) and piecewise discontinuous
 //               polynomials (pressure p).
 //
-//               The example demonstrates the use of the BlockOperator class, as
+//               The example demonstrates the use of the BlockMatrix class, as
 //               well as the collective saving of several grid functions in
 //               VisIt (visit.llnl.gov) and ParaView (paraview.org) formats.
 //

examples/ex5p.cpp

@@ -29,7 +29,7 @@
 //               finite elements (velocity u) and piecewise discontinuous
 //               polynomials (pressure p).
 //
-//               The example demonstrates the use of the BlockOperator class, as
+//               The example demonstrates the use of the BlockMatrix class, as
 //               well as the collective saving of several grid functions in
 //               VisIt (visit.llnl.gov) and ParaView (paraview.org) formats.
 //               Optional saving with ADIOS2 (adios2.readthedocs.io) streams is

examples/ex6p.cpp

@@ -4,11 +4,9 @@
 //
 // Sample runs:  mpirun -np 4 ex6p -m ../data/square-disc.mesh -o 1
 //               mpirun -np 4 ex6p -m ../data/square-disc.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/square-disc.mesh -o 2 -ns
 //               mpirun -np 4 ex6p -m ../data/square-disc-nurbs.mesh -o 2
 //               mpirun -np 4 ex6p -m ../data/star.mesh -o 3
 //               mpirun -np 4 ex6p -m ../data/escher.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/escher.mesh -o 2 -ns
 //               mpirun -np 4 ex6p -m ../data/fichera.mesh -o 2
 //               mpirun -np 4 ex6p -m ../data/disc-nurbs.mesh -o 2
 //               mpirun -np 4 ex6p -m ../data/ball-nurbs.mesh
@@ -16,7 +14,6 @@
 //               mpirun -np 4 ex6p -m ../data/star-surf.mesh -o 2
 //               mpirun -np 4 ex6p -m ../data/square-disc-surf.mesh -o 2
 //               mpirun -np 4 ex6p -m ../data/amr-quad.mesh
-//               mpirun -np 4 ex6p --restart
 //
 // Device sample runs:
 //               mpirun -np 4 ex6p -pa -d cuda
@@ -37,8 +34,8 @@
 //               The example demonstrates MFEM's capability to work with both
 //               conforming and nonconforming refinements, in 2D and 3D, on
 //               linear, curved and surface meshes. Interpolation of functions
-//               from coarse to fine meshes, restarting from a checkpoint, as
-//               well as persistent GLVis visualization are also illustrated.
+//               from coarse to fine meshes, as well as persistent GLVis
+//               visualization are also illustrated.
 //
 //               We recommend viewing Example 1 before viewing this example.
 
@@ -62,9 +59,6 @@ int main(int argc, char *argv[])
    int order = 1;
    bool pa = false;
    const char *device_config = "cpu";
-   bool nc_simplices = false;
-   int max_dofs = 100000;
-   bool restart = false;
    bool visualization = true;
 
    OptionsParser args(argc, argv);
@@ -76,14 +70,6 @@ int main(int argc, char *argv[])
                   "--no-partial-assembly", "Enable Partial Assembly.");
    args.AddOption(&device_config, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
-   args.AddOption(&nc_simplices, "-ns", "--nonconforming-simplices",
-                  "-cs", "--conforming-simplices",
-                  "For simplicial meshes, enable/disable nonconforming"
-                  " refinement");
-   args.AddOption(&max_dofs, "-md", "--max-dofs",
-                  "Stop after reaching this many degrees of freedom.");
-   args.AddOption(&restart, "-res", "--restart", "-no-res", "--no-restart",
-                  "Restart computation from the last checkpoint.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -107,52 +93,39 @@ int main(int argc, char *argv[])
    Device device(device_config);
    if (myid == 0) { device.Print(); }
 
-   ParMesh *pmesh;
-   if (!restart)
+   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+   int sdim = mesh->SpaceDimension();
+
+   // 5. Refine the serial mesh on all processors to increase the resolution.
+   //    Also project a NURBS mesh to a piecewise-quadratic curved mesh. Make
+   //    sure that the mesh is non-conforming.
+   if (mesh->NURBSext)
    {
-      // 4. Read the (serial) mesh from the given mesh file on all processors.
-      //    We can handle triangular, quadrilateral, tetrahedral, hexahedral,
-      //    surface and volume meshes with the same code.
-      Mesh mesh(mesh_file, 1, 1);
-
-      // 5. Refine the serial mesh on all processors to increase the resolution.
-      //    Also project a NURBS mesh to a piecewise-quadratic curved mesh. Make
-      //    sure that the mesh is non-conforming.
-      if (mesh.NURBSext)
-      {
-         mesh.UniformRefinement();
-         mesh.SetCurvature(2);
-      }
-      mesh.EnsureNCMesh(nc_simplices);
-
-      // 6. Define a parallel mesh by partitioning the serial mesh.
-      //    Once the parallel mesh is defined, the serial mesh can be deleted.
-      pmesh = new ParMesh(MPI_COMM_WORLD, mesh);
-   }
-   else
-   {
-      // 7. We can also restart the computation by loading the mesh from a
-      //    previously saved check-point.
-      string fname(MakeParFilename("ex6p-checkpoint.", myid));
-      ifstream ifs(fname);
-      MFEM_VERIFY(ifs.good(), "Checkpoint file " << fname << " not found.");
-      pmesh = new ParMesh(MPI_COMM_WORLD, ifs);
+      mesh->UniformRefinement();
+      mesh->SetCurvature(2);
    }
+   mesh->EnsureNCMesh();
 
-   int dim = pmesh->Dimension();
-   int sdim = pmesh->SpaceDimension();
+   // 6. Define a parallel mesh by partitioning the serial mesh.
+   //    Once the parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh pmesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
 
-   MFEM_VERIFY(pmesh->bdr_attributes.Size() > 0,
+   MFEM_VERIFY(pmesh.bdr_attributes.Size() > 0,
                "Boundary attributes required in the mesh.");
-   Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+   Array<int> ess_bdr(pmesh.bdr_attributes.Max());
    ess_bdr = 1;
 
-   // 8. Define a finite element space on the mesh. The polynomial order is
+   // 7. Define a finite element space on the mesh. The polynomial order is
    //    one (linear) by default, but this can be changed on the command line.
    H1_FECollection fec(order, dim);
-   ParFiniteElementSpace fespace(pmesh, &fec);
+   ParFiniteElementSpace fespace(&pmesh, &fec);
 
-   // 9. As in Example 1p, we set up bilinear and linear forms corresponding to
+   // 8. As in Example 1p, we set up bilinear and linear forms corresponding to
    //    the Laplace problem -\Delta u = 1. We don't assemble the discrete
    //    problem yet, this will be done in the main loop.
    ParBilinearForm a(&fespace);
@@ -169,12 +142,12 @@ int main(int argc, char *argv[])
    a.AddDomainIntegrator(integ);
    b.AddDomainIntegrator(new DomainLFIntegrator(one));
 
-   // 10. The solution vector x and the associated finite element grid function
-   //     will be maintained over the AMR iterations. We initialize it to zero.
+   // 9. The solution vector x and the associated finite element grid function
+   //    will be maintained over the AMR iterations. We initialize it to zero.
    ParGridFunction x(&fespace);
    x = 0;
 
-   // 11. Connect to GLVis.
+   // 10. Connect to GLVis.
    char vishost[] = "localhost";
    int  visport   = 19916;
 
@@ -196,29 +169,30 @@ int main(int argc, char *argv[])
       sout.precision(8);
    }
 
-   // 12. Set up an error estimator. Here we use the Zienkiewicz-Zhu estimator
+   // 11. Set up an error estimator. Here we use the Zienkiewicz-Zhu estimator
    //     with L2 projection in the smoothing step to better handle hanging
    //     nodes and parallel partitioning. We need to supply a space for the
    //     discontinuous flux (L2) and a space for the smoothed flux (H(div) is
    //     used here).
    L2_FECollection flux_fec(order, dim);
-   ParFiniteElementSpace flux_fes(pmesh, &flux_fec, sdim);
+   ParFiniteElementSpace flux_fes(&pmesh, &flux_fec, sdim);
    RT_FECollection smooth_flux_fec(order-1, dim);
-   ParFiniteElementSpace smooth_flux_fes(pmesh, &smooth_flux_fec);
+   ParFiniteElementSpace smooth_flux_fes(&pmesh, &smooth_flux_fec);
    // Another possible option for the smoothed flux space:
    // H1_FECollection smooth_flux_fec(order, dim);
-   // ParFiniteElementSpace smooth_flux_fes(pmesh, &smooth_flux_fec, dim);
+   // ParFiniteElementSpace smooth_flux_fes(&pmesh, &smooth_flux_fec, dim);
    L2ZienkiewiczZhuEstimator estimator(*integ, x, flux_fes, smooth_flux_fes);
 
-   // 13. A refiner selects and refines elements based on a refinement strategy.
+   // 12. A refiner selects and refines elements based on a refinement strategy.
    //     The strategy here is to refine elements with errors larger than a
    //     fraction of the maximum element error. Other strategies are possible.
    //     The refiner will call the given error estimator.
    ThresholdRefiner refiner(estimator);
    refiner.SetTotalErrorFraction(0.7);
 
-   // 14. The main AMR loop. In each iteration we solve the problem on the
+   // 13. The main AMR loop. In each iteration we solve the problem on the
    //     current mesh, visualize the solution, and refine the mesh.
+   const int max_dofs = 100000;
    for (int it = 0; ; it++)
    {
       HYPRE_Int global_dofs = fespace.GlobalTrueVSize();
@@ -228,19 +202,19 @@ int main(int argc, char *argv[])
          cout << "Number of unknowns: " << global_dofs << endl;
       }
 
-      // 15. Assemble the right-hand side and determine the list of true
+      // 14. Assemble the right-hand side and determine the list of true
       //     (i.e. parallel conforming) essential boundary dofs.
       Array<int> ess_tdof_list;
       fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
       b.Assemble();
 
-      // 16. Assemble the stiffness matrix. Note that MFEM doesn't care at this
+      // 15. Assemble the stiffness matrix. Note that MFEM doesn't care at this
       //     point that the mesh is nonconforming and parallel.  The FE space is
       //     considered 'cut' along hanging edges/faces, and also across
       //     processor boundaries.
       a.Assemble();
 
-      // 17. Create the parallel linear system: eliminate boundary conditions.
+      // 16. Create the parallel linear system: eliminate boundary conditions.
       //     The system will be solved for true (unconstrained/unique) DOFs only.
       OperatorPtr A;
       Vector B, X;
@@ -248,7 +222,7 @@ int main(int argc, char *argv[])
       const int copy_interior = 1;
       a.FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
 
-      // 18. Solve the linear system A X = B.
+      // 17. Solve the linear system A X = B.
       //     * With full assembly, use the BoomerAMG preconditioner from hypre.
       //     * With partial assembly, use a diagonal preconditioner.
       Solver *M = NULL;
@@ -271,19 +245,19 @@ int main(int argc, char *argv[])
       cg.Mult(B, X);
       delete M;
 
-      // 19. Switch back to the host and extract the parallel grid function
+      // 18. Switch back to the host and extract the parallel grid function
       //     corresponding to the finite element approximation X. This is the
       //     local solution on each processor.
       a.RecoverFEMSolution(X, b, x);
 
-      // 20. Send the solution by socket to a GLVis server.
+      // 19. Send the solution by socket to a GLVis server.
       if (visualization)
       {
          sout << "parallel " << num_procs << " " << myid << "\n";
-         sout << "solution\n" << *pmesh << x << flush;
+         sout << "solution\n" << pmesh << x << flush;
       }
 
-      if (global_dofs >= max_dofs)
+      if (global_dofs > max_dofs)
       {
          if (myid == 0)
          {
@@ -292,11 +266,11 @@ int main(int argc, char *argv[])
          break;
       }
 
-      // 21. Call the refiner to modify the mesh. The refiner calls the error
+      // 20. Call the refiner to modify the mesh. The refiner calls the error
       //     estimator to obtain element errors, then it selects elements to be
       //     refined and finally it modifies the mesh. The Stop() method can be
       //     used to determine if a stopping criterion was met.
-      refiner.Apply(*pmesh);
+      refiner.Apply(pmesh);
       if (refiner.Stop())
       {
          if (myid == 0)
@@ -306,7 +280,7 @@ int main(int argc, char *argv[])
          break;
       }
 
-      // 22. Update the finite element space (recalculate the number of DOFs,
+      // 21. Update the finite element space (recalculate the number of DOFs,
       //     etc.) and create a grid function update matrix. Apply the matrix
       //     to any GridFunctions over the space. In this case, the update
       //     matrix is an interpolation matrix so the updated GridFunction will
@@ -314,11 +288,11 @@ int main(int argc, char *argv[])
       fespace.Update();
       x.Update();
 
-      // 23. Load balance the mesh, and update the space and solution. Currently
+      // 22. Load balance the mesh, and update the space and solution. Currently
       //     available only for nonconforming meshes.
-      if (pmesh->Nonconforming())
+      if (pmesh.Nonconforming())
       {
-         pmesh->Rebalance();
+         pmesh.Rebalance();
 
          // Update the space and the GridFunction. This time the update matrix
          // redistributes the GridFunction among the processors.
@@ -326,30 +300,12 @@ int main(int argc, char *argv[])
          x.Update();
       }
 
-      // 24. Inform also the bilinear and linear forms that the space has
+      // 23. Inform also the bilinear and linear forms that the space has
       //     changed.
       a.Update();
       b.Update();
-
-      // 25. Save the current state of the mesh every 5 iterations. The
-      //     computation can be restarted from this point. Note that unlike in
-      //     visualization, we need to use the 'ParPrint' method to save all
-      //     internal parallel data structures.
-      if ((it + 1) % 5 == 0)
-      {
-         ofstream ofs(MakeParFilename("ex6p-checkpoint.", myid));
-         ofs.precision(8);
-         pmesh->ParPrint(ofs);
-
-         if (myid == 0)
-         {
-            cout << "\nCheckpoint saved." << endl;
-         }
-      }
    }
 
-   delete pmesh;
-
    MPI_Finalize();
    return 0;
 }

examples/ex9.cpp

@@ -290,16 +290,15 @@ int main(int argc, char *argv[])
       k.SetAssemblyLevel(AssemblyLevel::FULL);
    }
    m.AddDomainIntegrator(new MassIntegrator);
-   constexpr double alpha = -1.0;
-   k.AddDomainIntegrator(new ConvectionIntegrator(velocity, alpha));
+   k.AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
    k.AddInteriorFaceIntegrator(
-      new NonconservativeDGTraceIntegrator(velocity, alpha));
+      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
    k.AddBdrFaceIntegrator(
-      new NonconservativeDGTraceIntegrator(velocity, alpha));
+      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
 
    LinearForm b(&fes);
    b.AddBdrFaceIntegrator(
-      new BoundaryFlowIntegrator(inflow, velocity, alpha));
+      new BoundaryFlowIntegrator(inflow, velocity, -1.0, -0.5));
 
    m.Assemble();
    int skip_zeros = 0;
@@ -450,7 +449,7 @@ FE_Evolution::FE_Evolution(BilinearForm &_M, BilinearForm &_K, const Vector &_b)
    : TimeDependentOperator(_M.Height()), M(_M), K(_K), b(_b), z(_M.Height())
 {
    Array<int> ess_tdof_list;
-   if (M.GetAssemblyLevel() == AssemblyLevel::LEGACY)
+   if (M.GetAssemblyLevel() == AssemblyLevel::LEGACYFULL)
    {
       M_prec = new DSmoother(M.SpMat());
       M_solver.SetOperator(M.SpMat());

examples/ex9p.cpp

@@ -64,66 +64,6 @@ double inflow_function(const Vector &x);
 // Mesh bounding box
 Vector bb_min, bb_max;
 
-// Type of preconditioner for implicit time integrator
-enum class PrecType : int
-{
-   ILU = 0,
-   AIR = 1
-};
-
-#if MFEM_HYPRE_VERSION >= 21800
-// Algebraic multigrid preconditioner for advective problems based on
-// approximate ideal restriction (AIR). Most effective when matrix is
-// first scaled by DG block inverse, and AIR applied to scaled matrix.
-// See https://doi.org/10.1137/17M1144350.
-class AIR_prec : public Solver
-{
-private:
-   const HypreParMatrix *A;
-   // Copy of A scaled by block-diagonal inverse
-   HypreParMatrix A_s;
-
-   HypreBoomerAMG *AIR_solver;
-   int blocksize;
-
-public:
-   AIR_prec(int blocksize_) : AIR_solver(NULL), blocksize(blocksize_) { }
-
-   void SetOperator(const Operator &op)
-   {
-      width = op.Width();
-      height = op.Height();
-
-      A = dynamic_cast<const HypreParMatrix *>(&op);
-      MFEM_VERIFY(A != NULL, "AIR_prec requires a HypreParMatrix.")
-
-      // Scale A by block-diagonal inverse
-      BlockInverseScale(A, &A_s, NULL, NULL, blocksize,
-                        BlockInverseScaleJob::MATRIX_ONLY);
-      delete AIR_solver;
-      AIR_solver = new HypreBoomerAMG(A_s);
-      AIR_solver->SetAdvectiveOptions(1, "", "FA");
-      AIR_solver->SetPrintLevel(0);
-      AIR_solver->SetMaxLevels(50);
-   }
-
-   virtual void Mult(const Vector &x, Vector &y) const
-   {
-      // Scale the rhs by block inverse and solve system
-      HypreParVector z_s;
-      BlockInverseScale(A, NULL, &x, &z_s, blocksize,
-                        BlockInverseScaleJob::RHS_ONLY);
-      AIR_solver->Mult(z_s, y);
-   }
-
-   ~AIR_prec()
-   {
-      delete AIR_solver;
-   }
-};
-#endif
-
-
 class DG_Solver : public Solver
 {
 private:
@@ -131,37 +71,24 @@ private:
    SparseMatrix M_diag;
    HypreParMatrix *A;
    GMRESSolver linear_solver;
-   Solver *prec;
+   BlockILU prec;
    double dt;
 public:
-   DG_Solver(HypreParMatrix &M_, HypreParMatrix &K_, const FiniteElementSpace &fes,
-             PrecType prec_type)
+   DG_Solver(HypreParMatrix &M_, HypreParMatrix &K_, const FiniteElementSpace &fes)
       : M(M_),
         K(K_),
         A(NULL),
         linear_solver(M.GetComm()),
+        prec(fes.GetFE(0)->GetDof(),
+             BlockILU::Reordering::MINIMUM_DISCARDED_FILL),
         dt(-1.0)
    {
-      int block_size = fes.GetFE(0)->GetDof();
-      if (prec_type == PrecType::ILU)
-      {
-         prec = new BlockILU(block_size,
-                             BlockILU::Reordering::MINIMUM_DISCARDED_FILL);
-      }
-      else if (prec_type == PrecType::AIR)
-      {
-#if MFEM_HYPRE_VERSION >= 21800
-         prec = new AIR_prec(block_size);
-#else
-         MFEM_ABORT("Must have MFEM_HYPRE_VERSION >= 21800 to use AIR.\n");
-#endif
-      }
       linear_solver.iterative_mode = false;
       linear_solver.SetRelTol(1e-9);
       linear_solver.SetAbsTol(0.0);
       linear_solver.SetMaxIter(100);
       linear_solver.SetPrintLevel(0);
-      linear_solver.SetPreconditioner(*prec);
+      linear_solver.SetPreconditioner(prec);
 
       M.GetDiag(M_diag);
    }
@@ -194,12 +121,10 @@ public:
 
    ~DG_Solver()
    {
-      delete prec;
       delete A;
    }
 };
 
-
 /** A time-dependent operator for the right-hand side of the ODE. The DG weak
     form of du/dt = -v.grad(u) is M du/dt = K u + b, where M and K are the mass
     and advection matrices, and b describes the flow on the boundary. This can
@@ -217,8 +142,7 @@ private:
    mutable Vector z;
 
 public:
-   FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K, const Vector &_b,
-                PrecType prec_type);
+   FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K, const Vector &_b);
 
    virtual void Mult(const Vector &x, Vector &y) const;
    virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);
@@ -230,9 +154,10 @@ public:
 int main(int argc, char *argv[])
 {
    // 1. Initialize MPI.
-   MPI_Session mpi;
-   int num_procs = mpi.WorldSize();
-   int myid = mpi.WorldRank();
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
 
    // 2. Parse command-line options.
    problem = 0;
@@ -253,11 +178,7 @@ int main(int argc, char *argv[])
    bool adios2 = false;
    bool binary = false;
    int vis_steps = 5;
-#if MFEM_HYPRE_VERSION >= 21800
-   PrecType prec_type = PrecType::AIR;
-#else
-   PrecType prec_type = PrecType::ILU;
-#endif
+
    int precision = 8;
    cout.precision(precision);
 
@@ -291,8 +212,6 @@ int main(int argc, char *argv[])
                   "Final time; start time is 0.");
    args.AddOption(&dt, "-dt", "--time-step",
                   "Time step.");
-   args.AddOption((int *)&prec_type, "-pt", "--prec-type", "Preconditioner for "
-                  "implicit solves. 0 for ILU, 1 for pAIR-AMG.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
@@ -313,19 +232,20 @@ int main(int argc, char *argv[])
    args.Parse();
    if (!args.Good())
    {
-      if (mpi.Root())
+      if (myid == 0)
       {
          args.PrintUsage(cout);
       }
+      MPI_Finalize();
       return 1;
    }
-   if (mpi.Root())
+   if (myid == 0)
    {
       args.PrintOptions(cout);
    }
 
    Device device(device_config);
-   if (mpi.Root()) { device.Print(); }
+   if (myid == 0) { device.Print(); }
 
    // 3. Read the serial mesh from the given mesh file on all processors. We can
    //    handle geometrically periodic meshes in this code.
@@ -352,11 +272,12 @@ int main(int argc, char *argv[])
       case 23: ode_solver = new SDIRK23Solver; break;
       case 24: ode_solver = new SDIRK34Solver; break;
       default:
-         if (mpi.Root())
+         if (myid == 0)
          {
             cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
          }
          delete mesh;
+         MPI_Finalize();
          return 3;
    }
 
@@ -390,7 +311,7 @@ int main(int argc, char *argv[])
    ParFiniteElementSpace *fes = new ParFiniteElementSpace(pmesh, &fec);
 
    HYPRE_Int global_vSize = fes->GlobalTrueVSize();
-   if (mpi.Root())
+   if (myid == 0)
    {
       cout << "Number of unknowns: " << global_vSize << endl;
    }
@@ -421,16 +342,15 @@ int main(int argc, char *argv[])
    }
 
    m->AddDomainIntegrator(new MassIntegrator);
-   constexpr double alpha = -1.0;
-   k->AddDomainIntegrator(new ConvectionIntegrator(velocity, alpha));
+   k->AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
    k->AddInteriorFaceIntegrator(
-      new NonconservativeDGTraceIntegrator(velocity, alpha));
+      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
    k->AddBdrFaceIntegrator(
-      new NonconservativeDGTraceIntegrator(velocity, alpha));
+      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
 
    ParLinearForm *b = new ParLinearForm(fes);
    b->AddBdrFaceIntegrator(
-      new BoundaryFlowIntegrator(inflow, velocity, alpha));
+      new BoundaryFlowIntegrator(inflow, velocity, -1.0, -0.5));
 
    int skip_zeros = 0;
    m->Assemble();
@@ -531,11 +451,11 @@ int main(int argc, char *argv[])
       sout.open(vishost, visport);
       if (!sout)
       {
-         if (mpi.Root())
+         if (myid == 0)
             cout << "Unable to connect to GLVis server at "
                  << vishost << ':' << visport << endl;
          visualization = false;
-         if (mpi.Root())
+         if (myid == 0)
          {
             cout << "GLVis visualization disabled.\n";
          }
@@ -547,7 +467,7 @@ int main(int argc, char *argv[])
          sout << "solution\n" << *pmesh << *u;
          sout << "pause\n";
          sout << flush;
-         if (mpi.Root())
+         if (myid == 0)
             cout << "GLVis visualization paused."
                  << " Press space (in the GLVis window) to resume it.\n";
       }
@@ -556,7 +476,7 @@ int main(int argc, char *argv[])
    // 10. Define the time-dependent evolution operator describing the ODE
    //     right-hand side, and perform time-integration (looping over the time
    //     iterations, ti, with a time-step dt).
-   FE_Evolution adv(*m, *k, *B, prec_type);
+   FE_Evolution adv(*m, *k, *B);
 
    double t = 0.0;
    adv.SetTime(t);
@@ -573,7 +493,7 @@ int main(int argc, char *argv[])
 
       if (done || ti % vis_steps == 0)
       {
-         if (mpi.Root())
+         if (myid == 0)
          {
             cout << "time step: " << ti << ", time: " << t << endl;
          }
@@ -644,18 +564,20 @@ int main(int argc, char *argv[])
 #endif
    delete dc;
 
+   MPI_Finalize();
    return 0;
 }
 
 
 // Implementation of class FE_Evolution
 FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
-                           const Vector &_b, PrecType prec_type)
-   : TimeDependentOperator(_M.Height()), b(_b),
+                           const Vector &_b)
+   : TimeDependentOperator(_M.Height()),
+     b(_b),
      M_solver(_M.ParFESpace()->GetComm()),
      z(_M.Height())
 {
-   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACY)
+   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACYFULL)
    {
       M.Reset(_M.ParallelAssemble(), true);
       K.Reset(_K.ParallelAssemble(), true);
@@ -669,14 +591,14 @@ FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
    M_solver.SetOperator(*M);
 
    Array<int> ess_tdof_list;
-   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACY)
+   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACYFULL)
    {
       HypreParMatrix &M_mat = *M.As<HypreParMatrix>();
       HypreParMatrix &K_mat = *K.As<HypreParMatrix>();
       HypreSmoother *hypre_prec = new HypreSmoother(M_mat, HypreSmoother::Jacobi);
       M_prec = hypre_prec;
 
-      dg_solver = new DG_Solver(M_mat, K_mat, *_M.FESpace(), prec_type);
+      dg_solver = new DG_Solver(M_mat, K_mat, *_M.FESpace());
    }
    else
    {
@@ -692,10 +614,6 @@ FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
    M_solver.SetPrintLevel(0);
 }
 
-// Solve the equation:
-//    u_t = M^{-1}(Ku + b),
-// by solving associated linear system
-//    (M - dt*K) d = K*u + b
 void FE_Evolution::ImplicitSolve(const double dt, const Vector &x, Vector &k)
 {
    K->Mult(x, z);

examples/ginkgo/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

examples/ginkgo/makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

examples/hiop/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

examples/hiop/makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

examples/href_exs/href_exs.cpp

@@ -1,471 +0,0 @@
-// h-refinement examples
-//
-// Compile with: make href_exs
-//
-// Sample runs:  
-//
-// Description: Make h-refinement data sets for neural net
-//              comparison
-
-
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Exact smooth analytic solution for convergence study
-double u_exact(const Vector &);
-void u_grad_exact(const Vector &, Vector &);
-double u_exact_2(const Vector &);
-void u_grad_exact_2(const Vector &, Vector &);
-double u_exact_3(const Vector &);
-void u_grad_exact_3(const Vector &, Vector &);
-
-void convergenceStudy(const char *mesh_file, int num_ref, int &order,
-                      double &l2_err_prev, double &h1_err_prev, bool &visualization, 
-                      int &exact, int &solvePDE, bool static_cond)
-{
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 4. Refine the mesh num_ref times
-
-
-    for (int l = 1; l < num_ref+1; l++)
-    {
-        mesh->UniformRefinement();
-    }
-
-
-   // 5. Define a finite element space on the mesh. Here we use continuous
-   //    Lagrange finite elements of the specified order.
-
-   FiniteElementCollection *fec;
-   if (order == 1)
-   {
-      fec = new H1_FECollection(1, 2);
-   }
-   else if (order > 1)
-   {
-      fec = new H1_FECollection(order, 2);
-   }
-   else if (order < 0)
-   {
-      // fec = new H1_FECollection(-order, 2, BasisType::Positive);
-      fec = new H1_FECollection(-order, 2);
-   }
-   else
-   {
-     cout << "Error - something went wrong in processing order input." << endl;
-     fec = NULL;
-   }
-
-   // Set exact solution
-
-   // exact == 1 case:
-   FunctionCoefficient *u1 = new FunctionCoefficient(u_exact);
-   VectorFunctionCoefficient *(u1_grad) = new VectorFunctionCoefficient(dim, u_grad_exact);
-   
-   // exact == 2 case:
-   FunctionCoefficient *u2 = new FunctionCoefficient(u_exact_2);
-   VectorFunctionCoefficient *u2_grad = new VectorFunctionCoefficient(dim, u_grad_exact_2);
-
-   // exact == 3 case:
-   FunctionCoefficient *u3 = new FunctionCoefficient(u_exact_3);
-   VectorFunctionCoefficient *u3_grad = new VectorFunctionCoefficient(dim, u_grad_exact_3);
-
-   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
-
-   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
-   //    In this example, the boundary conditions are defined by marking all
-   //    the boundary attributes from the mesh as essential (Dirichlet) and
-   //    converting them to a list of true dofs.
-
-   // this variable may not be right for more general meshes than lattices?
-   int gotNdofs = fespace->GetNDofs();
-
-   Array<int> ess_tdof_list;
-   Array<int> ess_bdr(mesh->bdr_attributes.Max());
-   ess_bdr = 1;
-
-
-   if (solvePDE==1)
-   {
-      if (mesh->bdr_attributes.Size())
-      {
-         fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-   }
-   // For L2 Projection:
-   // Do not get boundary dofs
-
-   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
-   //    the basis functions in the finite element fespace.
-
-   LinearForm *b = new LinearForm(fespace);
-
-   if (solvePDE==1)
-   {
-      ConstantCoefficient zero(0.0);
-      b->AddDomainIntegrator(new DomainLFIntegrator(zero));
-   }
-   else // L2 Projection
-   {
-      if (exact == 1)
-      {
-         b->AddDomainIntegrator(new DomainLFIntegrator(*u1));
-      }
-      else if (exact == 2) 
-      {
-         b->AddDomainIntegrator(new DomainLFIntegrator(*u2));
-      }
-      else // exact == 3
-      {
-         b->AddDomainIntegrator(new DomainLFIntegrator(*u3));
-      }
-   }
-
-   b->Assemble();
-   
-   // 8. Define the solution vector x as a finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero,
-   //    which satisfies the boundary conditions.
-   GridFunction x(fespace);
-   x=0.0;
-
-   // 9. Set up the bilinear form a(.,.) on the finite element space
-   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //    domain integrator.
-   BilinearForm *a = new BilinearForm(fespace);
-
-   // DiffusionIntegrator *my_diff_integrator = new DiffusionIntegrator;
-   // MassIntegrator *my_mass_integrator = new MassIntegrator;
-
-   if (solvePDE==1)
-   {   
-      if (exact == 1)
-      {
-         x.ProjectBdrCoefficient(*u1, ess_bdr);
-      }
-      else if (exact == 2)
-      {
-         x.ProjectBdrCoefficient(*u2, ess_bdr);
-      }
-      else if (exact == 3)
-      {
-         x.ProjectBdrCoefficient(*u3, ess_bdr);
-      }
-      a->AddDomainIntegrator(new DiffusionIntegrator);
-   }
-   else // L2 Projection
-   {
-      a->AddDomainIntegrator(new MassIntegrator);
-   }
-   
-   // 10. Assemble the bilinear form and the corresponding linear system,
-   //     applying any necessary transformations such as: eliminating boundary
-   //     conditions, applying conforming constraints for non-conforming AMR,
-   //     static condensation, etc.
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
-
-   OperatorPtr A;
-   Vector B, X;
-
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-   
-   // cout << "Size of linear system: " << A->Height() << endl;
-  
-   // 11. Solve the linear system A X = B.
-
-   GSSmoother M((SparseMatrix&)(*A));
-   X = 0.0;
-   PCG(*A, M, B, X, 0, 200, 1e-24, 0.0);
-   
-   // 12. Recover the solution as a finite element grid function.
-   a->RecoverFEMSolution(X, *b, x);
-
-
-   // 13. Save the refined mesh and the solution. This output can be viewed later
-   //     using GLVis: "glvis -m refined.mesh -g sol.gf".
-   ofstream mesh_ofs("refined.mesh");
-   mesh_ofs.precision(8);
-   mesh->Print(mesh_ofs);
-   ofstream sol_ofs("sol.gf");
-   sol_ofs.precision(8);
-   x.Save(sol_ofs);
-
-
-   // 14. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *mesh << x << flush;
-   }
-   
-
-
-   // Compute and print the L^2 and H^1 norms of the error.
-   ConstantCoefficient one(1.0);
-   
-   double l2_err = 0;
-   double h1_err = 0;
-
-   if (exact == 1)
-   {
-      l2_err = x.ComputeL2Error(*u1);
-      h1_err = x.ComputeH1Error(u1, u1_grad, &one, 1.0, 1);
-   }
-   else if (exact == 2)
-   {
-      l2_err = x.ComputeL2Error(*u2);
-      h1_err = x.ComputeH1Error(u2, u2_grad, &one, 1.0, 1);
-   }
-   else if (exact == 3)
-   {
-      l2_err = x.ComputeL2Error(*u3);
-      h1_err = x.ComputeH1Error(u3, u3_grad, &one, 1.0, 1);
-   }
-
-   double l2_rate, h1_rate;
-
-   if (num_ref != 0)
-   {
-      l2_rate = -log(l2_err/l2_err_prev) / log(2);
-      h1_rate = -log(h1_err/h1_err_prev) / log(2);
-   }
-   else
-   {
-      l2_rate = 0.0;
-      h1_rate = 0.0;
-   }
-
-   int one_over_h = mesh->GetNE();
-   one_over_h = sqrt(one_over_h);
-
-   cout << setw(16) << gotNdofs << setw(16) << one_over_h << setw(
-           16) << l2_err << setw( 16) << l2_rate;
-   cout << setw(16) << h1_err << setw(16) << h1_rate << endl;
-
-   l2_err_prev = l2_err;
-   h1_err_prev = h1_err;
-
-
-   // Save the data to pass to neural network
-   cout << "Mesh vertex array = " << endl;
-   cout <<  mesh->GetVertex(0) << endl;
-
-   // 15. Free the used memory.
-   // delete pcg;
-   // delete amg;
-   // delete my_diff_integrator;
-   // delete my_mass_integrator;
-   delete a; 
-   delete b;
-   delete fespace;
-   delete u1;
-   delete u1_grad;
-   delete u2;
-   delete u2_grad;
-   delete u3;
-   delete u3_grad;
-   delete fec;
-   delete mesh;
-
-   return;
-}
-
-double u_exact(const Vector &x)
-{
-   return(x(0)+x(1));
-}
-
-void u_grad_exact(const Vector &x, Vector &u)
-{
-   u(0) = 1;
-   u(1) = 1;
-}
-
-double u_exact_2(const Vector &x)
-{
-   return sin(x(1))*exp(x(0));
-}
-
-void u_grad_exact_2(const Vector &x, Vector &u)
-{
-   u(0) = sin(x(1))*exp(x(0));
-   u(1) = cos(x(1))*exp(x(0));
-}
-
-double u_exact_3(const Vector &x)
-{
-    // return (x(0)*x(0) + (0.5)*x(1)*x(1));
-    int m=10;
-    double total = sin(x(0))* pow( sin(  x(0)*x(0) / M_PI  ), 2*m  );
-    total += sin(x(1))* pow( sin( 2*x(1)*x(1) / M_PI  ), 2*m  );
-    total *= -1;
-    return total;
-}
-
-void u_grad_exact_3(const Vector &x, Vector &u)
-{
-    // presumes m=10
-    u(0)  = - (40.0 * x(0) * cos( x(0)*x(0)/M_PI ) * sin(x(0)) * pow(sin( x(0)*x(0)/M_PI ),19)) / M_PI;
-    u(0) += - cos(x(0))*pow(sin( x(0)*x(0)/M_PI ),20);
-    u(1)  = - (80.0 * x(1) * cos( 2.0 * x(1)*x(1)/M_PI ) * sin(x(1)) * pow(sin( 2.0 * x(1)*x(1)/M_PI ),19)   ) / M_PI;
-    u(1) += - cos(x(1))*pow(sin( 2.0 * x(1)*x(1)/M_PI ),20);
-}
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   int total_refinements = 0;
-
-   // const char *mesh_file = "../../data/twoSquare.mesh";
-   // const char *mesh_file = "../../data/star-q3.mesh";
-   const char *mesh_file = "../../data/inline-oneTri.mesh";
-   int order = 1;
-   bool static_cond = false;
-   const char *device_config = "cpu";
-   bool visualization = false;
-   int exact = 3;
-   int solvePDE = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&total_refinements, "-r", "--refine",
-                  "Number of refinements to do.");   
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   //args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-   //             "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&exact, "-e", "--exact", 
-                  "Choice of exact solution. 1=constant 1; 2=sin(x)e^y; 3=michalewicz.");
-   args.AddOption(&solvePDE, "-L", "--L2Project",
-                  "Solve a PDE (1) or do L2 Projection (2)");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   if (order != 1)
-   {
-      cout << "Only allowing order 1 triangle elements for now." << endl;
-      return 1;
-   }
-   else if (order == 1)
-   {
-      cout << "Using H1 triangular elements of order " << order << "." << endl;
-   }
-
-   if (solvePDE == 1)
-   {
-      cout << "Approximating solution to Laplace problem with ";
-      if (exact == 1)
-      {
-        cout << "exact solution u(x,y)=x+y" << endl;
-      }
-      else if (exact == 2)
-      {
-         cout << "exact solution u(x,y)=sin(y)e^x" << endl;
-      }
-      else if (exact == 3)
-      {
-         cout << endl << "Michalewicz is not harmonic. Use option -L 2 to do L2 projection instead."
-              << endl;
-         return 1;
-      }
-      else
-      {
-         cout << endl << "*** Wrong usage of exact solution parameter (-e)"
-              << endl;
-         return 1;
-      }
-   }
-   else if (solvePDE == 2)
-   {
-      cout << "Doing L^2 projection of basis with right hand side ";
-      if (exact == 1)
-      {
-        cout << "u(x,y)=x+y" << endl;
-      }
-      else if (exact == 2)
-      {
-         cout << "u(x,y)=sin(y)e^x" << endl;
-      }
-      else if (exact == 3)
-      {
-         cout << "u(x,y)=michalewicz function" << endl;
-      }
-      else
-      {
-         cout << endl << "*** Wrong usage of exact solution parameter (-e)"
-              << endl;
-         return 1;
-      }
-   }
-   else
-   {
-      cout << "Wrong usage of solve vs. L2 Projection option -L."
-           << endl;
-      return 1;
-   }
-
-
-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // Set output options and print header
-   cout.precision(4);
-
-   cout << "----------------------------------------------------------------------------------------"
-        << endl;
-   cout << left << setw(16) << "DOFs "<< setw(16) <<"1/h "<< setw(
-           16) << "L^2 error "<< setw(16);
-   cout << "L^2 rate "<< setw(16) << "H^1 error "<< setw(16) << "H^1 rate" << endl;
-   cout << "----------------------------------------------------------------------------------------"
-        << endl;
-
-   double l2_err_prev = 0.0;
-   double h1_err_prev = 0.0;
-
-   // 3. Read the mesh from the given mesh file.
-   // Run last round with vis, if desired.
-
-   // can use this as a max DoF tolerance:  (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
-
-   // Loop over number of refinements for convergence study
-
-    bool noVisYet = false;
-    for (int i = 0; i < (total_refinements); i++)
-    {
-        convergenceStudy(mesh_file, i, order, l2_err_prev, h1_err_prev, noVisYet, 
-        exact, solvePDE, static_cond);
-    }
-
-    convergenceStudy(mesh_file, total_refinements, order, l2_err_prev, h1_err_prev, visualization, 
-            exact, solvePDE, static_cond);
-
-    return 0;
-}

examples/makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -144,7 +144,7 @@ clean-build:
 	rm -rf *.dSYM *.TVD.*breakpoints
 
 clean-exec:
-	@rm -f refined.mesh displaced.mesh mesh.* ex5.mesh ex6p-checkpoint.*
+	@rm -f refined.mesh displaced.mesh mesh.* ex5.mesh
 	@rm -rf Example5* Example9* Example15* Example16* Example23* ParaView
 	@rm -f sphere_refined.* sol.* sol_u.* sol_p.* sol_r.* sol_i.*
 	@rm -f ex9.mesh ex9-mesh.* ex9-init.* ex9-final.*

examples/petsc/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

examples/petsc/makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

examples/pumi/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

examples/pumi/makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #

examples/sundials/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #