modifications

Merge remote-tracking branch 'origin/master' into PrmBlockNonlinearForm
fixing bugs
2021-02-24 13:54:33 -08:00 · 2021-01-26 15:35:17 -08:00 · 2020-12-29 19:35:48 -08:00 · 2020-12-23 12:37:18 -08:00 · 2020-12-23 12:36:16 -08:00 · 2020-12-21 14:49:58 -08:00
586 changed files with 11130 additions and 21018 deletions
@@ -1,42 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-name: repo-check
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
-
-jobs:
-  copyright-check:
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: checkout mfem
-      uses: actions/checkout@v2
-      with:
-        path: mfem
-
-    - name: copyright check
-      run: |
-        cd mfem
-        if git grep -l "^#.*\-2020" > matches.txt
-        then
-          echo "Please update the following files to Copyright (c) 2010-2021:"
-          cat matches.txt
-          exit 1
-        else
-          echo "No outdated copyright found."
-        fi
@@ -66,7 +66,6 @@ examples/sol_u.*
 examples/sol_p.*
 examples/sol_r.*
 examples/sol_i.*
-examples/ex6p-checkpoint.*
 examples/ex9.mesh
 examples/ex9-mesh.*
 examples/ex9-init.*
@@ -238,9 +237,6 @@ miniapps/performance/refined.mesh
 miniapps/performance/mesh.*
 miniapps/performance/sol.*

-miniapps/shifted/distance
-miniapps/shifted/ParaViewDistance
-
 miniapps/tools/display-basis
 miniapps/tools/load-dc
 miniapps/tools/convert-dc
@@ -268,8 +264,6 @@ miniapps/toys/lissajous.mesh
 miniapps/toys/lissajous.gf
 miniapps/toys/mondrian.mesh

-miniapps/solvers/block-solvers
-
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -23,6 +23,10 @@
 # also means that the BUILD_ROOT directory will never be cleaned.
 # TODO: add a clean-up mechanism
 #
+# BUILD_PATH:
+# In BUILD_ROOT, we want to separate builds depending on the machine used,
+# typically because we build on a set of dependencies per machine.
+#
 # REBASELINE:
 # Defines the default choice for updating the saved baseline results. By default
 # the baseline can only be updated from the master branch. This variable offers
@@ -41,8 +45,9 @@

 variables:
  BUILD_ROOT: ${CI_BUILDS_DIR}/${CI_PROJECT_NAME}_${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID}
+  BUILD_PATH: ${CI_BUILDS_DIR}/${CI_PROJECT_NAME}_${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID}/${PLAT}/${TOOLCHAIN}
  REBASELINE: "NO"
-  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
+  MFEM_ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
  TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
  TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
  ARTIFACTS_DIR: artifacts
@@ -51,18 +56,23 @@ variables:
 # points, however, we use "needs" keyword to express the DAG of jobs for more
 # efficiency.
 # - We use setup phase to download content outside of mfem directory.
-# - Allocate/Release is where quartz resources are allocated/released once for all.
-# - Build and Test is where we build and MFEM for multiple toolchains.
+# - Allocate is where quartz resources are allocated once for all.
+# - Libs is where dependencies are built.
+# - Build is where we build MFEM for multiple toolchains.
+# - Test is where we perform a first set of tests on MFEM build.
 # - Baseline_checks gathers baseline-type test suites execution
 # - Baseline_publish, only available on master, allows to update baseline
 #   results
+# - Deallocate releases quartz resources
 stages:
-  - q_allocate_resources
-  - q_build_and_test
-  - q_release_resources
-  - l_build_and_test
-  - c_build_and_test
  - setup
+  - allocate
+  - libs
+  - build
+  - test
+  - deallocate
+  - lassen_build
+  - lassen_test
  - baseline_check
  - baseline_publish

@@ -85,25 +95,92 @@ setup:
    - if [ ! -d "tests" ]; then git clone ${TESTS_REPO}; fi
    - cd tpls && git pull && cd ..
    - cd tests && git pull && cd ..
-  needs: []

-.build_toss_3_x86_64_ib_script:
-  script:
-    - export THREADS=12
-    - echo ${ALLOC_NAME}
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - echo ${JOBID}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 30 -N 1 tests/gitlab/build_and_test
+# Share configurations (templates) for toolchain configuration.
+# Note: Any job using this template shouldn't redefine "before_script" key.
+.with_gcc_6_1_0:
+  variables:
+    TOOLCHAIN: gcc_6_1_0
+    CXX: g++
+    CC: gcc
+  before_script:
+    - module load gcc/6.1.0

-.build_toss_3_x86_64_ib_corona_script:
-  script:
-    - srun -p mi60 -t 15 -N 1 tests/gitlab/build_and_test
+# On lassen
+.with_gcc_8_3_1:
+  variables:
+    TOOLCHAIN: gcc_8_3_1
+    CXX: g++
+    CC: gcc
+  before_script:
+    - module load gcc/8.3.1

-# Lassen and Butte use a different job scheduler (spectrum lsf) that does not
-# allow pre-allocation the same way slurm does.
-.build_blueos_3_ppc64le_ib_script:
+.with_gcc_4_9_3:
+  variables:
+    TOOLCHAIN: gcc_4_9_3
+    CXX: g++
+    CC: gcc
+  before_script:
+    - module load gcc/4.9.3
+
+.if_srun_alloc_for_make: &if_srun_alloc_for_make |
+  if [[ "${MPIEXEC}" == "srun" ]]
+  then
+    export JOB_ID=$(squeue -h --name=${MFEM_ALLOC_NAME} --format=%A)
+    [[ -n "JOB_ID" ]] && export JOB_ID="--jobid=${JOB_ID}"
+    RESOURCES="--extra-node-info=1:${MAKE_PAR}:1 --cpus-per-task=${MAKE_PAR} --ntasks=1"
+    # Allocation prefix:
+    export EXEC_PREFIX="srun ${JOB_ID} ${RESOURCES}"
+  fi
+
+# Build of dependencies and mfem:
+# if on quartz, uses a part of preallocated resources
+# Note: Those jobs don't rely on MFEM repository, hence the use of
+# "GIT_STRATEGY: none" preventing its clone.
+.build_hypre:
+  variables:
+    EXEC_PREFIX: ""
+    GIT_STRATEGY: none
+  stage: libs
  script:
-    - lalloc 1 -W 15 tests/gitlab/build_and_test
+    - cd ${BUILD_ROOT}/tpls
+    - mkdir -p ${BUILD_PATH}
+    - make clean
+    - *if_srun_alloc_for_make
+    - ${EXEC_PREFIX} make -j ${MAKE_PAR} hypre PREFIX=${BUILD_PATH}/hypre/src/ BUILD_DIR=${BUILD_PATH} LOG_DIR=${BUILD_PATH}
+
+.build_metis:
+  variables:
+    EXEC_PREFIX: ""
+    GIT_STRATEGY: none
+  stage: libs
+  script:
+    - cd ${BUILD_ROOT}/tpls
+    - mkdir -p ${BUILD_PATH}
+    - *if_srun_alloc_for_make
+    - ${EXEC_PREFIX} make -j ${MAKE_PAR} ${METIS} PREFIX=${BUILD_PATH}/ BUILD_DIR=${BUILD_PATH} LOG_DIR=${BUILD_PATH} CC=${CC} CXX=${CXX}
+
+# Building MFEM
+.build_mfem:
+  variables:
+    EXEC_PREFIX: ""
+  stage: build
+  script:
+    - mkdir -p ${BUILD_PATH}
+    - cp -r ${CI_PROJECT_DIR} ${BUILD_PATH}/${CI_PROJECT_NAME}_${MFEM_USE_MPI}_${MFEM_DEBUG}
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_${MFEM_USE_MPI}_${MFEM_DEBUG}
+    - *if_srun_alloc_for_make
+    - make config CXX=${CXX} MPICXX=mpicxx MFEM_USE_MPI=${MFEM_USE_MPI} MFEM_DEBUG=${MFEM_DEBUG} MFEM_MPIEXEC="${EXEC_PREFIX}" MFEM_MPIEXEC_NP=${MPIEXEC_NP}
+    - ${EXEC_PREFIX} make -j ${MAKE_PAR} all
+
+# TODO: Could this phase use a parallel run?
+.sanitycheck_mfem:
+  stage: test
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_${MFEM_USE_MPI}_${MFEM_DEBUG}
+    - make test

 # Shared script for baseline and sample-run-baseline, the value of BASELINE_TEST
 # differentiates between the two tests.
@@ -125,7 +202,7 @@ setup:
  cd tests
  mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
  # run
-  srun --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${ADDITIONAL_DIR}"
+  salloc --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${ADDITIONAL_DIR}"
  # post
  mkdir ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}
  if [[ -s ${_glob_err} ]]
@@ -228,4 +305,4 @@ setup:
 # The list on jobs is defined in machine-specific files.
 include:
  - local: .gitlab/quartz.yml
-    #  - local: .gitlab/lassen.yml
+  - local: .gitlab/lassen.yml
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -15,85 +15,131 @@
  tags:
    - shell
    - quartz
-  rules:
-    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"' #run except if ...
-      when: never
-    - if: '$CI_JOB_NAME =~ /release_resources/'
-      when: always
-    - when: on_success
+  variables:
+    PLAT: quartz
+    MPIEXEC: srun
+    MPIEXEC_NP: "-n"
+    MAKE_PAR: 6
+    BASELINE_PAR: 18

 # Allocate
-q_allocate_resources:
+allocate_quartz:
  variables:
    GIT_STRATEGY: none
  extends: .on_quartz
-  stage: q_allocate_resources
+  stage: allocate
  script:
-    - salloc --exclusive --nodes=1 --partition=pdebug --time=30 --no-shell --job-name=${ALLOC_NAME}
+    - |
+      if [[ "${MPIEXEC}" == "srun" ]]
+      then
+      salloc --nodes=1 --extra-node-info=2:18:1 --partition=pdebug --time=30 --no-shell --job-name=${MFEM_ALLOC_NAME}
+      fi
  timeout: 6h

 # Release
-q_release_resources:
+deallocate_quartz:
  variables:
    GIT_STRATEGY: none
  extends: .on_quartz
-  stage: q_release_resources
+  stage: deallocate
  script:
-    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
+    - |
+      if [[ "${MPIEXEC}" == "srun" ]]
+      then
+      export JOB_ID=$(squeue -h --name=${MFEM_ALLOC_NAME} --format=%A)
+      ([[ -n "${JOB_ID}" ]] && scancel ${JOB_ID})
+      fi
+  when: always

-# Spack helped builds
-# Generic quartz build job, extending build script
-.build_and_test_on_quartz:
-  extends: [.build_toss_3_x86_64_ib_script, .on_quartz]
-  stage: q_build_and_test
+# Build external libraries (tpls)
+build_hypre_gcc_6_1_0_quartz:
+  extends: [.build_hypre, .with_gcc_6_1_0, .on_quartz]
+
+build_parmetis_gcc_6_1_0_quartz:
+  extends: [.build_metis, .with_gcc_6_1_0, .on_quartz]
+  variables:
+    METIS: parmetis
+
+build_metis4_gcc_6_1_0_quartz:
+  extends: [.build_metis, .with_gcc_6_1_0, .on_quartz]
+  variables:
+    METIS: metis4
+
+build_metis5_gcc_6_1_0_quartz:
+  extends: [.build_metis, .with_gcc_6_1_0, .on_quartz]
+  variables:
+    METIS: metis5
+  allow_failure: true

 # Build MFEM
-debug_ser_gcc_4_9_3:
+build_mfem_debug_ser_gcc_4_9_3_quartz:
+  extends: [.build_mfem, .with_gcc_4_9_3, .on_quartz]
  variables:
-    SPEC: "%gcc@4.9.3 +debug~mpi"
-  extends: .build_and_test_on_quartz
+    MFEM_DEBUG: "YES"
+    MFEM_USE_MPI: "NO"

-debug_ser_gcc_6_1_0:
+build_mfem_debug_ser_gcc_6_1_0_quartz:
+  extends: [.build_mfem, .with_gcc_6_1_0, .on_quartz]
  variables:
-    SPEC: "%gcc@6.1.0 +debug~mpi"
-  extends: .build_and_test_on_quartz
+    MFEM_DEBUG: "YES"
+    MFEM_USE_MPI: "NO"

-debug_par_gcc_6_1_0:
+build_mfem_debug_par_gcc_6_1_0_quartz:
+  extends: [.build_mfem, .with_gcc_6_1_0, .on_quartz]
  variables:
-    SPEC: "%gcc@6.1.0 +debug+mpi"
-  extends: .build_and_test_on_quartz
+    MFEM_DEBUG: "YES"
+    MFEM_USE_MPI: "YES"
+  needs: [build_hypre_gcc_6_1_0_quartz, build_parmetis_gcc_6_1_0_quartz]

-opt_ser_gcc_6_1_0:
+build_mfem_opt_ser_gcc_6_1_0_quartz:
+  extends: [.build_mfem, .with_gcc_6_1_0, .on_quartz]
  variables:
-    SPEC: "%gcc@6.1.0 ~mpi"
-  extends: .build_and_test_on_quartz
+    MFEM_DEBUG: "NO"
+    MFEM_USE_MPI: "NO"

-opt_par_gcc_6_1_0:
+build_mfem_opt_par_gcc_6_1_0_quartz:
+  extends: [.build_mfem, .with_gcc_6_1_0, .on_quartz]
  variables:
-    SPEC: "%gcc@6.1.0"
-  extends: .build_and_test_on_quartz
+    MFEM_DEBUG: "NO"
+    MFEM_USE_MPI: "YES"
+  needs: [build_hypre_gcc_6_1_0_quartz, build_parmetis_gcc_6_1_0_quartz]

-opt_par_gcc_6_1_0_sundials:
-  variables:
-    SPEC: "%gcc@6.1.0 +sundials"
-  extends: .build_and_test_on_quartz
+### Tests

-opt_par_gcc_6_1_0_petsc:
+# Sanity check
+sanitycheck_mfem_debug_ser_gcc_4_9_3_quartz:
+  extends: [.sanitycheck_mfem, .with_gcc_4_9_3, .on_quartz]
  variables:
-    SPEC: "%gcc@6.1.0 +petsc ^petsc+mumps"
-  extends: .build_and_test_on_quartz
+    MFEM_DEBUG: "YES"
+    MFEM_USE_MPI: "NO"
+  needs: [build_mfem_debug_ser_gcc_4_9_3_quartz, allocate_quartz]

-opt_par_gcc_6_1_0_pumi:
+sanitycheck_mfem_debug_ser_gcc_6_1_0_quartz:
+  extends: [.sanitycheck_mfem, .with_gcc_6_1_0, .on_quartz]
  variables:
-    SPEC: "%gcc@6.1.0 +pumi"
-  extends: .build_and_test_on_quartz
+    MFEM_DEBUG: "YES"
+    MFEM_USE_MPI: "NO"
+  needs: [build_mfem_debug_ser_gcc_6_1_0_quartz, allocate_quartz]
+
+sanitycheck_opt_ser_mfem_gcc_6_1_0_quartz:
+  extends: [.sanitycheck_mfem, .with_gcc_6_1_0, .on_quartz]
+  variables:
+    MFEM_DEBUG: "NO"
+    MFEM_USE_MPI: "NO"
+  needs: [build_mfem_opt_ser_gcc_6_1_0_quartz, allocate_quartz]
+
+sanitycheck_opt_par_mfem_gcc_6_1_0_quartz:
+  extends: [.sanitycheck_mfem, .with_gcc_6_1_0, .on_quartz]
+  variables:
+    MFEM_DEBUG: "NO"
+    MFEM_USE_MPI: "YES"
+  needs: [build_mfem_opt_par_gcc_6_1_0_quartz, allocate_quartz]

 # Baseline
 baselinecheck_mfem_intel_quartz:
  extends: [.baselinecheck_mfem, .on_quartz]
-  needs: [setup]
+  needs: [allocate_quartz]

 baselinepublish_mfem_quartz:
-  extends: [.on_quartz, .rebaseline_mfem]
+  extends: [.rebaseline_mfem, .on_quartz]
  needs: [baselinecheck_mfem_intel_quartz]
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -11,21 +11,6 @@
 Version 4.2.1 (development)
 ===========================

- Added high-order matrix-free auxiliary Maxwell solver for H(curl) problems,
-  as described in Barker and Kolev 2020 (https://doi.org/10.1002/nla.2348). See
-  Example 3p and linalg/auxiliary.?pp.
-
- Added a new miniapp block-solvers that compares the performance of various
-  solvers for mixed finite element discretization of the second order scalar
-  elliptic equations. Currently available solvers in the miniapp include a
-  block-diagonal preconditioner that is based on approximate Schur complement
-  (implemented in ex5p), and a newly implemented solver DivFreeSolver, which
-  exploits a multilevel decomposition of the Raviart-Thomas space and its
-  divergence-free subspace. See the miniapps/solvers directory for more details.
-
- Added a new miniapp for computing (signed) distance functions to a point
-  source or zero level set. See miniapps/shifted/distance.cpp.
-
 - Added matrix-free GPU-enabled implementations of GradientInterpolator and
  IdentityInterpolator.

@@ -35,14 +20,6 @@ Version 4.2.1 (development)
 - Added three ESDIRK time integrators: implicit trapezoid rule, L-stable
  ESDIRK-32, and A-stable ESDIRK-33.

- Introduced a new non-conforming mesh format that fixes known inconsistencies
-  of legacy "MFEM mesh v1.1" NC format and works consistently in both serial and
-  parallel. ParMesh::ParPrint can now print non-conforming AMR meshes that can
-  be used to restart a parallel AMR computation. Example 6p has been extended to
-  demonstrate restarting from a previously saved checkpoint. Note that parallel
-  NC data files are compatible with serial code, e.g., can be viewed with serial
-  GLVis. Loading of legacy NC mesh files is still supported.
-
 - Added a "scaled Jacobian" visualization option in the Mesh Explorer miniapp to
  help identify elements with poor mesh quality.

@@ -58,11 +35,9 @@ Version 4.2.1 (development)
 - Implemented a filter method for the Navier miniapp to stabilize highly
  turbulent flows in direct numerical simulation.

- Added HIP support to the CMake build system.
-
 - Added support for reading high-order Lagrange meshes in VTK format. Arbitrary-
  orders and all element types are supported. See the VTK blog for more info:
-  https://blog.kitware.com/wp-content/uploads/2018/09/Source_Issue_43.pdf.
+  https://blog.kitware.com/wp-content/uploads/2018/09/Source_Issue_43.pdf

 - Added support for reading VTK meshes in XML format.

@@ -74,9 +49,6 @@ Version 4.2.1 (development)
 - Added new mesh quality metrics and improved the untangling capabilities of the
  TMOP-based mesh optimization algorithms.

- Added convective and skew-symmetric integrators for the nonlinear term in the
-  Navier-Stokes equations.
-
 - Changed the interface for the error estimator.

 - Implemented the parallel Kelly error indicator for scalar-valued problems.
@@ -94,34 +66,9 @@ Version 4.2.1 (development)
 - Implemented an adaptive linear solver tolerance option for NewtonSolver based
  on the algorithm of Eisenstat and Walker.

- Added support for nonscalar coefficient with VectorDiffusionIntegrator.
-
 - Extending support for L2 basis functions using MapTypes VALUE and INTEGRAL in
  linear interpolators and GridFunction "GetValue" methods.

- Variable order spaces, p- and hp-refinement. This is the initial (serial)
-  support for variable-order FiniteElementCollection and FiniteElementSpace.
-  The new method FiniteElementSpace::SetElementOrder can be called to set an
-  arbitrary order for each mesh element. The conforming interpolation matrix
-  will now automatically constrain p- and hp- interfaces, enabling general
-  hp-refinement in both 2D and 3D, on uniform or mixed NC meshes. Support for
-  parallel variable-order spaces will follow shortly.
-
- Added support for creating refined meshes for all element types (e.g. by
-  splitting high-order elements into low-order refined elements), including
-  mixed meshes. The LOR Transfer miniapp (miniapps/tools/lor-transfer.cpp) now
-  supports meshes with any element geometry.
-
- Gitlab CI: use Spack (and Uberenv) to automate the build of TPLs.
-
-libCEED integration improvements
--------------------------------
- Refactor the libCEED integration
-
- Add support for VectorCoefficient with libCEED backends.
-
- Add support for ConvectionIntegrator, and VectorConvectionNLFIntegrator with libCEED backends.
-

 Version 4.2, released on October 30, 2020
 =========================================
@@ -994,7 +941,7 @@ New and updated examples and miniapps

 - Added a new meshing miniapp, Shaper, that can be used to resolve complicated
  material interfaces by mesh refinement, e.g. as a tool for initial mesh
-  generation from prescribed "material()" function. Both conforming and
+  generation from prescribed "material()" function.  Both conforming and
  non-conforming (isotropic and anisotropic) refinements are supported.

 - Added a new meshing miniapp, Mesh Optimizer, that demonstrates the use of TMOP
@@ -1006,7 +953,7 @@ Discretization improvements
 ---------------------------
 - Added a FindPoints method of the Mesh and ParMesh classes that returns the
  elements that contain a given set of points, together with the coordinates of
-  the points in the reference space of the corresponding element. In parallel,
+  the points in the reference space of the corresponding element.  In parallel,
  if a point is shared by multiple processors, only one of them will mark that
  point as found. Note that the current implementation of this method is not
  optimal and/or 100% reliable. See the mesh-explorer miniapp for an example.
@@ -1333,7 +1280,7 @@ New and improved linear solvers
  which is a sparse direct solver for distributed memory architectures. As such
  it can only be enabled along with MFEM_USE_MPI. When MFEM is configured with
  MFEM_USE_SUPERLU, one also needs to alter the version of METIS, since SuperLU
-  requires ParMETIS (which comes packaged with a serial version of METIS). See
+  requires ParMETIS (which comes packaged with a serial version of METIS).  See
  http://crd-legacy.lbl.gov/~xiaoye/SuperLU for SuperLU_DIST details.

 - Added a wrapper for the KLU solver in SuiteSparse see
@@ -1744,7 +1691,7 @@ New and updated examples
  (ADS) in hypre.

 - Modified Example 1 to use isoparametric discretization (use the FE space from
-  the mesh) including NURBS meshes and spaces. Updated Example 2 to support
+  the mesh) including NURBS meshes and spaces.  Updated Example 2 to support
  arbitrary order spaces. Updated all examples to work with NURBS meshes and
  spaces, as well as to not use projection onto discontinuous polynomial spaces
  for visualization (this is now handled directly in GLVis when necessary).
@@ -1836,7 +1783,7 @@ Version 1.1, released on Sep 13, 2010
 New MFEM format for general meshes
 ----------------------------------
 - New MFEM mesh v1.0 format with uniform structure for any dimension and support
-  for curved meshes including in 3D. Class Mesh will recognize and read the new
+  for curved meshes including in 3D.  Class Mesh will recognize and read the new
  format (in addition to all previously used formats) and Mesh::Print uses the
  new format by default. The old print function was renamed to Mesh::PrintXG.

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -88,9 +88,6 @@ include("${CMAKE_CURRENT_SOURCE_DIR}/config/XSDKDefaults.cmake")
 # Enable languages.
 enable_language(CXX)
 if (MFEM_USE_CUDA)
-   if (MFEM_USE_HIP)
-      message(FATAL_ERROR " *** MFEM_USE_HIP cannot be combined with MFEM_USE_CUDA.")
-   endif()
   # MFEM_USE_CUDA requires CMake 3.8 or newer (for direct CUDA support)
   cmake_minimum_required(VERSION 3.8 FATAL_ERROR)
   # Use ${CMAKE_CXX_COMPILER} as the cuda host compiler.
@@ -360,15 +357,6 @@ if (MFEM_USE_UMPIRE)
   find_package(UMPIRE REQUIRED)
 endif()

-# AMD HIP
-if (MFEM_USE_HIP)
-  find_package(HIP REQUIRED)
-  if (HIP_ARCH)
-    message(STATUS "Using HIP architecture: ${HIP_ARCH}")
-    list(APPEND HIP_HIPCC_FLAGS "--amdgpu-target=${HIP_ARCH}")
-  endif()
-endif()
-
 # ADIOS2 for parallel I/O
 if (MFEM_USE_ADIOS2)
   find_package(ADIOS2 REQUIRED)
@@ -444,9 +432,9 @@ foreach(DIR IN LISTS MFEM_SOURCE_DIRS)
 endforeach()

 if (MFEM_USE_CUDA)
-  set_source_files_properties(${SOURCES} PROPERTIES LANGUAGE CUDA)
-elseif(MFEM_USE_HIP)
-  set_source_files_properties(${SOURCES} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
+   foreach(file IN LISTS SOURCES)
+      set_property(SOURCE ${file} PROPERTY LANGUAGE CUDA)
+   endforeach()
 endif()

 add_subdirectory(config)
@@ -465,7 +453,7 @@ set(MFEM_INSTALL_DIR ${CMAKE_INSTALL_PREFIX} CACHE PATH
    "The MFEM install directory" FORCE)

 # Declaring the library
-mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
+add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
 # message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
 if (CMAKE_VERSION VERSION_GREATER 2.8.11)
  target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES})
@@ -622,7 +610,7 @@ endif()
 if (MFEM_USE_CEED)
  install(DIRECTORY ${MFEM_SOURCE_DIRS}
    DESTINATION ${INSTALL_INCLUDE_DIR}/mfem
-    FILES_MATCHING PATTERN "fem/ceed/*.h")
+    FILES_MATCHING PATTERN "fem/libceed/*.h")
 endif()

 # Install ${HEADERS}
@@ -104,7 +104,7 @@ The MFEM source code has the following structure:
  │   ├── sundials
  |   └── superlu
  ├── fem
-  │   └── ceed
+  │   └── libceed
  ├── general
  ├── linalg
  │   └── simd
@@ -118,13 +118,10 @@ The MFEM source code has the following structure:
  │   ├── navier
  │   ├── nurbs
  │   ├── performance
-  │   ├── shifted
-  │   ├── solvers
  │   ├── tools
  │   └── toys
  └── tests
      ├── convergence
-      ├── gitlab
      ├── par-mesh-format
      ├── scripts
      └── unit
@@ -564,9 +561,7 @@ MFEM uses a `master`/`next`-branch workflow as described below:
  Computing systems. Gitlab pipeline status is reported in the corresponding
  GitHub pull request.

- In Gitlab pipelines, TPLs (dependencies) are built using Spack, driven by Uberenv.
-
- No change to the MFEM repo can be made on this instance.
+- No change can be made on this instance.

 ## Automated Testing

@@ -692,7 +692,7 @@ The specific libraries and their options are:
 - CUDA (optional), used when MFEM_USE_CUDA = YES.
  URL: https://developer.nvidia.com/cuda-toolkit
  Options: CUDA_CXX, CUDA_ARCH, CUDA_OPT, CUDA_LIB.
-  Versions: CUDA >= 10.1.168.
+  Versions: CUDA >= 9.1, older versions may work too.

 - HIP (optional), used when MFEM_USE_HIP = YES.
  URL: https://rocm.github.io/ROCmInstall.html
@@ -1,6 +1,6 @@
 BSD 3-Clause License

-Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC
+Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC
 All rights reserved.

 Redistribution and use in source and binary forms, with or without
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -132,10 +132,6 @@
 // Requires a CUDA compiler (nvcc).
 #cmakedefine MFEM_USE_CUDA

-// Build the HIP-enabled version of the MFEM library.
-// Requires a HIP compiler (hipcc).
-#cmakedefine MFEM_USE_HIP
-
 // Enable MFEM functionality based on the RAJA library
 #cmakedefine MFEM_USE_RAJA

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,692 +0,0 @@
-###############################################################################
-# FindHIP.cmake
-###############################################################################
-include(CheckCXXCompilerFlag)
-###############################################################################
-# SET: Variable defaults
-###############################################################################
-# User defined flags
-set(HIP_HIPCC_FLAGS "" CACHE STRING "Semicolon delimited flags for HIPCC")
-set(HIP_HCC_FLAGS "" CACHE STRING "Semicolon delimited flags for HCC")
-set(HIP_CLANG_FLAGS "" CACHE STRING "Semicolon delimited flags for CLANG")
-set(HIP_NVCC_FLAGS "" CACHE STRING "Semicolon delimted flags for NVCC")
-mark_as_advanced(HIP_HIPCC_FLAGS HIP_HCC_FLAGS HIP_CLANG_FLAGS HIP_NVCC_FLAGS)
-
-set(_hip_configuration_types ${CMAKE_CONFIGURATION_TYPES} ${CMAKE_BUILD_TYPE} Debug MinSizeRel Release RelWithDebInfo)
-list(REMOVE_DUPLICATES _hip_configuration_types)
-foreach(config ${_hip_configuration_types})
-    string(TOUPPER ${config} config_upper)
-    set(HIP_HIPCC_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for HIPCC")
-    set(HIP_HCC_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for HCC")
-    set(HIP_CLANG_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for CLANG")
-    set(HIP_NVCC_FLAGS_${config_upper} "" CACHE STRING "Semicolon delimited flags for NVCC")
-    mark_as_advanced(HIP_HIPCC_FLAGS_${config_upper} HIP_HCC_FLAGS_${config_upper} HIP_CLANG_FLAGS_${config_upper} HIP_NVCC_FLAGS_${config_upper})
-endforeach()
-option(HIP_HOST_COMPILATION_CPP "Host code compilation mode" ON)
-option(HIP_VERBOSE_BUILD "Print out the commands run while compiling the HIP source file.  With the Makefile generator this defaults to VERBOSE variable specified on the command line, but can be forced on with this option." OFF)
-mark_as_advanced(HIP_HOST_COMPILATION_CPP)
-
-###############################################################################
-# FIND: HIP and associated helper binaries
-###############################################################################
-
-get_filename_component(_IMPORT_PREFIX "${CMAKE_CURRENT_LIST_DIR}/../" REALPATH)
-
-# HIP is supported on Linux only
-if(UNIX AND NOT APPLE AND NOT CYGWIN)
-    # Search for HIP installation
-    if(NOT HIP_ROOT_DIR)
-        # Search in user specified path first
-        find_path(
-            HIP_ROOT_DIR
-            NAMES bin/hipconfig
-            PATHS
-            "$ENV{ROCM_PATH}/hip"
-            ENV HIP_PATH
-            ${_IMPORT_PREFIX}
-            /opt/rocm/hip
-            DOC "HIP installed location"
-            NO_DEFAULT_PATH
-            )
-        if(NOT EXISTS ${HIP_ROOT_DIR})
-            if(HIP_FIND_REQUIRED)
-                message(FATAL_ERROR "Specify HIP_ROOT_DIR")
-            elseif(NOT HIP_FIND_QUIETLY)
-                message("HIP_ROOT_DIR not found or specified")
-            endif()
-        endif()
-        # And push it back to the cache
-        set(HIP_ROOT_DIR ${HIP_ROOT_DIR} CACHE PATH "HIP installed location" FORCE)
-    endif()
-
-    # Find HIPCC executable
-    find_program(
-        HIP_HIPCC_EXECUTABLE
-        NAMES hipcc
-        PATHS
-        "${HIP_ROOT_DIR}"
-        ENV ROCM_PATH
-        ENV HIP_PATH
-        /opt/rocm
-        /opt/rocm/hip
-        PATH_SUFFIXES bin
-        NO_DEFAULT_PATH
-        )
-    if(NOT HIP_HIPCC_EXECUTABLE)
-        # Now search in default paths
-        find_program(HIP_HIPCC_EXECUTABLE hipcc)
-    endif()
-    mark_as_advanced(HIP_HIPCC_EXECUTABLE)
-
-    # Find HIPCONFIG executable
-    find_program(
-        HIP_HIPCONFIG_EXECUTABLE
-        NAMES hipconfig
-        PATHS
-        "${HIP_ROOT_DIR}"
-        ENV ROCM_PATH
-        ENV HIP_PATH
-        /opt/rocm
-        /opt/rocm/hip
-        PATH_SUFFIXES bin
-        NO_DEFAULT_PATH
-        )
-    if(NOT HIP_HIPCONFIG_EXECUTABLE)
-        # Now search in default paths
-        find_program(HIP_HIPCONFIG_EXECUTABLE hipconfig)
-    endif()
-    mark_as_advanced(HIP_HIPCONFIG_EXECUTABLE)
-
-    # Find HIPCC_CMAKE_LINKER_HELPER executable
-    find_program(
-        HIP_HIPCC_CMAKE_LINKER_HELPER
-        NAMES hipcc_cmake_linker_helper
-        PATHS
-        "${HIP_ROOT_DIR}"
-        ENV ROCM_PATH
-        ENV HIP_PATH
-        /opt/rocm
-        /opt/rocm/hip
-        PATH_SUFFIXES bin
-        NO_DEFAULT_PATH
-        )
-    if(NOT HIP_HIPCC_CMAKE_LINKER_HELPER)
-        # Now search in default paths
-        find_program(HIP_HIPCC_CMAKE_LINKER_HELPER hipcc_cmake_linker_helper)
-    endif()
-    mark_as_advanced(HIP_HIPCC_CMAKE_LINKER_HELPER)
-
-    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_VERSION)
-        # Compute the version
-        execute_process(
-            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --version
-            OUTPUT_VARIABLE _hip_version
-            ERROR_VARIABLE _hip_error
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            ERROR_STRIP_TRAILING_WHITESPACE
-            )
-        if(NOT _hip_error)
-            set(HIP_VERSION ${_hip_version} CACHE STRING "Version of HIP as computed from hipcc")
-        else()
-            set(HIP_VERSION "0.0.0" CACHE STRING "Version of HIP as computed by FindHIP()")
-        endif()
-        mark_as_advanced(HIP_VERSION)
-    endif()
-    if(HIP_VERSION)
-        string(REPLACE "." ";" _hip_version_list "${HIP_VERSION}")
-        list(GET _hip_version_list 0 HIP_VERSION_MAJOR)
-        list(GET _hip_version_list 1 HIP_VERSION_MINOR)
-        list(GET _hip_version_list 2 HIP_VERSION_PATCH)
-        set(HIP_VERSION_STRING "${HIP_VERSION}")
-    endif()
-
-    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_PLATFORM)
-        # Compute the platform
-        execute_process(
-            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --platform
-            OUTPUT_VARIABLE _hip_platform
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            )
-        set(HIP_PLATFORM ${_hip_platform} CACHE STRING "HIP platform as computed by hipconfig")
-        mark_as_advanced(HIP_PLATFORM)
-    endif()
-
-    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_COMPILER)
-        # Compute the compiler
-        execute_process(
-            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --compiler
-            OUTPUT_VARIABLE _hip_compiler
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            )
-        set(HIP_COMPILER ${_hip_compiler} CACHE STRING "HIP compiler as computed by hipconfig")
-        mark_as_advanced(HIP_COMPILER)
-    endif()
-
-    if(HIP_HIPCONFIG_EXECUTABLE AND NOT HIP_RUNTIME)
-        # Compute the runtime
-        execute_process(
-            COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --runtime
-            OUTPUT_VARIABLE _hip_runtime
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            )
-        set(HIP_RUNTIME ${_hip_runtime} CACHE STRING "HIP runtime as computed by hipconfig")
-        mark_as_advanced(HIP_RUNTIME)
-    endif()
-endif()
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(
-    HIP
-    REQUIRED_VARS
-    HIP_ROOT_DIR
-    HIP_HIPCC_EXECUTABLE
-    HIP_HIPCONFIG_EXECUTABLE
-    HIP_PLATFORM
-    HIP_COMPILER
-    HIP_RUNTIME
-    VERSION_VAR HIP_VERSION
-    )
-
-###############################################################################
-# Set HIP CMAKE Flags
-###############################################################################
-# Copy the invocation styles from CXX to HIP
-set(CMAKE_HIP_ARCHIVE_CREATE ${CMAKE_CXX_ARCHIVE_CREATE})
-set(CMAKE_HIP_ARCHIVE_APPEND ${CMAKE_CXX_ARCHIVE_APPEND})
-set(CMAKE_HIP_ARCHIVE_FINISH ${CMAKE_CXX_ARCHIVE_FINISH})
-set(CMAKE_SHARED_LIBRARY_SONAME_HIP_FLAG ${CMAKE_SHARED_LIBRARY_SONAME_CXX_FLAG})
-set(CMAKE_SHARED_LIBRARY_CREATE_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS})
-set(CMAKE_SHARED_LIBRARY_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_CXX_FLAGS})
-#set(CMAKE_SHARED_LIBRARY_LINK_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS})
-set(CMAKE_SHARED_LIBRARY_RUNTIME_HIP_FLAG ${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG})
-set(CMAKE_SHARED_LIBRARY_RUNTIME_HIP_FLAG_SEP ${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG_SEP})
-set(CMAKE_SHARED_LIBRARY_LINK_STATIC_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_STATIC_CXX_FLAGS})
-set(CMAKE_SHARED_LIBRARY_LINK_DYNAMIC_HIP_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_DYNAMIC_CXX_FLAGS})
-
-set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "")
-set(HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS "")
-
-if("${HIP_COMPILER}" STREQUAL "nvcc")
-    # Set the CMake Flags to use the nvcc Compiler.
-    set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
-    set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <LINK_LIBRARIES> -shared" )
-    set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-elseif("${HIP_COMPILER}" STREQUAL "hcc")
-    # Set the CMake Flags to use the hcc Compiler.
-    set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
-    set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <LINK_LIBRARIES> -shared" )
-    set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-elseif("${HIP_COMPILER}" STREQUAL "clang")
-    #Number of parallel jobs by default is 1
-    if(NOT DEFINED HIP_CLANG_NUM_PARALLEL_JOBS)
-      set(HIP_CLANG_NUM_PARALLEL_JOBS 1)
-    endif()
-    #Add support for parallel build and link
-    if(${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
-      check_cxx_compiler_flag("-parallel-jobs=1" HIP_CLANG_SUPPORTS_PARALLEL_JOBS)
-    endif()
-    if(HIP_CLANG_NUM_PARALLEL_JOBS GREATER 1)
-      if(${HIP_CLANG_SUPPORTS_PARALLEL_JOBS})
-        set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "-Wno-format-nonliteral -parallel-jobs=${HIP_CLANG_NUM_PARALLEL_JOBS}")
-        set(HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS "-parallel-jobs=${HIP_CLANG_NUM_PARALLEL_JOBS}")
-      else()
-        message("clang compiler doesn't support parallel jobs")
-      endif()
-    endif()
-
-    # Set the CMake Flags to use the HIP-Clang Compiler.
-    set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
-    set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <LINK_LIBRARIES> -shared" )
-    set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-
-    if("${HIP_RUNTIME}" STREQUAL "rocclr")
-      if(TARGET host)
-        message(STATUS "host interface - found")
-        set(HIP_HOST_INTERFACE host)
-      endif()
-    endif()
-endif()
-
-###############################################################################
-# MACRO: Locate helper files
-###############################################################################
-macro(HIP_FIND_HELPER_FILE _name _extension)
-    set(_hip_full_name "${_name}.${_extension}")
-    get_filename_component(CMAKE_CURRENT_LIST_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH)
-    set(HIP_${_name} "${CMAKE_CURRENT_LIST_DIR}/FindHIP/${_hip_full_name}")
-    if(NOT EXISTS "${HIP_${_name}}")
-        set(error_message "${_hip_full_name} not found in ${CMAKE_CURRENT_LIST_DIR}/FindHIP")
-        if(HIP_FIND_REQUIRED)
-            message(FATAL_ERROR "${error_message}")
-        else()
-            if(NOT HIP_FIND_QUIETLY)
-                message(STATUS "${error_message}")
-            endif()
-        endif()
-    endif()
-    # Set this variable as internal, so the user isn't bugged with it.
-    set(HIP_${_name} ${HIP_${_name}} CACHE INTERNAL "Location of ${_full_name}" FORCE)
-endmacro()
-
-###############################################################################
-hip_find_helper_file(run_make2cmake cmake)
-hip_find_helper_file(run_hipcc cmake)
-###############################################################################
-
-###############################################################################
-# MACRO: Reset compiler flags
-###############################################################################
-macro(HIP_RESET_FLAGS)
-    unset(HIP_HIPCC_FLAGS)
-    unset(HIP_HCC_FLAGS)
-    unset(HIP_CLANG_FLAGS)
-    unset(HIP_NVCC_FLAGS)
-    foreach(config ${_hip_configuration_types})
-        string(TOUPPER ${config} config_upper)
-        unset(HIP_HIPCC_FLAGS_${config_upper})
-        unset(HIP_HCC_FLAGS_${config_upper})
-        unset(HIP_CLANG_FLAGS_${config_upper})
-        unset(HIP_NVCC_FLAGS_${config_upper})
-    endforeach()
-endmacro()
-
-###############################################################################
-# MACRO: Separate the options from the sources
-###############################################################################
-macro(HIP_GET_SOURCES_AND_OPTIONS _sources _cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options)
-    set(${_sources})
-    set(${_cmake_options})
-    set(${_hipcc_options})
-    set(${_hcc_options})
-    set(${_clang_options})
-    set(${_nvcc_options})
-    set(_hipcc_found_options FALSE)
-    set(_hcc_found_options FALSE)
-    set(_clang_found_options FALSE)
-    set(_nvcc_found_options FALSE)
-    foreach(arg ${ARGN})
-        if("x${arg}" STREQUAL "xHIPCC_OPTIONS")
-            set(_hipcc_found_options TRUE)
-            set(_hcc_found_options FALSE)
-            set(_clang_found_options FALSE)
-            set(_nvcc_found_options FALSE)
-        elseif("x${arg}" STREQUAL "xHCC_OPTIONS")
-            set(_hipcc_found_options FALSE)
-            set(_hcc_found_options TRUE)
-            set(_clang_found_options FALSE)
-            set(_nvcc_found_options FALSE)
-        elseif("x${arg}" STREQUAL "xCLANG_OPTIONS")
-            set(_hipcc_found_options FALSE)
-            set(_hcc_found_options FALSE)
-            set(_clang_found_options TRUE)
-            set(_nvcc_found_options FALSE)
-        elseif("x${arg}" STREQUAL "xNVCC_OPTIONS")
-            set(_hipcc_found_options FALSE)
-            set(_hcc_found_options FALSE)
-            set(_clang_found_options FALSE)
-            set(_nvcc_found_options TRUE)
-        elseif(
-                "x${arg}" STREQUAL "xEXCLUDE_FROM_ALL" OR
-                "x${arg}" STREQUAL "xSTATIC" OR
-                "x${arg}" STREQUAL "xSHARED" OR
-                "x${arg}" STREQUAL "xMODULE"
-                )
-            list(APPEND ${_cmake_options} ${arg})
-        else()
-            if(_hipcc_found_options)
-                list(APPEND ${_hipcc_options} ${arg})
-            elseif(_hcc_found_options)
-                list(APPEND ${_hcc_options} ${arg})
-            elseif(_clang_found_options)
-                list(APPEND ${_clang_options} ${arg})
-            elseif(_nvcc_found_options)
-                list(APPEND ${_nvcc_options} ${arg})
-            else()
-                # Assume this is a file
-                list(APPEND ${_sources} ${arg})
-            endif()
-        endif()
-    endforeach()
-endmacro()
-
-###############################################################################
-# MACRO: Add include directories to pass to the hipcc command
-###############################################################################
-set(HIP_HIPCC_INCLUDE_ARGS_USER "")
-macro(HIP_INCLUDE_DIRECTORIES)
-    foreach(dir ${ARGN})
-        list(APPEND HIP_HIPCC_INCLUDE_ARGS_USER $<$<BOOL:${dir}>:-I${dir}>)
-    endforeach()
-endmacro()
-
-###############################################################################
-# FUNCTION: Helper to avoid clashes of files with the same basename but different paths
-###############################################################################
-function(HIP_COMPUTE_BUILD_PATH path build_path)
-    # Convert to cmake style paths
-    file(TO_CMAKE_PATH "${path}" bpath)
-    if(IS_ABSOLUTE "${bpath}")
-        string(FIND "${bpath}" "${CMAKE_CURRENT_BINARY_DIR}" _binary_dir_pos)
-        if(_binary_dir_pos EQUAL 0)
-            file(RELATIVE_PATH bpath "${CMAKE_CURRENT_BINARY_DIR}" "${bpath}")
-        else()
-            file(RELATIVE_PATH bpath "${CMAKE_CURRENT_SOURCE_DIR}" "${bpath}")
-        endif()
-    endif()
-
-    # Remove leading /
-    string(REGEX REPLACE "^[/]+" "" bpath "${bpath}")
-    # Avoid absolute paths by removing ':'
-    string(REPLACE ":" "_" bpath "${bpath}")
-    # Avoid relative paths that go up the tree
-    string(REPLACE "../" "__/" bpath "${bpath}")
-    # Avoid spaces
-    string(REPLACE " " "_" bpath "${bpath}")
-    # Strip off the filename
-    get_filename_component(bpath "${bpath}" PATH)
-
-    set(${build_path} "${bpath}" PARENT_SCOPE)
-endfunction()
-
-###############################################################################
-# MACRO: Parse OPTIONS from ARGN & set variables prefixed by _option_prefix
-###############################################################################
-macro(HIP_PARSE_HIPCC_OPTIONS _option_prefix)
-    set(_hip_found_config)
-    foreach(arg ${ARGN})
-        # Determine if we are dealing with a per-configuration flag
-        foreach(config ${_hip_configuration_types})
-            string(TOUPPER ${config} config_upper)
-            if(arg STREQUAL "${config_upper}")
-                set(_hip_found_config _${arg})
-                # Clear arg to prevent it from being processed anymore
-                set(arg)
-            endif()
-        endforeach()
-        if(arg)
-            list(APPEND ${_option_prefix}${_hip_found_config} "${arg}")
-        endif()
-    endforeach()
-endmacro()
-
-###############################################################################
-# MACRO: Try and include dependency file if it exists
-###############################################################################
-macro(HIP_INCLUDE_HIPCC_DEPENDENCIES dependency_file)
-    set(HIP_HIPCC_DEPEND)
-    set(HIP_HIPCC_DEPEND_REGENERATE FALSE)
-
-    # Create the dependency file if it doesn't exist
-    if(NOT EXISTS ${dependency_file})
-        file(WRITE ${dependency_file} "# Generated by: FindHIP.cmake. Do not edit.\n")
-    endif()
-    # Include the dependency file
-    include(${dependency_file})
-
-    # Verify the existence of all the included files
-    if(HIP_HIPCC_DEPEND)
-        foreach(f ${HIP_HIPCC_DEPEND})
-            if(NOT EXISTS ${f})
-                # If they aren't there, regenerate the file again
-                set(HIP_HIPCC_DEPEND_REGENERATE TRUE)
-            endif()
-        endforeach()
-    else()
-        # No dependencies, so regenerate the file
-        set(HIP_HIPCC_DEPEND_REGENERATE TRUE)
-    endif()
-
-    # Regenerate the dependency file if needed
-    if(HIP_HIPCC_DEPEND_REGENERATE)
-        set(HIP_HIPCC_DEPEND ${dependency_file})
-        file(WRITE ${dependency_file} "# Generated by: FindHIP.cmake. Do not edit.\n")
-    endif()
-endmacro()
-
-###############################################################################
-# MACRO: Prepare cmake commands for the target
-###############################################################################
-macro(HIP_PREPARE_TARGET_COMMANDS _target _format _generated_files _source_files)
-    set(_hip_flags "")
-    string(TOUPPER "${CMAKE_BUILD_TYPE}" _hip_build_configuration)
-    if(HIP_HOST_COMPILATION_CPP)
-        set(HIP_C_OR_CXX CXX)
-    else()
-        set(HIP_C_OR_CXX C)
-    endif()
-    set(generated_extension ${CMAKE_${HIP_C_OR_CXX}_OUTPUT_EXTENSION})
-
-    # Initialize list of includes with those specified by the user. Append with
-    # ones specified to cmake directly.
-    set(HIP_HIPCC_INCLUDE_ARGS ${HIP_HIPCC_INCLUDE_ARGS_USER})
-
-    # Add the include directories
-    set(include_directories_generator "$<TARGET_PROPERTY:${_target},INCLUDE_DIRECTORIES>")
-    list(APPEND HIP_HIPCC_INCLUDE_ARGS "$<$<BOOL:${include_directories_generator}>:-I$<JOIN:${include_directories_generator}, -I>>")
-
-    get_directory_property(_hip_include_directories INCLUDE_DIRECTORIES)
-    list(REMOVE_DUPLICATES _hip_include_directories)
-    if(_hip_include_directories)
-        foreach(dir ${_hip_include_directories})
-            list(APPEND HIP_HIPCC_INCLUDE_ARGS $<$<BOOL:${dir}>:-I${dir}>)
-        endforeach()
-    endif()
-
-    HIP_GET_SOURCES_AND_OPTIONS(_hip_sources _hip_cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options ${ARGN})
-    HIP_PARSE_HIPCC_OPTIONS(HIP_HIPCC_FLAGS ${_hipcc_options})
-    HIP_PARSE_HIPCC_OPTIONS(HIP_HCC_FLAGS ${_hcc_options})
-    HIP_PARSE_HIPCC_OPTIONS(HIP_CLANG_FLAGS ${_clang_options})
-    HIP_PARSE_HIPCC_OPTIONS(HIP_NVCC_FLAGS ${_nvcc_options})
-
-    # Add the compile definitions
-    set(compile_definition_generator "$<TARGET_PROPERTY:${_target},COMPILE_DEFINITIONS>")
-    list(APPEND HIP_HIPCC_FLAGS "$<$<BOOL:${compile_definition_generator}>:-D$<JOIN:${compile_definition_generator}, -D>>")
-
-    # Check if we are building shared library.
-    set(_hip_build_shared_libs FALSE)
-    list(FIND _hip_cmake_options SHARED _hip_found_SHARED)
-    list(FIND _hip_cmake_options MODULE _hip_found_MODULE)
-    if(_hip_found_SHARED GREATER -1 OR _hip_found_MODULE GREATER -1)
-        set(_hip_build_shared_libs TRUE)
-    endif()
-    list(FIND _hip_cmake_options STATIC _hip_found_STATIC)
-    if(_hip_found_STATIC GREATER -1)
-        set(_hip_build_shared_libs FALSE)
-    endif()
-
-    # If we are building a shared library, add extra flags to HIP_HIPCC_FLAGS
-    if(_hip_build_shared_libs)
-        list(APPEND HIP_HCC_FLAGS "-fPIC")
-        list(APPEND HIP_CLANG_FLAGS "-fPIC")
-        list(APPEND HIP_NVCC_FLAGS "--shared -Xcompiler '-fPIC'")
-    endif()
-
-    # Set host compiler
-    set(HIP_HOST_COMPILER "${CMAKE_${HIP_C_OR_CXX}_COMPILER}")
-
-    # Set compiler flags
-    set(_HIP_HOST_FLAGS "set(CMAKE_HOST_FLAGS ${CMAKE_${HIP_C_OR_CXX}_FLAGS})")
-    set(_HIP_HIPCC_FLAGS "set(HIP_HIPCC_FLAGS ${HIP_HIPCC_FLAGS})")
-    set(_HIP_HCC_FLAGS "set(HIP_HCC_FLAGS ${HIP_HCC_FLAGS})")
-    set(_HIP_CLANG_FLAGS "set(HIP_CLANG_FLAGS ${HIP_CLANG_FLAGS})")
-    set(_HIP_NVCC_FLAGS "set(HIP_NVCC_FLAGS ${HIP_NVCC_FLAGS})")
-    foreach(config ${_hip_configuration_types})
-        string(TOUPPER ${config} config_upper)
-        set(_HIP_HOST_FLAGS "${_HIP_HOST_FLAGS}\nset(CMAKE_HOST_FLAGS_${config_upper} ${CMAKE_${HIP_C_OR_CXX}_FLAGS_${config_upper}})")
-        set(_HIP_HIPCC_FLAGS "${_HIP_HIPCC_FLAGS}\nset(HIP_HIPCC_FLAGS_${config_upper} ${HIP_HIPCC_FLAGS_${config_upper}})")
-        set(_HIP_HCC_FLAGS "${_HIP_HCC_FLAGS}\nset(HIP_HCC_FLAGS_${config_upper} ${HIP_HCC_FLAGS_${config_upper}})")
-        set(_HIP_CLANG_FLAGS "${_HIP_CLANG_FLAGS}\nset(HIP_CLANG_FLAGS_${config_upper} ${HIP_CLANG_FLAGS_${config_upper}})")
-        set(_HIP_NVCC_FLAGS "${_HIP_NVCC_FLAGS}\nset(HIP_NVCC_FLAGS_${config_upper} ${HIP_NVCC_FLAGS_${config_upper}})")
-    endforeach()
-
-    # Reset the output variable
-    set(_hip_generated_files "")
-    set(_hip_source_files "")
-
-    # Iterate over all arguments and create custom commands for all source files
-    foreach(file ${ARGN})
-        # Ignore any file marked as a HEADER_FILE_ONLY
-        get_source_file_property(_is_header ${file} HEADER_FILE_ONLY)
-        # Allow per source file overrides of the format. Also allows compiling non .cu files.
-        get_source_file_property(_hip_source_format ${file} HIP_SOURCE_PROPERTY_FORMAT)
-        if((${file} MATCHES "\\.cu$" OR _hip_source_format) AND NOT _is_header)
-            set(host_flag FALSE)
-        else()
-            set(host_flag TRUE)
-        endif()
-
-        if(NOT host_flag)
-            # Determine output directory
-            HIP_COMPUTE_BUILD_PATH("${file}" hip_build_path)
-            set(hip_compile_output_dir "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${_target}.dir/${hip_build_path}")
-
-            get_filename_component(basename ${file} NAME)
-            set(generated_file_path "${hip_compile_output_dir}/${CMAKE_CFG_INTDIR}")
-            set(generated_file_basename "${_target}_generated_${basename}${generated_extension}")
-
-            # Set file names
-            set(generated_file "${generated_file_path}/${generated_file_basename}")
-            set(cmake_dependency_file "${hip_compile_output_dir}/${generated_file_basename}.depend")
-            set(custom_target_script_pregen "${hip_compile_output_dir}/${generated_file_basename}.cmake.pre-gen")
-            set(custom_target_script "${hip_compile_output_dir}/${generated_file_basename}.cmake")
-
-            # Set properties for object files
-            set_source_files_properties("${generated_file}"
-                PROPERTIES
-                EXTERNAL_OBJECT true # This is an object file not to be compiled, but only be linked
-                )
-
-            # Don't add CMAKE_CURRENT_SOURCE_DIR if the path is already an absolute path
-            get_filename_component(file_path "${file}" PATH)
-            if(IS_ABSOLUTE "${file_path}")
-                set(source_file "${file}")
-            else()
-                set(source_file "${CMAKE_CURRENT_SOURCE_DIR}/${file}")
-            endif()
-
-            # Bring in the dependencies
-            HIP_INCLUDE_HIPCC_DEPENDENCIES(${cmake_dependency_file})
-
-            # Configure the build script
-            configure_file("${HIP_run_hipcc}" "${custom_target_script_pregen}" @ONLY)
-            file(GENERATE
-                OUTPUT "${custom_target_script}"
-                INPUT "${custom_target_script_pregen}"
-                )
-            set(main_dep DEPENDS ${source_file})
-            if(CMAKE_GENERATOR MATCHES "Makefiles")
-                set(verbose_output "$(VERBOSE)")
-            elseif(HIP_VERBOSE_BUILD)
-                set(verbose_output ON)
-            else()
-                set(verbose_output OFF)
-            endif()
-
-            # Create up the comment string
-            file(RELATIVE_PATH generated_file_relative_path "${CMAKE_BINARY_DIR}" "${generated_file}")
-            set(hip_build_comment_string "Building HIPCC object ${generated_file_relative_path}")
-
-            # Build the generated file and dependency file
-            add_custom_command(
-                OUTPUT ${generated_file}
-                # These output files depend on the source_file and the contents of cmake_dependency_file
-                ${main_dep}
-                DEPENDS ${HIP_HIPCC_DEPEND}
-                DEPENDS ${custom_target_script}
-                # Make sure the output directory exists before trying to write to it.
-                COMMAND ${CMAKE_COMMAND} -E make_directory "${generated_file_path}"
-                COMMAND ${CMAKE_COMMAND} ARGS
-                -D verbose:BOOL=${verbose_output}
-                -D build_configuration:STRING=${_hip_build_configuration}
-                -D "generated_file:STRING=${generated_file}"
-                -P "${custom_target_script}"
-                WORKING_DIRECTORY "${hip_compile_output_dir}"
-                COMMENT "${hip_build_comment_string}"
-                )
-
-            # Make sure the build system knows the file is generated
-            set_source_files_properties(${generated_file} PROPERTIES GENERATED TRUE)
-            list(APPEND _hip_generated_files ${generated_file})
-            list(APPEND _hip_source_files ${file})
-        endif()
-    endforeach()
-
-    # Set the return parameter
-    set(${_generated_files} ${_hip_generated_files})
-    set(${_source_files} ${_hip_source_files})
-endmacro()
-
-###############################################################################
-# HIP_ADD_EXECUTABLE
-###############################################################################
-macro(HIP_ADD_EXECUTABLE hip_target)
-    # Separate the sources from the options
-    HIP_GET_SOURCES_AND_OPTIONS(_sources _cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options ${ARGN})
-    HIP_PREPARE_TARGET_COMMANDS(${hip_target} OBJ _generated_files _source_files ${_sources} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} CLANG_OPTIONS ${_clang_options} NVCC_OPTIONS ${_nvcc_options})
-    if(_source_files)
-        list(REMOVE_ITEM _sources ${_source_files})
-    endif()
-    if("${HIP_COMPILER}" STREQUAL "hcc")
-        if("x${HCC_HOME}" STREQUAL "x")
-            if (DEFINED ENV{ROCM_PATH})
-                set(HCC_HOME "$ENV{ROCM_PATH}/hcc")
-            elseif(DEFINED ENV{HIP_PATH})
-                set(HCC_HOME "$ENV{HIP_PATH}/../hcc")
-            else()
-                set(HCC_HOME "/opt/rocm/hcc")
-            endif()
-        endif()
-        set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-    elseif("${HIP_COMPILER}" STREQUAL "clang")
-        if("x${HIP_CLANG_PATH}" STREQUAL "x")
-            if(DEFINED ENV{HIP_CLANG_PATH})
-                set(HIP_CLANG_PATH $ENV{HIP_CLANG_PATH})
-            elseif(DEFINED ENV{ROCM_PATH})
-                set(HIP_CLANG_PATH "$ENV{ROCM_PATH}/llvm/bin")
-            elseif(DEFINED ENV{HIP_PATH})
-                set(HIP_CLANG_PATH "$ENV{HIP_PATH}/../llvm/bin")
-            else()
-                set(HIP_CLANG_PATH "/opt/rocm/llvm/bin")
-            endif()
-        endif()
-        set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-    else()
-        set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
-    endif()
-    if ("${_sources}" STREQUAL "")
-        add_executable(${hip_target} ${_cmake_options} ${_generated_files} "")
-    else()
-        add_executable(${hip_target} ${_cmake_options} ${_generated_files} ${_sources})
-    endif()
-    set_target_properties(${hip_target} PROPERTIES LINKER_LANGUAGE HIP)
-    # Link with host
-    if (HIP_HOST_INTERFACE)
-        # hip rt should be rocclr, compiler should be clang
-        target_link_libraries(${hip_target} ${HIP_HOST_INTERFACE})
-    endif()
-endmacro()
-
-###############################################################################
-# HIP_ADD_LIBRARY
-###############################################################################
-macro(HIP_ADD_LIBRARY hip_target)
-    # Separate the sources from the options
-    HIP_GET_SOURCES_AND_OPTIONS(_sources _cmake_options _hipcc_options _hcc_options _clang_options _nvcc_options ${ARGN})
-    HIP_PREPARE_TARGET_COMMANDS(${hip_target} OBJ _generated_files _source_files ${_sources} ${_cmake_options} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} CLANG_OPTIONS ${_clang_options} NVCC_OPTIONS ${_nvcc_options})
-    if(_source_files)
-        list(REMOVE_ITEM _sources ${_source_files})
-    endif()
-    if ("${_sources}" STREQUAL "")
-        add_library(${hip_target} ${_cmake_options} ${_generated_files} "")
-    else()
-        add_library(${hip_target} ${_cmake_options} ${_generated_files} ${_sources})
-    endif()
-    set_target_properties(${hip_target} PROPERTIES LINKER_LANGUAGE ${HIP_C_OR_CXX})
-    # Link with host
-    if (HIP_HOST_INTERFACE)
-        # hip rt should be rocclr, compiler should be clang
-        target_link_libraries(${hip_target} ${HIP_HOST_INTERFACE})
-    endif()
-endmacro()
-
-# vim: ts=4:sw=4:expandtab:smartindent
@@ -1,182 +0,0 @@
-###############################################################################
-# Runs commands using HIPCC
-###############################################################################
-
-###############################################################################
-# This file runs the hipcc commands to produce the desired output file
-# along with the dependency file needed by CMake to compute dependencies.
-#
-# Input variables:
-#
-# verbose:BOOL=<>               OFF: Be as quiet as possible (default)
-#                               ON : Describe each step
-# build_configuration:STRING=<> Build configuration. Defaults to Debug.
-# generated_file:STRING=<>      File to generate. Mandatory argument.
-
-if(NOT build_configuration)
-    set(build_configuration Debug)
-endif()
-if(NOT generated_file)
-    message(FATAL_ERROR "You must specify generated_file on the command line")
-endif()
-
-# Set these up as variables to make reading the generated file easier
-set(HIP_HIPCC_EXECUTABLE "@HIP_HIPCC_EXECUTABLE@") # path
-set(HIP_HIPCONFIG_EXECUTABLE "@HIP_HIPCONFIG_EXECUTABLE@") #path
-set(HIP_HOST_COMPILER "@HIP_HOST_COMPILER@") # path
-set(CMAKE_COMMAND "@CMAKE_COMMAND@") # path
-set(HIP_run_make2cmake "@HIP_run_make2cmake@") # path
-set(HCC_HOME "@HCC_HOME@") #path
-set(HIP_CLANG_PATH "@HIP_CLANG_PATH@") #path
-set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "@HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS@")
-
-@HIP_HOST_FLAGS@
-@_HIP_HIPCC_FLAGS@
-@_HIP_HCC_FLAGS@
-@_HIP_CLANG_FLAGS@
-@_HIP_NVCC_FLAGS@
-#Needed to bring the HIP_HIPCC_INCLUDE_ARGS variable in scope
-set(HIP_HIPCC_INCLUDE_ARGS @HIP_HIPCC_INCLUDE_ARGS@) # list
-
-set(cmake_dependency_file "@cmake_dependency_file@") # path
-set(source_file "@source_file@") # path
-set(host_flag "@host_flag@") # bool
-
-# Determine compiler and compiler flags
-execute_process(COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --platform OUTPUT_VARIABLE HIP_PLATFORM OUTPUT_STRIP_TRAILING_WHITESPACE)
-execute_process(COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --compiler OUTPUT_VARIABLE HIP_COMPILER OUTPUT_STRIP_TRAILING_WHITESPACE)
-execute_process(COMMAND ${HIP_HIPCONFIG_EXECUTABLE} --runtime OUTPUT_VARIABLE HIP_RUNTIME OUTPUT_STRIP_TRAILING_WHITESPACE)
-if(NOT host_flag)
-    set(__CC ${HIP_HIPCC_EXECUTABLE})
-    if("${HIP_PLATFORM}" STREQUAL "amd")
-        if("${HIP_COMPILER}" STREQUAL "hcc")
-            if(NOT "x${HCC_HOME}" STREQUAL "x")
-                set(ENV{HCC_HOME} ${HCC_HOME})
-            endif()
-            set(__CC_FLAGS ${HIP_HIPCC_FLAGS} ${HIP_HCC_FLAGS} ${HIP_HIPCC_FLAGS_${build_configuration}} ${HIP_HCC_FLAGS_${build_configuration}})
-        elseif("${HIP_COMPILER}" STREQUAL "clang")
-            if(NOT "x${HIP_CLANG_PATH}" STREQUAL "x")
-                set(ENV{HIP_CLANG_PATH} ${HIP_CLANG_PATH})
-            endif()
-            # Temporarily include HIP_HCC_FLAGS for HIP-Clang for PyTorch builds
-            set(__CC_FLAGS ${HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS} ${HIP_HIPCC_FLAGS} ${HIP_HCC_FLAGS} ${HIP_CLANG_FLAGS} ${HIP_HIPCC_FLAGS_${build_configuration}} ${HIP_HCC_FLAGS_${build_configuration}} ${HIP_CLANG_FLAGS_${build_configuration}})
-        endif()
-    else()
-        set(__CC_FLAGS ${HIP_HIPCC_FLAGS} ${HIP_NVCC_FLAGS} ${HIP_HIPCC_FLAGS_${build_configuration}} ${HIP_NVCC_FLAGS_${build_configuration}})
-    endif()
-else()
-    set(__CC ${HIP_HOST_COMPILER})
-    set(__CC_FLAGS ${CMAKE_HOST_FLAGS} ${CMAKE_HOST_FLAGS_${build_configuration}})
-endif()
-set(__CC_INCLUDES ${HIP_HIPCC_INCLUDE_ARGS})
-
-# hip_execute_process - Executes a command with optional command echo and status message.
-#   status     - Status message to print if verbose is true
-#   command    - COMMAND argument from the usual execute_process argument structure
-#   ARGN       - Remaining arguments are the command with arguments
-#   HIP_result - Return value from running the command
-macro(hip_execute_process status command)
-    set(_command ${command})
-    if(NOT "x${_command}" STREQUAL "xCOMMAND")
-        message(FATAL_ERROR "Malformed call to hip_execute_process.  Missing COMMAND as second argument. (command = ${command})")
-    endif()
-    if(verbose)
-        execute_process(COMMAND "${CMAKE_COMMAND}" -E echo -- ${status})
-        # Build command string to print
-        set(hip_execute_process_string)
-        foreach(arg ${ARGN})
-            # Escape quotes if any
-            string(REPLACE "\"" "\\\"" arg ${arg})
-            # Surround args with spaces with quotes
-            if(arg MATCHES " ")
-                list(APPEND hip_execute_process_string "\"${arg}\"")
-            else()
-                list(APPEND hip_execute_process_string ${arg})
-            endif()
-        endforeach()
-        # Echo the command
-        execute_process(COMMAND ${CMAKE_COMMAND} -E echo ${hip_execute_process_string})
-    endif()
-    # Run the command
-    execute_process(COMMAND ${ARGN} RESULT_VARIABLE HIP_result)
-endmacro()
-
-# Delete the target file
-hip_execute_process(
-    "Removing ${generated_file}"
-    COMMAND "${CMAKE_COMMAND}" -E remove "${generated_file}"
-    )
-
-# Generate the dependency file
-hip_execute_process(
-    "Generating dependency file: ${cmake_dependency_file}.pre"
-    COMMAND "${__CC}"
-    -M
-    "${source_file}"
-    -o "${cmake_dependency_file}.pre"
-    ${__CC_FLAGS}
-    ${__CC_INCLUDES}
-    )
-
-if(HIP_result)
-    message(FATAL_ERROR "Error generating ${generated_file}")
-endif()
-
-# Generate the cmake readable dependency file to a temp file
-hip_execute_process(
-    "Generating temporary cmake readable file: ${cmake_dependency_file}.tmp"
-    COMMAND "${CMAKE_COMMAND}"
-    -D "input_file:FILEPATH=${cmake_dependency_file}.pre"
-    -D "output_file:FILEPATH=${cmake_dependency_file}.tmp"
-    -D "verbose=${verbose}"
-    -P "${HIP_run_make2cmake}"
-    )
-
-if(HIP_result)
-    message(FATAL_ERROR "Error generating ${generated_file}")
-endif()
-
-# Copy the file if it is different
-hip_execute_process(
-    "Copy if different ${cmake_dependency_file}.tmp to ${cmake_dependency_file}"
-    COMMAND "${CMAKE_COMMAND}" -E copy_if_different "${cmake_dependency_file}.tmp" "${cmake_dependency_file}"
-    )
-
-if(HIP_result)
-    message(FATAL_ERROR "Error generating ${generated_file}")
-endif()
-
-# Delete the temporary file
-hip_execute_process(
-    "Removing ${cmake_dependency_file}.tmp and ${cmake_dependency_file}.pre"
-    COMMAND "${CMAKE_COMMAND}" -E remove "${cmake_dependency_file}.tmp" "${cmake_dependency_file}.pre"
-    )
-
-if(HIP_result)
-    message(FATAL_ERROR "Error generating ${generated_file}")
-endif()
-
-# Generate the output file
-hip_execute_process(
-    "Generating ${generated_file}"
-    COMMAND "${__CC}"
-    -c
-    "${source_file}"
-    -o "${generated_file}"
-    ${__CC_FLAGS}
-    ${__CC_INCLUDES}
-    )
-
-if(HIP_result)
-    # Make sure that we delete the output file
-    hip_execute_process(
-        "Removing ${generated_file}"
-        COMMAND "${CMAKE_COMMAND}" -E remove "${generated_file}"
-        )
-    message(FATAL_ERROR "Error generating file ${generated_file}")
-else()
-    if(verbose)
-        message("Generated ${generated_file} successfully.")
-    endif()
-endif()
-# vim: ts=4:sw=4:expandtab:smartindent
@@ -1,50 +0,0 @@
-###############################################################################
-# Computes dependencies using HIPCC
-###############################################################################
-
-###############################################################################
-# This file converts dependency files generated using hipcc to a format that
-# cmake can understand.
-
-# Input variables:
-#
-# input_file:STRING=<> Dependency file to parse. Required argument
-# output_file:STRING=<> Output file to generate. Required argument
-
-if(NOT input_file OR NOT output_file)
-    message(FATAL_ERROR "You must specify input_file and output_file on the command line")
-endif()
-
-file(READ ${input_file} depend_text)
-
-if (NOT "${depend_text}" STREQUAL "")
-    string(REPLACE " /" "\n/" depend_text ${depend_text})
-    string(REGEX REPLACE "^.*:" "" depend_text ${depend_text})
-    string(REGEX REPLACE "[ \\\\]*\n" ";" depend_text ${depend_text})
-
-    set(dependency_list "")
-
-    foreach(file ${depend_text})
-        string(REGEX REPLACE "^ +" "" file ${file})
-        if(NOT EXISTS "${file}")
-            message(WARNING " Removing non-existent dependency file: ${file}")
-            set(file "")
-        endif()
-
-        if(NOT IS_DIRECTORY "${file}")
-            get_filename_component(file_absolute "${file}" ABSOLUTE)
-            list(APPEND dependency_list "${file_absolute}")
-        endif()
-    endforeach()
-endif()
-
-# Remove the duplicate entries and sort them.
-list(REMOVE_DUPLICATES dependency_list)
-list(SORT dependency_list)
-
-foreach(file ${dependency_list})
-    set(hip_hipcc_depend "${hip_hipcc_depend} \"${file}\"\n")
-endforeach()
-
-file(WRITE ${output_file} "# Generated by: FindHIP.cmake. Do not edit.\nSET(HIP_HIPCC_DEPEND\n ${hip_hipcc_depend})\n\n")
-# vim: ts=4:sw=4:expandtab:smartindent
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -43,24 +43,6 @@ function(convert_filenames_to_full_paths NAMES)
  set(${NAMES} ${tmp_names} PARENT_SCOPE)
 endfunction()

-# Wrapper for add_executable that calls the HIP wrapper if applicable
-macro(mfem_add_executable NAME)
-  if (MFEM_USE_HIP)
-    hip_add_executable(${NAME} ${ARGN})
-  else()
-    add_executable(${NAME} ${ARGN})
-  endif()
-endmacro()
-
-# Wrapper for add_library that calls the HIP wrapper if applicable
-macro(mfem_add_library NAME)
-  if (MFEM_USE_HIP)
-    hip_add_library(${NAME} ${ARGN})
-  else()
-    add_library(${NAME} ${ARGN})
-  endif()
-endmacro()
-
 # Simple shortcut to add_custom_target() with option to add the target to the
 # main target.
 function(add_mfem_target TARGET_NAME ADD_TO_ALL)
@@ -74,7 +56,7 @@ function(add_mfem_target TARGET_NAME ADD_TO_ALL)
 endfunction()

 # Add mfem examples
-macro(add_mfem_examples EXE_SRCS)
+function(add_mfem_examples EXE_SRCS)
  set(EXE_PREFIX "")
  set(EXE_PREREQUISITE "")
  set(EXE_NEEDED_BY "")
@@ -90,15 +72,13 @@ macro(add_mfem_examples EXE_SRCS)
  foreach(SRC_FILE IN LISTS ${EXE_SRCS})
    # If CUDA is enabled, tag source files to be compiled with nvcc.
    if (MFEM_USE_CUDA)
-      set_source_files_properties(${SRC_FILE} PROPERTIES LANGUAGE CUDA)
-    elseif(MFEM_USE_HIP)
-      set_source_files_properties(${SRC_FILE} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
+      set_property(SOURCE ${SRC_FILE} PROPERTY LANGUAGE CUDA)
    endif()

    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)

    string(REPLACE ".cpp" "" EXE_NAME "${EXE_PREFIX}${SRC_FILENAME}")
-    mfem_add_executable(${EXE_NAME} ${SRC_FILE})
+    add_executable(${EXE_NAME} ${SRC_FILE})
    add_dependencies(${MFEM_ALL_EXAMPLES_TARGET_NAME} ${EXE_NAME})
    if (EXE_NEEDED_BY)
      add_dependencies(${EXE_NEEDED_BY} ${EXE_NAME})
@@ -127,10 +107,10 @@ macro(add_mfem_examples EXE_SRCS)
      endif()
    endif()
  endforeach(SRC_FILE)
-endmacro()
+endfunction()

 # A slightly more versatile function for adding miniapps to MFEM
-macro(add_mfem_miniapp MFEM_EXE_NAME)
+function(add_mfem_miniapp MFEM_EXE_NAME)
  # Parse the input arguments looking for the things we need
  set(POSSIBLE_ARGS "MAIN" "EXTRA_SOURCES" "EXTRA_HEADERS" "EXTRA_OPTIONS" "EXTRA_DEFINES" "LIBRARIES")
  set(CURRENT_ARG)
@@ -146,7 +126,8 @@ macro(add_mfem_miniapp MFEM_EXE_NAME)

  # If CUDA is enabled, tag source files to be compiled with nvcc.
  if (MFEM_USE_CUDA)
-    set_source_files_properties(${MAIN_LIST} ${EXTRA_SOURCES_LIST} PROPERTIES LANGUAGE CUDA)
+    set_property(SOURCE ${MAIN_LIST} ${EXTRA_SOURCES_LIST}
+      PROPERTY LANGUAGE CUDA)
    if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.12.0)
      list(TRANSFORM EXTRA_OPTIONS_LIST PREPEND "-Xcompiler=")
    else()
@@ -156,13 +137,11 @@ macro(add_mfem_miniapp MFEM_EXE_NAME)
      endforeach()
      set(EXTRA_OPTIONS_LIST ${LIST_})
    endif()
-  elseif(MFEM_USE_HIP)
-    set_source_files_properties(${MAIN_LIST} ${EXTRA_SOURCES_LIST} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
  endif()

  # Actually add the executable
-  mfem_add_executable(${MFEM_EXE_NAME} ${MAIN_LIST}
-      ${EXTRA_SOURCES_LIST} ${EXTRA_HEADERS_LIST})
+  add_executable(${MFEM_EXE_NAME} ${MAIN_LIST}
+    ${EXTRA_SOURCES_LIST} ${EXTRA_HEADERS_LIST})
  add_dependencies(${MFEM_ALL_MINIAPPS_TARGET_NAME} ${MFEM_EXE_NAME})
  add_dependencies(${MFEM_EXE_NAME} ${MFEM_EXEC_PREREQUISITES_TARGET_NAME})

@@ -215,7 +194,7 @@ macro(add_mfem_miniapp MFEM_EXE_NAME)
        LINK_FLAGS "${MPI_CXX_LINK_FLAGS}")
    endif()
  endif()
-endmacro()
+endfunction()


 # Auxiliary function, used in mfem_find_package().
@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -76,9 +76,6 @@ option(MFEM_ENABLE_MINIAPPS "Build all of the miniapps" OFF)
 # Set the target CUDA architecture
 set(CUDA_ARCH "sm_60" CACHE STRING "Target CUDA architecture.")

-# Set the target HIP architecture
-set(HIP_ARCH "gfx900" CACHE STRING "Target HIP architecture.")
-
 set(MFEM_DIR ${CMAKE_CURRENT_SOURCE_DIR})

 # The *_DIR paths below will be the first place searched for the corresponding
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,6 +1,6 @@
 #!/bin/bash

-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,257 +1,241 @@
-MFEM NC mesh v1.0
+MFEM mesh v1.1

-# NCMesh supported geometry types:
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
 # TRIANGLE    = 2
 # SQUARE      = 3
 # TETRAHEDRON = 4
 # CUBE        = 5
-# PRISM       = 6
+#

 dimension
 3

-# rank attr geom ref_type nodes/children
 elements
-137
-1 1 5 7 1 18 35 52 69 86 103 120
-1 1 5 7 2 3 4 5 6 7 8 17
-0 1 5 0 0 27 81 30 35 82 87 85
-0 1 5 0 27 8 28 81 82 36 83 87
-0 1 5 0 81 28 20 29 87 83 37 84
-0 1 5 0 30 81 29 11 85 87 84 38
-0 1 5 0 35 82 87 85 16 31 86 34
-0 1 5 0 82 36 83 87 31 21 32 86
-1 1 5 7 9 10 11 12 13 14 15 16
-0 1 5 0 87 88 179 91 92 180 185 183
-0 1 5 0 88 83 89 179 180 94 181 185
-0 1 5 0 179 89 37 90 185 181 98 182
-0 1 5 0 91 179 90 84 183 185 182 99
-0 1 5 0 92 180 185 183 86 93 184 97
-0 1 5 0 180 94 181 185 93 32 95 184
-0 1 5 0 185 181 98 182 184 95 26 96
-0 1 5 0 183 185 182 99 97 184 96 33
-0 1 5 0 85 87 84 38 34 86 33 24
-1 1 5 7 19 20 21 22 23 24 25 26
-0 1 5 0 8 39 100 28 36 101 105 83
-0 1 5 0 39 1 40 100 101 45 102 105
-0 1 5 0 100 40 9 41 105 102 46 103
-0 1 5 0 28 100 41 20 83 105 103 37
-0 1 5 0 36 101 105 83 21 42 104 32
-0 1 5 0 101 45 102 105 42 17 43 104
-0 1 5 0 105 102 46 103 104 43 22 44
-1 1 5 7 27 28 29 30 31 32 33 34
-0 1 5 0 83 106 186 89 94 187 191 181
-0 1 5 0 106 105 107 186 187 110 188 191
-0 1 5 0 186 107 103 108 191 188 112 189
-0 1 5 0 89 186 108 37 181 191 189 98
-0 1 5 0 94 187 191 181 32 109 190 95
-0 1 5 0 187 110 188 191 109 104 111 190
-0 1 5 0 191 188 112 189 190 111 44 113
-0 1 5 0 181 191 189 98 95 190 113 26
-1 1 5 7 36 37 38 39 40 49 50 51
-0 1 5 0 20 41 114 49 37 103 119 117
-0 1 5 0 41 9 47 114 103 46 115 119
-0 1 5 0 114 47 3 48 119 115 53 116
-0 1 5 0 49 114 48 10 117 119 116 54
-1 1 5 7 41 42 43 44 45 46 47 48
-0 1 5 0 37 108 192 122 98 189 197 195
-0 1 5 0 108 103 120 192 189 112 193 197
-0 1 5 0 192 120 119 121 197 193 126 194
-0 1 5 0 122 192 121 117 195 197 194 127
-0 1 5 0 98 189 197 195 26 113 196 125
-0 1 5 0 189 112 193 197 113 44 123 196
-0 1 5 0 197 193 126 194 196 123 118 124
-0 1 5 0 195 197 194 127 125 196 124 52
-0 1 5 0 103 46 115 119 44 22 50 118
-0 1 5 0 119 115 53 116 118 50 18 51
-0 1 5 0 117 119 116 54 52 118 51 23
-1 1 5 7 53 54 55 56 57 58 67 68
-0 1 5 0 11 29 128 56 38 84 132 130
-0 1 5 0 29 20 49 128 84 37 117 132
-0 1 5 0 128 49 10 55 132 117 54 129
-0 1 5 0 56 128 55 2 130 132 129 59
-0 1 5 0 38 84 132 130 24 33 131 58
-1 1 5 7 59 60 61 62 63 64 65 66
-0 1 5 0 84 90 198 133 99 182 202 200
-0 1 5 0 90 37 122 198 182 98 195 202
-0 1 5 0 198 122 117 134 202 195 127 199
-0 1 5 0 133 198 134 132 200 202 199 136
-0 1 5 0 99 182 202 200 33 96 201 135
-0 1 5 0 182 98 195 202 96 26 125 201
-0 1 5 0 202 195 127 199 201 125 52 137
-0 1 5 0 200 202 199 136 135 201 137 131
-0 1 5 0 132 117 54 129 131 52 23 57
-0 1 5 0 130 132 129 59 58 131 57 19
-1 1 5 7 70 71 72 81 82 83 84 85
-0 1 5 0 16 31 86 34 64 172 175 173
-0 1 5 0 31 21 32 86 172 65 164 175
-1 1 5 7 73 74 75 76 77 78 79 80
-0 1 5 0 86 93 184 97 176 219 222 220
-0 1 5 0 93 32 95 184 219 169 216 222
-0 1 5 0 184 95 26 96 222 216 150 203
-0 1 5 0 97 184 96 33 220 222 203 145
-0 1 5 0 176 219 222 220 175 177 221 178
-0 1 5 0 219 169 216 222 177 164 171 221
-0 1 5 0 222 216 150 203 221 171 66 147
-0 1 5 0 220 222 203 145 178 221 147 138
-0 1 5 0 34 86 33 24 173 175 138 67
-0 1 5 0 64 172 175 173 4 60 174 63
-0 1 5 0 172 65 164 175 60 12 61 174
-0 1 5 0 175 164 66 138 174 61 25 62
-0 1 5 0 173 175 138 67 63 174 62 15
-1 1 5 7 87 88 89 90 99 100 101 102
-0 1 5 0 21 42 104 32 65 162 166 164
-0 1 5 0 42 17 43 104 162 71 163 166
-0 1 5 0 104 43 22 44 166 163 72 152
-1 1 5 7 91 92 93 94 95 96 97 98
-0 1 5 0 32 109 190 95 169 214 218 216
-0 1 5 0 109 104 111 190 214 168 215 218
-0 1 5 0 190 111 44 113 218 215 160 209
-0 1 5 0 95 190 113 26 216 218 209 150
-0 1 5 0 169 214 218 216 164 167 217 171
-0 1 5 0 214 168 215 218 167 166 170 217
-0 1 5 0 218 215 160 209 217 170 152 157
-0 1 5 0 216 218 209 150 171 217 157 66
-0 1 5 0 65 162 166 164 12 68 165 61
-0 1 5 0 162 71 163 166 68 5 69 165
-0 1 5 0 166 163 72 152 165 69 13 70
-0 1 5 0 164 166 152 66 61 165 70 25
-1 1 5 7 104 113 114 115 116 117 118 119
-1 1 5 7 105 106 107 108 109 110 111 112
-0 1 5 0 26 113 196 125 150 209 213 204
-0 1 5 0 113 44 123 196 209 160 210 213
-0 1 5 0 196 123 118 124 213 210 161 211
-0 1 5 0 125 196 124 52 204 213 211 151
-0 1 5 0 150 209 213 204 66 157 212 148
-0 1 5 0 209 160 210 213 157 152 158 212
-0 1 5 0 213 210 161 211 212 158 156 159
-0 1 5 0 204 213 211 151 148 212 159 139
-0 1 5 0 44 22 50 118 152 72 153 156
-0 1 5 0 118 50 18 51 156 153 76 154
-0 1 5 0 52 118 51 23 139 156 154 77
-0 1 5 0 66 152 156 139 25 70 155 75
-0 1 5 0 152 72 153 156 70 13 73 155
-0 1 5 0 156 153 76 154 155 73 7 74
-0 1 5 0 139 156 154 77 75 155 74 14
-1 1 5 7 121 122 131 132 133 134 135 136
-0 1 5 0 24 33 131 58 67 138 143 141
-1 1 5 7 123 124 125 126 127 128 129 130
-0 1 5 0 33 96 201 135 145 203 208 206
-0 1 5 0 96 26 125 201 203 150 204 208
-0 1 5 0 201 125 52 137 208 204 151 205
-0 1 5 0 135 201 137 131 206 208 205 146
-0 1 5 0 145 203 208 206 138 147 207 144
-0 1 5 0 203 150 204 208 147 66 148 207
-0 1 5 0 208 204 151 205 207 148 139 149
-0 1 5 0 206 208 205 146 144 207 149 143
-0 1 5 0 131 52 23 57 143 139 77 140
-0 1 5 0 58 131 57 19 141 143 140 80
-0 1 5 0 67 138 143 141 15 62 142 79
-0 1 5 0 138 66 139 143 62 25 75 142
-0 1 5 0 143 139 77 140 142 75 14 78
-0 1 5 0 141 143 140 80 79 142 78 6
+120
+1 5 0 27 81 30 35 82 87 85
+1 5 30 81 29 11 85 87 84 38
+1 5 85 87 84 38 34 86 33 24
+1 5 35 82 87 85 16 31 86 34
+1 5 82 36 83 87 31 21 32 86
+1 5 180 94 181 185 93 32 95 184
+1 5 92 180 185 183 86 93 184 97
+1 5 183 185 182 99 97 184 96 33
+1 5 185 181 98 182 184 95 26 96
+1 5 179 89 37 90 185 181 98 182
+1 5 91 179 90 84 183 185 182 99
+1 5 87 88 179 91 92 180 185 183
+1 5 88 83 89 179 180 94 181 185
+1 5 81 28 20 29 87 83 37 84
+1 5 27 8 28 81 82 36 83 87
+1 5 8 39 100 28 36 101 105 83
+1 5 36 101 105 83 21 42 104 32
+1 5 101 45 102 105 42 17 43 104
+1 5 39 1 40 100 101 45 102 105
+1 5 100 40 9 41 105 102 46 103
+1 5 105 102 46 103 104 43 22 44
+1 5 186 107 103 108 191 188 112 189
+1 5 106 105 107 186 187 110 188 191
+1 5 187 110 188 191 109 104 111 190
+1 5 191 188 112 189 190 111 44 113
+1 5 181 191 189 98 95 190 113 26
+1 5 94 187 191 181 32 109 190 95
+1 5 83 106 186 89 94 187 191 181
+1 5 89 186 108 37 181 191 189 98
+1 5 28 100 41 20 83 105 103 37
+1 5 20 41 114 49 37 103 119 117
+1 5 37 108 192 122 98 189 197 195
+1 5 122 192 121 117 195 197 194 127
+1 5 195 197 194 127 125 196 124 52
+1 5 98 189 197 195 26 113 196 125
+1 5 189 112 193 197 113 44 123 196
+1 5 197 193 126 194 196 123 118 124
+1 5 192 120 119 121 197 193 126 194
+1 5 108 103 120 192 189 112 193 197
+1 5 103 46 115 119 44 22 50 118
+1 5 41 9 47 114 103 46 115 119
+1 5 114 47 3 48 119 115 53 116
+1 5 119 115 53 116 118 50 18 51
+1 5 117 119 116 54 52 118 51 23
+1 5 49 114 48 10 117 119 116 54
+1 5 128 49 10 55 132 117 54 129
+1 5 56 128 55 2 130 132 129 59
+1 5 11 29 128 56 38 84 132 130
+1 5 29 20 49 128 84 37 117 132
+1 5 84 90 198 133 99 182 202 200
+1 5 90 37 122 198 182 98 195 202
+1 5 198 122 117 134 202 195 127 199
+1 5 133 198 134 132 200 202 199 136
+1 5 200 202 199 136 135 201 137 131
+1 5 202 195 127 199 201 125 52 137
+1 5 182 98 195 202 96 26 125 201
+1 5 99 182 202 200 33 96 201 135
+1 5 38 84 132 130 24 33 131 58
+1 5 130 132 129 59 58 131 57 19
+1 5 132 117 54 129 131 52 23 57
+1 5 131 52 23 57 143 139 77 140
+1 5 58 131 57 19 141 143 140 80
+1 5 24 33 131 58 67 138 143 141
+1 5 33 96 201 135 145 203 208 206
+1 5 96 26 125 201 203 150 204 208
+1 5 201 125 52 137 208 204 151 205
+1 5 135 201 137 131 206 208 205 146
+1 5 206 208 205 146 144 207 149 143
+1 5 208 204 151 205 207 148 139 149
+1 5 203 150 204 208 147 66 148 207
+1 5 145 203 208 206 138 147 207 144
+1 5 138 66 139 143 62 25 75 142
+1 5 67 138 143 141 15 62 142 79
+1 5 141 143 140 80 79 142 78 6
+1 5 143 139 77 140 142 75 14 78
+1 5 139 156 154 77 75 155 74 14
+1 5 52 118 51 23 139 156 154 77
+1 5 118 50 18 51 156 153 76 154
+1 5 156 153 76 154 155 73 7 74
+1 5 152 72 153 156 70 13 73 155
+1 5 44 22 50 118 152 72 153 156
+1 5 209 160 210 213 157 152 158 212
+1 5 213 210 161 211 212 158 156 159
+1 5 196 123 118 124 213 210 161 211
+1 5 113 44 123 196 209 160 210 213
+1 5 26 113 196 125 150 209 213 204
+1 5 125 196 124 52 204 213 211 151
+1 5 204 213 211 151 148 212 159 139
+1 5 150 209 213 204 66 157 212 148
+1 5 66 152 156 139 25 70 155 75
+1 5 164 166 152 66 61 165 70 25
+1 5 216 218 209 150 171 217 157 66
+1 5 169 214 218 216 164 167 217 171
+1 5 32 109 190 95 169 214 218 216
+1 5 95 190 113 26 216 218 209 150
+1 5 190 111 44 113 218 215 160 209
+1 5 109 104 111 190 214 168 215 218
+1 5 214 168 215 218 167 166 170 217
+1 5 218 215 160 209 217 170 152 157
+1 5 104 43 22 44 166 163 72 152
+1 5 166 163 72 152 165 69 13 70
+1 5 162 71 163 166 68 5 69 165
+1 5 42 17 43 104 162 71 163 166
+1 5 21 42 104 32 65 162 166 164
+1 5 65 162 166 164 12 68 165 61
+1 5 172 65 164 175 60 12 61 174
+1 5 175 164 66 138 174 61 25 62
+1 5 219 169 216 222 177 164 171 221
+1 5 176 219 222 220 175 177 221 178
+1 5 220 222 203 145 178 221 147 138
+1 5 222 216 150 203 221 171 66 147
+1 5 184 95 26 96 222 216 150 203
+1 5 97 184 96 33 220 222 203 145
+1 5 86 93 184 97 176 219 222 220
+1 5 93 32 95 184 219 169 216 222
+1 5 31 21 32 86 172 65 164 175
+1 5 16 31 86 34 64 172 175 173
+1 5 34 86 33 24 173 175 138 67
+1 5 173 175 138 67 63 174 62 15
+1 5 64 172 175 173 4 60 174 63

-# attr geom nodes
 boundary
 96
 1 3 30 81 27 0
 2 3 0 27 82 35
 5 3 30 0 35 85
-1 3 81 28 8 27
-2 3 27 8 36 82
-1 3 29 20 28 81
 1 3 11 29 81 30
 5 3 11 30 85 38
+5 3 38 85 34 24
 2 3 35 82 31 16
 5 3 85 35 16 34
 2 3 82 36 21 31
-5 3 38 85 34 24
+1 3 29 20 28 81
+1 3 81 28 8 27
+2 3 27 8 36 82
 1 3 28 100 39 8
 2 3 8 39 101 36
+2 3 36 101 42 21
+2 3 101 45 17 42
+3 3 45 102 43 17
 1 3 100 40 1 39
 2 3 39 1 45 101
 3 3 1 40 102 45
 1 3 41 9 40 100
 3 3 40 9 46 102
-1 3 20 41 100 28
-2 3 36 101 42 21
-2 3 101 45 17 42
-3 3 45 102 43 17
 3 3 102 46 22 43
+1 3 20 41 100 28
 1 3 49 114 41 20
+3 3 46 115 50 22
 1 3 114 47 9 41
 3 3 9 47 115 46
 1 3 48 3 47 114
 3 3 47 3 53 115
 4 3 3 48 116 53
-1 3 10 48 114 49
-4 3 48 10 54 116
-3 3 46 115 50 22
 3 3 115 53 18 50
 4 3 53 116 51 18
 4 3 116 54 23 51
-1 3 56 128 29 11
-5 3 56 11 38 130
-1 3 128 49 20 29
+1 3 10 48 114 49
+4 3 48 10 54 116
 1 3 55 10 49 128
 4 3 10 55 129 54
 1 3 2 55 128 56
 4 3 55 2 59 129
 5 3 2 56 130 59
+1 3 56 128 29 11
+5 3 56 11 38 130
+1 3 128 49 20 29
 5 3 130 38 24 58
-4 3 54 129 57 23
 4 3 129 59 19 57
 5 3 59 130 58 19
-2 3 16 31 172 64
-5 3 34 16 64 173
-2 3 31 21 65 172
-5 3 24 34 173 67
-2 3 64 172 60 4
-5 3 173 64 4 63
-6 3 4 60 174 63
-2 3 172 65 12 60
-6 3 60 12 61 174
-6 3 174 61 25 62
-5 3 67 173 63 15
-6 3 63 174 62 15
-2 3 21 42 162 65
-2 3 42 17 71 162
-3 3 17 43 163 71
-3 3 43 22 72 163
-2 3 65 162 68 12
-6 3 12 68 165 61
-2 3 162 71 5 68
-3 3 71 163 69 5
-6 3 68 5 69 165
-3 3 163 72 13 69
-6 3 165 69 13 70
-6 3 61 165 70 25
-3 3 22 50 153 72
-3 3 50 18 76 153
-4 3 18 51 154 76
-4 3 51 23 77 154
-6 3 25 70 155 75
-3 3 72 153 73 13
-6 3 70 13 73 155
-3 3 153 76 7 73
-4 3 76 154 74 7
-6 3 155 73 7 74
-4 3 154 77 14 74
-6 3 75 155 74 14
-5 3 58 24 67 141
+4 3 54 129 57 23
 4 3 23 57 140 77
 4 3 57 19 80 140
 5 3 19 58 141 80
+5 3 58 24 67 141
+6 3 62 25 75 142
 5 3 141 67 15 79
 6 3 15 62 142 79
-6 3 62 25 75 142
-4 3 77 140 78 14
-6 3 142 75 14 78
 4 3 140 80 6 78
 5 3 80 141 79 6
 6 3 79 142 78 6
+4 3 77 140 78 14
+6 3 142 75 14 78
+4 3 154 77 14 74
+6 3 75 155 74 14
+4 3 51 23 77 154
+3 3 50 18 76 153
+4 3 18 51 154 76
+3 3 153 76 7 73
+4 3 76 154 74 7
+6 3 155 73 7 74
+3 3 72 153 73 13
+6 3 70 13 73 155
+3 3 22 50 153 72
+6 3 25 70 155 75
+6 3 61 165 70 25
+3 3 43 22 72 163
+3 3 163 72 13 69
+6 3 165 69 13 70
+2 3 162 71 5 68
+3 3 71 163 69 5
+6 3 68 5 69 165
+2 3 42 17 71 162
+3 3 17 43 163 71
+2 3 21 42 162 65
+2 3 65 162 68 12
+6 3 12 68 165 61
+2 3 172 65 12 60
+6 3 60 12 61 174
+6 3 174 61 25 62
+2 3 31 21 65 172
+2 3 16 31 172 64
+5 3 34 16 64 173
+5 3 24 34 173 67
+5 3 67 173 63 15
+6 3 63 174 62 15
+2 3 64 172 60 4
+5 3 173 64 4 63
+6 3 4 60 174 63

-# vert_id p1 p2
 vertex_parents
 215
 8 0 1
@@ -470,9 +454,28 @@ vertex_parents
 221 171 178
 222 203 219

-# top-level node coordinates
-coordinates
-8
+coarse_elements
+17
+7 11 12 9 10 6 5 8 7
+7 0 14 13 1 3 4 120 2
+7 27 22 21 28 26 23 24 25
+7 15 18 19 29 16 17 20 122
+7 31 38 37 32 34 35 36 33
+7 30 40 41 44 124 39 42 43
+7 49 50 51 52 56 55 54 53
+7 47 48 45 46 57 126 59 58
+7 113 114 111 112 108 107 110 109
+7 116 115 128 117 119 105 106 118
+7 93 96 95 94 92 97 98 91
+7 103 102 99 130 104 101 100 90
+7 85 84 83 86 88 81 82 87
+7 132 80 77 76 89 79 78 75
+7 63 64 65 66 70 69 68 67
+7 62 134 60 61 72 71 74 73
+7 121 123 125 127 129 131 133 135
+
+vertices
+223
 3
 0 0 0
 1 0 0
@@ -482,5 +485,218 @@ coordinates
 1 0 1
 0 1 1
 1 1 1
-
-mfem_mesh_end
+0.5 0 0
+1 0.5 0
+0.5 1 0
+0 0.5 0
+0.5 0 1
+1 0.5 1
+0.5 1 1
+0 0.5 1
+0 0 0.5
+1 0 0.5
+1 1 0.5
+0 1 0.5
+0.5 0.5 0
+0.5 0 0.5
+1 0.5 0.5
+0.5 1 0.5
+0 0.5 0.5
+0.5 0.5 1
+0.5 0.5 0.5
+0.25 0 0
+0.5 0.25 0
+0.25 0.5 0
+0 0.25 0
+0.25 0 0.5
+0.5 0.25 0.5
+0.25 0.5 0.5
+0 0.25 0.5
+0 0 0.25
+0.5 0 0.25
+0.5 0.5 0.25
+0 0.5 0.25
+0.75 0 0
+1 0.25 0
+0.75 0.5 0
+0.75 0 0.5
+1 0.25 0.5
+0.75 0.5 0.5
+1 0 0.25
+1 0.5 0.25
+1 0.75 0
+0.75 1 0
+0.5 0.75 0
+1 0.75 0.5
+0.75 1 0.5
+0.5 0.75 0.5
+1 1 0.25
+0.5 1 0.25
+0.25 1 0
+0 0.75 0
+0.25 1 0.5
+0 0.75 0.5
+0 1 0.25
+0.25 0 1
+0.5 0.25 1
+0.25 0.5 1
+0 0.25 1
+0 0 0.75
+0.5 0 0.75
+0.5 0.5 0.75
+0 0.5 0.75
+0.75 0 1
+1 0.25 1
+0.75 0.5 1
+1 0 0.75
+1 0.5 0.75
+1 0.75 1
+0.75 1 1
+0.5 0.75 1
+1 1 0.75
+0.5 1 0.75
+0.25 1 1
+0 0.75 1
+0 1 0.75
+0.25 0.25 0
+0.25 0 0.25
+0.5 0.25 0.25
+0.25 0.5 0.25
+0 0.25 0.25
+0.25 0.25 0.5
+0.25 0.25 0.25
+0.375 0.25 0.25
+0.5 0.375 0.25
+0.375 0.5 0.25
+0.25 0.375 0.25
+0.25 0.25 0.375
+0.375 0.25 0.5
+0.5 0.25 0.375
+0.5 0.375 0.5
+0.375 0.5 0.5
+0.25 0.375 0.5
+0.5 0.5 0.375
+0.25 0.5 0.375
+0.75 0.25 0
+0.75 0 0.25
+1 0.25 0.25
+0.75 0.5 0.25
+0.75 0.25 0.5
+0.75 0.25 0.25
+0.625 0.25 0.25
+0.75 0.375 0.25
+0.625 0.5 0.25
+0.625 0.25 0.5
+0.75 0.25 0.375
+0.75 0.375 0.5
+0.75 0.5 0.375
+0.625 0.5 0.5
+0.75 0.75 0
+1 0.75 0.25
+0.75 1 0.25
+0.5 0.75 0.25
+0.75 0.75 0.5
+0.75 0.75 0.25
+0.75 0.625 0.25
+0.625 0.75 0.25
+0.5 0.625 0.25
+0.75 0.625 0.5
+0.625 0.75 0.5
+0.5 0.625 0.5
+0.75 0.75 0.375
+0.5 0.75 0.375
+0.25 0.75 0
+0.25 1 0.25
+0 0.75 0.25
+0.25 0.75 0.5
+0.25 0.75 0.25
+0.25 0.625 0.25
+0.375 0.75 0.25
+0.25 0.625 0.5
+0.25 0.75 0.375
+0.375 0.75 0.5
+0.25 0.5 0.75
+0.5 0.75 0.75
+0.25 1 0.75
+0 0.75 0.75
+0.25 0.75 1
+0.25 0.75 0.75
+0.25 0.625 0.75
+0.25 0.5 0.625
+0.25 0.75 0.625
+0.375 0.5 0.75
+0.5 0.625 0.75
+0.375 0.75 0.75
+0.5 0.5 0.625
+0.5 0.75 0.625
+0.75 0.5 0.75
+1 0.75 0.75
+0.75 1 0.75
+0.75 0.75 1
+0.75 0.75 0.75
+0.625 0.5 0.75
+0.75 0.625 0.75
+0.625 0.75 0.75
+0.75 0.5 0.625
+0.75 0.75 0.625
+0.75 0 0.75
+1 0.25 0.75
+0.5 0.25 0.75
+0.75 0.25 1
+0.75 0.25 0.75
+0.625 0.25 0.75
+0.75 0.25 0.625
+0.5 0.25 0.625
+0.75 0.375 0.75
+0.5 0.375 0.75
+0.25 0 0.75
+0 0.25 0.75
+0.25 0.25 1
+0.25 0.25 0.75
+0.25 0.25 0.625
+0.375 0.25 0.75
+0.25 0.375 0.75
+0.375 0.375 0.25
+0.375 0.25 0.375
+0.5 0.375 0.375
+0.375 0.5 0.375
+0.25 0.375 0.375
+0.375 0.375 0.5
+0.375 0.375 0.375
+0.625 0.375 0.25
+0.625 0.25 0.375
+0.75 0.375 0.375
+0.625 0.5 0.375
+0.625 0.375 0.5
+0.625 0.375 0.375
+0.625 0.625 0.25
+0.75 0.625 0.375
+0.625 0.75 0.375
+0.5 0.625 0.375
+0.625 0.625 0.5
+0.625 0.625 0.375
+0.375 0.625 0.25
+0.375 0.75 0.375
+0.25 0.625 0.375
+0.375 0.625 0.5
+0.375 0.625 0.375
+0.375 0.5 0.625
+0.5 0.625 0.625
+0.375 0.75 0.625
+0.25 0.625 0.625
+0.375 0.625 0.75
+0.375 0.625 0.625
+0.625 0.5 0.625
+0.75 0.625 0.625
+0.625 0.75 0.625
+0.625 0.625 0.75
+0.625 0.625 0.625
+0.625 0.25 0.625
+0.75 0.375 0.625
+0.5 0.375 0.625
+0.625 0.375 0.75
+0.625 0.375 0.625
+0.375 0.25 0.625
+0.25 0.375 0.625
+0.375 0.375 0.75
+0.375 0.375 0.625
@@ -1,63 +1,56 @@
-MFEM NC mesh v1.0
+MFEM mesh v1.1

-# NCMesh supported geometry types:
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
 # TRIANGLE    = 2
 # SQUARE      = 3
 # TETRAHEDRON = 4
 # CUBE        = 5
-# PRISM       = 6
+#

 dimension
 2

-# rank attr geom ref_type nodes/children
 elements
-37
-1 1 3 3 1 10 19 28
-1 1 3 3 2 3 4 9
-0 1 3 0 0 9 21 12
-0 1 3 0 9 4 10 21
-1 1 3 3 5 6 7 8
-0 1 3 0 21 22 37 25
-0 1 3 0 22 10 23 37
-0 1 3 0 37 23 8 24
-0 1 3 0 25 37 24 11
-0 1 3 0 12 21 11 7
-1 1 3 3 11 12 13 14
-0 1 3 0 4 13 26 10
-0 1 3 0 13 1 14 26
-0 1 3 0 26 14 5 15
-1 1 3 3 15 16 17 18
-0 1 3 0 10 27 38 23
-0 1 3 0 27 26 28 38
-0 1 3 0 38 28 15 29
-0 1 3 0 23 38 29 8
-1 1 3 3 20 25 26 27
-1 1 3 3 21 22 23 24
-0 1 3 0 8 29 39 33
-0 1 3 0 29 15 31 39
-0 1 3 0 39 31 30 32
-0 1 3 0 33 39 32 18
-0 1 3 0 15 5 16 30
-0 1 3 0 30 16 3 17
-0 1 3 0 18 30 17 6
-1 1 3 3 29 30 35 36
-0 1 3 0 7 11 34 20
-1 1 3 3 31 32 33 34
-0 1 3 0 11 24 40 35
-0 1 3 0 24 8 33 40
-0 1 3 0 40 33 18 36
-0 1 3 0 35 40 36 34
-0 1 3 0 34 18 6 19
-0 1 3 0 20 34 19 2
+28
+1 3 0 9 21 12
+1 3 12 21 11 7
+1 3 21 22 37 25
+1 3 25 37 24 11
+1 3 37 23 8 24
+1 3 22 10 23 37
+1 3 9 4 10 21
+1 3 4 13 26 10
+1 3 13 1 14 26
+1 3 26 14 5 15
+1 3 38 28 15 29
+1 3 27 26 28 38
+1 3 10 27 38 23
+1 3 23 38 29 8
+1 3 8 29 39 33
+1 3 33 39 32 18
+1 3 39 31 30 32
+1 3 29 15 31 39
+1 3 15 5 16 30
+1 3 30 16 3 17
+1 3 18 30 17 6
+1 3 34 18 6 19
+1 3 40 33 18 36
+1 3 24 8 33 40
+1 3 11 24 40 35
+1 3 35 40 36 34
+1 3 7 11 34 20
+1 3 20 34 19 2

-# attr geom nodes
 boundary
 16
 1 1 0 9
 4 1 12 0
-1 1 9 4
 4 1 7 12
+1 1 9 4
 1 1 4 13
 1 1 13 1
 2 1 1 14
@@ -66,12 +59,11 @@ boundary
 2 1 16 3
 3 1 3 17
 3 1 17 6
-4 1 20 7
 3 1 6 19
+4 1 20 7
 3 1 19 2
 4 1 2 20

-# vert_id p1 p2
 vertex_parents
 37
 4 0 1
@@ -112,13 +104,59 @@ vertex_parents
 39 29 32
 40 24 36

-# top-level node coordinates
-coordinates
-4
+coarse_elements
+9
+3 2 5 4 3
+3 0 6 28 1
+3 12 11 10 13
+3 7 8 9 30
+3 14 17 16 15
+3 32 18 19 20
+3 24 23 22 25
+3 26 34 21 27
+3 29 31 33 35
+
+vertices
+41
 2
 0 0
 1 0
 0 1
 1 1
-
-mfem_mesh_end
+0.5 0
+1 0.5
+0.5 1
+0 0.5
+0.5 0.5
+0.25 0
+0.5 0.25
+0.25 0.5
+0 0.25
+0.75 0
+1 0.25
+0.75 0.5
+1 0.75
+0.75 1
+0.5 0.75
+0.25 1
+0 0.75
+0.25 0.25
+0.375 0.25
+0.5 0.375
+0.375 0.5
+0.25 0.375
+0.75 0.25
+0.625 0.25
+0.75 0.375
+0.625 0.5
+0.75 0.75
+0.75 0.625
+0.625 0.75
+0.5 0.625
+0.25 0.75
+0.25 0.625
+0.375 0.75
+0.375 0.375
+0.625 0.375
+0.625 0.625
+0.375 0.625
@@ -1,42 +1,51 @@
-MFEM NC mesh v1.0
+MFEM mesh v1.1

-# NCMesh supported geometry types:
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
 # TRIANGLE    = 2
 # SQUARE      = 3
 # TETRAHEDRON = 4
 # CUBE        = 5
-# PRISM       = 6
+#

 dimension
 2

-# rank attr geom ref_type nodes/children
 elements
-20
-0 1 3 0 1 2 11 10
-0 1 3 0 2 3 12 11
-0 1 3 0 3 4 13 12
-0 2 3 0 4 5 14 13
-0 2 3 0 5 6 15 14
-0 2 3 0 6 7 16 15
-0 2 3 0 7 8 17 16
-1 1 3 3 8 13 14 15
-1 1 3 3 9 10 11 12
-0 1 3 0 0 23 30 26
-0 1 3 0 23 18 24 30
-0 1 3 0 30 24 22 25
-0 1 3 0 26 30 25 21
-0 1 3 0 18 1 19 22
-0 1 3 0 22 19 10 20
-1 1 3 3 16 17 18 19
-0 1 3 0 21 25 31 29
-0 1 3 0 25 22 27 31
-0 1 3 0 31 27 20 28
-0 1 3 0 29 31 28 9
+17
+1 3 0 23 30 26
+1 3 26 30 25 21
+1 3 30 24 22 25
+1 3 23 18 24 30
+1 3 18 1 19 22
+1 3 22 19 10 20
+1 3 31 27 20 28
+1 3 25 22 27 31
+1 3 21 25 31 29
+1 3 29 31 28 9
+1 3 1 2 11 10
+1 3 2 3 12 11
+1 3 3 4 13 12
+2 3 4 5 14 13
+2 3 5 6 15 14
+2 3 6 7 16 15
+2 3 7 8 17 16

-# attr geom nodes
 boundary
 25
+3 1 0 23
+1 1 26 0
+1 1 21 26
+3 1 23 18
+3 1 18 1
+3 1 10 20
+3 1 20 28
+1 1 29 21
+3 1 28 9
+1 1 9 29
 3 1 1 2
 3 1 11 10
 3 1 2 3
@@ -52,18 +61,7 @@ boundary
 3 1 7 8
 2 1 8 17
 3 1 17 16
-3 1 0 23
-1 1 26 0
-3 1 23 18
-1 1 21 26
-3 1 18 1
-3 1 10 20
-1 1 29 21
-3 1 20 28
-3 1 28 9
-1 1 9 29

-# vert_id p1 p2
 vertex_parents
 14
 18 0 1
@@ -81,21 +79,14 @@ vertex_parents
 30 23 25
 31 25 28

-# root element orientation
-root_state
-8
-1
-1
-1
-1
-1
-1
-0
-0
+coarse_elements
+3
+3 0 3 2 1
+3 8 7 6 9
+3 17 4 5 18

-# top-level node coordinates
-coordinates
-18
+vertices
+32
 2
 0 0
 1 0
@@ -115,5 +106,17 @@ coordinates
 6 1
 7 1
 8 1
-
-mfem_mesh_end
+0.5 0
+1 0.5
+0.5 1
+0 0.5
+0.5 0.5
+0.25 0
+0.5 0.25
+0.25 0.5
+0 0.25
+0.5 0.75
+0.25 1
+0 0.75
+0.25 0.25
+0.25 0.75
@@ -1,7 +0,0 @@
-MFEM INLINE mesh v1.0
-
-type = tri
-nx = 1
-ny = 1
-sx = 3.14
-sy = 3.14
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -781,8 +781,6 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/miniapps/navier \
                         @MFEM_SOURCE_DIR@/miniapps/nurbs \
                         @MFEM_SOURCE_DIR@/miniapps/performance \
-                         @MFEM_SOURCE_DIR@/miniapps/shifted \
-                         @MFEM_SOURCE_DIR@/miniapps/solvers \
                         @MFEM_SOURCE_DIR@/miniapps/tools \
                         @MFEM_SOURCE_DIR@/miniapps/toys

@@ -176,8 +176,6 @@ namespace mfem {
 * - <a class="el" href="findpts_8cpp_source.html">Find Points</a>: evaluate grid function in physical space, <a class="el" href="findpts_8cpp_source.html">serial</a> and <a class="el" href="pfindpts_8cpp_source.html">parallel</a> versions
 * - <a class="el" href="field-diff_8cpp_source.html">Field Diff</a>: compare grid functions on different meshes
 * - <a class="el" href="field-interp_8cpp_source.html">Field Interp</a>: transfer a grid functions between meshes
- * - <a class="el" href="distance_8cpp_source.html">Distance</a>: finite element distance function solver
- * - <a class="el" href="distance_8cpp_source.html">Block Solvers</a>: comparison of saddle point system solvers
 * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
 * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
 *
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -34,6 +34,7 @@ list(APPEND ALL_EXE_SRCS
  ex25.cpp
  ex26.cpp
  ex27.cpp
+  ex91.cpp
  )

 if (MFEM_USE_MPI)
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -69,10 +69,7 @@ int main(int argc, char *argv[])
   bool static_cond = false;
   bool pa = false;
   const char *device_config = "cpu";
-   bool visualization = true;
-#ifdef MFEM_USE_AMGX
-   bool useAmgX = false;
-#endif
+   bool visualization = 1;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -90,11 +87,6 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
-#ifdef MFEM_USE_AMGX
-   args.AddOption(&useAmgX, "-amgx", "--useAmgX", "-no-amgx",
-                  "--no-useAmgX",
-                  "Enable or disable AmgX in MatrixFreeAMS.");
-#endif

   args.Parse();
   if (!args.Good())
@@ -167,10 +159,9 @@ int main(int argc, char *argv[])
   //    by marking all the boundary attributes from the mesh as essential
   //    (Dirichlet) and converting them to a list of true dofs.
   Array<int> ess_tdof_list;
-   Array<int> ess_bdr;
   if (pmesh->bdr_attributes.Size())
   {
-      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
      ess_bdr = 1;
      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }
@@ -214,20 +205,20 @@ int main(int argc, char *argv[])
   Vector B, X;
   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);

-   // 13. Solve the system AX=B using PCG with an AMS preconditioner.
-   if (pa)
+   // 13. Solve the system AX=B using PCG with the AMS preconditioner from hypre
+   //     (in the full assembly case) or CG with Jacobi preconditioner (in the
+   //     partial assembly case).
+
+   if (pa) // Jacobi preconditioning in partial assembly mode
   {
-#ifdef MFEM_USE_AMGX
-      MatrixFreeAMS ams(*a, *A, *fespace, muinv, sigma, NULL, ess_bdr, useAmgX);
-#else
-      MatrixFreeAMS ams(*a, *A, *fespace, muinv, sigma, NULL, ess_bdr);
-#endif
+      OperatorJacobiSmoother Jacobi(*a, ess_tdof_list);
+
      CGSolver cg(MPI_COMM_WORLD);
      cg.SetRelTol(1e-12);
      cg.SetMaxIter(1000);
      cg.SetPrintLevel(1);
      cg.SetOperator(*A);
-      cg.SetPreconditioner(ams);
+      cg.SetPreconditioner(Jacobi);
      cg.Mult(B, X);
   }
   else
@@ -29,7 +29,7 @@
 //               finite elements (velocity u) and piecewise discontinuous
 //               polynomials (pressure p).
 //
-//               The example demonstrates the use of the BlockOperator class, as
+//               The example demonstrates the use of the BlockMatrix class, as
 //               well as the collective saving of several grid functions in
 //               VisIt (visit.llnl.gov) and ParaView (paraview.org) formats.
 //
@@ -29,7 +29,7 @@
 //               finite elements (velocity u) and piecewise discontinuous
 //               polynomials (pressure p).
 //
-//               The example demonstrates the use of the BlockOperator class, as
+//               The example demonstrates the use of the BlockMatrix class, as
 //               well as the collective saving of several grid functions in
 //               VisIt (visit.llnl.gov) and ParaView (paraview.org) formats.
 //               Optional saving with ADIOS2 (adios2.readthedocs.io) streams is
@@ -4,11 +4,9 @@
 //
 // Sample runs:  mpirun -np 4 ex6p -m ../data/square-disc.mesh -o 1
 //               mpirun -np 4 ex6p -m ../data/square-disc.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/square-disc.mesh -o 2 -ns
 //               mpirun -np 4 ex6p -m ../data/square-disc-nurbs.mesh -o 2
 //               mpirun -np 4 ex6p -m ../data/star.mesh -o 3
 //               mpirun -np 4 ex6p -m ../data/escher.mesh -o 2
-//               mpirun -np 4 ex6p -m ../data/escher.mesh -o 2 -ns
 //               mpirun -np 4 ex6p -m ../data/fichera.mesh -o 2
 //               mpirun -np 4 ex6p -m ../data/disc-nurbs.mesh -o 2
 //               mpirun -np 4 ex6p -m ../data/ball-nurbs.mesh
@@ -16,7 +14,6 @@
 //               mpirun -np 4 ex6p -m ../data/star-surf.mesh -o 2
 //               mpirun -np 4 ex6p -m ../data/square-disc-surf.mesh -o 2
 //               mpirun -np 4 ex6p -m ../data/amr-quad.mesh
-//               mpirun -np 4 ex6p --restart
 //
 // Device sample runs:
 //               mpirun -np 4 ex6p -pa -d cuda
@@ -37,8 +34,8 @@
 //               The example demonstrates MFEM's capability to work with both
 //               conforming and nonconforming refinements, in 2D and 3D, on
 //               linear, curved and surface meshes. Interpolation of functions
-//               from coarse to fine meshes, restarting from a checkpoint, as
-//               well as persistent GLVis visualization are also illustrated.
+//               from coarse to fine meshes, as well as persistent GLVis
+//               visualization are also illustrated.
 //
 //               We recommend viewing Example 1 before viewing this example.

@@ -62,9 +59,6 @@ int main(int argc, char *argv[])
   int order = 1;
   bool pa = false;
   const char *device_config = "cpu";
-   bool nc_simplices = false;
-   int max_dofs = 100000;
-   bool restart = false;
   bool visualization = true;

   OptionsParser args(argc, argv);
@@ -76,14 +70,6 @@ int main(int argc, char *argv[])
                  "--no-partial-assembly", "Enable Partial Assembly.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&nc_simplices, "-ns", "--nonconforming-simplices",
-                  "-cs", "--conforming-simplices",
-                  "For simplicial meshes, enable/disable nonconforming"
-                  " refinement");
-   args.AddOption(&max_dofs, "-md", "--max-dofs",
-                  "Stop after reaching this many degrees of freedom.");
-   args.AddOption(&restart, "-res", "--restart", "-no-res", "--no-restart",
-                  "Restart computation from the last checkpoint.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -107,52 +93,39 @@ int main(int argc, char *argv[])
   Device device(device_config);
   if (myid == 0) { device.Print(); }

-   ParMesh *pmesh;
-   if (!restart)
+   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+   int sdim = mesh->SpaceDimension();
+
+   // 5. Refine the serial mesh on all processors to increase the resolution.
+   //    Also project a NURBS mesh to a piecewise-quadratic curved mesh. Make
+   //    sure that the mesh is non-conforming.
+   if (mesh->NURBSext)
   {
-      // 4. Read the (serial) mesh from the given mesh file on all processors.
-      //    We can handle triangular, quadrilateral, tetrahedral, hexahedral,
-      //    surface and volume meshes with the same code.
-      Mesh mesh(mesh_file, 1, 1);
-
-      // 5. Refine the serial mesh on all processors to increase the resolution.
-      //    Also project a NURBS mesh to a piecewise-quadratic curved mesh. Make
-      //    sure that the mesh is non-conforming.
-      if (mesh.NURBSext)
-      {
-         mesh.UniformRefinement();
-         mesh.SetCurvature(2);
-      }
-      mesh.EnsureNCMesh(nc_simplices);
-
-      // 6. Define a parallel mesh by partitioning the serial mesh.
-      //    Once the parallel mesh is defined, the serial mesh can be deleted.
-      pmesh = new ParMesh(MPI_COMM_WORLD, mesh);
-   }
-   else
-   {
-      // 7. We can also restart the computation by loading the mesh from a
-      //    previously saved check-point.
-      string fname(MakeParFilename("ex6p-checkpoint.", myid));
-      ifstream ifs(fname);
-      MFEM_VERIFY(ifs.good(), "Checkpoint file " << fname << " not found.");
-      pmesh = new ParMesh(MPI_COMM_WORLD, ifs);
+      mesh->UniformRefinement();
+      mesh->SetCurvature(2);
   }
+   mesh->EnsureNCMesh();

-   int dim = pmesh->Dimension();
-   int sdim = pmesh->SpaceDimension();
+   // 6. Define a parallel mesh by partitioning the serial mesh.
+   //    Once the parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh pmesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;

-   MFEM_VERIFY(pmesh->bdr_attributes.Size() > 0,
+   MFEM_VERIFY(pmesh.bdr_attributes.Size() > 0,
               "Boundary attributes required in the mesh.");
-   Array<int> ess_bdr(pmesh->bdr_attributes.Max());
+   Array<int> ess_bdr(pmesh.bdr_attributes.Max());
   ess_bdr = 1;

-   // 8. Define a finite element space on the mesh. The polynomial order is
+   // 7. Define a finite element space on the mesh. The polynomial order is
   //    one (linear) by default, but this can be changed on the command line.
   H1_FECollection fec(order, dim);
-   ParFiniteElementSpace fespace(pmesh, &fec);
+   ParFiniteElementSpace fespace(&pmesh, &fec);

-   // 9. As in Example 1p, we set up bilinear and linear forms corresponding to
+   // 8. As in Example 1p, we set up bilinear and linear forms corresponding to
   //    the Laplace problem -\Delta u = 1. We don't assemble the discrete
   //    problem yet, this will be done in the main loop.
   ParBilinearForm a(&fespace);
@@ -169,12 +142,12 @@ int main(int argc, char *argv[])
   a.AddDomainIntegrator(integ);
   b.AddDomainIntegrator(new DomainLFIntegrator(one));

-   // 10. The solution vector x and the associated finite element grid function
-   //     will be maintained over the AMR iterations. We initialize it to zero.
+   // 9. The solution vector x and the associated finite element grid function
+   //    will be maintained over the AMR iterations. We initialize it to zero.
   ParGridFunction x(&fespace);
   x = 0;

-   // 11. Connect to GLVis.
+   // 10. Connect to GLVis.
   char vishost[] = "localhost";
   int  visport   = 19916;

@@ -196,29 +169,30 @@ int main(int argc, char *argv[])
      sout.precision(8);
   }

-   // 12. Set up an error estimator. Here we use the Zienkiewicz-Zhu estimator
+   // 11. Set up an error estimator. Here we use the Zienkiewicz-Zhu estimator
   //     with L2 projection in the smoothing step to better handle hanging
   //     nodes and parallel partitioning. We need to supply a space for the
   //     discontinuous flux (L2) and a space for the smoothed flux (H(div) is
   //     used here).
   L2_FECollection flux_fec(order, dim);
-   ParFiniteElementSpace flux_fes(pmesh, &flux_fec, sdim);
+   ParFiniteElementSpace flux_fes(&pmesh, &flux_fec, sdim);
   RT_FECollection smooth_flux_fec(order-1, dim);
-   ParFiniteElementSpace smooth_flux_fes(pmesh, &smooth_flux_fec);
+   ParFiniteElementSpace smooth_flux_fes(&pmesh, &smooth_flux_fec);
   // Another possible option for the smoothed flux space:
   // H1_FECollection smooth_flux_fec(order, dim);
-   // ParFiniteElementSpace smooth_flux_fes(pmesh, &smooth_flux_fec, dim);
+   // ParFiniteElementSpace smooth_flux_fes(&pmesh, &smooth_flux_fec, dim);
   L2ZienkiewiczZhuEstimator estimator(*integ, x, flux_fes, smooth_flux_fes);

-   // 13. A refiner selects and refines elements based on a refinement strategy.
+   // 12. A refiner selects and refines elements based on a refinement strategy.
   //     The strategy here is to refine elements with errors larger than a
   //     fraction of the maximum element error. Other strategies are possible.
   //     The refiner will call the given error estimator.
   ThresholdRefiner refiner(estimator);
   refiner.SetTotalErrorFraction(0.7);

-   // 14. The main AMR loop. In each iteration we solve the problem on the
+   // 13. The main AMR loop. In each iteration we solve the problem on the
   //     current mesh, visualize the solution, and refine the mesh.
+   const int max_dofs = 100000;
   for (int it = 0; ; it++)
   {
      HYPRE_Int global_dofs = fespace.GlobalTrueVSize();
@@ -228,19 +202,19 @@ int main(int argc, char *argv[])
         cout << "Number of unknowns: " << global_dofs << endl;
      }

-      // 15. Assemble the right-hand side and determine the list of true
+      // 14. Assemble the right-hand side and determine the list of true
      //     (i.e. parallel conforming) essential boundary dofs.
      Array<int> ess_tdof_list;
      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
      b.Assemble();

-      // 16. Assemble the stiffness matrix. Note that MFEM doesn't care at this
+      // 15. Assemble the stiffness matrix. Note that MFEM doesn't care at this
      //     point that the mesh is nonconforming and parallel.  The FE space is
      //     considered 'cut' along hanging edges/faces, and also across
      //     processor boundaries.
      a.Assemble();

-      // 17. Create the parallel linear system: eliminate boundary conditions.
+      // 16. Create the parallel linear system: eliminate boundary conditions.
      //     The system will be solved for true (unconstrained/unique) DOFs only.
      OperatorPtr A;
      Vector B, X;
@@ -248,7 +222,7 @@ int main(int argc, char *argv[])
      const int copy_interior = 1;
      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);

-      // 18. Solve the linear system A X = B.
+      // 17. Solve the linear system A X = B.
      //     * With full assembly, use the BoomerAMG preconditioner from hypre.
      //     * With partial assembly, use a diagonal preconditioner.
      Solver *M = NULL;
@@ -271,19 +245,19 @@ int main(int argc, char *argv[])
      cg.Mult(B, X);
      delete M;

-      // 19. Switch back to the host and extract the parallel grid function
+      // 18. Switch back to the host and extract the parallel grid function
      //     corresponding to the finite element approximation X. This is the
      //     local solution on each processor.
      a.RecoverFEMSolution(X, b, x);

-      // 20. Send the solution by socket to a GLVis server.
+      // 19. Send the solution by socket to a GLVis server.
      if (visualization)
      {
         sout << "parallel " << num_procs << " " << myid << "\n";
-         sout << "solution\n" << *pmesh << x << flush;
+         sout << "solution\n" << pmesh << x << flush;
      }

-      if (global_dofs >= max_dofs)
+      if (global_dofs > max_dofs)
      {
         if (myid == 0)
         {
@@ -292,11 +266,11 @@ int main(int argc, char *argv[])
         break;
      }

-      // 21. Call the refiner to modify the mesh. The refiner calls the error
+      // 20. Call the refiner to modify the mesh. The refiner calls the error
      //     estimator to obtain element errors, then it selects elements to be
      //     refined and finally it modifies the mesh. The Stop() method can be
      //     used to determine if a stopping criterion was met.
-      refiner.Apply(*pmesh);
+      refiner.Apply(pmesh);
      if (refiner.Stop())
      {
         if (myid == 0)
@@ -306,7 +280,7 @@ int main(int argc, char *argv[])
         break;
      }

-      // 22. Update the finite element space (recalculate the number of DOFs,
+      // 21. Update the finite element space (recalculate the number of DOFs,
      //     etc.) and create a grid function update matrix. Apply the matrix
      //     to any GridFunctions over the space. In this case, the update
      //     matrix is an interpolation matrix so the updated GridFunction will
@@ -314,11 +288,11 @@ int main(int argc, char *argv[])
      fespace.Update();
      x.Update();

-      // 23. Load balance the mesh, and update the space and solution. Currently
+      // 22. Load balance the mesh, and update the space and solution. Currently
      //     available only for nonconforming meshes.
-      if (pmesh->Nonconforming())
+      if (pmesh.Nonconforming())
      {
-         pmesh->Rebalance();
+         pmesh.Rebalance();

         // Update the space and the GridFunction. This time the update matrix
         // redistributes the GridFunction among the processors.
@@ -326,30 +300,12 @@ int main(int argc, char *argv[])
         x.Update();
      }

-      // 24. Inform also the bilinear and linear forms that the space has
+      // 23. Inform also the bilinear and linear forms that the space has
      //     changed.
      a.Update();
      b.Update();
-
-      // 25. Save the current state of the mesh every 5 iterations. The
-      //     computation can be restarted from this point. Note that unlike in
-      //     visualization, we need to use the 'ParPrint' method to save all
-      //     internal parallel data structures.
-      if ((it + 1) % 5 == 0)
-      {
-         ofstream ofs(MakeParFilename("ex6p-checkpoint.", myid));
-         ofs.precision(8);
-         pmesh->ParPrint(ofs);
-
-         if (myid == 0)
-         {
-            cout << "\nCheckpoint saved." << endl;
-         }
-      }
   }

-   delete pmesh;
-
   MPI_Finalize();
   return 0;
 }
@@ -450,7 +450,7 @@ FE_Evolution::FE_Evolution(BilinearForm &_M, BilinearForm &_K, const Vector &_b)
   : TimeDependentOperator(_M.Height()), M(_M), K(_K), b(_b), z(_M.Height())
 {
   Array<int> ess_tdof_list;
-   if (M.GetAssemblyLevel() == AssemblyLevel::LEGACY)
+   if (M.GetAssemblyLevel() == AssemblyLevel::LEGACYFULL)
   {
      M_prec = new DSmoother(M.SpMat());
      M_solver.SetOperator(M.SpMat());
@@ -0,0 +1,672 @@
+#include"mfem.hpp"
+
+#include<memory>
+#include<iostream>
+#include<fstream>
+
+namespace mfem {
+
+
+class LinDiffQFunc
+{
+public:
+    LinDiffQFunc(mfem::Coefficient& dd, mfem::Coefficient& ll, double gg_,
+                 double pp0_, double pp1_):diff(dd),load(ll),gg(gg_),pp0(pp0_),pp1(pp1_)
+    {
+
+    }
+
+    double QEnergy(ElementTransformation &T,
+                  const IntegrationPoint &ip,
+                  mfem::Vector& param, mfem::Vector& uu)
+    {
+        double dd=diff.Eval(T,ip);
+        double ll=load.Eval(T,ip);
+
+        double rho0=param[0];
+        double rho1=param[1];
+        double fd=dd*std::pow(rho0,pp0)*std::pow(rho1,pp1);
+
+        double rez = 0.5*(uu[0]*uu[0]+uu[1]*uu[1]+uu[2]*uu[2])*fd
+                        + 0.5*gg*uu[3]*uu[3] -uu[3]*ll;
+
+        return rez;
+    }
+
+    void QResidual(ElementTransformation &T,
+                   const IntegrationPoint &ip,
+                   mfem::Vector& param, mfem::Vector& uu, mfem::Vector& rr)
+    {
+        rr.SetSize(4);
+        double dd=diff.Eval(T,ip);
+        double ll=load.Eval(T,ip);
+
+        double rho0=param[0];
+        double rho1=param[1];
+
+        double fd=dd*std::pow(rho0,pp0)*std::pow(rho1,pp1);
+
+        rr[0]=uu[0]*fd;
+        rr[1]=uu[1]*fd;
+        rr[2]=uu[2]*fd;
+        rr[3]=gg*uu[3]-ll;
+    }
+
+    void AQResidual(ElementTransformation &T,
+                    const IntegrationPoint &ip,
+                    mfem::Vector& param,
+                    mfem::Vector& uu, mfem::Vector& aa, mfem::Vector& rr)
+    {
+         rr.SetSize(2);
+         double dd=diff.Eval(T,ip);
+         double ll=load.Eval(T,ip);
+
+         double rho0=param[0];
+         double rho1=param[1];
+
+         double fd0=dd*pp0*std::pow(rho0,pp0-1.0)*std::pow(rho1,pp1);
+         double fd1=dd*std::pow(rho0,pp0)*pp1*std::pow(rho1,pp1-1.0);
+
+         rr[0] = (aa[0]*uu[0]+aa[1]*uu[1]+aa[2]*uu[2])*fd0;
+         rr[1] = (aa[0]*uu[0]+aa[1]*uu[1]+aa[2]*uu[2])*fd1;
+
+    }
+
+    void QGradResidual(ElementTransformation &T,
+                       const IntegrationPoint &ip,
+                       mfem::Vector& param, mfem::Vector& uu, mfem::DenseMatrix& hh)
+    {
+        hh.SetSize(4);
+        double dd=diff.Eval(T,ip);
+        //double ll=load.Eval(T,ip);
+
+
+        double rho0=param[0];
+        double rho1=param[1];
+
+        double fd=dd*std::pow(rho0,pp0)*std::pow(rho1,pp1);
+        hh=0.0;
+
+        hh(0,0)=fd;
+        hh(1,1)=fd;
+        hh(2,2)=fd;
+        hh(3,3)=gg;
+    }
+
+
+private:
+    mfem::Coefficient& diff;
+    mfem::Coefficient& load;
+    double gg;
+    double pp0;
+    double pp1;
+};
+
+
+class PrmBlockLSFEMDiffusion: public PrmBlockNonlinearFormIntegrator
+{
+public:
+    PrmBlockLSFEMDiffusion(LinDiffQFunc& qfun_)
+    {
+        qfunc=&qfun_;
+    }
+
+    /// Compute the local energy
+    virtual double GetElementEnergy(const Array<const FiniteElement *>&el,
+                                    const Array<const FiniteElement *>&pel,
+                                    ElementTransformation &Tr,
+                                    const Array<const Vector *>&elfun,
+                                    const Array<const Vector *>&pelfun)
+    {
+        int dof_u0 = el[0]->GetDof();
+        int dof_r0 = pel[0]->GetDof();
+        int dof_r1 = pel[1]->GetDof();
+
+        int dim = el[0]->GetDim();
+        int spaceDim = Tr.GetSpaceDim();
+        if (dim != spaceDim)
+        {
+            mfem::mfem_error(" PrmBlockLSFEMDiffusion::GetElementEnergy"
+                             " is not defined on manifold meshes");
+        }
+
+        //shape functions
+        Vector shu0(dof_u0);
+        Vector shr0(dof_r0);
+        Vector shr1(dof_r1);
+        DenseMatrix dsu0(dof_u0,dim);
+        DenseMatrix B(dof_u0, 4);
+        B=0.0;
+
+        double w;
+
+        Vector param(2); param=0.0;
+        Vector uu(4); uu=0.0;
+
+        double energy =0.0;
+
+        const IntegrationRule *ir = nullptr;
+        if(ir==nullptr){
+            int order= 2 * el[0]->GetOrder() + Tr.OrderGrad(el[0])
+                                +pel[0]->GetOrder()+pel[1]->GetOrder();
+            ir=&IntRules.Get(Tr.GetGeometryType(),order);
+        }
+
+        for (int i = 0; i < ir->GetNPoints(); i++)
+        {
+            const IntegrationPoint &ip = ir->IntPoint(i);
+            Tr.SetIntPoint(&ip);
+            w=Tr.Weight();
+            w = ip.weight * w;
+
+            el[0]->CalcPhysDShape(Tr,dsu0);
+            el[0]->CalcPhysShape(Tr,shu0);
+            pel[0]->CalcPhysShape(Tr,shr0);
+            pel[1]->CalcPhysShape(Tr,shr1);
+
+            param[0]=shr0*(*pelfun[0]);
+            param[1]=shr1*(*pelfun[1]);
+
+            //set the matrix B
+            for(int jj=0;jj<dim;jj++)
+            {
+                B.SetCol(jj,dsu0.GetColumn(jj));
+            }
+            B.SetCol(3,shu0);
+            B.MultTranspose(*elfun[0],uu);
+            energy=energy+w * qfunc->QEnergy(Tr,ip,param,uu);
+        }
+        return energy;
+    }
+
+    /// Perform the local action of the BlockNonlinearFormIntegrator
+    virtual void AssembleElementVector(const Array<const FiniteElement *> &el,
+                                       const Array<const FiniteElement *>&pel,
+                                       ElementTransformation &Tr,
+                                       const Array<const Vector *> &elfun,
+                                       const Array<const Vector *>&pelfun,
+                                       const Array<Vector *> &elvec)
+    {
+        int dof_u0 = el[0]->GetDof();
+        int dof_r0 = pel[0]->GetDof();
+        int dof_r1 = pel[1]->GetDof();
+
+        int dim = el[0]->GetDim();
+
+        elvec[0]->SetSize(dof_u0);
+        *elvec[0]=0.0;
+        int spaceDim = Tr.GetSpaceDim();
+        if (dim != spaceDim)
+        {
+            mfem::mfem_error(" PrmBlockLSFEMDiffusion::AssembleElementVector"
+                             " is not defined on manifold meshes");
+        }
+
+        //shape functions
+        Vector shu0(dof_u0);
+        Vector shr0(dof_r0);
+        Vector shr1(dof_r1);
+        DenseMatrix dsu0(dof_u0,dim);
+        DenseMatrix B(dof_u0, 4);
+        B=0.0;
+
+        double w;
+
+        Vector param(2); param=0.0;
+        Vector uu(4); uu=0.0;
+        Vector rr;
+        Vector lvec; lvec.SetSize(dof_u0);
+
+        const IntegrationRule *ir = nullptr;
+        if(ir==nullptr){
+            int order= 2 * el[0]->GetOrder() + Tr.OrderGrad(el[0])
+                                +pel[0]->GetOrder()+pel[1]->GetOrder();
+            ir=&IntRules.Get(Tr.GetGeometryType(),order);
+        }
+
+        for (int i = 0; i < ir->GetNPoints(); i++)
+        {
+            const IntegrationPoint &ip = ir->IntPoint(i);
+            Tr.SetIntPoint(&ip);
+            w=Tr.Weight();
+            w = ip.weight * w;
+
+            el[0]->CalcPhysDShape(Tr,dsu0);
+            el[0]->CalcPhysShape(Tr,shu0);
+            pel[0]->CalcPhysShape(Tr,shr0);
+            pel[1]->CalcPhysShape(Tr,shr1);
+
+            param[0]=shr0*(*pelfun[0]);
+            param[1]=shr1*(*pelfun[1]);
+
+            //set the matrix B
+            for(int jj=0;jj<dim;jj++)
+            {
+                B.SetCol(jj,dsu0.GetColumn(jj));
+            }
+            B.SetCol(3,shu0);
+            B.MultTranspose(*elfun[0],uu);
+            qfunc->QResidual(Tr,ip,param, uu, rr);
+
+            B.Mult(rr,lvec);
+            elvec[0]->Add(w,lvec);
+        }
+
+    }
+
+    virtual void AssembleFaceVector(const Array<const FiniteElement *> &el1,
+                                    const Array<const FiniteElement *> &el2,
+                                    const Array<const FiniteElement *> &pel1,
+                                    const Array<const FiniteElement *> &pel2,
+                                    FaceElementTransformations &Tr,
+                                    const Array<const Vector *> &elfun,
+                                    const Array<const Vector *>&pelfun,
+                                    const Array<Vector *> &elvect)
+    {
+
+    }
+
+    /// Assemble the local gradient matrix
+    virtual void AssembleElementGrad(const Array<const FiniteElement*> &el,
+                                     const Array<const FiniteElement *>&pel,
+                                     ElementTransformation &Tr,
+                                     const Array<const Vector *> &elfun,
+                                     const Array<const Vector *>&pelfun,
+                                     const Array2D<DenseMatrix *> &elmats)
+    {
+        int dof_u0 = el[0]->GetDof();
+        int dof_r0 = pel[0]->GetDof();
+        int dof_r1 = pel[1]->GetDof();
+
+        int dim = el[0]->GetDim();
+
+        //elmats[0]->Size(dof_u0, dof_u0);
+        //*elmats[0]=0.0;
+
+        DenseMatrix* K=elmats(0,0);
+        K->SetSize(dof_u0,dof_u0);
+        (*K)=0.0;
+
+        int spaceDim = Tr.GetSpaceDim();
+        if (dim != spaceDim)
+        {
+            mfem::mfem_error(" PrmBlockLSFEMDiffusion::AssembleElementVector"
+                             " is not defined on manifold meshes");
+        }
+
+        //shape functions
+        Vector shu0(dof_u0);
+        Vector shr0(dof_r0);
+        Vector shr1(dof_r1);
+        DenseMatrix dsu0(dof_u0,dim);
+        DenseMatrix B(dof_u0, 4);
+        DenseMatrix A(dof_u0, 4);
+        B=0.0;
+
+        double w;
+
+        Vector param(2); param=0.0;
+        Vector uu(4); uu=0.0;
+        DenseMatrix hh;
+        Vector lvec; lvec.SetSize(dof_u0);
+
+        const IntegrationRule *ir = nullptr;
+        if(ir==nullptr){
+            int order= 2 * el[0]->GetOrder() + Tr.OrderGrad(el[0])
+                                +pel[0]->GetOrder()+pel[1]->GetOrder();
+            ir=&IntRules.Get(Tr.GetGeometryType(),order);
+        }
+
+        for (int i = 0; i < ir->GetNPoints(); i++)
+        {
+            const IntegrationPoint &ip = ir->IntPoint(i);
+            Tr.SetIntPoint(&ip);
+            w = Tr.Weight();
+            w = ip.weight * w;
+
+            el[0]->CalcPhysDShape(Tr,dsu0);
+            el[0]->CalcPhysShape(Tr,shu0);
+            pel[0]->CalcPhysShape(Tr,shr0);
+            pel[1]->CalcPhysShape(Tr,shr1);
+
+            param[0]=shr0*(*pelfun[0]);
+            param[1]=shr1*(*pelfun[1]);
+
+            //set the matrix B
+            for(int jj=0;jj<dim;jj++)
+            {
+                B.SetCol(jj,dsu0.GetColumn(jj));
+            }
+            B.SetCol(3,shu0);
+            B.MultTranspose(*elfun[0],uu);
+            qfunc->QGradResidual(Tr,ip,param,uu,hh);
+            Mult(B,hh,A);
+            AddMult_a_ABt(w,A,B,*K);
+        }
+    }
+
+    virtual void AssembleFaceGrad(const Array<const FiniteElement *>&el1,
+                                  const Array<const FiniteElement *>&el2,
+                                  const Array<const FiniteElement *> &pel1,
+                                  const Array<const FiniteElement *> &pel2,
+                                  FaceElementTransformations &Tr,
+                                  const Array<const Vector *> &elfun,
+                                  const Array<const Vector *>&pelfun,
+                                  const Array2D<DenseMatrix *> &elmats)
+    {
+
+    }
+
+    virtual void AssemblePrmElementVector(const Array<const FiniteElement *> &el,
+                                          const Array<const FiniteElement *> &pel,
+                                          ElementTransformation &Tr,
+                                          const Array<const Vector *> &elfun,
+                                          const Array<const Vector *> &alfun,
+                                          const Array<const Vector *> &pelfun,
+                                          const Array<Vector *> &elvec)
+    {
+        int dof_u0 = el[0]->GetDof();
+        int dof_r0 = pel[0]->GetDof();
+        int dof_r1 = pel[1]->GetDof();
+
+        int dim = el[0]->GetDim();
+
+        Vector& e0 = *(elvec[0]);
+        Vector& e1 = *(elvec[1]);
+
+        e0.SetSize(dof_r0);
+        e0=0.0;
+        e1.SetSize(dof_r1);
+        e1=0.0;
+
+        int spaceDim = Tr.GetSpaceDim();
+        if (dim != spaceDim)
+        {
+            mfem::mfem_error(" PrmBlockLSFEMDiffusion::AssembleElementVector"
+                             " is not defined on manifold meshes");
+        }
+
+        //shape functions
+        Vector shu0(dof_u0);
+        Vector shr0(dof_r0);
+        Vector shr1(dof_r1);
+        DenseMatrix dsu0(dof_u0,dim);
+        DenseMatrix B(dof_u0, 4);
+        B=0.0;
+
+        double w;
+
+        Vector param(2); param=0.0;
+        Vector uu(4); uu=0.0;
+        Vector aa(4); aa=0.0;
+        Vector rr;
+        Vector lvec0; lvec0.SetSize(dof_r0);
+        Vector lvec1; lvec1.SetSize(dof_r1);
+
+        const IntegrationRule *ir = nullptr;
+        if(ir==nullptr){
+            int order= 2 * el[0]->GetOrder() + Tr.OrderGrad(el[0])
+                                +pel[0]->GetOrder()+pel[1]->GetOrder();
+            ir=&IntRules.Get(Tr.GetGeometryType(),order);
+        }
+
+        for (int i = 0; i < ir->GetNPoints(); i++)
+        {
+            const IntegrationPoint &ip = ir->IntPoint(i);
+            Tr.SetIntPoint(&ip);
+            w=Tr.Weight();
+            w = ip.weight * w;
+
+            el[0]->CalcPhysDShape(Tr,dsu0);
+            el[0]->CalcPhysShape(Tr,shu0);
+            pel[0]->CalcPhysShape(Tr,shr0);
+            pel[1]->CalcPhysShape(Tr,shr1);
+
+            param[0]=shr0*(*pelfun[0]);
+            param[1]=shr1*(*pelfun[1]);
+
+            //set the matrix B
+            for(int jj=0;jj<dim;jj++)
+            {
+                B.SetCol(jj,dsu0.GetColumn(jj));
+            }
+            B.SetCol(3,shu0);
+            B.MultTranspose(*elfun[0],uu);
+            B.MultTranspose(*alfun[0],aa);
+
+            qfunc->AQResidual(Tr, ip, param, uu, aa, rr);
+
+            lvec0=shr0;
+            lvec0*=rr[0];
+            lvec1=shr1;
+            lvec1*=rr[1];
+
+            e0.Add(w,lvec0);
+            e1.Add(w,lvec1);
+        }
+
+    }
+
+    virtual void AssemblePrmFaceVector(const Array<const FiniteElement *> &el1,
+                                       const Array<const FiniteElement *> &el2,
+                                       const Array<const FiniteElement *> &pel1,
+                                       const Array<const FiniteElement *> &pel2,
+                                       FaceElementTransformations &Tr,
+                                       const Array<const Vector *> &elfun,
+                                       const Array<const Vector *> &alfun,
+                                       const Array<const Vector *> &pelfun,
+                                       const Array<Vector *> &elvect)
+    {
+
+    }
+
+
+
+private:
+    LinDiffQFunc* qfunc;
+};
+
+}
+
+int main(int argc, char *argv[])
+{
+    const char *mesh_file = "../../data/beam-tet.mesh";
+    int ser_ref_levels = 1;
+    int order = 2;
+    bool visualization = true;
+    double newton_rel_tol = 1e-4;
+    double newton_abs_tol = 1e-6;
+    int newton_iter = 10;
+    int print_level = 0;
+
+    mfem::OptionsParser args(argc, argv);
+    args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
+    args.AddOption(&ser_ref_levels,
+                     "-rs",
+                     "--refine-serial",
+                     "Number of times to refine the mesh uniformly in serial.");
+    args.AddOption(&order,
+                      "-o",
+                      "--order",
+                      "Order (degree) of the finite elements.");
+    args.AddOption(&visualization,
+                      "-vis",
+                      "--visualization",
+                      "-no-vis",
+                      "--no-visualization",
+                      "Enable or disable GLVis visualization.");
+    args.AddOption(&newton_rel_tol,
+                      "-rel",
+                      "--relative-tolerance",
+                      "Relative tolerance for the Newton solve.");
+    args.AddOption(&newton_abs_tol,
+                      "-abs",
+                      "--absolute-tolerance",
+                      "Absolute tolerance for the Newton solve.");
+    args.AddOption(&newton_iter,
+                      "-it",
+                      "--newton-iterations",
+                      "Maximum iterations for the Newton solve.");
+
+    args.Parse();
+    if (!args.Good())
+    {
+        args.PrintUsage(std::cout);
+        return 1;
+    }
+    args.PrintOptions(std::cout);
+    // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+    //    can handle triangular, quadrilateral, tetrahedral and hexahedral meshes
+    //    with the same code.
+    mfem::Mesh *mesh = new mfem::Mesh(mesh_file, 1, 1);
+    int dim = mesh->Dimension();
+
+    // 4. Refine the mesh in serial to increase the resolution. In this example
+    //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
+    //    a command-line parameter.
+    for (int lev = 0; lev < ser_ref_levels; lev++)
+    {
+        mesh->UniformRefinement();
+    }
+
+    /// Define the q-function
+    mfem::ConstantCoefficient* dc=new mfem::ConstantCoefficient(1.0);
+    mfem::ConstantCoefficient* lc=new mfem::ConstantCoefficient(1.0);
+    mfem::LinDiffQFunc* qfun=new mfem::LinDiffQFunc(*dc,*lc,1.0,1.0,1.0);
+
+    mfem::H1_FECollection fec00(order, dim);
+    mfem::L2_FECollection fec01(order, dim);
+    mfem::FiniteElementSpace* bfes00=new mfem::FiniteElementSpace(mesh,&fec00,1,mfem::Ordering::byVDIM);
+    mfem::FiniteElementSpace* pfes00=new mfem::FiniteElementSpace(mesh,&fec00,1,mfem::Ordering::byVDIM);
+    mfem::FiniteElementSpace* pfes01=new mfem::FiniteElementSpace(mesh,&fec01,1,mfem::Ordering::byVDIM);
+
+    /// Define parametric nonlinear form
+    mfem::Array<mfem::FiniteElementSpace*> bfes;
+    mfem::Array<mfem::FiniteElementSpace*> pfes;
+
+    bfes.Append(bfes00);
+    pfes.Append(pfes00);
+    pfes.Append(pfes01);
+
+    mfem::PrmBlockNonlinearForm* nf=new mfem::PrmBlockNonlinearForm(bfes,pfes);
+    nf->AddDomainIntegrator(new mfem::PrmBlockLSFEMDiffusion(*qfun));
+
+    /// Define the grid functions
+    mfem::GridFunction* bgf00=new mfem::GridFunction(bfes00);
+    mfem::GridFunction* pgf00=new mfem::GridFunction(pfes00);
+    mfem::GridFunction* pgf01=new mfem::GridFunction(pfes01);
+    mfem::GridFunction* ggf00=new mfem::GridFunction(pfes00);
+    mfem::GridFunction* ggf01=new mfem::GridFunction(pfes01);
+
+    *bgf00=0.0;
+    *pgf00=1.0;
+    *pgf01=1.0;
+
+    mfem::BlockVector solbv; solbv.Update(nf->GetBlockTrueOffsets()); solbv=0.0;
+    mfem::BlockVector resbv; resbv.Update(nf->GetBlockTrueOffsets()); resbv=0.0;
+    mfem::BlockVector adjbv; adjbv.Update(nf->GetBlockTrueOffsets()); adjbv=0.0;
+    mfem::BlockVector prmbv; prmbv.Update(nf->PrmGetBlockTrueOffsets()); prmbv=1.0;
+    mfem::BlockVector grdbv; grdbv.Update(nf->PrmGetBlockTrueOffsets()); grdbv=0.0;
+
+    bgf00->SetFromTrueDofs(solbv.GetBlock(0));
+    pgf00->SetFromTrueDofs(prmbv.GetBlock(0));
+    pgf01->SetFromTrueDofs(prmbv.GetBlock(1));
+
+    nf->SetPrmFields(prmbv);
+    double energy = nf->GetEnergy(solbv);
+
+    nf->Mult(solbv,resbv);
+    std::cout<<"Norm res="<<resbv.Norml2()<<std::endl;
+
+    //mfem::Operator& K=nf->GetGradient(solbv);
+    std::cout<<"energy ="<< energy<<std::endl;
+
+    nf->SetStateFields(solbv);
+    nf->SetAdjointFields(adjbv);
+    nf->PrmMult(prmbv,grdbv);
+
+
+    //set the BC for the physics
+    mfem::Array<mfem::Array<int> *> ess_bdr;
+    mfem::Array<mfem::Vector*>      ess_rhs;
+    ess_bdr.Append(new mfem::Array<int>(mesh->bdr_attributes.Max()));
+    ess_rhs.Append(nullptr);
+    (*ess_bdr[0]) = 1;
+    nf->SetEssentialBC(ess_bdr,ess_rhs);
+
+    //define the solvers
+    mfem::UMFPackSolver* umfsolv=new mfem::UMFPackSolver();
+
+    mfem::GMRESSolver *gmres;
+    gmres = new mfem::GMRESSolver();
+    gmres->SetAbsTol(newton_abs_tol/10);
+    gmres->SetRelTol(newton_rel_tol/10);
+    gmres->SetMaxIter(100);
+    gmres->SetPrintLevel(print_level);
+    //gmres->SetPreconditioner(*prec);
+
+
+    mfem::NewtonSolver *ns;
+    ns = new mfem::NewtonSolver();
+    ns->iterative_mode = true;
+    ns->SetSolver(*gmres);
+    ns->SetOperator(*nf);
+    ns->SetPrintLevel(print_level);
+    ns->SetRelTol(newton_rel_tol);
+    ns->SetAbsTol(newton_abs_tol);
+    ns->SetMaxIter(newton_iter);
+
+    mfem::Vector b; //RHS is zero
+    solbv=0.0;
+    ns->Mult(b, solbv);
+
+
+    nf->SetStateFields(solbv);
+    nf->SetAdjointFields(solbv);
+    nf->PrmMult(prmbv,grdbv);
+
+
+
+
+    mfem::ParaViewDataCollection *dacol = new mfem::ParaViewDataCollection("Example91",
+                                                                      mesh);
+
+    ggf00->SetFromTrueDofs(grdbv.GetBlock(0));
+    ggf01->SetFromTrueDofs(grdbv.GetBlock(1));
+    pgf00->SetFromTrueDofs(solbv.GetBlock(0));
+
+    dacol->SetLevelsOfDetail(order);
+    dacol->RegisterField("sol", pgf00);
+    dacol->RegisterField("grad00", ggf00);
+    dacol->RegisterField("grad01", ggf01);
+
+    dacol->SetTime(1.0);
+    dacol->SetCycle(1);
+    dacol->Save();
+
+    delete dacol;
+
+    delete ns;
+    delete umfsolv;
+    delete gmres;
+    delete ess_bdr[0];
+
+    delete bgf00;
+    delete pgf00;
+    delete pgf01;
+    delete ggf00;
+    delete ggf01;
+
+
+    delete nf;
+    delete pfes01;
+    delete pfes00;
+    delete bfes00;
+
+    delete qfun;
+    delete lc;
+    delete dc;
+
+    delete mesh;
+
+}
@@ -655,7 +655,7 @@ FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
     M_solver(_M.ParFESpace()->GetComm()),
     z(_M.Height())
 {
-   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACY)
+   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACYFULL)
   {
      M.Reset(_M.ParallelAssemble(), true);
      K.Reset(_K.ParallelAssemble(), true);
@@ -669,7 +669,7 @@ FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
   M_solver.SetOperator(*M);

   Array<int> ess_tdof_list;
-   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACY)
+   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACYFULL)
   {
      HypreParMatrix &M_mat = *M.As<HypreParMatrix>();
      HypreParMatrix &K_mat = *K.As<HypreParMatrix>();
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,471 +0,0 @@
-// h-refinement examples
-//
-// Compile with: make href_exs
-//
-// Sample runs:  
-//
-// Description: Make h-refinement data sets for neural net
-//              comparison
-
-
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Exact smooth analytic solution for convergence study
-double u_exact(const Vector &);
-void u_grad_exact(const Vector &, Vector &);
-double u_exact_2(const Vector &);
-void u_grad_exact_2(const Vector &, Vector &);
-double u_exact_3(const Vector &);
-void u_grad_exact_3(const Vector &, Vector &);
-
-void convergenceStudy(const char *mesh_file, int num_ref, int &order,
-                      double &l2_err_prev, double &h1_err_prev, bool &visualization, 
-                      int &exact, int &solvePDE, bool static_cond)
-{
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 4. Refine the mesh num_ref times
-
-
-    for (int l = 1; l < num_ref+1; l++)
-    {
-        mesh->UniformRefinement();
-    }
-
-
-   // 5. Define a finite element space on the mesh. Here we use continuous
-   //    Lagrange finite elements of the specified order.
-
-   FiniteElementCollection *fec;
-   if (order == 1)
-   {
-      fec = new H1_FECollection(1, 2);
-   }
-   else if (order > 1)
-   {
-      fec = new H1_FECollection(order, 2);
-   }
-   else if (order < 0)
-   {
-      // fec = new H1_FECollection(-order, 2, BasisType::Positive);
-      fec = new H1_FECollection(-order, 2);
-   }
-   else
-   {
-     cout << "Error - something went wrong in processing order input." << endl;
-     fec = NULL;
-   }
-
-   // Set exact solution
-
-   // exact == 1 case:
-   FunctionCoefficient *u1 = new FunctionCoefficient(u_exact);
-   VectorFunctionCoefficient *(u1_grad) = new VectorFunctionCoefficient(dim, u_grad_exact);
-   
-   // exact == 2 case:
-   FunctionCoefficient *u2 = new FunctionCoefficient(u_exact_2);
-   VectorFunctionCoefficient *u2_grad = new VectorFunctionCoefficient(dim, u_grad_exact_2);
-
-   // exact == 3 case:
-   FunctionCoefficient *u3 = new FunctionCoefficient(u_exact_3);
-   VectorFunctionCoefficient *u3_grad = new VectorFunctionCoefficient(dim, u_grad_exact_3);
-
-   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
-
-   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
-   //    In this example, the boundary conditions are defined by marking all
-   //    the boundary attributes from the mesh as essential (Dirichlet) and
-   //    converting them to a list of true dofs.
-
-   // this variable may not be right for more general meshes than lattices?
-   int gotNdofs = fespace->GetNDofs();
-
-   Array<int> ess_tdof_list;
-   Array<int> ess_bdr(mesh->bdr_attributes.Max());
-   ess_bdr = 1;
-
-
-   if (solvePDE==1)
-   {
-      if (mesh->bdr_attributes.Size())
-      {
-         fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-   }
-   // For L2 Projection:
-   // Do not get boundary dofs
-
-   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
-   //    the basis functions in the finite element fespace.
-
-   LinearForm *b = new LinearForm(fespace);
-
-   if (solvePDE==1)
-   {
-      ConstantCoefficient zero(0.0);
-      b->AddDomainIntegrator(new DomainLFIntegrator(zero));
-   }
-   else // L2 Projection
-   {
-      if (exact == 1)
-      {
-         b->AddDomainIntegrator(new DomainLFIntegrator(*u1));
-      }
-      else if (exact == 2) 
-      {
-         b->AddDomainIntegrator(new DomainLFIntegrator(*u2));
-      }
-      else // exact == 3
-      {
-         b->AddDomainIntegrator(new DomainLFIntegrator(*u3));
-      }
-   }
-
-   b->Assemble();
-   
-   // 8. Define the solution vector x as a finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero,
-   //    which satisfies the boundary conditions.
-   GridFunction x(fespace);
-   x=0.0;
-
-   // 9. Set up the bilinear form a(.,.) on the finite element space
-   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //    domain integrator.
-   BilinearForm *a = new BilinearForm(fespace);
-
-   // DiffusionIntegrator *my_diff_integrator = new DiffusionIntegrator;
-   // MassIntegrator *my_mass_integrator = new MassIntegrator;
-
-   if (solvePDE==1)
-   {   
-      if (exact == 1)
-      {
-         x.ProjectBdrCoefficient(*u1, ess_bdr);
-      }
-      else if (exact == 2)
-      {
-         x.ProjectBdrCoefficient(*u2, ess_bdr);
-      }
-      else if (exact == 3)
-      {
-         x.ProjectBdrCoefficient(*u3, ess_bdr);
-      }
-      a->AddDomainIntegrator(new DiffusionIntegrator);
-   }
-   else // L2 Projection
-   {
-      a->AddDomainIntegrator(new MassIntegrator);
-   }
-   
-   // 10. Assemble the bilinear form and the corresponding linear system,
-   //     applying any necessary transformations such as: eliminating boundary
-   //     conditions, applying conforming constraints for non-conforming AMR,
-   //     static condensation, etc.
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
-
-   OperatorPtr A;
-   Vector B, X;
-
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-   
-   // cout << "Size of linear system: " << A->Height() << endl;
-  
-   // 11. Solve the linear system A X = B.
-
-   GSSmoother M((SparseMatrix&)(*A));
-   X = 0.0;
-   PCG(*A, M, B, X, 0, 200, 1e-24, 0.0);
-   
-   // 12. Recover the solution as a finite element grid function.
-   a->RecoverFEMSolution(X, *b, x);
-
-
-   // 13. Save the refined mesh and the solution. This output can be viewed later
-   //     using GLVis: "glvis -m refined.mesh -g sol.gf".
-   ofstream mesh_ofs("refined.mesh");
-   mesh_ofs.precision(8);
-   mesh->Print(mesh_ofs);
-   ofstream sol_ofs("sol.gf");
-   sol_ofs.precision(8);
-   x.Save(sol_ofs);
-
-
-   // 14. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *mesh << x << flush;
-   }
-   
-
-
-   // Compute and print the L^2 and H^1 norms of the error.
-   ConstantCoefficient one(1.0);
-   
-   double l2_err = 0;
-   double h1_err = 0;
-
-   if (exact == 1)
-   {
-      l2_err = x.ComputeL2Error(*u1);
-      h1_err = x.ComputeH1Error(u1, u1_grad, &one, 1.0, 1);
-   }
-   else if (exact == 2)
-   {
-      l2_err = x.ComputeL2Error(*u2);
-      h1_err = x.ComputeH1Error(u2, u2_grad, &one, 1.0, 1);
-   }
-   else if (exact == 3)
-   {
-      l2_err = x.ComputeL2Error(*u3);
-      h1_err = x.ComputeH1Error(u3, u3_grad, &one, 1.0, 1);
-   }
-
-   double l2_rate, h1_rate;
-
-   if (num_ref != 0)
-   {
-      l2_rate = -log(l2_err/l2_err_prev) / log(2);
-      h1_rate = -log(h1_err/h1_err_prev) / log(2);
-   }
-   else
-   {
-      l2_rate = 0.0;
-      h1_rate = 0.0;
-   }
-
-   int one_over_h = mesh->GetNE();
-   one_over_h = sqrt(one_over_h);
-
-   cout << setw(16) << gotNdofs << setw(16) << one_over_h << setw(
-           16) << l2_err << setw( 16) << l2_rate;
-   cout << setw(16) << h1_err << setw(16) << h1_rate << endl;
-
-   l2_err_prev = l2_err;
-   h1_err_prev = h1_err;
-
-
-   // Save the data to pass to neural network
-   cout << "Mesh vertex array = " << endl;
-   cout <<  mesh->GetVertex(0) << endl;
-
-   // 15. Free the used memory.
-   // delete pcg;
-   // delete amg;
-   // delete my_diff_integrator;
-   // delete my_mass_integrator;
-   delete a; 
-   delete b;
-   delete fespace;
-   delete u1;
-   delete u1_grad;
-   delete u2;
-   delete u2_grad;
-   delete u3;
-   delete u3_grad;
-   delete fec;
-   delete mesh;
-
-   return;
-}
-
-double u_exact(const Vector &x)
-{
-   return(x(0)+x(1));
-}
-
-void u_grad_exact(const Vector &x, Vector &u)
-{
-   u(0) = 1;
-   u(1) = 1;
-}
-
-double u_exact_2(const Vector &x)
-{
-   return sin(x(1))*exp(x(0));
-}
-
-void u_grad_exact_2(const Vector &x, Vector &u)
-{
-   u(0) = sin(x(1))*exp(x(0));
-   u(1) = cos(x(1))*exp(x(0));
-}
-
-double u_exact_3(const Vector &x)
-{
-    // return (x(0)*x(0) + (0.5)*x(1)*x(1));
-    int m=10;
-    double total = sin(x(0))* pow( sin(  x(0)*x(0) / M_PI  ), 2*m  );
-    total += sin(x(1))* pow( sin( 2*x(1)*x(1) / M_PI  ), 2*m  );
-    total *= -1;
-    return total;
-}
-
-void u_grad_exact_3(const Vector &x, Vector &u)
-{
-    // presumes m=10
-    u(0)  = - (40.0 * x(0) * cos( x(0)*x(0)/M_PI ) * sin(x(0)) * pow(sin( x(0)*x(0)/M_PI ),19)) / M_PI;
-    u(0) += - cos(x(0))*pow(sin( x(0)*x(0)/M_PI ),20);
-    u(1)  = - (80.0 * x(1) * cos( 2.0 * x(1)*x(1)/M_PI ) * sin(x(1)) * pow(sin( 2.0 * x(1)*x(1)/M_PI ),19)   ) / M_PI;
-    u(1) += - cos(x(1))*pow(sin( 2.0 * x(1)*x(1)/M_PI ),20);
-}
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   int total_refinements = 0;
-
-   // const char *mesh_file = "../../data/twoSquare.mesh";
-   // const char *mesh_file = "../../data/star-q3.mesh";
-   const char *mesh_file = "../../data/inline-oneTri.mesh";
-   int order = 1;
-   bool static_cond = false;
-   const char *device_config = "cpu";
-   bool visualization = false;
-   int exact = 3;
-   int solvePDE = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&total_refinements, "-r", "--refine",
-                  "Number of refinements to do.");   
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   //args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-   //             "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&exact, "-e", "--exact", 
-                  "Choice of exact solution. 1=constant 1; 2=sin(x)e^y; 3=michalewicz.");
-   args.AddOption(&solvePDE, "-L", "--L2Project",
-                  "Solve a PDE (1) or do L2 Projection (2)");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   if (order != 1)
-   {
-      cout << "Only allowing order 1 triangle elements for now." << endl;
-      return 1;
-   }
-   else if (order == 1)
-   {
-      cout << "Using H1 triangular elements of order " << order << "." << endl;
-   }
-
-   if (solvePDE == 1)
-   {
-      cout << "Approximating solution to Laplace problem with ";
-      if (exact == 1)
-      {
-        cout << "exact solution u(x,y)=x+y" << endl;
-      }
-      else if (exact == 2)
-      {
-         cout << "exact solution u(x,y)=sin(y)e^x" << endl;
-      }
-      else if (exact == 3)
-      {
-         cout << endl << "Michalewicz is not harmonic. Use option -L 2 to do L2 projection instead."
-              << endl;
-         return 1;
-      }
-      else
-      {
-         cout << endl << "*** Wrong usage of exact solution parameter (-e)"
-              << endl;
-         return 1;
-      }
-   }
-   else if (solvePDE == 2)
-   {
-      cout << "Doing L^2 projection of basis with right hand side ";
-      if (exact == 1)
-      {
-        cout << "u(x,y)=x+y" << endl;
-      }
-      else if (exact == 2)
-      {
-         cout << "u(x,y)=sin(y)e^x" << endl;
-      }
-      else if (exact == 3)
-      {
-         cout << "u(x,y)=michalewicz function" << endl;
-      }
-      else
-      {
-         cout << endl << "*** Wrong usage of exact solution parameter (-e)"
-              << endl;
-         return 1;
-      }
-   }
-   else
-   {
-      cout << "Wrong usage of solve vs. L2 Projection option -L."
-           << endl;
-      return 1;
-   }
-
-
-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // Set output options and print header
-   cout.precision(4);
-
-   cout << "----------------------------------------------------------------------------------------"
-        << endl;
-   cout << left << setw(16) << "DOFs "<< setw(16) <<"1/h "<< setw(
-           16) << "L^2 error "<< setw(16);
-   cout << "L^2 rate "<< setw(16) << "H^1 error "<< setw(16) << "H^1 rate" << endl;
-   cout << "----------------------------------------------------------------------------------------"
-        << endl;
-
-   double l2_err_prev = 0.0;
-   double h1_err_prev = 0.0;
-
-   // 3. Read the mesh from the given mesh file.
-   // Run last round with vis, if desired.
-
-   // can use this as a max DoF tolerance:  (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
-
-   // Loop over number of refinements for convergence study
-
-    bool noVisYet = false;
-    for (int i = 0; i < (total_refinements); i++)
-    {
-        convergenceStudy(mesh_file, i, order, l2_err_prev, h1_err_prev, noVisYet, 
-        exact, solvePDE, static_cond);
-    }
-
-    convergenceStudy(mesh_file, total_refinements, order, l2_err_prev, h1_err_prev, visualization, 
-            exact, solvePDE, static_cond);
-
-    return 0;
-}
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -144,7 +144,7 @@ clean-build:
 	rm -rf *.dSYM *.TVD.*breakpoints

 clean-exec:
-	@rm -f refined.mesh displaced.mesh mesh.* ex5.mesh ex6p-checkpoint.*
+	@rm -f refined.mesh displaced.mesh mesh.* ex5.mesh
 	@rm -rf Example5* Example9* Example15* Example16* Example23* ParaView
 	@rm -f sphere_refined.* sol.* sol_u.* sol_p.* sol_r.* sol_i.*
 	@rm -f ex9.mesh ex9-mesh.* ex9-init.* ex9-final.*
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -475,7 +475,7 @@ FE_Evolution::FE_Evolution(BilinearForm &_M, BilinearForm &_K, const Vector &_b)
   : TimeDependentOperator(_M.Height()), M(_M), K(_K), b(_b), z(_M.Height())
 {
   Array<int> ess_tdof_list;
-   if (M.GetAssemblyLevel() == AssemblyLevel::LEGACY)
+   if (M.GetAssemblyLevel() == AssemblyLevel::LEGACYFULL)
   {
      M_prec = new DSmoother(M.SpMat());
      M_solver.SetOperator(M.SpMat());
@@ -595,7 +595,7 @@ FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
     M_solver(_M.ParFESpace()->GetComm()),
     z(_M.Height())
 {
-   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACY)
+   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACYFULL)
   {
      M.Reset(_M.ParallelAssemble(), true);
      K.Reset(_K.ParallelAssemble(), true);
@@ -609,7 +609,7 @@ FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
   M_solver.SetOperator(*M);

   Array<int> ess_tdof_list;
-   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACY)
+   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACYFULL)
   {
      HypreParMatrix &M_mat = *M.As<HypreParMatrix>();
      HypreParMatrix &K_mat = *K.As<HypreParMatrix>();
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 # at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 # LICENSE and NOTICE for details. LLNL-CODE-806117.
 #
@@ -14,7 +14,6 @@ set(SRCS
  bilinearform_ext.cpp
  bilininteg.cpp
  bilininteg_br2.cpp
-  bilininteg_convection_mf.cpp
  bilininteg_convection_pa.cpp
  bilininteg_convection_ea.cpp
  bilininteg_dgtrace_pa.cpp
@@ -48,12 +47,9 @@ set(SRCS
  gridfunc.cpp
  hybridization.cpp
  intrules.cpp
-  ceed/convection.cpp
-  ceed/diffusion.cpp
-  ceed/nlconvection.cpp
-  ceed/mass.cpp
-  ceed/operator.cpp
-  ceed/util.cpp
+  libceed/ceed.cpp
+  libceed/diffusion.cpp
+  libceed/mass.cpp
  linearform.cpp
  lininteg.cpp
  multigrid.cpp
@@ -62,7 +58,6 @@ set(SRCS
  nonlininteg.cpp
  fespacehierarchy.cpp
  nonlininteg_vectorconvection.cpp
-  nonlininteg_vectorconvection_mf.cpp
  quadinterpolator.cpp
  quadinterpolator_face.cpp
  restriction.cpp
@@ -71,6 +66,7 @@ set(SRCS
  tmop_tools.cpp
  gslib.cpp
  transfer.cpp
+  prmnonlinearform.cpp
  )

 set(HDRS
@@ -91,14 +87,9 @@ set(HDRS
  gridfunc.hpp
  hybridization.hpp
  intrules.hpp
-  ceed/coefficient.hpp
-  ceed/convection.hpp
-  ceed/diffusion.hpp
-  ceed/integrator.hpp
-  ceed/mass.hpp
-  ceed/nlconvection.hpp
-  ceed/operator.hpp
-  ceed/util.hpp
+  libceed/ceed.hpp
+  libceed/diffusion.hpp
+  libceed/mass.hpp
  linearform.hpp
  lininteg.hpp
  multigrid.hpp
@@ -122,6 +113,7 @@ set(HDRS
  tmop_tools.hpp
  gslib.hpp
  transfer.hpp
+  prmnonlinearform.hpp
  )

 if (MFEM_USE_SIDRE)
@@ -146,7 +138,8 @@ if (MFEM_USE_MPI)
    pgridfunc.cpp
    plinearform.cpp
    pnonlinearform.cpp
-    prestriction.cpp)
+    prestriction.cpp
+    pprmnonlinearform.cpp)
  # If this list (HDRS -> HEADERS) is used for install, we probably want the
  # headers added all the time.
  list(APPEND HDRS
@@ -155,7 +148,8 @@ if (MFEM_USE_MPI)
    pgridfunc.hpp
    plinearform.hpp
    pnonlinearform.hpp
-    prestriction.hpp)
+    prestriction.hpp
+    pprmnonlinearform.hpp)
 endif()

 convert_filenames_to_full_paths(SRCS)
@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -76,7 +76,7 @@ BilinearForm::BilinearForm(FiniteElementSpace * f)
   precompute_sparsity = 0;
   diag_policy = DIAG_KEEP;

-   assembly = AssemblyLevel::LEGACY;
+   assembly = AssemblyLevel::LEGACYFULL;
   batch = 1;
   ext = NULL;
 }
@@ -94,7 +94,7 @@ BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)
   precompute_sparsity = ps;
   diag_policy = DIAG_KEEP;

-   assembly = AssemblyLevel::LEGACY;
+   assembly = AssemblyLevel::LEGACYFULL;
   batch = 1;
   ext = NULL;

@@ -121,7 +121,7 @@ void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
   assembly = assembly_level;
   switch (assembly)
   {
-      case AssemblyLevel::LEGACY:
+      case AssemblyLevel::LEGACYFULL:
         break;
      case AssemblyLevel::FULL:
         ext = new FABilinearFormExtension(this);
@@ -143,7 +143,7 @@ void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
 void BilinearForm::EnableStaticCondensation()
 {
   delete static_cond;
-   if (assembly != AssemblyLevel::LEGACY)
+   if (assembly != AssemblyLevel::LEGACYFULL)
   {
      static_cond = NULL;
      MFEM_WARNING("Static condensation not supported for this assembly level");
@@ -168,7 +168,7 @@ void BilinearForm::EnableHybridization(FiniteElementSpace *constr_space,
                                       const Array<int> &ess_tdof_list)
 {
   delete hybridization;
-   if (assembly != AssemblyLevel::LEGACY)
+   if (assembly != AssemblyLevel::LEGACYFULL)
   {
      delete constr_integ;
      hybridization = NULL;
@@ -223,7 +223,7 @@ MatrixInverse * BilinearForm::Inverse() const

 void BilinearForm::Finalize (int skip_zeros)
 {
-   if (assembly == AssemblyLevel::LEGACY)
+   if (assembly == AssemblyLevel::LEGACYFULL)
   {
      if (!static_cond) { mat->Finalize(skip_zeros); }
      if (mat_e) { mat_e->Finalize(skip_zeros); }
@@ -1109,7 +1109,7 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
   mat = NULL;
   mat_e = NULL;
   extern_bfs = 0;
-   assembly = AssemblyLevel::LEGACY;
+   assembly = AssemblyLevel::LEGACYFULL;
   ext = NULL;
 }

@@ -1134,7 +1134,7 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
   bbfi_marker = mbf->bbfi_marker;
   btfbfi_marker = mbf->btfbfi_marker;

-   assembly = AssemblyLevel::LEGACY;
+   assembly = AssemblyLevel::LEGACYFULL;
   ext = NULL;
 }

@@ -1147,7 +1147,7 @@ void MixedBilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
   assembly = assembly_level;
   switch (assembly)
   {
-      case AssemblyLevel::LEGACY:
+      case AssemblyLevel::LEGACYFULL:
         break;
      case AssemblyLevel::FULL:
         // ext = new FAMixedBilinearFormExtension(this);
@@ -1219,7 +1219,7 @@ void MixedBilinearForm::AddMultTranspose(const Vector & x, Vector & y,

 MatrixInverse * MixedBilinearForm::Inverse() const
 {
-   if (assembly != AssemblyLevel::LEGACY)
+   if (assembly != AssemblyLevel::LEGACYFULL)
   {
      MFEM_WARNING("MixedBilinearForm::Inverse not possible with this assembly level!");
      return NULL;
@@ -1232,7 +1232,7 @@ MatrixInverse * MixedBilinearForm::Inverse() const

 void MixedBilinearForm::Finalize (int skip_zeros)
 {
-   if (assembly == AssemblyLevel::LEGACY)
+   if (assembly == AssemblyLevel::LEGACYFULL)
   {
      mat -> Finalize (skip_zeros);
   }
@@ -1509,7 +1509,7 @@ void MixedBilinearForm::AssembleDiagonal_ADAt(const Vector &D,

 void MixedBilinearForm::ConformingAssemble()
 {
-   if (assembly != AssemblyLevel::LEGACY)
+   if (assembly != AssemblyLevel::LEGACYFULL)
   {
      MFEM_WARNING("Conforming assemble not supported for this assembly level!");
      return;
@@ -1779,7 +1779,7 @@ void DiscreteLinearOperator::SetAssemblyLevel(AssemblyLevel assembly_level)
   assembly = assembly_level;
   switch (assembly)
   {
-      case AssemblyLevel::LEGACY:
+      case AssemblyLevel::LEGACYFULL:
      case AssemblyLevel::FULL:
         // Use the original implementation for now
         break;
@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -29,13 +29,8 @@ namespace mfem
    form classes derived from Operator. */
 enum class AssemblyLevel
 {
-   /// In the case of a BilinearForm LEGACY corresponds to a fully assembled
-   /// form, i.e. a global sparse matrix in MFEM, Hypre or PETSC format.
-   /// In the case of a NonlinearForm LEGACY corresponds to an operator that
-   /// is fully evaluated on the fly.
-   /// This assembly level is ALWAYS performed on the host.
-   LEGACY = 0,
-   /// @deprecated Use LEGACY instead.
+   /// Legacy fully assembled form, i.e. a global sparse matrix in MFEM, Hypre
+   /// or PETSC format. This assembly is ALWAYS performed on the host.
   LEGACYFULL = 0,
   /// Fully assembled form, i.e. a global sparse matrix in MFEM format. This
   /// assembly is compatible with device execution.
@@ -58,8 +53,6 @@ enum class AssemblyLevel
    SetAssemblyLevel() function. */
 class BilinearForm : public Matrix
 {
-   friend FABilinearFormExtension;
-
 protected:
   /// Sparse matrix \f$ M \f$ to be associated with the form. Owned.
   SparseMatrix *mat;
@@ -129,7 +122,7 @@ protected:
      static_cond = NULL; hybridization = NULL;
      precompute_sparsity = 0;
      diag_policy = DIAG_KEEP;
-      assembly = AssemblyLevel::LEGACY;
+      assembly = AssemblyLevel::LEGACYFULL;
      batch = 1;
      ext = NULL;
   }
@@ -164,7 +157,7 @@ public:
   /// Set the desired assembly level.
   /** Valid choices are:

-       - AssemblyLevel::LEGACY (default)
+       - AssemblyLevel::LEGACYFULL (default)
       - AssemblyLevel::FULL
       - AssemblyLevel::PARTIAL
       - AssemblyLevel::ELEMENT
@@ -770,7 +763,7 @@ public:
   /// Sets all sparse values of \f$ M \f$ to @a a.
   void operator=(const double a) { *mat = a; }

-   /// Set the desired assembly level. The default is AssemblyLevel::LEGACY.
+   /// Set the desired assembly level. The default is AssemblyLevel::LEGACYFULL.
   /** This method must be called before assembly. */
   void SetAssemblyLevel(AssemblyLevel assembly_level);

@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -14,9 +14,8 @@

 #include "../general/forall.hpp"
 #include "bilinearform.hpp"
-#include "pbilinearform.hpp"
+#include "libceed/ceed.hpp"
 #include "pgridfunc.hpp"
-#include "ceed/util.hpp"

 namespace mfem
 {
@@ -822,14 +821,20 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
 // Data and methods for fully-assembled bilinear forms
 FABilinearFormExtension::FABilinearFormExtension(BilinearForm *form)
   : EABilinearFormExtension(form),
-     mat(a->mat)
+     mat(form->FESpace()->GetVSize(),form->FESpace()->GetVSize(),0),
+     face_mat(form->FESpace()->GetVSize(),0,0),
+     use_face_mat(false)
 {
 #ifdef MFEM_USE_MPI
-   ParFiniteElementSpace *pfes = nullptr;
-   if ( a->GetFBFI()->Size()>0 &&
-        (pfes = dynamic_cast<ParFiniteElementSpace*>(form->FESpace())) )
+   if ( ParFiniteElementSpace* pfes =
+           dynamic_cast<ParFiniteElementSpace*>(form->FESpace()) )
   {
-      pfes->ExchangeFaceNbrData();
+      if (pfes->IsDGSpace())
+      {
+         use_face_mat = true;
+         pfes->ExchangeFaceNbrData();
+         face_mat.SetWidth(pfes->GetFaceNbrVSize());
+      }
   }
 #endif
 }
@@ -838,225 +843,111 @@ void FABilinearFormExtension::Assemble()
 {
   EABilinearFormExtension::Assemble();
   FiniteElementSpace &fes = *a->FESpace();
-   int width = fes.GetVSize();
-   int height = fes.GetVSize();
-   bool keep_nbr_block = false;
-#ifdef MFEM_USE_MPI
-   ParFiniteElementSpace *pfes = nullptr;
-   if ( a->GetFBFI()->Size()>0 &&
-        (pfes = dynamic_cast<ParFiniteElementSpace*>(&fes)) )
+   if (fes.IsDGSpace())
   {
-      pfes->ExchangeFaceNbrData();
-      width += pfes->GetFaceNbrVSize();
-      dg_x.SetSize(width);
-      ParBilinearForm *pb = nullptr;
-      if ((pb = dynamic_cast<ParBilinearForm*>(a)) && (pb->keep_nbr_block))
+      const L2ElementRestriction *restE =
+         static_cast<const L2ElementRestriction*>(elem_restrict);
+      const L2FaceRestriction *restF =
+         static_cast<const L2FaceRestriction*>(int_face_restrict_lex);
+      // 1. Fill I
+      //  1.1 Increment with restE
+      restE->FillI(mat);
+      //  1.2 Increment with restF
+      if (restF) { restF->FillI(mat, face_mat); }
+      //  1.3 Sum the non-zeros in I
+      auto h_I = mat.HostReadWriteI();
+      int cpt = 0;
+      const int vd = fes.GetVDim();
+      const int ndofs = ne*elemDofs*vd;
+      for (int i = 0; i < ndofs; i++)
      {
-         height += pfes->GetFaceNbrVSize();
-         dg_y.SetSize(height);
-         keep_nbr_block = true;
+         const int nnz = h_I[i];
+         h_I[i] = cpt;
+         cpt += nnz;
      }
-   }
-#endif
-   if (a->mat) // We reuse the sparse matrix memory
-   {
-      if (fes.IsDGSpace())
+      const int nnz = cpt;
+      h_I[ndofs] = nnz;
+      mat.GetMemoryJ().New(nnz, mat.GetMemoryJ().GetMemoryType());
+      mat.GetMemoryData().New(nnz, mat.GetMemoryData().GetMemoryType());
+      if (use_face_mat && restF)
      {
-         const L2ElementRestriction *restE =
-            static_cast<const L2ElementRestriction*>(elem_restrict);
-         const L2FaceRestriction *restF =
-            static_cast<const L2FaceRestriction*>(int_face_restrict_lex);
-         // 1. Fill J and Data
-         // 1.1 Fill J and Data with Elem ea_data
-         restE->FillJAndData(ea_data, *mat);
-         // 1.2 Fill J and Data with Face ea_data_ext
-         if (restF) { restF->FillJAndData(ea_data_ext, *mat, keep_nbr_block); }
-         // 1.3 Shift indirections in I back to original
-         auto I = mat->HostReadWriteI();
-         for (int i = height; i > 0; i--)
-         {
-            I[i] = I[i-1];
-         }
-         I[0] = 0;
-      }
-      else
-      {
-         const ElementRestriction &rest =
-            static_cast<const ElementRestriction&>(*elem_restrict);
-         rest.FillJAndData(ea_data, *mat);
-      }
-   }
-   else // We create, compute the sparsity, and fill the sparse matrix
-   {
-      mat = new SparseMatrix(height, width, 0);
-      if (fes.IsDGSpace())
-      {
-         const L2ElementRestriction *restE =
-            static_cast<const L2ElementRestriction*>(elem_restrict);
-         const L2FaceRestriction *restF =
-            static_cast<const L2FaceRestriction*>(int_face_restrict_lex);
-         // 1. Fill I
-         mat->GetMemoryI().New(height+1, mat->GetMemoryI().GetMemoryType());
-         //  1.1 Increment with restE
-         restE->FillI(*mat);
-         //  1.2 Increment with restF
-         if (restF) { restF->FillI(*mat, keep_nbr_block); }
-         //  1.3 Sum the non-zeros in I
-         auto h_I = mat->HostReadWriteI();
+         auto h_I_face = face_mat.HostReadWriteI();
         int cpt = 0;
-         for (int i = 0; i < height; i++)
+         for (int i = 0; i < ndofs; i++)
         {
-            const int nnz = h_I[i];
-            h_I[i] = cpt;
+            const int nnz = h_I_face[i];
+            h_I_face[i] = cpt;
            cpt += nnz;
         }
-         const int nnz = cpt;
-         h_I[height] = nnz;
-         mat->GetMemoryJ().New(nnz, mat->GetMemoryJ().GetMemoryType());
-         mat->GetMemoryData().New(nnz, mat->GetMemoryData().GetMemoryType());
-         // 2. Fill J and Data
-         // 2.1 Fill J and Data with Elem ea_data
-         restE->FillJAndData(ea_data, *mat);
-         // 2.2 Fill J and Data with Face ea_data_ext
-         if (restF) { restF->FillJAndData(ea_data_ext, *mat, keep_nbr_block); }
-         // 2.3 Shift indirections in I back to original
-         auto I = mat->HostReadWriteI();
-         for (int i = height; i > 0; i--)
+         const int nnz_face = cpt;
+         h_I_face[ndofs] = nnz_face;
+         face_mat.GetMemoryJ().New(nnz_face,
+                                   face_mat.GetMemoryJ().GetMemoryType());
+         face_mat.GetMemoryData().New(nnz_face,
+                                      face_mat.GetMemoryData().GetMemoryType());
+      }
+      // 2. Fill J and Data
+      // 2.1 Fill J and Data with Elem ea_data
+      restE->FillJAndData(ea_data, mat);
+      // 2.2 Fill J and Data with Face ea_data_ext
+      if (restF) { restF->FillJAndData(ea_data_ext, mat, face_mat); }
+      // 2.3 Shift indirections in I back to original
+      auto I = mat.HostReadWriteI();
+      for (int i = ndofs; i > 0; i--)
+      {
+         I[i] = I[i-1];
+      }
+      I[0] = 0;
+      if (use_face_mat && restF)
+      {
+         auto I_face = face_mat.HostReadWriteI();
+         for (int i = ndofs; i > 0; i--)
         {
-            I[i] = I[i-1];
+            I_face[i] = I_face[i-1];
         }
-         I[0] = 0;
-      }
-      else // continuous Galerkin case
-      {
-         const ElementRestriction &rest =
-            static_cast<const ElementRestriction&>(*elem_restrict);
-         rest.FillSparseMatrix(ea_data, *mat);
-      }
-      a->mat = mat;
-   }
-}
-
-void FABilinearFormExtension::DGMult(const Vector &x, Vector &y) const
-{
-#ifdef MFEM_USE_MPI
-   const ParFiniteElementSpace *pfes;
-   if ( (pfes = dynamic_cast<const ParFiniteElementSpace*>(testFes)) )
-   {
-      // DG Prolongation
-      ParGridFunction x_gf;
-      x_gf.MakeRef(const_cast<ParFiniteElementSpace*>(pfes),
-                   const_cast<Vector&>(x),0);
-      x_gf.ExchangeFaceNbrData();
-      Vector &shared_x = x_gf.FaceNbrData();
-      const int local_size = a->FESpace()->GetVSize();
-      auto dg_x_ptr = dg_x.Write();
-      auto x_ptr = x.Read();
-      MFEM_FORALL(i,local_size,
-      {
-         dg_x_ptr[i] = x_ptr[i];
-      });
-      const int shared_size = shared_x.Size();
-      auto shared_x_ptr = shared_x.Read();
-      MFEM_FORALL(i,shared_size,
-      {
-         dg_x_ptr[local_size+i] = shared_x_ptr[i];
-      });
-      ParBilinearForm *pform = nullptr;
-      if ((pform = dynamic_cast<ParBilinearForm*>(a)) && (pform->keep_nbr_block))
-      {
-         mat->Mult(dg_x, dg_y);
-         // DG Restriction
-         auto dg_y_ptr = dg_y.Read();
-         auto y_ptr = y.ReadWrite();
-         MFEM_FORALL(i,local_size,
-         {
-            y_ptr[i] += dg_y_ptr[i];
-         });
-      }
-      else
-      {
-         mat->Mult(dg_x, y);
+         I_face[0] = 0;
      }
   }
-   else
-#endif
+   else // continuous Galerkin case
   {
-      mat->Mult(x, y);
+      const ElementRestriction &rest =
+         static_cast<const ElementRestriction&>(*elem_restrict);
+      rest.FillSparseMatrix(ea_data, mat);
   }
 }

 void FABilinearFormExtension::Mult(const Vector &x, Vector &y) const
 {
-   if ( a->GetFBFI()->Size()>0 )
-   {
-      DGMult(x, y);
-   }
-   else
-   {
-      mat->Mult(x, y);
-   }
-}
-
-void FABilinearFormExtension::DGMultTranspose(const Vector &x, Vector &y) const
-{
+   mat.Mult(x, y);
 #ifdef MFEM_USE_MPI
-   const ParFiniteElementSpace *pfes;
-   if ( (pfes = dynamic_cast<const ParFiniteElementSpace*>(testFes)) )
+   if (const ParFiniteElementSpace *pfes =
+          dynamic_cast<const ParFiniteElementSpace*>(testFes))
   {
-      // DG Prolongation
      ParGridFunction x_gf;
      x_gf.MakeRef(const_cast<ParFiniteElementSpace*>(pfes),
                   const_cast<Vector&>(x),0);
      x_gf.ExchangeFaceNbrData();
      Vector &shared_x = x_gf.FaceNbrData();
-      const int local_size = a->FESpace()->GetVSize();
-      auto dg_x_ptr = dg_x.Write();
-      auto x_ptr = x.Read();
-      MFEM_FORALL(i,local_size,
-      {
-         dg_x_ptr[i] = x_ptr[i];
-      });
-      const int shared_size = shared_x.Size();
-      auto shared_x_ptr = shared_x.Read();
-      MFEM_FORALL(i,shared_size,
-      {
-         dg_x_ptr[local_size+i] = shared_x_ptr[i];
-      });
-      ParBilinearForm *pb = nullptr;
-      if ((pb = dynamic_cast<ParBilinearForm*>(a)) && (pb->keep_nbr_block))
-      {
-         mat->MultTranspose(dg_x, dg_y);
-         // DG Restriction
-         auto dg_y_ptr = dg_y.Read();
-         auto y_ptr = y.ReadWrite();
-         MFEM_FORALL(i,local_size,
-         {
-            y_ptr[i] += dg_y_ptr[i];
-         });
-      }
-      else
-      {
-         mat->MultTranspose(dg_x, y);
-      }
+      if (shared_x.Size()) { face_mat.AddMult(shared_x, y); }
   }
-   else
 #endif
-   {
-      mat->MultTranspose(x, y);
-   }
 }

 void FABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
 {
-   if ( a->GetFBFI()->Size()>0 )
+   mat.MultTranspose(x, y);
+#ifdef MFEM_USE_MPI
+   if (const ParFiniteElementSpace *pfes =
+          dynamic_cast<const ParFiniteElementSpace*>(testFes))
   {
-      DGMultTranspose(x, y);
-   }
-   else
-   {
-      mat->MultTranspose(x, y);
+      ParGridFunction x_gf;
+      x_gf.MakeRef(const_cast<ParFiniteElementSpace*>(pfes),
+                   const_cast<Vector&>(x),0);
+      x_gf.ExchangeFaceNbrData();
+      Vector &shared_x = x_gf.FaceNbrData();
+      if (shared_x.Size()) { face_mat.AddMultTranspose(shared_x, y); }
   }
+#endif
 }


@@ -1,4 +1,4 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
 // at the Lawrence Livermore National Laboratory. All Rights reserved. See files
 // LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
@@ -118,8 +118,10 @@ public:
 class FABilinearFormExtension : public EABilinearFormExtension
 {
 private:
-   SparseMatrix *mat;
-   mutable Vector dg_x, dg_y;
+   SparseMatrix mat;
+   /// face_mat handles parallelism for DG face terms.
+   SparseMatrix face_mat;
+   bool use_face_mat;

 public:
   FABilinearFormExtension(BilinearForm *form);
@@ -127,11 +129,6 @@ public:
   void Assemble();
   void Mult(const Vector &x, Vector &y) const;
   void MultTranspose(const Vector &x, Vector &y) const;
-
-   /** DGMult and DGMultTranspose use the extended L-vector to perform the
-       computation. */
-   void DGMult(const Vector &x, Vector &y) const;
-   void DGMultTranspose(const Vector &x, Vector &y) const;
 };

 /// Data and methods for matrix-free bilinear forms
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
blaz	cd6eca6ada	modifications	2021-02-24 13:54:33 -08:00
blaz	e823646017	Merge remote-tracking branch 'origin/master' into PrmBlockNonlinearForm	2021-01-26 15:35:17 -08:00
blaz	c0b3dc0097	fixing bugs	2020-12-29 19:35:48 -08:00
blaz	108956ea49	serial example	2020-12-23 12:37:18 -08:00
blaz	b581a194e2	bug cleaning	2020-12-23 12:36:16 -08:00
blaz	dc044d8e93	new features	2020-12-21 14:49:58 -08:00
blaz	a614cac02c	Initial implementation of parametric block nonlinear form	2020-12-16 09:45:19 -08:00
blaz	b01b4956cf	Initial interface definition for the Parametric Block integrators	2020-12-10 18:44:46 -08:00