Some mods for compiling

minor mods for MFEM
Added framework for IMEX BDF and RK; not included in MFEM config/setup yet
2021-08-18 09:42:36 -06:00 · 2021-08-18 09:08:24 -06:00 · 2021-08-18 09:03:18 -06:00 · 2021-08-15 18:16:37 -07:00 · 2021-08-15 18:16:01 -07:00 · 2021-08-10 18:09:33 -07:00
182 changed files with 10827 additions and 2527 deletions
@@ -47,26 +47,36 @@ jobs:
    strategy:
      matrix:
        os: [ubuntu-18.04, macos-10.15]
-        target: [debug, optim]
-        mpi: [sequential, parallel]
+        target: [dbg, opt]
+        mpi: [seq, par]
        build-system: [make]
-        # 'include' allows us to
-        # - add a variable without creating a new matrix dimension.
-        # - add a new combination ('build-system: cmake' case here)
+        hypre-target: [int32]
+        # 'include' allows us to:
+        # - Add a variable to all jobs without creating a new matrix dimension.
+        #   Codecov is defined that way.
+        # - Add a new combination.
+        #   'build-system: cmake' and 'hypre-target: int64'
        #
        # note: we will gather coverage info for any non-debug run except the
        # CMake build.
        include:
-          - target: debug
+          - target: dbg
            codecov: NO
-          - target: optim
+          - target: opt
            codecov: YES
          - os: ubuntu-18.04
-            target: optim
+            target: opt
            codecov: NO
-            mpi: parallel
+            mpi: par
            build-system: cmake
-    name: ${{ matrix.os }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.build-system }}
+            hypre-target: int32
+          - os: ubuntu-18.04
+            target: opt
+            codecov: NO
+            mpi: par
+            build-system: make
+            hypre-target: int64
+    name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}

    runs-on: ${{ matrix.os }}

@@ -92,7 +102,7 @@ jobs:
    # TODO: It would be nice to have only one step, e.g. with a dedicated
    # action, but I (@adrienbernede) don't see how at the moment.
    - name: get MPI (Linux)
-      if: matrix.mpi == 'parallel' && matrix.os == 'ubuntu-18.04'
+      if: matrix.mpi == 'par' && matrix.os == 'ubuntu-18.04'
      run: |
        sudo apt-get install mpich libmpich-dev
        export MAKE_CXX_FLAG="MPICXX=mpic++"
@@ -103,11 +113,11 @@ jobs:
        sudo apt-get install lcov

    - name: Set up Homebrew
-      if: ( matrix.mpi == 'parallel' || matrix.codecov == 'YES' ) && matrix.os == 'macos-10.15'
+      if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-10.15'
      uses: Homebrew/actions/setup-homebrew@c4aafe8c4620bf08883dd4679c374f11e73329d3

    - name: get MPI (MacOS)
-      if: matrix.mpi == 'parallel' && matrix.os == 'macos-10.15'
+      if: matrix.mpi == 'par' && matrix.os == 'macos-10.15'
      run: |
        export HOMEBREW_NO_INSTALL_CLEANUP=1
        brew install openmpi
@@ -123,39 +133,40 @@ jobs:
    # Install will only run on cache miss.
    - name: cache hypre
      id: hypre-cache
-      if: matrix.mpi == 'parallel'
+      if: matrix.mpi == 'par'
      uses: actions/cache@v2
      with:
        path: ${{ env.HYPRE_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2
+        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.0

    - name: get hypre
-      if: matrix.mpi == 'parallel' && steps.hypre-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-hypre@v1.0
+      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true'
+      uses: mfem/github-actions/build-hypre@v2.0
      with:
-        hypre-archive: ${{ env.HYPRE_ARCHIVE }}
-        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
+        archive: ${{ env.HYPRE_ARCHIVE }}
+        dir: ${{ env.HYPRE_TOP_DIR }}
+        target: ${{ matrix.hypre-target }}

    # Get Metis through cache, or build it.
    # Install will only run on cache miss.
    - name: cache metis
      id: metis-cache
-      if: matrix.mpi == 'parallel'
+      if: matrix.mpi == 'par'
      uses: actions/cache@v2
      with:
        path: ${{ env.METIS_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2
+        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.0

    - name: install metis
-      if: matrix.mpi == 'parallel' && steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@v1.0
+      if: matrix.mpi == 'par' && steps.metis-cache.outputs.cache-hit != 'true'
+      uses: mfem/github-actions/build-metis@v2.0
      with:
-        metis-archive: ${{ env.METIS_ARCHIVE }}
-        metis-dir: ${{ env.METIS_TOP_DIR }}
+        archive: ${{ env.METIS_ARCHIVE }}
+        dir: ${{ env.METIS_TOP_DIR }}

    # MFEM build and test
    - name: build
-      uses: mfem/github-actions/build-mfem@v1.0
+      uses: mfem/github-actions/build-mfem@v2.0
      with:
        os: ${{ matrix.os }}
        target: ${{ matrix.target }}
@@ -168,17 +179,17 @@ jobs:

    # Run checks (and only checks) on debug targets
    - name: checks
-      if: matrix.build-system == 'make' && matrix.target == 'debug'
+      if: matrix.build-system == 'make' && matrix.target == 'dbg'
      run: |
        cd ${{ env.MFEM_TOP_DIR }}  && make check

    - name: unit tests
-      if: matrix.build-system == 'make' && matrix.target == 'optim'
+      if: matrix.build-system == 'make' && matrix.target == 'opt'
      run: |
        cd ${{ env.MFEM_TOP_DIR }} && make unittest

    - name: tests
-      if: matrix.build-system == 'make' && matrix.target == 'optim'
+      if: matrix.build-system == 'make' && matrix.target == 'opt'
      run: |
        cd ${{ env.MFEM_TOP_DIR }} && make test

@@ -190,8 +201,8 @@ jobs:
    # Code coverage (process and upload reports)
    - name: codecov
      if: matrix.codecov == 'YES'
-      uses: mfem/github-actions/upload-coverage@v1.0
+      uses: mfem/github-actions/upload-coverage@v2.0
      with:
-        name: ${{ matrix.os }}-${{ matrix.mpi }}
+        name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
        project_dir: ${{ env.MFEM_TOP_DIR }}
        directories: "fem general linalg mesh"
@@ -53,32 +53,33 @@ jobs:
      uses: actions/cache@v2
      with:
        path: ${{ env.HYPRE_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2
+        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.0

    - name: Get Hypre
      if: steps.hypre-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-hypre@master
+      uses: mfem/github-actions/build-hypre@v2.0
      with:
-        hypre-archive: ${{ env.HYPRE_ARCHIVE }}
-        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
+        archive: ${{ env.HYPRE_ARCHIVE }}
+        dir: ${{ env.HYPRE_TOP_DIR }}
+        target: int32

    - name: Cache Metis Install
      id: metis-cache
      uses: actions/cache@v2
      with:
        path: ${{ env.METIS_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2
+        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.0

    - name: Install Metis
      if: steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@master
+      uses: mfem/github-actions/build-metis@v2.0
      with:
-        metis-archive: ${{ env.METIS_ARCHIVE }}
-        metis-dir: ${{ env.METIS_TOP_DIR }}
+        archive: ${{ env.METIS_ARCHIVE }}
+        dir: ${{ env.METIS_TOP_DIR }}

    # MFEM build and test
    - name: build-mfem
-      uses: mfem/github-actions/build-mfem@master
+      uses: mfem/github-actions/build-mfem@v2.0
      with:
        os: ${{ runner.os }}
        target: optim
@@ -28,49 +28,24 @@ jobs:
        access_token: ${{ github.token }}
    - name: checkout mfem
      uses: actions/checkout@v2
-      with:
-        path: mfem

    - name: copyright check
      id: copyright
      run: |
-        cd mfem
-        if git grep -l "^\(#\|//\).*\(\-2020\|\ 2010,\)" > matches.txt
-        then
-          echo "Please update the following files to Copyright (c) 2010-2021:"
-          cat matches.txt
-          exit 1
-        else
-          echo "No outdated copyright found."
-        fi
+        ./config/githooks/pre-push --copyright
+
      continue-on-error: true

    - name: license check
      id: license
      run: |
-        cd mfem
-        if git grep -li "^\(#\|//\).*GNU\ Lesser\ General\ Public\ License" > matches.txt
-        then
-          echo "Please update the following files to the BSD-3 license:"
-          cat matches.txt
-          exit 1
-        else
-          echo "No GNU GPL license found."
-        fi
+        ./config/githooks/pre-push --license
      continue-on-error: true

    - name: release check
      id: release
      run: |
-        cd mfem
-        if git grep -l "^\(#\|//\).*LLNL\-CODE\-443211" > matches.txt
-        then
-          echo "Please update the following files to LLNL-CODE-806117:"
-          cat matches.txt
-          exit 1
-        else
-          echo "No outdated release number found."
-        fi
+        ./config/githooks/pre-push --release
      continue-on-error: true

    - name: wrap-up
@@ -100,8 +75,7 @@ jobs:

    - name: style check
      run: |
-        cd tests/scripts
-        ./runtest code-style
+        ./config/githooks/pre-push --style

  documentation:
    runs-on: ubuntu-18.04
@@ -133,5 +107,4 @@ jobs:
      run: |
        git fetch origin master:master
        git checkout -b gh-actions-branch-history
-        cd tests/scripts
-        ./runtest branch-history
+        ./config/githooks/pre-push --history
@@ -26,6 +26,7 @@ CMakeFiles/
 config/_config.hpp
 config/config.mk
 config/sample-runs-build.log
+config/user.mk
 doc/CodeDocumentation.conf
 doc/CodeDocumentation.html
 doc/CodeDocumentation
@@ -251,6 +252,7 @@ miniapps/shifted/ParaViewDistance
 miniapps/shifted/diffusion
 miniapps/shifted/diffusion.mesh
 miniapps/shifted/diffusion.gf
+miniapps/shifted/ParaViewDiffusion

 miniapps/tools/display-basis
 miniapps/tools/load-dc
@@ -295,6 +297,7 @@ tests/unit/psedov_tests_*
 tests/unit/tmop_pa_tests_*
 tests/unit/ptmop_pa_tests_*
 tests/unit/ceed_tests
+tests/unit/debug_device_tests

 # Test script output
 tests/scripts/*.err
@@ -48,39 +48,58 @@ variables:
  TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
  TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
  AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
+  MFEM_DATA_REPO: https://github.com/mfem/data.git
  ARTIFACTS_DIR: artifacts

-# The pipeline is divided into stages. Usually, these are also synchronization
-# points, however, we use "needs" keyword to express the DAG of jobs for more
-# efficiency.
-# - We use setup phase to download content outside of mfem directory.
+# The pipeline is divided into stages. Usually, jobs in a given stage wait for
+# the preceding stages to complete before to start. However, we sometimes use
+# the "needs" keyword and express the DAG of jobs for more efficiency.
+# - We use setup and setup_baseline phases to download content outside of mfem
+#   directory.
 # - Allocate/Release is where quartz resources are allocated/released once for all.
 # - Build and Test is where we build and MFEM for multiple toolchains.
 # - Baseline_checks gathers baseline-type test suites execution
 # - Baseline_publish, only available on master, allows to update baseline
 #   results
 stages:
+  - setup
  - q_allocate_resources
  - q_build_and_test
  - q_release_resources
  - l_build_and_test
  - c_build_and_test
-  - setup
+  - setup_baseline
  - baseline_check
  - baseline_to_autotest
  - baseline_publish

-# The setup job in setup stage don't rely on MFEM git repo. It prepares a
-# pipeline-wide working directory downloading/updating external repos.
-# TODO: updating tests and tpls is not necessary anymore since pipelines are
-# now using unique directories so repo are never shared with another pipeline.
-# This is not memory efficient (we keep a lot of data), hence this reminder.
-# Setup
+# setup clones the mfem/data repo in ${BUILD_ROOT}. The build_and_test script
+# then symlinks the repo to the parent directory of the MFEM source directory.
+# Unit tests that depend on the mfem/data repo will then detect that this
+# directory is present and be enabled.
 setup:
  tags:
    - shell
    - quartz
  stage: setup
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
+    - if [ ! -d data ]; then git clone ${MFEM_DATA_REPO}; fi
+
+# The setup_baseline job in setup stage_baseline doesn't rely on MFEM git repo.
+# It prepares a pipeline-wide working directory downloading/updating external
+# repos. TODO: updating tests and tpls is not necessary anymore since pipelines
+# are now using unique directories so repo are never shared with another
+# pipeline. This is not memory efficient (we keep a lot of data), hence this
+# reminder.
+# Note: This job can start immediately.
+setup_baseline:
+  tags:
+    - shell
+    - quartz
+  stage: setup_baseline
  variables:
    GIT_STRATEGY: none
  script:
@@ -106,10 +125,10 @@ setup:
  script:
    - srun -p mi60 -t 15 -N 1 tests/gitlab/build_and_test

-# Lassen uses a different job scheduler (spectrum lsf) that does not
-# allow pre-allocation the same way slurm does.
-# We use pdebug queue on lassen to speed-up the allocation.
-# However this would not be scalable to multiple builds.
+# Lassen uses a different job scheduler (spectrum lsf) that does not allow
+# pre-allocation the same way slurm does.  We use pdebug queue on lassen to
+# speed-up the allocation.  However this would not be scalable to multiple
+# builds.
 .build_blueos_3_ppc64le_ib_script:
  script:
    - lalloc 1 -W 30 -q pdebug tests/gitlab/build_and_test
@@ -22,12 +22,13 @@

 # Spack helped builds
 # Generic lassen build job, extending build script
+# Note: Lassen jobs can start as soon as the setup job is complete.
 .build_and_test_on_lassen:
  extends: [.build_blueos_3_ppc64le_ib_script, .on_lassen]
  stage: l_build_and_test
-  needs: []
+  needs: [setup]

 opt_mpi_cuda_xl_16_1_1_8:
  variables:
-    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=sm_70"
+    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=70"
  extends: .build_and_test_on_lassen
@@ -16,13 +16,13 @@
    - shell
    - quartz
  rules:
-    # Don’t run quartz jobs if...
+    # Don't run quartz jobs if...
    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"'
      when: never
-    # Don’t run autotest update if...
+    # Don't run autotest update if...
    - if: '$CI_JOB_NAME =~ /update_autotest/ && $AUTOTEST != "YES"'
      when: never
-    # Don’t run autotest update if...
+    # Don't run autotest update if...
    - if: '$CI_JOB_NAME =~ /q_report/ && $AUTOTEST != "YES"'
      when: never
    # Report success on success status
@@ -37,6 +37,18 @@
    # Default is to run if previous stage succeeded
    - when: on_success

+# This is a yaml anchor, it can be used to avoid duplication like here.
+# The code below will simply be pasted wherever the anchor is placed.
+.safe_create_rundir: &safe_create_rundir |
+  if ! mkdir ${rundir}; then
+    n=1
+    while ! mkdir ${rundir}_${n}
+    do
+      n=$((n+1))
+    done
+    rundir=${rundir}_${n}
+  fi
+
 # Allocate
 q_allocate_resources:
  variables:
@@ -65,10 +77,11 @@ q_report_success:
  stage: q_release_resources
  script:
    - echo "Can only run if all the quartz jobs passed"
-    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
    - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - mkdir -p ${rundir}
+    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
+    - *safe_create_rundir
    - echo "The Quartz jobs were successful" > ${rundir}/gitlab.out
+    - echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
    - git add ${rundir}
    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
    - git push origin master
@@ -80,10 +93,11 @@ q_report_failure:
  stage: q_release_resources
  script:
    - echo "Runs if there was at least one failure on quartz"
-    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
    - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - mkdir -p ${rundir}
+    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
+    - *safe_create_rundir
    - echo "There was an error while running CI on Quartz" > ${rundir}/gitlab.err
+    - echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
    - cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
    - git add ${rundir}
    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
@@ -136,24 +150,29 @@ opt_par_gcc_6_1_0_pumi:
    SPEC: "%gcc@6.1.0 +pumi"
  extends: .build_and_test_on_quartz

-# Baseline
+# Baseline jobs form an independent set of jobs. We use `needs:[]` to specify
+# that "setup-baseline" can start immediately. Then, we have to use needs for
+# each one of the baseline jobs, otherwise they will wait for the rest of the
+# pipeline.
+
 baselinecheck_mfem_intel_quartz:
  extends: [.baselinecheck_mfem, .on_quartz]
-  needs: [setup]
+  needs: [setup_baseline]

 update_autotest:
  extends: [.on_quartz]
  needs: [baselinecheck_mfem_intel_quartz]
  stage: baseline_to_autotest
  script:
-    - rundir="quartz/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
    - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - mkdir -p ${rundir}
+    - rundir="quartz/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
+    - *safe_create_rundir
    - cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
    # We create an autotest-email.html file, because that's how we signal that there was a diff (temporary).
    - |
      if [[ -f ${rundir}/*.err ]]
      then
+        echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/*.err
        cp ${rundir}/*.err ${rundir}/autotest-email.html
      fi
    - git add ${rundir}
@@ -1,469 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-language: cpp
-
-os: linux
-dist: bionic
-
-stages:
-  - checks
-  - tests
-  - optional
-
-env:
-  global:
-    - HYPRE_ARCHIVE=v2.19.0.tar.gz
-      HYPRE_URL=https://github.com/hypre-space/hypre/archive/$HYPRE_ARCHIVE
-      HYPRE_TOP_DIR=hypre-2.19.0
-
-jobs:
-  include:
-
-    # ========================
-    #         Checks
-    # ========================
-    # - code-style
-    # - documentation
-    # - gitignore
-
-    - stage: checks
-      os: linux
-      dist: xenial
-      name: "code-style"
-      addons:
-        apt:
-          packages:
-            - astyle=2.05.1-0ubuntu1
-      script:
-        - cd ${TRAVIS_BUILD_DIR}
-        - cd tests/scripts
-        - ./runtest code-style
-
-    - stage: checks
-      os: linux
-      name: "documentation"
-      addons:
-        apt:
-          packages:
-            - doxygen
-            - graphviz
-      script:
-        - cd ${TRAVIS_BUILD_DIR}
-        - cd tests/scripts
-        - ./runtest documentation
-
-    - stage: checks
-      os: linux
-      name: "gitignore"
-      addons:
-        apt:
-          packages:
-            - mpich
-            - libmpich-dev
-      env: MPI=YES
-      before_script:
-        - cd ${TRAVIS_BUILD_DIR}
-        - mpicxx -v
-        - make config MFEM_USE_MPI=YES MFEM_MPI_NP=2
-        - make all -j3
-        - make test-noclean
-      script:
-        - cd tests/scripts
-        - ./runtest gitignore
-      cache:
-        ccache: true
-        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
-          - $TRAVIS_BUILD_DIR/../metis-4.0
-      before_cache:
-        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
-          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
-          rm -f Lib/*.{c,o}
-
-    # ========================
-    #  Optional Checks/Tests
-    # ========================
-    # - branch-history
-
-    - stage: optional
-      name: "branch-history"
-      if: branch != next
-      # need full git history for the binary/big files check
-      git:
-        depth: false
-      script:
-        - cd ${TRAVIS_BUILD_DIR}
-        # update master
-        - git fetch origin master:master
-        # checkout a branch (otherwise Travis works in detached head)
-        - git checkout -b travis_tests
-        - cd tests/scripts
-        - ./runtest branch-history
-
-    # ========================
-    #       Linux tests
-    # ========================
-    # - serial + debug
-    # - serial
-    # - parallel + debug
-    # - parallel
-
-    - stage: tests
-      os: linux
-      compiler: gcc
-      name: "Linux: Serial + Debug"
-      env: DEBUG=YES
-           MPI=NO
-           CODECOV=NO
-           MFEM_TEST_TARGET=check
-      cache:
-        ccache: true
-
-    - os: linux
-      compiler: gcc
-      name: "Linux: Serial"
-      env: DEBUG=NO
-           MPI=NO
-           CODECOV=NO
-           MFEM_TEST_TARGET=test
-      cache:
-        ccache: true
-
-    - os: linux
-      compiler: gcc
-      name: "Linux: Parallel + Debug"
-      addons:
-        apt:
-          # sources:
-            # - ubuntu-toolchain-r-test
-          packages:
-            # GCC 4.9
-            # - g++-4.9
-            # MPICH
-            - mpich
-            - libmpich-dev
-            # OpenMPI
-            # - openmpi-bin
-            # - libopenmpi-dev
-      env: DEBUG=YES
-           MPI=YES
-           CODECOV=NO
-           MFEM_TEST_TARGET=check
-           NPROCS=2
-      cache:
-        ccache: true
-        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
-          - $TRAVIS_BUILD_DIR/../metis-4.0
-      before_cache:
-        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
-          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
-          rm -f Lib/*.{c,o}
-
-    - os: linux
-      compiler: gcc
-      name: "Linux: Parallel"
-      addons:
-        apt:
-          # sources:
-            # - ubuntu-toolchain-r-test
-          packages:
-            # GCC 4.9
-            # - g++-4.9
-            # MPICH
-            - mpich
-            - libmpich-dev
-            # OpenMPI
-            # - openmpi-bin
-            # - libopenmpi-dev
-      env: DEBUG=NO
-           MPI=YES
-           CODECOV=YES
-           MFEM_TEST_TARGET=test
-           NPROCS=2
-      cache:
-        ccache: true
-        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
-          - $TRAVIS_BUILD_DIR/../metis-4.0
-      before_cache:
-        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
-          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
-          rm -f Lib/*.{c,o}
-
-    - os: linux
-      compiler: gcc
-      name: "Linux: Parallel (cmake)"
-      addons:
-        apt:
-          packages:
-            - mpich
-            - libmpich-dev
-      env: MPI=YES
-           NPROCS=2
-      script:
-        - cd ${TRAVIS_BUILD_DIR}
-        - mkdir ${TRAVIS_BUILD_DIR}/build
-        - cd ${TRAVIS_BUILD_DIR}/build
-        - cmake ..
-             -DMFEM_USE_MPI=ON
-             -DHYPRE_DIR=${TRAVIS_BUILD_DIR}/../$HYPRE_TOP_DIR/src/hypre
-             -DMFEM_MPI_NP=$NPROCS
-        - make -j3 mfem examples
-        - cd ${TRAVIS_BUILD_DIR}/build/tests/unit
-        - make -j3
-        - ctest --output-on-failure
-      cache:
-        ccache: true
-        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
-          - $TRAVIS_BUILD_DIR/../metis-4.0
-      before_cache:
-        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
-          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
-          rm -f Lib/*.{c,o}
-
-    # ========================
-    #      Mac OS X tests
-    # ========================
-    # - serial + debug
-    # - serial
-    # - parallel + debug
-    # - parallel
-
-    - os: osx
-      osx_image: xcode11.2
-      compiler: clang
-      name: "Mac: Serial + Debug"
-      addons:
-        homebrew:
-          packages:
-            - ccache
-      env: DEBUG=YES
-           MPI=NO
-           CODECOV=NO
-           MFEM_TEST_TARGET=check
-      cache:
-        ccache: true
-
-    - os: osx
-      osx_image: xcode11.2
-      compiler: clang
-      name: "Mac: Serial"
-      addons:
-        homebrew:
-          packages:
-            - ccache
-      env: DEBUG=NO
-           MPI=NO
-           CODECOV=NO
-           MFEM_TEST_TARGET=test
-      cache:
-        ccache: true
-
-    - os: osx
-      osx_image: xcode11.2
-      compiler: clang
-      name: "Mac: Parallel + Debug"
-      addons:
-        homebrew:
-          packages:
-            - ccache
-      env: DEBUG=YES
-           MPI=YES
-           CODECOV=NO
-           MFEM_TEST_TARGET=check
-           NPROCS=4
-           TMPDIR=/tmp
-      cache:
-        ccache: true
-        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
-          - $TRAVIS_BUILD_DIR/../metis-4.0
-          - $HOME/local-cached
-      before_cache:
-        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
-          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
-          rm -f Lib/*.{c,o}
-
-    - os: osx
-      osx_image: xcode11.2
-      compiler: clang
-      name: "Mac: Parallel"
-      addons:
-        homebrew:
-          packages:
-            - ccache
-      env: DEBUG=NO
-           MPI=YES
-           CODECOV=YES
-           MFEM_TEST_TARGET=test
-           NPROCS=4
-           TMPDIR=/tmp
-      cache:
-        ccache: true
-        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
-          - $TRAVIS_BUILD_DIR/../metis-4.0
-          - $HOME/local-cached
-      before_cache:
-        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
-          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
-          rm -f Lib/*.{c,o}
-
-before_install:
-   # No addon for brew yet, have to install OSX packages this way.
-   # - if [ $TRAVIS_OS_NAME == "osx" ] && [ $MPI == "YES" ]; then
-   #      brew install open-mpi;
-   #   fi
-
-   # Disable ccache while building dependencies that are cached:
-   - echo "before \$PATH = $PATH";
-     export PATH=${PATH//\/usr\/lib\/ccache:/};
-     echo "after  \$PATH = $PATH"
-
-   # On Mac OS X, build and cache OpenMPI 2.1.6:
-   - if [ $TRAVIS_OS_NAME == "osx" ] && [ $MPI == "YES" ]; then
-        if [ ! -e $HOME/local-cached/bin/mpicc ]; then
-           mkdir -p $HOME/builds && cd $HOME/builds &&
-           wget https://download.open-mpi.org/release/open-mpi/v2.1/openmpi-2.1.6.tar.bz2 &&
-           tar jxf openmpi-2.1.6.tar.bz2 &&
-           mkdir openmpi-build && cd openmpi-build &&
-           ../openmpi-2.1.6/configure --prefix=$HOME/local-cached &&
-           make -j3 all && make install;
-        fi;
-        PATH=$HOME/local-cached/bin:$PATH;
-        cd $TRAVIS_BUILD_DIR;
-     fi
-
-   # Update environment to find g++ 4.9 installation first.
-   # - if [ $TRAVIS_OS_NAME == "linux" ]; then
-   #      mkdir -p latest-gcc-symlinks;
-   #      ln -s /usr/bin/g++-4.9 latest-gcc-symlinks/g++;
-   #      ln -s /usr/bin/gcc-4.9 latest-gcc-symlinks/gcc;
-   #      ln -s /usr/bin/gcov-4.9 latest-gcc-symlinks/gcov;
-   #      export PATH=$PWD/latest-gcc-symlinks:$PATH;
-   #   fi
-
-   # Install tool to upload code coverage reports to coveralls.io
-   - if [ "$CODECOV" == "YES" ]; then
-        export PYTHONUSERBASE=$HOME/local;
-        pip install --user cpp-coveralls;
-        pip install --user pyyaml;
-        PATH=$HOME/local/bin:$PATH;
-     fi
-
-install:
-   # Set MPI compilers, print compiler version
-   - if [ $MPI == "YES" ]; then
-        if [ "$TRAVIS_OS_NAME" == "linux" ]; then
-           export MPICH_CC="$CC";
-           export MPICH_CXX="$CXX";
-        else
-           export OMPI_CC="$CC";
-           export OMPI_CXX="$CXX";
-           mpic++ --showme:version;
-        fi;
-        mpic++ -v;
-     else
-        $CXX -v;
-     fi
-
-   # Back out of the mfem directory to install the libraries
-   - cd ..
-
-   # hypre
-   - if [ $MPI == "YES" ]; then
-        if [ ! -e $HYPRE_TOP_DIR/src/hypre/lib/libHYPRE.a ]; then
-           wget $HYPRE_URL;
-           rm -rf $HYPRE_TOP_DIR;
-           tar xvzf $HYPRE_ARCHIVE;
-           cd $HYPRE_TOP_DIR/src;
-           ./configure --disable-fortran CC=mpicc CXX=mpic++;
-           make -j3;
-           cd ../..;
-        else
-            echo "Reusing cached $HYPRE_TOP_DIR/";
-        fi;
-        ln -s $HYPRE_TOP_DIR hypre;
-     else
-        echo "Serial build, not using hypre";
-     fi
-
-   # METIS, use a mirror because the original source server is not always up.
-   # Original url:
-   #    http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/OLD/metis-4.0.3.tar.gz
-   - if [ $MPI == "YES" ]; then
-        if [ ! -e metis-4.0/libmetis.a ]; then
-           wget https://mfem.github.io/tpls/metis-4.0.3.tar.gz;
-           tar xvzf metis-4.0.3.tar.gz;
-           make -j3 -C metis-4.0.3/Lib CC="$CC" OPTFLAGS="-O2";
-           rm -rf metis-4.0;
-           mv metis-4.0.3 metis-4.0;
-        else
-           echo "Reusing cached metis-4.0/";
-        fi;
-     fi
-
-   # Re-enable ccache on linux; enable ccache on mac os:
-   - if [ $TRAVIS_OS_NAME == "linux" ]; then
-        export PATH="/usr/lib/ccache:$PATH";
-     else
-        if [ $TRAVIS_OS_NAME == "osx" ]; then
-           export PATH="/usr/local/opt/ccache/libexec:$PATH";
-        fi;
-     fi
-
-   - printf "which \$CC  = "; which $CC;
-     printf "which \$CXX = "; which $CXX
-
-script:
-   # Compiler
-   - if [ $MPI == "YES" ]; then
-        export MYCXX=mpic++;
-        export MAKE_CXX_FLAG=MPICXX=$MYCXX;
-     else
-        export MYCXX="$CXX";
-        export MAKE_CXX_FLAG=CXX=$MYCXX;
-     fi
-
-   # Print the compiler version
-   - $MYCXX -v
-
-   # Set some variables
-   - cd $TRAVIS_BUILD_DIR;
-     CPPFLAGS="";
-     SKIP_TEST_DIRS="";
-     if [ "$CODECOV" == "YES" ]; then
-        CPPFLAGS="--coverage -g";
-     fi;
-     if [ "$TRAVIS_OS_NAME" != "linux" ] || [ "$DEBUG" == "YES" ]; then
-        CPPFLAGS+=" -pedantic -Wall -Werror";
-     fi
-
-   # Configure the library
-   - make config MFEM_USE_MPI=$MPI MFEM_DEBUG=$DEBUG $MAKE_CXX_FLAG
-        MFEM_MPI_NP=$NPROCS CPPFLAGS="$CPPFLAGS"
-   # Show the configuration
-   - make info
-   # Build the library
-   - make -j3
-   # Build the examples and the miniapps
-   - make -j3 all
-   # Run tests
-   - make $MFEM_TEST_TARGET SKIP_TEST_DIRS="$SKIP_TEST_DIRS"
-
-after_success:
-   - if [ "$CODECOV" == "YES" ]; then
-        coveralls --include fem --include general --include linalg --include
-           mesh --exclude /usr --gcov-options '\-lp' --root $TRAVIS_BUILD_DIR;
-     fi
@@ -8,40 +8,72 @@
                               https://mfem.org


-Version 4.2.1 (development)
+Version 4.3.1 (development)
 ===========================
- Added initial support for GPU-accelerated versions of PETSc that works with
-  MFEM_USE_CUDA if PETSc has been configured with CUDA support. Examples 1 and 9
-  in the examples/petsc directory have been modified to work with --device cuda.
-  Examples with GAMG (ex1p) and SLEPc (ex11p) are also provided.
+- Added support for hr-adaptivity using TMOP-based error estimator.

- Memory management:
-  * Added method Device::SetMemoryTypes that can be used to change the default
-    host and device MemoryTypes before Device setup.
-  * In class MemoryManager, added methods GetDualMemoryType and
-    SetDualMemoryType; dual MemoryTypes are used to determine the second
-    MemoryType (host or device) when only one MemoryType is specified in methods
-    of class Memory.
-  * Added Memory constructor for setting both the host and device MemoryTypes.
-  * Switched the default behavior of device memory allocations so that they
-    are deferred until the device pointer is needed.
-  * Added a second Umpire device MemoryType, DEVICE_UMPIRE_2, with
-    corresponding allocator that can be set with the method
-    MemoryManager::SetUmpireDevice2AllocatorName.
-  * Added HOST_PINNED MemoryType and a pinned host allocator for CUDA and HIP.

- Added support for Caliper: a library to integrate performance profiling
-  capabilities into applications. See examples/caliper for more details.
+Version 4.3, released on July 29, 2021
+======================================

- Added support for explicit vectorization in the high-performance templated
-  code for Fujitsu's A64FX ARM microprocessor architecture.
+Discretization improvements
+---------------------------
+- Variable order spaces, p- and hp-refinement. This is the initial (serial)
+  support for variable-order FiniteElementCollection and FiniteElementSpace.
+  The new method FiniteElementSpace::SetElementOrder can be called to set an
+  arbitrary order for each mesh element. The conforming interpolation matrix
+  will now automatically constrain p- and hp- interfaces, enabling general
+  hp-refinement in both 2D and 3D, on uniform or mixed NC meshes. Support for
+  parallel variable-order spaces will follow shortly.

- Added AlgebraicCeedSolver that does matrix-free algebraic p-multigrid for
-  diffusion problems with the Ceed backend.
+- Extended the support for field transfer between high-order and low-order
+  refined finite element spaces to include: dual fields and H1 fields (both
+  primary and dual). These are illustrated in the lor-transfer miniapp.

- Introduced new options for the mesh-explorer miniapp to visualize the actual
-  element attributes in parallel meshes while retaining the visualization of
-  the domain decomposition.
+- Improved libCEED integration, including support for VectorCoefficient,
+  ConvectionIntegrator, and VectorConvectionNLFIntegrator with libCEED backends.
+
+- Extending support for L2 basis functions using MapTypes VALUE and INTEGRAL in
+  linear interpolators and GridFunction "GetValue" methods.
+
+- Changed the interface for the error estimator and implemented the Kelly error
+  indicator for scalar-valued problems, supported in serial and parallel builds.
+
+- Added support for the "BR2" discontinuous Galerkin discretization for
+  diffusion via DGDiffusionBR2Integrator (see Example 14/14p).
+
+- Added convective and skew-symmetric integrators for the nonlinear term in the
+  Navier-Stokes equations.
+
+- Added new classes DenseSymmetricMatrix and SymmetricMatrixCoefficient for
+  efficient evaluation of symmetric matrix coefficients. This replaces the now
+  deprecated EvalSymmetric in MatrixCoefficient. Added DiagonalMatrixCoefficient
+  for clarity, which is a typedef of VectorCoefficient.
+
+- Added support for nonscalar coefficient with VectorDiffusionIntegrator.
+
+Linear and nonlinear solvers
+----------------------------
+- Added support for AMG preconditioners on GPUs based on the hypre library
+  (version 2.22.0 or later). These include BoomerAMG, AMS and ADS and most
+  MFEM examples that use hypre have been ported to support this functionality.
+  The GPU preconditioners require that both hypre and MFEM are built with CUDA
+  support. Hypre builds with CUDA and unified memory are also supported and
+  can be used with `-d cuda:uvm` as a command-line option.
+
+- Added support for AMG preconditioners for non-symmetric systems (e.g.
+  advection-dominated problems) using hypre's approximate ideal restriction
+  (AIR) AMG. Requires hypre version 2.14.0 or newer. Usage is illustrated in
+  example 9/9p.
+
+- Added new functionality for constructing low-order refined discretizations and
+  solvers, see the LORDiscretization and LORSolver classes. A new basis type for
+  H(curl) and H(div) spaces is introduced to give spectral equivalence. This
+  functionality is illustrated in the LOR solvers miniapp in miniapps/solvers.
+
+- Generalized the Multigrid class to support non-geometric multigrid. Previous
+  functionality, based on FiniteElementSpaceHierarchy, is now available in the
+  derived class GeometricMultigrid.

 - Introduced solver interface for linear problems with constraints, a few
  concrete solvers that implement the interface, and a demonstration of their
@@ -52,19 +84,18 @@ Version 4.2.1 (development)
  as described in Barker and Kolev 2020 (https://doi.org/10.1002/nla.2348). See
  Example 3p and linalg/auxiliary.?pp.

- Added a new miniapp block-solvers that compares the performance of various
-  solvers for mixed finite element discretization of the second order scalar
-  elliptic equations. Currently available solvers in the miniapp include a
-  block-diagonal preconditioner that is based on approximate Schur complement
-  (implemented in ex5p), and a newly implemented solver DivFreeSolver, which
-  exploits a multilevel decomposition of the Raviart-Thomas space and its
-  divergence-free subspace. See the miniapps/solvers directory for more details.
+- Improved interface for using the Ginkgo library, including: support for matrix-
+  free operators in Ginkgo solvers, new wrappers for Ginkgo preconditioners, HIP
+  support, and reduction of unnecessary data copies.

- Added a new miniapp for computing (signed) distance functions to a point
-  source or zero level set. See miniapps/shifted/distance.cpp.
+- Added initial support for hypre's mixed integer (mixedint) capability, which
+  uses different data types for local and global indices in order to save memory
+  in large problems. This capability requires that hypre was configured with the
+  --enable-mixedint option. Note that this option is currently tested only in
+  ex1p, ex3p, and ex4p, and may not work in more general settings.

- Added matrix-free GPU-enabled implementations of GradientInterpolator and
-  IdentityInterpolator.
+- Added AlgebraicCeedSolver that does matrix-free algebraic p-multigrid for
+  diffusion problems with the Ceed backend.

 - Added interface to MUMPS direct solver. Its usage is demonstrated in ex25p.
  See http://mumps.enseeiht.fr/ for more details. Supported versions >= 5.1.1.
@@ -72,6 +103,17 @@ Version 4.2.1 (development)
 - Added three ESDIRK time integrators: implicit trapezoid rule, L-stable
  ESDIRK-32, and A-stable ESDIRK-33.

+- Implemented a variable step-size IMEX (VSSIMEX) method for the Navier miniapp.
+
+- Implemented an adaptive linear solver tolerance option for NewtonSolver based
+  on the algorithm of Eisenstat and Walker.
+
+Meshing improvements
+--------------------
+- Added support for reading high-order Lagrange meshes in VTK format. Arbitrary-
+  orders and all element types are supported. See the VTK blog for more info:
+  https://blog.kitware.com/wp-content/uploads/2018/09/Source_Issue_43.pdf.
+
 - Introduced a new non-conforming mesh format that fixes known inconsistencies
  of legacy "MFEM mesh v1.1" NC format and works consistently in both serial and
  parallel. ParMesh::ParPrint can now print non-conforming AMR meshes that can
@@ -80,109 +122,26 @@ Version 4.2.1 (development)
  NC data files are compatible with serial code, e.g., can be viewed with serial
  GLVis. Loading of legacy NC mesh files is still supported.

- Added support for 1D non-conforming meshes (which can be useful for parallel
-  load balancing and derefinement).
-
- Added a "scaled Jacobian" visualization option in the Mesh Explorer miniapp to
-  help identify elements with poor mesh quality.
-
- Added support for the "BR2" discontinuous Galerkin discretization for
-  diffusion via DGDiffusionBR2Integrator (see Example 14/14p).
-
- Generalized the Multigrid class to support non-geometric multigrid. The
-  previous functionality, based on FiniteElementSpaceHierarchy, is now available
-  in the derived class GeometricMultigrid.
-
- Upgraded the Catch unit test framework from version 2.13.0 to version 2.13.2.
-
- The TMOP mesh optimization algorithms were extended to GPU:
-   - QualityMetric #1, #2, #7 and #77 are available in 2D, #302, #303, #315
-     and #321 in 3D
-   - Both AnalyticAdaptTC and DiscreteAdaptTC TargetConstructor are available
-   - Kernels for normalization and limiting have been added
-   - The AdvectorCG now also supports AssemblyLevel::PARTIAL
-
- Added a new command line boolean option (`--all`) to the unit tests to launch
-  *all* non-regression tests.
-
- Added support for different modes of QuadratureInterpolator on GPU.
-  The layout (QVectorLayout::byNODES|byVDIM) and the tensor products modes can
-  be enabled before calling the Mult, Values, Derivatives, PhysDerivatives and
-  Determinants methods.
-
- Implemented a filter method for the Navier miniapp to stabilize highly
-  turbulent flows in direct numerical simulation.
-
- Added HIP support to the CMake build system.
-
- Added support for reading high-order Lagrange meshes in VTK format. Arbitrary-
-  orders and all element types are supported. See the VTK blog for more info:
-  https://blog.kitware.com/wp-content/uploads/2018/09/Source_Issue_43.pdf.
-
- Added support for reading VTK meshes in XML format.
-
- Added partial assembly and device support to Example 25/25p, with diagonal
-  preconditioning.
-
- Implemented a variable step-size IMEX (VSSIMEX) method for the Navier miniapp.
+- Added FMS support (https://github.com/CEED/FMS) to mfem. FMS can represent
+  unstructured high-order meshes with general high-order finite element fields
+  on them. When enabled, mfem can convert data collections to/from FMS data
+  collections in memory. In addition, an FMS data collection class was added so
+  the convert-dc miniapp can read and generate data files in FMS format.

 - Added new mesh quality metrics and improved the untangling capabilities of the
  TMOP-based mesh optimization algorithms.

- Added convective and skew-symmetric integrators for the nonlinear term in the
-  Navier-Stokes equations.
-
- Added new miniapp directory mtop/ with optimization-oriented block parametric
-  non-linear form and abstract integrators. Two new miniapps, ParHeat and
-  SeqHeat, demonstrate parallel and sequential implementation of gradients
-  evaluation for linear diffusion with discrete density.
-
- Changed the interface for the error estimator.
-
- Implemented the Kelly error indicator for scalar-valued problems, supported
-  in serial and parallel builds.
-
- Added new classes DenseSymmetricMatrix and SymmetricMatrixCoefficient for
-  efficient evaluation of symmetric matrix coefficients. This replaces the now
-  deprecated EvalSymmetric in MatrixCoefficient. Added DiagonalMatrixCoefficient
-  for clarity, which is a typedef of VectorCoefficient.
-
- Added support for AMG preconditioners for non-symmetric systems (e.g.
-  advection-dominated problems) using hypre's approximate ideal restriction
-  (AIR) AMG. Requires hypre version 2.14.0 or newer. Usage is illustrated in
-  example 9/9p.
-
- Implemented an adaptive linear solver tolerance option for NewtonSolver based
-  on the algorithm of Eisenstat and Walker.
-
- Added support for nonscalar coefficient with VectorDiffusionIntegrator.
-
- Extending support for L2 basis functions using MapTypes VALUE and INTEGRAL in
-  linear interpolators and GridFunction "GetValue" methods.
-
- Variable order spaces, p- and hp-refinement. This is the initial (serial)
-  support for variable-order FiniteElementCollection and FiniteElementSpace.
-  The new method FiniteElementSpace::SetElementOrder can be called to set an
-  arbitrary order for each mesh element. The conforming interpolation matrix
-  will now automatically constrain p- and hp- interfaces, enabling general
-  hp-refinement in both 2D and 3D, on uniform or mixed NC meshes. Support for
-  parallel variable-order spaces will follow shortly.
+- The TMOP mesh optimization algorithms were extended to GPU:
+  * QualityMetric 1, 2, 7, 77 are available in 2D, 302, 303, 315, 321 in 3D
+  * Both AnalyticAdaptTC and DiscreteAdaptTC TargetConstructor are available
+  * Kernels for normalization and limiting have been added
+  * The AdvectorCG now also supports AssemblyLevel::PARTIAL

 - Added support for creating refined meshes for all element types (e.g. by
  splitting high-order elements into low-order refined elements), including
  mixed meshes. The LOR Transfer miniapp (miniapps/tools/lor-transfer.cpp) now
  supports meshes with any element geometry.

- Gitlab CI: use Spack (and Uberenv) to automate the build of TPLs.
-
- Added new miniapps demonstrating: 1) the use of GSLIB for overlapping grids,
-  see gslib/schwarz_ex1, and 2) coupling different physics in different domains,
-  see navier/cht. Note that gslib v1.0.7 is require (see INSTALL for details).
-
- Added a new, very simple example (ex0 and parallel version ex0p). This
-  example solves a simple Poisson problem using H1 elements (the same problem as
-  ex1), but is intended to be extremely simple and approachable for new users.
-
 - Meshes consisting of any type of elements (including mixed meshes) can be
  converted to all-simplex meshes using Mesh::MakeSimplicial.

@@ -195,56 +154,147 @@ Version 4.2.1 (development)
  requisite periodic vertex mappings can be created with
  Mesh::CreatePeriodicVertexMapping.

- Added support for transferring dual fields between high-order and low-order
-  refined finite element spaces using the transposed versions of the
-  L2ProjectionGridTransfer operators. This functionality is illustrated in the
-  lor-transfer miniapp.
-
- Improved interface for using the Ginkgo library, including: support for matrix-
-  free operators in Ginkgo solvers, new wrappers for Ginkgo preconditioners, HIP
-  support, and reduction of unnecessary data copies.
-
- Added initial support for hypre's mixed integer (mixedint) capability, which
-  uses different data types for local and global indices in order to save memory
-  in large problems. This capability requires that hypre was configured with the
-  --enable-mixedint option. Note that this option is currently tested only in
-  ex1p and may not work in more general settings.
-
- Added support for transferring fields (primary and dual) between high-order
-  and low-order refined H1 finite element spaces using the
-  L2ProjectionH1GridTransfer operators. This functionality is demonstrated
-  through the lor-transfer miniapp when run with the -h1 option.
-
- Added new functionality for constructing low-order refined discretizations and
-  solvers, see the LORDiscretization and LORSolver classes. A new basis type for
-  H(curl) and H(div) spaces is introduced to give spectral equivalence. This
-  functionality is illustrated in the LOR solvers miniapp in miniapps/solvers.
+- Added support for 1D non-conforming meshes (which can be useful for parallel
+  load balancing and derefinement).

 - Added sample meshes in the `data` subdirectory showing the reference elements
  of the six currently supported element types; ref-segment.mesh,
  ref-triangle.mesh, ref-square.mesh, ref-tetrahedron.mesh, ref-cube.mesh, and
  ref-prism.mesh.

+High-performance computing
+--------------------------
+- Added initial support for GPU-accelerated versions of PETSc that works with
+  MFEM_USE_CUDA if PETSc has been configured with CUDA support. Examples 1 and 9
+  in the examples/petsc directory have been modified to work with --device cuda.
+  Examples with GAMG (ex1p) and SLEPc (ex11p) are also provided.
+
+- Added support for explicit vectorization in the high-performance templated
+  code for Fujitsu's A64FX ARM microprocessor architecture.
+
+- Added support for different modes of QuadratureInterpolator on GPU.
+  The layout (QVectorLayout::byNODES|byVDIM) and the tensor products modes can
+  be enabled before calling the Mult, Values, Derivatives, PhysDerivatives and
+  Determinants methods.
+
+- Added method Device::SetMemoryTypes that can be used to change the default
+  host and device MemoryTypes before Device setup.
+
+- In class MemoryManager, added methods GetDualMemoryType and SetDualMemoryType;
+  dual MemoryTypes are used to determine the second MemoryType (host or device)
+  when only one MemoryType is specified in methods of class Memory.
+
+- Added Memory constructor for setting both the host and device MemoryTypes.
+
+- Switched the default behavior of device memory allocations so that they are
+  deferred until the device pointer is needed.
+
+- Added a second Umpire device MemoryType, DEVICE_UMPIRE_2, with corresponding
+  allocator that can be set with the method SetUmpireDevice2AllocatorName.
+
+- Added HOST_PINNED MemoryType and a pinned host allocator for CUDA and HIP.
+
+- Added matrix-free GPU-enabled implementations of GradientInterpolator and
+  IdentityInterpolator.
+
+New and updated examples and miniapps
+-------------------------------------
+- Added a new, very simple example (ex0 and parallel version ex0p). This example
+  solves a simple Poisson problem using H1 elements (the same problem as ex1),
+  but is intended to be extremely simple and approachable for new users.
+
+- Added new miniapps demonstrating: 1) the use of GSLIB for overlapping grids,
+  see gslib/schwarz_ex1, and 2) coupling different physics in different domains,
+  see navier/cht. Note that gslib v1.0.7 is require (see INSTALL for details).
+
+- Added a new miniapp for computing (signed) distance functions to a point
+  source or zero level set. See miniapps/shifted/distance.cpp.
+
 - Added a high-order extension of the shifted boundary method to solve PDEs on
  non body-fitted meshes. This is illustrated in the new Shifted Diffusion
  miniapp, see miniapps/shifted/diffusion.cpp.

+- Added new miniapp directory mtop/ with optimization-oriented block parametric
+  non-linear form and abstract integrators. Two new miniapps, ParHeat and
+  SeqHeat, demonstrate parallel and sequential implementation of gradients
+  evaluation for linear diffusion with discrete density.
+
+- Added a new miniapp block-solvers that compares the performance of various
+  solvers for mixed finite element discretization of the second order scalar
+  elliptic equations. Currently available solvers in the miniapp include a
+  block-diagonal preconditioner that is based on approximate Schur complement
+  (implemented in ex5p), and a newly implemented solver DivFreeSolver, which
+  exploits a multilevel decomposition of the Raviart-Thomas space and its
+  divergence-free subspace. See the miniapps/solvers directory for more details.
+
+- Introduced new options for the mesh-explorer miniapp to visualize the actual
+  element attributes in parallel meshes while retaining the visualization of the
+  domain decomposition.
+
+- Added partial assembly and device support to Example 25/25p, with diagonal
+  preconditioning.
+
+- Implemented a filter method for the Navier miniapp to stabilize highly
+  turbulent flows in direct numerical simulation.
+
+Improved testing
+----------------
+- Transitioned from Travis to GitHub Action for testing/CI on GitHub.
+
+- Use Spack (and Uberenv) to automate TPL building in LLNL GitLab tests.
+
+- Extended `make test` to include GPU tests when MFEM is built with CUDA or HIP
+  support.
+
+- Added a set of suggested git hooks for developers in config/githooks.
+
+- Added support for Caliper: a library to integrate performance profiling
+  capabilities into applications. See examples/caliper for more details.
+
+- Added a new command line boolean option (`--all`) to the unit tests to launch
+  *all* non-regression tests.
+
+- Upgraded the Catch unit test framework from version 2.13.0 to version 2.13.2.
+
+Miscellaneous
+-------------
+- The following integrations have updated minimum version requirements:
+  * CUDA >= 10.1.168
+  * Ginkgo >= 1.4.0
+  * GSLIB >= 1.0.7
+  * HIOP >= 0.4
+  * HYPRE >= 2.20.0 for mixedint support
+  * HYPRE >= 2.22.0 for CUDA support
+  * libCEED >= 0.8
+  * PETSc >= 3.15.0 for CUDA support
+  * RAJA >= 0.13.0
+  see INSTALL for more details.
+
+- Added a "scaled Jacobian" visualization option in the Mesh Explorer miniapp to
+  help identify elements with poor mesh quality.
+
+- Added support for reading VTK meshes in XML format.
+
 - Added makefile rule to generate TAGS table for vi or Emacs users.

-libCEED integration improvements
--------------------------------
- Refactor the libCEED integration
+- Added HIP support to the CMake build system.

- Add support for VectorCoefficient with libCEED backends.
+- Various other simplifications, extensions, and bugfixes in the code.

- Add support for ConvectionIntegrator, and VectorConvectionNLFIntegrator with
-  libCEED backends.
+API changes
+-----------
+- Added an abstract interface `mfem::FaceRestriction` for `H1FaceRestriction`
+  and `L2FaceRestriction`.
+  In order to conform with the semantic of `MultTranspose` in `mfem::Operator`,
+  `mfem::FaceRestriction::MultTranspose` now sets instead of adding values, and
+  `mfem::FaceRestriction::AddMultTranspose` should replace previous calls to
+  `mfem::FaceRestriction::MultTranspose`.


 Version 4.2, released on October 30, 2020
 =========================================

-High-Performance Computing
+High-performance computing
 --------------------------
 - Added support for explicit vectorization in the high-performance templated
  code, which can now take advantage of specific classes on the following
@@ -326,9 +376,6 @@ Linear and nonlinear solvers
  matrix with the function HypreParMatrixFromBlocks. This could be useful for
  solving block systems with parallel direct solvers such as STRUMPACK.

- Added CUDA support for SUNDIALS ODE integrators. See the updated SUNDIALS
-  modification of Example 9/9p.
-
 - Added wrappers for hypre's flexible GMRES solver and the new parallel ILU
  preconditioner. The latter requires hypre version 2.19.0 or later.

@@ -439,7 +486,7 @@ New and updated examples and miniapps
    L2, with partial assembly support in Example 24/24p.
  * Weak Dirichlet boundary conditions (Nitsche) to the NURBS miniapp.

-Data management and Visualization
+Data management and visualization
 ---------------------------------
 - Added support for ADIOS2 for parallel I/O with ParaView visualization. See
  Examples 5, 9, 12, 16. The classes adios2stream and ADIOS2DataCollection
@@ -16,7 +16,7 @@ set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH

 # Require C++11 and disable compiler-specific extensions
 set(CMAKE_CXX_STANDARD 11)
-if (MFEM_USE_GINKGO) 
+if (MFEM_USE_GINKGO)
  set(CMAKE_CXX_STANDARD 14)
 endif()
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -54,7 +54,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.2.1)
+set(${PROJECT_NAME}_VERSION 4.3.1)

 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -102,7 +102,7 @@ if (MFEM_USE_CUDA)
   endif()
   enable_language(CUDA)
   set(CMAKE_CUDA_STANDARD 11)
-   if (MFEM_USE_GINKGO) 
+   if (MFEM_USE_GINKGO)
     set(CMAKE_CUDA_STANDARD 14)
   endif()
   set(CMAKE_CUDA_STANDARD_REQUIRED ON)
@@ -246,6 +246,7 @@ if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
    message(FATAL_ERROR " *** MFEM_USE_LEGACY_OPENMP requires MFEM_THREAD_SAFE=ON.")
  endif()
  find_package(OpenMP REQUIRED)
+  set(OPENMP_LIBRARIES ${OpenMP_CXX_LIBRARIES})
 endif()

 # SuiteSparse (before SUNDIALS which may depend on KLU)
@@ -330,6 +331,10 @@ if (MFEM_USE_CONDUIT)
    find_package(Conduit REQUIRED conduit relay blueprint )
 endif()

+if (MFEM_USE_FMS)
+    find_package(FMS REQUIRED fms )
+endif()
+
 # Axom/Sidre
 if (MFEM_USE_SIDRE)
  find_package(Axom REQUIRED Axom)
@@ -424,9 +429,10 @@ endif()
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
 set(MFEM_TPLS MPI_CXX OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS PETSC
-    SLEPC MESQUITE MUMPS STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
+    SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB NETCDF
    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
    CUSPARSE MKL_CPARDISO AMGX CALIPER)
+
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")
@@ -445,6 +451,9 @@ include_directories(${TPL_INCLUDE_DIRS})
 if (OPENMP_FOUND)
  message(STATUS "MFEM: using package OpenMP")
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+  if (MFEM_USE_CUDA)
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=${OpenMP_CXX_FLAGS}")
+  endif()
 endif()

 message(STATUS "MFEM build type: CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}")
@@ -4,7 +4,9 @@

 <p align="center">
 <a href="https://github.com/mfem/mfem/blob/master/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-BSD-brightgreen.svg"></a>
-<a href="https://travis-ci.org/mfem/mfem"><img alt="Build Status" src="https://travis-ci.org/mfem/mfem.svg?branch=master"></a>
+<a href="https://github.com/mfem/mfem/actions?query=workflow%3Arepo-check+branch%3Amaster"><img alt="Repo check" src="https://github.com/mfem/mfem/actions/workflows/repo-check.yml/badge.svg?branch=master"></a>
+<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuild-analysis+branch%3Amaster"><img alt="Build Analysis" src="https://github.com/mfem/mfem/actions/workflows/mfem-analysis.yml/badge.svg?branch=master"></a>
+<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuilds-and-tests+branch%3Amaster"><img alt="Builds and Tests" src="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml/badge.svg?branch=master"></a>
 <a href="https://ci.appveyor.com/project/mfem/mfem"><img alt="Build Status" src="https://ci.appveyor.com/api/projects/status/19non9sqm6msi2wy?svg=true"></a>
 <a href="https://mfem.github.io/doxygen/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
 </p>
@@ -63,6 +65,8 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
  development branches off `mfem:master`.
 - Please follow the [developer guidelines](#developer-guidelines), in particular
  with regards to documentation and code styling.
+- Please do not commit large/binary files to the central repository (use a fork
+  instead).
 - Pull requests  should be issued toward `mfem:master`. Make sure
  to check the items off the [Pull Request Checklist](#pull-request-checklist).
 - When your contribution is fully working and ready to be reviewed, add
@@ -71,6 +75,7 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
  reviewers to evaluate the changes.
 - The reviewers have 3 weeks to evaluate the PR and work with the author to
  fix issues and implement improvements.
+- During review there should be no force pushes/rewriting history in the branch.
 - After approval, MFEM developers merge the PR manually in the [mfem:next branch](#masternext-workflow).
 - After a week of testing in `mfem:next`, the original PR is merged in `mfem:master`.
 - We use [milestones](https://github.com/mfem/mfem/milestones) to coordinate the
@@ -91,8 +96,8 @@ The MFEM source code has the following structure:
 ```
  .
  ├── config
-  │   └── cmake
-  │       └── ...
+  │   ├── cmake
+  │   └── githooks
  ├── data
  ├── doc
  ├── examples
@@ -129,10 +134,10 @@ The MFEM source code has the following structure:
  └── tests
      ├── convergence
      ├── gitlab
+      ├── mem_manager
      ├── par-mesh-format
      ├── scripts
      └── unit
-          └── ...
 ```

 #### Main directories and classes
@@ -363,6 +368,10 @@ Before you can start, you need a GitHub account, here are a few suggestions:
  two reviewers to evaluate the changes. The reviewers have 3 weeks to evaluate
  the PR and work with the author to implement improvements and fix issues.

+- Once the `ready-for-review` label has been applied and reviewers have been
+  assigned, the PR is considered under review. To help with the review process
+  there should be no force pushes/rewriting history in the branch.
+
 - After approval, the PR is [tested](#masternext-workflow) for a week with
  other approved PRs in the `mfem:next` branch.

@@ -370,16 +379,20 @@ Before you can start, you need a GitHub account, here are a few suggestions:
  `mfem:next`, see the [README](tests/scripts/README) file in that directory
  for more details.

- Track the Travis CI, Github Actions and Appveyor [continuous integration](#automated-testing)
+- Track the GitHub Actions and Appveyor [continuous integration](#automated-testing)
  builds at the end of the PR. These should generally run clean, so address any
  errors as soon as possible. Please ask if you are unsure how to do that.

- Note that some tests, such as the `branch-history` check in Travis and Github
-  Actions are safeguards that are allowed to fail in certain cases.
+- Note that some tests, such as the `branch-history` check in GitHub Actions
+  are safeguards that are allowed to fail in certain cases.

 - Other tests, such as the `code-style`, `documentation` and `gitignore`
-  checks in Travis and Github Actions enforce MFEM-specific rules which are
-  explained in the error messages and the `tests/scripts` directory.
+  checks in GitHub Actions enforce MFEM-specific rules which are explained in
+  the error messages and the `tests/scripts` directory.
+
+- Also note that the tests `branch-history` and `repos-checks` found in GitHub
+  Actions can be triggered automatically before each push using git hooks. See
+  the [git hooks README](config/githooks/README.md) for a detailed explanation.

 - If triggered, track the status of the LLNL GitLab tests. If failing, ask
  one of the _LLNL developers_ for details.
@@ -399,7 +412,7 @@ Before a PR can be merged, it should satisfy the following:
    - [ ] Does `make` or `cmake` have a new target?
    - [ ] Did the requirements or the installation process change? *(rare)*
 - [ ] Update continuous integration server configurations if necessary (e.g. with new version requirements for each of MFEM's dependencies)
-    - [ ] `.travis.yml`
+    - [ ] `.github`
    - [ ] `.appveyor.yml`
 - [ ] Update `.gitignore`:
    - [ ] Check if `make distclean; git status` shows any files that were generated from the source by the project (not an IDE) but we don't want to track in the repository.
@@ -516,7 +529,7 @@ MFEM uses a `master`/`next`-branch workflow as described below:
    - [ ] `doc/CodeDocumentation.conf.in`
 - [ ] Check that version requirements for each of MFEM's dependencies are documented in `INSTALL` and up-to-date
 - [ ] Check that continuous integration server configurations reflect the dependency version requirements of the new release
-    - [ ] `.travis.yml`
+    - [ ] `.github`
    - [ ] `.appveyor.yml`
 - [ ] Update the `CHANGELOG` to organize all release contributions
 - [ ] Review the whole source code once over
@@ -578,14 +591,17 @@ MFEM uses a `master`/`next`-branch workflow as described below:
 MFEM has several levels of automated testing running on GitHub, as well as on
 local Mac and Linux workstations, and Livermore Computing clusters at LLNL.

-### Linux and Mac smoke tests
-We use Travis CI and Github Actions to drive the default tests on the `master`
-and `next` branches. See the `.travis` file and the logs at
-[https://travis-ci.org/mfem/mfem](https://travis-ci.org/mfem/mfem).
+In addition, developers can set local git hooks to run some quick checks on
+commit or push, see the [README](config/githooks/README.md) in the `config/githooks`
+directory.

-Testing using Travis CI and Github Actions should be kept lightweight, as there
-is a time constraint on jobs. Two virtual machines are configured - Mac (OS X)
-and Linux.
+### Linux and Mac smoke tests
+We use GitHub Actions to drive the default tests on the `master` and `next`
+branches. See the `.github/workflows` files and the logs at
+[https://github.com/mfem/mfem/actions](https://github.com/mfem/mfem/actions).
+
+Testing using GitHub Actions should be kept lightweight, as there is a time
+constraint on jobs. Two virtual machines are configured - Mac (OS X) and Linux.

 - Tests on the `master` branch are triggered whenever a PR is issued on this branch.
 - Tests on the `next` branch are currently scheduled to run each night.
@@ -474,7 +474,7 @@ MFEM_USE_HIP = YES/NO
   Enables support for AMD devices in MFEM. HIP is a heterogeneous-compute
   interface for portability developed by AMD that can target both AMD and
   NVIDIA GPUs. The variable HIP_ARCH is used to specify the AMD GPU processor
-   used during compilation (by default, HIP_ARCH=gfx900).  When enabled, this
+   used during compilation (by default, HIP_ARCH=gfx900). When enabled, this
   option uses the HIP_* build options, see below.

 MFEM_USE_RAJA = YES/NO
@@ -516,6 +516,13 @@ MFEM_USE_CALIPER = YES/NO
   profiling at runtime with Caliper's configuration API. Alternatively, one
   can configure Caliper through environment variables or config files.

+MFEM_USE_FMS = YES/NO
+   Enables support for the FMS library which consists of the DataCollection
+   sub-class mfem::FMSDataCollection for I/O in FMS formats, see the header file
+   fem/fmsdatacollection.hpp. In addition, this option enables in-memory
+   convetion routines between FMS's FmsDataCollection structure and MFEM's
+   DataCollection class, see the header file fem/fmsconvert.hpp.
+
 MFEM_BUILD_TAG = (any value)
   An optional tag to characterize the build. Exported to config/config.mk.
   Can be used to identify the MFEM build from other makefiles.
@@ -540,8 +547,9 @@ The specific libraries and their options are:
  See also the "Specific options for hypre" section at the end of this file.
  URL: https://github.com/hypre-space/hypre and https://www.llnl.gov/casc/hypre
  Options: HYPRE_OPT, HYPRE_LIB.
-  Versions: HYPRE >= 2.10.0b,
-            HYPRE >= 2.20.0 for '--enable-mixedint' support.
+  Versions: HYPRE >= 2.10.0b  (HYPRE built without CUDA)
+            HYPRE >= 2.20.0   (HYPRE built with '--enable-mixedint')
+            HYPRE >= 2.22.0   (HYPRE built with CUDA)

 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
  MFEM_USE_METIS_5 = YES (default is to use METIS 4).
@@ -615,7 +623,7 @@ The specific libraries and their options are:
  and dependencies of specific modules, see the Ginkgo webpage below.
  URL: https://ginkgo-project.github.io
  Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or Debug).
-  Versions: Ginkgo >= 1.4.0. 
+  Versions: Ginkgo >= 1.4.0.

 - AmgX (optional), used when MFEM_USE_AMGX = YES.
  URL: https://github.com/NVIDIA/AMGX
@@ -753,6 +761,11 @@ The specific libraries and their options are:
  URL: https://zlib.net
  Options: ZLIB_OPT, ZLIB_LIB.

+- FMS (optional), used when MFEM_USE_FMS = YES.
+  URL: https://github.com/CEED/FMS
+  Options: FMS_OPT, FMS_LIB.
+  Versions: FMS >= 0.2.
+
 Building with CMake
 ===================
 The MFEM build system consists of two steps: configuration and compilation.
@@ -884,6 +897,7 @@ MFEM_USE_RAJA
 MFEM_USE_UMPIRE
 MFEM_USE_SIDRE
 MFEM_USE_CALIPER
+MFEM_USE_FMS

 The following options are CMake specific:

@@ -938,6 +952,7 @@ The CMake build system adds auto-detection for the following packages/libraries:
 - UMPIRE
 - AXOM - Used when MFEM_USE_SIDRE is enabled
 - CALIPER
+ - FMS

 The following built-in CMake packages are also used:

@@ -955,7 +970,7 @@ config/config.hpp.in:

   cp config/config.hpp.in config/_config.hpp

-The file config/_config.hpp can then be edited to enable desired options.  The
+The file config/_config.hpp can then be edited to enable desired options. The
 MFEM library is simply a combination of all object files obtained by compiling
 the .cpp source files in the source directories: general, linalg, mesh, and fem.

@@ -963,7 +978,7 @@ the .cpp source files in the source directories: general, linalg, mesh, and fem.
 Specifying an MPI job launcher
 ==============================
 By default, MFEM will use 'mpirun -np #' to launch any of its parallel tests or
-miniapps, where # is the number of MPI tasks.  An alternate MPI launcher can be
+miniapps, where # is the number of MPI tasks. An alternate MPI launcher can be
 provided by setting the MFEM_MPIEXEC and MFEM_MPIEXEC_NP config variables.

 MFEM will expect the launcher command, plus the command line option to allow it
@@ -256,6 +256,10 @@ IF (DEFINED TPL_ENABLE_SIDRE)
  SET(MFEM_USE_SIDRE ${TPL_ENABLE_SIDRE} CACHE BOOL "Enable Axom/Sidre usage" FORCE)
 ENDIF()

+IF (DEFINED TPL_ENABLE_FMS)
+  SET(MFEM_USE_FMS ${TPL_ENABLE_FMS} CACHE BOOL "Enable FMS usage" FORCE)
+ENDIF()
+
 IF (DEFINED TPL_ENABLE_CONDUIT)
  SET(MFEM_USE_CONDUIT ${TPL_ENABLE_CONDUIT} CACHE BOOL "Enable Conduit usage" FORCE)
 ENDIF()
@@ -44,6 +44,7 @@ set(MFEM_USE_PETSC @MFEM_USE_PETSC@)
 set(MFEM_USE_SLEPC @MFEM_USE_SLEPC@)
 set(MFEM_USE_MPFR @MFEM_USE_MPFR@)
 set(MFEM_USE_SIDRE @MFEM_USE_SIDRE@)
+set(MFEM_USE_FMS @MFEM_USE_FMS@)
 set(MFEM_USE_CONDUIT @MFEM_USE_CONDUIT@)
 set(MFEM_USE_PUMI @MFEM_USE_PUMI@)
 set(MFEM_USE_CUDA @MFEM_USE_CUDA@)
@@ -119,6 +119,9 @@
 // Enable the use of SIMD in the high performance templated classes
 #cmakedefine MFEM_USE_SIMD

+// Enable MFEM functionality based on the FMS library
+#cmakedefine MFEM_USE_FMS
+
 // Enable MFEM functionality based on Conduit
 #cmakedefine MFEM_USE_CONDUIT

@@ -0,0 +1,20 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - FMS_FOUND
+#   - FMS_LIBRARIES
+#   - FMS_INCLUDE_DIRS
+
+include(MfemCmakeUtilities)
+mfem_find_package(FMS FMS FMS_DIR
+  "include" fms.h "lib" fms
+  "Paths to headers required by FMS." "Libraries required by FMS.")
@@ -117,6 +117,9 @@
 // Enable the use of SIMD in the high performance templated classes
 // #define MFEM_USE_SIMD

+// Enable FMS support
+// #define MFEM_USE_FMS
+
 // Enable Conduit support
 // #define MFEM_USE_CONDUIT

@@ -43,6 +43,7 @@ MFEM_USE_PETSC         = @MFEM_USE_PETSC@
 MFEM_USE_SLEPC         = @MFEM_USE_SLEPC@
 MFEM_USE_MPFR          = @MFEM_USE_MPFR@
 MFEM_USE_SIDRE         = @MFEM_USE_SIDRE@
+MFEM_USE_FMS           = @MFEM_USE_FMS@
 MFEM_USE_CONDUIT       = @MFEM_USE_CONDUIT@
 MFEM_USE_PUMI          = @MFEM_USE_PUMI@
 MFEM_USE_HIOP          = @MFEM_USE_HIOP@
@@ -45,6 +45,7 @@ option(MFEM_USE_PETSC "Enable PETSc support." OFF)
 option(MFEM_USE_SLEPC "Enable SLEPc support." OFF)
 option(MFEM_USE_MPFR "Enable MPFR usage." OFF)
 option(MFEM_USE_SIDRE "Enable Axom/Sidre usage" OFF)
+option(MFEM_USE_FMS "Enable FMS usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
 option(MFEM_USE_PUMI "Enable PUMI" OFF)
 option(MFEM_USE_HIOP "Enable HiOp" OFF)
@@ -96,6 +97,11 @@ set(HYPRE_DIR "${MFEM_DIR}/../hypre/src/hypre" CACHE PATH
 # If hypre was compiled to depend on BLAS and LAPACK:
 # set(HYPRE_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
 #     "Packages that HYPRE depends on.")
+if (MFEM_USE_CUDA)
+   # This is only necessary when hypre is built with cuda:
+   set(HYPRE_REQUIRED_LIBRARIES "-lcusparse" "-lcurand" CACHE STRING
+       "Libraries that HYPRE depends on.")
+endif()

 set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library.")

@@ -132,10 +138,10 @@ set(MUMPS_DIR "${MFEM_DIR}/../MUMPS_5.2.0" CACHE PATH
    "Path to the MUMPS library.")
 # Packages required by MUMPS, depending on how it was compiled.
 set(MUMPS_REQUIRED_PACKAGES "MPI" "BLAS" "METIS" "ScaLAPACK" CACHE STRING
-    "Additional packages required by MUMPS.")    
+    "Additional packages required by MUMPS.")
 # If the MPI package does not find all required Fortran libraries:
 # set(MUMPS_REQUIRED_LIBRARIES "gfortran" "mpi_mpifh" CACHE STRING
-#     "Additional libraries required by MUMPS.")    
+#     "Additional libraries required by MUMPS.")

 set(STRUMPACK_DIR "${MFEM_DIR}/../STRUMPACK-build" CACHE PATH
    "Path to the STRUMPACK library.")
@@ -187,6 +193,12 @@ set(SLEPC_ARCH "arch-linux2-c-debug" CACHE STRING "SLEPC build architecture.")

 set(MPFR_DIR "" CACHE PATH "Path to the MPFR library.")

+set(FMS_DIR "${MFEM_DIR}/../fms" CACHE PATH
+    "Path to the FMS library.")
+# If FMS is built with Conduit:
+# set(FMS_REQUIRED_PACKAGES "Conduit/relay" CACHE STRING
+#     "Additional packages required by FMS.")
+
 set(CONDUIT_DIR "${MFEM_DIR}/../conduit" CACHE PATH
    "Path to the Conduit library.")

@@ -136,6 +136,7 @@ MFEM_USE_PETSC         = NO
 MFEM_USE_SLEPC         = NO
 MFEM_USE_MPFR          = NO
 MFEM_USE_SIDRE         = NO
+MFEM_USE_FMS           = NO
 MFEM_USE_CONDUIT       = NO
 MFEM_USE_PUMI          = NO
 MFEM_USE_HIOP          = NO
@@ -174,6 +175,10 @@ LIBUNWIND_LIB = $(if $(NOTMAC),-lunwind -ldl,)
 HYPRE_DIR = @MFEM_DIR@/../hypre/src/hypre
 HYPRE_OPT = -I$(HYPRE_DIR)/include
 HYPRE_LIB = -L$(HYPRE_DIR)/lib -lHYPRE
+ifeq (YES,$(MFEM_USE_CUDA))
+   # This is only necessary when hypre is built with cuda:
+   HYPRE_LIB += -lcusparse -lcurand
+endif

 # METIS library configuration
 ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK)$(MFEM_USE_MUMPS),NONONO)
@@ -357,6 +362,11 @@ endif
 MPFR_OPT =
 MPFR_LIB = -lmpfr

+# FMS and required libraries configuration
+FMS_DIR = $(MFEM_DIR)/../fms
+FMS_OPT = -I$(FMS_DIR)/include
+FMS_LIB = -Wl,-rpath,$(FMS_DIR)/lib -L$(FMS_DIR)/lib -lfms
+
 # Conduit and required libraries configuration
 CONDUIT_DIR = @MFEM_DIR@/../conduit
 CONDUIT_OPT = -I$(CONDUIT_DIR)/include/conduit
@@ -0,0 +1,41 @@
+                    Finite Element Discretization Library
+                                   __
+                       _ __ ___   / _|  ___  _ __ ___
+                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
+                      | | | | | ||  _||  __/| | | | | |
+                      |_| |_| |_||_|   \___||_| |_| |_|
+
+                               https://mfem.org
+
+
+This directory contains recommended git hooks, which are scripts that can be
+used to improve your development experience with MFEM:
+
+### The hooks
+
+* `pre-commit` is a hook that will be applied before each commit and run
+`astyle` on the code. This will ensure that your changes comply with the MFEM
+code styling guidelines.
+
+* `pre-push` is a hook that will be applied before each push to run a quick set
+of tests that verify that your files headers are in compliance, and that you did
+not add any large files to the repo.
+
+### Setup
+
+To setup the git hooks, run `make hooks`, which creates symlinks to the hooks in
+the `.git/hooks` directory. Individual hooks can be enabled by manually creating
+symlinks.
+
+(You may also copy the scripts directly and customize them further, but this way
+you may miss additional updates in the future.)
+
+### Failures
+
+The `branch-history` check can fail in some cases when the history is OK. For
+example, when a large number of files were modified for a legitimate reason, or
+when a picture was added for documentation.
+
+If that is the case, make sure the failure is indeed justified, and rerun the
+push command with the `--no-verify` option. This will skip the hooks, allowing
+you to push those changes.
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+# Apply automated code formatting
+make -C $(git rev-parse --show-toplevel) style
@@ -0,0 +1,107 @@
+#!/bin/bash
+
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+option=${1:-""}
+
+if [[ "${option}" == "--help" ]]; then
+  echo "This script runs checks on the repository."
+  echo "It has 2 modes: with and without an option."
+  echo ""
+  echo "Options are used in GitHub Actions and can be:"
+  echo "  --copyright"
+  echo "  --license"
+  echo "  --release"
+  echo "  --style"
+  echo "  --history"
+  echo ""
+  echo "As a githook, the script is used without options."
+  echo "In that case, it will run all the checks except style."
+  echo ""
+  echo "Use --help to print this help message."
+fi
+
+cd $(git rev-parse --show-toplevel)
+
+# copyright check
+copyright=true
+if [[ "${option}" == "--copyright" || "${option}" == "" ]]; then
+  if git grep -l "^\(#\|//\).*\(\-2020\|\ 2010,\)" > matches.txt; then
+    echo "Please update the following files to Copyright (c) 2010-2021:"
+    cat matches.txt
+    copyright=false
+  fi
+fi
+
+# license check
+license=true
+if [[ "${option}" == "--license" || "${option}" == "" ]]; then
+  if git grep -li "^\(#\|//\).*GNU\ Lesser\ General\ Public\ License" > matches.txt; then
+    echo "Please update the following files to the BSD-3 license:"
+    cat matches.txt
+    license=false
+  fi
+fi
+
+# release check
+release=true
+if [[ "${option}" == "--release" || "${option}" == "" ]]; then
+  if git grep -l "^\(#\|//\).*LLNL\-CODE\-443211" > matches.txt
+  then
+    echo "Please update the following files to LLNL-CODE-806117:"
+    cat matches.txt
+    release=false
+  fi
+fi
+
+# wrap-up
+code=0
+if ! $copyright ; then
+  echo "copyright check failed, unroll log for details"
+  code=1
+fi
+if ! $license ; then
+  echo "license check failed, unroll log for details"
+  code=1
+fi
+if ! $release ; then
+  echo "release check failed, unroll log for details"
+  code=1
+fi
+
+# `code-style` is not just a check, it will actually reformat the code if
+# necessary. This means that if one pushes while the repo is in dirty state
+# (changes not staged), those changes may be mixed with format changes.
+# To activate this, you will need to hard-copy this hook script in the hook
+# directory and uncomment only then. (See README.md)
+#
+## style check
+#if [[ "${option}" == "--style" || "${option}" == "" ]]; then
+if [[ "${option}" == "--style" ]]; then
+  if which astyle && [[ "$(astyle --version)" == "Artistic Style Version 2.05.1" ]]; then
+    cd tests/scripts
+    if ! ./runtest code-style; then code=1; fi
+    cd -
+  else
+    echo "Warning: astyle not found or version is not 2.05.1"
+  fi
+fi
+
+# branch-history
+if [[ "${option}" == "--history" || "${option}" == "" ]]; then
+  git fetch origin master:master
+  cd tests/scripts
+  if ! ./runtest branch-history; then code=1; fi
+  cd -
+fi
+
+exit $code
@@ -57,22 +57,27 @@ TIMECMD := $(word 1,$(TIMECMD))
 ifneq (,$(filter test%,$(MAKECMDGOALS)))
   MAKEFLAGS += -k
 endif
-# Test runs of the examples/miniapps with parameters - check exit code
+# Test runs of the examples/miniapps with parameters - check exit code:
+# 0 means success, 255 means the test was skipped, anything else means error
 mfem-test = \
   printf "   $(3) [$(2) $(1) ... ]: "; \
   $(call $(TIMEFUN),$(TIMECMD),$(2) ./$(1) $(if $(5),,-no-vis )$(4) \
     > $(1).stderr 2>&1); \
-   if [ "$$3" = 0 ]; \
-   then $(PRINT_OK); else $(PRINT_FAILED); cat $(1).stderr; fi; \
-   rm -f $(1).stderr; exit $$3
+   err="$$3"; \
+   if [ "$$3" = 0 ]; then $(PRINT_OK); \
+   else if [ "$$3" = 255 ]; then $(PRINT_SKIP); err=0; \
+   else $(PRINT_FAILED); cat $(1).stderr; fi; fi; \
+   rm -f $(1).stderr; exit $$err

 # Test runs of the examples/miniapps - check exit code and if a file exists
+# See mfem-test for the interpretation of the error code
 mfem-test-file = \
   printf "   $(3) [$(2) $(1) ... ]: "; \
   $(call $(TIMEFUN),$(TIMECMD),$(2) ./$(1) -no-vis > $(1).stderr 2>&1); \
   err="$$3"; \
-   if [ "$$3" = 0 ] && [ -e $(4) ]; \
-   then $(PRINT_OK); else $(PRINT_FAILED); cat $(1).stderr; err=64; fi; \
+   if [ "$$3" = 0 ] && [ -e $(4) ]; then $(PRINT_OK); \
+   else if [ "$$3" = 255 ] && [ -e $(4) ]; then $(PRINT_SKIP); err=0; \
+   else $(PRINT_FAILED); cat $(1).stderr; err=64; fi; fi; \
   rm -f $(1).stderr; exit $$err

 .PHONY: test test-par-YES test-par-NO test-ser test-par test-clean test-print
@@ -80,6 +85,26 @@ mfem-test-file = \
 # What sets of tests to run in serial and parallel
 test-par-YES: $(PAR_$(MFEM_TESTS):=-test-par) $(SEQ_$(MFEM_TESTS):=-test-seq)
 test-par-NO:  $(SEQ_$(MFEM_TESTS):=-test-seq)
+ifeq ($(MFEM_USE_CUDA),YES)
+.PHONY: test-par-YES-cuda test-par-NO-cuda test-ser-cuda test-par-cuda test-cuda
+test-par-YES: test-par-YES-cuda
+test-par-NO:  test-par-NO-cuda
+test-par-YES-cuda: test-par-cuda test-ser-cuda
+test-par-NO-cuda:  test-ser-cuda
+test-ser-cuda: $(SEQ_DEVICE_$(MFEM_TESTS):=-test-seq-cuda)
+test-par-cuda: $(PAR_DEVICE_$(MFEM_TESTS):=-test-par-cuda)
+test-cuda: test-par-$(MFEM_USE_MPI)-cuda clean-exec
+endif
+ifeq ($(MFEM_USE_HIP),YES)
+.PHONY: test-par-YES-hip test-par-NO-hip test-ser-hip test-par-hip test-hip
+test-par-YES: test-par-YES-hip
+test-par-NO:  test-par-NO-hip
+test-par-YES-hip: test-par-hip test-ser-hip
+test-par-NO-hip:  test-ser-hip
+test-ser-hip: $(SEQ_DEVICE_$(MFEM_TESTS):=-test-seq-hip)
+test-par-hip: $(PAR_DEVICE_$(MFEM_TESTS):=-test-par-hip)
+test-hip: test-par-$(MFEM_USE_MPI)-hip clean-exec
+endif
 test-ser:     test-par-NO
 test-par:     test-par-YES
 test:         all test-par-$(MFEM_USE_MPI) clean-exec
@@ -0,0 +1,246 @@
+FMS: 100
+DataCollection/Name: star
+DataCollection/NumberOfFieldDescriptors: 1
+DataCollection/FieldDescriptors/0/Name: CoordsDescriptor
+DataCollection/FieldDescriptors/0/ComponentName: volume
+DataCollection/FieldDescriptors/0/Type: 0
+DataCollection/FieldDescriptors/0/FixedOrder/Size: 3
+DataCollection/FieldDescriptors/0/FixedOrder/Type: FMS_UINT64
+DataCollection/FieldDescriptors/0/FixedOrder/Values: [0, 1, 3]
+DataCollection/FieldDescriptors/0/NumDofs: 211
+DataCollection/NumberOfFields: 1
+DataCollection/Fields/0/Name: Coords
+DataCollection/Fields/0/LayoutType: 0
+DataCollection/Fields/0/NumberOfVectorComponents: 2
+DataCollection/Fields/0/FieldDescriptorName: CoordsDescriptor
+DataCollection/Fields/0/Data/Size: 422
+DataCollection/Fields/0/Data/Type: FMS_DOUBLE
+DataCollection/Fields/0/Data/Values: [-0.016886, 1.000000, 0.309017, 
+1.309020, -0.809017, -0.500000, 
+-0.809017, -1.618030, 0.309017, 
+-0.500000, 1.309020, 0.519420, 
+1.154510, 0.809019, 0.147680, 
+-0.095492, -0.654508, -0.415586, 
+-1.213520, -1.213520, -0.392210, 
+-0.654508, -0.095492, 0.139949, 
+0.809019, 1.154510, 0.660184, 
+-0.264063, -0.800064, -0.231060, 
+0.663691, 0.183114, 0.317639, 
+0.543082, 0.598483, 0.345112, 
+0.478298, 0.027703, 0.095229, 
+0.012368, -0.092534, -0.334412, 
+-0.313767, -0.140526, -0.293881, 
+-0.534056, -0.660290, -0.537646, 
+-0.655590, -0.121396, -0.274504, 
+-0.346497, -0.296570, 0.004737, 
+-0.098835, 0.069287, 0.082675, 
+0.318799, 0.467183, 0.564505, 
+0.595190, 0.846237, 0.671735, 
+1.051500, 1.103010, 0.964008, 
+0.821603, 1.257520, 1.206010, 
+1.142350, 0.975686, 0.781273, 
+0.717257, 0.475684, 0.642352, 
+0.268930, 0.211049, 0.174181, 
+0.039345, -0.147746, -0.177481, 
+-0.365164, -0.230328, -0.551503, 
+-0.603005, -0.497587, -0.389864, 
+-0.757514, -0.706011, -0.675487, 
+-0.528946, -0.943851, -1.078690, 
+-1.087600, -0.955467, -1.483190, 
+-1.348360, -1.483190, -1.348360, 
+-1.085930, -0.938010, -0.943851, 
+-1.078690, -0.681476, -0.540944, 
+-0.757514, -0.706011, -0.540614, 
+-0.367058, -0.551503, -0.603005, 
+-0.365164, -0.230328, -0.138552, 
+-0.206896, 0.174181, 0.039345, 
+0.268468, 0.222269, 0.475684, 
+0.642352, 0.759791, 0.719381, 
+1.142350, 0.975686, 1.257520, 
+1.206010, 0.972837, 0.836119, 
+1.051500, 1.103010, 0.214572, 
+0.407449, 0.288323, 0.449827, 
+-0.086700, -0.027358, -0.200560, 
+-0.166595, -0.271802, -0.418426, 
+-0.426131, -0.551441, -0.096117, 
+-0.206969, -0.027946, -0.184969, 
+0.211136, 0.260131, 0.407172, 
+0.430781, 0.718277, 0.885068, 
+0.753103, 0.957692, 0.866273, 
+1.024530, 0.934099, 1.093820, 
+0.348422, 0.524463, 0.404903, 
+0.587376, 0.054525, 0.146431, 
+-0.078026, -0.007795, -0.329488, 
+-0.302967, -0.488115, -0.439332, 
+-0.498515, -0.453527, -0.633059, 
+-0.570251, -0.655787, -0.791132, 
+-0.802013, -0.956872, -1.094720, 
+-1.208560, -1.207720, -1.339910, 
+-0.693371, -0.795404, -0.803594, 
+-0.945068, -0.464668, -0.631721, 
+-0.458968, -0.547876, -0.332878, 
+-0.485696, -0.286385, -0.408481, 
+0.053021, -0.058200, 0.110846, 
+-0.002086, 0.381384, 0.416784, 
+0.551133, 0.613261, 0.872474, 
+0.901208, 1.038300, 1.084660, 
+0.737459, 0.751250, 0.890002, 
+0.915210, 0.010915, 0.000000, 
+0.951057, 0.951057, 0.587785, 
+1.538840, -0.587785, 0.000000, 
+-0.951057, -1.538840, -0.951057, 
+-0.015847, 0.475529, 0.951057, 
+0.492248, 1.244950, 1.063310, 
+0.274399, 0.293893, -0.293892, 
+-0.296404, -1.063310, -1.244950, 
+-0.453865, -0.951057, -0.475529, 
+0.466620, 0.792932, -0.013913, 
+-0.748783, -0.497528, 0.021382, 
+-0.017158, 0.172591, 0.330125, 
+0.458568, 0.457971, 0.137740, 
+0.299049, 0.588394, 0.667324, 
+0.432341, 0.634346, 0.117322, 
+0.193603, 0.211702, 0.098278, 
+-0.199438, -0.077304, -0.082243, 
+-0.216296, -0.458634, -0.592374, 
+-0.563926, -0.680404, -0.135751, 
+-0.302942, -0.469005, -0.453640, 
+-0.182727, -0.314240, 0.024270, 
+0.021546, 0.158510, 0.317019, 
+0.485799, 0.492951, 0.792548, 
+0.634038, 0.951057, 0.951057, 
+0.777915, 0.613430, 0.951057, 
+0.951057, 0.793994, 0.635800, 
+1.049020, 1.146990, 1.084480, 
+0.924310, 1.440880, 1.342910, 
+1.380330, 1.221820, 0.948209, 
+0.856297, 0.746293, 0.904802, 
+0.476242, 0.393234, 0.489821, 
+0.391857, 0.194471, 0.075751, 
+0.097964, 0.195929, -0.097964, 
+-0.195928, -0.173234, -0.078922, 
+-0.489821, -0.391856, -0.467007, 
+-0.397859, -0.746293, -0.904802, 
+-0.945206, -0.849559, -1.380330, 
+-1.221820, -1.440880, -1.342910, 
+-1.100830, -0.923191, -1.049020, 
+-1.146990, -0.774515, -0.621542, 
+-0.951057, -0.951057, -0.803055, 
+-0.635255, -0.951057, -0.951057, 
+-0.792548, -0.634038, -0.454301, 
+-0.479369, -0.158510, -0.317019, 
+0.149331, 0.178643, 0.295860, 
+0.306275, 0.246225, 0.406610, 
+0.367954, 0.498458, -0.014929, 
+0.084917, -0.094272, -0.000726, 
+-0.245374, -0.340755, -0.435351, 
+-0.490564, -0.176355, -0.294974, 
+-0.148010, -0.328246, 0.136248, 
+0.182883, 0.328957, 0.309903, 
+0.646446, 0.622546, 0.800859, 
+0.801803, 0.643087, 0.654473, 
+0.796963, 0.816799, 0.748189, 
+0.898148, 0.819092, 0.970033, 
+1.049540, 1.162660, 1.115310, 
+1.266700, 0.539960, 0.728396, 
+0.656318, 0.783152, 0.302716, 
+0.403858, 0.219006, 0.309252, 
+0.009667, 0.107161, -0.118211, 
+0.011786, -0.284704, -0.179858, 
+-0.411567, -0.296676, -0.570168, 
+-0.635348, -0.726784, -0.793211, 
+-1.016860, -1.112970, -1.170910, 
+-1.281530, -0.737860, -0.851723, 
+-0.883732, -0.995070, -0.653153, 
+-0.769939, -0.631918, -0.797688, 
+-0.616289, -0.806819, -0.638485, 
+-0.790356, -0.136399, -0.322769, 
+-0.165339, -0.309622]
+DataCollection/Mesh/PartitionInfo/Size: 2
+DataCollection/Mesh/PartitionInfo/Type: FMS_UINT64
+DataCollection/Mesh/PartitionInfo/Values: [0, 1]
+DataCollection/Mesh/NumDomainNames: 1
+DataCollection/Mesh/NumComponents: 1
+DataCollection/Mesh/NumTags: 0
+DataCollection/Mesh/DomainNames/0/Name: Domain
+DataCollection/Mesh/DomainNames/0/NumDomains: 1
+DataCollection/Mesh/DomainNames/0/Domains/0/Dimension: 2
+DataCollection/Mesh/DomainNames/0/Domains/0/NumVertices: 31
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/EntityType: FMS_EDGE
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/NumEntities: 50
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Size: 100
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Type: FMS_INT32
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Values: [11, 0, 26, 
+11, 26, 14, 
+14, 0, 27, 
+14, 27, 17, 
+17, 0, 28, 
+17, 28, 20, 
+20, 0, 29, 
+20, 29, 23, 
+23, 0, 30, 
+23, 30, 11, 
+11, 1, 12, 
+1, 26, 12, 
+12, 3, 13, 
+3, 26, 13, 
+13, 2, 14, 
+2, 15, 2, 
+27, 15, 15, 
+5, 16, 5, 
+27, 16, 16, 
+4, 17, 4, 
+18, 4, 28, 
+18, 18, 7, 
+19, 7, 28, 
+19, 19, 6, 
+20, 6, 21, 
+6, 29, 21, 
+21, 9, 22, 
+9, 29, 22, 
+22, 8, 23, 
+8, 24, 8, 
+30, 24, 24, 
+10, 25, 10, 
+30, 25, 25, 1]
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/EntityType: FMS_QUADRILATERAL
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/NumEntities: 20
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Size: 80
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Type: FMS_INT32
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Values: [0, 1, 2, 
+3, 3, 4, 
+5, 6, 6, 
+7, 8, 9, 
+9, 10, 11, 
+12, 12, 13, 
+14, 0, 15, 
+16, 17, 1, 
+17, 18, 19, 
+20, 2, 20, 
+21, 22, 22, 
+23, 24, 4, 
+24, 25, 26, 
+27, 5, 27, 
+28, 29, 29, 
+30, 31, 7, 
+31, 32, 33, 
+34, 8, 34, 
+35, 36, 36, 
+37, 38, 10, 
+38, 39, 40, 
+41, 11, 41, 
+42, 43, 43, 
+44, 45, 13, 
+45, 46, 47, 
+48, 14, 48, 
+49, 15]
+DataCollection/Mesh/Components/0/Name: volume
+DataCollection/Mesh/Components/0/Dimension: 2
+DataCollection/Mesh/Components/0/NumEntities: 20
+DataCollection/Mesh/Components/0/Coordinates: Coords
+DataCollection/Mesh/Components/0/NumParts: 1
+DataCollection/Mesh/Components/0/Parts/0/DomainName: Domain
+DataCollection/Mesh/Components/0/Parts/0/DomainID: 0
+DataCollection/Mesh/Components/0/Parts/0/FullDomain: Yes
+DataCollection/Mesh/Components/0/Relations/Size: 0
+DataCollection/Mesh/Components/0/Relations/Type: FMS_UINT64
@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.

-PROJECT_NUMBER         = v4.2.1
+PROJECT_NUMBER         = v4.3.1

 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -84,8 +84,9 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})

-  if (NOT (${TEST_NAME} MATCHES "ex0p?"))
-    set(THIS_TEST_OPTIONS "-no-vis")
+  set(THIS_TEST_OPTIONS "-no-vis")
+  if (${TEST_NAME} MATCHES "ex0p?")
+    set(THIS_TEST_OPTIONS)
  endif()
  if (${TEST_NAME} MATCHES "ex10p*")
    list(APPEND THIS_TEST_OPTIONS "-tf" "5")
@@ -107,6 +108,34 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
  endif()
 endforeach()

+# Add CUDA/HIP tests.
+set(DEVICE_EXAMPLES
+    # serial examples with device support:
+    ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
+    # parallel examples with device support:
+    ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p ex24p ex25p ex26p)
+set(MFEM_TEST_DEVICE)
+if (MFEM_USE_CUDA)
+  set(MFEM_TEST_DEVICE "cuda")
+elseif (MFEM_USE_HIP)
+  set(MFEM_TEST_DEVICE "hip")
+endif()
+if (MFEM_TEST_DEVICE)
+  foreach(TEST_NAME ${DEVICE_EXAMPLES})
+    set(THIS_TEST_OPTIONS "-no-vis" "-d" "${MFEM_TEST_DEVICE}")
+    if (NOT (${TEST_NAME} MATCHES ".*p$"))
+      add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_ser
+        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+    elseif (MFEM_USE_MPI)
+      add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_np=${MFEM_MPI_NP}
+        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+        ${MPIEXEC_PREFLAGS}
+        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+        ${MPIEXEC_POSTFLAGS})
+    endif()
+  endforeach()
+endif()
+
 # If STRUMPACK is enabled, add a test run that uses it.
 if (MFEM_USE_STRUMPACK)
  add_test(NAME ex11p_strumpack_np=${MFEM_MPI_NP}
@@ -0,0 +1,18 @@
+                    Finite Element Discretization Library
+                                   __
+                       _ __ ___   / _|  ___  _ __ ___
+                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
+                      | | | | | ||  _||  __/| | | | | |
+                      |_| |_| |_||_|   \___||_| |_| |_|
+
+                               https://mfem.org
+
+This directory contains modifications of the example codes that illustrate the
+use of MFEM features based on the Caliper performance profiling library.
+
+To build these examples, make sure that MFEM is configured with the option
+"MFEM_USE_CALIPER = YES", see the top-level INSTALL file for details (version
+2.5.0 of Caliper is recommended, though older versions may work too).
+
+We recommend comparing the original example codes with the corresponding files
+in the current directory.
@@ -55,6 +55,7 @@ int main(int argc, char *argv[])
   int order = 1;
   int nev = 5;
   bool visualization = 1;
+   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -71,6 +72,8 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -86,13 +89,18 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   // 3. Read the (serial) mesh from the given mesh file on all processors. We
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   if (myid == 0) { device.Print(); }
+
+   // 4. Read the (serial) mesh from the given mesh file on all processors. We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   // 5. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ref_levels' of uniform refinement (2 by default, or
   //    specified on the command line with -rs).
   for (int lev = 0; lev < ser_ref_levels; lev++)
@@ -100,7 +108,7 @@ int main(int argc, char *argv[])
      mesh->UniformRefinement();
   }

-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution (1 time by
   //    default, or specified on the command line with -rp). Once the parallel
   //    mesh is defined, the serial mesh can be deleted.
@@ -112,7 +120,7 @@ int main(int argc, char *argv[])
   }
   pmesh->ReorientTetMesh();

-   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   // 7. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Nedelec finite elements of the specified order.
   FiniteElementCollection *fec = new ND_FECollection(order, dim);
   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
@@ -122,7 +130,7 @@ int main(int argc, char *argv[])
      cout << "Number of unknowns: " << size << endl;
   }

-   // 7. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
+   // 8. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
   //    element space. The first corresponds to the curl curl, while the second
   //    is a simple mass matrix needed on the right hand side of the
   //    generalized eigenvalue problem below. The boundary conditions are
@@ -164,7 +172,7 @@ int main(int argc, char *argv[])
   delete a;
   delete m;

-   // 8. Define and configure the AME eigensolver and the AMS preconditioner for
+   // 9. Define and configure the AME eigensolver and the AMS preconditioner for
   //    A to be used within the solver. Set the matrices which define the
   //    generalized eigenproblem A x = lambda M x.
   HypreAMS *ams = new HypreAMS(*A,fespace);
@@ -180,15 +188,15 @@ int main(int argc, char *argv[])
   ame->SetMassMatrix(*M);
   ame->SetOperator(*A);

-   // 9. Compute the eigenmodes and extract the array of eigenvalues. Define a
-   //    parallel grid function to represent each of the eigenmodes returned by
-   //    the solver.
+   // 10. Compute the eigenmodes and extract the array of eigenvalues. Define a
+   //     parallel grid function to represent each of the eigenmodes returned by
+   //     the solver.
   Array<double> eigenvalues;
   ame->Solve();
   ame->GetEigenvalues(eigenvalues);
   ParGridFunction x(fespace);

-   // 10. Save the refined mesh and the modes in parallel. This output can be
+   // 11. Save the refined mesh and the modes in parallel. This output can be
   //     viewed later using GLVis: "glvis -np <np> -m mesh -g mode".
   {
      ostringstream mesh_name, mode_name;
@@ -213,7 +221,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 11. Send the solution by socket to a GLVis server.
+   // 12. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -253,7 +261,7 @@ int main(int argc, char *argv[])
      mode_sock.close();
   }

-   // 12. Free the used memory.
+   // 13. Free the used memory.
   delete ame;
   delete ams;
   delete M;
@@ -196,6 +196,12 @@ void InitialDeformation(const Vector &x, Vector &y);

 int main(int argc, char *argv[])
 {
+#ifdef HYPRE_USING_CUDA
+   cout << "\nAs of mfem-4.3 and hypre-2.22.0 (July 2021) this example\n"
+        << "is NOT supported with the CUDA version of hypre.\n\n";
+   return 255;
+#endif
+
   // 1. Initialize MPI
   MPI_Session mpi;
   const int myid = mpi.WorldRank();
@@ -438,16 +444,20 @@ JacobianPreconditioner::JacobianPreconditioner(Array<ParFiniteElementSpace *>
 void JacobianPreconditioner::Mult(const Vector &k, Vector &y) const
 {
   // Extract the blocks from the input and output vectors
-   Vector disp_in(k.GetData() + block_trueOffsets[0],
-                  block_trueOffsets[1]-block_trueOffsets[0]);
-   Vector pres_in(k.GetData() + block_trueOffsets[1],
-                  block_trueOffsets[2]-block_trueOffsets[1]);
-
-   Vector disp_out(y.GetData() + block_trueOffsets[0],
+   Vector disp_in;
+   disp_in.MakeRef(const_cast<Vector&>(k), block_trueOffsets[0],
                   block_trueOffsets[1]-block_trueOffsets[0]);
-   Vector pres_out(y.GetData() + block_trueOffsets[1],
+   Vector pres_in;
+   pres_in.MakeRef(const_cast<Vector&>(k), block_trueOffsets[1],
                   block_trueOffsets[2]-block_trueOffsets[1]);

+   Vector disp_out;
+   disp_out.MakeRef(y, block_trueOffsets[0],
+                    block_trueOffsets[1]-block_trueOffsets[0]);
+   Vector pres_out;
+   pres_out.MakeRef(y, block_trueOffsets[1],
+                    block_trueOffsets[2]-block_trueOffsets[1]);
+
   Vector temp(block_trueOffsets[1]-block_trueOffsets[0]);
   Vector temp2(block_trueOffsets[1]-block_trueOffsets[0]);

@@ -459,6 +469,9 @@ void JacobianPreconditioner::Mult(const Vector &k, Vector &y) const
   subtract(disp_in, temp, temp2);

   stiff_pcg->Mult(temp2, disp_out);
+
+   disp_out.SyncAliasMemory(y);
+   pres_out.SyncAliasMemory(y);
 }

 void JacobianPreconditioner::SetOperator(const Operator &op)
@@ -473,7 +486,10 @@ void JacobianPreconditioner::SetOperator(const Operator &op)

      if (!spaces[0]->GetParMesh()->Nonconforming())
      {
+#ifndef HYPRE_USING_CUDA
+         // Not available yet when hypre is built with CUDA
         stiff_prec_amg->SetElasticityOptions(spaces[0]);
+#endif
      }

      stiff_prec = stiff_prec_amg;
@@ -89,7 +89,8 @@ int main(int argc, char *argv[])
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
 #ifdef MFEM_USE_CEED
-   args.AddOption(&algebraic_ceed, "-a", "--algebraic", "-no-a", "--no-algebraic",
+   args.AddOption(&algebraic_ceed, "-a", "--algebraic",
+                  "-no-a", "--no-algebraic",
                  "Use algebraic Ceed solver");
 #endif
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -197,15 +198,15 @@ int main(int argc, char *argv[])
   b.AddDomainIntegrator(new DomainLFIntegrator(one));
   b.Assemble();

-   // 10. Define the solution vector x as a parallel finite element grid function
-   //     corresponding to fespace. Initialize x with initial guess of zero,
-   //     which satisfies the boundary conditions.
+   // 10. Define the solution vector x as a parallel finite element grid
+   //     function corresponding to fespace. Initialize x with initial guess of
+   //     zero, which satisfies the boundary conditions.
   ParGridFunction x(&fespace);
   x = 0.0;

   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //     domain integrator.
+   //     corresponding to the Laplacian operator -Delta, by adding the
+   //     Diffusion domain integrator.
   ParBilinearForm a(&fespace);
   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   a.AddDomainIntegrator(new DiffusionIntegrator(one));
@@ -105,7 +105,7 @@ private:
      Vector diag(fespace.GetTrueVSize());
      bfs.Last()->AssembleDiagonal(diag);

-      Solver* smoother = new OperatorChebyshevSmoother(opr.Ptr(), diag,
+      Solver* smoother = new OperatorChebyshevSmoother(*opr, diag,
                                                       *essentialTrueDofs.Last(), 2);
      AddLevel(opr.Ptr(), smoother, true, true);
   }
@@ -115,7 +115,7 @@ private:
      Vector diag(fespace.GetTrueVSize());
      bfs.Last()->AssembleDiagonal(diag);

-      Solver* smoother = new OperatorChebyshevSmoother(opr.Ptr(), diag,
+      Solver* smoother = new OperatorChebyshevSmoother(*opr, diag,
                                                       *essentialTrueDofs.Last(), 2, fespace.GetParMesh()->GetComm());

      AddLevel(opr.Ptr(), smoother, true, true);
@@ -81,6 +81,12 @@ Mesh * build_trapezoid_mesh(double offset)

 int main(int argc, char *argv[])
 {
+#ifdef HYPRE_USING_CUDA
+   cout << "\nAs of mfem-4.3 and hypre-2.22.0 (July 2021) this example\n"
+        << "is NOT supported with the CUDA version of hypre.\n\n";
+   return 255;
+#endif
+
   // 1. Initialize MPI.
   int num_procs, myid;
   MPI_Init(&argc, &argv);
@@ -360,6 +366,7 @@ int main(int argc, char *argv[])
   }
   delete pmesh;

+   // HYPRE_Finalize();
   MPI_Finalize();

   return 0;
@@ -61,6 +61,7 @@ int main(int argc, char *argv[])
   bool visualization = 1;
   bool amg_elast = 0;
   bool reorder_space = false;
+   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -78,6 +79,8 @@ int main(int argc, char *argv[])
                  "Enable or disable GLVis visualization.");
   args.AddOption(&reorder_space, "-nodes", "--by-nodes", "-vdim", "--by-vdim",
                  "Use byNODES ordering of vector space instead of byVDIM");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -93,7 +96,12 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   if (myid == 0) { device.Print(); }
+
+   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
@@ -109,14 +117,14 @@ int main(int argc, char *argv[])
      return 3;
   }

-   // 4. Select the order of the finite element discretization space. For NURBS
+   // 5. Select the order of the finite element discretization space. For NURBS
   //    meshes, we increase the order by degree elevation.
   if (mesh->NURBSext)
   {
      mesh->DegreeElevate(order, order);
   }

-   // 5. Refine the serial mesh on all processors to increase the resolution. In
+   // 6. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ref_levels' of uniform refinement. We choose
   //    'ref_levels' to be the largest number that gives a final mesh with no
   //    more than 1,000 elements.
@@ -129,7 +137,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 7. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -142,7 +150,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   // 8. Define a parallel finite element space on the parallel mesh. Here we
   //    use vector finite elements, i.e. dim copies of a scalar finite element
   //    space. We use the ordering by vector dimension (the last argument of
   //    the FiniteElementSpace constructor) which is expected in the systems
@@ -175,7 +183,7 @@ int main(int argc, char *argv[])
           << "Assembling: " << flush;
   }

-   // 8. Determine the list of true (i.e. parallel conforming) essential
+   // 9. Determine the list of true (i.e. parallel conforming) essential
   //    boundary dofs. In this example, the boundary conditions are defined by
   //    marking only boundary attribute 1 from the mesh as essential and
   //    converting it to a list of true dofs.
@@ -184,14 +192,14 @@ int main(int argc, char *argv[])
   ess_bdr[0] = 1;
   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);

-   // 9. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system. In this case, b_i equals the
-   //    boundary integral of f*phi_i where f represents a "pull down" force on
-   //    the Neumann part of the boundary and phi_i are the basis functions in
-   //    the finite element fespace. The force is defined by the object f, which
-   //    is a vector of Coefficient objects. The fact that f is non-zero on
-   //    boundary attribute 2 is indicated by the use of piece-wise constants
-   //    coefficient for its last component.
+   // 10. Set up the parallel linear form b(.) which corresponds to the
+   //     right-hand side of the FEM linear system. In this case, b_i equals the
+   //     boundary integral of f*phi_i where f represents a "pull down" force on
+   //     the Neumann part of the boundary and phi_i are the basis functions in
+   //     the finite element fespace. The force is defined by the object f, which
+   //     is a vector of Coefficient objects. The fact that f is non-zero on
+   //     boundary attribute 2 is indicated by the use of piece-wise constants
+   //     coefficient for its last component.
   VectorArrayCoefficient f(dim);
   for (int i = 0; i < dim-1; i++)
   {
@@ -212,13 +220,13 @@ int main(int argc, char *argv[])
   }
   b->Assemble();

-   // 10. Define the solution vector x as a parallel finite element grid
+   // 11. Define the solution vector x as a parallel finite element grid
   //     function corresponding to fespace. Initialize x with initial guess of
   //     zero, which satisfies the boundary conditions.
   ParGridFunction x(fespace);
   x = 0.0;

-   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
+   // 12. Set up the parallel bilinear form a(.,.) on the finite element space
   //     corresponding to the linear elasticity integrator with piece-wise
   //     constants coefficient lambda and mu.
   Vector lambda(pmesh->attributes.Max());
@@ -233,7 +241,7 @@ int main(int argc, char *argv[])
   ParBilinearForm *a = new ParBilinearForm(fespace);
   a->AddDomainIntegrator(new ElasticityIntegrator(lambda_func, mu_func));

-   // 12. Assemble the parallel bilinear form and the corresponding linear
+   // 13. Assemble the parallel bilinear form and the corresponding linear
   //     system, applying any necessary transformations such as: parallel
   //     assembly, eliminating boundary conditions, applying conforming
   //     constraints for non-conforming AMR, static condensation, etc.
@@ -250,7 +258,7 @@ int main(int argc, char *argv[])
      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
   }

-   // 13. Define and apply a parallel PCG solver for A X = B with the BoomerAMG
+   // 14. Define and apply a parallel PCG solver for A X = B with the BoomerAMG
   //     preconditioner from hypre.
   HypreBoomerAMG *amg = new HypreBoomerAMG(A);
   if (amg_elast && !a->StaticCondensationIsEnabled())
@@ -268,11 +276,11 @@ int main(int argc, char *argv[])
   pcg->SetPreconditioner(*amg);
   pcg->Mult(B, X);

-   // 14. Recover the parallel grid function corresponding to X. This is the
+   // 15. Recover the parallel grid function corresponding to X. This is the
   //     local finite element solution on each processor.
   a->RecoverFEMSolution(X, *b, x);

-   // 15. For non-NURBS meshes, make the mesh curved based on the finite element
+   // 16. For non-NURBS meshes, make the mesh curved based on the finite element
   //     space. This means that we define the mesh elements through a fespace
   //     based transformation of the reference element.  This allows us to save
   //     the displaced mesh as a curved mesh when using high-order finite
@@ -284,7 +292,7 @@ int main(int argc, char *argv[])
      pmesh->SetNodalFESpace(fespace);
   }

-   // 16. Save in parallel the displaced mesh and the inverted solution (which
+   // 17. Save in parallel the displaced mesh and the inverted solution (which
   //     gives the backward displacements to the original grid). This output
   //     can be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
   {
@@ -305,7 +313,7 @@ int main(int argc, char *argv[])
      x.Save(sol_ofs);
   }

-   // 17. Send the above data by socket to a GLVis server.  Use the "n" and "b"
+   // 18. Send the above data by socket to a GLVis server.  Use the "n" and "b"
   //     keys in GLVis to visualize the displacements.
   if (visualization)
   {
@@ -317,7 +325,7 @@ int main(int argc, char *argv[])
      sol_sock << "solution\n" << *pmesh << x << flush;
   }

-   // 18. Free the used memory.
+   // 19. Free the used memory.
   delete pcg;
   delete amg;
   delete a;
@@ -103,6 +103,7 @@ int main(int argc, char *argv[])
      {
         args.PrintUsage(cout);
      }
+      // HYPRE_Finalize();
      MPI_Finalize();
      return 1;
   }
@@ -197,6 +197,7 @@ int main(int argc, char *argv[])
      SparseMatrix &M(mVarf->SpMat());
      SparseMatrix &B(bVarf->SpMat());
      B *= -1.;
+      if (Device::IsEnabled()) { B.BuildTranspose(); }
      Bt = new TransposeOperator(&B);

      darcyOp.SetBlock(0,0, &M);
@@ -240,6 +241,7 @@ int main(int argc, char *argv[])
   {
      SparseMatrix &M(mVarf->SpMat());
      M.GetDiag(Md);
+      Md.HostReadWrite();

      SparseMatrix &B(bVarf->SpMat());
      MinvBt = Transpose(B);
@@ -287,12 +289,18 @@ int main(int argc, char *argv[])
   chrono.Stop();

   if (solver.GetConverged())
+   {
      std::cout << "MINRES converged in " << solver.GetNumIterations()
-                << " iterations with a residual norm of " << solver.GetFinalNorm() << ".\n";
+                << " iterations with a residual norm of "
+                << solver.GetFinalNorm() << ".\n";
+   }
   else
+   {
      std::cout << "MINRES did not converge in " << solver.GetNumIterations()
-                << " iterations. Residual norm is " << solver.GetFinalNorm() << ".\n";
-   std::cout << "MINRES solver took " << chrono.RealTime() << "s. \n";
+                << " iterations. Residual norm is " << solver.GetFinalNorm()
+                << ".\n";
+   }
+   std::cout << "MINRES solver took " << chrono.RealTime() << "s.\n";

   // 12. Create the grid functions u and p. Compute the L2 error norms.
   GridFunction u, p;
@@ -47,6 +47,7 @@ int main(int argc, char *argv[])
   int order = 2;
   bool always_snap = false;
   bool visualization = 1;
+   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&elem_type, "-e", "--elem",
@@ -65,6 +66,8 @@ int main(int argc, char *argv[])
                  "--snap-at-the-end",
                  "If true, snap nodes to the sphere initially and after each refinement "
                  "otherwise, snap only after the last refinement");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -80,7 +83,12 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   // 3. Generate an initial high-order (surface) mesh on the unit sphere. The
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   if (myid == 0) { device.Print(); }
+
+   // 4. Generate an initial high-order (surface) mesh on the unit sphere. The
   //    Mesh object represents a 2D mesh in 3 spatial dimensions. We first add
   //    the elements and the vertices of the mesh, and then make it high-order
   //    by specifying a finite element space for its nodes.
@@ -146,7 +154,7 @@ int main(int argc, char *argv[])
   FiniteElementSpace nodal_fes(mesh, &fec, mesh->SpaceDimension());
   mesh->SetNodalFESpace(&nodal_fes);

-   // 4. Refine the mesh while snapping nodes to the sphere. Number of parallel
+   // 5. Refine the mesh while snapping nodes to the sphere. Number of parallel
   //    refinements is fixed to 2.
   for (int l = 0; l <= ref_levels; l++)
   {
@@ -218,7 +226,7 @@ int main(int argc, char *argv[])
      SnapNodes(*pmesh);
   }

-   // 5. Define a finite element space on the mesh. Here we use isoparametric
+   // 6. Define a finite element space on the mesh. Here we use isoparametric
   //    finite elements -- the same as the mesh nodes.
   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, &fec);
   HYPRE_BigInt size = fespace->GlobalTrueVSize();
@@ -227,7 +235,7 @@ int main(int argc, char *argv[])
      cout << "Number of unknowns: " << size << endl;
   }

-   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
+   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
   //    the basis functions in the finite element fespace.
   ParLinearForm *b = new ParLinearForm(fespace);
@@ -237,27 +245,27 @@ int main(int argc, char *argv[])
   b->AddDomainIntegrator(new DomainLFIntegrator(rhs_coef));
   b->Assemble();

-   // 7. Define the solution vector x as a finite element grid function
+   // 8. Define the solution vector x as a finite element grid function
   //    corresponding to fespace. Initialize x with initial guess of zero.
   ParGridFunction x(fespace);
   x = 0.0;

-   // 8. Set up the bilinear form a(.,.) on the finite element space
+   // 9. Set up the bilinear form a(.,.) on the finite element space
   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
   //    and Mass domain integrators.
   ParBilinearForm *a = new ParBilinearForm(fespace);
   a->AddDomainIntegrator(new DiffusionIntegrator(one));
   a->AddDomainIntegrator(new MassIntegrator(one));

-   // 9. Assemble the parallel linear system, applying any transformations
-   //    such as: parallel assembly, applying conforming constraints, etc.
+   // 10. Assemble the parallel linear system, applying any transformations
+   //     such as: parallel assembly, applying conforming constraints, etc.
   a->Assemble();
   HypreParMatrix A;
   Vector B, X;
   Array<int> empty_tdof_list;
   a->FormLinearSystem(empty_tdof_list, x, *b, A, X, B);

-   // 10. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
+   // 11. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
   //     preconditioner from hypre. Extract the parallel grid function x
   //     corresponding to the finite element approximation X. This is the local
   //     solution on each processor.
@@ -273,14 +281,14 @@ int main(int argc, char *argv[])
   delete a;
   delete b;

-   // 11. Compute and print the L^2 norm of the error.
+   // 12. Compute and print the L^2 norm of the error.
   double err = x.ComputeL2Error(sol_coef);
   if (myid == 0)
   {
      cout << "\nL2 norm of error: " << err << endl;
   }

-   // 12. Save the refined mesh and the solution. This output can be viewed
+   // 13. Save the refined mesh and the solution. This output can be viewed
   //     later using GLVis: "glvis -np <np> -m sphere_refined -g sol".
   {
      ostringstream mesh_name, sol_name;
@@ -296,7 +304,7 @@ int main(int argc, char *argv[])
      x.Save(sol_ofs);
   }

-   // 13. Send the solution by socket to a GLVis server.
+   // 14. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -307,7 +315,7 @@ int main(int argc, char *argv[])
      sol_sock << "solution\n" << *pmesh << x << flush;
   }

-   // 14. Free the used memory.
+   // 15. Free the used memory.
   delete pcg;
   delete amg;
   delete fespace;
@@ -26,6 +26,9 @@ SEQ_EXAMPLES = ex0 ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 \
 PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
  ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
  ex25p ex26p ex27p ex28p ex29p
+SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
+PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p \
+  ex24p ex25p ex26p

 ifeq ($(MFEM_USE_MPI),NO)
   EXAMPLES = $(SEQ_EXAMPLES)
@@ -99,6 +102,14 @@ RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
 	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
 %-test-seq: %
 	@$(call mfem-test,$<,, Serial example)
+%-test-par-cuda: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel CUDA example,-d cuda)
+%-test-seq-cuda: %
+	@$(call mfem-test,$<,, Serial CUDA example,-d cuda)
+%-test-par-hip: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel HIP example,-d hip)
+%-test-seq-hip: %
+	@$(call mfem-test,$<,, Serial HIP example,-d hip)

 # Testing: Specific execution options
 ex0-test-seq: ex0
@@ -282,6 +282,10 @@ int main(int argc, char *argv[])
   superlu->SetOperator(*SLU_A);
   superlu->SetPrintStatistics(true);
   superlu->Mult(B, X);
+   superlu->DismantleGrid();
+
+   delete SLU_A;
+   delete superlu;

   // 14. Recover the parallel grid function corresponding to X. This is the
   //     local finite element solution on each processor.
@@ -105,6 +105,7 @@ set(SRCS
  tmop/tmop_pa_w3.cpp
  tmop/tmop_pa_w3_c0.cpp
  tmop_tools.cpp
+  tmop_amr.cpp
  gslib.cpp
  transfer.cpp
  lor.cpp
@@ -164,6 +165,7 @@ set(HDRS
  tmop.hpp
  tmop/tmop_pa.hpp
  tmop_tools.hpp
+  tmop_amr.hpp
  gslib.hpp
  transfer.hpp
  lor.hpp
@@ -184,6 +186,11 @@ if (MFEM_USE_ADIOS2)
  list(APPEND HDRS adios2datacollection.hpp)
 endif()

+if (MFEM_USE_FMS)
+  list(APPEND SRCS fmsdatacollection.cpp fmsconvert.cpp)
+  list(APPEND HDRS fmsdatacollection.hpp fmsconvert.hpp)
+endif()
+
 if (MFEM_USE_MPI)
  list(APPEND SRCS
    pbilinearform.cpp
@@ -31,7 +31,7 @@ void BilinearForm::AllocMat()
   const Table &elem_dof = fes->GetElementToDofTable();
   Table dof_dof;

-   if (fbfi.Size() > 0)
+   if (interior_face_integs.Size() > 0)
   {
      // the sparsity pattern is defined from the map: face->element->dof
      Table face_dof, dof_face;
@@ -99,15 +99,15 @@ BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)
   ext = NULL;

   // Copy the pointers to the integrators
-   dbfi = bf->dbfi;
+   domain_integs = bf->domain_integs;

-   bbfi = bf->bbfi;
-   bbfi_marker = bf->bbfi_marker;
+   boundary_integs = bf->boundary_integs;
+   boundary_integs_marker = bf->boundary_integs_marker;

-   fbfi = bf->fbfi;
+   interior_face_integs = bf->interior_face_integs;

-   bfbfi = bf->bfbfi;
-   bfbfi_marker = bf->bfbfi_marker;
+   boundary_face_integs = bf->boundary_face_integs;
+   boundary_face_integs_marker = bf->boundary_face_integs_marker;

   AllocMat();
 }
@@ -234,46 +234,47 @@ void BilinearForm::Finalize (int skip_zeros)

 void BilinearForm::AddDomainIntegrator(BilinearFormIntegrator *bfi)
 {
-   dbfi.Append(bfi);
-   dbfi_marker.Append(NULL); // NULL marker means apply everywhere
+   domain_integs.Append(bfi);
+   domain_integs_marker.Append(NULL); // NULL marker means apply everywhere
 }

 void BilinearForm::AddDomainIntegrator(BilinearFormIntegrator *bfi,
                                       Array<int> &elem_marker)
 {
-   dbfi.Append(bfi);
-   dbfi_marker.Append(&elem_marker);
+   domain_integs.Append(bfi);
+   domain_integs_marker.Append(&elem_marker);
 }

 void BilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi)
 {
-   bbfi.Append (bfi);
-   bbfi_marker.Append(NULL); // NULL marker means apply everywhere
+   boundary_integs.Append (bfi);
+   boundary_integs_marker.Append(NULL); // NULL marker means apply everywhere
 }

 void BilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi,
                                          Array<int> &bdr_marker)
 {
-   bbfi.Append (bfi);
-   bbfi_marker.Append(&bdr_marker);
+   boundary_integs.Append (bfi);
+   boundary_integs_marker.Append(&bdr_marker);
 }

 void BilinearForm::AddInteriorFaceIntegrator(BilinearFormIntegrator * bfi)
 {
-   fbfi.Append (bfi);
+   interior_face_integs.Append (bfi);
 }

 void BilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator *bfi)
 {
-   bfbfi.Append(bfi);
-   bfbfi_marker.Append(NULL); // NULL marker means apply everywhere
+   boundary_face_integs.Append(bfi);
+   // NULL marker means apply everywhere
+   boundary_face_integs_marker.Append(NULL);
 }

 void BilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator *bfi,
                                        Array<int> &bdr_marker)
 {
-   bfbfi.Append(bfi);
-   bfbfi_marker.Append(&bdr_marker);
+   boundary_face_integs.Append(bfi);
+   boundary_face_integs_marker.Append(&bdr_marker);
 }

 void BilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
@@ -285,14 +286,14 @@ void BilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
      return;
   }

-   if (dbfi.Size())
+   if (domain_integs.Size())
   {
      const FiniteElement &fe = *fes->GetFE(i);
      ElementTransformation *eltrans = fes->GetElementTransformation(i);
-      dbfi[0]->AssembleElementMatrix(fe, *eltrans, elmat);
-      for (int k = 1; k < dbfi.Size(); k++)
+      domain_integs[0]->AssembleElementMatrix(fe, *eltrans, elmat);
+      for (int k = 1; k < domain_integs.Size(); k++)
      {
-         dbfi[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
+         domain_integs[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
         elmat += elemmat;
      }
   }
@@ -306,14 +307,14 @@ void BilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)

 void BilinearForm::ComputeBdrElementMatrix(int i, DenseMatrix &elmat)
 {
-   if (bbfi.Size())
+   if (boundary_integs.Size())
   {
      const FiniteElement &be = *fes->GetBE(i);
      ElementTransformation *eltrans = fes->GetBdrElementTransformation(i);
-      bbfi[0]->AssembleElementMatrix(be, *eltrans, elmat);
-      for (int k = 1; k < bbfi.Size(); k++)
+      boundary_integs[0]->AssembleElementMatrix(be, *eltrans, elmat);
+      for (int k = 1; k < boundary_integs.Size(); k++)
      {
-         bbfi[k]->AssembleElementMatrix(be, *eltrans, elemmat);
+         boundary_integs[k]->AssembleElementMatrix(be, *eltrans, elemmat);
         elmat += elemmat;
      }
   }
@@ -407,13 +408,14 @@ void BilinearForm::Assemble(int skip_zeros)
   }
 #endif

-   if (dbfi.Size())
+   if (domain_integs.Size())
   {
-      for (int k = 0; k < dbfi.Size(); k++)
+      for (int k = 0; k < domain_integs.Size(); k++)
      {
-         if (dbfi_marker[k] != NULL)
+         if (domain_integs_marker[k] != NULL)
         {
-            MFEM_VERIFY(mesh->attributes.Size() == dbfi_marker[k]->Size(),
+            MFEM_VERIFY(mesh->attributes.Size() ==
+                        domain_integs_marker[k]->Size(),
                        "invalid element marker for domain integrator #"
                        << k << ", counting from zero");
         }
@@ -430,14 +432,14 @@ void BilinearForm::Assemble(int skip_zeros)
         else
         {
            elmat.SetSize(0);
-            for (int k = 0; k < dbfi.Size(); k++)
+            for (int k = 0; k < domain_integs.Size(); k++)
            {
-               if ( dbfi_marker[k] == NULL ||
-                    (*(dbfi_marker[k]))[elem_attr-1] == 1)
+               if ( domain_integs_marker[k] == NULL ||
+                    (*(domain_integs_marker[k]))[elem_attr-1] == 1)
               {
                  const FiniteElement &fe = *fes->GetFE(i);
                  eltrans = fes->GetElementTransformation(i);
-                  dbfi[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
+                  domain_integs[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
                  if (elmat.Size() == 0)
                  {
                     elmat = elemmat;
@@ -472,20 +474,20 @@ void BilinearForm::Assemble(int skip_zeros)
      }
   }

-   if (bbfi.Size())
+   if (boundary_integs.Size())
   {
      // Which boundary attributes need to be processed?
      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
                                 mesh->bdr_attributes.Max() : 0);
      bdr_attr_marker = 0;
-      for (int k = 0; k < bbfi.Size(); k++)
+      for (int k = 0; k < boundary_integs.Size(); k++)
      {
-         if (bbfi_marker[k] == NULL)
+         if (boundary_integs_marker[k] == NULL)
         {
            bdr_attr_marker = 1;
            break;
         }
-         Array<int> &bdr_marker = *bbfi_marker[k];
+         Array<int> &bdr_marker = *boundary_integs_marker[k];
         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
                     "invalid boundary marker for boundary integrator #"
                     << k << ", counting from zero");
@@ -504,21 +506,21 @@ void BilinearForm::Assemble(int skip_zeros)
         fes -> GetBdrElementVDofs (i, vdofs);
         eltrans = fes -> GetBdrElementTransformation (i);
         int k = 0;
-         for (; k < bbfi.Size(); k++)
+         for (; k < boundary_integs.Size(); k++)
         {
-            if (bbfi_marker[k] &&
-                (*bbfi_marker[k])[bdr_attr-1] == 0) { continue; }
+            if (boundary_integs_marker[k] &&
+                (*boundary_integs_marker[k])[bdr_attr-1] == 0) { continue; }

-            bbfi[k]->AssembleElementMatrix(be, *eltrans, elmat);
+            boundary_integs[k]->AssembleElementMatrix(be, *eltrans, elmat);
            k++;
            break;
         }
-         for (; k < bbfi.Size(); k++)
+         for (; k < boundary_integs.Size(); k++)
         {
-            if (bbfi_marker[k] &&
-                (*bbfi_marker[k])[bdr_attr-1] == 0) { continue; }
+            if (boundary_integs_marker[k] &&
+                (*boundary_integs_marker[k])[bdr_attr-1] == 0) { continue; }

-            bbfi[k]->AssembleElementMatrix(be, *eltrans, elemmat);
+            boundary_integs[k]->AssembleElementMatrix(be, *eltrans, elemmat);
            elmat += elemmat;
         }
         if (!static_cond)
@@ -536,7 +538,7 @@ void BilinearForm::Assemble(int skip_zeros)
      }
   }

-   if (fbfi.Size())
+   if (interior_face_integs.Size())
   {
      FaceElementTransformations *tr;
      Array<int> vdofs2;
@@ -550,18 +552,19 @@ void BilinearForm::Assemble(int skip_zeros)
            fes -> GetElementVDofs (tr -> Elem1No, vdofs);
            fes -> GetElementVDofs (tr -> Elem2No, vdofs2);
            vdofs.Append (vdofs2);
-            for (int k = 0; k < fbfi.Size(); k++)
+            for (int k = 0; k < interior_face_integs.Size(); k++)
            {
-               fbfi[k] -> AssembleFaceMatrix (*fes -> GetFE (tr -> Elem1No),
-                                              *fes -> GetFE (tr -> Elem2No),
-                                              *tr, elemmat);
+               interior_face_integs[k]->
+               AssembleFaceMatrix(*fes->GetFE(tr->Elem1No),
+                                  *fes->GetFE(tr->Elem2No),
+                                  *tr, elemmat);
               mat -> AddSubMatrix (vdofs, vdofs, elemmat, skip_zeros);
            }
         }
      }
   }

-   if (bfbfi.Size())
+   if (boundary_face_integs.Size())
   {
      FaceElementTransformations *tr;
      const FiniteElement *fe1, *fe2;
@@ -570,14 +573,14 @@ void BilinearForm::Assemble(int skip_zeros)
      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
                                 mesh->bdr_attributes.Max() : 0);
      bdr_attr_marker = 0;
-      for (int k = 0; k < bfbfi.Size(); k++)
+      for (int k = 0; k < boundary_face_integs.Size(); k++)
      {
-         if (bfbfi_marker[k] == NULL)
+         if (boundary_face_integs_marker[k] == NULL)
         {
            bdr_attr_marker = 1;
            break;
         }
-         Array<int> &bdr_marker = *bfbfi_marker[k];
+         Array<int> &bdr_marker = *boundary_face_integs_marker[k];
         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
                     "invalid boundary marker for boundary face integrator #"
                     << k << ", counting from zero");
@@ -601,12 +604,14 @@ void BilinearForm::Assemble(int skip_zeros)
            // but we can't dereference a NULL pointer, and we don't want to
            // actually make a fake element.
            fe2 = fe1;
-            for (int k = 0; k < bfbfi.Size(); k++)
+            for (int k = 0; k < boundary_face_integs.Size(); k++)
            {
-               if (bfbfi_marker[k] &&
-                   (*bfbfi_marker[k])[bdr_attr-1] == 0) { continue; }
+               if (boundary_face_integs_marker[k] &&
+                   (*boundary_face_integs_marker[k])[bdr_attr-1] == 0)
+               { continue; }

-               bfbfi[k] -> AssembleFaceMatrix (*fe1, *fe2, *tr, elemmat);
+               boundary_face_integs[k] -> AssembleFaceMatrix (*fe1, *fe2, *tr,
+                                                              elemmat);
               mat -> AddSubMatrix (vdofs, vdofs, elemmat, skip_zeros);
            }
         }
@@ -720,8 +725,8 @@ void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
      {
         // A, X and B point to the same data as mat, x and b
         EliminateVDofsInRHS(ess_tdof_list, x, b);
-         X.NewMemoryAndSize(x.GetMemory(), x.Size(), false);
-         B.NewMemoryAndSize(b.GetMemory(), b.Size(), false);
+         X.MakeRef(x, 0, x.Size());
+         B.MakeRef(b, 0, b.Size());
         if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
      }
   }
@@ -857,7 +862,7 @@ void BilinearForm::RecoverFEMSolution(const Vector &X,

 void BilinearForm::ComputeElementMatrices()
 {
-   if (element_matrices || dbfi.Size() == 0 || fes->GetNE() == 0)
+   if (element_matrices || domain_integs.Size() == 0 || fes->GetNE() == 0)
   {
      return;
   }
@@ -886,11 +891,11 @@ void BilinearForm::ComputeElementMatrices()
 #endif
      fes->GetElementTransformation(i, &eltrans);

-      dbfi[0]->AssembleElementMatrix(fe, eltrans, elmat);
-      for (int k = 1; k < dbfi.Size(); k++)
+      domain_integs[0]->AssembleElementMatrix(fe, eltrans, elmat);
+      for (int k = 1; k < domain_integs.Size(); k++)
      {
         // note: some integrators may not be thread-safe
-         dbfi[k]->AssembleElementMatrix(fe, eltrans, tmp);
+         domain_integs[k]->AssembleElementMatrix(fe, eltrans, tmp);
         elmat += tmp;
      }
      elmat.ClearExternalData();
@@ -1105,10 +1110,12 @@ BilinearForm::~BilinearForm()
   if (!extern_bfs)
   {
      int k;
-      for (k=0; k < dbfi.Size(); k++) { delete dbfi[k]; }
-      for (k=0; k < bbfi.Size(); k++) { delete bbfi[k]; }
-      for (k=0; k < fbfi.Size(); k++) { delete fbfi[k]; }
-      for (k=0; k < bfbfi.Size(); k++) { delete bfbfi[k]; }
+      for (k=0; k < domain_integs.Size(); k++) { delete domain_integs[k]; }
+      for (k=0; k < boundary_integs.Size(); k++) { delete boundary_integs[k]; }
+      for (k=0; k < interior_face_integs.Size(); k++)
+      { delete interior_face_integs[k]; }
+      for (k=0; k < boundary_face_integs.Size(); k++)
+      { delete boundary_face_integs[k]; }
   }

   delete ext;
@@ -1141,13 +1148,13 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
   ext = NULL;

   // Copy the pointers to the integrators
-   dbfi = mbf->dbfi;
-   bbfi = mbf->bbfi;
-   tfbfi = mbf->tfbfi;
-   btfbfi = mbf->btfbfi;
+   domain_integs = mbf->domain_integs;
+   boundary_integs = mbf->boundary_integs;
+   trace_face_integs = mbf->trace_face_integs;
+   boundary_trace_face_integs = mbf->boundary_trace_face_integs;

-   bbfi_marker = mbf->bbfi_marker;
-   btfbfi_marker = mbf->btfbfi_marker;
+   boundary_integs_marker = mbf->boundary_integs_marker;
+   boundary_trace_face_integs_marker = mbf->boundary_trace_face_integs_marker;

   assembly = AssemblyLevel::LEGACY;
   ext = NULL;
@@ -1236,7 +1243,8 @@ MatrixInverse * MixedBilinearForm::Inverse() const
 {
   if (assembly != AssemblyLevel::LEGACY)
   {
-      MFEM_WARNING("MixedBilinearForm::Inverse not possible with this assembly level!");
+      MFEM_WARNING("MixedBilinearForm::Inverse not possible with this "
+                   "assembly level!");
      return NULL;
   }
   else
@@ -1267,38 +1275,39 @@ void MixedBilinearForm::GetBlocks(Array2D<SparseMatrix *> &blocks) const

 void MixedBilinearForm::AddDomainIntegrator (BilinearFormIntegrator * bfi)
 {
-   dbfi.Append (bfi);
+   domain_integs.Append (bfi);
 }

 void MixedBilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi)
 {
-   bbfi.Append (bfi);
-   bbfi_marker.Append(NULL); // NULL marker means apply everywhere
+   boundary_integs.Append (bfi);
+   boundary_integs_marker.Append(NULL); // NULL marker means apply everywhere
 }

 void MixedBilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi,
                                               Array<int> &bdr_marker)
 {
-   bbfi.Append (bfi);
-   bbfi_marker.Append(&bdr_marker);
+   boundary_integs.Append (bfi);
+   boundary_integs_marker.Append(&bdr_marker);
 }

 void MixedBilinearForm::AddTraceFaceIntegrator (BilinearFormIntegrator * bfi)
 {
-   tfbfi.Append (bfi);
+   trace_face_integs.Append (bfi);
 }

 void MixedBilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator *bfi)
 {
-   btfbfi.Append(bfi);
-   btfbfi_marker.Append(NULL); // NULL marker means apply everywhere
+   boundary_trace_face_integs.Append(bfi);
+   // NULL marker means apply everywhere
+   boundary_trace_face_integs_marker.Append(NULL);
 }

 void MixedBilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator *bfi,
                                                  Array<int> &bdr_marker)
 {
-   btfbfi.Append(bfi);
-   btfbfi_marker.Append(&bdr_marker);
+   boundary_trace_face_integs.Append(bfi);
+   boundary_trace_face_integs_marker.Append(&bdr_marker);
 }

 void MixedBilinearForm::Assemble (int skip_zeros)
@@ -1320,37 +1329,37 @@ void MixedBilinearForm::Assemble (int skip_zeros)
      mat = new SparseMatrix(height, width);
   }

-   if (dbfi.Size())
+   if (domain_integs.Size())
   {
      for (int i = 0; i < test_fes -> GetNE(); i++)
      {
         trial_fes -> GetElementVDofs (i, tr_vdofs);
         test_fes  -> GetElementVDofs (i, te_vdofs);
         eltrans = test_fes -> GetElementTransformation (i);
-         for (int k = 0; k < dbfi.Size(); k++)
+         for (int k = 0; k < domain_integs.Size(); k++)
         {
-            dbfi[k] -> AssembleElementMatrix2 (*trial_fes -> GetFE(i),
-                                               *test_fes  -> GetFE(i),
-                                               *eltrans, elemmat);
+            domain_integs[k] -> AssembleElementMatrix2 (*trial_fes -> GetFE(i),
+                                                        *test_fes  -> GetFE(i),
+                                                        *eltrans, elemmat);
            mat -> AddSubMatrix (te_vdofs, tr_vdofs, elemmat, skip_zeros);
         }
      }
   }

-   if (bbfi.Size())
+   if (boundary_integs.Size())
   {
      // Which boundary attributes need to be processed?
      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
                                 mesh->bdr_attributes.Max() : 0);
      bdr_attr_marker = 0;
-      for (int k = 0; k < bbfi.Size(); k++)
+      for (int k = 0; k < boundary_integs.Size(); k++)
      {
-         if (bbfi_marker[k] == NULL)
+         if (boundary_integs_marker[k] == NULL)
         {
            bdr_attr_marker = 1;
            break;
         }
-         Array<int> &bdr_marker = *bbfi_marker[k];
+         Array<int> &bdr_marker = *boundary_integs_marker[k];
         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
                     "invalid boundary marker for boundary integrator #"
                     << k << ", counting from zero");
@@ -1368,20 +1377,20 @@ void MixedBilinearForm::Assemble (int skip_zeros)
         trial_fes -> GetBdrElementVDofs (i, tr_vdofs);
         test_fes  -> GetBdrElementVDofs (i, te_vdofs);
         eltrans = test_fes -> GetBdrElementTransformation (i);
-         for (int k = 0; k < bbfi.Size(); k++)
+         for (int k = 0; k < boundary_integs.Size(); k++)
         {
-            if (bbfi_marker[k] &&
-                (*bbfi_marker[k])[bdr_attr-1] == 0) { continue; }
+            if (boundary_integs_marker[k] &&
+                (*boundary_integs_marker[k])[bdr_attr-1] == 0) { continue; }

-            bbfi[k] -> AssembleElementMatrix2 (*trial_fes -> GetBE(i),
-                                               *test_fes  -> GetBE(i),
-                                               *eltrans, elemmat);
+            boundary_integs[k]->AssembleElementMatrix2 (*trial_fes -> GetBE(i),
+                                                        *test_fes  -> GetBE(i),
+                                                        *eltrans, elemmat);
            mat -> AddSubMatrix (te_vdofs, tr_vdofs, elemmat, skip_zeros);
         }
      }
   }

-   if (tfbfi.Size())
+   if (trace_face_integs.Size())
   {
      FaceElementTransformations *ftr;
      Array<int> te_vdofs2;
@@ -1408,16 +1417,16 @@ void MixedBilinearForm::Assemble (int skip_zeros)
            // want to actually make a fake element.
            test_fe2 = test_fe1;
         }
-         for (int k = 0; k < tfbfi.Size(); k++)
+         for (int k = 0; k < trace_face_integs.Size(); k++)
         {
-            tfbfi[k]->AssembleFaceMatrix(*trial_face_fe, *test_fe1, *test_fe2,
-                                         *ftr, elemmat);
+            trace_face_integs[k]->AssembleFaceMatrix(*trial_face_fe, *test_fe1,
+                                                     *test_fe2, *ftr, elemmat);
            mat->AddSubMatrix(te_vdofs, tr_vdofs, elemmat, skip_zeros);
         }
      }
   }

-   if (btfbfi.Size())
+   if (boundary_trace_face_integs.Size())
   {
      FaceElementTransformations *ftr;
      Array<int> te_vdofs2;
@@ -1427,17 +1436,17 @@ void MixedBilinearForm::Assemble (int skip_zeros)
      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
                                 mesh->bdr_attributes.Max() : 0);
      bdr_attr_marker = 0;
-      for (int k = 0; k < btfbfi.Size(); k++)
+      for (int k = 0; k < boundary_trace_face_integs.Size(); k++)
      {
-         if (btfbfi_marker[k] == NULL)
+         if (boundary_trace_face_integs_marker[k] == NULL)
         {
            bdr_attr_marker = 1;
            break;
         }
-         Array<int> &bdr_marker = *btfbfi_marker[k];
+         Array<int> &bdr_marker = *boundary_trace_face_integs_marker[k];
         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
-                     "invalid boundary marker for boundary trace face integrator #"
-                     << k << ", counting from zero");
+                     "invalid boundary marker for boundary trace face"
+                     "integrator #" << k << ", counting from zero");
         for (int i = 0; i < bdr_attr_marker.Size(); i++)
         {
            bdr_attr_marker[i] |= bdr_marker[i];
@@ -1460,13 +1469,16 @@ void MixedBilinearForm::Assemble (int skip_zeros)
            // boundaries, but we can't dereference a NULL pointer, and we don't
            // want to actually make a fake element.
            test_fe2 = test_fe1;
-            for (int k = 0; k < btfbfi.Size(); k++)
+            for (int k = 0; k < boundary_trace_face_integs.Size(); k++)
            {
-               if (btfbfi_marker[k] &&
-                   (*btfbfi_marker[k])[bdr_attr-1] == 0) { continue; }
+               if (boundary_trace_face_integs_marker[k] &&
+                   (*boundary_trace_face_integs_marker[k])[bdr_attr-1] == 0)
+               { continue; }

-               btfbfi[k]->AssembleFaceMatrix(*trial_face_fe, *test_fe1, *test_fe2,
-                                             *ftr, elemmat);
+               boundary_trace_face_integs[k]->AssembleFaceMatrix(*trial_face_fe,
+                                                                 *test_fe1,
+                                                                 *test_fe2,
+                                                                 *ftr, elemmat);
               mat->AddSubMatrix(te_vdofs, tr_vdofs, elemmat, skip_zeros);
            }
         }
@@ -1557,15 +1569,17 @@ void MixedBilinearForm::ConformingAssemble()

 void MixedBilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
 {
-   if (dbfi.Size())
+   if (domain_integs.Size())
   {
      const FiniteElement &trial_fe = *trial_fes->GetFE(i);
      const FiniteElement &test_fe = *test_fes->GetFE(i);
      ElementTransformation *eltrans = test_fes->GetElementTransformation(i);
-      dbfi[0]->AssembleElementMatrix2(trial_fe, test_fe, *eltrans, elmat);
-      for (int k = 1; k < dbfi.Size(); k++)
+      domain_integs[0]->AssembleElementMatrix2(trial_fe, test_fe, *eltrans,
+                                               elmat);
+      for (int k = 1; k < domain_integs.Size(); k++)
      {
-         dbfi[k]->AssembleElementMatrix2(trial_fe, test_fe, *eltrans, elemmat);
+         domain_integs[k]->AssembleElementMatrix2(trial_fe, test_fe, *eltrans,
+                                                  elemmat);
         elmat += elemmat;
      }
   }
@@ -1580,15 +1594,17 @@ void MixedBilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)

 void MixedBilinearForm::ComputeBdrElementMatrix(int i, DenseMatrix &elmat)
 {
-   if (bbfi.Size())
+   if (boundary_integs.Size())
   {
      const FiniteElement &trial_be = *trial_fes->GetBE(i);
      const FiniteElement &test_be = *test_fes->GetBE(i);
      ElementTransformation *eltrans = test_fes->GetBdrElementTransformation(i);
-      bbfi[0]->AssembleElementMatrix2(trial_be, test_be, *eltrans, elmat);
-      for (int k = 1; k < bbfi.Size(); k++)
+      boundary_integs[0]->AssembleElementMatrix2(trial_be, test_be, *eltrans,
+                                                 elmat);
+      for (int k = 1; k < boundary_integs.Size(); k++)
      {
-         bbfi[k]->AssembleElementMatrix2(trial_be, test_be, *eltrans, elemmat);
+         boundary_integs[k]->AssembleElementMatrix2(trial_be, test_be, *eltrans,
+                                                    elemmat);
         elmat += elemmat;
      }
   }
@@ -1688,10 +1704,10 @@ void MixedBilinearForm::EliminateTestDofs (const Array<int> &bdr_attr_is_ess)
      }
 }

-void MixedBilinearForm::FormRectangularSystemMatrix(const Array<int>
-                                                    &trial_tdof_list,
-                                                    const Array<int> &test_tdof_list,
-                                                    OperatorHandle &A)
+void MixedBilinearForm::FormRectangularSystemMatrix(
+   const Array<int> &trial_tdof_list,
+   const Array<int> &test_tdof_list,
+   OperatorHandle &A)

 {
   if (ext)
@@ -1729,17 +1745,17 @@ void MixedBilinearForm::FormRectangularSystemMatrix(const Array<int>
   A.Reset(mat, false);
 }

-void MixedBilinearForm::FormRectangularLinearSystem(const Array<int>
-                                                    &trial_tdof_list,
-                                                    const Array<int> &test_tdof_list,
-                                                    Vector &x, Vector &b,
-                                                    OperatorHandle &A,
-                                                    Vector &X, Vector &B)
+void MixedBilinearForm::FormRectangularLinearSystem(
+   const Array<int> &trial_tdof_list,
+   const Array<int> &test_tdof_list,
+   Vector &x, Vector &b,
+   OperatorHandle &A,
+   Vector &X, Vector &B)
 {
   if (ext)
   {
-      ext->FormRectangularLinearSystem(trial_tdof_list, test_tdof_list, x, b, A, X,
-                                       B);
+      ext->FormRectangularLinearSystem(trial_tdof_list, test_tdof_list,
+                                       x, b, A, X, B);
      return;
   }

@@ -1777,10 +1793,13 @@ MixedBilinearForm::~MixedBilinearForm()
   if (!extern_bfs)
   {
      int i;
-      for (i = 0; i < dbfi.Size(); i++) { delete dbfi[i]; }
-      for (i = 0; i < bbfi.Size(); i++) { delete bbfi[i]; }
-      for (i = 0; i < tfbfi.Size(); i++) { delete tfbfi[i]; }
-      for (i = 0; i < btfbfi.Size(); i++) { delete btfbfi[i]; }
+      for (i = 0; i < domain_integs.Size(); i++) { delete domain_integs[i]; }
+      for (i = 0; i < boundary_integs.Size(); i++)
+      { delete boundary_integs[i]; }
+      for (i = 0; i < trace_face_integs.Size(); i++)
+      { delete trace_face_integs[i]; }
+      for (i = 0; i < boundary_trace_face_integs.Size(); i++)
+      { delete boundary_trace_face_integs[i]; }
   }
   delete ext;
 }
@@ -1830,7 +1849,7 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
      mat = new SparseMatrix(height, width);
   }

-   if (dbfi.Size() > 0)
+   if (domain_integs.Size() > 0)
   {
      for (int i = 0; i < test_fes->GetNE(); i++)
      {
@@ -1840,17 +1859,19 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
         dom_fe = trial_fes->GetFE(i);
         ran_fe = test_fes->GetFE(i);

-         dbfi[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, totelmat);
-         for (int j = 1; j < dbfi.Size(); j++)
+         domain_integs[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
+                                                  totelmat);
+         for (int j = 1; j < domain_integs.Size(); j++)
         {
-            dbfi[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, elmat);
+            domain_integs[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
+                                                     elmat);
            totelmat += elmat;
         }
         mat->SetSubMatrix(ran_vdofs, dom_vdofs, totelmat, skip_zeros);
      }
   }

-   if (tfbfi.Size())
+   if (trace_face_integs.Size())
   {
      const int nfaces = test_fes->GetMesh()->GetNumFaces();
      for (int i = 0; i < nfaces; i++)
@@ -1861,10 +1882,12 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
         dom_fe = trial_fes->GetFaceElement(i);
         ran_fe = test_fes->GetFaceElement(i);

-         tfbfi[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, totelmat);
-         for (int j = 1; j < tfbfi.Size(); j++)
+         trace_face_integs[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
+                                                      totelmat);
+         for (int j = 1; j < trace_face_integs.Size(); j++)
         {
-            tfbfi[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, elmat);
+            trace_face_integs[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
+                                                         elmat);
            totelmat += elmat;
         }
         mat->SetSubMatrix(ran_vdofs, dom_vdofs, totelmat, skip_zeros);
@@ -84,28 +84,29 @@ protected:
       the BilinearForm. */
   long sequence;

-   /** @brief Indicates the BilinearFormIntegrator%s stored in #dbfi, #bbfi,
-       #fbfi, and #bfbfi are owned by another BilinearForm. */
+   /** @brief Indicates the BilinearFormIntegrator%s stored in #domain_integs,
+       #boundary_integs, #interior_face_integs, and #boundary_face_integs are
+       owned by another BilinearForm. */
   int extern_bfs;

   /// Set of Domain Integrators to be applied.
-   Array<BilinearFormIntegrator*> dbfi;
+   Array<BilinearFormIntegrator*> domain_integs;
   /// Element attribute marker (should be of length mesh->attributes)
   /// Includes all by default.
   /// 0 - ignore attribute
   /// 1 - include attribute
-   Array<Array<int>*>             dbfi_marker;
+   Array<Array<int>*>             domain_integs_marker;

   /// Set of Boundary Integrators to be applied.
-   Array<BilinearFormIntegrator*> bbfi;
-   Array<Array<int>*>             bbfi_marker; ///< Entries are not owned.
+   Array<BilinearFormIntegrator*> boundary_integs;
+   Array<Array<int>*> boundary_integs_marker; ///< Entries are not owned.

   /// Set of interior face Integrators to be applied.
-   Array<BilinearFormIntegrator*> fbfi;
+   Array<BilinearFormIntegrator*> interior_face_integs;

   /// Set of boundary face Integrators to be applied.
-   Array<BilinearFormIntegrator*> bfbfi;
-   Array<Array<int>*>             bfbfi_marker; ///< Entries are not owned.
+   Array<BilinearFormIntegrator*> boundary_face_integs;
+   Array<Array<int>*> boundary_face_integs_marker; ///< Entries are not owned.

   DenseMatrix elemmat;
   Array<int>  vdofs;
@@ -231,24 +232,25 @@ public:
   void AllocateMatrix() { if (mat == NULL) { AllocMat(); } }

   /// Access all the integrators added with AddDomainIntegrator().
-   Array<BilinearFormIntegrator*> *GetDBFI() { return &dbfi; }
+   Array<BilinearFormIntegrator*> *GetDBFI() { return &domain_integs; }

   /// Access all the integrators added with AddBoundaryIntegrator().
-   Array<BilinearFormIntegrator*> *GetBBFI() { return &bbfi; }
+   Array<BilinearFormIntegrator*> *GetBBFI() { return &boundary_integs; }
   /** @brief Access all boundary markers added with AddBoundaryIntegrator().
       If no marker was specified when the integrator was added, the
       corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetBBFI_Marker() { return &bbfi_marker; }
+   Array<Array<int>*> *GetBBFI_Marker() { return &boundary_integs_marker; }

   /// Access all integrators added with AddInteriorFaceIntegrator().
-   Array<BilinearFormIntegrator*> *GetFBFI() { return &fbfi; }
+   Array<BilinearFormIntegrator*> *GetFBFI() { return &interior_face_integs; }

   /// Access all integrators added with AddBdrFaceIntegrator().
-   Array<BilinearFormIntegrator*> *GetBFBFI() { return &bfbfi; }
+   Array<BilinearFormIntegrator*> *GetBFBFI() { return &boundary_face_integs; }
   /** @brief Access all boundary markers added with AddBdrFaceIntegrator().
       If no marker was specified when the integrator was added, the
       corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetBFBFI_Marker() { return &bfbfi_marker; }
+   Array<Array<int>*> *GetBFBFI_Marker()
+   { return &boundary_face_integs_marker; }

   /// Returns a reference to: \f$ M_{ij} \f$
   const double &operator()(int i, int j) { return (*mat)(i,j); }
@@ -652,23 +654,25 @@ protected:
       Partial Assembly (PA), or Matrix Free assembly (MF). */
   MixedBilinearFormExtension *ext;

-   /** @brief Indicates the BilinearFormIntegrator%s stored in #dbfi, #bbfi,
-       #tfbfi and #btfbfi are owned by another MixedBilinearForm. */
+   /** @brief Indicates the BilinearFormIntegrator%s stored in #domain_integs,
+       #boundary_integs, #trace_face_integs and #boundary_trace_face_integs
+       are owned by another MixedBilinearForm. */
   int extern_bfs;

   /// Domain integrators.
-   Array<BilinearFormIntegrator*> dbfi;
+   Array<BilinearFormIntegrator*> domain_integs;

   /// Boundary integrators.
-   Array<BilinearFormIntegrator*> bbfi;
-   Array<Array<int>*>             bbfi_marker;///< Entries are not owned.
+   Array<BilinearFormIntegrator*> boundary_integs;
+   Array<Array<int>*> boundary_integs_marker; ///< Entries are not owned.

   /// Trace face (skeleton) integrators.
-   Array<BilinearFormIntegrator*> tfbfi;
+   Array<BilinearFormIntegrator*> trace_face_integs;

   /// Boundary trace face (skeleton) integrators.
-   Array<BilinearFormIntegrator*> btfbfi;
-   Array<Array<int>*>             btfbfi_marker;///< Entries are not owned.
+   Array<BilinearFormIntegrator*> boundary_trace_face_integs;
+   /// Entries are not owned.
+   Array<Array<int>*> boundary_trace_face_integs_marker;

   DenseMatrix elemmat;
   Array<int>  trial_vdofs, test_vdofs;
@@ -762,24 +766,26 @@ public:
                                   Array<int> &bdr_marker);

   /// Access all integrators added with AddDomainIntegrator().
-   Array<BilinearFormIntegrator*> *GetDBFI() { return &dbfi; }
+   Array<BilinearFormIntegrator*> *GetDBFI() { return &domain_integs; }

   /// Access all integrators added with AddBoundaryIntegrator().
-   Array<BilinearFormIntegrator*> *GetBBFI() { return &bbfi; }
+   Array<BilinearFormIntegrator*> *GetBBFI() { return &boundary_integs; }
   /** @brief Access all boundary markers added with AddBoundaryIntegrator().
       If no marker was specified when the integrator was added, the
       corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetBBFI_Marker() { return &bbfi_marker; }
+   Array<Array<int>*> *GetBBFI_Marker() { return &boundary_integs_marker; }

   /// Access all integrators added with AddTraceFaceIntegrator().
-   Array<BilinearFormIntegrator*> *GetTFBFI() { return &tfbfi; }
+   Array<BilinearFormIntegrator*> *GetTFBFI() { return &trace_face_integs; }

   /// Access all integrators added with AddBdrTraceFaceIntegrator().
-   Array<BilinearFormIntegrator*> *GetBTFBFI() { return &btfbfi; }
+   Array<BilinearFormIntegrator*> *GetBTFBFI()
+   { return &boundary_trace_face_integs; }
   /** @brief Access all boundary markers added with AddBdrTraceFaceIntegrator().
       If no marker was specified when the integrator was added, the
       corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetBTFBFI_Marker() { return &btfbfi_marker; }
+   Array<Array<int>*> *GetBTFBFI_Marker()
+   { return &boundary_trace_face_integs_marker; }

   /// Sets all sparse values of \f$ M \f$ to @a a.
   void operator=(const double a) { *mat = a; }
@@ -1004,7 +1010,7 @@ public:
   { AddTraceFaceIntegrator(di); }

   /// Access all interpolators added with AddDomainInterpolator().
-   Array<BilinearFormIntegrator*> *GetDI() { return &dbfi; }
+   Array<BilinearFormIntegrator*> *GetDI() { return &domain_integs; }

   /// Set the desired assembly level. The default is AssemblyLevel::FULL.
   /** This method must be called before assembly. */
@@ -160,7 +160,7 @@ void MFBilinearFormExtension::Mult(const Vector &x, Vector &y) const
         {
            intFaceIntegrators[i]->AddMultMF(faceIntX, faceIntY);
         }
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
+         int_face_restrict_lex->AddMultTranspose(faceIntY, y);
      }
   }

@@ -176,7 +176,7 @@ void MFBilinearFormExtension::Mult(const Vector &x, Vector &y) const
         {
            bdrFaceIntegrators[i]->AddMultMF(faceBdrX, faceBdrY);
         }
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
+         bdr_face_restrict_lex->AddMultTranspose(faceBdrY, y);
      }
   }
 }
@@ -217,7 +217,7 @@ void MFBilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         {
            intFaceIntegrators[i]->AddMultTransposeMF(faceIntX, faceIntY);
         }
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
+         int_face_restrict_lex->AddMultTranspose(faceIntY, y);
      }
   }

@@ -233,7 +233,7 @@ void MFBilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         {
            bdrFaceIntegrators[i]->AddMultTransposeMF(faceBdrX, faceBdrY);
         }
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
+         bdr_face_restrict_lex->AddMultTranspose(faceBdrY, y);
      }
   }
 }
@@ -417,7 +417,7 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
         {
            intFaceIntegrators[i]->AddMultPA(faceIntX, faceIntY);
         }
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
+         int_face_restrict_lex->AddMultTranspose(faceIntY, y);
      }
   }

@@ -433,7 +433,7 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
         {
            bdrFaceIntegrators[i]->AddMultPA(faceBdrX, faceBdrY);
         }
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
+         bdr_face_restrict_lex->AddMultTranspose(faceBdrY, y);
      }
   }
 }
@@ -474,7 +474,7 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         {
            intFaceIntegrators[i]->AddMultTransposePA(faceIntX, faceIntY);
         }
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
+         int_face_restrict_lex->AddMultTranspose(faceIntY, y);
      }
   }

@@ -490,7 +490,7 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         {
            bdrFaceIntegrators[i]->AddMultTransposePA(faceBdrX, faceBdrY);
         }
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
+         bdr_face_restrict_lex->AddMultTranspose(faceBdrY, y);
      }
   }
 }
@@ -657,7 +657,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
            Y(j, 0, f) += res;
         });
         // Apply the Interior Face Restriction transposed
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
+         int_face_restrict_lex->AddMultTranspose(faceIntY, y);
      }
   }

@@ -688,7 +688,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
            Y(j, f) += res;
         });
         // Apply the Boundary Face Restriction transposed
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
+         bdr_face_restrict_lex->AddMultTranspose(faceBdrY, y);
      }
   }
 }
@@ -783,7 +783,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
            Y(j, 0, f) += res;
         });
         // Apply the Interior Face Restriction transposed
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
+         int_face_restrict_lex->AddMultTranspose(faceIntY, y);
      }
   }

@@ -814,7 +814,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
            Y(j, f) += res;
         });
         // Apply the Boundary Face Restriction transposed
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
+         bdr_face_restrict_lex->AddMultTranspose(faceBdrY, y);
      }
   }
 }
@@ -72,8 +72,8 @@ protected:
   mutable Vector faceIntX, faceIntY;
   mutable Vector faceBdrX, faceBdrY;
   const Operator *elem_restrict; // Not owned
-   const Operator *int_face_restrict_lex; // Not owned
-   const Operator *bdr_face_restrict_lex; // Not owned
+   const FaceRestriction *int_face_restrict_lex; // Not owned
+   const FaceRestriction *bdr_face_restrict_lex; // Not owned

 public:
   PABilinearFormExtension(BilinearForm*);
@@ -143,8 +143,8 @@ protected:
   mutable Vector faceIntX, faceIntY;
   mutable Vector faceBdrX, faceBdrY;
   const Operator *elem_restrict; // Not owned
-   const Operator *int_face_restrict_lex; // Not owned
-   const Operator *bdr_face_restrict_lex; // Not owned
+   const FaceRestriction *int_face_restrict_lex; // Not owned
+   const FaceRestriction *bdr_face_restrict_lex; // Not owned

 public:
   MFBilinearFormExtension(BilinearForm *form);
@@ -175,6 +175,11 @@ void BilinearFormIntegrator::AssembleFaceVector(
   elmat.Mult(elfun, elvect);
 }

+void TransposeIntegrator::SetIntRule(const IntegrationRule *ir)
+{
+   IntRule = ir;
+   bfi->SetIntRule(ir);
+}

 void TransposeIntegrator::AssembleElementMatrix (
   const FiniteElement &el, ElementTransformation &Trans, DenseMatrix &elmat)
@@ -202,6 +207,12 @@ void TransposeIntegrator::AssembleFaceMatrix (
   elmat.Transpose (bfi_elmat);
 }

+void LumpedIntegrator::SetIntRule(const IntegrationRule *ir)
+{
+   IntRule = ir;
+   bfi->SetIntRule(ir);
+}
+
 void LumpedIntegrator::AssembleElementMatrix (
   const FiniteElement &el, ElementTransformation &Trans, DenseMatrix &elmat)
 {
@@ -209,6 +220,12 @@ void LumpedIntegrator::AssembleElementMatrix (
   elmat.Lump();
 }

+void InverseIntegrator::SetIntRule(const IntegrationRule *ir)
+{
+   IntRule = ir;
+   integrator->SetIntRule(ir);
+}
+
 void InverseIntegrator::AssembleElementMatrix(
   const FiniteElement &el, ElementTransformation &Trans, DenseMatrix &elmat)
 {
@@ -216,6 +233,15 @@ void InverseIntegrator::AssembleElementMatrix(
   elmat.Invert();
 }

+void SumIntegrator::SetIntRule(const IntegrationRule *ir)
+{
+   IntRule = ir;
+   for (int i = 0; i < integrators.Size(); i++)
+   {
+      integrators[i]->SetIntRule(ir);
+   }
+}
+
 void SumIntegrator::AssembleElementMatrix(
   const FiniteElement &el, ElementTransformation &Trans, DenseMatrix &elmat)
 {
@@ -1751,15 +1777,16 @@ void DerivativeIntegrator::AssembleElementMatrix2 (
   int dim = trial_fe.GetDim();
   int trial_nd = trial_fe.GetDof();
   int test_nd = test_fe.GetDof();
+   int spaceDim = Trans.GetSpaceDim();

   int i, l;
   double det;

   elmat.SetSize (test_nd,trial_nd);
   dshape.SetSize (trial_nd,dim);
-   dshapedxt.SetSize(trial_nd,dim);
+   dshapedxt.SetSize(trial_nd, spaceDim);
   dshapedxi.SetSize(trial_nd);
-   invdfdx.SetSize(dim);
+   invdfdx.SetSize(dim, spaceDim);
   shape.SetSize (test_nd);

   const IntegrationRule *ir = IntRule;
@@ -261,6 +261,8 @@ public:
   TransposeIntegrator (BilinearFormIntegrator *bfi_, int own_bfi_ = 1)
   { bfi = bfi_; own_bfi = own_bfi_; }

+   virtual void SetIntRule(const IntegrationRule *ir);
+
   virtual void AssembleElementMatrix(const FiniteElement &el,
                                      ElementTransformation &Trans,
                                      DenseMatrix &elmat);
@@ -328,6 +330,8 @@ public:
   LumpedIntegrator (BilinearFormIntegrator *bfi_, int own_bfi_ = 1)
   { bfi = bfi_; own_bfi = own_bfi_; }

+   virtual void SetIntRule(const IntegrationRule *ir);
+
   virtual void AssembleElementMatrix(const FiniteElement &el,
                                      ElementTransformation &Trans,
                                      DenseMatrix &elmat);
@@ -346,6 +350,8 @@ public:
   InverseIntegrator(BilinearFormIntegrator *integ, int own_integ = 1)
   { integrator = integ; own_integrator = own_integ; }

+   virtual void SetIntRule(const IntegrationRule *ir);
+
   virtual void AssembleElementMatrix(const FiniteElement &el,
                                      ElementTransformation &Trans,
                                      DenseMatrix &elmat);
@@ -364,6 +370,8 @@ private:
 public:
   SumIntegrator(int own_integs = 1) { own_integrators = own_integs; }

+   virtual void SetIntRule(const IntegrationRule *ir);
+
   void AddIntegrator(BilinearFormIntegrator *integ)
   { integrators.Append(integ); }

@@ -703,7 +711,7 @@ protected:
   {
      return "MixedScalarDerivativeIntegrator:  "
             "Trial and test spaces must both be scalar fields in 1D "
-             "and the trial space must implement CaldDShape.";
+             "and the trial space must implement CalcDShape.";
   }

   inline virtual void CalcTrialShape(const FiniteElement & trial_fe,
@@ -2928,11 +2936,11 @@ public:

    - F. Bassi and S. Rebay. A high order discontinuous Galerkin method for
      compressible turbulent flows. In B. Cockburn, G. E. Karniadakis, and
-      C.-W. Shu, editors, Discontinuous Galerkin Methods, pages 77–88. Springer
+      C.-W. Shu, editors, Discontinuous Galerkin Methods, pages 77-88. Springer
      Berlin Heidelberg, 2000.
    - D. N. Arnold, F. Brezzi, B. Cockburn, and L. D. Marini. Unified analysis
      of discontinuous Galerkin methods for elliptic problems. SIAM Journal on
-      Numerical Analysis, 39(5):1749–1779, 2002.
+      Numerical Analysis, 39(5):1749-1779, 2002.
 */
 class DGDiffusionBR2Integrator : public BilinearFormIntegrator
 {
@@ -143,7 +143,7 @@ Solver *BuildSmootherFromCeed(ConstrainedOperator &op, bool chebyshev)
   if (chebyshev)
   {
      const int cheb_order = 3;
-      out = new OperatorChebyshevSmoother(&op, t_diag, ess_tdofs, cheb_order);
+      out = new OperatorChebyshevSmoother(op, t_diag, ess_tdofs, cheb_order);
   }
   else
   {
@@ -1204,17 +1204,30 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
      });
      // Modify offdiagonal blocks (imaginary parts of the matrix) to conform
      // with standard essential BC treatment
-      ess_tdof_list.HostRead();
      if (A_i.Type() == Operator::Hypre_ParCSR)
      {
         HypreParMatrix * Ah;
         A_i.Get(Ah);
         hypre_ParCSRMatrix *Aih = *Ah;
+#ifndef HYPRE_USING_CUDA
+         ess_tdof_list.HostRead();
         for (int k = 0; k < n; k++)
         {
            const int j = ess_tdof_list[k];
            Aih->diag->data[Aih->diag->i[j]] = 0.0;
         }
+#else
+         Ah->HypreReadWrite();
+         const int *d_ess_tdof_list =
+            ess_tdof_list.GetMemory().Read(MemoryClass::DEVICE, n);
+         const int *d_diag_i = Aih->diag->i;
+         double *d_diag_data = Aih->diag->data;
+         CuWrap1D(n, [=] MFEM_DEVICE (int k)
+         {
+            const int j = d_ess_tdof_list[k];
+            d_diag_data[d_diag_i[j]] = 0.0;
+         });
+#endif
      }
      else
      {
@@ -17,6 +17,7 @@

 #include <cerrno>      // errno
 #include <sstream>
+#include <regex>

 #ifndef _WIN32
 #include <sys/stat.h>  // mkdir
@@ -764,7 +765,8 @@ ParaViewDataCollection::ParaViewDataCollection(const std::string&
   : DataCollection(collection_name, mesh_),
     levels_of_detail(1),
     pv_data_format(VTKFormat::BINARY),
-     high_order_output(false)
+     high_order_output(false),
+     restart_mode(false)
 {
 #ifdef MFEM_USE_ZLIB
   compression = -1; // default zlib compression level, equivalent to 6
@@ -842,17 +844,60 @@ void ParaViewDataCollection::Save()
   }
   // the directory is created

-   // create pvd file if needed
+   // create pvd file if needed. If we are not in restart mode, a new pvd file
+   // is always created. In restart mode, we keep any previously defined
+   // timestep values as long as they are less than the currently defined time.
+
   if (myid == 0 && !pvd_stream.is_open())
   {
      std::string dpath=GenerateCollectionPath();
      std::string pvdname=dpath+"/"+GeneratePVDFileName();
-      pvd_stream.open(pvdname.c_str(),std::ios::out);
-      // initialize the file
-      pvd_stream << "<?xml version=\"1.0\"?>\n";
-      pvd_stream << "<VTKFile type=\"Collection\" version=\"0.1\"";
-      pvd_stream << " byte_order=\"" << VTKByteOrder() << "\">\n";
-      pvd_stream << "<Collection>" << std::endl;
+
+      std::ifstream pvd_in;
+      if (restart_mode && (pvd_in.open(pvdname,std::ios::binary),pvd_in.good()))
+      {
+         // PVD file exists and restart mode enabled: preserve existing time
+         // steps less than the current time.
+         std::fstream::pos_type pos_begin = pvd_in.tellg();
+         std::fstream::pos_type pos_end = pos_begin;
+
+         std::regex regexp("timestep=\"([^[:space:]]+)\".*file=\"Cycle(\\d+)");
+         std::smatch match;
+
+         std::string line;
+         while (getline(pvd_in,line))
+         {
+            if (regex_search(line,match,regexp))
+            {
+               MFEM_ASSERT(match.size() == 3, "Unable to parse DataSet");
+               double tvalue = std::stod(match[1]);
+               if (tvalue >= GetTime()) { break; }
+               int cvalue = std::stoi(match[2]);
+               MFEM_VERIFY(cvalue < GetCycle(), "Cycle " << GetCycle() <<
+                           " is too small for restart mode: trying to overwrite"
+                           " existing data.");
+               pos_end = pvd_in.tellg();
+            }
+         }
+         size_t count = pos_end - pos_begin;
+         std::vector<char> buf(count);
+         pvd_in.clear();
+         pvd_in.seekg(pos_begin);
+         pvd_in.read(buf.data(), count);
+         pvd_in.close();
+         pvd_stream.open(pvdname.c_str(),std::ios::out);
+         pvd_stream.write(buf.data(), count);
+      }
+      else
+      {
+         // initialize new pvd file
+         pvd_stream.open(pvdname.c_str(),std::ios::out);
+         // initialize the file
+         pvd_stream << "<?xml version=\"1.0\"?>\n";
+         pvd_stream << "<VTKFile type=\"Collection\" version=\"0.1\"";
+         pvd_stream << " byte_order=\"" << VTKByteOrder() << "\">\n";
+         pvd_stream << "<Collection>" << std::endl;
+      }
   }

   // define the vtu file
@@ -1091,6 +1136,11 @@ void ParaViewDataCollection::SetCompression(bool compression_)
   }
 }

+void ParaViewDataCollection::UseRestartMode(bool restart_mode_)
+{
+   restart_mode = restart_mode_;
+}
+
 const char *ParaViewDataCollection::GetDataFormatString() const
 {
   if (pv_data_format == VTKFormat::ASCII)
@@ -488,6 +488,7 @@ private:
   std::fstream pvd_stream;
   VTKFormat pv_data_format;
   bool high_order_output;
+   bool restart_mode;

 protected:
   void SaveDataVTU(std::ostream &out, int ref);
@@ -545,6 +546,11 @@ public:
   /// by default). Reading high-order data requires ParaView 5.5 or later.
   void SetHighOrderOutput(bool high_order_output_);

+   /// Enable or disable restart mode. If restart is enabled, new writes will
+   /// preserve timestep metadata for any solutions prior to the currently
+   /// defined time.
+   void UseRestartMode(bool restart_mode_);
+
   /// Load the collection - not implemented in the ParaView writer
   virtual void Load(int cycle_ = 0) override;
 };
@@ -316,7 +316,7 @@ public:

   /// Set the desired print level, useful for debugging.
   /** The valid options are: -1 - never print (default); 0 - print only errors;
-       1 - print the first and last last iterations; 2 - print every iteration;
+       1 - print the first and last iterations; 2 - print every iteration;
       and 3 - print every iteration including point coordinates. */
   void SetPrintLevel(int pr_level) { print_level = pr_level; }

@@ -495,6 +495,7 @@ void ScalarFiniteElement::ScalarLocalRestriction(
      R *= 1.0 / Trans.Weight();
   }
 }
+
 const DofToQuad &ScalarFiniteElement::GetDofToQuad(const IntegrationRule &ir,
                                                   DofToQuad::Mode mode) const
 {
@@ -7948,7 +7949,27 @@ VectorTensorFiniteElement::VectorTensorFiniteElement(const int dims,
                         p, M, FunctionSpace::Qk),
     TensorBasisElement(dims, p, VerifyNodal(cbtype), dmtype),
     cbasis1d(poly1d.GetBasis(p, VerifyClosed(cbtype))),
-     obasis1d(poly1d.GetBasis(p - 1, VerifyOpen(obtype))) { }
+     obasis1d(poly1d.GetBasis(p - 1, VerifyOpen(obtype)))
+{
+   MFEM_VERIFY(dims > 1, "Constructor for VectorTensorFiniteElement with both "
+               "open and closed bases is not valid for 1D elements.");
+}
+
+VectorTensorFiniteElement::VectorTensorFiniteElement(const int dims,
+                                                     const int d,
+                                                     const int p,
+                                                     const int obtype,
+                                                     const int M,
+                                                     const DofMapType dmtype)
+   : VectorFiniteElement(dims, GetTensorProductGeometry(dims), d,
+                         p, M, FunctionSpace::Pk),
+     TensorBasisElement(dims, p, obtype, dmtype),
+     cbasis1d(poly1d.GetBasis(p, VerifyOpen(obtype))),
+     obasis1d(poly1d.GetBasis(p, VerifyOpen(obtype)))
+{
+   MFEM_VERIFY(dims == 1, "Constructor for VectorTensorFiniteElement without "
+               "closed basis is only valid for 1D elements.");
+}

 H1_SegmentElement::H1_SegmentElement(const int p, const int btype)
   : NodalTensorFiniteElement(1, p, VerifyClosed(btype), H1_DOF_MAP)
@@ -13055,9 +13076,8 @@ void ND_TriangleElement::CalcCurlShape(const IntegrationPoint &ip,
 const double ND_SegmentElement::tk[1] = { 1. };

 ND_SegmentElement::ND_SegmentElement(const int p, const int ob_type)
-   : VectorFiniteElement(1, Geometry::SEGMENT, p, p - 1,
-                         H_CURL, FunctionSpace::Pk),
-     obasis1d(poly1d.GetBasis(p - 1, VerifyOpen(ob_type))),
+   : VectorTensorFiniteElement(1, p, p - 1, ob_type, H_CURL,
+                               DofMapType::L2_DOF_MAP),
     dof2tk(dof)
 {
   if (obasis1d.IsIntegratedType()) { is_nodal = false; }
@@ -97,7 +97,7 @@ public:
      {
         "Gauss-Legendre", "Gauss-Lobatto", "Positive (Bernstein)",
         "Open uniform", "Closed uniform", "Open half uniform",
-         "Seredipity", "Closed Gauss-Legendre",
+         "Serendipity", "Closed Gauss-Legendre",
         "Integrated Gauss-Lobatto indicator"
      };
      return name[Check(b_type)];
@@ -1126,7 +1126,7 @@ public:
   { dofs = 1.0; }
 };

-/// A 1D quadractic finite element with uniformly spaced nodes
+/// A 1D quadratic finite element with uniformly spaced nodes
 class Quad1DFiniteElement : public NodalFiniteElement
 {
 public:
@@ -2239,6 +2239,11 @@ public:
                             const int cbtype, const int obtype,
                             const int M, const DofMapType dmtype);

+   // For 1D elements: there is only an "open basis", no "closed basis"
+   VectorTensorFiniteElement(const int dims, const int d, const int p,
+                             const int obtype, const int M,
+                             const DofMapType dmtype);
+
   const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
                                 DofToQuad::Mode mode) const;

@@ -3311,11 +3316,9 @@ public:


 /// Arbitrary order Nedelec elements in 1D on a segment
-class ND_SegmentElement : public VectorFiniteElement
+class ND_SegmentElement : public VectorTensorFiniteElement
 {
   static const double tk[1];
-
-   Poly_1D::Basis &obasis1d;
   Array<int> dof2tk;

 public:
@@ -34,6 +34,7 @@
 #include "staticcond.hpp"
 #include "tmop.hpp"
 #include "tmop_tools.hpp"
+#include "tmop_amr.hpp"
 #include "gslib.hpp"
 #include "restriction.hpp"
 #include "quadinterpolator.hpp"
@@ -64,4 +65,9 @@
 #include "adios2datacollection.hpp"
 #endif

+#ifdef MFEM_USE_FMS
+#include "fmsconvert.hpp"
+#include "fmsdatacollection.hpp"
+#endif
+
 #endif
@@ -1225,7 +1225,7 @@ const Operator *FiniteElementSpace::GetElementRestriction(
   return L2E_nat.Ptr();
 }

-const Operator *FiniteElementSpace::GetFaceRestriction(
+const FaceRestriction *FiniteElementSpace::GetFaceRestriction(
   ElementDofOrdering e_ordering, FaceType type, L2FaceValues mul) const
 {
   const bool is_dg_space = IsDGSpace();
@@ -1239,7 +1239,7 @@ const Operator *FiniteElementSpace::GetFaceRestriction(
   }
   else
   {
-      Operator* res;
+      FaceRestriction *res;
      if (is_dg_space)
      {
         res = new L2FaceRestriction(*this, e_ordering, type, m);
@@ -164,7 +164,7 @@ protected:
                + 8 * (int)std::get<3>(k);
      }
   };
-   using map_L2F = std::unordered_map<const key_face,Operator*,key_hash>;
+   using map_L2F = std::unordered_map<const key_face,FaceRestriction*,key_hash>;
   mutable map_L2F L2F;

   mutable Array<QuadratureInterpolator*> E2Q_array;
@@ -488,7 +488,7 @@ public:
   const Operator *GetElementRestriction(ElementDofOrdering e_ordering) const;

   /// Return an Operator that converts L-vectors to E-vectors on each face.
-   virtual const Operator *GetFaceRestriction(
+   virtual const FaceRestriction *GetFaceRestriction(
      ElementDofOrdering e_ordering, FaceType,
      L2FaceValues mul = L2FaceValues::DoubleValued) const;

@@ -0,0 +1,46 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef FMS_CONVERT
+#define FMS_CONVERT
+
+#include "../config/config.hpp"
+#include "datacollection.hpp"
+
+#ifdef MFEM_USE_FMS
+#include <fms.h>
+
+namespace mfem
+{
+
+/** In-memory conversion of FMS data collection to an MFEM data collection.
+    @param dc The FMS data collection to convert.
+    @param[out] mfem_dc A pointer to a new MFEM DataCollection containing the
+    FMS data.
+    @return 0 on success; non-zero on failure.
+*/
+int FmsDataCollectionToDataCollection(FmsDataCollection dc,
+                                      DataCollection **mfem_dc);
+
+/** In-memory conversion of MFEM data collection to an FMS data collection.
+    @param mfem_dc The MFEM data collection to convert.
+    @param[out] dc A pointer to a new FmsDataCollection containing the MFEM
+    data.
+    @return 0 on success; non-zero on failure.
+*/
+int DataCollectionToFmsDataCollection(DataCollection *mfem_dc,
+                                      FmsDataCollection *dc);
+
+} // namespace mfem
+
+#endif
+
+#endif
@@ -0,0 +1,167 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../config/config.hpp"
+
+#ifdef MFEM_USE_FMS
+
+#include "fem.hpp"
+#include "../general/text.hpp"
+
+#include <fmsio.h>
+
+#include <string>
+#include <sstream>
+
+namespace mfem
+{
+
+// class FMSDataCollection implementation
+
+FMSDataCollection::FMSDataCollection(const std::string& coll_name,
+                                     Mesh *mesh)
+   : DataCollection(coll_name, mesh),
+     fms_protocol("ascii")
+{
+   appendRankToFileName = false; // always include rank in file names
+   cycle = 0;                   // always include cycle in directory names
+}
+
+#ifdef MFEM_USE_MPI
+
+FMSDataCollection::FMSDataCollection(MPI_Comm comm,
+                                     const std::string& coll_name,
+                                     Mesh *mesh)
+   : DataCollection(coll_name, mesh),
+     fms_protocol("ascii")
+{
+   m_comm = comm;
+   MPI_Comm_rank(comm, &myid);
+   MPI_Comm_size(comm, &num_procs);
+   appendRankToFileName = true; // always include rank in file names
+   cycle = 0;                   // always include cycle in directory names
+}
+
+#endif
+
+FMSDataCollection::~FMSDataCollection()
+{
+   // empty
+}
+
+void FMSDataCollection::Save()
+{
+   // Convert this to FmsDataCollection.
+
+   FmsDataCollection dc;
+   if (DataCollectionToFmsDataCollection(this, &dc) == 0)
+   {
+      std::string root(RootFileName());
+      int err = FmsIOWrite(root.c_str(), fms_protocol.c_str(), dc);
+      FmsDataCollectionDestroy(&dc);
+      if (err)
+      {
+         MFEM_ABORT("Error creating FMS file: " << root);
+      }
+   }
+   else
+   {
+      MFEM_ABORT("Error converting data collection");
+   }
+}
+
+void FMSDataCollection::Load(int cycle)
+{
+   DeleteAll();
+   this->cycle = cycle;
+
+   FmsDataCollection dc;
+   std::string root(RootFileName());
+   int err = FmsIORead(root.c_str(), fms_protocol.c_str(), &dc);
+
+   if (err == 0)
+   {
+      DataCollection *mdc = nullptr;
+      if (FmsDataCollectionToDataCollection(dc,&mdc) == 0)
+      {
+         // Tell the data collection we read that it does not own data.
+         // We will steal its data.
+         mdc->SetOwnData(false);
+
+         SetCycle(mdc->GetCycle());
+         SetTime(mdc->GetTime());
+         SetTimeStep(mdc->GetTimeStep());
+         name = mdc->GetCollectionName();
+
+         // Set mdc's mesh as our mesh.
+         SetMesh(mdc->GetMesh());
+
+         // Set mdc's fields/qfields as ours.
+         std::vector<std::string> names;
+         for (const auto &pair : mdc->GetFieldMap())
+         {
+            names.push_back(pair.first);
+            RegisterField(pair.first, pair.second);
+         }
+         for (const auto &name : names)
+         {
+            mdc->DeregisterField(name);
+         }
+
+         names.clear();
+         for (const auto &pair : mdc->GetQFieldMap())
+         {
+            names.push_back(pair.first);
+            RegisterQField(pair.first, pair.second);
+         }
+         for (const auto &name : names)
+         {
+            mdc->DeregisterField(name);
+         }
+
+         // Indicate that we own the data.
+         SetOwnData(true);
+
+         // Delete mdc. We stole its contents.
+         delete mdc;
+      }
+      FmsDataCollectionDestroy(&dc);
+   }
+   else
+   {
+      MFEM_ABORT("Error reading data collection: " << root);
+   }
+}
+
+void FMSDataCollection::SetProtocol(const std::string &protocol)
+{
+   fms_protocol = protocol;
+}
+
+std::string FMSDataCollection::RootFileName()
+{
+   std::string res;
+   if (pad_digits_cycle)
+   {
+      res = prefix_path + name + "_" +
+            to_padded_string(cycle, pad_digits_cycle) +
+            ".fms";
+   }
+   else
+   {
+      res = prefix_path + name + ".fms";
+   }
+   return res;
+}
+
+} // namespace mfem
+
+#endif
@@ -0,0 +1,74 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_FMSDATACOLLECTION
+#define MFEM_FMSDATACOLLECTION
+
+#include "../config/config.hpp"
+
+#ifdef MFEM_USE_FMS
+
+#include "datacollection.hpp"
+#include <fms.h>
+
+namespace mfem
+{
+
+/** @brief Data collection that uses FMS. */
+/** FMSDataCollection lets MFEM read/write data using FMS.
+
+    For more information, see:
+    - FMS project, https://ceed.exascaleproject.org/fms/
+*/
+
+/// Data collection with FMS I/O routines
+class FMSDataCollection : public DataCollection
+{
+protected:
+   // file name helpers
+
+   /// Returns file name for the current cycle
+   std::string RootFileName();
+
+   // holds currently active i/o protocol
+   std::string fms_protocol;
+
+public:
+   /// Constructor. The collection name is used when saving the data.
+   /** If @a mesh is NULL, then the mesh can be set later by calling either
+       SetMesh() or Load(). The latter works only in serial. */
+   FMSDataCollection(const std::string& collection_name,
+                     Mesh *mesh = NULL);
+#ifdef MFEM_USE_MPI
+   /// Construct a parallel FMSDataCollection.
+   FMSDataCollection(MPI_Comm comm, const std::string& collection_name,
+                     Mesh *mesh = NULL);
+#endif
+
+   /// We will delete the mesh and fields if we own them
+   virtual ~FMSDataCollection();
+
+   /// Set the FMS relay i/o protocol to use
+   /** Supported options: ascii (default), json, yaml, hdf5 */
+   void SetProtocol(const std::string &protocol);
+
+   /// Save the collection and a FMS blueprint root file
+   virtual void Save();
+
+   /// Load the collection based blueprint data
+   virtual void Load(int cycle = 0);
+};
+
+} // namespace mfem
+
+#endif
+
+#endif
@@ -218,7 +218,7 @@ void GridFunction::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)

 void GridFunction::MakeTRef(FiniteElementSpace *f, double *tv)
 {
-   if (!f->GetProlongationMatrix())
+   if (IsIdentityProlongation(f->GetProlongationMatrix()))
   {
      MakeRef(f, tv);
      t_vec.NewDataAndSize(tv, size);
@@ -232,7 +232,8 @@ void GridFunction::MakeTRef(FiniteElementSpace *f, double *tv)

 void GridFunction::MakeTRef(FiniteElementSpace *f, Vector &tv, int tv_offset)
 {
-   if (!f->GetProlongationMatrix())
+   tv.UseDevice(true);
+   if (IsIdentityProlongation(f->GetProlongationMatrix()))
   {
      MakeRef(f, tv, tv_offset);
      t_vec.NewMemoryAndSize(data, size, false);
@@ -241,10 +242,7 @@ void GridFunction::MakeTRef(FiniteElementSpace *f, Vector &tv, int tv_offset)
   {
      MFEM_ASSERT(tv.Size() >= tv_offset + f->GetTrueVSize(), "");
      SetSpace(f); // works in parallel
-      tv.UseDevice(true);
-      const int tv_size = f->GetTrueVSize();
-      t_vec.NewMemoryAndSize(Memory<double>(tv.GetMemory(), tv_offset, tv_size),
-                             tv_size, true);
+      t_vec.MakeRef(tv, tv_offset, f->GetTrueVSize());
   }
 }

@@ -334,10 +332,10 @@ int GridFunction::VectorDim() const
 void GridFunction::GetTrueDofs(Vector &tv) const
 {
   const SparseMatrix *R = fes->GetRestrictionMatrix();
-   if (!R)
+   if (!R || IsIdentityProlongation(fes->GetProlongationMatrix()))
   {
-      // R is identity -> make tv a reference to *this
-      tv.MakeRef(const_cast<GridFunction &>(*this), 0, size);
+      // R is identity
+      tv = *this; // no real copy if 'tv' and '*this' use the same data
   }
   else
   {
@@ -95,6 +95,12 @@ public:
      : Vector(data, f->GetVSize())
   { fes = f; fec = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }

+   /** @brief Construct a GridFunction using previously allocated Vector @a base
+       starting at the given offset, @a base_offset. */
+   GridFunction(FiniteElementSpace *f, Vector &base, int base_offset = 0)
+      : Vector(base, base_offset, f->GetVSize())
+   { fes = f; fec = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }
+
   /// Construct a GridFunction on the given Mesh, using the data from @a input.
   /** The content of @a input should be in the format created by the method
       Save(). The reconstructed FiniteElementSpace and FiniteElementCollection
@@ -130,9 +136,7 @@ public:
       or set. */
   Vector &GetTrueVector() { return t_vec; }

-   /// @brief Extract the true-dofs from the GridFunction. If all dofs are true,
-   /// then `tv` will be set to point to the data of `*this`.
-   /** @warning This method breaks const-ness when all dofs are true. */
+   /// Extract the true-dofs from the GridFunction.
   void GetTrueDofs(Vector &tv) const;

   /// Shortcut for calling GetTrueDofs() with GetTrueVector() as argument.
@@ -19,7 +19,11 @@
 #pragma GCC diagnostic ignored "-Wunused-function"
 #endif

+// External GSLIB header (the MFEM header is gslib.hpp)
+namespace gslib
+{
 #include "gslib.h"
+}

 #ifdef MFEM_HAVE_GCC_PRAGMA_DIAGNOSTIC
 #pragma GCC diagnostic pop
@@ -34,13 +38,13 @@ FindPointsGSLIB::FindPointsGSLIB()
     dim(-1), points_cnt(0), setupflag(false), default_interp_value(0),
     avgtype(AvgType::ARITHMETIC)
 {
-   gsl_comm = new comm;
-   cr       = new crystal;
+   gsl_comm = new gslib::comm;
+   cr       = new gslib::crystal;
 #ifdef MFEM_USE_MPI
   int initialized;
   MPI_Initialized(&initialized);
   if (!initialized) { MPI_Init(NULL, NULL); }
-   MPI_Comm comm = MPI_COMM_WORLD;;
+   MPI_Comm comm = MPI_COMM_WORLD;
   comm_init(gsl_comm, comm);
 #else
   comm_init(gsl_comm, 0);
@@ -62,8 +66,8 @@ FindPointsGSLIB::FindPointsGSLIB(MPI_Comm comm_)
     dim(-1), points_cnt(0), setupflag(false), default_interp_value(0),
     avgtype(AvgType::ARITHMETIC)
 {
-   gsl_comm = new comm;
-   cr      = new crystal;
+   gsl_comm = new gslib::comm;
+   cr      = new gslib::crystal;
   comm_init(gsl_comm, comm_);
 }
 #endif
@@ -606,7 +610,12 @@ void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
      {
         if (gsl_code[i] == 1) { indl2.Append(i); }
      }
-      if (indl2.Size() == 0) { return; } // no points on element borders
+      int borderPts = indl2.Size();
+#ifdef MFEM_USE_MPI
+      MPI_Allreduce(MPI_IN_PLACE, &borderPts, 1, MPI_INT, MPI_SUM, gsl_comm->c);
+#endif
+      if (borderPts == 0) { return; } // no points on element borders
+

      Vector field_out_l2(field_out.Size());
      VectorGridFunctionCoefficient field_in_dg(&field_in);
@@ -728,7 +737,7 @@ void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
      }

      // Pack data to send via crystal router
-      struct array *outpt = new array;
+      struct gslib::array *outpt = new gslib::array;
      struct out_pt { double r[3], ival; uint index, el, proc; };
      struct out_pt *pt;
      array_init(struct out_pt, outpt, nptsend);
@@ -788,7 +797,7 @@ void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
         }

         // Save index and proc data in a struct
-         struct array *savpt = new array;
+         struct gslib::array *savpt = new gslib::array;
         struct sav_pt { uint index, proc; };
         struct sav_pt *spt;
         array_init(struct sav_pt, savpt, npt);
@@ -806,7 +815,7 @@ void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
         delete outpt;

         // Copy data from save struct to send struct and send component wise
-         struct array *sendpt = new array;
+         struct gslib::array *sendpt = new gslib::array;
         struct send_pt { double ival; uint index, proc; };
         struct send_pt *sdpt;
         for (int j = 0; j < ncomp; j++)
@@ -17,11 +17,13 @@

 #ifdef MFEM_USE_GSLIB

+namespace gslib
+{
 struct comm;
 struct findpts_data_2;
 struct findpts_data_3;
-struct array;
 struct crystal;
+}

 namespace mfem
 {
@@ -50,10 +52,10 @@ public:
 protected:
   Mesh *mesh, *meshsplit;
   IntegrationRule *ir_simplex;    // IntegrationRule to split quads/hex -> simplex
-   struct findpts_data_2 *fdata2D; // gslib's internal data
-   struct findpts_data_3 *fdata3D; // gslib's internal data
-   struct crystal *cr;             // gslib's internal data
-   struct comm *gsl_comm;          // gslib's internal data
+   struct gslib::findpts_data_2 *fdata2D; // gslib's internal data
+   struct gslib::findpts_data_3 *fdata3D; // gslib's internal data
+   struct gslib::crystal *cr;             // gslib's internal data
+   struct gslib::comm *gsl_comm;          // gslib's internal data
   int dim, points_cnt;
   Array<unsigned int> gsl_code, gsl_proc, gsl_elem, gsl_mfem_elem;
   Vector gsl_mesh, gsl_ref, gsl_dist, gsl_mfem_ref;
@@ -34,6 +34,7 @@ IntegrationRule::IntegrationRule(IntegrationRule &irx, IntegrationRule &iry)
   nx = irx.GetNPoints();
   ny = iry.GetNPoints();
   SetSize(nx * ny);
+   SetPointIndices();

   for (j = 0; j < ny; j++)
   {
@@ -48,8 +49,6 @@ IntegrationRule::IntegrationRule(IntegrationRule &irx, IntegrationRule &iry)
         ip.weight = ipx.weight * ipy.weight;
      }
   }
-
-   SetPointIndices();
 }

 IntegrationRule::IntegrationRule(IntegrationRule &irx, IntegrationRule &iry,
@@ -59,6 +58,7 @@ IntegrationRule::IntegrationRule(IntegrationRule &irx, IntegrationRule &iry,
   const int ny = iry.GetNPoints();
   const int nz = irz.GetNPoints();
   SetSize(nx*ny*nz);
+   SetPointIndices();

   for (int iz = 0; iz < nz; ++iz)
   {
@@ -78,8 +78,6 @@ IntegrationRule::IntegrationRule(IntegrationRule &irx, IntegrationRule &iry,
         }
      }
   }
-
-   SetPointIndices();
 }

 const Array<double> &IntegrationRule::GetWeights() const
@@ -125,6 +123,7 @@ void IntegrationRule::GrundmannMollerSimplexRule(int s, int n)
   }
   np /= f;
   SetSize(np);
+   SetPointIndices();

   int pt = 0;
   for (int i = 0; i <= s; i++)
@@ -375,6 +374,7 @@ public:
 void QuadratureFunctions1D::GaussLegendre(const int np, IntegrationRule* ir)
 {
   ir->SetSize(np);
+   ir->SetPointIndices();

   switch (np)
   {
@@ -477,6 +477,7 @@ void QuadratureFunctions1D::GaussLobatto(const int np, IntegrationRule* ir)
   */

   ir->SetSize(np);
+   ir->SetPointIndices();
   if ( np == 1 )
   {
      ir->IntPoint(0).Set1w(0.5, 1.0);
@@ -576,6 +577,7 @@ void QuadratureFunctions1D::GaussLobatto(const int np, IntegrationRule* ir)
 void QuadratureFunctions1D::OpenUniform(const int np, IntegrationRule* ir)
 {
   ir->SetSize(np);
+   ir->SetPointIndices();

   // The Newton-Cotes quadrature is based on weights that integrate exactly the
   // interpolatory polynomial through the equally spaced quadrature points.
@@ -591,6 +593,7 @@ void QuadratureFunctions1D::ClosedUniform(const int np,
                                          IntegrationRule* ir)
 {
   ir->SetSize(np);
+   ir->SetPointIndices();
   if ( np == 1 ) // allow this case as "closed"
   {
      ir->IntPoint(0).Set1w(0.5, 1.0);
@@ -608,6 +611,7 @@ void QuadratureFunctions1D::ClosedUniform(const int np,
 void QuadratureFunctions1D::OpenHalfUniform(const int np, IntegrationRule* ir)
 {
   ir->SetSize(np);
+   ir->SetPointIndices();

   // Open half points: the centers of np uniform intervals
   for (int i = 0; i < np ; ++i)
@@ -621,6 +625,7 @@ void QuadratureFunctions1D::OpenHalfUniform(const int np, IntegrationRule* ir)
 void QuadratureFunctions1D::ClosedGL(const int np, IntegrationRule* ir)
 {
   ir->SetSize(np);
+   ir->SetPointIndices();
   ir->IntPoint(0).x = 0.0;
   ir->IntPoint(np-1).x = 1.0;

@@ -96,9 +96,6 @@ private:
       by request with the method GetWeights(). */
   mutable Array<double> weights;

-   /// Sets the indices of each quadrature point on initialization.
-   void SetPointIndices();
-
   /// Define n-simplex rule (triangle/tetrahedron for n=2/3) of order (2s+1)
   void GrundmannMollerSimplexRule(int s, int n = 3);

@@ -227,6 +224,11 @@ public:
      }
   }

+   /// Sets the indices of each quadrature point on initialization.
+   /** Note that most calls to IntegrationRule::SetSize should be paired with a
+       call to SetPointIndices in order for the indices to be set correctly. */
+   void SetPointIndices();
+
   /// Tensor product of two 1D integration rules
   IntegrationRule(IntegrationRule &irx, IntegrationRule &iry);

@@ -26,14 +26,14 @@ LinearForm::LinearForm(FiniteElementSpace *f, LinearForm *lf)
   extern_lfs = 1;

   // Copy the pointers to the integrators
-   dlfi = lf->dlfi;
+   domain_integs = lf->domain_integs;

-   dlfi_delta = lf->dlfi_delta;
+   domain_delta_integs = lf->domain_delta_integs;

-   blfi = lf->blfi;
+   boundary_integs = lf->boundary_integs;

-   flfi = lf->flfi;
-   flfi_marker = lf->flfi_marker;
+   boundary_face_integs = lf->boundary_face_integs;
+   boundary_face_integs_marker = lf->boundary_face_integs_marker;
 }

 void LinearForm::AddDomainIntegrator(LinearFormIntegrator *lfi)
@@ -42,13 +42,13 @@ void LinearForm::AddDomainIntegrator(LinearFormIntegrator *lfi)
      dynamic_cast<DeltaLFIntegrator *>(lfi);
   if (!maybe_delta || !maybe_delta->IsDelta())
   {
-      dlfi.Append(lfi);
+      domain_integs.Append(lfi);
   }
   else
   {
-      dlfi_delta.Append(maybe_delta);
+      domain_delta_integs.Append(maybe_delta);
   }
-   dlfi_marker.Append(NULL);
+   domain_integs_marker.Append(NULL);
 }

 void LinearForm::AddDomainIntegrator(LinearFormIntegrator *lfi,
@@ -58,44 +58,45 @@ void LinearForm::AddDomainIntegrator(LinearFormIntegrator *lfi,
      dynamic_cast<DeltaLFIntegrator *>(lfi);
   if (!maybe_delta || !maybe_delta->IsDelta())
   {
-      dlfi.Append(lfi);
+      domain_integs.Append(lfi);
   }
   else
   {
-      dlfi_delta.Append(maybe_delta);
+      domain_delta_integs.Append(maybe_delta);
   }
-   dlfi_marker.Append(&elem_marker);
+   domain_integs_marker.Append(&elem_marker);
 }

 void LinearForm::AddBoundaryIntegrator (LinearFormIntegrator * lfi)
 {
-   blfi.Append (lfi);
-   blfi_marker.Append(NULL); // NULL -> all attributes are active
+   boundary_integs.Append (lfi);
+   boundary_integs_marker.Append(NULL); // NULL -> all attributes are active
 }

 void LinearForm::AddBoundaryIntegrator (LinearFormIntegrator * lfi,
                                        Array<int> &bdr_attr_marker)
 {
-   blfi.Append (lfi);
-   blfi_marker.Append(&bdr_attr_marker);
+   boundary_integs.Append (lfi);
+   boundary_integs_marker.Append(&bdr_attr_marker);
 }

 void LinearForm::AddBdrFaceIntegrator (LinearFormIntegrator * lfi)
 {
-   flfi.Append(lfi);
-   flfi_marker.Append(NULL); // NULL -> all attributes are active
+   boundary_face_integs.Append(lfi);
+   // NULL -> all attributes are active
+   boundary_face_integs_marker.Append(NULL);
 }

 void LinearForm::AddBdrFaceIntegrator(LinearFormIntegrator *lfi,
                                      Array<int> &bdr_attr_marker)
 {
-   flfi.Append(lfi);
-   flfi_marker.Append(&bdr_attr_marker);
+   boundary_face_integs.Append(lfi);
+   boundary_face_integs_marker.Append(&bdr_attr_marker);
 }

 void LinearForm::AddInteriorFaceIntegrator(LinearFormIntegrator *lfi)
 {
-   iflfi.Append(lfi);
+   interior_face_integs.Append(lfi);
 }

 void LinearForm::Assemble()
@@ -112,14 +113,14 @@ void LinearForm::Assemble()
   // The first use of AddElementVector() below will move it back to host
   // because both 'vdofs' and 'elemvect' are on host.

-   if (dlfi.Size())
+   if (domain_integs.Size())
   {
-      for (int k = 0; k < dlfi.Size(); k++)
+      for (int k = 0; k < domain_integs.Size(); k++)
      {
-         if (dlfi_marker[k] != NULL)
+         if (domain_integs_marker[k] != NULL)
         {
            MFEM_VERIFY(fes->GetMesh()->attributes.Size() ==
-                        dlfi_marker[k]->Size(),
+                        domain_integs_marker[k]->Size(),
                        "invalid element marker for domain linear form "
                        "integrator #" << k << ", counting from zero");
         }
@@ -128,14 +129,15 @@ void LinearForm::Assemble()
      for (i = 0; i < fes -> GetNE(); i++)
      {
         int elem_attr = fes->GetMesh()->GetAttribute(i);
-         for (int k = 0; k < dlfi.Size(); k++)
+         for (int k = 0; k < domain_integs.Size(); k++)
         {
-            if ( dlfi_marker[k] == NULL ||
-                 (*(dlfi_marker[k]))[elem_attr-1] == 1 )
+            if ( domain_integs_marker[k] == NULL ||
+                 (*(domain_integs_marker[k]))[elem_attr-1] == 1 )
            {
               fes -> GetElementVDofs (i, vdofs);
               eltrans = fes -> GetElementTransformation (i);
-               dlfi[k]->AssembleRHSElementVect(*fes->GetFE(i), *eltrans, elemvect);
+               domain_integs[k]->AssembleRHSElementVect(*fes->GetFE(i),
+                                                        *eltrans, elemvect);
               AddElementVector (vdofs, elemvect);
            }
         }
@@ -143,7 +145,7 @@ void LinearForm::Assemble()
   }
   AssembleDelta();

-   if (blfi.Size())
+   if (boundary_integs.Size())
   {
      Mesh *mesh = fes->GetMesh();

@@ -151,14 +153,14 @@ void LinearForm::Assemble()
      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
                                 mesh->bdr_attributes.Max() : 0);
      bdr_attr_marker = 0;
-      for (int k = 0; k < blfi.Size(); k++)
+      for (int k = 0; k < boundary_integs.Size(); k++)
      {
-         if (blfi_marker[k] == NULL)
+         if (boundary_integs_marker[k] == NULL)
         {
            bdr_attr_marker = 1;
            break;
         }
-         Array<int> &bdr_marker = *blfi_marker[k];
+         Array<int> &bdr_marker = *boundary_integs_marker[k];
         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
                     "invalid boundary marker for boundary integrator #"
                     << k << ", counting from zero");
@@ -174,18 +176,19 @@ void LinearForm::Assemble()
         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
         fes -> GetBdrElementVDofs (i, vdofs);
         eltrans = fes -> GetBdrElementTransformation (i);
-         for (int k=0; k < blfi.Size(); k++)
+         for (int k=0; k < boundary_integs.Size(); k++)
         {
-            if (blfi_marker[k] &&
-                (*blfi_marker[k])[bdr_attr-1] == 0) { continue; }
+            if (boundary_integs_marker[k] &&
+                (*boundary_integs_marker[k])[bdr_attr-1] == 0) { continue; }

-            blfi[k]->AssembleRHSElementVect(*fes->GetBE(i), *eltrans, elemvect);
+            boundary_integs[k]->AssembleRHSElementVect(*fes->GetBE(i),
+                                                       *eltrans, elemvect);

            AddElementVector (vdofs, elemvect);
         }
      }
   }
-   if (flfi.Size())
+   if (boundary_face_integs.Size())
   {
      FaceElementTransformations *tr;
      Mesh *mesh = fes->GetMesh();
@@ -194,14 +197,14 @@ void LinearForm::Assemble()
      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
                                 mesh->bdr_attributes.Max() : 0);
      bdr_attr_marker = 0;
-      for (int k = 0; k < flfi.Size(); k++)
+      for (int k = 0; k < boundary_face_integs.Size(); k++)
      {
-         if (flfi_marker[k] == NULL)
+         if (boundary_face_integs_marker[k] == NULL)
         {
            bdr_attr_marker = 1;
            break;
         }
-         Array<int> &bdr_marker = *flfi_marker[k];
+         Array<int> &bdr_marker = *boundary_face_integs_marker[k];
         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
                     "invalid boundary marker for boundary face integrator #"
                     << k << ", counting from zero");
@@ -220,24 +223,26 @@ void LinearForm::Assemble()
         if (tr != NULL)
         {
            fes -> GetElementVDofs (tr -> Elem1No, vdofs);
-            for (int k = 0; k < flfi.Size(); k++)
+            for (int k = 0; k < boundary_face_integs.Size(); k++)
            {
-               if (flfi_marker[k] &&
-                   (*flfi_marker[k])[bdr_attr-1] == 0) { continue; }
+               if (boundary_face_integs_marker[k] &&
+                   (*boundary_face_integs_marker[k])[bdr_attr-1] == 0)
+               { continue; }

-               flfi[k] -> AssembleRHSElementVect (*fes->GetFE(tr -> Elem1No),
-                                                  *tr, elemvect);
+               boundary_face_integs[k]->
+               AssembleRHSElementVect(*fes->GetFE(tr->Elem1No),
+                                      *tr, elemvect);
               AddElementVector (vdofs, elemvect);
            }
         }
      }
   }

-   if (iflfi.Size())
+   if (interior_face_integs.Size())
   {
      Mesh *mesh = fes->GetMesh();

-      for (int k = 0; k < iflfi.Size(); k++)
+      for (int k = 0; k < interior_face_integs.Size(); k++)
      {
         for (i = 0; i < mesh->GetNumFaces(); i++)
         {
@@ -249,9 +254,10 @@ void LinearForm::Assemble()
               Array<int> vdofs2;
               fes -> GetElementVDofs (tr -> Elem2No, vdofs2);
               vdofs.Append(vdofs2);
-               iflfi[k] -> AssembleRHSElementVect (*fes->GetFE(tr -> Elem1No),
-                                                   *fes->GetFE(tr -> Elem2No),
-                                                   *tr, elemvect);
+               interior_face_integs[k]->
+               AssembleRHSElementVect(*fes->GetFE(tr->Elem1No),
+                                      *fes->GetFE(tr->Elem2No),
+                                      *tr, elemvect);
               AddElementVector (vdofs, elemvect);
            }
         }
@@ -261,56 +267,55 @@ void LinearForm::Assemble()

 void LinearForm::Update(FiniteElementSpace *f, Vector &v, int v_offset)
 {
+   MFEM_ASSERT(v.Size() >= v_offset + f->GetVSize(), "");
   fes = f;
-   NewMemoryAndSize(Memory<double>(v.GetMemory(), v_offset, f->GetVSize()),
-                    f->GetVSize(), false);
+   v.UseDevice(true);
+   this->Vector::MakeRef(v, v_offset, fes->GetVSize());
   ResetDeltaLocations();
 }

 void LinearForm::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
 {
-   MFEM_ASSERT(v.Size() >= v_offset + f->GetVSize(), "");
-   fes = f;
-   v.UseDevice(true);
-   this->Vector::MakeRef(v, v_offset, fes->GetVSize());
+   Update(f, v, v_offset);
 }

 void LinearForm::AssembleDelta()
 {
-   if (dlfi_delta.Size() == 0) { return; }
+   if (domain_delta_integs.Size() == 0) { return; }

   if (!HaveDeltaLocations())
   {
      int sdim = fes->GetMesh()->SpaceDimension();
      Vector center;
-      DenseMatrix centers(sdim, dlfi_delta.Size());
+      DenseMatrix centers(sdim, domain_delta_integs.Size());
      for (int i = 0; i < centers.Width(); i++)
      {
         centers.GetColumnReference(i, center);
-         dlfi_delta[i]->GetDeltaCenter(center);
+         domain_delta_integs[i]->GetDeltaCenter(center);
         MFEM_VERIFY(center.Size() == sdim,
                     "Point dim " << center.Size() <<
                     " does not match space dim " << sdim);
      }
-      fes->GetMesh()->FindPoints(centers, dlfi_delta_elem_id, dlfi_delta_ip);
+      fes->GetMesh()->FindPoints(centers, domain_delta_integs_elem_id,
+                                 domain_delta_integs_ip);
   }

   Array<int> vdofs;
   Vector elemvect;
-   for (int i = 0; i < dlfi_delta.Size(); i++)
+   for (int i = 0; i < domain_delta_integs.Size(); i++)
   {
-      int elem_id = dlfi_delta_elem_id[i];
+      int elem_id = domain_delta_integs_elem_id[i];
      // The delta center may be outside of this sub-domain, or
      // (Par)Mesh::FindPoints() failed to find this point:
      if (elem_id < 0) { continue; }

-      const IntegrationPoint &ip = dlfi_delta_ip[i];
+      const IntegrationPoint &ip = domain_delta_integs_ip[i];
      ElementTransformation &Trans = *fes->GetElementTransformation(elem_id);
      Trans.SetIntPoint(&ip);

      fes->GetElementVDofs(elem_id, vdofs);
-      dlfi_delta[i]->AssembleDeltaElementVect(*fes->GetFE(elem_id), Trans,
-                                              elemvect);
+      domain_delta_integs[i]->AssembleDeltaElementVect(*fes->GetFE(elem_id),
+                                                       Trans, elemvect);
      AddElementVector(vdofs, elemvect);
   }
 }
@@ -333,11 +338,14 @@ LinearForm::~LinearForm()
   if (!extern_lfs)
   {
      int k;
-      for (k=0; k < dlfi_delta.Size(); k++) { delete dlfi_delta[k]; }
-      for (k=0; k < dlfi.Size(); k++) { delete dlfi[k]; }
-      for (k=0; k < blfi.Size(); k++) { delete blfi[k]; }
-      for (k=0; k < flfi.Size(); k++) { delete flfi[k]; }
-      for (k=0; k < iflfi.Size(); k++) { delete iflfi[k]; }
+      for (k=0; k < domain_delta_integs.Size(); k++)
+      { delete domain_delta_integs[k]; }
+      for (k=0; k < domain_integs.Size(); k++) { delete domain_integs[k]; }
+      for (k=0; k < boundary_integs.Size(); k++) { delete boundary_integs[k]; }
+      for (k=0; k < boundary_face_integs.Size(); k++)
+      { delete boundary_face_integs[k]; }
+      for (k=0; k < interior_face_integs.Size(); k++)
+      { delete interior_face_integs[k]; }
   }
 }

@@ -26,43 +26,46 @@ protected:
   /// FE space on which the LinearForm lives. Not owned.
   FiniteElementSpace *fes;

-   /** @brief Indicates the LinearFormIntegrator%s stored in #dlfi, #dlfi_delta,
-       #blfi, and #flfi are owned by another LinearForm. */
+   /** @brief Indicates the LinearFormIntegrator%s stored in #domain_integs,
+       #domain_delta_integs, #boundary_integs, and #boundary_face_integs are
+       owned by another LinearForm. */
   int extern_lfs;

   /// Set of Domain Integrators to be applied.
-   Array<LinearFormIntegrator*> dlfi;
+   Array<LinearFormIntegrator*> domain_integs;
   /// Element attribute marker (should be of length mesh->attributes)
   /// Includes all by default.
   /// 0 - ignore attribute
   /// 1 - include attribute
-   Array<Array<int>*>           dlfi_marker;
+   Array<Array<int>*>           domain_integs_marker;

   /// Separate array for integrators with delta function coefficients.
-   Array<DeltaLFIntegrator*> dlfi_delta;
+   Array<DeltaLFIntegrator*>    domain_delta_integs;

   /// Set of Boundary Integrators to be applied.
-   Array<LinearFormIntegrator*> blfi;
-   Array<Array<int>*>           blfi_marker; ///< Entries are not owned.
+   Array<LinearFormIntegrator*> boundary_integs;
+   /// Entries are not owned.
+   Array<Array<int>*>           boundary_integs_marker;

   /// Set of Boundary Face Integrators to be applied.
-   Array<LinearFormIntegrator*> flfi;
-   Array<Array<int>*>           flfi_marker; ///< Entries are not owned.
+   Array<LinearFormIntegrator*> boundary_face_integs;
+   Array<Array<int>*> boundary_face_integs_marker; ///< Entries not owned.

   /// Set of Internal Face Integrators to be applied.
-   Array<LinearFormIntegrator*> iflfi;
+   Array<LinearFormIntegrator*> interior_face_integs;

   /// The element ids where the centers of the delta functions lie
-   Array<int> dlfi_delta_elem_id;
+   Array<int> domain_delta_integs_elem_id;

   /// The reference coordinates where the centers of the delta functions lie
-   Array<IntegrationPoint> dlfi_delta_ip;
+   Array<IntegrationPoint> domain_delta_integs_ip;

   /// If true, the delta locations are not (re)computed during assembly.
-   bool HaveDeltaLocations() { return (dlfi_delta_elem_id.Size() != 0); }
+   bool HaveDeltaLocations()
+   { return (domain_delta_integs_elem_id.Size() != 0); }

   /// Force (re)computation of delta locations.
-   void ResetDeltaLocations() { dlfi_delta_elem_id.SetSize(0); }
+   void ResetDeltaLocations() { domain_delta_integs_elem_id.SetSize(0); }

 private:
   /// Copy construction is not supported; body is undefined.
@@ -150,22 +153,25 @@ public:
   /** @brief Access all integrators added with AddDomainIntegrator() which are
       not DeltaLFIntegrator%s or they are DeltaLFIntegrator%s with non-delta
       coefficients. */
-   Array<LinearFormIntegrator*> *GetDLFI() { return &dlfi; }
+   Array<LinearFormIntegrator*> *GetDLFI() { return &domain_integs; }

   /** @brief Access all integrators added with AddDomainIntegrator() which are
       DeltaLFIntegrator%s with delta coefficients. */
-   Array<DeltaLFIntegrator*> *GetDLFI_Delta() { return &dlfi_delta; }
+   Array<DeltaLFIntegrator*> *GetDLFI_Delta() { return &domain_delta_integs; }

   /// Access all integrators added with AddBoundaryIntegrator().
-   Array<LinearFormIntegrator*> *GetBLFI() { return &blfi; }
+   Array<LinearFormIntegrator*> *GetBLFI() { return &boundary_integs; }

   /// Access all integrators added with AddBdrFaceIntegrator().
-   Array<LinearFormIntegrator*> *GetFLFI() { return &flfi; }
+   Array<LinearFormIntegrator*> *GetFLFI() { return &boundary_face_integs; }
+
+   /// Access all integrators added with AddInteriorFaceIntegrator().
+   Array<LinearFormIntegrator*> *GetIFLFI() { return &interior_face_integs; }

   /** @brief Access all boundary markers added with AddBdrFaceIntegrator().
       If no marker was specified when the integrator was added, the
       corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetFLFI_Marker() { return &flfi_marker; }
+   Array<Array<int>*> *GetFLFI_Marker() { return &boundary_face_integs_marker; }

   /// Assembles the linear form i.e. sums over all domain/bdr integrators.
   void Assemble();
@@ -630,7 +630,7 @@ double BlockNonlinearForm::GetEnergyBlocked(const BlockVector &bx) const

 double BlockNonlinearForm::GetEnergy(const Vector &x) const
 {
-   xs.Update(x.GetData(), block_offsets);
+   xs.Update(const_cast<Vector&>(x), block_offsets);
   return GetEnergyBlocked(xs);
 }

@@ -646,7 +646,9 @@ void BlockNonlinearForm::MultBlocked(const BlockVector &bx,
   Array<const FiniteElement *> fe2(fes.Size());
   ElementTransformation *T;

+   by.UseDevice(true);
   by = 0.0;
+   by.SyncToBlocks();
   for (int s=0; s<fes.Size(); ++s)
   {
      el_x_const[s] = el_x[s] = new Vector();
@@ -785,6 +787,8 @@ void BlockNonlinearForm::MultBlocked(const BlockVector &bx,
      delete el_y[s];
      delete el_x[s];
   }
+
+   by.SyncFromBlocks();
 }

 const BlockVector &BlockNonlinearForm::Prolongate(const BlockVector &bx) const
@@ -805,8 +809,8 @@ const BlockVector &BlockNonlinearForm::Prolongate(const BlockVector &bx) const

 void BlockNonlinearForm::Mult(const Vector &x, Vector &y) const
 {
-   BlockVector bx(x.GetData(), block_trueOffsets);
-   BlockVector by(y.GetData(), block_trueOffsets);
+   BlockVector bx(const_cast<Vector&>(x), block_trueOffsets);
+   BlockVector by(y, block_trueOffsets);

   const BlockVector &pbx = Prolongate(bx);
   if (needs_prolongation)
@@ -815,8 +819,8 @@ void BlockNonlinearForm::Mult(const Vector &x, Vector &y) const
   }
   BlockVector &pby = needs_prolongation ? aux2 : by;

-   xs.Update(pbx.GetData(), block_offsets);
-   ys.Update(pby.GetData(), block_offsets);
+   xs.Update(const_cast<BlockVector&>(pbx), block_offsets);
+   ys.Update(pby, block_offsets);
   MultBlocked(xs, ys);

   for (int s = 0; s < fes.Size(); s++)
@@ -979,6 +983,8 @@ void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const

            for (int k = 0; k < bfnfi.Size(); ++k)
            {
+               if (bfnfi_marker[k] &&
+                   (*bfnfi_marker[k])[bdr_attr-1] == 0) { continue; }
               bfnfi[k]->AssembleFaceGrad(fe, fe2, *tr, el_x_const, elmats);
               for (int l=0; l<fes.Size(); ++l)
               {
@@ -1019,7 +1025,7 @@ void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const

 Operator &BlockNonlinearForm::GetGradient(const Vector &x) const
 {
-   BlockVector bx(x.GetData(), block_trueOffsets);
+   BlockVector bx(const_cast<Vector&>(x), block_trueOffsets);
   const BlockVector &pbx = Prolongate(bx);

   ComputeGradientBlocked(pbx);
@@ -40,10 +40,10 @@ protected:
 public:
   /** @brief Prescribe a fixed IntegrationRule to use (when @a ir != NULL) or
       let the integrator choose (when @a ir == NULL). */
-   void SetIntRule(const IntegrationRule *ir) { IntRule = ir; }
+   virtual void SetIntRule(const IntegrationRule *ir) { IntRule = ir; }

   /// Prescribe a fixed IntegrationRule to use.
-   void SetIntegrationRule(const IntegrationRule &irule) { IntRule = &irule; }
+   void SetIntegrationRule(const IntegrationRule &ir) { SetIntRule(&ir); }

   /// Set the memory type used for GeometricFactors and other large allocations
   /// in PA extensions.
@@ -121,7 +121,7 @@ public:
       @param[in,out] y  The result Vector: @f$ y += G x @f$. */
   virtual void AddMultGradPA(const Vector &x, Vector &y) const;

-   /// Method for computing the diagonal of the gradient with partial assmebly.
+   /// Method for computing the diagonal of the gradient with partial assembly.
   /** The result Vector @a diag is an E-Vector. This method can be called only
       after the method AssembleGradPA() has been called.

@@ -130,7 +130,7 @@ void ParBilinearForm::ParallelAssemble(OperatorHandle &A, SparseMatrix *A_local)

   OperatorHandle dA(A.Type()), Ph(A.Type()), hdA;

-   if (fbfi.Size() == 0)
+   if (interior_face_integs.Size() == 0)
   {
      // construct a parallel block-diagonal matrix 'A' based on 'a'
      dA.MakeSquareBlockDiag(pfes->GetComm(), pfes->GlobalVSize(),
@@ -214,11 +214,12 @@ void ParBilinearForm::AssembleSharedFaces(int skip_zeros)
         }
      }
      vdofs_all.Append(vdofs2);
-      for (int k = 0; k < fbfi.Size(); k++)
+      for (int k = 0; k < interior_face_integs.Size(); k++)
      {
-         fbfi[k]->AssembleFaceMatrix(*pfes->GetFE(T->Elem1No),
-                                     *pfes->GetFaceNbrFE(Elem2NbrNo),
-                                     *T, elemmat);
+         interior_face_integs[k]->
+         AssembleFaceMatrix(*pfes->GetFE(T->Elem1No),
+                            *pfes->GetFaceNbrFE(Elem2NbrNo),
+                            *T, elemmat);
         if (keep_nbr_block)
         {
            mat->AddSubMatrix(vdofs_all, vdofs_all, elemmat, skip_zeros);
@@ -233,7 +234,7 @@ void ParBilinearForm::AssembleSharedFaces(int skip_zeros)

 void ParBilinearForm::Assemble(int skip_zeros)
 {
-   if (fbfi.Size())
+   if (interior_face_integs.Size())
   {
      pfes->ExchangeFaceNbrData();
      if (!ext && mat == NULL)
@@ -244,7 +245,7 @@ void ParBilinearForm::Assemble(int skip_zeros)

   BilinearForm::Assemble(skip_zeros);

-   if (!ext && fbfi.Size() > 0)
+   if (!ext && interior_face_integs.Size() > 0)
   {
      AssembleSharedFaces(skip_zeros);
   }
@@ -316,9 +317,6 @@ ParallelEliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
 void ParBilinearForm::TrueAddMult(const Vector &x, Vector &y, const double a)
 const
 {
-   MFEM_VERIFY(fbfi.Size() == 0, "the case of interior face integrators is not"
-               " implemented");
-
   if (X.ParFESpace() != pfes)
   {
      X.SetSpace(pfes);
@@ -332,6 +330,9 @@ const
   }
   else
   {
+      MFEM_VERIFY(interior_face_integs.Size() == 0,
+                  "the case of interior face integrators is not"
+                  " implemented");
      mat->Mult(X, Y);
   }
   pfes->Dof_TrueDof_Matrix()->MultTranspose(a, Y, 1.0, y);
@@ -471,7 +472,7 @@ void ParBilinearForm::RecoverFEMSolution(
   else
   {
      // Apply conforming prolongation
-      x.SetSize(P.Height());
+      x.SetSize(P.Height(), GetHypreMemoryType());
      P.Mult(X, x);
   }
 }
@@ -515,7 +515,7 @@ const FiniteElement *ParFiniteElementSpace::GetFE(int i) const
   else { return FiniteElementSpace::GetFE(i); }
 }

-const Operator *ParFiniteElementSpace::GetFaceRestriction(
+const FaceRestriction *ParFiniteElementSpace::GetFaceRestriction(
   ElementDofOrdering e_ordering, FaceType type, L2FaceValues mul) const
 {
   const bool is_dg_space = IsDGSpace();
@@ -529,7 +529,7 @@ const Operator *ParFiniteElementSpace::GetFaceRestriction(
   }
   else
   {
-      Operator* res;
+      FaceRestriction *res;
      if (is_dg_space)
      {
         res = new ParL2FaceRestriction(*this, e_ordering, type, m);
@@ -2885,19 +2885,10 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,

   HypreParMatrix* R;
   R = new HypreParMatrix(MyComm, dof_offsets[nrk], old_dof_offsets[nrk],
-                          dof_offsets, old_dof_offsets, diag, offd, cmap);
+                          dof_offsets, old_dof_offsets, diag, offd, cmap,
+                          true);

-#ifndef HYPRE_BIGINT
-   diag->LoseData();
-   offd->LoseData();
-#else
-   diag->SetDataOwner(false);
-   offd->SetDataOwner(false);
-#endif
-   delete diag;
-   delete offd;
-
-   R->SetOwnerFlags(3, 3, 1);
+   R->SetOwnerFlags(R->OwnsDiag(), R->OwnsOffd(), 1);

   return R;
 }
@@ -291,7 +291,7 @@ public:
   /** Returns pointer to the FiniteElement in the FiniteElementCollection
       associated with i'th element in the mesh object. If @a i is greater than
       or equal to the number of local mesh elements, @a i will be interpreted
-       as a shifted index of a face neigbor element. */
+       as a shifted index of a face neighbor element. */
   virtual const FiniteElement *GetFE(int i) const;

   /** Returns an Operator that converts L-vectors to E-vectors on each face.
@@ -299,7 +299,7 @@ public:
       presence of shared faces. Shared faces are treated as interior faces,
       the returned operator handles the communication needed to get the
       shared face values from other MPI ranks */
-   virtual const Operator *GetFaceRestriction(
+   virtual const FaceRestriction *GetFaceRestriction(
      ElementDofOrdering e_ordering, FaceType type,
      L2FaceValues mul = L2FaceValues::DoubleValued) const;

@@ -65,6 +65,11 @@ public:
   ParGridFunction(ParFiniteElementSpace *pf, double *data) :
      GridFunction(pf, data), pfes(pf) { }

+   /** @brief Construct a ParGridFunction using previously allocated Vector
+       @a base starting at the given offset, @a base_offset. */
+   ParGridFunction(ParFiniteElementSpace *pf, Vector &base, int base_offset = 0)
+      : GridFunction(pf, base, base_offset), pfes(pf) { }
+
   /// Construct a ParGridFunction using a GridFunction as external data.
   /** The parallel space @a *pf and the space used by @a *gf should match. The
       data from @a *gf is used as the local data of the ParGridFunction on each
@@ -47,7 +47,7 @@ void ParLinearForm::Assemble()
 {
   LinearForm::Assemble();

-   if (iflfi.Size())
+   if (interior_face_integs.Size())
   {
      pfes->ExchangeFaceNbrData();
      AssembleSharedFaces();
@@ -59,10 +59,10 @@ void ParLinearForm::AssembleSharedFaces()
   Array<int> vdofs;
   Vector elemvect;

-   if (iflfi.Size())
+   if (interior_face_integs.Size())
   {
      ParMesh *pmesh = pfes->GetParMesh();
-      for (int k = 0; k < iflfi.Size(); k++)
+      for (int k = 0; k < interior_face_integs.Size(); k++)
      {
         for (int i = 0; i < pmesh->GetNSharedFaces(); i++)
         {
@@ -73,9 +73,10 @@ void ParLinearForm::AssembleSharedFaces()
            {
               int Elem2Nbr = tr->Elem2No - pmesh->GetNE();
               fes -> GetElementVDofs (tr -> Elem1No, vdofs);
-               iflfi[0] -> AssembleRHSElementVect (*fes->GetFE(tr -> Elem1No),
-                                                   *pfes->GetFaceNbrFE(Elem2Nbr),
-                                                   *tr, elemvect);
+               interior_face_integs[k]->
+               AssembleRHSElementVect(*fes->GetFE(tr->Elem1No),
+                                      *pfes->GetFaceNbrFE(Elem2Nbr),
+                                      *tr, elemvect);
               AddElementVector (vdofs, elemvect);
            }
         }
@@ -218,7 +218,8 @@ void ParBlockNonlinearForm::SetEssentialBC(const

 double ParBlockNonlinearForm::GetEnergy(const Vector &x) const
 {
-   xs_true.Update(x.GetData(), block_trueOffsets);
+   // xs_true is not modified, so const_cast is okay
+   xs_true.Update(const_cast<Vector &>(x), block_trueOffsets);
   xs.Update(block_offsets);

   for (int s = 0; s < fes.Size(); ++s)
@@ -237,8 +238,9 @@ double ParBlockNonlinearForm::GetEnergy(const Vector &x) const

 void ParBlockNonlinearForm::Mult(const Vector &x, Vector &y) const
 {
-   xs_true.Update(x.GetData(), block_trueOffsets);
-   ys_true.Update(y.GetData(), block_trueOffsets);
+   // xs_true is not modified, so const_cast is okay
+   xs_true.Update(const_cast<Vector &>(x), block_trueOffsets);
+   ys_true.Update(y, block_trueOffsets);
   xs.Update(block_offsets);
   ys.Update(block_offsets);

@@ -262,13 +264,17 @@ void ParBlockNonlinearForm::Mult(const Vector &x, Vector &y) const

      ys_true.GetBlock(s).SetSubVector(*ess_tdofs[s], 0.0);
   }
+
+   ys_true.SyncFromBlocks();
+   y.SyncMemory(ys_true);
 }

 /// Return the local gradient matrix for the given true-dof vector x
 const BlockOperator & ParBlockNonlinearForm::GetLocalGradient(
   const Vector &x) const
 {
-   xs_true.Update(x.GetData(), block_trueOffsets);
+   // xs_true is not modified, so const_cast is okay
+   xs_true.Update(const_cast<Vector &>(x), block_trueOffsets);
   xs.Update(block_offsets);

   for (int s=0; s<fes.Size(); ++s)
@@ -277,7 +283,8 @@ const BlockOperator & ParBlockNonlinearForm::GetLocalGradient(
         xs_true.GetBlock(s), xs.GetBlock(s));
   }

-   BlockNonlinearForm::ComputeGradientBlocked(xs); // (re)assemble Grad with b.c.
+   // (re)assemble Grad without b.c. into 'Grads'
+   BlockNonlinearForm::ComputeGradientBlocked(xs);

   delete BlockGrad;
   BlockGrad = new BlockOperator(block_offsets);
@@ -33,6 +33,10 @@ ParL2FaceRestriction::ParL2FaceRestriction(const ParFiniteElementSpace &fes,
   // If fespace == L2
   const ParFiniteElementSpace &pfes =
      static_cast<const ParFiniteElementSpace&>(this->fes);
+
+   // Ensure the face neighbor data is constructed
+   pfes.GetParMesh()->ExchangeFaceNbrData();
+
   const FiniteElement *fe = pfes.GetFE(0);
   const TensorBasisElement *tfe = dynamic_cast<const TensorBasisElement*>(fe);
   MFEM_VERIFY(tfe != NULL &&
@@ -13,6 +13,13 @@
 #include "gridfunc.hpp"
 #include "fespace.hpp"
 #include "../general/forall.hpp"
+#include <climits>
+
+#ifdef MFEM_USE_MPI
+
+#include "pfespace.hpp"
+
+#endif

 namespace mfem
 {
@@ -267,35 +274,25 @@ void ElementRestriction::FillSparseMatrix(const Vector &mat_ea,
   FillJAndData(mat_ea, mat);
 }

-template <int MaxNbNbr>
 static MFEM_HOST_DEVICE int GetMinElt(const int *my_elts, const int nbElts,
                                      const int *nbr_elts, const int nbrNbElts)
 {
-   // Building the intersection
-   int inter[MaxNbNbr];
-   int cpt = 0;
+   // Find the minimal element index found in both my_elts[] and nbr_elts[]
+   int min_el = INT_MAX;
   for (int i = 0; i < nbElts; i++)
   {
      const int e_i = my_elts[i];
+      if (e_i >= min_el) { continue; }
      for (int j = 0; j < nbrNbElts; j++)
      {
         if (e_i==nbr_elts[j])
         {
-            inter[cpt] = e_i;
-            cpt++;
+            min_el = e_i; // we already know e_i < min_el
+            break;
         }
      }
   }
-   // Finding the minimum
-   int min = inter[0];
-   for (int i = 1; i < cpt; i++)
-   {
-      if (inter[i] < min)
-      {
-         min = inter[i];
-      }
-   }
-   return min;
+   return min_el;
 }

 /** Returns the index where a non-zero entry should be added and increment the
@@ -355,7 +352,7 @@ int ElementRestriction::FillI(SparseMatrix &mat) const
                  const int elt = j_E/elt_dofs;
                  j_elts[e_j] = elt;
               }
-               int min_e = GetMinElt<Max>(i_elts, i_nbElts, j_elts, j_nbElts);
+               int min_e = GetMinElt(i_elts, i_nbElts, j_elts, j_nbElts);
               if (e == min_e) // add the nnz only once
               {
                  GetAndIncrementNnzIndex(i_L, I);
@@ -434,7 +431,7 @@ void ElementRestriction::FillJAndData(const Vector &ea_data,
                  j_elts[e_j] = elt;
                  j_B[e_j]    = j_E%elt_dofs;
               }
-               int min_e = GetMinElt<Max>(i_elts, i_nbElts, j_elts, j_nbElts);
+               int min_e = GetMinElt(i_elts, i_nbElts, j_elts, j_nbElts);
               if (e == min_e) // add the nnz only once
               {
                  double val = 0.0;
@@ -684,6 +681,19 @@ H1FaceRestriction::H1FaceRestriction(const FiniteElementSpace &fes,
     gather_indices(nf*dof)
 {
   if (nf==0) { return; }
+
+#ifdef MFEM_USE_MPI
+
+   // If the underlying finite element space is parallel, ensure the face
+   // neighbor information is generated.
+   if (const ParFiniteElementSpace *pfes
+       = dynamic_cast<const ParFiniteElementSpace*>(&fes))
+   {
+      pfes->GetParMesh()->ExchangeFaceNbrData();
+   }
+
+#endif
+
   // If fespace == H1
   const FiniteElement *fe = fes.GetFE(0);
   const TensorBasisElement *tfe = dynamic_cast<const TensorBasisElement*>(fe);
@@ -847,7 +857,7 @@ void H1FaceRestriction::Mult(const Vector& x, Vector& y) const
   });
 }

-void H1FaceRestriction::MultTranspose(const Vector& x, Vector& y) const
+void H1FaceRestriction::AddMultTranspose(const Vector& x, Vector& y) const
 {
   // Assumes all elements have the same number of dofs
   const int nd = dof;
@@ -856,7 +866,7 @@ void H1FaceRestriction::MultTranspose(const Vector& x, Vector& y) const
   auto d_offsets = offsets.Read();
   auto d_indices = gather_indices.Read();
   auto d_x = Reshape(x.Read(), nd, vd, nf);
-   auto d_y = Reshape(y.Write(), t?vd:ndofs, t?ndofs:vd);
+   auto d_y = Reshape(y.ReadWrite(), t?vd:ndofs, t?ndofs:vd);
   MFEM_FORALL(i, ndofs,
   {
      const int offset = d_offsets[i];
@@ -1267,7 +1277,7 @@ void L2FaceRestriction::Mult(const Vector& x, Vector& y) const
   }
 }

-void L2FaceRestriction::MultTranspose(const Vector& x, Vector& y) const
+void L2FaceRestriction::AddMultTranspose(const Vector& x, Vector& y) const
 {
   // Assumes all elements have the same number of dofs
   const int nd = dof;
@@ -1280,7 +1290,7 @@ void L2FaceRestriction::MultTranspose(const Vector& x, Vector& y) const
   if (m == L2FaceValues::DoubleValued)
   {
      auto d_x = Reshape(x.Read(), nd, vd, 2, nf);
-      auto d_y = Reshape(y.Write(), t?vd:ndofs, t?ndofs:vd);
+      auto d_y = Reshape(y.ReadWrite(), t?vd:ndofs, t?ndofs:vd);
      MFEM_FORALL(i, ndofs,
      {
         const int offset = d_offsets[i];
@@ -1304,7 +1314,7 @@ void L2FaceRestriction::MultTranspose(const Vector& x, Vector& y) const
   else
   {
      auto d_x = Reshape(x.Read(), nd, vd, nf);
-      auto d_y = Reshape(y.Write(), t?vd:ndofs, t?ndofs:vd);
+      auto d_y = Reshape(y.ReadWrite(), t?vd:ndofs, t?ndofs:vd);
      MFEM_FORALL(i, ndofs,
      {
         const int offset = d_offsets[i];
@@ -21,10 +21,6 @@ namespace mfem
 class FiniteElementSpace;
 enum class ElementDofOrdering;

-/** An enum type to specify if only e1 value is requested (SingleValued) or both
-    e1 and e2 (DoubleValued). */
-enum class L2FaceValues : bool {SingleValued, DoubleValued};
-
 /// Operator that converts FiniteElementSpace L-vectors to E-vectors.
 /** Objects of this type are typically created and owned by FiniteElementSpace
    objects, see FiniteElementSpace::GetElementRestriction(). */
@@ -104,10 +100,75 @@ public:
   void FillJAndData(const Vector &ea_data, SparseMatrix &mat) const;
 };

-/// Operator that extracts Face degrees of freedom.
+/** An enum type to specify if only e1 value is requested (SingleValued) or both
+    e1 and e2 (DoubleValued). */
+enum class L2FaceValues : bool {SingleValued, DoubleValued};
+
+/** @brief Base class for operators that extracts Face degrees of freedom.
+
+    In order to compute quantities on the faces of a mesh, it is often useful to
+    extract the degrees of freedom on the faces of the elements. This class
+    provides an interface for such operations.
+
+    If the FiniteElementSpace is ordered by Ordering::byVDIM, then the expected
+    format for the L-vector is (vdim x ndofs), otherwise if Ordering::byNODES
+    the expected format is (ndofs x vdim), where ndofs is the total number of
+    degrees of freedom.
+    Since FiniteElementSpace can either be continuous or discontinuous, the
+    degrees of freedom on a face can either be single valued or double valued,
+    this is what we refer to as the multiplicity and is represented by the
+    L2FaceValues enum type.
+    The format of the output face E-vector of degrees of freedom is
+    (face_dofs x vdim x multiplicity x nfaces), where face_dofs is the number of
+    degrees of freedom on each face, and nfaces the number of faces of the
+    requested FaceType (see FiniteElementSpace::GetNFbyType).
+
+    @note Objects of this type are typically created and owned by
+    FiniteElementSpace objects, see FiniteElementSpace::GetFaceRestriction(). */
+class FaceRestriction : public Operator
+{
+public:
+   FaceRestriction(): Operator() { }
+
+   FaceRestriction(int h, int w): Operator(h, w) { }
+
+   virtual ~FaceRestriction() { }
+
+   /** @brief Extract the face degrees of freedom from @a x into @a y.
+
+       @param[in]  x The L-vector of degrees of freedom.
+       @param[out] y The degrees of freedom on the face, corresponding to a face
+                     E-vector.
+   */
+   void Mult(const Vector &x, Vector &y) const override = 0;
+
+   /** @brief Add the face degrees of freedom @a x to the element degrees of
+       freedom @a y.
+
+       @param[in]     x The face degrees of freedom on the face.
+       @param[in,out] y The L-vector of degrees of freedom to which we add the
+                        face degrees of freedom.
+   */
+   virtual void AddMultTranspose(const Vector &x, Vector &y) const = 0;
+
+   /** @brief Set the face degrees of freedom in the element degrees of freedom
+       @a y to the values given in @a x.
+
+       @param[in]     x The face degrees of freedom on the face.
+       @param[in,out] y The L-vector of degrees of freedom to which we add the
+                        face degrees of freedom.
+   */
+   void MultTranspose(const Vector &x, Vector &y) const override
+   {
+      y = 0.0;
+      AddMultTranspose(x, y);
+   }
+};
+
+/// Operator that extracts Face degrees of freedom for H1 FiniteElementSpaces.
 /** Objects of this type are typically created and owned by FiniteElementSpace
    objects, see FiniteElementSpace::GetFaceRestriction(). */
-class H1FaceRestriction : public Operator
+class H1FaceRestriction : public FaceRestriction
 {
 protected:
   const FiniteElementSpace &fes;
@@ -122,16 +183,42 @@ protected:
   Array<int> gather_indices;

 public:
-   H1FaceRestriction(const FiniteElementSpace&, const ElementDofOrdering,
-                     const FaceType);
-   void Mult(const Vector &x, Vector &y) const;
-   void MultTranspose(const Vector &x, Vector &y) const;
+   /** @brief Constructor for a H1FaceRestriction.
+
+       @param[in] fes The FiniteElementSpace on which this H1FaceRestriction
+                      operates.
+       @param[in] ordering The requested output ordering of the
+                           H1FaceRestriction, either Native or Lexicographic.
+       @param[in] type The requested type of faces on which this operator
+                       extracts the degrees of freedom, either Interior or
+                       Boundary.
+   */
+   H1FaceRestriction(const FiniteElementSpace& fes,
+                     const ElementDofOrdering ordering,
+                     const FaceType type);
+
+   /** @brief Extract the face degrees of freedom from @a x into @a y.
+
+       @param[in]  x The L-vector of degrees of freedom.
+       @param[out] y The degrees of freedom on the face, corresponding to a face
+                     E-vector.
+   */
+   void Mult(const Vector &x, Vector &y) const override;
+
+   /** @brief Add the face degrees of freedom @a x to the element degrees of
+       freedom @a y.
+
+       @param[in]     x The face degrees of freedom on the face.
+       @param[in,out] y The L-vector of degrees of freedom to which we add the
+                        face degrees of freedom.
+   */
+   void AddMultTranspose(const Vector &x, Vector &y) const override;
 };

-/// Operator that extracts Face degrees of freedom.
+/// Operator that extracts Face degrees of freedom on L2 FiniteElementSpaces.
 /** Objects of this type are typically created and owned by FiniteElementSpace
    objects, see FiniteElementSpace::GetFaceRestriction(). */
-class L2FaceRestriction : public Operator
+class L2FaceRestriction : public FaceRestriction
 {
 protected:
   const FiniteElementSpace &fes;
@@ -154,19 +241,38 @@ protected:
                     const L2FaceValues m = L2FaceValues::DoubleValued);

 public:
-   L2FaceRestriction(const FiniteElementSpace&, const ElementDofOrdering,
+   L2FaceRestriction(const FiniteElementSpace&,
+                     const ElementDofOrdering,
                     const FaceType,
                     const L2FaceValues m = L2FaceValues::DoubleValued);
-   virtual void Mult(const Vector &x, Vector &y) const;
-   void MultTranspose(const Vector &x, Vector &y) const;
+
+   /** @brief Extract the face degrees of freedom from @a x into @a y.
+
+       @param[in]  x The L-vector of degrees of freedom.
+       @param[out] y The degrees of freedom on the face, corresponding to a face
+                     E-vector.
+   */
+   void Mult(const Vector &x, Vector &y) const override;
+
+   /** @brief Add the face degrees of freedom @a x to the element degrees of
+       freedom @a y.
+
+       @param[in]     x The face degrees of freedom on the face.
+       @param[in,out] y The L-vector of degrees of freedom to which we add the
+                        face degrees of freedom.
+   */
+   void AddMultTranspose(const Vector &x, Vector &y) const override;
+
   /** Fill the I array of SparseMatrix corresponding to the sparsity pattern
       given by this L2FaceRestriction. */
   virtual void FillI(SparseMatrix &mat, const bool keep_nbr_block = false) const;
+
   /** Fill the J and Data arrays of SparseMatrix corresponding to the sparsity
       pattern given by this L2FaceRestriction, and the values of ea_data. */
   virtual void FillJAndData(const Vector &ea_data,
                             SparseMatrix &mat,
                             const bool keep_nbr_block = false) const;
+
   /// This methods adds the DG face matrices to the element matrices.
   void AddFaceMatricesToElementMatrices(Vector &fea_data,
                                         Vector &ea_data) const;
@@ -34,6 +34,7 @@ void TMOP_Combo_QualityMetric::EvalP(const DenseMatrix &Jpt,
                                     DenseMatrix &P) const
 {
   DenseMatrix Pt(P.Size());
+   P = 0.0;
   for (int i = 0; i < tmop_q_arr.Size(); i++)
   {
      tmop_q_arr[i]->EvalP(Jpt, Pt);
@@ -50,6 +51,7 @@ void TMOP_Combo_QualityMetric::AssembleH(const DenseMatrix &Jpt,
   DenseMatrix At(A.Size());
   for (int i = 0; i < tmop_q_arr.Size(); i++)
   {
+      At = 0.0;
      tmop_q_arr[i]->AssembleH(Jpt, DS, weight, At);
      At *= wt_arr[i];
      A += At;
@@ -1312,33 +1314,61 @@ static inline void device_copy(double *d_dest, const double *d_src, int size)
 } // namespace internal

 #ifdef MFEM_USE_MPI
-void DiscreteAdaptTC::FinalizeParDiscreteTargetSpec(const ParGridFunction
-                                                    &tspec_)
+void DiscreteAdaptTC::FinalizeParDiscreteTargetSpec(const ParGridFunction &t)
 {
   MFEM_VERIFY(adapt_eval, "SetAdaptivityEvaluator() has not been called!")
   MFEM_VERIFY(ncomp > 0, "No target specifications have been set!");

-   ParFiniteElementSpace *ptspec_fes = tspec_.ParFESpace();
+   ParFiniteElementSpace *ptspec_fes = t.ParFESpace();

   adapt_eval->SetParMetaInfo(*ptspec_fes->GetParMesh(),
                              *ptspec_fes->FEColl(), ncomp);
-   adapt_eval->SetInitialField(*tspec_fes->GetMesh()->GetNodes(), tspec);
+   adapt_eval->SetInitialField(*ptspec_fes->GetMesh()->GetNodes(), tspec);

   tspec_sav = tspec;

   delete tspec_fesv;
-   tspec_fesv = new FiniteElementSpace(tspec_fes->GetMesh(),
-                                       tspec_fes->FEColl(), ncomp);
+   tspec_fesv = new FiniteElementSpace(ptspec_fes->GetMesh(),
+                                       ptspec_fes->FEColl(), ncomp);
+
+   delete ptspec_fesv;
+   ptspec_fesv = new ParFiniteElementSpace(ptspec_fes->GetParMesh(),
+                                           ptspec_fes->FEColl(), ncomp);
+
+   delete tspec_pgf;
+   tspec_pgf = new ParGridFunction(ptspec_fesv, tspec);
+   tspec_gf = tspec_pgf;
+}
+
+void DiscreteAdaptTC::ParUpdateAfterMeshTopologyChange()
+{
+   ptspec_fesv->Update();
+   if (tspec_fesv)
+   {
+      delete tspec_fesv;
+      tspec_fesv = new FiniteElementSpace(ptspec_fesv->GetMesh(),
+                                          ptspec_fesv->FEColl(), ncomp);
+   }
+   tspec_pgf->Update();
+   tspec_gf = tspec_pgf;
+   tspec.SetDataAndSize(tspec_pgf->GetData(), tspec_pgf->Size());
+   tspec_sav = tspec;
+
+   adapt_eval->SetParMetaInfo(*ptspec_fesv->GetParMesh(),
+                              *ptspec_fesv->FEColl(), ncomp);
+   adapt_eval->SetInitialField(*ptspec_fesv->GetMesh()->GetNodes(), tspec);
 }

 void DiscreteAdaptTC::SetTspecAtIndex(int idx, const ParGridFunction &tspec_)
 {
-   const int vdim     = tspec_.FESpace()->GetVDim(),
-             dof_cnt  = tspec_.Size()/vdim;
+   const int vdim = tspec_.FESpace()->GetVDim(),
+             ndof = tspec_.FESpace()->GetNDofs();
+   MFEM_VERIFY(ndof == tspec.Size()/ncomp, "Inconsistency in SetTspecAtIndex.");
+
   const auto tspec__d = tspec_.Read();
   auto tspec_d = tspec.ReadWrite();
-   const int offset = idx*dof_cnt;
-   internal::device_copy(tspec_d + offset, tspec__d, dof_cnt*vdim);
+   const int offset = idx*ndof;
+   internal::device_copy(tspec_d + offset, tspec__d, ndof*vdim);
   FinalizeParDiscreteTargetSpec(tspec_);
 }

@@ -1358,78 +1388,71 @@ void DiscreteAdaptTC::SetParDiscreteTargetSkew(const ParGridFunction &tspec_)
   FinalizeParDiscreteTargetSpec(tspec_);
 }

-void DiscreteAdaptTC::SetParDiscreteTargetAspectRatio(const ParGridFunction
-                                                      &tspec_)
+void DiscreteAdaptTC::SetParDiscreteTargetAspectRatio(const ParGridFunction &ar)
 {
-   if (aspectratioidx > -1) { SetTspecAtIndex(aspectratioidx, tspec_); return; }
+   if (aspectratioidx > -1) { SetTspecAtIndex(aspectratioidx, ar); return; }
   aspectratioidx = ncomp;
-   SetDiscreteTargetBase(tspec_);
-   FinalizeParDiscreteTargetSpec(tspec_);
+   SetDiscreteTargetBase(ar);
+   FinalizeParDiscreteTargetSpec(ar);
 }

-void DiscreteAdaptTC::SetParDiscreteTargetOrientation(const ParGridFunction
-                                                      &tspec_)
+void DiscreteAdaptTC::SetParDiscreteTargetOrientation(const ParGridFunction &o)
 {
-   if (orientationidx > -1) { SetTspecAtIndex(orientationidx, tspec_); return; }
+   if (orientationidx > -1) { SetTspecAtIndex(orientationidx, o); return; }
   orientationidx = ncomp;
-   SetDiscreteTargetBase(tspec_);
-   FinalizeParDiscreteTargetSpec(tspec_);
+   SetDiscreteTargetBase(o);
+   FinalizeParDiscreteTargetSpec(o);
 }

 void DiscreteAdaptTC::SetParDiscreteTargetSpec(const ParGridFunction &tspec_)
 {
   SetParDiscreteTargetSize(tspec_);
-   FinalizeParDiscreteTargetSpec(tspec_);
 }
 #endif // MFEM_USE_MPI

 void DiscreteAdaptTC::SetDiscreteTargetBase(const GridFunction &tspec_)
 {
-   const int vdim     = tspec_.FESpace()->GetVDim(),
-             dof_cnt  = tspec_.Size()/vdim;
+   const int vdim = tspec_.FESpace()->GetVDim(),
+             ndof = tspec_.FESpace()->GetNDofs();

   ncomp += vdim;

-   delete tspec_fes;
-   tspec_fes = new FiniteElementSpace(tspec_.FESpace()->GetMesh(),
-                                      tspec_.FESpace()->FEColl(), 1);
-
   // need to append data to tspec
   // make a copy of tspec->tspec_temp, increase its size, and
   // copy data from tspec_temp -> tspec, then add new entries
   Vector tspec_temp = tspec;
   tspec.UseDevice(true);
   tspec_sav.UseDevice(true);
-   tspec.SetSize(ncomp*dof_cnt);
+   tspec.SetSize(ncomp*ndof);

   const auto tspec_temp_d = tspec_temp.Read();
   auto tspec_d = tspec.ReadWrite();
   internal::device_copy(tspec_d, tspec_temp_d, tspec_temp.Size());

   const auto tspec__d = tspec_.Read();
-   const int offset = (ncomp-vdim)*dof_cnt;
-   internal::device_copy(tspec_d + offset, tspec__d, dof_cnt*vdim);
+   const int offset = (ncomp-vdim)*ndof;
+   internal::device_copy(tspec_d + offset, tspec__d, ndof*vdim);
 }

 void DiscreteAdaptTC::SetTspecAtIndex(int idx, const GridFunction &tspec_)
 {
-   const int vdim     = tspec_.FESpace()->GetVDim(),
-             dof_cnt  = tspec_.Size()/vdim;
+   const int vdim = tspec_.FESpace()->GetVDim(),
+             ndof = tspec_.FESpace()->GetNDofs();
+   MFEM_VERIFY(ndof == tspec.Size()/ncomp, "Inconsistency in SetTargetSpec.");

   const auto tspec__d = tspec_.Read();
   auto tspec_d = tspec.ReadWrite();
-   const int offset = idx*dof_cnt;
-   internal::device_copy(tspec_d + offset, tspec__d, dof_cnt*vdim);
-   FinalizeSerialDiscreteTargetSpec();
+   const int offset = idx*ndof;
+   internal::device_copy(tspec_d + offset, tspec__d, ndof*vdim);
+   FinalizeSerialDiscreteTargetSpec(tspec_);
 }

 void DiscreteAdaptTC::SetSerialDiscreteTargetSize(const GridFunction &tspec_)
 {
-
   if (sizeidx > -1) { SetTspecAtIndex(sizeidx, tspec_); return; }
   sizeidx = ncomp;
   SetDiscreteTargetBase(tspec_);
-   FinalizeSerialDiscreteTargetSpec();
+   FinalizeSerialDiscreteTargetSpec(tspec_);
 }

 void DiscreteAdaptTC::SetSerialDiscreteTargetSkew(const GridFunction &tspec_)
@@ -1437,32 +1460,31 @@ void DiscreteAdaptTC::SetSerialDiscreteTargetSkew(const GridFunction &tspec_)
   if (skewidx > -1) { SetTspecAtIndex(skewidx, tspec_); return; }
   skewidx = ncomp;
   SetDiscreteTargetBase(tspec_);
-   FinalizeSerialDiscreteTargetSpec();
+   FinalizeSerialDiscreteTargetSpec(tspec_);
 }

-void DiscreteAdaptTC::SetSerialDiscreteTargetAspectRatio(
-   const GridFunction &tspec_)
+void DiscreteAdaptTC::SetSerialDiscreteTargetAspectRatio(const GridFunction &ar)
 {
-   if (aspectratioidx > -1) { SetTspecAtIndex(aspectratioidx, tspec_); return; }
+   if (aspectratioidx > -1) { SetTspecAtIndex(aspectratioidx, ar); return; }
   aspectratioidx = ncomp;
-   SetDiscreteTargetBase(tspec_);
-   FinalizeSerialDiscreteTargetSpec();
+   SetDiscreteTargetBase(ar);
+   FinalizeSerialDiscreteTargetSpec(ar);
 }

-void DiscreteAdaptTC::SetSerialDiscreteTargetOrientation(
-   const GridFunction &tspec_)
+void DiscreteAdaptTC::SetSerialDiscreteTargetOrientation(const GridFunction &o)
 {
-   if (orientationidx > -1) { SetTspecAtIndex(orientationidx, tspec_); return; }
+   if (orientationidx > -1) { SetTspecAtIndex(orientationidx, o); return; }
   orientationidx = ncomp;
-   SetDiscreteTargetBase(tspec_);
-   FinalizeSerialDiscreteTargetSpec();
+   SetDiscreteTargetBase(o);
+   FinalizeSerialDiscreteTargetSpec(o);
 }

-void DiscreteAdaptTC::FinalizeSerialDiscreteTargetSpec()
+void DiscreteAdaptTC::FinalizeSerialDiscreteTargetSpec(const GridFunction &t)
 {
   MFEM_VERIFY(adapt_eval, "SetAdaptivityEvaluator() has not been called!")
   MFEM_VERIFY(ncomp > 0, "No target specifications have been set!");

+   const FiniteElementSpace *tspec_fes = t.FESpace();
   adapt_eval->SetSerialMetaInfo(*tspec_fes->GetMesh(),
                                 *tspec_fes->FEColl(), ncomp);
   adapt_eval->SetInitialField(*tspec_fes->GetMesh()->GetNodes(), tspec);
@@ -1472,12 +1494,40 @@ void DiscreteAdaptTC::FinalizeSerialDiscreteTargetSpec()
   delete tspec_fesv;
   tspec_fesv = new FiniteElementSpace(tspec_fes->GetMesh(),
                                       tspec_fes->FEColl(), ncomp);
+
+   delete tspec_gf;
+   tspec_gf = new GridFunction(tspec_fesv, tspec);
+}
+
+void DiscreteAdaptTC::GetDiscreteTargetSpec(GridFunction &tspec_, int idx)
+{
+   if (idx < 0) { return; }
+   const int ndof = tspec_.FESpace()->GetNDofs(),
+             vdim = tspec_.FESpace()->GetVDim();
+   MFEM_VERIFY(ndof == tspec.Size()/ncomp,
+               "Inconsistency in GetSerialDiscreteTargetSpec.");
+
+   for (int i = 0; i < ndof*vdim; i++)
+   {
+      tspec_(i) = tspec(i + idx*ndof);
+   }
+}
+
+void DiscreteAdaptTC::UpdateAfterMeshTopologyChange()
+{
+   tspec_fesv->Update();
+   tspec_gf->Update();
+   tspec.SetDataAndSize(tspec_gf->GetData(), tspec_gf->Size());
+   tspec_sav = tspec;
+
+   adapt_eval->SetSerialMetaInfo(*tspec_fesv->GetMesh(),
+                                 *tspec_fesv->FEColl(), ncomp);
+   adapt_eval->SetInitialField(*tspec_fesv->GetMesh()->GetNodes(), tspec);
 }

 void DiscreteAdaptTC::SetSerialDiscreteTargetSpec(const GridFunction &tspec_)
 {
   SetSerialDiscreteTargetSize(tspec_);
-   FinalizeSerialDiscreteTargetSpec();
 }


@@ -1507,7 +1557,7 @@ void DiscreteAdaptTC::UpdateTargetSpecificationAtNode(const FiniteElement &el,
   MFEM_VERIFY(tspec.Size() > 0, "Target specification is not set!");

   Array<int> dofs;
-   tspec_fes->GetElementDofs(T.ElementNo, dofs);
+   tspec_fesv->GetElementDofs(T.ElementNo, dofs);
   const int cnt = tspec.Size()/ncomp; // dofs per scalar-field

   for (int i = 0; i < ncomp; i++)
@@ -1522,7 +1572,7 @@ void DiscreteAdaptTC::RestoreTargetSpecificationAtNode(ElementTransformation &T,
   MFEM_VERIFY(tspec.Size() > 0, "Target specification is not set!");

   Array<int> dofs;
-   tspec_fes->GetElementDofs(T.ElementNo, dofs);
+   tspec_fesv->GetElementDofs(T.ElementNo, dofs);
   const int cnt = tspec.Size()/ncomp;
   for (int i = 0; i < ncomp; i++)
   {
@@ -1530,6 +1580,40 @@ void DiscreteAdaptTC::RestoreTargetSpecificationAtNode(ElementTransformation &T,
   }
 }

+void DiscreteAdaptTC::SetTspecFromIntRule(int e_id,
+                                          const IntegrationRule &intrule)
+{
+   switch (target_type)
+   {
+      case IDEAL_SHAPE_GIVEN_SIZE:
+      case GIVEN_SHAPE_AND_SIZE:
+      {
+         const int ndofs = tspec_fesv->GetFE(e_id)->GetDof(),
+                   ntspec_dofs = ndofs*ncomp;
+
+         Vector tspec_vals(ntspec_dofs);
+
+         Array<int> dofs;
+         tspec_fesv->GetElementVDofs(e_id, dofs);
+         tspec.GetSubVector(dofs, tspec_vals);
+         DenseMatrix tr;
+         tspec_gf->GetVectorValues(e_id, intrule, tspec_refine, tr);
+         tspec_refine.Transpose();
+         break;
+      }
+      default:
+         MFEM_ABORT("Incompatible target type for discrete adaptation!");
+   }
+}
+
+void DiscreteAdaptTC::SetTspecDataForDerefinement(FiniteElementSpace *fes)
+{
+   coarse_tspec_fesv = fes;
+   const Operator *c_op = fes->GetUpdateOperator();
+   tspec_derefine.SetSize(c_op->Height());
+   c_op->Mult(tspec, tspec_derefine);
+}
+
 void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
                                            const IntegrationRule &ir,
                                            const Vector &elfun,
@@ -1540,6 +1624,8 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
             nqp = ir.GetNPoints();
   Jtrcomp.SetSize(dim, dim, 4*nqp);

+   FiniteElementSpace *src_fes = tspec_fesv;
+
   switch (target_type)
   {
      case IDEAL_SHAPE_GIVEN_SIZE:
@@ -1548,7 +1634,7 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
         const DenseMatrix &Wideal =
            Geometries.GetGeomToPerfGeomJac(fe.GetGeomType());
         const int dim = Wideal.Height(),
-                   ndofs = tspec_fes->GetFE(e_id)->GetDof(),
+                   ndofs = tspec_fesv->GetFE(e_id)->GetDof(),
                   ntspec_dofs = ndofs*ncomp;

         Vector shape(ndofs), tspec_vals(ntspec_dofs), par_vals,
@@ -1559,11 +1645,29 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
         tspec_fesv->GetElementVDofs(e_id, dofs);
         tspec.UseDevice(true);
         tspec.GetSubVector(dofs, tspec_vals);
+         if (tspec_refine.NumCols() > 0) // Refinement
+         {
+            MFEM_VERIFY(amr_el >= 0, " Target being constructed for an AMR element.");
+            for (int i = 0; i < ncomp; i++)
+            {
+               for (int j = 0; j < ndofs; j++)
+               {
+                  tspec_vals(j + i*ndofs) = tspec_refine(j + amr_el*ndofs, i);
+               }
+            }
+         }
+         else if (tspec_derefine.Size() > 0) // Derefinement
+         {
+            dofs.SetSize(0);
+            coarse_tspec_fesv->GetElementVDofs(e_id, dofs);
+            tspec_derefine.GetSubVector(dofs, tspec_vals);
+            src_fes = coarse_tspec_fesv;
+         }

         for (int q = 0; q < nqp; q++)
         {
            const IntegrationPoint &ip = ir.IntPoint(q);
-            tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
+            src_fes->GetFE(e_id)->CalcShape(ip, shape);
            Jtr(q) = Wideal; // Initialize to identity
            for (int d = 0; d < 4; d++)
            {
@@ -1574,9 +1678,16 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
            if (sizeidx != -1) // Set size
            {
               par_vals.SetDataAndSize(tspec_vals.GetData()+sizeidx*ndofs, ndofs);
-               const double min_size = par_vals.Min();
-               MFEM_VERIFY(min_size > 0.0,
-                           "Non-positive size propagated in the target definition.");
+               double min_size = par_vals.Min();//0.001; //
+               if (lim_min_size > 0.)
+               {
+                  min_size = lim_min_size;
+               }
+               else
+               {
+                  MFEM_VERIFY(min_size > 0.0,
+                              "Non-positive size propagated in the target definition.");
+               }
               const double size = std::max(shape * par_vals, min_size);
               Jtr(q).Set(std::pow(size, 1.0/dim), Jtr(q));
               DenseMatrix Jtrcomp_q(Jtrcomp.GetData(0 + 4*q), dim, dim);
@@ -1591,6 +1702,9 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
               {
                  par_vals.SetDataAndSize(tspec_vals.GetData()+
                                          aspectratioidx*ndofs, ndofs);
+                  const double min_size = par_vals.Min();
+                  MFEM_VERIFY(min_size > 0.0,
+                              "Non-positive aspect-ratio propagated in the target definition.");

                  const double aspectratio = shape * par_vals;
                  D_rho = 0.;
@@ -1775,7 +1889,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,

               grad_phys.Mult(par_vals, grad_ptr_c1);
               Vector grad_q(dim);
-               tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
+               tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
               grad_e_c1.MultTranspose(shape, grad_q);

               const double min_size = par_vals.Min();
@@ -1808,7 +1922,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,

                  grad_phys.Mult(par_vals, grad_ptr_c1);
                  Vector grad_q(dim);
-                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
+                  tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
                  grad_e_c1.MultTranspose(shape, grad_q);

                  const double aspectratio = shape * par_vals;
@@ -1839,7 +1953,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,
                  grad_phys.Mult(par_vals_c2, grad_ptr_c2);
                  grad_phys.Mult(par_vals_c3, grad_ptr_c3);
                  Vector grad_q1(dim), grad_q2(dim), grad_q3(dim);
-                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
+                  tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
                  grad_e_c1.MultTranspose(shape, grad_q1);
                  grad_e_c2.MultTranspose(shape, grad_q2);
                  grad_e_c3.MultTranspose(shape, grad_q3);
@@ -1878,7 +1992,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,

                  grad_phys.Mult(par_vals, grad_ptr_c1);
                  Vector grad_q(dim);
-                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
+                  tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
                  grad_e_c1.MultTranspose(shape, grad_q);

                  const double skew = shape * par_vals;
@@ -1911,7 +2025,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,
                  grad_phys.Mult(par_vals_c2, grad_ptr_c2);
                  grad_phys.Mult(par_vals_c3, grad_ptr_c3);
                  Vector grad_q1(dim), grad_q2(dim), grad_q3(dim);
-                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
+                  tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
                  grad_e_c1.MultTranspose(shape, grad_q1);
                  grad_e_c2.MultTranspose(shape, grad_q2);
                  grad_e_c3.MultTranspose(shape, grad_q3);
@@ -1958,7 +2072,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,

                  grad_phys.Mult(par_vals, grad_ptr_c1);
                  Vector grad_q(dim);
-                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
+                  tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
                  grad_e_c1.MultTranspose(shape, grad_q);

                  const double theta = shape * par_vals;
@@ -1989,7 +2103,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,
                  grad_phys.Mult(par_vals_c2, grad_ptr_c2);
                  grad_phys.Mult(par_vals_c3, grad_ptr_c3);
                  Vector grad_q1(dim), grad_q2(dim), grad_q3(dim);
-                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
+                  tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
                  grad_e_c1.MultTranspose(shape, grad_q1);
                  grad_e_c2.MultTranspose(shape, grad_q2);
                  grad_e_c3.MultTranspose(shape, grad_q3);
@@ -2069,7 +2183,7 @@ void DiscreteAdaptTC::UpdateGradientTargetSpecification(const Vector &x,
 {
   if (use_flag && good_tspec_grad) { return; }

-   const int dim = tspec_fes->GetFE(0)->GetDim(),
+   const int dim = tspec_fesv->GetFE(0)->GetDim(),
             cnt = x.Size()/dim;

   tspec_pert1h.SetSize(x.Size()*ncomp);
@@ -2095,7 +2209,7 @@ void DiscreteAdaptTC::UpdateHessianTargetSpecification(const Vector &x,

   if (use_flag && good_tspec_hess) { return; }

-   const int dim    = tspec_fes->GetFE(0)->GetDim(),
+   const int dim    = tspec_fesv->GetFE(0)->GetDim(),
             cnt    = x.Size()/dim,
             totmix = 1+2*(dim-2);

@@ -2143,6 +2257,16 @@ void DiscreteAdaptTC::UpdateHessianTargetSpecification(const Vector &x,
   good_tspec_hess = use_flag;
 }

+DiscreteAdaptTC::~DiscreteAdaptTC()
+{
+   delete tspec_gf;
+   delete adapt_eval;
+   delete tspec_fesv;
+#ifdef MFEM_USE_MPI
+   delete ptspec_fesv;
+#endif
+}
+
 void AdaptivityEvaluator::SetSerialMetaInfo(const Mesh &m,
                                            const FiniteElementCollection &fec,
                                            int num_comp)
@@ -2256,6 +2380,7 @@ void TMOP_Integrator::EnableAdaptiveLimiting(const ParGridFunction &z0,
                                             AdaptivityEvaluator &ae)
 {
   zeta_0 = &z0;
+   pzeta_0 = &z0;
   delete zeta;
   zeta   = new GridFunction(z0);
   coeff_zeta = &coeff;
@@ -2268,6 +2393,33 @@ void TMOP_Integrator::EnableAdaptiveLimiting(const ParGridFunction &z0,
 }
 #endif

+void TMOP_Integrator::UpdateAfterMeshTopologyChange()
+{
+   if (zeta)
+   {
+      zeta->Update();
+      adapt_eval->SetSerialMetaInfo(*zeta->FESpace()->GetMesh(),
+                                    *zeta->FESpace()->FEColl(), 1);
+      adapt_eval->SetInitialField
+      (*zeta->FESpace()->GetMesh()->GetNodes(), *zeta);
+   }
+}
+
+#ifdef MFEM_USE_MPI
+void TMOP_Integrator::ParUpdateAfterMeshTopologyChange()
+{
+   if (zeta)
+   {
+      zeta->Update();
+      adapt_eval->SetParMetaInfo(*pzeta_0->ParFESpace()->GetParMesh(),
+                                 *pzeta_0->ParFESpace()->FEColl(), 1);
+      adapt_eval->SetInitialField
+      (*zeta->FESpace()->GetMesh()->GetNodes(), *zeta);
+   }
+}
+#endif
+
+
 double TMOP_Integrator::GetElementEnergy(const FiniteElement &el,
                                         ElementTransformation &T,
                                         const Vector &elfun)
@@ -2376,6 +2528,145 @@ double TMOP_Integrator::GetElementEnergy(const FiniteElement &el,

   return energy;
 }
+
+double TMOP_Integrator::GetRefinementElementEnergy(const FiniteElement &el,
+                                                   ElementTransformation &T,
+                                                   const Vector &elfun,
+                                                   const IntegrationRule &irule)
+{
+   int dof = el.GetDof(), dim = el.GetDim(),
+       NEsplit = elfun.Size() / (dof*dim), el_id = T.ElementNo;
+   double energy = 0.;
+
+   TargetConstructor *tc = const_cast<TargetConstructor *>(targetC);
+   DiscreteAdaptTC *dtc = dynamic_cast<DiscreteAdaptTC *>(tc);
+   // For DiscreteAdaptTC the GridFunctions used to set the targets must be
+   // mapped onto the fine elements.
+   if (dtc) { dtc->SetTspecFromIntRule(el_id, irule); }
+
+   for (int e = 0; e < NEsplit; e++)
+   {
+      DSh.SetSize(dof, dim);
+      Jrt.SetSize(dim);
+      Jpr.SetSize(dim);
+      Jpt.SetSize(dim);
+      Vector elfun_child(dof*dim);
+      for (int i = 0; i < dof; i++)
+      {
+         for (int d = 0; d < dim; d++)
+         {
+            // elfun is (xe1,xe2,...xen,ye1,ye2...yen) and has nodal coordinates
+            // for all the children element of the parent element being considered.
+            // So we must index and get (xek, yek) i.e. nodal coordinates for
+            // the fine element being considered.
+            elfun_child(i + d*dof) = elfun(i + e*dof + d*dof*NEsplit);
+         }
+      }
+      PMatI.UseExternalData(elfun_child.GetData(), dof, dim);
+
+      const IntegrationRule &ir = EnergyIntegrationRule(el);
+
+      double el_energy = 0;
+      DenseTensor Jtr(dim, dim, ir.GetNPoints());
+      if (dtc)
+      {
+         // This is used to index into the tspec vector inside DiscreteAdaptTC.
+         dtc->SetRefinementSubElement(e);
+      }
+      targetC->ComputeElementTargets(el_id, el, ir, elfun_child, Jtr);
+
+      // Define ref->physical transformation, wn a Coefficient is specified.
+      IsoparametricTransformation *Tpr = NULL;
+      if (coeff1 || coeff0)
+      {
+         Tpr = new IsoparametricTransformation;
+         Tpr->SetFE(&el);
+         Tpr->ElementNo = T.ElementNo;
+         Tpr->ElementType = ElementTransformation::ELEMENT;
+         Tpr->Attribute = T.Attribute;
+         Tpr->GetPointMat().Transpose(PMatI); // PointMat = PMatI^T
+      }
+
+      for (int i = 0; i < ir.GetNPoints(); i++)
+      {
+         const IntegrationPoint &ip = ir.IntPoint(i);
+         const DenseMatrix &Jtr_i = Jtr(i);
+         h_metric->SetTargetJacobian(Jtr_i);
+         CalcInverse(Jtr_i, Jrt);
+         const double weight = ip.weight * Jtr_i.Det();
+
+         el.CalcDShape(ip, DSh);
+         MultAtB(PMatI, DSh, Jpr);
+         Mult(Jpr, Jrt, Jpt);
+
+         double val = metric_normal * h_metric->EvalW(Jpt);
+         if (coeff1) { val *= coeff1->Eval(*Tpr, ip); }
+
+         el_energy += weight * val;
+         delete Tpr;
+      }
+      energy += el_energy;
+   }
+   energy /= NEsplit;
+
+   if (dtc) { dtc->ResetRefinementTspecData(); }
+
+   return energy;
+}
+
+double TMOP_Integrator::GetDerefinementElementEnergy(const FiniteElement &el,
+                                                     ElementTransformation &T,
+                                                     const Vector &elfun)
+{
+   int dof = el.GetDof(), dim = el.GetDim();
+   double energy = 0.;
+
+   DSh.SetSize(dof, dim);
+   Jrt.SetSize(dim);
+   Jpr.SetSize(dim);
+   Jpt.SetSize(dim);
+   PMatI.UseExternalData(elfun.GetData(), dof, dim);
+
+   const IntegrationRule &ir = EnergyIntegrationRule(el);
+
+   energy = 0.0;
+   DenseTensor Jtr(dim, dim, ir.GetNPoints());
+   targetC->ComputeElementTargets(T.ElementNo, el, ir, elfun, Jtr);
+
+   // Define ref->physical transformation, wn a Coefficient is specified.
+   IsoparametricTransformation *Tpr = NULL;
+   if (coeff1)
+   {
+      Tpr = new IsoparametricTransformation;
+      Tpr->SetFE(&el);
+      Tpr->ElementNo = T.ElementNo;
+      Tpr->ElementType = ElementTransformation::ELEMENT;
+      Tpr->Attribute = T.Attribute;
+      Tpr->GetPointMat().Transpose(PMatI); // PointMat = PMatI^T
+   }
+
+   for (int i = 0; i < ir.GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir.IntPoint(i);
+      const DenseMatrix &Jtr_i = Jtr(i);
+      h_metric->SetTargetJacobian(Jtr_i);
+      CalcInverse(Jtr_i, Jrt);
+      const double weight = ip.weight * Jtr_i.Det();
+
+      el.CalcDShape(ip, DSh);
+      MultAtB(PMatI, DSh, Jpr);
+      Mult(Jpr, Jrt, Jpt);
+
+      double val = metric_normal * h_metric->EvalW(Jpt);
+      if (coeff1) { val *= coeff1->Eval(*Tpr, ip); }
+
+      energy += weight * val;
+   }
+
+   delete Tpr;
+   return energy;
+}
+
 void TMOP_Integrator::AssembleElementVector(const FiniteElement &el,
                                            ElementTransformation &T,
                                            const Vector &elfun, Vector &elvect)
@@ -3037,7 +3328,7 @@ void TMOP_Integrator::ComputeMinJac(const Vector &x,
   dx = detv_avg_min / dxscale;
 }

-void TMOP_Integrator::UpdateAfterMeshChange(const Vector &new_x)
+void TMOP_Integrator::UpdateAfterMeshPositionChange(const Vector &new_x)
 {
   if (discr_tc)
   {
@@ -3166,6 +3457,32 @@ void TMOPComboIntegrator::AssembleElementGrad(const FiniteElement &el,
   }
 }

+double TMOPComboIntegrator::GetRefinementElementEnergy(const FiniteElement &el,
+                                                       ElementTransformation &T,
+                                                       const Vector &elfun,
+                                                       const IntegrationRule &irule)
+{
+   double energy= 0.0;
+   for (int i = 0; i < tmopi.Size(); i++)
+   {
+      energy += tmopi[i]->GetRefinementElementEnergy(el, T, elfun, irule);
+   }
+   return energy;
+}
+
+double TMOPComboIntegrator::GetDerefinementElementEnergy(
+   const FiniteElement &el,
+   ElementTransformation &T,
+   const Vector &elfun)
+{
+   double energy= 0.0;
+   for (int i = 0; i < tmopi.Size(); i++)
+   {
+      energy += tmopi[i]->GetDerefinementElementEnergy(el, T, elfun);
+   }
+   return energy;
+}
+
 void TMOPComboIntegrator::EnableNormalization(const GridFunction &x)
 {
   const int cnt = tmopi.Size();
@@ -371,6 +371,8 @@ public:
      AddQualityMetric(sh_metric, 1.-gamma_);
      AddQualityMetric(sz_metric, gamma_);
   }
+   virtual int Id() const { return 80; }
+   double GetGamma() const { return gamma; }

   virtual ~TMOP_Metric_080() { delete sh_metric; delete sz_metric; }
 };
@@ -590,6 +592,52 @@ public:
   virtual int Id() const { return 321; }
 };

+/// 3D barrier Shape+Size (VS) metric (polyconvex).
+class TMOP_Metric_332 : public TMOP_Combo_QualityMetric
+{
+protected:
+   double gamma;
+   TMOP_QualityMetric *sh_metric, *sz_metric;
+
+public:
+   TMOP_Metric_332(double gamma_) : gamma(gamma_),
+      sh_metric(new TMOP_Metric_302),
+      sz_metric(new TMOP_Metric_315)
+   {
+      // (1-gamma) mu_302 + gamma mu_315
+      AddQualityMetric(sh_metric, 1.-gamma_);
+      AddQualityMetric(sz_metric, gamma_);
+   }
+
+   virtual int Id() const { return 332; }
+   double GetGamma() const { return gamma; }
+
+   virtual ~TMOP_Metric_332() { delete sh_metric; delete sz_metric; }
+};
+
+/// 3D barrier Shape+Size (VS) metric (polyconvex).
+class TMOP_Metric_333 : public TMOP_Combo_QualityMetric
+{
+protected:
+   double gamma;
+   TMOP_QualityMetric *sh_metric, *sz_metric;
+
+public:
+   TMOP_Metric_333(double gamma_) : gamma(gamma_),
+      sh_metric(new TMOP_Metric_302),
+      sz_metric(new TMOP_Metric_316)
+   {
+      // (1-gamma) mu_302 + gamma mu_316
+      AddQualityMetric(sh_metric, 1.-gamma_);
+      AddQualityMetric(sz_metric, gamma_);
+   }
+
+   virtual int Id() const { return 333; }
+   double GetGamma() const { return gamma; }
+
+   virtual ~TMOP_Metric_333() { delete sh_metric; delete sz_metric; }
+};
+
 /// Shifted barrier form of 3D metric 16 (volume, ideal barrier metric), 3D
 class TMOP_Metric_352 : public TMOP_QualityMetric
 {
@@ -1009,14 +1057,31 @@ protected:
   // eta1(x+h,y), eta2(x+h,y) ... etan(x+h,y), eta1(x,y+h), eta2(x,y+h) ...
   // same for tspec_pert2h and tspec_pertmix.

+   // DenseMatrix to hold target_spec values for the (children of the)
+   // element being refined to consider for h-refinement.
+   DenseMatrix tspec_refine;
+   // Vector to hold the target_spec values for the coarse version of the
+   // current mesh. Used for derefinement decision with hr-adaptivity.
+   Vector tspec_derefine;
+
   // Components of Target Jacobian at each quadrature point of an element. This
   // is required for computation of the derivative using chain rule.
   mutable DenseTensor Jtrcomp;

   // Note: do not use the Nodes of this space as they may not be on the
   // positions corresponding to the values of tspec.
-   const FiniteElementSpace *tspec_fes;
-   const FiniteElementSpace *tspec_fesv;
+   FiniteElementSpace *tspec_fesv;         //owned
+   FiniteElementSpace *coarse_tspec_fesv;  //not owned, derefinement FESpace
+   GridFunction *tspec_gf;                 //owned, uses tspec and tspec_fes
+   // discrete adaptivity
+#ifdef MFEM_USE_MPI
+   ParFiniteElementSpace *ptspec_fesv;     //owned, needed for derefinement to
+   // get update operator.
+   ParGridFunction *tspec_pgf;             // similar to tspec_gf
+#endif
+
+   int amr_el;
+   double lim_min_size;

   // These flags can be used by outside functions to avoid recomputing the
   // tspec and tspec_perth fields again on the same mesh.
@@ -1028,7 +1093,7 @@ protected:

   void SetDiscreteTargetBase(const GridFunction &tspec_);
   void SetTspecAtIndex(int idx, const GridFunction &tspec_);
-   void FinalizeSerialDiscreteTargetSpec();
+   void FinalizeSerialDiscreteTargetSpec(const GridFunction &tspec_);
 #ifdef MFEM_USE_MPI
   void SetTspecAtIndex(int idx, const ParGridFunction &tspec_);
   void FinalizeParDiscreteTargetSpec(const ParGridFunction &tspec_);
@@ -1040,16 +1105,16 @@ public:
        ncomp(0),
        sizeidx(-1), skewidx(-1), aspectratioidx(-1), orientationidx(-1),
        tspec(), tspec_sav(), tspec_pert1h(), tspec_pert2h(), tspec_pertmix(),
-        tspec_fes(NULL), tspec_fesv(NULL),
+        tspec_refine(), tspec_derefine(),
+        tspec_fesv(NULL), coarse_tspec_fesv(NULL), tspec_gf(NULL),
+#ifdef MFEM_USE_MPI
+        ptspec_fesv(NULL), tspec_pgf(NULL),
+#endif
+        amr_el(-1), lim_min_size(-0.1),
        good_tspec(false), good_tspec_grad(false), good_tspec_hess(false),
        adapt_eval(NULL) { }

-   virtual ~DiscreteAdaptTC()
-   {
-      delete adapt_eval;
-      delete tspec_fes;
-      delete tspec_fesv;
-   }
+   virtual ~DiscreteAdaptTC();

   /** @name Target specification methods.
       The following methods are used to specify geometric parameters of the
@@ -1080,6 +1145,20 @@ public:
   void ResetUpdateFlags()
   { good_tspec = good_tspec_grad = good_tspec_hess = false; }

+   /// Get one of the discrete fields from tspec.
+   void GetDiscreteTargetSpec(GridFunction &tspec_, int idx);
+   /// Get the FESpace associated with tspec.
+   FiniteElementSpace *GetTSpecFESpace() { return tspec_fesv; }
+   /// Get the entire tspec.
+   GridFunction *GetTSpecData() { return tspec_gf; }
+   /// Update all discrete fields based on tspec and update for AMR
+   void UpdateAfterMeshTopologyChange();
+
+#ifdef MFEM_USE_MPI
+   ParFiniteElementSpace *GetTSpecParFESpace() { return ptspec_fesv; }
+   void ParUpdateAfterMeshTopologyChange();
+#endif
+
   /** Used to update the target specification after the mesh has changed. The
       new mesh positions are given by new_x. If @a use_flags is true, repeated
       calls won't do anything until ResetUpdateFlags() is called. */
@@ -1136,6 +1215,36 @@ public:
                                              const Vector &elfun,
                                              IsoparametricTransformation &Tpr,
                                              DenseTensor &dJtr) const;
+
+   // Generates tspec_vals for target construction using intrule
+   // Used for the refinement component in hr-adaptivity.
+   void SetTspecFromIntRule(int e_id, const IntegrationRule &intrule);
+
+   // Targets based on discrete functions can result in invalid (negative)
+   // size at the quadrature points. This method can be used to set a
+   // minimum target size.
+   void SetMinSizeForTargets(double min_size_) { lim_min_size = min_size_; }
+
+   /// Computes target specification data with respect to the coarse FE space.
+   void SetTspecDataForDerefinement(FiniteElementSpace *fes);
+
+   // Reset refinement data associated with h-adaptivity component.
+   void ResetRefinementTspecData()
+   {
+      tspec_refine.Clear();
+      amr_el = -1;
+   }
+
+   // Reset derefinement data associated with h-adaptivity component.
+   void ResetDerefinementTspecData()
+   {
+      tspec_derefine.Destroy();
+      coarse_tspec_fesv = NULL;
+   }
+
+   // Used to specify the fine element for determining energy of children of a
+   // parent element.
+   void SetRefinementSubElement(int amr_el_) { amr_el = amr_el_; }
 };

 class TMOPNewtonSolver;
@@ -1153,6 +1262,7 @@ protected:
   friend class TMOPNewtonSolver;
   friend class TMOPComboIntegrator;

+   TMOP_QualityMetric *h_metric;
   TMOP_QualityMetric *metric;        // not owned
   const TargetConstructor *targetC;  // not owned

@@ -1179,6 +1289,9 @@ protected:

   // Adaptive limiting.
   const GridFunction *zeta_0;       // Not owned.
+#ifdef MFEM_USE_MPI
+   const ParGridFunction *pzeta_0;
+#endif
   GridFunction *zeta;               // Owned. Updated by adapt_eval.
   Coefficient *coeff_zeta;          // Not owned.
   AdaptivityEvaluator *adapt_eval;  // Not owned.
@@ -1289,7 +1402,7 @@ protected:
 #endif
   void ComputeMinJac(const Vector &x, const FiniteElementSpace &fes);

-   void UpdateAfterMeshChange(const Vector &new_x);
+   void UpdateAfterMeshPositionChange(const Vector &new_x);

   void DisableLimiting()
   {
@@ -1347,11 +1460,13 @@ protected:
   void ComputeAllElementTargets(const Vector &xe = Vector()) const;

 public:
-   /** @param[in] m  TMOP_QualityMetric that will be integrated (not owned).
-       @param[in] tc Target-matrix construction algorithm to use (not owned). */
-   TMOP_Integrator(TMOP_QualityMetric *m, TargetConstructor *tc)
-      : metric(m), targetC(tc), IntegRules(NULL), integ_order(-1),
-        coeff1(NULL), metric_normal(1.0),
+   /** @param[in] m    TMOP_QualityMetric for r-adaptivity (not owned).
+       @param[in] tc   Target-matrix construction algorithm to use (not owned).
+       @param[in] hm   TMOP_QualityMetric for h-adaptivity (not owned). */
+   TMOP_Integrator(TMOP_QualityMetric *m, TargetConstructor *tc,
+                   TMOP_QualityMetric *hm)
+      : h_metric(hm), metric(m), targetC(tc), IntegRules(NULL),
+        integ_order(-1), coeff1(NULL), metric_normal(1.0),
        nodes0(NULL), coeff0(NULL),
        lim_dist(NULL), lim_func(NULL), lim_normal(1.0),
        zeta_0(NULL), zeta(NULL), coeff_zeta(NULL), adapt_eval(NULL),
@@ -1359,6 +1474,9 @@ public:
        fdflag(false), dxscale(1.0e3), fd_call_flag(false), exact_action(false)
   { PA.enabled = false; }

+   TMOP_Integrator(TMOP_QualityMetric *m, TargetConstructor *tc)
+      : TMOP_Integrator(m, tc, m) { }
+
   ~TMOP_Integrator();

   /// Release the device memory of large PA allocations. This will copy device
@@ -1430,6 +1548,22 @@ public:
                                   ElementTransformation &T,
                                   const Vector &elfun);

+   /** @brief Computes the mean of the energies of the given element's children.
+
+       In addition to the inputs for GetElementEnergy, this function requires an
+       IntegrationRule to be specified that will give the decomposition of the
+       given element based on the refinement type being considered. */
+   virtual double GetRefinementElementEnergy(const FiniteElement &el,
+                                             ElementTransformation &T,
+                                             const Vector &elfun,
+                                             const IntegrationRule &irule);
+
+   /// This function is similar to GetElementEnergy, but ignores components
+   /// such as limiting etc. to compute the element energy.
+   virtual double GetDerefinementElementEnergy(const FiniteElement &el,
+                                               ElementTransformation &T,
+                                               const Vector &elfun);
+
   virtual void AssembleElementVector(const FiniteElement &el,
                                      ElementTransformation &T,
                                      const Vector &elfun, Vector &elvect);
@@ -1438,6 +1572,13 @@ public:
                                    ElementTransformation &T,
                                    const Vector &elfun, DenseMatrix &elmat);

+   TMOP_QualityMetric &GetAMRQualityMetric() { return *h_metric; }
+
+   void UpdateAfterMeshTopologyChange();
+#ifdef MFEM_USE_MPI
+   void ParUpdateAfterMeshTopologyChange();
+#endif
+
   // PA extension
   using NonlinearFormIntegrator::AssemblePA;
   virtual void AssemblePA(const FiniteElementSpace&);
@@ -1516,6 +1657,15 @@ public:
                                    ElementTransformation &T,
                                    const Vector &elfun, DenseMatrix &elmat);

+   virtual double GetRefinementElementEnergy(const FiniteElement &el,
+                                             ElementTransformation &T,
+                                             const Vector &elfun,
+                                             const IntegrationRule &irule);
+
+   virtual double GetDerefinementElementEnergy(const FiniteElement &el,
+                                               ElementTransformation &T,
+                                               const Vector &elfun);
+
   /// Normalization factor that considers all integrators in the combination.
   void EnableNormalization(const GridFunction &x);
 #ifdef MFEM_USE_MPI
@@ -150,9 +150,49 @@ void EvalH_077(const int e, const int qx, const int qy,
   }
 }

+static MFEM_HOST_DEVICE inline
+void EvalH_080(const int e, const int qx, const int qy,
+               const double weight, const double gamma, const double *Jpt,
+               DeviceTensor<7,double> H)
+{
+   // h_80 = (1-gamma) h_2 + gamma h_77.
+
+   constexpr int DIM = 2;
+   double ddI1[4], ddI1b[4], dI2[4], dI2b[4], ddI2[4];
+   kernels::InvariantsEvaluator2D ie(Args()
+                                     .J(Jpt)
+                                     .dI2(dI2)
+                                     .ddI1(ddI1)
+                                     .ddI1b(ddI1b)
+                                     .dI2b(dI2b)
+                                     .ddI2(ddI2));
+
+   const double I2 = ie.Get_I2(), I2inv_sq = 1.0 / (I2 * I2);
+   ConstDeviceMatrix di2(ie.Get_dI2(),DIM,DIM);
+   for (int i = 0; i < DIM; i++)
+   {
+      for (int j = 0; j < DIM; j++)
+      {
+         ConstDeviceMatrix ddi1b(ie.Get_ddI1b(i,j),DIM,DIM);
+         ConstDeviceMatrix ddi2(ie.Get_ddI2(i,j),DIM,DIM);
+         for (int r = 0; r < DIM; r++)
+         {
+            for (int c = 0; c < DIM; c++)
+            {
+               H(r,c,i,j,qx,qy,e) =
+                  (1.0 - gamma) * 0.5 * weight * ddi1b(r,c) +
+                  gamma * ( weight * 0.5 * (1.0 - I2inv_sq) * ddi2(r,c) +
+                            weight * (I2inv_sq / I2) * di2(r,c) * di2(i,j) );
+            }
+         }
+      }
+   }
+}
+
 MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_2D,
                           const Vector &x_,
                           const double metric_normal,
+                           const double metric_param,
                           const int mid,
                           const int NE,
                           const Array<double> &w_,
@@ -163,7 +203,7 @@ MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_2D,
                           const int d1d,
                           const int q1d)
 {
-   MFEM_VERIFY(mid == 1 || mid == 2 || mid == 7 || mid == 77,
+   MFEM_VERIFY(mid == 1 || mid == 2 || mid == 7 || mid == 77 || mid == 80,
               "Metric not yet implemented!");

   constexpr int DIM = 2;
@@ -222,6 +262,7 @@ MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_2D,
            if (mid ==  2) { EvalH_002(e,qx,qy,weight,Jpt,H); }
            if (mid ==  7) { EvalH_007(e,qx,qy,weight,Jpt,H); }
            if (mid == 77) { EvalH_077(e,qx,qy,weight,Jpt,H); }
+            if (mid == 80) { EvalH_080(e,qx,qy,weight,metric_param,Jpt,H); }
         } // qx
      } // qy
   });
@@ -241,7 +282,10 @@ void TMOP_Integrator::AssembleGradPA_2D(const Vector &X) const
   const Array<double> &G = PA.maps->G;
   Vector &H = PA.H;

-   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_2D,id,X,mn,M,N,W,B,G,J,H);
+   double mp = 0.0;
+   if (auto m = dynamic_cast<TMOP_Metric_080 *>(metric)) { mp = m->GetGamma(); }
+
+   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_2D,id,X,mn,mp,M,N,W,B,G,J,H);
 }

 } // namespace mfem
@@ -181,8 +181,58 @@ void EvalH_321(const int e, const int qx, const int qy, const int qz,
   }
 }

+// H_332 = (1-gamma) H_302 + gamma H_315
+static MFEM_HOST_DEVICE inline
+void EvalH_332(const int e, const int qx, const int qy, const int qz,
+               const double weight, const double gamma,
+               const double *J, DeviceTensor<8,double> dP)
+{
+   double B[9];
+   double         dI1b[9],          ddI1b[9];
+   double dI2[9], dI2b[9], ddI2[9], ddI2b[9];
+   double         dI3b[9],          ddI3b[9];
+   constexpr int DIM = 3;
+   kernels::InvariantsEvaluator3D ie(Args()
+                                     .J(J).B(B)
+                                     .dI1b(dI1b).ddI1b(ddI1b)
+                                     .dI2(dI2).dI2b(dI2b).ddI2(ddI2).ddI2b(ddI2b)
+                                     .dI3b(dI3b).ddI3b(ddI3b));
+   double sign_detJ;
+   const double c1 = weight/9.;
+   const double I1b = ie.Get_I1b();
+   const double I2b = ie.Get_I2b();
+   const double I3b = ie.Get_I3b(sign_detJ);
+   ConstDeviceMatrix di1b(ie.Get_dI1b(),DIM,DIM);
+   ConstDeviceMatrix di2b(ie.Get_dI2b(),DIM,DIM);
+   ConstDeviceMatrix di3b(ie.Get_dI3b(sign_detJ),DIM,DIM);
+   for (int i = 0; i < DIM; i++)
+   {
+      for (int j = 0; j < DIM; j++)
+      {
+         ConstDeviceMatrix ddi1b(ie.Get_ddI1b(i,j),DIM,DIM);
+         ConstDeviceMatrix ddi2b(ie.Get_ddI2b(i,j),DIM,DIM);
+         ConstDeviceMatrix ddi3b(ie.Get_ddI3b(i,j),DIM,DIM);
+         for (int r = 0; r < DIM; r++)
+         {
+            for (int c = 0; c < DIM; c++)
+            {
+               const double dp_302 =
+                  (di2b(r,c)*di1b(i,j) + di1b(r,c)*di2b(i,j))
+                  + ddi2b(r,c)*I1b
+                  + ddi1b(r,c)*I2b;
+               const double dp_315 = 2.0 * weight * (I3b - 1.0) * ddi3b(r,c) +
+                                     2.0 * weight * di3b(r,c) * di3b(i,j);
+               dP(r,c,i,j,qx,qy,qz,e) = (1.0 - gamma) * c1 * dp_302 +
+                                        gamma * dp_315;
+            }
+         }
+      }
+   }
+}
+
 MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_3D,
                           const double metric_normal,
+                           const double metric_param,
                           const int mid,
                           const Vector &x_,
                           const int NE,
@@ -194,8 +244,8 @@ MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_3D,
                           const int d1d,
                           const int q1d)
 {
-   MFEM_VERIFY(mid == 302 || mid == 303 || mid == 315 || mid == 321 ,
-               "3D metric not yet implemented!");
+   MFEM_VERIFY(mid == 302 || mid == 303 || mid == 315 ||
+               mid == 321 || mid == 332, "3D metric not yet implemented!");

   constexpr int DIM = 3;
   const int D1D = T_D1D ? T_D1D : d1d;
@@ -255,6 +305,7 @@ MFEM_REGISTER_TMOP_KERNELS(void, SetupGradPA_3D,
               if (mid == 303) { EvalH_303(e,qx,qy,qz,weight,Jpt,H); }
               if (mid == 315) { EvalH_315(e,qx,qy,qz,weight,Jpt,H); }
               if (mid == 321) { EvalH_321(e,qx,qy,qz,weight,Jpt,H); }
+               if (mid == 332) { EvalH_332(e,qx,qy,qz,weight,metric_param,Jpt,H); }
            } // qx
         } // qy
      } // qz
@@ -275,7 +326,10 @@ void TMOP_Integrator::AssembleGradPA_3D(const Vector &X) const
   const Array<double> &G = PA.maps->G;
   Vector &H = PA.H;

-   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_3D,id,mn,M,X,N,W,B,G,J,H);
+   double mp = 0.0;
+   if (auto m = dynamic_cast<TMOP_Metric_332 *>(metric)) { mp = m->GetGamma(); }
+
+   MFEM_LAUNCH_TMOP_KERNEL(SetupGradPA_3D,id,mn,mp,M,X,N,W,B,G,J,H);
 }

 } // namespace mfem
@@ -58,8 +58,24 @@ void EvalP_077(const double *Jpt, double *P)
   kernels::Set(2,2, 0.5 * (1.0 - 1.0 / (I2 * I2)), ie.Get_dI2(), P);
 }

+static MFEM_HOST_DEVICE inline
+void EvalP_080(const double *Jpt, double gamma, double *P)
+{
+   // p_80 = (1-gamma) p_2 + gamma p_77.
+
+   double dI1b[4], dI2[4], dI2b[4];
+   kernels::InvariantsEvaluator2D ie(Args().J(Jpt).
+                                     dI1b(dI1b).dI2(dI2).dI2b(dI2b));
+
+   kernels::Set(2,2, (1.0 - gamma) * 1./2., ie.Get_dI1b(), P);
+
+   const double I2 = ie.Get_I2();
+   kernels::Add(2,2, gamma * 0.5 * (1.0 - 1.0 / (I2 * I2)), ie.Get_dI2(), P);
+}
+
 MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_2D,
                           const double metric_normal,
+                           const double metric_param,
                           const int mid,
                           const int NE,
                           const DenseTensor &j_,
@@ -71,7 +87,7 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_2D,
                           const int d1d,
                           const int q1d)
 {
-   MFEM_VERIFY(mid == 1 || mid == 2 || mid == 7 || mid == 77,
+   MFEM_VERIFY(mid == 1 || mid == 2 || mid == 7 || mid == 77 || mid == 80,
               "Metric not yet implemented!");

   constexpr int DIM = 2;
@@ -132,6 +148,7 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_2D,
            if (mid ==  2) { EvalP_002(Jpt, P); }
            if (mid ==  7) { EvalP_007(Jpt, P); }
            if (mid == 77) { EvalP_077(Jpt, P); }
+            if (mid == 80) { EvalP_080(Jpt, metric_param, P); }
            for (int i = 0; i < 4; i++) { P[i] *= weight; }

            // PMatO += DS . P^t += DSh . (Jrt . P^t)
@@ -160,7 +177,10 @@ void TMOP_Integrator::AddMultPA_2D(const Vector &X, Vector &Y) const
   const Array<double> &G = PA.maps->G;
   const double mn = metric_normal;

-   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_2D,id,mn,M,N,J,W,B,G,X,Y);
+   double mp = 0.0;
+   if (auto m = dynamic_cast<TMOP_Metric_080 *>(metric)) { mp = m->GetGamma(); }
+
+   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_2D,id,mn,mp,M,N,J,W,B,G,X,Y);
 }

 } // namespace mfem
@@ -75,8 +75,29 @@ void EvalP_321(const double *J, double *P)
   kernels::Add(3,3, ie.Get_dI1(), P);
 }

+// P_332 = (1-gamma) P_302 + gamma P_315.
+static MFEM_HOST_DEVICE inline
+void EvalP_332(const double *J, double gamma, double *P)
+{
+   double B[9];
+   double dI1b[9], dI2[9], dI2b[9], dI3b[9];
+   kernels::InvariantsEvaluator3D ie(Args()
+                                     .J(J).B(B)
+                                     .dI1b(dI1b)
+                                     .dI2(dI2).dI2b(dI2b)
+                                     .dI3b(dI3b));
+   const double alpha = (1.0 - gamma) * ie.Get_I1b()/9.;
+   const double beta = (1.0 - gamma) * ie.Get_I2b()/9.;
+   kernels::Add(3,3, alpha, ie.Get_dI2b(), beta, ie.Get_dI1b(), P);
+
+   double sign_detJ;
+   const double I3b = ie.Get_I3b(sign_detJ);
+   kernels::Add(3,3, gamma * 2.0 * (I3b - 1.0), ie.Get_dI3b(sign_detJ), P);
+}
+
 MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_3D,
                           const double metric_normal,
+                           double metric_param,
                           const int mid,
                           const int NE,
                           const DenseTensor &j_,
@@ -88,8 +109,8 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_3D,
                           const int d1d,
                           const int q1d)
 {
-   MFEM_VERIFY(mid == 302 || mid == 303 || mid == 315 || mid == 321 ,
-               "3D metric not yet implemented!");
+   MFEM_VERIFY(mid == 302 || mid == 303 || mid == 315 ||
+               mid == 321 || mid == 332, "3D metric not yet implemented!");

   constexpr int DIM = 3;
   const int D1D = T_D1D ? T_D1D : d1d;
@@ -146,10 +167,11 @@ MFEM_REGISTER_TMOP_KERNELS(void, AddMultPA_Kernel_3D,

               // metric->EvalP(Jpt, P);
               double P[9];
-               if (mid == 302) { EvalP_302(Jpt,P); }
-               if (mid == 303) { EvalP_303(Jpt,P); }
-               if (mid == 315) { EvalP_315(Jpt,P); }
-               if (mid == 321) { EvalP_321(Jpt,P); }
+               if (mid == 302) { EvalP_302(Jpt, P); }
+               if (mid == 303) { EvalP_303(Jpt, P); }
+               if (mid == 315) { EvalP_315(Jpt, P); }
+               if (mid == 321) { EvalP_321(Jpt, P); }
+               if (mid == 332) { EvalP_332(Jpt, metric_param, P); }
               for (int i = 0; i < 9; i++) { P[i] *= weight; }

               // Y += DS . P^t += DSh . (Jrt . P^t)
@@ -180,7 +202,10 @@ void TMOP_Integrator::AddMultPA_3D(const Vector &X, Vector &Y) const
   const Array<double> &G = PA.maps->G;
   const double mn = metric_normal;

-   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_3D,id,mn,M,N,J,W,B,G,X,Y);
+   double mp = 0.0;
+   if (auto m = dynamic_cast<TMOP_Metric_332 *>(metric)) { mp = m->GetGamma(); }
+
+   MFEM_LAUNCH_TMOP_KERNEL(AddMultPA_Kernel_3D,id,mn,mp,M,N,J,W,B,G,X,Y);
 }

 } // namespace mfem
@@ -50,8 +50,15 @@ double EvalW_077(const double *Jpt)
   return 0.5*(I2b*I2b + 1./(I2b*I2b) - 2.);
 }

+static MFEM_HOST_DEVICE inline
+double EvalW_080(const double *Jpt, double gamma)
+{
+   return (1.0 - gamma) * EvalW_002(Jpt) + gamma * EvalW_077(Jpt);
+}
+
 MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_2D,
                           const double metric_normal,
+                           const double metric_param,
                           const int mid,
                           const int NE,
                           const DenseTensor &j_,
@@ -64,7 +71,7 @@ MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_2D,
                           const int d1d,
                           const int q1d)
 {
-   MFEM_VERIFY(mid == 1 || mid == 2 || mid == 7 || mid == 77,
+   MFEM_VERIFY(mid == 1 || mid == 2 || mid == 7 || mid == 77 || mid == 80,
               "2D metric not yet implemented!");

   constexpr int DIM = 2;
@@ -125,7 +132,8 @@ MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_2D,
            mid ==  1 ? EvalW_001(Jpt) :
            mid ==  2 ? EvalW_002(Jpt) :
            mid ==  7 ? EvalW_007(Jpt) :
-            mid == 77 ? EvalW_077(Jpt) : 0.0;
+            mid == 77 ? EvalW_077(Jpt) :
+            mid == 80 ? EvalW_080(Jpt, metric_param) : 0.0;

            E(qx,qy,e) = weight * EvalW;
         }
@@ -141,7 +149,7 @@ double TMOP_Integrator::GetLocalStateEnergyPA_2D(const Vector &X) const
   const int D1D = PA.maps->ndof;
   const int Q1D = PA.maps->nqpt;
   const int id = (D1D << 4 ) | Q1D;
-   const double m = metric_normal;
+   const double mn = metric_normal;
   const DenseTensor &J = PA.Jtr;
   const Array<double> &W = PA.ir->GetWeights();
   const Array<double> &B = PA.maps->B;
@@ -149,7 +157,10 @@ double TMOP_Integrator::GetLocalStateEnergyPA_2D(const Vector &X) const
   const Vector &O = PA.O;
   Vector &E = PA.E;

-   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_2D,id,m,M,N,J,W,B,G,X,O,E);
+   double mp = 0.0;
+   if (auto m = dynamic_cast<TMOP_Metric_080 *>(metric)) { mp = m->GetGamma(); }
+
+   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_2D,id,mn,mp,M,N,J,W,B,G,X,O,E);
 }

 } // namespace mfem
@@ -58,8 +58,15 @@ double EvalW_321(const double *J)
   return ie.Get_I1() + ie.Get_I2()/ie.Get_I3() - 6.0;
 }

+static MFEM_HOST_DEVICE inline
+double EvalW_332(const double *J, double gamma)
+{
+   return (1.0 - gamma) * EvalW_302(J) + gamma * EvalW_315(J);
+}
+
 MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_3D,
                           const double metric_normal,
+                           const double metric_param,
                           const int mid,
                           const int NE,
                           const DenseTensor &j_,
@@ -72,8 +79,8 @@ MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_3D,
                           const int d1d,
                           const int q1d)
 {
-   MFEM_VERIFY(mid == 302 || mid == 303 || mid == 315 || mid == 321 ,
-               "3D metric not yet implemented!");
+   MFEM_VERIFY(mid == 302 || mid == 303 || mid == 315 ||
+               mid == 321 || mid == 332, "3D metric not yet implemented!");

   constexpr int DIM = 3;
   const int D1D = T_D1D ? T_D1D : d1d;
@@ -134,7 +141,8 @@ MFEM_REGISTER_TMOP_KERNELS(double, EnergyPA_3D,
               mid == 302 ? EvalW_302(Jpt) :
               mid == 303 ? EvalW_303(Jpt) :
               mid == 315 ? EvalW_315(Jpt) :
-               mid == 321 ? EvalW_321(Jpt) : 0.0;
+               mid == 321 ? EvalW_321(Jpt) :
+               mid == 332 ? EvalW_332(Jpt, metric_param) : 0.0;

               E(qx,qy,qz,e) = weight * EvalW;
            }
@@ -159,7 +167,10 @@ double TMOP_Integrator::GetLocalStateEnergyPA_3D(const Vector &X) const
   const Vector &O = PA.O;
   Vector &E = PA.E;

-   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_3D,id,mn,M,N,J,W,B,G,O,X,E);
+   double mp = 0.0;
+   if (auto m = dynamic_cast<TMOP_Metric_332 *>(metric)) { mp = m->GetGamma(); }
+
+   MFEM_LAUNCH_TMOP_KERNEL(EnergyPA_3D,id,mn,mp,M,N,J,W,B,G,O,X,E);
 }

 } // namespace mfem
@@ -0,0 +1,896 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop_amr.hpp"
+
+namespace mfem
+{
+
+using namespace mfem;
+
+void TMOPRefinerEstimator::ComputeEstimates()
+{
+   bool iso = false;
+   bool aniso = false;
+   if (amrmetric == 1 || amrmetric == 2 || amrmetric == 58)
+   {
+      aniso = true;
+   }
+   if (amrmetric == 55 || amrmetric == 56 || amrmetric == 77 ||
+       amrmetric == 315 || amrmetric == 316 || amrmetric == 321)
+   {
+      iso = true;
+   }
+   if (amrmetric == 7 || amrmetric == 9)
+   {
+      iso = true; aniso = true;
+   }
+
+   MFEM_VERIFY(iso || aniso, "Metric type not supported in hr-adaptivity.");
+
+   const int dim = mesh->Dimension();
+   const int num_ref_types = 3 + 4*(dim-2);
+   const int NEorig = mesh->GetNE();
+
+   aniso_flags.SetSize(NEorig);
+   error_estimates.SetSize(NEorig);
+   Vector amr_base_energy(NEorig), amr_temp_energy(NEorig);
+   error_estimates = 1.*std::numeric_limits<float>::max();
+   aniso_flags = -1;
+   GetTMOPRefinementEnergy(0, amr_base_energy);
+
+   for (int i = 1; i < num_ref_types+1; i++)
+   {
+      if ( dim == 2 && i < 3  && aniso != true ) { continue; }
+      if ( dim == 2 && i == 3 && iso   != true ) { continue; }
+      if ( dim == 3 && i < 7  && aniso != true ) { continue; }
+      if ( dim == 3 && i == 7 && iso   != true ) { continue; }
+
+      GetTMOPRefinementEnergy(i, amr_temp_energy);
+
+      for (int e = 0; e < NEorig; e++)
+      {
+         if ( amr_temp_energy(e) < error_estimates(e) )
+         {
+            error_estimates(e) = amr_temp_energy(e);
+            aniso_flags[e] = i;
+         }
+      }
+   }
+   error_estimates *= energy_scaling_factor;
+
+   if (spat_gf)
+   {
+      L2_FECollection avg_fec(0, mesh->Dimension());
+      FiniteElementSpace avg_fes(spat_gf->FESpace()->GetMesh(), &avg_fec);
+      GridFunction elem_avg(&avg_fes);
+      spat_gf->GetElementAverages(elem_avg);
+      for (int i = 0; i < amr_base_energy.Size(); i++)
+      {
+         if (elem_avg(i) < spat_gf_critical) { amr_base_energy(i) = 0.; }
+      }
+   }
+
+   error_estimates -= amr_base_energy;
+   error_estimates *= -1; // error = E(parent) - scaling_factor*mean(E(children))
+   current_sequence = mesh->GetSequence();
+}
+
+void TMOPRefinerEstimator::GetTMOPRefinementEnergy(int reftype,
+                                                   Vector &el_energy_vec)
+{
+   const FiniteElementSpace *fes = mesh->GetNodalFESpace();
+   const int NE = fes->GetNE();
+   GridFunction *xdof = mesh->GetNodes();
+   xdof->SetTrueVector();
+   xdof->SetFromTrueVector();
+
+   el_energy_vec.SetSize(NE);
+   el_energy_vec = std::numeric_limits<float>::max();
+
+   for (int e = 0; e < NE; e++)
+   {
+      Geometry::Type gtype = fes->GetFE(e)->GetGeomType();
+      DenseMatrix tr, xsplit;
+      IntegrationRule *irule = NULL;
+
+      if ( (gtype == Geometry::TRIANGLE && reftype > 0 && reftype < 3) ||
+           (gtype == Geometry::CUBE && reftype > 0 && reftype < 7) ||
+           (gtype == Geometry::TETRAHEDRON && reftype > 0 && reftype < 7) )
+      {
+         continue;
+      }
+
+      switch (gtype)
+      {
+         case Geometry::TRIANGLE:
+         {
+            int ref_access = reftype == 0 ? 0 : 1;
+            xdof->GetVectorValues(e, *TriIntRule[ref_access], xsplit, tr);
+            irule = TriIntRule[ref_access];
+            break;
+         }
+         case Geometry::TETRAHEDRON:
+         {
+            int ref_access = reftype == 0 ? 0 : 1;
+            xdof->GetVectorValues(e, *TetIntRule[ref_access], xsplit, tr);
+            irule = TetIntRule[ref_access];
+            break;
+         }
+         case Geometry::SQUARE:
+         {
+            MFEM_VERIFY(QuadIntRule[reftype], " Integration rule does not exist.");
+            xdof->GetVectorValues(e, *QuadIntRule[reftype], xsplit, tr);
+            irule = QuadIntRule[reftype];
+            break;
+         }
+         case Geometry::CUBE:
+         {
+            int ref_access = reftype == 0 ? 0 : 1;
+            xdof->GetVectorValues(e, *HexIntRule[ref_access], xsplit, tr);
+            irule = HexIntRule[ref_access];
+            break;
+         }
+         default:
+            MFEM_ABORT("Incompatible geometry type!");
+      }
+      xsplit.Transpose();
+
+      el_energy_vec(e) = 0.; // Re-set to 0
+
+      // The data format is xe1,xe2,..xen,ye1,ye2..yen.
+      // We will reformat it inside GetRefinementElementEnergy
+      Vector elfun(xsplit.GetData(), xsplit.NumCols()*xsplit.NumRows());
+
+      Array<NonlinearFormIntegrator*> &integs = *(nlf->GetDNFI());
+      TMOP_Integrator *ti  = NULL;
+      TMOPComboIntegrator *co = NULL;
+      for (int i = 0; i < integs.Size(); i++)
+      {
+         ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
+         if (ti)
+         {
+            el_energy_vec(e) = ti->GetRefinementElementEnergy(*fes->GetFE(e),
+                                                              *mesh->GetElementTransformation(e),
+                                                              elfun,
+                                                              *irule);
+         }
+         co = dynamic_cast<TMOPComboIntegrator *>(integs[i]);
+         if (co)
+         {
+            Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
+            for (int j = 0; j < ati.Size(); j++)
+            {
+               el_energy_vec(e) += ati[j]->GetRefinementElementEnergy(*fes->GetFE(e),
+                                                                      *mesh->GetElementTransformation(e),
+                                                                      elfun,
+                                                                      *irule);
+            }
+         }
+      }
+   }
+}
+
+void TMOPRefinerEstimator::SetHexIntRules()
+{
+   HexIntRule.SetSize(1+1);
+   // Reftype = 0 -> original element
+   Mesh meshsplit = Mesh::MakeCartesian3D(1, 1, 1, Element::HEXAHEDRON);
+   Mesh base_mesh_copy(meshsplit);
+   HexIntRule[0] = SetIntRulesFromMesh(meshsplit);
+   meshsplit.Clear();
+
+   // Reftype = 7
+   for (int i = 7; i < 8; i++)
+   {
+      Array<Refinement> marked_elements;
+      Mesh mesh_ref(base_mesh_copy);
+      for (int e = 0; e < mesh_ref.GetNE(); e++)
+      {
+         marked_elements.Append(Refinement(e, i));
+      }
+      mesh_ref.GeneralRefinement(marked_elements, 1, 0);
+      HexIntRule[1] = SetIntRulesFromMesh(mesh_ref);
+      mesh_ref.Clear();
+   }
+}
+
+void TMOPRefinerEstimator::SetQuadIntRules()
+{
+   QuadIntRule.SetSize(3+1);
+
+   // Reftype = 0 -> original element
+   Mesh meshsplit = Mesh::MakeCartesian2D(1, 1, Element::QUADRILATERAL);
+   Mesh base_mesh_copy(meshsplit);
+   QuadIntRule[0] = SetIntRulesFromMesh(meshsplit);
+   meshsplit.Clear();
+
+   // Reftype = 1-3
+   for (int i = 1; i < 4; i++)
+   {
+      Array<Refinement> marked_elements;
+      Mesh mesh_ref(base_mesh_copy);
+      for (int e = 0; e < mesh_ref.GetNE(); e++)
+      {
+         marked_elements.Append(Refinement(e, i));
+      }
+      mesh_ref.GeneralRefinement(marked_elements, 1, 0);
+      QuadIntRule[i] = SetIntRulesFromMesh(mesh_ref);
+      mesh_ref.Clear();
+   }
+}
+
+void TMOPRefinerEstimator::SetTriIntRules()
+{
+   TriIntRule.SetSize(1+1);
+
+   // Reftype = 0 // original element
+   const int Nvert = 3, NEsplit = 1;
+   Mesh meshsplit(2, Nvert, NEsplit, 0 ,2);
+   const double tri_v[3][2] =
+   {
+      {0, 0}, {1, 0}, {0, 1}
+   };
+   const int tri_e[1][3] =
+   {
+      {0, 1, 2}
+   };
+
+   for (int j = 0; j < Nvert; j++)
+   {
+      meshsplit.AddVertex(tri_v[j]);
+   }
+   meshsplit.AddTriangle(tri_e[0], 1);
+   meshsplit.FinalizeTriMesh(1, 1, true);
+
+   Mesh base_mesh_copy(meshsplit);
+   TriIntRule[0] = SetIntRulesFromMesh(meshsplit);
+   meshsplit.Clear();
+
+   // no anisotropic refinements for triangle
+   // Reftype = 3
+   for (int i = 1; i < 2; i++)
+   {
+      Array<Refinement> marked_elements;
+      Mesh mesh_ref(base_mesh_copy);
+      for (int e = 0; e < mesh_ref.GetNE(); e++)
+      {
+         marked_elements.Append(Refinement(e, i));
+      }
+      mesh_ref.GeneralRefinement(marked_elements, 1, 0);
+      TriIntRule[i] = SetIntRulesFromMesh(mesh_ref);
+      mesh_ref.Clear();
+   }
+}
+
+void TMOPRefinerEstimator::SetTetIntRules()
+{
+   TetIntRule.SetSize(1+1);
+
+   // Reftype = 0 // original element
+   const int Nvert = 4, NEsplit = 1;
+   Mesh meshsplit(3, Nvert, NEsplit, 0, 3);
+   const double tet_v[4][3] =
+   {
+      {0, 0, 0}, {1, 0, 0}, {0, 1, 0}, {0, 0, 1}
+   };
+   const int tet_e[1][4] =
+   {
+      {0, 1, 2, 3}
+   };
+
+   for (int j = 0; j < Nvert; j++)
+   {
+      meshsplit.AddVertex(tet_v[j]);
+   }
+   meshsplit.AddTet(tet_e[0], 1);
+   meshsplit.FinalizeTetMesh(1, 1, true);
+
+   Mesh base_mesh_copy(meshsplit);
+   TetIntRule[0] = SetIntRulesFromMesh(meshsplit);
+   meshsplit.Clear();
+
+   // no anisotropic refinements for triangle
+   // Reftype = 7
+   for (int i = 1; i < 2; i++)
+   {
+      Array<Refinement> marked_elements;
+      Mesh mesh_ref(base_mesh_copy);
+      for (int e = 0; e < mesh_ref.GetNE(); e++)
+      {
+         marked_elements.Append(Refinement(e, i)); //ref_type will default to 7
+      }
+      mesh_ref.GeneralRefinement(marked_elements, 1, 0);
+      TetIntRule[i] = SetIntRulesFromMesh(mesh_ref);
+      mesh_ref.Clear();
+   }
+}
+
+IntegrationRule* TMOPRefinerEstimator::SetIntRulesFromMesh(Mesh &meshsplit)
+{
+   const int dim = meshsplit.Dimension();
+   H1_FECollection fec(order, dim);
+   FiniteElementSpace nodal_fes(&meshsplit, &fec, dim);
+   meshsplit.SetNodalFESpace(&nodal_fes);
+
+   const int NEsplit = meshsplit.GetNE();
+   const int dof_cnt = nodal_fes.GetFE(0)->GetDof(),
+             pts_cnt = NEsplit * dof_cnt;
+
+   DenseMatrix pos(dof_cnt, dim);
+   Vector posV(pos.Data(), dof_cnt * dim);
+   Array<int> xdofs(dof_cnt * dim);
+
+   // Create an IntegrationRule on the nodes of the reference submesh.
+   IntegrationRule *irule = new IntegrationRule(pts_cnt);
+   GridFunction *nodesplit = meshsplit.GetNodes();
+
+   int pt_id = 0;
+   for (int i = 0; i < NEsplit; i++)
+   {
+      nodal_fes.GetElementVDofs(i, xdofs);
+      nodesplit->GetSubVector(xdofs, posV);
+      for (int j = 0; j < dof_cnt; j++)
+      {
+         if (dim == 2)
+         {
+            irule->IntPoint(pt_id).Set2(pos(j, 0), pos(j, 1));
+         }
+         else if (dim == 3)
+         {
+            irule->IntPoint(pt_id).Set3(pos(j, 0), pos(j, 1), pos(j, 2));
+         }
+         pt_id++;
+      }
+   }
+   return irule;
+}
+
+bool TMOPDeRefinerEstimator::GetDerefineEnergyForIntegrator(
+   TMOP_Integrator &tmopi,
+   Vector &fine_energy)
+{
+   DiscreteAdaptTC *tcd = tmopi.GetDiscreteAdaptTC();
+   fine_energy.SetSize(mesh->GetNE());
+
+   if (serial)
+   {
+      Mesh meshcopy(*mesh);
+      FiniteElementSpace *tcdfes = NULL;
+      if (tcd)
+      {
+         tcdfes = new FiniteElementSpace(*tcd->GetTSpecFESpace(), &meshcopy);
+      }
+
+      Vector local_err(meshcopy.GetNE());
+      local_err = 0.;
+      double threshold = std::numeric_limits<float>::max();
+      meshcopy.DerefineByError(local_err, threshold, 0, 1);
+
+      if (meshcopy.GetGlobalNE() == mesh->GetGlobalNE())
+      {
+         delete tcdfes;
+         return false;
+      }
+
+      if (tcd)
+      {
+         tcdfes->Update();
+         tcd->SetTspecDataForDerefinement(tcdfes);
+      }
+
+      Vector coarse_energy(meshcopy.GetNE());
+      GetTMOPDerefinementEnergy(meshcopy, tmopi, coarse_energy);
+      if (tcd) { tcd->ResetDerefinementTspecData(); }
+      GetTMOPDerefinementEnergy(*mesh, tmopi, fine_energy);
+
+      const CoarseFineTransformations &dtrans =
+         meshcopy.ncmesh->GetDerefinementTransforms();
+      Table coarse_to_fine;
+      dtrans.GetCoarseToFineMap(meshcopy, coarse_to_fine);
+
+      for (int pe = 0; pe < coarse_to_fine.Size(); pe++)
+      {
+         Array<int> tabrow;
+         coarse_to_fine.GetRow(pe, tabrow);
+         int nchild = tabrow.Size();
+         double parent_energy = coarse_energy(pe);
+         for (int fe = 0; fe < nchild; fe++)
+         {
+            int child = tabrow[fe];
+            MFEM_VERIFY(child < mesh->GetNE(), " invalid coarse to fine mapping");
+            fine_energy(child) -= parent_energy;
+         }
+      }
+      delete tcdfes;
+   }
+   else
+   {
+#ifdef MFEM_USE_MPI
+      ParMesh meshcopy(*pmesh);
+      ParFiniteElementSpace *tcdfes = NULL;
+      if (tcd)
+      {
+         tcdfes = new ParFiniteElementSpace(*tcd->GetTSpecParFESpace(), meshcopy);
+      }
+
+      Vector local_err(meshcopy.GetNE());
+      local_err = 0.;
+      double threshold = std::numeric_limits<float>::max();
+      meshcopy.DerefineByError(local_err, threshold, 0, 1);
+
+      if (meshcopy.GetGlobalNE() == pmesh->GetGlobalNE())
+      {
+         delete tcdfes;
+         return false;
+      }
+
+      if (tcd)
+      {
+         tcdfes->Update();
+         tcd->SetTspecDataForDerefinement(tcdfes);
+      }
+
+      Vector coarse_energy(meshcopy.GetNE());
+      GetTMOPDerefinementEnergy(meshcopy, tmopi, coarse_energy);
+      if (tcd) { tcd->ResetDerefinementTspecData(); }
+      GetTMOPDerefinementEnergy(*pmesh, tmopi, fine_energy);
+
+      const CoarseFineTransformations &dtrans =
+         meshcopy.pncmesh->GetDerefinementTransforms();
+      Table coarse_to_fine;
+      dtrans.GetCoarseToFineMap(meshcopy, coarse_to_fine);
+
+      for (int pe = 0; pe < meshcopy.GetNE(); pe++)
+      {
+         Array<int> tabrow;
+         coarse_to_fine.GetRow(pe, tabrow);
+         int nchild = tabrow.Size();
+         double parent_energy = coarse_energy(pe);
+         for (int fe = 0; fe < nchild; fe++)
+         {
+            int child = tabrow[fe];
+            MFEM_VERIFY(child < pmesh->GetNE(), " invalid coarse to fine mapping");
+            fine_energy(child) -= parent_energy;
+         }
+      }
+      delete tcdfes;
+#endif
+   }
+
+   // error_estimate(e) = energy(parent_of_e)-energy(e)
+   // Negative energy means derefinement is desirable.
+   fine_energy *= -1;
+   return true;
+}
+
+void TMOPDeRefinerEstimator::ComputeEstimates()
+{
+   Array<NonlinearFormIntegrator*> &integs = *(nlf->GetDNFI());
+   TMOP_Integrator *ti  = NULL;
+   TMOPComboIntegrator *co = NULL;
+   error_estimates.SetSize(mesh->GetNE());
+   error_estimates = 0.;
+   Vector fine_energy(mesh->GetNE());
+
+   for (int i = 0; i < integs.Size(); i++)
+   {
+      ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
+      if (ti)
+      {
+         bool deref = GetDerefineEnergyForIntegrator(*ti, fine_energy);
+         if (!deref) { error_estimates = 1; return; }
+         error_estimates += fine_energy;
+      }
+      co = dynamic_cast<TMOPComboIntegrator *>(integs[i]);
+      if (co)
+      {
+         Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
+         for (int j = 0; j < ati.Size(); j++)
+         {
+            bool deref = GetDerefineEnergyForIntegrator(*ati[j], fine_energy);
+            if (!deref) { error_estimates = 1; return; }
+            error_estimates += fine_energy;
+         }
+      }
+   }
+}
+
+void TMOPDeRefinerEstimator::GetTMOPDerefinementEnergy(Mesh &cmesh,
+                                                       TMOP_Integrator &tmopi,
+                                                       Vector &el_energy_vec)
+{
+   const int cNE = cmesh.GetNE();
+   el_energy_vec.SetSize(cNE);
+   const FiniteElementSpace *fespace = cmesh.GetNodalFESpace();
+
+   GridFunction *cxdof = cmesh.GetNodes();
+
+   Array<int> vdofs;
+   Vector el_x;
+   const FiniteElement *fe;
+   ElementTransformation *T;
+
+   for (int j = 0; j < cNE; j++)
+   {
+      fe = fespace->GetFE(j);
+      fespace->GetElementVDofs(j, vdofs);
+      T = cmesh.GetElementTransformation(j);
+      cxdof->GetSubVector(vdofs, el_x);
+      el_energy_vec(j) = tmopi.GetDerefinementElementEnergy(*fe, *T, el_x);
+   }
+}
+
+
+TMOPHRSolver::TMOPHRSolver(Mesh &mesh_, NonlinearForm &nlf_,
+                           TMOPNewtonSolver &tmopns_, GridFunction &x_,
+                           bool move_bnd_, bool hradaptivity_,
+                           int mesh_poly_deg_, int amr_metric_id_,
+                           int hr_iter_, int h_per_r_iter_) :
+   mesh(&mesh_), nlf(&nlf_), tmopns(&tmopns_), x(&x_),
+   gridfuncarr(), fespacearr(),
+   move_bnd(move_bnd_), hradaptivity(hradaptivity_),
+   mesh_poly_deg(mesh_poly_deg_), amr_metric_id(amr_metric_id_),
+   serial(true), hr_iter(hr_iter_), h_per_r_iter(h_per_r_iter_)
+{
+   if (!hradaptivity) { return; }
+   tmop_r_est = new TMOPRefinerEstimator(*mesh, *nlf, mesh_poly_deg,
+                                         amr_metric_id);
+   tmop_r = new ThresholdRefiner(*tmop_r_est);
+   tmop_r->SetTotalErrorFraction(0.0);
+   tmop_r_est->SetEnergyScalingFactor(1.);
+   tmop_dr_est= new TMOPDeRefinerEstimator(*mesh, *nlf);
+   tmop_dr = new ThresholdDerefiner(*tmop_dr_est);
+   AddGridFunctionForUpdate(x);
+}
+
+#ifdef MFEM_USE_MPI
+TMOPHRSolver::TMOPHRSolver(ParMesh &pmesh_, ParNonlinearForm &pnlf_,
+                           TMOPNewtonSolver &tmopns_, ParGridFunction &px_,
+                           bool move_bnd_, bool hradaptivity_,
+                           int mesh_poly_deg_, int amr_metric_id_,
+                           int hr_iter_, int h_per_r_iter_) :
+   mesh(&pmesh_), nlf(&pnlf_), tmopns(&tmopns_), x(&px_),
+   gridfuncarr(), fespacearr(),
+   move_bnd(move_bnd_), hradaptivity(hradaptivity_),
+   mesh_poly_deg(mesh_poly_deg_), amr_metric_id(amr_metric_id_),
+   pmesh(&pmesh_), pnlf(&pnlf_), pgridfuncarr(), pfespacearr(),
+   serial(false), hr_iter(hr_iter_), h_per_r_iter(h_per_r_iter_)
+{
+   if (!hradaptivity) { return; }
+   tmop_r_est = new TMOPRefinerEstimator(*pmesh, *pnlf, mesh_poly_deg,
+                                         amr_metric_id);
+   tmop_r = new ThresholdRefiner(*tmop_r_est);
+   tmop_r->SetTotalErrorFraction(0.0);
+   tmop_r_est->SetEnergyScalingFactor(1.);
+   tmop_dr_est= new TMOPDeRefinerEstimator(*pmesh, *pnlf);
+   tmop_dr = new ThresholdDerefiner(*tmop_dr_est);
+   AddGridFunctionForUpdate(&px_);
+}
+#endif
+
+void TMOPHRSolver::Mult()
+{
+   Vector b(0);
+   int myid = 0;
+   if (serial)
+   {
+      tmopns->SetOperator(*nlf);
+   }
+   else
+   {
+#ifdef MFEM_USE_MPI
+      myid = pnlf->ParFESpace()->GetMyRank();
+      tmopns->SetOperator(*pnlf);
+#endif
+   }
+   if (!hradaptivity)
+   {
+      tmopns->Mult(b, x->GetTrueVector());
+      if (tmopns->GetConverged() == false)
+      {
+         if (myid == 0) { mfem::out << "Nonlinear solver: rtol not achieved.\n"; }
+      }
+      x->SetFromTrueVector();
+      return;
+   }
+
+   bool radaptivity = true;
+
+   tmop_dr->Reset();
+   tmop_r->Reset();
+
+   if (serial)
+   {
+      for (int i_hr = 0; i_hr < hr_iter; i_hr++)
+      {
+         if (!radaptivity)
+         {
+            break;
+         }
+         mfem::out << i_hr << " r-adaptivity iteration.\n";
+
+         tmopns->SetOperator(*nlf);
+         tmopns->Mult(b, x->GetTrueVector());
+         x->SetFromTrueVector();
+
+         mfem::out << "TMOP energy after r-adaptivity: " <<
+                   nlf->GetGridFunctionEnergy(*x)/mesh->GetNE() <<
+                   ", Elements: " << mesh->GetNE() << std::endl;
+
+         for (int i_h = 0; i_h < h_per_r_iter; i_h++)
+         {
+            // Derefinement step.
+            if (mesh->ncmesh)
+            {
+               tmop_dr->Apply(*mesh);
+               Update();
+            }
+            mfem::out << "TMOP energy after derefinement: " <<
+                      nlf->GetGridFunctionEnergy(*x)/mesh->GetNE() <<
+                      ", Elements: " << mesh->GetNE() << std::endl;
+
+            // Refinement step.
+            tmop_r->Apply(*mesh);
+            Update();
+            mfem::out << "TMOP energy after   refinement: " <<
+                      nlf->GetGridFunctionEnergy(*x)/mesh->GetNE() <<
+                      ", Elements: " << mesh->GetNE() << std::endl;
+
+            if (!tmop_dr->Derefined() && tmop_r->Stop())
+            {
+               radaptivity = false;
+               mfem::out << "AMR stopping criterion satisfied. Stop.\n";
+               break;
+            }
+         } //n_h
+      } //n_hr
+   }
+   else
+   {
+#ifdef MFEM_USE_MPI
+      int NEGlob;
+      double tmopenergy;
+      for (int i_hr = 0; i_hr < hr_iter; i_hr++)
+      {
+         if (!radaptivity)
+         {
+            break;
+         }
+         if (myid == 0) { mfem::out << i_hr << " r-adaptivity iteration.\n"; }
+         tmopns->SetOperator(*pnlf);
+         tmopns->Mult(b, x->GetTrueVector());
+         x->SetFromTrueVector();
+
+         NEGlob = pmesh->GetGlobalNE();
+         tmopenergy = pnlf->GetParGridFunctionEnergy(*x) / NEGlob;
+         if (myid == 0)
+         {
+            mfem::out << "TMOP energy after r-adaptivity: " << tmopenergy <<
+                      ", Elements: " << NEGlob << std::endl;
+         }
+
+         for (int i_h = 0; i_h < h_per_r_iter; i_h++)
+         {
+            // Derefinement step.
+            if (pmesh->pncmesh)
+            {
+               RebalanceParNCMesh();
+               ParUpdate();
+
+               tmop_dr->Apply(*pmesh);
+               ParUpdate();
+            }
+            NEGlob = pmesh->GetGlobalNE();
+            tmopenergy = pnlf->GetParGridFunctionEnergy(*x) / NEGlob;
+            if (myid == 0)
+            {
+               mfem::out << "TMOP energy after derefinement: " << tmopenergy <<
+                         ", Elements: " << NEGlob << std::endl;
+            }
+
+            // Refinement step.
+            tmop_r->Apply(*pmesh);
+            ParUpdate();
+            NEGlob = pmesh->GetGlobalNE();
+            tmopenergy = pnlf->GetParGridFunctionEnergy(*x) / NEGlob;
+            if (myid == 0)
+            {
+               mfem::out << "TMOP energy after   refinement: " << tmopenergy <<
+                         ", Elements: " << NEGlob << std::endl;
+            }
+
+            if (!tmop_dr->Derefined() && tmop_r->Stop())
+            {
+               radaptivity = false;
+               if (myid == 0)
+               {
+                  mfem::out << "AMR stopping criterion satisfied. Stop.\n";
+               }
+               break;
+            }
+         } //n_r limit
+      } //n_hr
+#endif
+   }
+}
+
+#ifdef MFEM_USE_MPI
+void TMOPHRSolver::RebalanceParNCMesh()
+{
+   ParNCMesh *pncmesh = pmesh->pncmesh;
+   if (pncmesh)
+   {
+      const Table &dreftable = pncmesh->GetDerefinementTable();
+      Array<int> drefs, new_ranks;
+      for (int i = 0; i < dreftable.Size(); i++)
+      {
+         drefs.Append(i);
+      }
+      pncmesh->GetFineToCoarsePartitioning(drefs, new_ranks);
+      pmesh->Rebalance(new_ranks);
+   }
+}
+#endif
+
+void TMOPHRSolver::Update()
+{
+   // Update FESpace
+   for (int i = 0; i < fespacearr.Size(); i++)
+   {
+      fespacearr[i]->Update();
+   }
+   // Update nodal GF
+   for (int i = 0; i < gridfuncarr.Size(); i++)
+   {
+      gridfuncarr[i]->Update();
+      gridfuncarr[i]->SetTrueVector();
+      gridfuncarr[i]->SetFromTrueVector();
+   }
+
+   // Update Discrete Indicator for all the TMOP_Integrators in NonLinearForm
+   Array<NonlinearFormIntegrator*> &integs = *(nlf->GetDNFI());
+   TMOP_Integrator *ti  = NULL;
+   TMOPComboIntegrator *co = NULL;
+   DiscreteAdaptTC *dtc = NULL;
+   for (int i = 0; i < integs.Size(); i++)
+   {
+      ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
+      if (ti)
+      {
+         ti->UpdateAfterMeshTopologyChange();
+         dtc = ti->GetDiscreteAdaptTC();
+         if (dtc) { dtc->UpdateAfterMeshTopologyChange(); }
+      }
+      co = dynamic_cast<TMOPComboIntegrator *>(integs[i]);
+      if (co)
+      {
+         Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
+         for (int j = 0; j < ati.Size(); j++)
+         {
+            ati[j]->UpdateAfterMeshTopologyChange();
+            dtc = ati[j]->GetDiscreteAdaptTC();
+            if (dtc) { dtc->UpdateAfterMeshTopologyChange(); }
+         }
+      }
+   }
+
+   // Update the Nonlinear form and set Essential BC.
+   UpdateNonlinearFormAndBC(mesh, nlf);
+}
+
+#ifdef MFEM_USE_MPI
+void TMOPHRSolver::ParUpdate()
+{
+   // Update FESpace
+   for (int i = 0; i < pfespacearr.Size(); i++)
+   {
+      pfespacearr[i]->Update();
+   }
+   // Update nodal GF
+   for (int i = 0; i < pgridfuncarr.Size(); i++)
+   {
+      pgridfuncarr[i]->Update();
+      pgridfuncarr[i]->SetTrueVector();
+      pgridfuncarr[i]->SetFromTrueVector();
+   }
+
+   // Update Discrete Indicator
+   Array<NonlinearFormIntegrator*> &integs = *(nlf->GetDNFI());
+   TMOP_Integrator *ti  = NULL;
+   TMOPComboIntegrator *co = NULL;
+   DiscreteAdaptTC *dtc = NULL;
+   for (int i = 0; i < integs.Size(); i++)
+   {
+      ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
+      if (ti)
+      {
+         ti->ParUpdateAfterMeshTopologyChange();
+         dtc = ti->GetDiscreteAdaptTC();
+         if (dtc) { dtc->ParUpdateAfterMeshTopologyChange(); }
+      }
+      co = dynamic_cast<TMOPComboIntegrator *>(integs[i]);
+      if (co)
+      {
+         Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
+         for (int j = 0; j < ati.Size(); j++)
+         {
+            ati[j]->ParUpdateAfterMeshTopologyChange();
+            dtc = ati[j]->GetDiscreteAdaptTC();
+            if (dtc) { dtc->ParUpdateAfterMeshTopologyChange(); }
+         }
+      }
+   }
+
+   // Update the Nonlinear form and set Essential BC.
+   UpdateNonlinearFormAndBC(pmesh, pnlf);
+}
+#endif
+
+void TMOPHRSolver::UpdateNonlinearFormAndBC(Mesh *mesh, NonlinearForm *nlf)
+{
+   const FiniteElementSpace &fes = *mesh->GetNodalFESpace();
+
+   // Update Nonlinear form and Set Essential BC
+   nlf->Update();
+   const int dim = fes.GetFE(0)->GetDim();
+   if (move_bnd == false)
+   {
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      nlf->SetEssentialBC(ess_bdr);
+   }
+   else
+   {
+      const int nd  = fes.GetBE(0)->GetDof();
+      int n = 0;
+      for (int i = 0; i < mesh->GetNBE(); i++)
+      {
+         const int attr = mesh->GetBdrElement(i)->GetAttribute();
+         MFEM_VERIFY(!(dim == 2 && attr == 3),
+                     "Boundary attribute 3 must be used only for 3D meshes. "
+                     "Adjust the attributes (1/2/3/4 for fixed x/y/z/all "
+                     "components, rest for free nodes), or use -fix-bnd.");
+         if (attr == 1 || attr == 2 || attr == 3) { n += nd; }
+         if (attr == 4) { n += nd * dim; }
+      }
+      Array<int> ess_vdofs(n), vdofs;
+      n = 0;
+      for (int i = 0; i < mesh->GetNBE(); i++)
+      {
+         const int attr = mesh->GetBdrElement(i)->GetAttribute();
+         fes.GetBdrElementVDofs(i, vdofs);
+         if (attr == 1) // Fix x components.
+         {
+            for (int j = 0; j < nd; j++)
+            { ess_vdofs[n++] = vdofs[j]; }
+         }
+         else if (attr == 2) // Fix y components.
+         {
+            for (int j = 0; j < nd; j++)
+            { ess_vdofs[n++] = vdofs[j+nd]; }
+         }
+         else if (attr == 3) // Fix z components.
+         {
+            for (int j = 0; j < nd; j++)
+            { ess_vdofs[n++] = vdofs[j+2*nd]; }
+         }
+         else if (attr == 4) // Fix all components.
+         {
+            for (int j = 0; j < vdofs.Size(); j++)
+            { ess_vdofs[n++] = vdofs[j]; }
+         }
+      }
+      nlf->SetEssentialVDofs(ess_vdofs);
+   }
+}
+
+}
@@ -0,0 +1,284 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_TMOP_AMR_HPP
+#define MFEM_TMOP_AMR_HPP
+
+#include "tmop_tools.hpp"
+#include "nonlinearform.hpp"
+#include "pnonlinearform.hpp"
+#include "estimators.hpp"
+#include "../mesh/mesh_operators.hpp"
+
+namespace mfem
+{
+
+class TMOPRefinerEstimator : public AnisotropicErrorEstimator
+{
+protected:
+   Mesh *mesh; // not owned
+   NonlinearForm *nlf; // not owned
+   int order;
+   int amrmetric;
+   Array<IntegrationRule *> TriIntRule, QuadIntRule, TetIntRule, HexIntRule;
+   long current_sequence;
+   Vector error_estimates;
+   Array<int> aniso_flags;
+   // An element is refined only if
+   // [mean TMOPEnergy(children)]*energy_scaling_factor < TMOPEnergy(parent)
+   double energy_scaling_factor;
+   GridFunction *spat_gf;   // If specified, can be used to specify the
+   double spat_gf_critical; // region where hr-adaptivity is done.
+
+   /// Check if the mesh of the solution was modified.
+   bool MeshIsModified()
+   {
+      long mesh_sequence = mesh->GetSequence();
+      MFEM_ASSERT(mesh_sequence >= current_sequence, "");
+      return (mesh_sequence > current_sequence);
+   }
+
+   /// Compute the element error estimates. For an element E in the mesh,
+   /// error(E) = TMOPEnergy(E)*energy_scaling_factor-Mean(TMOPEnergy(ChildofE)),
+   /// where TMOPEnergy of Children of E is obtained by assuming the element E
+   /// is refined using the refinement type being considered based on the TMOP
+   /// mesh quality metric.
+   void ComputeEstimates();
+
+   /// Construct the integration rules to model how each element type is split
+   /// using different refinement types. ref_type = 0 is the original element
+   /// and reftype \ in [1, 7] represent different refinement type based on
+   /// NCMesh class.
+   void SetQuadIntRules(); // supports ref_type = 1 to 3.
+   void SetTriIntRules(); // currently supports only isotropic refinement.
+   void SetHexIntRules(); // currently supports only isotropic refinement.
+   void SetTetIntRules(); // currently supports only isotropic refinement.
+
+   /// Get TMOP energy for each element corresponding to the refinement type
+   /// specified.
+   void GetTMOPRefinementEnergy(int reftype, Vector &el_energy_vec);
+
+   /// Use a mesh to setup an integration rule that will mimic the different
+   /// refinement types.
+   IntegrationRule* SetIntRulesFromMesh(Mesh &meshsplit);
+public:
+   TMOPRefinerEstimator(Mesh &mesh_, NonlinearForm &nlf_, int order_,
+                        int amrmetric_) :
+      mesh(&mesh_), nlf(&nlf_), order(order_), amrmetric(amrmetric_),
+      TriIntRule(0), QuadIntRule(0), TetIntRule(0), HexIntRule(0),
+      current_sequence(-1), error_estimates(), aniso_flags(),
+      energy_scaling_factor(1.), spat_gf(NULL), spat_gf_critical(0.)
+   {
+      if (mesh->Dimension() == 2)
+      {
+         SetQuadIntRules();
+         SetTriIntRules();
+      }
+      else
+      {
+         SetHexIntRules();
+         SetTetIntRules();
+      }
+   }
+
+   ~TMOPRefinerEstimator()
+   {
+      for (int i = 0; i < QuadIntRule.Size(); i++) { delete QuadIntRule[i]; }
+      for (int i = 0; i < TriIntRule.Size();  i++) { delete TriIntRule[i]; }
+      for (int i = 0; i < HexIntRule.Size();  i++) { delete HexIntRule[i]; }
+      for (int i = 0; i < TetIntRule.Size();  i++) { delete TetIntRule[i]; }
+   }
+
+   /// Get TMOP-based errors for each element in the mesh computed based on the
+   /// refinement types being considered.
+   virtual const Vector &GetLocalErrors()
+   {
+      if (MeshIsModified()) { ComputeEstimates(); }
+      return error_estimates;
+   }
+   /// For anisotropic refinements, get the refinement type (e.g., x or y)
+   virtual const Array<int> &GetAnisotropicFlags()
+   {
+      if (MeshIsModified()) { ComputeEstimates(); }
+      return aniso_flags;
+   }
+
+   /// Scaling factor for the TMOP refinement energy. An element is refined if
+   /// [mean TMOPEnergy(children)]*energy_scaling_factor < TMOPEnergy(parent)
+   void SetEnergyScalingFactor(double scale) { energy_scaling_factor = scale; }
+
+   /// Spatial indicator function (eta) that can be used to prevent elements
+   /// from being refined even if the energy criterion is met. Using this,
+   /// an element E is not refined if mean(@a spat_gf(E)) < @a spat_gf_critical.
+   void SetSpatialIndicator(GridFunction &spat_gf_,
+                            double spat_gf_critical_ = 0.5)
+   { spat_gf = &spat_gf_; spat_gf_critical = spat_gf_critical_; }
+   void SetSpatialIndicatorCritical(double val_) { spat_gf_critical = val_; }
+
+   /// Reset the error estimator.
+   virtual void Reset() { current_sequence = -1; }
+};
+
+class TMOPDeRefinerEstimator : public ErrorEstimator
+{
+protected:
+   Mesh *mesh;
+   NonlinearForm *nlf;
+#ifdef MFEM_USE_MPI
+   ParMesh *pmesh;
+   ParNonlinearForm *pnlf;
+#endif
+   int order;
+   int amrmetric;
+   long current_sequence;
+   Vector error_estimates;
+   bool serial;
+
+   /// Check if the mesh of the solution was modified.
+   bool MeshIsModified()
+   {
+      long mesh_sequence = mesh->GetSequence();
+      MFEM_ASSERT(mesh_sequence >= current_sequence, "");
+      return (mesh_sequence > current_sequence);
+   }
+
+   /// Compute the element error estimates. For a given element E in the mesh,
+   /// error(E) = TMOPEnergy(parent_of_E)-TMOPEnergy(E). Children element of an
+   /// element are derefined if the mean TMOP energy of children is greated than
+   /// the TMOP energy associated with their parent.
+   void ComputeEstimates();
+
+   void GetTMOPDerefinementEnergy(Mesh &cmesh,
+                                  TMOP_Integrator &tmopi,
+                                  Vector &el_energy_vec);
+
+   bool GetDerefineEnergyForIntegrator(TMOP_Integrator &tmopi,
+                                       Vector &fine_energy);
+public:
+   TMOPDeRefinerEstimator(Mesh &mesh_, NonlinearForm &nlf_) :
+      mesh(&mesh_), nlf(&nlf_),
+      current_sequence(-1), error_estimates(), serial(true)   { }
+#ifdef MFEM_USE_MPI
+   TMOPDeRefinerEstimator(ParMesh &pmesh_, ParNonlinearForm &pnlf_) :
+      mesh(&pmesh_), nlf(&pnlf_), pmesh(&pmesh_), pnlf(&pnlf_),
+      current_sequence(-1), error_estimates(), serial(false) { }
+#endif
+
+   ~TMOPDeRefinerEstimator() { }
+
+   virtual const Vector &GetLocalErrors()
+   {
+      if (MeshIsModified()) { ComputeEstimates(); }
+      return error_estimates;
+   }
+
+   /// Reset the error estimator.
+   virtual void Reset() { current_sequence = -1; }
+};
+
+// hr-adaptivity using TMOP.
+// If hr-adaptivity is disabled, r-adaptivity is done once using the
+// TMOPNewtonSolver.
+// Otherwise, "hr_iter" iterations of r-adaptivity are done followed by
+// "h_per_r_iter" iterations of h-adaptivity after each r-adaptivity iteration.
+// The solver terminates early if an h-adaptivity iteration does not
+// refine/derefine any element in the mesh.
+class TMOPHRSolver
+{
+protected:
+   Mesh *mesh;
+   NonlinearForm *nlf;
+   TMOPNewtonSolver *tmopns;
+   GridFunction *x;
+   Array<GridFunction *> gridfuncarr;
+   Array<FiniteElementSpace *> fespacearr;
+   bool move_bnd, hradaptivity;
+   const int mesh_poly_deg, amr_metric_id;
+#ifdef MFEM_USE_MPI
+   ParMesh *pmesh;
+   ParNonlinearForm *pnlf;
+   Array<ParGridFunction *> pgridfuncarr;
+   Array<ParFiniteElementSpace *> pfespacearr;
+#endif
+   bool serial;
+
+   // All are owned.
+   TMOPRefinerEstimator *tmop_r_est;
+   ThresholdRefiner *tmop_r;
+   TMOPDeRefinerEstimator *tmop_dr_est;
+   ThresholdDerefiner *tmop_dr;
+
+   int hr_iter, h_per_r_iter;
+
+   void Update();
+#ifdef MFEM_USE_MPI
+   void ParUpdate();
+#endif
+   void UpdateNonlinearFormAndBC(Mesh *mesh, NonlinearForm *nlf);
+
+#ifdef MFEM_USE_MPI
+   // Rebalance ParMesh such that all the children elements are moved to the same
+   // MPI rank where the parent will be if the mesh were to be derefined.
+   void RebalanceParNCMesh();
+#endif
+
+public:
+   TMOPHRSolver(Mesh &mesh_, NonlinearForm &nlf_,
+                TMOPNewtonSolver &tmopns_, GridFunction &x_,
+                bool move_bnd_, bool hradaptivity_,
+                int mesh_poly_deg_, int amr_metric_id_,
+                int hr_iter_ = 5, int h_per_r_iter_ = 1);
+#ifdef MFEM_USE_MPI
+   TMOPHRSolver(ParMesh &pmesh_, ParNonlinearForm &pnlf_,
+                TMOPNewtonSolver &tmopns_, ParGridFunction &x_,
+                bool move_bnd_, bool hradaptivity_,
+                int mesh_poly_deg_, int amr_metric_id_,
+                int hr_iter_ = 5, int h_per_r_iter_ = 1);
+#endif
+
+   void Mult();
+
+   /// These are used to update spaces and functions that are not owned by the
+   /// TMOPIntegrator or DiscreteAdaptTC. The owned ones are updated in the
+   /// functions UpdateAfterMeshTopologyChange() of both classes.
+   void AddGridFunctionForUpdate(GridFunction *gf) { gridfuncarr.Append(gf); }
+   void AddFESpaceForUpdate(FiniteElementSpace *fes) { fespacearr.Append(fes); }
+
+#ifdef MFEM_USE_MPI
+   void AddGridFunctionForUpdate(ParGridFunction *pgf_)
+   {
+      pgridfuncarr.Append(pgf_);
+   }
+   void AddFESpaceForUpdate(ParFiniteElementSpace *pfes_)
+   {
+      pfespacearr.Append(pfes_);
+   }
+#endif
+
+   ~TMOPHRSolver()
+   {
+      if (!hradaptivity) { return; }
+      delete tmop_dr;
+      delete tmop_dr_est;
+      delete tmop_r;
+      delete tmop_r_est;
+   }
+
+   /// Total number of hr-adaptivity iterations. At each iteration, we do an
+   /// r-adaptivity iteration followed by a number of h-adaptivity iterations.
+   void SetHRAdaptivityIterations(int iter) { hr_iter = iter; }
+
+   /// Total number of h-adaptivity iterations per r-adaptivity iteration.
+   void SetHAdaptivityIterations(int iter) { h_per_r_iter = iter; }
+};
+
+}
+#endif
@@ -407,6 +407,8 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
   {
      // Needed for the line search below. The untangling metrics see this
      // reference to detect deteriorations.
+      MFEM_VERIFY(min_det_ptr != NULL, " Initial mesh was valid, but"
+                  " intermediate mesh is invalid. Contact TMOP Developers.");
      *min_det_ptr = untangle_factor * min_detT_in;
   }

@@ -576,7 +578,7 @@ void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
         ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
         if (ti)
         {
-            ti->UpdateAfterMeshChange(x_loc);
+            ti->UpdateAfterMeshPositionChange(x_loc);
            ti->ComputeFDh(x_loc, *pfesc);
            UpdateDiscreteTC(*ti, x_loc);
         }
@@ -586,7 +588,7 @@ void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
            Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
            for (int j = 0; j < ati.Size(); j++)
            {
-               ati[j]->UpdateAfterMeshChange(x_loc);
+               ati[j]->UpdateAfterMeshPositionChange(x_loc);
               ati[j]->ComputeFDh(x_loc, *pfesc);
               UpdateDiscreteTC(*ati[j], x_loc);
            }
@@ -613,7 +615,7 @@ void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
         ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
         if (ti)
         {
-            ti->UpdateAfterMeshChange(x_loc);
+            ti->UpdateAfterMeshPositionChange(x_loc);
            ti->ComputeFDh(x_loc, *fesc);
            UpdateDiscreteTC(*ti, x_loc);
         }
@@ -623,7 +625,7 @@ void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
            Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
            for (int j = 0; j < ati.Size(); j++)
            {
-               ati[j]->UpdateAfterMeshChange(x_loc);
+               ati[j]->UpdateAfterMeshPositionChange(x_loc);
               ati[j]->ComputeFDh(x_loc, *fesc);
               UpdateDiscreteTC(*ati[j], x_loc);
            }
@@ -339,6 +339,10 @@ inline bool operator!=(const Array<T> &LHS, const Array<T> &RHS)
 }


+/// Utility function similar to std::as_const in c++17.
+template <typename T> const T &AsConst(T &a) { return a; }
+
+
 template <class T>
 class Array2D;

@@ -71,7 +71,7 @@ static const unsigned char b64table[] =
   255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255
 };

-void DecodeBase64(const char *src, size_t len, std::vector<unsigned char> &buf)
+void DecodeBase64(const char *src, size_t len, std::vector<char> &buf)
 {
   const unsigned char *in = (const unsigned char *)src;
   buf.clear();
@@ -79,7 +79,7 @@ void DecodeBase64(const char *src, size_t len, std::vector<unsigned char> &buf)
   for (size_t i=0; i<len; ++i) { if (b64table[in[i]] != 255) { ++count; } }
   if (count % 4 != 0) { return; }
   buf.resize(3*len/4);
-   unsigned char *out = buf.data();
+   unsigned char *out = (unsigned char *)buf.data();
   count = 0;
   int pad = 0;
   unsigned char c[4];
@@ -97,8 +97,10 @@ void DecodeBase64(const char *src, size_t len, std::vector<unsigned char> &buf)
         count = pad = 0;
      }
   }
-   buf.resize(out - buf.data());
+   buf.resize(out - (unsigned char *)buf.data());
 }

+size_t NumBase64Chars(size_t nbytes) { return ((4*nbytes/3) + 3) & ~3; }
+
 } // namespace mfem::bin_io
 } // namespace mfem
@@ -50,6 +50,7 @@ inline T read(const char *buf)
   return value;
 }

+/// Append the binary representation of @a val to the byte buffer @a vec.
 template <typename T>
 void AppendBytes(std::vector<char> &vec, const T &val)
 {
@@ -57,9 +58,17 @@ void AppendBytes(std::vector<char> &vec, const T &val)
   vec.insert(vec.end(), ptr, ptr + sizeof(T));
 }

-void WriteBase64(std::ostream &out, const void *bytes, size_t length);
+/// Given a buffer @a buf of length @a nbytes, encode the data in base-64
+/// format, and write the encoded data to the output stream @a out.
+void WriteBase64(std::ostream &out, const void *bytes, size_t nbytes);

-void DecodeBase64(const char *src, size_t len, std::vector<unsigned char> &buf);
+/// Decode @a len base-64 encoded characters in the buffer @a src, and store the
+/// resulting decoded data in @a buf. @a buf will be resized as needed.
+void DecodeBase64(const char *src, size_t len, std::vector<char> &buf);
+
+/// Return the number of characters needed to encode @a nbytes in base-64. This
+/// is equal to 4*nbytes/3, rounded up to the nearest multiple of 4.
+size_t NumBase64Chars(size_t nbytes);

 } // namespace mfem::bin_io

@@ -92,7 +92,7 @@ void mfem_backtrace(int mode, int depth)
   int err = unw_getcontext(&uc);
   err = err ? err : unw_init_local(&cursor, &uc);

-   Array<unw_word_t> addrs;
+   Array<unw_word_t> addrs(MemoryType::HOST);
   while (unw_step(&cursor) > 0 && addrs.Size() != depth)
   {
      err = err ? err : unw_get_proc_name(&cursor, name, UNW_NAME_LEN, &offp);
@@ -45,6 +45,9 @@
 #endif
 #endif // MFEM_USE_UMPIRE

+// Internal debug option, useful for tracking some memory manager operations.
+// #define MFEM_TRACK_MEM_MANAGER
+
 namespace mfem
 {

@@ -63,6 +66,27 @@ MemoryType GetMemoryType(MemoryClass mc)
 }


+bool MemoryClassContainsType(MemoryClass mc, MemoryType mt)
+{
+   switch (mc)
+   {
+      case MemoryClass::HOST: return IsHostMemory(mt);
+      case MemoryClass::HOST_32:
+         return (mt == MemoryType::HOST_32 ||
+                 mt == MemoryType::HOST_64 ||
+                 mt == MemoryType::HOST_DEBUG);
+      case MemoryClass::HOST_64:
+         return (mt == MemoryType::HOST_64 ||
+                 mt == MemoryType::HOST_DEBUG);
+      case MemoryClass::DEVICE: return IsDeviceMemory(mt);
+      case MemoryClass::MANAGED:
+         return (mt == MemoryType::MANAGED);
+   }
+   MFEM_ABORT("invalid MemoryClass");
+   return false;
+}
+
+
 static void MFEM_VERIFY_TYPES(const MemoryType h_mt, const MemoryType d_mt)
 {
   MFEM_VERIFY(IsHostMemory(h_mt), "h_mt = " << (int)h_mt);
@@ -147,10 +171,12 @@ struct Memory
 /// Alias class that holds the base memory region and the offset
 struct Alias
 {
-   Memory *const mem;
-   const size_t offset, bytes;
+   Memory *mem;
+   size_t offset;
   size_t counter;
-   const MemoryType h_mt;
+   // 'h_mt' is already stored in 'mem', however, we use this field for type
+   // checking since the alias may be dangling, i.e. 'mem' may be invalid.
+   MemoryType h_mt;
 };

 /// Maps for the Memory and the Alias classes
@@ -558,7 +584,7 @@ public:
 #ifdef MFEM_USE_HIP
      return HipMemcpyHtoD(dst, src, bytes);
 #endif
-      //rm.copy(dst, const_cast<void*>(src), bytes); return dst;
+      // rm.copy(dst, const_cast<void*>(src), bytes); return dst;
   }
   void *DtoD(void* dst, const void* src, size_t bytes) override
   {
@@ -568,7 +594,7 @@ public:
 #ifdef MFEM_USE_HIP
      return HipMemcpyDtoD(dst, src, bytes);
 #endif
-      //rm.copy(dst, const_cast<void*>(src), bytes); return dst;
+      // rm.copy(dst, const_cast<void*>(src), bytes); return dst;
   }
   void *DtoH(void *dst, const void *src, size_t bytes) override
   {
@@ -578,7 +604,7 @@ public:
 #ifdef MFEM_USE_HIP
      return HipMemcpyDtoH(dst, src, bytes);
 #endif
-      //rm.copy(dst, const_cast<void*>(src), bytes); return dst;
+      // rm.copy(dst, const_cast<void*>(src), bytes); return dst;
   }
 };
 #else
@@ -759,7 +785,7 @@ void *MemoryManager::New_(void *h_tmp, size_t bytes, MemoryType h_mt,
   // mm.InsertDevice(nullptr, h_ptr, bytes, h_mt, d_mt); // non-lazy dev alloc

   // MFEM_VERIFY_TYPES(h_mt, mt); // done by mm.Insert() above
-   CheckHostMemoryType_(h_mt, h_ptr);
+   CheckHostMemoryType_(h_mt, h_ptr, false);

   return h_ptr;
 }
@@ -770,7 +796,7 @@ void *MemoryManager::Register_(void *ptr, void *h_tmp, size_t bytes,
 {
   MFEM_CONTRACT_VAR(alias);
   MFEM_ASSERT(exists, "Internal error!");
-   MFEM_ASSERT(!alias, "Cannot register an alias!");
+   MFEM_VERIFY(!alias, "Cannot register an alias!");
   const bool is_host_mem = IsHostMemory(mt);
   const MemType h_mt = is_host_mem ? mt : GetDualMemoryType(mt);
   const MemType d_mt = is_host_mem ? MemoryType::DEFAULT : mt;
@@ -798,20 +824,21 @@ void *MemoryManager::Register_(void *ptr, void *h_tmp, size_t bytes,
   }
   else // DEVICE TYPES
   {
-      MFEM_VERIFY(ptr, "cannot register NULL device pointer");
+      MFEM_VERIFY(ptr || bytes == 0,
+                  "cannot register NULL device pointer with bytes = " << bytes);
      if (h_tmp == nullptr) { ctrl->Host(h_mt)->Alloc(&h_ptr, bytes); }
      else { h_ptr = h_tmp; }
      mm.InsertDevice(ptr, h_ptr, bytes, h_mt, d_mt);
      flags = own ? flags | Mem::OWNS_DEVICE : flags & ~Mem::OWNS_DEVICE;
      flags |= (Mem::OWNS_HOST | Mem::VALID_DEVICE);
   }
-   CheckHostMemoryType_(h_mt, h_ptr);
+   CheckHostMemoryType_(h_mt, h_ptr, alias);
   return h_ptr;
 }

-void MemoryManager::Register_(void *h_ptr, void *d_ptr, size_t bytes,
-                              MemoryType h_mt, MemoryType d_mt,
-                              bool own, bool alias, unsigned &flags)
+void MemoryManager::Register2_(void *h_ptr, void *d_ptr, size_t bytes,
+                               MemoryType h_mt, MemoryType d_mt,
+                               bool own, bool alias, unsigned &flags)
 {
   MFEM_CONTRACT_VAR(alias);
   MFEM_ASSERT(exists, "Internal error!");
@@ -826,12 +853,14 @@ void MemoryManager::Register_(void *h_ptr, void *d_ptr, size_t bytes,

   flags |= Mem::REGISTERED | Mem::OWNS_INTERNAL;

+   MFEM_VERIFY(d_ptr || bytes == 0,
+               "cannot register NULL device pointer with bytes = " << bytes);
   mm.InsertDevice(d_ptr, h_ptr, bytes, h_mt, d_mt);
   flags = (own ? flags | (Mem::OWNS_HOST | Mem::OWNS_DEVICE) :
            flags & ~(Mem::OWNS_HOST | Mem::OWNS_DEVICE)) |
           Mem::VALID_HOST;

-   CheckHostMemoryType_(h_mt, h_ptr);
+   CheckHostMemoryType_(h_mt, h_ptr, alias);
 }

 void MemoryManager::Alias_(void *base_h_ptr, size_t offset, size_t bytes,
@@ -871,37 +900,40 @@ void MemoryManager::SetDeviceMemoryType_(void *h_ptr, unsigned flags,
   }
 }

-MemoryType MemoryManager::Delete_(void *h_ptr, MemoryType mt, unsigned flags)
+MemoryType MemoryManager::Delete_(void *h_ptr, MemoryType h_mt, unsigned flags)
 {
   const bool alias = flags & Mem::ALIAS;
   const bool registered = flags & Mem::REGISTERED;
   const bool owns_host = flags & Mem::OWNS_HOST;
   const bool owns_device = flags & Mem::OWNS_DEVICE;
   const bool owns_internal = flags & Mem::OWNS_INTERNAL;
-   MFEM_ASSERT(registered || IsHostMemory(mt),"");
+   MFEM_ASSERT(IsHostMemory(h_mt), "invalid h_mt = " << (int)h_mt);
+   // MFEM_ASSERT(registered || IsHostMemory(h_mt),"");
   MFEM_ASSERT(!owns_device || owns_internal, "invalid Memory state");
-   if (!mm.exists || !registered) { return mt; }
+   MFEM_ASSERT(registered || !(owns_host || owns_device || owns_internal),
+               "invalid Memory state");
+   if (!mm.exists || !registered) { return h_mt; }
   if (alias)
   {
      if (owns_internal)
      {
-         const MemoryType h_mt = maps->aliases.at(h_ptr).h_mt;
-         MFEM_ASSERT(mt == h_mt,"");
+         MFEM_ASSERT(mm.IsAlias(h_ptr), "");
+         MFEM_ASSERT(h_mt == maps->aliases.at(h_ptr).h_mt, "");
         mm.EraseAlias(h_ptr);
-         return h_mt;
      }
   }
   else // Known
   {
-      const MemoryType h_mt = mt;
-      MFEM_ASSERT(!owns_internal ||
-                  mt == maps->memories.at(h_ptr).h_mt,"");
      if (owns_host && (h_mt != MemoryType::HOST))
      { ctrl->Host(h_mt)->Dealloc(h_ptr); }
-      if (owns_internal) { mm.Erase(h_ptr, owns_device); }
-      return h_mt;
+      if (owns_internal)
+      {
+         MFEM_ASSERT(mm.IsKnown(h_ptr), "");
+         MFEM_ASSERT(h_mt == maps->memories.at(h_ptr).h_mt, "");
+         mm.Erase(h_ptr, owns_device);
+      }
   }
-   return mt;
+   return h_mt;
 }

 void MemoryManager::DeleteDevice_(void *h_ptr, unsigned & flags)
@@ -923,15 +955,19 @@ bool MemoryManager::MemoryClassCheck_(MemoryClass mc, void *h_ptr,
      MFEM_VERIFY(bytes == 0, "Trying to access NULL with size " << bytes);
      return true;
   }
-
-   const bool known = mm.IsKnown(h_ptr);
-   const bool alias = mm.IsAlias(h_ptr);
-   const bool check = known || ((flags & Mem::ALIAS) && alias);
-   MFEM_VERIFY(check, "Unknown host pointer: " << h_ptr);
-   const internal::Memory &mem =
-      (flags & Mem::ALIAS) ?
-      *maps->aliases.at(h_ptr).mem : maps->memories.at(h_ptr);
-   MemoryType d_mt = mem.d_mt;
+   MemoryType d_mt;
+   if (!(flags & Mem::ALIAS))
+   {
+      auto iter = maps->memories.find(h_ptr);
+      MFEM_VERIFY(iter != maps->memories.end(), "internal error");
+      d_mt = iter->second.d_mt;
+   }
+   else
+   {
+      auto iter = maps->aliases.find(h_ptr);
+      MFEM_VERIFY(iter != maps->aliases.end(), "internal error");
+      d_mt = iter->second.mem->d_mt;
+   }
   if (d_mt == MemoryType::DEFAULT) { d_mt = GetDualMemoryType(h_mt); }
   switch (mc)
   {
@@ -969,7 +1005,7 @@ bool MemoryManager::MemoryClassCheck_(MemoryClass mc, void *h_ptr,
 void *MemoryManager::ReadWrite_(void *h_ptr, MemoryType h_mt, MemoryClass mc,
                                size_t bytes, unsigned &flags)
 {
-   MemoryManager::CheckHostMemoryType_(h_mt, h_ptr);
+   if (h_ptr) { CheckHostMemoryType_(h_mt, h_ptr, flags & Mem::ALIAS); }
   if (bytes > 0) { MFEM_VERIFY(flags & Mem::REGISTERED,""); }
   MFEM_ASSERT(MemoryClassCheck_(mc, h_ptr, h_mt, bytes, flags),"");
   if (IsHostMemory(GetMemoryType(mc)) && mc < MemoryClass::DEVICE)
@@ -993,7 +1029,7 @@ void *MemoryManager::ReadWrite_(void *h_ptr, MemoryType h_mt, MemoryClass mc,
 const void *MemoryManager::Read_(void *h_ptr, MemoryType h_mt, MemoryClass mc,
                                 size_t bytes, unsigned &flags)
 {
-   CheckHostMemoryType_(h_mt, h_ptr);
+   if (h_ptr) { CheckHostMemoryType_(h_mt, h_ptr, flags & Mem::ALIAS); }
   if (bytes > 0) { MFEM_VERIFY(flags & Mem::REGISTERED,""); }
   MFEM_ASSERT(MemoryClassCheck_(mc, h_ptr, h_mt, bytes, flags),"");
   if (IsHostMemory(GetMemoryType(mc)) && mc < MemoryClass::DEVICE)
@@ -1017,7 +1053,7 @@ const void *MemoryManager::Read_(void *h_ptr, MemoryType h_mt, MemoryClass mc,
 void *MemoryManager::Write_(void *h_ptr, MemoryType h_mt, MemoryClass mc,
                            size_t bytes, unsigned &flags)
 {
-   CheckHostMemoryType_(h_mt, h_ptr);
+   if (h_ptr) { CheckHostMemoryType_(h_mt, h_ptr, flags & Mem::ALIAS); }
   if (bytes > 0) { MFEM_VERIFY(flags & Mem::REGISTERED,""); }
   MFEM_ASSERT(MemoryClassCheck_(mc, h_ptr, h_mt, bytes, flags),"");
   if (IsHostMemory(GetMemoryType(mc)) && mc < MemoryClass::DEVICE)
@@ -1061,22 +1097,20 @@ void MemoryManager::SyncAlias_(const void *base_h_ptr, void *alias_h_ptr,
                 (base_flags & (Mem::VALID_HOST | Mem::VALID_DEVICE));
 }

-MemoryType MemoryManager::GetDeviceMemoryType_(void *h_ptr)
+MemoryType MemoryManager::GetDeviceMemoryType_(void *h_ptr, bool alias)
 {
   if (mm.exists)
   {
-      const bool known = mm.IsKnown(h_ptr);
-      if (known)
+      if (!alias)
      {
-         internal::Memory &mem = maps->memories.at(h_ptr);
-         return mem.d_mt;
-      }
-      const bool alias = mm.IsAlias(h_ptr);
-      if (alias)
-      {
-         internal::Memory *mem = maps->aliases.at(h_ptr).mem;
-         return mem->d_mt;
+         auto iter = maps->memories.find(h_ptr);
+         MFEM_ASSERT(iter != maps->memories.end(), "internal error");
+         return iter->second.d_mt;
      }
+      // alias == true
+      auto iter = maps->aliases.find(h_ptr);
+      MFEM_ASSERT(iter != maps->aliases.end(), "internal error");
+      return iter->second.mem->d_mt;
   }
   MFEM_ABORT("internal error");
   return MemoryManager::host_mem_type;
@@ -1086,7 +1120,7 @@ MemoryType MemoryManager::GetHostMemoryType_(void *h_ptr)
 {
   if (!mm.exists) { return MemoryManager::host_mem_type; }
   if (mm.IsKnown(h_ptr)) { return maps->memories.at(h_ptr).h_mt; }
-   if (mm.IsAlias(h_ptr)) { return maps->aliases.at(h_ptr).mem->h_mt; }
+   if (mm.IsAlias(h_ptr)) { return maps->aliases.at(h_ptr).h_mt; }
   return MemoryManager::host_mem_type;
 }

@@ -1137,7 +1171,7 @@ void MemoryManager::Copy_(void *dst_h_ptr, const void *src_h_ptr,
      {
         if (dst_h_ptr != src_d_ptr && bytes != 0)
         {
-            internal::Memory &src_d_base = maps->memories.at(src_d_ptr);
+            internal::Memory &src_d_base = maps->memories.at(src_h_ptr);
            MemoryType src_d_mt = src_d_base.d_mt;
            ctrl->Device(src_d_mt)->DtoH(dst_h_ptr, src_d_ptr, bytes);
         }
@@ -1240,6 +1274,10 @@ bool MemoryManager::IsAlias_(const void *h_ptr)
 void MemoryManager::Insert(void *h_ptr, size_t bytes,
                           MemoryType h_mt, MemoryType d_mt)
 {
+#ifdef MFEM_TRACK_MEM_MANAGER
+   mfem::out << "[mfem memory manager]: registering h_ptr: " << h_ptr
+             << ", bytes: " << bytes << std::endl;
+#endif
   if (h_ptr == NULL)
   {
      MFEM_VERIFY(bytes == 0, "Trying to add NULL with size " << bytes);
@@ -1254,8 +1292,14 @@ void MemoryManager::Insert(void *h_ptr, size_t bytes,
   if (res.second == false)
   {
      auto &m = res.first->second;
-      MFEM_VERIFY(m.bytes >= bytes && m.h_mt == h_mt && m.d_mt == d_mt,
+      MFEM_VERIFY(m.bytes >= bytes && m.h_mt == h_mt &&
+                  (m.d_mt == d_mt || (d_mt == MemoryType::DEFAULT &&
+                                      m.d_mt == GetDualMemoryType(h_mt))),
                  "Address already present with different attributes!");
+#ifdef MFEM_TRACK_MEM_MANAGER
+      mfem::out << "[mfem memory manager]: repeated registration of h_ptr: "
+                << h_ptr << std::endl;
+#endif
   }
 #endif
 }
@@ -1267,7 +1311,7 @@ void MemoryManager::InsertDevice(void *d_ptr, void *h_ptr, size_t bytes,
   MFEM_ASSERT(h_ptr != NULL, "internal error");
   Insert(h_ptr, bytes, h_mt, d_mt);
   internal::Memory &mem = maps->memories.at(h_ptr);
-   if (d_ptr == NULL) { ctrl->Device(d_mt)->Alloc(mem); }
+   if (d_ptr == NULL && bytes != 0) { ctrl->Device(d_mt)->Alloc(mem); }
   else { mem.d_ptr = d_ptr; }
 }

@@ -1276,6 +1320,11 @@ void MemoryManager::InsertAlias(const void *base_ptr, void *alias_ptr,
 {
   size_t offset = static_cast<size_t>(static_cast<const char*>(alias_ptr) -
                                       static_cast<const char*>(base_ptr));
+#ifdef MFEM_TRACK_MEM_MANAGER
+   mfem::out << "[mfem memory manager]: registering alias of base_ptr: "
+             << base_ptr << ", offset: " << offset << ", bytes: " << bytes
+             << ", base is alias: " << base_is_alias << std::endl;
+#endif
   if (!base_ptr)
   {
      MFEM_VERIFY(offset == 0,
@@ -1288,26 +1337,33 @@ void MemoryManager::InsertAlias(const void *base_ptr, void *alias_ptr,
      MFEM_ASSERT(alias.mem,"");
      base_ptr = alias.mem->h_ptr;
      offset += alias.offset;
+#ifdef MFEM_TRACK_MEM_MANAGER
+      mfem::out << "[mfem memory manager]: real base_ptr: " << base_ptr
+                << std::endl;
+#endif
   }
   internal::Memory &mem = maps->memories.at(base_ptr);
+   MFEM_VERIFY(offset + bytes <= mem.bytes, "invalid alias");
   auto res =
      maps->aliases.emplace(alias_ptr,
-                            internal::Alias{&mem, offset, bytes, 1, mem.h_mt});
+                            internal::Alias{&mem, offset, 1, mem.h_mt});
   if (res.second == false) // alias_ptr was already in the map
   {
-      if (res.first->second.mem != &mem || res.first->second.offset != offset)
-      {
-         mfem_error("alias already exists with different base/offset!");
-      }
-      else
-      {
-         res.first->second.counter++;
-      }
+      internal::Alias &alias = res.first->second;
+      // Update the alias data in case the existing alias is dangling
+      alias.mem = &mem;
+      alias.offset = offset;
+      alias.h_mt = mem.h_mt;
+      alias.counter++;
   }
 }

 void MemoryManager::Erase(void *h_ptr, bool free_dev_ptr)
 {
+#ifdef MFEM_TRACK_MEM_MANAGER
+   mfem::out << "[mfem memory manager]: un-registering h_ptr: " << h_ptr
+             << std::endl;
+#endif
   if (!h_ptr) { return; }
   auto mem_map_iter = maps->memories.find(h_ptr);
   if (mem_map_iter == maps->memories.end()) { mfem_error("Unknown pointer!"); }
@@ -1321,10 +1377,6 @@ void MemoryManager::EraseDevice(void *h_ptr)
   if (!h_ptr) { return; }
   auto mem_map_iter = maps->memories.find(h_ptr);
   if (mem_map_iter == maps->memories.end()) { mfem_error("Unknown pointer!"); }
-   if (maps->aliases.find(h_ptr) != maps->aliases.end())
-   {
-      mfem_error("cannot delete aliased obj!");
-   }
   internal::Memory &mem = mem_map_iter->second;
   if (mem.d_ptr) { ctrl->Device(mem.d_mt)->Dealloc(mem);}
   mem.d_ptr = nullptr;
@@ -1332,6 +1384,10 @@ void MemoryManager::EraseDevice(void *h_ptr)

 void MemoryManager::EraseAlias(void *alias_ptr)
 {
+#ifdef MFEM_TRACK_MEM_MANAGER
+   mfem::out << "[mfem memory manager]: un-registering alias_ptr: " << alias_ptr
+             << std::endl;
+#endif
   if (!alias_ptr) { return; }
   auto alias_map_iter = maps->aliases.find(alias_ptr);
   if (alias_map_iter == maps->aliases.end()) { mfem_error("Unknown alias!"); }
@@ -1355,14 +1411,14 @@ void *MemoryManager::GetDevicePtr(const void *h_ptr, size_t bytes,
   if (!mem.d_ptr)
   {
      if (d_mt == MemoryType::DEFAULT) { d_mt = GetDualMemoryType(h_mt); }
-      ctrl->Device(d_mt)->Alloc(mem);
+      if (mem.bytes) { ctrl->Device(d_mt)->Alloc(mem); }
   }
   // Aliases might have done some protections
-   ctrl->Device(d_mt)->Unprotect(mem);
+   if (mem.d_ptr) { ctrl->Device(d_mt)->Unprotect(mem); }
   if (copy_data)
   {
      MFEM_ASSERT(bytes <= mem.bytes, "invalid copy size");
-      ctrl->Device(d_mt)->HtoD(mem.d_ptr, h_ptr, bytes);
+      if (bytes) { ctrl->Device(d_mt)->HtoD(mem.d_ptr, h_ptr, bytes); }
   }
   ctrl->Host(h_mt)->Protect(mem, bytes);
   return mem.d_ptr;
@@ -1388,16 +1444,17 @@ void *MemoryManager::GetAliasDevicePtr(const void *alias_ptr, size_t bytes,
   if (!mem.d_ptr)
   {
      if (d_mt == MemoryType::DEFAULT) { d_mt = GetDualMemoryType(h_mt); }
-      ctrl->Device(d_mt)->Alloc(mem);
+      if (mem.bytes) { ctrl->Device(d_mt)->Alloc(mem); }
   }
   void *alias_h_ptr = static_cast<char*>(mem.h_ptr) + offset;
   void *alias_d_ptr = static_cast<char*>(mem.d_ptr) + offset;
   MFEM_ASSERT(alias_h_ptr == alias_ptr, "internal error");
-   MFEM_ASSERT(bytes <= alias.bytes, "internal error");
+   MFEM_ASSERT(offset + bytes <= mem.bytes, "internal error");
   mem.d_rw = mem.h_rw = false;
-   ctrl->Device(d_mt)->AliasUnprotect(alias_d_ptr, bytes);
+   if (mem.d_ptr) { ctrl->Device(d_mt)->AliasUnprotect(alias_d_ptr, bytes); }
   ctrl->Host(h_mt)->AliasUnprotect(alias_ptr, bytes);
-   if (copy) { ctrl->Device(d_mt)->HtoD(alias_d_ptr, alias_h_ptr, bytes); }
+   if (copy && mem.d_ptr)
+   { ctrl->Device(d_mt)->HtoD(alias_d_ptr, alias_h_ptr, bytes); }
   ctrl->Host(h_mt)->AliasProtect(alias_ptr, bytes);
   return alias_d_ptr;
 }
@@ -1504,6 +1561,23 @@ void MemoryManager::Configure(const MemoryType host_mt,
 void MemoryManager::Destroy()
 {
   MFEM_VERIFY(exists, "MemoryManager has already been destroyed!");
+#ifdef MFEM_TRACK_MEM_MANAGER
+   size_t num_memories = maps->memories.size();
+   size_t num_aliases = maps->aliases.size();
+   if (num_memories != 0 || num_aliases != 0)
+   {
+      MFEM_WARNING("...\n\t number of registered pointers: " << num_memories
+                   << "\n\t number of registered aliases : " << num_aliases);
+   }
+#endif
+   // Keep for debugging purposes:
+#if 0
+   mfem::out << "Destroying the MemoryManager ...\n"
+             << "remaining registered pointers : "
+             << maps->memories.size() << '\n'
+             << "remaining registered aliases  : "
+             << maps->aliases.size() << '\n';
+#endif
   for (auto& n : maps->memories)
   {
      internal::Memory &mem = n.second;
@@ -1554,7 +1628,6 @@ int MemoryManager::PrintAliases(std::ostream &out)
      out << "\nalias: key " << n.first << ", "
          << "h_ptr " << alias.mem->h_ptr << ", "
          << "offset " << alias.offset << ", "
-          << "bytes  " << alias.bytes << ", "
          << "counter " << alias.counter;
      n_out++;
   }
@@ -1569,7 +1642,13 @@ int MemoryManager::CompareHostAndDevice_(void *h_ptr, size_t size,
                 mm.GetAliasDevicePtr(h_ptr, size, false) :
                 mm.GetDevicePtr(h_ptr, size, false);
   char *h_buf = new char[size];
+#ifdef MFEM_USE_CUDA
   CuMemcpyDtoH(h_buf, d_ptr, size);
+#elif MFE_USE_HIP
+   HipMemcpyDtoH(h_buf, d_ptr, size);
+#else
+   std::memcpy(h_buf, d_ptr, size);
+#endif
   int res = std::memcmp(h_ptr, h_buf, size);
   delete [] h_buf;
   return res;
@@ -1591,13 +1670,24 @@ void MemoryPrintFlags(unsigned flags)
         << std::endl;
 }

-void MemoryManager::CheckHostMemoryType_(MemoryType h_mt, void *h_ptr)
+void MemoryManager::CheckHostMemoryType_(MemoryType h_mt, void *h_ptr,
+                                         bool alias)
 {
   if (!mm.exists) {return;}
-   const bool known = mm.IsKnown(h_ptr);
-   const bool alias = mm.IsAlias(h_ptr);
-   if (known) { MFEM_VERIFY(h_mt == maps->memories.at(h_ptr).h_mt,""); }
-   if (alias) { MFEM_VERIFY(h_mt == maps->aliases.at(h_ptr).mem->h_mt,""); }
+   if (!alias)
+   {
+      auto it = maps->memories.find(h_ptr);
+      MFEM_VERIFY(it != maps->memories.end(),
+                  "host pointer is not registered: h_ptr = " << h_ptr);
+      MFEM_VERIFY(h_mt == it->second.h_mt, "host pointer MemoryType mismatch");
+   }
+   else
+   {
+      auto it = maps->aliases.find(h_ptr);
+      MFEM_VERIFY(it != maps->aliases.end(),
+                  "alias pointer is not registered: h_ptr = " << h_ptr);
+      MFEM_VERIFY(h_mt == it->second.h_mt, "alias pointer MemoryType mismatch");
+   }
 }

 MemoryManager mm;
@@ -17,6 +17,9 @@
 #include <cstring> // std::memcpy
 #include <type_traits> // std::is_const
 #include <cstddef> // std::max_align_t
+#ifdef MFEM_USE_MPI
+#include <HYPRE_config.h> // HYPRE_USING_CUDA
+#endif

 namespace mfem
 {
@@ -90,6 +93,9 @@ inline bool IsDeviceMemory(MemoryType mt)
 /// Return a suitable MemoryType for a given MemoryClass.
 MemoryType GetMemoryType(MemoryClass mc);

+/// Return true iff the MemoryType @a mt is contained in the MemoryClass @a mc.
+bool MemoryClassContainsType(MemoryClass mc, MemoryType mt);
+
 /// Return a suitable MemoryClass from a pair of MemoryClass%es.
 /** Note: this operation is commutative, i.e. a*b = b*a, associative, i.e.
    (a*b)*c = a*(b*c), and has an identity element: MemoryClass::HOST.
@@ -463,6 +469,13 @@ public:
       returned. */
   inline MemoryType GetMemoryType() const;

+   /// Return the host MemoryType of the Memory object.
+   inline MemoryType GetHostMemoryType() const { return h_mt; }
+
+   /** @brief Return the device MemoryType of the Memory object. If the device
+       MemoryType is not set, return MemoryType::DEFAULT. */
+   inline MemoryType GetDeviceMemoryType() const;
+
   /** @brief Return true if host pointer is valid */
   inline bool HostIsValid() const;

@@ -481,8 +494,7 @@ public:
   /// Copy @a size entries from @a *this to @a dest.
   /** The given @a size should not exceed the Capacity() of @a *this and the
       destination, @a dest. */
-   inline void CopyTo(Memory &dest, int size) const
-   { dest.CopyFrom(*this, size); }
+   inline void CopyTo(Memory &dest, int size) const;

   /// Copy @a size entries from @a *this to the host pointer @a dest.
   /** The given @a size should not exceed the Capacity() of @a *this. */
@@ -596,9 +608,9 @@ private: // Static methods used by the Memory<T> class
                          bool own, bool alias, unsigned &flags);

   /// Register a pair of external host and device pointers
-   static void Register_(void *h_ptr, void *d_ptr, size_t bytes,
-                         MemoryType h_mt, MemoryType d_mt,
-                         bool own, bool alias, unsigned &flags);
+   static void Register2_(void *h_ptr, void *d_ptr, size_t bytes,
+                          MemoryType h_mt, MemoryType d_mt,
+                          bool own, bool alias, unsigned &flags);

   /// Register an alias. Note: base_h_ptr may be an alias.
   static void Alias_(void *base_h_ptr, size_t offset, size_t bytes,
@@ -635,13 +647,13 @@ private: // Static methods used by the Memory<T> class

   /// Return the type the of the currently valid memory.
   /// If more than one types are valid, return a device type.
-   static MemoryType GetDeviceMemoryType_(void *h_ptr);
+   static MemoryType GetDeviceMemoryType_(void *h_ptr, bool alias);

   /// Return the type the of the host memory.
   static MemoryType GetHostMemoryType_(void *h_ptr);

   /// Verify that h_mt and h_ptr's h_mt (memory or alias) are equal.
-   static void CheckHostMemoryType_(MemoryType h_mt, void *h_ptr);
+   static void CheckHostMemoryType_(MemoryType h_mt, void *h_ptr, bool alias);

   /// Copy entries from valid memory type to valid memory type.
   ///  Both dest_h_ptr and src_h_ptr are registered host pointers.
@@ -851,15 +863,21 @@ inline void Memory<T>::Wrap(T *ptr, int size, bool own)
 {
   h_ptr = ptr;
   capacity = size;
-   const size_t bytes = size*sizeof(T);
   flags = (own ? OWNS_HOST : 0) | VALID_HOST;
   h_mt = MemoryManager::GetHostMemoryType();
 #ifdef MFEM_DEBUG
   if (own && MemoryManager::Exists())
-   { MFEM_VERIFY(h_mt == MemoryManager::GetHostMemoryType_(h_ptr),""); }
+   {
+      MemoryType h_ptr_mt = MemoryManager::GetHostMemoryType_(h_ptr);
+      MFEM_VERIFY(h_mt == h_ptr_mt,
+                  "h_mt = " << (int)h_mt << ", h_ptr_mt = " << (int)h_ptr_mt);
+   }
 #endif
   if (own && h_mt != MemoryType::HOST)
-   { MemoryManager::Register_(ptr, ptr, bytes, h_mt, own, false, flags); }
+   {
+      const size_t bytes = size*sizeof(T);
+      MemoryManager::Register_(ptr, ptr, bytes, h_mt, own, false, flags);
+   }
 }

 template <typename T>
@@ -880,7 +898,7 @@ inline void Memory<T>::Wrap(T *ptr, int size, MemoryType mt, bool own)
   else
   {
      h_mt = MemoryManager::GetDualMemoryType(mt);
-      h_ptr = (h_mt == MemoryType::HOST) ? new T[size] : nullptr;
+      h_ptr = (h_mt == MemoryType::HOST) ? NewHOST(size) : nullptr;
   }
   flags = 0;
   h_ptr = (T*)MemoryManager::Register_(ptr, h_ptr, size*sizeof(T), mt,
@@ -897,23 +915,51 @@ inline void Memory<T>::Wrap(T *ptr, T *d_ptr, int size, MemoryType mt, bool own)
   MFEM_ASSERT(IsHostMemory(h_mt),"");
   const size_t bytes = size*sizeof(T);
   const MemoryType d_mt = MemoryManager::GetDualMemoryType(h_mt);
-   MemoryManager::Register_(h_ptr, d_ptr, bytes, h_mt, d_mt, own, false, flags);
+   MemoryManager::Register2_(h_ptr, d_ptr, bytes, h_mt, d_mt,
+                             own, false, flags);
 }

 template <typename T>
 inline void Memory<T>::MakeAlias(const Memory &base, int offset, int size)
 {
+   MFEM_ASSERT(0 <= offset, "invalid offset = " << offset);
+   MFEM_ASSERT(0 <= size, "invalid size = " << size);
+   MFEM_ASSERT(offset + size <= base.capacity,
+               "invalid offset + size = " << offset + size
+               << " > base capacity = " << base.capacity);
   capacity = size;
   h_mt = base.h_mt;
   h_ptr = base.h_ptr + offset;
   if (!(base.flags & REGISTERED))
-   { flags = (base.flags | ALIAS) & ~(OWNS_HOST | OWNS_DEVICE); }
-   else
   {
-      const size_t s_bytes = size*sizeof(T);
-      const size_t o_bytes = offset*sizeof(T);
-      MemoryManager::Alias_(base.h_ptr, o_bytes, s_bytes, base.flags, flags);
+      if (
+#ifndef HYPRE_USING_CUDA
+         // If the following condition is true then MemoryManager::Exists()
+         // should also be true:
+         IsDeviceMemory(MemoryManager::GetDeviceMemoryType())
+#else
+         // When HYPRE_USING_CUDA is defined we always register the 'base' if
+         // the MemoryManager::Exists():
+         MemoryManager::Exists()
+#endif
+      )
+      {
+         // Register 'base':
+         MemoryManager::Register_(base.h_ptr, nullptr, base.capacity*sizeof(T),
+                                  base.h_mt, base.flags & OWNS_HOST,
+                                  base.flags & ALIAS, base.flags);
+      }
+      else
+      {
+         // Copy the flags from 'base', setting the ALIAS flag to true, and
+         // setting both OWNS_HOST and OWNS_DEVICE to false:
+         flags = (base.flags | ALIAS) & ~(OWNS_HOST | OWNS_DEVICE);
+         return;
+      }
   }
+   const size_t s_bytes = size*sizeof(T);
+   const size_t o_bytes = offset*sizeof(T);
+   MemoryManager::Alias_(base.h_ptr, o_bytes, s_bytes, base.flags, flags);
 }

 template <typename T>
@@ -1069,7 +1115,14 @@ template <typename T>
 inline MemoryType Memory<T>::GetMemoryType() const
 {
   if (!(flags & VALID_DEVICE)) { return h_mt; }
-   return MemoryManager::GetDeviceMemoryType_(h_ptr);
+   return MemoryManager::GetDeviceMemoryType_(h_ptr, flags & ALIAS);
+}
+
+template <typename T>
+inline MemoryType Memory<T>::GetDeviceMemoryType() const
+{
+   if (!(flags & REGISTERED)) { return MemoryType::DEFAULT; }
+   return MemoryManager::GetDeviceMemoryType_(h_ptr, flags & ALIAS);
 }

 template <typename T>
@@ -1087,6 +1140,7 @@ inline bool Memory<T>::DeviceIsValid() const
 template <typename T>
 inline void Memory<T>::CopyFrom(const Memory &src, int size)
 {
+   MFEM_VERIFY(src.capacity>=size && capacity>=size, "Incorrect size");
   if (!(flags & REGISTERED) && !(src.flags & REGISTERED))
   {
      if (h_ptr != src.h_ptr && size != 0)
@@ -1106,6 +1160,7 @@ inline void Memory<T>::CopyFrom(const Memory &src, int size)
 template <typename T>
 inline void Memory<T>::CopyFromHost(const T *src, int size)
 {
+   MFEM_VERIFY(capacity>=size, "Incorrect size");
   if (!(flags & REGISTERED))
   {
      if (h_ptr != src && size != 0)
@@ -1122,9 +1177,17 @@ inline void Memory<T>::CopyFromHost(const T *src, int size)
   }
 }

+template <typename T>
+inline void Memory<T>::CopyTo(Memory &dest, int size) const
+{
+   MFEM_VERIFY(capacity>=size, "Incorrect size");
+   dest.CopyFrom(*this, size);
+}
+
 template <typename T>
 inline void Memory<T>::CopyToHost(T *dest, int size) const
 {
+   MFEM_VERIFY(capacity>=size, "Incorrect size");
   if (!(flags & REGISTERED))
   {
      if (h_ptr != dest && size != 0)
@@ -208,11 +208,14 @@ void AmgXSolver::DefaultParameters(const AMGX_MODE amgxMode_,
                    " \"config_version\": 2, \n"
                    " \"solver\": { \n"
                    "   \"solver\": \"AMG\", \n"
+                    "   \"scope\": \"main\", \n"
+                    "   \"smoother\": \"JACOBI_L1\", \n"
                    "   \"presweeps\": 1, \n"
-                    "   \"postsweeps\": 1, \n"
                    "   \"interpolator\": \"D2\", \n"
-                    "   \"max_iters\": 2, \n"
-                    "   \"convergence\": \"ABSOLUTE\", \n"
+                    "   \"max_row_sum\" : 0.9, \n"
+                    "   \"strength_threshold\" : 0.25, \n"
+                    "   \"postsweeps\": 1, \n"
+                    "   \"max_iters\": 1, \n"
                    "   \"cycle\": \"V\"";
      if (verbose)
      {
@@ -239,22 +242,21 @@ void AmgXSolver::DefaultParameters(const AMGX_MODE amgxMode_,
                    "     \"solver\": \"AMG\", \n"
                    "     \"smoother\": { \n"
                    "     \"scope\": \"jacobi\", \n"
-                    "     \"solver\": \"BLOCK_JACOBI\", \n"
-                    "     \"relaxation_factor\": 0.7 \n"
+                    "     \"solver\": \"JACOBI_L1\" \n"
                    "       }, \n"
                    "     \"presweeps\": 1, \n"
                    "     \"interpolator\": \"D2\", \n"
                    "     \"max_row_sum\" : 0.9, \n"
                    "     \"strength_threshold\" : 0.25, \n"
-                    "     \"max_iters\": 2, \n"
+                    "     \"max_iters\": 1, \n"
                    "     \"scope\": \"amg\", \n"
                    "     \"max_levels\": 100, \n"
                    "     \"cycle\": \"V\", \n"
                    "     \"postsweeps\": 1 \n"
                    "    }, \n"
                    "  \"solver\": \"PCG\", \n"
-                    "  \"max_iters\": 100, \n"
-                    "  \"convergence\": \"RELATIVE_MAX\", \n"
+                    "  \"max_iters\": 150, \n"
+                    "  \"convergence\": \"RELATIVE_INI_CORE\", \n"
                    "  \"scope\": \"main\", \n"
                    "  \"tolerance\": 1e-12, \n"
                    "  \"monitor_residual\": 1, \n"
@@ -97,11 +97,6 @@ void BlockOperator::Mult (const Vector & x, Vector & y) const
   {
      yblock.GetBlock(iRow).SyncAliasMemory(y);
   }
-
-   // Destroy alias vectors to prevent dangling aliases when the base vectors
-   // are deleted
-   for (int i=0; i < xblock.NumBlocks(); ++i) { xblock.GetBlock(i).Destroy(); }
-   for (int i=0; i < yblock.NumBlocks(); ++i) { yblock.GetBlock(i).Destroy(); }
 }

 // Action of the transpose operator
@@ -133,11 +128,6 @@ void BlockOperator::MultTranspose (const Vector & x, Vector & y) const
   {
      yblock.GetBlock(iRow).SyncAliasMemory(y);
   }
-
-   // Destroy alias vectors to prevent dangling aliases when the base vectors
-   // are deleted
-   for (int i=0; i < xblock.NumBlocks(); ++i) { xblock.GetBlock(i).Destroy(); }
-   for (int i=0; i < yblock.NumBlocks(); ++i) { yblock.GetBlock(i).Destroy(); }
 }

 BlockOperator::~BlockOperator()
@@ -208,11 +198,6 @@ void BlockDiagonalPreconditioner::Mult (const Vector & x, Vector & y) const
   {
      yblock.GetBlock(i).SyncAliasMemory(y);
   }
-
-   // Destroy alias vectors to prevent dangling aliases when the base vectors
-   // are deleted
-   for (int i=0; i < xblock.NumBlocks(); ++i) { xblock.GetBlock(i).Destroy(); }
-   for (int i=0; i < yblock.NumBlocks(); ++i) { yblock.GetBlock(i).Destroy(); }
 }

 // Action of the transpose operator
@@ -244,11 +229,6 @@ void BlockDiagonalPreconditioner::MultTranspose (const Vector & x,
   {
      yblock.GetBlock(i).SyncAliasMemory(y);
   }
-
-   // Destroy alias vectors to prevent dangling aliases when the base vectors
-   // are deleted
-   for (int i=0; i < xblock.NumBlocks(); ++i) { xblock.GetBlock(i).Destroy(); }
-   for (int i=0; i < yblock.NumBlocks(); ++i) { yblock.GetBlock(i).Destroy(); }
 }

 BlockDiagonalPreconditioner::~BlockDiagonalPreconditioner()
@@ -51,6 +51,12 @@ public:
    */
   BlockOperator(const Array<int> & row_offsets, const Array<int> & col_offsets);

+   /// Copy assignment is not supported
+   BlockOperator &operator=(const BlockOperator &) = delete;
+
+   /// Move assignment is not supported
+   BlockOperator &operator=(BlockOperator &&) = delete;
+
   //! Add block op in the block-entry (iblock, iblock).
   /**
    * iblock: The block will be inserted in location (iblock, iblock).
--- a/Show More
+++ b/Show More