Merge branch 'eigen-dev' of github.com:mfem/mfem into eigen-dev

Minors.
Fixed MFEM build dir path.
2021-09-03 12:38:04 +02:00 · 2021-09-03 12:27:49 +02:00 · 2021-09-03 12:27:49 +02:00 · 2021-09-03 12:27:49 +02:00 · 2021-09-03 12:27:10 +02:00 · 2021-09-03 12:27:10 +02:00
252 changed files with 22350 additions and 4187 deletions
@@ -47,26 +47,36 @@ jobs:
    strategy:
      matrix:
        os: [ubuntu-18.04, macos-10.15]
-        target: [debug, optim]
-        mpi: [sequential, parallel]
+        target: [dbg, opt]
+        mpi: [seq, par]
        build-system: [make]
-        # 'include' allows us to
-        # - add a variable without creating a new matrix dimension.
-        # - add a new combination ('build-system: cmake' case here)
+        hypre-target: [int32]
+        # 'include' allows us to:
+        # - Add a variable to all jobs without creating a new matrix dimension.
+        #   Codecov is defined that way.
+        # - Add a new combination.
+        #   'build-system: cmake' and 'hypre-target: int64'
        #
        # note: we will gather coverage info for any non-debug run except the
        # CMake build.
        include:
-          - target: debug
+          - target: dbg
            codecov: NO
-          - target: optim
+          - target: opt
            codecov: YES
          - os: ubuntu-18.04
-            target: optim
+            target: opt
            codecov: NO
-            mpi: parallel
+            mpi: par
            build-system: cmake
-    name: ${{ matrix.os }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.build-system }}
+            hypre-target: int32
+          - os: ubuntu-18.04
+            target: opt
+            codecov: NO
+            mpi: par
+            build-system: make
+            hypre-target: int64
+    name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}

    runs-on: ${{ matrix.os }}

@@ -92,7 +102,7 @@ jobs:
    # TODO: It would be nice to have only one step, e.g. with a dedicated
    # action, but I (@adrienbernede) don't see how at the moment.
    - name: get MPI (Linux)
-      if: matrix.mpi == 'parallel' && matrix.os == 'ubuntu-18.04'
+      if: matrix.mpi == 'par' && matrix.os == 'ubuntu-18.04'
      run: |
        sudo apt-get install mpich libmpich-dev
        export MAKE_CXX_FLAG="MPICXX=mpic++"
@@ -103,11 +113,11 @@ jobs:
        sudo apt-get install lcov

    - name: Set up Homebrew
-      if: ( matrix.mpi == 'parallel' || matrix.codecov == 'YES' ) && matrix.os == 'macos-10.15'
+      if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-10.15'
      uses: Homebrew/actions/setup-homebrew@c4aafe8c4620bf08883dd4679c374f11e73329d3

    - name: get MPI (MacOS)
-      if: matrix.mpi == 'parallel' && matrix.os == 'macos-10.15'
+      if: matrix.mpi == 'par' && matrix.os == 'macos-10.15'
      run: |
        export HOMEBREW_NO_INSTALL_CLEANUP=1
        brew install openmpi
@@ -123,39 +133,40 @@ jobs:
    # Install will only run on cache miss.
    - name: cache hypre
      id: hypre-cache
-      if: matrix.mpi == 'parallel'
+      if: matrix.mpi == 'par'
      uses: actions/cache@v2
      with:
        path: ${{ env.HYPRE_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2
+        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.0

    - name: get hypre
-      if: matrix.mpi == 'parallel' && steps.hypre-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-hypre@v1.0
+      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true'
+      uses: mfem/github-actions/build-hypre@v2.0
      with:
-        hypre-archive: ${{ env.HYPRE_ARCHIVE }}
-        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
+        archive: ${{ env.HYPRE_ARCHIVE }}
+        dir: ${{ env.HYPRE_TOP_DIR }}
+        target: ${{ matrix.hypre-target }}

    # Get Metis through cache, or build it.
    # Install will only run on cache miss.
    - name: cache metis
      id: metis-cache
-      if: matrix.mpi == 'parallel'
+      if: matrix.mpi == 'par'
      uses: actions/cache@v2
      with:
        path: ${{ env.METIS_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2
+        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.0

    - name: install metis
-      if: matrix.mpi == 'parallel' && steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@v1.0
+      if: matrix.mpi == 'par' && steps.metis-cache.outputs.cache-hit != 'true'
+      uses: mfem/github-actions/build-metis@v2.0
      with:
-        metis-archive: ${{ env.METIS_ARCHIVE }}
-        metis-dir: ${{ env.METIS_TOP_DIR }}
+        archive: ${{ env.METIS_ARCHIVE }}
+        dir: ${{ env.METIS_TOP_DIR }}

    # MFEM build and test
    - name: build
-      uses: mfem/github-actions/build-mfem@v1.0
+      uses: mfem/github-actions/build-mfem@v2.0
      with:
        os: ${{ matrix.os }}
        target: ${{ matrix.target }}
@@ -168,17 +179,17 @@ jobs:

    # Run checks (and only checks) on debug targets
    - name: checks
-      if: matrix.build-system == 'make' && matrix.target == 'debug'
+      if: matrix.build-system == 'make' && matrix.target == 'dbg'
      run: |
        cd ${{ env.MFEM_TOP_DIR }}  && make check

    - name: unit tests
-      if: matrix.build-system == 'make' && matrix.target == 'optim'
+      if: matrix.build-system == 'make' && matrix.target == 'opt'
      run: |
        cd ${{ env.MFEM_TOP_DIR }} && make unittest

    - name: tests
-      if: matrix.build-system == 'make' && matrix.target == 'optim'
+      if: matrix.build-system == 'make' && matrix.target == 'opt'
      run: |
        cd ${{ env.MFEM_TOP_DIR }} && make test

@@ -190,8 +201,8 @@ jobs:
    # Code coverage (process and upload reports)
    - name: codecov
      if: matrix.codecov == 'YES'
-      uses: mfem/github-actions/upload-coverage@v1.0
+      uses: mfem/github-actions/upload-coverage@v2.0
      with:
-        name: ${{ matrix.os }}-${{ matrix.mpi }}
+        name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
        project_dir: ${{ env.MFEM_TOP_DIR }}
        directories: "fem general linalg mesh"
@@ -53,32 +53,33 @@ jobs:
      uses: actions/cache@v2
      with:
        path: ${{ env.HYPRE_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2
+        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.0

    - name: Get Hypre
      if: steps.hypre-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-hypre@master
+      uses: mfem/github-actions/build-hypre@v2.0
      with:
-        hypre-archive: ${{ env.HYPRE_ARCHIVE }}
-        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
+        archive: ${{ env.HYPRE_ARCHIVE }}
+        dir: ${{ env.HYPRE_TOP_DIR }}
+        target: int32

    - name: Cache Metis Install
      id: metis-cache
      uses: actions/cache@v2
      with:
        path: ${{ env.METIS_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2
+        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.0

    - name: Install Metis
      if: steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@master
+      uses: mfem/github-actions/build-metis@v2.0
      with:
-        metis-archive: ${{ env.METIS_ARCHIVE }}
-        metis-dir: ${{ env.METIS_TOP_DIR }}
+        archive: ${{ env.METIS_ARCHIVE }}
+        dir: ${{ env.METIS_TOP_DIR }}

    # MFEM build and test
    - name: build-mfem
-      uses: mfem/github-actions/build-mfem@master
+      uses: mfem/github-actions/build-mfem@v2.0
      with:
        os: ${{ runner.os }}
        target: optim
@@ -28,49 +28,24 @@ jobs:
        access_token: ${{ github.token }}
    - name: checkout mfem
      uses: actions/checkout@v2
-      with:
-        path: mfem

    - name: copyright check
      id: copyright
      run: |
-        cd mfem
-        if git grep -l "^\(#\|//\).*\(\-2020\|\ 2010,\)" > matches.txt
-        then
-          echo "Please update the following files to Copyright (c) 2010-2021:"
-          cat matches.txt
-          exit 1
-        else
-          echo "No outdated copyright found."
-        fi
+        ./config/githooks/pre-push --copyright
+
      continue-on-error: true

    - name: license check
      id: license
      run: |
-        cd mfem
-        if git grep -li "^\(#\|//\).*GNU\ Lesser\ General\ Public\ License" > matches.txt
-        then
-          echo "Please update the following files to the BSD-3 license:"
-          cat matches.txt
-          exit 1
-        else
-          echo "No GNU GPL license found."
-        fi
+        ./config/githooks/pre-push --license
      continue-on-error: true

    - name: release check
      id: release
      run: |
-        cd mfem
-        if git grep -l "^\(#\|//\).*LLNL\-CODE\-443211" > matches.txt
-        then
-          echo "Please update the following files to LLNL-CODE-806117:"
-          cat matches.txt
-          exit 1
-        else
-          echo "No outdated release number found."
-        fi
+        ./config/githooks/pre-push --release
      continue-on-error: true

    - name: wrap-up
@@ -100,8 +75,7 @@ jobs:

    - name: style check
      run: |
-        cd tests/scripts
-        ./runtest code-style
+        ./config/githooks/pre-push --style

  documentation:
    runs-on: ubuntu-18.04
@@ -133,5 +107,4 @@ jobs:
      run: |
        git fetch origin master:master
        git checkout -b gh-actions-branch-history
-        cd tests/scripts
-        ./runtest branch-history
+        ./config/githooks/pre-push --history
@@ -26,6 +26,7 @@ CMakeFiles/
 config/_config.hpp
 config/config.mk
 config/sample-runs-build.log
+config/user.mk
 doc/CodeDocumentation.conf
 doc/CodeDocumentation.html
 doc/CodeDocumentation
@@ -144,6 +145,14 @@ examples/petsc/velocity.*
 examples/petsc/elastic_energy.*
 examples/petsc/mode_*

+examples/arpack/ex11
+examples/arpack/mode_*
+examples/arpack/ex11.mesh
+
+examples/spectra/ex11
+examples/spectra/mode_*
+examples/spectra/ex11.mesh
+
 examples/pumi/ex1
 examples/pumi/ex[126]p
 examples/pumi/refined.mesh
@@ -251,6 +260,7 @@ miniapps/shifted/ParaViewDistance
 miniapps/shifted/diffusion
 miniapps/shifted/diffusion.mesh
 miniapps/shifted/diffusion.gf
+miniapps/shifted/ParaViewDiffusion

 miniapps/tools/display-basis
 miniapps/tools/load-dc
@@ -295,6 +305,7 @@ tests/unit/psedov_tests_*
 tests/unit/tmop_pa_tests_*
 tests/unit/ptmop_pa_tests_*
 tests/unit/ceed_tests
+tests/unit/debug_device_tests

 # Test script output
 tests/scripts/*.err
@@ -307,7 +318,13 @@ tests/convergence/prates
 tests/par-mesh-format/ex1p

 # VPATH builds
-build-*/*
+build-*/
+
+# User config
+user-*
+
+# VSCode
+.vscode 

 # PETSc automated build
 petsc-build/*
@@ -48,39 +48,59 @@ variables:
  TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
  TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
  AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
+  MFEM_DATA_REPO: https://github.com/mfem/data.git
  ARTIFACTS_DIR: artifacts
+  SLURM_OVERLAP: 1

-# The pipeline is divided into stages. Usually, these are also synchronization
-# points, however, we use "needs" keyword to express the DAG of jobs for more
-# efficiency.
-# - We use setup phase to download content outside of mfem directory.
+# The pipeline is divided into stages. Usually, jobs in a given stage wait for
+# the preceding stages to complete before to start. However, we sometimes use
+# the "needs" keyword and express the DAG of jobs for more efficiency.
+# - We use setup and setup_baseline phases to download content outside of mfem
+#   directory.
 # - Allocate/Release is where quartz resources are allocated/released once for all.
 # - Build and Test is where we build and MFEM for multiple toolchains.
 # - Baseline_checks gathers baseline-type test suites execution
 # - Baseline_publish, only available on master, allows to update baseline
 #   results
 stages:
+  - setup
  - q_allocate_resources
  - q_build_and_test
  - q_release_resources
  - l_build_and_test
  - c_build_and_test
-  - setup
+  - setup_baseline
  - baseline_check
  - baseline_to_autotest
  - baseline_publish

-# The setup job in setup stage don't rely on MFEM git repo. It prepares a
-# pipeline-wide working directory downloading/updating external repos.
-# TODO: updating tests and tpls is not necessary anymore since pipelines are
-# now using unique directories so repo are never shared with another pipeline.
-# This is not memory efficient (we keep a lot of data), hence this reminder.
-# Setup
+# setup clones the mfem/data repo in ${BUILD_ROOT}. The build_and_test script
+# then symlinks the repo to the parent directory of the MFEM source directory.
+# Unit tests that depend on the mfem/data repo will then detect that this
+# directory is present and be enabled.
 setup:
  tags:
    - shell
    - quartz
  stage: setup
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
+    - if [ ! -d data ]; then git clone ${MFEM_DATA_REPO}; fi
+
+# The setup_baseline job in setup stage_baseline doesn't rely on MFEM git repo.
+# It prepares a pipeline-wide working directory downloading/updating external
+# repos. TODO: updating tests and tpls is not necessary anymore since pipelines
+# are now using unique directories so repo are never shared with another
+# pipeline. This is not memory efficient (we keep a lot of data), hence this
+# reminder.
+# Note: This job can start immediately.
+setup_baseline:
+  tags:
+    - shell
+    - quartz
+  stage: setup_baseline
  variables:
    GIT_STRATEGY: none
  script:
@@ -106,10 +126,10 @@ setup:
  script:
    - srun -p mi60 -t 15 -N 1 tests/gitlab/build_and_test

-# Lassen uses a different job scheduler (spectrum lsf) that does not
-# allow pre-allocation the same way slurm does.
-# We use pdebug queue on lassen to speed-up the allocation.
-# However this would not be scalable to multiple builds.
+# Lassen uses a different job scheduler (spectrum lsf) that does not allow
+# pre-allocation the same way slurm does.  We use pdebug queue on lassen to
+# speed-up the allocation.  However this would not be scalable to multiple
+# builds.
 .build_blueos_3_ppc64le_ib_script:
  script:
    - lalloc 1 -W 30 -q pdebug tests/gitlab/build_and_test
@@ -22,12 +22,13 @@

 # Spack helped builds
 # Generic lassen build job, extending build script
+# Note: Lassen jobs can start as soon as the setup job is complete.
 .build_and_test_on_lassen:
  extends: [.build_blueos_3_ppc64le_ib_script, .on_lassen]
  stage: l_build_and_test
-  needs: []
+  needs: [setup]

 opt_mpi_cuda_xl_16_1_1_8:
  variables:
-    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=sm_70"
+    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=70"
  extends: .build_and_test_on_lassen
@@ -16,13 +16,13 @@
    - shell
    - quartz
  rules:
-    # Don’t run quartz jobs if...
+    # Don't run quartz jobs if...
    - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"'
      when: never
-    # Don’t run autotest update if...
+    # Don't run autotest update if...
    - if: '$CI_JOB_NAME =~ /update_autotest/ && $AUTOTEST != "YES"'
      when: never
-    # Don’t run autotest update if...
+    # Don't run autotest update if...
    - if: '$CI_JOB_NAME =~ /q_report/ && $AUTOTEST != "YES"'
      when: never
    # Report success on success status
@@ -37,6 +37,18 @@
    # Default is to run if previous stage succeeded
    - when: on_success

+# This is a yaml anchor, it can be used to avoid duplication like here.
+# The code below will simply be pasted wherever the anchor is placed.
+.safe_create_rundir: &safe_create_rundir |
+  if ! mkdir ${rundir}; then
+    n=1
+    while ! mkdir ${rundir}_${n}
+    do
+      n=$((n+1))
+    done
+    rundir=${rundir}_${n}
+  fi
+
 # Allocate
 q_allocate_resources:
  variables:
@@ -65,10 +77,11 @@ q_report_success:
  stage: q_release_resources
  script:
    - echo "Can only run if all the quartz jobs passed"
-    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
    - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - mkdir -p ${rundir}
+    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
+    - *safe_create_rundir
    - echo "The Quartz jobs were successful" > ${rundir}/gitlab.out
+    - echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
    - git add ${rundir}
    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
    - git push origin master
@@ -80,10 +93,11 @@ q_report_failure:
  stage: q_release_resources
  script:
    - echo "Runs if there was at least one failure on quartz"
-    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
    - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - mkdir -p ${rundir}
+    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
+    - *safe_create_rundir
    - echo "There was an error while running CI on Quartz" > ${rundir}/gitlab.err
+    - echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
    - cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
    - git add ${rundir}
    - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
@@ -136,24 +150,29 @@ opt_par_gcc_6_1_0_pumi:
    SPEC: "%gcc@6.1.0 +pumi"
  extends: .build_and_test_on_quartz

-# Baseline
+# Baseline jobs form an independent set of jobs. We use `needs:[]` to specify
+# that "setup-baseline" can start immediately. Then, we have to use needs for
+# each one of the baseline jobs, otherwise they will wait for the rest of the
+# pipeline.
+
 baselinecheck_mfem_intel_quartz:
  extends: [.baselinecheck_mfem, .on_quartz]
-  needs: [setup]
+  needs: [setup_baseline]

 update_autotest:
  extends: [.on_quartz]
  needs: [baselinecheck_mfem_intel_quartz]
  stage: baseline_to_autotest
  script:
-    - rundir="quartz/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
    - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - mkdir -p ${rundir}
+    - rundir="quartz/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
+    - *safe_create_rundir
    - cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
    # We create an autotest-email.html file, because that's how we signal that there was a diff (temporary).
    - |
      if [[ -f ${rundir}/*.err ]]
      then
+        echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/*.err
        cp ${rundir}/*.err ${rundir}/autotest-email.html
      fi
    - git add ${rundir}
@@ -1,469 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-language: cpp
-
-os: linux
-dist: bionic
-
-stages:
-  - checks
-  - tests
-  - optional
-
-env:
-  global:
-    - HYPRE_ARCHIVE=v2.19.0.tar.gz
-      HYPRE_URL=https://github.com/hypre-space/hypre/archive/$HYPRE_ARCHIVE
-      HYPRE_TOP_DIR=hypre-2.19.0
-
-jobs:
-  include:
-
-    # ========================
-    #         Checks
-    # ========================
-    # - code-style
-    # - documentation
-    # - gitignore
-
-    - stage: checks
-      os: linux
-      dist: xenial
-      name: "code-style"
-      addons:
-        apt:
-          packages:
-            - astyle=2.05.1-0ubuntu1
-      script:
-        - cd ${TRAVIS_BUILD_DIR}
-        - cd tests/scripts
-        - ./runtest code-style
-
-    - stage: checks
-      os: linux
-      name: "documentation"
-      addons:
-        apt:
-          packages:
-            - doxygen
-            - graphviz
-      script:
-        - cd ${TRAVIS_BUILD_DIR}
-        - cd tests/scripts
-        - ./runtest documentation
-
-    - stage: checks
-      os: linux
-      name: "gitignore"
-      addons:
-        apt:
-          packages:
-            - mpich
-            - libmpich-dev
-      env: MPI=YES
-      before_script:
-        - cd ${TRAVIS_BUILD_DIR}
-        - mpicxx -v
-        - make config MFEM_USE_MPI=YES MFEM_MPI_NP=2
-        - make all -j3
-        - make test-noclean
-      script:
-        - cd tests/scripts
-        - ./runtest gitignore
-      cache:
-        ccache: true
-        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
-          - $TRAVIS_BUILD_DIR/../metis-4.0
-      before_cache:
-        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
-          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
-          rm -f Lib/*.{c,o}
-
-    # ========================
-    #  Optional Checks/Tests
-    # ========================
-    # - branch-history
-
-    - stage: optional
-      name: "branch-history"
-      if: branch != next
-      # need full git history for the binary/big files check
-      git:
-        depth: false
-      script:
-        - cd ${TRAVIS_BUILD_DIR}
-        # update master
-        - git fetch origin master:master
-        # checkout a branch (otherwise Travis works in detached head)
-        - git checkout -b travis_tests
-        - cd tests/scripts
-        - ./runtest branch-history
-
-    # ========================
-    #       Linux tests
-    # ========================
-    # - serial + debug
-    # - serial
-    # - parallel + debug
-    # - parallel
-
-    - stage: tests
-      os: linux
-      compiler: gcc
-      name: "Linux: Serial + Debug"
-      env: DEBUG=YES
-           MPI=NO
-           CODECOV=NO
-           MFEM_TEST_TARGET=check
-      cache:
-        ccache: true
-
-    - os: linux
-      compiler: gcc
-      name: "Linux: Serial"
-      env: DEBUG=NO
-           MPI=NO
-           CODECOV=NO
-           MFEM_TEST_TARGET=test
-      cache:
-        ccache: true
-
-    - os: linux
-      compiler: gcc
-      name: "Linux: Parallel + Debug"
-      addons:
-        apt:
-          # sources:
-            # - ubuntu-toolchain-r-test
-          packages:
-            # GCC 4.9
-            # - g++-4.9
-            # MPICH
-            - mpich
-            - libmpich-dev
-            # OpenMPI
-            # - openmpi-bin
-            # - libopenmpi-dev
-      env: DEBUG=YES
-           MPI=YES
-           CODECOV=NO
-           MFEM_TEST_TARGET=check
-           NPROCS=2
-      cache:
-        ccache: true
-        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
-          - $TRAVIS_BUILD_DIR/../metis-4.0
-      before_cache:
-        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
-          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
-          rm -f Lib/*.{c,o}
-
-    - os: linux
-      compiler: gcc
-      name: "Linux: Parallel"
-      addons:
-        apt:
-          # sources:
-            # - ubuntu-toolchain-r-test
-          packages:
-            # GCC 4.9
-            # - g++-4.9
-            # MPICH
-            - mpich
-            - libmpich-dev
-            # OpenMPI
-            # - openmpi-bin
-            # - libopenmpi-dev
-      env: DEBUG=NO
-           MPI=YES
-           CODECOV=YES
-           MFEM_TEST_TARGET=test
-           NPROCS=2
-      cache:
-        ccache: true
-        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
-          - $TRAVIS_BUILD_DIR/../metis-4.0
-      before_cache:
-        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
-          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
-          rm -f Lib/*.{c,o}
-
-    - os: linux
-      compiler: gcc
-      name: "Linux: Parallel (cmake)"
-      addons:
-        apt:
-          packages:
-            - mpich
-            - libmpich-dev
-      env: MPI=YES
-           NPROCS=2
-      script:
-        - cd ${TRAVIS_BUILD_DIR}
-        - mkdir ${TRAVIS_BUILD_DIR}/build
-        - cd ${TRAVIS_BUILD_DIR}/build
-        - cmake ..
-             -DMFEM_USE_MPI=ON
-             -DHYPRE_DIR=${TRAVIS_BUILD_DIR}/../$HYPRE_TOP_DIR/src/hypre
-             -DMFEM_MPI_NP=$NPROCS
-        - make -j3 mfem examples
-        - cd ${TRAVIS_BUILD_DIR}/build/tests/unit
-        - make -j3
-        - ctest --output-on-failure
-      cache:
-        ccache: true
-        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
-          - $TRAVIS_BUILD_DIR/../metis-4.0
-      before_cache:
-        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
-          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
-          rm -f Lib/*.{c,o}
-
-    # ========================
-    #      Mac OS X tests
-    # ========================
-    # - serial + debug
-    # - serial
-    # - parallel + debug
-    # - parallel
-
-    - os: osx
-      osx_image: xcode11.2
-      compiler: clang
-      name: "Mac: Serial + Debug"
-      addons:
-        homebrew:
-          packages:
-            - ccache
-      env: DEBUG=YES
-           MPI=NO
-           CODECOV=NO
-           MFEM_TEST_TARGET=check
-      cache:
-        ccache: true
-
-    - os: osx
-      osx_image: xcode11.2
-      compiler: clang
-      name: "Mac: Serial"
-      addons:
-        homebrew:
-          packages:
-            - ccache
-      env: DEBUG=NO
-           MPI=NO
-           CODECOV=NO
-           MFEM_TEST_TARGET=test
-      cache:
-        ccache: true
-
-    - os: osx
-      osx_image: xcode11.2
-      compiler: clang
-      name: "Mac: Parallel + Debug"
-      addons:
-        homebrew:
-          packages:
-            - ccache
-      env: DEBUG=YES
-           MPI=YES
-           CODECOV=NO
-           MFEM_TEST_TARGET=check
-           NPROCS=4
-           TMPDIR=/tmp
-      cache:
-        ccache: true
-        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
-          - $TRAVIS_BUILD_DIR/../metis-4.0
-          - $HOME/local-cached
-      before_cache:
-        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
-          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
-          rm -f Lib/*.{c,o}
-
-    - os: osx
-      osx_image: xcode11.2
-      compiler: clang
-      name: "Mac: Parallel"
-      addons:
-        homebrew:
-          packages:
-            - ccache
-      env: DEBUG=NO
-           MPI=YES
-           CODECOV=YES
-           MFEM_TEST_TARGET=test
-           NPROCS=4
-           TMPDIR=/tmp
-      cache:
-        ccache: true
-        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
-          - $TRAVIS_BUILD_DIR/../metis-4.0
-          - $HOME/local-cached
-      before_cache:
-        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
-          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
-          rm -f Lib/*.{c,o}
-
-before_install:
-   # No addon for brew yet, have to install OSX packages this way.
-   # - if [ $TRAVIS_OS_NAME == "osx" ] && [ $MPI == "YES" ]; then
-   #      brew install open-mpi;
-   #   fi
-
-   # Disable ccache while building dependencies that are cached:
-   - echo "before \$PATH = $PATH";
-     export PATH=${PATH//\/usr\/lib\/ccache:/};
-     echo "after  \$PATH = $PATH"
-
-   # On Mac OS X, build and cache OpenMPI 2.1.6:
-   - if [ $TRAVIS_OS_NAME == "osx" ] && [ $MPI == "YES" ]; then
-        if [ ! -e $HOME/local-cached/bin/mpicc ]; then
-           mkdir -p $HOME/builds && cd $HOME/builds &&
-           wget https://download.open-mpi.org/release/open-mpi/v2.1/openmpi-2.1.6.tar.bz2 &&
-           tar jxf openmpi-2.1.6.tar.bz2 &&
-           mkdir openmpi-build && cd openmpi-build &&
-           ../openmpi-2.1.6/configure --prefix=$HOME/local-cached &&
-           make -j3 all && make install;
-        fi;
-        PATH=$HOME/local-cached/bin:$PATH;
-        cd $TRAVIS_BUILD_DIR;
-     fi
-
-   # Update environment to find g++ 4.9 installation first.
-   # - if [ $TRAVIS_OS_NAME == "linux" ]; then
-   #      mkdir -p latest-gcc-symlinks;
-   #      ln -s /usr/bin/g++-4.9 latest-gcc-symlinks/g++;
-   #      ln -s /usr/bin/gcc-4.9 latest-gcc-symlinks/gcc;
-   #      ln -s /usr/bin/gcov-4.9 latest-gcc-symlinks/gcov;
-   #      export PATH=$PWD/latest-gcc-symlinks:$PATH;
-   #   fi
-
-   # Install tool to upload code coverage reports to coveralls.io
-   - if [ "$CODECOV" == "YES" ]; then
-        export PYTHONUSERBASE=$HOME/local;
-        pip install --user cpp-coveralls;
-        pip install --user pyyaml;
-        PATH=$HOME/local/bin:$PATH;
-     fi
-
-install:
-   # Set MPI compilers, print compiler version
-   - if [ $MPI == "YES" ]; then
-        if [ "$TRAVIS_OS_NAME" == "linux" ]; then
-           export MPICH_CC="$CC";
-           export MPICH_CXX="$CXX";
-        else
-           export OMPI_CC="$CC";
-           export OMPI_CXX="$CXX";
-           mpic++ --showme:version;
-        fi;
-        mpic++ -v;
-     else
-        $CXX -v;
-     fi
-
-   # Back out of the mfem directory to install the libraries
-   - cd ..
-
-   # hypre
-   - if [ $MPI == "YES" ]; then
-        if [ ! -e $HYPRE_TOP_DIR/src/hypre/lib/libHYPRE.a ]; then
-           wget $HYPRE_URL;
-           rm -rf $HYPRE_TOP_DIR;
-           tar xvzf $HYPRE_ARCHIVE;
-           cd $HYPRE_TOP_DIR/src;
-           ./configure --disable-fortran CC=mpicc CXX=mpic++;
-           make -j3;
-           cd ../..;
-        else
-            echo "Reusing cached $HYPRE_TOP_DIR/";
-        fi;
-        ln -s $HYPRE_TOP_DIR hypre;
-     else
-        echo "Serial build, not using hypre";
-     fi
-
-   # METIS, use a mirror because the original source server is not always up.
-   # Original url:
-   #    http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/OLD/metis-4.0.3.tar.gz
-   - if [ $MPI == "YES" ]; then
-        if [ ! -e metis-4.0/libmetis.a ]; then
-           wget https://mfem.github.io/tpls/metis-4.0.3.tar.gz;
-           tar xvzf metis-4.0.3.tar.gz;
-           make -j3 -C metis-4.0.3/Lib CC="$CC" OPTFLAGS="-O2";
-           rm -rf metis-4.0;
-           mv metis-4.0.3 metis-4.0;
-        else
-           echo "Reusing cached metis-4.0/";
-        fi;
-     fi
-
-   # Re-enable ccache on linux; enable ccache on mac os:
-   - if [ $TRAVIS_OS_NAME == "linux" ]; then
-        export PATH="/usr/lib/ccache:$PATH";
-     else
-        if [ $TRAVIS_OS_NAME == "osx" ]; then
-           export PATH="/usr/local/opt/ccache/libexec:$PATH";
-        fi;
-     fi
-
-   - printf "which \$CC  = "; which $CC;
-     printf "which \$CXX = "; which $CXX
-
-script:
-   # Compiler
-   - if [ $MPI == "YES" ]; then
-        export MYCXX=mpic++;
-        export MAKE_CXX_FLAG=MPICXX=$MYCXX;
-     else
-        export MYCXX="$CXX";
-        export MAKE_CXX_FLAG=CXX=$MYCXX;
-     fi
-
-   # Print the compiler version
-   - $MYCXX -v
-
-   # Set some variables
-   - cd $TRAVIS_BUILD_DIR;
-     CPPFLAGS="";
-     SKIP_TEST_DIRS="";
-     if [ "$CODECOV" == "YES" ]; then
-        CPPFLAGS="--coverage -g";
-     fi;
-     if [ "$TRAVIS_OS_NAME" != "linux" ] || [ "$DEBUG" == "YES" ]; then
-        CPPFLAGS+=" -pedantic -Wall -Werror";
-     fi
-
-   # Configure the library
-   - make config MFEM_USE_MPI=$MPI MFEM_DEBUG=$DEBUG $MAKE_CXX_FLAG
-        MFEM_MPI_NP=$NPROCS CPPFLAGS="$CPPFLAGS"
-   # Show the configuration
-   - make info
-   # Build the library
-   - make -j3
-   # Build the examples and the miniapps
-   - make -j3 all
-   # Run tests
-   - make $MFEM_TEST_TARGET SKIP_TEST_DIRS="$SKIP_TEST_DIRS"
-
-after_success:
-   - if [ "$CODECOV" == "YES" ]; then
-        coveralls --include fem --include general --include linalg --include
-           mesh --exclude /usr --gcov-options '\-lp' --root $TRAVIS_BUILD_DIR;
-     fi
@@ -8,40 +8,90 @@
                               https://mfem.org


-Version 4.2.1 (development)
+Version 4.3.1 (development)
 ===========================
- Added initial support for GPU-accelerated versions of PETSc that works with
-  MFEM_USE_CUDA if PETSc has been configured with CUDA support. Examples 1 and 9
-  in the examples/petsc directory have been modified to work with --device cuda.
-  Examples with GAMG (ex1p) and SLEPc (ex11p) are also provided.
+- Added support for hr-adaptivity using TMOP-based error estimator.

- Memory management:
-  * Added method Device::SetMemoryTypes that can be used to change the default
-    host and device MemoryTypes before Device setup.
-  * In class MemoryManager, added methods GetDualMemoryType and
-    SetDualMemoryType; dual MemoryTypes are used to determine the second
-    MemoryType (host or device) when only one MemoryType is specified in methods
-    of class Memory.
-  * Added Memory constructor for setting both the host and device MemoryTypes.
-  * Switched the default behavior of device memory allocations so that they
-    are deferred until the device pointer is needed.
-  * Added a second Umpire device MemoryType, DEVICE_UMPIRE_2, with
-    corresponding allocator that can be set with the method
-    MemoryManager::SetUmpireDevice2AllocatorName.
-  * Added HOST_PINNED MemoryType and a pinned host allocator for CUDA and HIP.
+- Adding lowest order Nedelec and Raviart-Thomas basis functions on wedge
+  shaped elements.

- Added support for Caliper: a library to integrate performance profiling
-  capabilities into applications. See examples/caliper for more details.
+- Added initial support for meshes with pyramidal elements, including several
+  pyramidal meshes in the data/ directory and support for the lowest order H1,
+  Nedelec, Raviart-Thomas, and L2 basis functions on pyramids.

- Added support for explicit vectorization in the high-performance templated
-  code for Fujitsu's A64FX ARM microprocessor architecture.
+- Updated the hypre interface according to changes in hypre-2.22.1. The ADS
+  solver is now fully working on GPUs.

- Added AlgebraicCeedSolver that does matrix-free algebraic p-multigrid for
-  diffusion problems with the Ceed backend.
+- Tetrahedral meshes no longer need to be reordered to support high order
+  Nedelec basis functions. This will allow future support for Nedelec basis
+  functions on wedges and pyramids which are not amenable to reordering. The
+  ReorientTetMesh method of the Mesh and ParMesh classes has been deprecated.

- Introduced new options for the mesh-explorer miniapp to visualize the actual
-  element attributes in parallel meshes while retaining the visualization of
-  the domain decomposition.
+- Gmsh meshes where all elements have zero physical tag (the default Gmsh
+  output format if no physical groups are defined) are now successfully loaded,
+  and elements are reassigned attribute number 1.
+
+Version 4.3, released on July 29, 2021
+======================================
+
+Discretization improvements
+---------------------------
+- Variable order spaces, p- and hp-refinement. This is the initial (serial)
+  support for variable-order FiniteElementCollection and FiniteElementSpace.
+  The new method FiniteElementSpace::SetElementOrder can be called to set an
+  arbitrary order for each mesh element. The conforming interpolation matrix
+  will now automatically constrain p- and hp- interfaces, enabling general
+  hp-refinement in both 2D and 3D, on uniform or mixed NC meshes. Support for
+  parallel variable-order spaces will follow shortly.
+
+- Extended the support for field transfer between high-order and low-order
+  refined finite element spaces to include: dual fields and H1 fields (both
+  primary and dual). These are illustrated in the lor-transfer miniapp.
+
+- Improved libCEED integration, including support for VectorCoefficient,
+  ConvectionIntegrator, and VectorConvectionNLFIntegrator with libCEED backends.
+
+- Extending support for L2 basis functions using MapTypes VALUE and INTEGRAL in
+  linear interpolators and GridFunction "GetValue" methods.
+
+- Changed the interface for the error estimator and implemented the Kelly error
+  indicator for scalar-valued problems, supported in serial and parallel builds.
+
+- Added support for the "BR2" discontinuous Galerkin discretization for
+  diffusion via DGDiffusionBR2Integrator (see Example 14/14p).
+
+- Added convective and skew-symmetric integrators for the nonlinear term in the
+  Navier-Stokes equations.
+
+- Added new classes DenseSymmetricMatrix and SymmetricMatrixCoefficient for
+  efficient evaluation of symmetric matrix coefficients. This replaces the now
+  deprecated EvalSymmetric in MatrixCoefficient. Added DiagonalMatrixCoefficient
+  for clarity, which is a typedef of VectorCoefficient.
+
+- Added support for nonscalar coefficient with VectorDiffusionIntegrator.
+
+Linear and nonlinear solvers
+----------------------------
+- Added support for AMG preconditioners on GPUs based on the hypre library
+  (version 2.22.0 or later). These include BoomerAMG, AMS and ADS and most
+  MFEM examples that use hypre have been ported to support this functionality.
+  The GPU preconditioners require that both hypre and MFEM are built with CUDA
+  support. Hypre builds with CUDA and unified memory are also supported and
+  can be used with `-d cuda:uvm` as a command-line option.
+
+- Added support for AMG preconditioners for non-symmetric systems (e.g.
+  advection-dominated problems) using hypre's approximate ideal restriction
+  (AIR) AMG. Requires hypre version 2.14.0 or newer. Usage is illustrated in
+  example 9/9p.
+
+- Added new functionality for constructing low-order refined discretizations and
+  solvers, see the LORDiscretization and LORSolver classes. A new basis type for
+  H(curl) and H(div) spaces is introduced to give spectral equivalence. This
+  functionality is illustrated in the LOR solvers miniapp in miniapps/solvers.
+
+- Generalized the Multigrid class to support non-geometric multigrid. Previous
+  functionality, based on FiniteElementSpaceHierarchy, is now available in the
+  derived class GeometricMultigrid.

 - Introduced solver interface for linear problems with constraints, a few
  concrete solvers that implement the interface, and a demonstration of their
@@ -52,19 +102,18 @@ Version 4.2.1 (development)
  as described in Barker and Kolev 2020 (https://doi.org/10.1002/nla.2348). See
  Example 3p and linalg/auxiliary.?pp.

- Added a new miniapp block-solvers that compares the performance of various
-  solvers for mixed finite element discretization of the second order scalar
-  elliptic equations. Currently available solvers in the miniapp include a
-  block-diagonal preconditioner that is based on approximate Schur complement
-  (implemented in ex5p), and a newly implemented solver DivFreeSolver, which
-  exploits a multilevel decomposition of the Raviart-Thomas space and its
-  divergence-free subspace. See the miniapps/solvers directory for more details.
+- Improved interface for using the Ginkgo library, including: support for matrix-
+  free operators in Ginkgo solvers, new wrappers for Ginkgo preconditioners, HIP
+  support, and reduction of unnecessary data copies.

- Added a new miniapp for computing (signed) distance functions to a point
-  source or zero level set. See miniapps/shifted/distance.cpp.
+- Added initial support for hypre's mixed integer (mixedint) capability, which
+  uses different data types for local and global indices in order to save memory
+  in large problems. This capability requires that hypre was configured with the
+  --enable-mixedint option. Note that this option is currently tested only in
+  ex1p, ex3p, and ex4p, and may not work in more general settings.

- Added matrix-free GPU-enabled implementations of GradientInterpolator and
-  IdentityInterpolator.
+- Added AlgebraicCeedSolver that does matrix-free algebraic p-multigrid for
+  diffusion problems with the Ceed backend.

 - Added interface to MUMPS direct solver. Its usage is demonstrated in ex25p.
  See http://mumps.enseeiht.fr/ for more details. Supported versions >= 5.1.1.
@@ -72,6 +121,17 @@ Version 4.2.1 (development)
 - Added three ESDIRK time integrators: implicit trapezoid rule, L-stable
  ESDIRK-32, and A-stable ESDIRK-33.

+- Implemented a variable step-size IMEX (VSSIMEX) method for the Navier miniapp.
+
+- Implemented an adaptive linear solver tolerance option for NewtonSolver based
+  on the algorithm of Eisenstat and Walker.
+
+Meshing improvements
+--------------------
+- Added support for reading high-order Lagrange meshes in VTK format. Arbitrary-
+  orders and all element types are supported. See the VTK blog for more info:
+  https://blog.kitware.com/wp-content/uploads/2018/09/Source_Issue_43.pdf.
+
 - Introduced a new non-conforming mesh format that fixes known inconsistencies
  of legacy "MFEM mesh v1.1" NC format and works consistently in both serial and
  parallel. ParMesh::ParPrint can now print non-conforming AMR meshes that can
@@ -80,109 +140,26 @@ Version 4.2.1 (development)
  NC data files are compatible with serial code, e.g., can be viewed with serial
  GLVis. Loading of legacy NC mesh files is still supported.

- Added support for 1D non-conforming meshes (which can be useful for parallel
-  load balancing and derefinement).
-
- Added a "scaled Jacobian" visualization option in the Mesh Explorer miniapp to
-  help identify elements with poor mesh quality.
-
- Added support for the "BR2" discontinuous Galerkin discretization for
-  diffusion via DGDiffusionBR2Integrator (see Example 14/14p).
-
- Generalized the Multigrid class to support non-geometric multigrid. The
-  previous functionality, based on FiniteElementSpaceHierarchy, is now available
-  in the derived class GeometricMultigrid.
-
- Upgraded the Catch unit test framework from version 2.13.0 to version 2.13.2.
-
- The TMOP mesh optimization algorithms were extended to GPU:
-   - QualityMetric #1, #2, #7 and #77 are available in 2D, #302, #303, #315
-     and #321 in 3D
-   - Both AnalyticAdaptTC and DiscreteAdaptTC TargetConstructor are available
-   - Kernels for normalization and limiting have been added
-   - The AdvectorCG now also supports AssemblyLevel::PARTIAL
-
- Added a new command line boolean option (`--all`) to the unit tests to launch
-  *all* non-regression tests.
-
- Added support for different modes of QuadratureInterpolator on GPU.
-  The layout (QVectorLayout::byNODES|byVDIM) and the tensor products modes can
-  be enabled before calling the Mult, Values, Derivatives, PhysDerivatives and
-  Determinants methods.
-
- Implemented a filter method for the Navier miniapp to stabilize highly
-  turbulent flows in direct numerical simulation.
-
- Added HIP support to the CMake build system.
-
- Added support for reading high-order Lagrange meshes in VTK format. Arbitrary-
-  orders and all element types are supported. See the VTK blog for more info:
-  https://blog.kitware.com/wp-content/uploads/2018/09/Source_Issue_43.pdf.
-
- Added support for reading VTK meshes in XML format.
-
- Added partial assembly and device support to Example 25/25p, with diagonal
-  preconditioning.
-
- Implemented a variable step-size IMEX (VSSIMEX) method for the Navier miniapp.
+- Added FMS support (https://github.com/CEED/FMS) to mfem. FMS can represent
+  unstructured high-order meshes with general high-order finite element fields
+  on them. When enabled, mfem can convert data collections to/from FMS data
+  collections in memory. In addition, an FMS data collection class was added so
+  the convert-dc miniapp can read and generate data files in FMS format.

 - Added new mesh quality metrics and improved the untangling capabilities of the
  TMOP-based mesh optimization algorithms.

- Added convective and skew-symmetric integrators for the nonlinear term in the
-  Navier-Stokes equations.
-
- Added new miniapp directory mtop/ with optimization-oriented block parametric
-  non-linear form and abstract integrators. Two new miniapps, ParHeat and
-  SeqHeat, demonstrate parallel and sequential implementation of gradients
-  evaluation for linear diffusion with discrete density.
-
- Changed the interface for the error estimator.
-
- Implemented the Kelly error indicator for scalar-valued problems, supported
-  in serial and parallel builds.
-
- Added new classes DenseSymmetricMatrix and SymmetricMatrixCoefficient for
-  efficient evaluation of symmetric matrix coefficients. This replaces the now
-  deprecated EvalSymmetric in MatrixCoefficient. Added DiagonalMatrixCoefficient
-  for clarity, which is a typedef of VectorCoefficient.
-
- Added support for AMG preconditioners for non-symmetric systems (e.g.
-  advection-dominated problems) using hypre's approximate ideal restriction
-  (AIR) AMG. Requires hypre version 2.14.0 or newer. Usage is illustrated in
-  example 9/9p.
-
- Implemented an adaptive linear solver tolerance option for NewtonSolver based
-  on the algorithm of Eisenstat and Walker.
-
- Added support for nonscalar coefficient with VectorDiffusionIntegrator.
-
- Extending support for L2 basis functions using MapTypes VALUE and INTEGRAL in
-  linear interpolators and GridFunction "GetValue" methods.
-
- Variable order spaces, p- and hp-refinement. This is the initial (serial)
-  support for variable-order FiniteElementCollection and FiniteElementSpace.
-  The new method FiniteElementSpace::SetElementOrder can be called to set an
-  arbitrary order for each mesh element. The conforming interpolation matrix
-  will now automatically constrain p- and hp- interfaces, enabling general
-  hp-refinement in both 2D and 3D, on uniform or mixed NC meshes. Support for
-  parallel variable-order spaces will follow shortly.
+- The TMOP mesh optimization algorithms were extended to GPU:
+  * QualityMetric 1, 2, 7, 77 are available in 2D, 302, 303, 315, 321 in 3D
+  * Both AnalyticAdaptTC and DiscreteAdaptTC TargetConstructor are available
+  * Kernels for normalization and limiting have been added
+  * The AdvectorCG now also supports AssemblyLevel::PARTIAL

 - Added support for creating refined meshes for all element types (e.g. by
  splitting high-order elements into low-order refined elements), including
  mixed meshes. The LOR Transfer miniapp (miniapps/tools/lor-transfer.cpp) now
  supports meshes with any element geometry.

- Gitlab CI: use Spack (and Uberenv) to automate the build of TPLs.
-
- Added new miniapps demonstrating: 1) the use of GSLIB for overlapping grids,
-  see gslib/schwarz_ex1, and 2) coupling different physics in different domains,
-  see navier/cht. Note that gslib v1.0.7 is require (see INSTALL for details).
-
- Added a new, very simple example (ex0 and parallel version ex0p). This
-  example solves a simple Poisson problem using H1 elements (the same problem as
-  ex1), but is intended to be extremely simple and approachable for new users.
-
 - Meshes consisting of any type of elements (including mixed meshes) can be
  converted to all-simplex meshes using Mesh::MakeSimplicial.

@@ -195,56 +172,147 @@ Version 4.2.1 (development)
  requisite periodic vertex mappings can be created with
  Mesh::CreatePeriodicVertexMapping.

- Added support for transferring dual fields between high-order and low-order
-  refined finite element spaces using the transposed versions of the
-  L2ProjectionGridTransfer operators. This functionality is illustrated in the
-  lor-transfer miniapp.
-
- Improved interface for using the Ginkgo library, including: support for matrix-
-  free operators in Ginkgo solvers, new wrappers for Ginkgo preconditioners, HIP
-  support, and reduction of unnecessary data copies.
-
- Added initial support for hypre's mixed integer (mixedint) capability, which
-  uses different data types for local and global indices in order to save memory
-  in large problems. This capability requires that hypre was configured with the
-  --enable-mixedint option. Note that this option is currently tested only in
-  ex1p and may not work in more general settings.
-
- Added support for transferring fields (primary and dual) between high-order
-  and low-order refined H1 finite element spaces using the
-  L2ProjectionH1GridTransfer operators. This functionality is demonstrated
-  through the lor-transfer miniapp when run with the -h1 option.
-
- Added new functionality for constructing low-order refined discretizations and
-  solvers, see the LORDiscretization and LORSolver classes. A new basis type for
-  H(curl) and H(div) spaces is introduced to give spectral equivalence. This
-  functionality is illustrated in the LOR solvers miniapp in miniapps/solvers.
+- Added support for 1D non-conforming meshes (which can be useful for parallel
+  load balancing and derefinement).

 - Added sample meshes in the `data` subdirectory showing the reference elements
  of the six currently supported element types; ref-segment.mesh,
  ref-triangle.mesh, ref-square.mesh, ref-tetrahedron.mesh, ref-cube.mesh, and
  ref-prism.mesh.

+High-performance computing
+--------------------------
+- Added initial support for GPU-accelerated versions of PETSc that works with
+  MFEM_USE_CUDA if PETSc has been configured with CUDA support. Examples 1 and 9
+  in the examples/petsc directory have been modified to work with --device cuda.
+  Examples with GAMG (ex1p) and SLEPc (ex11p) are also provided.
+
+- Added support for explicit vectorization in the high-performance templated
+  code for Fujitsu's A64FX ARM microprocessor architecture.
+
+- Added support for different modes of QuadratureInterpolator on GPU.
+  The layout (QVectorLayout::byNODES|byVDIM) and the tensor products modes can
+  be enabled before calling the Mult, Values, Derivatives, PhysDerivatives and
+  Determinants methods.
+
+- Added method Device::SetMemoryTypes that can be used to change the default
+  host and device MemoryTypes before Device setup.
+
+- In class MemoryManager, added methods GetDualMemoryType and SetDualMemoryType;
+  dual MemoryTypes are used to determine the second MemoryType (host or device)
+  when only one MemoryType is specified in methods of class Memory.
+
+- Added Memory constructor for setting both the host and device MemoryTypes.
+
+- Switched the default behavior of device memory allocations so that they are
+  deferred until the device pointer is needed.
+
+- Added a second Umpire device MemoryType, DEVICE_UMPIRE_2, with corresponding
+  allocator that can be set with the method SetUmpireDevice2AllocatorName.
+
+- Added HOST_PINNED MemoryType and a pinned host allocator for CUDA and HIP.
+
+- Added matrix-free GPU-enabled implementations of GradientInterpolator and
+  IdentityInterpolator.
+
+New and updated examples and miniapps
+-------------------------------------
+- Added a new, very simple example (ex0 and parallel version ex0p). This example
+  solves a simple Poisson problem using H1 elements (the same problem as ex1),
+  but is intended to be extremely simple and approachable for new users.
+
+- Added new miniapps demonstrating: 1) the use of GSLIB for overlapping grids,
+  see gslib/schwarz_ex1, and 2) coupling different physics in different domains,
+  see navier/cht. Note that gslib v1.0.7 is require (see INSTALL for details).
+
+- Added a new miniapp for computing (signed) distance functions to a point
+  source or zero level set. See miniapps/shifted/distance.cpp.
+
 - Added a high-order extension of the shifted boundary method to solve PDEs on
  non body-fitted meshes. This is illustrated in the new Shifted Diffusion
  miniapp, see miniapps/shifted/diffusion.cpp.

+- Added new miniapp directory mtop/ with optimization-oriented block parametric
+  non-linear form and abstract integrators. Two new miniapps, ParHeat and
+  SeqHeat, demonstrate parallel and sequential implementation of gradients
+  evaluation for linear diffusion with discrete density.
+
+- Added a new miniapp block-solvers that compares the performance of various
+  solvers for mixed finite element discretization of the second order scalar
+  elliptic equations. Currently available solvers in the miniapp include a
+  block-diagonal preconditioner that is based on approximate Schur complement
+  (implemented in ex5p), and a newly implemented solver DivFreeSolver, which
+  exploits a multilevel decomposition of the Raviart-Thomas space and its
+  divergence-free subspace. See the miniapps/solvers directory for more details.
+
+- Introduced new options for the mesh-explorer miniapp to visualize the actual
+  element attributes in parallel meshes while retaining the visualization of the
+  domain decomposition.
+
+- Added partial assembly and device support to Example 25/25p, with diagonal
+  preconditioning.
+
+- Implemented a filter method for the Navier miniapp to stabilize highly
+  turbulent flows in direct numerical simulation.
+
+Improved testing
+----------------
+- Transitioned from Travis to GitHub Action for testing/CI on GitHub.
+
+- Use Spack (and Uberenv) to automate TPL building in LLNL GitLab tests.
+
+- Extended `make test` to include GPU tests when MFEM is built with CUDA or HIP
+  support.
+
+- Added a set of suggested git hooks for developers in config/githooks.
+
+- Added support for Caliper: a library to integrate performance profiling
+  capabilities into applications. See examples/caliper for more details.
+
+- Added a new command line boolean option (`--all`) to the unit tests to launch
+  *all* non-regression tests.
+
+- Upgraded the Catch unit test framework from version 2.13.0 to version 2.13.2.
+
+Miscellaneous
+-------------
+- The following integrations have updated minimum version requirements:
+  * CUDA >= 10.1.168
+  * Ginkgo >= 1.4.0
+  * GSLIB >= 1.0.7
+  * HIOP >= 0.4
+  * HYPRE >= 2.20.0 for mixedint support
+  * HYPRE >= 2.22.0 for CUDA support
+  * libCEED >= 0.8
+  * PETSc >= 3.15.0 for CUDA support
+  * RAJA >= 0.13.0
+  see INSTALL for more details.
+
+- Added a "scaled Jacobian" visualization option in the Mesh Explorer miniapp to
+  help identify elements with poor mesh quality.
+
+- Added support for reading VTK meshes in XML format.
+
 - Added makefile rule to generate TAGS table for vi or Emacs users.

-libCEED integration improvements
--------------------------------
- Refactor the libCEED integration
+- Added HIP support to the CMake build system.

- Add support for VectorCoefficient with libCEED backends.
+- Various other simplifications, extensions, and bugfixes in the code.

- Add support for ConvectionIntegrator, and VectorConvectionNLFIntegrator with
-  libCEED backends.
+API changes
+-----------
+- Added an abstract interface `mfem::FaceRestriction` for `H1FaceRestriction`
+  and `L2FaceRestriction`.
+  In order to conform with the semantic of `MultTranspose` in `mfem::Operator`,
+  `mfem::FaceRestriction::MultTranspose` now sets instead of adding values, and
+  `mfem::FaceRestriction::AddMultTranspose` should replace previous calls to
+  `mfem::FaceRestriction::MultTranspose`.


 Version 4.2, released on October 30, 2020
 =========================================

-High-Performance Computing
+High-performance computing
 --------------------------
 - Added support for explicit vectorization in the high-performance templated
  code, which can now take advantage of specific classes on the following
@@ -326,9 +394,6 @@ Linear and nonlinear solvers
  matrix with the function HypreParMatrixFromBlocks. This could be useful for
  solving block systems with parallel direct solvers such as STRUMPACK.

- Added CUDA support for SUNDIALS ODE integrators. See the updated SUNDIALS
-  modification of Example 9/9p.
-
 - Added wrappers for hypre's flexible GMRES solver and the new parallel ILU
  preconditioner. The latter requires hypre version 2.19.0 or later.

@@ -439,7 +504,7 @@ New and updated examples and miniapps
    L2, with partial assembly support in Example 24/24p.
  * Weak Dirichlet boundary conditions (Nitsche) to the NURBS miniapp.

-Data management and Visualization
+Data management and visualization
 ---------------------------------
 - Added support for ADIOS2 for parallel I/O with ParaView visualization. See
  Examples 5, 9, 12, 16. The classes adios2stream and ADIOS2DataCollection
@@ -0,0 +1,66 @@
+cff-version: 1.2.0
+message: "If you use MFEM, please cite it as follows."
+authors:
+- family-names: "MFEM Team"
+title: "MFEM: Modular Finite Element Methods [Software]"
+doi: 10.11578/dc.20171025.1248
+url: "https://mfem.org"
+preferred-citation:
+  type: article
+  authors:
+  - family-names: "Anderson"
+    given-names: "Robert"
+    orcid: "https://orcid.org/0000-0002-3508-9944"
+  - family-names: "Andrej"
+    given-names: "Julian"
+    orcid: "https://orcid.org/0000-0001-7661-4840"
+  - family-names: "Barker"
+    given-names: "Andrew"
+    orcid: "https://orcid.org/0000-0003-3572-911X"
+  - family-names: "Bramwell"
+    given-names: "Jamie"
+  - family-names: "Camier"
+    given-names: "Jean-Sylvain"
+    orcid: "https://orcid.org/0000-0003-2421-1999"
+  - family-names: "Cerveny"
+    given-names: "Jakub"
+    orcid: "https://orcid.org/0000-0003-4231-2531"
+  - family-names: "Dobrev"
+    given-names: "Veselin"
+    orcid: "https://orcid.org/0000-0003-1793-5622"
+  - family-names: "Dudouit"
+    given-names: "Yohann"
+    orcid: "https://orcid.org/0000-0001-5831-561X"
+  - family-names: "Fisher"
+    given-names: "Aaron"
+  - family-names: "Kolev"
+    given-names: "Tzanio"
+    orcid: "https://orcid.org/0000-0002-2810-3090"
+  - family-names: "Pazner"
+    given-names: "Will"
+    orcid: "https://orcid.org/0000-0003-4885-2934"
+  - family-names: "Stowell"
+    given-names: "Mark"
+    orcid: "https://orcid.org/0000-0002-5389-7435"
+  - family-names: "Tomov"
+    given-names: "Vladimir"
+    orcid: "https://orcid.org/0000-0002-1846-6816"
+  - family-names: "Akkerman"
+    given-names: "Ido"
+    orcid: "https://orcid.org/0000-0002-5937-0300"
+  - family-names: "Dahm"
+    given-names: "Johann"
+    orcid: "https://orcid.org/0000-0001-9657-3564"
+  - family-names: "Medina"
+    given-names: "David"
+  - family-names: "Zampini"
+    given-names: "Stefano"
+    orcid: "https://orcid.org/0000-0002-0435-0433"
+  doi: "10.1016/j.camwa.2020.06.009"
+  journal: "Computers \\& Mathematics with Applications"
+  month: 1
+  start: 42 # First page number
+  end: 74 # Last page number
+  title: "MFEM: A Modular Finite Element Methods Library"
+  volume: 81
+  year: 2021
@@ -16,7 +16,7 @@ set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH

 # Require C++11 and disable compiler-specific extensions
 set(CMAKE_CXX_STANDARD 11)
-if (MFEM_USE_GINKGO) 
+if (MFEM_USE_GINKGO)
  set(CMAKE_CXX_STANDARD 14)
 endif()
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -54,7 +54,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.2.1)
+set(${PROJECT_NAME}_VERSION 4.3.1)

 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -102,7 +102,7 @@ if (MFEM_USE_CUDA)
   endif()
   enable_language(CUDA)
   set(CMAKE_CUDA_STANDARD 11)
-   if (MFEM_USE_GINKGO) 
+   if (MFEM_USE_GINKGO)
     set(CMAKE_CUDA_STANDARD 14)
   endif()
   set(CMAKE_CUDA_STANDARD_REQUIRED ON)
@@ -246,6 +246,7 @@ if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
    message(FATAL_ERROR " *** MFEM_USE_LEGACY_OPENMP requires MFEM_THREAD_SAFE=ON.")
  endif()
  find_package(OpenMP REQUIRED)
+  set(OPENMP_LIBRARIES ${OpenMP_CXX_LIBRARIES})
 endif()

 # SuiteSparse (before SUNDIALS which may depend on KLU)
@@ -330,6 +331,10 @@ if (MFEM_USE_CONDUIT)
    find_package(Conduit REQUIRED conduit relay blueprint )
 endif()

+if (MFEM_USE_FMS)
+    find_package(FMS REQUIRED fms )
+endif()
+
 # Axom/Sidre
 if (MFEM_USE_SIDRE)
  find_package(Axom REQUIRED Axom)
@@ -424,9 +429,10 @@ endif()
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
 set(MFEM_TPLS MPI_CXX OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS PETSC
-    SLEPC MESQUITE MUMPS STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
+    SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB NETCDF
    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
    CUSPARSE MKL_CPARDISO AMGX CALIPER)
+
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")
@@ -445,6 +451,9 @@ include_directories(${TPL_INCLUDE_DIRS})
 if (OPENMP_FOUND)
  message(STATUS "MFEM: using package OpenMP")
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+  if (MFEM_USE_CUDA)
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=${OpenMP_CXX_FLAGS}")
+  endif()
 endif()

 message(STATUS "MFEM build type: CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}")
@@ -4,7 +4,9 @@

 <p align="center">
 <a href="https://github.com/mfem/mfem/blob/master/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-BSD-brightgreen.svg"></a>
-<a href="https://travis-ci.org/mfem/mfem"><img alt="Build Status" src="https://travis-ci.org/mfem/mfem.svg?branch=master"></a>
+<a href="https://github.com/mfem/mfem/actions?query=workflow%3Arepo-check+branch%3Amaster"><img alt="Repo check" src="https://github.com/mfem/mfem/actions/workflows/repo-check.yml/badge.svg?branch=master"></a>
+<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuild-analysis+branch%3Amaster"><img alt="Build Analysis" src="https://github.com/mfem/mfem/actions/workflows/mfem-analysis.yml/badge.svg?branch=master"></a>
+<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuilds-and-tests+branch%3Amaster"><img alt="Builds and Tests" src="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml/badge.svg?branch=master"></a>
 <a href="https://ci.appveyor.com/project/mfem/mfem"><img alt="Build Status" src="https://ci.appveyor.com/api/projects/status/19non9sqm6msi2wy?svg=true"></a>
 <a href="https://mfem.github.io/doxygen/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
 </p>
@@ -63,6 +65,8 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
  development branches off `mfem:master`.
 - Please follow the [developer guidelines](#developer-guidelines), in particular
  with regards to documentation and code styling.
+- Please do not commit large/binary files to the central repository (use a fork
+  instead).
 - Pull requests  should be issued toward `mfem:master`. Make sure
  to check the items off the [Pull Request Checklist](#pull-request-checklist).
 - When your contribution is fully working and ready to be reviewed, add
@@ -71,6 +75,7 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
  reviewers to evaluate the changes.
 - The reviewers have 3 weeks to evaluate the PR and work with the author to
  fix issues and implement improvements.
+- During review there should be no force pushes/rewriting history in the branch.
 - After approval, MFEM developers merge the PR manually in the [mfem:next branch](#masternext-workflow).
 - After a week of testing in `mfem:next`, the original PR is merged in `mfem:master`.
 - We use [milestones](https://github.com/mfem/mfem/milestones) to coordinate the
@@ -91,8 +96,8 @@ The MFEM source code has the following structure:
 ```
  .
  ├── config
-  │   └── cmake
-  │       └── ...
+  │   ├── cmake
+  │   └── githooks
  ├── data
  ├── doc
  ├── examples
@@ -129,10 +134,10 @@ The MFEM source code has the following structure:
  └── tests
      ├── convergence
      ├── gitlab
+      ├── mem_manager
      ├── par-mesh-format
      ├── scripts
      └── unit
-          └── ...
 ```

 #### Main directories and classes
@@ -363,6 +368,10 @@ Before you can start, you need a GitHub account, here are a few suggestions:
  two reviewers to evaluate the changes. The reviewers have 3 weeks to evaluate
  the PR and work with the author to implement improvements and fix issues.

+- Once the `ready-for-review` label has been applied and reviewers have been
+  assigned, the PR is considered under review. To help with the review process
+  there should be no force pushes/rewriting history in the branch.
+
 - After approval, the PR is [tested](#masternext-workflow) for a week with
  other approved PRs in the `mfem:next` branch.

@@ -370,16 +379,20 @@ Before you can start, you need a GitHub account, here are a few suggestions:
  `mfem:next`, see the [README](tests/scripts/README) file in that directory
  for more details.

- Track the Travis CI, Github Actions and Appveyor [continuous integration](#automated-testing)
+- Track the GitHub Actions and Appveyor [continuous integration](#automated-testing)
  builds at the end of the PR. These should generally run clean, so address any
  errors as soon as possible. Please ask if you are unsure how to do that.

- Note that some tests, such as the `branch-history` check in Travis and Github
-  Actions are safeguards that are allowed to fail in certain cases.
+- Note that some tests, such as the `branch-history` check in GitHub Actions
+  are safeguards that are allowed to fail in certain cases.

 - Other tests, such as the `code-style`, `documentation` and `gitignore`
-  checks in Travis and Github Actions enforce MFEM-specific rules which are
-  explained in the error messages and the `tests/scripts` directory.
+  checks in GitHub Actions enforce MFEM-specific rules which are explained in
+  the error messages and the `tests/scripts` directory.
+
+- Also note that the tests `branch-history` and `repos-checks` found in GitHub
+  Actions can be triggered automatically before each push using git hooks. See
+  the [git hooks README](config/githooks/README.md) for a detailed explanation.

 - If triggered, track the status of the LLNL GitLab tests. If failing, ask
  one of the _LLNL developers_ for details.
@@ -399,7 +412,7 @@ Before a PR can be merged, it should satisfy the following:
    - [ ] Does `make` or `cmake` have a new target?
    - [ ] Did the requirements or the installation process change? *(rare)*
 - [ ] Update continuous integration server configurations if necessary (e.g. with new version requirements for each of MFEM's dependencies)
-    - [ ] `.travis.yml`
+    - [ ] `.github`
    - [ ] `.appveyor.yml`
 - [ ] Update `.gitignore`:
    - [ ] Check if `make distclean; git status` shows any files that were generated from the source by the project (not an IDE) but we don't want to track in the repository.
@@ -516,7 +529,7 @@ MFEM uses a `master`/`next`-branch workflow as described below:
    - [ ] `doc/CodeDocumentation.conf.in`
 - [ ] Check that version requirements for each of MFEM's dependencies are documented in `INSTALL` and up-to-date
 - [ ] Check that continuous integration server configurations reflect the dependency version requirements of the new release
-    - [ ] `.travis.yml`
+    - [ ] `.github`
    - [ ] `.appveyor.yml`
 - [ ] Update the `CHANGELOG` to organize all release contributions
 - [ ] Review the whole source code once over
@@ -578,14 +591,17 @@ MFEM uses a `master`/`next`-branch workflow as described below:
 MFEM has several levels of automated testing running on GitHub, as well as on
 local Mac and Linux workstations, and Livermore Computing clusters at LLNL.

-### Linux and Mac smoke tests
-We use Travis CI and Github Actions to drive the default tests on the `master`
-and `next` branches. See the `.travis` file and the logs at
-[https://travis-ci.org/mfem/mfem](https://travis-ci.org/mfem/mfem).
+In addition, developers can set local git hooks to run some quick checks on
+commit or push, see the [README](config/githooks/README.md) in the `config/githooks`
+directory.

-Testing using Travis CI and Github Actions should be kept lightweight, as there
-is a time constraint on jobs. Two virtual machines are configured - Mac (OS X)
-and Linux.
+### Linux and Mac smoke tests
+We use GitHub Actions to drive the default tests on the `master` and `next`
+branches. See the `.github/workflows` files and the logs at
+[https://github.com/mfem/mfem/actions](https://github.com/mfem/mfem/actions).
+
+Testing using GitHub Actions should be kept lightweight, as there is a time
+constraint on jobs. Two virtual machines are configured - Mac (OS X) and Linux.

 - Tests on the `master` branch are triggered whenever a PR is issued on this branch.
 - Tests on the `next` branch are currently scheduled to run each night.
@@ -474,7 +474,7 @@ MFEM_USE_HIP = YES/NO
   Enables support for AMD devices in MFEM. HIP is a heterogeneous-compute
   interface for portability developed by AMD that can target both AMD and
   NVIDIA GPUs. The variable HIP_ARCH is used to specify the AMD GPU processor
-   used during compilation (by default, HIP_ARCH=gfx900).  When enabled, this
+   used during compilation (by default, HIP_ARCH=gfx900). When enabled, this
   option uses the HIP_* build options, see below.

 MFEM_USE_RAJA = YES/NO
@@ -516,6 +516,13 @@ MFEM_USE_CALIPER = YES/NO
   profiling at runtime with Caliper's configuration API. Alternatively, one
   can configure Caliper through environment variables or config files.

+MFEM_USE_FMS = YES/NO
+   Enables support for the FMS library which consists of the DataCollection
+   sub-class mfem::FMSDataCollection for I/O in FMS formats, see the header file
+   fem/fmsdatacollection.hpp. In addition, this option enables in-memory
+   convetion routines between FMS's FmsDataCollection structure and MFEM's
+   DataCollection class, see the header file fem/fmsconvert.hpp.
+
 MFEM_BUILD_TAG = (any value)
   An optional tag to characterize the build. Exported to config/config.mk.
   Can be used to identify the MFEM build from other makefiles.
@@ -540,8 +547,9 @@ The specific libraries and their options are:
  See also the "Specific options for hypre" section at the end of this file.
  URL: https://github.com/hypre-space/hypre and https://www.llnl.gov/casc/hypre
  Options: HYPRE_OPT, HYPRE_LIB.
-  Versions: HYPRE >= 2.10.0b,
-            HYPRE >= 2.20.0 for '--enable-mixedint' support.
+  Versions: HYPRE >= 2.10.0b  (HYPRE built without CUDA)
+            HYPRE >= 2.20.0   (HYPRE built with '--enable-mixedint')
+            HYPRE >= 2.22.1   (HYPRE built with CUDA)

 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
  MFEM_USE_METIS_5 = YES (default is to use METIS 4).
@@ -615,7 +623,7 @@ The specific libraries and their options are:
  and dependencies of specific modules, see the Ginkgo webpage below.
  URL: https://ginkgo-project.github.io
  Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or Debug).
-  Versions: Ginkgo >= 1.4.0. 
+  Versions: Ginkgo >= 1.4.0.

 - AmgX (optional), used when MFEM_USE_AMGX = YES.
  URL: https://github.com/NVIDIA/AMGX
@@ -753,6 +761,11 @@ The specific libraries and their options are:
  URL: https://zlib.net
  Options: ZLIB_OPT, ZLIB_LIB.

+- FMS (optional), used when MFEM_USE_FMS = YES.
+  URL: https://github.com/CEED/FMS
+  Options: FMS_OPT, FMS_LIB.
+  Versions: FMS >= 0.2.
+
 Building with CMake
 ===================
 The MFEM build system consists of two steps: configuration and compilation.
@@ -884,6 +897,7 @@ MFEM_USE_RAJA
 MFEM_USE_UMPIRE
 MFEM_USE_SIDRE
 MFEM_USE_CALIPER
+MFEM_USE_FMS

 The following options are CMake specific:

@@ -938,6 +952,7 @@ The CMake build system adds auto-detection for the following packages/libraries:
 - UMPIRE
 - AXOM - Used when MFEM_USE_SIDRE is enabled
 - CALIPER
+ - FMS

 The following built-in CMake packages are also used:

@@ -955,7 +970,7 @@ config/config.hpp.in:

   cp config/config.hpp.in config/_config.hpp

-The file config/_config.hpp can then be edited to enable desired options.  The
+The file config/_config.hpp can then be edited to enable desired options. The
 MFEM library is simply a combination of all object files obtained by compiling
 the .cpp source files in the source directories: general, linalg, mesh, and fem.

@@ -963,7 +978,7 @@ the .cpp source files in the source directories: general, linalg, mesh, and fem.
 Specifying an MPI job launcher
 ==============================
 By default, MFEM will use 'mpirun -np #' to launch any of its parallel tests or
-miniapps, where # is the number of MPI tasks.  An alternate MPI launcher can be
+miniapps, where # is the number of MPI tasks. An alternate MPI launcher can be
 provided by setting the MFEM_MPIEXEC and MFEM_MPIEXEC_NP config variables.

 MFEM will expect the launcher command, plus the command line option to allow it
@@ -256,6 +256,10 @@ IF (DEFINED TPL_ENABLE_SIDRE)
  SET(MFEM_USE_SIDRE ${TPL_ENABLE_SIDRE} CACHE BOOL "Enable Axom/Sidre usage" FORCE)
 ENDIF()

+IF (DEFINED TPL_ENABLE_FMS)
+  SET(MFEM_USE_FMS ${TPL_ENABLE_FMS} CACHE BOOL "Enable FMS usage" FORCE)
+ENDIF()
+
 IF (DEFINED TPL_ENABLE_CONDUIT)
  SET(MFEM_USE_CONDUIT ${TPL_ENABLE_CONDUIT} CACHE BOOL "Enable Conduit usage" FORCE)
 ENDIF()
@@ -44,6 +44,7 @@ set(MFEM_USE_PETSC @MFEM_USE_PETSC@)
 set(MFEM_USE_SLEPC @MFEM_USE_SLEPC@)
 set(MFEM_USE_MPFR @MFEM_USE_MPFR@)
 set(MFEM_USE_SIDRE @MFEM_USE_SIDRE@)
+set(MFEM_USE_FMS @MFEM_USE_FMS@)
 set(MFEM_USE_CONDUIT @MFEM_USE_CONDUIT@)
 set(MFEM_USE_PUMI @MFEM_USE_PUMI@)
 set(MFEM_USE_CUDA @MFEM_USE_CUDA@)
@@ -119,6 +119,9 @@
 // Enable the use of SIMD in the high performance templated classes
 #cmakedefine MFEM_USE_SIMD

+// Enable MFEM functionality based on the FMS library
+#cmakedefine MFEM_USE_FMS
+
 // Enable MFEM functionality based on Conduit
 #cmakedefine MFEM_USE_CONDUIT

@@ -0,0 +1,20 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - FMS_FOUND
+#   - FMS_LIBRARIES
+#   - FMS_INCLUDE_DIRS
+
+include(MfemCmakeUtilities)
+mfem_find_package(FMS FMS FMS_DIR
+  "include" fms.h "lib" fms
+  "Paths to headers required by FMS." "Libraries required by FMS.")
@@ -91,6 +91,12 @@
 // Enable MFEM functionality based on the SuiteSparse library.
 // #define MFEM_USE_SUITESPARSE

+// Enable MFEM functionality based on the ARPACK library.
+// #define MFEM_USE_ARPACK
+
+// Enable MFEM functionality based on the SPECTRA library.
+// #define MFEM_USE_SPECTRA
+
 // Enable MFEM functionality based on the SuperLU library.
 // #define MFEM_USE_SUPERLU
 // #define MFEM_USE_SUPERLU5
@@ -117,6 +123,9 @@
 // Enable the use of SIMD in the high performance templated classes
 // #define MFEM_USE_SIMD

+// Enable FMS support
+// #define MFEM_USE_FMS
+
 // Enable Conduit support
 // #define MFEM_USE_CONDUIT

@@ -31,6 +31,8 @@ MFEM_TIMER_TYPE        = @MFEM_TIMER_TYPE@
 MFEM_USE_SUNDIALS      = @MFEM_USE_SUNDIALS@
 MFEM_USE_MESQUITE      = @MFEM_USE_MESQUITE@
 MFEM_USE_SUITESPARSE   = @MFEM_USE_SUITESPARSE@
+MFEM_USE_ARPACK        = @MFEM_USE_ARPACK@
+MFEM_USE_SPECTRA       = @MFEM_USE_SPECTRA@
 MFEM_USE_SUPERLU       = @MFEM_USE_SUPERLU@
 MFEM_USE_SUPERLU5      = @MFEM_USE_SUPERLU5@
 MFEM_USE_MUMPS         = @MFEM_USE_MUMPS@
@@ -43,6 +45,7 @@ MFEM_USE_PETSC         = @MFEM_USE_PETSC@
 MFEM_USE_SLEPC         = @MFEM_USE_SLEPC@
 MFEM_USE_MPFR          = @MFEM_USE_MPFR@
 MFEM_USE_SIDRE         = @MFEM_USE_SIDRE@
+MFEM_USE_FMS           = @MFEM_USE_FMS@
 MFEM_USE_CONDUIT       = @MFEM_USE_CONDUIT@
 MFEM_USE_PUMI          = @MFEM_USE_PUMI@
 MFEM_USE_HIOP          = @MFEM_USE_HIOP@
@@ -45,6 +45,7 @@ option(MFEM_USE_PETSC "Enable PETSc support." OFF)
 option(MFEM_USE_SLEPC "Enable SLEPc support." OFF)
 option(MFEM_USE_MPFR "Enable MPFR usage." OFF)
 option(MFEM_USE_SIDRE "Enable Axom/Sidre usage" OFF)
+option(MFEM_USE_FMS "Enable FMS usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
 option(MFEM_USE_PUMI "Enable PUMI" OFF)
 option(MFEM_USE_HIOP "Enable HiOp" OFF)
@@ -96,6 +97,11 @@ set(HYPRE_DIR "${MFEM_DIR}/../hypre/src/hypre" CACHE PATH
 # If hypre was compiled to depend on BLAS and LAPACK:
 # set(HYPRE_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
 #     "Packages that HYPRE depends on.")
+if (MFEM_USE_CUDA)
+   # This is only necessary when hypre is built with cuda:
+   set(HYPRE_REQUIRED_LIBRARIES "-lcusparse" "-lcurand" CACHE STRING
+       "Libraries that HYPRE depends on.")
+endif()

 set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library.")

@@ -132,10 +138,10 @@ set(MUMPS_DIR "${MFEM_DIR}/../MUMPS_5.2.0" CACHE PATH
    "Path to the MUMPS library.")
 # Packages required by MUMPS, depending on how it was compiled.
 set(MUMPS_REQUIRED_PACKAGES "MPI" "BLAS" "METIS" "ScaLAPACK" CACHE STRING
-    "Additional packages required by MUMPS.")    
+    "Additional packages required by MUMPS.")
 # If the MPI package does not find all required Fortran libraries:
 # set(MUMPS_REQUIRED_LIBRARIES "gfortran" "mpi_mpifh" CACHE STRING
-#     "Additional libraries required by MUMPS.")    
+#     "Additional libraries required by MUMPS.")

 set(STRUMPACK_DIR "${MFEM_DIR}/../STRUMPACK-build" CACHE PATH
    "Path to the STRUMPACK library.")
@@ -187,6 +193,12 @@ set(SLEPC_ARCH "arch-linux2-c-debug" CACHE STRING "SLEPC build architecture.")

 set(MPFR_DIR "" CACHE PATH "Path to the MPFR library.")

+set(FMS_DIR "${MFEM_DIR}/../fms" CACHE PATH
+    "Path to the FMS library.")
+# If FMS is built with Conduit:
+# set(FMS_REQUIRED_PACKAGES "Conduit/relay" CACHE STRING
+#     "Additional packages required by FMS.")
+
 set(CONDUIT_DIR "${MFEM_DIR}/../conduit" CACHE PATH
    "Path to the Conduit library.")

@@ -136,6 +136,7 @@ MFEM_USE_PETSC         = NO
 MFEM_USE_SLEPC         = NO
 MFEM_USE_MPFR          = NO
 MFEM_USE_SIDRE         = NO
+MFEM_USE_FMS           = NO
 MFEM_USE_CONDUIT       = NO
 MFEM_USE_PUMI          = NO
 MFEM_USE_HIOP          = NO
@@ -150,6 +151,8 @@ MFEM_USE_UMPIRE        = NO
 MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
 MFEM_USE_MKL_CPARDISO  = NO
+MFEM_USE_ARPACK        = NO
+MFEM_USE_SPECTRA       = NO

 # MPI library compile and link flags
 # These settings are used only when building MFEM with MPI + HIP
@@ -174,6 +177,10 @@ LIBUNWIND_LIB = $(if $(NOTMAC),-lunwind -ldl,)
 HYPRE_DIR = @MFEM_DIR@/../hypre/src/hypre
 HYPRE_OPT = -I$(HYPRE_DIR)/include
 HYPRE_LIB = -L$(HYPRE_DIR)/lib -lHYPRE
+ifeq (YES,$(MFEM_USE_CUDA))
+   # This is only necessary when hypre is built with cuda:
+   HYPRE_LIB += -lcusparse -lcurand
+endif

 # METIS library configuration
 ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK)$(MFEM_USE_MUMPS),NONONO)
@@ -323,6 +330,19 @@ NETCDF_LIB = $(XLINKER)-rpath,$(NETCDF_DIR)/lib -L$(NETCDF_DIR)/lib\
 $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib\
 -lnetcdf -lhdf5_hl -lhdf5 $(ZLIB_LIB)

+# ARPACK library configuration
+ARPACK_DIR = @MFEM_DIR@/../ARPACK
+ARPACK_OPT = -I$(ARPACK_DIR)
+ARPACK_LIB = -L$(ARPACK_DIR) -lparpack -larpack
+
+# EIGEN library configuration
+EIGEN_DIR = @MFEM_DIR@/../eigen
+EIGEN_OPT = -I$(EIGEN_DIR)
+
+# SPECTRA library configuration
+SPECTRA_DIR = @MFEM_DIR@/../spectra/include
+SPECTRA_OPT = -I$(SPECTRA_DIR) $(EIGEN_OPT)
+
 # PETSc library configuration (version greater or equal to 3.8 or the dev branch)
 PETSC_ARCH := arch-linux2-c-debug
 PETSC_DIR  := $(MFEM_DIR)/../petsc/$(PETSC_ARCH)
@@ -357,6 +377,11 @@ endif
 MPFR_OPT =
 MPFR_LIB = -lmpfr

+# FMS and required libraries configuration
+FMS_DIR = $(MFEM_DIR)/../fms
+FMS_OPT = -I$(FMS_DIR)/include
+FMS_LIB = -Wl,-rpath,$(FMS_DIR)/lib -L$(FMS_DIR)/lib -lfms
+
 # Conduit and required libraries configuration
 CONDUIT_DIR = @MFEM_DIR@/../conduit
 CONDUIT_OPT = -I$(CONDUIT_DIR)/include/conduit
@@ -0,0 +1,41 @@
+                    Finite Element Discretization Library
+                                   __
+                       _ __ ___   / _|  ___  _ __ ___
+                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
+                      | | | | | ||  _||  __/| | | | | |
+                      |_| |_| |_||_|   \___||_| |_| |_|
+
+                               https://mfem.org
+
+
+This directory contains recommended git hooks, which are scripts that can be
+used to improve your development experience with MFEM:
+
+### The hooks
+
+* `pre-commit` is a hook that will be applied before each commit and run
+`astyle` on the code. This will ensure that your changes comply with the MFEM
+code styling guidelines.
+
+* `pre-push` is a hook that will be applied before each push to run a quick set
+of tests that verify that your files headers are in compliance, and that you did
+not add any large files to the repo.
+
+### Setup
+
+To setup the git hooks, run `make hooks`, which creates symlinks to the hooks in
+the `.git/hooks` directory. Individual hooks can be enabled by manually creating
+symlinks.
+
+(You may also copy the scripts directly and customize them further, but this way
+you may miss additional updates in the future.)
+
+### Failures
+
+The `branch-history` check can fail in some cases when the history is OK. For
+example, when a large number of files were modified for a legitimate reason, or
+when a picture was added for documentation.
+
+If that is the case, make sure the failure is indeed justified, and rerun the
+push command with the `--no-verify` option. This will skip the hooks, allowing
+you to push those changes.
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+# Apply automated code formatting
+make -C $(git rev-parse --show-toplevel) style
@@ -0,0 +1,107 @@
+#!/bin/bash
+
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+option=${1:-""}
+
+if [[ "${option}" == "--help" ]]; then
+  echo "This script runs checks on the repository."
+  echo "It has 2 modes: with and without an option."
+  echo ""
+  echo "Options are used in GitHub Actions and can be:"
+  echo "  --copyright"
+  echo "  --license"
+  echo "  --release"
+  echo "  --style"
+  echo "  --history"
+  echo ""
+  echo "As a githook, the script is used without options."
+  echo "In that case, it will run all the checks except style."
+  echo ""
+  echo "Use --help to print this help message."
+fi
+
+cd $(git rev-parse --show-toplevel)
+
+# copyright check
+copyright=true
+if [[ "${option}" == "--copyright" || "${option}" == "" ]]; then
+  if git grep -l "^\(#\|//\).*\(\-2020\|\ 2010,\)" > matches.txt; then
+    echo "Please update the following files to Copyright (c) 2010-2021:"
+    cat matches.txt
+    copyright=false
+  fi
+fi
+
+# license check
+license=true
+if [[ "${option}" == "--license" || "${option}" == "" ]]; then
+  if git grep -li "^\(#\|//\).*GNU\ Lesser\ General\ Public\ License" > matches.txt; then
+    echo "Please update the following files to the BSD-3 license:"
+    cat matches.txt
+    license=false
+  fi
+fi
+
+# release check
+release=true
+if [[ "${option}" == "--release" || "${option}" == "" ]]; then
+  if git grep -l "^\(#\|//\).*LLNL\-CODE\-443211" > matches.txt
+  then
+    echo "Please update the following files to LLNL-CODE-806117:"
+    cat matches.txt
+    release=false
+  fi
+fi
+
+# wrap-up
+code=0
+if ! $copyright ; then
+  echo "copyright check failed, unroll log for details"
+  code=1
+fi
+if ! $license ; then
+  echo "license check failed, unroll log for details"
+  code=1
+fi
+if ! $release ; then
+  echo "release check failed, unroll log for details"
+  code=1
+fi
+
+# `code-style` is not just a check, it will actually reformat the code if
+# necessary. This means that if one pushes while the repo is in dirty state
+# (changes not staged), those changes may be mixed with format changes.
+# To activate this, you will need to hard-copy this hook script in the hook
+# directory and uncomment only then. (See README.md)
+#
+## style check
+#if [[ "${option}" == "--style" || "${option}" == "" ]]; then
+if [[ "${option}" == "--style" ]]; then
+  if which astyle && [[ "$(astyle --version)" == "Artistic Style Version 2.05.1" ]]; then
+    cd tests/scripts
+    if ! ./runtest code-style; then code=1; fi
+    cd -
+  else
+    echo "Warning: astyle not found or version is not 2.05.1"
+  fi
+fi
+
+# branch-history
+if [[ "${option}" == "--history" || "${option}" == "" ]]; then
+  git fetch origin master:master
+  cd tests/scripts
+  if ! ./runtest branch-history; then code=1; fi
+  cd -
+fi
+
+exit $code
@@ -57,22 +57,27 @@ TIMECMD := $(word 1,$(TIMECMD))
 ifneq (,$(filter test%,$(MAKECMDGOALS)))
   MAKEFLAGS += -k
 endif
-# Test runs of the examples/miniapps with parameters - check exit code
+# Test runs of the examples/miniapps with parameters - check exit code:
+# 0 means success, 255 means the test was skipped, anything else means error
 mfem-test = \
   printf "   $(3) [$(2) $(1) ... ]: "; \
   $(call $(TIMEFUN),$(TIMECMD),$(2) ./$(1) $(if $(5),,-no-vis )$(4) \
     > $(1).stderr 2>&1); \
-   if [ "$$3" = 0 ]; \
-   then $(PRINT_OK); else $(PRINT_FAILED); cat $(1).stderr; fi; \
-   rm -f $(1).stderr; exit $$3
+   err="$$3"; \
+   if [ "$$3" = 0 ]; then $(PRINT_OK); \
+   else if [ "$$3" = 255 ]; then $(PRINT_SKIP); err=0; \
+   else $(PRINT_FAILED); cat $(1).stderr; fi; fi; \
+   rm -f $(1).stderr; exit $$err

 # Test runs of the examples/miniapps - check exit code and if a file exists
+# See mfem-test for the interpretation of the error code
 mfem-test-file = \
   printf "   $(3) [$(2) $(1) ... ]: "; \
   $(call $(TIMEFUN),$(TIMECMD),$(2) ./$(1) -no-vis > $(1).stderr 2>&1); \
   err="$$3"; \
-   if [ "$$3" = 0 ] && [ -e $(4) ]; \
-   then $(PRINT_OK); else $(PRINT_FAILED); cat $(1).stderr; err=64; fi; \
+   if [ "$$3" = 0 ] && [ -e $(4) ]; then $(PRINT_OK); \
+   else if [ "$$3" = 255 ] && [ -e $(4) ]; then $(PRINT_SKIP); err=0; \
+   else $(PRINT_FAILED); cat $(1).stderr; err=64; fi; fi; \
   rm -f $(1).stderr; exit $$err

 .PHONY: test test-par-YES test-par-NO test-ser test-par test-clean test-print
@@ -80,6 +85,26 @@ mfem-test-file = \
 # What sets of tests to run in serial and parallel
 test-par-YES: $(PAR_$(MFEM_TESTS):=-test-par) $(SEQ_$(MFEM_TESTS):=-test-seq)
 test-par-NO:  $(SEQ_$(MFEM_TESTS):=-test-seq)
+ifeq ($(MFEM_USE_CUDA),YES)
+.PHONY: test-par-YES-cuda test-par-NO-cuda test-ser-cuda test-par-cuda test-cuda
+test-par-YES: test-par-YES-cuda
+test-par-NO:  test-par-NO-cuda
+test-par-YES-cuda: test-par-cuda test-ser-cuda
+test-par-NO-cuda:  test-ser-cuda
+test-ser-cuda: $(SEQ_DEVICE_$(MFEM_TESTS):=-test-seq-cuda)
+test-par-cuda: $(PAR_DEVICE_$(MFEM_TESTS):=-test-par-cuda)
+test-cuda: test-par-$(MFEM_USE_MPI)-cuda clean-exec
+endif
+ifeq ($(MFEM_USE_HIP),YES)
+.PHONY: test-par-YES-hip test-par-NO-hip test-ser-hip test-par-hip test-hip
+test-par-YES: test-par-YES-hip
+test-par-NO:  test-par-NO-hip
+test-par-YES-hip: test-par-hip test-ser-hip
+test-par-NO-hip:  test-ser-hip
+test-ser-hip: $(SEQ_DEVICE_$(MFEM_TESTS):=-test-seq-hip)
+test-par-hip: $(PAR_DEVICE_$(MFEM_TESTS):=-test-par-hip)
+test-hip: test-par-$(MFEM_USE_MPI)-hip clean-exec
+endif
 test-ser:     test-par-NO
 test-par:     test-par-YES
 test:         all test-par-$(MFEM_USE_MPI) clean-exec
@@ -0,0 +1,9 @@
+MFEM INLINE mesh v1.0
+
+type = pyramid
+nx = 4
+ny = 4
+nz = 4
+sx = 1.0
+sy = 1.0
+sz = 1.0
@@ -0,0 +1,43 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+# PYRAMID     = 7
+#
+
+dimension
+3
+
+elements
+2
+1 7 4 3 2 1 0
+1 7 1 2 3 4 5
+
+boundary
+8
+1 2 0 2 1
+2 2 0 3 2
+3 2 0 4 3
+4 2 0 1 4
+5 2 1 2 5
+6 2 2 3 5
+7 2 3 4 5
+8 2 4 1 5
+
+vertices
+6
+3
+0 0 -1
+1 0 0
+0 1 0
+-1 0 0
+0 -1 0
+0 0 1
@@ -0,0 +1,38 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+# PYRAMID     = 7
+#
+
+dimension
+3
+
+elements
+1
+1 7 0 1 2 3 4
+
+boundary
+5
+1 3 3 2 1 0
+2 2 0 1 4
+3 2 1 2 4
+4 2 2 3 4
+5 2 3 0 4
+
+vertices
+5
+3
+0 0 0
+1 0 0
+1 1 0
+0 1 0
+0 0 1
@@ -0,0 +1,47 @@
+Mesh.Algorithm = 6;
+
+lc = 0.1;
+Point(1) = {0.0,0.0,0.0,lc};
+Point(2) = {1,0.0,0.0,lc};
+Point(3) = {0,1,0.0,lc};
+Circle(1) = {2,1,3};
+Point(4) = {-1,0,0.0,lc};
+Point(5) = {0,-1,0.0,lc};
+Circle(2) = {3,1,4};
+Circle(3) = {4,1,5};
+Circle(4) = {5,1,2};
+Point(6) = {0,0,-1,lc};
+Point(7) = {0,0,1,lc};
+Circle(5) = {3,1,6};
+Circle(6) = {6,1,5};
+Circle(7) = {5,1,7};
+Circle(8) = {7,1,3};
+Circle(9) = {2,1,7};
+Circle(10) = {7,1,4};
+Circle(11) = {4,1,6};
+Circle(12) = {6,1,2};
+Curve Loop(13) = {2,8,-10};
+Surface(14) = {13};
+Curve Loop(15) = {10,3,7};
+Surface(16) = {15};
+Curve Loop(17) = {-8,-9,1};
+Surface(18) = {17};
+Curve Loop(19) = {-11,-2,5};
+Surface(20) = {19};
+Curve Loop(21) = {-5,-12,-1};
+Surface(22) = {21};
+Curve Loop(23) = {-3,11,6};
+Surface(24) = {23};
+Curve Loop(25) = {-7,4,9};
+Surface(26) = {25};
+Curve Loop(27) = {-4,12,-6};
+Surface(28) = {27};
+Surface Loop(29) = {28,26,16,14,20,24,22,18};
+Volume(30) = {29};
+
+Physical Surface(1) = {28,26,16,14,20,24,22,18};
+Physical Volume(2) = 30;
+
+// Generate 2D mesh
+Mesh 2;
+Mesh.MshFileVersion = 2.2;
@@ -0,0 +1,246 @@
+FMS: 100
+DataCollection/Name: star
+DataCollection/NumberOfFieldDescriptors: 1
+DataCollection/FieldDescriptors/0/Name: CoordsDescriptor
+DataCollection/FieldDescriptors/0/ComponentName: volume
+DataCollection/FieldDescriptors/0/Type: 0
+DataCollection/FieldDescriptors/0/FixedOrder/Size: 3
+DataCollection/FieldDescriptors/0/FixedOrder/Type: FMS_UINT64
+DataCollection/FieldDescriptors/0/FixedOrder/Values: [0, 1, 3]
+DataCollection/FieldDescriptors/0/NumDofs: 211
+DataCollection/NumberOfFields: 1
+DataCollection/Fields/0/Name: Coords
+DataCollection/Fields/0/LayoutType: 0
+DataCollection/Fields/0/NumberOfVectorComponents: 2
+DataCollection/Fields/0/FieldDescriptorName: CoordsDescriptor
+DataCollection/Fields/0/Data/Size: 422
+DataCollection/Fields/0/Data/Type: FMS_DOUBLE
+DataCollection/Fields/0/Data/Values: [-0.016886, 1.000000, 0.309017, 
+1.309020, -0.809017, -0.500000, 
+-0.809017, -1.618030, 0.309017, 
+-0.500000, 1.309020, 0.519420, 
+1.154510, 0.809019, 0.147680, 
+-0.095492, -0.654508, -0.415586, 
+-1.213520, -1.213520, -0.392210, 
+-0.654508, -0.095492, 0.139949, 
+0.809019, 1.154510, 0.660184, 
+-0.264063, -0.800064, -0.231060, 
+0.663691, 0.183114, 0.317639, 
+0.543082, 0.598483, 0.345112, 
+0.478298, 0.027703, 0.095229, 
+0.012368, -0.092534, -0.334412, 
+-0.313767, -0.140526, -0.293881, 
+-0.534056, -0.660290, -0.537646, 
+-0.655590, -0.121396, -0.274504, 
+-0.346497, -0.296570, 0.004737, 
+-0.098835, 0.069287, 0.082675, 
+0.318799, 0.467183, 0.564505, 
+0.595190, 0.846237, 0.671735, 
+1.051500, 1.103010, 0.964008, 
+0.821603, 1.257520, 1.206010, 
+1.142350, 0.975686, 0.781273, 
+0.717257, 0.475684, 0.642352, 
+0.268930, 0.211049, 0.174181, 
+0.039345, -0.147746, -0.177481, 
+-0.365164, -0.230328, -0.551503, 
+-0.603005, -0.497587, -0.389864, 
+-0.757514, -0.706011, -0.675487, 
+-0.528946, -0.943851, -1.078690, 
+-1.087600, -0.955467, -1.483190, 
+-1.348360, -1.483190, -1.348360, 
+-1.085930, -0.938010, -0.943851, 
+-1.078690, -0.681476, -0.540944, 
+-0.757514, -0.706011, -0.540614, 
+-0.367058, -0.551503, -0.603005, 
+-0.365164, -0.230328, -0.138552, 
+-0.206896, 0.174181, 0.039345, 
+0.268468, 0.222269, 0.475684, 
+0.642352, 0.759791, 0.719381, 
+1.142350, 0.975686, 1.257520, 
+1.206010, 0.972837, 0.836119, 
+1.051500, 1.103010, 0.214572, 
+0.407449, 0.288323, 0.449827, 
+-0.086700, -0.027358, -0.200560, 
+-0.166595, -0.271802, -0.418426, 
+-0.426131, -0.551441, -0.096117, 
+-0.206969, -0.027946, -0.184969, 
+0.211136, 0.260131, 0.407172, 
+0.430781, 0.718277, 0.885068, 
+0.753103, 0.957692, 0.866273, 
+1.024530, 0.934099, 1.093820, 
+0.348422, 0.524463, 0.404903, 
+0.587376, 0.054525, 0.146431, 
+-0.078026, -0.007795, -0.329488, 
+-0.302967, -0.488115, -0.439332, 
+-0.498515, -0.453527, -0.633059, 
+-0.570251, -0.655787, -0.791132, 
+-0.802013, -0.956872, -1.094720, 
+-1.208560, -1.207720, -1.339910, 
+-0.693371, -0.795404, -0.803594, 
+-0.945068, -0.464668, -0.631721, 
+-0.458968, -0.547876, -0.332878, 
+-0.485696, -0.286385, -0.408481, 
+0.053021, -0.058200, 0.110846, 
+-0.002086, 0.381384, 0.416784, 
+0.551133, 0.613261, 0.872474, 
+0.901208, 1.038300, 1.084660, 
+0.737459, 0.751250, 0.890002, 
+0.915210, 0.010915, 0.000000, 
+0.951057, 0.951057, 0.587785, 
+1.538840, -0.587785, 0.000000, 
+-0.951057, -1.538840, -0.951057, 
+-0.015847, 0.475529, 0.951057, 
+0.492248, 1.244950, 1.063310, 
+0.274399, 0.293893, -0.293892, 
+-0.296404, -1.063310, -1.244950, 
+-0.453865, -0.951057, -0.475529, 
+0.466620, 0.792932, -0.013913, 
+-0.748783, -0.497528, 0.021382, 
+-0.017158, 0.172591, 0.330125, 
+0.458568, 0.457971, 0.137740, 
+0.299049, 0.588394, 0.667324, 
+0.432341, 0.634346, 0.117322, 
+0.193603, 0.211702, 0.098278, 
+-0.199438, -0.077304, -0.082243, 
+-0.216296, -0.458634, -0.592374, 
+-0.563926, -0.680404, -0.135751, 
+-0.302942, -0.469005, -0.453640, 
+-0.182727, -0.314240, 0.024270, 
+0.021546, 0.158510, 0.317019, 
+0.485799, 0.492951, 0.792548, 
+0.634038, 0.951057, 0.951057, 
+0.777915, 0.613430, 0.951057, 
+0.951057, 0.793994, 0.635800, 
+1.049020, 1.146990, 1.084480, 
+0.924310, 1.440880, 1.342910, 
+1.380330, 1.221820, 0.948209, 
+0.856297, 0.746293, 0.904802, 
+0.476242, 0.393234, 0.489821, 
+0.391857, 0.194471, 0.075751, 
+0.097964, 0.195929, -0.097964, 
+-0.195928, -0.173234, -0.078922, 
+-0.489821, -0.391856, -0.467007, 
+-0.397859, -0.746293, -0.904802, 
+-0.945206, -0.849559, -1.380330, 
+-1.221820, -1.440880, -1.342910, 
+-1.100830, -0.923191, -1.049020, 
+-1.146990, -0.774515, -0.621542, 
+-0.951057, -0.951057, -0.803055, 
+-0.635255, -0.951057, -0.951057, 
+-0.792548, -0.634038, -0.454301, 
+-0.479369, -0.158510, -0.317019, 
+0.149331, 0.178643, 0.295860, 
+0.306275, 0.246225, 0.406610, 
+0.367954, 0.498458, -0.014929, 
+0.084917, -0.094272, -0.000726, 
+-0.245374, -0.340755, -0.435351, 
+-0.490564, -0.176355, -0.294974, 
+-0.148010, -0.328246, 0.136248, 
+0.182883, 0.328957, 0.309903, 
+0.646446, 0.622546, 0.800859, 
+0.801803, 0.643087, 0.654473, 
+0.796963, 0.816799, 0.748189, 
+0.898148, 0.819092, 0.970033, 
+1.049540, 1.162660, 1.115310, 
+1.266700, 0.539960, 0.728396, 
+0.656318, 0.783152, 0.302716, 
+0.403858, 0.219006, 0.309252, 
+0.009667, 0.107161, -0.118211, 
+0.011786, -0.284704, -0.179858, 
+-0.411567, -0.296676, -0.570168, 
+-0.635348, -0.726784, -0.793211, 
+-1.016860, -1.112970, -1.170910, 
+-1.281530, -0.737860, -0.851723, 
+-0.883732, -0.995070, -0.653153, 
+-0.769939, -0.631918, -0.797688, 
+-0.616289, -0.806819, -0.638485, 
+-0.790356, -0.136399, -0.322769, 
+-0.165339, -0.309622]
+DataCollection/Mesh/PartitionInfo/Size: 2
+DataCollection/Mesh/PartitionInfo/Type: FMS_UINT64
+DataCollection/Mesh/PartitionInfo/Values: [0, 1]
+DataCollection/Mesh/NumDomainNames: 1
+DataCollection/Mesh/NumComponents: 1
+DataCollection/Mesh/NumTags: 0
+DataCollection/Mesh/DomainNames/0/Name: Domain
+DataCollection/Mesh/DomainNames/0/NumDomains: 1
+DataCollection/Mesh/DomainNames/0/Domains/0/Dimension: 2
+DataCollection/Mesh/DomainNames/0/Domains/0/NumVertices: 31
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/EntityType: FMS_EDGE
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/NumEntities: 50
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Size: 100
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Type: FMS_INT32
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Values: [11, 0, 26, 
+11, 26, 14, 
+14, 0, 27, 
+14, 27, 17, 
+17, 0, 28, 
+17, 28, 20, 
+20, 0, 29, 
+20, 29, 23, 
+23, 0, 30, 
+23, 30, 11, 
+11, 1, 12, 
+1, 26, 12, 
+12, 3, 13, 
+3, 26, 13, 
+13, 2, 14, 
+2, 15, 2, 
+27, 15, 15, 
+5, 16, 5, 
+27, 16, 16, 
+4, 17, 4, 
+18, 4, 28, 
+18, 18, 7, 
+19, 7, 28, 
+19, 19, 6, 
+20, 6, 21, 
+6, 29, 21, 
+21, 9, 22, 
+9, 29, 22, 
+22, 8, 23, 
+8, 24, 8, 
+30, 24, 24, 
+10, 25, 10, 
+30, 25, 25, 1]
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/EntityType: FMS_QUADRILATERAL
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/NumEntities: 20
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Size: 80
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Type: FMS_INT32
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Values: [0, 1, 2, 
+3, 3, 4, 
+5, 6, 6, 
+7, 8, 9, 
+9, 10, 11, 
+12, 12, 13, 
+14, 0, 15, 
+16, 17, 1, 
+17, 18, 19, 
+20, 2, 20, 
+21, 22, 22, 
+23, 24, 4, 
+24, 25, 26, 
+27, 5, 27, 
+28, 29, 29, 
+30, 31, 7, 
+31, 32, 33, 
+34, 8, 34, 
+35, 36, 36, 
+37, 38, 10, 
+38, 39, 40, 
+41, 11, 41, 
+42, 43, 43, 
+44, 45, 13, 
+45, 46, 47, 
+48, 14, 48, 
+49, 15]
+DataCollection/Mesh/Components/0/Name: volume
+DataCollection/Mesh/Components/0/Dimension: 2
+DataCollection/Mesh/Components/0/NumEntities: 20
+DataCollection/Mesh/Components/0/Coordinates: Coords
+DataCollection/Mesh/Components/0/NumParts: 1
+DataCollection/Mesh/Components/0/Parts/0/DomainName: Domain
+DataCollection/Mesh/Components/0/Parts/0/DomainID: 0
+DataCollection/Mesh/Components/0/Parts/0/FullDomain: Yes
+DataCollection/Mesh/Components/0/Relations/Size: 0
+DataCollection/Mesh/Components/0/Relations/Type: FMS_UINT64
@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.

-PROJECT_NUMBER         = v4.2.1
+PROJECT_NUMBER         = v4.3.1

 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -84,8 +84,9 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})

-  if (NOT (${TEST_NAME} MATCHES "ex0p?"))
-    set(THIS_TEST_OPTIONS "-no-vis")
+  set(THIS_TEST_OPTIONS "-no-vis")
+  if (${TEST_NAME} MATCHES "ex0p?")
+    set(THIS_TEST_OPTIONS)
  endif()
  if (${TEST_NAME} MATCHES "ex10p*")
    list(APPEND THIS_TEST_OPTIONS "-tf" "5")
@@ -107,6 +108,34 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
  endif()
 endforeach()

+# Add CUDA/HIP tests.
+set(DEVICE_EXAMPLES
+    # serial examples with device support:
+    ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
+    # parallel examples with device support:
+    ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p ex24p ex25p ex26p)
+set(MFEM_TEST_DEVICE)
+if (MFEM_USE_CUDA)
+  set(MFEM_TEST_DEVICE "cuda")
+elseif (MFEM_USE_HIP)
+  set(MFEM_TEST_DEVICE "hip")
+endif()
+if (MFEM_TEST_DEVICE)
+  foreach(TEST_NAME ${DEVICE_EXAMPLES})
+    set(THIS_TEST_OPTIONS "-no-vis" "-d" "${MFEM_TEST_DEVICE}")
+    if (NOT (${TEST_NAME} MATCHES ".*p$"))
+      add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_ser
+        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+    elseif (MFEM_USE_MPI)
+      add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_np=${MFEM_MPI_NP}
+        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+        ${MPIEXEC_PREFLAGS}
+        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+        ${MPIEXEC_POSTFLAGS})
+    endif()
+  endforeach()
+endif()
+
 # If STRUMPACK is enabled, add a test run that uses it.
 if (MFEM_USE_STRUMPACK)
  add_test(NAME ex11p_strumpack_np=${MFEM_MPI_NP}
@@ -1,301 +0,0 @@
-
-#include "mesh_partition.hpp"
-
-
-Subdomain::Subdomain(const Mesh & mesh0_) 
-: mesh0(&mesh0_), dim(mesh0->Dimension()), sdim(mesh0->SpaceDimension()) 
-{ 
-   // MFEM_VERIFY(dim == 3, "Only 3D domains for now are supported");
-   if(mesh0->NURBSext)
-   {
-      MFEM_ABORT("Nurbs meshes are not supported yet");
-   }
-}
-
-void Subdomain::BuildSubMesh(const Array<int> & elems, const entity_type & etype)
-{
-   // if mesh nodes are defined we use them for the vertices
-   // otherwise we use the vertices them selfs
-   cout << "entity type " << etype << endl;
-
-   int nv = mesh0->GetNV();
-   int subelems = elems.Size();
-   Array<int> vmarker(nv); vmarker = 0;
-   int numvertices = 0;
-   for (int ie = 0; ie<subelems; ie++)
-   {
-      int el = elems[ie];
-      Array<int> vertices;
-      switch (etype)
-      {
-      case 0: mesh0->GetElementVertices(el,vertices); break;
-      case 1: mesh0->GetFaceVertices(el,vertices); break;
-      default:
-         MFEM_ABORT("Wrong entity type choice");
-         break;
-      }
-      for (int iv=0; iv<vertices.Size(); iv++)
-      {
-         int v = vertices[iv];
-         if (vmarker[v]) continue;
-         vmarker[v] = 1;
-         numvertices++;
-      }
-   }
-   cout << "Num of new vertices: " << numvertices << endl;
-
-   // Construct new mesh
-   Mesh * meshptr = nullptr;
-   switch (etype)
-   {
-   case 0: 
-      mesh = new Mesh(dim,numvertices, subelems); 
-      element_map = elems;
-      meshptr = mesh;
-      break;
-   case 1: 
-      surface_mesh = new Mesh(dim-1,numvertices, subelems,0,sdim);
-      surface_element_map = elems;
-      meshptr = surface_mesh;
-      break;
-      default:
-         MFEM_ABORT("Wrong entity type choice");
-         break;
-   }
-
-   Vector values;
-   const GridFunction * nodes0 = mesh0->GetNodes();
-   int vk = 0;
-   // if (nodes0) // this is NOT NEEDED here
-   // {
-   //    vcoords.SetSize(sdim, mesh0->GetNV());
-   //    for (int i = 0; i< sdim; i++)
-   //    {
-   //       nodes0->GetNodalValues(values,i+1);
-   //       vcoords.SetRow(i,values);
-   //       cout << "values size = " << values.Size() << endl;
-   //    }
-   //    for (int iv = 0; iv<mesh0->GetNV(); ++iv)
-   //    {
-   //       if (!vmarker[iv]) continue;
-   //       meshptr->AddVertex(vcoords.GetColumn(iv));
-   //       vmarker[iv] = ++vk;
-   //    }
-   // }
-   // else
-   {
-      for (int iv = 0; iv<mesh0->GetNV(); ++iv)
-      {
-         if (!vmarker[iv]) continue;
-         meshptr->AddVertex(mesh0->GetVertex(iv));
-         vmarker[iv] = ++vk;
-      }
-   }
-   // Add elements
-   for (int ie = 0; ie<subelems; ie++)
-   {
-      const Element * el = nullptr;
-      switch (etype)
-      {
-         case 0: el = mesh0->GetElement(elems[ie]); break;
-         case 1: el = mesh0->GetFace(elems[ie]); break;
-         default: MFEM_ABORT("Wrong entity type choice"); break;
-      }
-      Element * nel = meshptr->NewElement(el->GetGeometryType());
-      int nv0 = el->GetNVertices();
-      const int * v0 = el->GetVertices();
-      Array<int> v1(nv0);
-      for (int i=0; i<nv0; i++)
-      {
-         v1[i] = vmarker[v0[i]]-1;
-      }   
-      nel->SetVertices(v1.GetData());
-      meshptr->AddElement(nel);
-   }
-
-   meshptr->FinalizeTopology();
-   
-   if (nodes0)
-   {
-      cout << "nodes not null" << endl;
-      // Extract Nodes GridFunction and determine its type
-      const FiniteElementSpace * fes0 = nodes0->FESpace();
-
-      Ordering::Type ordering = fes0->GetOrdering();
-      int order = fes0->FEColl()->GetOrder();
-      bool discont = fes0->IsDGSpace();   
-
-      cout << "discont = " << discont << endl;
-      // Set curvature of the same type as original mesh
-      meshptr->SetCurvature(order, discont, sdim, ordering);
-
-      const FiniteElementSpace * fes1 = meshptr->GetNodalFESpace();
-      GridFunction * nodes = meshptr->GetNodes();
-
-      Array<int> vdofs0;
-      Array<int> vdofs;
-      Vector loc_vec;
-      
-      // Copy nodes to submesh
-      for (int e = 0; e < elems.Size(); e++)
-      {
-         fes1->GetElementVDofs(e, vdofs);
-         switch (etype)
-         {
-         case 0: 
-            fes0->GetElementVDofs(elems[e], vdofs0); 
-            nodes0->GetSubVector(vdofs0, loc_vec);
-            break;
-         case 1: 
-            if (!discont)
-            {
-               fes0->GetFaceVDofs(elems[e], vdofs0); 
-               nodes0->GetSubVector(vdofs0, loc_vec);
-            }
-            else
-            {
-               const FiniteElement * el = fes1->GetFE(e);
-               const IntegrationRule & ir = el->GetNodes(); 
-               int np = ir.GetNPoints();
-               FaceElementTransformations * Tr = 
-                  const_cast<Mesh *>(mesh0)->GetFaceElementTransformations(elems[e]);
-               int el1 = Tr->Elem1No;
-               loc_vec.SetSize(vdofs.Size());
-               for (int i = 0; i<np; i++)
-               {
-                  Tr->SetAllIntPoints(&ir[i]);
-                  const IntegrationPoint & ip = Tr->GetElement1IntPoint();
-                  Vector val;
-                  nodes0->GetVectorValue(el1,ip,val);
-                  for (int j = 0; j<val.Size(); j++)
-                  {
-                     loc_vec[i+j*np] = val[j];
-                  }
-               }
-            }
-            break;
-         default:
-            MFEM_ABORT("Wrong entity type choice");
-            break;
-         }
-         nodes->SetSubVector(vdofs, loc_vec);
-      }
-   }
-   
-   meshptr->Finalize();
-}
-
-
-void Subdomain::BuildDofMap(const entity_type & etype)
-{
-   Array<int> elems;
-   FiniteElementSpace * fesptr = nullptr;
-   const FiniteElementCollection *fec = fes0->FEColl();
-   switch(etype)
-   {
-   case 0: 
-      fesptr = new FiniteElementSpace(mesh,fec);
-      elems = element_map;
-      break;
-   case 1: 
-      fesptr = new FiniteElementSpace(surface_mesh,fec);
-      elems = surface_element_map;
-      break;
-   default:
-      MFEM_ABORT("Wrong entity type choice");
-      break;
-   }
-   
-   Array<int> dofs(fesptr->GetVSize());
-   for (int iel = 0; iel<elems.Size(); ++iel)
-   {
-      // index in the global mesh
-      int iel_idx = elems[iel];
-      // get the dofs of this element
-      Array<int> ldofs;
-      Array<int> gdofs;
-      switch(etype)
-      {
-         case 0: fes0->GetElementVDofs(iel_idx,gdofs); break;
-         case 1: fes0->GetFaceVDofs(iel_idx,gdofs); break;
-         default:  MFEM_ABORT("Wrong entity type");  break;
-      }
-      fesptr->GetElementDofs(iel,ldofs);
-         // the sizes have to match
-      MFEM_VERIFY(gdofs.Size() == ldofs.Size(),
-                  "Size inconsistency");
-     // loop through the dofs and take into account the signs;
-      int ndof = ldofs.Size();
-      for (int i = 0; i<ndof; ++i)
-      {
-         int ldof_ = ldofs[i];
-         int gdof_ = gdofs[i];
-         int ldof = (ldof_ >= 0) ? ldof_ : abs(ldof_) - 1;
-         int gdof = (gdof_ >= 0) ? gdof_ : abs(gdof_) - 1;
-         dofs[ldof] = gdof;
-      }
-   }
-   switch(etype)
-   {
-   case 0: 
-      dof_map = dofs; 
-      fes = fesptr;
-      break;
-   case 1: 
-      surface_dof_map = dofs; 
-      surface_fes = fesptr;
-      break;
-   default:  
-      MFEM_ABORT("Wrong entity type");  break;
-   }
-}
-
-void Subdomain::BuildProlongationMatrix(const entity_type & etype)
-{
-   Array<int> dofs;
-   SparseMatrix * Ptr = nullptr;
-   switch (etype)
-   {
-   case 0: 
-      if (!dof_map.Size()) BuildDofMap(etype);
-      dofs = dof_map; 
-      Ptr = P;
-      break;
-   case 1: 
-      if (!surface_dof_map.Size()) BuildDofMap(etype);
-      dofs = surface_dof_map; 
-      Ptr = Pf;
-      break;
-   default: 
-      MFEM_ABORT("Wrong entity type"); 
-      break;
-   }
-
-   int height = fes0->GetVSize();
-   int width = dofs.Size();
-   Ptr = new SparseMatrix(height,width);
-   for (int i = 0; i< dofs.Size(); i++)
-   {
-      int j = dofs[i];
-      Ptr->Set(j,i,1.);
-   }
-   Ptr->Finalize();
-
-   switch (etype)
-   {
-      case 0: P = Ptr; break;
-      case 1: Pf = Ptr; break;
-      default: MFEM_ABORT("Wrong entity type"); break;
-   }
-
-}
-
-Subdomain::~Subdomain()
-{
-   delete mesh;
-   delete surface_mesh;
-   delete fes; 
-   delete surface_fes;
-   delete P; 
-   delete Pf; 
-}
@@ -1,106 +0,0 @@
-#pragma once
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-using namespace std;
-using namespace mfem;
-
-
-class Subdomain
-{
-public:
-   enum entity_type
-   {
-      volume, 
-      surface
-   };
-private:
-   const Mesh *mesh0=nullptr;
-   int dim, sdim;
-   const FiniteElementSpace *fes0=nullptr;
-   DenseMatrix vcoords;
-   Mesh *mesh=nullptr; // Submesh 
-   Mesh *surface_mesh=nullptr; //Surface mesh 
-   FiniteElementSpace *fes=nullptr; // FE Space on the submesh
-   FiniteElementSpace *surface_fes=nullptr; // FE Space on the submesh
-
-   Array<int> element_map, surface_element_map;
-   Array<int> dof_map, surface_dof_map;
-   SparseMatrix * P=nullptr; 
-   SparseMatrix * Pf=nullptr; 
-   void BuildDofMap(const entity_type & etype);
-   void BuildProlongationMatrix(const entity_type & etype);
-   void BuildSubMesh(const Array<int> & elems, const entity_type & etype);
-
-public:
-   Subdomain(const Mesh & mesh_);
-   Mesh * GetSubMesh(const Array<int> & elems)
-   {
-      if(!mesh) BuildSubMesh(elems, entity_type::volume);
-      return mesh; 
-   }
-   Mesh * GetSurfaceMesh(const Array<int> & surface_elems)
-   {
-      if (!surface_mesh) BuildSubMesh(surface_elems, entity_type::surface);
-      return surface_mesh;
-   }
-   void SetFESpace(const FiniteElementSpace & fes0_)
-   {
-      fes0 = &fes0_;
-   }
-   void GetElementMap(Array<int> & element_map_)
-   {
-      element_map_ = element_map;
-   }
-   void GetFaceElementMap(Array<int> & surface_element_map_)
-   {
-      surface_element_map_ = surface_element_map;
-   }
-   void GetDofMap(Array<int> & dof_map_)
-   {
-      if (!dof_map.Size()) BuildDofMap(entity_type::volume);
-      dof_map_ = dof_map;
-   }
-   void GetSurfaceDofMap(Array<int> & surface_dof_map_)
-   {
-      if (!surface_dof_map.Size()) BuildDofMap(entity_type::surface);
-      surface_dof_map_ = surface_dof_map;
-   }
-   SparseMatrix * GetProlonationMatrix()
-   {
-      if (!P) BuildProlongationMatrix(entity_type::volume);
-      return P;
-   }
-   SparseMatrix * GetSurfaceProlonationMatrix()
-   {
-      if (!Pf) BuildProlongationMatrix(entity_type::surface);
-      return Pf;
-   }
-   FiniteElementSpace * GetSubFESpace(const entity_type & etype)
-   {
-      switch (etype)
-      {
-      case 0: 
-         if (!fes)
-         {
-            MFEM_VERIFY(mesh, "Volume mesh not built");
-            BuildDofMap(etype);
-         }
-         return fes; 
-         break;
-      case 1: 
-         if (!surface_fes)
-         {
-            MFEM_VERIFY(surface_mesh, "Surface mesh not built");
-            BuildDofMap(etype);
-         }
-         return surface_fes; 
-         break;
-      default: 
-         MFEM_ABORT("Wrong entity type"); 
-         return nullptr; 
-         break;
-      }
-   }
-   ~Subdomain();
-};
@@ -1,96 +0,0 @@
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command line options
-   const char *mesh_file = "../../data/periodic-annulus-sector.msh";
-   int order = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order", "Finite element polynomial degree");
-   args.ParseCheck();
-
-   // 2. Read the mesh from the given mesh file, and refine once uniformly.
-   Mesh orig_mesh(mesh_file);
-   orig_mesh.CheckElementOrientation(true);
-   orig_mesh.CheckBdrElementOrientation(true);
-
-   // mesh.EnsureNodes();
-   // mesh.UniformRefinement();
-   // Array<int> elems({1,3,21,10,20,2,0});
-   Array<int> elems({0,1,2,3});
-   // int nel = orig_mesh.GetNE();
-   int nel = elems.Size();
-   // Array<int> elems(nel);
-   // for (int i = 0; i<nel; i++)
-   // {
-      // elems[i] = i;
-   // }
-   // elems.Print();
-
-   Mesh new_mesh = Mesh::ExtractMesh(orig_mesh,elems);
-   new_mesh.CheckElementOrientation(true);
-   new_mesh.CheckBdrElementOrientation(true);
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream mesh0_sock(vishost, visport);
-      mesh0_sock.precision(8);
-      mesh0_sock << "mesh\n" << orig_mesh << "keys n \n" << flush;
-
-      socketstream mesh1_sock(vishost, visport);
-      mesh1_sock.precision(8);
-      mesh1_sock << "mesh\n" << new_mesh << "keys n \n" << flush;
-   }
-
-   Array<int> faces;
-   for (int i = 0; i<orig_mesh.GetNBE(); i++)
-   {
-      if (orig_mesh.GetBdrAttribute(i) >= 1)
-         faces.Append(orig_mesh.GetBdrFace(i));
-   }
-      // faces.Append(orig_mesh.GetBdrFace(1));
-      // faces.Append(orig_mesh.GetBdrFace(2));
-
-   Mesh surface_mesh = Mesh::ExtractSurfaceMesh(orig_mesh,faces);
-   surface_mesh.CheckElementOrientation(true);
-   surface_mesh.CheckBdrElementOrientation(true);
-
-   surface_mesh.Print();
-
-   if (surface_mesh.Dimension() > 1)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-
-      socketstream mesh2_sock(vishost, visport);
-      mesh2_sock.precision(8);
-      mesh2_sock << "mesh\n" << surface_mesh << "keys n \n" << flush;
-   }
-   else
-   {
-      ParaViewDataCollection paraview_dc("surf_mesh", &surface_mesh);
-      paraview_dc.SetPrefixPath("ParaView");
-      paraview_dc.SetLevelsOfDetail(3);
-      paraview_dc.SetCycle(0);
-      paraview_dc.SetDataFormat(VTKFormat::BINARY);
-      paraview_dc.SetHighOrderOutput(true);
-      paraview_dc.SetTime(0.0); // set the time
-      H1_FECollection fec(order,surface_mesh.Dimension());
-      FiniteElementSpace fespace(&surface_mesh,&fec);
-      GridFunction gf(&fespace);
-      gf.Randomize();
-      paraview_dc.RegisterField("solution",&gf);
-      paraview_dc.Save();
-   }
-
-   return 0;
-}
-
@@ -1,180 +0,0 @@
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-#include "mesh_partition.hpp"
-
-using namespace std;
-using namespace mfem;
-
-double sin_func(const Vector & x);
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command line options
-   const char *mesh_file = "../../data/periodic-annulus-sector.msh";
-   int order = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order", "Finite element polynomial degree");
-   args.ParseCheck();
-
-   // 2. Read the mesh from the given mesh file, and refine once uniformly.
-   Mesh mesh(mesh_file);
-   // mesh.EnsureNodes();
-   // mesh.UniformRefinement();
-   // Array<int> elems0({0,1,2,3});
-   int nel = mesh.GetNE();
-   // int nel = 5;
-   Array<int> elems0(nel/2);
-   for (int i = 0; i<nel/2; i++)
-   {
-      elems0[i] = i;
-   }
-   elems0.Print();
-   // elems0.Append(24);
-   // elems0.Append(23);
-   // elems0.Append(26);
-   Subdomain subdomain0(mesh);
-
-   Mesh * submesh = subdomain0.GetSubMesh(elems0);
-
-
-   // cout << "number of boundary elements = " << mesh.GetNBE() << endl;
-   Array<int> faces(mesh.GetNBE()/2);
-   for (int i = 0; i<mesh.GetNBE()/2; i++)
-   {
-      faces[i] = mesh.GetBdrFace(i);
-   }
-   
-   Mesh * surfmesh = subdomain0.GetSurfaceMesh(faces);
-
-   H1_FECollection fec(order, mesh.Dimension());
-   FiniteElementSpace fespace(&mesh, &fec);
-   FunctionCoefficient coeff(sin_func);
-   GridFunction gf(&fespace);
-   gf.ProjectCoefficient(coeff);
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream mesh_sock(vishost, visport);
-      mesh_sock.precision(8);
-      // mesh_sock << "mesh\n" << mesh << "keys n \n" << flush;
-      mesh_sock << "solution\n" << mesh << gf <<  "keys jnmR \n" 
-      << "valuerange 0 1.0 \n" << flush;
-         // << flush;
-   }
-
-   subdomain0.SetFESpace(fespace);
-
-   SparseMatrix * P = subdomain0.GetProlonationMatrix();
-   FiniteElementSpace * elem_fes = 
-               subdomain0.GetSubFESpace(Subdomain::entity_type::volume);
-   GridFunction gf_e(elem_fes);
-   cout << "Size P = " << P->Height() << " x " << P->Width() << endl;
-   cout << "gf_e.Size = " << gf_e.Size() << endl;
-   cout << "gf.Size = " << gf.Size() << endl;
-   P->MultTranspose(gf,gf_e);
-
-   SparseMatrix * Pb = subdomain0.GetSurfaceProlonationMatrix();
-   FiniteElementSpace * bdr_elem_fes = 
-               subdomain0.GetSubFESpace(Subdomain::entity_type::surface);
-   GridFunction gf_b(bdr_elem_fes);
-   Pb->MultTranspose(gf,gf_b);
-
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      if (submesh)
-      {
-         socketstream mesh0_sock(vishost, visport);
-         mesh0_sock.precision(8);
-         // mesh0_sock << "mesh\n" << *submesh << "keys n \n" << flush;
-         mesh0_sock << "solution\n" << *submesh << gf_e << "keys nmR \n"
-         << "valuerange 0 1.0 \n" << flush;
-         // << flush;
-      }
-      if (surfmesh && mesh.Dimension() == 3)
-      {
-         socketstream mesh1_sock(vishost, visport);
-         mesh1_sock.precision(8);
-         // mesh1_sock << "mesh\n" << *bdrmesh0 << "keys n \n" << flush;
-         mesh1_sock << "solution\n" << *surfmesh << gf_b
-         << "valuerange 0 1.0 \n" << flush;
-         // << flush;
-      }
-   }
-
-   // ParaViewDataCollection paraview_dc("mesh_partition", surfmesh);
-   // paraview_dc.SetPrefixPath("ParaView");
-   // const FiniteElementSpace * fes_ = surfmesh->GetNodalFESpace();
-   // int ord = (fes_) ? fes_->GetOrder(0) : order;
-   // paraview_dc.SetLevelsOfDetail(ord);
-   // paraview_dc.SetCycle(0);
-   // paraview_dc.SetDataFormat(VTKFormat::BINARY);
-   // paraview_dc.SetHighOrderOutput(true);
-   // paraview_dc.SetTime(0.0); // set the time
-   // paraview_dc.RegisterField("solution",&gf_b);
-   // paraview_dc.Save();
-
-
-
-   // // ---------------------------------------------------------
-   // FiniteElementCollection *fec1 = new H1_FECollection(order, submesh->Dimension());
-   // FiniteElementSpace fespace1(submesh, fec1);
-   // Array<int> ess_tdof_list;
-   // if (submesh->bdr_attributes.Size())
-   // {
-   //    Array<int> ess_bdr(mesh.bdr_attributes.Max());
-   //    ess_bdr = 1;
-   //    fespace1.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   // }
-
-   // LinearForm b(&fespace1);
-   // ConstantCoefficient one(1.0);
-   // b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   // b.Assemble();
-
-   // GridFunction x(&fespace1);
-   // x = 0.0;
-
-   // // 9. Set up the bilinear form a(.,.) on the finite element space
-   // //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   // //    domain integrator.
-   // BilinearForm a(&fespace1);
-   // a.AddDomainIntegrator(new DiffusionIntegrator(one));
-   // a.Assemble();
-   // OperatorPtr A;
-   // Vector B, X;
-   // a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-   // cout << "Size of linear system: " << A->Height() << endl;
-   // // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
-   // GSSmoother M((SparseMatrix&)(*A));
-   // PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
-   // // 12. Recover the solution as a finite element grid function.
-   // a.RecoverFEMSolution(X, b, x);
-   // {
-   //    char vishost[] = "localhost";
-   //    int  visport   = 19916;
-   //    socketstream sol_sock2(vishost, visport);
-   //    sol_sock2.precision(8);
-   //    sol_sock2 << "solution\n" << *submesh << x << flush;
-   // }
-
-   return 0;
-}
-
-double sin_func(const Vector & x)
-{
-   Vector c(x.Size());
-   c.Randomize();
-   // double dotp = c*x;
-   // return (sin(10.0*M_PI*dotp));
-   // return sin(2.*M_PI*x[0]);
-   // return 1.-x[1]*x[1]/4.0;
-   // return (0.5-x[1])*(0.5-x[1]);
-   return x[1];
-   // double r = sqrt(x[0]*x[0] + x[1]*x[1] + x[2]*x[2]);
-   // return r;
-}
@@ -1,178 +0,0 @@
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-#include "mesh_partition.hpp"
-
-using namespace std;
-using namespace mfem;
-
-double sin_func(const Vector & x);
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command line options
-   const char *mesh_file = "../data/star.mesh";
-   int order = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order", "Finite element polynomial degree");
-   args.ParseCheck();
-
-   // 2. Read the mesh from the given mesh file, and refine once uniformly.
-   Mesh mesh(mesh_file);
-   // mesh.Print(cout);
-   mesh.UniformRefinement();
-
-   // mesh.EnsureNodes();
-   // Array<int> elems0({0,4,8,12,16});
-   // Array<int> elems0({0,4,8,12,16});
-   // Array<int> elems0({6,7,8});
-   Array<int> elems0({0,1,2,3,4});
-   // Array<int> elems0({7,6,17,20,21,22});
-   // Array<int> elems0({0,1,2,3,4});
-   // Array<int> elems0({104,103,86,109});
-   // elems0.Print(cout, elems0.Size());
-   Subdomain subdomain0(mesh);
-   Mesh * submesh0 = subdomain0.GetSubMesh(elems0);
-
-   // Array<int> bdrelems0({8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23});
-   cout << "number of boundary elements = " << mesh.GetNBE() << endl;
-   Array<int> bdrelems0({0,1});
-   Array<int> faces(bdrelems0.Size());
-   for (int i = 0; i<bdrelems0.Size(); i++)
-   {
-      faces[i] = mesh.GetBdrFace(bdrelems0[i]);
-   }
-   Mesh * surfmesh0 = subdomain0.GetSurfaceMesh(faces);
-
-   H1_FECollection fec(order, mesh.Dimension());
-   FiniteElementSpace fespace(&mesh, &fec);
-   FunctionCoefficient coeff(sin_func);
-   GridFunction gf(&fespace);
-   gf.ProjectCoefficient(coeff);
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream mesh_sock(vishost, visport);
-      mesh_sock.precision(8);
-      mesh_sock << "solution\n" << mesh << gf 
-      << "valuerange -1.0 1.0 \n" << flush;
-   }
-
-   subdomain0.SetFESpace(fespace);
-
-   SparseMatrix * P = subdomain0.GetProlonationMatrix();
-   FiniteElementSpace * elem_fes = 
-               subdomain0.GetSubFESpace(Subdomain::entity_type::volume);
-   GridFunction gf_e(elem_fes);
-   P->MultTranspose(gf,gf_e);
-
-   SparseMatrix * Pf = subdomain0.GetSurfaceProlonationMatrix();
-   FiniteElementSpace * face_elem_fes = 
-               subdomain0.GetSubFESpace(Subdomain::entity_type::surface);
-   GridFunction gf_f(face_elem_fes);
-   Pf->MultTranspose(gf,gf_f);
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      if (submesh0)
-      {
-         socketstream mesh0_sock(vishost, visport);
-         mesh0_sock.precision(8);
-         // mesh0_sock << "mesh\n" << *submesh0 << "keys n \n" << flush;
-         mesh0_sock << "solution\n" << *submesh0 << gf_e 
-         << "valuerange -1.0 1.0 \n" << flush;
-      }
-      if (surfmesh0 && mesh.Dimension()==3)
-      {
-         socketstream mesh1_sock(vishost, visport);
-         mesh1_sock.precision(8);
-         // mesh1_sock << "mesh\n" << *bdrmesh0 << "keys n \n" << flush;
-         mesh1_sock << "solution\n" << *surfmesh0 << gf_f 
-         << "valuerange -1.0 1.0 \n" << flush;
-      }
-   }
-
-   // Array<int> bdr_faces;
-   // for (int i =0; i<mesh.GetNBE(); i++)
-   // {
-   //    int attr = mesh.GetBdrAttribute(i);
-   //    if (attr == 4)
-   //    {
-   //       bdr_faces.Append(mesh.GetBdrFace(i));
-   //    }
-   // }
-
-   // H1_FECollection fec(order, mesh.Dimension());
-   // FiniteElementSpace fespace(&mesh, &fec);
-   // FunctionCoefficient coeff(sin_func);
-   // GridFunction gf(&fespace);
-   // gf.ProjectCoefficient(coeff);
-   // {
-   //    char vishost[] = "localhost";
-   //    int  visport   = 19916;
-   //    socketstream mesh_sock(vishost, visport);
-   //    mesh_sock.precision(8);
-   //    // mesh_sock << "mesh\n" << mesh << "keys n \n" << flush;
-   //    mesh_sock << "solution\n" << mesh << gf << flush; 
-   //    // << "valuerange -5000.0 5000.0 \n" << flush;
-   //    // << "valuerange -1.0 1.0 \n" << flush;
-   // }
-
-   // Subdomain subdomain1(mesh);
-   // subdomain1.SetFESpace(fespace);
-   // Mesh * bdrmesh0 = subdomain1.GetBdrSurfaceMesh(bdr_faces);
-   // SparseMatrix * Pb = subdomain1.GetBdrProlonationMatrix();
-   // FiniteElementSpace * bdr_elem_fes = 
-   //             subdomain1.GetSubFESpace(Subdomain::entity_type::bdr);
-   // GridFunction gf_f(bdr_elem_fes);
-   // Pb->MultTranspose(gf,gf_f);
-
-
-
-   // // gf.Print();
-   // // gf_f.Print();
-   // // bdrmesh0->Print(cout);
-
-   // if (bdrmesh0)
-   // {
-   //    char vishost[] = "localhost";
-   //    int  visport   = 19916;
-   //    socketstream mesh1_sock(vishost, visport);
-   //    mesh1_sock.precision(8);
-   //    // mesh1_sock << "mesh\n" << *bdrmesh0 << "keys n \n" << flush;
-   //    mesh1_sock << "solution\n" << *bdrmesh0 << gf_f << flush;
-   //    // << "valuerange -5000.0 5000.0 \n" << flush;
-   // }
-
-   ParaViewDataCollection paraview_dc("mesh_partition", surfmesh0);
-   paraview_dc.SetPrefixPath("ParaView");
-   const FiniteElementSpace * fes_ = surfmesh0->GetNodalFESpace();
-   int ord = (fes_) ? fes_->GetOrder(0) : order;
-   paraview_dc.SetLevelsOfDetail(5);
-   paraview_dc.SetCycle(0);
-   paraview_dc.SetDataFormat(VTKFormat::BINARY);
-   paraview_dc.SetHighOrderOutput(true);
-   paraview_dc.SetTime(0.0); // set the time
-   paraview_dc.RegisterField("solution",&gf_f);
-   paraview_dc.Save();
-
-
-   return 0;
-}
-
-double sin_func(const Vector & x)
-{
-   Vector c(x.Size());
-   c.Randomize();
-   // double dotp = c*x;
-   double dotp = x.Sum();
-   // return (sin(10.0*M_PI*dotp));
-   // return sin(2.*M_PI*x[0]);
-   // return 1.-x[1]*x[1]/4.0;
-   return (0.5-x[1])*(0.5-x[1]);
-   // double r = sqrt(x[0]*x[0] + x[1]*x[1] + x[2]*x[2]);
-   // return r;
-}
@@ -0,0 +1,286 @@
+//                       MFEM Example 11 - Serial Version
+//
+// Compile with: make ex11
+//
+// Sample runs:  ex11 -m ../data/square-disc.mesh
+//               ex11 -m ../data/star.mesh
+//               ex11 -m ../data/star-mixed.mesh
+//               ex11 -m ../data/periodic-annulus-sector.msh
+//               ex11 -m ../data/square-disc-p2.vtk -o 2
+//               ex11 -m ../data/square-disc-p3.mesh -o 3
+//               ex11 -m ../data/square-disc-nurbs.mesh -o -1
+//               ex11 -m ../data/disc-nurbs.mesh -o -1 -n 20
+//               ex11 -m ../data/star-surf.mesh
+//               ex11 -m ../data/square-disc-surf.mesh
+//               ex11 -m ../data/inline-segment.mesh
+//               ex11 -m ../data/inline-quad.mesh
+//               ex11 -m ../data/inline-tri.mesh
+//               ex11 -m ../data/amr-quad.mesh
+//               ex11 -m ../data/amr-hex.mesh
+//               ex11 -m ../data/mobius-strip.mesh -n 8
+//
+// Description:  This example code demonstrates the use of MFEM to solve the
+//               eigenvalue problem -Delta u = lambda u with homogeneous
+//               Dirichlet boundary conditions.
+//
+//               We compute a number of the lowest eigenmodes by discretizing
+//               the Laplacian and Mass operators using a FE space of the
+//               specified order, or an isoparametric/isogeometric space if
+//               order < 1 (quadratic for quadratic curvilinear mesh, NURBS for
+//               NURBS mesh, etc.)
+//
+//               The example highlights the use of the ARPACK eigenvalue solver
+//               (regular inverse mode). Reusing a single GLVis visualization
+//               window for multiple eigenfunctions is also illustrated.
+//
+//               We recommend viewing Example 1 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../data/star.mesh";
+   int ser_ref_levels = 3;
+   int order = 1;
+   int nev = 5;
+   double dbc_eig = 1e3;
+   bool visualization = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&nev, "-n", "--num-eigs",
+                  "Number of desired eigenmodes.");
+   args.AddOption(&dbc_eig, "-d", "--dbc-eig",
+                  "Eigenvalues associated with Dirichlet BC "
+                  "(should be larger than the maximum desired eigenvalue).");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 2. Read the (serial) mesh from the given mesh file on all processors. We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh;
+   ifstream imesh(mesh_file);
+   if (!imesh)
+   {
+      cerr << "\nCan not open mesh file: " << mesh_file << '\n' << endl;
+      return 2;
+   }
+   mesh = new Mesh(imesh, 1, 1);
+   imesh.close();
+   int dim = mesh->Dimension();
+
+   // 3. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement (2 by default, or
+   //    specified on the command line with -rs).
+   for (int lev = 0; lev < ser_ref_levels; lev++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 4. Define a finite element space on the mesh. Here we
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+   }
+   else if (mesh->GetNodes())
+   {
+      fec = mesh->GetNodes()->OwnFEC();
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+   int size = fespace->GetVSize();
+
+   cout << "Number of unknowns: " << size << endl;
+
+   // 5. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
+   //    element space. The first corresponds to the Laplacian operator -Delta,
+   //    while the second is a simple mass matrix needed on the right hand side
+   //    of the generalized eigenvalue problem below. The boundary conditions
+   //    are implemented by elimination with special values on the diagonal to
+   //    shift the Dirichlet eigenvalues out of the computational range. After
+   //    serial and parallel assembly we extract the corresponding parallel
+   //    matrices A and M.
+   ConstantCoefficient one(1.0);
+   Array<int> ess_bdr;
+   if (mesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+   }
+
+   BilinearForm *a = new BilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+   if (mesh->bdr_attributes.Size() == 0)
+   {
+      // Add a mass term if the mesh has no boundary, e.g. periodic mesh or
+      // closed surface.
+      a->AddDomainIntegrator(new MassIntegrator(one));
+   }
+   a->Assemble();
+   if (mesh->bdr_attributes.Size() != 0)
+   {
+      a->EliminateEssentialBCDiag(ess_bdr, dbc_eig);
+   }
+   a->Finalize();
+
+   BilinearForm *m = new BilinearForm(fespace);
+   m->AddDomainIntegrator(new MassIntegrator(one));
+   m->Assemble();
+   if (mesh->bdr_attributes.Size() != 0)
+   {
+      // shift the eigenvalue corresponding to eliminated dofs to a large value
+      m->EliminateEssentialBCDiag(ess_bdr, 1.0);
+   }
+   m->Finalize();
+
+   // 6. Define and configure the ARPACK eigensolver
+   ArPackSym * arpack = new ArPackSym();
+   Solver    * solver = NULL;
+
+#ifndef MFEM_USE_SUITESPARSE
+   // 7. Define a simple symmetric Gauss-Seidel preconditioner and use it to
+   //    solve the system A X = B with PCG.
+   cout << "Building CGSolver" << endl;
+   GSSmoother M(m->SpMat());
+   CGSolver * cg_solver = new CGSolver;
+   cg_solver->SetPreconditioner(M);
+   cg_solver->SetRelTol(1.0e-12);
+   solver = cg_solver;
+#else
+   // 7. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+   cout << "Building UMFPackSolver" << endl;
+   UMFPackSolver * umf_solver = new UMFPackSolver;
+   umf_solver->Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+   solver = umf_solver;
+#endif
+   solver->SetOperator(m->SpMat());
+
+   arpack->SetNumModes(nev);
+   arpack->SetMaxIter(400);
+   arpack->SetTol(1e-8);
+   arpack->SetMode(2);
+   arpack->SetPrintLevel(2);
+
+   arpack->SetOperator(*a);
+   arpack->SetMassMatrix(*m);
+   arpack->SetSolver(*solver);
+
+   // 8. Compute the eigenmodes and extract the array of eigenvalues. Define a
+   //    parallel grid function to represent each of the eigenmodes returned by
+   //    the solver.
+   Array<double> eigenvalues;
+   arpack->Solve();
+   arpack->GetEigenvalues(eigenvalues);
+
+   cout << endl;
+   std::ios::fmtflags old_fmt = cout.flags();
+   cout.setf(std::ios::scientific);
+   std::streamsize old_prec = cout.precision(14);
+   for (int i=0; i<nev; i++)
+   {
+      cout << "Eigenvalue lambda   " << eigenvalues[i] << endl;
+   }
+   cout.precision(old_prec);
+   cout.flags(old_fmt);
+   cout << endl;
+
+   GridFunction x(fespace);
+
+   // 9. Save the refined mesh and the modes in parallel. This output can be
+   //    viewed later using GLVis: "glvis -np <np> -m mesh -g mode".
+   {
+      ostringstream mesh_name, mode_name;
+      mesh_name << "ex11.mesh";
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      mesh->Print(mesh_ofs);
+
+      for (int i=0; i<nev; i++)
+      {
+         // convert eigenvector from HypreParVector to ParGridFunction
+         x = arpack->GetEigenvector(i);
+
+         mode_name << "mode_" << setfill('0') << setw(2) << i;
+
+         ofstream mode_ofs(mode_name.str().c_str());
+         mode_ofs.precision(8);
+         x.Save(mode_ofs);
+         mode_name.str("");
+      }
+   }
+
+   // 10. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream mode_sock(vishost, visport);
+      mode_sock.precision(8);
+
+      for (int i=0; i<nev; i++)
+      {
+         cout << "Eigenmode " << i+1 << '/' << nev
+              << ", Lambda = " << eigenvalues[i] << endl;
+
+         // convert eigenvector from HypreParVector to ParGridFunction
+         x = arpack->GetEigenvector(i);
+
+         mode_sock << "solution\n" << *mesh << x << flush
+                   << "window_title 'Eigenmode " << i+1 << '/' << nev
+                   << ", Lambda = " << eigenvalues[i] << "'" << endl;
+
+         char c;
+         cout << "press (q)uit or (c)ontinue --> " << flush;
+         cin >> c;
+
+         if (c != 'c')
+         {
+            break;
+         }
+      }
+      mode_sock.close();
+   }
+
+   // 11. Free the used memory.
+   delete arpack;
+   delete solver;
+   delete m;
+   delete a;
+
+   delete fespace;
+   if (order > 0)
+   {
+      delete fec;
+   }
+   delete mesh;
+
+   return 0;
+}
@@ -0,0 +1,69 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../..
+MFEM_BUILD_DIR ?= ../..
+SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/arpack/,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = ex11
+PAR_EXAMPLES = 
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES)
+endif
+RC_FILES = $(patsubst $(SRC)%,%,$(wildcard $(SRC)rc_*))
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES)
+
+# Examples depend on their corresponding rc_* files:
+make-rc-rule = $(1): | $(filter rc_$(1)%,$(RC_FILES))
+$(foreach ex,$(EXAMPLES),$(eval $(call make-rc-rule,$(ex))))
+
+# Rules to copy the rc_* files when building out-of-source:
+ifneq ($(SRC),)
+$(RC_FILES): %: $(SRC)%
+	cp -pf $(<) .
+endif
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean: clean-build clean-exec
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:
+	@rm -rf mesh.* sol.* sol_p.* sol_u.* Example5*
+	@rm -f ex9-mesh.* ex9-init.* ex9-final.* Example9*
+	@rm -f deformed.* velocity.* elastic_energy.*
@@ -0,0 +1,18 @@
+                    Finite Element Discretization Library
+                                   __
+                       _ __ ___   / _|  ___  _ __ ___
+                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
+                      | | | | | ||  _||  __/| | | | | |
+                      |_| |_| |_||_|   \___||_| |_| |_|
+
+                               https://mfem.org
+
+This directory contains modifications of the example codes that illustrate the
+use of MFEM features based on the Caliper performance profiling library.
+
+To build these examples, make sure that MFEM is configured with the option
+"MFEM_USE_CALIPER = YES", see the top-level INSTALL file for details (version
+2.5.0 of Caliper is recommended, though older versions may work too).
+
+We recommend comparing the original example codes with the corresponding files
+in the current directory.
@@ -206,9 +206,9 @@ int main(int argc, char *argv[])
   cout << "Size of linear system: " << A->Height() << endl;

   // 11. Solve the linear system A X = B.
-   MFEM_PERF_BEGIN("Solve A X=B");
   if (!pa)
   {
+      MFEM_PERF_SCOPE("Solve A X=B (FA)");
 #ifndef MFEM_USE_SUITESPARSE
      // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
      GSSmoother M((SparseMatrix&)(*A));
@@ -223,6 +223,7 @@ int main(int argc, char *argv[])
   }
   else // Jacobi preconditioning in partial assembly mode
   {
+      MFEM_PERF_SCOPE("Solve A X=B (PA)");
      if (UsesTensorBasis(fespace))
      {
         OperatorJacobiSmoother M(a, ess_tdof_list);
@@ -233,7 +234,6 @@ int main(int argc, char *argv[])
         CG(*A, B, X, 1, 400, 1e-12, 0.0);
      }
   }
-   MFEM_PERF_END("Solve A X=B");
   // 12. Recover the solution as a finite element grid function.
   a.RecoverFEMSolution(X, b, x);

@@ -231,28 +231,29 @@ int main(int argc, char *argv[])
   // 13. Solve the linear system A X = B.
   //     * With full assembly, use the BoomerAMG preconditioner from hypre.
   //     * With partial assembly, use Jacobi smoothing, for now.
-   MFEM_PERF_BEGIN("Solve A X = B");
-   Solver *prec = NULL;
-   if (pa)
   {
-      if (UsesTensorBasis(fespace))
+      MFEM_PERF_SCOPE("Solve A X=B");
+      Solver *prec = NULL;
+      if (pa)
      {
-         prec = new OperatorJacobiSmoother(a, ess_tdof_list);
+         if (UsesTensorBasis(fespace))
+         {
+            prec = new OperatorJacobiSmoother(a, ess_tdof_list);
+         }
      }
+      else
+      {
+         prec = new HypreBoomerAMG;
+      }
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(2000);
+      cg.SetPrintLevel(1);
+      if (prec) { cg.SetPreconditioner(*prec); }
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      delete prec;
   }
-   else
-   {
-      prec = new HypreBoomerAMG;
-   }
-   CGSolver cg(MPI_COMM_WORLD);
-   cg.SetRelTol(1e-12);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(1);
-   if (prec) { cg.SetPreconditioner(*prec); }
-   cg.SetOperator(*A);
-   cg.Mult(B, X);
-   delete prec;
-   MFEM_PERF_END("Solve A X = B");
   // 14. Recover the parallel grid function corresponding to X. This is the
   //     local finite element solution on each processor.
   a.RecoverFEMSolution(X, b, x);
@@ -9,6 +9,7 @@
 //               ex1 -m ../data/fichera.mesh
 //               ex1 -m ../data/fichera-mixed.mesh
 //               ex1 -m ../data/toroid-wedge.mesh
+//               ex1 -m ../data/octahedron.mesh -o 1
 //               ex1 -m ../data/periodic-annulus-sector.msh
 //               ex1 -m ../data/periodic-torus-sector.msh
 //               ex1 -m ../data/square-disc-p2.vtk -o 2
@@ -55,6 +55,7 @@ int main(int argc, char *argv[])
   int order = 1;
   int nev = 5;
   bool visualization = 1;
+   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -71,6 +72,8 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -86,13 +89,18 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   // 3. Read the (serial) mesh from the given mesh file on all processors. We
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   if (myid == 0) { device.Print(); }
+
+   // 4. Read the (serial) mesh from the given mesh file on all processors. We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   // 5. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ref_levels' of uniform refinement (2 by default, or
   //    specified on the command line with -rs).
   for (int lev = 0; lev < ser_ref_levels; lev++)
@@ -100,7 +108,7 @@ int main(int argc, char *argv[])
      mesh->UniformRefinement();
   }

-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution (1 time by
   //    default, or specified on the command line with -rp). Once the parallel
   //    mesh is defined, the serial mesh can be deleted.
@@ -110,9 +118,8 @@ int main(int argc, char *argv[])
   {
      pmesh->UniformRefinement();
   }
-   pmesh->ReorientTetMesh();

-   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   // 7. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Nedelec finite elements of the specified order.
   FiniteElementCollection *fec = new ND_FECollection(order, dim);
   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
@@ -122,7 +129,7 @@ int main(int argc, char *argv[])
      cout << "Number of unknowns: " << size << endl;
   }

-   // 7. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
+   // 8. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
   //    element space. The first corresponds to the curl curl, while the second
   //    is a simple mass matrix needed on the right hand side of the
   //    generalized eigenvalue problem below. The boundary conditions are
@@ -164,7 +171,7 @@ int main(int argc, char *argv[])
   delete a;
   delete m;

-   // 8. Define and configure the AME eigensolver and the AMS preconditioner for
+   // 9. Define and configure the AME eigensolver and the AMS preconditioner for
   //    A to be used within the solver. Set the matrices which define the
   //    generalized eigenproblem A x = lambda M x.
   HypreAMS *ams = new HypreAMS(*A,fespace);
@@ -180,15 +187,15 @@ int main(int argc, char *argv[])
   ame->SetMassMatrix(*M);
   ame->SetOperator(*A);

-   // 9. Compute the eigenmodes and extract the array of eigenvalues. Define a
-   //    parallel grid function to represent each of the eigenmodes returned by
-   //    the solver.
+   // 10. Compute the eigenmodes and extract the array of eigenvalues. Define a
+   //     parallel grid function to represent each of the eigenmodes returned by
+   //     the solver.
   Array<double> eigenvalues;
   ame->Solve();
   ame->GetEigenvalues(eigenvalues);
   ParGridFunction x(fespace);

-   // 10. Save the refined mesh and the modes in parallel. This output can be
+   // 11. Save the refined mesh and the modes in parallel. This output can be
   //     viewed later using GLVis: "glvis -np <np> -m mesh -g mode".
   {
      ostringstream mesh_name, mode_name;
@@ -213,7 +220,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 11. Send the solution by socket to a GLVis server.
+   // 12. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -253,7 +260,7 @@ int main(int argc, char *argv[])
      mode_sock.close();
   }

-   // 12. Free the used memory.
+   // 13. Free the used memory.
   delete ame;
   delete ams;
   delete M;
@@ -24,7 +24,10 @@
 //               class ConductionOperator defining C(u)), as well as their
 //               implicit time integration. Note that implementing the method
 //               ConductionOperator::ImplicitSolve is the only requirement for
-//               high-order implicit (SDIRK) time integration.
+//               high-order implicit (SDIRK) time integration. In this example,
+//               the diffusion operator is linearized by evaluating with the
+//               lagged solution from the previous timestep, so there is only
+//               a linear solve.
 //
 //               We recommend viewing examples 2, 9 and 10 before viewing this
 //               example.
@@ -326,8 +329,8 @@ ConductionOperator::ConductionOperator(FiniteElementSpace &f, double al,
 void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
 {
   // Compute:
-   //    du_dt = M^{-1}*-K(u)
-   // for du_dt
+   //    du_dt = M^{-1}*-Ku
+   // for du_dt, where K is linearized by using u from the previous timestep
   Kmat.Mult(u, z);
   z.Neg(); // z = -z
   M_solver.Mult(z, du_dt);
@@ -338,7 +341,7 @@ void ConductionOperator::ImplicitSolve(const double dt,
 {
   // Solve the equation:
   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
-   // for du_dt
+   // for du_dt, where K is linearized by using u from the previous timestep
   if (!T)
   {
      T = Add(1.0, Mmat, dt, Kmat);
@@ -24,8 +24,11 @@
 //               class ConductionOperator defining C(u)), as well as their
 //               implicit time integration. Note that implementing the method
 //               ConductionOperator::ImplicitSolve is the only requirement for
-//               high-order implicit (SDIRK) time integration. Optional saving
-//               with ADIOS2 (adios2.readthedocs.io) is also illustrated.
+//               high-order implicit (SDIRK) time integration. In this example,
+//               the diffusion operator is linearized by evaluating with the
+//               lagged solution from the previous timestep, so there is only
+//               a linear solve. Optional saving with ADIOS2
+//               (adios2.readthedocs.io) is also illustrated.
 //
 //               We recommend viewing examples 2, 9 and 10 before viewing this
 //               example.
@@ -420,8 +423,8 @@ ConductionOperator::ConductionOperator(ParFiniteElementSpace &f, double al,
 void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
 {
   // Compute:
-   //    du_dt = M^{-1}*-K(u)
-   // for du_dt
+   //    du_dt = M^{-1}*-Ku
+   // for du_dt, where K is linearized by using u from the previous timestep
   Kmat.Mult(u, z);
   z.Neg(); // z = -z
   M_solver.Mult(z, du_dt);
@@ -432,7 +435,7 @@ void ConductionOperator::ImplicitSolve(const double dt,
 {
   // Solve the equation:
   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
-   // for du_dt
+   // for du_dt, where K is linearized by using u from the previous timestep
   if (!T)
   {
      T = Add(1.0, Mmat, dt, Kmat);
@@ -196,6 +196,12 @@ void InitialDeformation(const Vector &x, Vector &y);

 int main(int argc, char *argv[])
 {
+#ifdef HYPRE_USING_CUDA
+   cout << "\nAs of mfem-4.3 and hypre-2.22.0 (July 2021) this example\n"
+        << "is NOT supported with the CUDA version of hypre.\n\n";
+   return 255;
+#endif
+
   // 1. Initialize MPI
   MPI_Session mpi;
   const int myid = mpi.WorldRank();
@@ -438,16 +444,20 @@ JacobianPreconditioner::JacobianPreconditioner(Array<ParFiniteElementSpace *>
 void JacobianPreconditioner::Mult(const Vector &k, Vector &y) const
 {
   // Extract the blocks from the input and output vectors
-   Vector disp_in(k.GetData() + block_trueOffsets[0],
-                  block_trueOffsets[1]-block_trueOffsets[0]);
-   Vector pres_in(k.GetData() + block_trueOffsets[1],
-                  block_trueOffsets[2]-block_trueOffsets[1]);
-
-   Vector disp_out(y.GetData() + block_trueOffsets[0],
+   Vector disp_in;
+   disp_in.MakeRef(const_cast<Vector&>(k), block_trueOffsets[0],
                   block_trueOffsets[1]-block_trueOffsets[0]);
-   Vector pres_out(y.GetData() + block_trueOffsets[1],
+   Vector pres_in;
+   pres_in.MakeRef(const_cast<Vector&>(k), block_trueOffsets[1],
                   block_trueOffsets[2]-block_trueOffsets[1]);

+   Vector disp_out;
+   disp_out.MakeRef(y, block_trueOffsets[0],
+                    block_trueOffsets[1]-block_trueOffsets[0]);
+   Vector pres_out;
+   pres_out.MakeRef(y, block_trueOffsets[1],
+                    block_trueOffsets[2]-block_trueOffsets[1]);
+
   Vector temp(block_trueOffsets[1]-block_trueOffsets[0]);
   Vector temp2(block_trueOffsets[1]-block_trueOffsets[0]);

@@ -459,6 +469,9 @@ void JacobianPreconditioner::Mult(const Vector &k, Vector &y) const
   subtract(disp_in, temp, temp2);

   stiff_pcg->Mult(temp2, disp_out);
+
+   disp_out.SyncAliasMemory(y);
+   pres_out.SyncAliasMemory(y);
 }

 void JacobianPreconditioner::SetOperator(const Operator &op)
@@ -473,7 +486,10 @@ void JacobianPreconditioner::SetOperator(const Operator &op)

      if (!spaces[0]->GetParMesh()->Nonconforming())
      {
+#ifndef HYPRE_USING_CUDA
+         // Not available yet when hypre is built with CUDA
         stiff_prec_amg->SetElasticityOptions(spaces[0]);
+#endif
      }

      stiff_prec = stiff_prec_amg;
@@ -9,6 +9,7 @@
 //               mpirun -np 4 ex1p -m ../data/fichera.mesh
 //               mpirun -np 4 ex1p -m ../data/fichera-mixed.mesh
 //               mpirun -np 4 ex1p -m ../data/toroid-wedge.mesh
+//               mpirun -np 4 ex1p -m ../data/octahedron.mesh -o 1
 //               mpirun -np 4 ex1p -m ../data/periodic-annulus-sector.msh
 //               mpirun -np 4 ex1p -m ../data/periodic-torus-sector.msh
 //               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
@@ -89,7 +90,8 @@ int main(int argc, char *argv[])
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
 #ifdef MFEM_USE_CEED
-   args.AddOption(&algebraic_ceed, "-a", "--algebraic", "-no-a", "--no-algebraic",
+   args.AddOption(&algebraic_ceed, "-a", "--algebraic",
+                  "-no-a", "--no-algebraic",
                  "Use algebraic Ceed solver");
 #endif
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -197,15 +199,15 @@ int main(int argc, char *argv[])
   b.AddDomainIntegrator(new DomainLFIntegrator(one));
   b.Assemble();

-   // 10. Define the solution vector x as a parallel finite element grid function
-   //     corresponding to fespace. Initialize x with initial guess of zero,
-   //     which satisfies the boundary conditions.
+   // 10. Define the solution vector x as a parallel finite element grid
+   //     function corresponding to fespace. Initialize x with initial guess of
+   //     zero, which satisfies the boundary conditions.
   ParGridFunction x(&fespace);
   x = 0.0;

   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //     domain integrator.
+   //     corresponding to the Laplacian operator -Delta, by adding the
+   //     Diffusion domain integrator.
   ParBilinearForm a(&fespace);
   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   a.AddDomainIntegrator(new DiffusionIntegrator(one));
@@ -13,6 +13,8 @@
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 1
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 2
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 2 -pa
+//               ex22 -m ../data/inline-wedge.mesh -o 1
+//               ex22 -m ../data/inline-pyramid.mesh -o 1
 //               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0
 //
 // Device sample runs:
@@ -13,6 +13,8 @@
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 1
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 2
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 1 -p 2 -pa
+//               mpirun -np 4 ex22p -m ../data/inline-wedge.mesh -o 1
+//               mpirun -np 4 ex22p -m ../data/inline-pyramid.mesh -o 1
 //               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0
 //
 // Device sample runs:
@@ -113,7 +113,6 @@ int main(int argc, char *argv[])
         mesh->UniformRefinement();
      }
   }
-   mesh->ReorientTetMesh();

   // 5. Define a finite element space on the mesh. Here we use Nedelec or
   //    Raviart-Thomas finite elements of the specified order.
@@ -141,7 +141,6 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
-   pmesh->ReorientTetMesh();

   // 7. Define a parallel finite element space on the parallel mesh. Here we
   //    use Nedelec or Raviart-Thomas finite elements of the specified order.
@@ -277,10 +277,8 @@ int main(int argc, char *argv[])
      mesh->UniformRefinement();
   }

-   // 6. Reorient mesh in case of a tet mesh
-   mesh->ReorientTetMesh();
-
-   // Set element attributes in order to distinguish elements in the PML region
+   // 6. Set element attributes in order to distinguish elements in the
+   //    PML region
   pml->SetAttributes(mesh);

   // 7. Define a finite element space on the mesh. Here we use the Nedelec
@@ -316,9 +316,6 @@ int main(int argc, char *argv[])
      }
   }

-   // 7a. Reorient mesh in case of a tet mesh
-   pmesh->ReorientTetMesh();
-
   // 8. Set element attributes in order to distinguish elements in the PML
   pml->SetAttributes(pmesh);

@@ -105,7 +105,7 @@ private:
      Vector diag(fespace.GetTrueVSize());
      bfs.Last()->AssembleDiagonal(diag);

-      Solver* smoother = new OperatorChebyshevSmoother(opr.Ptr(), diag,
+      Solver* smoother = new OperatorChebyshevSmoother(*opr, diag,
                                                       *essentialTrueDofs.Last(), 2);
      AddLevel(opr.Ptr(), smoother, true, true);
   }
@@ -115,7 +115,7 @@ private:
      Vector diag(fespace.GetTrueVSize());
      bfs.Last()->AssembleDiagonal(diag);

-      Solver* smoother = new OperatorChebyshevSmoother(opr.Ptr(), diag,
+      Solver* smoother = new OperatorChebyshevSmoother(*opr, diag,
                                                       *essentialTrueDofs.Last(), 2, fespace.GetParMesh()->GetComm());

      AddLevel(opr.Ptr(), smoother, true, true);
@@ -81,6 +81,12 @@ Mesh * build_trapezoid_mesh(double offset)

 int main(int argc, char *argv[])
 {
+#ifdef HYPRE_USING_CUDA
+   cout << "\nAs of mfem-4.3 and hypre-2.22.0 (July 2021) this example\n"
+        << "is NOT supported with the CUDA version of hypre.\n\n";
+   return 255;
+#endif
+
   // 1. Initialize MPI.
   int num_procs, myid;
   MPI_Init(&argc, &argv);
@@ -360,6 +366,7 @@ int main(int argc, char *argv[])
   }
   delete pmesh;

+   // HYPRE_Finalize();
   MPI_Finalize();

   return 0;
@@ -61,6 +61,7 @@ int main(int argc, char *argv[])
   bool visualization = 1;
   bool amg_elast = 0;
   bool reorder_space = false;
+   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -78,6 +79,8 @@ int main(int argc, char *argv[])
                  "Enable or disable GLVis visualization.");
   args.AddOption(&reorder_space, "-nodes", "--by-nodes", "-vdim", "--by-vdim",
                  "Use byNODES ordering of vector space instead of byVDIM");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -93,7 +96,12 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   if (myid == 0) { device.Print(); }
+
+   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
@@ -109,14 +117,14 @@ int main(int argc, char *argv[])
      return 3;
   }

-   // 4. Select the order of the finite element discretization space. For NURBS
+   // 5. Select the order of the finite element discretization space. For NURBS
   //    meshes, we increase the order by degree elevation.
   if (mesh->NURBSext)
   {
      mesh->DegreeElevate(order, order);
   }

-   // 5. Refine the serial mesh on all processors to increase the resolution. In
+   // 6. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ref_levels' of uniform refinement. We choose
   //    'ref_levels' to be the largest number that gives a final mesh with no
   //    more than 1,000 elements.
@@ -129,7 +137,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 7. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -142,7 +150,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   // 8. Define a parallel finite element space on the parallel mesh. Here we
   //    use vector finite elements, i.e. dim copies of a scalar finite element
   //    space. We use the ordering by vector dimension (the last argument of
   //    the FiniteElementSpace constructor) which is expected in the systems
@@ -175,7 +183,7 @@ int main(int argc, char *argv[])
           << "Assembling: " << flush;
   }

-   // 8. Determine the list of true (i.e. parallel conforming) essential
+   // 9. Determine the list of true (i.e. parallel conforming) essential
   //    boundary dofs. In this example, the boundary conditions are defined by
   //    marking only boundary attribute 1 from the mesh as essential and
   //    converting it to a list of true dofs.
@@ -184,14 +192,14 @@ int main(int argc, char *argv[])
   ess_bdr[0] = 1;
   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);

-   // 9. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system. In this case, b_i equals the
-   //    boundary integral of f*phi_i where f represents a "pull down" force on
-   //    the Neumann part of the boundary and phi_i are the basis functions in
-   //    the finite element fespace. The force is defined by the object f, which
-   //    is a vector of Coefficient objects. The fact that f is non-zero on
-   //    boundary attribute 2 is indicated by the use of piece-wise constants
-   //    coefficient for its last component.
+   // 10. Set up the parallel linear form b(.) which corresponds to the
+   //     right-hand side of the FEM linear system. In this case, b_i equals the
+   //     boundary integral of f*phi_i where f represents a "pull down" force on
+   //     the Neumann part of the boundary and phi_i are the basis functions in
+   //     the finite element fespace. The force is defined by the object f, which
+   //     is a vector of Coefficient objects. The fact that f is non-zero on
+   //     boundary attribute 2 is indicated by the use of piece-wise constants
+   //     coefficient for its last component.
   VectorArrayCoefficient f(dim);
   for (int i = 0; i < dim-1; i++)
   {
@@ -212,13 +220,13 @@ int main(int argc, char *argv[])
   }
   b->Assemble();

-   // 10. Define the solution vector x as a parallel finite element grid
+   // 11. Define the solution vector x as a parallel finite element grid
   //     function corresponding to fespace. Initialize x with initial guess of
   //     zero, which satisfies the boundary conditions.
   ParGridFunction x(fespace);
   x = 0.0;

-   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
+   // 12. Set up the parallel bilinear form a(.,.) on the finite element space
   //     corresponding to the linear elasticity integrator with piece-wise
   //     constants coefficient lambda and mu.
   Vector lambda(pmesh->attributes.Max());
@@ -233,7 +241,7 @@ int main(int argc, char *argv[])
   ParBilinearForm *a = new ParBilinearForm(fespace);
   a->AddDomainIntegrator(new ElasticityIntegrator(lambda_func, mu_func));

-   // 12. Assemble the parallel bilinear form and the corresponding linear
+   // 13. Assemble the parallel bilinear form and the corresponding linear
   //     system, applying any necessary transformations such as: parallel
   //     assembly, eliminating boundary conditions, applying conforming
   //     constraints for non-conforming AMR, static condensation, etc.
@@ -250,7 +258,7 @@ int main(int argc, char *argv[])
      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
   }

-   // 13. Define and apply a parallel PCG solver for A X = B with the BoomerAMG
+   // 14. Define and apply a parallel PCG solver for A X = B with the BoomerAMG
   //     preconditioner from hypre.
   HypreBoomerAMG *amg = new HypreBoomerAMG(A);
   if (amg_elast && !a->StaticCondensationIsEnabled())
@@ -268,11 +276,11 @@ int main(int argc, char *argv[])
   pcg->SetPreconditioner(*amg);
   pcg->Mult(B, X);

-   // 14. Recover the parallel grid function corresponding to X. This is the
+   // 15. Recover the parallel grid function corresponding to X. This is the
   //     local finite element solution on each processor.
   a->RecoverFEMSolution(X, *b, x);

-   // 15. For non-NURBS meshes, make the mesh curved based on the finite element
+   // 16. For non-NURBS meshes, make the mesh curved based on the finite element
   //     space. This means that we define the mesh elements through a fespace
   //     based transformation of the reference element.  This allows us to save
   //     the displaced mesh as a curved mesh when using high-order finite
@@ -284,7 +292,7 @@ int main(int argc, char *argv[])
      pmesh->SetNodalFESpace(fespace);
   }

-   // 16. Save in parallel the displaced mesh and the inverted solution (which
+   // 17. Save in parallel the displaced mesh and the inverted solution (which
   //     gives the backward displacements to the original grid). This output
   //     can be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
   {
@@ -305,7 +313,7 @@ int main(int argc, char *argv[])
      x.Save(sol_ofs);
   }

-   // 17. Send the above data by socket to a GLVis server.  Use the "n" and "b"
+   // 18. Send the above data by socket to a GLVis server.  Use the "n" and "b"
   //     keys in GLVis to visualize the displacements.
   if (visualization)
   {
@@ -317,7 +325,7 @@ int main(int argc, char *argv[])
      sol_sock << "solution\n" << *pmesh << x << flush;
   }

-   // 18. Free the used memory.
+   // 19. Free the used memory.
   delete pcg;
   delete amg;
   delete a;
@@ -16,6 +16,8 @@
 //               ex3 -m ../data/beam-hex-nurbs.mesh
 //               ex3 -m ../data/amr-hex.mesh
 //               ex3 -m ../data/fichera-amr.mesh
+//               ex3 -m ../data/ref-prism.mesh -o 1
+//               ex3 -m ../data/octahedron.mesh -o 1
 //               ex3 -m ../data/star-surf.mesh -o 1
 //               ex3 -m ../data/mobius-strip.mesh -f 0.1
 //               ex3 -m ../data/klein-bottle.mesh -f 0.1
@@ -113,7 +115,6 @@ int main(int argc, char *argv[])
         mesh->UniformRefinement();
      }
   }
-   mesh->ReorientTetMesh();

   // 5. Define a finite element space on the mesh. Here we use the Nedelec
   //    finite elements of the specified order.
@@ -16,6 +16,8 @@
 //               mpirun -np 4 ex3p -m ../data/beam-hex-nurbs.mesh
 //               mpirun -np 4 ex3p -m ../data/amr-quad.mesh -o 2
 //               mpirun -np 4 ex3p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex3p -m ../data/ref-prism.mesh -o 1
+//               mpirun -np 4 ex3p -m ../data/octahedron.mesh -o 1
 //               mpirun -np 4 ex3p -m ../data/star-surf.mesh -o 2
 //               mpirun -np 4 ex3p -m ../data/mobius-strip.mesh -o 2 -f 0.1
 //               mpirun -np 4 ex3p -m ../data/klein-bottle.mesh -o 2 -f 0.1
@@ -103,6 +105,7 @@ int main(int argc, char *argv[])
      {
         args.PrintUsage(cout);
      }
+      // HYPRE_Finalize();
      MPI_Finalize();
      return 1;
   }
@@ -138,9 +141,7 @@ int main(int argc, char *argv[])

   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
-   //    meshes need to be reoriented before we can define high-order Nedelec
-   //    spaces on them.
+   //    parallel mesh is defined, the serial mesh can be deleted.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
   delete mesh;
   {
@@ -150,7 +151,6 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
-   pmesh->ReorientTetMesh();

   // 7. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Nedelec finite elements of the specified order.
@@ -19,6 +19,8 @@
 //               ex4 -m ../data/amr-hex.mesh
 //               ex4 -m ../data/amr-hex.mesh -o 2 -hb
 //               ex4 -m ../data/fichera-amr.mesh -o 2 -sc
+//               ex4 -m ../data/ref-prism.mesh -o 1
+//               ex4 -m ../data/octahedron.mesh -o 1
 //               ex4 -m ../data/star-surf.mesh -o 1
 //
 // Device sample runs:
@@ -19,6 +19,8 @@
 //               mpirun -np 3 ex4p -m ../data/amr-quad.mesh -o 2 -hb
 //               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -sc
 //               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -hb
+//               mpirun -np 4 ex4p -m ../data/ref-prism.mesh -o 1
+//               mpirun -np 4 ex4p -m ../data/octahedron.mesh -o 1
 //               mpirun -np 4 ex4p -m ../data/star-surf.mesh -o 3 -hb
 //
 // Device sample runs:
@@ -135,9 +137,7 @@ int main(int argc, char *argv[])

   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
-   //    meshes need to be reoriented before we can define high-order Nedelec
-   //    spaces on them (this is needed in the ADS solver below).
+   //    parallel mesh is defined, the serial mesh can be deleted.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
   delete mesh;
   {
@@ -147,7 +147,6 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
-   pmesh->ReorientTetMesh();

   // 7. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Raviart-Thomas finite elements of the specified order.
@@ -197,6 +197,7 @@ int main(int argc, char *argv[])
      SparseMatrix &M(mVarf->SpMat());
      SparseMatrix &B(bVarf->SpMat());
      B *= -1.;
+      if (Device::IsEnabled()) { B.BuildTranspose(); }
      Bt = new TransposeOperator(&B);

      darcyOp.SetBlock(0,0, &M);
@@ -240,6 +241,7 @@ int main(int argc, char *argv[])
   {
      SparseMatrix &M(mVarf->SpMat());
      M.GetDiag(Md);
+      Md.HostReadWrite();

      SparseMatrix &B(bVarf->SpMat());
      MinvBt = Transpose(B);
@@ -287,12 +289,18 @@ int main(int argc, char *argv[])
   chrono.Stop();

   if (solver.GetConverged())
+   {
      std::cout << "MINRES converged in " << solver.GetNumIterations()
-                << " iterations with a residual norm of " << solver.GetFinalNorm() << ".\n";
+                << " iterations with a residual norm of "
+                << solver.GetFinalNorm() << ".\n";
+   }
   else
+   {
      std::cout << "MINRES did not converge in " << solver.GetNumIterations()
-                << " iterations. Residual norm is " << solver.GetFinalNorm() << ".\n";
-   std::cout << "MINRES solver took " << chrono.RealTime() << "s. \n";
+                << " iterations. Residual norm is " << solver.GetFinalNorm()
+                << ".\n";
+   }
+   std::cout << "MINRES solver took " << chrono.RealTime() << "s.\n";

   // 12. Create the grid functions u and p. Compute the L2 error norms.
   GridFunction u, p;
@@ -47,6 +47,7 @@ int main(int argc, char *argv[])
   int order = 2;
   bool always_snap = false;
   bool visualization = 1;
+   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&elem_type, "-e", "--elem",
@@ -65,6 +66,8 @@ int main(int argc, char *argv[])
                  "--snap-at-the-end",
                  "If true, snap nodes to the sphere initially and after each refinement "
                  "otherwise, snap only after the last refinement");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -80,7 +83,12 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   // 3. Generate an initial high-order (surface) mesh on the unit sphere. The
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   if (myid == 0) { device.Print(); }
+
+   // 4. Generate an initial high-order (surface) mesh on the unit sphere. The
   //    Mesh object represents a 2D mesh in 3 spatial dimensions. We first add
   //    the elements and the vertices of the mesh, and then make it high-order
   //    by specifying a finite element space for its nodes.
@@ -146,7 +154,7 @@ int main(int argc, char *argv[])
   FiniteElementSpace nodal_fes(mesh, &fec, mesh->SpaceDimension());
   mesh->SetNodalFESpace(&nodal_fes);

-   // 4. Refine the mesh while snapping nodes to the sphere. Number of parallel
+   // 5. Refine the mesh while snapping nodes to the sphere. Number of parallel
   //    refinements is fixed to 2.
   for (int l = 0; l <= ref_levels; l++)
   {
@@ -218,7 +226,7 @@ int main(int argc, char *argv[])
      SnapNodes(*pmesh);
   }

-   // 5. Define a finite element space on the mesh. Here we use isoparametric
+   // 6. Define a finite element space on the mesh. Here we use isoparametric
   //    finite elements -- the same as the mesh nodes.
   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, &fec);
   HYPRE_BigInt size = fespace->GlobalTrueVSize();
@@ -227,7 +235,7 @@ int main(int argc, char *argv[])
      cout << "Number of unknowns: " << size << endl;
   }

-   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
+   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
   //    the basis functions in the finite element fespace.
   ParLinearForm *b = new ParLinearForm(fespace);
@@ -237,27 +245,27 @@ int main(int argc, char *argv[])
   b->AddDomainIntegrator(new DomainLFIntegrator(rhs_coef));
   b->Assemble();

-   // 7. Define the solution vector x as a finite element grid function
+   // 8. Define the solution vector x as a finite element grid function
   //    corresponding to fespace. Initialize x with initial guess of zero.
   ParGridFunction x(fespace);
   x = 0.0;

-   // 8. Set up the bilinear form a(.,.) on the finite element space
+   // 9. Set up the bilinear form a(.,.) on the finite element space
   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
   //    and Mass domain integrators.
   ParBilinearForm *a = new ParBilinearForm(fespace);
   a->AddDomainIntegrator(new DiffusionIntegrator(one));
   a->AddDomainIntegrator(new MassIntegrator(one));

-   // 9. Assemble the parallel linear system, applying any transformations
-   //    such as: parallel assembly, applying conforming constraints, etc.
+   // 10. Assemble the parallel linear system, applying any transformations
+   //     such as: parallel assembly, applying conforming constraints, etc.
   a->Assemble();
   HypreParMatrix A;
   Vector B, X;
   Array<int> empty_tdof_list;
   a->FormLinearSystem(empty_tdof_list, x, *b, A, X, B);

-   // 10. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
+   // 11. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
   //     preconditioner from hypre. Extract the parallel grid function x
   //     corresponding to the finite element approximation X. This is the local
   //     solution on each processor.
@@ -273,14 +281,14 @@ int main(int argc, char *argv[])
   delete a;
   delete b;

-   // 11. Compute and print the L^2 norm of the error.
+   // 12. Compute and print the L^2 norm of the error.
   double err = x.ComputeL2Error(sol_coef);
   if (myid == 0)
   {
      cout << "\nL2 norm of error: " << err << endl;
   }

-   // 12. Save the refined mesh and the solution. This output can be viewed
+   // 13. Save the refined mesh and the solution. This output can be viewed
   //     later using GLVis: "glvis -np <np> -m sphere_refined -g sol".
   {
      ostringstream mesh_name, sol_name;
@@ -296,7 +304,7 @@ int main(int argc, char *argv[])
      x.Save(sol_ofs);
   }

-   // 13. Send the solution by socket to a GLVis server.
+   // 14. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -307,7 +315,7 @@ int main(int argc, char *argv[])
      sol_sock << "solution\n" << *pmesh << x << flush;
   }

-   // 14. Free the used memory.
+   // 15. Free the used memory.
   delete pcg;
   delete amg;
   delete fespace;
@@ -106,7 +106,6 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
-   pmesh->ReorientTetMesh();

   // 6. Define the trial, interfacial (trace) and test DPG spaces:
   //    - The trial space, x0_space, contains the non-interfacial unknowns and
@@ -26,6 +26,9 @@ SEQ_EXAMPLES = ex0 ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 \
 PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
  ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
  ex25p ex26p ex27p ex28p ex29p
+SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
+PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p \
+  ex24p ex25p ex26p

 ifeq ($(MFEM_USE_MPI),NO)
   EXAMPLES = $(SEQ_EXAMPLES)
@@ -99,6 +102,14 @@ RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
 	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
 %-test-seq: %
 	@$(call mfem-test,$<,, Serial example)
+%-test-par-cuda: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel CUDA example,-d cuda)
+%-test-seq-cuda: %
+	@$(call mfem-test,$<,, Serial CUDA example,-d cuda)
+%-test-par-hip: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel HIP example,-d hip)
+%-test-seq-hip: %
+	@$(call mfem-test,$<,, Serial HIP example,-d hip)

 # Testing: Specific execution options
 ex0-test-seq: ex0
@@ -121,9 +121,7 @@ int main(int argc, char *argv[])

   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
-   //    meshes need to be reoriented before we can define high-order Nedelec
-   //    spaces on them.
+   //    parallel mesh is defined, the serial mesh can be deleted.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
   delete mesh;
   {
@@ -133,7 +131,6 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
-   pmesh->ReorientTetMesh();

   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Nedelec finite elements of the specified order.
@@ -122,9 +122,7 @@ int main(int argc, char *argv[])

   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
-   //    meshes need to be reoriented before we can define high-order Nedelec
-   //    spaces on them (this is needed in the ADS solver below).
+   //    parallel mesh is defined, the serial mesh can be deleted.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
   delete mesh;
   {
@@ -134,7 +132,6 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
-   pmesh->ReorientTetMesh();

   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Raviart-Thomas finite elements of the specified order.
@@ -0,0 +1,250 @@
+//                       MFEM Example 11 - Serial Version
+//
+// Compile with: make ex11
+//
+// Sample runs:  ex11 -m ../data/square-disc.mesh
+//               ex11 -m ../data/star.mesh
+//               ex11 -m ../data/star-mixed.mesh
+//               ex11 -m ../data/periodic-annulus-sector.msh
+//               ex11 -m ../data/square-disc-p2.vtk -o 2
+//               ex11 -m ../data/square-disc-p3.mesh -o 3
+//               ex11 -m ../data/square-disc-nurbs.mesh -o -1
+//               ex11 -m ../data/disc-nurbs.mesh -o -1 -n 20
+//               ex11 -m ../data/star-surf.mesh
+//               ex11 -m ../data/square-disc-surf.mesh
+//               ex11 -m ../data/inline-segment.mesh
+//               ex11 -m ../data/inline-quad.mesh
+//               ex11 -m ../data/inline-tri.mesh
+//               ex11 -m ../data/amr-quad.mesh
+//               ex11 -m ../data/amr-hex.mesh
+//               ex11 -m ../data/mobius-strip.mesh -n 8
+//
+// Description:  This example code demonstrates the use of MFEM to solve the
+//               eigenvalue problem -Delta u = lambda u with homogeneous
+//               Dirichlet boundary conditions.
+//
+//               We compute a number of the lowest eigenmodes by discretizing
+//               the Laplacian and Mass operators using a FE space of the
+//               specified order, or an isoparametric/isogeometric space if
+//               order < 1 (quadratic for quadratic curvilinear mesh, NURBS for
+//               NURBS mesh, etc.)
+//
+//               The example highlights the use of the ARPACK eigenvalue solver
+//               (regular inverse mode). Reusing a single GLVis visualization
+//               window for multiple eigenfunctions is also illustrated.
+//
+//               We recommend viewing Example 1 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../data/star.mesh";
+   int ser_ref_levels = 1;
+   int order = 1;
+   int nev = 5;
+   double dbc_eig = 1e3;
+   bool visualization = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&nev, "-n", "--num-eigs",
+                  "Number of desired eigenmodes.");
+   args.AddOption(&dbc_eig, "-d", "--dbc-eig",
+                  "Eigenvalues associated with Dirichlet BC "
+                  "(should be larger than the maximum desired eigenvalue).");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 2. Read the (serial) mesh from the given mesh file on all processors. We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh;
+   ifstream imesh(mesh_file);
+   if (!imesh)
+   {
+      cerr << "\nCan not open mesh file: " << mesh_file << '\n' << endl;
+      return 2;
+   }
+   mesh = new Mesh(imesh, 1, 1);
+   imesh.close();
+   int dim = mesh->Dimension();
+
+   // 3. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement (2 by default, or
+   //    specified on the command line with -rs). 
+   for (int lev = 0; lev < ser_ref_levels; lev++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 4. Define a finite element space on the mesh. Here we
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+   }
+   else if (mesh->GetNodes())
+   {
+      fec = mesh->GetNodes()->OwnFEC();
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+   int size = fespace->GetVSize();
+
+   cout << "Number of unknowns: " << size << endl;
+
+   // 5. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
+   //    element space. The first corresponds to the Laplacian operator -Delta,
+   //    while the second is a simple mass matrix needed on the right hand side
+   //    of the generalized eigenvalue problem below. The boundary conditions
+   //    are implemented by elimination with special values on the diagonal to
+   //    shift the Dirichlet eigenvalues out of the computational range. After
+   //    serial and parallel assembly we extract the corresponding parallel
+   //    matrices A and M.
+   ConstantCoefficient one(1.0);
+   Array<int> ess_bdr;
+   if (mesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+   }
+
+   BilinearForm *a = new BilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+   if (mesh->bdr_attributes.Size() == 0)
+   {
+      // Add a mass term if the mesh has no boundary, e.g. periodic mesh or
+      // closed surface.
+      a->AddDomainIntegrator(new MassIntegrator(one));
+   }
+   a->Assemble();
+   if (mesh->bdr_attributes.Size() != 0)
+   {
+      a->EliminateEssentialBCDiag(ess_bdr, dbc_eig);
+   }
+   a->Finalize();
+
+   BilinearForm *m = new BilinearForm(fespace);
+   m->AddDomainIntegrator(new MassIntegrator(one));
+   m->Assemble();
+   if (mesh->bdr_attributes.Size() != 0)
+   {
+      // shift the eigenvalue corresponding to eliminated dofs to a large value
+      m->EliminateEssentialBCDiag(ess_bdr, 1.0);
+   }
+   m->Finalize();
+
+   // 6. Define and configure the SPECTRA eigensolver and solve problem
+   SpectraEigenSolver spectra;
+
+   spectra.SetNumModes(nev)
+      .SetKrylov(10)
+      .SetMaxIter(5000)
+      .SetTol(1e-5)
+      .SetOperators(*a, *m)
+      .Solve();
+
+   Eigen::VectorXd eigenvalues = spectra.GetEigenvalues(nev);
+
+   // 7. Define a grid function to represent each of the eigenmodes returned by the solver.
+   
+   GridFunction x(fespace);
+
+   // 8. Save the refined mesh and the modes in parallel. 
+   //    This output can be viewed later using GLVis: "glvis -np <np> -m mesh -g mode"
+   {
+      ostringstream mesh_name, mode_name;
+      mesh_name << "ex11.mesh";
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      mesh->Print(mesh_ofs);
+
+      for (int i = 0; i < nev; i++) {
+         // conver Eigen Vector to MFEM Vector
+         Vector eigenvector = VectorConverter<double>::from(spectra.GetEigenvector(i));
+         
+         // convert eigenvector from Vector to GridFunction
+         x = eigenvector;
+
+         mode_name << "mode_" << setfill('0') << setw(2) << i;
+
+         ofstream mode_ofs(mode_name.str().c_str());
+         mode_ofs.precision(8);
+         x.Save(mode_ofs);
+         mode_name.str("");
+      }
+   }
+
+   // 10. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream mode_sock(vishost, visport);
+      mode_sock.precision(8);
+
+      for (int i=0; i<nev; i++)
+      {
+         cout << "Eigenmode " << i+1 << '/' << nev
+              << ", Lambda = " << eigenvalues[i] << endl;
+
+         // convert eigenvector from HypreParVector to ParGridFunction
+         Vector eigenvector = VectorConverter<double>::from(spectra.GetEigenvector(i));
+         x = eigenvector;
+
+         mode_sock << "solution\n" << *mesh << x << flush
+                   << "window_title 'Eigenmode " << i+1 << '/' << nev
+                   << ", Lambda = " << eigenvalues[i] << "'" << endl;
+
+         char c;
+         cout << "press (q)uit or (c)ontinue --> " << flush;
+         cin >> c;
+
+         if (c != 'c')
+         {
+            break;
+         }
+      }
+      mode_sock.close();
+   }
+
+   // 10. Free the used memory.
+   delete m;
+   delete a;
+
+   delete fespace;
+   if (order > 0)
+   {
+      delete fec;
+   }
+   delete mesh;
+
+   return 0;
+}
@@ -0,0 +1,67 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../..
+MFEM_BUILD_DIR ?= ../..
+SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/spectra/,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = ex11
+PAR_EXAMPLES = 
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES)
+endif
+RC_FILES = $(patsubst $(SRC)%,%,$(wildcard $(SRC)rc_*))
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES)
+
+# Examples depend on their corresponding rc_* files:
+make-rc-rule = $(1): | $(filter rc_$(1)%,$(RC_FILES))
+$(foreach ex,$(EXAMPLES),$(eval $(call make-rc-rule,$(ex))))
+
+# Rules to copy the rc_* files when building out-of-source:
+ifneq ($(SRC),)
+$(RC_FILES): %: $(SRC)%
+	cp -pf $(<) .
+endif
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean: clean-build clean-exec
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:
+	@rm -rf *.mesh mode_*
@@ -282,6 +282,10 @@ int main(int argc, char *argv[])
   superlu->SetOperator(*SLU_A);
   superlu->SetPrintStatistics(true);
   superlu->Mult(B, X);
+   superlu->DismantleGrid();
+
+   delete SLU_A;
+   delete superlu;

   // 14. Recover the parallel grid function corresponding to X. This is the
   //     local finite element solution on each processor.
@@ -39,6 +39,7 @@ set(SRCS
  complex_fem.cpp
  convergence.cpp
  datacollection.cpp
+  doftrans.cpp
  eltrans.cpp
  estimators.cpp
  fe.cpp
@@ -105,6 +106,7 @@ set(SRCS
  tmop/tmop_pa_w3.cpp
  tmop/tmop_pa_w3_c0.cpp
  tmop_tools.cpp
+  tmop_amr.cpp
  gslib.cpp
  transfer.cpp
  lor.cpp
@@ -118,6 +120,7 @@ set(HDRS
  complex_fem.hpp
  convergence.hpp
  datacollection.hpp
+  doftrans.hpp
  eltrans.hpp
  estimators.hpp
  fe.hpp
@@ -164,6 +167,7 @@ set(HDRS
  tmop.hpp
  tmop/tmop_pa.hpp
  tmop_tools.hpp
+  tmop_amr.hpp
  gslib.hpp
  transfer.hpp
  lor.hpp
@@ -184,6 +188,11 @@ if (MFEM_USE_ADIOS2)
  list(APPEND HDRS adios2datacollection.hpp)
 endif()

+if (MFEM_USE_FMS)
+  list(APPEND SRCS fmsdatacollection.cpp fmsconvert.cpp)
+  list(APPEND HDRS fmsdatacollection.hpp fmsconvert.hpp)
+endif()
+
 if (MFEM_USE_MPI)
  list(APPEND SRCS
    pbilinearform.cpp
@@ -31,7 +31,7 @@ void BilinearForm::AllocMat()
   const Table &elem_dof = fes->GetElementToDofTable();
   Table dof_dof;

-   if (fbfi.Size() > 0)
+   if (interior_face_integs.Size() > 0)
   {
      // the sparsity pattern is defined from the map: face->element->dof
      Table face_dof, dof_face;
@@ -99,15 +99,15 @@ BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)
   ext = NULL;

   // Copy the pointers to the integrators
-   dbfi = bf->dbfi;
+   domain_integs = bf->domain_integs;

-   bbfi = bf->bbfi;
-   bbfi_marker = bf->bbfi_marker;
+   boundary_integs = bf->boundary_integs;
+   boundary_integs_marker = bf->boundary_integs_marker;

-   fbfi = bf->fbfi;
+   interior_face_integs = bf->interior_face_integs;

-   bfbfi = bf->bfbfi;
-   bfbfi_marker = bf->bfbfi_marker;
+   boundary_face_integs = bf->boundary_face_integs;
+   boundary_face_integs_marker = bf->boundary_face_integs_marker;

   AllocMat();
 }
@@ -234,46 +234,47 @@ void BilinearForm::Finalize (int skip_zeros)

 void BilinearForm::AddDomainIntegrator(BilinearFormIntegrator *bfi)
 {
-   dbfi.Append(bfi);
-   dbfi_marker.Append(NULL); // NULL marker means apply everywhere
+   domain_integs.Append(bfi);
+   domain_integs_marker.Append(NULL); // NULL marker means apply everywhere
 }

 void BilinearForm::AddDomainIntegrator(BilinearFormIntegrator *bfi,
                                       Array<int> &elem_marker)
 {
-   dbfi.Append(bfi);
-   dbfi_marker.Append(&elem_marker);
+   domain_integs.Append(bfi);
+   domain_integs_marker.Append(&elem_marker);
 }

 void BilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi)
 {
-   bbfi.Append (bfi);
-   bbfi_marker.Append(NULL); // NULL marker means apply everywhere
+   boundary_integs.Append (bfi);
+   boundary_integs_marker.Append(NULL); // NULL marker means apply everywhere
 }

 void BilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi,
                                          Array<int> &bdr_marker)
 {
-   bbfi.Append (bfi);
-   bbfi_marker.Append(&bdr_marker);
+   boundary_integs.Append (bfi);
+   boundary_integs_marker.Append(&bdr_marker);
 }

 void BilinearForm::AddInteriorFaceIntegrator(BilinearFormIntegrator * bfi)
 {
-   fbfi.Append (bfi);
+   interior_face_integs.Append (bfi);
 }

 void BilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator *bfi)
 {
-   bfbfi.Append(bfi);
-   bfbfi_marker.Append(NULL); // NULL marker means apply everywhere
+   boundary_face_integs.Append(bfi);
+   // NULL marker means apply everywhere
+   boundary_face_integs_marker.Append(NULL);
 }

 void BilinearForm::AddBdrFaceIntegrator(BilinearFormIntegrator *bfi,
                                        Array<int> &bdr_marker)
 {
-   bfbfi.Append(bfi);
-   bfbfi_marker.Append(&bdr_marker);
+   boundary_face_integs.Append(bfi);
+   boundary_face_integs_marker.Append(&bdr_marker);
 }

 void BilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
@@ -285,14 +286,14 @@ void BilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
      return;
   }

-   if (dbfi.Size())
+   if (domain_integs.Size())
   {
      const FiniteElement &fe = *fes->GetFE(i);
      ElementTransformation *eltrans = fes->GetElementTransformation(i);
-      dbfi[0]->AssembleElementMatrix(fe, *eltrans, elmat);
-      for (int k = 1; k < dbfi.Size(); k++)
+      domain_integs[0]->AssembleElementMatrix(fe, *eltrans, elmat);
+      for (int k = 1; k < domain_integs.Size(); k++)
      {
-         dbfi[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
+         domain_integs[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
         elmat += elemmat;
      }
   }
@@ -306,14 +307,14 @@ void BilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)

 void BilinearForm::ComputeBdrElementMatrix(int i, DenseMatrix &elmat)
 {
-   if (bbfi.Size())
+   if (boundary_integs.Size())
   {
      const FiniteElement &be = *fes->GetBE(i);
      ElementTransformation *eltrans = fes->GetBdrElementTransformation(i);
-      bbfi[0]->AssembleElementMatrix(be, *eltrans, elmat);
-      for (int k = 1; k < bbfi.Size(); k++)
+      boundary_integs[0]->AssembleElementMatrix(be, *eltrans, elmat);
+      for (int k = 1; k < boundary_integs.Size(); k++)
      {
-         bbfi[k]->AssembleElementMatrix(be, *eltrans, elemmat);
+         boundary_integs[k]->AssembleElementMatrix(be, *eltrans, elemmat);
         elmat += elemmat;
      }
   }
@@ -390,6 +391,7 @@ void BilinearForm::Assemble(int skip_zeros)
   }

   ElementTransformation *eltrans;
+   DofTransformation * doftrans;
   Mesh *mesh = fes -> GetMesh();
   DenseMatrix elmat, *elmat_p;

@@ -407,13 +409,14 @@ void BilinearForm::Assemble(int skip_zeros)
   }
 #endif

-   if (dbfi.Size())
+   if (domain_integs.Size())
   {
-      for (int k = 0; k < dbfi.Size(); k++)
+      for (int k = 0; k < domain_integs.Size(); k++)
      {
-         if (dbfi_marker[k] != NULL)
+         if (domain_integs_marker[k] != NULL)
         {
-            MFEM_VERIFY(mesh->attributes.Size() == dbfi_marker[k]->Size(),
+            MFEM_VERIFY(mesh->attributes.Size() ==
+                        domain_integs_marker[k]->Size(),
                        "invalid element marker for domain integrator #"
                        << k << ", counting from zero");
         }
@@ -422,7 +425,7 @@ void BilinearForm::Assemble(int skip_zeros)
      for (int i = 0; i < fes -> GetNE(); i++)
      {
         int elem_attr = fes->GetMesh()->GetAttribute(i);
-         fes->GetElementVDofs(i, vdofs);
+         doftrans = fes->GetElementVDofs(i, vdofs);
         if (element_matrices)
         {
            elmat_p = &(*element_matrices)(i);
@@ -430,14 +433,14 @@ void BilinearForm::Assemble(int skip_zeros)
         else
         {
            elmat.SetSize(0);
-            for (int k = 0; k < dbfi.Size(); k++)
+            for (int k = 0; k < domain_integs.Size(); k++)
            {
-               if ( dbfi_marker[k] == NULL ||
-                    (*(dbfi_marker[k]))[elem_attr-1] == 1)
+               if ( domain_integs_marker[k] == NULL ||
+                    (*(domain_integs_marker[k]))[elem_attr-1] == 1)
               {
                  const FiniteElement &fe = *fes->GetFE(i);
                  eltrans = fes->GetElementTransformation(i);
-                  dbfi[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
+                  domain_integs[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
                  if (elmat.Size() == 0)
                  {
                     elmat = elemmat;
@@ -456,6 +459,11 @@ void BilinearForm::Assemble(int skip_zeros)
            {
               elmat_p = &elmat;
            }
+            if (doftrans)
+            {
+               doftrans->TransformDual(elmat);
+            }
+            elmat_p = &elmat;
         }
         if (static_cond)
         {
@@ -472,20 +480,20 @@ void BilinearForm::Assemble(int skip_zeros)
      }
   }

-   if (bbfi.Size())
+   if (boundary_integs.Size())
   {
      // Which boundary attributes need to be processed?
      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
                                 mesh->bdr_attributes.Max() : 0);
      bdr_attr_marker = 0;
-      for (int k = 0; k < bbfi.Size(); k++)
+      for (int k = 0; k < boundary_integs.Size(); k++)
      {
-         if (bbfi_marker[k] == NULL)
+         if (boundary_integs_marker[k] == NULL)
         {
            bdr_attr_marker = 1;
            break;
         }
-         Array<int> &bdr_marker = *bbfi_marker[k];
+         Array<int> &bdr_marker = *boundary_integs_marker[k];
         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
                     "invalid boundary marker for boundary integrator #"
                     << k << ", counting from zero");
@@ -501,42 +509,47 @@ void BilinearForm::Assemble(int skip_zeros)
         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }

         const FiniteElement &be = *fes->GetBE(i);
-         fes -> GetBdrElementVDofs (i, vdofs);
+         doftrans = fes -> GetBdrElementVDofs (i, vdofs);
         eltrans = fes -> GetBdrElementTransformation (i);
         int k = 0;
-         for (; k < bbfi.Size(); k++)
+         for (; k < boundary_integs.Size(); k++)
         {
-            if (bbfi_marker[k] &&
-                (*bbfi_marker[k])[bdr_attr-1] == 0) { continue; }
+            if (boundary_integs_marker[k] &&
+                (*boundary_integs_marker[k])[bdr_attr-1] == 0) { continue; }

-            bbfi[k]->AssembleElementMatrix(be, *eltrans, elmat);
+            boundary_integs[k]->AssembleElementMatrix(be, *eltrans, elmat);
            k++;
            break;
         }
-         for (; k < bbfi.Size(); k++)
+         for (; k < boundary_integs.Size(); k++)
         {
-            if (bbfi_marker[k] &&
-                (*bbfi_marker[k])[bdr_attr-1] == 0) { continue; }
+            if (boundary_integs_marker[k] &&
+                (*boundary_integs_marker[k])[bdr_attr-1] == 0) { continue; }

-            bbfi[k]->AssembleElementMatrix(be, *eltrans, elemmat);
+            boundary_integs[k]->AssembleElementMatrix(be, *eltrans, elemmat);
            elmat += elemmat;
         }
+         if (doftrans)
+         {
+            doftrans->TransformDual(elmat);
+         }
+         elmat_p = &elmat;
         if (!static_cond)
         {
-            mat->AddSubMatrix(vdofs, vdofs, elmat, skip_zeros);
+            mat->AddSubMatrix(vdofs, vdofs, *elmat_p, skip_zeros);
            if (hybridization)
            {
-               hybridization->AssembleBdrMatrix(i, elmat);
+               hybridization->AssembleBdrMatrix(i, *elmat_p);
            }
         }
         else
         {
-            static_cond->AssembleBdrMatrix(i, elmat);
+            static_cond->AssembleBdrMatrix(i, *elmat_p);
         }
      }
   }

-   if (fbfi.Size())
+   if (interior_face_integs.Size())
   {
      FaceElementTransformations *tr;
      Array<int> vdofs2;
@@ -550,18 +563,19 @@ void BilinearForm::Assemble(int skip_zeros)
            fes -> GetElementVDofs (tr -> Elem1No, vdofs);
            fes -> GetElementVDofs (tr -> Elem2No, vdofs2);
            vdofs.Append (vdofs2);
-            for (int k = 0; k < fbfi.Size(); k++)
+            for (int k = 0; k < interior_face_integs.Size(); k++)
            {
-               fbfi[k] -> AssembleFaceMatrix (*fes -> GetFE (tr -> Elem1No),
-                                              *fes -> GetFE (tr -> Elem2No),
-                                              *tr, elemmat);
+               interior_face_integs[k]->
+               AssembleFaceMatrix(*fes->GetFE(tr->Elem1No),
+                                  *fes->GetFE(tr->Elem2No),
+                                  *tr, elemmat);
               mat -> AddSubMatrix (vdofs, vdofs, elemmat, skip_zeros);
            }
         }
      }
   }

-   if (bfbfi.Size())
+   if (boundary_face_integs.Size())
   {
      FaceElementTransformations *tr;
      const FiniteElement *fe1, *fe2;
@@ -570,14 +584,14 @@ void BilinearForm::Assemble(int skip_zeros)
      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
                                 mesh->bdr_attributes.Max() : 0);
      bdr_attr_marker = 0;
-      for (int k = 0; k < bfbfi.Size(); k++)
+      for (int k = 0; k < boundary_face_integs.Size(); k++)
      {
-         if (bfbfi_marker[k] == NULL)
+         if (boundary_face_integs_marker[k] == NULL)
         {
            bdr_attr_marker = 1;
            break;
         }
-         Array<int> &bdr_marker = *bfbfi_marker[k];
+         Array<int> &bdr_marker = *boundary_face_integs_marker[k];
         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
                     "invalid boundary marker for boundary face integrator #"
                     << k << ", counting from zero");
@@ -601,12 +615,14 @@ void BilinearForm::Assemble(int skip_zeros)
            // but we can't dereference a NULL pointer, and we don't want to
            // actually make a fake element.
            fe2 = fe1;
-            for (int k = 0; k < bfbfi.Size(); k++)
+            for (int k = 0; k < boundary_face_integs.Size(); k++)
            {
-               if (bfbfi_marker[k] &&
-                   (*bfbfi_marker[k])[bdr_attr-1] == 0) { continue; }
+               if (boundary_face_integs_marker[k] &&
+                   (*boundary_face_integs_marker[k])[bdr_attr-1] == 0)
+               { continue; }

-               bfbfi[k] -> AssembleFaceMatrix (*fe1, *fe2, *tr, elemmat);
+               boundary_face_integs[k] -> AssembleFaceMatrix (*fe1, *fe2, *tr,
+                                                              elemmat);
               mat -> AddSubMatrix (vdofs, vdofs, elemmat, skip_zeros);
            }
         }
@@ -720,8 +736,8 @@ void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
      {
         // A, X and B point to the same data as mat, x and b
         EliminateVDofsInRHS(ess_tdof_list, x, b);
-         X.NewMemoryAndSize(x.GetMemory(), x.Size(), false);
-         B.NewMemoryAndSize(b.GetMemory(), b.Size(), false);
+         X.MakeRef(x, 0, x.Size());
+         B.MakeRef(b, 0, b.Size());
         if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
      }
   }
@@ -857,7 +873,7 @@ void BilinearForm::RecoverFEMSolution(const Vector &X,

 void BilinearForm::ComputeElementMatrices()
 {
-   if (element_matrices || dbfi.Size() == 0 || fes->GetNE() == 0)
+   if (element_matrices || domain_integs.Size() == 0 || fes->GetNE() == 0)
   {
      return;
   }
@@ -886,11 +902,11 @@ void BilinearForm::ComputeElementMatrices()
 #endif
      fes->GetElementTransformation(i, &eltrans);

-      dbfi[0]->AssembleElementMatrix(fe, eltrans, elmat);
-      for (int k = 1; k < dbfi.Size(); k++)
+      domain_integs[0]->AssembleElementMatrix(fe, eltrans, elmat);
+      for (int k = 1; k < domain_integs.Size(); k++)
      {
         // note: some integrators may not be thread-safe
-         dbfi[k]->AssembleElementMatrix(fe, eltrans, tmp);
+         domain_integs[k]->AssembleElementMatrix(fe, eltrans, tmp);
         elmat += tmp;
      }
      elmat.ClearExternalData();
@@ -1105,10 +1121,12 @@ BilinearForm::~BilinearForm()
   if (!extern_bfs)
   {
      int k;
-      for (k=0; k < dbfi.Size(); k++) { delete dbfi[k]; }
-      for (k=0; k < bbfi.Size(); k++) { delete bbfi[k]; }
-      for (k=0; k < fbfi.Size(); k++) { delete fbfi[k]; }
-      for (k=0; k < bfbfi.Size(); k++) { delete bfbfi[k]; }
+      for (k=0; k < domain_integs.Size(); k++) { delete domain_integs[k]; }
+      for (k=0; k < boundary_integs.Size(); k++) { delete boundary_integs[k]; }
+      for (k=0; k < interior_face_integs.Size(); k++)
+      { delete interior_face_integs[k]; }
+      for (k=0; k < boundary_face_integs.Size(); k++)
+      { delete boundary_face_integs[k]; }
   }

   delete ext;
@@ -1141,13 +1159,13 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
   ext = NULL;

   // Copy the pointers to the integrators
-   dbfi = mbf->dbfi;
-   bbfi = mbf->bbfi;
-   tfbfi = mbf->tfbfi;
-   btfbfi = mbf->btfbfi;
+   domain_integs = mbf->domain_integs;
+   boundary_integs = mbf->boundary_integs;
+   trace_face_integs = mbf->trace_face_integs;
+   boundary_trace_face_integs = mbf->boundary_trace_face_integs;

-   bbfi_marker = mbf->bbfi_marker;
-   btfbfi_marker = mbf->btfbfi_marker;
+   boundary_integs_marker = mbf->boundary_integs_marker;
+   boundary_trace_face_integs_marker = mbf->boundary_trace_face_integs_marker;

   assembly = AssemblyLevel::LEGACY;
   ext = NULL;
@@ -1236,7 +1254,8 @@ MatrixInverse * MixedBilinearForm::Inverse() const
 {
   if (assembly != AssemblyLevel::LEGACY)
   {
-      MFEM_WARNING("MixedBilinearForm::Inverse not possible with this assembly level!");
+      MFEM_WARNING("MixedBilinearForm::Inverse not possible with this "
+                   "assembly level!");
      return NULL;
   }
   else
@@ -1267,38 +1286,39 @@ void MixedBilinearForm::GetBlocks(Array2D<SparseMatrix *> &blocks) const

 void MixedBilinearForm::AddDomainIntegrator (BilinearFormIntegrator * bfi)
 {
-   dbfi.Append (bfi);
+   domain_integs.Append (bfi);
 }

 void MixedBilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi)
 {
-   bbfi.Append (bfi);
-   bbfi_marker.Append(NULL); // NULL marker means apply everywhere
+   boundary_integs.Append (bfi);
+   boundary_integs_marker.Append(NULL); // NULL marker means apply everywhere
 }

 void MixedBilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi,
                                               Array<int> &bdr_marker)
 {
-   bbfi.Append (bfi);
-   bbfi_marker.Append(&bdr_marker);
+   boundary_integs.Append (bfi);
+   boundary_integs_marker.Append(&bdr_marker);
 }

 void MixedBilinearForm::AddTraceFaceIntegrator (BilinearFormIntegrator * bfi)
 {
-   tfbfi.Append (bfi);
+   trace_face_integs.Append (bfi);
 }

 void MixedBilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator *bfi)
 {
-   btfbfi.Append(bfi);
-   btfbfi_marker.Append(NULL); // NULL marker means apply everywhere
+   boundary_trace_face_integs.Append(bfi);
+   // NULL marker means apply everywhere
+   boundary_trace_face_integs_marker.Append(NULL);
 }

 void MixedBilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator *bfi,
                                                  Array<int> &bdr_marker)
 {
-   btfbfi.Append(bfi);
-   btfbfi_marker.Append(&bdr_marker);
+   boundary_trace_face_integs.Append(bfi);
+   boundary_trace_face_integs_marker.Append(&bdr_marker);
 }

 void MixedBilinearForm::Assemble (int skip_zeros)
@@ -1309,9 +1329,10 @@ void MixedBilinearForm::Assemble (int skip_zeros)
      return;
   }

-   Array<int> tr_vdofs, te_vdofs;
   ElementTransformation *eltrans;
-   DenseMatrix elemmat;
+   DofTransformation * dom_dof_trans;
+   DofTransformation * ran_dof_trans;
+   DenseMatrix elmat;

   Mesh *mesh = test_fes -> GetMesh();

@@ -1320,37 +1341,45 @@ void MixedBilinearForm::Assemble (int skip_zeros)
      mat = new SparseMatrix(height, width);
   }

-   if (dbfi.Size())
+   if (domain_integs.Size())
   {
      for (int i = 0; i < test_fes -> GetNE(); i++)
      {
-         trial_fes -> GetElementVDofs (i, tr_vdofs);
-         test_fes  -> GetElementVDofs (i, te_vdofs);
+         dom_dof_trans = trial_fes -> GetElementVDofs (i, trial_vdofs);
+         ran_dof_trans = test_fes  -> GetElementVDofs (i, test_vdofs);
         eltrans = test_fes -> GetElementTransformation (i);
-         for (int k = 0; k < dbfi.Size(); k++)
+
+         elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
+         elmat = 0.0;
+         for (int k = 0; k < domain_integs.Size(); k++)
         {
-            dbfi[k] -> AssembleElementMatrix2 (*trial_fes -> GetFE(i),
-                                               *test_fes  -> GetFE(i),
-                                               *eltrans, elemmat);
-            mat -> AddSubMatrix (te_vdofs, tr_vdofs, elemmat, skip_zeros);
+            domain_integs[k] -> AssembleElementMatrix2 (*trial_fes -> GetFE(i),
+                                                        *test_fes  -> GetFE(i),
+                                                        *eltrans, elemmat);
+            elmat += elemmat;
         }
+         if (ran_dof_trans || dom_dof_trans)
+         {
+            TransformDual(ran_dof_trans, dom_dof_trans, elmat);
+         }
+         mat -> AddSubMatrix (test_vdofs, trial_vdofs, elmat, skip_zeros);
      }
   }

-   if (bbfi.Size())
+   if (boundary_integs.Size())
   {
      // Which boundary attributes need to be processed?
      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
                                 mesh->bdr_attributes.Max() : 0);
      bdr_attr_marker = 0;
-      for (int k = 0; k < bbfi.Size(); k++)
+      for (int k = 0; k < boundary_integs.Size(); k++)
      {
-         if (bbfi_marker[k] == NULL)
+         if (boundary_integs_marker[k] == NULL)
         {
            bdr_attr_marker = 1;
            break;
         }
-         Array<int> &bdr_marker = *bbfi_marker[k];
+         Array<int> &bdr_marker = *boundary_integs_marker[k];
         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
                     "invalid boundary marker for boundary integrator #"
                     << k << ", counting from zero");
@@ -1365,40 +1394,48 @@ void MixedBilinearForm::Assemble (int skip_zeros)
         const int bdr_attr = mesh->GetBdrAttribute(i);
         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }

-         trial_fes -> GetBdrElementVDofs (i, tr_vdofs);
-         test_fes  -> GetBdrElementVDofs (i, te_vdofs);
+         dom_dof_trans = trial_fes -> GetBdrElementVDofs (i, trial_vdofs);
+         ran_dof_trans = test_fes  -> GetBdrElementVDofs (i, test_vdofs);
         eltrans = test_fes -> GetBdrElementTransformation (i);
-         for (int k = 0; k < bbfi.Size(); k++)
-         {
-            if (bbfi_marker[k] &&
-                (*bbfi_marker[k])[bdr_attr-1] == 0) { continue; }

-            bbfi[k] -> AssembleElementMatrix2 (*trial_fes -> GetBE(i),
-                                               *test_fes  -> GetBE(i),
-                                               *eltrans, elemmat);
-            mat -> AddSubMatrix (te_vdofs, tr_vdofs, elemmat, skip_zeros);
+         elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
+         elmat = 0.0;
+         for (int k = 0; k < boundary_integs.Size(); k++)
+         {
+            if (boundary_integs_marker[k] &&
+                (*boundary_integs_marker[k])[bdr_attr-1] == 0) { continue; }
+
+            boundary_integs[k]->AssembleElementMatrix2 (*trial_fes -> GetBE(i),
+                                                        *test_fes  -> GetBE(i),
+                                                        *eltrans, elemmat);
+            elmat += elemmat;
         }
+         if (ran_dof_trans || dom_dof_trans)
+         {
+            TransformDual(ran_dof_trans, dom_dof_trans, elmat);
+         }
+         mat -> AddSubMatrix (test_vdofs, trial_vdofs, elmat, skip_zeros);
      }
   }

-   if (tfbfi.Size())
+   if (trace_face_integs.Size())
   {
      FaceElementTransformations *ftr;
-      Array<int> te_vdofs2;
+      Array<int> test_vdofs2;
      const FiniteElement *trial_face_fe, *test_fe1, *test_fe2;

      int nfaces = mesh->GetNumFaces();
      for (int i = 0; i < nfaces; i++)
      {
         ftr = mesh->GetFaceElementTransformations(i);
-         trial_fes->GetFaceVDofs(i, tr_vdofs);
-         test_fes->GetElementVDofs(ftr->Elem1No, te_vdofs);
+         trial_fes->GetFaceVDofs(i, trial_vdofs);
+         test_fes->GetElementVDofs(ftr->Elem1No, test_vdofs);
         trial_face_fe = trial_fes->GetFaceElement(i);
         test_fe1 = test_fes->GetFE(ftr->Elem1No);
         if (ftr->Elem2No >= 0)
         {
-            test_fes->GetElementVDofs(ftr->Elem2No, te_vdofs2);
-            te_vdofs.Append(te_vdofs2);
+            test_fes->GetElementVDofs(ftr->Elem2No, test_vdofs2);
+            test_vdofs.Append(test_vdofs2);
            test_fe2 = test_fes->GetFE(ftr->Elem2No);
         }
         else
@@ -1408,16 +1445,16 @@ void MixedBilinearForm::Assemble (int skip_zeros)
            // want to actually make a fake element.
            test_fe2 = test_fe1;
         }
-         for (int k = 0; k < tfbfi.Size(); k++)
+         for (int k = 0; k < trace_face_integs.Size(); k++)
         {
-            tfbfi[k]->AssembleFaceMatrix(*trial_face_fe, *test_fe1, *test_fe2,
-                                         *ftr, elemmat);
-            mat->AddSubMatrix(te_vdofs, tr_vdofs, elemmat, skip_zeros);
+            trace_face_integs[k]->AssembleFaceMatrix(*trial_face_fe, *test_fe1,
+                                                     *test_fe2, *ftr, elemmat);
+            mat->AddSubMatrix(test_vdofs, trial_vdofs, elemmat, skip_zeros);
         }
      }
   }

-   if (btfbfi.Size())
+   if (boundary_trace_face_integs.Size())
   {
      FaceElementTransformations *ftr;
      Array<int> te_vdofs2;
@@ -1427,17 +1464,17 @@ void MixedBilinearForm::Assemble (int skip_zeros)
      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
                                 mesh->bdr_attributes.Max() : 0);
      bdr_attr_marker = 0;
-      for (int k = 0; k < btfbfi.Size(); k++)
+      for (int k = 0; k < boundary_trace_face_integs.Size(); k++)
      {
-         if (btfbfi_marker[k] == NULL)
+         if (boundary_trace_face_integs_marker[k] == NULL)
         {
            bdr_attr_marker = 1;
            break;
         }
-         Array<int> &bdr_marker = *btfbfi_marker[k];
+         Array<int> &bdr_marker = *boundary_trace_face_integs_marker[k];
         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
-                     "invalid boundary marker for boundary trace face integrator #"
-                     << k << ", counting from zero");
+                     "invalid boundary marker for boundary trace face"
+                     "integrator #" << k << ", counting from zero");
         for (int i = 0; i < bdr_attr_marker.Size(); i++)
         {
            bdr_attr_marker[i] |= bdr_marker[i];
@@ -1452,22 +1489,25 @@ void MixedBilinearForm::Assemble (int skip_zeros)
         ftr = mesh->GetBdrFaceTransformations(i);
         if (ftr)
         {
-            trial_fes->GetFaceVDofs(ftr->ElementNo, tr_vdofs);
-            test_fes->GetElementVDofs(ftr->Elem1No, te_vdofs);
+            trial_fes->GetFaceVDofs(ftr->ElementNo, trial_vdofs);
+            test_fes->GetElementVDofs(ftr->Elem1No, test_vdofs);
            trial_face_fe = trial_fes->GetFaceElement(ftr->ElementNo);
            test_fe1 = test_fes->GetFE(ftr->Elem1No);
            // The test_fe2 object is really a dummy and not used on the
            // boundaries, but we can't dereference a NULL pointer, and we don't
            // want to actually make a fake element.
            test_fe2 = test_fe1;
-            for (int k = 0; k < btfbfi.Size(); k++)
+            for (int k = 0; k < boundary_trace_face_integs.Size(); k++)
            {
-               if (btfbfi_marker[k] &&
-                   (*btfbfi_marker[k])[bdr_attr-1] == 0) { continue; }
+               if (boundary_trace_face_integs_marker[k] &&
+                   (*boundary_trace_face_integs_marker[k])[bdr_attr-1] == 0)
+               { continue; }

-               btfbfi[k]->AssembleFaceMatrix(*trial_face_fe, *test_fe1, *test_fe2,
-                                             *ftr, elemmat);
-               mat->AddSubMatrix(te_vdofs, tr_vdofs, elemmat, skip_zeros);
+               boundary_trace_face_integs[k]->AssembleFaceMatrix(*trial_face_fe,
+                                                                 *test_fe1,
+                                                                 *test_fe2,
+                                                                 *ftr, elemmat);
+               mat->AddSubMatrix(test_vdofs, trial_vdofs, elemmat, skip_zeros);
            }
         }
      }
@@ -1557,15 +1597,17 @@ void MixedBilinearForm::ConformingAssemble()

 void MixedBilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
 {
-   if (dbfi.Size())
+   if (domain_integs.Size())
   {
      const FiniteElement &trial_fe = *trial_fes->GetFE(i);
      const FiniteElement &test_fe = *test_fes->GetFE(i);
      ElementTransformation *eltrans = test_fes->GetElementTransformation(i);
-      dbfi[0]->AssembleElementMatrix2(trial_fe, test_fe, *eltrans, elmat);
-      for (int k = 1; k < dbfi.Size(); k++)
+      domain_integs[0]->AssembleElementMatrix2(trial_fe, test_fe, *eltrans,
+                                               elmat);
+      for (int k = 1; k < domain_integs.Size(); k++)
      {
-         dbfi[k]->AssembleElementMatrix2(trial_fe, test_fe, *eltrans, elemmat);
+         domain_integs[k]->AssembleElementMatrix2(trial_fe, test_fe, *eltrans,
+                                                  elemmat);
         elmat += elemmat;
      }
   }
@@ -1580,15 +1622,17 @@ void MixedBilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)

 void MixedBilinearForm::ComputeBdrElementMatrix(int i, DenseMatrix &elmat)
 {
-   if (bbfi.Size())
+   if (boundary_integs.Size())
   {
      const FiniteElement &trial_be = *trial_fes->GetBE(i);
      const FiniteElement &test_be = *test_fes->GetBE(i);
      ElementTransformation *eltrans = test_fes->GetBdrElementTransformation(i);
-      bbfi[0]->AssembleElementMatrix2(trial_be, test_be, *eltrans, elmat);
-      for (int k = 1; k < bbfi.Size(); k++)
+      boundary_integs[0]->AssembleElementMatrix2(trial_be, test_be, *eltrans,
+                                                 elmat);
+      for (int k = 1; k < boundary_integs.Size(); k++)
      {
-         bbfi[k]->AssembleElementMatrix2(trial_be, test_be, *eltrans, elemmat);
+         boundary_integs[k]->AssembleElementMatrix2(trial_be, test_be, *eltrans,
+                                                    elemmat);
         elmat += elemmat;
      }
   }
@@ -1688,10 +1732,10 @@ void MixedBilinearForm::EliminateTestDofs (const Array<int> &bdr_attr_is_ess)
      }
 }

-void MixedBilinearForm::FormRectangularSystemMatrix(const Array<int>
-                                                    &trial_tdof_list,
-                                                    const Array<int> &test_tdof_list,
-                                                    OperatorHandle &A)
+void MixedBilinearForm::FormRectangularSystemMatrix(
+   const Array<int> &trial_tdof_list,
+   const Array<int> &test_tdof_list,
+   OperatorHandle &A)

 {
   if (ext)
@@ -1729,17 +1773,17 @@ void MixedBilinearForm::FormRectangularSystemMatrix(const Array<int>
   A.Reset(mat, false);
 }

-void MixedBilinearForm::FormRectangularLinearSystem(const Array<int>
-                                                    &trial_tdof_list,
-                                                    const Array<int> &test_tdof_list,
-                                                    Vector &x, Vector &b,
-                                                    OperatorHandle &A,
-                                                    Vector &X, Vector &B)
+void MixedBilinearForm::FormRectangularLinearSystem(
+   const Array<int> &trial_tdof_list,
+   const Array<int> &test_tdof_list,
+   Vector &x, Vector &b,
+   OperatorHandle &A,
+   Vector &X, Vector &B)
 {
   if (ext)
   {
-      ext->FormRectangularLinearSystem(trial_tdof_list, test_tdof_list, x, b, A, X,
-                                       B);
+      ext->FormRectangularLinearSystem(trial_tdof_list, test_tdof_list,
+                                       x, b, A, X, B);
      return;
   }

@@ -1777,10 +1821,13 @@ MixedBilinearForm::~MixedBilinearForm()
   if (!extern_bfs)
   {
      int i;
-      for (i = 0; i < dbfi.Size(); i++) { delete dbfi[i]; }
-      for (i = 0; i < bbfi.Size(); i++) { delete bbfi[i]; }
-      for (i = 0; i < tfbfi.Size(); i++) { delete tfbfi[i]; }
-      for (i = 0; i < btfbfi.Size(); i++) { delete btfbfi[i]; }
+      for (i = 0; i < domain_integs.Size(); i++) { delete domain_integs[i]; }
+      for (i = 0; i < boundary_integs.Size(); i++)
+      { delete boundary_integs[i]; }
+      for (i = 0; i < trace_face_integs.Size(); i++)
+      { delete trace_face_integs[i]; }
+      for (i = 0; i < boundary_trace_face_integs.Size(); i++)
+      { delete boundary_trace_face_integs[i]; }
   }
   delete ext;
 }
@@ -1822,6 +1869,8 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)

   Array<int> dom_vdofs, ran_vdofs;
   ElementTransformation *T;
+   DofTransformation * dom_dof_trans;
+   DofTransformation * ran_dof_trans;
   const FiniteElement *dom_fe, *ran_fe;
   DenseMatrix totelmat, elmat;

@@ -1830,27 +1879,33 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
      mat = new SparseMatrix(height, width);
   }

-   if (dbfi.Size() > 0)
+   if (domain_integs.Size() > 0)
   {
      for (int i = 0; i < test_fes->GetNE(); i++)
      {
-         trial_fes->GetElementVDofs(i, dom_vdofs);
-         test_fes->GetElementVDofs(i, ran_vdofs);
+         dom_dof_trans = trial_fes->GetElementVDofs(i, dom_vdofs);
+         ran_dof_trans = test_fes->GetElementVDofs(i, ran_vdofs);
         T = test_fes->GetElementTransformation(i);
         dom_fe = trial_fes->GetFE(i);
         ran_fe = test_fes->GetFE(i);

-         dbfi[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, totelmat);
-         for (int j = 1; j < dbfi.Size(); j++)
+         domain_integs[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
+                                                  totelmat);
+         for (int j = 1; j < domain_integs.Size(); j++)
         {
-            dbfi[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, elmat);
+            domain_integs[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
+                                                     elmat);
            totelmat += elmat;
         }
+         if (ran_dof_trans || dom_dof_trans)
+         {
+            TransformPrimal(ran_dof_trans, dom_dof_trans, totelmat);
+         }
         mat->SetSubMatrix(ran_vdofs, dom_vdofs, totelmat, skip_zeros);
      }
   }

-   if (tfbfi.Size())
+   if (trace_face_integs.Size())
   {
      const int nfaces = test_fes->GetMesh()->GetNumFaces();
      for (int i = 0; i < nfaces; i++)
@@ -1861,10 +1916,12 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
         dom_fe = trial_fes->GetFaceElement(i);
         ran_fe = test_fes->GetFaceElement(i);

-         tfbfi[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, totelmat);
-         for (int j = 1; j < tfbfi.Size(); j++)
+         trace_face_integs[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
+                                                      totelmat);
+         for (int j = 1; j < trace_face_integs.Size(); j++)
         {
-            tfbfi[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, elmat);
+            trace_face_integs[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
+                                                         elmat);
            totelmat += elmat;
         }
         mat->SetSubMatrix(ran_vdofs, dom_vdofs, totelmat, skip_zeros);
@@ -84,28 +84,29 @@ protected:
       the BilinearForm. */
   long sequence;

-   /** @brief Indicates the BilinearFormIntegrator%s stored in #dbfi, #bbfi,
-       #fbfi, and #bfbfi are owned by another BilinearForm. */
+   /** @brief Indicates the BilinearFormIntegrator%s stored in #domain_integs,
+       #boundary_integs, #interior_face_integs, and #boundary_face_integs are
+       owned by another BilinearForm. */
   int extern_bfs;

   /// Set of Domain Integrators to be applied.
-   Array<BilinearFormIntegrator*> dbfi;
+   Array<BilinearFormIntegrator*> domain_integs;
   /// Element attribute marker (should be of length mesh->attributes)
   /// Includes all by default.
   /// 0 - ignore attribute
   /// 1 - include attribute
-   Array<Array<int>*>             dbfi_marker;
+   Array<Array<int>*>             domain_integs_marker;

   /// Set of Boundary Integrators to be applied.
-   Array<BilinearFormIntegrator*> bbfi;
-   Array<Array<int>*>             bbfi_marker; ///< Entries are not owned.
+   Array<BilinearFormIntegrator*> boundary_integs;
+   Array<Array<int>*> boundary_integs_marker; ///< Entries are not owned.

   /// Set of interior face Integrators to be applied.
-   Array<BilinearFormIntegrator*> fbfi;
+   Array<BilinearFormIntegrator*> interior_face_integs;

   /// Set of boundary face Integrators to be applied.
-   Array<BilinearFormIntegrator*> bfbfi;
-   Array<Array<int>*>             bfbfi_marker; ///< Entries are not owned.
+   Array<BilinearFormIntegrator*> boundary_face_integs;
+   Array<Array<int>*> boundary_face_integs_marker; ///< Entries are not owned.

   DenseMatrix elemmat;
   Array<int>  vdofs;
@@ -231,24 +232,25 @@ public:
   void AllocateMatrix() { if (mat == NULL) { AllocMat(); } }

   /// Access all the integrators added with AddDomainIntegrator().
-   Array<BilinearFormIntegrator*> *GetDBFI() { return &dbfi; }
+   Array<BilinearFormIntegrator*> *GetDBFI() { return &domain_integs; }

   /// Access all the integrators added with AddBoundaryIntegrator().
-   Array<BilinearFormIntegrator*> *GetBBFI() { return &bbfi; }
+   Array<BilinearFormIntegrator*> *GetBBFI() { return &boundary_integs; }
   /** @brief Access all boundary markers added with AddBoundaryIntegrator().
       If no marker was specified when the integrator was added, the
       corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetBBFI_Marker() { return &bbfi_marker; }
+   Array<Array<int>*> *GetBBFI_Marker() { return &boundary_integs_marker; }

   /// Access all integrators added with AddInteriorFaceIntegrator().
-   Array<BilinearFormIntegrator*> *GetFBFI() { return &fbfi; }
+   Array<BilinearFormIntegrator*> *GetFBFI() { return &interior_face_integs; }

   /// Access all integrators added with AddBdrFaceIntegrator().
-   Array<BilinearFormIntegrator*> *GetBFBFI() { return &bfbfi; }
+   Array<BilinearFormIntegrator*> *GetBFBFI() { return &boundary_face_integs; }
   /** @brief Access all boundary markers added with AddBdrFaceIntegrator().
       If no marker was specified when the integrator was added, the
       corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetBFBFI_Marker() { return &bfbfi_marker; }
+   Array<Array<int>*> *GetBFBFI_Marker()
+   { return &boundary_face_integs_marker; }

   /// Returns a reference to: \f$ M_{ij} \f$
   const double &operator()(int i, int j) { return (*mat)(i,j); }
@@ -652,23 +654,25 @@ protected:
       Partial Assembly (PA), or Matrix Free assembly (MF). */
   MixedBilinearFormExtension *ext;

-   /** @brief Indicates the BilinearFormIntegrator%s stored in #dbfi, #bbfi,
-       #tfbfi and #btfbfi are owned by another MixedBilinearForm. */
+   /** @brief Indicates the BilinearFormIntegrator%s stored in #domain_integs,
+       #boundary_integs, #trace_face_integs and #boundary_trace_face_integs
+       are owned by another MixedBilinearForm. */
   int extern_bfs;

   /// Domain integrators.
-   Array<BilinearFormIntegrator*> dbfi;
+   Array<BilinearFormIntegrator*> domain_integs;

   /// Boundary integrators.
-   Array<BilinearFormIntegrator*> bbfi;
-   Array<Array<int>*>             bbfi_marker;///< Entries are not owned.
+   Array<BilinearFormIntegrator*> boundary_integs;
+   Array<Array<int>*> boundary_integs_marker; ///< Entries are not owned.

   /// Trace face (skeleton) integrators.
-   Array<BilinearFormIntegrator*> tfbfi;
+   Array<BilinearFormIntegrator*> trace_face_integs;

   /// Boundary trace face (skeleton) integrators.
-   Array<BilinearFormIntegrator*> btfbfi;
-   Array<Array<int>*>             btfbfi_marker;///< Entries are not owned.
+   Array<BilinearFormIntegrator*> boundary_trace_face_integs;
+   /// Entries are not owned.
+   Array<Array<int>*> boundary_trace_face_integs_marker;

   DenseMatrix elemmat;
   Array<int>  trial_vdofs, test_vdofs;
@@ -762,24 +766,26 @@ public:
                                   Array<int> &bdr_marker);

   /// Access all integrators added with AddDomainIntegrator().
-   Array<BilinearFormIntegrator*> *GetDBFI() { return &dbfi; }
+   Array<BilinearFormIntegrator*> *GetDBFI() { return &domain_integs; }

   /// Access all integrators added with AddBoundaryIntegrator().
-   Array<BilinearFormIntegrator*> *GetBBFI() { return &bbfi; }
+   Array<BilinearFormIntegrator*> *GetBBFI() { return &boundary_integs; }
   /** @brief Access all boundary markers added with AddBoundaryIntegrator().
       If no marker was specified when the integrator was added, the
       corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetBBFI_Marker() { return &bbfi_marker; }
+   Array<Array<int>*> *GetBBFI_Marker() { return &boundary_integs_marker; }

   /// Access all integrators added with AddTraceFaceIntegrator().
-   Array<BilinearFormIntegrator*> *GetTFBFI() { return &tfbfi; }
+   Array<BilinearFormIntegrator*> *GetTFBFI() { return &trace_face_integs; }

   /// Access all integrators added with AddBdrTraceFaceIntegrator().
-   Array<BilinearFormIntegrator*> *GetBTFBFI() { return &btfbfi; }
+   Array<BilinearFormIntegrator*> *GetBTFBFI()
+   { return &boundary_trace_face_integs; }
   /** @brief Access all boundary markers added with AddBdrTraceFaceIntegrator().
       If no marker was specified when the integrator was added, the
       corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetBTFBFI_Marker() { return &btfbfi_marker; }
+   Array<Array<int>*> *GetBTFBFI_Marker()
+   { return &boundary_trace_face_integs_marker; }

   /// Sets all sparse values of \f$ M \f$ to @a a.
   void operator=(const double a) { *mat = a; }
@@ -1004,7 +1010,7 @@ public:
   { AddTraceFaceIntegrator(di); }

   /// Access all interpolators added with AddDomainInterpolator().
-   Array<BilinearFormIntegrator*> *GetDI() { return &dbfi; }
+   Array<BilinearFormIntegrator*> *GetDI() { return &domain_integs; }

   /// Set the desired assembly level. The default is AssemblyLevel::FULL.
   /** This method must be called before assembly. */
@@ -160,7 +160,7 @@ void MFBilinearFormExtension::Mult(const Vector &x, Vector &y) const
         {
            intFaceIntegrators[i]->AddMultMF(faceIntX, faceIntY);
         }
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
+         int_face_restrict_lex->AddMultTranspose(faceIntY, y);
      }
   }

@@ -176,7 +176,7 @@ void MFBilinearFormExtension::Mult(const Vector &x, Vector &y) const
         {
            bdrFaceIntegrators[i]->AddMultMF(faceBdrX, faceBdrY);
         }
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
+         bdr_face_restrict_lex->AddMultTranspose(faceBdrY, y);
      }
   }
 }
@@ -217,7 +217,7 @@ void MFBilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         {
            intFaceIntegrators[i]->AddMultTransposeMF(faceIntX, faceIntY);
         }
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
+         int_face_restrict_lex->AddMultTranspose(faceIntY, y);
      }
   }

@@ -233,7 +233,7 @@ void MFBilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         {
            bdrFaceIntegrators[i]->AddMultTransposeMF(faceBdrX, faceBdrY);
         }
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
+         bdr_face_restrict_lex->AddMultTranspose(faceBdrY, y);
      }
   }
 }
@@ -417,7 +417,7 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
         {
            intFaceIntegrators[i]->AddMultPA(faceIntX, faceIntY);
         }
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
+         int_face_restrict_lex->AddMultTranspose(faceIntY, y);
      }
   }

@@ -433,7 +433,7 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
         {
            bdrFaceIntegrators[i]->AddMultPA(faceBdrX, faceBdrY);
         }
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
+         bdr_face_restrict_lex->AddMultTranspose(faceBdrY, y);
      }
   }
 }
@@ -474,7 +474,7 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         {
            intFaceIntegrators[i]->AddMultTransposePA(faceIntX, faceIntY);
         }
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
+         int_face_restrict_lex->AddMultTranspose(faceIntY, y);
      }
   }

@@ -490,7 +490,7 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         {
            bdrFaceIntegrators[i]->AddMultTransposePA(faceBdrX, faceBdrY);
         }
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
+         bdr_face_restrict_lex->AddMultTranspose(faceBdrY, y);
      }
   }
 }
@@ -657,7 +657,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
            Y(j, 0, f) += res;
         });
         // Apply the Interior Face Restriction transposed
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
+         int_face_restrict_lex->AddMultTranspose(faceIntY, y);
      }
   }

@@ -688,7 +688,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
            Y(j, f) += res;
         });
         // Apply the Boundary Face Restriction transposed
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
+         bdr_face_restrict_lex->AddMultTranspose(faceBdrY, y);
      }
   }
 }
@@ -783,7 +783,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
            Y(j, 0, f) += res;
         });
         // Apply the Interior Face Restriction transposed
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
+         int_face_restrict_lex->AddMultTranspose(faceIntY, y);
      }
   }

@@ -814,7 +814,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
            Y(j, f) += res;
         });
         // Apply the Boundary Face Restriction transposed
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
+         bdr_face_restrict_lex->AddMultTranspose(faceBdrY, y);
      }
   }
 }
@@ -72,8 +72,8 @@ protected:
   mutable Vector faceIntX, faceIntY;
   mutable Vector faceBdrX, faceBdrY;
   const Operator *elem_restrict; // Not owned
-   const Operator *int_face_restrict_lex; // Not owned
-   const Operator *bdr_face_restrict_lex; // Not owned
+   const FaceRestriction *int_face_restrict_lex; // Not owned
+   const FaceRestriction *bdr_face_restrict_lex; // Not owned

 public:
   PABilinearFormExtension(BilinearForm*);
@@ -143,8 +143,8 @@ protected:
   mutable Vector faceIntX, faceIntY;
   mutable Vector faceBdrX, faceBdrY;
   const Operator *elem_restrict; // Not owned
-   const Operator *int_face_restrict_lex; // Not owned
-   const Operator *bdr_face_restrict_lex; // Not owned
+   const FaceRestriction *int_face_restrict_lex; // Not owned
+   const FaceRestriction *bdr_face_restrict_lex; // Not owned

 public:
   MFBilinearFormExtension(BilinearForm *form);
@@ -175,6 +175,11 @@ void BilinearFormIntegrator::AssembleFaceVector(
   elmat.Mult(elfun, elvect);
 }

+void TransposeIntegrator::SetIntRule(const IntegrationRule *ir)
+{
+   IntRule = ir;
+   bfi->SetIntRule(ir);
+}

 void TransposeIntegrator::AssembleElementMatrix (
   const FiniteElement &el, ElementTransformation &Trans, DenseMatrix &elmat)
@@ -202,6 +207,12 @@ void TransposeIntegrator::AssembleFaceMatrix (
   elmat.Transpose (bfi_elmat);
 }

+void LumpedIntegrator::SetIntRule(const IntegrationRule *ir)
+{
+   IntRule = ir;
+   bfi->SetIntRule(ir);
+}
+
 void LumpedIntegrator::AssembleElementMatrix (
   const FiniteElement &el, ElementTransformation &Trans, DenseMatrix &elmat)
 {
@@ -209,6 +220,12 @@ void LumpedIntegrator::AssembleElementMatrix (
   elmat.Lump();
 }

+void InverseIntegrator::SetIntRule(const IntegrationRule *ir)
+{
+   IntRule = ir;
+   integrator->SetIntRule(ir);
+}
+
 void InverseIntegrator::AssembleElementMatrix(
   const FiniteElement &el, ElementTransformation &Trans, DenseMatrix &elmat)
 {
@@ -216,6 +233,15 @@ void InverseIntegrator::AssembleElementMatrix(
   elmat.Invert();
 }

+void SumIntegrator::SetIntRule(const IntegrationRule *ir)
+{
+   IntRule = ir;
+   for (int i = 0; i < integrators.Size(); i++)
+   {
+      integrators[i]->SetIntRule(ir);
+   }
+}
+
 void SumIntegrator::AssembleElementMatrix(
   const FiniteElement &el, ElementTransformation &Trans, DenseMatrix &elmat)
 {
@@ -1751,15 +1777,16 @@ void DerivativeIntegrator::AssembleElementMatrix2 (
   int dim = trial_fe.GetDim();
   int trial_nd = trial_fe.GetDof();
   int test_nd = test_fe.GetDof();
+   int spaceDim = Trans.GetSpaceDim();

   int i, l;
   double det;

   elmat.SetSize (test_nd,trial_nd);
   dshape.SetSize (trial_nd,dim);
-   dshapedxt.SetSize(trial_nd,dim);
+   dshapedxt.SetSize(trial_nd, spaceDim);
   dshapedxi.SetSize(trial_nd);
-   invdfdx.SetSize(dim);
+   invdfdx.SetSize(dim, spaceDim);
   shape.SetSize (test_nd);

   const IntegrationRule *ir = IntRule;
@@ -261,6 +261,8 @@ public:
   TransposeIntegrator (BilinearFormIntegrator *bfi_, int own_bfi_ = 1)
   { bfi = bfi_; own_bfi = own_bfi_; }

+   virtual void SetIntRule(const IntegrationRule *ir);
+
   virtual void AssembleElementMatrix(const FiniteElement &el,
                                      ElementTransformation &Trans,
                                      DenseMatrix &elmat);
@@ -328,6 +330,8 @@ public:
   LumpedIntegrator (BilinearFormIntegrator *bfi_, int own_bfi_ = 1)
   { bfi = bfi_; own_bfi = own_bfi_; }

+   virtual void SetIntRule(const IntegrationRule *ir);
+
   virtual void AssembleElementMatrix(const FiniteElement &el,
                                      ElementTransformation &Trans,
                                      DenseMatrix &elmat);
@@ -346,6 +350,8 @@ public:
   InverseIntegrator(BilinearFormIntegrator *integ, int own_integ = 1)
   { integrator = integ; own_integrator = own_integ; }

+   virtual void SetIntRule(const IntegrationRule *ir);
+
   virtual void AssembleElementMatrix(const FiniteElement &el,
                                      ElementTransformation &Trans,
                                      DenseMatrix &elmat);
@@ -364,6 +370,8 @@ private:
 public:
   SumIntegrator(int own_integs = 1) { own_integrators = own_integs; }

+   virtual void SetIntRule(const IntegrationRule *ir);
+
   void AddIntegrator(BilinearFormIntegrator *integ)
   { integrators.Append(integ); }

@@ -703,7 +711,7 @@ protected:
   {
      return "MixedScalarDerivativeIntegrator:  "
             "Trial and test spaces must both be scalar fields in 1D "
-             "and the trial space must implement CaldDShape.";
+             "and the trial space must implement CalcDShape.";
   }

   inline virtual void CalcTrialShape(const FiniteElement & trial_fe,
@@ -2928,11 +2936,11 @@ public:

    - F. Bassi and S. Rebay. A high order discontinuous Galerkin method for
      compressible turbulent flows. In B. Cockburn, G. E. Karniadakis, and
-      C.-W. Shu, editors, Discontinuous Galerkin Methods, pages 77–88. Springer
+      C.-W. Shu, editors, Discontinuous Galerkin Methods, pages 77-88. Springer
      Berlin Heidelberg, 2000.
    - D. N. Arnold, F. Brezzi, B. Cockburn, and L. D. Marini. Unified analysis
      of discontinuous Galerkin methods for elliptic problems. SIAM Journal on
-      Numerical Analysis, 39(5):1749–1779, 2002.
+      Numerical Analysis, 39(5):1749-1779, 2002.
 */
 class DGDiffusionBR2Integrator : public BilinearFormIntegrator
 {
@@ -143,7 +143,7 @@ Solver *BuildSmootherFromCeed(ConstrainedOperator &op, bool chebyshev)
   if (chebyshev)
   {
      const int cheb_order = 3;
-      out = new OperatorChebyshevSmoother(&op, t_diag, ess_tdofs, cheb_order);
+      out = new OperatorChebyshevSmoother(op, t_diag, ess_tdofs, cheb_order);
   }
   else
   {
@@ -1204,17 +1204,30 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
      });
      // Modify offdiagonal blocks (imaginary parts of the matrix) to conform
      // with standard essential BC treatment
-      ess_tdof_list.HostRead();
      if (A_i.Type() == Operator::Hypre_ParCSR)
      {
         HypreParMatrix * Ah;
         A_i.Get(Ah);
         hypre_ParCSRMatrix *Aih = *Ah;
+#ifndef HYPRE_USING_CUDA
+         ess_tdof_list.HostRead();
         for (int k = 0; k < n; k++)
         {
            const int j = ess_tdof_list[k];
            Aih->diag->data[Aih->diag->i[j]] = 0.0;
         }
+#else
+         Ah->HypreReadWrite();
+         const int *d_ess_tdof_list =
+            ess_tdof_list.GetMemory().Read(MemoryClass::DEVICE, n);
+         const int *d_diag_i = Aih->diag->i;
+         double *d_diag_data = Aih->diag->data;
+         CuWrap1D(n, [=] MFEM_DEVICE (int k)
+         {
+            const int j = d_ess_tdof_list[k];
+            d_diag_data[d_diag_i[j]] = 0.0;
+         });
+#endif
      }
      else
      {
@@ -17,6 +17,7 @@

 #include <cerrno>      // errno
 #include <sstream>
+#include <regex>

 #ifndef _WIN32
 #include <sys/stat.h>  // mkdir
@@ -764,7 +765,8 @@ ParaViewDataCollection::ParaViewDataCollection(const std::string&
   : DataCollection(collection_name, mesh_),
     levels_of_detail(1),
     pv_data_format(VTKFormat::BINARY),
-     high_order_output(false)
+     high_order_output(false),
+     restart_mode(false)
 {
 #ifdef MFEM_USE_ZLIB
   compression = -1; // default zlib compression level, equivalent to 6
@@ -842,17 +844,60 @@ void ParaViewDataCollection::Save()
   }
   // the directory is created

-   // create pvd file if needed
+   // create pvd file if needed. If we are not in restart mode, a new pvd file
+   // is always created. In restart mode, we keep any previously defined
+   // timestep values as long as they are less than the currently defined time.
+
   if (myid == 0 && !pvd_stream.is_open())
   {
      std::string dpath=GenerateCollectionPath();
      std::string pvdname=dpath+"/"+GeneratePVDFileName();
-      pvd_stream.open(pvdname.c_str(),std::ios::out);
-      // initialize the file
-      pvd_stream << "<?xml version=\"1.0\"?>\n";
-      pvd_stream << "<VTKFile type=\"Collection\" version=\"0.1\"";
-      pvd_stream << " byte_order=\"" << VTKByteOrder() << "\">\n";
-      pvd_stream << "<Collection>" << std::endl;
+
+      std::ifstream pvd_in;
+      if (restart_mode && (pvd_in.open(pvdname,std::ios::binary),pvd_in.good()))
+      {
+         // PVD file exists and restart mode enabled: preserve existing time
+         // steps less than the current time.
+         std::fstream::pos_type pos_begin = pvd_in.tellg();
+         std::fstream::pos_type pos_end = pos_begin;
+
+         std::regex regexp("timestep=\"([^[:space:]]+)\".*file=\"Cycle(\\d+)");
+         std::smatch match;
+
+         std::string line;
+         while (getline(pvd_in,line))
+         {
+            if (regex_search(line,match,regexp))
+            {
+               MFEM_ASSERT(match.size() == 3, "Unable to parse DataSet");
+               double tvalue = std::stod(match[1]);
+               if (tvalue >= GetTime()) { break; }
+               int cvalue = std::stoi(match[2]);
+               MFEM_VERIFY(cvalue < GetCycle(), "Cycle " << GetCycle() <<
+                           " is too small for restart mode: trying to overwrite"
+                           " existing data.");
+               pos_end = pvd_in.tellg();
+            }
+         }
+         size_t count = pos_end - pos_begin;
+         std::vector<char> buf(count);
+         pvd_in.clear();
+         pvd_in.seekg(pos_begin);
+         pvd_in.read(buf.data(), count);
+         pvd_in.close();
+         pvd_stream.open(pvdname.c_str(),std::ios::out);
+         pvd_stream.write(buf.data(), count);
+      }
+      else
+      {
+         // initialize new pvd file
+         pvd_stream.open(pvdname.c_str(),std::ios::out);
+         // initialize the file
+         pvd_stream << "<?xml version=\"1.0\"?>\n";
+         pvd_stream << "<VTKFile type=\"Collection\" version=\"0.1\"";
+         pvd_stream << " byte_order=\"" << VTKByteOrder() << "\">\n";
+         pvd_stream << "<Collection>" << std::endl;
+      }
   }

   // define the vtu file
@@ -1091,6 +1136,11 @@ void ParaViewDataCollection::SetCompression(bool compression_)
   }
 }

+void ParaViewDataCollection::UseRestartMode(bool restart_mode_)
+{
+   restart_mode = restart_mode_;
+}
+
 const char *ParaViewDataCollection::GetDataFormatString() const
 {
   if (pv_data_format == VTKFormat::ASCII)
@@ -488,6 +488,7 @@ private:
   std::fstream pvd_stream;
   VTKFormat pv_data_format;
   bool high_order_output;
+   bool restart_mode;

 protected:
   void SaveDataVTU(std::ostream &out, int ref);
@@ -545,6 +546,11 @@ public:
   /// by default). Reading high-order data requires ParaView 5.5 or later.
   void SetHighOrderOutput(bool high_order_output_);

+   /// Enable or disable restart mode. If restart is enabled, new writes will
+   /// preserve timestep metadata for any solutions prior to the currently
+   /// defined time.
+   void UseRestartMode(bool restart_mode_);
+
   /// Load the collection - not implemented in the ParaView writer
   virtual void Load(int cycle_ = 0) override;
 };
@@ -0,0 +1,358 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "fem.hpp"
+
+namespace mfem
+{
+
+void DofTransformation::TransformPrimal(Vector &v) const
+{
+   TransformPrimal(v.GetData());
+}
+
+void DofTransformation::TransformPrimalCols(DenseMatrix &V) const
+{
+   for (int c=0; c<V.Width(); c++)
+   {
+      TransformPrimal(V.GetColumn(c));
+   }
+}
+
+void DofTransformation::TransformDual(Vector &v) const
+{
+   TransformDual(v.GetData());
+}
+
+void DofTransformation::TransformDual(DenseMatrix &V) const
+{
+   TransformDualCols(V);
+   TransformDualRows(V);
+}
+
+void DofTransformation::TransformDualRows(DenseMatrix &V) const
+{
+   Vector row;
+   for (int r=0; r<V.Height(); r++)
+   {
+      V.GetRow(r, row);
+      TransformDual(row);
+      V.SetRow(r, row);
+   }
+}
+
+void DofTransformation::TransformDualCols(DenseMatrix &V) const
+{
+   for (int c=0; c<V.Width(); c++)
+   {
+      TransformDual(V.GetColumn(c));
+   }
+}
+
+void DofTransformation::InvTransformPrimal(Vector &v) const
+{
+   InvTransformPrimal(v.GetData());
+}
+
+void TransformPrimal(const DofTransformation *ran_dof_trans,
+                     const DofTransformation *dom_dof_trans,
+                     DenseMatrix &elmat)
+{
+   if (ran_dof_trans && dom_dof_trans)
+   {
+      ran_dof_trans->TransformPrimalCols(elmat);
+      dom_dof_trans->TransformDualRows(elmat);
+   }
+   else if (ran_dof_trans)
+   {
+      ran_dof_trans->TransformPrimalCols(elmat);
+   }
+   else if (dom_dof_trans)
+   {
+      dom_dof_trans->TransformDualRows(elmat);
+   }
+   else
+   {
+      // If both transformations are NULL this function should not be called
+   }
+}
+
+void TransformDual(const DofTransformation *ran_dof_trans,
+                   const DofTransformation *dom_dof_trans,
+                   DenseMatrix &elmat)
+{
+   if (ran_dof_trans && dom_dof_trans)
+   {
+      ran_dof_trans->TransformDualCols(elmat);
+      dom_dof_trans->TransformDualRows(elmat);
+   }
+   else if (ran_dof_trans)
+   {
+      ran_dof_trans->TransformDualCols(elmat);
+   }
+   else if (dom_dof_trans)
+   {
+      dom_dof_trans->TransformDualRows(elmat);
+   }
+   else
+   {
+      // If both transformations are NULL this function should not be called
+   }
+}
+
+void VDofTransformation::TransformPrimal(double *v) const
+{
+   int size = doftrans_->Size();
+
+   if ((Ordering::Type)ordering_ == Ordering::byNODES || vdim_ == 1)
+   {
+      for (int i=0; i<vdim_; i++)
+      {
+         doftrans_->TransformPrimal(&v[i*size]);
+      }
+   }
+   else
+   {
+      Vector vec(size);
+      for (int i=0; i<vdim_; i++)
+      {
+         for (int j=0; j<size; j++)
+         {
+            vec(j) = v[j*vdim_+i];
+         }
+         doftrans_->TransformPrimal(vec);
+         for (int j=0; j<size; j++)
+         {
+            v[j*vdim_+i] = vec(j);
+         }
+      }
+   }
+}
+
+void VDofTransformation::InvTransformPrimal(double *v) const
+{
+   int size = doftrans_->Height();
+
+   if ((Ordering::Type)ordering_ == Ordering::byNODES)
+   {
+      for (int i=0; i<vdim_; i++)
+      {
+         doftrans_->InvTransformPrimal(&v[i*size]);
+      }
+   }
+   else
+   {
+      Vector vec(size);
+      for (int i=0; i<vdim_; i++)
+      {
+         for (int j=0; j<size; j++)
+         {
+            vec(j) = v[j*vdim_+i];
+         }
+         doftrans_->InvTransformPrimal(vec);
+         for (int j=0; j<size; j++)
+         {
+            v[j*vdim_+i] = vec(j);
+         }
+      }
+   }
+}
+
+void VDofTransformation::TransformDual(double *v) const
+{
+   int size = doftrans_->Size();
+
+   if ((Ordering::Type)ordering_ == Ordering::byNODES)
+   {
+      for (int i=0; i<vdim_; i++)
+      {
+         doftrans_->TransformDual(&v[i*size]);
+      }
+   }
+   else
+   {
+      Vector vec(size);
+      for (int i=0; i<vdim_; i++)
+      {
+         for (int j=0; j<size; j++)
+         {
+            vec(j) = v[j*vdim_+i];
+         }
+         doftrans_->TransformDual(vec);
+         for (int j=0; j<size; j++)
+         {
+            v[j*vdim_+i] = vec(j);
+         }
+      }
+   }
+}
+
+const double ND_DofTransformation::T_data[24] =
+{
+   1.0,  0.0,  0.0,  1.0,
+   -1.0, -1.0,  0.0,  1.0,
+   0.0,  1.0, -1.0, -1.0,
+   1.0,  0.0, -1.0, -1.0,
+   -1.0, -1.0,  1.0,  0.0,
+   0.0,  1.0,  1.0,  0.0
+};
+
+const DenseTensor ND_DofTransformation
+::T(const_cast<double*>(ND_DofTransformation::T_data), 2, 2, 6);
+
+const double ND_DofTransformation::TInv_data[24] =
+{
+   1.0,  0.0,  0.0,  1.0,
+   -1.0, -1.0,  0.0,  1.0,
+   -1.0, -1.0,  1.0,  0.0,
+   1.0,  0.0, -1.0, -1.0,
+   0.0,  1.0, -1.0, -1.0,
+   0.0,  1.0,  1.0,  0.0
+};
+
+const DenseTensor ND_DofTransformation
+::TInv(const_cast<double*>(TInv_data), 2, 2, 6);
+
+ND_DofTransformation::ND_DofTransformation(int size, int p)
+   : DofTransformation(size),
+     order(p)
+{
+}
+
+ND_TriDofTransformation::ND_TriDofTransformation(int p)
+   : ND_DofTransformation(p*(p + 2), p)
+{
+}
+
+void ND_TriDofTransformation::TransformPrimal(double *v) const
+{
+   int nedofs = order; // number of DoFs per edge
+   int nfdofs = order*(order-1); // number of DoFs per face
+
+   double data[2];
+   Vector v2(data, 2);
+
+   // Transform face DoFs
+   for (int f=0; f<1; f++)
+   {
+      for (int i=0; i<nfdofs/2; i++)
+      {
+         v2 = &v[3*nedofs + f*nfdofs + 2*i];
+         T(Fo[f]).Mult(v2, &v[3*nedofs + f*nfdofs + 2*i]);
+      }
+   }
+}
+
+void
+ND_TriDofTransformation::InvTransformPrimal(double *v) const
+{
+   int nedofs = order; // number of DoFs per edge
+   int nfdofs = order*(order-1); // number of DoFs per face
+
+   double data[2];
+   Vector v2(data, 2);
+
+   // Transform face DoFs
+   for (int f=0; f<1; f++)
+   {
+      for (int i=0; i<nfdofs/2; i++)
+      {
+         v2 = &v[3*nedofs + f*nfdofs + 2*i];
+         TInv(Fo[f]).Mult(v2, &v[3*nedofs + f*nfdofs + 2*i]);
+      }
+   }
+}
+
+void
+ND_TriDofTransformation::TransformDual(double *v) const
+{
+   int nedofs = order; // number of DoFs per edge
+   int nfdofs = order*(order-1); // number of DoFs per face
+
+   double data[2];
+   Vector v2(data, 2);
+
+   // Transform face DoFs
+   for (int f=0; f<1; f++)
+   {
+      for (int i=0; i<nfdofs/2; i++)
+      {
+         v2 = &v[3*nedofs + f*nfdofs + 2*i];
+         TInv(Fo[f]).MultTranspose(v2, &v[3*nedofs + f*nfdofs + 2*i]);
+      }
+   }
+}
+
+ND_TetDofTransformation::ND_TetDofTransformation(int p)
+   : ND_DofTransformation(p*(p + 2)*(p + 3)/2, p)
+{
+}
+
+void ND_TetDofTransformation::TransformPrimal(double *v) const
+{
+   int nedofs = order; // number of DoFs per edge
+   int nfdofs = order*(order-1); // number of DoFs per face
+
+   double data[2];
+   Vector v2(data, 2);
+
+   // Transform face DoFs
+   for (int f=0; f<4; f++)
+   {
+      for (int i=0; i<nfdofs/2; i++)
+      {
+         v2 = &v[6*nedofs + f*nfdofs + 2*i];
+         T(Fo[f]).Mult(v2, &v[6*nedofs + f*nfdofs + 2*i]);
+      }
+   }
+}
+
+void
+ND_TetDofTransformation::InvTransformPrimal(double *v) const
+{
+   int nedofs = order; // number of DoFs per edge
+   int nfdofs = order*(order-1); // number of DoFs per face
+
+   double data[2];
+   Vector v2(data, 2);
+
+   // Transform face DoFs
+   for (int f=0; f<4; f++)
+   {
+      for (int i=0; i<nfdofs/2; i++)
+      {
+         v2 = &v[6*nedofs + f*nfdofs + 2*i];
+         TInv(Fo[f]).Mult(v2, &v[6*nedofs + f*nfdofs + 2*i]);
+      }
+   }
+}
+
+void
+ND_TetDofTransformation::TransformDual(double *v) const
+{
+   int nedofs = order; // number of DoFs per edge
+   int nfdofs = order*(order-1); // number of DoFs per face
+
+   double data[2];
+   Vector v2(data, 2);
+
+   // Transform face DoFs
+   for (int f=0; f<4; f++)
+   {
+      for (int i=0; i<nfdofs/2; i++)
+      {
+         v2 = &v[6*nedofs + f*nfdofs + 2*i];
+         TInv(Fo[f]).MultTranspose(v2, &v[6*nedofs + f*nfdofs + 2*i]);
+      }
+   }
+}
+
+} // namespace mfem
@@ -0,0 +1,277 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_DOFTRANSFORM
+#define MFEM_DOFTRANSFORM
+
+#include "../config/config.hpp"
+#include "../linalg/linalg.hpp"
+#include "intrules.hpp"
+#include "fe.hpp"
+
+namespace mfem
+{
+
+/** The DofTransformation class is an abstract base class for a family of
+    transformations that map local degrees of freedom (DoFs), contained within
+    individual elements, to global degrees of freedom, stored within
+    GridFunction objects. These transformations are necessary to ensure that
+    basis functions in neighboring elements align correctly. Closely related but
+    complementary transformations are required for the entries stored in
+    LinearForm and BilinearForm objects. The DofTransformation class is designed
+    to apply the action of both of these types of DoF transformations.
+
+    Let the "primal transformation" be given by the operator T. This means that
+    given a local element vector v the data that must be placed into a
+    GridFunction object is v_t = T * v.
+
+    We also need the inverse of the primal transformation T^{-1} so that we can
+    recover the local element vector from data read out of a GridFunction
+    e.g. v = T^{-1} * v_t.
+
+    We need to preserve the action of our linear forms applied to primal
+    vectors. In other words, if f is the local vector computed by a linear
+    form then f * v = f_t * v_t (where "*" represents an inner product of
+    vectors). This requires that f_t = T^{-T} * f i.e. the "dual transform" is
+    given by the transpose of the inverse of the primal transformation.
+
+    For bilinear forms we require that v^T * A * v = v_t^T * A_t * v_t. This
+    implies that A_t = T^{-T} * A * T^{-1}. This can be accomplished by
+    performing dual transformations of the rows and columns of the matrix A.
+
+    For discrete linear operators the range must be modified with the primal
+    transformation rather than the dual transformation because the result is a
+    primal vector rather than a dual vector. This leads to the transformation
+    D_t = T * D * T^{-1}. This can be accomplished by using a primal
+    transformation on the columns of D and a dual transformation on its rows.
+*/
+class DofTransformation
+{
+protected:
+   int size_;
+
+   Array<int> Fo;
+
+   DofTransformation(int size)
+      : size_(size) {}
+
+public:
+
+   inline int Size() const { return size_; }
+   inline int Height() const { return size_; }
+   inline int NumRows() const { return size_; }
+   inline int Width() const { return size_; }
+   inline int NumCols() const { return size_; }
+
+   /** @brief Configure the transformation using face orientations for the
+       current element. */
+   /// The face_orientation array can be obtained from Mesh::GetElementFaces.
+   inline void SetFaceOrientations(const Array<int> & face_orientation)
+   { Fo = face_orientation; }
+
+   inline const Array<int> & GetFaceOrientations() const { return Fo; }
+
+   /** Transform local DoFs to align with the global DoFs. For example, this
+       transformation can be used to map the local vector computed by
+       FiniteElement::Project() to the transformed vector stored within a
+       GridFunction object. */
+   virtual void TransformPrimal(double *v) const = 0;
+   virtual void TransformPrimal(Vector &v) const;
+
+   /// Transform groups of DoFs stored as dense matrices
+   virtual void TransformPrimalCols(DenseMatrix &V) const;
+
+   /** Inverse transform local DoFs. Used to transform DoFs from a global vector
+       back to their element-local form. For example, this must be used to
+       transform the vector obtained using GridFunction::GetSubVector before it
+       can be used to compute a local interpolation.
+   */
+   virtual void InvTransformPrimal(double *v) const = 0;
+   virtual void InvTransformPrimal(Vector &v) const;
+
+   /** Transform dual DoFs as computed by a LinearFormIntegrator before summing
+       into a LinearForm object. */
+   virtual void TransformDual(double *v) const = 0;
+   virtual void TransformDual(Vector &v) const;
+
+   /** Transform a matrix of dual DoFs entries as computed by a
+       BilinearFormIntegrator before summing into a BilinearForm object. */
+   virtual void TransformDual(DenseMatrix &V) const;
+
+   /// Transform groups of dual DoFs stored as dense matrices
+   virtual void TransformDualRows(DenseMatrix &V) const;
+   virtual void TransformDualCols(DenseMatrix &V) const;
+
+   virtual ~DofTransformation() {}
+};
+
+/** Transform a matrix of DoFs entries from different finite element spaces as
+    computed by a DiscreteInterpolator before copying into a
+    DiscreteLinearOperator.
+*/
+void TransformPrimal(const DofTransformation *ran_dof_trans,
+                     const DofTransformation *dom_dof_trans,
+                     DenseMatrix &elmat);
+
+/** Transform a matrix of dual DoFs entries from different finite element spaces
+    as computed by a BilinearFormIntegrator before summing into a
+    MixedBilinearForm object.
+*/
+void TransformDual(const DofTransformation *ran_dof_trans,
+                   const DofTransformation *dom_dof_trans,
+                   DenseMatrix &elmat);
+
+/** The VDofTransformation class implements a nested transformation where an
+    arbitrary DofTransformation is replicated with a vdim >= 1.
+*/
+class VDofTransformation : public DofTransformation
+{
+private:
+   int vdim_;
+   int ordering_;
+   DofTransformation * doftrans_;
+
+public:
+   /** @brief Default constructor which requires that SetDofTransformation be
+       called before use. */
+   VDofTransformation(int vdim = 1, int ordering = 0)
+      : DofTransformation(0),
+        vdim_(vdim), ordering_(ordering),
+        doftrans_(NULL) {}
+
+   /// Constructor with a known DofTransformation
+   VDofTransformation(DofTransformation & doftrans, int vdim = 1,
+                      int ordering = 0)
+      : DofTransformation(vdim * doftrans.Size()),
+        vdim_(vdim), ordering_(ordering),
+        doftrans_(&doftrans) {}
+
+   /// Set or change the vdim parameter
+   inline void SetVDim(int vdim)
+   {
+      vdim_ = vdim;
+      if (doftrans_)
+      {
+         size_ = vdim_ * doftrans_->Size();
+      }
+   }
+
+   /// Return the current vdim value
+   inline int GetVDim() const { return vdim_; }
+
+   /// Set or change the nested DofTransformation object
+   inline void SetDofTransformation(DofTransformation & doftrans)
+   {
+      size_ = vdim_ * doftrans.Size();
+      doftrans_ = &doftrans;
+   }
+
+   /// Return the nested DofTransformation object
+   inline DofTransformation * GetDofTransformation() const { return doftrans_; }
+
+   inline void SetFaceOrientation(const Array<int> & face_orientation)
+   { Fo = face_orientation; doftrans_->SetFaceOrientations(face_orientation); }
+
+   using DofTransformation::TransformPrimal;
+   using DofTransformation::InvTransformPrimal;
+   using DofTransformation::TransformDual;
+
+   void TransformPrimal(double *v) const;
+   void InvTransformPrimal(double *v) const;
+   void TransformDual(double *v) const;
+};
+
+/** Abstract base class for high-order Nedelec spaces on elements with
+    triangular faces.
+
+    The Nedelec DoFs on the interior of triangular faces come in pairs which
+    share an interpolation point but have different vector directions. These
+    directions depend on the orientation of the face and can therefore differ in
+    neighboring elements. The mapping required to transform these DoFs can be
+    implemented as series of 2x2 linear transformations. The raw data for these
+    linear transformations is stored in the T_data and TInv_data arrays and can
+    be accessed as DenseMatrices using the GetFaceTransform() and
+    GetFaceInverseTransform() methods.
+*/
+class ND_DofTransformation : public DofTransformation
+{
+protected:
+   static const double T_data[24];
+   static const double TInv_data[24];
+   static const DenseTensor T, TInv;
+   int order;
+
+   ND_DofTransformation(int size, int order);
+
+public:
+   // Return the 2x2 transformation operator for the given face orientation
+   static const DenseMatrix & GetFaceTransform(int ori) { return T(ori); }
+
+   // Return the 2x2 inverse transformation operator
+   static const DenseMatrix & GetFaceInverseTransform(int ori)
+   { return TInv(ori); }
+};
+
+/// DoF transformation implementation for the Nedelec basis on triangles
+class ND_TriDofTransformation : public ND_DofTransformation
+{
+public:
+   ND_TriDofTransformation(int order);
+
+   using DofTransformation::TransformPrimal;
+   using DofTransformation::InvTransformPrimal;
+   using DofTransformation::TransformDual;
+
+   void TransformPrimal(double *v) const;
+
+   void InvTransformPrimal(double *v) const;
+
+   void TransformDual(double *v) const;
+};
+
+/// DoF transformation implementation for the Nedelec basis on tetrahedra
+class ND_TetDofTransformation : public ND_DofTransformation
+{
+public:
+   ND_TetDofTransformation(int order);
+
+   using DofTransformation::TransformPrimal;
+   using DofTransformation::InvTransformPrimal;
+   using DofTransformation::TransformDual;
+
+   void TransformPrimal(double *v) const;
+
+   void InvTransformPrimal(double *v) const;
+
+   void TransformDual(double *v) const;
+};
+
+/// DoF transformation implementation for the Nedelec basis on wedge elements
+/** TODO: (Under development) */
+class ND_WedgeDofTransformation : public ND_DofTransformation
+{
+public:
+   ND_WedgeDofTransformation(int order);
+
+   using DofTransformation::TransformPrimal;
+   using DofTransformation::InvTransformPrimal;
+   using DofTransformation::TransformDual;
+
+   void TransformPrimal(double *v) const;
+
+   void InvTransformPrimal(double *v) const;
+
+   void TransformDual(double *v) const;
+};
+
+} // namespace mfem
+
+#endif // MFEM_DOFTRANSFORM
@@ -380,6 +380,7 @@ void IsoparametricTransformation::SetIdentityTransformation(
      case Geometry::TETRAHEDRON : FElem = &TetrahedronFE; break;
      case Geometry::CUBE :        FElem = &HexahedronFE; break;
      case Geometry::PRISM :       FElem = &WedgeFE; break;
+      case Geometry::PYRAMID :     FElem = &PyramidFE; break;
      default:
         MFEM_ABORT("unknown Geometry::Type!");
   }
@@ -316,7 +316,7 @@ public:

   /// Set the desired print level, useful for debugging.
   /** The valid options are: -1 - never print (default); 0 - print only errors;
-       1 - print the first and last last iterations; 2 - print every iteration;
+       1 - print the first and last iterations; 2 - print every iteration;
       and 3 - print every iteration including point coordinates. */
   void SetPrintLevel(int pr_level) { print_level = pr_level; }

@@ -329,7 +329,7 @@ void KellyErrorEstimator::ComputeEstimates()
         error_estimates(e) = sqrt(factor * error_estimates(e));
      }

-      total_error = error_estimates.Sum();
+      total_error = error_estimates.Norml2();
      delete flux;
      return;
   }
@@ -452,9 +452,10 @@ void KellyErrorEstimator::ComputeEstimates()
   auto pfes = dynamic_cast<ParFiniteElementSpace*>(xfes);
   MFEM_VERIFY(pfes, "xfes is not a ParFiniteElementSpace pointer");

-   double process_local_error = error_estimates.Sum();
+   double process_local_error = pow(error_estimates.Norml2(),2.0);
   MPI_Allreduce(&process_local_error, &total_error, 1, MPI_DOUBLE,
                 MPI_SUM, pfes->GetComm());
+   total_error = sqrt(total_error);
 #endif // MFEM_USE_MPI
 }

@@ -97,7 +97,7 @@ public:
      {
         "Gauss-Legendre", "Gauss-Lobatto", "Positive (Bernstein)",
         "Open uniform", "Closed uniform", "Open half uniform",
-         "Seredipity", "Closed Gauss-Legendre",
+         "Serendipity", "Closed Gauss-Legendre",
         "Integrated Gauss-Lobatto indicator"
      };
      return name[Check(b_type)];
@@ -1126,7 +1126,7 @@ public:
   { dofs = 1.0; }
 };

-/// A 1D quadractic finite element with uniformly spaced nodes
+/// A 1D quadratic finite element with uniformly spaced nodes
 class Quad1DFiniteElement : public NodalFiniteElement
 {
 public:
@@ -1313,6 +1313,64 @@ public:
                           DenseMatrix &dshape) const;
 };

+/// A linear element defined on a triangular prism
+class LinearWedgeFiniteElement : public NodalFiniteElement
+{
+public:
+   /// Construct the LinearWedgeFiniteElement
+   LinearWedgeFiniteElement();
+
+   /** @brief virtual function which evaluates the values of all
+       shape functions at a given point ip and stores
+       them in the vector shape of dimension Dof (4) */
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+
+   /** @brief virtual function which evaluates the values of all
+       partial derivatives of all shape functions at a given
+       point ip and stores them in the matrix dshape (Dof x Dim) (4 x 3)
+       so that each row contains the derivatives of one shape function */
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+
+   virtual void ProjectDelta(int vertex, Vector &dofs) const
+   { dofs = 0.0; dofs(vertex) = 1.0; }
+
+   /** @brief Get the dofs associated with the given @a face.
+       @a *dofs is set to an internal array of the local dofc on the
+       face, while *ndofs is set to the number of dofs on that face.
+   */
+   virtual void GetFaceDofs(int face, int **dofs, int *ndofs) const;
+};
+
+/// A linear element defined on a square pyramid
+class LinearPyramidFiniteElement : public NodalFiniteElement
+{
+public:
+   /// Construct the LinearPyramidFiniteElement
+   LinearPyramidFiniteElement();
+
+   /** @brief virtual function which evaluates the values of all
+       shape functions at a given point ip and stores
+       them in the vector shape of dimension Dof (4) */
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+
+   /** @brief virtual function which evaluates the values of all
+       partial derivatives of all shape functions at a given
+       point ip and stores them in the matrix dshape (Dof x Dim) (4 x 3)
+       so that each row contains the derivatives of one shape function */
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+
+   virtual void ProjectDelta(int vertex, Vector &dofs) const
+   { dofs = 0.0; dofs(vertex) = 1.0; }
+
+   /** @brief Get the dofs associated with the given @a face.
+       @a *dofs is set to an internal array of the local dofc on the
+       face, while *ndofs is set to the number of dofs on that face.
+   */
+   virtual void GetFaceDofs(int face, int **dofs, int *ndofs) const;
+};
+
 /// A 2D constant element on a triangle
 class P0TriangleFiniteElement : public NodalFiniteElement
 {
@@ -1690,6 +1748,32 @@ public:
   { dofs(0) = 1.0; }
 };

+/// A 3D constant element on a wedge
+class P0WdgFiniteElement : public NodalFiniteElement
+{
+public:
+   /// Construct the P0WdgFiniteElement
+   P0WdgFiniteElement ();
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void ProjectDelta(int vertex, Vector &dofs) const
+   { dofs(0) = 1.0; }
+};
+
+/// A 3D constant element on a pyramid
+class P0PyrFiniteElement : public NodalFiniteElement
+{
+public:
+   /// Construct the P0PyrFiniteElement
+   P0PyrFiniteElement ();
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void ProjectDelta(int vertex, Vector &dofs) const
+   { dofs(0) = 1.0; }
+};
+
 /** @brief Tensor products of 1D Lagrange1DFiniteElement
    (only degree 2 is functional) */
 class LagrangeHexFiniteElement : public NodalFiniteElement
@@ -1828,6 +1912,10 @@ public:
   using FiniteElement::Project;
   virtual void Project (VectorCoefficient &vc,
                         ElementTransformation &Trans, Vector &dofs) const;
+
+   virtual void ProjectGrad(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &grad) const;
 };


@@ -1852,6 +1940,66 @@ public:
   using FiniteElement::Project;
   virtual void Project (VectorCoefficient &vc,
                         ElementTransformation &Trans, Vector &dofs) const;
+
+   virtual void ProjectGrad(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &grad) const;
+};
+
+
+/// A 3D 1st order Nedelec element on a wedge
+class Nedelec1WdgFiniteElement : public VectorFiniteElement
+{
+private:
+   static const double tk[9][3];
+
+public:
+   /// Construct the Nedelec1WdgFiniteElement
+   Nedelec1WdgFiniteElement();
+   virtual void CalcVShape(const IntegrationPoint &ip,
+                           DenseMatrix &shape) const;
+   virtual void CalcVShape(ElementTransformation &Trans,
+                           DenseMatrix &shape) const
+   { CalcVShape_ND(Trans, shape); }
+   virtual void CalcCurlShape(const IntegrationPoint &ip,
+                              DenseMatrix &curl_shape) const;
+   virtual void GetLocalInterpolation (ElementTransformation &Trans,
+                                       DenseMatrix &I) const;
+   using FiniteElement::Project;
+   virtual void Project (VectorCoefficient &vc,
+                         ElementTransformation &Trans, Vector &dofs) const;
+
+   virtual void ProjectGrad(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &grad) const;
+};
+
+
+/// A 3D 1st order Nedelec element on a pyramid
+class Nedelec1PyrFiniteElement : public VectorFiniteElement
+{
+private:
+   static const double tk[8][3];
+
+public:
+   /// Construct the Nedelec1PyrFiniteElement
+   Nedelec1PyrFiniteElement();
+   virtual void CalcVShape(const IntegrationPoint &ip,
+                           DenseMatrix &shape) const;
+   virtual void CalcVShape(ElementTransformation &Trans,
+                           DenseMatrix &shape) const
+   { CalcVShape_ND(Trans, shape); }
+   virtual void CalcCurlShape(const IntegrationPoint &ip,
+                              DenseMatrix &curl_shape) const;
+   virtual void GetLocalInterpolation (ElementTransformation &Trans,
+                                       DenseMatrix &I) const;
+   using FiniteElement::Project;
+   virtual void Project (VectorCoefficient &vc,
+                         ElementTransformation &Trans, Vector &dofs) const;
+
+   virtual void ProjectGrad(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &grad) const;
 };


@@ -1945,6 +2093,77 @@ public:
 };


+/// A 3D 0th order Raviert-Thomas element on a wedge
+class RT0WdgFiniteElement : public VectorFiniteElement
+{
+private:
+   static const double nk[5][3];
+
+public:
+   /// Construct the RT0WdgFiniteElement
+   RT0WdgFiniteElement();
+
+   virtual void CalcVShape(const IntegrationPoint &ip,
+                           DenseMatrix &shape) const;
+
+   virtual void CalcVShape(ElementTransformation &Trans,
+                           DenseMatrix &shape) const
+   { CalcVShape_RT(Trans, shape); }
+
+   virtual void CalcDivShape(const IntegrationPoint &ip,
+                             Vector &divshape) const;
+
+   virtual void GetLocalInterpolation (ElementTransformation &Trans,
+                                       DenseMatrix &I) const;
+
+   using FiniteElement::Project;
+
+   virtual void Project (VectorCoefficient &vc,
+                         ElementTransformation &Trans, Vector &dofs) const;
+
+   virtual void ProjectCurl(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &curl) const;
+};
+
+
+/// A 3D 0th order Raviert-Thomas element on a pyramid
+class RT0PyrFiniteElement : public VectorFiniteElement
+{
+private:
+   static const double nk[5][3];
+
+   // If true match RT0TetFiniteElement rather than RT_TetrahedronElement(0)
+   bool rt0;
+
+public:
+   /// Construct the RT0PyrFiniteElement
+   RT0PyrFiniteElement(bool rt0tets = true);
+
+   virtual void CalcVShape(const IntegrationPoint &ip,
+                           DenseMatrix &shape) const;
+
+   virtual void CalcVShape(ElementTransformation &Trans,
+                           DenseMatrix &shape) const
+   { CalcVShape_RT(Trans, shape); }
+
+   virtual void CalcDivShape(const IntegrationPoint &ip,
+                             Vector &divshape) const;
+
+   virtual void GetLocalInterpolation (ElementTransformation &Trans,
+                                       DenseMatrix &I) const;
+
+   using FiniteElement::Project;
+
+   virtual void Project (VectorCoefficient &vc,
+                         ElementTransformation &Trans, Vector &dofs) const;
+
+   virtual void ProjectCurl(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &curl) const;
+};
+
+
 class RotTriLinearHexFiniteElement : public NodalFiniteElement
 {
 public:
@@ -2239,6 +2458,11 @@ public:
                             const int cbtype, const int obtype,
                             const int M, const DofMapType dmtype);

+   // For 1D elements: there is only an "open basis", no "closed basis"
+   VectorTensorFiniteElement(const int dims, const int d, const int p,
+                             const int obtype, const int M,
+                             const DofMapType dmtype);
+
   const DofToQuad &GetDofToQuad(const IntegrationRule &ir,
                                 DofToQuad::Mode mode) const;

@@ -3311,11 +3535,9 @@ public:


 /// Arbitrary order Nedelec elements in 1D on a segment
-class ND_SegmentElement : public VectorFiniteElement
+class ND_SegmentElement : public VectorTensorFiniteElement
 {
   static const double tk[1];
-
-   Poly_1D::Basis &obasis1d;
   Array<int> dof2tk;

 public:
@@ -33,6 +33,9 @@ int FiniteElementCollection::HasFaceDofs(Geometry::Type geom, int p) const
      case Geometry::PRISM:
         return max(GetNumDof(Geometry::TRIANGLE, p),
                    GetNumDof(Geometry::SQUARE, p));
+      case Geometry::PYRAMID:
+         return max(GetNumDof(Geometry::TRIANGLE, p),
+                    GetNumDof(Geometry::SQUARE, p));
      default:
         MFEM_ABORT("unknown geometry type");
   }
@@ -574,6 +577,7 @@ LinearFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::TETRAHEDRON: return &TetrahedronFE;
      case Geometry::CUBE:        return &ParallelepipedFE;
      case Geometry::PRISM:       return &WedgeFE;
+      case Geometry::PYRAMID:     return &PyramidFE;
      default:
         mfem_error ("LinearFECollection: unknown geometry type.");
   }
@@ -591,6 +595,7 @@ int LinearFECollection::DofForGeometry(Geometry::Type GeomType) const
      case Geometry::TETRAHEDRON: return 0;
      case Geometry::CUBE:        return 0;
      case Geometry::PRISM:       return 0;
+      case Geometry::PYRAMID:     return 0;
      default:
         mfem_error ("LinearFECollection: unknown geometry type.");
   }
@@ -1240,6 +1245,7 @@ Const3DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::TETRAHEDRON: return &TetrahedronFE;
      case Geometry::CUBE:        return &ParallelepipedFE;
      case Geometry::PRISM:       return &WedgeFE;
+      case Geometry::PYRAMID:     return &PyramidFE;
      default:
         mfem_error ("Const3DFECollection: unknown geometry type.");
   }
@@ -1257,6 +1263,7 @@ int Const3DFECollection::DofForGeometry(Geometry::Type GeomType) const
      case Geometry::TETRAHEDRON: return 1;
      case Geometry::CUBE:        return 1;
      case Geometry::PRISM:       return 1;
+      case Geometry::PYRAMID:     return 1;
      default:
         mfem_error ("Const3DFECollection: unknown geometry type.");
   }
@@ -1277,6 +1284,8 @@ LinearDiscont3DFECollection::FiniteElementForGeometry(
   switch (GeomType)
   {
      case Geometry::TETRAHEDRON: return &TetrahedronFE;
+      case Geometry::PYRAMID:     return &PyramidFE;
+      case Geometry::PRISM:       return &WedgeFE;
      case Geometry::CUBE:        return &ParallelepipedFE;
      default:
         mfem_error ("LinearDiscont3DFECollection: unknown geometry type.");
@@ -1293,6 +1302,8 @@ int LinearDiscont3DFECollection::DofForGeometry(Geometry::Type GeomType) const
      case Geometry::TRIANGLE:    return 0;
      case Geometry::SQUARE:      return 0;
      case Geometry::TETRAHEDRON: return 4;
+      case Geometry::PYRAMID:     return 5;
+      case Geometry::PRISM:       return 6;
      case Geometry::CUBE:        return 8;
      default:
         mfem_error ("LinearDiscont3DFECollection: unknown geometry type.");
@@ -1394,6 +1405,8 @@ ND1_3DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
   {
      case Geometry::CUBE:        return &HexahedronFE;
      case Geometry::TETRAHEDRON: return &TetrahedronFE;
+      case Geometry::PRISM:       return &WedgeFE;
+      case Geometry::PYRAMID:     return &PyramidFE;
      default:
         mfem_error ("ND1_3DFECollection: unknown geometry type.");
   }
@@ -1410,6 +1423,8 @@ int ND1_3DFECollection::DofForGeometry(Geometry::Type GeomType) const
      case Geometry::SQUARE:      return 0;
      case Geometry::TETRAHEDRON: return 0;
      case Geometry::CUBE:        return 0;
+      case Geometry::PRISM:       return 0;
+      case Geometry::PYRAMID:     return 0;
      default:
         mfem_error ("ND1_3DFECollection: unknown geometry type.");
   }
@@ -1439,6 +1454,8 @@ RT0_3DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::SQUARE:      return &QuadrilateralFE;
      case Geometry::CUBE:        return &HexahedronFE;
      case Geometry::TETRAHEDRON: return &TetrahedronFE;
+      case Geometry::PRISM:       return &WedgeFE;
+      case Geometry::PYRAMID:     return &PyramidFE;
      default:
         mfem_error ("RT0_3DFECollection: unknown geometry type.");
   }
@@ -1455,6 +1472,8 @@ int RT0_3DFECollection::DofForGeometry(Geometry::Type GeomType) const
      case Geometry::SQUARE:      return 1;
      case Geometry::TETRAHEDRON: return 0;
      case Geometry::CUBE:        return 0;
+      case Geometry::PRISM:       return 0;
+      case Geometry::PYRAMID:     return 0;
      default:
         mfem_error ("RT0_3DFECollection: unknown geometry type.");
   }
@@ -1730,6 +1749,7 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype)
         H1_dof[Geometry::TETRAHEDRON] = (TriDof*pm3)/3;
         H1_dof[Geometry::CUBE] = QuadDof*pm1;
         H1_dof[Geometry::PRISM] = TriDof*pm1;
+         H1_dof[Geometry::PYRAMID] = 0;
         if (b_type == BasisType::Positive)
         {
            H1_Elements[Geometry::TETRAHEDRON] = new H1Pos_TetrahedronElement(p);
@@ -1743,6 +1763,7 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype)
            H1_Elements[Geometry::CUBE] = new H1_HexahedronElement(p, btype);
            H1_Elements[Geometry::PRISM] = new H1_WedgeElement(p, btype);
         }
+         H1_Elements[Geometry::PYRAMID] = new LinearPyramidFiniteElement;

         const int &TetDof = H1_dof[Geometry::TETRAHEDRON];
         TetDofOrd[0] = new int[24*TetDof];
@@ -1837,6 +1858,21 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype)
   }
 }

+const FiniteElement *
+H1_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
+{
+   if (GeomType != Geometry::PYRAMID || this->GetOrder() == 1)
+   {
+      return H1_Elements[GeomType];
+   }
+   else
+   {
+      MFEM_ABORT("H1 Pyramid basis functions are not yet supported "
+                 "for order > 1.");
+      return NULL;
+   }
+}
+
 const int *H1_FECollection::DofOrderForOrientation(Geometry::Type GeomType,
                                                   int Or) const
 {
@@ -2076,9 +2112,12 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
         L2_Elements[Geometry::CUBE] = new L2_HexahedronElement(p, btype);
         L2_Elements[Geometry::PRISM] = new L2_WedgeElement(p, btype);
      }
+      L2_Elements[Geometry::PYRAMID] = new P0PyrFiniteElement;
+
      L2_Elements[Geometry::TETRAHEDRON]->SetMapType(map_type);
      L2_Elements[Geometry::CUBE]->SetMapType(map_type);
      L2_Elements[Geometry::PRISM]->SetMapType(map_type);
+      L2_Elements[Geometry::PYRAMID]->SetMapType(map_type);
      // Trace element use the default Gauss-Legendre nodal points for positive basis
      if (b_type == BasisType::Positive)
      {
@@ -2199,6 +2238,21 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
   }
 }

+const FiniteElement *
+L2_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
+{
+   if (GeomType != Geometry::PYRAMID || this->GetOrder() == 0)
+   {
+      return L2_Elements[GeomType];
+   }
+   else
+   {
+      MFEM_ABORT("L2 Pyramid basis functions are not yet supported "
+                 "for order > 0.");
+      return NULL;
+   }
+}
+
 const int *L2_FECollection::DofOrderForOrientation(Geometry::Type GeomType,
                                                   int Or) const
 {
@@ -2290,6 +2344,12 @@ RT_FECollection::RT_FECollection(const int order, const int dim,

      RT_Elements[Geometry::CUBE] = new RT_HexahedronElement(p, cb_type, ob_type);
      RT_dof[Geometry::CUBE] = 3*p*pp1*pp1;
+
+      RT_Elements[Geometry::PRISM] = new RT0WdgFiniteElement;
+      RT_dof[Geometry::PRISM] = 0;
+
+      RT_Elements[Geometry::PYRAMID] = new RT0PyrFiniteElement(false);
+      RT_dof[Geometry::PYRAMID] = 0;
   }
   else
   {
@@ -2433,6 +2493,22 @@ void RT_FECollection::InitFaces(const int p, const int dim,
   }
 }

+const FiniteElement *
+RT_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
+{
+   if ((GeomType != Geometry::PRISM && GeomType != Geometry::PYRAMID) ||
+       this->GetOrder() == 1)
+   {
+      return RT_Elements[GeomType];
+   }
+   else
+   {
+      MFEM_ABORT("RT Wedge and Pyramid basis functions are not yet supported "
+                 "for order > 0.");
+      return NULL;
+   }
+}
+
 const int *RT_FECollection::DofOrderForOrientation(Geometry::Type GeomType,
                                                   int Or) const
 {
@@ -2656,18 +2732,31 @@ ND_FECollection::ND_FECollection(const int p, const int dim,
      {
         for (int i = 0; i + j <= pm2; i++)
         {
-            int k1 = p*pm1 - (p - j)*(pm1 - j) + 2*i;
-            int k2 = p*pm1 - (p - i)*(pm1 - i) + 2*j;
-            // (0,1,2)
-            TriDofOrd[0][k1  ] = k1;
-            TriDofOrd[0][k1+1] = k1 + 1;
-            // (0,2,1)
-            TriDofOrd[5][k1  ] = k2 + 1;
-            TriDofOrd[5][k1+1] = k2;
+            int k0 = p*pm1 - (p - j)*(pm1 - j) + 2*i;
+            int k1 = 2*pm2 - 2*i + ((2*p-3)-j)*j;
+            int k2 = 2*pm2 - 2*j + ((2*p-3)-i)*i;
+            int k3 = p*pm1 - 2 - 3*j - i - (i+j)*(i+j);
+            int k4 = p*pm1 - 2 - 3*i - j - (i+j)*(i+j);
+            int k5 = p*pm1 - (p - i)*(pm1 - i) + 2*j;

-            // The other orientations can not be supported with the current
-            // interface. The method Mesh::ReorientTetMesh will ensure that
-            // only orientations 0 and 5 are generated.
+            // (0,1,2)
+            TriDofOrd[0][k0  ] = k0;
+            TriDofOrd[0][k0+1] = k0 + 1;
+            // (1,0,2)
+            TriDofOrd[1][k0  ] = k1;
+            TriDofOrd[1][k0+1] = k1 + 1;
+            // (2,0,1)
+            TriDofOrd[2][k0  ] = k2;
+            TriDofOrd[2][k0+1] = k2 + 1;
+            // (2,1,0)
+            TriDofOrd[3][k0  ] = k3;
+            TriDofOrd[3][k0+1] = k3 + 1;
+            // (1,2,0)
+            TriDofOrd[4][k0  ] = k4;
+            TriDofOrd[4][k0+1] = k4 + 1;
+            // (0,2,1)
+            TriDofOrd[5][k0  ] = k5;
+            TriDofOrd[5][k0+1] = k5 + 1;
         }
      }
   }
@@ -2680,6 +2769,28 @@ ND_FECollection::ND_FECollection(const int p, const int dim,
      // TODO: cb_type and ob_type for tets
      ND_Elements[Geometry::TETRAHEDRON] = new ND_TetrahedronElement(p);
      ND_dof[Geometry::TETRAHEDRON] = p*pm1*pm2/2;
+
+      ND_Elements[Geometry::PRISM] = new Nedelec1WdgFiniteElement;
+      ND_dof[Geometry::PRISM] = 0;
+
+      ND_Elements[Geometry::PYRAMID] = new Nedelec1PyrFiniteElement;
+      ND_dof[Geometry::PYRAMID] = 0;
+   }
+}
+
+const FiniteElement *
+ND_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
+{
+   if ((GeomType != Geometry::PRISM && GeomType != Geometry::PYRAMID) ||
+       this->GetOrder() == 1)
+   {
+      return ND_Elements[GeomType];
+   }
+   else
+   {
+      MFEM_ABORT("ND Wedge and Pyramid basis functions are not yet supported "
+                 "for order > 1.");
+      return NULL;
   }
 }

@@ -2692,11 +2803,6 @@ const int *ND_FECollection::DofOrderForOrientation(Geometry::Type GeomType,
   }
   else if (GeomType == Geometry::TRIANGLE)
   {
-      if (Or != 0 && Or != 5)
-      {
-         MFEM_ABORT("triangle face orientation " << Or << " is not supported! "
-                    "Use Mesh::ReorientTetMesh to fix it.");
-      }
      return TriDofOrd[Or%6];
   }
   else if (GeomType == Geometry::SQUARE)
@@ -228,8 +228,7 @@ public:
                            const int btype = BasisType::GaussLobatto);

   virtual const FiniteElement *FiniteElementForGeometry(
-      Geometry::Type GeomType) const
-   { return H1_Elements[GeomType]; }
+      Geometry::Type GeomType) const;
   virtual int DofForGeometry(Geometry::Type GeomType) const
   { return H1_dof[GeomType]; }
   virtual const int *DofOrderForOrientation(Geometry::Type GeomType,
@@ -302,10 +301,7 @@ public:
                   const int map_type = FiniteElement::VALUE);

   virtual const FiniteElement *FiniteElementForGeometry(
-      Geometry::Type GeomType) const
-   {
-      return L2_Elements[GeomType];
-   }
+      Geometry::Type GeomType) const;
   virtual int DofForGeometry(Geometry::Type GeomType) const
   {
      if (L2_Elements[GeomType])
@@ -371,8 +367,7 @@ public:
                   const int ob_type = BasisType::GaussLegendre);

   virtual const FiniteElement *FiniteElementForGeometry(
-      Geometry::Type GeomType) const
-   { return RT_Elements[GeomType]; }
+      Geometry::Type GeomType) const;
   virtual int DofForGeometry(Geometry::Type GeomType) const
   { return RT_dof[GeomType]; }
   virtual const int *DofOrderForOrientation(Geometry::Type GeomType,
@@ -430,8 +425,7 @@ public:
                   const int ob_type = BasisType::GaussLegendre);

   virtual const FiniteElement *
-   FiniteElementForGeometry(Geometry::Type GeomType) const
-   { return ND_Elements[GeomType]; }
+   FiniteElementForGeometry(Geometry::Type GeomType) const;

   virtual int DofForGeometry(Geometry::Type GeomType) const
   { return ND_dof[GeomType]; }
@@ -529,9 +523,10 @@ private:
   const BiLinear2DFiniteElement QuadrilateralFE;
   const Linear3DFiniteElement TetrahedronFE;
   const TriLinear3DFiniteElement ParallelepipedFE;
-   const H1_WedgeElement WedgeFE;
+   const LinearWedgeFiniteElement WedgeFE;
+   const LinearPyramidFiniteElement PyramidFE;
 public:
-   LinearFECollection() : FiniteElementCollection(1), WedgeFE(1) { }
+   LinearFECollection() : FiniteElementCollection(1) { }

   virtual const FiniteElement *
   FiniteElementForGeometry(Geometry::Type GeomType) const;
@@ -936,10 +931,11 @@ class Const3DFECollection : public FiniteElementCollection
 private:
   const P0TetFiniteElement TetrahedronFE;
   const P0HexFiniteElement ParallelepipedFE;
-   const L2_WedgeElement WedgeFE;
+   const P0WdgFiniteElement WedgeFE;
+   const P0PyrFiniteElement PyramidFE;

 public:
-   Const3DFECollection() : FiniteElementCollection(0), WedgeFE(0) { }
+   Const3DFECollection() : FiniteElementCollection(0) { }

   virtual const FiniteElement *
   FiniteElementForGeometry(Geometry::Type GeomType) const;
@@ -960,6 +956,8 @@ class LinearDiscont3DFECollection : public FiniteElementCollection
 {
 private:
   const Linear3DFiniteElement TetrahedronFE;
+   const LinearPyramidFiniteElement PyramidFE;
+   const LinearWedgeFiniteElement WedgeFE;
   const TriLinear3DFiniteElement ParallelepipedFE;

 public:
@@ -1036,6 +1034,8 @@ class ND1_3DFECollection : public FiniteElementCollection
 private:
   const Nedelec1HexFiniteElement HexahedronFE;
   const Nedelec1TetFiniteElement TetrahedronFE;
+   const Nedelec1WdgFiniteElement WedgeFE;
+   const Nedelec1PyrFiniteElement PyramidFE;

 public:
   ND1_3DFECollection() : FiniteElementCollection(1) { }
@@ -1061,6 +1061,8 @@ private:
   const P0QuadFiniteElement QuadrilateralFE;
   const RT0HexFiniteElement HexahedronFE;
   const RT0TetFiniteElement TetrahedronFE;
+   const RT0WdgFiniteElement WedgeFE;
+   const RT0PyrFiniteElement PyramidFE;
 public:
   RT0_3DFECollection() : FiniteElementCollection(1) { }

@@ -16,6 +16,7 @@
 #include "geom.hpp"
 #include "fe.hpp"
 #include "fe_coll.hpp"
+#include "doftrans.hpp"
 #include "eltrans.hpp"
 #include "coefficient.hpp"
 #include "complex_fem.hpp"
@@ -34,6 +35,7 @@
 #include "staticcond.hpp"
 #include "tmop.hpp"
 #include "tmop_tools.hpp"
+#include "tmop_amr.hpp"
 #include "gslib.hpp"
 #include "restriction.hpp"
 #include "quadinterpolator.hpp"
@@ -64,4 +66,9 @@
 #include "adios2datacollection.hpp"
 #endif

+#ifdef MFEM_USE_FMS
+#include "fmsconvert.hpp"
+#include "fmsdatacollection.hpp"
+#endif
+
 #endif
@@ -58,9 +58,12 @@ DofsToVDofs<Ordering::byVDIM>(int ndofs, int vdim, Array<int> &dofs)

 FiniteElementSpace::FiniteElementSpace()
   : mesh(NULL), fec(NULL), vdim(0), ordering(Ordering::byNODES),
-     ndofs(0), nvdofs(0), nedofs(0), nfdofs(0), nbdofs(0), bdofs(NULL),
-     elem_dof(NULL), bdr_elem_dof(NULL), face_dof(NULL),
+     ndofs(0), nvdofs(0), nedofs(0), nfdofs(0), nbdofs(0),
+     bdofs(NULL),
+     elem_dof(NULL), elem_fos(NULL), bdr_elem_dof(NULL), bdr_elem_fos(NULL),
+     face_dof(NULL),
     NURBSext(NULL), own_ext(false),
+     DoFTrans(0), VDoFTrans(vdim, ordering),
     cP(NULL), cR(NULL), cR_hp(NULL), cP_is_set(false),
     Th(Operator::ANY_TYPE),
     sequence(0), mesh_sequence(0), orders_changed(false), relaxed_hp(false)
@@ -69,6 +72,7 @@ FiniteElementSpace::FiniteElementSpace()
 FiniteElementSpace::FiniteElementSpace(const FiniteElementSpace &orig,
                                       Mesh *mesh,
                                       const FiniteElementCollection *fec)
+   : VDoFTrans(orig.vdim, orig.ordering)
 {
   mesh = mesh ? mesh : orig.mesh;
   fec = fec ? fec : orig.fec;
@@ -259,16 +263,36 @@ void FiniteElementSpace::AdjustVDofs (Array<int> &vdofs)
   }
 }

-void FiniteElementSpace::GetElementVDofs(int i, Array<int> &vdofs) const
+DofTransformation *
+FiniteElementSpace::GetElementVDofs(int i, Array<int> &vdofs) const
 {
-   GetElementDofs(i, vdofs);
+   DofTransformation * doftrans = GetElementDofs(i, vdofs);
   DofsToVDofs(vdofs);
+   if (vdim == 1 || doftrans == NULL)
+   {
+      return doftrans;
+   }
+   else
+   {
+      VDoFTrans.SetDofTransformation(*doftrans);
+      return &VDoFTrans;
+   }
 }

-void FiniteElementSpace::GetBdrElementVDofs(int i, Array<int> &vdofs) const
+DofTransformation *
+FiniteElementSpace::GetBdrElementVDofs(int i, Array<int> &vdofs) const
 {
-   GetBdrElementDofs(i, vdofs);
+   DofTransformation * doftrans = GetBdrElementDofs(i, vdofs);
   DofsToVDofs(vdofs);
+   if (vdim == 1 || doftrans == NULL)
+   {
+      return doftrans;
+   }
+   else
+   {
+      VDoFTrans.SetDofTransformation(*doftrans);
+      return &VDoFTrans;
+   }
 }

 void FiniteElementSpace::GetFaceVDofs(int i, Array<int> &vdofs) const
@@ -307,21 +331,39 @@ void FiniteElementSpace::BuildElementToDofTable() const

   // TODO: can we call GetElementDofs only once per element?
   Table *el_dof = new Table;
+   Table *el_fos = (mesh->Dimension() > 2) ? (new Table) : NULL;
   Array<int> dofs;
+   Array<int> F, Fo;
   el_dof -> MakeI (mesh -> GetNE());
+   if (el_fos) { el_fos -> MakeI (mesh -> GetNE()); }
   for (int i = 0; i < mesh -> GetNE(); i++)
   {
      GetElementDofs (i, dofs);
      el_dof -> AddColumnsInRow (i, dofs.Size());
+
+      if (el_fos)
+      {
+         mesh->GetElementFaces(i, F, Fo);
+         el_fos -> AddColumnsInRow (i, Fo.Size());
+      }
   }
   el_dof -> MakeJ();
+   if (el_fos) { el_fos -> MakeJ(); }
   for (int i = 0; i < mesh -> GetNE(); i++)
   {
      GetElementDofs (i, dofs);
      el_dof -> AddConnections (i, (int *)dofs, dofs.Size());
+
+      if (el_fos)
+      {
+         mesh->GetElementFaces(i, F, Fo);
+         el_fos -> AddConnections (i, (int *)Fo, Fo.Size());
+      }
   }
   el_dof -> ShiftUpI();
+   if (el_fos) { el_fos -> ShiftUpI(); }
   elem_dof = el_dof;
+   elem_fos = el_fos;
 }

 void FiniteElementSpace::BuildBdrElementToDofTable() const
@@ -375,7 +417,9 @@ void FiniteElementSpace::BuildFaceToDofTable() const
 void FiniteElementSpace::RebuildElementToDofTable()
 {
   delete elem_dof;
+   delete elem_fos;
   elem_dof = NULL;
+   elem_fos = NULL;
   BuildElementToDofTable();
 }

@@ -1225,7 +1269,7 @@ const Operator *FiniteElementSpace::GetElementRestriction(
   return L2E_nat.Ptr();
 }

-const Operator *FiniteElementSpace::GetFaceRestriction(
+const FaceRestriction *FiniteElementSpace::GetFaceRestriction(
   ElementDofOrdering e_ordering, FaceType type, L2FaceValues mul) const
 {
   const bool is_dg_space = IsDGSpace();
@@ -1239,7 +1283,7 @@ const Operator *FiniteElementSpace::GetFaceRestriction(
   }
   else
   {
-      Operator* res;
+      FaceRestriction *res;
      if (is_dg_space)
      {
         res = new L2FaceRestriction(*this, e_ordering, type, m);
@@ -1315,8 +1359,10 @@ const FaceQuadratureInterpolator

 SparseMatrix *FiniteElementSpace::RefinementMatrix_main(
   const int coarse_ndofs, const Table &coarse_elem_dof,
-   const DenseTensor localP[]) const
+   const Table *coarse_elem_fos, const DenseTensor localP[]) const
 {
+   /// TODO: Implement DofTransformation support
+
   MFEM_VERIFY(mesh->GetLastOperation() == Mesh::REFINE, "");

   Array<int> dofs, coarse_dofs, coarse_vdofs;
@@ -1399,7 +1445,8 @@ void FiniteElementSpace::GetLocalRefinementMatrices(
 }

 SparseMatrix* FiniteElementSpace::RefinementMatrix(int old_ndofs,
-                                                   const Table* old_elem_dof)
+                                                   const Table* old_elem_dof,
+                                                   const Table* old_elem_fos)
 {
   MFEM_VERIFY(GetNE() >= old_elem_dof->Size(),
               "Previous mesh is not coarser.");
@@ -1412,13 +1459,16 @@ SparseMatrix* FiniteElementSpace::RefinementMatrix(int old_ndofs,
      GetLocalRefinementMatrices(elem_geoms[i], localP[elem_geoms[i]]);
   }

-   return RefinementMatrix_main(old_ndofs, *old_elem_dof, localP);
+   return RefinementMatrix_main(old_ndofs, *old_elem_dof, old_elem_fos,
+                                localP);
 }

 FiniteElementSpace::RefinementOperator::RefinementOperator
-(const FiniteElementSpace* fespace, Table* old_elem_dof, int old_ndofs)
+(const FiniteElementSpace* fespace, Table* old_elem_dof, Table* old_elem_fos,
+ int old_ndofs)
   : fespace(fespace)
   , old_elem_dof(old_elem_dof)
+   , old_elem_fos(old_elem_fos)
 {
   MFEM_VERIFY(fespace->GetNE() >= old_elem_dof->Size(),
               "Previous mesh is not coarser.");
@@ -1432,12 +1482,14 @@ FiniteElementSpace::RefinementOperator::RefinementOperator
   {
      fespace->GetLocalRefinementMatrices(elem_geoms[i], localP[elem_geoms[i]]);
   }
+
+   ConstructDoFTrans();
 }

 FiniteElementSpace::RefinementOperator::RefinementOperator(
   const FiniteElementSpace *fespace, const FiniteElementSpace *coarse_fes)
   : Operator(fespace->GetVSize(), coarse_fes->GetVSize()),
-     fespace(fespace), old_elem_dof(NULL)
+     fespace(fespace), old_elem_dof(NULL), old_elem_fos(NULL)
 {
   Mesh::GeometryList elem_geoms(*fespace->GetMesh());

@@ -1449,11 +1501,50 @@ FiniteElementSpace::RefinementOperator::RefinementOperator(

   // Make a copy of the coarse elem_dof Table.
   old_elem_dof = new Table(coarse_fes->GetElementToDofTable());
+
+   // Make a copy of the coarse elem_fos Table if it exists.
+   if (coarse_fes->GetElementToFaceOrientationTable())
+   {
+      old_elem_fos = new Table(*coarse_fes->GetElementToFaceOrientationTable());
+   }
+
+   ConstructDoFTrans();
 }

 FiniteElementSpace::RefinementOperator::~RefinementOperator()
 {
   delete old_elem_dof;
+   delete old_elem_fos;
+}
+
+void FiniteElementSpace::RefinementOperator
+::ConstructDoFTrans()
+{
+   old_DoFTrans.SetSize(Geometry::NUM_GEOMETRIES);
+   for (int i=0; i<old_DoFTrans.Size(); i++)
+   {
+      old_DoFTrans[i] = NULL;
+   }
+
+   const FiniteElementCollection *fec = fespace->FEColl();
+   if (dynamic_cast<const ND_FECollection*>(fec))
+   {
+      const FiniteElement * nd_tri =
+         fec->FiniteElementForGeometry(Geometry::TRIANGLE);
+      if (nd_tri)
+      {
+         old_DoFTrans[Geometry::TRIANGLE] =
+            new ND_TriDofTransformation(nd_tri->GetOrder());
+      }
+
+      const FiniteElement * nd_tet =
+         fec->FiniteElementForGeometry(Geometry::TETRAHEDRON);
+      if (nd_tet)
+      {
+         old_DoFTrans[Geometry::TETRAHEDRON] =
+            new ND_TetDofTransformation(nd_tet->GetOrder());
+      }
+   }
 }

 void FiniteElementSpace::RefinementOperator
@@ -1462,7 +1553,7 @@ void FiniteElementSpace::RefinementOperator
   Mesh* mesh = fespace->GetMesh();
   const CoarseFineTransformations &rtrans = mesh->GetRefinementTransforms();

-   Array<int> dofs, vdofs, old_dofs, old_vdofs;
+   Array<int> dofs, vdofs, old_dofs, old_vdofs, old_Fo;

   int vdim = fespace->GetVDim();
   int old_ndofs = width / vdim;
@@ -1477,18 +1568,53 @@ void FiniteElementSpace::RefinementOperator

      subY.SetSize(lP.Height());

-      fespace->GetElementDofs(k, dofs);
+      DofTransformation *doftrans = fespace->GetElementDofs(k, dofs);
      old_elem_dof->GetRow(emb.parent, old_dofs);

-      for (int vd = 0; vd < vdim; vd++)
+      if (!doftrans)
      {
-         dofs.Copy(vdofs);
-         fespace->DofsToVDofs(vd, vdofs);
-         old_dofs.Copy(old_vdofs);
-         fespace->DofsToVDofs(vd, old_vdofs, old_ndofs);
-         x.GetSubVector(old_vdofs, subX);
-         lP.Mult(subX, subY);
-         y.SetSubVector(vdofs, subY);
+         for (int vd = 0; vd < vdim; vd++)
+         {
+            dofs.Copy(vdofs);
+            fespace->DofsToVDofs(vd, vdofs);
+            old_dofs.Copy(old_vdofs);
+            fespace->DofsToVDofs(vd, old_vdofs, old_ndofs);
+            x.GetSubVector(old_vdofs, subX);
+            lP.Mult(subX, subY);
+            y.SetSubVector(vdofs, subY);
+         }
+      }
+      else
+      {
+         old_elem_fos->GetRow(emb.parent, old_Fo);
+         old_DoFTrans[geom]->SetFaceOrientations(old_Fo);
+
+         DofTransformation *new_doftrans = NULL;
+         VDofTransformation *vdoftrans =
+            dynamic_cast<VDofTransformation*>(doftrans);
+         if (vdoftrans)
+         {
+            new_doftrans = doftrans;
+            doftrans = vdoftrans->GetDofTransformation();
+         }
+
+         for (int vd = 0; vd < vdim; vd++)
+         {
+            dofs.Copy(vdofs);
+            fespace->DofsToVDofs(vd, vdofs);
+            old_dofs.Copy(old_vdofs);
+            fespace->DofsToVDofs(vd, old_vdofs, old_ndofs);
+            x.GetSubVector(old_vdofs, subX);
+            old_DoFTrans[geom]->InvTransformPrimal(subX);
+            lP.Mult(subX, subY);
+            doftrans->TransformPrimal(subY);
+            y.SetSubVector(vdofs, subY);
+         }
+
+         if (vdoftrans)
+         {
+            doftrans = new_doftrans;
+         }
      }
   }
 }
@@ -1504,12 +1630,12 @@ void FiniteElementSpace::RefinementOperator
   Array<char> processed(fespace->GetVSize());
   processed = 0;

-   Array<int> f_dofs, c_dofs, f_vdofs, c_vdofs;
+   Array<int> f_dofs, c_dofs, f_vdofs, c_vdofs, old_Fo;

   int vdim = fespace->GetVDim();
   int old_ndofs = width / vdim;

-   Vector subY, subX;
+   Vector subY, subX, subYt, subXt;

   for (int k = 0; k < mesh->GetNE(); k++)
   {
@@ -1517,30 +1643,77 @@ void FiniteElementSpace::RefinementOperator
      const Geometry::Type geom = mesh->GetElementBaseGeometry(k);
      const DenseMatrix &lP = localP[geom](emb.matrix);

-      fespace->GetElementDofs(k, f_dofs);
+      DofTransformation * doftrans = fespace->GetElementDofs(k, f_dofs);
      old_elem_dof->GetRow(emb.parent, c_dofs);

-      subY.SetSize(lP.Width());
-
-      for (int vd = 0; vd < vdim; vd++)
+      if (!doftrans)
      {
-         f_dofs.Copy(f_vdofs);
-         fespace->DofsToVDofs(vd, f_vdofs);
-         c_dofs.Copy(c_vdofs);
-         fespace->DofsToVDofs(vd, c_vdofs, old_ndofs);
+         subY.SetSize(lP.Width());

-         x.GetSubVector(f_vdofs, subX);
-
-         for (int p = 0; p < f_dofs.Size(); ++p)
+         for (int vd = 0; vd < vdim; vd++)
         {
-            if (processed[DecodeDof(f_dofs[p])])
+            f_dofs.Copy(f_vdofs);
+            fespace->DofsToVDofs(vd, f_vdofs);
+            c_dofs.Copy(c_vdofs);
+            fespace->DofsToVDofs(vd, c_vdofs, old_ndofs);
+
+            x.GetSubVector(f_vdofs, subX);
+
+            for (int p = 0; p < f_dofs.Size(); ++p)
            {
-               subX[p] = 0.0;
+               if (processed[DecodeDof(f_dofs[p])])
+               {
+                  subX[p] = 0.0;
+               }
            }
+
+            lP.MultTranspose(subX, subY);
+            y.AddElementVector(c_vdofs, subY);
+         }
+      }
+      else
+      {
+         subYt.SetSize(lP.Width());
+
+         old_elem_fos->GetRow(emb.parent, old_Fo);
+         old_DoFTrans[geom]->SetFaceOrientations(old_Fo);
+
+         DofTransformation *new_doftrans = NULL;
+         VDofTransformation *vdoftrans =
+            dynamic_cast<VDofTransformation*>(doftrans);
+         if (vdoftrans)
+         {
+            new_doftrans = doftrans;
+            doftrans = vdoftrans->GetDofTransformation();
         }

-         lP.MultTranspose(subX, subY);
-         y.AddElementVector(c_vdofs, subY);
+         for (int vd = 0; vd < vdim; vd++)
+         {
+            f_dofs.Copy(f_vdofs);
+            fespace->DofsToVDofs(vd, f_vdofs);
+            c_dofs.Copy(c_vdofs);
+            fespace->DofsToVDofs(vd, c_vdofs, old_ndofs);
+
+            x.GetSubVector(f_vdofs, subX);
+            old_DoFTrans[geom]->InvTransformPrimal(subX);
+
+            for (int p = 0; p < f_dofs.Size(); ++p)
+            {
+               if (processed[DecodeDof(f_dofs[p])])
+               {
+                  subX[p] = 0.0;
+               }
+            }
+
+            lP.MultTranspose(subX, subY);
+            doftrans->TransformPrimal(subY);
+            y.AddElementVector(c_vdofs, subY);
+         }
+
+         if (vdoftrans)
+         {
+            doftrans = new_doftrans;
+         }
      }

      for (int p = 0; p < f_dofs.Size(); ++p)
@@ -1550,6 +1723,7 @@ void FiniteElementSpace::RefinementOperator
   }
 }

+/// TODO: Implement DofTransformation support
 FiniteElementSpace::DerefinementOperator::DerefinementOperator(
   const FiniteElementSpace *f_fes, const FiniteElementSpace *c_fes,
   BilinearFormIntegrator *mass_integ)
@@ -1707,8 +1881,11 @@ void FiniteElementSpace::GetLocalDerefinementMatrices(Geometry::Type geom,
 }

 SparseMatrix* FiniteElementSpace::DerefinementMatrix(int old_ndofs,
-                                                     const Table* old_elem_dof)
+                                                     const Table* old_elem_dof,
+                                                     const Table* old_elem_fos)
 {
+   /// TODO: Implement DofTransformation support
+
   MFEM_VERIFY(Nonconforming(), "Not implemented for conforming meshes.");
   MFEM_VERIFY(old_ndofs, "Missing previous (finer) space.");
   MFEM_VERIFY(ndofs <= old_ndofs, "Previous space is not finer.");
@@ -1814,6 +1991,7 @@ void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
   this->ordering = (Ordering::Type) ordering;

   elem_dof = NULL;
+   elem_fos = NULL;
   face_dof = NULL;

   sequence = 0;
@@ -1841,6 +2019,8 @@ void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
      UpdateNURBS();
      cP = cR = cR_hp = NULL;
      cP_is_set = false;
+
+      ConstructDoFTrans();
   }
   else
   {
@@ -1848,9 +2028,41 @@ void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
      own_ext = 0;
      Construct();
   }
+
   BuildElementToDofTable();
 }

+void FiniteElementSpace::ConstructDoFTrans()
+{
+   DestroyDoFTrans();
+
+   VDoFTrans.SetVDim(vdim);
+   DoFTrans.SetSize(Geometry::NUM_GEOMETRIES);
+   for (int i=0; i<DoFTrans.Size(); i++)
+   {
+      DoFTrans[i] = NULL;
+   }
+   if (mesh->Dimension() < 3) { return; }
+   if (dynamic_cast<const ND_FECollection*>(fec))
+   {
+      const FiniteElement * nd_tri =
+         fec->FiniteElementForGeometry(Geometry::TRIANGLE);
+      if (nd_tri)
+      {
+         DoFTrans[Geometry::TRIANGLE] =
+            new ND_TriDofTransformation(nd_tri->GetOrder());
+      }
+
+      const FiniteElement * nd_tet =
+         fec->FiniteElementForGeometry(Geometry::TETRAHEDRON);
+      if (nd_tet)
+      {
+         DoFTrans[Geometry::TETRAHEDRON] =
+            new ND_TetDofTransformation(nd_tet->GetOrder());
+      }
+   }
+}
+
 NURBSExtension *FiniteElementSpace::StealNURBSext()
 {
   if (NURBSext && !own_ext)
@@ -1946,7 +2158,9 @@ void FiniteElementSpace::Construct()
               "Variable order space requires a nonconforming mesh.");

   elem_dof = NULL;
+   elem_fos = NULL;
   bdr_elem_dof = NULL;
+   bdr_elem_fos = NULL;
   face_dof = NULL;

   ndofs = 0;
@@ -2044,6 +2258,8 @@ void FiniteElementSpace::Construct()

   ndofs = nvdofs + nedofs + nfdofs + nbdofs;

+   ConstructDoFTrans();
+
   // record the current mesh sequence number to detect refinement etc.
   mesh_sequence = mesh->GetSequence();

@@ -2295,14 +2511,22 @@ int FiniteElementSpace::GetNVariants(int entity, int index) const
 static const char* msg_orders_changed =
   "Element orders changed, you need to Update() the space first.";

-void FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
+DofTransformation *
+FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
 {
   MFEM_VERIFY(!orders_changed, msg_orders_changed);

   if (elem_dof)
   {
      elem_dof->GetRow(elem, dofs);
-      return;
+
+      if (DoFTrans[mesh->GetElementBaseGeometry(elem)])
+      {
+         Array<int> Fo;
+         elem_fos -> GetRow (elem, Fo);
+         DoFTrans[mesh->GetElementBaseGeometry(elem)]->SetFaceOrientations(Fo);
+      }
+      return DoFTrans[mesh->GetElementBaseGeometry(elem)];
   }

   Array<int> V, E, Eo, F, Fo; // TODO: LocalArray
@@ -2326,6 +2550,11 @@ void FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
      {
         nfd += fec->GetNumDof(mesh->GetFaceGeometry(F[i]), order);
      }
+      if (DoFTrans[mesh->GetElementBaseGeometry(elem)])
+      {
+         DoFTrans[mesh->GetElementBaseGeometry(elem)]
+         -> SetFaceOrientations(Fo);
+      }
   }

   dofs.SetSize(0);
@@ -2383,6 +2612,7 @@ void FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
         dofs.Append(bbase + j);
      }
   }
+   return DoFTrans[mesh->GetElementBaseGeometry(elem)];
 }

 const FiniteElement *FiniteElementSpace::GetFE(int i) const
@@ -2415,18 +2645,27 @@ const FiniteElement *FiniteElementSpace::GetFE(int i) const
   return FE;
 }

-void FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const
+DofTransformation *
+FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const
 {
   MFEM_VERIFY(!orders_changed, msg_orders_changed);

   if (bdr_elem_dof)
   {
      bdr_elem_dof->GetRow(bel, dofs);
-      return;
+
+      if (DoFTrans[mesh->GetBdrElementBaseGeometry(bel)])
+      {
+         Array<int> Fo;
+         bdr_elem_fos -> GetRow (bel, Fo);
+         DoFTrans[mesh->GetBdrElementBaseGeometry(bel)]->
+         SetFaceOrientations(Fo);
+      }
+      return DoFTrans[mesh->GetBdrElementBaseGeometry(bel)];
   }

-   Array<int> V, E, Eo; // TODO: LocalArray
-   int F, Fo;
+   Array<int> V, E, Eo, Fo; // TODO: LocalArray
+   int F, oF;

   int dim = mesh->Dimension();
   auto geom = mesh->GetBdrElementGeometry(bel);
@@ -2445,7 +2684,17 @@ void FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const

   if (nv) { mesh->GetBdrElementVertices(bel, V); }
   if (ne) { mesh->GetBdrElementEdges(bel, E, Eo); }
-   if (nf) { mesh->GetBdrElementFace(bel, &F, &Fo); }
+   if (nf)
+   {
+      mesh->GetBdrElementFace(bel, &F, &oF);
+
+      if (DoFTrans[mesh->GetBdrElementBaseGeometry(bel)])
+      {
+         Fo.Append(oF);
+         DoFTrans[mesh->GetBdrElementBaseGeometry(bel)]->
+         SetFaceOrientations(Fo);
+      }
+   }

   dofs.SetSize(0);
   dofs.Reserve(nv*V.Size() + ne*E.Size() + nf);
@@ -2478,13 +2727,15 @@ void FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const
   if (nf) // face DOFs
   {
      int fbase = (var_face_dofs.Size() > 0) ? FindFaceDof(F, nf) : F*nf;
-      const int *ind = fec->GetDofOrdering(geom, order, Fo);
+      const int *ind = fec->GetDofOrdering(geom, order, oF);

      for (int j = 0; j < nf; j++)
      {
         dofs.Append(EncodeDof(nvdofs + nedofs + fbase, ind[j]));
      }
   }
+
+   return DoFTrans[mesh->GetBdrElementBaseGeometry(bel)];
 }

 int FiniteElementSpace::GetFaceDofs(int face, Array<int> &dofs,
@@ -2794,6 +3045,8 @@ void FiniteElementSpace::Destroy()
   }
   E2BFQ_array.SetSize(0);

+   DestroyDoFTrans();
+
   dof_elem_array.DeleteAll();
   dof_ldof_array.DeleteAll();

@@ -2806,7 +3059,9 @@ void FiniteElementSpace::Destroy()
   else
   {
      delete elem_dof;
+      delete elem_fos;
      delete bdr_elem_dof;
+      delete bdr_elem_fos;
      delete face_dof;

      delete [] bdofs;
@@ -2814,6 +3069,15 @@ void FiniteElementSpace::Destroy()
   ceed::RemoveBasisAndRestriction(this);
 }

+void FiniteElementSpace::DestroyDoFTrans()
+{
+   for (int i = 0; i < DoFTrans.Size(); i++)
+   {
+      delete DoFTrans[i];
+   }
+   DoFTrans.SetSize(0);
+}
+
 void FiniteElementSpace::GetTransferOperator(
   const FiniteElementSpace &coarse_fes, OperatorHandle &T) const
 {
@@ -2831,6 +3095,8 @@ void FiniteElementSpace::GetTransferOperator(
      }
      T.Reset(RefinementMatrix_main(coarse_fes.GetNDofs(),
                                    coarse_fes.GetElementToDofTable(),
+                                    coarse_fes.
+                                    GetElementToFaceOrientationTable(),
                                    localP));
   }
   else
@@ -2933,6 +3199,7 @@ void FiniteElementSpace::Update(bool want_transform)
   }

   Table* old_elem_dof = NULL;
+   Table* old_elem_fos = NULL;
   int old_ndofs;
   bool old_orders_changed = orders_changed;

@@ -2940,7 +3207,9 @@ void FiniteElementSpace::Update(bool want_transform)
   if (want_transform)
   {
      old_elem_dof = elem_dof;
+      old_elem_fos = elem_fos;
      elem_dof = NULL;
+      elem_fos = NULL;
      old_ndofs = ndofs;
   }

@@ -2966,15 +3235,18 @@ void FiniteElementSpace::Update(bool want_transform)
         {
            if (Th.Type() != Operator::MFEM_SPARSEMAT)
            {
-               Th.Reset(new RefinementOperator(this, old_elem_dof, old_ndofs));
+               Th.Reset(new RefinementOperator(this, old_elem_dof,
+                                               old_elem_fos, old_ndofs));
               // The RefinementOperator takes ownership of 'old_elem_dof', so
               // we no longer own it:
               old_elem_dof = NULL;
+               old_elem_fos = NULL;
            }
            else
            {
               // calculate fully assembled matrix
-               Th.Reset(RefinementMatrix(old_ndofs, old_elem_dof));
+               Th.Reset(RefinementMatrix(old_ndofs, old_elem_dof,
+                                         old_elem_fos));
            }
            break;
         }
@@ -2982,7 +3254,7 @@ void FiniteElementSpace::Update(bool want_transform)
         case Mesh::DEREFINE:
         {
            BuildConformingInterpolation();
-            Th.Reset(DerefinementMatrix(old_ndofs, old_elem_dof));
+            Th.Reset(DerefinementMatrix(old_ndofs, old_elem_dof, old_elem_fos));
            if (cP && cR)
            {
               Th.SetOperatorOwner(false);
@@ -2997,6 +3269,7 @@ void FiniteElementSpace::Update(bool want_transform)
      }

      delete old_elem_dof;
+      delete old_elem_fos;
   }
 }

@@ -16,6 +16,7 @@
 #include "../linalg/sparsemat.hpp"
 #include "../mesh/mesh.hpp"
 #include "fe_coll.hpp"
+#include "doftrans.hpp"
 #include "restriction.hpp"
 #include <iostream>
 #include <unordered_map>
@@ -128,7 +129,9 @@ protected:

   // precalculated DOFs for each element, boundary element, and face
   mutable Table *elem_dof; // owned (except in NURBS FE space)
+   mutable Table *elem_fos; // face orientations by element index
   mutable Table *bdr_elem_dof; // owned (except in NURBS FE space)
+   mutable Table *bdr_elem_fos; // bdr face orientations by bdr element index
   mutable Table *face_dof; // owned; in var-order space contains variant 0 DOFs

   Array<int> dof_elem_array, dof_ldof_array;
@@ -137,6 +140,9 @@ protected:
   int own_ext;
   mutable Array<int> face_to_be; // NURBS FE space only

+   Array<DofTransformation*> DoFTrans;
+   mutable VDofTransformation VDoFTrans;
+
   /** Matrix representing the prolongation from the global conforming dofs to
       a set of intermediate partially conforming dofs, e.g. the dofs associated
       with a "cut" space on a non-conforming mesh. */
@@ -164,7 +170,7 @@ protected:
                + 8 * (int)std::get<3>(k);
      }
   };
-   using map_L2F = std::unordered_map<const key_face,Operator*,key_hash>;
+   using map_L2F = std::unordered_map<const key_face,FaceRestriction*,key_hash>;
   mutable map_L2F L2F;

   mutable Array<QuadratureInterpolator*> E2Q_array;
@@ -189,6 +195,9 @@ protected:
   void Construct();
   void Destroy();

+   void ConstructDoFTrans();
+   void DestroyDoFTrans();
+
   void BuildElementToDofTable() const;
   void BuildBdrElementToDofTable() const;
   void BuildFaceToDofTable() const;
@@ -283,12 +292,19 @@ protected:
      const FiniteElementSpace* fespace;
      DenseTensor localP[Geometry::NumGeom];
      Table* old_elem_dof; // Owned.
+      Table* old_elem_fos; // Owned.
+
+      Array<DofTransformation*> old_DoFTrans;
+      mutable VDofTransformation old_VDoFTrans;
+
+      void ConstructDoFTrans();

   public:
      /** Construct the operator based on the elem_dof table of the original
          (coarse) space. The class takes ownership of the table. */
      RefinementOperator(const FiniteElementSpace* fespace,
-                         Table *old_elem_dof/*takes ownership*/, int old_ndofs);
+                         Table *old_elem_dof/*takes ownership*/,
+                         Table *old_elem_fos/*takes ownership*/, int old_ndofs);
      RefinementOperator(const FiniteElementSpace *fespace,
                         const FiniteElementSpace *coarse_fes);
      virtual void Mult(const Vector &x, Vector &y) const;
@@ -302,6 +318,7 @@ protected:
      const FiniteElementSpace *fine_fes; // Not owned.
      DenseTensor localR[Geometry::NumGeom];
      Table *coarse_elem_dof; // Owned.
+      // Table *coarse_elem_fos; // Owned.
      Table coarse_to_fine;
      Array<int> coarse_to_ref_type;
      Array<Geometry::Type> ref_type_to_geom;
@@ -323,6 +340,7 @@ protected:
       the same vector dimension, vdim. */
   SparseMatrix *RefinementMatrix_main(const int coarse_ndofs,
                                       const Table &coarse_elem_dof,
+                                       const Table *coarse_elem_fos,
                                       const DenseTensor localP[]) const;

   void GetLocalRefinementMatrices(Geometry::Type geom,
@@ -333,10 +351,12 @@ protected:
   /** Calculate explicit GridFunction interpolation matrix (after mesh
       refinement). NOTE: consider using the RefinementOperator class instead
       of the fully assembled matrix, which can take a lot of memory. */
-   SparseMatrix* RefinementMatrix(int old_ndofs, const Table* old_elem_dof);
+   SparseMatrix* RefinementMatrix(int old_ndofs, const Table* old_elem_dof,
+                                  const Table* old_elem_fos);

   /// Calculate GridFunction restriction matrix after mesh derefinement.
-   SparseMatrix* DerefinementMatrix(int old_ndofs, const Table* old_elem_dof);
+   SparseMatrix* DerefinementMatrix(int old_ndofs, const Table* old_elem_dof,
+                                    const Table* old_elem_fos);

   /** @brief Return in @a localP the local refinement matrices that map
       between fespaces after mesh refinement. */
@@ -488,7 +508,7 @@ public:
   const Operator *GetElementRestriction(ElementDofOrdering e_ordering) const;

   /// Return an Operator that converts L-vectors to E-vectors on each face.
-   virtual const Operator *GetFaceRestriction(
+   virtual const FaceRestriction *GetFaceRestriction(
      ElementDofOrdering e_ordering, FaceType,
      L2FaceValues mul = L2FaceValues::DoubleValued) const;

@@ -614,10 +634,11 @@ public:
   int GetBdrAttribute(int i) const { return mesh->GetBdrAttribute(i); }

   /// Returns indices of degrees of freedom of element 'elem'.
-   virtual void GetElementDofs(int elem, Array<int> &dofs) const;
+   virtual DofTransformation *GetElementDofs(int elem, Array<int> &dofs) const;

   /// Returns indices of degrees of freedom for boundary element 'bel'.
-   virtual void GetBdrElementDofs(int bel, Array<int> &dofs) const;
+   virtual DofTransformation *GetBdrElementDofs(int bel,
+                                                Array<int> &dofs) const;

   /** @brief Returns the indices of the degrees of freedom for the specified
       face, including the DOFs for the edges and the vertices of the face. */
@@ -666,10 +687,10 @@ public:
   static void AdjustVDofs(Array<int> &vdofs);

   /// Returns indexes of degrees of freedom in array dofs for i'th element.
-   void GetElementVDofs(int i, Array<int> &vdofs) const;
+   DofTransformation *GetElementVDofs(int i, Array<int> &vdofs) const;

   /// Returns indexes of degrees of freedom for i'th boundary element.
-   void GetBdrElementVDofs(int i, Array<int> &vdofs) const;
+   DofTransformation *GetBdrElementVDofs(int i, Array<int> &vdofs) const;

   /// Returns indexes of degrees of freedom for i'th face element (2D and 3D).
   void GetFaceVDofs(int i, Array<int> &vdofs) const;
@@ -695,6 +716,8 @@ public:
       is preserved. */
   void ReorderElementToDofTable();

+   const Table *GetElementToFaceOrientationTable() const { return elem_fos; }
+
   /** @brief Return a reference to the internal Table that stores the lists of
       scalar dofs, for each mesh element, as returned by GetElementDofs(). */
   const Table &GetElementToDofTable() const { return *elem_dof; }
@@ -0,0 +1,46 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef FMS_CONVERT
+#define FMS_CONVERT
+
+#include "../config/config.hpp"
+#include "datacollection.hpp"
+
+#ifdef MFEM_USE_FMS
+#include <fms.h>
+
+namespace mfem
+{
+
+/** In-memory conversion of FMS data collection to an MFEM data collection.
+    @param dc The FMS data collection to convert.
+    @param[out] mfem_dc A pointer to a new MFEM DataCollection containing the
+    FMS data.
+    @return 0 on success; non-zero on failure.
+*/
+int FmsDataCollectionToDataCollection(FmsDataCollection dc,
+                                      DataCollection **mfem_dc);
+
+/** In-memory conversion of MFEM data collection to an FMS data collection.
+    @param mfem_dc The MFEM data collection to convert.
+    @param[out] dc A pointer to a new FmsDataCollection containing the MFEM
+    data.
+    @return 0 on success; non-zero on failure.
+*/
+int DataCollectionToFmsDataCollection(DataCollection *mfem_dc,
+                                      FmsDataCollection *dc);
+
+} // namespace mfem
+
+#endif
+
+#endif
@@ -0,0 +1,167 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../config/config.hpp"
+
+#ifdef MFEM_USE_FMS
+
+#include "fem.hpp"
+#include "../general/text.hpp"
+
+#include <fmsio.h>
+
+#include <string>
+#include <sstream>
+
+namespace mfem
+{
+
+// class FMSDataCollection implementation
+
+FMSDataCollection::FMSDataCollection(const std::string& coll_name,
+                                     Mesh *mesh)
+   : DataCollection(coll_name, mesh),
+     fms_protocol("ascii")
+{
+   appendRankToFileName = false; // always include rank in file names
+   cycle = 0;                   // always include cycle in directory names
+}
+
+#ifdef MFEM_USE_MPI
+
+FMSDataCollection::FMSDataCollection(MPI_Comm comm,
+                                     const std::string& coll_name,
+                                     Mesh *mesh)
+   : DataCollection(coll_name, mesh),
+     fms_protocol("ascii")
+{
+   m_comm = comm;
+   MPI_Comm_rank(comm, &myid);
+   MPI_Comm_size(comm, &num_procs);
+   appendRankToFileName = true; // always include rank in file names
+   cycle = 0;                   // always include cycle in directory names
+}
+
+#endif
+
+FMSDataCollection::~FMSDataCollection()
+{
+   // empty
+}
+
+void FMSDataCollection::Save()
+{
+   // Convert this to FmsDataCollection.
+
+   FmsDataCollection dc;
+   if (DataCollectionToFmsDataCollection(this, &dc) == 0)
+   {
+      std::string root(RootFileName());
+      int err = FmsIOWrite(root.c_str(), fms_protocol.c_str(), dc);
+      FmsDataCollectionDestroy(&dc);
+      if (err)
+      {
+         MFEM_ABORT("Error creating FMS file: " << root);
+      }
+   }
+   else
+   {
+      MFEM_ABORT("Error converting data collection");
+   }
+}
+
+void FMSDataCollection::Load(int cycle)
+{
+   DeleteAll();
+   this->cycle = cycle;
+
+   FmsDataCollection dc;
+   std::string root(RootFileName());
+   int err = FmsIORead(root.c_str(), fms_protocol.c_str(), &dc);
+
+   if (err == 0)
+   {
+      DataCollection *mdc = nullptr;
+      if (FmsDataCollectionToDataCollection(dc,&mdc) == 0)
+      {
+         // Tell the data collection we read that it does not own data.
+         // We will steal its data.
+         mdc->SetOwnData(false);
+
+         SetCycle(mdc->GetCycle());
+         SetTime(mdc->GetTime());
+         SetTimeStep(mdc->GetTimeStep());
+         name = mdc->GetCollectionName();
+
+         // Set mdc's mesh as our mesh.
+         SetMesh(mdc->GetMesh());
+
+         // Set mdc's fields/qfields as ours.
+         std::vector<std::string> names;
+         for (const auto &pair : mdc->GetFieldMap())
+         {
+            names.push_back(pair.first);
+            RegisterField(pair.first, pair.second);
+         }
+         for (const auto &name : names)
+         {
+            mdc->DeregisterField(name);
+         }
+
+         names.clear();
+         for (const auto &pair : mdc->GetQFieldMap())
+         {
+            names.push_back(pair.first);
+            RegisterQField(pair.first, pair.second);
+         }
+         for (const auto &name : names)
+         {
+            mdc->DeregisterField(name);
+         }
+
+         // Indicate that we own the data.
+         SetOwnData(true);
+
+         // Delete mdc. We stole its contents.
+         delete mdc;
+      }
+      FmsDataCollectionDestroy(&dc);
+   }
+   else
+   {
+      MFEM_ABORT("Error reading data collection: " << root);
+   }
+}
+
+void FMSDataCollection::SetProtocol(const std::string &protocol)
+{
+   fms_protocol = protocol;
+}
+
+std::string FMSDataCollection::RootFileName()
+{
+   std::string res;
+   if (pad_digits_cycle)
+   {
+      res = prefix_path + name + "_" +
+            to_padded_string(cycle, pad_digits_cycle) +
+            ".fms";
+   }
+   else
+   {
+      res = prefix_path + name + ".fms";
+   }
+   return res;
+}
+
+} // namespace mfem
+
+#endif
@@ -0,0 +1,74 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_FMSDATACOLLECTION
+#define MFEM_FMSDATACOLLECTION
+
+#include "../config/config.hpp"
+
+#ifdef MFEM_USE_FMS
+
+#include "datacollection.hpp"
+#include <fms.h>
+
+namespace mfem
+{
+
+/** @brief Data collection that uses FMS. */
+/** FMSDataCollection lets MFEM read/write data using FMS.
+
+    For more information, see:
+    - FMS project, https://ceed.exascaleproject.org/fms/
+*/
+
+/// Data collection with FMS I/O routines
+class FMSDataCollection : public DataCollection
+{
+protected:
+   // file name helpers
+
+   /// Returns file name for the current cycle
+   std::string RootFileName();
+
+   // holds currently active i/o protocol
+   std::string fms_protocol;
+
+public:
+   /// Constructor. The collection name is used when saving the data.
+   /** If @a mesh is NULL, then the mesh can be set later by calling either
+       SetMesh() or Load(). The latter works only in serial. */
+   FMSDataCollection(const std::string& collection_name,
+                     Mesh *mesh = NULL);
+#ifdef MFEM_USE_MPI
+   /// Construct a parallel FMSDataCollection.
+   FMSDataCollection(MPI_Comm comm, const std::string& collection_name,
+                     Mesh *mesh = NULL);
+#endif
+
+   /// We will delete the mesh and fields if we own them
+   virtual ~FMSDataCollection();
+
+   /// Set the FMS relay i/o protocol to use
+   /** Supported options: ascii (default), json, yaml, hdf5 */
+   void SetProtocol(const std::string &protocol);
+
+   /// Save the collection and a FMS blueprint root file
+   virtual void Save();
+
+   /// Load the collection based blueprint data
+   virtual void Load(int cycle = 0);
+};
+
+} // namespace mfem
+
+#endif
+
+#endif
@@ -11,15 +11,19 @@

 #include "fem.hpp"
 #include "../mesh/wedge.hpp"
+#include "../mesh/pyramid.hpp"

 namespace mfem
 {

 const char *Geometry::Name[NumGeom] =
-{ "Point", "Segment", "Triangle", "Square", "Tetrahedron", "Cube", "Prism" };
+{
+   "Point", "Segment", "Triangle", "Square", "Tetrahedron", "Cube", "Prism",
+   "Pyramid"
+};

 const double Geometry::Volume[NumGeom] =
-{ 1.0, 1.0, 0.5, 1.0, 1./6, 1.0, 0.5 };
+{ 1.0, 1.0, 0.5, 1.0, 1./6, 1.0, 0.5, 1./3 };

 Geometry::Geometry()
 {
@@ -139,6 +143,28 @@ Geometry::Geometry()
   GeomVert[6]->IntPoint(5).y = 1.0;
   GeomVert[6]->IntPoint(5).z = 1.0;

+   // Vertices for Geometry::PYRAMID
+   GeomVert[7] = new IntegrationRule(5);
+   GeomVert[7]->IntPoint(0).x = 0.0;
+   GeomVert[7]->IntPoint(0).y = 0.0;
+   GeomVert[7]->IntPoint(0).z = 0.0;
+
+   GeomVert[7]->IntPoint(1).x = 1.0;
+   GeomVert[7]->IntPoint(1).y = 0.0;
+   GeomVert[7]->IntPoint(1).z = 0.0;
+
+   GeomVert[7]->IntPoint(2).x = 1.0;
+   GeomVert[7]->IntPoint(2).y = 1.0;
+   GeomVert[7]->IntPoint(2).z = 0.0;
+
+   GeomVert[7]->IntPoint(3).x = 0.0;
+   GeomVert[7]->IntPoint(3).y = 1.0;
+   GeomVert[7]->IntPoint(3).z = 0.0;
+
+   GeomVert[7]->IntPoint(4).x = 0.0;
+   GeomVert[7]->IntPoint(4).y = 0.0;
+   GeomVert[7]->IntPoint(4).z = 1.0;
+
   GeomCenter[POINT].x = 0.0;
   GeomCenter[POINT].y = 0.0;
   GeomCenter[POINT].z = 0.0;
@@ -167,6 +193,10 @@ Geometry::Geometry()
   GeomCenter[PRISM].y = 1.0 / 3.0;
   GeomCenter[PRISM].z = 0.5;

+   GeomCenter[PYRAMID].x = 0.375;
+   GeomCenter[PYRAMID].y = 0.375;
+   GeomCenter[PYRAMID].z = 0.25;
+
   GeomToPerfGeomJac[POINT]       = NULL;
   GeomToPerfGeomJac[SEGMENT]     = new DenseMatrix(1);
   GeomToPerfGeomJac[TRIANGLE]    = new DenseMatrix(2);
@@ -174,6 +204,7 @@ Geometry::Geometry()
   GeomToPerfGeomJac[TETRAHEDRON] = new DenseMatrix(3);
   GeomToPerfGeomJac[CUBE]        = new DenseMatrix(3);
   GeomToPerfGeomJac[PRISM]       = new DenseMatrix(3);
+   GeomToPerfGeomJac[PYRAMID]     = new DenseMatrix(3);

   PerfGeomToGeomJac[POINT]       = NULL;
   PerfGeomToGeomJac[SEGMENT]     = NULL;
@@ -182,6 +213,7 @@ Geometry::Geometry()
   PerfGeomToGeomJac[TETRAHEDRON] = new DenseMatrix(3);
   PerfGeomToGeomJac[CUBE]        = NULL;
   PerfGeomToGeomJac[PRISM]       = new DenseMatrix(3);
+   PerfGeomToGeomJac[PYRAMID]     = new DenseMatrix(3);

   GeomToPerfGeomJac[SEGMENT]->Diag(1.0, 1);
   {
@@ -210,6 +242,14 @@ Geometry::Geometry()
      *GeomToPerfGeomJac[PRISM] = pri_T.Jacobian();
      CalcInverse(pri_T.Jacobian(), *PerfGeomToGeomJac[PRISM]);
   }
+   {
+      IsoparametricTransformation pyr_T;
+      pyr_T.SetFE(&PyramidFE);
+      GetPerfPointMat (PYRAMID, pyr_T.GetPointMat());
+      pyr_T.SetIntPoint(&GeomCenter[PYRAMID]);
+      *GeomToPerfGeomJac[PYRAMID] = pyr_T.Jacobian();
+      CalcInverse(pyr_T.Jacobian(), *PerfGeomToGeomJac[PYRAMID]);
+   }
 }

 Geometry::~Geometry()
@@ -233,6 +273,7 @@ const IntegrationRule * Geometry::GetVertices(int GeomType)
      case Geometry::TETRAHEDRON: return GeomVert[4];
      case Geometry::CUBE:        return GeomVert[5];
      case Geometry::PRISM:       return GeomVert[6];
+      case Geometry::PYRAMID:     return GeomVert[7];
      default:
         mfem_error ("Geometry::GetVertices(...)");
   }
@@ -310,6 +351,25 @@ void Geometry::GetRandomPoint(int GeomType, IntegrationPoint &ip)
            ip.y = 1.0 - ip.y;
         }
         break;
+      case Geometry::PYRAMID:
+         ip.x = double(rand()) / RAND_MAX;
+         ip.y = double(rand()) / RAND_MAX;
+         ip.z = double(rand()) / RAND_MAX;
+         if (ip.x + ip.z > 1.0 && ip.y < ip.x)
+         {
+            double x = ip.x;
+            ip.x = ip.y;
+            ip.y = 1.0 - ip.z;
+            ip.z = 1.0 - x;
+         }
+         else if (ip.y + ip.z > 1.0)
+         {
+            double z = ip.z;
+            ip.z = 1.0 - ip.y;
+            ip.y = ip.x;
+            ip.x = 1.0 - z;
+         }
+         break;
      default:
         MFEM_ABORT("Unknown type of reference element!");
   }
@@ -371,6 +431,10 @@ bool Geometry::CheckPoint(int GeomType, const IntegrationPoint &ip)
         if (ip.x < 0.0 || ip.y < 0.0 || ip.x+ip.y > 1.0 ||
             ip.z < 0.0 || ip.z > 1.0) { return false; }
         break;
+      case Geometry::PYRAMID:
+         if (ip.x < 0.0 || ip.y < 0.0 || ip.x+ip.z > 1.0 || ip.y+ip.z > 1.0 ||
+             ip.z < 0.0 || ip.z > 1.0) { return false; }
+         break;
      default:
         MFEM_ABORT("Unknown type of reference element!");
   }
@@ -441,6 +505,17 @@ bool Geometry::CheckPoint(int GeomType, const IntegrationPoint &ip, double eps)
            return false;
         }
         break;
+      case Geometry::PYRAMID:
+         if (internal::FuzzyLT(ip.x, 0.0, eps)
+             || internal::FuzzyLT(ip.y, 0.0, eps)
+             || internal::FuzzyGT(ip.x+ip.z, 1.0, eps)
+             || internal::FuzzyGT(ip.y+ip.z, 1.0, eps)
+             || internal::FuzzyLT(ip.z, 0.0, eps)
+             || internal::FuzzyGT(ip.z, 1.0, eps) )
+         {
+            return false;
+         }
+         break;
      default:
         MFEM_ABORT("Unknown type of reference element!");
   }
@@ -555,6 +630,16 @@ bool Geometry::ProjectPoint(int GeomType, const IntegrationPoint &beg,
         double lbeg[5] = { beg.x, beg.y, beg.z, 1.0-beg.x-beg.y, 1.0-beg.z };
         return internal::IntersectSegment<5,3>(lbeg, lend, end);
      }
+      case Geometry::PYRAMID:
+      {
+         double lend[6] = { end.x, end.y, end.z,
+                            1.0-end.x-end.z, 1.0-end.y-end.z, 1.0-end.z
+                          };
+         double lbeg[6] = { beg.x, beg.y, beg.z,
+                            1.0-beg.x-beg.z, 1.0-beg.y-beg.z, 1.0-beg.z
+                          };
+         return internal::IntersectSegment<6,3>(lbeg, lend, end);
+      }
      default:
         MFEM_ABORT("Unknown type of reference element!");
   }
@@ -652,6 +737,43 @@ bool Geometry::ProjectPoint(int GeomType, IntegrationPoint &ip)
         return in_tri && in_z;
      }

+      case PYRAMID:
+      {
+         if (ip.x < 0.0)
+         {
+            ip.x = 0.0;
+            internal::ProjectTriangle(ip.y, ip.z);
+            return false;
+         }
+         if (ip.y < 0.0)
+         {
+            ip.y = 0.0;
+            internal::ProjectTriangle(ip.x, ip.z);
+            return false;
+         }
+         if (ip.z < 0.0)
+         {
+            ip.z = 0.0;
+            if (ip.x > 1.0) { ip.x = 1.0; }
+            if (ip.y > 1.0) { ip.y = 1.0; }
+            return false;
+         }
+         if (ip.x >= ip.y)
+         {
+            bool in_y = true;
+            bool in_tri = internal::ProjectTriangle(ip.x, ip.z);
+            if (ip.y > ip.z) { in_y = false; ip.y = ip.z; }
+            return in_tri && in_y;
+         }
+         else
+         {
+            bool in_x = true;
+            bool in_tri = internal::ProjectTriangle(ip.y, ip.z);
+            if (ip.x > ip.z) { in_x = false; ip.x = ip.z; }
+            return in_tri && in_x;
+         }
+      }
+
      default:
         MFEM_ABORT("Reference element type is not supported!");
   }
@@ -726,6 +848,17 @@ void Geometry::GetPerfPointMat(int GeomType, DenseMatrix &pm)
      }
      break;

+      case Geometry::PYRAMID:
+      {
+         pm.SetSize (3, 5);
+         pm(0,0) = 0.0;  pm(1,0) = 0.0;  pm(2,0) = 0.0;
+         pm(0,1) = 1.0;  pm(1,1) = 0.0;  pm(2,1) = 0.0;
+         pm(0,2) = 1.0;  pm(1,2) = 1.0;  pm(2,2) = 0.0;
+         pm(0,3) = 0.0;  pm(1,3) = 1.0;  pm(2,3) = 0.0;
+         pm(0,4) = 0.5;  pm(1,4) = 0.5;  pm(2,4) = 0.7071067811865475;
+      }
+      break;
+
      default:
         mfem_error ("Geometry::GetPerfPointMat (...)");
   }
@@ -744,13 +877,13 @@ void Geometry::JacToPerfJac(int GeomType, const DenseMatrix &J,
   }
 }

-const int Geometry::NumBdrArray[NumGeom] = { 0, 2, 3, 4, 4, 6, 5 };
-const int Geometry::Dimension[NumGeom] = { 0, 1, 2, 2, 3, 3, 3 };
+const int Geometry::NumBdrArray[NumGeom] = { 0, 2, 3, 4, 4, 6, 5, 5 };
+const int Geometry::Dimension[NumGeom] = { 0, 1, 2, 2, 3, 3, 3, 3 };
 const int Geometry::DimStart[MaxDim+2] =
 { POINT, SEGMENT, TRIANGLE, TETRAHEDRON, NUM_GEOMETRIES };
-const int Geometry::NumVerts[NumGeom] = { 1, 2, 3, 4, 4, 8, 6 };
-const int Geometry::NumEdges[NumGeom] = { 0, 1, 3, 4, 6, 12, 9 };
-const int Geometry::NumFaces[NumGeom] = { 0, 0, 1, 1, 4, 6, 5 };
+const int Geometry::NumVerts[NumGeom] = { 1, 2, 3, 4, 4, 8, 6, 5 };
+const int Geometry::NumEdges[NumGeom] = { 0, 1, 3, 4, 6, 12, 9, 8 };
+const int Geometry::NumFaces[NumGeom] = { 0, 0, 1, 1, 4, 6, 5, 5 };

 const int Geometry::
 Constants<Geometry::POINT>::Orient[1][1] = {{0}};
@@ -897,6 +1030,30 @@ Constants<Geometry::PRISM>::VertToVert::J[9][2] =
   {5, 4}                   // 4,5:4
 };

+const int Geometry::
+Constants<Geometry::PYRAMID>::Edges[8][2] =
+{{0, 1}, {1, 2}, {3, 2}, {0, 3}, {0, 4}, {1, 4}, {2, 4}, {3, 4}};
+const int Geometry::
+Constants<Geometry::PYRAMID>::FaceTypes[5] =
+{
+   Geometry::SQUARE,
+   Geometry::TRIANGLE, Geometry::TRIANGLE,
+   Geometry::TRIANGLE, Geometry::TRIANGLE
+};
+const int Geometry::
+Constants<Geometry::PYRAMID>::FaceVert[5][4] =
+{{3, 2, 1, 0}, {0, 1, 4, -1}, {1, 2, 4, -1}, {2, 3, 4, -1}, {3, 0, 4, -1}};
+const int Geometry::
+Constants<Geometry::PYRAMID>::VertToVert::I[5] = {0, 3, 5, 7, 8};
+const int Geometry::
+Constants<Geometry::PYRAMID>::VertToVert::J[8][2] =
+{
+   {1, 0}, {3, 3}, {4, 4}, // 0,1:0   0,3:3  0,4:4
+   {2, 1}, {4, 5},         // 1,2:1   1,4:5
+   {3,-3}, {4, 6},         // 2,3:-3  2,4:6
+   {4, 7}                  // 3,4:7
+};
+

 GeometryRefiner::GeometryRefiner()
 {
@@ -1262,6 +1419,104 @@ RefinedGeometry * GeometryRefiner::Refine(Geometry::Type Geom,
         return RG;
      }

+      case Geometry::PYRAMID:
+      {
+         const int n = Times;
+         RG = new RefinedGeometry ((n+1)*(n+2)*(2*n+3)/6,
+                                   5*n*(2*n-1)*(2*n+1)/3, 0);
+         RG->Times = Times;
+         RG->ETimes = ETimes;
+         RG->Type = type;
+         // enumerate and define the vertices
+         m = 0;
+         for (k = 0; k <= n; k++)
+         {
+            const double *cpij =
+               poly1d.GetPoints(Times - k, BasisType::GetNodalBasis(type));
+            for (j = 0; j <= n - k; j++)
+               for (i = 0; i <= n - k; i++)
+               {
+                  IntegrationPoint &ip = RG->RefPts.IntPoint(m);
+                  if (type == 0)
+                  {
+                     ip.x = (n > k) ? (double(i) / (n - k)) : 0.0;
+                     ip.y = (n > k) ? (double(j) / (n - k)) : 0.0;
+                     ip.z = double(k) / n;
+                  }
+                  else
+                  {
+                     ip.x = cpij[i] * (1.0 - cp[k]);
+                     ip.y = cpij[j] * (1.0 - cp[k]);
+                     ip.z = cp[k];
+                  }
+                  m++;
+               }
+         }
+         if (m != (n+1)*(n+2)*(2*n+3)/6)
+         {
+            mfem_error("GeometryRefiner::Refine() for PYRAMID #1");
+         }
+         // elements
+         Array<int> &G = RG->RefGeoms;
+         m = 0;
+         for (k = 0; k < n; k++)
+         {
+            int lk = k * (k * (2 * k - 6 * n - 9) + 6 * n * (n + 3) + 13) / 6;
+            int lkp1 = (k + 1) *
+                       (k * (2 * k - 6 * n -5) + 6 * n * (n + 2) + 6) / 6;
+            for (j = 0; j < n - k; j++)
+            {
+               for (i = 0; i < n - k; i++)
+               {
+                  G[m++] = lk + j * (n - k + 1) + i;
+                  G[m++] = lk + j * (n - k + 1) + i + 1;
+                  G[m++] = lk + (j + 1) * (n - k + 1) + i + 1;
+                  G[m++] = lk + (j + 1) * (n - k + 1) + i;
+                  G[m++] = lkp1 + j * (n - k) + i;
+               }
+            }
+            for (j = 0; j < n - k - 1; j++)
+            {
+               for (i = 0; i < n - k - 1; i++)
+               {
+                  G[m++] = lkp1 + j * (n - k) + i;
+                  G[m++] = lkp1 + (j + 1) * (n - k) + i;
+                  G[m++] = lkp1 + (j + 1) * (n - k) + i + 1;
+                  G[m++] = lkp1 + j * (n - k) + i + 1;
+                  G[m++] = lk + (j + 1) * (n - k + 1) + i + 1;
+               }
+            }
+            for (j = 0; j < n - k; j++)
+            {
+               for (i = 0; i < n - k - 1; i++)
+               {
+                  G[m++] = lk + j * (n - k + 1) + i + 1;
+                  G[m++] = lk + (j + 1) * (n - k + 1) + i + 1;
+                  G[m++] = lkp1 + j * (n - k) + i;
+                  G[m++] = lkp1 + j * (n - k) + i + 1;
+                  G[m++] = -1;
+               }
+            }
+            for (j = 0; j < n - k - 1; j++)
+            {
+               for (i = 0; i < n - k; i++)
+               {
+                  G[m++] = lk + (j + 1) * (n - k + 1) + i;
+                  G[m++] = lk + (j + 1) * (n - k + 1) + i + 1;
+                  G[m++] = lkp1 + (j + 1) * (n - k) + i;
+                  G[m++] = lkp1 + j * (n - k) + i;
+                  G[m++] = -1;
+               }
+            }
+         }
+         if (m != 5*n*(2*n-1)*(2*n+1)/3)
+         {
+            mfem_error("GeometryRefiner::Refine() for PYRAMID #2");
+         }
+         RGeom[Geometry::PYRAMID].Append(RG);
+         return RG;
+      }
+
      case Geometry::PRISM:
      {
         const int n = Times;
--- a/Show More
+++ b/Show More