Adding first draft of MixedSesquilinearForm class

Merge pull request #2517 from mfem/hiop-inttypes-fix
Fixing HiOp int types
2021-09-16 20:22:32 -07:00 · 2021-09-16 08:28:45 -07:00 · 2021-09-16 08:28:31 -07:00 · 2021-09-14 18:02:26 -07:00 · 2021-09-14 18:02:01 -07:00 · 2021-09-13 12:25:02 -07:00
134 changed files with 8972 additions and 904 deletions
@@ -63,7 +63,7 @@ jobs:
        exit 1

  code-style:
-    runs-on: ubuntu-16.04 # needed for astyle 2.05.1
+    runs-on: ubuntu-18.04

    steps:
    - name: checkout mfem
@@ -71,7 +71,7 @@ jobs:

    - name: get astyle
      run: |
-        sudo apt-get install astyle=2.05.1-0ubuntu1
+        sudo apt-get install astyle=3.1-1ubuntu2

    - name: style check
      run: |
@@ -16,7 +16,7 @@
 # We define the following GitLab pipeline variables:
 #
 # BUILD_ROOT:
-# The path to the shared resources between all jobs. For example, external
+# The path to the shared resource between all jobs. For example, external
 # repositories like 'tests' and 'tpls' are cloned here. Also, 'tpls' is built
 # once for all targets, so that build happen here. The BUILD_ROOT is unique to
 # the pipeline, preventing any form of concurrency with other pipelines. This
@@ -30,7 +30,7 @@
 #
 # MFEM_ALLOC_NAME:
 # On LLNL's quartz, there is only one allocation shared among jobs in order to
-# save time and resources. This allocation has to be uniquely named so that we
+# save time and resource. This allocation has to be uniquely named so that we
 # are sure to retrieve it.
 #
 # TPLS_REPO & TESTS_REPO:
@@ -50,24 +50,27 @@ variables:
  AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
  MFEM_DATA_REPO: https://github.com/mfem/data.git
  ARTIFACTS_DIR: artifacts
+  SLURM_OVERLAP: 1

 # The pipeline is divided into stages. Usually, jobs in a given stage wait for
 # the preceding stages to complete before to start. However, we sometimes use
 # the "needs" keyword and express the DAG of jobs for more efficiency.
 # - We use setup and setup_baseline phases to download content outside of mfem
 #   directory.
-# - Allocate/Release is where quartz resources are allocated/released once for all.
+# - Allocate/Release is where quartz resource are allocated/released once for all.
 # - Build and Test is where we build and MFEM for multiple toolchains.
 # - Baseline_checks gathers baseline-type test suites execution
 # - Baseline_publish, only available on master, allows to update baseline
 #   results
 stages:
  - setup
-  - q_allocate_resources
+  - q_allocate_resource
  - q_build_and_test
-  - q_release_resources
+  - q_release_resource
  - l_build_and_test
+  - c_allocate_resource
  - c_build_and_test
+  - c_release_resource
  - setup_baseline
  - baseline_check
  - baseline_to_autotest
@@ -119,11 +122,11 @@ setup_baseline:
    - echo ${ALLOC_NAME}
    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
    - echo ${JOBID}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 30 -N 1 tests/gitlab/build_and_test
+    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 30 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --build-root "${BUILD_ROOT}" --data

 .build_toss_3_x86_64_ib_corona_script:
  script:
-    - srun -p mi60 -t 15 -N 1 tests/gitlab/build_and_test
+    - srun -p mi60 -t 15 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --build-root "${BUILD_ROOT}" --data

 # Lassen uses a different job scheduler (spectrum lsf) that does not allow
 # pre-allocation the same way slurm does.  We use pdebug queue on lassen to
@@ -131,7 +134,7 @@ setup_baseline:
 # builds.
 .build_blueos_3_ppc64le_ib_script:
  script:
-    - lalloc 1 -W 30 -q pdebug tests/gitlab/build_and_test
+    - lalloc 1 -W 30 -q pdebug tests/gitlab/build_and_test --spec "${SPEC}" --build-root "${BUILD_ROOT}" --data

 # Shared script for baseline and sample-run-baseline, the value of BASELINE_TEST
 # differentiates between the two tests.
@@ -252,3 +255,4 @@ setup_baseline:
 include:
  - local: .gitlab/quartz.yml
  - local: .gitlab/lassen.yml
+  - local: .gitlab/corona.yml
@@ -0,0 +1,127 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# GitLab pipeline configuration for the Corona machine at LLNL
+
+.on_corona:
+  tags:
+    - shell
+    - corona
+  rules:
+    # Don’t run corona jobs if...
+    # Note: This makes corona an "opt-in" machine. To activate builds on corona
+    # for a given GitLab clone of MFEM, go to Setting/CI-CD/variables, and set
+    # "ON_CORONA" to "ON". An LC account on for corona is required to trigger a
+    # pipeline there.
+    - if: '$CI_COMMIT_BRANCH =~ /_cnone/ || $ON_CORONA != "ON"'
+      when: never
+    # Don’t run autotest update if...
+    - if: '$CI_JOB_NAME =~ /update_autotest/ && $AUTOTEST != "YES"'
+      when: never
+    # Don’t run autotest update if...
+    - if: '$CI_JOB_NAME =~ /c_report/ && $AUTOTEST != "YES"'
+      when: never
+    # Report success on success status
+    - if: '$CI_JOB_NAME =~ /c_report_success/ && $AUTOTEST == "YES"'
+      when: on_success
+    # Report failure on failure status
+    - if: '$CI_JOB_NAME =~ /c_report_failure/ && $AUTOTEST == "YES"'
+      when: on_failure
+    # Always release resource
+    - if: '$CI_JOB_NAME =~ /release_resource/'
+      when: always
+    # Default is to run if previous stage succeeded
+    - when: on_success
+
+# This is a yaml anchor, it can be used to avoid duplication like here.
+# The code below will simply be pasted wherever the anchor is placed.
+.safe_create_rundir: &safe_create_rundir |
+  if ! mkdir ${rundir}; then
+    n=1
+    while ! mkdir ${rundir}_${n}
+    do
+      n=$((n+1))
+    done
+    rundir=${rundir}_${n}
+  fi
+
+# Allocate
+c_allocate_resource:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_corona
+  stage: c_allocate_resource
+  script:
+    - salloc --exclusive --nodes=1 --partition=mi60 --time=30 --no-shell --job-name=${ALLOC_NAME}
+  timeout: 6h
+  needs: [setup]
+
+# Release
+c_release_resource:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_corona
+  stage: c_release_resource
+  script:
+    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
+    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
+  needs: [rocm_gcc_8.3.1]
+
+# Release
+c_report_success:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_corona
+  stage: c_release_resource
+  script:
+    - echo "Can only run if all the corona jobs passed"
+    - cd ${AUTOTEST_ROOT}/autotest && git pull
+    - rundir="corona/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
+    - mkdir -p corona
+    - *safe_create_rundir
+    - echo "The Corona jobs were successful" > ${rundir}/gitlab.out
+    - echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
+    - git add ${rundir}
+    - git commit -am "Gitlab CI log for baseline on corona with intel ($(date +%Y-%m-%d))"
+    - git push origin master
+  needs: [rocm_gcc_8.3.1]
+
+c_report_failure:
+  variables:
+    GIT_STRATEGY: none
+  extends: .on_corona
+  stage: c_release_resource
+  script:
+    - echo "Runs if there was at least one failure on corona"
+    - cd ${AUTOTEST_ROOT}/autotest && git pull
+    - rundir="corona/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
+    - mkdir -p corona
+    - *safe_create_rundir
+    - echo "There was an error while running CI on Corona" > ${rundir}/gitlab.err
+    - cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
+    - git add ${rundir}
+    - git commit -am "Gitlab CI log for baseline on corona with intel ($(date +%Y-%m-%d))"
+    - git push origin master
+  needs: [rocm_gcc_8.3.1]
+
+# Spack helped builds
+# Generic corona build job, extending build script
+.build_and_test_on_corona:
+  extends: [.build_toss_3_x86_64_ib_corona_script, .on_corona]
+  stage: c_build_and_test
+
+# Build MFEM
+rocm_gcc_8.3.1:
+  variables:
+    SPEC: "@develop%gcc@8.3.1+rocm amdgpu_target=gfx906"
+  extends: .build_and_test_on_corona
+  needs: [c_allocate_resource]
+
@@ -30,5 +30,5 @@

 opt_mpi_cuda_xl_16_1_1_8:
  variables:
-    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=sm_70"
+    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=70"
  extends: .build_and_test_on_lassen
@@ -31,8 +31,8 @@
    # Report failure on failure status
    - if: '$CI_JOB_NAME =~ /q_report_failure/ && $AUTOTEST == "YES"'
      when: on_failure
-    # Always release resources
-    - if: '$CI_JOB_NAME =~ /release_resources/'
+    # Always release resource
+    - if: '$CI_JOB_NAME =~ /release_resource/'
      when: always
    # Default is to run if previous stage succeeded
    - when: on_success
@@ -50,21 +50,21 @@
  fi

 # Allocate
-q_allocate_resources:
+q_allocate_resource:
  variables:
    GIT_STRATEGY: none
  extends: .on_quartz
-  stage: q_allocate_resources
+  stage: q_allocate_resource
  script:
    - salloc --exclusive --nodes=1 --partition=pdebug --time=30 --no-shell --job-name=${ALLOC_NAME}
  timeout: 6h

 # Release
-q_release_resources:
+q_release_resource:
  variables:
    GIT_STRATEGY: none
  extends: .on_quartz
-  stage: q_release_resources
+  stage: q_release_resource
  script:
    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
    - ([[ -n "${JOBID}" ]] && scancel ${JOBID})
@@ -74,11 +74,12 @@ q_report_success:
  variables:
    GIT_STRATEGY: none
  extends: .on_quartz
-  stage: q_release_resources
+  stage: q_release_resource
  script:
    - echo "Can only run if all the quartz jobs passed"
    - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
+    - rundir="quartz/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
+    - mkdir -p quartz
    - *safe_create_rundir
    - echo "The Quartz jobs were successful" > ${rundir}/gitlab.out
    - echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
@@ -90,11 +91,12 @@ q_report_failure:
  variables:
    GIT_STRATEGY: none
  extends: .on_quartz
-  stage: q_release_resources
+  stage: q_release_resource
  script:
    - echo "Runs if there was at least one failure on quartz"
    - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
+    - rundir="quartz/$(date +%Y-%m-%d)-gitlab-ci-${CI_COMMIT_REF_SLUG}"
+    - mkdir -p quartz
    - *safe_create_rundir
    - echo "There was an error while running CI on Quartz" > ${rundir}/gitlab.err
    - echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
@@ -165,7 +167,8 @@ update_autotest:
  stage: baseline_to_autotest
  script:
    - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - rundir="quartz/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
+    - rundir="quartz/$(date +%Y-%m-%d)-gitlab-baseline-${CI_COMMIT_REF_SLUG}"
+    - mkdir -p quartz
    - *safe_create_rundir
    - cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
    # We create an autotest-email.html file, because that's how we signal that there was a diff (temporary).
@@ -10,6 +10,33 @@

 Version 4.3.1 (development)
 ===========================
+- Switched from Artistic Style (astyle) version 2.05.1 to version 3.1 for code
+  formatting. See the "make style" target.
+
+- Added support for hr-adaptivity using TMOP-based error estimator.
+
+- Added 'double' atomicAdd implementation for previous versions of CUDA.
+
+- Adding lowest order Nedelec and Raviart-Thomas basis functions on wedge
+  shaped elements.
+
+- Added initial support for meshes with pyramidal elements, including several
+  pyramidal meshes in the data/ directory and support for the lowest order H1,
+  Nedelec, Raviart-Thomas, and L2 basis functions on pyramids.
+
+- Updated the hypre interface according to changes in hypre-2.22.1. The ADS
+  solver is now fully working on GPUs.
+
+- Tetrahedral meshes no longer need to be reordered to support high order
+  Nedelec basis functions. This will allow future support for Nedelec basis
+  functions on wedges and pyramids which are not amenable to reordering. The
+  ReorientTetMesh method of the Mesh and ParMesh classes has been deprecated.
+
+- Gmsh meshes where all elements have zero physical tag (the default Gmsh
+  output format if no physical groups are defined) are now successfully loaded,
+  and elements are reassigned attribute number 1.
+
+- Fixed several MinGW build issues on Windows.


 Version 4.3, released on July 29, 2021
@@ -0,0 +1,66 @@
+cff-version: 1.2.0
+message: "If you use MFEM, please cite it as follows."
+authors:
+- family-names: "MFEM Team"
+title: "MFEM: Modular Finite Element Methods [Software]"
+doi: 10.11578/dc.20171025.1248
+url: "https://mfem.org"
+preferred-citation:
+  type: article
+  authors:
+  - family-names: "Anderson"
+    given-names: "Robert"
+    orcid: "https://orcid.org/0000-0002-3508-9944"
+  - family-names: "Andrej"
+    given-names: "Julian"
+    orcid: "https://orcid.org/0000-0001-7661-4840"
+  - family-names: "Barker"
+    given-names: "Andrew"
+    orcid: "https://orcid.org/0000-0003-3572-911X"
+  - family-names: "Bramwell"
+    given-names: "Jamie"
+  - family-names: "Camier"
+    given-names: "Jean-Sylvain"
+    orcid: "https://orcid.org/0000-0003-2421-1999"
+  - family-names: "Cerveny"
+    given-names: "Jakub"
+    orcid: "https://orcid.org/0000-0003-4231-2531"
+  - family-names: "Dobrev"
+    given-names: "Veselin"
+    orcid: "https://orcid.org/0000-0003-1793-5622"
+  - family-names: "Dudouit"
+    given-names: "Yohann"
+    orcid: "https://orcid.org/0000-0001-5831-561X"
+  - family-names: "Fisher"
+    given-names: "Aaron"
+  - family-names: "Kolev"
+    given-names: "Tzanio"
+    orcid: "https://orcid.org/0000-0002-2810-3090"
+  - family-names: "Pazner"
+    given-names: "Will"
+    orcid: "https://orcid.org/0000-0003-4885-2934"
+  - family-names: "Stowell"
+    given-names: "Mark"
+    orcid: "https://orcid.org/0000-0002-5389-7435"
+  - family-names: "Tomov"
+    given-names: "Vladimir"
+    orcid: "https://orcid.org/0000-0002-1846-6816"
+  - family-names: "Akkerman"
+    given-names: "Ido"
+    orcid: "https://orcid.org/0000-0002-5937-0300"
+  - family-names: "Dahm"
+    given-names: "Johann"
+    orcid: "https://orcid.org/0000-0001-9657-3564"
+  - family-names: "Medina"
+    given-names: "David"
+  - family-names: "Zampini"
+    given-names: "Stefano"
+    orcid: "https://orcid.org/0000-0002-0435-0433"
+  doi: "10.1016/j.camwa.2020.06.009"
+  journal: "Computers \\& Mathematics with Applications"
+  month: 1
+  start: 42 # First page number
+  end: 74 # Last page number
+  title: "MFEM: A Modular Finite Element Methods Library"
+  volume: 81
+  year: 2021
@@ -90,6 +90,11 @@ include("${CMAKE_CURRENT_SOURCE_DIR}/config/XSDKDefaults.cmake")

 # Enable languages.
 enable_language(CXX)
+if (MINGW)
+   # MinGW GCC does not expose the functions jn/_jn, yn/_yn (used in Example
+   # 25/25p) unless we use '-std=gnu++11':
+   set(CMAKE_CXX_EXTENSIONS ON)
+endif()
 if (MFEM_USE_CUDA)
   if (MFEM_USE_HIP)
      message(FATAL_ERROR " *** MFEM_USE_HIP cannot be combined with MFEM_USE_CUDA.")
@@ -428,10 +433,10 @@ endif()
 #    With newer versions of SuiteSparse which include METIS header using 64-bit
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
-set(MFEM_TPLS MPI_CXX OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS PETSC
+set(MFEM_TPLS OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS PETSC
    SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB NETCDF
    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
-    CUSPARSE MKL_CPARDISO AMGX CALIPER)
+    CUSPARSE MKL_CPARDISO AMGX CALIPER MPI_CXX)

 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
@@ -496,13 +501,9 @@ set(MFEM_INSTALL_DIR ${CMAKE_INSTALL_PREFIX} CACHE PATH
 # Declaring the library
 mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
 # message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
-if (CMAKE_VERSION VERSION_GREATER 2.8.11)
-  target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES})
-else()
-  target_link_libraries(mfem ${TPL_LIBRARIES})
-endif()
+target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES})
 if (MINGW)
-  target_link_libraries(mfem ws2_32)
+  target_link_libraries(mfem PRIVATE ws2_32)
 endif()
 set_target_properties(mfem PROPERTIES VERSION "${mfem_VERSION}")
 set_target_properties(mfem PROPERTIES SOVERSION "${mfem_VERSION}")
@@ -327,7 +327,7 @@ Before you can start, you need a GitHub account, here are a few suggestions:
    documentation in source comments.
  - Consistent code styling is enforced with `make style` in the top-level
    directory. This requires [Artistic Style](http://astyle.sourceforge.net) (we
-    specifically use version 2.05.1). See also the file `config/mfem.astylerc`.
+    specifically use version 3.1). See also the file `config/mfem.astylerc`.
  - Use `mfem::out` and `mfem::err` instead of `std::cout` and `std::cerr` in
    internal library code. (You can use `std` in examples and miniapps.)
  - When manually resolving conflicts during a merge, make sure to mention the
@@ -549,7 +549,7 @@ The specific libraries and their options are:
  Options: HYPRE_OPT, HYPRE_LIB.
  Versions: HYPRE >= 2.10.0b  (HYPRE built without CUDA)
            HYPRE >= 2.20.0   (HYPRE built with '--enable-mixedint')
-            HYPRE >= 2.22.0   (HYPRE built with CUDA)
+            HYPRE >= 2.22.1   (HYPRE built with CUDA)

 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
  MFEM_USE_METIS_5 = YES (default is to use METIS 4).
@@ -692,7 +692,7 @@ The specific libraries and their options are:
 - HiOp (optional), used when MFEM_USE_HIOP = YES.
  URL: https://github.com/LLNL/hiop
  Options: HIOP_OPT, HIOP_LIB.
-  Versions: HIOP >= 0.4.
+  Versions: HIOP >= 0.4.6.

 - GSLIB (optional), used when MFEM_USE_GSLIB = YES. The gslib library must be
  built prior to the MFEM build, as follows: download gslib-1.0.7, untar it at
@@ -87,12 +87,12 @@ fi
 ## style check
 #if [[ "${option}" == "--style" || "${option}" == "" ]]; then
 if [[ "${option}" == "--style" ]]; then
-  if which astyle && [[ "$(astyle --version)" == "Artistic Style Version 2.05.1" ]]; then
+  if which astyle && [[ "$(astyle --version)" == "Artistic Style Version 3.1" ]]; then
    cd tests/scripts
    if ! ./runtest code-style; then code=1; fi
    cd -
  else
-    echo "Warning: astyle not found or version is not 2.05.1"
+    echo "Warning: astyle not found or version is not 3.1"
  fi
 fi

@@ -0,0 +1,9 @@
+MFEM INLINE mesh v1.0
+
+type = pyramid
+nx = 4
+ny = 4
+nz = 4
+sx = 1.0
+sy = 1.0
+sz = 1.0
@@ -0,0 +1,43 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+# PYRAMID     = 7
+#
+
+dimension
+3
+
+elements
+2
+1 7 4 3 2 1 0
+1 7 1 2 3 4 5
+
+boundary
+8
+1 2 0 2 1
+2 2 0 3 2
+3 2 0 4 3
+4 2 0 1 4
+5 2 1 2 5
+6 2 2 3 5
+7 2 3 4 5
+8 2 4 1 5
+
+vertices
+6
+3
+0 0 -1
+1 0 0
+0 1 0
+-1 0 0
+0 -1 0
+0 0 1
@@ -0,0 +1,38 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+# PYRAMID     = 7
+#
+
+dimension
+3
+
+elements
+1
+1 7 0 1 2 3 4
+
+boundary
+5
+1 3 3 2 1 0
+2 2 0 1 4
+3 2 1 2 4
+4 2 2 3 4
+5 2 3 0 4
+
+vertices
+5
+3
+0 0 0
+1 0 0
+1 1 0
+0 1 0
+0 0 1
@@ -206,9 +206,9 @@ int main(int argc, char *argv[])
   cout << "Size of linear system: " << A->Height() << endl;

   // 11. Solve the linear system A X = B.
-   MFEM_PERF_BEGIN("Solve A X=B");
   if (!pa)
   {
+      MFEM_PERF_SCOPE("Solve A X=B (FA)");
 #ifndef MFEM_USE_SUITESPARSE
      // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
      GSSmoother M((SparseMatrix&)(*A));
@@ -223,6 +223,7 @@ int main(int argc, char *argv[])
   }
   else // Jacobi preconditioning in partial assembly mode
   {
+      MFEM_PERF_SCOPE("Solve A X=B (PA)");
      if (UsesTensorBasis(fespace))
      {
         OperatorJacobiSmoother M(a, ess_tdof_list);
@@ -233,7 +234,6 @@ int main(int argc, char *argv[])
         CG(*A, B, X, 1, 400, 1e-12, 0.0);
      }
   }
-   MFEM_PERF_END("Solve A X=B");
   // 12. Recover the solution as a finite element grid function.
   a.RecoverFEMSolution(X, b, x);

@@ -231,28 +231,29 @@ int main(int argc, char *argv[])
   // 13. Solve the linear system A X = B.
   //     * With full assembly, use the BoomerAMG preconditioner from hypre.
   //     * With partial assembly, use Jacobi smoothing, for now.
-   MFEM_PERF_BEGIN("Solve A X = B");
-   Solver *prec = NULL;
-   if (pa)
   {
-      if (UsesTensorBasis(fespace))
+      MFEM_PERF_SCOPE("Solve A X=B");
+      Solver *prec = NULL;
+      if (pa)
      {
-         prec = new OperatorJacobiSmoother(a, ess_tdof_list);
+         if (UsesTensorBasis(fespace))
+         {
+            prec = new OperatorJacobiSmoother(a, ess_tdof_list);
+         }
      }
+      else
+      {
+         prec = new HypreBoomerAMG;
+      }
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(2000);
+      cg.SetPrintLevel(1);
+      if (prec) { cg.SetPreconditioner(*prec); }
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      delete prec;
   }
-   else
-   {
-      prec = new HypreBoomerAMG;
-   }
-   CGSolver cg(MPI_COMM_WORLD);
-   cg.SetRelTol(1e-12);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(1);
-   if (prec) { cg.SetPreconditioner(*prec); }
-   cg.SetOperator(*A);
-   cg.Mult(B, X);
-   delete prec;
-   MFEM_PERF_END("Solve A X = B");
   // 14. Recover the parallel grid function corresponding to X. This is the
   //     local finite element solution on each processor.
   a.RecoverFEMSolution(X, b, x);
@@ -9,6 +9,7 @@
 //               ex1 -m ../data/fichera.mesh
 //               ex1 -m ../data/fichera-mixed.mesh
 //               ex1 -m ../data/toroid-wedge.mesh
+//               ex1 -m ../data/octahedron.mesh -o 1
 //               ex1 -m ../data/periodic-annulus-sector.msh
 //               ex1 -m ../data/periodic-torus-sector.msh
 //               ex1 -m ../data/square-disc-p2.vtk -o 2
@@ -118,7 +118,6 @@ int main(int argc, char *argv[])
   {
      pmesh->UniformRefinement();
   }
-   pmesh->ReorientTetMesh();

   // 7. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Nedelec finite elements of the specified order.
@@ -24,7 +24,10 @@
 //               class ConductionOperator defining C(u)), as well as their
 //               implicit time integration. Note that implementing the method
 //               ConductionOperator::ImplicitSolve is the only requirement for
-//               high-order implicit (SDIRK) time integration.
+//               high-order implicit (SDIRK) time integration. In this example,
+//               the diffusion operator is linearized by evaluating with the
+//               lagged solution from the previous timestep, so there is only
+//               a linear solve.
 //
 //               We recommend viewing examples 2, 9 and 10 before viewing this
 //               example.
@@ -326,8 +329,8 @@ ConductionOperator::ConductionOperator(FiniteElementSpace &f, double al,
 void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
 {
   // Compute:
-   //    du_dt = M^{-1}*-K(u)
-   // for du_dt
+   //    du_dt = M^{-1}*-Ku
+   // for du_dt, where K is linearized by using u from the previous timestep
   Kmat.Mult(u, z);
   z.Neg(); // z = -z
   M_solver.Mult(z, du_dt);
@@ -338,7 +341,7 @@ void ConductionOperator::ImplicitSolve(const double dt,
 {
   // Solve the equation:
   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
-   // for du_dt
+   // for du_dt, where K is linearized by using u from the previous timestep
   if (!T)
   {
      T = Add(1.0, Mmat, dt, Kmat);
@@ -24,8 +24,11 @@
 //               class ConductionOperator defining C(u)), as well as their
 //               implicit time integration. Note that implementing the method
 //               ConductionOperator::ImplicitSolve is the only requirement for
-//               high-order implicit (SDIRK) time integration. Optional saving
-//               with ADIOS2 (adios2.readthedocs.io) is also illustrated.
+//               high-order implicit (SDIRK) time integration. In this example,
+//               the diffusion operator is linearized by evaluating with the
+//               lagged solution from the previous timestep, so there is only
+//               a linear solve. Optional saving with ADIOS2
+//               (adios2.readthedocs.io) is also illustrated.
 //
 //               We recommend viewing examples 2, 9 and 10 before viewing this
 //               example.
@@ -420,8 +423,8 @@ ConductionOperator::ConductionOperator(ParFiniteElementSpace &f, double al,
 void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
 {
   // Compute:
-   //    du_dt = M^{-1}*-K(u)
-   // for du_dt
+   //    du_dt = M^{-1}*-Ku
+   // for du_dt, where K is linearized by using u from the previous timestep
   Kmat.Mult(u, z);
   z.Neg(); // z = -z
   M_solver.Mult(z, du_dt);
@@ -432,7 +435,7 @@ void ConductionOperator::ImplicitSolve(const double dt,
 {
   // Solve the equation:
   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
-   // for du_dt
+   // for du_dt, where K is linearized by using u from the previous timestep
   if (!T)
   {
      T = Add(1.0, Mmat, dt, Kmat);
@@ -9,6 +9,7 @@
 //               mpirun -np 4 ex1p -m ../data/fichera.mesh
 //               mpirun -np 4 ex1p -m ../data/fichera-mixed.mesh
 //               mpirun -np 4 ex1p -m ../data/toroid-wedge.mesh
+//               mpirun -np 4 ex1p -m ../data/octahedron.mesh -o 1
 //               mpirun -np 4 ex1p -m ../data/periodic-annulus-sector.msh
 //               mpirun -np 4 ex1p -m ../data/periodic-torus-sector.msh
 //               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
@@ -13,6 +13,8 @@
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 1
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 2
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 2 -pa
+//               ex22 -m ../data/inline-wedge.mesh -o 1
+//               ex22 -m ../data/inline-pyramid.mesh -o 1
 //               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0
 //
 // Device sample runs:
@@ -13,6 +13,8 @@
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 1
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 2
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 1 -p 2 -pa
+//               mpirun -np 4 ex22p -m ../data/inline-wedge.mesh -o 1
+//               mpirun -np 4 ex22p -m ../data/inline-pyramid.mesh -o 1
 //               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0
 //
 // Device sample runs:
@@ -113,7 +113,6 @@ int main(int argc, char *argv[])
         mesh->UniformRefinement();
      }
   }
-   mesh->ReorientTetMesh();

   // 5. Define a finite element space on the mesh. Here we use Nedelec or
   //    Raviart-Thomas finite elements of the specified order.
@@ -141,7 +141,6 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
-   pmesh->ReorientTetMesh();

   // 7. Define a parallel finite element space on the parallel mesh. Here we
   //    use Nedelec or Raviart-Thomas finite elements of the specified order.
@@ -92,7 +92,7 @@ class PMLDiagMatrixCoefficient : public VectorCoefficient
 {
 private:
   CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML * , Vector &);
+   void (*Function)(const Vector &, CartesianPML *, Vector &);
 public:
   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
                                              Vector &),
@@ -277,10 +277,8 @@ int main(int argc, char *argv[])
      mesh->UniformRefinement();
   }

-   // 6. Reorient mesh in case of a tet mesh
-   mesh->ReorientTetMesh();
-
-   // Set element attributes in order to distinguish elements in the PML region
+   // 6. Set element attributes in order to distinguish elements in the
+   //    PML region
   pml->SetAttributes(mesh);

   // 7. Define a finite element space on the mesh. Here we use the Nedelec
@@ -92,7 +92,7 @@ class PMLDiagMatrixCoefficient : public VectorCoefficient
 {
 private:
   CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML * , Vector &);
+   void (*Function)(const Vector &, CartesianPML *, Vector &);
 public:
   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
                                              Vector &),
@@ -316,9 +316,6 @@ int main(int argc, char *argv[])
      }
   }

-   // 7a. Reorient mesh in case of a tet mesh
-   pmesh->ReorientTetMesh();
-
   // 8. Set element attributes in order to distinguish elements in the PML
   pml->SetAttributes(pmesh);

@@ -295,17 +295,7 @@ int main(int argc, char *argv[])
   //     element solution.
   a.RecoverFEMSolution(X, b, u);

-   // 13. Build a mass matrix to help solve for n.Grad(u) where 'n' is a surface
-   //     normal.
-   BilinearForm m(&fespace);
-   m.AddDomainIntegrator(new MassIntegrator);
-   m.Assemble();
-
-   ess_tdof_list.SetSize(0);
-   OperatorPtr M;
-   m.FormSystemMatrix(ess_tdof_list, M);
-
-   // 14. Compute the various boundary integrals.
+   // 13. Compute the various boundary integrals.
   mfem::out << endl
             << "Verifying boundary conditions" << endl
             << "=============================" << endl;
@@ -361,7 +351,7 @@ int main(int argc, char *argv[])
                << " error " << err << endl;
   }

-   // 15. Save the refined mesh and the solution. This output can be viewed
+   // 14. Save the refined mesh and the solution. This output can be viewed
   //     later using GLVis: "glvis -m refined.mesh -g sol.gf".
   {
      ofstream mesh_ofs("refined.mesh");
@@ -372,7 +362,7 @@ int main(int argc, char *argv[])
      u.Save(sol_ofs);
   }

-   // 16. Send the solution by socket to a GLVis server.
+   // 15. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      string title_str = h1 ? "H1" : "DG";
@@ -385,7 +375,7 @@ int main(int argc, char *argv[])
               << " keys 'mmc'" << flush;
   }

-   // 17. Free the used memory.
+   // 16. Free the used memory.
   delete fec;
   delete mesh;

@@ -314,17 +314,7 @@ int main(int argc, char *argv[])
   //     local finite element solution on each processor.
   a.RecoverFEMSolution(X, b, u);

-   // 14. Build a mass matrix to help solve for n.Grad(u) where 'n' is a surface
-   //     normal.
-   ParBilinearForm m(&fespace);
-   m.AddDomainIntegrator(new MassIntegrator);
-   m.Assemble();
-
-   ess_tdof_list.SetSize(0);
-   OperatorPtr M;
-   m.FormSystemMatrix(ess_tdof_list, M);
-
-   // 15. Compute the various boundary integrals.
+   // 14. Compute the various boundary integrals.
   mfem::out << endl
             << "Verifying boundary conditions" << endl
             << "=============================" << endl;
@@ -380,7 +370,7 @@ int main(int argc, char *argv[])
                << " error " << err << endl;
   }

-   // 16. Save the refined mesh and the solution in parallel. This output can be
+   // 15. Save the refined mesh and the solution in parallel. This output can be
   //     viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
   {
      ostringstream mesh_name, sol_name;
@@ -396,7 +386,7 @@ int main(int argc, char *argv[])
      u.Save(sol_ofs);
   }

-   // 17. Send the solution by socket to a GLVis server.
+   // 16. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      string title_str = h1 ? "H1" : "DG";
@@ -411,7 +401,7 @@ int main(int argc, char *argv[])
               << " keys 'mmc'" << flush;
   }

-   // 18. Free the used memory.
+   // 17. Free the used memory.
   delete fec;

   return 0;
@@ -16,6 +16,8 @@
 //               ex3 -m ../data/beam-hex-nurbs.mesh
 //               ex3 -m ../data/amr-hex.mesh
 //               ex3 -m ../data/fichera-amr.mesh
+//               ex3 -m ../data/ref-prism.mesh -o 1
+//               ex3 -m ../data/octahedron.mesh -o 1
 //               ex3 -m ../data/star-surf.mesh -o 1
 //               ex3 -m ../data/mobius-strip.mesh -f 0.1
 //               ex3 -m ../data/klein-bottle.mesh -f 0.1
@@ -113,7 +115,6 @@ int main(int argc, char *argv[])
         mesh->UniformRefinement();
      }
   }
-   mesh->ReorientTetMesh();

   // 5. Define a finite element space on the mesh. Here we use the Nedelec
   //    finite elements of the specified order.
@@ -16,6 +16,8 @@
 //               mpirun -np 4 ex3p -m ../data/beam-hex-nurbs.mesh
 //               mpirun -np 4 ex3p -m ../data/amr-quad.mesh -o 2
 //               mpirun -np 4 ex3p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex3p -m ../data/ref-prism.mesh -o 1
+//               mpirun -np 4 ex3p -m ../data/octahedron.mesh -o 1
 //               mpirun -np 4 ex3p -m ../data/star-surf.mesh -o 2
 //               mpirun -np 4 ex3p -m ../data/mobius-strip.mesh -o 2 -f 0.1
 //               mpirun -np 4 ex3p -m ../data/klein-bottle.mesh -o 2 -f 0.1
@@ -139,9 +141,7 @@ int main(int argc, char *argv[])

   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
-   //    meshes need to be reoriented before we can define high-order Nedelec
-   //    spaces on them.
+   //    parallel mesh is defined, the serial mesh can be deleted.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
   delete mesh;
   {
@@ -151,7 +151,6 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
-   pmesh->ReorientTetMesh();

   // 7. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Nedelec finite elements of the specified order.
@@ -19,6 +19,8 @@
 //               ex4 -m ../data/amr-hex.mesh
 //               ex4 -m ../data/amr-hex.mesh -o 2 -hb
 //               ex4 -m ../data/fichera-amr.mesh -o 2 -sc
+//               ex4 -m ../data/ref-prism.mesh -o 1
+//               ex4 -m ../data/octahedron.mesh -o 1
 //               ex4 -m ../data/star-surf.mesh -o 1
 //
 // Device sample runs:
@@ -19,6 +19,8 @@
 //               mpirun -np 3 ex4p -m ../data/amr-quad.mesh -o 2 -hb
 //               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -sc
 //               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -hb
+//               mpirun -np 4 ex4p -m ../data/ref-prism.mesh -o 1
+//               mpirun -np 4 ex4p -m ../data/octahedron.mesh -o 1
 //               mpirun -np 4 ex4p -m ../data/star-surf.mesh -o 3 -hb
 //
 // Device sample runs:
@@ -135,9 +137,7 @@ int main(int argc, char *argv[])

   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
-   //    meshes need to be reoriented before we can define high-order Nedelec
-   //    spaces on them (this is needed in the ADS solver below).
+   //    parallel mesh is defined, the serial mesh can be deleted.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
   delete mesh;
   {
@@ -147,7 +147,6 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
-   pmesh->ReorientTetMesh();

   // 7. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Raviart-Thomas finite elements of the specified order.
@@ -106,7 +106,6 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
-   pmesh->ReorientTetMesh();

   // 6. Define the trial, interfacial (trace) and test DPG spaces:
   //    - The trial space, x0_space, contains the non-interfacial unknowns and
@@ -121,9 +121,7 @@ int main(int argc, char *argv[])

   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
-   //    meshes need to be reoriented before we can define high-order Nedelec
-   //    spaces on them.
+   //    parallel mesh is defined, the serial mesh can be deleted.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
   delete mesh;
   {
@@ -133,7 +131,6 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
-   pmesh->ReorientTetMesh();

   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Nedelec finite elements of the specified order.
@@ -122,9 +122,7 @@ int main(int argc, char *argv[])

   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
-   //    meshes need to be reoriented before we can define high-order Nedelec
-   //    spaces on them (this is needed in the ADS solver below).
+   //    parallel mesh is defined, the serial mesh can be deleted.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
   delete mesh;
   {
@@ -134,7 +132,6 @@ int main(int argc, char *argv[])
         pmesh->UniformRefinement();
      }
   }
-   pmesh->ReorientTetMesh();

   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Raviart-Thomas finite elements of the specified order.
@@ -39,6 +39,7 @@ set(SRCS
  complex_fem.cpp
  convergence.cpp
  datacollection.cpp
+  doftrans.cpp
  eltrans.cpp
  estimators.cpp
  fe.cpp
@@ -105,6 +106,7 @@ set(SRCS
  tmop/tmop_pa_w3.cpp
  tmop/tmop_pa_w3_c0.cpp
  tmop_tools.cpp
+  tmop_amr.cpp
  gslib.cpp
  transfer.cpp
  lor.cpp
@@ -118,6 +120,7 @@ set(HDRS
  complex_fem.hpp
  convergence.hpp
  datacollection.hpp
+  doftrans.hpp
  eltrans.hpp
  estimators.hpp
  fe.hpp
@@ -164,6 +167,7 @@ set(HDRS
  tmop.hpp
  tmop/tmop_pa.hpp
  tmop_tools.hpp
+  tmop_amr.hpp
  gslib.hpp
  transfer.hpp
  lor.hpp
@@ -391,6 +391,7 @@ void BilinearForm::Assemble(int skip_zeros)
   }

   ElementTransformation *eltrans;
+   DofTransformation * doftrans;
   Mesh *mesh = fes -> GetMesh();
   DenseMatrix elmat, *elmat_p;

@@ -424,7 +425,7 @@ void BilinearForm::Assemble(int skip_zeros)
      for (int i = 0; i < fes -> GetNE(); i++)
      {
         int elem_attr = fes->GetMesh()->GetAttribute(i);
-         fes->GetElementVDofs(i, vdofs);
+         doftrans = fes->GetElementVDofs(i, vdofs);
         if (element_matrices)
         {
            elmat_p = &(*element_matrices)(i);
@@ -458,6 +459,11 @@ void BilinearForm::Assemble(int skip_zeros)
            {
               elmat_p = &elmat;
            }
+            if (doftrans)
+            {
+               doftrans->TransformDual(elmat);
+            }
+            elmat_p = &elmat;
         }
         if (static_cond)
         {
@@ -503,7 +509,7 @@ void BilinearForm::Assemble(int skip_zeros)
         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }

         const FiniteElement &be = *fes->GetBE(i);
-         fes -> GetBdrElementVDofs (i, vdofs);
+         doftrans = fes -> GetBdrElementVDofs (i, vdofs);
         eltrans = fes -> GetBdrElementTransformation (i);
         int k = 0;
         for (; k < boundary_integs.Size(); k++)
@@ -523,17 +529,22 @@ void BilinearForm::Assemble(int skip_zeros)
            boundary_integs[k]->AssembleElementMatrix(be, *eltrans, elemmat);
            elmat += elemmat;
         }
+         if (doftrans)
+         {
+            doftrans->TransformDual(elmat);
+         }
+         elmat_p = &elmat;
         if (!static_cond)
         {
-            mat->AddSubMatrix(vdofs, vdofs, elmat, skip_zeros);
+            mat->AddSubMatrix(vdofs, vdofs, *elmat_p, skip_zeros);
            if (hybridization)
            {
-               hybridization->AssembleBdrMatrix(i, elmat);
+               hybridization->AssembleBdrMatrix(i, *elmat_p);
            }
         }
         else
         {
-            static_cond->AssembleBdrMatrix(i, elmat);
+            static_cond->AssembleBdrMatrix(i, *elmat_p);
         }
      }
   }
@@ -1318,9 +1329,10 @@ void MixedBilinearForm::Assemble (int skip_zeros)
      return;
   }

-   Array<int> tr_vdofs, te_vdofs;
   ElementTransformation *eltrans;
-   DenseMatrix elemmat;
+   DofTransformation * dom_dof_trans;
+   DofTransformation * ran_dof_trans;
+   DenseMatrix elmat;

   Mesh *mesh = test_fes -> GetMesh();

@@ -1333,16 +1345,24 @@ void MixedBilinearForm::Assemble (int skip_zeros)
   {
      for (int i = 0; i < test_fes -> GetNE(); i++)
      {
-         trial_fes -> GetElementVDofs (i, tr_vdofs);
-         test_fes  -> GetElementVDofs (i, te_vdofs);
+         dom_dof_trans = trial_fes -> GetElementVDofs (i, trial_vdofs);
+         ran_dof_trans = test_fes  -> GetElementVDofs (i, test_vdofs);
         eltrans = test_fes -> GetElementTransformation (i);
+
+         elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
+         elmat = 0.0;
         for (int k = 0; k < domain_integs.Size(); k++)
         {
            domain_integs[k] -> AssembleElementMatrix2 (*trial_fes -> GetFE(i),
                                                        *test_fes  -> GetFE(i),
                                                        *eltrans, elemmat);
-            mat -> AddSubMatrix (te_vdofs, tr_vdofs, elemmat, skip_zeros);
+            elmat += elemmat;
         }
+         if (ran_dof_trans || dom_dof_trans)
+         {
+            TransformDual(ran_dof_trans, dom_dof_trans, elmat);
+         }
+         mat -> AddSubMatrix (test_vdofs, trial_vdofs, elmat, skip_zeros);
      }
   }

@@ -1374,9 +1394,12 @@ void MixedBilinearForm::Assemble (int skip_zeros)
         const int bdr_attr = mesh->GetBdrAttribute(i);
         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }

-         trial_fes -> GetBdrElementVDofs (i, tr_vdofs);
-         test_fes  -> GetBdrElementVDofs (i, te_vdofs);
+         dom_dof_trans = trial_fes -> GetBdrElementVDofs (i, trial_vdofs);
+         ran_dof_trans = test_fes  -> GetBdrElementVDofs (i, test_vdofs);
         eltrans = test_fes -> GetBdrElementTransformation (i);
+
+         elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
+         elmat = 0.0;
         for (int k = 0; k < boundary_integs.Size(); k++)
         {
            if (boundary_integs_marker[k] &&
@@ -1385,29 +1408,34 @@ void MixedBilinearForm::Assemble (int skip_zeros)
            boundary_integs[k]->AssembleElementMatrix2 (*trial_fes -> GetBE(i),
                                                        *test_fes  -> GetBE(i),
                                                        *eltrans, elemmat);
-            mat -> AddSubMatrix (te_vdofs, tr_vdofs, elemmat, skip_zeros);
+            elmat += elemmat;
         }
+         if (ran_dof_trans || dom_dof_trans)
+         {
+            TransformDual(ran_dof_trans, dom_dof_trans, elmat);
+         }
+         mat -> AddSubMatrix (test_vdofs, trial_vdofs, elmat, skip_zeros);
      }
   }

   if (trace_face_integs.Size())
   {
      FaceElementTransformations *ftr;
-      Array<int> te_vdofs2;
+      Array<int> test_vdofs2;
      const FiniteElement *trial_face_fe, *test_fe1, *test_fe2;

      int nfaces = mesh->GetNumFaces();
      for (int i = 0; i < nfaces; i++)
      {
         ftr = mesh->GetFaceElementTransformations(i);
-         trial_fes->GetFaceVDofs(i, tr_vdofs);
-         test_fes->GetElementVDofs(ftr->Elem1No, te_vdofs);
+         trial_fes->GetFaceVDofs(i, trial_vdofs);
+         test_fes->GetElementVDofs(ftr->Elem1No, test_vdofs);
         trial_face_fe = trial_fes->GetFaceElement(i);
         test_fe1 = test_fes->GetFE(ftr->Elem1No);
         if (ftr->Elem2No >= 0)
         {
-            test_fes->GetElementVDofs(ftr->Elem2No, te_vdofs2);
-            te_vdofs.Append(te_vdofs2);
+            test_fes->GetElementVDofs(ftr->Elem2No, test_vdofs2);
+            test_vdofs.Append(test_vdofs2);
            test_fe2 = test_fes->GetFE(ftr->Elem2No);
         }
         else
@@ -1421,7 +1449,7 @@ void MixedBilinearForm::Assemble (int skip_zeros)
         {
            trace_face_integs[k]->AssembleFaceMatrix(*trial_face_fe, *test_fe1,
                                                     *test_fe2, *ftr, elemmat);
-            mat->AddSubMatrix(te_vdofs, tr_vdofs, elemmat, skip_zeros);
+            mat->AddSubMatrix(test_vdofs, trial_vdofs, elemmat, skip_zeros);
         }
      }
   }
@@ -1461,8 +1489,8 @@ void MixedBilinearForm::Assemble (int skip_zeros)
         ftr = mesh->GetBdrFaceTransformations(i);
         if (ftr)
         {
-            trial_fes->GetFaceVDofs(ftr->ElementNo, tr_vdofs);
-            test_fes->GetElementVDofs(ftr->Elem1No, te_vdofs);
+            trial_fes->GetFaceVDofs(ftr->ElementNo, trial_vdofs);
+            test_fes->GetElementVDofs(ftr->Elem1No, test_vdofs);
            trial_face_fe = trial_fes->GetFaceElement(ftr->ElementNo);
            test_fe1 = test_fes->GetFE(ftr->Elem1No);
            // The test_fe2 object is really a dummy and not used on the
@@ -1479,7 +1507,7 @@ void MixedBilinearForm::Assemble (int skip_zeros)
                                                                 *test_fe1,
                                                                 *test_fe2,
                                                                 *ftr, elemmat);
-               mat->AddSubMatrix(te_vdofs, tr_vdofs, elemmat, skip_zeros);
+               mat->AddSubMatrix(test_vdofs, trial_vdofs, elemmat, skip_zeros);
            }
         }
      }
@@ -1841,6 +1869,8 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)

   Array<int> dom_vdofs, ran_vdofs;
   ElementTransformation *T;
+   DofTransformation * dom_dof_trans;
+   DofTransformation * ran_dof_trans;
   const FiniteElement *dom_fe, *ran_fe;
   DenseMatrix totelmat, elmat;

@@ -1853,8 +1883,8 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
   {
      for (int i = 0; i < test_fes->GetNE(); i++)
      {
-         trial_fes->GetElementVDofs(i, dom_vdofs);
-         test_fes->GetElementVDofs(i, ran_vdofs);
+         dom_dof_trans = trial_fes->GetElementVDofs(i, dom_vdofs);
+         ran_dof_trans = test_fes->GetElementVDofs(i, ran_vdofs);
         T = test_fes->GetElementTransformation(i);
         dom_fe = trial_fes->GetFE(i);
         ran_fe = test_fes->GetFE(i);
@@ -1867,6 +1897,10 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
                                                     elmat);
            totelmat += elmat;
         }
+         if (ran_dof_trans || dom_dof_trans)
+         {
+            TransformPrimal(ran_dof_trans, dom_dof_trans, totelmat);
+         }
         mat->SetSubMatrix(ran_vdofs, dom_vdofs, totelmat, skip_zeros);
      }
   }
@@ -747,6 +747,7 @@ void DiffusionIntegrator::AssembleElementMatrix
 #ifdef MFEM_THREAD_SAFE
   DenseMatrix dshape(nd, dim), dshapedxt(nd, spaceDim);
   DenseMatrix dshapedxt_m(nd, MQ ? spaceDim : 0);
+   DenseMatrix M(MQ ? spaceDim : 0);
   Vector D(VQ ? VQ->GetVDim() : 0);
 #else
   dshape.SetSize(nd, dim);
@@ -125,7 +125,7 @@ void PADiffusionSetup2D<2>(const int Q1D,
               D(qx,qy,0,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
               D(qx,qy,1,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
               D(qx,qy,2,e) = w_detJ * (symmetric ? (-J21*R12 + J11*R22) :
-               (J22*R12 - J12*R22)); // 2,2 or 1,2
+                                        (J22*R12 - J12*R22)); // 2,2 or 1,2
               if (!symmetric)
               {
                  D(qx,qy,3,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
@@ -186,7 +186,7 @@ void PAHcurlMassAssembleDiagonal2D(const int D1D,
                  const double wy = (c == 1) ? Bo(qy,dy) : Bc(qy,dy);

                  mass[qx] += wy * wy * ((c == 0) ? op(qx,qy,0,e) :
-                  op(qx,qy,symmetric ? 2 : 3, e));
+                                         op(qx,qy,symmetric ? 2 : 3, e));
               }
            }

@@ -237,7 +237,7 @@ void PAHcurlMassAssembleDiagonal3D(const int D1D,
         const int D1Dx = (c == 0) ? D1D - 1 : D1D;

         const int opc = (c == 0) ? 0 : ((c == 1) ? (symmetric ? 3 : 4) :
-         (symmetric ? 5 : 8));
+                                         (symmetric ? 5 : 8));

         double mass[MAX_Q1D];

@@ -178,10 +178,10 @@ int CeedATPMGElemRestriction(int order,
                  {
                     left_in_edof = i*P1d + 0;
                     right_in_edof = i*P1d + (P1d - 1);
-                     left_in_ldof = in_elem_dof[left_in_edof*in_layout[0]
-                                                + e*in_layout[2]] + rounding_guard;
-                     right_in_ldof = in_elem_dof[right_in_edof*in_layout[0]
-                                                 + e*in_layout[2]] + rounding_guard;
+                     left_in_ldof = in_elem_dof[left_in_edof*in_layout[0]+e*in_layout[2]]
+                                    + rounding_guard;
+                     right_in_ldof = in_elem_dof[right_in_edof*in_layout[0]+e*in_layout[2]]
+                                     + rounding_guard;
                     coarse_i = coarse_1d_edof(i, P1d, coarse_P1d);
                     coarse_j = (left_in_ldof < right_in_ldof) ?
                                coarse_1d_edof(j, P1d, coarse_P1d) : reverse_coarse_1d_edof(j, P1d, coarse_P1d);
@@ -190,10 +190,10 @@ int CeedATPMGElemRestriction(int order,
                  {
                     left_in_edof = 0*P1d + j;
                     right_in_edof = (P1d - 1)*P1d + j;
-                     left_in_ldof = in_elem_dof[left_in_edof*in_layout[0]
-                                                + e*in_layout[2]] + rounding_guard;
-                     right_in_ldof = in_elem_dof[right_in_edof*in_layout[0]
-                                                 + e*in_layout[2]] + rounding_guard;
+                     left_in_ldof = in_elem_dof[left_in_edof*in_layout[0]+e*in_layout[2]]
+                                    + rounding_guard;
+                     right_in_ldof = in_elem_dof[right_in_edof*in_layout[0]+e*in_layout[2]]
+                                     + rounding_guard;
                     coarse_i = (left_in_ldof < right_in_ldof) ?
                                coarse_1d_edof(i, P1d, coarse_P1d) : reverse_coarse_1d_edof(i, P1d, coarse_P1d);
                     coarse_j = coarse_1d_edof(j, P1d, coarse_P1d);
@@ -234,8 +234,8 @@ int CeedATPMGElemRestriction(int order,
                  // Determine topology; is this edof on the outside of the element
                  // in the i, j, or k direction?
                  int in_edof = i*P1d*P1d + j*P1d + k;
-                  int in_ldof = in_elem_dof[in_edof*in_layout[0] +
-                                            e*in_layout[2]] + rounding_guard;
+                  int in_ldof = in_elem_dof[in_edof*in_layout[0]+e*in_layout[2]]
+                                + rounding_guard;
                  bool i_edge = (i == 0 || i == P1d - 1);
                  bool j_edge = (j == 0 || j == P1d - 1);
                  bool k_edge = (k == 0 || k == P1d - 1);
@@ -265,10 +265,10 @@ int CeedATPMGElemRestriction(int order,
                     {
                        left_in_edof = 0*P1d*P1d + j*P1d + k;
                        right_in_edof = (P1d - 1)*P1d*P1d + j*P1d + k;
-                        left_in_ldof = in_elem_dof[left_in_edof*in_layout[0] +
-                                                   e*in_layout[2]] + rounding_guard;
-                        right_in_ldof = in_elem_dof[right_in_edof*in_layout[0] +
-                                                    e*in_layout[2]] + rounding_guard;
+                        left_in_ldof = in_elem_dof[left_in_edof*in_layout[0]+e*in_layout[2]]
+                                       + rounding_guard;
+                        right_in_ldof = in_elem_dof[right_in_edof*in_layout[0]+e*in_layout[2]]
+                                        + rounding_guard;
                        coarse_i = (left_in_ldof < right_in_ldof) ?
                                   coarse_1d_edof(i, P1d, coarse_P1d) : reverse_coarse_1d_edof(i, P1d, coarse_P1d);
                        coarse_j = coarse_1d_edof(j, P1d, coarse_P1d);
@@ -278,10 +278,10 @@ int CeedATPMGElemRestriction(int order,
                     {
                        left_in_edof = i*P1d*P1d + 0*P1d + k;
                        right_in_edof = i*P1d*P1d + (P1d - 1)*P1d + k;
-                        left_in_ldof = in_elem_dof[left_in_edof*in_layout[0] +
-                                                   e*in_layout[2]] + rounding_guard;
-                        right_in_ldof = in_elem_dof[right_in_edof*in_layout[0] +
-                                                    e*in_layout[2]] + rounding_guard;
+                        left_in_ldof = in_elem_dof[left_in_edof*in_layout[0]+e*in_layout[2]]
+                                       + rounding_guard;
+                        right_in_ldof = in_elem_dof[right_in_edof*in_layout[0]+e*in_layout[2]]
+                                        + rounding_guard;
                        coarse_i = coarse_1d_edof(i, P1d, coarse_P1d);
                        coarse_j = (left_in_ldof < right_in_ldof) ?
                                   coarse_1d_edof(j, P1d, coarse_P1d) : reverse_coarse_1d_edof(j, P1d, coarse_P1d);
@@ -296,10 +296,10 @@ int CeedATPMGElemRestriction(int order,
                        }
                        left_in_edof = i*P1d*P1d + j*P1d + 0;
                        right_in_edof = i*P1d*P1d + j*P1d + (P1d - 1);
-                        left_in_ldof = in_elem_dof[left_in_edof*in_layout[0] +
-                                                   e*in_layout[2]] + rounding_guard;
-                        right_in_ldof = in_elem_dof[right_in_edof*in_layout[0] +
-                                                    e*in_layout[2]] + rounding_guard;
+                        left_in_ldof = in_elem_dof[left_in_edof*in_layout[0]+e*in_layout[2]]
+                                       + rounding_guard;
+                        right_in_ldof = in_elem_dof[right_in_edof*in_layout[0]+e*in_layout[2]]
+                                        + rounding_guard;
                        coarse_i = coarse_1d_edof(i, P1d, coarse_P1d);
                        coarse_j = coarse_1d_edof(j, P1d, coarse_P1d);
                        coarse_k = (left_in_ldof < right_in_ldof) ?
@@ -323,14 +323,14 @@ int CeedATPMGElemRestriction(int order,
                        bottom_right_edof = i*P1d*P1d + 0*P1d + (P1d - 1);
                        top_right_edof = i*P1d*P1d + (P1d - 1)*P1d + (P1d - 1);
                        top_left_edof = i*P1d*P1d + (P1d - 1)*P1d + 0;
-                        bottom_left_ldof = in_elem_dof[bottom_left_edof*in_layout[0] +
-                                                       e*in_layout[2]] + rounding_guard;
-                        bottom_right_ldof = in_elem_dof[bottom_right_edof*in_layout[0] +
-                                                        e*in_layout[2]] + rounding_guard;
-                        top_right_ldof = in_elem_dof[top_right_edof*in_layout[0] +
-                                                     e*in_layout[2]] + rounding_guard;
-                        top_left_ldof = in_elem_dof[top_left_edof*in_layout[0] +
-                                                    e*in_layout[2]] + rounding_guard;
+                        bottom_left_ldof = in_elem_dof[bottom_left_edof*in_layout[0]+e*in_layout[2]]
+                                           + rounding_guard;
+                        bottom_right_ldof = in_elem_dof[bottom_right_edof*in_layout[0]+e*in_layout[2]]
+                                            + rounding_guard;
+                        top_right_ldof = in_elem_dof[top_right_edof*in_layout[0]+e*in_layout[2]]
+                                         + rounding_guard;
+                        top_left_ldof = in_elem_dof[top_left_edof*in_layout[0]+e*in_layout[2]]
+                                        + rounding_guard;
                        int m = min4(bottom_left_ldof, bottom_right_ldof, top_right_ldof,
                                     top_left_ldof);
                        coarse_i = coarse_1d_edof(i, P1d, coarse_P1d);
@@ -361,14 +361,14 @@ int CeedATPMGElemRestriction(int order,
                        bottom_right_edof = 0*P1d*P1d + j*P1d + (P1d - 1);
                        top_right_edof = (P1d - 1)*P1d*P1d + j*P1d + (P1d - 1);
                        top_left_edof = (P1d - 1)*P1d*P1d + j*P1d + 0;
-                        bottom_left_ldof = in_elem_dof[bottom_left_edof*in_layout[0] +
-                                                       e*in_layout[2]] + rounding_guard;
-                        bottom_right_ldof = in_elem_dof[bottom_right_edof*in_layout[0] +
-                                                        e*in_layout[2]] + rounding_guard;
-                        top_right_ldof = in_elem_dof[top_right_edof*in_layout[0] +
-                                                     e*in_layout[2]] + rounding_guard;
-                        top_left_ldof = in_elem_dof[top_left_edof*in_layout[0] +
-                                                    e*in_layout[2]] + rounding_guard;
+                        bottom_left_ldof = in_elem_dof[bottom_left_edof*in_layout[0]+e*in_layout[2]]
+                                           + rounding_guard;
+                        bottom_right_ldof = in_elem_dof[bottom_right_edof*in_layout[0]+e*in_layout[2]]
+                                            + rounding_guard;
+                        top_right_ldof = in_elem_dof[top_right_edof*in_layout[0]+e*in_layout[2]]
+                                         + rounding_guard;
+                        top_left_ldof = in_elem_dof[top_left_edof*in_layout[0]+e*in_layout[2]]
+                                        + rounding_guard;
                        int m = min4(bottom_left_ldof, bottom_right_ldof, top_right_ldof,
                                     top_left_ldof);
                        coarse_j = coarse_1d_edof(j, P1d, coarse_P1d);
@@ -404,14 +404,14 @@ int CeedATPMGElemRestriction(int order,
                        bottom_right_edof = 0*P1d*P1d + (P1d - 1)*P1d + k;
                        top_right_edof = (P1d - 1)*P1d*P1d + (P1d - 1)*P1d + k;
                        top_left_edof = (P1d - 1)*P1d*P1d + 0*P1d + k;
-                        bottom_left_ldof = in_elem_dof[bottom_left_edof*in_layout[0] +
-                                                       e*in_layout[2]] + rounding_guard;
-                        bottom_right_ldof = in_elem_dof[bottom_right_edof*in_layout[0] +
-                                                        e*in_layout[2]] + rounding_guard;
-                        top_right_ldof = in_elem_dof[top_right_edof*in_layout[0] +
-                                                     e*in_layout[2]] + rounding_guard;
-                        top_left_ldof = in_elem_dof[top_left_edof*in_layout[0] +
-                                                    e*in_layout[2]] + rounding_guard;
+                        bottom_left_ldof = in_elem_dof[bottom_left_edof*in_layout[0]+e*in_layout[2]]
+                                           + rounding_guard;
+                        bottom_right_ldof = in_elem_dof[bottom_right_edof*in_layout[0]+e*in_layout[2]]
+                                            + rounding_guard;
+                        top_right_ldof = in_elem_dof[top_right_edof*in_layout[0]+e*in_layout[2]]
+                                         + rounding_guard;
+                        top_left_ldof = in_elem_dof[top_left_edof*in_layout[0]+e*in_layout[2]]
+                                        + rounding_guard;
                        int m = min4(bottom_left_ldof, bottom_right_ldof,
                                     top_right_ldof, top_left_ldof);
                        coarse_k = coarse_1d_edof(k, P1d, coarse_P1d);
@@ -431,6 +431,7 @@ MatrixArrayCoefficient::~MatrixArrayCoefficient ()
 void MatrixArrayCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
                                  const IntegrationPoint &ip)
 {
+   K.SetSize(height, width);
   for (int i = 0; i < height; i++)
   {
      for (int j = 0; j < width; j++)
@@ -635,6 +635,320 @@ SesquilinearForm::Update(FiniteElementSpace *nfes)
 }


+bool MixedSesquilinearForm::RealInteg()
+{
+   int nint = blfr->GetTFBFI()->Size() + blfr->GetDBFI()->Size() +
+              blfr->GetBBFI()->Size() + blfr->GetBTFBFI()->Size();
+   return (nint != 0);
+}
+
+bool MixedSesquilinearForm::ImagInteg()
+{
+   int nint = blfi->GetTFBFI()->Size() + blfi->GetDBFI()->Size() +
+              blfi->GetBBFI()->Size() + blfi->GetBTFBFI()->Size();
+   return (nint != 0);
+}
+
+MixedSesquilinearForm::MixedSesquilinearForm(FiniteElementSpace *tr_f,
+                                             FiniteElementSpace *te_f,
+                                             ComplexOperator::Convention
+                                             convention)
+   : conv(convention),
+     blfr(new MixedBilinearForm(tr_f, te_f)),
+     blfi(new MixedBilinearForm(tr_f, te_f))
+{}
+
+MixedSesquilinearForm::MixedSesquilinearForm(FiniteElementSpace *tr_f,
+                                             FiniteElementSpace *te_f,
+                                             MixedBilinearForm *bfr,
+                                             MixedBilinearForm *bfi,
+                                             ComplexOperator::Convention
+                                             convention)
+   : conv(convention),
+     blfr(new MixedBilinearForm(tr_f, te_f, bfr)),
+     blfi(new MixedBilinearForm(tr_f, te_f, bfi))
+{}
+
+MixedSesquilinearForm::~MixedSesquilinearForm()
+{
+   delete blfr;
+   delete blfi;
+}
+
+void
+MixedSesquilinearForm::AddDomainIntegrator(BilinearFormIntegrator *bfi_real,
+                                           BilinearFormIntegrator *bfi_imag)
+{
+   if (bfi_real) { blfr->AddDomainIntegrator(bfi_real); }
+   if (bfi_imag) { blfi->AddDomainIntegrator(bfi_imag); }
+}
+
+void
+MixedSesquilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator *bfi_real,
+                                             BilinearFormIntegrator *bfi_imag)
+{
+   if (bfi_real) { blfr->AddBoundaryIntegrator(bfi_real); }
+   if (bfi_imag) { blfi->AddBoundaryIntegrator(bfi_imag); }
+}
+
+void
+MixedSesquilinearForm::AddBoundaryIntegrator(BilinearFormIntegrator *bfi_real,
+                                             BilinearFormIntegrator *bfi_imag,
+                                             Array<int> & bdr_marker)
+{
+   if (bfi_real) { blfr->AddBoundaryIntegrator(bfi_real, bdr_marker); }
+   if (bfi_imag) { blfi->AddBoundaryIntegrator(bfi_imag, bdr_marker); }
+}
+
+void
+MixedSesquilinearForm::AddTraceFaceIntegrator(BilinearFormIntegrator
+                                              *bfi_real,
+                                              BilinearFormIntegrator
+                                              *bfi_imag)
+{
+   if (bfi_real) { blfr->AddTraceFaceIntegrator(bfi_real); }
+   if (bfi_imag) { blfi->AddTraceFaceIntegrator(bfi_imag); }
+}
+
+void
+MixedSesquilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator
+                                                 *bfi_real,
+                                                 BilinearFormIntegrator
+                                                 *bfi_imag)
+{
+   if (bfi_real) { blfr->AddBdrTraceFaceIntegrator(bfi_real); }
+   if (bfi_imag) { blfi->AddBdrTraceFaceIntegrator(bfi_imag); }
+}
+
+void
+MixedSesquilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator
+                                                 *bfi_real,
+                                                 BilinearFormIntegrator
+                                                 *bfi_imag,
+                                                 Array<int> &bdr_marker)
+{
+   if (bfi_real) { blfr->AddBdrTraceFaceIntegrator(bfi_real, bdr_marker); }
+   if (bfi_imag) { blfi->AddBdrTraceFaceIntegrator(bfi_imag, bdr_marker); }
+}
+
+void
+MixedSesquilinearForm::Assemble(int skip_zeros)
+{
+   blfr->Assemble(skip_zeros);
+   blfi->Assemble(skip_zeros);
+}
+
+void
+MixedSesquilinearForm::Finalize(int skip_zeros)
+{
+   blfr->Finalize(skip_zeros);
+   blfi->Finalize(skip_zeros);
+}
+
+ComplexSparseMatrix *
+MixedSesquilinearForm::AssembleComplexSparseMatrix()
+{
+   return new ComplexSparseMatrix(&blfr->SpMat(),
+                                  &blfi->SpMat(),
+                                  false, false, conv);
+}
+
+void
+MixedSesquilinearForm::FormRectangularLinearSystem(const Array<int>
+                                                   &trial_tdof_list,
+                                                   const Array<int>
+                                                   &test_tdof_list,
+                                                   Vector &x, Vector &b,
+                                                   OperatorHandle &A,
+                                                   Vector &X, Vector &B)
+{
+   FiniteElementSpace *tr_fes = blfr->TrialFESpace();
+   FiniteElementSpace *te_fes = blfr->TestFESpace();
+   const int tr_vsize = tr_fes->GetVSize();
+   const int te_vsize = te_fes->GetVSize();
+
+   // Allocate temporary vector
+   Vector b_0;
+   b_0.UseDevice(true);
+   b_0.SetSize(tr_vsize);
+   b_0 = 0.0;
+
+   // Extract the real and imaginary parts of the input vectors
+   MFEM_ASSERT(x.Size() == 2 * te_vsize,
+               "Input GridFunction of incorrect size!");
+   x.Read();
+   Vector x_r; x_r.MakeRef(x, 0, te_vsize);
+   Vector x_i; x_i.MakeRef(x, te_vsize, te_vsize);
+
+   MFEM_ASSERT(b.Size() == 2 * tr_vsize, "Input LinearForm of incorrect size!");
+   b.Read();
+   Vector b_r; b_r.MakeRef(b, 0, tr_vsize);
+   Vector b_i; b_i.MakeRef(b, tr_vsize, tr_vsize);
+
+   if (conv == ComplexOperator::BLOCK_SYMMETRIC) { b_i *= -1.0; }
+
+   const int tr_tvsize = tr_fes->GetTrueVSize();
+   const int te_tvsize = te_fes->GetTrueVSize();
+   OperatorHandle A_r, A_i;
+
+   X.UseDevice(true);
+   X.SetSize(2 * te_tvsize);
+   X = 0.0;
+
+   B.UseDevice(true);
+   B.SetSize(2 * tr_tvsize);
+   B = 0.0;
+
+   Vector X_r; X_r.MakeRef(X, 0, te_tvsize);
+   Vector X_i; X_i.MakeRef(X, te_tvsize, te_tvsize);
+   Vector B_r; B_r.MakeRef(B, 0, tr_tvsize);
+   Vector B_i; B_i.MakeRef(B, tr_tvsize, tr_tvsize);
+
+   Vector X_0, B_0;
+
+   if (RealInteg())
+   {
+      b_0 = b_r;
+      blfr->FormRectangularLinearSystem(trial_tdof_list, test_tdof_list,
+                                        x_r, b_0, A_r, X_0, B_0);
+      X_r = X_0; B_r = B_0;
+
+      b_0 = b_i;
+      blfr->FormRectangularLinearSystem(trial_tdof_list, test_tdof_list,
+                                        x_i, b_0, A_r, X_0, B_0);
+      X_i = X_0; B_i = B_0;
+
+      if (ImagInteg())
+      {
+         b_0 = 0.0;
+         blfi->FormRectangularLinearSystem(trial_tdof_list, test_tdof_list,
+                                           x_i, b_0, A_i, X_0, B_0);
+         B_r -= B_0;
+
+         b_0 = 0.0;
+         blfi->FormRectangularLinearSystem(trial_tdof_list, test_tdof_list,
+                                           x_r, b_0, A_i, X_0, B_0);
+         B_i += B_0;
+      }
+   }
+   else if (ImagInteg())
+   {
+      b_0 = b_i;
+      blfi->FormRectangularLinearSystem(trial_tdof_list, test_tdof_list,
+                                        x_r, b_0, A_i, X_0, B_0);
+      X_r = X_0; B_i = B_0;
+
+      b_0 = b_r; b_0 *= -1.0;
+      blfi->FormRectangularLinearSystem(trial_tdof_list, test_tdof_list,
+                                        x_i, b_0, A_i, X_0, B_0);
+      X_i = X_0; B_r = B_0; B_r *= -1.0;
+   }
+   else
+   {
+      MFEM_ABORT("Real and Imaginary part of the Sesquilinear form are empty");
+   }
+
+   if (conv == ComplexOperator::BLOCK_SYMMETRIC)
+   {
+      B_i *= -1.0;
+      b_i *= -1.0;
+   }
+
+   x_r.SyncAliasMemory(x);
+   x_i.SyncAliasMemory(x);
+   b_r.SyncAliasMemory(b);
+   b_i.SyncAliasMemory(b);
+
+   X_r.SyncAliasMemory(X);
+   X_i.SyncAliasMemory(X);
+   B_r.SyncAliasMemory(B);
+   B_i.SyncAliasMemory(B);
+
+   // A = A_r + i A_i
+   A.Clear();
+   if ( A_r.Type() == Operator::MFEM_SPARSEMAT ||
+        A_i.Type() == Operator::MFEM_SPARSEMAT )
+   {
+      ComplexSparseMatrix * A_sp =
+         new ComplexSparseMatrix(A_r.As<SparseMatrix>(),
+                                 A_i.As<SparseMatrix>(),
+                                 A_r.OwnsOperator(),
+                                 A_i.OwnsOperator(),
+                                 conv);
+      A.Reset<ComplexSparseMatrix>(A_sp, true);
+   }
+   else
+   {
+      ComplexOperator * A_op =
+         new ComplexOperator(A_r.Ptr(),
+                             A_i.Ptr(),
+                             A_r.OwnsOperator(),
+                             A_i.OwnsOperator(),
+                             conv);
+      A.Reset<ComplexOperator>(A_op, true);
+   }
+   A_r.SetOperatorOwner(false);
+   A_i.SetOperatorOwner(false);
+}
+
+void
+MixedSesquilinearForm::FormRectangularSystemMatrix(const Array<int>
+                                                   &trial_tdof_list,
+                                                   const Array<int>
+                                                   &test_tdof_list,
+                                                   OperatorHandle &A)
+
+{
+   OperatorHandle A_r, A_i;
+   if (RealInteg())
+   {
+      blfr->FormRectangularSystemMatrix(trial_tdof_list, test_tdof_list, A_r);
+   }
+   if (ImagInteg())
+   {
+      blfi->FormRectangularSystemMatrix(trial_tdof_list, test_tdof_list, A_i);
+   }
+   if (!RealInteg() && !ImagInteg())
+   {
+      MFEM_ABORT("Both Real and Imaginary part of the mixed Sesquilinear form "
+                 "are empty");
+   }
+
+   // A = A_r + i A_i
+   A.Clear();
+   if ( A_r.Type() == Operator::MFEM_SPARSEMAT ||
+        A_i.Type() == Operator::MFEM_SPARSEMAT )
+   {
+      ComplexSparseMatrix * A_sp =
+         new ComplexSparseMatrix(A_r.As<SparseMatrix>(),
+                                 A_i.As<SparseMatrix>(),
+                                 A_r.OwnsOperator(),
+                                 A_i.OwnsOperator(),
+                                 conv);
+      A.Reset<ComplexSparseMatrix>(A_sp, true);
+   }
+   else
+   {
+      ComplexOperator * A_op =
+         new ComplexOperator(A_r.Ptr(),
+                             A_i.Ptr(),
+                             A_r.OwnsOperator(),
+                             A_i.OwnsOperator(),
+                             conv);
+      A.Reset<ComplexOperator>(A_op, true);
+   }
+   A_r.SetOperatorOwner(false);
+   A_i.SetOperatorOwner(false);
+}
+
+void
+MixedSesquilinearForm::Update()
+{
+   if ( blfr ) { blfr->Update(); }
+   if ( blfi ) { blfi->Update(); }
+}
+
+
 #ifdef MFEM_USE_MPI

 ParComplexGridFunction::ParComplexGridFunction(ParFiniteElementSpace *pfes)
@@ -324,6 +324,150 @@ public:
   virtual ~SesquilinearForm();
 };

+/** Class for mixed sesquilinear form
+
+    A mixed sesquilinear form is a generalization of a mixed bilinear form to
+    complex-valued fields. Sesquilinear forms are linear in the second argument
+    but the first argument involves a complex conjugate in the sense that:
+
+                a(alpha u, beta v) = conj(alpha) beta a(u, v)
+
+    The 'Mixed' keyword indicates that the domain and range spaces may differ
+    leading to rectangular operators.
+
+    The @a convention argument in the class's constructor is documented in the
+    mfem::ComplexOperator class found in linalg/complex_operator.hpp.
+
+    When supplying integrators to the MixedSesquilinearForm either the real or
+    imaginary integrator can be NULL. This indicates that the corresponding
+    portion of the complex-valued material coefficient is equal to zero.
+*/
+class MixedSesquilinearForm
+{
+private:
+   ComplexOperator::Convention conv;
+
+   MixedBilinearForm *blfr;
+   MixedBilinearForm *blfi;
+
+   /* These methods check if the real/imag parts of the sesquilinear form are
+      not empty */
+   bool RealInteg();
+   bool ImagInteg();
+
+public:
+   MixedSesquilinearForm(FiniteElementSpace *tr_fes,
+                         FiniteElementSpace *te_fes,
+                         ComplexOperator::Convention
+                         convention = ComplexOperator::HERMITIAN);
+   /** @brief Create a MixedSesquilinearForm given a trial space,
+       FiniteElementSpace @a tr_fes, and a test space,
+       FiniteElementSpace @a te_fes, using the same integrators as the
+       MixedBilinearForms @a bfr and @a bfi .
+
+       The pointers @a tr_fes and @a te_fes are not owned by the newly
+       constructed object.
+
+       The integrators are copied as pointers and they are not owned by the
+       newly constructed MixedSesquilinearForm. */
+   MixedSesquilinearForm(FiniteElementSpace *tr_fes,
+                         FiniteElementSpace *te_fes,
+                         MixedBilinearForm *bfr, MixedBilinearForm *bfi,
+                         ComplexOperator::Convention
+                         convention = ComplexOperator::HERMITIAN);
+
+   ComplexOperator::Convention GetConvention() const { return conv; }
+   void SetConvention(const ComplexOperator::Convention &
+                      convention) { conv = convention; }
+
+   /// Set the desired assembly level.
+   /** Valid choices are:
+
+       - AssemblyLevel::LEGACY (default)
+       - AssemblyLevel::FULL
+       - AssemblyLevel::PARTIAL
+       - AssemblyLevel::ELEMENT
+       - AssemblyLevel::NONE
+
+       This method must be called before assembly. */
+   void SetAssemblyLevel(AssemblyLevel assembly_level)
+   {
+      blfr->SetAssemblyLevel(assembly_level);
+      blfi->SetAssemblyLevel(assembly_level);
+   }
+
+   MixedBilinearForm & real() { return *blfr; }
+   MixedBilinearForm & imag() { return *blfi; }
+   const MixedBilinearForm & real() const { return *blfr; }
+   const MixedBilinearForm & imag() const { return *blfi; }
+
+   /// Adds new Domain Integrator.
+   void AddDomainIntegrator(BilinearFormIntegrator *bfi_real,
+                            BilinearFormIntegrator *bfi_imag);
+
+   /// Adds new Boundary Integrator.
+   void AddBoundaryIntegrator(BilinearFormIntegrator *bfi_real,
+                              BilinearFormIntegrator *bfi_imag);
+
+   /// Adds new Boundary Integrator, restricted to specific boundary attributes.
+   void AddBoundaryIntegrator(BilinearFormIntegrator *bfi_real,
+                              BilinearFormIntegrator *bfi_imag,
+                              Array<int> &bdr_marker);
+
+   /// Adds new interior Face Integrator. Assumes ownership of @a bfi.
+   void AddTraceFaceIntegrator(BilinearFormIntegrator *bfi_real,
+                               BilinearFormIntegrator *bfi_imag);
+
+   /// Adds new boundary Face Integrator. Assumes ownership of @a bfi.
+   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator *bfi_real,
+                                  BilinearFormIntegrator *bfi_imag);
+
+   /** @brief Adds new boundary Face Integrator, restricted to specific boundary
+       attributes.
+
+       Assumes ownership of @a bfi.
+
+       The array @a bdr_marker is stored internally as a pointer to the given
+       Array<int> object. */
+   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator *bfi_real,
+                                  BilinearFormIntegrator *bfi_imag,
+                                  Array<int> &bdr_marker);
+
+   /// Assemble the local matrix
+   void Assemble(int skip_zeros = 1);
+
+   /// Finalizes the matrix initialization.
+   void Finalize(int skip_zeros = 1);
+
+   /// Returns the matrix assembled on the true dofs, i.e. P^t A P.
+   /** The returned matrix has to be deleted by the caller. */
+   ComplexSparseMatrix *AssembleComplexSparseMatrix();
+
+   /// Return the trial FE space associated with the BilinearForm.
+   FiniteElementSpace *TrialFESpace() { return blfr->TrialFESpace(); }
+   /// Read-only access to the associated trial FiniteElementSpace.
+   const FiniteElementSpace *TrialFESpace() const
+   { return blfr->TrialFESpace(); }
+
+   /// Return the test FE space associated with the BilinearForm.
+   FiniteElementSpace *TestFESpace() { return blfr->TestFESpace(); }
+   /// Read-only access to the associated test FiniteElementSpace.
+   const FiniteElementSpace *TestFESpace() const { return blfr->TestFESpace(); }
+
+   void FormRectangularLinearSystem(const Array<int> &trial_tdof_list,
+                                    const Array<int> &test_tdof_list,
+                                    Vector &x, Vector &b,
+                                    OperatorHandle &A, Vector &X, Vector &B);
+
+   void FormRectangularSystemMatrix(const Array<int> &trial_tdof_list,
+                                    const Array<int> &test_tdof_list,
+                                    OperatorHandle &A);
+
+   virtual void Update();
+
+   virtual ~MixedSesquilinearForm();
+};
+
 #ifdef MFEM_USE_MPI

 /// Class for parallel complex-valued grid function - real + imaginary part
@@ -482,7 +482,7 @@ void VisItDataCollection::SaveRootFile()
   std::string root_name = prefix_path + name + "_" +
                           to_padded_string(cycle, pad_digits_cycle) +
                           ".mfem_root";
-   std::ofstream root_file(root_name.c_str());
+   std::ofstream root_file(root_name);
   root_file << GetVisItRootString();
   if (!root_file)
   {
@@ -548,7 +548,7 @@ void VisItDataCollection::Load(int cycle_)

 void VisItDataCollection::LoadVisItRootFile(const std::string& root_name)
 {
-   std::ifstream root_file(root_name.c_str());
+   std::ifstream root_file(root_name);
   std::stringstream buffer;
   buffer << root_file.rdbuf();
   if (!buffer)
@@ -853,6 +853,7 @@ void ParaViewDataCollection::Save()
      std::string dpath=GenerateCollectionPath();
      std::string pvdname=dpath+"/"+GeneratePVDFileName();

+      bool write_header = true;
      std::ifstream pvd_in;
      if (restart_mode && (pvd_in.open(pvdname,std::ios::binary),pvd_in.good()))
      {
@@ -879,20 +880,34 @@ void ParaViewDataCollection::Save()
               pos_end = pvd_in.tellg();
            }
         }
+         // Since pvd_in is opened in binary mode, count will store the number
+         // of bytes from the beginning of the file until the desired insertion
+         // point (in text mode on Windows this is not the case).
         size_t count = pos_end - pos_begin;
-         std::vector<char> buf(count);
-         pvd_in.clear();
-         pvd_in.seekg(pos_begin);
-         pvd_in.read(buf.data(), count);
-         pvd_in.close();
-         pvd_stream.open(pvdname.c_str(),std::ios::out);
-         pvd_stream.write(buf.data(), count);
+         if (count != 0)
+         {
+            write_header = false;
+            std::vector<char> buf(count);
+            // Read the contents of the PVD file, from the beginning to the
+            // insertion point.
+            pvd_in.clear();
+            pvd_in.seekg(pos_begin);
+            pvd_in.read(buf.data(), count);
+            pvd_in.close();
+            // Open the PVD file in truncate mode to delete the previous
+            // contents. Open in binary mode to write the data buffer without
+            // converting \r\n to \r\r\n on Windows.
+            pvd_stream.open(pvdname,std::ios::out|std::ios::trunc|std::ios::binary);
+            pvd_stream.write(buf.data(), count);
+            // Close and reopen the file in text mode, appending to the end.
+            pvd_stream.close();
+            pvd_stream.open(pvdname,std::ios::in|std::ios::out|std::ios::ate);
+         }
      }
-      else
+      if (write_header)
      {
-         // initialize new pvd file
-         pvd_stream.open(pvdname.c_str(),std::ios::out);
-         // initialize the file
+         // Initialize new pvd file.
+         pvd_stream.open(pvdname,std::ios::out|std::ios::trunc);
         pvd_stream << "<?xml version=\"1.0\"?>\n";
         pvd_stream << "<VTKFile type=\"Collection\" version=\"0.1\"";
         pvd_stream << " byte_order=\"" << VTKByteOrder() << "\">\n";
@@ -904,7 +919,7 @@ void ParaViewDataCollection::Save()
   {
      std::string fname = GenerateCollectionPath()+"/"+GenerateVTUPath()+"/"
                          +GenerateVTUFileName();
-      std::fstream out(fname.c_str(), std::ios::out);
+      std::fstream out(fname, std::ios::out);
      out.precision(precision);
      SaveDataVTU(out,levels_of_detail);
      out.close();
@@ -915,7 +930,7 @@ void ParaViewDataCollection::Save()
   {
      std::string fname = GenerateCollectionPath()+"/"+GeneratePVTUPath()+"/"
                          +GeneratePVTUFileName();
-      std::fstream out(fname.c_str(), std::ios::out);
+      std::fstream out(fname, std::ios::out);

      out << "<?xml version=\"1.0\"?>\n";
      out << "<VTKFile type=\"PUnstructuredGrid\"";
@@ -973,6 +988,7 @@ void ParaViewDataCollection::Save()
      pvd_stream << "<DataSet timestep=\"" << GetTime();  // GetCycle();
      pvd_stream << "\" group=\"\" part=\"" << 0 << "\" file=\"";
      pvd_stream << fname << "\"/>\n";
+      pvd_stream.flush();
      std::fstream::pos_type pos = pvd_stream.tellp();
      pvd_stream << "</Collection>\n";
      pvd_stream << "</VTKFile>" << std::endl;
@@ -0,0 +1,358 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "fem.hpp"
+
+namespace mfem
+{
+
+void DofTransformation::TransformPrimal(Vector &v) const
+{
+   TransformPrimal(v.GetData());
+}
+
+void DofTransformation::TransformPrimalCols(DenseMatrix &V) const
+{
+   for (int c=0; c<V.Width(); c++)
+   {
+      TransformPrimal(V.GetColumn(c));
+   }
+}
+
+void DofTransformation::TransformDual(Vector &v) const
+{
+   TransformDual(v.GetData());
+}
+
+void DofTransformation::TransformDual(DenseMatrix &V) const
+{
+   TransformDualCols(V);
+   TransformDualRows(V);
+}
+
+void DofTransformation::TransformDualRows(DenseMatrix &V) const
+{
+   Vector row;
+   for (int r=0; r<V.Height(); r++)
+   {
+      V.GetRow(r, row);
+      TransformDual(row);
+      V.SetRow(r, row);
+   }
+}
+
+void DofTransformation::TransformDualCols(DenseMatrix &V) const
+{
+   for (int c=0; c<V.Width(); c++)
+   {
+      TransformDual(V.GetColumn(c));
+   }
+}
+
+void DofTransformation::InvTransformPrimal(Vector &v) const
+{
+   InvTransformPrimal(v.GetData());
+}
+
+void TransformPrimal(const DofTransformation *ran_dof_trans,
+                     const DofTransformation *dom_dof_trans,
+                     DenseMatrix &elmat)
+{
+   if (ran_dof_trans && dom_dof_trans)
+   {
+      ran_dof_trans->TransformPrimalCols(elmat);
+      dom_dof_trans->TransformDualRows(elmat);
+   }
+   else if (ran_dof_trans)
+   {
+      ran_dof_trans->TransformPrimalCols(elmat);
+   }
+   else if (dom_dof_trans)
+   {
+      dom_dof_trans->TransformDualRows(elmat);
+   }
+   else
+   {
+      // If both transformations are NULL this function should not be called
+   }
+}
+
+void TransformDual(const DofTransformation *ran_dof_trans,
+                   const DofTransformation *dom_dof_trans,
+                   DenseMatrix &elmat)
+{
+   if (ran_dof_trans && dom_dof_trans)
+   {
+      ran_dof_trans->TransformDualCols(elmat);
+      dom_dof_trans->TransformDualRows(elmat);
+   }
+   else if (ran_dof_trans)
+   {
+      ran_dof_trans->TransformDualCols(elmat);
+   }
+   else if (dom_dof_trans)
+   {
+      dom_dof_trans->TransformDualRows(elmat);
+   }
+   else
+   {
+      // If both transformations are NULL this function should not be called
+   }
+}
+
+void VDofTransformation::TransformPrimal(double *v) const
+{
+   int size = doftrans_->Size();
+
+   if ((Ordering::Type)ordering_ == Ordering::byNODES || vdim_ == 1)
+   {
+      for (int i=0; i<vdim_; i++)
+      {
+         doftrans_->TransformPrimal(&v[i*size]);
+      }
+   }
+   else
+   {
+      Vector vec(size);
+      for (int i=0; i<vdim_; i++)
+      {
+         for (int j=0; j<size; j++)
+         {
+            vec(j) = v[j*vdim_+i];
+         }
+         doftrans_->TransformPrimal(vec);
+         for (int j=0; j<size; j++)
+         {
+            v[j*vdim_+i] = vec(j);
+         }
+      }
+   }
+}
+
+void VDofTransformation::InvTransformPrimal(double *v) const
+{
+   int size = doftrans_->Height();
+
+   if ((Ordering::Type)ordering_ == Ordering::byNODES)
+   {
+      for (int i=0; i<vdim_; i++)
+      {
+         doftrans_->InvTransformPrimal(&v[i*size]);
+      }
+   }
+   else
+   {
+      Vector vec(size);
+      for (int i=0; i<vdim_; i++)
+      {
+         for (int j=0; j<size; j++)
+         {
+            vec(j) = v[j*vdim_+i];
+         }
+         doftrans_->InvTransformPrimal(vec);
+         for (int j=0; j<size; j++)
+         {
+            v[j*vdim_+i] = vec(j);
+         }
+      }
+   }
+}
+
+void VDofTransformation::TransformDual(double *v) const
+{
+   int size = doftrans_->Size();
+
+   if ((Ordering::Type)ordering_ == Ordering::byNODES)
+   {
+      for (int i=0; i<vdim_; i++)
+      {
+         doftrans_->TransformDual(&v[i*size]);
+      }
+   }
+   else
+   {
+      Vector vec(size);
+      for (int i=0; i<vdim_; i++)
+      {
+         for (int j=0; j<size; j++)
+         {
+            vec(j) = v[j*vdim_+i];
+         }
+         doftrans_->TransformDual(vec);
+         for (int j=0; j<size; j++)
+         {
+            v[j*vdim_+i] = vec(j);
+         }
+      }
+   }
+}
+
+const double ND_DofTransformation::T_data[24] =
+{
+   1.0,  0.0,  0.0,  1.0,
+   -1.0, -1.0,  0.0,  1.0,
+   0.0,  1.0, -1.0, -1.0,
+   1.0,  0.0, -1.0, -1.0,
+   -1.0, -1.0,  1.0,  0.0,
+   0.0,  1.0,  1.0,  0.0
+};
+
+const DenseTensor ND_DofTransformation
+::T(const_cast<double*>(ND_DofTransformation::T_data), 2, 2, 6);
+
+const double ND_DofTransformation::TInv_data[24] =
+{
+   1.0,  0.0,  0.0,  1.0,
+   -1.0, -1.0,  0.0,  1.0,
+   -1.0, -1.0,  1.0,  0.0,
+   1.0,  0.0, -1.0, -1.0,
+   0.0,  1.0, -1.0, -1.0,
+   0.0,  1.0,  1.0,  0.0
+};
+
+const DenseTensor ND_DofTransformation
+::TInv(const_cast<double*>(TInv_data), 2, 2, 6);
+
+ND_DofTransformation::ND_DofTransformation(int size, int p)
+   : DofTransformation(size),
+     order(p)
+{
+}
+
+ND_TriDofTransformation::ND_TriDofTransformation(int p)
+   : ND_DofTransformation(p*(p + 2), p)
+{
+}
+
+void ND_TriDofTransformation::TransformPrimal(double *v) const
+{
+   int nedofs = order; // number of DoFs per edge
+   int nfdofs = order*(order-1); // number of DoFs per face
+
+   double data[2];
+   Vector v2(data, 2);
+
+   // Transform face DoFs
+   for (int f=0; f<1; f++)
+   {
+      for (int i=0; i<nfdofs/2; i++)
+      {
+         v2 = &v[3*nedofs + f*nfdofs + 2*i];
+         T(Fo[f]).Mult(v2, &v[3*nedofs + f*nfdofs + 2*i]);
+      }
+   }
+}
+
+void
+ND_TriDofTransformation::InvTransformPrimal(double *v) const
+{
+   int nedofs = order; // number of DoFs per edge
+   int nfdofs = order*(order-1); // number of DoFs per face
+
+   double data[2];
+   Vector v2(data, 2);
+
+   // Transform face DoFs
+   for (int f=0; f<1; f++)
+   {
+      for (int i=0; i<nfdofs/2; i++)
+      {
+         v2 = &v[3*nedofs + f*nfdofs + 2*i];
+         TInv(Fo[f]).Mult(v2, &v[3*nedofs + f*nfdofs + 2*i]);
+      }
+   }
+}
+
+void
+ND_TriDofTransformation::TransformDual(double *v) const
+{
+   int nedofs = order; // number of DoFs per edge
+   int nfdofs = order*(order-1); // number of DoFs per face
+
+   double data[2];
+   Vector v2(data, 2);
+
+   // Transform face DoFs
+   for (int f=0; f<1; f++)
+   {
+      for (int i=0; i<nfdofs/2; i++)
+      {
+         v2 = &v[3*nedofs + f*nfdofs + 2*i];
+         TInv(Fo[f]).MultTranspose(v2, &v[3*nedofs + f*nfdofs + 2*i]);
+      }
+   }
+}
+
+ND_TetDofTransformation::ND_TetDofTransformation(int p)
+   : ND_DofTransformation(p*(p + 2)*(p + 3)/2, p)
+{
+}
+
+void ND_TetDofTransformation::TransformPrimal(double *v) const
+{
+   int nedofs = order; // number of DoFs per edge
+   int nfdofs = order*(order-1); // number of DoFs per face
+
+   double data[2];
+   Vector v2(data, 2);
+
+   // Transform face DoFs
+   for (int f=0; f<4; f++)
+   {
+      for (int i=0; i<nfdofs/2; i++)
+      {
+         v2 = &v[6*nedofs + f*nfdofs + 2*i];
+         T(Fo[f]).Mult(v2, &v[6*nedofs + f*nfdofs + 2*i]);
+      }
+   }
+}
+
+void
+ND_TetDofTransformation::InvTransformPrimal(double *v) const
+{
+   int nedofs = order; // number of DoFs per edge
+   int nfdofs = order*(order-1); // number of DoFs per face
+
+   double data[2];
+   Vector v2(data, 2);
+
+   // Transform face DoFs
+   for (int f=0; f<4; f++)
+   {
+      for (int i=0; i<nfdofs/2; i++)
+      {
+         v2 = &v[6*nedofs + f*nfdofs + 2*i];
+         TInv(Fo[f]).Mult(v2, &v[6*nedofs + f*nfdofs + 2*i]);
+      }
+   }
+}
+
+void
+ND_TetDofTransformation::TransformDual(double *v) const
+{
+   int nedofs = order; // number of DoFs per edge
+   int nfdofs = order*(order-1); // number of DoFs per face
+
+   double data[2];
+   Vector v2(data, 2);
+
+   // Transform face DoFs
+   for (int f=0; f<4; f++)
+   {
+      for (int i=0; i<nfdofs/2; i++)
+      {
+         v2 = &v[6*nedofs + f*nfdofs + 2*i];
+         TInv(Fo[f]).MultTranspose(v2, &v[6*nedofs + f*nfdofs + 2*i]);
+      }
+   }
+}
+
+} // namespace mfem
@@ -0,0 +1,277 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_DOFTRANSFORM
+#define MFEM_DOFTRANSFORM
+
+#include "../config/config.hpp"
+#include "../linalg/linalg.hpp"
+#include "intrules.hpp"
+#include "fe.hpp"
+
+namespace mfem
+{
+
+/** The DofTransformation class is an abstract base class for a family of
+    transformations that map local degrees of freedom (DoFs), contained within
+    individual elements, to global degrees of freedom, stored within
+    GridFunction objects. These transformations are necessary to ensure that
+    basis functions in neighboring elements align correctly. Closely related but
+    complementary transformations are required for the entries stored in
+    LinearForm and BilinearForm objects. The DofTransformation class is designed
+    to apply the action of both of these types of DoF transformations.
+
+    Let the "primal transformation" be given by the operator T. This means that
+    given a local element vector v the data that must be placed into a
+    GridFunction object is v_t = T * v.
+
+    We also need the inverse of the primal transformation T^{-1} so that we can
+    recover the local element vector from data read out of a GridFunction
+    e.g. v = T^{-1} * v_t.
+
+    We need to preserve the action of our linear forms applied to primal
+    vectors. In other words, if f is the local vector computed by a linear
+    form then f * v = f_t * v_t (where "*" represents an inner product of
+    vectors). This requires that f_t = T^{-T} * f i.e. the "dual transform" is
+    given by the transpose of the inverse of the primal transformation.
+
+    For bilinear forms we require that v^T * A * v = v_t^T * A_t * v_t. This
+    implies that A_t = T^{-T} * A * T^{-1}. This can be accomplished by
+    performing dual transformations of the rows and columns of the matrix A.
+
+    For discrete linear operators the range must be modified with the primal
+    transformation rather than the dual transformation because the result is a
+    primal vector rather than a dual vector. This leads to the transformation
+    D_t = T * D * T^{-1}. This can be accomplished by using a primal
+    transformation on the columns of D and a dual transformation on its rows.
+*/
+class DofTransformation
+{
+protected:
+   int size_;
+
+   Array<int> Fo;
+
+   DofTransformation(int size)
+      : size_(size) {}
+
+public:
+
+   inline int Size() const { return size_; }
+   inline int Height() const { return size_; }
+   inline int NumRows() const { return size_; }
+   inline int Width() const { return size_; }
+   inline int NumCols() const { return size_; }
+
+   /** @brief Configure the transformation using face orientations for the
+       current element. */
+   /// The face_orientation array can be obtained from Mesh::GetElementFaces.
+   inline void SetFaceOrientations(const Array<int> & face_orientation)
+   { Fo = face_orientation; }
+
+   inline const Array<int> & GetFaceOrientations() const { return Fo; }
+
+   /** Transform local DoFs to align with the global DoFs. For example, this
+       transformation can be used to map the local vector computed by
+       FiniteElement::Project() to the transformed vector stored within a
+       GridFunction object. */
+   virtual void TransformPrimal(double *v) const = 0;
+   virtual void TransformPrimal(Vector &v) const;
+
+   /// Transform groups of DoFs stored as dense matrices
+   virtual void TransformPrimalCols(DenseMatrix &V) const;
+
+   /** Inverse transform local DoFs. Used to transform DoFs from a global vector
+       back to their element-local form. For example, this must be used to
+       transform the vector obtained using GridFunction::GetSubVector before it
+       can be used to compute a local interpolation.
+   */
+   virtual void InvTransformPrimal(double *v) const = 0;
+   virtual void InvTransformPrimal(Vector &v) const;
+
+   /** Transform dual DoFs as computed by a LinearFormIntegrator before summing
+       into a LinearForm object. */
+   virtual void TransformDual(double *v) const = 0;
+   virtual void TransformDual(Vector &v) const;
+
+   /** Transform a matrix of dual DoFs entries as computed by a
+       BilinearFormIntegrator before summing into a BilinearForm object. */
+   virtual void TransformDual(DenseMatrix &V) const;
+
+   /// Transform groups of dual DoFs stored as dense matrices
+   virtual void TransformDualRows(DenseMatrix &V) const;
+   virtual void TransformDualCols(DenseMatrix &V) const;
+
+   virtual ~DofTransformation() {}
+};
+
+/** Transform a matrix of DoFs entries from different finite element spaces as
+    computed by a DiscreteInterpolator before copying into a
+    DiscreteLinearOperator.
+*/
+void TransformPrimal(const DofTransformation *ran_dof_trans,
+                     const DofTransformation *dom_dof_trans,
+                     DenseMatrix &elmat);
+
+/** Transform a matrix of dual DoFs entries from different finite element spaces
+    as computed by a BilinearFormIntegrator before summing into a
+    MixedBilinearForm object.
+*/
+void TransformDual(const DofTransformation *ran_dof_trans,
+                   const DofTransformation *dom_dof_trans,
+                   DenseMatrix &elmat);
+
+/** The VDofTransformation class implements a nested transformation where an
+    arbitrary DofTransformation is replicated with a vdim >= 1.
+*/
+class VDofTransformation : public DofTransformation
+{
+private:
+   int vdim_;
+   int ordering_;
+   DofTransformation * doftrans_;
+
+public:
+   /** @brief Default constructor which requires that SetDofTransformation be
+       called before use. */
+   VDofTransformation(int vdim = 1, int ordering = 0)
+      : DofTransformation(0),
+        vdim_(vdim), ordering_(ordering),
+        doftrans_(NULL) {}
+
+   /// Constructor with a known DofTransformation
+   VDofTransformation(DofTransformation & doftrans, int vdim = 1,
+                      int ordering = 0)
+      : DofTransformation(vdim * doftrans.Size()),
+        vdim_(vdim), ordering_(ordering),
+        doftrans_(&doftrans) {}
+
+   /// Set or change the vdim parameter
+   inline void SetVDim(int vdim)
+   {
+      vdim_ = vdim;
+      if (doftrans_)
+      {
+         size_ = vdim_ * doftrans_->Size();
+      }
+   }
+
+   /// Return the current vdim value
+   inline int GetVDim() const { return vdim_; }
+
+   /// Set or change the nested DofTransformation object
+   inline void SetDofTransformation(DofTransformation & doftrans)
+   {
+      size_ = vdim_ * doftrans.Size();
+      doftrans_ = &doftrans;
+   }
+
+   /// Return the nested DofTransformation object
+   inline DofTransformation * GetDofTransformation() const { return doftrans_; }
+
+   inline void SetFaceOrientation(const Array<int> & face_orientation)
+   { Fo = face_orientation; doftrans_->SetFaceOrientations(face_orientation); }
+
+   using DofTransformation::TransformPrimal;
+   using DofTransformation::InvTransformPrimal;
+   using DofTransformation::TransformDual;
+
+   void TransformPrimal(double *v) const;
+   void InvTransformPrimal(double *v) const;
+   void TransformDual(double *v) const;
+};
+
+/** Abstract base class for high-order Nedelec spaces on elements with
+    triangular faces.
+
+    The Nedelec DoFs on the interior of triangular faces come in pairs which
+    share an interpolation point but have different vector directions. These
+    directions depend on the orientation of the face and can therefore differ in
+    neighboring elements. The mapping required to transform these DoFs can be
+    implemented as series of 2x2 linear transformations. The raw data for these
+    linear transformations is stored in the T_data and TInv_data arrays and can
+    be accessed as DenseMatrices using the GetFaceTransform() and
+    GetFaceInverseTransform() methods.
+*/
+class ND_DofTransformation : public DofTransformation
+{
+protected:
+   static const double T_data[24];
+   static const double TInv_data[24];
+   static const DenseTensor T, TInv;
+   int order;
+
+   ND_DofTransformation(int size, int order);
+
+public:
+   // Return the 2x2 transformation operator for the given face orientation
+   static const DenseMatrix & GetFaceTransform(int ori) { return T(ori); }
+
+   // Return the 2x2 inverse transformation operator
+   static const DenseMatrix & GetFaceInverseTransform(int ori)
+   { return TInv(ori); }
+};
+
+/// DoF transformation implementation for the Nedelec basis on triangles
+class ND_TriDofTransformation : public ND_DofTransformation
+{
+public:
+   ND_TriDofTransformation(int order);
+
+   using DofTransformation::TransformPrimal;
+   using DofTransformation::InvTransformPrimal;
+   using DofTransformation::TransformDual;
+
+   void TransformPrimal(double *v) const;
+
+   void InvTransformPrimal(double *v) const;
+
+   void TransformDual(double *v) const;
+};
+
+/// DoF transformation implementation for the Nedelec basis on tetrahedra
+class ND_TetDofTransformation : public ND_DofTransformation
+{
+public:
+   ND_TetDofTransformation(int order);
+
+   using DofTransformation::TransformPrimal;
+   using DofTransformation::InvTransformPrimal;
+   using DofTransformation::TransformDual;
+
+   void TransformPrimal(double *v) const;
+
+   void InvTransformPrimal(double *v) const;
+
+   void TransformDual(double *v) const;
+};
+
+/// DoF transformation implementation for the Nedelec basis on wedge elements
+/** TODO: (Under development) */
+class ND_WedgeDofTransformation : public ND_DofTransformation
+{
+public:
+   ND_WedgeDofTransformation(int order);
+
+   using DofTransformation::TransformPrimal;
+   using DofTransformation::InvTransformPrimal;
+   using DofTransformation::TransformDual;
+
+   void TransformPrimal(double *v) const;
+
+   void InvTransformPrimal(double *v) const;
+
+   void TransformDual(double *v) const;
+};
+
+} // namespace mfem
+
+#endif // MFEM_DOFTRANSFORM
@@ -380,6 +380,7 @@ void IsoparametricTransformation::SetIdentityTransformation(
      case Geometry::TETRAHEDRON : FElem = &TetrahedronFE; break;
      case Geometry::CUBE :        FElem = &HexahedronFE; break;
      case Geometry::PRISM :       FElem = &WedgeFE; break;
+      case Geometry::PYRAMID :     FElem = &PyramidFE; break;
      default:
         MFEM_ABORT("unknown Geometry::Type!");
   }
@@ -329,7 +329,7 @@ void KellyErrorEstimator::ComputeEstimates()
         error_estimates(e) = sqrt(factor * error_estimates(e));
      }

-      total_error = error_estimates.Sum();
+      total_error = error_estimates.Norml2();
      delete flux;
      return;
   }
@@ -452,9 +452,10 @@ void KellyErrorEstimator::ComputeEstimates()
   auto pfes = dynamic_cast<ParFiniteElementSpace*>(xfes);
   MFEM_VERIFY(pfes, "xfes is not a ParFiniteElementSpace pointer");

-   double process_local_error = error_estimates.Sum();
+   double process_local_error = pow(error_estimates.Norml2(),2.0);
   MPI_Allreduce(&process_local_error, &total_error, 1, MPI_DOUBLE,
                 MPI_SUM, pfes->GetComm());
+   total_error = sqrt(total_error);
 #endif // MFEM_USE_MPI
 }

@@ -1313,6 +1313,64 @@ public:
                           DenseMatrix &dshape) const;
 };

+/// A linear element defined on a triangular prism
+class LinearWedgeFiniteElement : public NodalFiniteElement
+{
+public:
+   /// Construct the LinearWedgeFiniteElement
+   LinearWedgeFiniteElement();
+
+   /** @brief virtual function which evaluates the values of all
+       shape functions at a given point ip and stores
+       them in the vector shape of dimension Dof (4) */
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+
+   /** @brief virtual function which evaluates the values of all
+       partial derivatives of all shape functions at a given
+       point ip and stores them in the matrix dshape (Dof x Dim) (4 x 3)
+       so that each row contains the derivatives of one shape function */
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+
+   virtual void ProjectDelta(int vertex, Vector &dofs) const
+   { dofs = 0.0; dofs(vertex) = 1.0; }
+
+   /** @brief Get the dofs associated with the given @a face.
+       @a *dofs is set to an internal array of the local dofc on the
+       face, while *ndofs is set to the number of dofs on that face.
+   */
+   virtual void GetFaceDofs(int face, int **dofs, int *ndofs) const;
+};
+
+/// A linear element defined on a square pyramid
+class LinearPyramidFiniteElement : public NodalFiniteElement
+{
+public:
+   /// Construct the LinearPyramidFiniteElement
+   LinearPyramidFiniteElement();
+
+   /** @brief virtual function which evaluates the values of all
+       shape functions at a given point ip and stores
+       them in the vector shape of dimension Dof (4) */
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+
+   /** @brief virtual function which evaluates the values of all
+       partial derivatives of all shape functions at a given
+       point ip and stores them in the matrix dshape (Dof x Dim) (4 x 3)
+       so that each row contains the derivatives of one shape function */
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+
+   virtual void ProjectDelta(int vertex, Vector &dofs) const
+   { dofs = 0.0; dofs(vertex) = 1.0; }
+
+   /** @brief Get the dofs associated with the given @a face.
+       @a *dofs is set to an internal array of the local dofc on the
+       face, while *ndofs is set to the number of dofs on that face.
+   */
+   virtual void GetFaceDofs(int face, int **dofs, int *ndofs) const;
+};
+
 /// A 2D constant element on a triangle
 class P0TriangleFiniteElement : public NodalFiniteElement
 {
@@ -1690,6 +1748,32 @@ public:
   { dofs(0) = 1.0; }
 };

+/// A 3D constant element on a wedge
+class P0WdgFiniteElement : public NodalFiniteElement
+{
+public:
+   /// Construct the P0WdgFiniteElement
+   P0WdgFiniteElement ();
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void ProjectDelta(int vertex, Vector &dofs) const
+   { dofs(0) = 1.0; }
+};
+
+/// A 3D constant element on a pyramid
+class P0PyrFiniteElement : public NodalFiniteElement
+{
+public:
+   /// Construct the P0PyrFiniteElement
+   P0PyrFiniteElement ();
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void ProjectDelta(int vertex, Vector &dofs) const
+   { dofs(0) = 1.0; }
+};
+
 /** @brief Tensor products of 1D Lagrange1DFiniteElement
    (only degree 2 is functional) */
 class LagrangeHexFiniteElement : public NodalFiniteElement
@@ -1828,6 +1912,10 @@ public:
   using FiniteElement::Project;
   virtual void Project (VectorCoefficient &vc,
                         ElementTransformation &Trans, Vector &dofs) const;
+
+   virtual void ProjectGrad(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &grad) const;
 };


@@ -1852,6 +1940,66 @@ public:
   using FiniteElement::Project;
   virtual void Project (VectorCoefficient &vc,
                         ElementTransformation &Trans, Vector &dofs) const;
+
+   virtual void ProjectGrad(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &grad) const;
+};
+
+
+/// A 3D 1st order Nedelec element on a wedge
+class Nedelec1WdgFiniteElement : public VectorFiniteElement
+{
+private:
+   static const double tk[9][3];
+
+public:
+   /// Construct the Nedelec1WdgFiniteElement
+   Nedelec1WdgFiniteElement();
+   virtual void CalcVShape(const IntegrationPoint &ip,
+                           DenseMatrix &shape) const;
+   virtual void CalcVShape(ElementTransformation &Trans,
+                           DenseMatrix &shape) const
+   { CalcVShape_ND(Trans, shape); }
+   virtual void CalcCurlShape(const IntegrationPoint &ip,
+                              DenseMatrix &curl_shape) const;
+   virtual void GetLocalInterpolation (ElementTransformation &Trans,
+                                       DenseMatrix &I) const;
+   using FiniteElement::Project;
+   virtual void Project (VectorCoefficient &vc,
+                         ElementTransformation &Trans, Vector &dofs) const;
+
+   virtual void ProjectGrad(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &grad) const;
+};
+
+
+/// A 3D 1st order Nedelec element on a pyramid
+class Nedelec1PyrFiniteElement : public VectorFiniteElement
+{
+private:
+   static const double tk[8][3];
+
+public:
+   /// Construct the Nedelec1PyrFiniteElement
+   Nedelec1PyrFiniteElement();
+   virtual void CalcVShape(const IntegrationPoint &ip,
+                           DenseMatrix &shape) const;
+   virtual void CalcVShape(ElementTransformation &Trans,
+                           DenseMatrix &shape) const
+   { CalcVShape_ND(Trans, shape); }
+   virtual void CalcCurlShape(const IntegrationPoint &ip,
+                              DenseMatrix &curl_shape) const;
+   virtual void GetLocalInterpolation (ElementTransformation &Trans,
+                                       DenseMatrix &I) const;
+   using FiniteElement::Project;
+   virtual void Project (VectorCoefficient &vc,
+                         ElementTransformation &Trans, Vector &dofs) const;
+
+   virtual void ProjectGrad(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &grad) const;
 };


@@ -1945,6 +2093,77 @@ public:
 };


+/// A 3D 0th order Raviert-Thomas element on a wedge
+class RT0WdgFiniteElement : public VectorFiniteElement
+{
+private:
+   static const double nk[5][3];
+
+public:
+   /// Construct the RT0WdgFiniteElement
+   RT0WdgFiniteElement();
+
+   virtual void CalcVShape(const IntegrationPoint &ip,
+                           DenseMatrix &shape) const;
+
+   virtual void CalcVShape(ElementTransformation &Trans,
+                           DenseMatrix &shape) const
+   { CalcVShape_RT(Trans, shape); }
+
+   virtual void CalcDivShape(const IntegrationPoint &ip,
+                             Vector &divshape) const;
+
+   virtual void GetLocalInterpolation (ElementTransformation &Trans,
+                                       DenseMatrix &I) const;
+
+   using FiniteElement::Project;
+
+   virtual void Project (VectorCoefficient &vc,
+                         ElementTransformation &Trans, Vector &dofs) const;
+
+   virtual void ProjectCurl(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &curl) const;
+};
+
+
+/// A 3D 0th order Raviert-Thomas element on a pyramid
+class RT0PyrFiniteElement : public VectorFiniteElement
+{
+private:
+   static const double nk[5][3];
+
+   // If true match RT0TetFiniteElement rather than RT_TetrahedronElement(0)
+   bool rt0;
+
+public:
+   /// Construct the RT0PyrFiniteElement
+   RT0PyrFiniteElement(bool rt0tets = true);
+
+   virtual void CalcVShape(const IntegrationPoint &ip,
+                           DenseMatrix &shape) const;
+
+   virtual void CalcVShape(ElementTransformation &Trans,
+                           DenseMatrix &shape) const
+   { CalcVShape_RT(Trans, shape); }
+
+   virtual void CalcDivShape(const IntegrationPoint &ip,
+                             Vector &divshape) const;
+
+   virtual void GetLocalInterpolation (ElementTransformation &Trans,
+                                       DenseMatrix &I) const;
+
+   using FiniteElement::Project;
+
+   virtual void Project (VectorCoefficient &vc,
+                         ElementTransformation &Trans, Vector &dofs) const;
+
+   virtual void ProjectCurl(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &curl) const;
+};
+
+
 class RotTriLinearHexFiniteElement : public NodalFiniteElement
 {
 public:
@@ -33,6 +33,9 @@ int FiniteElementCollection::HasFaceDofs(Geometry::Type geom, int p) const
      case Geometry::PRISM:
         return max(GetNumDof(Geometry::TRIANGLE, p),
                    GetNumDof(Geometry::SQUARE, p));
+      case Geometry::PYRAMID:
+         return max(GetNumDof(Geometry::TRIANGLE, p),
+                    GetNumDof(Geometry::SQUARE, p));
      default:
         MFEM_ABORT("unknown geometry type");
   }
@@ -574,6 +577,7 @@ LinearFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::TETRAHEDRON: return &TetrahedronFE;
      case Geometry::CUBE:        return &ParallelepipedFE;
      case Geometry::PRISM:       return &WedgeFE;
+      case Geometry::PYRAMID:     return &PyramidFE;
      default:
         mfem_error ("LinearFECollection: unknown geometry type.");
   }
@@ -591,6 +595,7 @@ int LinearFECollection::DofForGeometry(Geometry::Type GeomType) const
      case Geometry::TETRAHEDRON: return 0;
      case Geometry::CUBE:        return 0;
      case Geometry::PRISM:       return 0;
+      case Geometry::PYRAMID:     return 0;
      default:
         mfem_error ("LinearFECollection: unknown geometry type.");
   }
@@ -1240,6 +1245,7 @@ Const3DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::TETRAHEDRON: return &TetrahedronFE;
      case Geometry::CUBE:        return &ParallelepipedFE;
      case Geometry::PRISM:       return &WedgeFE;
+      case Geometry::PYRAMID:     return &PyramidFE;
      default:
         mfem_error ("Const3DFECollection: unknown geometry type.");
   }
@@ -1257,6 +1263,7 @@ int Const3DFECollection::DofForGeometry(Geometry::Type GeomType) const
      case Geometry::TETRAHEDRON: return 1;
      case Geometry::CUBE:        return 1;
      case Geometry::PRISM:       return 1;
+      case Geometry::PYRAMID:     return 1;
      default:
         mfem_error ("Const3DFECollection: unknown geometry type.");
   }
@@ -1277,6 +1284,8 @@ LinearDiscont3DFECollection::FiniteElementForGeometry(
   switch (GeomType)
   {
      case Geometry::TETRAHEDRON: return &TetrahedronFE;
+      case Geometry::PYRAMID:     return &PyramidFE;
+      case Geometry::PRISM:       return &WedgeFE;
      case Geometry::CUBE:        return &ParallelepipedFE;
      default:
         mfem_error ("LinearDiscont3DFECollection: unknown geometry type.");
@@ -1293,6 +1302,8 @@ int LinearDiscont3DFECollection::DofForGeometry(Geometry::Type GeomType) const
      case Geometry::TRIANGLE:    return 0;
      case Geometry::SQUARE:      return 0;
      case Geometry::TETRAHEDRON: return 4;
+      case Geometry::PYRAMID:     return 5;
+      case Geometry::PRISM:       return 6;
      case Geometry::CUBE:        return 8;
      default:
         mfem_error ("LinearDiscont3DFECollection: unknown geometry type.");
@@ -1394,6 +1405,8 @@ ND1_3DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
   {
      case Geometry::CUBE:        return &HexahedronFE;
      case Geometry::TETRAHEDRON: return &TetrahedronFE;
+      case Geometry::PRISM:       return &WedgeFE;
+      case Geometry::PYRAMID:     return &PyramidFE;
      default:
         mfem_error ("ND1_3DFECollection: unknown geometry type.");
   }
@@ -1410,6 +1423,8 @@ int ND1_3DFECollection::DofForGeometry(Geometry::Type GeomType) const
      case Geometry::SQUARE:      return 0;
      case Geometry::TETRAHEDRON: return 0;
      case Geometry::CUBE:        return 0;
+      case Geometry::PRISM:       return 0;
+      case Geometry::PYRAMID:     return 0;
      default:
         mfem_error ("ND1_3DFECollection: unknown geometry type.");
   }
@@ -1439,6 +1454,8 @@ RT0_3DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
      case Geometry::SQUARE:      return &QuadrilateralFE;
      case Geometry::CUBE:        return &HexahedronFE;
      case Geometry::TETRAHEDRON: return &TetrahedronFE;
+      case Geometry::PRISM:       return &WedgeFE;
+      case Geometry::PYRAMID:     return &PyramidFE;
      default:
         mfem_error ("RT0_3DFECollection: unknown geometry type.");
   }
@@ -1455,6 +1472,8 @@ int RT0_3DFECollection::DofForGeometry(Geometry::Type GeomType) const
      case Geometry::SQUARE:      return 1;
      case Geometry::TETRAHEDRON: return 0;
      case Geometry::CUBE:        return 0;
+      case Geometry::PRISM:       return 0;
+      case Geometry::PYRAMID:     return 0;
      default:
         mfem_error ("RT0_3DFECollection: unknown geometry type.");
   }
@@ -1730,6 +1749,7 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype)
         H1_dof[Geometry::TETRAHEDRON] = (TriDof*pm3)/3;
         H1_dof[Geometry::CUBE] = QuadDof*pm1;
         H1_dof[Geometry::PRISM] = TriDof*pm1;
+         H1_dof[Geometry::PYRAMID] = 0;
         if (b_type == BasisType::Positive)
         {
            H1_Elements[Geometry::TETRAHEDRON] = new H1Pos_TetrahedronElement(p);
@@ -1743,6 +1763,7 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype)
            H1_Elements[Geometry::CUBE] = new H1_HexahedronElement(p, btype);
            H1_Elements[Geometry::PRISM] = new H1_WedgeElement(p, btype);
         }
+         H1_Elements[Geometry::PYRAMID] = new LinearPyramidFiniteElement;

         const int &TetDof = H1_dof[Geometry::TETRAHEDRON];
         TetDofOrd[0] = new int[24*TetDof];
@@ -1837,6 +1858,21 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype)
   }
 }

+const FiniteElement *
+H1_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
+{
+   if (GeomType != Geometry::PYRAMID || this->GetOrder() == 1)
+   {
+      return H1_Elements[GeomType];
+   }
+   else
+   {
+      MFEM_ABORT("H1 Pyramid basis functions are not yet supported "
+                 "for order > 1.");
+      return NULL;
+   }
+}
+
 const int *H1_FECollection::DofOrderForOrientation(Geometry::Type GeomType,
                                                   int Or) const
 {
@@ -2076,9 +2112,12 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
         L2_Elements[Geometry::CUBE] = new L2_HexahedronElement(p, btype);
         L2_Elements[Geometry::PRISM] = new L2_WedgeElement(p, btype);
      }
+      L2_Elements[Geometry::PYRAMID] = new P0PyrFiniteElement;
+
      L2_Elements[Geometry::TETRAHEDRON]->SetMapType(map_type);
      L2_Elements[Geometry::CUBE]->SetMapType(map_type);
      L2_Elements[Geometry::PRISM]->SetMapType(map_type);
+      L2_Elements[Geometry::PYRAMID]->SetMapType(map_type);
      // Trace element use the default Gauss-Legendre nodal points for positive basis
      if (b_type == BasisType::Positive)
      {
@@ -2199,6 +2238,21 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
   }
 }

+const FiniteElement *
+L2_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
+{
+   if (GeomType != Geometry::PYRAMID || this->GetOrder() == 0)
+   {
+      return L2_Elements[GeomType];
+   }
+   else
+   {
+      MFEM_ABORT("L2 Pyramid basis functions are not yet supported "
+                 "for order > 0.");
+      return NULL;
+   }
+}
+
 const int *L2_FECollection::DofOrderForOrientation(Geometry::Type GeomType,
                                                   int Or) const
 {
@@ -2290,6 +2344,12 @@ RT_FECollection::RT_FECollection(const int order, const int dim,

      RT_Elements[Geometry::CUBE] = new RT_HexahedronElement(p, cb_type, ob_type);
      RT_dof[Geometry::CUBE] = 3*p*pp1*pp1;
+
+      RT_Elements[Geometry::PRISM] = new RT0WdgFiniteElement;
+      RT_dof[Geometry::PRISM] = 0;
+
+      RT_Elements[Geometry::PYRAMID] = new RT0PyrFiniteElement(false);
+      RT_dof[Geometry::PYRAMID] = 0;
   }
   else
   {
@@ -2433,6 +2493,22 @@ void RT_FECollection::InitFaces(const int p, const int dim,
   }
 }

+const FiniteElement *
+RT_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
+{
+   if ((GeomType != Geometry::PRISM && GeomType != Geometry::PYRAMID) ||
+       this->GetOrder() == 1)
+   {
+      return RT_Elements[GeomType];
+   }
+   else
+   {
+      MFEM_ABORT("RT Wedge and Pyramid basis functions are not yet supported "
+                 "for order > 0.");
+      return NULL;
+   }
+}
+
 const int *RT_FECollection::DofOrderForOrientation(Geometry::Type GeomType,
                                                   int Or) const
 {
@@ -2656,18 +2732,31 @@ ND_FECollection::ND_FECollection(const int p, const int dim,
      {
         for (int i = 0; i + j <= pm2; i++)
         {
-            int k1 = p*pm1 - (p - j)*(pm1 - j) + 2*i;
-            int k2 = p*pm1 - (p - i)*(pm1 - i) + 2*j;
-            // (0,1,2)
-            TriDofOrd[0][k1  ] = k1;
-            TriDofOrd[0][k1+1] = k1 + 1;
-            // (0,2,1)
-            TriDofOrd[5][k1  ] = k2 + 1;
-            TriDofOrd[5][k1+1] = k2;
+            int k0 = p*pm1 - (p - j)*(pm1 - j) + 2*i;
+            int k1 = 2*pm2 - 2*i + ((2*p-3)-j)*j;
+            int k2 = 2*pm2 - 2*j + ((2*p-3)-i)*i;
+            int k3 = p*pm1 - 2 - 3*j - i - (i+j)*(i+j);
+            int k4 = p*pm1 - 2 - 3*i - j - (i+j)*(i+j);
+            int k5 = p*pm1 - (p - i)*(pm1 - i) + 2*j;

-            // The other orientations can not be supported with the current
-            // interface. The method Mesh::ReorientTetMesh will ensure that
-            // only orientations 0 and 5 are generated.
+            // (0,1,2)
+            TriDofOrd[0][k0  ] = k0;
+            TriDofOrd[0][k0+1] = k0 + 1;
+            // (1,0,2)
+            TriDofOrd[1][k0  ] = k1;
+            TriDofOrd[1][k0+1] = k1 + 1;
+            // (2,0,1)
+            TriDofOrd[2][k0  ] = k2;
+            TriDofOrd[2][k0+1] = k2 + 1;
+            // (2,1,0)
+            TriDofOrd[3][k0  ] = k3;
+            TriDofOrd[3][k0+1] = k3 + 1;
+            // (1,2,0)
+            TriDofOrd[4][k0  ] = k4;
+            TriDofOrd[4][k0+1] = k4 + 1;
+            // (0,2,1)
+            TriDofOrd[5][k0  ] = k5;
+            TriDofOrd[5][k0+1] = k5 + 1;
         }
      }
   }
@@ -2680,6 +2769,28 @@ ND_FECollection::ND_FECollection(const int p, const int dim,
      // TODO: cb_type and ob_type for tets
      ND_Elements[Geometry::TETRAHEDRON] = new ND_TetrahedronElement(p);
      ND_dof[Geometry::TETRAHEDRON] = p*pm1*pm2/2;
+
+      ND_Elements[Geometry::PRISM] = new Nedelec1WdgFiniteElement;
+      ND_dof[Geometry::PRISM] = 0;
+
+      ND_Elements[Geometry::PYRAMID] = new Nedelec1PyrFiniteElement;
+      ND_dof[Geometry::PYRAMID] = 0;
+   }
+}
+
+const FiniteElement *
+ND_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
+{
+   if ((GeomType != Geometry::PRISM && GeomType != Geometry::PYRAMID) ||
+       this->GetOrder() == 1)
+   {
+      return ND_Elements[GeomType];
+   }
+   else
+   {
+      MFEM_ABORT("ND Wedge and Pyramid basis functions are not yet supported "
+                 "for order > 1.");
+      return NULL;
   }
 }

@@ -2692,11 +2803,6 @@ const int *ND_FECollection::DofOrderForOrientation(Geometry::Type GeomType,
   }
   else if (GeomType == Geometry::TRIANGLE)
   {
-      if (Or != 0 && Or != 5)
-      {
-         MFEM_ABORT("triangle face orientation " << Or << " is not supported! "
-                    "Use Mesh::ReorientTetMesh to fix it.");
-      }
      return TriDofOrd[Or%6];
   }
   else if (GeomType == Geometry::SQUARE)
@@ -228,8 +228,7 @@ public:
                            const int btype = BasisType::GaussLobatto);

   virtual const FiniteElement *FiniteElementForGeometry(
-      Geometry::Type GeomType) const
-   { return H1_Elements[GeomType]; }
+      Geometry::Type GeomType) const;
   virtual int DofForGeometry(Geometry::Type GeomType) const
   { return H1_dof[GeomType]; }
   virtual const int *DofOrderForOrientation(Geometry::Type GeomType,
@@ -302,10 +301,7 @@ public:
                   const int map_type = FiniteElement::VALUE);

   virtual const FiniteElement *FiniteElementForGeometry(
-      Geometry::Type GeomType) const
-   {
-      return L2_Elements[GeomType];
-   }
+      Geometry::Type GeomType) const;
   virtual int DofForGeometry(Geometry::Type GeomType) const
   {
      if (L2_Elements[GeomType])
@@ -371,8 +367,7 @@ public:
                   const int ob_type = BasisType::GaussLegendre);

   virtual const FiniteElement *FiniteElementForGeometry(
-      Geometry::Type GeomType) const
-   { return RT_Elements[GeomType]; }
+      Geometry::Type GeomType) const;
   virtual int DofForGeometry(Geometry::Type GeomType) const
   { return RT_dof[GeomType]; }
   virtual const int *DofOrderForOrientation(Geometry::Type GeomType,
@@ -430,8 +425,7 @@ public:
                   const int ob_type = BasisType::GaussLegendre);

   virtual const FiniteElement *
-   FiniteElementForGeometry(Geometry::Type GeomType) const
-   { return ND_Elements[GeomType]; }
+   FiniteElementForGeometry(Geometry::Type GeomType) const;

   virtual int DofForGeometry(Geometry::Type GeomType) const
   { return ND_dof[GeomType]; }
@@ -529,9 +523,10 @@ private:
   const BiLinear2DFiniteElement QuadrilateralFE;
   const Linear3DFiniteElement TetrahedronFE;
   const TriLinear3DFiniteElement ParallelepipedFE;
-   const H1_WedgeElement WedgeFE;
+   const LinearWedgeFiniteElement WedgeFE;
+   const LinearPyramidFiniteElement PyramidFE;
 public:
-   LinearFECollection() : FiniteElementCollection(1), WedgeFE(1) { }
+   LinearFECollection() : FiniteElementCollection(1) { }

   virtual const FiniteElement *
   FiniteElementForGeometry(Geometry::Type GeomType) const;
@@ -936,10 +931,11 @@ class Const3DFECollection : public FiniteElementCollection
 private:
   const P0TetFiniteElement TetrahedronFE;
   const P0HexFiniteElement ParallelepipedFE;
-   const L2_WedgeElement WedgeFE;
+   const P0WdgFiniteElement WedgeFE;
+   const P0PyrFiniteElement PyramidFE;

 public:
-   Const3DFECollection() : FiniteElementCollection(0), WedgeFE(0) { }
+   Const3DFECollection() : FiniteElementCollection(0) { }

   virtual const FiniteElement *
   FiniteElementForGeometry(Geometry::Type GeomType) const;
@@ -960,6 +956,8 @@ class LinearDiscont3DFECollection : public FiniteElementCollection
 {
 private:
   const Linear3DFiniteElement TetrahedronFE;
+   const LinearPyramidFiniteElement PyramidFE;
+   const LinearWedgeFiniteElement WedgeFE;
   const TriLinear3DFiniteElement ParallelepipedFE;

 public:
@@ -1036,6 +1034,8 @@ class ND1_3DFECollection : public FiniteElementCollection
 private:
   const Nedelec1HexFiniteElement HexahedronFE;
   const Nedelec1TetFiniteElement TetrahedronFE;
+   const Nedelec1WdgFiniteElement WedgeFE;
+   const Nedelec1PyrFiniteElement PyramidFE;

 public:
   ND1_3DFECollection() : FiniteElementCollection(1) { }
@@ -1061,6 +1061,8 @@ private:
   const P0QuadFiniteElement QuadrilateralFE;
   const RT0HexFiniteElement HexahedronFE;
   const RT0TetFiniteElement TetrahedronFE;
+   const RT0WdgFiniteElement WedgeFE;
+   const RT0PyrFiniteElement PyramidFE;
 public:
   RT0_3DFECollection() : FiniteElementCollection(1) { }

@@ -16,6 +16,7 @@
 #include "geom.hpp"
 #include "fe.hpp"
 #include "fe_coll.hpp"
+#include "doftrans.hpp"
 #include "eltrans.hpp"
 #include "coefficient.hpp"
 #include "complex_fem.hpp"
@@ -34,6 +35,7 @@
 #include "staticcond.hpp"
 #include "tmop.hpp"
 #include "tmop_tools.hpp"
+#include "tmop_amr.hpp"
 #include "gslib.hpp"
 #include "restriction.hpp"
 #include "quadinterpolator.hpp"
@@ -58,9 +58,12 @@ DofsToVDofs<Ordering::byVDIM>(int ndofs, int vdim, Array<int> &dofs)

 FiniteElementSpace::FiniteElementSpace()
   : mesh(NULL), fec(NULL), vdim(0), ordering(Ordering::byNODES),
-     ndofs(0), nvdofs(0), nedofs(0), nfdofs(0), nbdofs(0), bdofs(NULL),
-     elem_dof(NULL), bdr_elem_dof(NULL), face_dof(NULL),
+     ndofs(0), nvdofs(0), nedofs(0), nfdofs(0), nbdofs(0),
+     bdofs(NULL),
+     elem_dof(NULL), elem_fos(NULL), bdr_elem_dof(NULL), bdr_elem_fos(NULL),
+     face_dof(NULL),
     NURBSext(NULL), own_ext(false),
+     DoFTrans(0), VDoFTrans(vdim, ordering),
     cP(NULL), cR(NULL), cR_hp(NULL), cP_is_set(false),
     Th(Operator::ANY_TYPE),
     sequence(0), mesh_sequence(0), orders_changed(false), relaxed_hp(false)
@@ -69,6 +72,7 @@ FiniteElementSpace::FiniteElementSpace()
 FiniteElementSpace::FiniteElementSpace(const FiniteElementSpace &orig,
                                       Mesh *mesh,
                                       const FiniteElementCollection *fec)
+   : VDoFTrans(orig.vdim, orig.ordering)
 {
   mesh = mesh ? mesh : orig.mesh;
   fec = fec ? fec : orig.fec;
@@ -259,16 +263,36 @@ void FiniteElementSpace::AdjustVDofs (Array<int> &vdofs)
   }
 }

-void FiniteElementSpace::GetElementVDofs(int i, Array<int> &vdofs) const
+DofTransformation *
+FiniteElementSpace::GetElementVDofs(int i, Array<int> &vdofs) const
 {
-   GetElementDofs(i, vdofs);
+   DofTransformation * doftrans = GetElementDofs(i, vdofs);
   DofsToVDofs(vdofs);
+   if (vdim == 1 || doftrans == NULL)
+   {
+      return doftrans;
+   }
+   else
+   {
+      VDoFTrans.SetDofTransformation(*doftrans);
+      return &VDoFTrans;
+   }
 }

-void FiniteElementSpace::GetBdrElementVDofs(int i, Array<int> &vdofs) const
+DofTransformation *
+FiniteElementSpace::GetBdrElementVDofs(int i, Array<int> &vdofs) const
 {
-   GetBdrElementDofs(i, vdofs);
+   DofTransformation * doftrans = GetBdrElementDofs(i, vdofs);
   DofsToVDofs(vdofs);
+   if (vdim == 1 || doftrans == NULL)
+   {
+      return doftrans;
+   }
+   else
+   {
+      VDoFTrans.SetDofTransformation(*doftrans);
+      return &VDoFTrans;
+   }
 }

 void FiniteElementSpace::GetFaceVDofs(int i, Array<int> &vdofs) const
@@ -307,21 +331,39 @@ void FiniteElementSpace::BuildElementToDofTable() const

   // TODO: can we call GetElementDofs only once per element?
   Table *el_dof = new Table;
+   Table *el_fos = (mesh->Dimension() > 2) ? (new Table) : NULL;
   Array<int> dofs;
+   Array<int> F, Fo;
   el_dof -> MakeI (mesh -> GetNE());
+   if (el_fos) { el_fos -> MakeI (mesh -> GetNE()); }
   for (int i = 0; i < mesh -> GetNE(); i++)
   {
      GetElementDofs (i, dofs);
      el_dof -> AddColumnsInRow (i, dofs.Size());
+
+      if (el_fos)
+      {
+         mesh->GetElementFaces(i, F, Fo);
+         el_fos -> AddColumnsInRow (i, Fo.Size());
+      }
   }
   el_dof -> MakeJ();
+   if (el_fos) { el_fos -> MakeJ(); }
   for (int i = 0; i < mesh -> GetNE(); i++)
   {
      GetElementDofs (i, dofs);
      el_dof -> AddConnections (i, (int *)dofs, dofs.Size());
+
+      if (el_fos)
+      {
+         mesh->GetElementFaces(i, F, Fo);
+         el_fos -> AddConnections (i, (int *)Fo, Fo.Size());
+      }
   }
   el_dof -> ShiftUpI();
+   if (el_fos) { el_fos -> ShiftUpI(); }
   elem_dof = el_dof;
+   elem_fos = el_fos;
 }

 void FiniteElementSpace::BuildBdrElementToDofTable() const
@@ -375,7 +417,9 @@ void FiniteElementSpace::BuildFaceToDofTable() const
 void FiniteElementSpace::RebuildElementToDofTable()
 {
   delete elem_dof;
+   delete elem_fos;
   elem_dof = NULL;
+   elem_fos = NULL;
   BuildElementToDofTable();
 }

@@ -1315,8 +1359,10 @@ const FaceQuadratureInterpolator

 SparseMatrix *FiniteElementSpace::RefinementMatrix_main(
   const int coarse_ndofs, const Table &coarse_elem_dof,
-   const DenseTensor localP[]) const
+   const Table *coarse_elem_fos, const DenseTensor localP[]) const
 {
+   /// TODO: Implement DofTransformation support
+
   MFEM_VERIFY(mesh->GetLastOperation() == Mesh::REFINE, "");

   Array<int> dofs, coarse_dofs, coarse_vdofs;
@@ -1399,7 +1445,8 @@ void FiniteElementSpace::GetLocalRefinementMatrices(
 }

 SparseMatrix* FiniteElementSpace::RefinementMatrix(int old_ndofs,
-                                                   const Table* old_elem_dof)
+                                                   const Table* old_elem_dof,
+                                                   const Table* old_elem_fos)
 {
   MFEM_VERIFY(GetNE() >= old_elem_dof->Size(),
               "Previous mesh is not coarser.");
@@ -1412,13 +1459,16 @@ SparseMatrix* FiniteElementSpace::RefinementMatrix(int old_ndofs,
      GetLocalRefinementMatrices(elem_geoms[i], localP[elem_geoms[i]]);
   }

-   return RefinementMatrix_main(old_ndofs, *old_elem_dof, localP);
+   return RefinementMatrix_main(old_ndofs, *old_elem_dof, old_elem_fos,
+                                localP);
 }

 FiniteElementSpace::RefinementOperator::RefinementOperator
-(const FiniteElementSpace* fespace, Table* old_elem_dof, int old_ndofs)
+(const FiniteElementSpace* fespace, Table* old_elem_dof, Table* old_elem_fos,
+ int old_ndofs)
   : fespace(fespace)
   , old_elem_dof(old_elem_dof)
+   , old_elem_fos(old_elem_fos)
 {
   MFEM_VERIFY(fespace->GetNE() >= old_elem_dof->Size(),
               "Previous mesh is not coarser.");
@@ -1432,12 +1482,14 @@ FiniteElementSpace::RefinementOperator::RefinementOperator
   {
      fespace->GetLocalRefinementMatrices(elem_geoms[i], localP[elem_geoms[i]]);
   }
+
+   ConstructDoFTrans();
 }

 FiniteElementSpace::RefinementOperator::RefinementOperator(
   const FiniteElementSpace *fespace, const FiniteElementSpace *coarse_fes)
   : Operator(fespace->GetVSize(), coarse_fes->GetVSize()),
-     fespace(fespace), old_elem_dof(NULL)
+     fespace(fespace), old_elem_dof(NULL), old_elem_fos(NULL)
 {
   Mesh::GeometryList elem_geoms(*fespace->GetMesh());

@@ -1449,11 +1501,50 @@ FiniteElementSpace::RefinementOperator::RefinementOperator(

   // Make a copy of the coarse elem_dof Table.
   old_elem_dof = new Table(coarse_fes->GetElementToDofTable());
+
+   // Make a copy of the coarse elem_fos Table if it exists.
+   if (coarse_fes->GetElementToFaceOrientationTable())
+   {
+      old_elem_fos = new Table(*coarse_fes->GetElementToFaceOrientationTable());
+   }
+
+   ConstructDoFTrans();
 }

 FiniteElementSpace::RefinementOperator::~RefinementOperator()
 {
   delete old_elem_dof;
+   delete old_elem_fos;
+}
+
+void FiniteElementSpace::RefinementOperator
+::ConstructDoFTrans()
+{
+   old_DoFTrans.SetSize(Geometry::NUM_GEOMETRIES);
+   for (int i=0; i<old_DoFTrans.Size(); i++)
+   {
+      old_DoFTrans[i] = NULL;
+   }
+
+   const FiniteElementCollection *fec = fespace->FEColl();
+   if (dynamic_cast<const ND_FECollection*>(fec))
+   {
+      const FiniteElement * nd_tri =
+         fec->FiniteElementForGeometry(Geometry::TRIANGLE);
+      if (nd_tri)
+      {
+         old_DoFTrans[Geometry::TRIANGLE] =
+            new ND_TriDofTransformation(nd_tri->GetOrder());
+      }
+
+      const FiniteElement * nd_tet =
+         fec->FiniteElementForGeometry(Geometry::TETRAHEDRON);
+      if (nd_tet)
+      {
+         old_DoFTrans[Geometry::TETRAHEDRON] =
+            new ND_TetDofTransformation(nd_tet->GetOrder());
+      }
+   }
 }

 void FiniteElementSpace::RefinementOperator
@@ -1462,7 +1553,7 @@ void FiniteElementSpace::RefinementOperator
   Mesh* mesh = fespace->GetMesh();
   const CoarseFineTransformations &rtrans = mesh->GetRefinementTransforms();

-   Array<int> dofs, vdofs, old_dofs, old_vdofs;
+   Array<int> dofs, vdofs, old_dofs, old_vdofs, old_Fo;

   int vdim = fespace->GetVDim();
   int old_ndofs = width / vdim;
@@ -1477,18 +1568,53 @@ void FiniteElementSpace::RefinementOperator

      subY.SetSize(lP.Height());

-      fespace->GetElementDofs(k, dofs);
+      DofTransformation *doftrans = fespace->GetElementDofs(k, dofs);
      old_elem_dof->GetRow(emb.parent, old_dofs);

-      for (int vd = 0; vd < vdim; vd++)
+      if (!doftrans)
      {
-         dofs.Copy(vdofs);
-         fespace->DofsToVDofs(vd, vdofs);
-         old_dofs.Copy(old_vdofs);
-         fespace->DofsToVDofs(vd, old_vdofs, old_ndofs);
-         x.GetSubVector(old_vdofs, subX);
-         lP.Mult(subX, subY);
-         y.SetSubVector(vdofs, subY);
+         for (int vd = 0; vd < vdim; vd++)
+         {
+            dofs.Copy(vdofs);
+            fespace->DofsToVDofs(vd, vdofs);
+            old_dofs.Copy(old_vdofs);
+            fespace->DofsToVDofs(vd, old_vdofs, old_ndofs);
+            x.GetSubVector(old_vdofs, subX);
+            lP.Mult(subX, subY);
+            y.SetSubVector(vdofs, subY);
+         }
+      }
+      else
+      {
+         old_elem_fos->GetRow(emb.parent, old_Fo);
+         old_DoFTrans[geom]->SetFaceOrientations(old_Fo);
+
+         DofTransformation *new_doftrans = NULL;
+         VDofTransformation *vdoftrans =
+            dynamic_cast<VDofTransformation*>(doftrans);
+         if (vdoftrans)
+         {
+            new_doftrans = doftrans;
+            doftrans = vdoftrans->GetDofTransformation();
+         }
+
+         for (int vd = 0; vd < vdim; vd++)
+         {
+            dofs.Copy(vdofs);
+            fespace->DofsToVDofs(vd, vdofs);
+            old_dofs.Copy(old_vdofs);
+            fespace->DofsToVDofs(vd, old_vdofs, old_ndofs);
+            x.GetSubVector(old_vdofs, subX);
+            old_DoFTrans[geom]->InvTransformPrimal(subX);
+            lP.Mult(subX, subY);
+            doftrans->TransformPrimal(subY);
+            y.SetSubVector(vdofs, subY);
+         }
+
+         if (vdoftrans)
+         {
+            doftrans = new_doftrans;
+         }
      }
   }
 }
@@ -1504,12 +1630,12 @@ void FiniteElementSpace::RefinementOperator
   Array<char> processed(fespace->GetVSize());
   processed = 0;

-   Array<int> f_dofs, c_dofs, f_vdofs, c_vdofs;
+   Array<int> f_dofs, c_dofs, f_vdofs, c_vdofs, old_Fo;

   int vdim = fespace->GetVDim();
   int old_ndofs = width / vdim;

-   Vector subY, subX;
+   Vector subY, subX, subYt, subXt;

   for (int k = 0; k < mesh->GetNE(); k++)
   {
@@ -1517,30 +1643,77 @@ void FiniteElementSpace::RefinementOperator
      const Geometry::Type geom = mesh->GetElementBaseGeometry(k);
      const DenseMatrix &lP = localP[geom](emb.matrix);

-      fespace->GetElementDofs(k, f_dofs);
+      DofTransformation * doftrans = fespace->GetElementDofs(k, f_dofs);
      old_elem_dof->GetRow(emb.parent, c_dofs);

-      subY.SetSize(lP.Width());
-
-      for (int vd = 0; vd < vdim; vd++)
+      if (!doftrans)
      {
-         f_dofs.Copy(f_vdofs);
-         fespace->DofsToVDofs(vd, f_vdofs);
-         c_dofs.Copy(c_vdofs);
-         fespace->DofsToVDofs(vd, c_vdofs, old_ndofs);
+         subY.SetSize(lP.Width());

-         x.GetSubVector(f_vdofs, subX);
-
-         for (int p = 0; p < f_dofs.Size(); ++p)
+         for (int vd = 0; vd < vdim; vd++)
         {
-            if (processed[DecodeDof(f_dofs[p])])
+            f_dofs.Copy(f_vdofs);
+            fespace->DofsToVDofs(vd, f_vdofs);
+            c_dofs.Copy(c_vdofs);
+            fespace->DofsToVDofs(vd, c_vdofs, old_ndofs);
+
+            x.GetSubVector(f_vdofs, subX);
+
+            for (int p = 0; p < f_dofs.Size(); ++p)
            {
-               subX[p] = 0.0;
+               if (processed[DecodeDof(f_dofs[p])])
+               {
+                  subX[p] = 0.0;
+               }
            }
+
+            lP.MultTranspose(subX, subY);
+            y.AddElementVector(c_vdofs, subY);
+         }
+      }
+      else
+      {
+         subYt.SetSize(lP.Width());
+
+         old_elem_fos->GetRow(emb.parent, old_Fo);
+         old_DoFTrans[geom]->SetFaceOrientations(old_Fo);
+
+         DofTransformation *new_doftrans = NULL;
+         VDofTransformation *vdoftrans =
+            dynamic_cast<VDofTransformation*>(doftrans);
+         if (vdoftrans)
+         {
+            new_doftrans = doftrans;
+            doftrans = vdoftrans->GetDofTransformation();
         }

-         lP.MultTranspose(subX, subY);
-         y.AddElementVector(c_vdofs, subY);
+         for (int vd = 0; vd < vdim; vd++)
+         {
+            f_dofs.Copy(f_vdofs);
+            fespace->DofsToVDofs(vd, f_vdofs);
+            c_dofs.Copy(c_vdofs);
+            fespace->DofsToVDofs(vd, c_vdofs, old_ndofs);
+
+            x.GetSubVector(f_vdofs, subX);
+            old_DoFTrans[geom]->InvTransformPrimal(subX);
+
+            for (int p = 0; p < f_dofs.Size(); ++p)
+            {
+               if (processed[DecodeDof(f_dofs[p])])
+               {
+                  subX[p] = 0.0;
+               }
+            }
+
+            lP.MultTranspose(subX, subY);
+            doftrans->TransformPrimal(subY);
+            y.AddElementVector(c_vdofs, subY);
+         }
+
+         if (vdoftrans)
+         {
+            doftrans = new_doftrans;
+         }
      }

      for (int p = 0; p < f_dofs.Size(); ++p)
@@ -1550,6 +1723,7 @@ void FiniteElementSpace::RefinementOperator
   }
 }

+/// TODO: Implement DofTransformation support
 FiniteElementSpace::DerefinementOperator::DerefinementOperator(
   const FiniteElementSpace *f_fes, const FiniteElementSpace *c_fes,
   BilinearFormIntegrator *mass_integ)
@@ -1707,8 +1881,11 @@ void FiniteElementSpace::GetLocalDerefinementMatrices(Geometry::Type geom,
 }

 SparseMatrix* FiniteElementSpace::DerefinementMatrix(int old_ndofs,
-                                                     const Table* old_elem_dof)
+                                                     const Table* old_elem_dof,
+                                                     const Table* old_elem_fos)
 {
+   /// TODO: Implement DofTransformation support
+
   MFEM_VERIFY(Nonconforming(), "Not implemented for conforming meshes.");
   MFEM_VERIFY(old_ndofs, "Missing previous (finer) space.");
   MFEM_VERIFY(ndofs <= old_ndofs, "Previous space is not finer.");
@@ -1814,6 +1991,7 @@ void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
   this->ordering = (Ordering::Type) ordering;

   elem_dof = NULL;
+   elem_fos = NULL;
   face_dof = NULL;

   sequence = 0;
@@ -1841,6 +2019,8 @@ void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
      UpdateNURBS();
      cP = cR = cR_hp = NULL;
      cP_is_set = false;
+
+      ConstructDoFTrans();
   }
   else
   {
@@ -1848,9 +2028,41 @@ void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
      own_ext = 0;
      Construct();
   }
+
   BuildElementToDofTable();
 }

+void FiniteElementSpace::ConstructDoFTrans()
+{
+   DestroyDoFTrans();
+
+   VDoFTrans.SetVDim(vdim);
+   DoFTrans.SetSize(Geometry::NUM_GEOMETRIES);
+   for (int i=0; i<DoFTrans.Size(); i++)
+   {
+      DoFTrans[i] = NULL;
+   }
+   if (mesh->Dimension() < 3) { return; }
+   if (dynamic_cast<const ND_FECollection*>(fec))
+   {
+      const FiniteElement * nd_tri =
+         fec->FiniteElementForGeometry(Geometry::TRIANGLE);
+      if (nd_tri)
+      {
+         DoFTrans[Geometry::TRIANGLE] =
+            new ND_TriDofTransformation(nd_tri->GetOrder());
+      }
+
+      const FiniteElement * nd_tet =
+         fec->FiniteElementForGeometry(Geometry::TETRAHEDRON);
+      if (nd_tet)
+      {
+         DoFTrans[Geometry::TETRAHEDRON] =
+            new ND_TetDofTransformation(nd_tet->GetOrder());
+      }
+   }
+}
+
 NURBSExtension *FiniteElementSpace::StealNURBSext()
 {
   if (NURBSext && !own_ext)
@@ -1946,7 +2158,9 @@ void FiniteElementSpace::Construct()
               "Variable order space requires a nonconforming mesh.");

   elem_dof = NULL;
+   elem_fos = NULL;
   bdr_elem_dof = NULL;
+   bdr_elem_fos = NULL;
   face_dof = NULL;

   ndofs = 0;
@@ -2044,6 +2258,8 @@ void FiniteElementSpace::Construct()

   ndofs = nvdofs + nedofs + nfdofs + nbdofs;

+   ConstructDoFTrans();
+
   // record the current mesh sequence number to detect refinement etc.
   mesh_sequence = mesh->GetSequence();

@@ -2295,14 +2511,22 @@ int FiniteElementSpace::GetNVariants(int entity, int index) const
 static const char* msg_orders_changed =
   "Element orders changed, you need to Update() the space first.";

-void FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
+DofTransformation *
+FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
 {
   MFEM_VERIFY(!orders_changed, msg_orders_changed);

   if (elem_dof)
   {
      elem_dof->GetRow(elem, dofs);
-      return;
+
+      if (DoFTrans[mesh->GetElementBaseGeometry(elem)])
+      {
+         Array<int> Fo;
+         elem_fos -> GetRow (elem, Fo);
+         DoFTrans[mesh->GetElementBaseGeometry(elem)]->SetFaceOrientations(Fo);
+      }
+      return DoFTrans[mesh->GetElementBaseGeometry(elem)];
   }

   Array<int> V, E, Eo, F, Fo; // TODO: LocalArray
@@ -2326,6 +2550,11 @@ void FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
      {
         nfd += fec->GetNumDof(mesh->GetFaceGeometry(F[i]), order);
      }
+      if (DoFTrans[mesh->GetElementBaseGeometry(elem)])
+      {
+         DoFTrans[mesh->GetElementBaseGeometry(elem)]
+         -> SetFaceOrientations(Fo);
+      }
   }

   dofs.SetSize(0);
@@ -2383,6 +2612,7 @@ void FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
         dofs.Append(bbase + j);
      }
   }
+   return DoFTrans[mesh->GetElementBaseGeometry(elem)];
 }

 const FiniteElement *FiniteElementSpace::GetFE(int i) const
@@ -2415,18 +2645,27 @@ const FiniteElement *FiniteElementSpace::GetFE(int i) const
   return FE;
 }

-void FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const
+DofTransformation *
+FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const
 {
   MFEM_VERIFY(!orders_changed, msg_orders_changed);

   if (bdr_elem_dof)
   {
      bdr_elem_dof->GetRow(bel, dofs);
-      return;
+
+      if (DoFTrans[mesh->GetBdrElementBaseGeometry(bel)])
+      {
+         Array<int> Fo;
+         bdr_elem_fos -> GetRow (bel, Fo);
+         DoFTrans[mesh->GetBdrElementBaseGeometry(bel)]->
+         SetFaceOrientations(Fo);
+      }
+      return DoFTrans[mesh->GetBdrElementBaseGeometry(bel)];
   }

-   Array<int> V, E, Eo; // TODO: LocalArray
-   int F, Fo;
+   Array<int> V, E, Eo, Fo; // TODO: LocalArray
+   int F, oF;

   int dim = mesh->Dimension();
   auto geom = mesh->GetBdrElementGeometry(bel);
@@ -2445,7 +2684,17 @@ void FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const

   if (nv) { mesh->GetBdrElementVertices(bel, V); }
   if (ne) { mesh->GetBdrElementEdges(bel, E, Eo); }
-   if (nf) { mesh->GetBdrElementFace(bel, &F, &Fo); }
+   if (nf)
+   {
+      mesh->GetBdrElementFace(bel, &F, &oF);
+
+      if (DoFTrans[mesh->GetBdrElementBaseGeometry(bel)])
+      {
+         Fo.Append(oF);
+         DoFTrans[mesh->GetBdrElementBaseGeometry(bel)]->
+         SetFaceOrientations(Fo);
+      }
+   }

   dofs.SetSize(0);
   dofs.Reserve(nv*V.Size() + ne*E.Size() + nf);
@@ -2478,13 +2727,15 @@ void FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const
   if (nf) // face DOFs
   {
      int fbase = (var_face_dofs.Size() > 0) ? FindFaceDof(F, nf) : F*nf;
-      const int *ind = fec->GetDofOrdering(geom, order, Fo);
+      const int *ind = fec->GetDofOrdering(geom, order, oF);

      for (int j = 0; j < nf; j++)
      {
         dofs.Append(EncodeDof(nvdofs + nedofs + fbase, ind[j]));
      }
   }
+
+   return DoFTrans[mesh->GetBdrElementBaseGeometry(bel)];
 }

 int FiniteElementSpace::GetFaceDofs(int face, Array<int> &dofs,
@@ -2794,6 +3045,8 @@ void FiniteElementSpace::Destroy()
   }
   E2BFQ_array.SetSize(0);

+   DestroyDoFTrans();
+
   dof_elem_array.DeleteAll();
   dof_ldof_array.DeleteAll();

@@ -2806,7 +3059,9 @@ void FiniteElementSpace::Destroy()
   else
   {
      delete elem_dof;
+      delete elem_fos;
      delete bdr_elem_dof;
+      delete bdr_elem_fos;
      delete face_dof;

      delete [] bdofs;
@@ -2814,6 +3069,15 @@ void FiniteElementSpace::Destroy()
   ceed::RemoveBasisAndRestriction(this);
 }

+void FiniteElementSpace::DestroyDoFTrans()
+{
+   for (int i = 0; i < DoFTrans.Size(); i++)
+   {
+      delete DoFTrans[i];
+   }
+   DoFTrans.SetSize(0);
+}
+
 void FiniteElementSpace::GetTransferOperator(
   const FiniteElementSpace &coarse_fes, OperatorHandle &T) const
 {
@@ -2831,6 +3095,8 @@ void FiniteElementSpace::GetTransferOperator(
      }
      T.Reset(RefinementMatrix_main(coarse_fes.GetNDofs(),
                                    coarse_fes.GetElementToDofTable(),
+                                    coarse_fes.
+                                    GetElementToFaceOrientationTable(),
                                    localP));
   }
   else
@@ -2933,6 +3199,7 @@ void FiniteElementSpace::Update(bool want_transform)
   }

   Table* old_elem_dof = NULL;
+   Table* old_elem_fos = NULL;
   int old_ndofs;
   bool old_orders_changed = orders_changed;

@@ -2940,7 +3207,9 @@ void FiniteElementSpace::Update(bool want_transform)
   if (want_transform)
   {
      old_elem_dof = elem_dof;
+      old_elem_fos = elem_fos;
      elem_dof = NULL;
+      elem_fos = NULL;
      old_ndofs = ndofs;
   }

@@ -2966,15 +3235,18 @@ void FiniteElementSpace::Update(bool want_transform)
         {
            if (Th.Type() != Operator::MFEM_SPARSEMAT)
            {
-               Th.Reset(new RefinementOperator(this, old_elem_dof, old_ndofs));
+               Th.Reset(new RefinementOperator(this, old_elem_dof,
+                                               old_elem_fos, old_ndofs));
               // The RefinementOperator takes ownership of 'old_elem_dof', so
               // we no longer own it:
               old_elem_dof = NULL;
+               old_elem_fos = NULL;
            }
            else
            {
               // calculate fully assembled matrix
-               Th.Reset(RefinementMatrix(old_ndofs, old_elem_dof));
+               Th.Reset(RefinementMatrix(old_ndofs, old_elem_dof,
+                                         old_elem_fos));
            }
            break;
         }
@@ -2982,7 +3254,7 @@ void FiniteElementSpace::Update(bool want_transform)
         case Mesh::DEREFINE:
         {
            BuildConformingInterpolation();
-            Th.Reset(DerefinementMatrix(old_ndofs, old_elem_dof));
+            Th.Reset(DerefinementMatrix(old_ndofs, old_elem_dof, old_elem_fos));
            if (cP && cR)
            {
               Th.SetOperatorOwner(false);
@@ -2997,6 +3269,7 @@ void FiniteElementSpace::Update(bool want_transform)
      }

      delete old_elem_dof;
+      delete old_elem_fos;
   }
 }

@@ -16,6 +16,7 @@
 #include "../linalg/sparsemat.hpp"
 #include "../mesh/mesh.hpp"
 #include "fe_coll.hpp"
+#include "doftrans.hpp"
 #include "restriction.hpp"
 #include <iostream>
 #include <unordered_map>
@@ -128,7 +129,9 @@ protected:

   // precalculated DOFs for each element, boundary element, and face
   mutable Table *elem_dof; // owned (except in NURBS FE space)
+   mutable Table *elem_fos; // face orientations by element index
   mutable Table *bdr_elem_dof; // owned (except in NURBS FE space)
+   mutable Table *bdr_elem_fos; // bdr face orientations by bdr element index
   mutable Table *face_dof; // owned; in var-order space contains variant 0 DOFs

   Array<int> dof_elem_array, dof_ldof_array;
@@ -137,6 +140,9 @@ protected:
   int own_ext;
   mutable Array<int> face_to_be; // NURBS FE space only

+   Array<DofTransformation*> DoFTrans;
+   mutable VDofTransformation VDoFTrans;
+
   /** Matrix representing the prolongation from the global conforming dofs to
       a set of intermediate partially conforming dofs, e.g. the dofs associated
       with a "cut" space on a non-conforming mesh. */
@@ -189,6 +195,9 @@ protected:
   void Construct();
   void Destroy();

+   void ConstructDoFTrans();
+   void DestroyDoFTrans();
+
   void BuildElementToDofTable() const;
   void BuildBdrElementToDofTable() const;
   void BuildFaceToDofTable() const;
@@ -283,12 +292,19 @@ protected:
      const FiniteElementSpace* fespace;
      DenseTensor localP[Geometry::NumGeom];
      Table* old_elem_dof; // Owned.
+      Table* old_elem_fos; // Owned.
+
+      Array<DofTransformation*> old_DoFTrans;
+      mutable VDofTransformation old_VDoFTrans;
+
+      void ConstructDoFTrans();

   public:
      /** Construct the operator based on the elem_dof table of the original
          (coarse) space. The class takes ownership of the table. */
      RefinementOperator(const FiniteElementSpace* fespace,
-                         Table *old_elem_dof/*takes ownership*/, int old_ndofs);
+                         Table *old_elem_dof/*takes ownership*/,
+                         Table *old_elem_fos/*takes ownership*/, int old_ndofs);
      RefinementOperator(const FiniteElementSpace *fespace,
                         const FiniteElementSpace *coarse_fes);
      virtual void Mult(const Vector &x, Vector &y) const;
@@ -302,6 +318,7 @@ protected:
      const FiniteElementSpace *fine_fes; // Not owned.
      DenseTensor localR[Geometry::NumGeom];
      Table *coarse_elem_dof; // Owned.
+      // Table *coarse_elem_fos; // Owned.
      Table coarse_to_fine;
      Array<int> coarse_to_ref_type;
      Array<Geometry::Type> ref_type_to_geom;
@@ -323,6 +340,7 @@ protected:
       the same vector dimension, vdim. */
   SparseMatrix *RefinementMatrix_main(const int coarse_ndofs,
                                       const Table &coarse_elem_dof,
+                                       const Table *coarse_elem_fos,
                                       const DenseTensor localP[]) const;

   void GetLocalRefinementMatrices(Geometry::Type geom,
@@ -333,10 +351,12 @@ protected:
   /** Calculate explicit GridFunction interpolation matrix (after mesh
       refinement). NOTE: consider using the RefinementOperator class instead
       of the fully assembled matrix, which can take a lot of memory. */
-   SparseMatrix* RefinementMatrix(int old_ndofs, const Table* old_elem_dof);
+   SparseMatrix* RefinementMatrix(int old_ndofs, const Table* old_elem_dof,
+                                  const Table* old_elem_fos);

   /// Calculate GridFunction restriction matrix after mesh derefinement.
-   SparseMatrix* DerefinementMatrix(int old_ndofs, const Table* old_elem_dof);
+   SparseMatrix* DerefinementMatrix(int old_ndofs, const Table* old_elem_dof,
+                                    const Table* old_elem_fos);

   /** @brief Return in @a localP the local refinement matrices that map
       between fespaces after mesh refinement. */
@@ -614,10 +634,11 @@ public:
   int GetBdrAttribute(int i) const { return mesh->GetBdrAttribute(i); }

   /// Returns indices of degrees of freedom of element 'elem'.
-   virtual void GetElementDofs(int elem, Array<int> &dofs) const;
+   virtual DofTransformation *GetElementDofs(int elem, Array<int> &dofs) const;

   /// Returns indices of degrees of freedom for boundary element 'bel'.
-   virtual void GetBdrElementDofs(int bel, Array<int> &dofs) const;
+   virtual DofTransformation *GetBdrElementDofs(int bel,
+                                                Array<int> &dofs) const;

   /** @brief Returns the indices of the degrees of freedom for the specified
       face, including the DOFs for the edges and the vertices of the face. */
@@ -666,10 +687,10 @@ public:
   static void AdjustVDofs(Array<int> &vdofs);

   /// Returns indexes of degrees of freedom in array dofs for i'th element.
-   void GetElementVDofs(int i, Array<int> &vdofs) const;
+   DofTransformation *GetElementVDofs(int i, Array<int> &vdofs) const;

   /// Returns indexes of degrees of freedom for i'th boundary element.
-   void GetBdrElementVDofs(int i, Array<int> &vdofs) const;
+   DofTransformation *GetBdrElementVDofs(int i, Array<int> &vdofs) const;

   /// Returns indexes of degrees of freedom for i'th face element (2D and 3D).
   void GetFaceVDofs(int i, Array<int> &vdofs) const;
@@ -695,6 +716,8 @@ public:
       is preserved. */
   void ReorderElementToDofTable();

+   const Table *GetElementToFaceOrientationTable() const { return elem_fos; }
+
   /** @brief Return a reference to the internal Table that stores the lists of
       scalar dofs, for each mesh element, as returned by GetElementDofs(). */
   const Table &GetElementToDofTable() const { return *elem_dof; }
@@ -11,15 +11,19 @@

 #include "fem.hpp"
 #include "../mesh/wedge.hpp"
+#include "../mesh/pyramid.hpp"

 namespace mfem
 {

 const char *Geometry::Name[NumGeom] =
-{ "Point", "Segment", "Triangle", "Square", "Tetrahedron", "Cube", "Prism" };
+{
+   "Point", "Segment", "Triangle", "Square", "Tetrahedron", "Cube", "Prism",
+   "Pyramid"
+};

 const double Geometry::Volume[NumGeom] =
-{ 1.0, 1.0, 0.5, 1.0, 1./6, 1.0, 0.5 };
+{ 1.0, 1.0, 0.5, 1.0, 1./6, 1.0, 0.5, 1./3 };

 Geometry::Geometry()
 {
@@ -139,6 +143,28 @@ Geometry::Geometry()
   GeomVert[6]->IntPoint(5).y = 1.0;
   GeomVert[6]->IntPoint(5).z = 1.0;

+   // Vertices for Geometry::PYRAMID
+   GeomVert[7] = new IntegrationRule(5);
+   GeomVert[7]->IntPoint(0).x = 0.0;
+   GeomVert[7]->IntPoint(0).y = 0.0;
+   GeomVert[7]->IntPoint(0).z = 0.0;
+
+   GeomVert[7]->IntPoint(1).x = 1.0;
+   GeomVert[7]->IntPoint(1).y = 0.0;
+   GeomVert[7]->IntPoint(1).z = 0.0;
+
+   GeomVert[7]->IntPoint(2).x = 1.0;
+   GeomVert[7]->IntPoint(2).y = 1.0;
+   GeomVert[7]->IntPoint(2).z = 0.0;
+
+   GeomVert[7]->IntPoint(3).x = 0.0;
+   GeomVert[7]->IntPoint(3).y = 1.0;
+   GeomVert[7]->IntPoint(3).z = 0.0;
+
+   GeomVert[7]->IntPoint(4).x = 0.0;
+   GeomVert[7]->IntPoint(4).y = 0.0;
+   GeomVert[7]->IntPoint(4).z = 1.0;
+
   GeomCenter[POINT].x = 0.0;
   GeomCenter[POINT].y = 0.0;
   GeomCenter[POINT].z = 0.0;
@@ -167,6 +193,10 @@ Geometry::Geometry()
   GeomCenter[PRISM].y = 1.0 / 3.0;
   GeomCenter[PRISM].z = 0.5;

+   GeomCenter[PYRAMID].x = 0.375;
+   GeomCenter[PYRAMID].y = 0.375;
+   GeomCenter[PYRAMID].z = 0.25;
+
   GeomToPerfGeomJac[POINT]       = NULL;
   GeomToPerfGeomJac[SEGMENT]     = new DenseMatrix(1);
   GeomToPerfGeomJac[TRIANGLE]    = new DenseMatrix(2);
@@ -174,6 +204,7 @@ Geometry::Geometry()
   GeomToPerfGeomJac[TETRAHEDRON] = new DenseMatrix(3);
   GeomToPerfGeomJac[CUBE]        = new DenseMatrix(3);
   GeomToPerfGeomJac[PRISM]       = new DenseMatrix(3);
+   GeomToPerfGeomJac[PYRAMID]     = new DenseMatrix(3);

   PerfGeomToGeomJac[POINT]       = NULL;
   PerfGeomToGeomJac[SEGMENT]     = NULL;
@@ -182,6 +213,7 @@ Geometry::Geometry()
   PerfGeomToGeomJac[TETRAHEDRON] = new DenseMatrix(3);
   PerfGeomToGeomJac[CUBE]        = NULL;
   PerfGeomToGeomJac[PRISM]       = new DenseMatrix(3);
+   PerfGeomToGeomJac[PYRAMID]     = new DenseMatrix(3);

   GeomToPerfGeomJac[SEGMENT]->Diag(1.0, 1);
   {
@@ -210,6 +242,14 @@ Geometry::Geometry()
      *GeomToPerfGeomJac[PRISM] = pri_T.Jacobian();
      CalcInverse(pri_T.Jacobian(), *PerfGeomToGeomJac[PRISM]);
   }
+   {
+      IsoparametricTransformation pyr_T;
+      pyr_T.SetFE(&PyramidFE);
+      GetPerfPointMat (PYRAMID, pyr_T.GetPointMat());
+      pyr_T.SetIntPoint(&GeomCenter[PYRAMID]);
+      *GeomToPerfGeomJac[PYRAMID] = pyr_T.Jacobian();
+      CalcInverse(pyr_T.Jacobian(), *PerfGeomToGeomJac[PYRAMID]);
+   }
 }

 Geometry::~Geometry()
@@ -233,6 +273,7 @@ const IntegrationRule * Geometry::GetVertices(int GeomType)
      case Geometry::TETRAHEDRON: return GeomVert[4];
      case Geometry::CUBE:        return GeomVert[5];
      case Geometry::PRISM:       return GeomVert[6];
+      case Geometry::PYRAMID:     return GeomVert[7];
      default:
         mfem_error ("Geometry::GetVertices(...)");
   }
@@ -310,6 +351,25 @@ void Geometry::GetRandomPoint(int GeomType, IntegrationPoint &ip)
            ip.y = 1.0 - ip.y;
         }
         break;
+      case Geometry::PYRAMID:
+         ip.x = double(rand()) / RAND_MAX;
+         ip.y = double(rand()) / RAND_MAX;
+         ip.z = double(rand()) / RAND_MAX;
+         if (ip.x + ip.z > 1.0 && ip.y < ip.x)
+         {
+            double x = ip.x;
+            ip.x = ip.y;
+            ip.y = 1.0 - ip.z;
+            ip.z = 1.0 - x;
+         }
+         else if (ip.y + ip.z > 1.0)
+         {
+            double z = ip.z;
+            ip.z = 1.0 - ip.y;
+            ip.y = ip.x;
+            ip.x = 1.0 - z;
+         }
+         break;
      default:
         MFEM_ABORT("Unknown type of reference element!");
   }
@@ -371,6 +431,10 @@ bool Geometry::CheckPoint(int GeomType, const IntegrationPoint &ip)
         if (ip.x < 0.0 || ip.y < 0.0 || ip.x+ip.y > 1.0 ||
             ip.z < 0.0 || ip.z > 1.0) { return false; }
         break;
+      case Geometry::PYRAMID:
+         if (ip.x < 0.0 || ip.y < 0.0 || ip.x+ip.z > 1.0 || ip.y+ip.z > 1.0 ||
+             ip.z < 0.0 || ip.z > 1.0) { return false; }
+         break;
      default:
         MFEM_ABORT("Unknown type of reference element!");
   }
@@ -441,6 +505,17 @@ bool Geometry::CheckPoint(int GeomType, const IntegrationPoint &ip, double eps)
            return false;
         }
         break;
+      case Geometry::PYRAMID:
+         if (internal::FuzzyLT(ip.x, 0.0, eps)
+             || internal::FuzzyLT(ip.y, 0.0, eps)
+             || internal::FuzzyGT(ip.x+ip.z, 1.0, eps)
+             || internal::FuzzyGT(ip.y+ip.z, 1.0, eps)
+             || internal::FuzzyLT(ip.z, 0.0, eps)
+             || internal::FuzzyGT(ip.z, 1.0, eps) )
+         {
+            return false;
+         }
+         break;
      default:
         MFEM_ABORT("Unknown type of reference element!");
   }
@@ -555,6 +630,16 @@ bool Geometry::ProjectPoint(int GeomType, const IntegrationPoint &beg,
         double lbeg[5] = { beg.x, beg.y, beg.z, 1.0-beg.x-beg.y, 1.0-beg.z };
         return internal::IntersectSegment<5,3>(lbeg, lend, end);
      }
+      case Geometry::PYRAMID:
+      {
+         double lend[6] = { end.x, end.y, end.z,
+                            1.0-end.x-end.z, 1.0-end.y-end.z, 1.0-end.z
+                          };
+         double lbeg[6] = { beg.x, beg.y, beg.z,
+                            1.0-beg.x-beg.z, 1.0-beg.y-beg.z, 1.0-beg.z
+                          };
+         return internal::IntersectSegment<6,3>(lbeg, lend, end);
+      }
      default:
         MFEM_ABORT("Unknown type of reference element!");
   }
@@ -652,6 +737,43 @@ bool Geometry::ProjectPoint(int GeomType, IntegrationPoint &ip)
         return in_tri && in_z;
      }

+      case PYRAMID:
+      {
+         if (ip.x < 0.0)
+         {
+            ip.x = 0.0;
+            internal::ProjectTriangle(ip.y, ip.z);
+            return false;
+         }
+         if (ip.y < 0.0)
+         {
+            ip.y = 0.0;
+            internal::ProjectTriangle(ip.x, ip.z);
+            return false;
+         }
+         if (ip.z < 0.0)
+         {
+            ip.z = 0.0;
+            if (ip.x > 1.0) { ip.x = 1.0; }
+            if (ip.y > 1.0) { ip.y = 1.0; }
+            return false;
+         }
+         if (ip.x >= ip.y)
+         {
+            bool in_y = true;
+            bool in_tri = internal::ProjectTriangle(ip.x, ip.z);
+            if (ip.y > ip.z) { in_y = false; ip.y = ip.z; }
+            return in_tri && in_y;
+         }
+         else
+         {
+            bool in_x = true;
+            bool in_tri = internal::ProjectTriangle(ip.y, ip.z);
+            if (ip.x > ip.z) { in_x = false; ip.x = ip.z; }
+            return in_tri && in_x;
+         }
+      }
+
      default:
         MFEM_ABORT("Reference element type is not supported!");
   }
@@ -726,6 +848,17 @@ void Geometry::GetPerfPointMat(int GeomType, DenseMatrix &pm)
      }
      break;

+      case Geometry::PYRAMID:
+      {
+         pm.SetSize (3, 5);
+         pm(0,0) = 0.0;  pm(1,0) = 0.0;  pm(2,0) = 0.0;
+         pm(0,1) = 1.0;  pm(1,1) = 0.0;  pm(2,1) = 0.0;
+         pm(0,2) = 1.0;  pm(1,2) = 1.0;  pm(2,2) = 0.0;
+         pm(0,3) = 0.0;  pm(1,3) = 1.0;  pm(2,3) = 0.0;
+         pm(0,4) = 0.5;  pm(1,4) = 0.5;  pm(2,4) = 0.7071067811865475;
+      }
+      break;
+
      default:
         mfem_error ("Geometry::GetPerfPointMat (...)");
   }
@@ -744,13 +877,13 @@ void Geometry::JacToPerfJac(int GeomType, const DenseMatrix &J,
   }
 }

-const int Geometry::NumBdrArray[NumGeom] = { 0, 2, 3, 4, 4, 6, 5 };
-const int Geometry::Dimension[NumGeom] = { 0, 1, 2, 2, 3, 3, 3 };
+const int Geometry::NumBdrArray[NumGeom] = { 0, 2, 3, 4, 4, 6, 5, 5 };
+const int Geometry::Dimension[NumGeom] = { 0, 1, 2, 2, 3, 3, 3, 3 };
 const int Geometry::DimStart[MaxDim+2] =
 { POINT, SEGMENT, TRIANGLE, TETRAHEDRON, NUM_GEOMETRIES };
-const int Geometry::NumVerts[NumGeom] = { 1, 2, 3, 4, 4, 8, 6 };
-const int Geometry::NumEdges[NumGeom] = { 0, 1, 3, 4, 6, 12, 9 };
-const int Geometry::NumFaces[NumGeom] = { 0, 0, 1, 1, 4, 6, 5 };
+const int Geometry::NumVerts[NumGeom] = { 1, 2, 3, 4, 4, 8, 6, 5 };
+const int Geometry::NumEdges[NumGeom] = { 0, 1, 3, 4, 6, 12, 9, 8 };
+const int Geometry::NumFaces[NumGeom] = { 0, 0, 1, 1, 4, 6, 5, 5 };

 const int Geometry::
 Constants<Geometry::POINT>::Orient[1][1] = {{0}};
@@ -897,6 +1030,30 @@ Constants<Geometry::PRISM>::VertToVert::J[9][2] =
   {5, 4}                   // 4,5:4
 };

+const int Geometry::
+Constants<Geometry::PYRAMID>::Edges[8][2] =
+{{0, 1}, {1, 2}, {3, 2}, {0, 3}, {0, 4}, {1, 4}, {2, 4}, {3, 4}};
+const int Geometry::
+Constants<Geometry::PYRAMID>::FaceTypes[5] =
+{
+   Geometry::SQUARE,
+   Geometry::TRIANGLE, Geometry::TRIANGLE,
+   Geometry::TRIANGLE, Geometry::TRIANGLE
+};
+const int Geometry::
+Constants<Geometry::PYRAMID>::FaceVert[5][4] =
+{{3, 2, 1, 0}, {0, 1, 4, -1}, {1, 2, 4, -1}, {2, 3, 4, -1}, {3, 0, 4, -1}};
+const int Geometry::
+Constants<Geometry::PYRAMID>::VertToVert::I[5] = {0, 3, 5, 7, 8};
+const int Geometry::
+Constants<Geometry::PYRAMID>::VertToVert::J[8][2] =
+{
+   {1, 0}, {3, 3}, {4, 4}, // 0,1:0   0,3:3  0,4:4
+   {2, 1}, {4, 5},         // 1,2:1   1,4:5
+   {3,-3}, {4, 6},         // 2,3:-3  2,4:6
+   {4, 7}                  // 3,4:7
+};
+

 GeometryRefiner::GeometryRefiner()
 {
@@ -1262,6 +1419,104 @@ RefinedGeometry * GeometryRefiner::Refine(Geometry::Type Geom,
         return RG;
      }

+      case Geometry::PYRAMID:
+      {
+         const int n = Times;
+         RG = new RefinedGeometry ((n+1)*(n+2)*(2*n+3)/6,
+                                   5*n*(2*n-1)*(2*n+1)/3, 0);
+         RG->Times = Times;
+         RG->ETimes = ETimes;
+         RG->Type = type;
+         // enumerate and define the vertices
+         m = 0;
+         for (k = 0; k <= n; k++)
+         {
+            const double *cpij =
+               poly1d.GetPoints(Times - k, BasisType::GetNodalBasis(type));
+            for (j = 0; j <= n - k; j++)
+               for (i = 0; i <= n - k; i++)
+               {
+                  IntegrationPoint &ip = RG->RefPts.IntPoint(m);
+                  if (type == 0)
+                  {
+                     ip.x = (n > k) ? (double(i) / (n - k)) : 0.0;
+                     ip.y = (n > k) ? (double(j) / (n - k)) : 0.0;
+                     ip.z = double(k) / n;
+                  }
+                  else
+                  {
+                     ip.x = cpij[i] * (1.0 - cp[k]);
+                     ip.y = cpij[j] * (1.0 - cp[k]);
+                     ip.z = cp[k];
+                  }
+                  m++;
+               }
+         }
+         if (m != (n+1)*(n+2)*(2*n+3)/6)
+         {
+            mfem_error("GeometryRefiner::Refine() for PYRAMID #1");
+         }
+         // elements
+         Array<int> &G = RG->RefGeoms;
+         m = 0;
+         for (k = 0; k < n; k++)
+         {
+            int lk = k * (k * (2 * k - 6 * n - 9) + 6 * n * (n + 3) + 13) / 6;
+            int lkp1 = (k + 1) *
+                       (k * (2 * k - 6 * n -5) + 6 * n * (n + 2) + 6) / 6;
+            for (j = 0; j < n - k; j++)
+            {
+               for (i = 0; i < n - k; i++)
+               {
+                  G[m++] = lk + j * (n - k + 1) + i;
+                  G[m++] = lk + j * (n - k + 1) + i + 1;
+                  G[m++] = lk + (j + 1) * (n - k + 1) + i + 1;
+                  G[m++] = lk + (j + 1) * (n - k + 1) + i;
+                  G[m++] = lkp1 + j * (n - k) + i;
+               }
+            }
+            for (j = 0; j < n - k - 1; j++)
+            {
+               for (i = 0; i < n - k - 1; i++)
+               {
+                  G[m++] = lkp1 + j * (n - k) + i;
+                  G[m++] = lkp1 + (j + 1) * (n - k) + i;
+                  G[m++] = lkp1 + (j + 1) * (n - k) + i + 1;
+                  G[m++] = lkp1 + j * (n - k) + i + 1;
+                  G[m++] = lk + (j + 1) * (n - k + 1) + i + 1;
+               }
+            }
+            for (j = 0; j < n - k; j++)
+            {
+               for (i = 0; i < n - k - 1; i++)
+               {
+                  G[m++] = lk + j * (n - k + 1) + i + 1;
+                  G[m++] = lk + (j + 1) * (n - k + 1) + i + 1;
+                  G[m++] = lkp1 + j * (n - k) + i;
+                  G[m++] = lkp1 + j * (n - k) + i + 1;
+                  G[m++] = -1;
+               }
+            }
+            for (j = 0; j < n - k - 1; j++)
+            {
+               for (i = 0; i < n - k; i++)
+               {
+                  G[m++] = lk + (j + 1) * (n - k + 1) + i;
+                  G[m++] = lk + (j + 1) * (n - k + 1) + i + 1;
+                  G[m++] = lkp1 + (j + 1) * (n - k) + i;
+                  G[m++] = lkp1 + j * (n - k) + i;
+                  G[m++] = -1;
+               }
+            }
+         }
+         if (m != 5*n*(2*n-1)*(2*n+1)/3)
+         {
+            mfem_error("GeometryRefiner::Refine() for PYRAMID #2");
+         }
+         RGeom[Geometry::PYRAMID].Append(RG);
+         return RG;
+      }
+
      case Geometry::PRISM:
      {
         const int n = Times;
@@ -27,6 +27,7 @@ namespace mfem
    Geometry::TETRAHEDRON - w/ vert. (0,0,0),(1,0,0),(0,1,0),(0,0,1)
    Geometry::CUBE - the unit cube
    Geometry::PRISM - w/ vert. (0,0,0),(1,0,0),(0,1,0),(0,0,1),(1,0,1),(0,1,1)
+    Geometry::PYRAMID - w/ vert. (0,0,0),(1,0,0),(1,1,0),(0,1,0),(0,0,1)
 */
 class Geometry
 {
@@ -34,7 +35,7 @@ public:
   enum Type
   {
      INVALID = -1,
-      POINT = 0, SEGMENT, TRIANGLE, SQUARE, TETRAHEDRON, CUBE, PRISM,
+      POINT = 0, SEGMENT, TRIANGLE, SQUARE, TETRAHEDRON, CUBE, PRISM, PYRAMID,
      NUM_GEOMETRIES
   };

@@ -251,7 +252,26 @@ template <> struct Geometry::Constants<Geometry::PRISM>
   };
 };

-// Defined in fe.cpp to ensure construction after 'mfem::WedgeFE'.
+template <> struct Geometry::Constants<Geometry::PYRAMID>
+{
+   static const int Dimension = 3;
+   static const int NumVert = 5;
+   static const int NumEdges = 8;
+   static const int Edges[NumEdges][2];
+   static const int NumFaces = 5;
+   static const int FaceTypes[NumFaces];
+   static const int MaxFaceVert = 4;
+   static const int FaceVert[NumFaces][MaxFaceVert];
+   // Upper-triangular part of the local vertex-to-vertex graph.
+   struct VertToVert
+   {
+      static const int I[NumVert];
+      static const int J[NumEdges][2]; // {end,edge_idx}
+   };
+};
+
+// Defined in fe.cpp to ensure construction after 'mfem::TriangleFE' and
+// `mfem::TetrahedronFE`.
 extern Geometry Geometries;


@@ -255,6 +255,8 @@ void GridFunction::SumFluxAndCount(BilinearFormIntegrator &blfi,
   GridFunction &u = *this;

   ElementTransformation *Transf;
+   DofTransformation *udoftrans;
+   DofTransformation *fdoftrans;

   FiniteElementSpace *ufes = u.FESpace();
   FiniteElementSpace *ffes = flux.FESpace();
@@ -274,15 +276,23 @@ void GridFunction::SumFluxAndCount(BilinearFormIntegrator &blfi,
         continue;
      }

-      ufes->GetElementVDofs(i, udofs);
-      ffes->GetElementVDofs(i, fdofs);
+      udoftrans = ufes->GetElementVDofs(i, udofs);
+      fdoftrans = ffes->GetElementVDofs(i, fdofs);

      u.GetSubVector(udofs, ul);
+      if (udoftrans)
+      {
+         udoftrans->InvTransformPrimal(ul);
+      }

      Transf = ufes->GetElementTransformation(i);
      blfi.ComputeElementFlux(*ufes->GetFE(i), *Transf, ul,
                              *ffes->GetFE(i), fl, wcoef);

+      if (fdoftrans)
+      {
+         fdoftrans->TransformPrimal(fl);
+      }
      flux.AddElementVector(fdofs, fl);

      FiniteElementSpace::AdjustVDofs(fdofs);
@@ -364,7 +374,7 @@ void GridFunction::GetNodalValues(int i, Array<double> &nval, int vdim) const

   int k;

-   fes->GetElementVDofs(i, vdofs);
+   DofTransformation * doftrans = fes->GetElementVDofs(i, vdofs);
   const FiniteElement *FElem = fes->GetFE(i);
   const IntegrationRule *ElemVert =
      Geometries.GetVertices(FElem->GetGeomType());
@@ -374,6 +384,10 @@ void GridFunction::GetNodalValues(int i, Array<double> &nval, int vdim) const
   vdim--;
   Vector loc_data;
   GetSubVector(vdofs, loc_data);
+   if (doftrans)
+   {
+      doftrans->InvTransformPrimal(loc_data);
+   }

   if (FElem->GetRangeType() == FiniteElement::SCALAR)
   {
@@ -403,7 +417,7 @@ double GridFunction::GetValue(int i, const IntegrationPoint &ip, int vdim)
 const
 {
   Array<int> dofs;
-   fes->GetElementDofs(i, dofs);
+   DofTransformation * doftrans = fes->GetElementDofs(i, dofs);
   fes->DofsToVDofs(vdim-1, dofs);
   Vector DofVal(dofs.Size()), LocVec;
   const FiniteElement *fe = fes->GetFE(i);
@@ -418,6 +432,10 @@ const
      fe->CalcPhysShape(*Tr, DofVal);
   }
   GetSubVector(dofs, LocVec);
+   if (doftrans)
+   {
+      doftrans->InvTransformPrimal(LocVec);
+   }

   return (DofVal * LocVec);
 }
@@ -428,9 +446,13 @@ void GridFunction::GetVectorValue(int i, const IntegrationPoint &ip,
   const FiniteElement *FElem = fes->GetFE(i);
   int dof = FElem->GetDof();
   Array<int> vdofs;
-   fes->GetElementVDofs(i, vdofs);
+   DofTransformation * doftrans = fes->GetElementVDofs(i, vdofs);
   Vector loc_data;
   GetSubVector(vdofs, loc_data);
+   if (doftrans)
+   {
+      doftrans->InvTransformPrimal(loc_data);
+   }
   if (FElem->GetRangeType() == FiniteElement::SCALAR)
   {
      Vector shape(dof);
@@ -470,12 +492,16 @@ const
   Array<int> dofs;
   int n = ir.GetNPoints();
   vals.SetSize(n);
-   fes->GetElementDofs(i, dofs);
+   DofTransformation * doftrans = fes->GetElementDofs(i, dofs);
   fes->DofsToVDofs(vdim-1, dofs);
   const FiniteElement *FElem = fes->GetFE(i);
   int dof = FElem->GetDof();
   Vector DofVal(dof), loc_data(dof);
   GetSubVector(dofs, loc_data);
+   if (doftrans)
+   {
+      doftrans->InvTransformPrimal(loc_data);
+   }
   if (FElem->GetMapType() == FiniteElement::VALUE)
   {
      for (int k = 0; k < n; k++)
@@ -861,11 +887,12 @@ void GridFunction::GetVectorValue(ElementTransformation &T,

   Array<int> vdofs;
   const FiniteElement *fe = NULL;
+   DofTransformation * doftrans = NULL;

   switch (T.ElementType)
   {
      case ElementTransformation::ELEMENT:
-         fes->GetElementVDofs(T.ElementNo, vdofs);
+         doftrans = fes->GetElementVDofs(T.ElementNo, vdofs);
         fe = fes->GetFE(T.ElementNo);
         break;
      case ElementTransformation::EDGE:
@@ -956,6 +983,10 @@ void GridFunction::GetVectorValue(ElementTransformation &T,
   int dof = fe->GetDof();
   Vector loc_data;
   GetSubVector(vdofs, loc_data);
+   if (doftrans)
+   {
+      doftrans->InvTransformPrimal(loc_data);
+   }
   if (fe->GetRangeType() == FiniteElement::SCALAR)
   {
      Vector shape(dof);
@@ -998,10 +1029,14 @@ void GridFunction::GetVectorValues(ElementTransformation &T,
   int dof = FElem->GetDof();

   Array<int> vdofs;
-   fes->GetElementVDofs(T.ElementNo, vdofs);
-
+   DofTransformation * doftrans = fes->GetElementVDofs(T.ElementNo, vdofs);
   Vector loc_data;
   GetSubVector(vdofs, loc_data);
+   if (doftrans)
+   {
+      doftrans->InvTransformPrimal(loc_data);
+   }
+
   int nip = ir.GetNPoints();

   if (FElem->GetRangeType() == FiniteElement::SCALAR)
@@ -1089,6 +1124,8 @@ void GridFunction::GetValuesFrom(const GridFunction &orig_func)
   // Without averaging ...

   const FiniteElementSpace *orig_fes = orig_func.FESpace();
+   DofTransformation * doftrans;
+   DofTransformation * orig_doftrans;
   Array<int> vdofs, orig_vdofs;
   Vector shape, loc_values, orig_loc_values;
   int i, j, d, ne, dof, odof, vdim;
@@ -1097,9 +1134,13 @@ void GridFunction::GetValuesFrom(const GridFunction &orig_func)
   vdim = fes->GetVDim();
   for (i = 0; i < ne; i++)
   {
-      fes->GetElementVDofs(i, vdofs);
-      orig_fes->GetElementVDofs(i, orig_vdofs);
+      doftrans = fes->GetElementVDofs(i, vdofs);
+      orig_doftrans = orig_fes->GetElementVDofs(i, orig_vdofs);
      orig_func.GetSubVector(orig_vdofs, orig_loc_values);
+      if (orig_doftrans)
+      {
+         orig_doftrans->InvTransformPrimal(orig_loc_values);
+      }
      const FiniteElement *fe = fes->GetFE(i);
      const FiniteElement *orig_fe = orig_fes->GetFE(i);
      dof = fe->GetDof();
@@ -1117,6 +1158,10 @@ void GridFunction::GetValuesFrom(const GridFunction &orig_func)
               shape * ((const double *)orig_loc_values + d * odof) ;
         }
      }
+      if (doftrans)
+      {
+         doftrans->TransformPrimal(loc_values);
+      }
      SetSubVector(vdofs, loc_values);
   }
 }
@@ -1126,8 +1171,10 @@ void GridFunction::GetBdrValuesFrom(const GridFunction &orig_func)
   // Without averaging ...

   const FiniteElementSpace *orig_fes = orig_func.FESpace();
+   // DofTransformation * doftrans;
+   // DofTransformation * orig_doftrans;
   Array<int> vdofs, orig_vdofs;
-   Vector shape, loc_values, orig_loc_values;
+   Vector shape, loc_values, loc_values_t, orig_loc_values, orig_loc_values_t;
   int i, j, d, nbe, dof, odof, vdim;

   nbe = fes->GetNBE();
@@ -1165,37 +1212,33 @@ void GridFunction::GetVectorFieldValues(
   Array<int> vdofs;
   ElementTransformation *transf;

-   int d, j, k, n, sdim, dof, ind;
+   int d, k, n, sdim, dof;

   n = ir.GetNPoints();
-   fes->GetElementVDofs(i, vdofs);
+   DofTransformation * doftrans = fes->GetElementVDofs(i, vdofs);
   const FiniteElement *fe = fes->GetFE(i);
   dof = fe->GetDof();
   sdim = fes->GetMesh()->SpaceDimension();
-   int *dofs = &vdofs[comp*dof];
+   // int *dofs = &vdofs[comp*dof];
   transf = fes->GetElementTransformation(i);
   transf->Transform(ir, tr);
   vals.SetSize(n, sdim);
   DenseMatrix vshape(dof, sdim);
-   double a;
+   Vector loc_data, val(sdim);
+   GetSubVector(vdofs, loc_data);
+   if (doftrans)
+   {
+      doftrans->InvTransformPrimal(loc_data);
+   }
   for (k = 0; k < n; k++)
   {
      const IntegrationPoint &ip = ir.IntPoint(k);
      transf->SetIntPoint(&ip);
      fe->CalcVShape(*transf, vshape);
+      vshape.MultTranspose(loc_data, val);
      for (d = 0; d < sdim; d++)
      {
-         a = 0.0;
-         for (j = 0; j < dof; j++)
-            if ( (ind=dofs[j]) >= 0 )
-            {
-               a += vshape(j, d) * data[ind];
-            }
-            else
-            {
-               a -= vshape(j, d) * data[-1-ind];
-            }
-         vals(k, d) = a;
+         vals(k,d) = val(d);
      }
   }
 }
@@ -1365,9 +1408,13 @@ void GridFunction::GetVectorGradientHat(
   const FiniteElement *FElem = fes->GetFE(elNo);
   int dim = FElem->GetDim(), dof = FElem->GetDof();
   Array<int> vdofs;
-   fes->GetElementVDofs(elNo, vdofs);
+   DofTransformation * doftrans = fes->GetElementVDofs(elNo, vdofs);
   Vector loc_data;
   GetSubVector(vdofs, loc_data);
+   if (doftrans)
+   {
+      doftrans->InvTransformPrimal(loc_data);
+   }
   // assuming scalar FE
   int vdim = fes->GetVDim();
   DenseMatrix dshape(dof, dim);
@@ -1406,9 +1453,13 @@ double GridFunction::GetDivergence(ElementTransformation &T) const
         {
            // Assuming RT-type space
            Array<int> dofs;
-            fes->GetElementDofs(elNo, dofs);
+            DofTransformation * doftrans = fes->GetElementDofs(elNo, dofs);
            Vector loc_data, divshape(fe->GetDof());
            GetSubVector(dofs, loc_data);
+            if (doftrans)
+            {
+               doftrans->InvTransformPrimal(loc_data);
+            }
            fe->CalcDivShape(T.GetIntPoint(), divshape);
            return (loc_data * divshape) / T.Weight();
         }
@@ -1499,9 +1550,13 @@ void GridFunction::GetCurl(ElementTransformation &T, Vector &curl) const
         {
            // Assuming ND-type space
            Array<int> dofs;
-            fes->GetElementDofs(elNo, dofs);
+            DofTransformation * doftrans = fes->GetElementDofs(elNo, dofs);
            Vector loc_data;
            GetSubVector(dofs, loc_data);
+            if (doftrans)
+            {
+               doftrans->InvTransformPrimal(loc_data);
+            }
            DenseMatrix curl_shape(fe->GetDof(), fe->GetDim() == 3 ? 3 : 1);
            fe->CalcCurlShape(T.GetIntPoint(), curl_shape);
            curl.SetSize(curl_shape.Width());
@@ -1643,8 +1698,12 @@ void GridFunction::GetGradients(ElementTransformation &tr,
   DenseMatrix dshape(fe->GetDof(), fe->GetDim());
   Vector lval, gh(fe->GetDim()), gcol;
   Array<int> dofs;
-   fes->GetElementDofs(elNo, dofs);
+   DofTransformation * doftrans = fes->GetElementDofs(elNo, dofs);
   GetSubVector(dofs, lval);
+   if (doftrans)
+   {
+      doftrans->InvTransformPrimal(lval);
+   }
   grad.SetSize(fe->GetDim(), ir.GetNPoints());
   for (int i = 0; i < ir.GetNPoints(); i++)
   {
@@ -1724,6 +1783,8 @@ void GridFunction::GetElementAverages(GridFunction &avgs) const
 {
   MassIntegrator Mi;
   DenseMatrix loc_mass;
+   DofTransformation * te_doftrans;
+   DofTransformation * tr_doftrans;
   Array<int> te_dofs, tr_dofs;
   Vector loc_avgs, loc_this;
   Vector int_psi(avgs.Size());
@@ -1734,11 +1795,19 @@ void GridFunction::GetElementAverages(GridFunction &avgs) const
   {
      Mi.AssembleElementMatrix2(*fes->GetFE(i), *avgs.FESpace()->GetFE(i),
                                *fes->GetElementTransformation(i), loc_mass);
-      fes->GetElementDofs(i, tr_dofs);
-      avgs.FESpace()->GetElementDofs(i, te_dofs);
+      tr_doftrans = fes->GetElementDofs(i, tr_dofs);
+      te_doftrans = avgs.FESpace()->GetElementDofs(i, te_dofs);
      GetSubVector(tr_dofs, loc_this);
+      if (tr_doftrans)
+      {
+         tr_doftrans->InvTransformPrimal(loc_this);
+      }
      loc_avgs.SetSize(te_dofs.Size());
      loc_mass.Mult(loc_this, loc_avgs);
+      if (te_doftrans)
+      {
+         te_doftrans->TransformPrimal(loc_avgs);
+      }
      avgs.AddElementVector(te_dofs, loc_avgs);
      loc_this = 1.0; // assume the local basis for 'this' sums to 1
      loc_mass.Mult(loc_this, loc_avgs);
@@ -1753,8 +1822,12 @@ void GridFunction::GetElementAverages(GridFunction &avgs) const
 void GridFunction::GetElementDofValues(int el, Vector &dof_vals) const
 {
   Array<int> dof_idx;
-   fes->GetElementVDofs(el, dof_idx);
+   DofTransformation * doftrans = fes->GetElementVDofs(el, dof_idx);
   GetSubVector(dof_idx, dof_vals);
+   if (doftrans)
+   {
+      doftrans->InvTransformPrimal(dof_vals);
+   }
 }

 void GridFunction::ProjectGridFunction(const GridFunction &src)
@@ -1790,13 +1863,21 @@ void GridFunction::ProjectGridFunction(const GridFunction &src)
         cached_geom = geom;
      }

-      src.fes->GetElementVDofs(i, src_vdofs);
+      DofTransformation * src_doftrans = src.fes->GetElementVDofs(i, src_vdofs);
      src.GetSubVector(src_vdofs, src_lvec);
+      if (src_doftrans)
+      {
+         src_doftrans->InvTransformPrimal(src_lvec);
+      }
      for (int vd = 0; vd < vdim; vd++)
      {
         P.Mult(&src_lvec[vd*P.Width()], &dest_lvec[vd*P.Height()]);
      }
-      fes->GetElementVDofs(i, dest_vdofs);
+      DofTransformation * doftrans = fes->GetElementVDofs(i, dest_vdofs);
+      if (doftrans)
+      {
+         doftrans->TransformPrimal(dest_lvec);
+      }
      SetSubVector(dest_vdofs, dest_lvec);
   }
 }
@@ -1805,10 +1886,15 @@ void GridFunction::ImposeBounds(int i, const Vector &weights,
                                const Vector &lo_, const Vector &hi_)
 {
   Array<int> vdofs;
-   fes->GetElementVDofs(i, vdofs);
+   DofTransformation * doftrans = fes->GetElementVDofs(i, vdofs);
   int size = vdofs.Size();
   Vector vals, new_vals(size);
+
   GetSubVector(vdofs, vals);
+   if (doftrans)
+   {
+      doftrans->InvTransformPrimal(vals);
+   }

   MFEM_ASSERT(weights.Size() == size, "Different # of weights and dofs.");
   MFEM_ASSERT(lo_.Size() == size, "Different # of lower bounds and dofs.");
@@ -1825,6 +1911,10 @@ void GridFunction::ImposeBounds(int i, const Vector &weights,
   slbqp.SetPrintLevel(0); // print messages only if not converged
   slbqp.Mult(vals, new_vals);

+   if (doftrans)
+   {
+      doftrans->TransformPrimal(new_vals);
+   }
   SetSubVector(vdofs, new_vals);
 }

@@ -1832,10 +1922,14 @@ void GridFunction::ImposeBounds(int i, const Vector &weights,
                                double min_, double max_)
 {
   Array<int> vdofs;
-   fes->GetElementVDofs(i, vdofs);
+   DofTransformation * doftrans = fes->GetElementVDofs(i, vdofs);
   int size = vdofs.Size();
   Vector vals, new_vals(size);
   GetSubVector(vdofs, vals);
+   if (doftrans)
+   {
+      doftrans->InvTransformPrimal(vals);
+   }

   double max_val = vals.Max();
   double min_val = vals.Min();
@@ -1843,6 +1937,10 @@ void GridFunction::ImposeBounds(int i, const Vector &weights,
   if (max_val <= min_)
   {
      new_vals = min_;
+      if (doftrans)
+      {
+         doftrans->TransformPrimal(new_vals);
+      }
      SetSubVector(vdofs, new_vals);
      return;
   }
@@ -2278,6 +2376,7 @@ void GridFunction::ProjectDeltaCoefficient(DeltaCoefficient &delta_coeff,
 void GridFunction::ProjectCoefficient(Coefficient &coeff)
 {
   DeltaCoefficient *delta_c = dynamic_cast<DeltaCoefficient *>(&coeff);
+   DofTransformation * doftrans = NULL;

   if (delta_c == NULL)
   {
@@ -2286,9 +2385,13 @@ void GridFunction::ProjectCoefficient(Coefficient &coeff)

      for (int i = 0; i < fes->GetNE(); i++)
      {
-         fes->GetElementVDofs(i, vdofs);
+         doftrans = fes->GetElementVDofs(i, vdofs);
         vals.SetSize(vdofs.Size());
         fes->GetFE(i)->Project(coeff, *fes->GetElementTransformation(i), vals);
+         if (doftrans)
+         {
+            doftrans->TransformPrimal(vals);
+         }
         SetSubVector(vdofs, vals);
      }
   }
@@ -2334,11 +2437,17 @@ void GridFunction::ProjectCoefficient(VectorCoefficient &vcoeff)
   Array<int> vdofs;
   Vector vals;

+   DofTransformation * doftrans = NULL;
+
   for (i = 0; i < fes->GetNE(); i++)
   {
-      fes->GetElementVDofs(i, vdofs);
+      doftrans = fes->GetElementVDofs(i, vdofs);
      vals.SetSize(vdofs.Size());
      fes->GetFE(i)->Project(vcoeff, *fes->GetElementTransformation(i), vals);
+      if (doftrans)
+      {
+         doftrans->TransformPrimal(vals);
+      }
      SetSubVector(vdofs, vals);
   }
 }
@@ -2401,6 +2510,7 @@ void GridFunction::ProjectCoefficient(Coefficient *coeff[])
   double val;
   const FiniteElement *fe;
   ElementTransformation *transf;
+   // DofTransformation * doftrans;
   Array<int> vdofs;

   vdim = fes->GetVDim();
@@ -2410,6 +2520,7 @@ void GridFunction::ProjectCoefficient(Coefficient *coeff[])
      fdof = fe->GetDof();
      transf = fes->GetElementTransformation(i);
      const IntegrationRule &ir = fe->GetNodes();
+      // doftrans = fes->GetElementVDofs(i, vdofs);
      fes->GetElementVDofs(i, vdofs);
      for (j = 0; j < fdof; j++)
      {
@@ -95,6 +95,12 @@ public:
      : Vector(data, f->GetVSize())
   { fes = f; fec = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }

+   /** @brief Construct a GridFunction using previously allocated Vector @a base
+       starting at the given offset, @a base_offset. */
+   GridFunction(FiniteElementSpace *f, Vector &base, int base_offset = 0)
+      : Vector(base, base_offset, f->GetVSize())
+   { fes = f; fec = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }
+
   /// Construct a GridFunction on the given Mesh, using the data from @a input.
   /** The content of @a input should be in the format created by the method
       Save(). The reconstructed FiniteElementSpace and FiniteElementCollection
@@ -610,7 +610,12 @@ void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
      {
         if (gsl_code[i] == 1) { indl2.Append(i); }
      }
-      if (indl2.Size() == 0) { return; } // no points on element borders
+      int borderPts = indl2.Size();
+#ifdef MFEM_USE_MPI
+      MPI_Allreduce(MPI_IN_PLACE, &borderPts, 1, MPI_INT, MPI_SUM, gsl_comm->c);
+#endif
+      if (borderPts == 0) { return; } // no points on element borders
+

      Vector field_out_l2(field_out.Size());
      VectorGridFunctionCoefficient field_in_dg(&field_in);
@@ -910,6 +910,9 @@ IntegrationRules::IntegrationRules(int Ref, int type_):
   TetrahedronIntRules.SetSize(32, h_mt);
   TetrahedronIntRules = NULL;

+   PyramidIntRules.SetSize(32, h_mt);
+   PyramidIntRules = NULL;
+
   PrismIntRules.SetSize(32, h_mt);
   PrismIntRules = NULL;

@@ -930,6 +933,7 @@ const IntegrationRule &IntegrationRules::Get(int GeomType, int Order)
      case Geometry::TETRAHEDRON: ir_array = &TetrahedronIntRules; break;
      case Geometry::CUBE:        ir_array = &CubeIntRules; break;
      case Geometry::PRISM:       ir_array = &PrismIntRules; break;
+      case Geometry::PYRAMID:     ir_array = &PyramidIntRules; break;
      default:
         mfem_error("IntegrationRules::Get(...) : Unknown geometry type!");
         ir_array = NULL;
@@ -951,7 +955,7 @@ const IntegrationRule &IntegrationRules::Get(int GeomType, int Order)
            IntegrationRule *ir = GenerateIntegrationRule(GeomType, Order);
            int RealOrder = Order;
            while (RealOrder+1 < ir_array->Size() &&
-            /*  */ (*ir_array)[RealOrder+1] == ir)
+                   (*ir_array)[RealOrder+1] == ir)
            {
               RealOrder++;
            }
@@ -976,6 +980,7 @@ void IntegrationRules::Set(int GeomType, int Order, IntegrationRule &IntRule)
      case Geometry::TETRAHEDRON: ir_array = &TetrahedronIntRules; break;
      case Geometry::CUBE:        ir_array = &CubeIntRules; break;
      case Geometry::PRISM:       ir_array = &PrismIntRules; break;
+      case Geometry::PYRAMID:     ir_array = &PyramidIntRules; break;
      default:
         mfem_error("IntegrationRules::Set(...) : Unknown geometry type!");
         ir_array = NULL;
@@ -1019,6 +1024,7 @@ IntegrationRules::~IntegrationRules()
   DeleteIntRuleArray(TetrahedronIntRules);
   DeleteIntRuleArray(CubeIntRules);
   DeleteIntRuleArray(PrismIntRules);
+   DeleteIntRuleArray(PyramidIntRules);
 }


@@ -1041,6 +1047,8 @@ IntegrationRule *IntegrationRules::GenerateIntegrationRule(int GeomType,
         return CubeIntegrationRule(Order);
      case Geometry::PRISM:
         return PrismIntegrationRule(Order);
+      case Geometry::PYRAMID:
+         return PyramidIntegrationRule(Order);
      default:
         mfem_error("IntegrationRules::Set(...) : Unknown geometry type!");
         return NULL;
@@ -1648,6 +1656,30 @@ IntegrationRule *IntegrationRules::TetrahedronIntegrationRule(int Order)
   }
 }

+// Integration rules for reference pyramid
+IntegrationRule *IntegrationRules::PyramidIntegrationRule(int Order)
+{
+   // This is a simple integration rule adapted from an integration
+   // rule for a cube which seems to be adequate for now. When we
+   // implement high order finite elements for pyramids we should
+   // revisit this and see if we can improve upon it.
+   const IntegrationRule &irc = Get(Geometry::CUBE, Order);
+   int npts = irc.GetNPoints();
+   AllocIntRule(PyramidIntRules, Order);
+   PyramidIntRules[Order] = new IntegrationRule(npts);
+
+   for (int k=0; k<npts; k++)
+   {
+      const IntegrationPoint & ipc = irc.IntPoint(k);
+      IntegrationPoint & ipp = PyramidIntRules[Order]->IntPoint(k);
+      ipp.x = ipc.x * (1.0 - ipc.z);
+      ipp.y = ipc.y * (1.0 - ipc.z);
+      ipp.z = ipc.z;
+      ipp.weight = ipc.weight / 3.0;
+   }
+   return PyramidIntRules[Order];
+}
+
 // Integration rules for reference prism
 IntegrationRule *IntegrationRules::PrismIntegrationRule(int Order)
 {
@@ -323,6 +323,7 @@ private:
   Array<IntegrationRule *> TriangleIntRules;
   Array<IntegrationRule *> SquareIntRules;
   Array<IntegrationRule *> TetrahedronIntRules;
+   Array<IntegrationRule *> PyramidIntRules;
   Array<IntegrationRule *> PrismIntRules;
   Array<IntegrationRule *> CubeIntRules;

@@ -351,6 +352,7 @@ private:
   IntegrationRule *TriangleIntegrationRule(int Order);
   IntegrationRule *SquareIntegrationRule(int Order);
   IntegrationRule *TetrahedronIntegrationRule(int Order);
+   IntegrationRule *PyramidIntegrationRule(int Order);
   IntegrationRule *PrismIntegrationRule(int Order);
   IntegrationRule *CubeIntegrationRule(int Order);

@@ -103,6 +103,7 @@ void LinearForm::Assemble()
 {
   Array<int> vdofs;
   ElementTransformation *eltrans;
+   DofTransformation *doftrans;
   Vector elemvect;

   int i;
@@ -134,10 +135,14 @@ void LinearForm::Assemble()
            if ( domain_integs_marker[k] == NULL ||
                 (*(domain_integs_marker[k]))[elem_attr-1] == 1 )
            {
-               fes -> GetElementVDofs (i, vdofs);
+               doftrans = fes -> GetElementVDofs (i, vdofs);
               eltrans = fes -> GetElementTransformation (i);
               domain_integs[k]->AssembleRHSElementVect(*fes->GetFE(i),
                                                        *eltrans, elemvect);
+               if (doftrans)
+               {
+                  doftrans->TransformDual(elemvect);
+               }
               AddElementVector (vdofs, elemvect);
            }
         }
@@ -174,7 +179,7 @@ void LinearForm::Assemble()
      {
         const int bdr_attr = mesh->GetBdrAttribute(i);
         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
-         fes -> GetBdrElementVDofs (i, vdofs);
+         doftrans = fes -> GetBdrElementVDofs (i, vdofs);
         eltrans = fes -> GetBdrElementTransformation (i);
         for (int k=0; k < boundary_integs.Size(); k++)
         {
@@ -184,6 +189,10 @@ void LinearForm::Assemble()
            boundary_integs[k]->AssembleRHSElementVect(*fes->GetBE(i),
                                                       *eltrans, elemvect);

+            if (doftrans)
+            {
+               doftrans->TransformDual(elemvect);
+            }
            AddElementVector (vdofs, elemvect);
         }
      }
@@ -317,14 +317,11 @@ ParallelEliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
 void ParBilinearForm::TrueAddMult(const Vector &x, Vector &y, const double a)
 const
 {
-   MFEM_VERIFY(interior_face_integs.Size() == 0,
-               "the case of interior face integrators is not"
-               " implemented");
-
   if (X.ParFESpace() != pfes)
   {
      X.SetSpace(pfes);
      Y.SetSpace(pfes);
+      Ytmp.SetSize(pfes->GetTrueVSize());
   }

   X.Distribute(&x);
@@ -334,9 +331,13 @@ const
   }
   else
   {
+      MFEM_VERIFY(interior_face_integs.Size() == 0,
+                  "the case of interior face integrators is not"
+                  " implemented");
      mat->Mult(X, Y);
   }
-   pfes->Dof_TrueDof_Matrix()->MultTranspose(a, Y, 1.0, y);
+   pfes->GetProlongationMatrix()->MultTranspose(Y, Ytmp);
+   y.Add(a,Ytmp);
 }

 void ParBilinearForm::FormLinearSystem(
@@ -33,6 +33,7 @@ protected:

   /// Auxiliary objects used in TrueAddMult().
   mutable ParGridFunction X, Y;
+   mutable Vector Ytmp;

   OperatorHandle p_mat, p_mat_e;

@@ -128,6 +128,9 @@ void ParFiniteElementSpace::ParInit(ParMesh *pm)
   {
      ApplyLDofSigns(*elem_dof);
   }
+
+   // Check for shared trianglular faces with interior Nedelec DoFs
+   CheckNDSTriaDofs();
 }

 void ParFiniteElementSpace::Construct()
@@ -464,32 +467,53 @@ void ParFiniteElementSpace::ApplyLDofSigns(Table &el_dof) const
   ApplyLDofSigns(all_dofs);
 }

-void ParFiniteElementSpace::GetElementDofs(int i, Array<int> &dofs) const
+DofTransformation *
+ParFiniteElementSpace::GetElementDofs(int i, Array<int> &dofs) const
 {
   if (elem_dof)
   {
      elem_dof->GetRow(i, dofs);
-      return;
+
+      if (DoFTrans[mesh->GetElementBaseGeometry(i)])
+      {
+         Array<int> Fo;
+         elem_fos->GetRow(i, Fo);
+         DoFTrans[mesh->GetElementBaseGeometry(i)]->SetFaceOrientations(Fo);
+         return DoFTrans[mesh->GetElementBaseGeometry(i)];
+      }
+      return NULL;
   }
-   FiniteElementSpace::GetElementDofs(i, dofs);
+   DofTransformation * doftrans = FiniteElementSpace::GetElementDofs(i, dofs);
   if (Conforming())
   {
      ApplyLDofSigns(dofs);
   }
+   return doftrans;
 }

-void ParFiniteElementSpace::GetBdrElementDofs(int i, Array<int> &dofs) const
+DofTransformation *
+ParFiniteElementSpace::GetBdrElementDofs(int i, Array<int> &dofs) const
 {
   if (bdr_elem_dof)
   {
      bdr_elem_dof->GetRow(i, dofs);
-      return;
+
+      if (DoFTrans[mesh->GetBdrElementBaseGeometry(i)])
+      {
+         Array<int> Fo;
+         bdr_elem_fos -> GetRow (i, Fo);
+         DoFTrans[mesh->GetBdrElementBaseGeometry(i)]->SetFaceOrientations(Fo);
+         return DoFTrans[mesh->GetBdrElementBaseGeometry(i)];
+      }
+      return NULL;
   }
-   FiniteElementSpace::GetBdrElementDofs(i, dofs);
+   DofTransformation * doftrans =
+      FiniteElementSpace::GetBdrElementDofs(i, dofs);
   if (Conforming())
   {
      ApplyLDofSigns(dofs);
   }
+   return doftrans;
 }

 int ParFiniteElementSpace::GetFaceDofs(int i, Array<int> &dofs,
@@ -657,59 +681,266 @@ void ParFiniteElementSpace::GenerateGlobalOffsets() const
   }
 }

+void ParFiniteElementSpace::CheckNDSTriaDofs()
+{
+   // Check for Nedelec basis
+   bool nd_basis = dynamic_cast<const ND_FECollection*>(fec);
+   if (!nd_basis)
+   {
+      nd_strias = false;
+      return;
+   }
+
+   // Check for interior face dofs on triangles (the use of TETRAHEDRON
+   // is not an error)
+   bool nd_fdof  = fec->HasFaceDofs(Geometry::TETRAHEDRON,
+                                    GetMaxElementOrder());
+   if (!nd_fdof)
+   {
+      nd_strias = false;
+      return;
+   }
+
+   // Check for shared triangle faces
+   bool strias   = false;
+   {
+      int ngrps = pmesh->GetNGroups();
+      for (int g = 1; g < ngrps; g++)
+      {
+         strias |= pmesh->GroupNTriangles(g);
+      }
+   }
+
+   // Combine results
+   int loc_nd_strias = strias ? 1 : 0;
+   int glb_nd_strias = 0;
+   MPI_Allreduce(&loc_nd_strias, &glb_nd_strias, 1,
+                 MPI_INTEGER, MPI_SUM, MyComm);
+   nd_strias = glb_nd_strias > 0;
+}
+
 void ParFiniteElementSpace::Build_Dof_TrueDof_Matrix() const // matrix P
 {
   MFEM_ASSERT(Conforming(), "wrong code path");

   if (P) { return; }

-   int ldof  = GetVSize();
-   int ltdof = TrueVSize();
-
-   HYPRE_Int *i_diag = Memory<HYPRE_Int>(ldof+1);
-   HYPRE_Int *j_diag = Memory<HYPRE_Int>(ltdof);
-   int diag_counter;
-
-   HYPRE_Int *i_offd = Memory<HYPRE_Int>(ldof+1);
-   HYPRE_Int *j_offd = Memory<HYPRE_Int>(ldof-ltdof);
-   int offd_counter;
-
-   HYPRE_BigInt *cmap = Memory<HYPRE_BigInt>(ldof-ltdof);
-
-   HYPRE_BigInt *col_starts = GetTrueDofOffsets();
-   HYPRE_BigInt *row_starts = GetDofOffsets();
-
-   Array<Pair<HYPRE_BigInt, int> > cmap_j_offd(ldof-ltdof);
-
-   i_diag[0] = i_offd[0] = 0;
-   diag_counter = offd_counter = 0;
-   for (int i = 0; i < ldof; i++)
+   if (!nd_strias)
   {
-      int ltdof = GetLocalTDofNumber(i);
-      if (ltdof >= 0)
+      // Safe to assume 1-1 correspondence between shared dofs
+      int ldof  = GetVSize();
+      int ltdof = TrueVSize();
+
+      HYPRE_Int *i_diag = Memory<HYPRE_Int>(ldof+1);
+      HYPRE_Int *j_diag = Memory<HYPRE_Int>(ltdof);
+      int diag_counter;
+
+      HYPRE_Int *i_offd = Memory<HYPRE_Int>(ldof+1);
+      HYPRE_Int *j_offd = Memory<HYPRE_Int>(ldof-ltdof);
+      int offd_counter;
+
+      HYPRE_BigInt *cmap   = Memory<HYPRE_BigInt>(ldof-ltdof);
+
+      HYPRE_BigInt *col_starts = GetTrueDofOffsets();
+      HYPRE_BigInt *row_starts = GetDofOffsets();
+
+      Array<Pair<HYPRE_BigInt, int> > cmap_j_offd(ldof-ltdof);
+
+      i_diag[0] = i_offd[0] = 0;
+      diag_counter = offd_counter = 0;
+      for (int i = 0; i < ldof; i++)
      {
-         j_diag[diag_counter++] = ltdof;
+         int ltdof = GetLocalTDofNumber(i);
+         if (ltdof >= 0)
+         {
+            j_diag[diag_counter++] = ltdof;
+         }
+         else
+         {
+            cmap_j_offd[offd_counter].one = GetGlobalTDofNumber(i);
+            cmap_j_offd[offd_counter].two = offd_counter;
+            offd_counter++;
+         }
+         i_diag[i+1] = diag_counter;
+         i_offd[i+1] = offd_counter;
      }
-      else
+
+      SortPairs<HYPRE_BigInt, int>(cmap_j_offd, offd_counter);
+
+      for (int i = 0; i < offd_counter; i++)
      {
-         cmap_j_offd[offd_counter].one = GetGlobalTDofNumber(i);
-         cmap_j_offd[offd_counter].two = offd_counter;
-         offd_counter++;
+         cmap[i] = cmap_j_offd[i].one;
+         j_offd[cmap_j_offd[i].two] = i;
      }
-      i_diag[i+1] = diag_counter;
-      i_offd[i+1] = offd_counter;
+
+      P = new HypreParMatrix(MyComm, MyRank, NRanks, row_starts, col_starts,
+                             i_diag, j_diag, i_offd, j_offd,
+                             cmap, offd_counter);
   }
-
-   SortPairs<HYPRE_BigInt, int>(cmap_j_offd, offd_counter);
-
-   for (int i = 0; i < offd_counter; i++)
+   else
   {
-      cmap[i] = cmap_j_offd[i].one;
-      j_offd[cmap_j_offd[i].two] = i;
-   }
+      // Some shared dofs will be linear combinations of others
+      int ldof  = GetVSize();
+      int ltdof = TrueVSize();

-   P = new HypreParMatrix(MyComm, MyRank, NRanks, row_starts, col_starts,
-                          i_diag, j_diag, i_offd, j_offd, cmap, offd_counter);
+      HYPRE_Int gdof  = -1;
+      HYPRE_Int gtdof = -1;
+
+      MPI_Allreduce(&ldof, &gdof, 1, HYPRE_MPI_INT, MPI_SUM, MyComm);
+      MPI_Allreduce(&ltdof, &gtdof, 1, HYPRE_MPI_INT, MPI_SUM, MyComm);
+
+      // Ensure face orientations have been communicated
+      pmesh->ExchangeFaceNbrData();
+
+      // Locate and count non-zeros in off-diagonal portion of P
+      int nnz_offd = 0;
+      Array<int> ldsize(ldof); ldsize = 0;
+      Array<int> ltori(ldof);  ltori = 0; // Local triangle orientations
+      {
+         int ngrps = pmesh->GetNGroups();
+         int nedofs = fec->DofForGeometry(Geometry::SEGMENT);
+         Array<int> sdofs;
+         for (int g = 1; g < ngrps; g++)
+         {
+            if (pmesh->gtopo.IAmMaster(g))
+            {
+               continue;
+            }
+            for (int ei=0; ei<pmesh->GroupNEdges(g); ei++)
+            {
+               this->GetSharedEdgeDofs(g, ei, sdofs);
+               for (int i=0; i<sdofs.Size(); i++)
+               {
+                  int ind = (sdofs[i]>=0) ? sdofs[i] : (-sdofs[i]-1);
+                  if (ldsize[ind] == 0) { nnz_offd++; }
+                  ldsize[ind] = 1;
+               }
+            }
+            for (int fi=0; fi<pmesh->GroupNTriangles(g); fi++)
+            {
+               int face, ori, info1, info2;
+               pmesh->GroupTriangle(g, fi, face, ori);
+               pmesh->GetFaceInfos(face, &info1, &info2);
+               this->GetSharedTriangleDofs(g, fi, sdofs);
+               for (int i=0; i<3*nedofs; i++)
+               {
+                  int ind = (sdofs[i]>=0) ? sdofs[i] : (-sdofs[i]-1);
+                  if (ldsize[ind] == 0) { nnz_offd++; }
+                  ldsize[ind] = 1;
+               }
+               for (int i=3*nedofs; i<sdofs.Size(); i++)
+               {
+                  if (ldsize[sdofs[i]] == 0) { nnz_offd += 2; }
+                  ldsize[sdofs[i]] = 2;
+                  ltori[sdofs[i]]  = info2 % 64;
+               }
+            }
+            for (int fi=0; fi<pmesh->GroupNQuadrilaterals(g); fi++)
+            {
+               this->GetSharedQuadrilateralDofs(g, fi, sdofs);
+               for (int i=0; i<sdofs.Size(); i++)
+               {
+                  int ind = (sdofs[i]>=0) ? sdofs[i] : (-sdofs[i]-1);
+                  if (ldsize[ind] == 0) { nnz_offd++; }
+                  ldsize[ind] = 1;
+               }
+            }
+         }
+      }
+
+      HYPRE_Int *i_diag = new HYPRE_Int[ldof+1];
+      HYPRE_Int *j_diag = new HYPRE_Int[ltdof];
+      double    *d_diag = new double[ltdof];
+      int diag_counter;
+
+      HYPRE_Int *i_offd = new HYPRE_Int[ldof+1];
+      HYPRE_Int *j_offd = new HYPRE_Int[nnz_offd];
+      double    *d_offd = new double[nnz_offd];
+      int offd_counter;
+
+      HYPRE_BigInt *cmap   = new HYPRE_BigInt[ldof-ltdof];
+
+      HYPRE_BigInt *col_starts = GetTrueDofOffsets();
+      HYPRE_BigInt *row_starts = GetDofOffsets();
+
+      Array<Pair<HYPRE_BigInt, int> > cmap_j_offd(ldof-ltdof);
+
+      i_diag[0] = i_offd[0] = 0;
+      diag_counter = offd_counter = 0;
+      int offd_col_counter = 0;
+      for (int i = 0; i < ldof; i++)
+      {
+         int ltdof = GetLocalTDofNumber(i);
+         if (ltdof >= 0)
+         {
+            j_diag[diag_counter]   = ltdof;
+            d_diag[diag_counter++] = 1.0;
+         }
+         else
+         {
+            if (ldsize[i] == 1)
+            {
+               cmap_j_offd[offd_col_counter].one = GetGlobalTDofNumber(i);
+               cmap_j_offd[offd_col_counter].two = offd_counter;
+               offd_counter++;
+               offd_col_counter++;
+            }
+            else
+            {
+               cmap_j_offd[offd_col_counter].one = GetGlobalTDofNumber(i);
+               cmap_j_offd[offd_col_counter].two = offd_counter;
+               offd_counter += 2;
+               offd_col_counter++;
+               i_diag[i+1] = diag_counter;
+               i_offd[i+1] = offd_counter;
+               i++;
+               cmap_j_offd[offd_col_counter].one = GetGlobalTDofNumber(i);
+               cmap_j_offd[offd_col_counter].two = offd_counter;
+               offd_counter += 2;
+               offd_col_counter++;
+            }
+         }
+         i_diag[i+1] = diag_counter;
+         i_offd[i+1] = offd_counter;
+      }
+
+      SortPairs<HYPRE_BigInt, int>(cmap_j_offd, offd_col_counter);
+
+      for (int i = 0; i < nnz_offd; i++)
+      {
+         j_offd[i] = -1;
+         d_offd[i] = 0.0;
+      }
+
+      for (int i = 0; i < offd_col_counter; i++)
+      {
+         cmap[i] = cmap_j_offd[i].one;
+         j_offd[cmap_j_offd[i].two] = i;
+      }
+
+      for (int i = 0; i < ldof; i++)
+      {
+         if (i_offd[i+1] == i_offd[i] + 1)
+         {
+            d_offd[i_offd[i]] = 1.0;
+         }
+         else if (i_offd[i+1] == i_offd[i] + 2)
+         {
+            const double * T = ND_DofTransformation
+                               ::GetFaceTransform(ltori[i]).GetData();
+            j_offd[i_offd[i] + 1] = j_offd[i_offd[i]] + 1;
+            d_offd[i_offd[i]] = T[0]; d_offd[i_offd[i] + 1] = T[2];
+            i++;
+            j_offd[i_offd[i] + 1] = j_offd[i_offd[i]];
+            j_offd[i_offd[i]] = j_offd[i_offd[i] + 1] - 1;
+            d_offd[i_offd[i]] = T[1]; d_offd[i_offd[i] + 1] = T[3];
+         }
+      }
+
+      P = new HypreParMatrix(MyComm, gdof, gtdof, row_starts, col_starts,
+                             i_diag, j_diag, d_diag, i_offd, j_offd, d_offd,
+                             offd_col_counter, cmap);
+   }

   SparseMatrix Pdiag;
   P->GetDiag(Pdiag);
@@ -913,6 +1144,8 @@ const Operator *ParFiniteElementSpace::GetProlongationMatrix() const
   {
      if (Pconf) { return Pconf; }

+      if (nd_strias) { return Dof_TrueDof_Matrix(); }
+
      if (NRanks == 1)
      {
         Pconf = new IdentityOperator(GetTrueVSize());
@@ -1216,10 +1449,29 @@ void ParFiniteElementSpace::ExchangeFaceNbrData()
   delete [] requests;
 }

-void ParFiniteElementSpace::GetFaceNbrElementVDofs(
+DofTransformation *ParFiniteElementSpace::GetFaceNbrElementVDofs(
   int i, Array<int> &vdofs) const
 {
   face_nbr_element_dof.GetRow(i, vdofs);
+
+   DofTransformation *doftrans = NULL;
+   Geometry::Type geom = GetFaceNbrFE(i)->GetGeomType();
+   if (DoFTrans[geom])
+   {
+      Array<int> F, Fo;
+      pmesh->GetFaceNbrElementFaces(pmesh->GetNE() + i, F, Fo);
+      doftrans = DoFTrans[geom];
+      doftrans->SetFaceOrientations(Fo);
+   }
+   if (vdim == 1 || doftrans == NULL)
+   {
+      return doftrans;
+   }
+   else
+   {
+      VDoFTrans.SetDofTransformation(*doftrans);
+      return &VDoFTrans;
+   }
 }

 void ParFiniteElementSpace::GetFaceNbrFaceVDofs(int i, Array<int> &vdofs) const
@@ -1278,12 +1530,17 @@ const FiniteElement *ParFiniteElementSpace::GetFaceNbrFaceFE(int i) const

 void ParFiniteElementSpace::Lose_Dof_TrueDof_Matrix()
 {
+   P -> StealData();
+#if MFEM_HYPRE_VERSION <= 22200
   hypre_ParCSRMatrix *csrP = (hypre_ParCSRMatrix*)(*P);
   hypre_ParCSRMatrixOwnsRowStarts(csrP) = 1;
   hypre_ParCSRMatrixOwnsColStarts(csrP) = 1;
-   P -> StealData();
   dof_offsets.LoseData();
   tdof_offsets.LoseData();
+#else
+   dof_offsets.DeleteAll();
+   tdof_offsets.DeleteAll();
+#endif
 }

 void ParFiniteElementSpace::ConstructTrueDofs()
@@ -2526,7 +2783,8 @@ static int_type* make_j_array(int_type* I, int nrows)

 HypreParMatrix*
 ParFiniteElementSpace::RebalanceMatrix(int old_ndofs,
-                                       const Table* old_elem_dof)
+                                       const Table* old_elem_dof,
+                                       const Table* old_elem_fos)
 {
   MFEM_VERIFY(Nonconforming(), "Only supported for nonconforming meshes.");
   MFEM_VERIFY(old_dof_offsets.Size(), "ParFiniteElementSpace::Update needs to "
@@ -2651,7 +2909,8 @@ struct DerefDofMessage

 HypreParMatrix*
 ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
-                                                  const Table* old_elem_dof)
+                                                  const Table* old_elem_dof,
+                                                  const Table *old_elem_fos)
 {
   int nrk = HYPRE_AssumedPartitionCheck() ? 2 : NRanks;

@@ -3007,13 +3266,16 @@ void ParFiniteElementSpace::Update(bool want_transform)
   }

   Table* old_elem_dof = NULL;
+   Table* old_elem_fos = NULL;
   int old_ndofs;

   // save old DOF table
   if (want_transform)
   {
      old_elem_dof = elem_dof;
+      old_elem_fos = elem_fos;
      elem_dof = NULL;
+      elem_fos = NULL;
      old_ndofs = ndofs;
      Swap(dof_offsets, old_dof_offsets);
   }
@@ -3035,22 +3297,25 @@ void ParFiniteElementSpace::Update(bool want_transform)
         {
            if (Th.Type() != Operator::MFEM_SPARSEMAT)
            {
-               Th.Reset(new RefinementOperator(this, old_elem_dof, old_ndofs));
+               Th.Reset(new RefinementOperator(this, old_elem_dof,
+                                               old_elem_fos, old_ndofs));
               // The RefinementOperator takes ownership of 'old_elem_dofs', so
               // we no longer own it:
               old_elem_dof = NULL;
+               old_elem_fos = NULL;
            }
            else
            {
               // calculate fully assembled matrix
-               Th.Reset(RefinementMatrix(old_ndofs, old_elem_dof));
+               Th.Reset(RefinementMatrix(old_ndofs, old_elem_dof, old_elem_fos));
            }
            break;
         }

         case Mesh::DEREFINE:
         {
-            Th.Reset(ParallelDerefinementMatrix(old_ndofs, old_elem_dof));
+            Th.Reset(ParallelDerefinementMatrix(old_ndofs, old_elem_dof,
+                                                old_elem_fos));
            if (Nonconforming())
            {
               Th.SetOperatorOwner(false);
@@ -3062,7 +3327,7 @@ void ParFiniteElementSpace::Update(bool want_transform)

         case Mesh::REBALANCE:
         {
-            Th.Reset(RebalanceMatrix(old_ndofs, old_elem_dof));
+            Th.Reset(RebalanceMatrix(old_ndofs, old_elem_dof, old_elem_fos));
            break;
         }

@@ -3071,6 +3336,7 @@ void ParFiniteElementSpace::Update(bool want_transform)
      }

      delete old_elem_dof;
+      delete old_elem_fos;
   }
 }

@@ -87,6 +87,12 @@ private:
       this is a TransposeOperator wrapping R. */
   mutable Operator *R_transpose;

+   /// Flag indicating the existence of shared triangles with interior ND dofs
+   bool nd_strias;
+
+   /// Resets nd_strias flag at constuction or after rebalancing
+   void CheckNDSTriaDofs();
+
   ParNURBSExtension *pNURBSext() const
   { return dynamic_cast<ParNURBSExtension *>(NURBSext); }

@@ -174,14 +180,16 @@ private:
       The result is a parallel permutation matrix that can be used to update
       all grid functions defined on this space. */
   HypreParMatrix* RebalanceMatrix(int old_ndofs,
-                                   const Table* old_elem_dof);
+                                   const Table* old_elem_dof,
+                                   const Table* old_elem_fos);

   /** Calculate a GridFunction restriction matrix after mesh derefinement.
       The matrix is constructed so that the new grid function interpolates
       the original function, i.e., the original function is evaluated at the
       nodes of the coarse function. */
   HypreParMatrix* ParallelDerefinementMatrix(int old_ndofs,
-                                              const Table *old_elem_dof);
+                                              const Table *old_elem_dof,
+                                              const Table *old_elem_fos);

   /// Updates the internal mesh pointer. @warning @a new_mesh must be
   /// <b>topologically identical</b> to the existing mesh. Used if the address
@@ -202,6 +210,8 @@ public:
   int num_face_nbr_dofs;
   // Face-neighbor-element to face-neighbor dof
   Table face_nbr_element_dof;
+   // Face-neighbor-element face orientations
+   Table face_nbr_element_fos;
   // Face-neighbor to ldof in the face-neighbor numbering
   Table face_nbr_ldof;
   // The global ldof indices of the face-neighbor dofs
@@ -279,10 +289,10 @@ public:
   virtual int GetTrueVSize() const { return ltdof_size; }

   /// Returns indexes of degrees of freedom in array dofs for i'th element.
-   virtual void GetElementDofs(int i, Array<int> &dofs) const;
+   virtual DofTransformation *GetElementDofs(int i, Array<int> &dofs) const;

   /// Returns indexes of degrees of freedom for i'th boundary element.
-   virtual void GetBdrElementDofs(int i, Array<int> &dofs) const;
+   virtual DofTransformation *GetBdrElementDofs(int i, Array<int> &dofs) const;

   /** Returns the indexes of the degrees of freedom for i'th face
       including the dofs for the edges and the vertices of the face. */
@@ -382,7 +392,7 @@ public:
   // Face-neighbor functions
   void ExchangeFaceNbrData();
   int GetFaceNbrVSize() const { return num_face_nbr_dofs; }
-   void GetFaceNbrElementVDofs(int i, Array<int> &vdofs) const;
+   DofTransformation *GetFaceNbrElementVDofs(int i, Array<int> &vdofs) const;
   void GetFaceNbrFaceVDofs(int i, Array<int> &vdofs) const;
   const FiniteElement *GetFaceNbrFE(int i) const;
   const FiniteElement *GetFaceNbrFaceFE(int i) const;
@@ -397,6 +407,8 @@ public:
   bool Conforming() const { return pmesh->pncmesh == NULL && !nonconf_P; }
   bool Nonconforming() const { return pmesh->pncmesh != NULL || nonconf_P; }

+   bool SharedNDTriangleDofs() const { return nd_strias; }
+
   // Transfer parallel true-dof data from coarse_fes, defined on a coarse mesh,
   // to this FE space, defined on a refined mesh. See full documentation in the
   // base class, FiniteElementSpace::GetTrueTransferOperator.
@@ -325,9 +325,14 @@ void ParGridFunction::GetVectorValue(int i, const IntegrationPoint &ip,
   if (nbr_el_no >= 0)
   {
      Array<int> dofs;
-      pfes->GetFaceNbrElementVDofs(nbr_el_no, dofs);
+      DofTransformation * doftrans = pfes->GetFaceNbrElementVDofs(nbr_el_no,
+                                                                  dofs);
      Vector loc_data;
      face_nbr_data.GetSubVector(dofs, loc_data);
+      if (doftrans)
+      {
+         doftrans->InvTransformPrimal(loc_data);
+      }
      const FiniteElement *FElem = pfes->GetFaceNbrFE(nbr_el_no);
      int dof = FElem->GetDof();
      if (FElem->GetRangeType() == FiniteElement::SCALAR)
@@ -437,12 +442,17 @@ void ParGridFunction::GetVectorValue(ElementTransformation &T,
   }

   Array<int> vdofs;
-   pfes->GetFaceNbrElementVDofs(nbr_el_no, vdofs);
+   DofTransformation * doftrans = pfes->GetFaceNbrElementVDofs(nbr_el_no,
+                                                               vdofs);
   const FiniteElement *fe = pfes->GetFaceNbrFE(nbr_el_no);

   int dof = fe->GetDof();
   Vector loc_data;
   face_nbr_data.GetSubVector(vdofs, loc_data);
+   if (doftrans)
+   {
+      doftrans->InvTransformPrimal(loc_data);
+   }
   if (fe->GetRangeType() == FiniteElement::SCALAR)
   {
      Vector shape(dof);
@@ -65,6 +65,11 @@ public:
   ParGridFunction(ParFiniteElementSpace *pf, double *data) :
      GridFunction(pf, data), pfes(pf) { }

+   /** @brief Construct a ParGridFunction using previously allocated Vector
+       @a base starting at the given offset, @a base_offset. */
+   ParGridFunction(ParFiniteElementSpace *pf, Vector &base, int base_offset = 0)
+      : GridFunction(pf, base, base_offset), pfes(pf) { }
+
   /// Construct a ParGridFunction using a GridFunction as external data.
   /** The parallel space @a *pf and the space used by @a *gf should match. The
       data from @a *gf is used as the local data of the ParGridFunction on each
@@ -33,6 +33,10 @@ ParL2FaceRestriction::ParL2FaceRestriction(const ParFiniteElementSpace &fes,
   // If fespace == L2
   const ParFiniteElementSpace &pfes =
      static_cast<const ParFiniteElementSpace&>(this->fes);
+
+   // Ensure the face neighbor data is constructed
+   pfes.GetParMesh()->ExchangeFaceNbrData();
+
   const FiniteElement *fe = pfes.GetFE(0);
   const TensorBasisElement *tfe = dynamic_cast<const TensorBasisElement*>(fe);
   MFEM_VERIFY(tfe != NULL &&
@@ -15,6 +15,12 @@
 #include "../general/forall.hpp"
 #include <climits>

+#ifdef MFEM_USE_MPI
+
+#include "pfespace.hpp"
+
+#endif
+
 namespace mfem
 {

@@ -161,8 +167,8 @@ void ElementRestriction::MultTranspose(const Vector& x, Vector& y) const
         for (int j = offset; j < nextOffset; ++j)
         {
            const int idx_j = (d_indices[j] >= 0) ? d_indices[j] : -1 - d_indices[j];
-            dofValue += (d_indices[j] >= 0) ? d_x(idx_j % nd, c,
-            idx_j / nd) : -d_x(idx_j % nd, c, idx_j / nd);
+            dofValue += ((d_indices[j] >= 0) ? d_x(idx_j % nd, c, idx_j / nd) :
+                         -d_x(idx_j % nd, c, idx_j / nd));
         }
         d_y(t?c:i,t?i:c) = dofValue;
      }
@@ -675,6 +681,19 @@ H1FaceRestriction::H1FaceRestriction(const FiniteElementSpace &fes,
     gather_indices(nf*dof)
 {
   if (nf==0) { return; }
+
+#ifdef MFEM_USE_MPI
+
+   // If the underlying finite element space is parallel, ensure the face
+   // neighbor information is generated.
+   if (const ParFiniteElementSpace *pfes
+       = dynamic_cast<const ParFiniteElementSpace*>(&fes))
+   {
+      pfes->GetParMesh()->ExchangeFaceNbrData();
+   }
+
+#endif
+
   // If fespace == H1
   const FiniteElement *fe = fes.GetFE(0);
   const TensorBasisElement *tfe = dynamic_cast<const TensorBasisElement*>(fe);
@@ -1314,33 +1314,61 @@ static inline void device_copy(double *d_dest, const double *d_src, int size)
 } // namespace internal

 #ifdef MFEM_USE_MPI
-void DiscreteAdaptTC::FinalizeParDiscreteTargetSpec(const ParGridFunction
-                                                    &tspec_)
+void DiscreteAdaptTC::FinalizeParDiscreteTargetSpec(const ParGridFunction &t)
 {
   MFEM_VERIFY(adapt_eval, "SetAdaptivityEvaluator() has not been called!")
   MFEM_VERIFY(ncomp > 0, "No target specifications have been set!");

-   ParFiniteElementSpace *ptspec_fes = tspec_.ParFESpace();
+   ParFiniteElementSpace *ptspec_fes = t.ParFESpace();

   adapt_eval->SetParMetaInfo(*ptspec_fes->GetParMesh(),
                              *ptspec_fes->FEColl(), ncomp);
-   adapt_eval->SetInitialField(*tspec_fes->GetMesh()->GetNodes(), tspec);
+   adapt_eval->SetInitialField(*ptspec_fes->GetMesh()->GetNodes(), tspec);

   tspec_sav = tspec;

   delete tspec_fesv;
-   tspec_fesv = new FiniteElementSpace(tspec_fes->GetMesh(),
-                                       tspec_fes->FEColl(), ncomp);
+   tspec_fesv = new FiniteElementSpace(ptspec_fes->GetMesh(),
+                                       ptspec_fes->FEColl(), ncomp);
+
+   delete ptspec_fesv;
+   ptspec_fesv = new ParFiniteElementSpace(ptspec_fes->GetParMesh(),
+                                           ptspec_fes->FEColl(), ncomp);
+
+   delete tspec_pgf;
+   tspec_pgf = new ParGridFunction(ptspec_fesv, tspec);
+   tspec_gf = tspec_pgf;
+}
+
+void DiscreteAdaptTC::ParUpdateAfterMeshTopologyChange()
+{
+   ptspec_fesv->Update();
+   if (tspec_fesv)
+   {
+      delete tspec_fesv;
+      tspec_fesv = new FiniteElementSpace(ptspec_fesv->GetMesh(),
+                                          ptspec_fesv->FEColl(), ncomp);
+   }
+   tspec_pgf->Update();
+   tspec_gf = tspec_pgf;
+   tspec.SetDataAndSize(tspec_pgf->GetData(), tspec_pgf->Size());
+   tspec_sav = tspec;
+
+   adapt_eval->SetParMetaInfo(*ptspec_fesv->GetParMesh(),
+                              *ptspec_fesv->FEColl(), ncomp);
+   adapt_eval->SetInitialField(*ptspec_fesv->GetMesh()->GetNodes(), tspec);
 }

 void DiscreteAdaptTC::SetTspecAtIndex(int idx, const ParGridFunction &tspec_)
 {
-   const int vdim     = tspec_.FESpace()->GetVDim(),
-             dof_cnt  = tspec_.Size()/vdim;
+   const int vdim = tspec_.FESpace()->GetVDim(),
+             ndof = tspec_.FESpace()->GetNDofs();
+   MFEM_VERIFY(ndof == tspec.Size()/ncomp, "Inconsistency in SetTspecAtIndex.");
+
   const auto tspec__d = tspec_.Read();
   auto tspec_d = tspec.ReadWrite();
-   const int offset = idx*dof_cnt;
-   internal::device_copy(tspec_d + offset, tspec__d, dof_cnt*vdim);
+   const int offset = idx*ndof;
+   internal::device_copy(tspec_d + offset, tspec__d, ndof*vdim);
   FinalizeParDiscreteTargetSpec(tspec_);
 }

@@ -1360,78 +1388,71 @@ void DiscreteAdaptTC::SetParDiscreteTargetSkew(const ParGridFunction &tspec_)
   FinalizeParDiscreteTargetSpec(tspec_);
 }

-void DiscreteAdaptTC::SetParDiscreteTargetAspectRatio(const ParGridFunction
-                                                      &tspec_)
+void DiscreteAdaptTC::SetParDiscreteTargetAspectRatio(const ParGridFunction &ar)
 {
-   if (aspectratioidx > -1) { SetTspecAtIndex(aspectratioidx, tspec_); return; }
+   if (aspectratioidx > -1) { SetTspecAtIndex(aspectratioidx, ar); return; }
   aspectratioidx = ncomp;
-   SetDiscreteTargetBase(tspec_);
-   FinalizeParDiscreteTargetSpec(tspec_);
+   SetDiscreteTargetBase(ar);
+   FinalizeParDiscreteTargetSpec(ar);
 }

-void DiscreteAdaptTC::SetParDiscreteTargetOrientation(const ParGridFunction
-                                                      &tspec_)
+void DiscreteAdaptTC::SetParDiscreteTargetOrientation(const ParGridFunction &o)
 {
-   if (orientationidx > -1) { SetTspecAtIndex(orientationidx, tspec_); return; }
+   if (orientationidx > -1) { SetTspecAtIndex(orientationidx, o); return; }
   orientationidx = ncomp;
-   SetDiscreteTargetBase(tspec_);
-   FinalizeParDiscreteTargetSpec(tspec_);
+   SetDiscreteTargetBase(o);
+   FinalizeParDiscreteTargetSpec(o);
 }

 void DiscreteAdaptTC::SetParDiscreteTargetSpec(const ParGridFunction &tspec_)
 {
   SetParDiscreteTargetSize(tspec_);
-   FinalizeParDiscreteTargetSpec(tspec_);
 }
 #endif // MFEM_USE_MPI

 void DiscreteAdaptTC::SetDiscreteTargetBase(const GridFunction &tspec_)
 {
-   const int vdim     = tspec_.FESpace()->GetVDim(),
-             dof_cnt  = tspec_.Size()/vdim;
+   const int vdim = tspec_.FESpace()->GetVDim(),
+             ndof = tspec_.FESpace()->GetNDofs();

   ncomp += vdim;

-   delete tspec_fes;
-   tspec_fes = new FiniteElementSpace(tspec_.FESpace()->GetMesh(),
-                                      tspec_.FESpace()->FEColl(), 1);
-
   // need to append data to tspec
   // make a copy of tspec->tspec_temp, increase its size, and
   // copy data from tspec_temp -> tspec, then add new entries
   Vector tspec_temp = tspec;
   tspec.UseDevice(true);
   tspec_sav.UseDevice(true);
-   tspec.SetSize(ncomp*dof_cnt);
+   tspec.SetSize(ncomp*ndof);

   const auto tspec_temp_d = tspec_temp.Read();
   auto tspec_d = tspec.ReadWrite();
   internal::device_copy(tspec_d, tspec_temp_d, tspec_temp.Size());

   const auto tspec__d = tspec_.Read();
-   const int offset = (ncomp-vdim)*dof_cnt;
-   internal::device_copy(tspec_d + offset, tspec__d, dof_cnt*vdim);
+   const int offset = (ncomp-vdim)*ndof;
+   internal::device_copy(tspec_d + offset, tspec__d, ndof*vdim);
 }

 void DiscreteAdaptTC::SetTspecAtIndex(int idx, const GridFunction &tspec_)
 {
-   const int vdim     = tspec_.FESpace()->GetVDim(),
-             dof_cnt  = tspec_.Size()/vdim;
+   const int vdim = tspec_.FESpace()->GetVDim(),
+             ndof = tspec_.FESpace()->GetNDofs();
+   MFEM_VERIFY(ndof == tspec.Size()/ncomp, "Inconsistency in SetTargetSpec.");

   const auto tspec__d = tspec_.Read();
   auto tspec_d = tspec.ReadWrite();
-   const int offset = idx*dof_cnt;
-   internal::device_copy(tspec_d + offset, tspec__d, dof_cnt*vdim);
-   FinalizeSerialDiscreteTargetSpec();
+   const int offset = idx*ndof;
+   internal::device_copy(tspec_d + offset, tspec__d, ndof*vdim);
+   FinalizeSerialDiscreteTargetSpec(tspec_);
 }

 void DiscreteAdaptTC::SetSerialDiscreteTargetSize(const GridFunction &tspec_)
 {
-
   if (sizeidx > -1) { SetTspecAtIndex(sizeidx, tspec_); return; }
   sizeidx = ncomp;
   SetDiscreteTargetBase(tspec_);
-   FinalizeSerialDiscreteTargetSpec();
+   FinalizeSerialDiscreteTargetSpec(tspec_);
 }

 void DiscreteAdaptTC::SetSerialDiscreteTargetSkew(const GridFunction &tspec_)
@@ -1439,32 +1460,31 @@ void DiscreteAdaptTC::SetSerialDiscreteTargetSkew(const GridFunction &tspec_)
   if (skewidx > -1) { SetTspecAtIndex(skewidx, tspec_); return; }
   skewidx = ncomp;
   SetDiscreteTargetBase(tspec_);
-   FinalizeSerialDiscreteTargetSpec();
+   FinalizeSerialDiscreteTargetSpec(tspec_);
 }

-void DiscreteAdaptTC::SetSerialDiscreteTargetAspectRatio(
-   const GridFunction &tspec_)
+void DiscreteAdaptTC::SetSerialDiscreteTargetAspectRatio(const GridFunction &ar)
 {
-   if (aspectratioidx > -1) { SetTspecAtIndex(aspectratioidx, tspec_); return; }
+   if (aspectratioidx > -1) { SetTspecAtIndex(aspectratioidx, ar); return; }
   aspectratioidx = ncomp;
-   SetDiscreteTargetBase(tspec_);
-   FinalizeSerialDiscreteTargetSpec();
+   SetDiscreteTargetBase(ar);
+   FinalizeSerialDiscreteTargetSpec(ar);
 }

-void DiscreteAdaptTC::SetSerialDiscreteTargetOrientation(
-   const GridFunction &tspec_)
+void DiscreteAdaptTC::SetSerialDiscreteTargetOrientation(const GridFunction &o)
 {
-   if (orientationidx > -1) { SetTspecAtIndex(orientationidx, tspec_); return; }
+   if (orientationidx > -1) { SetTspecAtIndex(orientationidx, o); return; }
   orientationidx = ncomp;
-   SetDiscreteTargetBase(tspec_);
-   FinalizeSerialDiscreteTargetSpec();
+   SetDiscreteTargetBase(o);
+   FinalizeSerialDiscreteTargetSpec(o);
 }

-void DiscreteAdaptTC::FinalizeSerialDiscreteTargetSpec()
+void DiscreteAdaptTC::FinalizeSerialDiscreteTargetSpec(const GridFunction &t)
 {
   MFEM_VERIFY(adapt_eval, "SetAdaptivityEvaluator() has not been called!")
   MFEM_VERIFY(ncomp > 0, "No target specifications have been set!");

+   const FiniteElementSpace *tspec_fes = t.FESpace();
   adapt_eval->SetSerialMetaInfo(*tspec_fes->GetMesh(),
                                 *tspec_fes->FEColl(), ncomp);
   adapt_eval->SetInitialField(*tspec_fes->GetMesh()->GetNodes(), tspec);
@@ -1474,12 +1494,40 @@ void DiscreteAdaptTC::FinalizeSerialDiscreteTargetSpec()
   delete tspec_fesv;
   tspec_fesv = new FiniteElementSpace(tspec_fes->GetMesh(),
                                       tspec_fes->FEColl(), ncomp);
+
+   delete tspec_gf;
+   tspec_gf = new GridFunction(tspec_fesv, tspec);
+}
+
+void DiscreteAdaptTC::GetDiscreteTargetSpec(GridFunction &tspec_, int idx)
+{
+   if (idx < 0) { return; }
+   const int ndof = tspec_.FESpace()->GetNDofs(),
+             vdim = tspec_.FESpace()->GetVDim();
+   MFEM_VERIFY(ndof == tspec.Size()/ncomp,
+               "Inconsistency in GetSerialDiscreteTargetSpec.");
+
+   for (int i = 0; i < ndof*vdim; i++)
+   {
+      tspec_(i) = tspec(i + idx*ndof);
+   }
+}
+
+void DiscreteAdaptTC::UpdateAfterMeshTopologyChange()
+{
+   tspec_fesv->Update();
+   tspec_gf->Update();
+   tspec.SetDataAndSize(tspec_gf->GetData(), tspec_gf->Size());
+   tspec_sav = tspec;
+
+   adapt_eval->SetSerialMetaInfo(*tspec_fesv->GetMesh(),
+                                 *tspec_fesv->FEColl(), ncomp);
+   adapt_eval->SetInitialField(*tspec_fesv->GetMesh()->GetNodes(), tspec);
 }

 void DiscreteAdaptTC::SetSerialDiscreteTargetSpec(const GridFunction &tspec_)
 {
   SetSerialDiscreteTargetSize(tspec_);
-   FinalizeSerialDiscreteTargetSpec();
 }


@@ -1509,7 +1557,7 @@ void DiscreteAdaptTC::UpdateTargetSpecificationAtNode(const FiniteElement &el,
   MFEM_VERIFY(tspec.Size() > 0, "Target specification is not set!");

   Array<int> dofs;
-   tspec_fes->GetElementDofs(T.ElementNo, dofs);
+   tspec_fesv->GetElementDofs(T.ElementNo, dofs);
   const int cnt = tspec.Size()/ncomp; // dofs per scalar-field

   for (int i = 0; i < ncomp; i++)
@@ -1524,7 +1572,7 @@ void DiscreteAdaptTC::RestoreTargetSpecificationAtNode(ElementTransformation &T,
   MFEM_VERIFY(tspec.Size() > 0, "Target specification is not set!");

   Array<int> dofs;
-   tspec_fes->GetElementDofs(T.ElementNo, dofs);
+   tspec_fesv->GetElementDofs(T.ElementNo, dofs);
   const int cnt = tspec.Size()/ncomp;
   for (int i = 0; i < ncomp; i++)
   {
@@ -1532,6 +1580,40 @@ void DiscreteAdaptTC::RestoreTargetSpecificationAtNode(ElementTransformation &T,
   }
 }

+void DiscreteAdaptTC::SetTspecFromIntRule(int e_id,
+                                          const IntegrationRule &intrule)
+{
+   switch (target_type)
+   {
+      case IDEAL_SHAPE_GIVEN_SIZE:
+      case GIVEN_SHAPE_AND_SIZE:
+      {
+         const int ndofs = tspec_fesv->GetFE(e_id)->GetDof(),
+                   ntspec_dofs = ndofs*ncomp;
+
+         Vector tspec_vals(ntspec_dofs);
+
+         Array<int> dofs;
+         tspec_fesv->GetElementVDofs(e_id, dofs);
+         tspec.GetSubVector(dofs, tspec_vals);
+         DenseMatrix tr;
+         tspec_gf->GetVectorValues(e_id, intrule, tspec_refine, tr);
+         tspec_refine.Transpose();
+         break;
+      }
+      default:
+         MFEM_ABORT("Incompatible target type for discrete adaptation!");
+   }
+}
+
+void DiscreteAdaptTC::SetTspecDataForDerefinement(FiniteElementSpace *fes)
+{
+   coarse_tspec_fesv = fes;
+   const Operator *c_op = fes->GetUpdateOperator();
+   tspec_derefine.SetSize(c_op->Height());
+   c_op->Mult(tspec, tspec_derefine);
+}
+
 void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
                                            const IntegrationRule &ir,
                                            const Vector &elfun,
@@ -1542,6 +1624,8 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
             nqp = ir.GetNPoints();
   Jtrcomp.SetSize(dim, dim, 4*nqp);

+   FiniteElementSpace *src_fes = tspec_fesv;
+
   switch (target_type)
   {
      case IDEAL_SHAPE_GIVEN_SIZE:
@@ -1550,7 +1634,7 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
         const DenseMatrix &Wideal =
            Geometries.GetGeomToPerfGeomJac(fe.GetGeomType());
         const int dim = Wideal.Height(),
-                   ndofs = tspec_fes->GetFE(e_id)->GetDof(),
+                   ndofs = tspec_fesv->GetFE(e_id)->GetDof(),
                   ntspec_dofs = ndofs*ncomp;

         Vector shape(ndofs), tspec_vals(ntspec_dofs), par_vals,
@@ -1561,11 +1645,29 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
         tspec_fesv->GetElementVDofs(e_id, dofs);
         tspec.UseDevice(true);
         tspec.GetSubVector(dofs, tspec_vals);
+         if (tspec_refine.NumCols() > 0) // Refinement
+         {
+            MFEM_VERIFY(amr_el >= 0, " Target being constructed for an AMR element.");
+            for (int i = 0; i < ncomp; i++)
+            {
+               for (int j = 0; j < ndofs; j++)
+               {
+                  tspec_vals(j + i*ndofs) = tspec_refine(j + amr_el*ndofs, i);
+               }
+            }
+         }
+         else if (tspec_derefine.Size() > 0) // Derefinement
+         {
+            dofs.SetSize(0);
+            coarse_tspec_fesv->GetElementVDofs(e_id, dofs);
+            tspec_derefine.GetSubVector(dofs, tspec_vals);
+            src_fes = coarse_tspec_fesv;
+         }

         for (int q = 0; q < nqp; q++)
         {
            const IntegrationPoint &ip = ir.IntPoint(q);
-            tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
+            src_fes->GetFE(e_id)->CalcShape(ip, shape);
            Jtr(q) = Wideal; // Initialize to identity
            for (int d = 0; d < 4; d++)
            {
@@ -1576,9 +1678,16 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
            if (sizeidx != -1) // Set size
            {
               par_vals.SetDataAndSize(tspec_vals.GetData()+sizeidx*ndofs, ndofs);
-               const double min_size = par_vals.Min();
-               MFEM_VERIFY(min_size > 0.0,
-                           "Non-positive size propagated in the target definition.");
+               double min_size = par_vals.Min();//0.001; //
+               if (lim_min_size > 0.)
+               {
+                  min_size = lim_min_size;
+               }
+               else
+               {
+                  MFEM_VERIFY(min_size > 0.0,
+                              "Non-positive size propagated in the target definition.");
+               }
               const double size = std::max(shape * par_vals, min_size);
               Jtr(q).Set(std::pow(size, 1.0/dim), Jtr(q));
               DenseMatrix Jtrcomp_q(Jtrcomp.GetData(0 + 4*q), dim, dim);
@@ -1593,6 +1702,9 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
               {
                  par_vals.SetDataAndSize(tspec_vals.GetData()+
                                          aspectratioidx*ndofs, ndofs);
+                  const double min_size = par_vals.Min();
+                  MFEM_VERIFY(min_size > 0.0,
+                              "Non-positive aspect-ratio propagated in the target definition.");

                  const double aspectratio = shape * par_vals;
                  D_rho = 0.;
@@ -1777,7 +1889,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,

               grad_phys.Mult(par_vals, grad_ptr_c1);
               Vector grad_q(dim);
-               tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
+               tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
               grad_e_c1.MultTranspose(shape, grad_q);

               const double min_size = par_vals.Min();
@@ -1810,7 +1922,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,

                  grad_phys.Mult(par_vals, grad_ptr_c1);
                  Vector grad_q(dim);
-                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
+                  tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
                  grad_e_c1.MultTranspose(shape, grad_q);

                  const double aspectratio = shape * par_vals;
@@ -1841,7 +1953,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,
                  grad_phys.Mult(par_vals_c2, grad_ptr_c2);
                  grad_phys.Mult(par_vals_c3, grad_ptr_c3);
                  Vector grad_q1(dim), grad_q2(dim), grad_q3(dim);
-                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
+                  tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
                  grad_e_c1.MultTranspose(shape, grad_q1);
                  grad_e_c2.MultTranspose(shape, grad_q2);
                  grad_e_c3.MultTranspose(shape, grad_q3);
@@ -1880,7 +1992,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,

                  grad_phys.Mult(par_vals, grad_ptr_c1);
                  Vector grad_q(dim);
-                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
+                  tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
                  grad_e_c1.MultTranspose(shape, grad_q);

                  const double skew = shape * par_vals;
@@ -1913,7 +2025,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,
                  grad_phys.Mult(par_vals_c2, grad_ptr_c2);
                  grad_phys.Mult(par_vals_c3, grad_ptr_c3);
                  Vector grad_q1(dim), grad_q2(dim), grad_q3(dim);
-                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
+                  tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
                  grad_e_c1.MultTranspose(shape, grad_q1);
                  grad_e_c2.MultTranspose(shape, grad_q2);
                  grad_e_c3.MultTranspose(shape, grad_q3);
@@ -1960,7 +2072,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,

                  grad_phys.Mult(par_vals, grad_ptr_c1);
                  Vector grad_q(dim);
-                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
+                  tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
                  grad_e_c1.MultTranspose(shape, grad_q);

                  const double theta = shape * par_vals;
@@ -1991,7 +2103,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,
                  grad_phys.Mult(par_vals_c2, grad_ptr_c2);
                  grad_phys.Mult(par_vals_c3, grad_ptr_c3);
                  Vector grad_q1(dim), grad_q2(dim), grad_q3(dim);
-                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
+                  tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
                  grad_e_c1.MultTranspose(shape, grad_q1);
                  grad_e_c2.MultTranspose(shape, grad_q2);
                  grad_e_c3.MultTranspose(shape, grad_q3);
@@ -2071,7 +2183,7 @@ void DiscreteAdaptTC::UpdateGradientTargetSpecification(const Vector &x,
 {
   if (use_flag && good_tspec_grad) { return; }

-   const int dim = tspec_fes->GetFE(0)->GetDim(),
+   const int dim = tspec_fesv->GetFE(0)->GetDim(),
             cnt = x.Size()/dim;

   tspec_pert1h.SetSize(x.Size()*ncomp);
@@ -2097,7 +2209,7 @@ void DiscreteAdaptTC::UpdateHessianTargetSpecification(const Vector &x,

   if (use_flag && good_tspec_hess) { return; }

-   const int dim    = tspec_fes->GetFE(0)->GetDim(),
+   const int dim    = tspec_fesv->GetFE(0)->GetDim(),
             cnt    = x.Size()/dim,
             totmix = 1+2*(dim-2);

@@ -2145,6 +2257,16 @@ void DiscreteAdaptTC::UpdateHessianTargetSpecification(const Vector &x,
   good_tspec_hess = use_flag;
 }

+DiscreteAdaptTC::~DiscreteAdaptTC()
+{
+   delete tspec_gf;
+   delete adapt_eval;
+   delete tspec_fesv;
+#ifdef MFEM_USE_MPI
+   delete ptspec_fesv;
+#endif
+}
+
 void AdaptivityEvaluator::SetSerialMetaInfo(const Mesh &m,
                                            const FiniteElementCollection &fec,
                                            int num_comp)
@@ -2258,6 +2380,7 @@ void TMOP_Integrator::EnableAdaptiveLimiting(const ParGridFunction &z0,
                                             AdaptivityEvaluator &ae)
 {
   zeta_0 = &z0;
+   pzeta_0 = &z0;
   delete zeta;
   zeta   = new GridFunction(z0);
   coeff_zeta = &coeff;
@@ -2270,6 +2393,33 @@ void TMOP_Integrator::EnableAdaptiveLimiting(const ParGridFunction &z0,
 }
 #endif

+void TMOP_Integrator::UpdateAfterMeshTopologyChange()
+{
+   if (zeta)
+   {
+      zeta->Update();
+      adapt_eval->SetSerialMetaInfo(*zeta->FESpace()->GetMesh(),
+                                    *zeta->FESpace()->FEColl(), 1);
+      adapt_eval->SetInitialField
+      (*zeta->FESpace()->GetMesh()->GetNodes(), *zeta);
+   }
+}
+
+#ifdef MFEM_USE_MPI
+void TMOP_Integrator::ParUpdateAfterMeshTopologyChange()
+{
+   if (zeta)
+   {
+      zeta->Update();
+      adapt_eval->SetParMetaInfo(*pzeta_0->ParFESpace()->GetParMesh(),
+                                 *pzeta_0->ParFESpace()->FEColl(), 1);
+      adapt_eval->SetInitialField
+      (*zeta->FESpace()->GetMesh()->GetNodes(), *zeta);
+   }
+}
+#endif
+
+
 double TMOP_Integrator::GetElementEnergy(const FiniteElement &el,
                                         ElementTransformation &T,
                                         const Vector &elfun)
@@ -2378,6 +2528,145 @@ double TMOP_Integrator::GetElementEnergy(const FiniteElement &el,

   return energy;
 }
+
+double TMOP_Integrator::GetRefinementElementEnergy(const FiniteElement &el,
+                                                   ElementTransformation &T,
+                                                   const Vector &elfun,
+                                                   const IntegrationRule &irule)
+{
+   int dof = el.GetDof(), dim = el.GetDim(),
+       NEsplit = elfun.Size() / (dof*dim), el_id = T.ElementNo;
+   double energy = 0.;
+
+   TargetConstructor *tc = const_cast<TargetConstructor *>(targetC);
+   DiscreteAdaptTC *dtc = dynamic_cast<DiscreteAdaptTC *>(tc);
+   // For DiscreteAdaptTC the GridFunctions used to set the targets must be
+   // mapped onto the fine elements.
+   if (dtc) { dtc->SetTspecFromIntRule(el_id, irule); }
+
+   for (int e = 0; e < NEsplit; e++)
+   {
+      DSh.SetSize(dof, dim);
+      Jrt.SetSize(dim);
+      Jpr.SetSize(dim);
+      Jpt.SetSize(dim);
+      Vector elfun_child(dof*dim);
+      for (int i = 0; i < dof; i++)
+      {
+         for (int d = 0; d < dim; d++)
+         {
+            // elfun is (xe1,xe2,...xen,ye1,ye2...yen) and has nodal coordinates
+            // for all the children element of the parent element being considered.
+            // So we must index and get (xek, yek) i.e. nodal coordinates for
+            // the fine element being considered.
+            elfun_child(i + d*dof) = elfun(i + e*dof + d*dof*NEsplit);
+         }
+      }
+      PMatI.UseExternalData(elfun_child.GetData(), dof, dim);
+
+      const IntegrationRule &ir = EnergyIntegrationRule(el);
+
+      double el_energy = 0;
+      DenseTensor Jtr(dim, dim, ir.GetNPoints());
+      if (dtc)
+      {
+         // This is used to index into the tspec vector inside DiscreteAdaptTC.
+         dtc->SetRefinementSubElement(e);
+      }
+      targetC->ComputeElementTargets(el_id, el, ir, elfun_child, Jtr);
+
+      // Define ref->physical transformation, wn a Coefficient is specified.
+      IsoparametricTransformation *Tpr = NULL;
+      if (coeff1 || coeff0)
+      {
+         Tpr = new IsoparametricTransformation;
+         Tpr->SetFE(&el);
+         Tpr->ElementNo = T.ElementNo;
+         Tpr->ElementType = ElementTransformation::ELEMENT;
+         Tpr->Attribute = T.Attribute;
+         Tpr->GetPointMat().Transpose(PMatI); // PointMat = PMatI^T
+      }
+
+      for (int i = 0; i < ir.GetNPoints(); i++)
+      {
+         const IntegrationPoint &ip = ir.IntPoint(i);
+         const DenseMatrix &Jtr_i = Jtr(i);
+         h_metric->SetTargetJacobian(Jtr_i);
+         CalcInverse(Jtr_i, Jrt);
+         const double weight = ip.weight * Jtr_i.Det();
+
+         el.CalcDShape(ip, DSh);
+         MultAtB(PMatI, DSh, Jpr);
+         Mult(Jpr, Jrt, Jpt);
+
+         double val = metric_normal * h_metric->EvalW(Jpt);
+         if (coeff1) { val *= coeff1->Eval(*Tpr, ip); }
+
+         el_energy += weight * val;
+         delete Tpr;
+      }
+      energy += el_energy;
+   }
+   energy /= NEsplit;
+
+   if (dtc) { dtc->ResetRefinementTspecData(); }
+
+   return energy;
+}
+
+double TMOP_Integrator::GetDerefinementElementEnergy(const FiniteElement &el,
+                                                     ElementTransformation &T,
+                                                     const Vector &elfun)
+{
+   int dof = el.GetDof(), dim = el.GetDim();
+   double energy = 0.;
+
+   DSh.SetSize(dof, dim);
+   Jrt.SetSize(dim);
+   Jpr.SetSize(dim);
+   Jpt.SetSize(dim);
+   PMatI.UseExternalData(elfun.GetData(), dof, dim);
+
+   const IntegrationRule &ir = EnergyIntegrationRule(el);
+
+   energy = 0.0;
+   DenseTensor Jtr(dim, dim, ir.GetNPoints());
+   targetC->ComputeElementTargets(T.ElementNo, el, ir, elfun, Jtr);
+
+   // Define ref->physical transformation, wn a Coefficient is specified.
+   IsoparametricTransformation *Tpr = NULL;
+   if (coeff1)
+   {
+      Tpr = new IsoparametricTransformation;
+      Tpr->SetFE(&el);
+      Tpr->ElementNo = T.ElementNo;
+      Tpr->ElementType = ElementTransformation::ELEMENT;
+      Tpr->Attribute = T.Attribute;
+      Tpr->GetPointMat().Transpose(PMatI); // PointMat = PMatI^T
+   }
+
+   for (int i = 0; i < ir.GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir.IntPoint(i);
+      const DenseMatrix &Jtr_i = Jtr(i);
+      h_metric->SetTargetJacobian(Jtr_i);
+      CalcInverse(Jtr_i, Jrt);
+      const double weight = ip.weight * Jtr_i.Det();
+
+      el.CalcDShape(ip, DSh);
+      MultAtB(PMatI, DSh, Jpr);
+      Mult(Jpr, Jrt, Jpt);
+
+      double val = metric_normal * h_metric->EvalW(Jpt);
+      if (coeff1) { val *= coeff1->Eval(*Tpr, ip); }
+
+      energy += weight * val;
+   }
+
+   delete Tpr;
+   return energy;
+}
+
 void TMOP_Integrator::AssembleElementVector(const FiniteElement &el,
                                            ElementTransformation &T,
                                            const Vector &elfun, Vector &elvect)
@@ -3039,7 +3328,7 @@ void TMOP_Integrator::ComputeMinJac(const Vector &x,
   dx = detv_avg_min / dxscale;
 }

-void TMOP_Integrator::UpdateAfterMeshChange(const Vector &new_x)
+void TMOP_Integrator::UpdateAfterMeshPositionChange(const Vector &new_x)
 {
   if (discr_tc)
   {
@@ -3168,6 +3457,32 @@ void TMOPComboIntegrator::AssembleElementGrad(const FiniteElement &el,
   }
 }

+double TMOPComboIntegrator::GetRefinementElementEnergy(const FiniteElement &el,
+                                                       ElementTransformation &T,
+                                                       const Vector &elfun,
+                                                       const IntegrationRule &irule)
+{
+   double energy= 0.0;
+   for (int i = 0; i < tmopi.Size(); i++)
+   {
+      energy += tmopi[i]->GetRefinementElementEnergy(el, T, elfun, irule);
+   }
+   return energy;
+}
+
+double TMOPComboIntegrator::GetDerefinementElementEnergy(
+   const FiniteElement &el,
+   ElementTransformation &T,
+   const Vector &elfun)
+{
+   double energy= 0.0;
+   for (int i = 0; i < tmopi.Size(); i++)
+   {
+      energy += tmopi[i]->GetDerefinementElementEnergy(el, T, elfun);
+   }
+   return energy;
+}
+
 void TMOPComboIntegrator::EnableNormalization(const GridFunction &x)
 {
   const int cnt = tmopi.Size();
@@ -1057,14 +1057,31 @@ protected:
   // eta1(x+h,y), eta2(x+h,y) ... etan(x+h,y), eta1(x,y+h), eta2(x,y+h) ...
   // same for tspec_pert2h and tspec_pertmix.

+   // DenseMatrix to hold target_spec values for the (children of the)
+   // element being refined to consider for h-refinement.
+   DenseMatrix tspec_refine;
+   // Vector to hold the target_spec values for the coarse version of the
+   // current mesh. Used for derefinement decision with hr-adaptivity.
+   Vector tspec_derefine;
+
   // Components of Target Jacobian at each quadrature point of an element. This
   // is required for computation of the derivative using chain rule.
   mutable DenseTensor Jtrcomp;

   // Note: do not use the Nodes of this space as they may not be on the
   // positions corresponding to the values of tspec.
-   const FiniteElementSpace *tspec_fes;
-   const FiniteElementSpace *tspec_fesv;
+   FiniteElementSpace *tspec_fesv;         //owned
+   FiniteElementSpace *coarse_tspec_fesv;  //not owned, derefinement FESpace
+   GridFunction *tspec_gf;                 //owned, uses tspec and tspec_fes
+   // discrete adaptivity
+#ifdef MFEM_USE_MPI
+   ParFiniteElementSpace *ptspec_fesv;     //owned, needed for derefinement to
+   // get update operator.
+   ParGridFunction *tspec_pgf;             // similar to tspec_gf
+#endif
+
+   int amr_el;
+   double lim_min_size;

   // These flags can be used by outside functions to avoid recomputing the
   // tspec and tspec_perth fields again on the same mesh.
@@ -1076,7 +1093,7 @@ protected:

   void SetDiscreteTargetBase(const GridFunction &tspec_);
   void SetTspecAtIndex(int idx, const GridFunction &tspec_);
-   void FinalizeSerialDiscreteTargetSpec();
+   void FinalizeSerialDiscreteTargetSpec(const GridFunction &tspec_);
 #ifdef MFEM_USE_MPI
   void SetTspecAtIndex(int idx, const ParGridFunction &tspec_);
   void FinalizeParDiscreteTargetSpec(const ParGridFunction &tspec_);
@@ -1088,16 +1105,16 @@ public:
        ncomp(0),
        sizeidx(-1), skewidx(-1), aspectratioidx(-1), orientationidx(-1),
        tspec(), tspec_sav(), tspec_pert1h(), tspec_pert2h(), tspec_pertmix(),
-        tspec_fes(NULL), tspec_fesv(NULL),
+        tspec_refine(), tspec_derefine(),
+        tspec_fesv(NULL), coarse_tspec_fesv(NULL), tspec_gf(NULL),
+#ifdef MFEM_USE_MPI
+        ptspec_fesv(NULL), tspec_pgf(NULL),
+#endif
+        amr_el(-1), lim_min_size(-0.1),
        good_tspec(false), good_tspec_grad(false), good_tspec_hess(false),
        adapt_eval(NULL) { }

-   virtual ~DiscreteAdaptTC()
-   {
-      delete adapt_eval;
-      delete tspec_fes;
-      delete tspec_fesv;
-   }
+   virtual ~DiscreteAdaptTC();

   /** @name Target specification methods.
       The following methods are used to specify geometric parameters of the
@@ -1128,6 +1145,20 @@ public:
   void ResetUpdateFlags()
   { good_tspec = good_tspec_grad = good_tspec_hess = false; }

+   /// Get one of the discrete fields from tspec.
+   void GetDiscreteTargetSpec(GridFunction &tspec_, int idx);
+   /// Get the FESpace associated with tspec.
+   FiniteElementSpace *GetTSpecFESpace() { return tspec_fesv; }
+   /// Get the entire tspec.
+   GridFunction *GetTSpecData() { return tspec_gf; }
+   /// Update all discrete fields based on tspec and update for AMR
+   void UpdateAfterMeshTopologyChange();
+
+#ifdef MFEM_USE_MPI
+   ParFiniteElementSpace *GetTSpecParFESpace() { return ptspec_fesv; }
+   void ParUpdateAfterMeshTopologyChange();
+#endif
+
   /** Used to update the target specification after the mesh has changed. The
       new mesh positions are given by new_x. If @a use_flags is true, repeated
       calls won't do anything until ResetUpdateFlags() is called. */
@@ -1184,6 +1215,36 @@ public:
                                              const Vector &elfun,
                                              IsoparametricTransformation &Tpr,
                                              DenseTensor &dJtr) const;
+
+   // Generates tspec_vals for target construction using intrule
+   // Used for the refinement component in hr-adaptivity.
+   void SetTspecFromIntRule(int e_id, const IntegrationRule &intrule);
+
+   // Targets based on discrete functions can result in invalid (negative)
+   // size at the quadrature points. This method can be used to set a
+   // minimum target size.
+   void SetMinSizeForTargets(double min_size_) { lim_min_size = min_size_; }
+
+   /// Computes target specification data with respect to the coarse FE space.
+   void SetTspecDataForDerefinement(FiniteElementSpace *fes);
+
+   // Reset refinement data associated with h-adaptivity component.
+   void ResetRefinementTspecData()
+   {
+      tspec_refine.Clear();
+      amr_el = -1;
+   }
+
+   // Reset derefinement data associated with h-adaptivity component.
+   void ResetDerefinementTspecData()
+   {
+      tspec_derefine.Destroy();
+      coarse_tspec_fesv = NULL;
+   }
+
+   // Used to specify the fine element for determining energy of children of a
+   // parent element.
+   void SetRefinementSubElement(int amr_el_) { amr_el = amr_el_; }
 };

 class TMOPNewtonSolver;
@@ -1201,6 +1262,7 @@ protected:
   friend class TMOPNewtonSolver;
   friend class TMOPComboIntegrator;

+   TMOP_QualityMetric *h_metric;
   TMOP_QualityMetric *metric;        // not owned
   const TargetConstructor *targetC;  // not owned

@@ -1227,6 +1289,9 @@ protected:

   // Adaptive limiting.
   const GridFunction *zeta_0;       // Not owned.
+#ifdef MFEM_USE_MPI
+   const ParGridFunction *pzeta_0;
+#endif
   GridFunction *zeta;               // Owned. Updated by adapt_eval.
   Coefficient *coeff_zeta;          // Not owned.
   AdaptivityEvaluator *adapt_eval;  // Not owned.
@@ -1337,7 +1402,7 @@ protected:
 #endif
   void ComputeMinJac(const Vector &x, const FiniteElementSpace &fes);

-   void UpdateAfterMeshChange(const Vector &new_x);
+   void UpdateAfterMeshPositionChange(const Vector &new_x);

   void DisableLimiting()
   {
@@ -1395,11 +1460,13 @@ protected:
   void ComputeAllElementTargets(const Vector &xe = Vector()) const;

 public:
-   /** @param[in] m  TMOP_QualityMetric that will be integrated (not owned).
-       @param[in] tc Target-matrix construction algorithm to use (not owned). */
-   TMOP_Integrator(TMOP_QualityMetric *m, TargetConstructor *tc)
-      : metric(m), targetC(tc), IntegRules(NULL), integ_order(-1),
-        coeff1(NULL), metric_normal(1.0),
+   /** @param[in] m    TMOP_QualityMetric for r-adaptivity (not owned).
+       @param[in] tc   Target-matrix construction algorithm to use (not owned).
+       @param[in] hm   TMOP_QualityMetric for h-adaptivity (not owned). */
+   TMOP_Integrator(TMOP_QualityMetric *m, TargetConstructor *tc,
+                   TMOP_QualityMetric *hm)
+      : h_metric(hm), metric(m), targetC(tc), IntegRules(NULL),
+        integ_order(-1), coeff1(NULL), metric_normal(1.0),
        nodes0(NULL), coeff0(NULL),
        lim_dist(NULL), lim_func(NULL), lim_normal(1.0),
        zeta_0(NULL), zeta(NULL), coeff_zeta(NULL), adapt_eval(NULL),
@@ -1407,6 +1474,9 @@ public:
        fdflag(false), dxscale(1.0e3), fd_call_flag(false), exact_action(false)
   { PA.enabled = false; }

+   TMOP_Integrator(TMOP_QualityMetric *m, TargetConstructor *tc)
+      : TMOP_Integrator(m, tc, m) { }
+
   ~TMOP_Integrator();

   /// Release the device memory of large PA allocations. This will copy device
@@ -1478,6 +1548,22 @@ public:
                                   ElementTransformation &T,
                                   const Vector &elfun);

+   /** @brief Computes the mean of the energies of the given element's children.
+
+       In addition to the inputs for GetElementEnergy, this function requires an
+       IntegrationRule to be specified that will give the decomposition of the
+       given element based on the refinement type being considered. */
+   virtual double GetRefinementElementEnergy(const FiniteElement &el,
+                                             ElementTransformation &T,
+                                             const Vector &elfun,
+                                             const IntegrationRule &irule);
+
+   /// This function is similar to GetElementEnergy, but ignores components
+   /// such as limiting etc. to compute the element energy.
+   virtual double GetDerefinementElementEnergy(const FiniteElement &el,
+                                               ElementTransformation &T,
+                                               const Vector &elfun);
+
   virtual void AssembleElementVector(const FiniteElement &el,
                                      ElementTransformation &T,
                                      const Vector &elfun, Vector &elvect);
@@ -1486,6 +1572,13 @@ public:
                                    ElementTransformation &T,
                                    const Vector &elfun, DenseMatrix &elmat);

+   TMOP_QualityMetric &GetAMRQualityMetric() { return *h_metric; }
+
+   void UpdateAfterMeshTopologyChange();
+#ifdef MFEM_USE_MPI
+   void ParUpdateAfterMeshTopologyChange();
+#endif
+
   // PA extension
   using NonlinearFormIntegrator::AssemblePA;
   virtual void AssemblePA(const FiniteElementSpace&);
@@ -1564,6 +1657,15 @@ public:
                                    ElementTransformation &T,
                                    const Vector &elfun, DenseMatrix &elmat);

+   virtual double GetRefinementElementEnergy(const FiniteElement &el,
+                                             ElementTransformation &T,
+                                             const Vector &elfun,
+                                             const IntegrationRule &irule);
+
+   virtual double GetDerefinementElementEnergy(const FiniteElement &el,
+                                               ElementTransformation &T,
+                                               const Vector &elfun);
+
   /// Normalization factor that considers all integrators in the combination.
   void EnableNormalization(const GridFunction &x);
 #ifdef MFEM_USE_MPI
@@ -0,0 +1,896 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "tmop_amr.hpp"
+
+namespace mfem
+{
+
+using namespace mfem;
+
+void TMOPRefinerEstimator::ComputeEstimates()
+{
+   bool iso = false;
+   bool aniso = false;
+   if (amrmetric == 1 || amrmetric == 2 || amrmetric == 58)
+   {
+      aniso = true;
+   }
+   if (amrmetric == 55 || amrmetric == 56 || amrmetric == 77 ||
+       amrmetric == 315 || amrmetric == 316 || amrmetric == 321)
+   {
+      iso = true;
+   }
+   if (amrmetric == 7 || amrmetric == 9)
+   {
+      iso = true; aniso = true;
+   }
+
+   MFEM_VERIFY(iso || aniso, "Metric type not supported in hr-adaptivity.");
+
+   const int dim = mesh->Dimension();
+   const int num_ref_types = 3 + 4*(dim-2);
+   const int NEorig = mesh->GetNE();
+
+   aniso_flags.SetSize(NEorig);
+   error_estimates.SetSize(NEorig);
+   Vector amr_base_energy(NEorig), amr_temp_energy(NEorig);
+   error_estimates = 1.*std::numeric_limits<float>::max();
+   aniso_flags = -1;
+   GetTMOPRefinementEnergy(0, amr_base_energy);
+
+   for (int i = 1; i < num_ref_types+1; i++)
+   {
+      if ( dim == 2 && i < 3  && aniso != true ) { continue; }
+      if ( dim == 2 && i == 3 && iso   != true ) { continue; }
+      if ( dim == 3 && i < 7  && aniso != true ) { continue; }
+      if ( dim == 3 && i == 7 && iso   != true ) { continue; }
+
+      GetTMOPRefinementEnergy(i, amr_temp_energy);
+
+      for (int e = 0; e < NEorig; e++)
+      {
+         if ( amr_temp_energy(e) < error_estimates(e) )
+         {
+            error_estimates(e) = amr_temp_energy(e);
+            aniso_flags[e] = i;
+         }
+      }
+   }
+   error_estimates *= energy_scaling_factor;
+
+   if (spat_gf)
+   {
+      L2_FECollection avg_fec(0, mesh->Dimension());
+      FiniteElementSpace avg_fes(spat_gf->FESpace()->GetMesh(), &avg_fec);
+      GridFunction elem_avg(&avg_fes);
+      spat_gf->GetElementAverages(elem_avg);
+      for (int i = 0; i < amr_base_energy.Size(); i++)
+      {
+         if (elem_avg(i) < spat_gf_critical) { amr_base_energy(i) = 0.; }
+      }
+   }
+
+   error_estimates -= amr_base_energy;
+   error_estimates *= -1; // error = E(parent) - scaling_factor*mean(E(children))
+   current_sequence = mesh->GetSequence();
+}
+
+void TMOPRefinerEstimator::GetTMOPRefinementEnergy(int reftype,
+                                                   Vector &el_energy_vec)
+{
+   const FiniteElementSpace *fes = mesh->GetNodalFESpace();
+   const int NE = fes->GetNE();
+   GridFunction *xdof = mesh->GetNodes();
+   xdof->SetTrueVector();
+   xdof->SetFromTrueVector();
+
+   el_energy_vec.SetSize(NE);
+   el_energy_vec = std::numeric_limits<float>::max();
+
+   for (int e = 0; e < NE; e++)
+   {
+      Geometry::Type gtype = fes->GetFE(e)->GetGeomType();
+      DenseMatrix tr, xsplit;
+      IntegrationRule *irule = NULL;
+
+      if ( (gtype == Geometry::TRIANGLE && reftype > 0 && reftype < 3) ||
+           (gtype == Geometry::CUBE && reftype > 0 && reftype < 7) ||
+           (gtype == Geometry::TETRAHEDRON && reftype > 0 && reftype < 7) )
+      {
+         continue;
+      }
+
+      switch (gtype)
+      {
+         case Geometry::TRIANGLE:
+         {
+            int ref_access = reftype == 0 ? 0 : 1;
+            xdof->GetVectorValues(e, *TriIntRule[ref_access], xsplit, tr);
+            irule = TriIntRule[ref_access];
+            break;
+         }
+         case Geometry::TETRAHEDRON:
+         {
+            int ref_access = reftype == 0 ? 0 : 1;
+            xdof->GetVectorValues(e, *TetIntRule[ref_access], xsplit, tr);
+            irule = TetIntRule[ref_access];
+            break;
+         }
+         case Geometry::SQUARE:
+         {
+            MFEM_VERIFY(QuadIntRule[reftype], " Integration rule does not exist.");
+            xdof->GetVectorValues(e, *QuadIntRule[reftype], xsplit, tr);
+            irule = QuadIntRule[reftype];
+            break;
+         }
+         case Geometry::CUBE:
+         {
+            int ref_access = reftype == 0 ? 0 : 1;
+            xdof->GetVectorValues(e, *HexIntRule[ref_access], xsplit, tr);
+            irule = HexIntRule[ref_access];
+            break;
+         }
+         default:
+            MFEM_ABORT("Incompatible geometry type!");
+      }
+      xsplit.Transpose();
+
+      el_energy_vec(e) = 0.; // Re-set to 0
+
+      // The data format is xe1,xe2,..xen,ye1,ye2..yen.
+      // We will reformat it inside GetRefinementElementEnergy
+      Vector elfun(xsplit.GetData(), xsplit.NumCols()*xsplit.NumRows());
+
+      Array<NonlinearFormIntegrator*> &integs = *(nlf->GetDNFI());
+      TMOP_Integrator *ti  = NULL;
+      TMOPComboIntegrator *co = NULL;
+      for (int i = 0; i < integs.Size(); i++)
+      {
+         ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
+         if (ti)
+         {
+            el_energy_vec(e) = ti->GetRefinementElementEnergy(*fes->GetFE(e),
+                                                              *mesh->GetElementTransformation(e),
+                                                              elfun,
+                                                              *irule);
+         }
+         co = dynamic_cast<TMOPComboIntegrator *>(integs[i]);
+         if (co)
+         {
+            Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
+            for (int j = 0; j < ati.Size(); j++)
+            {
+               el_energy_vec(e) += ati[j]->GetRefinementElementEnergy(*fes->GetFE(e),
+                                                                      *mesh->GetElementTransformation(e),
+                                                                      elfun,
+                                                                      *irule);
+            }
+         }
+      }
+   }
+}
+
+void TMOPRefinerEstimator::SetHexIntRules()
+{
+   HexIntRule.SetSize(1+1);
+   // Reftype = 0 -> original element
+   Mesh meshsplit = Mesh::MakeCartesian3D(1, 1, 1, Element::HEXAHEDRON);
+   Mesh base_mesh_copy(meshsplit);
+   HexIntRule[0] = SetIntRulesFromMesh(meshsplit);
+   meshsplit.Clear();
+
+   // Reftype = 7
+   for (int i = 7; i < 8; i++)
+   {
+      Array<Refinement> marked_elements;
+      Mesh mesh_ref(base_mesh_copy);
+      for (int e = 0; e < mesh_ref.GetNE(); e++)
+      {
+         marked_elements.Append(Refinement(e, i));
+      }
+      mesh_ref.GeneralRefinement(marked_elements, 1, 0);
+      HexIntRule[1] = SetIntRulesFromMesh(mesh_ref);
+      mesh_ref.Clear();
+   }
+}
+
+void TMOPRefinerEstimator::SetQuadIntRules()
+{
+   QuadIntRule.SetSize(3+1);
+
+   // Reftype = 0 -> original element
+   Mesh meshsplit = Mesh::MakeCartesian2D(1, 1, Element::QUADRILATERAL);
+   Mesh base_mesh_copy(meshsplit);
+   QuadIntRule[0] = SetIntRulesFromMesh(meshsplit);
+   meshsplit.Clear();
+
+   // Reftype = 1-3
+   for (int i = 1; i < 4; i++)
+   {
+      Array<Refinement> marked_elements;
+      Mesh mesh_ref(base_mesh_copy);
+      for (int e = 0; e < mesh_ref.GetNE(); e++)
+      {
+         marked_elements.Append(Refinement(e, i));
+      }
+      mesh_ref.GeneralRefinement(marked_elements, 1, 0);
+      QuadIntRule[i] = SetIntRulesFromMesh(mesh_ref);
+      mesh_ref.Clear();
+   }
+}
+
+void TMOPRefinerEstimator::SetTriIntRules()
+{
+   TriIntRule.SetSize(1+1);
+
+   // Reftype = 0 // original element
+   const int Nvert = 3, NEsplit = 1;
+   Mesh meshsplit(2, Nvert, NEsplit, 0, 2);
+   const double tri_v[3][2] =
+   {
+      {0, 0}, {1, 0}, {0, 1}
+   };
+   const int tri_e[1][3] =
+   {
+      {0, 1, 2}
+   };
+
+   for (int j = 0; j < Nvert; j++)
+   {
+      meshsplit.AddVertex(tri_v[j]);
+   }
+   meshsplit.AddTriangle(tri_e[0], 1);
+   meshsplit.FinalizeTriMesh(1, 1, true);
+
+   Mesh base_mesh_copy(meshsplit);
+   TriIntRule[0] = SetIntRulesFromMesh(meshsplit);
+   meshsplit.Clear();
+
+   // no anisotropic refinements for triangle
+   // Reftype = 3
+   for (int i = 1; i < 2; i++)
+   {
+      Array<Refinement> marked_elements;
+      Mesh mesh_ref(base_mesh_copy);
+      for (int e = 0; e < mesh_ref.GetNE(); e++)
+      {
+         marked_elements.Append(Refinement(e, i));
+      }
+      mesh_ref.GeneralRefinement(marked_elements, 1, 0);
+      TriIntRule[i] = SetIntRulesFromMesh(mesh_ref);
+      mesh_ref.Clear();
+   }
+}
+
+void TMOPRefinerEstimator::SetTetIntRules()
+{
+   TetIntRule.SetSize(1+1);
+
+   // Reftype = 0 // original element
+   const int Nvert = 4, NEsplit = 1;
+   Mesh meshsplit(3, Nvert, NEsplit, 0, 3);
+   const double tet_v[4][3] =
+   {
+      {0, 0, 0}, {1, 0, 0}, {0, 1, 0}, {0, 0, 1}
+   };
+   const int tet_e[1][4] =
+   {
+      {0, 1, 2, 3}
+   };
+
+   for (int j = 0; j < Nvert; j++)
+   {
+      meshsplit.AddVertex(tet_v[j]);
+   }
+   meshsplit.AddTet(tet_e[0], 1);
+   meshsplit.FinalizeTetMesh(1, 1, true);
+
+   Mesh base_mesh_copy(meshsplit);
+   TetIntRule[0] = SetIntRulesFromMesh(meshsplit);
+   meshsplit.Clear();
+
+   // no anisotropic refinements for triangle
+   // Reftype = 7
+   for (int i = 1; i < 2; i++)
+   {
+      Array<Refinement> marked_elements;
+      Mesh mesh_ref(base_mesh_copy);
+      for (int e = 0; e < mesh_ref.GetNE(); e++)
+      {
+         marked_elements.Append(Refinement(e, i)); //ref_type will default to 7
+      }
+      mesh_ref.GeneralRefinement(marked_elements, 1, 0);
+      TetIntRule[i] = SetIntRulesFromMesh(mesh_ref);
+      mesh_ref.Clear();
+   }
+}
+
+IntegrationRule* TMOPRefinerEstimator::SetIntRulesFromMesh(Mesh &meshsplit)
+{
+   const int dim = meshsplit.Dimension();
+   H1_FECollection fec(order, dim);
+   FiniteElementSpace nodal_fes(&meshsplit, &fec, dim);
+   meshsplit.SetNodalFESpace(&nodal_fes);
+
+   const int NEsplit = meshsplit.GetNE();
+   const int dof_cnt = nodal_fes.GetFE(0)->GetDof(),
+             pts_cnt = NEsplit * dof_cnt;
+
+   DenseMatrix pos(dof_cnt, dim);
+   Vector posV(pos.Data(), dof_cnt * dim);
+   Array<int> xdofs(dof_cnt * dim);
+
+   // Create an IntegrationRule on the nodes of the reference submesh.
+   IntegrationRule *irule = new IntegrationRule(pts_cnt);
+   GridFunction *nodesplit = meshsplit.GetNodes();
+
+   int pt_id = 0;
+   for (int i = 0; i < NEsplit; i++)
+   {
+      nodal_fes.GetElementVDofs(i, xdofs);
+      nodesplit->GetSubVector(xdofs, posV);
+      for (int j = 0; j < dof_cnt; j++)
+      {
+         if (dim == 2)
+         {
+            irule->IntPoint(pt_id).Set2(pos(j, 0), pos(j, 1));
+         }
+         else if (dim == 3)
+         {
+            irule->IntPoint(pt_id).Set3(pos(j, 0), pos(j, 1), pos(j, 2));
+         }
+         pt_id++;
+      }
+   }
+   return irule;
+}
+
+bool TMOPDeRefinerEstimator::GetDerefineEnergyForIntegrator(
+   TMOP_Integrator &tmopi,
+   Vector &fine_energy)
+{
+   DiscreteAdaptTC *tcd = tmopi.GetDiscreteAdaptTC();
+   fine_energy.SetSize(mesh->GetNE());
+
+   if (serial)
+   {
+      Mesh meshcopy(*mesh);
+      FiniteElementSpace *tcdfes = NULL;
+      if (tcd)
+      {
+         tcdfes = new FiniteElementSpace(*tcd->GetTSpecFESpace(), &meshcopy);
+      }
+
+      Vector local_err(meshcopy.GetNE());
+      local_err = 0.;
+      double threshold = std::numeric_limits<float>::max();
+      meshcopy.DerefineByError(local_err, threshold, 0, 1);
+
+      if (meshcopy.GetGlobalNE() == mesh->GetGlobalNE())
+      {
+         delete tcdfes;
+         return false;
+      }
+
+      if (tcd)
+      {
+         tcdfes->Update();
+         tcd->SetTspecDataForDerefinement(tcdfes);
+      }
+
+      Vector coarse_energy(meshcopy.GetNE());
+      GetTMOPDerefinementEnergy(meshcopy, tmopi, coarse_energy);
+      if (tcd) { tcd->ResetDerefinementTspecData(); }
+      GetTMOPDerefinementEnergy(*mesh, tmopi, fine_energy);
+
+      const CoarseFineTransformations &dtrans =
+         meshcopy.ncmesh->GetDerefinementTransforms();
+      Table coarse_to_fine;
+      dtrans.GetCoarseToFineMap(meshcopy, coarse_to_fine);
+
+      for (int pe = 0; pe < coarse_to_fine.Size(); pe++)
+      {
+         Array<int> tabrow;
+         coarse_to_fine.GetRow(pe, tabrow);
+         int nchild = tabrow.Size();
+         double parent_energy = coarse_energy(pe);
+         for (int fe = 0; fe < nchild; fe++)
+         {
+            int child = tabrow[fe];
+            MFEM_VERIFY(child < mesh->GetNE(), " invalid coarse to fine mapping");
+            fine_energy(child) -= parent_energy;
+         }
+      }
+      delete tcdfes;
+   }
+   else
+   {
+#ifdef MFEM_USE_MPI
+      ParMesh meshcopy(*pmesh);
+      ParFiniteElementSpace *tcdfes = NULL;
+      if (tcd)
+      {
+         tcdfes = new ParFiniteElementSpace(*tcd->GetTSpecParFESpace(), meshcopy);
+      }
+
+      Vector local_err(meshcopy.GetNE());
+      local_err = 0.;
+      double threshold = std::numeric_limits<float>::max();
+      meshcopy.DerefineByError(local_err, threshold, 0, 1);
+
+      if (meshcopy.GetGlobalNE() == pmesh->GetGlobalNE())
+      {
+         delete tcdfes;
+         return false;
+      }
+
+      if (tcd)
+      {
+         tcdfes->Update();
+         tcd->SetTspecDataForDerefinement(tcdfes);
+      }
+
+      Vector coarse_energy(meshcopy.GetNE());
+      GetTMOPDerefinementEnergy(meshcopy, tmopi, coarse_energy);
+      if (tcd) { tcd->ResetDerefinementTspecData(); }
+      GetTMOPDerefinementEnergy(*pmesh, tmopi, fine_energy);
+
+      const CoarseFineTransformations &dtrans =
+         meshcopy.pncmesh->GetDerefinementTransforms();
+      Table coarse_to_fine;
+      dtrans.GetCoarseToFineMap(meshcopy, coarse_to_fine);
+
+      for (int pe = 0; pe < meshcopy.GetNE(); pe++)
+      {
+         Array<int> tabrow;
+         coarse_to_fine.GetRow(pe, tabrow);
+         int nchild = tabrow.Size();
+         double parent_energy = coarse_energy(pe);
+         for (int fe = 0; fe < nchild; fe++)
+         {
+            int child = tabrow[fe];
+            MFEM_VERIFY(child < pmesh->GetNE(), " invalid coarse to fine mapping");
+            fine_energy(child) -= parent_energy;
+         }
+      }
+      delete tcdfes;
+#endif
+   }
+
+   // error_estimate(e) = energy(parent_of_e)-energy(e)
+   // Negative energy means derefinement is desirable.
+   fine_energy *= -1;
+   return true;
+}
+
+void TMOPDeRefinerEstimator::ComputeEstimates()
+{
+   Array<NonlinearFormIntegrator*> &integs = *(nlf->GetDNFI());
+   TMOP_Integrator *ti  = NULL;
+   TMOPComboIntegrator *co = NULL;
+   error_estimates.SetSize(mesh->GetNE());
+   error_estimates = 0.;
+   Vector fine_energy(mesh->GetNE());
+
+   for (int i = 0; i < integs.Size(); i++)
+   {
+      ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
+      if (ti)
+      {
+         bool deref = GetDerefineEnergyForIntegrator(*ti, fine_energy);
+         if (!deref) { error_estimates = 1; return; }
+         error_estimates += fine_energy;
+      }
+      co = dynamic_cast<TMOPComboIntegrator *>(integs[i]);
+      if (co)
+      {
+         Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
+         for (int j = 0; j < ati.Size(); j++)
+         {
+            bool deref = GetDerefineEnergyForIntegrator(*ati[j], fine_energy);
+            if (!deref) { error_estimates = 1; return; }
+            error_estimates += fine_energy;
+         }
+      }
+   }
+}
+
+void TMOPDeRefinerEstimator::GetTMOPDerefinementEnergy(Mesh &cmesh,
+                                                       TMOP_Integrator &tmopi,
+                                                       Vector &el_energy_vec)
+{
+   const int cNE = cmesh.GetNE();
+   el_energy_vec.SetSize(cNE);
+   const FiniteElementSpace *fespace = cmesh.GetNodalFESpace();
+
+   GridFunction *cxdof = cmesh.GetNodes();
+
+   Array<int> vdofs;
+   Vector el_x;
+   const FiniteElement *fe;
+   ElementTransformation *T;
+
+   for (int j = 0; j < cNE; j++)
+   {
+      fe = fespace->GetFE(j);
+      fespace->GetElementVDofs(j, vdofs);
+      T = cmesh.GetElementTransformation(j);
+      cxdof->GetSubVector(vdofs, el_x);
+      el_energy_vec(j) = tmopi.GetDerefinementElementEnergy(*fe, *T, el_x);
+   }
+}
+
+
+TMOPHRSolver::TMOPHRSolver(Mesh &mesh_, NonlinearForm &nlf_,
+                           TMOPNewtonSolver &tmopns_, GridFunction &x_,
+                           bool move_bnd_, bool hradaptivity_,
+                           int mesh_poly_deg_, int amr_metric_id_,
+                           int hr_iter_, int h_per_r_iter_) :
+   mesh(&mesh_), nlf(&nlf_), tmopns(&tmopns_), x(&x_),
+   gridfuncarr(), fespacearr(),
+   move_bnd(move_bnd_), hradaptivity(hradaptivity_),
+   mesh_poly_deg(mesh_poly_deg_), amr_metric_id(amr_metric_id_),
+   serial(true), hr_iter(hr_iter_), h_per_r_iter(h_per_r_iter_)
+{
+   if (!hradaptivity) { return; }
+   tmop_r_est = new TMOPRefinerEstimator(*mesh, *nlf, mesh_poly_deg,
+                                         amr_metric_id);
+   tmop_r = new ThresholdRefiner(*tmop_r_est);
+   tmop_r->SetTotalErrorFraction(0.0);
+   tmop_r_est->SetEnergyScalingFactor(1.);
+   tmop_dr_est= new TMOPDeRefinerEstimator(*mesh, *nlf);
+   tmop_dr = new ThresholdDerefiner(*tmop_dr_est);
+   AddGridFunctionForUpdate(x);
+}
+
+#ifdef MFEM_USE_MPI
+TMOPHRSolver::TMOPHRSolver(ParMesh &pmesh_, ParNonlinearForm &pnlf_,
+                           TMOPNewtonSolver &tmopns_, ParGridFunction &px_,
+                           bool move_bnd_, bool hradaptivity_,
+                           int mesh_poly_deg_, int amr_metric_id_,
+                           int hr_iter_, int h_per_r_iter_) :
+   mesh(&pmesh_), nlf(&pnlf_), tmopns(&tmopns_), x(&px_),
+   gridfuncarr(), fespacearr(),
+   move_bnd(move_bnd_), hradaptivity(hradaptivity_),
+   mesh_poly_deg(mesh_poly_deg_), amr_metric_id(amr_metric_id_),
+   pmesh(&pmesh_), pnlf(&pnlf_), pgridfuncarr(), pfespacearr(),
+   serial(false), hr_iter(hr_iter_), h_per_r_iter(h_per_r_iter_)
+{
+   if (!hradaptivity) { return; }
+   tmop_r_est = new TMOPRefinerEstimator(*pmesh, *pnlf, mesh_poly_deg,
+                                         amr_metric_id);
+   tmop_r = new ThresholdRefiner(*tmop_r_est);
+   tmop_r->SetTotalErrorFraction(0.0);
+   tmop_r_est->SetEnergyScalingFactor(1.);
+   tmop_dr_est= new TMOPDeRefinerEstimator(*pmesh, *pnlf);
+   tmop_dr = new ThresholdDerefiner(*tmop_dr_est);
+   AddGridFunctionForUpdate(&px_);
+}
+#endif
+
+void TMOPHRSolver::Mult()
+{
+   Vector b(0);
+   int myid = 0;
+   if (serial)
+   {
+      tmopns->SetOperator(*nlf);
+   }
+   else
+   {
+#ifdef MFEM_USE_MPI
+      myid = pnlf->ParFESpace()->GetMyRank();
+      tmopns->SetOperator(*pnlf);
+#endif
+   }
+   if (!hradaptivity)
+   {
+      tmopns->Mult(b, x->GetTrueVector());
+      if (tmopns->GetConverged() == false)
+      {
+         if (myid == 0) { mfem::out << "Nonlinear solver: rtol not achieved.\n"; }
+      }
+      x->SetFromTrueVector();
+      return;
+   }
+
+   bool radaptivity = true;
+
+   tmop_dr->Reset();
+   tmop_r->Reset();
+
+   if (serial)
+   {
+      for (int i_hr = 0; i_hr < hr_iter; i_hr++)
+      {
+         if (!radaptivity)
+         {
+            break;
+         }
+         mfem::out << i_hr << " r-adaptivity iteration.\n";
+
+         tmopns->SetOperator(*nlf);
+         tmopns->Mult(b, x->GetTrueVector());
+         x->SetFromTrueVector();
+
+         mfem::out << "TMOP energy after r-adaptivity: " <<
+                   nlf->GetGridFunctionEnergy(*x)/mesh->GetNE() <<
+                   ", Elements: " << mesh->GetNE() << std::endl;
+
+         for (int i_h = 0; i_h < h_per_r_iter; i_h++)
+         {
+            // Derefinement step.
+            if (mesh->ncmesh)
+            {
+               tmop_dr->Apply(*mesh);
+               Update();
+            }
+            mfem::out << "TMOP energy after derefinement: " <<
+                      nlf->GetGridFunctionEnergy(*x)/mesh->GetNE() <<
+                      ", Elements: " << mesh->GetNE() << std::endl;
+
+            // Refinement step.
+            tmop_r->Apply(*mesh);
+            Update();
+            mfem::out << "TMOP energy after   refinement: " <<
+                      nlf->GetGridFunctionEnergy(*x)/mesh->GetNE() <<
+                      ", Elements: " << mesh->GetNE() << std::endl;
+
+            if (!tmop_dr->Derefined() && tmop_r->Stop())
+            {
+               radaptivity = false;
+               mfem::out << "AMR stopping criterion satisfied. Stop.\n";
+               break;
+            }
+         } //n_h
+      } //n_hr
+   }
+   else
+   {
+#ifdef MFEM_USE_MPI
+      int NEGlob;
+      double tmopenergy;
+      for (int i_hr = 0; i_hr < hr_iter; i_hr++)
+      {
+         if (!radaptivity)
+         {
+            break;
+         }
+         if (myid == 0) { mfem::out << i_hr << " r-adaptivity iteration.\n"; }
+         tmopns->SetOperator(*pnlf);
+         tmopns->Mult(b, x->GetTrueVector());
+         x->SetFromTrueVector();
+
+         NEGlob = pmesh->GetGlobalNE();
+         tmopenergy = pnlf->GetParGridFunctionEnergy(*x) / NEGlob;
+         if (myid == 0)
+         {
+            mfem::out << "TMOP energy after r-adaptivity: " << tmopenergy <<
+                      ", Elements: " << NEGlob << std::endl;
+         }
+
+         for (int i_h = 0; i_h < h_per_r_iter; i_h++)
+         {
+            // Derefinement step.
+            if (pmesh->pncmesh)
+            {
+               RebalanceParNCMesh();
+               ParUpdate();
+
+               tmop_dr->Apply(*pmesh);
+               ParUpdate();
+            }
+            NEGlob = pmesh->GetGlobalNE();
+            tmopenergy = pnlf->GetParGridFunctionEnergy(*x) / NEGlob;
+            if (myid == 0)
+            {
+               mfem::out << "TMOP energy after derefinement: " << tmopenergy <<
+                         ", Elements: " << NEGlob << std::endl;
+            }
+
+            // Refinement step.
+            tmop_r->Apply(*pmesh);
+            ParUpdate();
+            NEGlob = pmesh->GetGlobalNE();
+            tmopenergy = pnlf->GetParGridFunctionEnergy(*x) / NEGlob;
+            if (myid == 0)
+            {
+               mfem::out << "TMOP energy after   refinement: " << tmopenergy <<
+                         ", Elements: " << NEGlob << std::endl;
+            }
+
+            if (!tmop_dr->Derefined() && tmop_r->Stop())
+            {
+               radaptivity = false;
+               if (myid == 0)
+               {
+                  mfem::out << "AMR stopping criterion satisfied. Stop.\n";
+               }
+               break;
+            }
+         } //n_r limit
+      } //n_hr
+#endif
+   }
+}
+
+#ifdef MFEM_USE_MPI
+void TMOPHRSolver::RebalanceParNCMesh()
+{
+   ParNCMesh *pncmesh = pmesh->pncmesh;
+   if (pncmesh)
+   {
+      const Table &dreftable = pncmesh->GetDerefinementTable();
+      Array<int> drefs, new_ranks;
+      for (int i = 0; i < dreftable.Size(); i++)
+      {
+         drefs.Append(i);
+      }
+      pncmesh->GetFineToCoarsePartitioning(drefs, new_ranks);
+      pmesh->Rebalance(new_ranks);
+   }
+}
+#endif
+
+void TMOPHRSolver::Update()
+{
+   // Update FESpace
+   for (int i = 0; i < fespacearr.Size(); i++)
+   {
+      fespacearr[i]->Update();
+   }
+   // Update nodal GF
+   for (int i = 0; i < gridfuncarr.Size(); i++)
+   {
+      gridfuncarr[i]->Update();
+      gridfuncarr[i]->SetTrueVector();
+      gridfuncarr[i]->SetFromTrueVector();
+   }
+
+   // Update Discrete Indicator for all the TMOP_Integrators in NonLinearForm
+   Array<NonlinearFormIntegrator*> &integs = *(nlf->GetDNFI());
+   TMOP_Integrator *ti  = NULL;
+   TMOPComboIntegrator *co = NULL;
+   DiscreteAdaptTC *dtc = NULL;
+   for (int i = 0; i < integs.Size(); i++)
+   {
+      ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
+      if (ti)
+      {
+         ti->UpdateAfterMeshTopologyChange();
+         dtc = ti->GetDiscreteAdaptTC();
+         if (dtc) { dtc->UpdateAfterMeshTopologyChange(); }
+      }
+      co = dynamic_cast<TMOPComboIntegrator *>(integs[i]);
+      if (co)
+      {
+         Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
+         for (int j = 0; j < ati.Size(); j++)
+         {
+            ati[j]->UpdateAfterMeshTopologyChange();
+            dtc = ati[j]->GetDiscreteAdaptTC();
+            if (dtc) { dtc->UpdateAfterMeshTopologyChange(); }
+         }
+      }
+   }
+
+   // Update the Nonlinear form and set Essential BC.
+   UpdateNonlinearFormAndBC(mesh, nlf);
+}
+
+#ifdef MFEM_USE_MPI
+void TMOPHRSolver::ParUpdate()
+{
+   // Update FESpace
+   for (int i = 0; i < pfespacearr.Size(); i++)
+   {
+      pfespacearr[i]->Update();
+   }
+   // Update nodal GF
+   for (int i = 0; i < pgridfuncarr.Size(); i++)
+   {
+      pgridfuncarr[i]->Update();
+      pgridfuncarr[i]->SetTrueVector();
+      pgridfuncarr[i]->SetFromTrueVector();
+   }
+
+   // Update Discrete Indicator
+   Array<NonlinearFormIntegrator*> &integs = *(nlf->GetDNFI());
+   TMOP_Integrator *ti  = NULL;
+   TMOPComboIntegrator *co = NULL;
+   DiscreteAdaptTC *dtc = NULL;
+   for (int i = 0; i < integs.Size(); i++)
+   {
+      ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
+      if (ti)
+      {
+         ti->ParUpdateAfterMeshTopologyChange();
+         dtc = ti->GetDiscreteAdaptTC();
+         if (dtc) { dtc->ParUpdateAfterMeshTopologyChange(); }
+      }
+      co = dynamic_cast<TMOPComboIntegrator *>(integs[i]);
+      if (co)
+      {
+         Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
+         for (int j = 0; j < ati.Size(); j++)
+         {
+            ati[j]->ParUpdateAfterMeshTopologyChange();
+            dtc = ati[j]->GetDiscreteAdaptTC();
+            if (dtc) { dtc->ParUpdateAfterMeshTopologyChange(); }
+         }
+      }
+   }
+
+   // Update the Nonlinear form and set Essential BC.
+   UpdateNonlinearFormAndBC(pmesh, pnlf);
+}
+#endif
+
+void TMOPHRSolver::UpdateNonlinearFormAndBC(Mesh *mesh, NonlinearForm *nlf)
+{
+   const FiniteElementSpace &fes = *mesh->GetNodalFESpace();
+
+   // Update Nonlinear form and Set Essential BC
+   nlf->Update();
+   const int dim = fes.GetFE(0)->GetDim();
+   if (move_bnd == false)
+   {
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+      nlf->SetEssentialBC(ess_bdr);
+   }
+   else
+   {
+      const int nd  = fes.GetBE(0)->GetDof();
+      int n = 0;
+      for (int i = 0; i < mesh->GetNBE(); i++)
+      {
+         const int attr = mesh->GetBdrElement(i)->GetAttribute();
+         MFEM_VERIFY(!(dim == 2 && attr == 3),
+                     "Boundary attribute 3 must be used only for 3D meshes. "
+                     "Adjust the attributes (1/2/3/4 for fixed x/y/z/all "
+                     "components, rest for free nodes), or use -fix-bnd.");
+         if (attr == 1 || attr == 2 || attr == 3) { n += nd; }
+         if (attr == 4) { n += nd * dim; }
+      }
+      Array<int> ess_vdofs(n), vdofs;
+      n = 0;
+      for (int i = 0; i < mesh->GetNBE(); i++)
+      {
+         const int attr = mesh->GetBdrElement(i)->GetAttribute();
+         fes.GetBdrElementVDofs(i, vdofs);
+         if (attr == 1) // Fix x components.
+         {
+            for (int j = 0; j < nd; j++)
+            { ess_vdofs[n++] = vdofs[j]; }
+         }
+         else if (attr == 2) // Fix y components.
+         {
+            for (int j = 0; j < nd; j++)
+            { ess_vdofs[n++] = vdofs[j+nd]; }
+         }
+         else if (attr == 3) // Fix z components.
+         {
+            for (int j = 0; j < nd; j++)
+            { ess_vdofs[n++] = vdofs[j+2*nd]; }
+         }
+         else if (attr == 4) // Fix all components.
+         {
+            for (int j = 0; j < vdofs.Size(); j++)
+            { ess_vdofs[n++] = vdofs[j]; }
+         }
+      }
+      nlf->SetEssentialVDofs(ess_vdofs);
+   }
+}
+
+}
@@ -0,0 +1,284 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_TMOP_AMR_HPP
+#define MFEM_TMOP_AMR_HPP
+
+#include "tmop_tools.hpp"
+#include "nonlinearform.hpp"
+#include "pnonlinearform.hpp"
+#include "estimators.hpp"
+#include "../mesh/mesh_operators.hpp"
+
+namespace mfem
+{
+
+class TMOPRefinerEstimator : public AnisotropicErrorEstimator
+{
+protected:
+   Mesh *mesh; // not owned
+   NonlinearForm *nlf; // not owned
+   int order;
+   int amrmetric;
+   Array<IntegrationRule *> TriIntRule, QuadIntRule, TetIntRule, HexIntRule;
+   long current_sequence;
+   Vector error_estimates;
+   Array<int> aniso_flags;
+   // An element is refined only if
+   // [mean TMOPEnergy(children)]*energy_scaling_factor < TMOPEnergy(parent)
+   double energy_scaling_factor;
+   GridFunction *spat_gf;   // If specified, can be used to specify the
+   double spat_gf_critical; // region where hr-adaptivity is done.
+
+   /// Check if the mesh of the solution was modified.
+   bool MeshIsModified()
+   {
+      long mesh_sequence = mesh->GetSequence();
+      MFEM_ASSERT(mesh_sequence >= current_sequence, "");
+      return (mesh_sequence > current_sequence);
+   }
+
+   /// Compute the element error estimates. For an element E in the mesh,
+   /// error(E) = TMOPEnergy(E)*energy_scaling_factor-Mean(TMOPEnergy(ChildofE)),
+   /// where TMOPEnergy of Children of E is obtained by assuming the element E
+   /// is refined using the refinement type being considered based on the TMOP
+   /// mesh quality metric.
+   void ComputeEstimates();
+
+   /// Construct the integration rules to model how each element type is split
+   /// using different refinement types. ref_type = 0 is the original element
+   /// and reftype \ in [1, 7] represent different refinement type based on
+   /// NCMesh class.
+   void SetQuadIntRules(); // supports ref_type = 1 to 3.
+   void SetTriIntRules(); // currently supports only isotropic refinement.
+   void SetHexIntRules(); // currently supports only isotropic refinement.
+   void SetTetIntRules(); // currently supports only isotropic refinement.
+
+   /// Get TMOP energy for each element corresponding to the refinement type
+   /// specified.
+   void GetTMOPRefinementEnergy(int reftype, Vector &el_energy_vec);
+
+   /// Use a mesh to setup an integration rule that will mimic the different
+   /// refinement types.
+   IntegrationRule* SetIntRulesFromMesh(Mesh &meshsplit);
+public:
+   TMOPRefinerEstimator(Mesh &mesh_, NonlinearForm &nlf_, int order_,
+                        int amrmetric_) :
+      mesh(&mesh_), nlf(&nlf_), order(order_), amrmetric(amrmetric_),
+      TriIntRule(0), QuadIntRule(0), TetIntRule(0), HexIntRule(0),
+      current_sequence(-1), error_estimates(), aniso_flags(),
+      energy_scaling_factor(1.), spat_gf(NULL), spat_gf_critical(0.)
+   {
+      if (mesh->Dimension() == 2)
+      {
+         SetQuadIntRules();
+         SetTriIntRules();
+      }
+      else
+      {
+         SetHexIntRules();
+         SetTetIntRules();
+      }
+   }
+
+   ~TMOPRefinerEstimator()
+   {
+      for (int i = 0; i < QuadIntRule.Size(); i++) { delete QuadIntRule[i]; }
+      for (int i = 0; i < TriIntRule.Size();  i++) { delete TriIntRule[i]; }
+      for (int i = 0; i < HexIntRule.Size();  i++) { delete HexIntRule[i]; }
+      for (int i = 0; i < TetIntRule.Size();  i++) { delete TetIntRule[i]; }
+   }
+
+   /// Get TMOP-based errors for each element in the mesh computed based on the
+   /// refinement types being considered.
+   virtual const Vector &GetLocalErrors()
+   {
+      if (MeshIsModified()) { ComputeEstimates(); }
+      return error_estimates;
+   }
+   /// For anisotropic refinements, get the refinement type (e.g., x or y)
+   virtual const Array<int> &GetAnisotropicFlags()
+   {
+      if (MeshIsModified()) { ComputeEstimates(); }
+      return aniso_flags;
+   }
+
+   /// Scaling factor for the TMOP refinement energy. An element is refined if
+   /// [mean TMOPEnergy(children)]*energy_scaling_factor < TMOPEnergy(parent)
+   void SetEnergyScalingFactor(double scale) { energy_scaling_factor = scale; }
+
+   /// Spatial indicator function (eta) that can be used to prevent elements
+   /// from being refined even if the energy criterion is met. Using this,
+   /// an element E is not refined if mean(@a spat_gf(E)) < @a spat_gf_critical.
+   void SetSpatialIndicator(GridFunction &spat_gf_,
+                            double spat_gf_critical_ = 0.5)
+   { spat_gf = &spat_gf_; spat_gf_critical = spat_gf_critical_; }
+   void SetSpatialIndicatorCritical(double val_) { spat_gf_critical = val_; }
+
+   /// Reset the error estimator.
+   virtual void Reset() { current_sequence = -1; }
+};
+
+class TMOPDeRefinerEstimator : public ErrorEstimator
+{
+protected:
+   Mesh *mesh;
+   NonlinearForm *nlf;
+#ifdef MFEM_USE_MPI
+   ParMesh *pmesh;
+   ParNonlinearForm *pnlf;
+#endif
+   int order;
+   int amrmetric;
+   long current_sequence;
+   Vector error_estimates;
+   bool serial;
+
+   /// Check if the mesh of the solution was modified.
+   bool MeshIsModified()
+   {
+      long mesh_sequence = mesh->GetSequence();
+      MFEM_ASSERT(mesh_sequence >= current_sequence, "");
+      return (mesh_sequence > current_sequence);
+   }
+
+   /// Compute the element error estimates. For a given element E in the mesh,
+   /// error(E) = TMOPEnergy(parent_of_E)-TMOPEnergy(E). Children element of an
+   /// element are derefined if the mean TMOP energy of children is greated than
+   /// the TMOP energy associated with their parent.
+   void ComputeEstimates();
+
+   void GetTMOPDerefinementEnergy(Mesh &cmesh,
+                                  TMOP_Integrator &tmopi,
+                                  Vector &el_energy_vec);
+
+   bool GetDerefineEnergyForIntegrator(TMOP_Integrator &tmopi,
+                                       Vector &fine_energy);
+public:
+   TMOPDeRefinerEstimator(Mesh &mesh_, NonlinearForm &nlf_) :
+      mesh(&mesh_), nlf(&nlf_),
+      current_sequence(-1), error_estimates(), serial(true)   { }
+#ifdef MFEM_USE_MPI
+   TMOPDeRefinerEstimator(ParMesh &pmesh_, ParNonlinearForm &pnlf_) :
+      mesh(&pmesh_), nlf(&pnlf_), pmesh(&pmesh_), pnlf(&pnlf_),
+      current_sequence(-1), error_estimates(), serial(false) { }
+#endif
+
+   ~TMOPDeRefinerEstimator() { }
+
+   virtual const Vector &GetLocalErrors()
+   {
+      if (MeshIsModified()) { ComputeEstimates(); }
+      return error_estimates;
+   }
+
+   /// Reset the error estimator.
+   virtual void Reset() { current_sequence = -1; }
+};
+
+// hr-adaptivity using TMOP.
+// If hr-adaptivity is disabled, r-adaptivity is done once using the
+// TMOPNewtonSolver.
+// Otherwise, "hr_iter" iterations of r-adaptivity are done followed by
+// "h_per_r_iter" iterations of h-adaptivity after each r-adaptivity iteration.
+// The solver terminates early if an h-adaptivity iteration does not
+// refine/derefine any element in the mesh.
+class TMOPHRSolver
+{
+protected:
+   Mesh *mesh;
+   NonlinearForm *nlf;
+   TMOPNewtonSolver *tmopns;
+   GridFunction *x;
+   Array<GridFunction *> gridfuncarr;
+   Array<FiniteElementSpace *> fespacearr;
+   bool move_bnd, hradaptivity;
+   const int mesh_poly_deg, amr_metric_id;
+#ifdef MFEM_USE_MPI
+   ParMesh *pmesh;
+   ParNonlinearForm *pnlf;
+   Array<ParGridFunction *> pgridfuncarr;
+   Array<ParFiniteElementSpace *> pfespacearr;
+#endif
+   bool serial;
+
+   // All are owned.
+   TMOPRefinerEstimator *tmop_r_est;
+   ThresholdRefiner *tmop_r;
+   TMOPDeRefinerEstimator *tmop_dr_est;
+   ThresholdDerefiner *tmop_dr;
+
+   int hr_iter, h_per_r_iter;
+
+   void Update();
+#ifdef MFEM_USE_MPI
+   void ParUpdate();
+#endif
+   void UpdateNonlinearFormAndBC(Mesh *mesh, NonlinearForm *nlf);
+
+#ifdef MFEM_USE_MPI
+   // Rebalance ParMesh such that all the children elements are moved to the same
+   // MPI rank where the parent will be if the mesh were to be derefined.
+   void RebalanceParNCMesh();
+#endif
+
+public:
+   TMOPHRSolver(Mesh &mesh_, NonlinearForm &nlf_,
+                TMOPNewtonSolver &tmopns_, GridFunction &x_,
+                bool move_bnd_, bool hradaptivity_,
+                int mesh_poly_deg_, int amr_metric_id_,
+                int hr_iter_ = 5, int h_per_r_iter_ = 1);
+#ifdef MFEM_USE_MPI
+   TMOPHRSolver(ParMesh &pmesh_, ParNonlinearForm &pnlf_,
+                TMOPNewtonSolver &tmopns_, ParGridFunction &x_,
+                bool move_bnd_, bool hradaptivity_,
+                int mesh_poly_deg_, int amr_metric_id_,
+                int hr_iter_ = 5, int h_per_r_iter_ = 1);
+#endif
+
+   void Mult();
+
+   /// These are used to update spaces and functions that are not owned by the
+   /// TMOPIntegrator or DiscreteAdaptTC. The owned ones are updated in the
+   /// functions UpdateAfterMeshTopologyChange() of both classes.
+   void AddGridFunctionForUpdate(GridFunction *gf) { gridfuncarr.Append(gf); }
+   void AddFESpaceForUpdate(FiniteElementSpace *fes) { fespacearr.Append(fes); }
+
+#ifdef MFEM_USE_MPI
+   void AddGridFunctionForUpdate(ParGridFunction *pgf_)
+   {
+      pgridfuncarr.Append(pgf_);
+   }
+   void AddFESpaceForUpdate(ParFiniteElementSpace *pfes_)
+   {
+      pfespacearr.Append(pfes_);
+   }
+#endif
+
+   ~TMOPHRSolver()
+   {
+      if (!hradaptivity) { return; }
+      delete tmop_dr;
+      delete tmop_dr_est;
+      delete tmop_r;
+      delete tmop_r_est;
+   }
+
+   /// Total number of hr-adaptivity iterations. At each iteration, we do an
+   /// r-adaptivity iteration followed by a number of h-adaptivity iterations.
+   void SetHRAdaptivityIterations(int iter) { hr_iter = iter; }
+
+   /// Total number of h-adaptivity iterations per r-adaptivity iteration.
+   void SetHAdaptivityIterations(int iter) { h_per_r_iter = iter; }
+};
+
+}
+#endif
@@ -407,6 +407,8 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
   {
      // Needed for the line search below. The untangling metrics see this
      // reference to detect deteriorations.
+      MFEM_VERIFY(min_det_ptr != NULL, " Initial mesh was valid, but"
+                  " intermediate mesh is invalid. Contact TMOP Developers.");
      *min_det_ptr = untangle_factor * min_detT_in;
   }

@@ -576,7 +578,7 @@ void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
         ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
         if (ti)
         {
-            ti->UpdateAfterMeshChange(x_loc);
+            ti->UpdateAfterMeshPositionChange(x_loc);
            ti->ComputeFDh(x_loc, *pfesc);
            UpdateDiscreteTC(*ti, x_loc);
         }
@@ -586,7 +588,7 @@ void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
            Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
            for (int j = 0; j < ati.Size(); j++)
            {
-               ati[j]->UpdateAfterMeshChange(x_loc);
+               ati[j]->UpdateAfterMeshPositionChange(x_loc);
               ati[j]->ComputeFDh(x_loc, *pfesc);
               UpdateDiscreteTC(*ati[j], x_loc);
            }
@@ -613,7 +615,7 @@ void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
         ti = dynamic_cast<TMOP_Integrator *>(integs[i]);
         if (ti)
         {
-            ti->UpdateAfterMeshChange(x_loc);
+            ti->UpdateAfterMeshPositionChange(x_loc);
            ti->ComputeFDh(x_loc, *fesc);
            UpdateDiscreteTC(*ti, x_loc);
         }
@@ -623,7 +625,7 @@ void TMOPNewtonSolver::ProcessNewState(const Vector &x) const
            Array<TMOP_Integrator *> ati = co->GetTMOPIntegrators();
            for (int j = 0; j < ati.Size(); j++)
            {
-               ati[j]->UpdateAfterMeshChange(x_loc);
+               ati[j]->UpdateAfterMeshPositionChange(x_loc);
               ati[j]->ComputeFDh(x_loc, *fesc);
               UpdateDiscreteTC(*ati[j], x_loc);
            }
@@ -175,7 +175,8 @@ const Operator &InterpolationGridTransfer::ForwardOperator()
                                            localP[elem_geoms[i]]);
      }
      F.Reset(ran_fes.RefinementMatrix_main(
-                 dom_fes.GetNDofs(), dom_fes.GetElementToDofTable(), localP));
+                 dom_fes.GetNDofs(), dom_fes.GetElementToDofTable(),
+                 dom_fes.GetElementToFaceOrientationTable(), localP));
   }
   else
   {
@@ -21,12 +21,16 @@
 #define MFEM_PERF_FUNCTION CALI_CXX_MARK_FUNCTION
 #define MFEM_PERF_BEGIN(s) CALI_MARK_BEGIN(s)
 #define MFEM_PERF_END(s) CALI_MARK_END(s)
+#define MFEM_PERF_SCOPE(name) \
+  cali::Annotation::Guard cali_autogenerated_guard_name(cali::Annotation("function").begin(std::string(name).c_str()))
+

 #else

 #define MFEM_PERF_FUNCTION
 #define MFEM_PERF_BEGIN(s)
 #define MFEM_PERF_END(s)
+#define MFEM_PERF_SCOPE(name)

 #endif

@@ -58,6 +58,23 @@
 #define MFEM_FOREACH_THREAD(i,k,N) for(int i=0; i<N; i++)
 #endif

+// 'double' atomicAdd implementation for previous versions of CUDA
+#if defined(MFEM_USE_CUDA) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 600
+MFEM_DEVICE double atomicAdd(double *add, double val)
+{
+   unsigned long long int *ptr = (unsigned long long int *) add;
+   unsigned long long int old = *ptr, reg;
+   do
+   {
+      reg = old;
+      old = atomicCAS(ptr, reg,
+                      __double_as_longlong(val + __longlong_as_double(reg)));
+   }
+   while (reg != old);
+   return __longlong_as_double(old);
+}
+#endif
+
 template <typename T>
 MFEM_HOST_DEVICE T AtomicAdd(T &add, const T val)
 {
@@ -66,6 +83,9 @@ MFEM_HOST_DEVICE T AtomicAdd(T &add, const T val)
   return atomicAdd(&add,val);
 #else
   T old = add;
+#ifdef MFEM_USE_OPENMP
+   #pragma omp atomic
+#endif
   add += val;
   return old;
 #endif
@@ -13,6 +13,7 @@
 #define MFEM_FORALL_HPP

 #include "../config/config.hpp"
+#include "annotation.hpp"
 #include "error.hpp"
 #include "backends.hpp"
 #include "device.hpp"
@@ -23,7 +23,9 @@
 #include <unistd.h>
 #else
 #include <winsock.h>
+#ifdef _MSC_VER
 typedef int ssize_t;
+#endif
 typedef int socklen_t;
 #define close closesocket
 // Link with ws2_32.lib
@@ -494,8 +494,7 @@ public:
   /// Copy @a size entries from @a *this to @a dest.
   /** The given @a size should not exceed the Capacity() of @a *this and the
       destination, @a dest. */
-   inline void CopyTo(Memory &dest, int size) const
-   { dest.CopyFrom(*this, size); }
+   inline void CopyTo(Memory &dest, int size) const;

   /// Copy @a size entries from @a *this to the host pointer @a dest.
   /** The given @a size should not exceed the Capacity() of @a *this. */
@@ -923,6 +922,11 @@ inline void Memory<T>::Wrap(T *ptr, T *d_ptr, int size, MemoryType mt, bool own)
 template <typename T>
 inline void Memory<T>::MakeAlias(const Memory &base, int offset, int size)
 {
+   MFEM_ASSERT(0 <= offset, "invalid offset = " << offset);
+   MFEM_ASSERT(0 <= size, "invalid size = " << size);
+   MFEM_ASSERT(offset + size <= base.capacity,
+               "invalid offset + size = " << offset + size
+               << " > base capacity = " << base.capacity);
   capacity = size;
   h_mt = base.h_mt;
   h_ptr = base.h_ptr + offset;
@@ -1136,6 +1140,7 @@ inline bool Memory<T>::DeviceIsValid() const
 template <typename T>
 inline void Memory<T>::CopyFrom(const Memory &src, int size)
 {
+   MFEM_VERIFY(src.capacity>=size && capacity>=size, "Incorrect size");
   if (!(flags & REGISTERED) && !(src.flags & REGISTERED))
   {
      if (h_ptr != src.h_ptr && size != 0)
@@ -1155,6 +1160,7 @@ inline void Memory<T>::CopyFrom(const Memory &src, int size)
 template <typename T>
 inline void Memory<T>::CopyFromHost(const T *src, int size)
 {
+   MFEM_VERIFY(capacity>=size, "Incorrect size");
   if (!(flags & REGISTERED))
   {
      if (h_ptr != src && size != 0)
@@ -1171,9 +1177,17 @@ inline void Memory<T>::CopyFromHost(const T *src, int size)
   }
 }

+template <typename T>
+inline void Memory<T>::CopyTo(Memory &dest, int size) const
+{
+   MFEM_VERIFY(capacity>=size, "Incorrect size");
+   dest.CopyFrom(*this, size);
+}
+
 template <typename T>
 inline void Memory<T>::CopyToHost(T *dest, int size) const
 {
+   MFEM_VERIFY(capacity>=size, "Incorrect size");
   if (!(flags & REGISTERED))
   {
      if (h_ptr != dest && size != 0)
@@ -28,7 +28,9 @@
 #define closesocket (::close)
 #else
 #include <winsock.h>
+#ifdef _MSC_VER
 typedef int ssize_t;
+#endif
 // Link with ws2_32.lib
 #pragma comment(lib, "ws2_32.lib")
 #endif
@@ -118,6 +118,11 @@ int STable3D::Index (int r, int c, int f) const
 {
   STable3DNode *node;

+   if (r >= Size)
+   {
+      return -1;
+   }
+
   Sort3 (r, c, f);

   for (node = Rows[r]; node != NULL; node = node->Prev)
@@ -787,10 +787,16 @@ HypreParMatrix * ComplexHypreParMatrix::GetSystemMatrix() const
                                           2 * num_cols_offd, cmap,
                                           true);

+#if MFEM_HYPRE_VERSION <= 22200
   // Give the new matrix ownership of row_starts and col_starts
   hypre_ParCSRMatrix *hA = (hypre_ParCSRMatrix*)(*A);
+
   hypre_ParCSRMatrixSetRowStartsOwner(hA,1);
   hypre_ParCSRMatrixSetColStartsOwner(hA,1);
+#else
+   mfem_hypre_TFree_host(row_starts);
+   mfem_hypre_TFree_host(col_starts);
+#endif

   return A;
 }
@@ -763,6 +763,13 @@ public:
      tdata.New(i*j*k);
   }

+   DenseTensor(double *d, int i, int j, int k)
+      : Mk(NULL, i, j)
+   {
+      nk = k;
+      tdata.Wrap(d, i*j*k, false);
+   }
+
   DenseTensor(int i, int j, int k, MemoryType mt)
      : Mk(NULL, i, j)
   {
@@ -22,7 +22,7 @@ using namespace hiop;
 namespace mfem
 {

-bool HiopOptimizationProblem::get_prob_sizes(long long &n, long long &m)
+bool HiopOptimizationProblem::get_prob_sizes(size_type &n, size_type &m)
 {
   n = ntdofs_glob;
   m = problem.GetNumConstraints();
@@ -30,7 +30,7 @@ bool HiopOptimizationProblem::get_prob_sizes(long long &n, long long &m)
   return true;
 }

-bool HiopOptimizationProblem::get_starting_point(const long long &n, double *x0)
+bool HiopOptimizationProblem::get_starting_point(const size_type &n, double *x0)
 {
   MFEM_ASSERT(x_start != NULL && ntdofs_loc == x_start->Size(),
               "Starting point is not set properly.");
@@ -40,7 +40,7 @@ bool HiopOptimizationProblem::get_starting_point(const long long &n, double *x0)
   return true;
 }

-bool HiopOptimizationProblem::get_vars_info(const long long &n,
+bool HiopOptimizationProblem::get_vars_info(const size_type &n,
                                            double *xlow, double *xupp,
                                            NonlinearityType *type)
 {
@@ -55,7 +55,7 @@ bool HiopOptimizationProblem::get_vars_info(const long long &n,
   return true;
 }

-bool HiopOptimizationProblem::get_cons_info(const long long &m,
+bool HiopOptimizationProblem::get_cons_info(const size_type &m,
                                            double *clow, double *cupp,
                                            NonlinearityType *type)
 {
@@ -79,7 +79,7 @@ bool HiopOptimizationProblem::get_cons_info(const long long &m,
   return true;
 }

-bool HiopOptimizationProblem::eval_f(const long long &n, const double *x,
+bool HiopOptimizationProblem::eval_f(const size_type &n, const double *x,
                                     bool new_x, double &obj_value)
 {
   MFEM_ASSERT(n == ntdofs_glob, "Global input mismatch.");
@@ -93,7 +93,7 @@ bool HiopOptimizationProblem::eval_f(const long long &n, const double *x,
   return true;
 }

-bool HiopOptimizationProblem::eval_grad_f(const long long &n, const double *x,
+bool HiopOptimizationProblem::eval_grad_f(const size_type &n, const double *x,
                                          bool new_x, double *gradf)
 {
   MFEM_ASSERT(n == ntdofs_glob, "Global input mismatch.");
@@ -108,9 +108,9 @@ bool HiopOptimizationProblem::eval_grad_f(const long long &n, const double *x,
   return true;
 }

-bool HiopOptimizationProblem::eval_cons(const long long &n, const long long &m,
-                                        const long long &num_cons,
-                                        const long long *idx_cons,
+bool HiopOptimizationProblem::eval_cons(const size_type &n, const size_type &m,
+                                        const size_type &num_cons,
+                                        const index_type *idx_cons,
                                        const double *x, bool new_x,
                                        double *cons)
 {
@@ -134,10 +134,10 @@ bool HiopOptimizationProblem::eval_cons(const long long &n, const long long &m,
   return true;
 }

-bool HiopOptimizationProblem::eval_Jac_cons(const long long &n,
-                                            const long long &m,
-                                            const long long &num_cons,
-                                            const long long *idx_cons,
+bool HiopOptimizationProblem::eval_Jac_cons(const size_type &n,
+                                            const size_type &m,
+                                            const size_type &num_cons,
+                                            const index_type *idx_cons,
                                            const double *x, bool new_x,
                                            double *Jac)
 {
@@ -165,16 +165,16 @@ bool HiopOptimizationProblem::eval_Jac_cons(const long long &n,
   return true;
 }

-bool HiopOptimizationProblem::get_vecdistrib_info(long long global_n,
-                                                  long long *cols)
+bool HiopOptimizationProblem::get_vecdistrib_info(size_type global_n,
+                                                  index_type *cols)
 {
 #ifdef MFEM_USE_MPI
   int nranks;
   MPI_Comm_size(comm, &nranks);

-   long long *sizes = new long long[nranks];
-   MPI_Allgather(&ntdofs_loc, 1, MPI_LONG_LONG_INT, sizes, 1,
-                 MPI_LONG_LONG_INT, comm);
+   size_type *sizes = new size_type[nranks];
+   MPI_Allgather(&ntdofs_loc, 1, MPI_HIOP_SIZE_TYPE, sizes, 1,
+                 MPI_HIOP_SIZE_TYPE, comm);
   cols[0] = 0;
   for (int r = 1; r <= nranks; r++)
   {
@@ -302,6 +302,8 @@ void HiopNlpOptimizer::Mult(const Vector &xt, Vector &x) const
   hiopInstance.options->SetNumericValue("fixed_var_tolerance", 1e-20);
   hiopInstance.options->SetNumericValue("fixed_var_perturb", 1e-9);

+   hiopInstance.options->SetNumericValue("mu0", 1e-1);
+
   // 0: no output; 3: not too much
   hiopInstance.options->SetIntegerValue("verbosity_level", print_level);

@@ -41,8 +41,8 @@ private:
   const OptimizationProblem &problem;

   // Local and global number of variables and constraints.
-   const long long ntdofs_loc, m_total;
-   long long ntdofs_glob;
+   const hiop::size_type ntdofs_loc, m_total;
+   hiop::size_type ntdofs_glob;

   // Initial guess.
   const Vector *x_start;
@@ -78,8 +78,7 @@ public:
        constr_vals(m_total), constr_grads(m_total, ntdofs_loc),
        constr_info_is_current(false)
   {
-      MPI_Allreduce(&ntdofs_loc, &ntdofs_glob, 1, MPI_LONG_LONG_INT,
-                    MPI_SUM, comm);
+      MPI_Allreduce(&ntdofs_loc, &ntdofs_glob, 1, MPI_HIOP_SIZE_TYPE, MPI_SUM, comm);
   }
 #endif

@@ -87,27 +86,27 @@ public:

   /** Extraction of problem dimensions:
    *  n is the number of variables, m is the number of constraints. */
-   virtual bool get_prob_sizes(long long int& n, long long int& m);
+   virtual bool get_prob_sizes(hiop::size_type& n, hiop::size_type& m);

   /** Provide an primal starting point. This point is subject to adjustments
    *  internally in HiOp. */
-   virtual bool get_starting_point(const long long &n, double *x0);
+   virtual bool get_starting_point(const hiop::size_type &n, double *x0);

-   virtual bool get_vars_info(const long long& n, double *xlow, double* xupp,
+   virtual bool get_vars_info(const hiop::size_type &n, double *xlow, double* xupp,
                              NonlinearityType* type);

   /** bounds on the constraints
    *  (clow<=-1e20 means no lower bound, cupp>=1e20 means no upper bound) */
-   virtual bool get_cons_info(const long long &m, double *clow, double *cupp,
+   virtual bool get_cons_info(const hiop::size_type &m, double *clow, double *cupp,
                              NonlinearityType* type);

   /** Objective function evaluation.
    *  Each rank returns the global objective value. */
-   virtual bool eval_f(const long long& n, const double *x, bool new_x,
+   virtual bool eval_f(const hiop::size_type &n, const double *x, bool new_x,
                       double& obj_value);

   /** Gradient of the objective function (local chunk). */
-   virtual bool eval_grad_f(const long long &n, const double *x, bool new_x,
+   virtual bool eval_grad_f(const hiop::size_type &n, const double *x, bool new_x,
                            double *gradf);

   /** Evaluates a subset of the constraints cons(x). The subset is of size
@@ -132,8 +131,9 @@ public:
    *  When MPI enabled, every rank populates cons, since the constraints are
    *  not distributed.
    */
-   virtual bool eval_cons(const long long &n, const long long &m,
-                          const long long &num_cons, const long long *idx_cons,
+   virtual bool eval_cons(const hiop::size_type &n, const hiop::size_type &m,
+                          const hiop::size_type &num_cons,
+                          const hiop::index_type *idx_cons,
                          const double *x, bool new_x, double *cons);

   /** Evaluates the Jacobian of the subset of constraints indicated by
@@ -147,9 +147,9 @@ public:
    *  When MPI enabled, each rank computes only the local columns of the
    *  Jacobian, that is the partials with respect to local variables.
    */
-   virtual bool eval_Jac_cons(const long long &n, const long long &m,
-                              const long long &num_cons,
-                              const long long *idx_cons,
+   virtual bool eval_Jac_cons(const hiop::size_type &n, const hiop::size_type &m,
+                              const hiop::size_type &num_cons,
+                              const hiop::index_type *idx_cons,
                              const double *x, bool new_x, double *Jac);

   /** Specifies column partitioning for distributed memory vectors.
@@ -158,7 +158,8 @@ public:
    *  Example: for a vector x of 6 entries (globally) on 3 ranks, the uniform
    *  column partitioning is cols=[0,2,4,6].
    */
-   virtual bool get_vecdistrib_info(long long global_n, long long *cols);
+   virtual bool get_vecdistrib_info(hiop::size_type global_n,
+                                    hiop::index_type *cols);

 #ifdef MFEM_USE_MPI
   virtual bool get_MPI_comm(MPI_Comm &comm_out)
@@ -88,7 +88,9 @@ HypreParVector::HypreParVector(MPI_Comm comm, HYPRE_BigInt glob_size,
 {
   x = hypre_ParVectorCreate(comm,glob_size,col);
   hypre_ParVectorInitialize(x);
+#if MFEM_HYPRE_VERSION <= 22200
   hypre_ParVectorSetPartitioningOwner(x,0);
+#endif
   // The data will be destroyed by hypre (this is the default)
   hypre_ParVectorSetDataOwner(x,1);
   hypre_SeqVectorSetDataOwner(hypre_ParVectorLocalVector(x),1);
@@ -105,7 +107,9 @@ HypreParVector::HypreParVector(MPI_Comm comm, HYPRE_BigInt glob_size,
   hypre_ParVectorSetDataOwner(x,1); // owns the seq vector
   hypre_Vector *x_loc = hypre_ParVectorLocalVector(x);
   hypre_SeqVectorSetDataOwner(x_loc,0);
+#if MFEM_HYPRE_VERSION <= 22200
   hypre_ParVectorSetPartitioningOwner(x,0);
+#endif
   double tmp = 0.0;
   hypre_VectorData(x_loc) = &tmp;
 #ifdef HYPRE_USING_CUDA
@@ -128,7 +132,9 @@ HypreParVector::HypreParVector(const HypreParVector &y) : Vector()
   x = hypre_ParVectorCreate(y.x -> comm, y.x -> global_size,
                             y.x -> partitioning);
   hypre_ParVectorInitialize(x);
+#if MFEM_HYPRE_VERSION <= 22200
   hypre_ParVectorSetPartitioningOwner(x,0);
+#endif
   hypre_ParVectorSetDataOwner(x,1);
   hypre_SeqVectorSetDataOwner(hypre_ParVectorLocalVector(x),1);
   _SetDataAndSize_();
@@ -162,7 +168,9 @@ HypreParVector::HypreParVector(ParFiniteElementSpace *pfes)
   x = hypre_ParVectorCreate(pfes->GetComm(), pfes->GlobalTrueVSize(),
                             pfes->GetTrueDofOffsets());
   hypre_ParVectorInitialize(x);
+#if MFEM_HYPRE_VERSION <= 22200
   hypre_ParVectorSetPartitioningOwner(x,0);
+#endif
   // The data will be destroyed by hypre (this is the default)
   hypre_ParVectorSetDataOwner(x,1);
   hypre_SeqVectorSetDataOwner(hypre_ParVectorLocalVector(x),1);
@@ -683,8 +691,10 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, HYPRE_BigInt glob_size,
   A = hypre_ParCSRMatrixCreate(comm, glob_size, glob_size, row_starts,
                                row_starts, 0, diag->NumNonZeroElems(), 0);
   hypre_ParCSRMatrixSetDataOwner(A,1);
+#if MFEM_HYPRE_VERSION <= 22200
   hypre_ParCSRMatrixSetRowStartsOwner(A,0);
   hypre_ParCSRMatrixSetColStartsOwner(A,0);
+#endif

   hypre_CSRMatrixSetDataOwner(A->diag,0);
   diagOwner = CopyCSR(diag, mem_diag, A->diag, false);
@@ -726,8 +736,10 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm,
                                row_starts, col_starts,
                                0, diag->NumNonZeroElems(), 0);
   hypre_ParCSRMatrixSetDataOwner(A,1);
+#if MFEM_HYPRE_VERSION <= 22200
   hypre_ParCSRMatrixSetRowStartsOwner(A,0);
   hypre_ParCSRMatrixSetColStartsOwner(A,0);
+#endif

   hypre_CSRMatrixSetDataOwner(A->diag,0);
   diagOwner = CopyCSR(diag, mem_diag, A->diag, false);
@@ -770,8 +782,10 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm,
                                offd->Width(), diag->NumNonZeroElems(),
                                offd->NumNonZeroElems());
   hypre_ParCSRMatrixSetDataOwner(A,1);
+#if MFEM_HYPRE_VERSION <= 22200
   hypre_ParCSRMatrixSetRowStartsOwner(A,0);
   hypre_ParCSRMatrixSetColStartsOwner(A,0);
+#endif

   hypre_CSRMatrixSetDataOwner(A->diag,0);
   diagOwner = CopyCSR(diag, mem_diag, A->diag, own_diag_offd);
@@ -817,8 +831,10 @@ HypreParMatrix::HypreParMatrix(
   A = hypre_ParCSRMatrixCreate(comm, global_num_rows, global_num_cols,
                                row_starts, col_starts, offd_num_cols, 0, 0);
   hypre_ParCSRMatrixSetDataOwner(A,1);
+#if MFEM_HYPRE_VERSION <= 22200
   hypre_ParCSRMatrixSetRowStartsOwner(A,0);
   hypre_ParCSRMatrixSetColStartsOwner(A,0);
+#endif

   HYPRE_Int local_num_rows = hypre_CSRMatrixNumRows(A->diag);

@@ -931,8 +947,10 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm,
   A = hypre_ParCSRMatrixCreate(comm, global_num_rows, global_num_cols,
                                row_starts, col_starts, 0, nnz, 0);
   hypre_ParCSRMatrixSetDataOwner(A,1);
+#if MFEM_HYPRE_VERSION <= 22200
   hypre_ParCSRMatrixSetRowStartsOwner(A,0);
   hypre_ParCSRMatrixSetColStartsOwner(A,0);
+#endif

   hypre_CSRMatrixSetDataOwner(A->diag,0);
   diagOwner = CopyBoolCSR(diag, mem_diag, A->diag);
@@ -989,8 +1007,10 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, int id, int np,
   }

   hypre_ParCSRMatrixSetDataOwner(A,1);
+#if MFEM_HYPRE_VERSION <= 22200
   hypre_ParCSRMatrixSetRowStartsOwner(A,0);
   hypre_ParCSRMatrixSetColStartsOwner(A,0);
+#endif

   mem_diag.data.New(diag_nnz);
   for (HYPRE_Int i = 0; i < diag_nnz; i++)
@@ -1177,6 +1197,13 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, int nrows,
   {
      hypre_CSRMatrixReorder(hypre_ParCSRMatrixDiag(A));
   }
+#if MFEM_HYPRE_VERSION > 22200
+   mfem_hypre_TFree_host(row_starts);
+   if (rows != cols)
+   {
+      mfem_hypre_TFree_host(col_starts);
+   }
+#endif
   hypre_MatvecCommPkgCreate(A);

   height = GetNumRows();
@@ -1263,6 +1290,7 @@ void HypreParMatrix::SetOwnerFlags(signed char diag, signed char offd,

 void HypreParMatrix::CopyRowStarts()
 {
+#if MFEM_HYPRE_VERSION <= 22200
   if (!A || hypre_ParCSRMatrixOwnsRowStarts(A) ||
       (hypre_ParCSRMatrixRowStarts(A) == hypre_ParCSRMatrixColStarts(A) &&
        hypre_ParCSRMatrixOwnsColStarts(A)))
@@ -1297,10 +1325,12 @@ void HypreParMatrix::CopyRowStarts()
      hypre_ParCSRMatrixColStarts(A) = new_row_starts;
      hypre_ParCSRMatrixOwnsColStarts(A) = 0;
   }
+#endif
 }

 void HypreParMatrix::CopyColStarts()
 {
+#if MFEM_HYPRE_VERSION <= 22200
   if (!A || hypre_ParCSRMatrixOwnsColStarts(A) ||
       (hypre_ParCSRMatrixRowStarts(A) == hypre_ParCSRMatrixColStarts(A) &&
        hypre_ParCSRMatrixOwnsRowStarts(A)))
@@ -1339,6 +1369,7 @@ void HypreParMatrix::CopyColStarts()
   {
      hypre_ParCSRMatrixOwnsColStarts(A) = 1;
   }
+#endif
 }

 void HypreParMatrix::GetDiag(Vector &diag) const
@@ -1791,9 +1822,14 @@ HypreParMatrix* HypreParMatrix::LeftDiagMult(const SparseMatrix &D,
                         DA_diag, DA_offd, new_col_map_offd,
                         own_diag_offd);

+#if MFEM_HYPRE_VERSION <= 22200
   // Give ownership of row_starts, col_starts, and col_map_offd to DA
   hypre_ParCSRMatrixSetRowStartsOwner(DA->A, 1);
   hypre_ParCSRMatrixSetColStartsOwner(DA->A, 1);
+#else
+   mfem_hypre_TFree_host(new_row_starts);
+   mfem_hypre_TFree_host(new_col_starts);
+#endif
   DA->colMapOwner = 1;

   return DA;
@@ -1948,18 +1984,22 @@ void HypreParMatrix::Threshold(double threshold)
   row_starts = hypre_ParCSRMatrixRowStarts(A);
   col_starts = hypre_ParCSRMatrixColStarts(A);

+#if MFEM_HYPRE_VERSION <= 22200
   bool old_owns_row = hypre_ParCSRMatrixOwnsRowStarts(A);
   bool old_owns_col = hypre_ParCSRMatrixOwnsColStarts(A);
+#endif
   HYPRE_BigInt global_num_rows = hypre_ParCSRMatrixGlobalNumRows(A);
   HYPRE_BigInt global_num_cols = hypre_ParCSRMatrixGlobalNumCols(A);
   parcsr_A_ptr = hypre_ParCSRMatrixCreate(comm, global_num_rows,
                                           global_num_cols,
                                           row_starts, col_starts,
                                           0, 0, 0);
+#if MFEM_HYPRE_VERSION <= 22200
   hypre_ParCSRMatrixOwnsRowStarts(parcsr_A_ptr) = old_owns_row;
   hypre_ParCSRMatrixOwnsColStarts(parcsr_A_ptr) = old_owns_col;
   hypre_ParCSRMatrixOwnsRowStarts(A) = 0;
   hypre_ParCSRMatrixOwnsColStarts(A) = 0;
+#endif

   csr_A = hypre_MergeDiagAndOffd(A);

@@ -1994,7 +2034,12 @@ void HypreParMatrix::Threshold(double threshold)

   hypre_ParCSRMatrixSetNumNonzeros(A);
   /* Make sure that the first entry in each row is the diagonal one. */
+#if MFEM_HYPRE_VERSION <= 22200
   if (row_starts == col_starts)
+#else
+   if ((row_starts[0] == col_starts[0]) &&
+       (row_starts[1] == col_starts[1]))
+#endif
   {
      hypre_CSRMatrixReorder(hypre_ParCSRMatrixDiag(A));
   }
@@ -2503,11 +2548,14 @@ HypreParMatrix * RAP(const HypreParMatrix *A, const HypreParMatrix *P)
      // hypre_ParCSRMatrixRAPKT
   }
 #else
+#if MFEM_HYPRE_VERSION <= 22200
   HYPRE_Int P_owns_its_col_starts =
      hypre_ParCSRMatrixOwnsColStarts((hypre_ParCSRMatrix*)(*P));
+#endif

   hypre_BoomerAMGBuildCoarseOperator(*P,*A,*P,&rap);

+#if MFEM_HYPRE_VERSION <= 22200
   /* Warning: hypre_BoomerAMGBuildCoarseOperator steals the col_starts
      from P (even if it does not own them)! */
   hypre_ParCSRMatrixSetRowStartsOwner(rap,0);
@@ -2516,6 +2564,7 @@ HypreParMatrix * RAP(const HypreParMatrix *A, const HypreParMatrix *P)
   {
      hypre_ParCSRMatrixSetColStartsOwner(*P, 1);
   }
+#endif
 #endif

   hypre_ParCSRMatrixSetNumNonzeros(rap);
@@ -2536,13 +2585,16 @@ HypreParMatrix * RAP(const HypreParMatrix * Rt, const HypreParMatrix *A,
      hypre_ParCSRMatrixDestroy(Q);
   }
 #else
+#if MFEM_HYPRE_VERSION <= 22200
   HYPRE_Int P_owns_its_col_starts =
      hypre_ParCSRMatrixOwnsColStarts((hypre_ParCSRMatrix*)(*P));
   HYPRE_Int Rt_owns_its_col_starts =
      hypre_ParCSRMatrixOwnsColStarts((hypre_ParCSRMatrix*)(*Rt));
+#endif

   hypre_BoomerAMGBuildCoarseOperator(*Rt,*A,*P,&rap);

+#if MFEM_HYPRE_VERSION <= 22200
   /* Warning: hypre_BoomerAMGBuildCoarseOperator steals the col_starts
      from Rt and P (even if they do not own them)! */
   hypre_ParCSRMatrixSetRowStartsOwner(rap,0);
@@ -2555,6 +2607,7 @@ HypreParMatrix * RAP(const HypreParMatrix * Rt, const HypreParMatrix *A,
   {
      hypre_ParCSRMatrixSetColStartsOwner(*Rt, 1);
   }
+#endif
 #endif

   hypre_ParCSRMatrixSetNumNonzeros(rap);
@@ -3144,8 +3197,12 @@ void HypreSmoother::SetOperator(const Operator &op)
      }
      else
      {
+#if MFEM_HYPRE_VERSION <= 22200
         min_eig_est = 0;
         hypre_ParCSRMaxEigEstimate(*A, poly_scale, &max_eig_est);
+#else
+         hypre_ParCSRMaxEigEstimate(*A, poly_scale, &max_eig_est, &min_eig_est);
+#endif
      }
      Z = new HypreParVector(*A);
   }
@@ -3159,8 +3216,12 @@ void HypreSmoother::SetOperator(const Operator &op)
      }
      else
      {
+#if MFEM_HYPRE_VERSION <= 22200
         min_eig_est = 0;
         hypre_ParCSRMaxEigEstimate(*A, poly_scale, &max_eig_est);
+#else
+         hypre_ParCSRMaxEigEstimate(*A, poly_scale, &max_eig_est, &min_eig_est);
+#endif
      }

      // The Taubin and FIR polynomials are defined on [0, 2]
@@ -4997,11 +5058,11 @@ HypreADS::HypreADS(const HypreParMatrix &A, ParFiniteElementSpace *face_fespace)
 void HypreADS::Init(ParFiniteElementSpace *face_fespace)
 {
   int cycle_type       = 11;
-   int rlx_type         = 2;
   int rlx_sweeps       = 1;
   double rlx_weight    = 1.0;
   double rlx_omega     = 1.0;
 #ifndef HYPRE_USING_CUDA
+   int rlx_type         = 2;
   int amg_coarsen_type = 10;
   int amg_agg_levels   = 1;
   int amg_rlx_type     = 8;
@@ -5009,6 +5070,7 @@ void HypreADS::Init(ParFiniteElementSpace *face_fespace)
   int amg_interp_type  = 6;
   int amg_Pmax         = 4;
 #else
+   int rlx_type         = 1;
   int amg_coarsen_type = 8;
   int amg_agg_levels   = 0;
   int amg_rlx_type     = 18;
@@ -508,8 +508,10 @@ void hypre_ParCSRMatrixEliminateAAe(hypre_ParCSRMatrix *A,
                                  hypre_ParCSRMatrixColStarts(A),
                                  0, 0, 0);

+#if MFEM_HYPRE_VERSION <= 22200
   hypre_ParCSRMatrixSetRowStartsOwner(*Ae, 0);
   hypre_ParCSRMatrixSetColStartsOwner(*Ae, 0);
+#endif

   hypre_CSRMatrix *Ae_diag = hypre_ParCSRMatrixDiag(*Ae);
   hypre_CSRMatrix *Ae_offd = hypre_ParCSRMatrixOffd(*Ae);
@@ -1002,10 +1004,17 @@ void hypre_ParCSRMatrixSplit(hypre_ParCSRMatrix *A,

      hypre_ParCSRMatrixOwnsData(blocks[i]) = 1;

+#if MFEM_HYPRE_VERSION <= 22200
      /* only the first block will own the row/col_starts */
      hypre_ParCSRMatrixOwnsRowStarts(blocks[i]) = !i;
      hypre_ParCSRMatrixOwnsColStarts(blocks[i]) = !i;
+#endif
   }
+
+#if MFEM_HYPRE_VERSION > 22200
+   mfem_hypre_TFree_host(row_starts);
+   mfem_hypre_TFree_host(col_starts);
+#endif
 }

 /* Based on hypre_CSRMatrixMatvec in hypre's csr_matvec.c */
@@ -1916,9 +1925,12 @@ hypre_ParCSRMatrixAdd(hypre_ParCSRMatrix *A,

   /* C owns diag, offd, and cmap. */
   hypre_ParCSRMatrixSetDataOwner(C, 1);
+
+#if MFEM_HYPRE_VERSION <= 22200
   /* C does not own row and column starts. */
   hypre_ParCSRMatrixSetRowStartsOwner(C, 0);
   hypre_ParCSRMatrixSetColStartsOwner(C, 0);
+#endif

   return C;
 }
@@ -33,7 +33,12 @@ int SparseMatrix::SparseMatrixCount = 0;
 cusparseHandle_t SparseMatrix::handle = nullptr;
 size_t SparseMatrix::bufferSize = 0;
 void * SparseMatrix::dBuffer = nullptr;
-#endif
+#  if CUSPARSE_VERSION >=  11400
+#    define MFEM_CUSPARSE_ALG CUSPARSE_SPMV_CSR_ALG1
+#  else
+#    define MFEM_CUSPARSE_ALG CUSPARSE_CSRMV_ALG1
+#  endif // CUSPARSE_VERSION >= 11400
+#endif // MFEM_USE_CUDA

 void SparseMatrix::InitCuSparse()
 {
@@ -679,25 +684,16 @@ void SparseMatrix::AddMult(const Vector &x, Vector &y, const double a) const
         cusparseCreateMatDescr(&matA_descr);
         cusparseSetMatIndexBase(matA_descr, CUSPARSE_INDEX_BASE_ZERO);
         cusparseSetMatType(matA_descr, CUSPARSE_MATRIX_TYPE_GENERAL);
-
 #endif
-
         initBuffers = true;
      }
      // Allocate kernel space. Buffer is shared between different sparsemats
      size_t newBufferSize = 0;

-#if CUDA_VERSION >= 11020
      cusparseSpMV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha,
                              matA_descr,
                              vecX_descr, &beta, vecY_descr, CUDA_R_64F,
-                              CUSPARSE_SPMV_CSR_ALG1, &newBufferSize);
-#elif CUDA_VERSION >= 10010
-      cusparseSpMV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha,
-                              matA_descr,
-                              vecX_descr, &beta, vecY_descr, CUDA_R_64F,
-                              CUSPARSE_CSRMV_ALG1, &newBufferSize);
-#endif
+                              MFEM_CUSPARSE_ALG, &newBufferSize);

      // Check if we need to resize
      if (newBufferSize > bufferSize)
@@ -707,30 +703,22 @@ void SparseMatrix::AddMult(const Vector &x, Vector &y, const double a) const
         CuMemAlloc(&dBuffer, bufferSize);
      }

-#if CUDA_VERSION >= 11020
+#if CUDA_VERSION >= 10010
      // Update input/output vectors
      cusparseDnVecSetValues(vecX_descr, const_cast<double *>(d_x));
      cusparseDnVecSetValues(vecY_descr, d_y);

      // Y = alpha A * X + beta * Y
      cusparseSpMV(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA_descr,
-                   vecX_descr, &beta, vecY_descr, CUDA_R_64F, CUSPARSE_SPMV_CSR_ALG1, dBuffer);
-#elif CUDA_VERSION >= 10010
-      // Update input/output vectors
-      cusparseDnVecSetValues(vecX_descr, const_cast<double *>(d_x));
-      cusparseDnVecSetValues(vecY_descr, d_y);
-
-      // Y = alpha A * X + beta * Y
-      cusparseSpMV(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, matA_descr,
-                   vecX_descr, &beta, vecY_descr, CUDA_R_64F, CUSPARSE_CSRMV_ALG1, dBuffer);
+                   vecX_descr, &beta, vecY_descr, CUDA_R_64F, MFEM_CUSPARSE_ALG, dBuffer);
 #else
      cusparseDcsrmv(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
                     Height(), Width(), J.Capacity(),
                     &alpha, matA_descr,
                     const_cast<double *>(d_A), const_cast<int *>(d_I), const_cast<int *>(d_J),
                     const_cast<double *>(d_x), &beta, d_y);
-#endif
-#endif
+#endif // CUDA_VERSION >= 10010
+#endif // MFEM_USE_CUDA
   }
   else
   {
@@ -2461,52 +2449,67 @@ void SparseMatrix::DiagScale(const Vector &b, Vector &x, double sc) const
   });
 }

+template <bool useFabs>
+static void JacobiDispatch(const Vector &b, const Vector &x0, Vector &x1,
+                           const Memory<int> &I, const Memory<int> &J,
+                           const Memory<double> &A, const int height,
+                           const double sc)
+{
+   const bool useDevice = b.UseDevice() || x0.UseDevice() || x1.UseDevice();
+
+   const auto bp  = b.Read(useDevice);
+   const auto x0p = x0.Read(useDevice);
+   auto       x1p = x1.Write(useDevice);
+
+   const auto Ip = Read(I, height+1, useDevice);
+   const auto Jp = Read(J, J.Capacity(), useDevice);
+   const auto Ap = Read(A, J.Capacity(), useDevice);
+
+   MFEM_FORALL_SWITCH(useDevice, i, height,
+   {
+      double resi = bp[i], norm = 0.0;
+      for (int j = Ip[i]; j < Ip[i+1]; j++)
+      {
+         resi -= Ap[j] * x0p[Jp[j]];
+         if (useFabs)
+         {
+            norm += fabs(Ap[j]);
+         }
+         else
+         {
+            norm += Ap[j];
+         }
+      }
+      if (norm > 0.0)
+      {
+         x1p[i] = x0p[i] + sc * resi / norm;
+      }
+      else
+      {
+         if (useFabs)
+         {
+            MFEM_ABORT_KERNEL("L1 norm of row is zero.");
+         }
+         else
+         {
+            MFEM_ABORT_KERNEL("sum of row is zero.");
+         }
+      }
+   });
+}
+
 void SparseMatrix::Jacobi2(const Vector &b, const Vector &x0, Vector &x1,
                           double sc) const
 {
   MFEM_VERIFY(Finalized(), "Matrix must be finalized.");
-
-   for (int i = 0; i < height; i++)
-   {
-      double resi = b(i), norm = 0.0;
-      for (int j = I[i]; j < I[i+1]; j++)
-      {
-         resi -= A[j] * x0(J[j]);
-         norm += fabs(A[j]);
-      }
-      if (norm > 0.0)
-      {
-         x1(i) = x0(i) + sc * resi / norm;
-      }
-      else
-      {
-         MFEM_ABORT("L1 norm of row " << i << " is zero.");
-      }
-   }
+   JacobiDispatch<true>(b,x0,x1,I,J,A,height,sc);
 }

 void SparseMatrix::Jacobi3(const Vector &b, const Vector &x0, Vector &x1,
                           double sc) const
 {
   MFEM_VERIFY(Finalized(), "Matrix must be finalized.");
-
-   for (int i = 0; i < height; i++)
-   {
-      double resi = b(i), sum = 0.0;
-      for (int j = I[i]; j < I[i+1]; j++)
-      {
-         resi -= A[j] * x0(J[j]);
-         sum  += A[j];
-      }
-      if (sum > 0.0)
-      {
-         x1(i) = x0(i) + sc * resi / sum;
-      }
-      else
-      {
-         MFEM_ABORT("sum of row " << i << " is zero.");
-      }
-   }
+   JacobiDispatch<false>(b,x0,x1,I,J,A,height,sc);
 }

 void SparseMatrix::AddSubMatrix(const Array<int> &rows, const Array<int> &cols,
@@ -1677,7 +1677,7 @@ int KINSolver::GradientMult(N_Vector v, N_Vector Jv, N_Vector u,

 // Wrapper for evaluating linear systems J u = b
 int KINSolver::LinSysSetup(N_Vector u, N_Vector, SUNMatrix J,
-                           void *, N_Vector , N_Vector )
+                           void *, N_Vector, N_Vector )
 {
   const SundialsNVector mfem_u(u);
   KINSolver *self = static_cast<KINSolver*>(GET_CONTENT(J));
@@ -82,6 +82,11 @@ public:
   Vector(double *data_, int size_)
   { data.Wrap(data_, size_, false); size = size_; }

+   /** @brief Create a Vector referencing a sub-vector of the Vector @a base
+       starting at the given offset, @a base_offset, and size @a size_. */
+   Vector(Vector &base, int base_offset, int size_)
+      : data(base.data, base_offset, size_), size(size_) { }
+
   /// Create a Vector of size @a size_ using MemoryType @a mt.
   Vector(int size_, MemoryType mt)
      : data(size_, mt), size(size_) { }
@@ -119,7 +119,7 @@ $(if $(word 2,$(SRC)),$(error Spaces in SRC = "$(SRC)" are not supported))
 MFEM_GIT_STRING = $(shell [ -d $(MFEM_DIR)/.git ] && git -C $(MFEM_DIR) \
   describe --all --long --abbrev=40 --dirty --always 2> /dev/null)

-EXAMPLE_SUBDIRS = amgx ginkgo hiop petsc pumi sundials superlu
+EXAMPLE_SUBDIRS = amgx caliper ginkgo hiop petsc pumi sundials superlu
 EXAMPLE_DIRS := examples $(addprefix examples/,$(EXAMPLE_SUBDIRS))
 EXAMPLE_TEST_DIRS := examples

@@ -703,7 +703,7 @@ status info:

 ASTYLE_BIN = astyle
 ASTYLE = $(ASTYLE_BIN) --options=$(SRC)config/mfem.astylerc
-ASTYLE_VER = "Artistic Style Version 2.05.1"
+ASTYLE_VER = "Artistic Style Version 3.1"
 FORMAT_FILES = $(foreach dir,$(DIRS) $(EM_DIRS) config,$(dir)/*.?pp)
 FORMAT_FILES += tests/unit/*.cpp
 UNIT_TESTS_SUBDIRS = general linalg mesh fem miniapps ceed
@@ -19,6 +19,7 @@ set(SRCS
  ncmesh.cpp
  nurbs.cpp
  point.cpp
+  pyramid.cpp
  quadrilateral.cpp
  segment.cpp
  tetrahedron.cpp
@@ -38,6 +39,7 @@ set(HDRS
  ncmesh.hpp
  nurbs.hpp
  point.hpp
+  pyramid.hpp
  quadrilateral.hpp
  segment.hpp
  tetrahedron.hpp
@@ -39,7 +39,7 @@ public:

   /// Constants for the classes derived from Element.
   enum Type { POINT, SEGMENT, TRIANGLE, QUADRILATERAL,
-               TETRAHEDRON, HEXAHEDRON, WEDGE
+               TETRAHEDRON, HEXAHEDRON, WEDGE, PYRAMID
             };

   /// Default element constructor.
@@ -75,7 +75,7 @@ void Mesh::GetElementCenter(int i, Vector &center)

 double Mesh::GetElementSize(ElementTransformation *T, int type)
 {
-   DenseMatrix J(Dim);
+   DenseMatrix J(spaceDim,Dim);

   Geometry::Type geom = T->GetGeometryType();
   T->SetIntPoint(&Geometries.GetCenter(geom));
@@ -83,7 +83,7 @@ double Mesh::GetElementSize(ElementTransformation *T, int type)

   if (type == 0)
   {
-      return pow(fabs(J.Det()), 1./Dim);
+      return pow(fabs(J.Weight()), 1./Dim);
   }
   else if (type == 1)
   {
@@ -102,7 +102,7 @@ double Mesh::GetElementSize(int i, int type)

 double Mesh::GetElementSize(int i, const Vector &dir)
 {
-   DenseMatrix J(Dim);
+   DenseMatrix J(spaceDim,Dim);
   Vector d_hat(Dim);
   GetElementJacobian(i, J);
   J.MultTranspose(dir, d_hat);
@@ -335,6 +335,7 @@ FiniteElement *Mesh::GetTransformationFEforElementType(Element::Type ElemType)
      case Element::TETRAHEDRON :    return &TetrahedronFE;
      case Element::HEXAHEDRON :     return &HexahedronFE;
      case Element::WEDGE :          return &WedgeFE;
+      case Element::PYRAMID :        return &PyramidFE;
      default:
         MFEM_ABORT("Unknown element type \"" << ElemType << "\"");
         break;
@@ -735,6 +736,31 @@ void Mesh::GetLocalTriToWdgTransformation(
   }
 }

+void Mesh::GetLocalTriToPyrTransformation(
+   IsoparametricTransformation &Transf, int i)
+{
+   DenseMatrix &locpm = Transf.GetPointMat();
+
+   Transf.SetFE(&TriangleFE);
+   //  (i/64) is the local face no. in the pyr
+   MFEM_VERIFY(i >= 64, "Local face index " << i/64
+               << " is not a triangular face of a pyramid.");
+   const int *pv = pyr_t::FaceVert[i/64];
+   //  (i%64) is the orientation of the pyramid face
+   //         w.r.t. the face element
+   const int *to = tri_t::Orient[i%64];
+   const IntegrationRule *PyrVert =
+      Geometries.GetVertices(Geometry::PYRAMID);
+   locpm.SetSize(3, 3);
+   for (int j = 0; j < 3; j++)
+   {
+      const IntegrationPoint &vert = PyrVert->IntPoint(pv[to[j]]);
+      locpm(0, j) = vert.x;
+      locpm(1, j) = vert.y;
+      locpm(2, j) = vert.z;
+   }
+}
+
 void Mesh::GetLocalQuadToHexTransformation(
   IsoparametricTransformation &Transf, int i)
 {
@@ -781,6 +807,29 @@ void Mesh::GetLocalQuadToWdgTransformation(
   }
 }

+void Mesh::GetLocalQuadToPyrTransformation(
+   IsoparametricTransformation &Transf, int i)
+{
+   DenseMatrix &locpm = Transf.GetPointMat();
+
+   Transf.SetFE(&QuadrilateralFE);
+   //  (i/64) is the local face no. in the pyr
+   MFEM_VERIFY(i < 64, "Local face index " << i/64
+               << " is not a quadrilateral face of a pyramid.");
+   const int *pv = pyr_t::FaceVert[i/64];
+   //  (i%64) is the orientation of the quad
+   const int *qo = quad_t::Orient[i%64];
+   const IntegrationRule *PyrVert = Geometries.GetVertices(Geometry::PYRAMID);
+   locpm.SetSize(3, 4);
+   for (int j = 0; j < 4; j++)
+   {
+      const IntegrationPoint &vert = PyrVert->IntPoint(pv[qo[j]]);
+      locpm(0, j) = vert.x;
+      locpm(1, j) = vert.y;
+      locpm(2, j) = vert.z;
+   }
+}
+
 const GeometricFactors* Mesh::GetGeometricFactors(const IntegrationRule& ir,
                                                  const int flags,
                                                  MemoryType d_mt)
@@ -862,10 +911,19 @@ void Mesh::GetLocalFaceTransformation(
         {
            GetLocalTriToTetTransformation(Transf, info);
         }
+         else if (elem_type == Element::WEDGE)
+         {
+            GetLocalTriToWdgTransformation(Transf, info);
+         }
+         else if (elem_type == Element::PYRAMID)
+         {
+            GetLocalTriToPyrTransformation(Transf, info);
+         }
         else
         {
-            MFEM_ASSERT(elem_type == Element::WEDGE, "");
-            GetLocalTriToWdgTransformation(Transf, info);
+            MFEM_ABORT("Mesh::GetLocalFaceTransformation not defined for "
+                       "face type " << face_type
+                       << " and element type " << elem_type << "\n");
         }
         break;

@@ -874,10 +932,19 @@ void Mesh::GetLocalFaceTransformation(
         {
            GetLocalQuadToHexTransformation(Transf, info);
         }
+         else if (elem_type == Element::WEDGE)
+         {
+            GetLocalQuadToWdgTransformation(Transf, info);
+         }
+         else if (elem_type == Element::PYRAMID)
+         {
+            GetLocalQuadToPyrTransformation(Transf, info);
+         }
         else
         {
-            MFEM_ASSERT(elem_type == Element::WEDGE, "");
-            GetLocalQuadToWdgTransformation(Transf, info);
+            MFEM_ABORT("Mesh::GetLocalFaceTransformation not defined for "
+                       "face type " << face_type
+                       << " and element type " << elem_type << "\n");
         }
         break;
   }
@@ -1370,6 +1437,20 @@ int Mesh::AddWedge(const int *vi, int attr)
   return NumOfElements++;
 }

+int Mesh::AddPyramid(int v1, int v2, int v3, int v4, int v5, int attr)
+{
+   CheckEnlarge(elements, NumOfElements);
+   elements[NumOfElements] = new Pyramid(v1, v2, v3, v4, v5, attr);
+   return NumOfElements++;
+}
+
+int Mesh::AddPyramid(const int *vi, int attr)
+{
+   CheckEnlarge(elements, NumOfElements);
+   elements[NumOfElements] = new Pyramid(vi, attr);
+   return NumOfElements++;
+}
+
 int Mesh::AddHex(int v1, int v2, int v3, int v4, int v5, int v6, int v7, int v8,
                 int attr)
 {
@@ -1423,6 +1504,25 @@ void Mesh::AddHexAsWedges(const int *vi, int attr)
   }
 }

+void Mesh::AddHexAsPyramids(const int *vi, int attr)
+{
+   static const int hex_to_pyr[6][5] =
+   {
+      { 0, 1, 2, 3, 8 }, { 0, 4, 5, 1, 8 }, { 1, 5, 6, 2, 8 },
+      { 2, 6, 7, 3, 8 }, { 3, 7, 4, 0, 8 }, { 7, 6, 5, 4, 8 }
+   };
+   int ti[5];
+
+   for (int i = 0; i < 6; i++)
+   {
+      for (int j = 0; j < 5; j++)
+      {
+         ti[j] = vi[hex_to_pyr[i][j]];
+      }
+      AddPyramid(ti, attr);
+   }
+}
+
 int Mesh::AddElement(Element *elem)
 {
   CheckEnlarge(elements, NumOfElements);
@@ -2692,11 +2792,16 @@ void Mesh::Make3D(int nx, int ny, int nz, Element::Type type,
      NElem *= 2;
      NBdrElem += 2*nx*ny;
   }
+   else if (type == Element::PYRAMID)
+   {
+      NElem *= 6;
+      NVert += nx * ny * nz;
+   }

   InitMesh(3, 3, NVert, NElem, NBdrElem);

   double coord[3];
-   int ind[8];
+   int ind[9];

   // Sets vertices and the corresponding coordinates
   for (z = 0; z <= nz; z++)
@@ -2712,8 +2817,25 @@ void Mesh::Make3D(int nx, int ny, int nz, Element::Type type,
         }
      }
   }
+   if (type == Element::PYRAMID)
+   {
+      for (z = 0; z < nz; z++)
+      {
+         coord[2] = (((double) z + 0.5) / nz) * sz;
+         for (y = 0; y < ny; y++)
+         {
+            coord[1] = (((double) y + 0.5 ) / ny) * sy;
+            for (x = 0; x < nx; x++)
+            {
+               coord[0] = (((double) x + 0.5 ) / nx) * sx;
+               AddVertex(coord);
+            }
+         }
+      }
+   }

 #define VTX(XC, YC, ZC) ((XC)+((YC)+(ZC)*(ny+1))*(nx+1))
+#define VTXP(XC, YC, ZC) ((nx+1)*(ny+1)*(nz+1)+(XC)+((YC)+(ZC)*ny)*nx)

   // Sets elements and the corresponding indices of vertices
   if (sfc_ordering && type == Element::HEXAHEDRON)
@@ -2728,6 +2850,7 @@ void Mesh::Make3D(int nx, int ny, int nz, Element::Type type,
         y = sfc[3*k + 1];
         z = sfc[3*k + 2];

+         // *INDENT-OFF*
         ind[0] = VTX(x  , y  , z  );
         ind[1] = VTX(x+1, y  , z  );
         ind[2] = VTX(x+1, y+1, z  );
@@ -2736,6 +2859,7 @@ void Mesh::Make3D(int nx, int ny, int nz, Element::Type type,
         ind[5] = VTX(x+1, y  , z+1);
         ind[6] = VTX(x+1, y+1, z+1);
         ind[7] = VTX(x  , y+1, z+1);
+         // *INDENT-ON*

         AddHex(ind, 1);
      }
@@ -2748,6 +2872,7 @@ void Mesh::Make3D(int nx, int ny, int nz, Element::Type type,
         {
            for (x = 0; x < nx; x++)
            {
+               // *INDENT-OFF*
               ind[0] = VTX(x  , y  , z  );
               ind[1] = VTX(x+1, y  , z  );
               ind[2] = VTX(x+1, y+1, z  );
@@ -2756,6 +2881,7 @@ void Mesh::Make3D(int nx, int ny, int nz, Element::Type type,
               ind[5] = VTX(x+1, y  , z+1);
               ind[6] = VTX(x+1, y+1, z+1);
               ind[7] = VTX(  x, y+1, z+1);
+               // *INDENT-ON*
               if (type == Element::TETRAHEDRON)
               {
                  AddHexAsTets(ind, 1);
@@ -2764,6 +2890,11 @@ void Mesh::Make3D(int nx, int ny, int nz, Element::Type type,
               {
                  AddHexAsWedges(ind, 1);
               }
+               else if (type == Element::PYRAMID)
+               {
+                  ind[8] = VTXP(x, y, z);
+                  AddHexAsPyramids(ind, 1);
+               }
               else
               {
                  AddHex(ind, 1);
@@ -2779,10 +2910,12 @@ void Mesh::Make3D(int nx, int ny, int nz, Element::Type type,
   {
      for (x = 0; x < nx; x++)
      {
+         // *INDENT-OFF*
         ind[0] = VTX(x  , y  , 0);
         ind[1] = VTX(x  , y+1, 0);
         ind[2] = VTX(x+1, y+1, 0);
         ind[3] = VTX(x+1, y  , 0);
+         // *INDENT-ON*
         if (type == Element::TETRAHEDRON)
         {
            AddBdrQuadAsTriangles(ind, 1);
@@ -2802,10 +2935,12 @@ void Mesh::Make3D(int nx, int ny, int nz, Element::Type type,
   {
      for (x = 0; x < nx; x++)
      {
+         // *INDENT-OFF*
         ind[0] = VTX(x  , y  , nz);
         ind[1] = VTX(x+1, y  , nz);
         ind[2] = VTX(x+1, y+1, nz);
         ind[3] = VTX(x  , y+1, nz);
+         // *INDENT-ON*
         if (type == Element::TETRAHEDRON)
         {
            AddBdrQuadAsTriangles(ind, 6);
@@ -2825,10 +2960,12 @@ void Mesh::Make3D(int nx, int ny, int nz, Element::Type type,
   {
      for (y = 0; y < ny; y++)
      {
+         // *INDENT-OFF*
         ind[0] = VTX(0  , y  , z  );
         ind[1] = VTX(0  , y  , z+1);
         ind[2] = VTX(0  , y+1, z+1);
         ind[3] = VTX(0  , y+1, z  );
+         // *INDENT-ON*
         if (type == Element::TETRAHEDRON)
         {
            AddBdrQuadAsTriangles(ind, 5);
@@ -2844,10 +2981,12 @@ void Mesh::Make3D(int nx, int ny, int nz, Element::Type type,
   {
      for (y = 0; y < ny; y++)
      {
+         // *INDENT-OFF*
         ind[0] = VTX(nx, y  , z  );
         ind[1] = VTX(nx, y+1, z  );
         ind[2] = VTX(nx, y+1, z+1);
         ind[3] = VTX(nx, y  , z+1);
+         // *INDENT-ON*
         if (type == Element::TETRAHEDRON)
         {
            AddBdrQuadAsTriangles(ind, 3);
@@ -2863,10 +3002,12 @@ void Mesh::Make3D(int nx, int ny, int nz, Element::Type type,
   {
      for (z = 0; z < nz; z++)
      {
+         // *INDENT-OFF*
         ind[0] = VTX(x  , 0, z  );
         ind[1] = VTX(x+1, 0, z  );
         ind[2] = VTX(x+1, 0, z+1);
         ind[3] = VTX(x  , 0, z+1);
+         // *INDENT-ON*
         if (type == Element::TETRAHEDRON)
         {
            AddBdrQuadAsTriangles(ind, 2);
@@ -2882,10 +3023,12 @@ void Mesh::Make3D(int nx, int ny, int nz, Element::Type type,
   {
      for (z = 0; z < nz; z++)
      {
+         // *INDENT-OFF*
         ind[0] = VTX(x  , ny, z  );
         ind[1] = VTX(x  , ny, z+1);
         ind[2] = VTX(x+1, ny, z+1);
         ind[3] = VTX(x+1, ny, z  );
+         // *INDENT-ON*
         if (type == Element::TETRAHEDRON)
         {
            AddBdrQuadAsTriangles(ind, 4);
@@ -3428,6 +3571,7 @@ Element *Mesh::NewElement(int geom)
 #endif
      case Geometry::CUBE:      return (new Hexahedron);
      case Geometry::PRISM:     return (new Wedge);
+      case Geometry::PYRAMID:   return (new Pyramid);
      default:
         MFEM_ABORT("invalid Geometry::Type, geom = " << geom);
   }
@@ -3520,6 +3664,15 @@ void Mesh::SetMeshGen()
            meshgen |= 4;
            break;

+         case Element::PYRAMID:
+            mesh_geoms |= (1 << Geometry::PYRAMID);
+            mesh_geoms |= (1 << Geometry::SQUARE);
+            mesh_geoms |= (1 << Geometry::TRIANGLE);
+            mesh_geoms |= (1 << Geometry::SEGMENT);
+            mesh_geoms |= (1 << Geometry::POINT);
+            meshgen |= 8;
+            break;
+
         default:
            MFEM_ABORT("invalid element type: " << type);
            break;
@@ -3974,6 +4127,12 @@ void Mesh::MakeRefined_(Mesh &orig_mesh, const Array<int> ref_factors,
      }
   }

+   if (Dim > 2)
+   {
+      GetElementToFaceTable(false);
+      GenerateFaces();
+   }
+
   // Add refined boundary elements
   for (int el = 0; el < orig_mesh.GetNBE(); el++)
   {
@@ -5054,6 +5213,19 @@ int Mesh::CheckElementOrientation(bool fix_it)
               }
               break;

+            case Element::PYRAMID:
+               // only check the Jacobian at the center of the element
+               GetElementJacobian(i, J);
+               if (J.Det() < 0.0)
+               {
+                  wo++;
+                  if (fix_it)
+                  {
+                     // how?
+                  }
+               }
+               break;
+
            case Element::HEXAHEDRON:
               // only check the Jacobian at the center of the element
               GetElementJacobian(i, J);
@@ -6136,6 +6308,22 @@ void Mesh::GenerateFaces()
               }
               break;
            }
+            case Element::PYRAMID:
+            {
+               for (int j = 0; j < 1; j++)
+               {
+                  const int *fv = pyr_t::FaceVert[j];
+                  AddQuadFaceElement(j, ef[j], i,
+                                     v[fv[0]], v[fv[1]], v[fv[2]], v[fv[3]]);
+               }
+               for (int j = 1; j < 5; j++)
+               {
+                  const int *fv = pyr_t::FaceVert[j];
+                  AddTriangleFaceElement(j, ef[j], i,
+                                         v[fv[0]], v[fv[1]], v[fv[2]]);
+               }
+               break;
+            }
            case Element::HEXAHEDRON:
            {
               for (int j = 0; j < 6; j++)
@@ -6176,9 +6364,12 @@ void Mesh::GenerateNCFaceInfo()
      const NCMesh::Master &master = list.masters[i];
      if (master.index >= nfaces) { continue; }

-      faces_info[master.index].NCFace = nc_faces_info.Size();
+      FaceInfo &master_fi = faces_info[master.index];
+      master_fi.NCFace = nc_faces_info.Size();
      nc_faces_info.Append(NCFaceInfo(false, master.local, NULL));
      // NOTE: one of the unused members stores local face no. to be used below
+      MFEM_ASSERT(master_fi.Elem2No == -1, "internal error");
+      MFEM_ASSERT(master_fi.Elem2Inf == -1, "internal error");
   }

   // add records for slave faces
@@ -6228,6 +6419,20 @@ STable3D *Mesh::GetFacesTable()
            }
            break;
         }
+         case Element::PYRAMID:
+         {
+            for (int j = 0; j < 1; j++)
+            {
+               const int *fv = pyr_t::FaceVert[j];
+               faces_tbl->Push4(v[fv[0]], v[fv[1]], v[fv[2]], v[fv[3]]);
+            }
+            for (int j = 1; j < 5; j++)
+            {
+               const int *fv = pyr_t::FaceVert[j];
+               faces_tbl->Push(v[fv[0]], v[fv[1]], v[fv[2]]);
+            }
+            break;
+         }
         case Element::WEDGE:
         {
            for (int j = 0; j < 2; j++)
@@ -6302,6 +6507,22 @@ STable3D *Mesh::GetElementToFaceTable(int ret_ftbl)
            }
            break;
         }
+         case Element::PYRAMID:
+         {
+            for (int j = 0; j < 1; j++)
+            {
+               const int *fv = pyr_t::FaceVert[j];
+               el_to_face->Push(
+                  i, faces_tbl->Push4(v[fv[0]], v[fv[1]], v[fv[2]], v[fv[3]]));
+            }
+            for (int j = 1; j < 5; j++)
+            {
+               const int *fv = pyr_t::FaceVert[j];
+               el_to_face->Push(
+                  i, faces_tbl->Push(v[fv[0]], v[fv[1]], v[fv[2]]));
+            }
+            break;
+         }
         case Element::HEXAHEDRON:
         {
            // find the face by the vertices with the smallest 3 numbers
@@ -7638,8 +7859,22 @@ void Mesh::UniformRefinement3D_base(Array<int> *f2qf_ptr, DSTable *v_to_v_p,
      }
   }

+   int pyr_counter = 0;
+   if (HasGeometry(Geometry::PYRAMID))
+   {
+      for (int i = 0; i < elements.Size(); i++)
+      {
+         if (elements[i]->GetType() == Element::PYRAMID)
+         {
+            pyr_counter++;
+         }
+      }
+   }
+
   // Map from edge-index to vertex-index, needed for ReorientTetMesh() for
   // parallel meshes.
+   // Note: with the removal of ReorientTetMesh() this may no longer
+   // be needed.  Unfortunately, it's hard to be sure.
   Array<int> e2v;
   if (HasGeometry(Geometry::TETRAHEDRON))
   {
@@ -7695,7 +7930,7 @@ void Mesh::UniformRefinement3D_base(Array<int> *f2qf_ptr, DSTable *v_to_v_p,
   Array<Element*> new_boundary;

   vertices.SetSize(oelem + hex_counter);
-   new_elements.SetSize(8 * NumOfElements);
+   new_elements.SetSize(8 * NumOfElements + 2 * pyr_counter);
   CoarseFineTr.embeddings.SetSize(new_elements.Size());

   hex_counter = 0;
@@ -7961,6 +8196,73 @@ void Mesh::UniformRefinement3D_base(Array<int> *f2qf_ptr, DSTable *v_to_v_p,
         }
         break;

+         case Element::PYRAMID:
+         {
+            const int *f = el_to_face->GetRow(i);
+            // pyr_counter++;
+
+            for (int fi = 0; fi < 1; fi++)
+            {
+               for (int k = 0; k < 4; k++)
+               {
+                  vv[k] = v[pyr_t::FaceVert[fi][k]];
+               }
+               AverageVertices(vv, 4, oface + f2qf[f[fi]]);
+            }
+
+            for (int ei = 0; ei < 8; ei++)
+            {
+               for (int k = 0; k < 2; k++)
+               {
+                  vv[k] = v[pyr_t::Edges[ei][k]];
+               }
+               AverageVertices(vv, 2, oedge+e[ei]);
+            }
+
+            const int qf0 = f2qf[f[0]];
+
+            new_elements[j++] =
+               new Pyramid(v[0], oedge+e[0], oface+qf0,
+                           oedge+e[3], oedge+e[4], attr);
+
+            new_elements[j++] =
+               new Pyramid(oedge+e[0], v[1], oedge+e[1],
+                           oface+qf0, oedge+e[5], attr);
+
+            new_elements[j++] =
+               new Pyramid(oface+qf0, oedge+e[1], v[2],
+                           oedge+e[2], oedge+e[6], attr);
+
+            new_elements[j++] =
+               new Pyramid(oedge+e[3], oface+qf0, oedge+e[2],
+                           v[3], oedge+e[7], attr);
+
+            new_elements[j++] =
+               new Pyramid(oedge+e[4], oedge+e[5], oedge+e[6],
+                           oedge+e[7], v[4], attr);
+
+            new_elements[j++] =
+               new Pyramid(oedge+e[7], oedge+e[6], oedge+e[5],
+                           oedge+e[4], oface+qf0, attr);
+
+            new_elements[j++] =
+               new Tetrahedron(oedge+e[0], oedge+e[4], oedge+e[5],
+                               oface+qf0, attr);
+
+            new_elements[j++] =
+               new Tetrahedron(oedge+e[1], oedge+e[5], oedge+e[6],
+                               oface+qf0, attr);
+
+            new_elements[j++] =
+               new Tetrahedron(oedge+e[2], oedge+e[6], oedge+e[7],
+                               oface+qf0, attr);
+
+            new_elements[j++] =
+               new Tetrahedron(oedge+e[3], oedge+e[7], oedge+e[4],
+                               oface+qf0, attr);
+         }
+         break;
+
         case Element::HEXAHEDRON:
         {
            const int *f = el_to_face->GetRow(i);
@@ -8092,7 +8394,7 @@ void Mesh::UniformRefinement3D_base(Array<int> *f2qf_ptr, DSTable *v_to_v_p,
   }
   mfem::Swap(boundary, new_boundary);

-   static const double A = 0.0, B = 0.5, C = 1.0;
+   static const double A = 0.0, B = 0.5, C = 1.0, D = -1.0;
   static double tet_children[3*4*16] =
   {
      A,A,A, B,A,A, A,B,A, A,A,B,
@@ -8118,6 +8420,19 @@ void Mesh::UniformRefinement3D_base(Array<int> *f2qf_ptr, DSTable *v_to_v_p,
      A,A,B, A,B,B, B,A,B, B,B,A,
      A,A,B, B,A,B, B,A,A, B,B,A
   };
+   static double pyr_children[3*5*10] =
+   {
+      A,A,A, B,A,A, B,B,A, A,B,A, A,A,B,
+      B,A,A, C,A,A, C,B,A, B,B,A, B,A,B,
+      B,B,A, C,B,A, C,C,A, B,C,A, B,B,B,
+      A,B,A, B,B,A, B,C,A, A,C,A, A,B,B,
+      A,A,B, B,A,B, B,B,B, A,B,B, A,A,C,
+      A,B,B, B,B,B, B,A,B, A,A,B, B,B,A,
+      B,A,A, A,A,B, B,A,B, B,B,A, D,D,D,
+      C,B,A, B,A,B, B,B,B, B,B,A, D,D,D,
+      B,C,A, B,B,B, A,B,B, B,B,A, D,D,D,
+      A,B,A, A,B,B, A,A,B, B,B,A, D,D,D
+   };
   static double pri_children[3*6*8] =
   {
      A,A,A, B,A,A, A,B,A, A,A,B, B,A,B, A,B,B,
@@ -8143,6 +8458,8 @@ void Mesh::UniformRefinement3D_base(Array<int> *f2qf_ptr, DSTable *v_to_v_p,

   CoarseFineTr.point_matrices[Geometry::TETRAHEDRON]
   .UseExternalData(tet_children, 3, 4, 16);
+   CoarseFineTr.point_matrices[Geometry::PYRAMID]
+   .UseExternalData(pyr_children, 3, 5, 10);
   CoarseFineTr.point_matrices[Geometry::PRISM]
   .UseExternalData(pri_children, 3, 6, 8);
   CoarseFineTr.point_matrices[Geometry::CUBE]
@@ -8159,7 +8476,7 @@ void Mesh::UniformRefinement3D_base(Array<int> *f2qf_ptr, DSTable *v_to_v_p,
   }

   NumOfVertices    = vertices.Size();
-   NumOfElements    = 8 * NumOfElements;
+   NumOfElements    = 8 * NumOfElements + 2 * pyr_counter;
   NumOfBdrElements = 4 * NumOfBdrElements;

   GetElementToFaceTable();
@@ -8743,7 +9060,7 @@ void Mesh::GeneralRefinement(const Array<Refinement> &refinements,
   else if (nonconforming < 0)
   {
      // determine if nonconforming refinement is suitable
-      if ((meshgen & 2) || (meshgen & 4))
+      if ((meshgen & 2) || (meshgen & 4) || (meshgen & 8))
      {
         nonconforming = 1; // tensor product elements and wedges
      }
@@ -9527,6 +9844,7 @@ void Mesh::Printer(std::ostream &out, std::string section_delimiter) const
       "# TETRAHEDRON = 4\n"
       "# CUBE        = 5\n"
       "# PRISM       = 6\n"
+       "# PYRAMID     = 7\n"
       "#\n";

   out << "\ndimension\n" << Dim;
@@ -107,18 +107,23 @@ protected:
   // element (slave or master, in the non-conforming case) that generated the
   // face.
   // Classification of a local (non-ghost) face based on its FaceInfo:
-   // - Elem2No >= 0 --> local internal face; can be either:
+   // - Elem2No >= 0 --> local interior face; can be either:
   //    - NCFace == -1 --> conforming face, or
-   //    - NCFace >= 0 --> non-conforming slave face.
+   //    - NCFace >= 0 --> non-conforming slave face; Elem2No is the index of
+   //      the master volume element; Elem2Inf%64 is 0, see the note in
+   //      Mesh::GenerateNCFaceInfo().
   // - Elem2No < 0 --> local "boundary" face; can be one of:
   //    - NCFace == -1 --> conforming face; can be either:
   //       - Elem2Inf < 0 --> true boundary face (no element on side 2)
   //       - Elem2Inf >= 0 --> shared face where element 2 is a face-neighbor
   //         element with index -1-Elem2No. This state is initialized by
   //         ParMesh::ExchangeFaceNbrData().
-   //    - NCFace >= 0 --> non-conforming master face. Elem2No is -1 or, in the
-   //      case of a shared face, -1-Elem2No is the index of one of the adjacent
-   //      (the last one?) slave ghost elements. Elem2Inf is -1.
+   //    - NCFace >= 0 --> non-conforming face; can be one of:
+   //       - Elem2Inf < 0 --> master non-conforming face, interior or shared;
+   //         In this case, Elem2No is -1; see GenerateNCFaceInfo().
+   //       - Elem2Inf >= 0 --> shared slave non-conforming face where element 2
+   //         is the master face-neighbor element with index -1-Elem2No; see
+   //         ParNCMesh::GetFaceNeighbors().
   //
   // A ghost face is a non-conforming face that is generated by a non-local,
   // i.e. ghost, element. A ghost face has index i in faces_info such that
@@ -130,7 +135,7 @@ protected:
   //   master side element, i.e. side 1 IS NOT the side that generated the
   //   face. Elem2No is < 0 and -1-Elem2No is the index of the ghost
   //   face-neighbor element that generated this slave ghost face. In this
-   //   case, Elem2Inf >= 0.
+   //   case, Elem2Inf >= 0 and NCFace >= 0.
   // Relevant methods: GenerateFaces(), GenerateNCFaceInfo(),
   //                   ParNCMesh::GetFaceNeighbors(),
   //                   ParMesh::ExchangeFaceNbrData()
@@ -177,6 +182,7 @@ protected:
   int own_nodes;

   static const int vtk_quadratic_tet[10];
+   static const int vtk_quadratic_pyramid[13];
   static const int vtk_quadratic_wedge[18];
   static const int vtk_quadratic_hex[27];

@@ -195,6 +201,7 @@ public:
   typedef Geometry::Constants<Geometry::TETRAHEDRON> tet_t;
   typedef Geometry::Constants<Geometry::CUBE>        hex_t;
   typedef Geometry::Constants<Geometry::PRISM>       pri_t;
+   typedef Geometry::Constants<Geometry::PYRAMID>     pyr_t;

   enum Operation { NONE, REFINE, DEREFINE, REBALANCE };

@@ -373,11 +380,17 @@ protected:
   void GetLocalTriToWdgTransformation (IsoparametricTransformation &loc,
                                        int i);
   /// Used in GetFaceElementTransformations (...)
+   void GetLocalTriToPyrTransformation (IsoparametricTransformation &loc,
+                                        int i);
+   /// Used in GetFaceElementTransformations (...)
   void GetLocalQuadToHexTransformation (IsoparametricTransformation &loc,
                                         int i);
   /// Used in GetFaceElementTransformations (...)
   void GetLocalQuadToWdgTransformation (IsoparametricTransformation &loc,
                                         int i);
+   /// Used in GetFaceElementTransformations (...)
+   void GetLocalQuadToPyrTransformation (IsoparametricTransformation &loc,
+                                         int i);

   /** Used in GetFaceElementTransformations to account for the fact that a
       slave face occupies only a portion of its master face. */
@@ -503,7 +516,7 @@ public:
   Mesh& operator=(Mesh &&mesh);

   /// Explicitly delete the copy assignment operator.
-   Mesh& operator=(Mesh &mesh) = delete;
+   Mesh& operator=(const Mesh &mesh) = delete;

   /** @name Named mesh constructors.

@@ -657,11 +670,15 @@ public:
   int AddWedge(int v1, int v2, int v3, int v4, int v5, int v6, int attr = 1);
   int AddWedge(const int *vi, int attr = 1);

+   int AddPyramid(int v1, int v2, int v3, int v4, int v5, int attr = 1);
+   int AddPyramid(const int *vi, int attr = 1);
+
   int AddHex(int v1, int v2, int v3, int v4, int v5, int v6, int v7, int v8,
              int attr = 1);
   int AddHex(const int *vi, int attr = 1);
   void AddHexAsTets(const int *vi, int attr = 1);
   void AddHexAsWedges(const int *vi, int attr = 1);
+   void AddHexAsPyramids(const int *vi, int attr = 1);

   /// The parameter @a elem should be allocated using the NewElement() method
   int AddElement(Element *elem);
@@ -829,10 +846,16 @@ public:

   /** @brief Get the mesh generator/type.

+       The purpose of this is to be able to quickly tell what type of elements
+       one has in the mesh. Examination of this bitmask along with knowledge
+       of the mesh dimension can be used to identify which element types are
+       present.
+
       @return A bitmask:
       - bit 0 - simplices are present in the mesh (triangles, tets),
       - bit 1 - tensor product elements are present in the mesh (quads, hexes),
       - bit 2 - the mesh has wedge elements.
+       - bit 3 - the mesh has pyramid elements.

       In parallel, the result takes into account elements on all processors.
   */
@@ -1227,7 +1250,7 @@ public:
       satisfy: v0 < min(v1, v2).

       @note Refinement does not work after a call to this method! */
-   virtual void ReorientTetMesh();
+   MFEM_DEPRECATED virtual void ReorientTetMesh();

   int *CartesianPartitioning(int nxyz[]);
   int *GeneratePartitioning(int nparts, int part_method = 1);
--- a/Show More
+++ b/Show More