Merge branch 'eigen-dev' of github.com:mfem/mfem into eigen-dev

Allow reading Gmsh meshes where all elements have attribute zero [gmsh-zero-attributes]

.gitignore

@@ -145,6 +145,14 @@ examples/petsc/velocity.*
 examples/petsc/elastic_energy.*
 examples/petsc/mode_*
 
+examples/arpack/ex11
+examples/arpack/mode_*
+examples/arpack/ex11.mesh
+
+examples/spectra/ex11
+examples/spectra/mode_*
+examples/spectra/ex11.mesh
+
 examples/pumi/ex1
 examples/pumi/ex[126]p
 examples/pumi/refined.mesh
@@ -252,6 +260,7 @@ miniapps/shifted/ParaViewDistance
 miniapps/shifted/diffusion
 miniapps/shifted/diffusion.mesh
 miniapps/shifted/diffusion.gf
+miniapps/shifted/ParaViewDiffusion
 
 miniapps/tools/display-basis
 miniapps/tools/load-dc
@@ -296,6 +305,7 @@ tests/unit/psedov_tests_*
 tests/unit/tmop_pa_tests_*
 tests/unit/ptmop_pa_tests_*
 tests/unit/ceed_tests
+tests/unit/debug_device_tests
 
 # Test script output
 tests/scripts/*.err
@@ -308,7 +318,13 @@ tests/convergence/prates
 tests/par-mesh-format/ex1p
 
 # VPATH builds
-build-*/*
+build-*/
+
+# User config
+user-*
+
+# VSCode
+.vscode 
 
 # PETSc automated build
 petsc-build/*

.gitlab-ci.yml

@@ -50,10 +50,11 @@ variables:
   AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
   MFEM_DATA_REPO: https://github.com/mfem/data.git
   ARTIFACTS_DIR: artifacts
+  SLURM_OVERLAP: 1
 
-# The pipeline is divided into stages. Usually, these are also synchronization
-# points, however, we use "needs" keyword to express the DAG of jobs for more
-# efficiency.
+# The pipeline is divided into stages. Usually, jobs in a given stage wait for
+# the preceding stages to complete before to start. However, we sometimes use
+# the "needs" keyword and express the DAG of jobs for more efficiency.
 # - We use setup and setup_baseline phases to download content outside of mfem
 #   directory.
 # - Allocate/Release is where quartz resources are allocated/released once for all.
@@ -87,7 +88,6 @@ setup:
   script:
     - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
     - if [ ! -d data ]; then git clone ${MFEM_DATA_REPO}; fi
-  needs: []
 
 # The setup_baseline job in setup stage_baseline doesn't rely on MFEM git repo.
 # It prepares a pipeline-wide working directory downloading/updating external
@@ -95,6 +95,7 @@ setup:
 # are now using unique directories so repo are never shared with another
 # pipeline. This is not memory efficient (we keep a lot of data), hence this
 # reminder.
+# Note: This job can start immediately.
 setup_baseline:
   tags:
     - shell
@@ -125,10 +126,10 @@ setup_baseline:
   script:
     - srun -p mi60 -t 15 -N 1 tests/gitlab/build_and_test
 
-# Lassen uses a different job scheduler (spectrum lsf) that does not
-# allow pre-allocation the same way slurm does.
-# We use pdebug queue on lassen to speed-up the allocation.
-# However this would not be scalable to multiple builds.
+# Lassen uses a different job scheduler (spectrum lsf) that does not allow
+# pre-allocation the same way slurm does.  We use pdebug queue on lassen to
+# speed-up the allocation.  However this would not be scalable to multiple
+# builds.
 .build_blueos_3_ppc64le_ib_script:
   script:
     - lalloc 1 -W 30 -q pdebug tests/gitlab/build_and_test

.gitlab/lassen.yml

@@ -22,6 +22,7 @@
 
 # Spack helped builds
 # Generic lassen build job, extending build script
+# Note: Lassen jobs can start as soon as the setup job is complete.
 .build_and_test_on_lassen:
   extends: [.build_blueos_3_ppc64le_ib_script, .on_lassen]
   stage: l_build_and_test
@@ -29,5 +30,5 @@
 
 opt_mpi_cuda_xl_16_1_1_8:
   variables:
-    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=sm_70"
+    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=70"
   extends: .build_and_test_on_lassen

.gitlab/quartz.yml

@@ -16,13 +16,13 @@
     - shell
     - quartz
   rules:
-    # Don’t run quartz jobs if...
+    # Don't run quartz jobs if...
     - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"'
       when: never
-    # Don’t run autotest update if...
+    # Don't run autotest update if...
     - if: '$CI_JOB_NAME =~ /update_autotest/ && $AUTOTEST != "YES"'
       when: never
-    # Don’t run autotest update if...
+    # Don't run autotest update if...
     - if: '$CI_JOB_NAME =~ /q_report/ && $AUTOTEST != "YES"'
       when: never
     # Report success on success status
@@ -37,6 +37,18 @@
     # Default is to run if previous stage succeeded
     - when: on_success
 
+# This is a yaml anchor, it can be used to avoid duplication like here.
+# The code below will simply be pasted wherever the anchor is placed.
+.safe_create_rundir: &safe_create_rundir |
+  if ! mkdir ${rundir}; then
+    n=1
+    while ! mkdir ${rundir}_${n}
+    do
+      n=$((n+1))
+    done
+    rundir=${rundir}_${n}
+  fi
+
 # Allocate
 q_allocate_resources:
   variables:
@@ -65,10 +77,11 @@ q_report_success:
   stage: q_release_resources
   script:
     - echo "Can only run if all the quartz jobs passed"
-    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
     - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - mkdir -p ${rundir}
+    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
+    - *safe_create_rundir
     - echo "The Quartz jobs were successful" > ${rundir}/gitlab.out
+    - echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
     - git add ${rundir}
     - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
     - git push origin master
@@ -80,10 +93,11 @@ q_report_failure:
   stage: q_release_resources
   script:
     - echo "Runs if there was at least one failure on quartz"
-    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
     - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - mkdir -p ${rundir}
+    - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
+    - *safe_create_rundir
     - echo "There was an error while running CI on Quartz" > ${rundir}/gitlab.err
+    - echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
     - cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
     - git add ${rundir}
     - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
@@ -94,7 +108,6 @@ q_report_failure:
 .build_and_test_on_quartz:
   extends: [.build_toss_3_x86_64_ib_script, .on_quartz]
   stage: q_build_and_test
-  needs: [setup]
 
 # Build MFEM
 debug_ser_gcc_4_9_3:
@@ -137,7 +150,11 @@ opt_par_gcc_6_1_0_pumi:
     SPEC: "%gcc@6.1.0 +pumi"
   extends: .build_and_test_on_quartz
 
-# Baseline
+# Baseline jobs form an independent set of jobs. We use `needs:[]` to specify
+# that "setup-baseline" can start immediately. Then, we have to use needs for
+# each one of the baseline jobs, otherwise they will wait for the rest of the
+# pipeline.
+
 baselinecheck_mfem_intel_quartz:
   extends: [.baselinecheck_mfem, .on_quartz]
   needs: [setup_baseline]
@@ -147,14 +164,15 @@ update_autotest:
   needs: [baselinecheck_mfem_intel_quartz]
   stage: baseline_to_autotest
   script:
-    - rundir="quartz/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
     - cd ${AUTOTEST_ROOT}/autotest && git pull
-    - mkdir -p ${rundir}
+    - rundir="quartz/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
+    - *safe_create_rundir
     - cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
     # We create an autotest-email.html file, because that's how we signal that there was a diff (temporary).
     - |
       if [[ -f ${rundir}/*.err ]]
       then
+        echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/*.err
         cp ${rundir}/*.err ${rundir}/autotest-email.html
       fi
     - git add ${rundir}

CHANGELOG

@@ -8,40 +8,90 @@
                                https://mfem.org
 
 
-Version 4.2.1 (development)
+Version 4.3.1 (development)
 ===========================
-- Added initial support for GPU-accelerated versions of PETSc that works with
-  MFEM_USE_CUDA if PETSc has been configured with CUDA support. Examples 1 and 9
-  in the examples/petsc directory have been modified to work with --device cuda.
-  Examples with GAMG (ex1p) and SLEPc (ex11p) are also provided.
+- Added support for hr-adaptivity using TMOP-based error estimator.
 
-- Memory management:
-  * Added method Device::SetMemoryTypes that can be used to change the default
-    host and device MemoryTypes before Device setup.
-  * In class MemoryManager, added methods GetDualMemoryType and
-    SetDualMemoryType; dual MemoryTypes are used to determine the second
-    MemoryType (host or device) when only one MemoryType is specified in methods
-    of class Memory.
-  * Added Memory constructor for setting both the host and device MemoryTypes.
-  * Switched the default behavior of device memory allocations so that they
-    are deferred until the device pointer is needed.
-  * Added a second Umpire device MemoryType, DEVICE_UMPIRE_2, with
-    corresponding allocator that can be set with the method
-    MemoryManager::SetUmpireDevice2AllocatorName.
-  * Added HOST_PINNED MemoryType and a pinned host allocator for CUDA and HIP.
+- Adding lowest order Nedelec and Raviart-Thomas basis functions on wedge
+  shaped elements.
 
-- Added support for Caliper: a library to integrate performance profiling
-  capabilities into applications. See examples/caliper for more details.
+- Added initial support for meshes with pyramidal elements, including several
+  pyramidal meshes in the data/ directory and support for the lowest order H1,
+  Nedelec, Raviart-Thomas, and L2 basis functions on pyramids.
 
-- Added support for explicit vectorization in the high-performance templated
-  code for Fujitsu's A64FX ARM microprocessor architecture.
+- Updated the hypre interface according to changes in hypre-2.22.1. The ADS
+  solver is now fully working on GPUs.
 
-- Added AlgebraicCeedSolver that does matrix-free algebraic p-multigrid for
-  diffusion problems with the Ceed backend.
+- Tetrahedral meshes no longer need to be reordered to support high order
+  Nedelec basis functions. This will allow future support for Nedelec basis
+  functions on wedges and pyramids which are not amenable to reordering. The
+  ReorientTetMesh method of the Mesh and ParMesh classes has been deprecated.
 
-- Introduced new options for the mesh-explorer miniapp to visualize the actual
-  element attributes in parallel meshes while retaining the visualization of
-  the domain decomposition.
+- Gmsh meshes where all elements have zero physical tag (the default Gmsh
+  output format if no physical groups are defined) are now successfully loaded,
+  and elements are reassigned attribute number 1.
+
+Version 4.3, released on July 29, 2021
+======================================
+
+Discretization improvements
+---------------------------
+- Variable order spaces, p- and hp-refinement. This is the initial (serial)
+  support for variable-order FiniteElementCollection and FiniteElementSpace.
+  The new method FiniteElementSpace::SetElementOrder can be called to set an
+  arbitrary order for each mesh element. The conforming interpolation matrix
+  will now automatically constrain p- and hp- interfaces, enabling general
+  hp-refinement in both 2D and 3D, on uniform or mixed NC meshes. Support for
+  parallel variable-order spaces will follow shortly.
+
+- Extended the support for field transfer between high-order and low-order
+  refined finite element spaces to include: dual fields and H1 fields (both
+  primary and dual). These are illustrated in the lor-transfer miniapp.
+
+- Improved libCEED integration, including support for VectorCoefficient,
+  ConvectionIntegrator, and VectorConvectionNLFIntegrator with libCEED backends.
+
+- Extending support for L2 basis functions using MapTypes VALUE and INTEGRAL in
+  linear interpolators and GridFunction "GetValue" methods.
+
+- Changed the interface for the error estimator and implemented the Kelly error
+  indicator for scalar-valued problems, supported in serial and parallel builds.
+
+- Added support for the "BR2" discontinuous Galerkin discretization for
+  diffusion via DGDiffusionBR2Integrator (see Example 14/14p).
+
+- Added convective and skew-symmetric integrators for the nonlinear term in the
+  Navier-Stokes equations.
+
+- Added new classes DenseSymmetricMatrix and SymmetricMatrixCoefficient for
+  efficient evaluation of symmetric matrix coefficients. This replaces the now
+  deprecated EvalSymmetric in MatrixCoefficient. Added DiagonalMatrixCoefficient
+  for clarity, which is a typedef of VectorCoefficient.
+
+- Added support for nonscalar coefficient with VectorDiffusionIntegrator.
+
+Linear and nonlinear solvers
+----------------------------
+- Added support for AMG preconditioners on GPUs based on the hypre library
+  (version 2.22.0 or later). These include BoomerAMG, AMS and ADS and most
+  MFEM examples that use hypre have been ported to support this functionality.
+  The GPU preconditioners require that both hypre and MFEM are built with CUDA
+  support. Hypre builds with CUDA and unified memory are also supported and
+  can be used with `-d cuda:uvm` as a command-line option.
+
+- Added support for AMG preconditioners for non-symmetric systems (e.g.
+  advection-dominated problems) using hypre's approximate ideal restriction
+  (AIR) AMG. Requires hypre version 2.14.0 or newer. Usage is illustrated in
+  example 9/9p.
+
+- Added new functionality for constructing low-order refined discretizations and
+  solvers, see the LORDiscretization and LORSolver classes. A new basis type for
+  H(curl) and H(div) spaces is introduced to give spectral equivalence. This
+  functionality is illustrated in the LOR solvers miniapp in miniapps/solvers.
+
+- Generalized the Multigrid class to support non-geometric multigrid. Previous
+  functionality, based on FiniteElementSpaceHierarchy, is now available in the
+  derived class GeometricMultigrid.
 
 - Introduced solver interface for linear problems with constraints, a few
   concrete solvers that implement the interface, and a demonstration of their
@@ -52,19 +102,18 @@ Version 4.2.1 (development)
   as described in Barker and Kolev 2020 (https://doi.org/10.1002/nla.2348). See
   Example 3p and linalg/auxiliary.?pp.
 
-- Added a new miniapp block-solvers that compares the performance of various
-  solvers for mixed finite element discretization of the second order scalar
-  elliptic equations. Currently available solvers in the miniapp include a
-  block-diagonal preconditioner that is based on approximate Schur complement
-  (implemented in ex5p), and a newly implemented solver DivFreeSolver, which
-  exploits a multilevel decomposition of the Raviart-Thomas space and its
-  divergence-free subspace. See the miniapps/solvers directory for more details.
+- Improved interface for using the Ginkgo library, including: support for matrix-
+  free operators in Ginkgo solvers, new wrappers for Ginkgo preconditioners, HIP
+  support, and reduction of unnecessary data copies.
 
-- Added a new miniapp for computing (signed) distance functions to a point
-  source or zero level set. See miniapps/shifted/distance.cpp.
+- Added initial support for hypre's mixed integer (mixedint) capability, which
+  uses different data types for local and global indices in order to save memory
+  in large problems. This capability requires that hypre was configured with the
+  --enable-mixedint option. Note that this option is currently tested only in
+  ex1p, ex3p, and ex4p, and may not work in more general settings.
 
-- Added matrix-free GPU-enabled implementations of GradientInterpolator and
-  IdentityInterpolator.
+- Added AlgebraicCeedSolver that does matrix-free algebraic p-multigrid for
+  diffusion problems with the Ceed backend.
 
 - Added interface to MUMPS direct solver. Its usage is demonstrated in ex25p.
   See http://mumps.enseeiht.fr/ for more details. Supported versions >= 5.1.1.
@@ -72,6 +121,17 @@ Version 4.2.1 (development)
 - Added three ESDIRK time integrators: implicit trapezoid rule, L-stable
   ESDIRK-32, and A-stable ESDIRK-33.
 
+- Implemented a variable step-size IMEX (VSSIMEX) method for the Navier miniapp.
+
+- Implemented an adaptive linear solver tolerance option for NewtonSolver based
+  on the algorithm of Eisenstat and Walker.
+
+Meshing improvements
+--------------------
+- Added support for reading high-order Lagrange meshes in VTK format. Arbitrary-
+  orders and all element types are supported. See the VTK blog for more info:
+  https://blog.kitware.com/wp-content/uploads/2018/09/Source_Issue_43.pdf.
+
 - Introduced a new non-conforming mesh format that fixes known inconsistencies
   of legacy "MFEM mesh v1.1" NC format and works consistently in both serial and
   parallel. ParMesh::ParPrint can now print non-conforming AMR meshes that can
@@ -80,113 +140,26 @@ Version 4.2.1 (development)
   NC data files are compatible with serial code, e.g., can be viewed with serial
   GLVis. Loading of legacy NC mesh files is still supported.
 
-- Added support for 1D non-conforming meshes (which can be useful for parallel
-  load balancing and derefinement).
-
-- Added a "scaled Jacobian" visualization option in the Mesh Explorer miniapp to
-  help identify elements with poor mesh quality.
-
-- Added support for the "BR2" discontinuous Galerkin discretization for
-  diffusion via DGDiffusionBR2Integrator (see Example 14/14p).
-
-- Generalized the Multigrid class to support non-geometric multigrid. The
-  previous functionality, based on FiniteElementSpaceHierarchy, is now available
-  in the derived class GeometricMultigrid.
-
-- Upgraded the Catch unit test framework from version 2.13.0 to version 2.13.2.
-
-- The TMOP mesh optimization algorithms were extended to GPU:
-   - QualityMetric #1, #2, #7 and #77 are available in 2D, #302, #303, #315
-     and #321 in 3D
-   - Both AnalyticAdaptTC and DiscreteAdaptTC TargetConstructor are available
-   - Kernels for normalization and limiting have been added
-   - The AdvectorCG now also supports AssemblyLevel::PARTIAL
-
-- Added a new command line boolean option (`--all`) to the unit tests to launch
-  *all* non-regression tests.
-
-- Added support for different modes of QuadratureInterpolator on GPU.
-  The layout (QVectorLayout::byNODES|byVDIM) and the tensor products modes can
-  be enabled before calling the Mult, Values, Derivatives, PhysDerivatives and
-  Determinants methods.
-
-- Implemented a filter method for the Navier miniapp to stabilize highly
-  turbulent flows in direct numerical simulation.
-
-- Added HIP support to the CMake build system.
-
-- Added support for reading high-order Lagrange meshes in VTK format. Arbitrary-
-  orders and all element types are supported. See the VTK blog for more info:
-  https://blog.kitware.com/wp-content/uploads/2018/09/Source_Issue_43.pdf.
-
-- Added support for reading VTK meshes in XML format.
-
-- Added partial assembly and device support to Example 25/25p, with diagonal
-  preconditioning.
-
-- Implemented a variable step-size IMEX (VSSIMEX) method for the Navier miniapp.
+- Added FMS support (https://github.com/CEED/FMS) to mfem. FMS can represent
+  unstructured high-order meshes with general high-order finite element fields
+  on them. When enabled, mfem can convert data collections to/from FMS data
+  collections in memory. In addition, an FMS data collection class was added so
+  the convert-dc miniapp can read and generate data files in FMS format.
 
 - Added new mesh quality metrics and improved the untangling capabilities of the
   TMOP-based mesh optimization algorithms.
 
-- Added convective and skew-symmetric integrators for the nonlinear term in the
-  Navier-Stokes equations.
-
-- Added new miniapp directory mtop/ with optimization-oriented block parametric
-  non-linear form and abstract integrators. Two new miniapps, ParHeat and
-  SeqHeat, demonstrate parallel and sequential implementation of gradients
-  evaluation for linear diffusion with discrete density.
-
-- Changed the interface for the error estimator.
-
-- Implemented the Kelly error indicator for scalar-valued problems, supported
-  in serial and parallel builds.
-
-- Added new classes DenseSymmetricMatrix and SymmetricMatrixCoefficient for
-  efficient evaluation of symmetric matrix coefficients. This replaces the now
-  deprecated EvalSymmetric in MatrixCoefficient. Added DiagonalMatrixCoefficient
-  for clarity, which is a typedef of VectorCoefficient.
-
-- Added support for AMG preconditioners for non-symmetric systems (e.g.
-  advection-dominated problems) using hypre's approximate ideal restriction
-  (AIR) AMG. Requires hypre version 2.14.0 or newer. Usage is illustrated in
-  example 9/9p.
-
-- Implemented an adaptive linear solver tolerance option for NewtonSolver based
-  on the algorithm of Eisenstat and Walker.
-
-- Added support for nonscalar coefficient with VectorDiffusionIntegrator.
-
-- Extending support for L2 basis functions using MapTypes VALUE and INTEGRAL in
-  linear interpolators and GridFunction "GetValue" methods.
-
-- Variable order spaces, p- and hp-refinement. This is the initial (serial)
-  support for variable-order FiniteElementCollection and FiniteElementSpace.
-  The new method FiniteElementSpace::SetElementOrder can be called to set an
-  arbitrary order for each mesh element. The conforming interpolation matrix
-  will now automatically constrain p- and hp- interfaces, enabling general
-  hp-refinement in both 2D and 3D, on uniform or mixed NC meshes. Support for
-  parallel variable-order spaces will follow shortly.
+- The TMOP mesh optimization algorithms were extended to GPU:
+  * QualityMetric 1, 2, 7, 77 are available in 2D, 302, 303, 315, 321 in 3D
+  * Both AnalyticAdaptTC and DiscreteAdaptTC TargetConstructor are available
+  * Kernels for normalization and limiting have been added
+  * The AdvectorCG now also supports AssemblyLevel::PARTIAL
 
 - Added support for creating refined meshes for all element types (e.g. by
   splitting high-order elements into low-order refined elements), including
   mixed meshes. The LOR Transfer miniapp (miniapps/tools/lor-transfer.cpp) now
   supports meshes with any element geometry.
 
-- Testing improvements:
-  * Transitioned from Travis to GitHub Action for testing/CI on GitHub.
-  * Effectively remove Travis from CI.
-  * Use Spack (and Uberenv) to automate TPL building in LLNL GitLab tests.
-  * Added a set of suggested git hooks for developers in config/githooks.
-
-- Added new miniapps demonstrating: 1) the use of GSLIB for overlapping grids,
-  see gslib/schwarz_ex1, and 2) coupling different physics in different domains,
-  see navier/cht. Note that gslib v1.0.7 is require (see INSTALL for details).
-
-- Added a new, very simple example (ex0 and parallel version ex0p). This
-  example solves a simple Poisson problem using H1 elements (the same problem as
-  ex1), but is intended to be extremely simple and approachable for new users.
-
 - Meshes consisting of any type of elements (including mixed meshes) can be
   converted to all-simplex meshes using Mesh::MakeSimplicial.
 
@@ -199,42 +172,133 @@ Version 4.2.1 (development)
   requisite periodic vertex mappings can be created with
   Mesh::CreatePeriodicVertexMapping.
 
-- Added support for transferring dual fields between high-order and low-order
-  refined finite element spaces using the transposed versions of the
-  L2ProjectionGridTransfer operators. This functionality is illustrated in the
-  lor-transfer miniapp.
-
-- Improved interface for using the Ginkgo library, including: support for matrix-
-  free operators in Ginkgo solvers, new wrappers for Ginkgo preconditioners, HIP
-  support, and reduction of unnecessary data copies.
-
-- Added initial support for hypre's mixed integer (mixedint) capability, which
-  uses different data types for local and global indices in order to save memory
-  in large problems. This capability requires that hypre was configured with the
-  --enable-mixedint option. Note that this option is currently tested only in
-  ex1p and may not work in more general settings.
-
-- Added support for transferring fields (primary and dual) between high-order
-  and low-order refined H1 finite element spaces using the
-  L2ProjectionH1GridTransfer operators. This functionality is demonstrated
-  through the lor-transfer miniapp when run with the -h1 option.
-
-- Added new functionality for constructing low-order refined discretizations and
-  solvers, see the LORDiscretization and LORSolver classes. A new basis type for
-  H(curl) and H(div) spaces is introduced to give spectral equivalence. This
-  functionality is illustrated in the LOR solvers miniapp in miniapps/solvers.
+- Added support for 1D non-conforming meshes (which can be useful for parallel
+  load balancing and derefinement).
 
 - Added sample meshes in the `data` subdirectory showing the reference elements
   of the six currently supported element types; ref-segment.mesh,
   ref-triangle.mesh, ref-square.mesh, ref-tetrahedron.mesh, ref-cube.mesh, and
   ref-prism.mesh.
 
+High-performance computing
+--------------------------
+- Added initial support for GPU-accelerated versions of PETSc that works with
+  MFEM_USE_CUDA if PETSc has been configured with CUDA support. Examples 1 and 9
+  in the examples/petsc directory have been modified to work with --device cuda.
+  Examples with GAMG (ex1p) and SLEPc (ex11p) are also provided.
+
+- Added support for explicit vectorization in the high-performance templated
+  code for Fujitsu's A64FX ARM microprocessor architecture.
+
+- Added support for different modes of QuadratureInterpolator on GPU.
+  The layout (QVectorLayout::byNODES|byVDIM) and the tensor products modes can
+  be enabled before calling the Mult, Values, Derivatives, PhysDerivatives and
+  Determinants methods.
+
+- Added method Device::SetMemoryTypes that can be used to change the default
+  host and device MemoryTypes before Device setup.
+
+- In class MemoryManager, added methods GetDualMemoryType and SetDualMemoryType;
+  dual MemoryTypes are used to determine the second MemoryType (host or device)
+  when only one MemoryType is specified in methods of class Memory.
+
+- Added Memory constructor for setting both the host and device MemoryTypes.
+
+- Switched the default behavior of device memory allocations so that they are
+  deferred until the device pointer is needed.
+
+- Added a second Umpire device MemoryType, DEVICE_UMPIRE_2, with corresponding
+  allocator that can be set with the method SetUmpireDevice2AllocatorName.
+
+- Added HOST_PINNED MemoryType and a pinned host allocator for CUDA and HIP.
+
+- Added matrix-free GPU-enabled implementations of GradientInterpolator and
+  IdentityInterpolator.
+
+New and updated examples and miniapps
+-------------------------------------
+- Added a new, very simple example (ex0 and parallel version ex0p). This example
+  solves a simple Poisson problem using H1 elements (the same problem as ex1),
+  but is intended to be extremely simple and approachable for new users.
+
+- Added new miniapps demonstrating: 1) the use of GSLIB for overlapping grids,
+  see gslib/schwarz_ex1, and 2) coupling different physics in different domains,
+  see navier/cht. Note that gslib v1.0.7 is require (see INSTALL for details).
+
+- Added a new miniapp for computing (signed) distance functions to a point
+  source or zero level set. See miniapps/shifted/distance.cpp.
+
 - Added a high-order extension of the shifted boundary method to solve PDEs on
   non body-fitted meshes. This is illustrated in the new Shifted Diffusion
   miniapp, see miniapps/shifted/diffusion.cpp.
 
+- Added new miniapp directory mtop/ with optimization-oriented block parametric
+  non-linear form and abstract integrators. Two new miniapps, ParHeat and
+  SeqHeat, demonstrate parallel and sequential implementation of gradients
+  evaluation for linear diffusion with discrete density.
+
+- Added a new miniapp block-solvers that compares the performance of various
+  solvers for mixed finite element discretization of the second order scalar
+  elliptic equations. Currently available solvers in the miniapp include a
+  block-diagonal preconditioner that is based on approximate Schur complement
+  (implemented in ex5p), and a newly implemented solver DivFreeSolver, which
+  exploits a multilevel decomposition of the Raviart-Thomas space and its
+  divergence-free subspace. See the miniapps/solvers directory for more details.
+
+- Introduced new options for the mesh-explorer miniapp to visualize the actual
+  element attributes in parallel meshes while retaining the visualization of the
+  domain decomposition.
+
+- Added partial assembly and device support to Example 25/25p, with diagonal
+  preconditioning.
+
+- Implemented a filter method for the Navier miniapp to stabilize highly
+  turbulent flows in direct numerical simulation.
+
+Improved testing
+----------------
+- Transitioned from Travis to GitHub Action for testing/CI on GitHub.
+
+- Use Spack (and Uberenv) to automate TPL building in LLNL GitLab tests.
+
+- Extended `make test` to include GPU tests when MFEM is built with CUDA or HIP
+  support.
+
+- Added a set of suggested git hooks for developers in config/githooks.
+
+- Added support for Caliper: a library to integrate performance profiling
+  capabilities into applications. See examples/caliper for more details.
+
+- Added a new command line boolean option (`--all`) to the unit tests to launch
+  *all* non-regression tests.
+
+- Upgraded the Catch unit test framework from version 2.13.0 to version 2.13.2.
+
+Miscellaneous
+-------------
+- The following integrations have updated minimum version requirements:
+  * CUDA >= 10.1.168
+  * Ginkgo >= 1.4.0
+  * GSLIB >= 1.0.7
+  * HIOP >= 0.4
+  * HYPRE >= 2.20.0 for mixedint support
+  * HYPRE >= 2.22.0 for CUDA support
+  * libCEED >= 0.8
+  * PETSc >= 3.15.0 for CUDA support
+  * RAJA >= 0.13.0
+  see INSTALL for more details.
+
+- Added a "scaled Jacobian" visualization option in the Mesh Explorer miniapp to
+  help identify elements with poor mesh quality.
+
+- Added support for reading VTK meshes in XML format.
+
 - Added makefile rule to generate TAGS table for vi or Emacs users.
 
+- Added HIP support to the CMake build system.
+
+- Various other simplifications, extensions, and bugfixes in the code.
+
 API changes
 -----------
 - Added an abstract interface `mfem::FaceRestriction` for `H1FaceRestriction`
@@ -244,20 +308,11 @@ API changes
   `mfem::FaceRestriction::AddMultTranspose` should replace previous calls to
   `mfem::FaceRestriction::MultTranspose`.
 
-libCEED integration improvements
---------------------------------
-- Refactor the libCEED integration
-
-- Add support for VectorCoefficient with libCEED backends.
-
-- Add support for ConvectionIntegrator, and VectorConvectionNLFIntegrator with
-  libCEED backends.
-
 
 Version 4.2, released on October 30, 2020
 =========================================
 
-High-Performance Computing
+High-performance computing
 --------------------------
 - Added support for explicit vectorization in the high-performance templated
   code, which can now take advantage of specific classes on the following
@@ -339,9 +394,6 @@ Linear and nonlinear solvers
   matrix with the function HypreParMatrixFromBlocks. This could be useful for
   solving block systems with parallel direct solvers such as STRUMPACK.
 
-- Added CUDA support for SUNDIALS ODE integrators. See the updated SUNDIALS
-  modification of Example 9/9p.
-
 - Added wrappers for hypre's flexible GMRES solver and the new parallel ILU
   preconditioner. The latter requires hypre version 2.19.0 or later.
 
@@ -452,7 +504,7 @@ New and updated examples and miniapps
     L2, with partial assembly support in Example 24/24p.
   * Weak Dirichlet boundary conditions (Nitsche) to the NURBS miniapp.
 
-Data management and Visualization
+Data management and visualization
 ---------------------------------
 - Added support for ADIOS2 for parallel I/O with ParaView visualization. See
   Examples 5, 9, 12, 16. The classes adios2stream and ADIOS2DataCollection

CITATION.cff

@@ -0,0 +1,66 @@
+cff-version: 1.2.0
+message: "If you use MFEM, please cite it as follows."
+authors:
+- family-names: "MFEM Team"
+title: "MFEM: Modular Finite Element Methods [Software]"
+doi: 10.11578/dc.20171025.1248
+url: "https://mfem.org"
+preferred-citation:
+  type: article
+  authors:
+  - family-names: "Anderson"
+    given-names: "Robert"
+    orcid: "https://orcid.org/0000-0002-3508-9944"
+  - family-names: "Andrej"
+    given-names: "Julian"
+    orcid: "https://orcid.org/0000-0001-7661-4840"
+  - family-names: "Barker"
+    given-names: "Andrew"
+    orcid: "https://orcid.org/0000-0003-3572-911X"
+  - family-names: "Bramwell"
+    given-names: "Jamie"
+  - family-names: "Camier"
+    given-names: "Jean-Sylvain"
+    orcid: "https://orcid.org/0000-0003-2421-1999"
+  - family-names: "Cerveny"
+    given-names: "Jakub"
+    orcid: "https://orcid.org/0000-0003-4231-2531"
+  - family-names: "Dobrev"
+    given-names: "Veselin"
+    orcid: "https://orcid.org/0000-0003-1793-5622"
+  - family-names: "Dudouit"
+    given-names: "Yohann"
+    orcid: "https://orcid.org/0000-0001-5831-561X"
+  - family-names: "Fisher"
+    given-names: "Aaron"
+  - family-names: "Kolev"
+    given-names: "Tzanio"
+    orcid: "https://orcid.org/0000-0002-2810-3090"
+  - family-names: "Pazner"
+    given-names: "Will"
+    orcid: "https://orcid.org/0000-0003-4885-2934"
+  - family-names: "Stowell"
+    given-names: "Mark"
+    orcid: "https://orcid.org/0000-0002-5389-7435"
+  - family-names: "Tomov"
+    given-names: "Vladimir"
+    orcid: "https://orcid.org/0000-0002-1846-6816"
+  - family-names: "Akkerman"
+    given-names: "Ido"
+    orcid: "https://orcid.org/0000-0002-5937-0300"
+  - family-names: "Dahm"
+    given-names: "Johann"
+    orcid: "https://orcid.org/0000-0001-9657-3564"
+  - family-names: "Medina"
+    given-names: "David"
+  - family-names: "Zampini"
+    given-names: "Stefano"
+    orcid: "https://orcid.org/0000-0002-0435-0433"
+  doi: "10.1016/j.camwa.2020.06.009"
+  journal: "Computers \\& Mathematics with Applications"
+  month: 1
+  start: 42 # First page number
+  end: 74 # Last page number
+  title: "MFEM: A Modular Finite Element Methods Library"
+  volume: 81
+  year: 2021

CMakeLists.txt

@@ -16,7 +16,7 @@ set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH
 
 # Require C++11 and disable compiler-specific extensions
 set(CMAKE_CXX_STANDARD 11)
-if (MFEM_USE_GINKGO) 
+if (MFEM_USE_GINKGO)
   set(CMAKE_CXX_STANDARD 14)
 endif()
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -54,7 +54,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.2.1)
+set(${PROJECT_NAME}_VERSION 4.3.1)
 
 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -102,7 +102,7 @@ if (MFEM_USE_CUDA)
    endif()
    enable_language(CUDA)
    set(CMAKE_CUDA_STANDARD 11)
-   if (MFEM_USE_GINKGO) 
+   if (MFEM_USE_GINKGO)
      set(CMAKE_CUDA_STANDARD 14)
    endif()
    set(CMAKE_CUDA_STANDARD_REQUIRED ON)
@@ -246,6 +246,7 @@ if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
     message(FATAL_ERROR " *** MFEM_USE_LEGACY_OPENMP requires MFEM_THREAD_SAFE=ON.")
   endif()
   find_package(OpenMP REQUIRED)
+  set(OPENMP_LIBRARIES ${OpenMP_CXX_LIBRARIES})
 endif()
 
 # SuiteSparse (before SUNDIALS which may depend on KLU)
@@ -330,6 +331,10 @@ if (MFEM_USE_CONDUIT)
     find_package(Conduit REQUIRED conduit relay blueprint )
 endif()
 
+if (MFEM_USE_FMS)
+    find_package(FMS REQUIRED fms )
+endif()
+
 # Axom/Sidre
 if (MFEM_USE_SIDRE)
   find_package(Axom REQUIRED Axom)
@@ -424,9 +429,10 @@ endif()
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
 set(MFEM_TPLS MPI_CXX OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS PETSC
-    SLEPC MESQUITE MUMPS STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
+    SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB NETCDF
     MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
     CUSPARSE MKL_CPARDISO AMGX CALIPER)
+
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")
@@ -445,6 +451,9 @@ include_directories(${TPL_INCLUDE_DIRS})
 if (OPENMP_FOUND)
   message(STATUS "MFEM: using package OpenMP")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+  if (MFEM_USE_CUDA)
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=${OpenMP_CXX_FLAGS}")
+  endif()
 endif()
 
 message(STATUS "MFEM build type: CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}")

CONTRIBUTING.md

@@ -97,7 +97,6 @@ The MFEM source code has the following structure:
   .
   ├── config
   │   ├── cmake
-  │   │    └── ...
   │   └── githooks
   ├── data
   ├── doc
@@ -135,10 +134,10 @@ The MFEM source code has the following structure:
   └── tests
       ├── convergence
       ├── gitlab
+      ├── mem_manager
       ├── par-mesh-format
       ├── scripts
       └── unit
-          └── ...
 ```
 
 #### Main directories and classes

INSTALL

@@ -474,7 +474,7 @@ MFEM_USE_HIP = YES/NO
    Enables support for AMD devices in MFEM. HIP is a heterogeneous-compute
    interface for portability developed by AMD that can target both AMD and
    NVIDIA GPUs. The variable HIP_ARCH is used to specify the AMD GPU processor
-   used during compilation (by default, HIP_ARCH=gfx900).  When enabled, this
+   used during compilation (by default, HIP_ARCH=gfx900). When enabled, this
    option uses the HIP_* build options, see below.
 
 MFEM_USE_RAJA = YES/NO
@@ -516,6 +516,13 @@ MFEM_USE_CALIPER = YES/NO
    profiling at runtime with Caliper's configuration API. Alternatively, one
    can configure Caliper through environment variables or config files.
 
+MFEM_USE_FMS = YES/NO
+   Enables support for the FMS library which consists of the DataCollection
+   sub-class mfem::FMSDataCollection for I/O in FMS formats, see the header file
+   fem/fmsdatacollection.hpp. In addition, this option enables in-memory
+   convetion routines between FMS's FmsDataCollection structure and MFEM's
+   DataCollection class, see the header file fem/fmsconvert.hpp.
+
 MFEM_BUILD_TAG = (any value)
    An optional tag to characterize the build. Exported to config/config.mk.
    Can be used to identify the MFEM build from other makefiles.
@@ -540,8 +547,9 @@ The specific libraries and their options are:
   See also the "Specific options for hypre" section at the end of this file.
   URL: https://github.com/hypre-space/hypre and https://www.llnl.gov/casc/hypre
   Options: HYPRE_OPT, HYPRE_LIB.
-  Versions: HYPRE >= 2.10.0b,
-            HYPRE >= 2.20.0 for '--enable-mixedint' support.
+  Versions: HYPRE >= 2.10.0b  (HYPRE built without CUDA)
+            HYPRE >= 2.20.0   (HYPRE built with '--enable-mixedint')
+            HYPRE >= 2.22.1   (HYPRE built with CUDA)
 
 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
   MFEM_USE_METIS_5 = YES (default is to use METIS 4).
@@ -615,7 +623,7 @@ The specific libraries and their options are:
   and dependencies of specific modules, see the Ginkgo webpage below.
   URL: https://ginkgo-project.github.io
   Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or Debug).
-  Versions: Ginkgo >= 1.4.0. 
+  Versions: Ginkgo >= 1.4.0.
 
 - AmgX (optional), used when MFEM_USE_AMGX = YES.
   URL: https://github.com/NVIDIA/AMGX
@@ -753,6 +761,11 @@ The specific libraries and their options are:
   URL: https://zlib.net
   Options: ZLIB_OPT, ZLIB_LIB.
 
+- FMS (optional), used when MFEM_USE_FMS = YES.
+  URL: https://github.com/CEED/FMS
+  Options: FMS_OPT, FMS_LIB.
+  Versions: FMS >= 0.2.
+
 Building with CMake
 ===================
 The MFEM build system consists of two steps: configuration and compilation.
@@ -884,6 +897,7 @@ MFEM_USE_RAJA
 MFEM_USE_UMPIRE
 MFEM_USE_SIDRE
 MFEM_USE_CALIPER
+MFEM_USE_FMS
 
 The following options are CMake specific:
 
@@ -938,6 +952,7 @@ The CMake build system adds auto-detection for the following packages/libraries:
  - UMPIRE
  - AXOM - Used when MFEM_USE_SIDRE is enabled
  - CALIPER
+ - FMS
 
 The following built-in CMake packages are also used:
 
@@ -955,7 +970,7 @@ config/config.hpp.in:
 
    cp config/config.hpp.in config/_config.hpp
 
-The file config/_config.hpp can then be edited to enable desired options.  The
+The file config/_config.hpp can then be edited to enable desired options. The
 MFEM library is simply a combination of all object files obtained by compiling
 the .cpp source files in the source directories: general, linalg, mesh, and fem.
 
@@ -963,7 +978,7 @@ the .cpp source files in the source directories: general, linalg, mesh, and fem.
 Specifying an MPI job launcher
 ==============================
 By default, MFEM will use 'mpirun -np #' to launch any of its parallel tests or
-miniapps, where # is the number of MPI tasks.  An alternate MPI launcher can be
+miniapps, where # is the number of MPI tasks. An alternate MPI launcher can be
 provided by setting the MFEM_MPIEXEC and MFEM_MPIEXEC_NP config variables.
 
 MFEM will expect the launcher command, plus the command line option to allow it

config/XSDKDefaults.cmake

@@ -256,6 +256,10 @@ IF (DEFINED TPL_ENABLE_SIDRE)
   SET(MFEM_USE_SIDRE ${TPL_ENABLE_SIDRE} CACHE BOOL "Enable Axom/Sidre usage" FORCE)
 ENDIF()
 
+IF (DEFINED TPL_ENABLE_FMS)
+  SET(MFEM_USE_FMS ${TPL_ENABLE_FMS} CACHE BOOL "Enable FMS usage" FORCE)
+ENDIF()
+
 IF (DEFINED TPL_ENABLE_CONDUIT)
   SET(MFEM_USE_CONDUIT ${TPL_ENABLE_CONDUIT} CACHE BOOL "Enable Conduit usage" FORCE)
 ENDIF()

config/cmake/MFEMConfig.cmake.in

@@ -44,6 +44,7 @@ set(MFEM_USE_PETSC @MFEM_USE_PETSC@)
 set(MFEM_USE_SLEPC @MFEM_USE_SLEPC@)
 set(MFEM_USE_MPFR @MFEM_USE_MPFR@)
 set(MFEM_USE_SIDRE @MFEM_USE_SIDRE@)
+set(MFEM_USE_FMS @MFEM_USE_FMS@)
 set(MFEM_USE_CONDUIT @MFEM_USE_CONDUIT@)
 set(MFEM_USE_PUMI @MFEM_USE_PUMI@)
 set(MFEM_USE_CUDA @MFEM_USE_CUDA@)

config/cmake/config.hpp.in

@@ -119,6 +119,9 @@
 // Enable the use of SIMD in the high performance templated classes
 #cmakedefine MFEM_USE_SIMD
 
+// Enable MFEM functionality based on the FMS library
+#cmakedefine MFEM_USE_FMS
+
 // Enable MFEM functionality based on Conduit
 #cmakedefine MFEM_USE_CONDUIT
 

config/cmake/modules/FindFMS.cmake

@@ -0,0 +1,20 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - FMS_FOUND
+#   - FMS_LIBRARIES
+#   - FMS_INCLUDE_DIRS
+
+include(MfemCmakeUtilities)
+mfem_find_package(FMS FMS FMS_DIR
+  "include" fms.h "lib" fms
+  "Paths to headers required by FMS." "Libraries required by FMS.")

config/config.hpp.in

@@ -91,6 +91,12 @@
 // Enable MFEM functionality based on the SuiteSparse library.
 // #define MFEM_USE_SUITESPARSE
 
+// Enable MFEM functionality based on the ARPACK library.
+// #define MFEM_USE_ARPACK
+
+// Enable MFEM functionality based on the SPECTRA library.
+// #define MFEM_USE_SPECTRA
+
 // Enable MFEM functionality based on the SuperLU library.
 // #define MFEM_USE_SUPERLU
 // #define MFEM_USE_SUPERLU5
@@ -117,6 +123,9 @@
 // Enable the use of SIMD in the high performance templated classes
 // #define MFEM_USE_SIMD
 
+// Enable FMS support
+// #define MFEM_USE_FMS
+
 // Enable Conduit support
 // #define MFEM_USE_CONDUIT
 

config/config.mk.in

@@ -31,6 +31,8 @@ MFEM_TIMER_TYPE        = @MFEM_TIMER_TYPE@
 MFEM_USE_SUNDIALS      = @MFEM_USE_SUNDIALS@
 MFEM_USE_MESQUITE      = @MFEM_USE_MESQUITE@
 MFEM_USE_SUITESPARSE   = @MFEM_USE_SUITESPARSE@
+MFEM_USE_ARPACK        = @MFEM_USE_ARPACK@
+MFEM_USE_SPECTRA       = @MFEM_USE_SPECTRA@
 MFEM_USE_SUPERLU       = @MFEM_USE_SUPERLU@
 MFEM_USE_SUPERLU5      = @MFEM_USE_SUPERLU5@
 MFEM_USE_MUMPS         = @MFEM_USE_MUMPS@
@@ -43,6 +45,7 @@ MFEM_USE_PETSC         = @MFEM_USE_PETSC@
 MFEM_USE_SLEPC         = @MFEM_USE_SLEPC@
 MFEM_USE_MPFR          = @MFEM_USE_MPFR@
 MFEM_USE_SIDRE         = @MFEM_USE_SIDRE@
+MFEM_USE_FMS           = @MFEM_USE_FMS@
 MFEM_USE_CONDUIT       = @MFEM_USE_CONDUIT@
 MFEM_USE_PUMI          = @MFEM_USE_PUMI@
 MFEM_USE_HIOP          = @MFEM_USE_HIOP@

config/defaults.cmake

@@ -45,6 +45,7 @@ option(MFEM_USE_PETSC "Enable PETSc support." OFF)
 option(MFEM_USE_SLEPC "Enable SLEPc support." OFF)
 option(MFEM_USE_MPFR "Enable MPFR usage." OFF)
 option(MFEM_USE_SIDRE "Enable Axom/Sidre usage" OFF)
+option(MFEM_USE_FMS "Enable FMS usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
 option(MFEM_USE_PUMI "Enable PUMI" OFF)
 option(MFEM_USE_HIOP "Enable HiOp" OFF)
@@ -96,6 +97,11 @@ set(HYPRE_DIR "${MFEM_DIR}/../hypre/src/hypre" CACHE PATH
 # If hypre was compiled to depend on BLAS and LAPACK:
 # set(HYPRE_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
 #     "Packages that HYPRE depends on.")
+if (MFEM_USE_CUDA)
+   # This is only necessary when hypre is built with cuda:
+   set(HYPRE_REQUIRED_LIBRARIES "-lcusparse" "-lcurand" CACHE STRING
+       "Libraries that HYPRE depends on.")
+endif()
 
 set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library.")
 
@@ -132,10 +138,10 @@ set(MUMPS_DIR "${MFEM_DIR}/../MUMPS_5.2.0" CACHE PATH
     "Path to the MUMPS library.")
 # Packages required by MUMPS, depending on how it was compiled.
 set(MUMPS_REQUIRED_PACKAGES "MPI" "BLAS" "METIS" "ScaLAPACK" CACHE STRING
-    "Additional packages required by MUMPS.")    
+    "Additional packages required by MUMPS.")
 # If the MPI package does not find all required Fortran libraries:
 # set(MUMPS_REQUIRED_LIBRARIES "gfortran" "mpi_mpifh" CACHE STRING
-#     "Additional libraries required by MUMPS.")    
+#     "Additional libraries required by MUMPS.")
 
 set(STRUMPACK_DIR "${MFEM_DIR}/../STRUMPACK-build" CACHE PATH
     "Path to the STRUMPACK library.")
@@ -187,6 +193,12 @@ set(SLEPC_ARCH "arch-linux2-c-debug" CACHE STRING "SLEPC build architecture.")
 
 set(MPFR_DIR "" CACHE PATH "Path to the MPFR library.")
 
+set(FMS_DIR "${MFEM_DIR}/../fms" CACHE PATH
+    "Path to the FMS library.")
+# If FMS is built with Conduit:
+# set(FMS_REQUIRED_PACKAGES "Conduit/relay" CACHE STRING
+#     "Additional packages required by FMS.")
+
 set(CONDUIT_DIR "${MFEM_DIR}/../conduit" CACHE PATH
     "Path to the Conduit library.")
 

config/defaults.mk

@@ -136,6 +136,7 @@ MFEM_USE_PETSC         = NO
 MFEM_USE_SLEPC         = NO
 MFEM_USE_MPFR          = NO
 MFEM_USE_SIDRE         = NO
+MFEM_USE_FMS           = NO
 MFEM_USE_CONDUIT       = NO
 MFEM_USE_PUMI          = NO
 MFEM_USE_HIOP          = NO
@@ -150,6 +151,8 @@ MFEM_USE_UMPIRE        = NO
 MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
 MFEM_USE_MKL_CPARDISO  = NO
+MFEM_USE_ARPACK        = NO
+MFEM_USE_SPECTRA       = NO
 
 # MPI library compile and link flags
 # These settings are used only when building MFEM with MPI + HIP
@@ -174,6 +177,10 @@ LIBUNWIND_LIB = $(if $(NOTMAC),-lunwind -ldl,)
 HYPRE_DIR = @MFEM_DIR@/../hypre/src/hypre
 HYPRE_OPT = -I$(HYPRE_DIR)/include
 HYPRE_LIB = -L$(HYPRE_DIR)/lib -lHYPRE
+ifeq (YES,$(MFEM_USE_CUDA))
+   # This is only necessary when hypre is built with cuda:
+   HYPRE_LIB += -lcusparse -lcurand
+endif
 
 # METIS library configuration
 ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK)$(MFEM_USE_MUMPS),NONONO)
@@ -323,6 +330,19 @@ NETCDF_LIB = $(XLINKER)-rpath,$(NETCDF_DIR)/lib -L$(NETCDF_DIR)/lib\
  $(XLINKER)-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib\
  -lnetcdf -lhdf5_hl -lhdf5 $(ZLIB_LIB)
 
+# ARPACK library configuration
+ARPACK_DIR = @MFEM_DIR@/../ARPACK
+ARPACK_OPT = -I$(ARPACK_DIR)
+ARPACK_LIB = -L$(ARPACK_DIR) -lparpack -larpack
+
+# EIGEN library configuration
+EIGEN_DIR = @MFEM_DIR@/../eigen
+EIGEN_OPT = -I$(EIGEN_DIR)
+
+# SPECTRA library configuration
+SPECTRA_DIR = @MFEM_DIR@/../spectra/include
+SPECTRA_OPT = -I$(SPECTRA_DIR) $(EIGEN_OPT)
+
 # PETSc library configuration (version greater or equal to 3.8 or the dev branch)
 PETSC_ARCH := arch-linux2-c-debug
 PETSC_DIR  := $(MFEM_DIR)/../petsc/$(PETSC_ARCH)
@@ -357,6 +377,11 @@ endif
 MPFR_OPT =
 MPFR_LIB = -lmpfr
 
+# FMS and required libraries configuration
+FMS_DIR = $(MFEM_DIR)/../fms
+FMS_OPT = -I$(FMS_DIR)/include
+FMS_LIB = -Wl,-rpath,$(FMS_DIR)/lib -L$(FMS_DIR)/lib -lfms
+
 # Conduit and required libraries configuration
 CONDUIT_DIR = @MFEM_DIR@/../conduit
 CONDUIT_OPT = -I$(CONDUIT_DIR)/include/conduit

config/test.mk

@@ -57,22 +57,27 @@ TIMECMD := $(word 1,$(TIMECMD))
 ifneq (,$(filter test%,$(MAKECMDGOALS)))
    MAKEFLAGS += -k
 endif
-# Test runs of the examples/miniapps with parameters - check exit code
+# Test runs of the examples/miniapps with parameters - check exit code:
+# 0 means success, 255 means the test was skipped, anything else means error
 mfem-test = \
    printf "   $(3) [$(2) $(1) ... ]: "; \
    $(call $(TIMEFUN),$(TIMECMD),$(2) ./$(1) $(if $(5),,-no-vis )$(4) \
      > $(1).stderr 2>&1); \
-   if [ "$$3" = 0 ]; \
-   then $(PRINT_OK); else $(PRINT_FAILED); cat $(1).stderr; fi; \
-   rm -f $(1).stderr; exit $$3
+   err="$$3"; \
+   if [ "$$3" = 0 ]; then $(PRINT_OK); \
+   else if [ "$$3" = 255 ]; then $(PRINT_SKIP); err=0; \
+   else $(PRINT_FAILED); cat $(1).stderr; fi; fi; \
+   rm -f $(1).stderr; exit $$err
 
 # Test runs of the examples/miniapps - check exit code and if a file exists
+# See mfem-test for the interpretation of the error code
 mfem-test-file = \
    printf "   $(3) [$(2) $(1) ... ]: "; \
    $(call $(TIMEFUN),$(TIMECMD),$(2) ./$(1) -no-vis > $(1).stderr 2>&1); \
    err="$$3"; \
-   if [ "$$3" = 0 ] && [ -e $(4) ]; \
-   then $(PRINT_OK); else $(PRINT_FAILED); cat $(1).stderr; err=64; fi; \
+   if [ "$$3" = 0 ] && [ -e $(4) ]; then $(PRINT_OK); \
+   else if [ "$$3" = 255 ] && [ -e $(4) ]; then $(PRINT_SKIP); err=0; \
+   else $(PRINT_FAILED); cat $(1).stderr; err=64; fi; fi; \
    rm -f $(1).stderr; exit $$err
 
 .PHONY: test test-par-YES test-par-NO test-ser test-par test-clean test-print
@@ -80,6 +85,26 @@ mfem-test-file = \
 # What sets of tests to run in serial and parallel
 test-par-YES: $(PAR_$(MFEM_TESTS):=-test-par) $(SEQ_$(MFEM_TESTS):=-test-seq)
 test-par-NO:  $(SEQ_$(MFEM_TESTS):=-test-seq)
+ifeq ($(MFEM_USE_CUDA),YES)
+.PHONY: test-par-YES-cuda test-par-NO-cuda test-ser-cuda test-par-cuda test-cuda
+test-par-YES: test-par-YES-cuda
+test-par-NO:  test-par-NO-cuda
+test-par-YES-cuda: test-par-cuda test-ser-cuda
+test-par-NO-cuda:  test-ser-cuda
+test-ser-cuda: $(SEQ_DEVICE_$(MFEM_TESTS):=-test-seq-cuda)
+test-par-cuda: $(PAR_DEVICE_$(MFEM_TESTS):=-test-par-cuda)
+test-cuda: test-par-$(MFEM_USE_MPI)-cuda clean-exec
+endif
+ifeq ($(MFEM_USE_HIP),YES)
+.PHONY: test-par-YES-hip test-par-NO-hip test-ser-hip test-par-hip test-hip
+test-par-YES: test-par-YES-hip
+test-par-NO:  test-par-NO-hip
+test-par-YES-hip: test-par-hip test-ser-hip
+test-par-NO-hip:  test-ser-hip
+test-ser-hip: $(SEQ_DEVICE_$(MFEM_TESTS):=-test-seq-hip)
+test-par-hip: $(PAR_DEVICE_$(MFEM_TESTS):=-test-par-hip)
+test-hip: test-par-$(MFEM_USE_MPI)-hip clean-exec
+endif
 test-ser:     test-par-NO
 test-par:     test-par-YES
 test:         all test-par-$(MFEM_USE_MPI) clean-exec

data/inline-pyramid.mesh

@@ -0,0 +1,9 @@
+MFEM INLINE mesh v1.0
+
+type = pyramid
+nx = 4
+ny = 4
+nz = 4
+sx = 1.0
+sy = 1.0
+sz = 1.0

data/octahedron.mesh

@@ -0,0 +1,43 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+# PYRAMID     = 7
+#
+
+dimension
+3
+
+elements
+2
+1 7 4 3 2 1 0
+1 7 1 2 3 4 5
+
+boundary
+8
+1 2 0 2 1
+2 2 0 3 2
+3 2 0 4 3
+4 2 0 1 4
+5 2 1 2 5
+6 2 2 3 5
+7 2 3 4 5
+8 2 4 1 5
+
+vertices
+6
+3
+0 0 -1
+1 0 0
+0 1 0
+-1 0 0
+0 -1 0
+0 0 1

data/ref-pyramid.mesh

@@ -0,0 +1,38 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+# PYRAMID     = 7
+#
+
+dimension
+3
+
+elements
+1
+1 7 0 1 2 3 4
+
+boundary
+5
+1 3 3 2 1 0
+2 2 0 1 4
+3 2 1 2 4
+4 2 2 3 4
+5 2 3 0 4
+
+vertices
+5
+3
+0 0 0
+1 0 0
+1 1 0
+0 1 0
+0 0 1

data/sphere.geo

@@ -0,0 +1,47 @@
+Mesh.Algorithm = 6;
+
+lc = 0.1;
+Point(1) = {0.0,0.0,0.0,lc};
+Point(2) = {1,0.0,0.0,lc};
+Point(3) = {0,1,0.0,lc};
+Circle(1) = {2,1,3};
+Point(4) = {-1,0,0.0,lc};
+Point(5) = {0,-1,0.0,lc};
+Circle(2) = {3,1,4};
+Circle(3) = {4,1,5};
+Circle(4) = {5,1,2};
+Point(6) = {0,0,-1,lc};
+Point(7) = {0,0,1,lc};
+Circle(5) = {3,1,6};
+Circle(6) = {6,1,5};
+Circle(7) = {5,1,7};
+Circle(8) = {7,1,3};
+Circle(9) = {2,1,7};
+Circle(10) = {7,1,4};
+Circle(11) = {4,1,6};
+Circle(12) = {6,1,2};
+Curve Loop(13) = {2,8,-10};
+Surface(14) = {13};
+Curve Loop(15) = {10,3,7};
+Surface(16) = {15};
+Curve Loop(17) = {-8,-9,1};
+Surface(18) = {17};
+Curve Loop(19) = {-11,-2,5};
+Surface(20) = {19};
+Curve Loop(21) = {-5,-12,-1};
+Surface(22) = {21};
+Curve Loop(23) = {-3,11,6};
+Surface(24) = {23};
+Curve Loop(25) = {-7,4,9};
+Surface(26) = {25};
+Curve Loop(27) = {-4,12,-6};
+Surface(28) = {27};
+Surface Loop(29) = {28,26,16,14,20,24,22,18};
+Volume(30) = {29};
+
+Physical Surface(1) = {28,26,16,14,20,24,22,18};
+Physical Volume(2) = 30;
+
+// Generate 2D mesh
+Mesh 2;
+Mesh.MshFileVersion = 2.2;

data/star-q3.fms

@@ -0,0 +1,246 @@
+FMS: 100
+DataCollection/Name: star
+DataCollection/NumberOfFieldDescriptors: 1
+DataCollection/FieldDescriptors/0/Name: CoordsDescriptor
+DataCollection/FieldDescriptors/0/ComponentName: volume
+DataCollection/FieldDescriptors/0/Type: 0
+DataCollection/FieldDescriptors/0/FixedOrder/Size: 3
+DataCollection/FieldDescriptors/0/FixedOrder/Type: FMS_UINT64
+DataCollection/FieldDescriptors/0/FixedOrder/Values: [0, 1, 3]
+DataCollection/FieldDescriptors/0/NumDofs: 211
+DataCollection/NumberOfFields: 1
+DataCollection/Fields/0/Name: Coords
+DataCollection/Fields/0/LayoutType: 0
+DataCollection/Fields/0/NumberOfVectorComponents: 2
+DataCollection/Fields/0/FieldDescriptorName: CoordsDescriptor
+DataCollection/Fields/0/Data/Size: 422
+DataCollection/Fields/0/Data/Type: FMS_DOUBLE
+DataCollection/Fields/0/Data/Values: [-0.016886, 1.000000, 0.309017, 
+1.309020, -0.809017, -0.500000, 
+-0.809017, -1.618030, 0.309017, 
+-0.500000, 1.309020, 0.519420, 
+1.154510, 0.809019, 0.147680, 
+-0.095492, -0.654508, -0.415586, 
+-1.213520, -1.213520, -0.392210, 
+-0.654508, -0.095492, 0.139949, 
+0.809019, 1.154510, 0.660184, 
+-0.264063, -0.800064, -0.231060, 
+0.663691, 0.183114, 0.317639, 
+0.543082, 0.598483, 0.345112, 
+0.478298, 0.027703, 0.095229, 
+0.012368, -0.092534, -0.334412, 
+-0.313767, -0.140526, -0.293881, 
+-0.534056, -0.660290, -0.537646, 
+-0.655590, -0.121396, -0.274504, 
+-0.346497, -0.296570, 0.004737, 
+-0.098835, 0.069287, 0.082675, 
+0.318799, 0.467183, 0.564505, 
+0.595190, 0.846237, 0.671735, 
+1.051500, 1.103010, 0.964008, 
+0.821603, 1.257520, 1.206010, 
+1.142350, 0.975686, 0.781273, 
+0.717257, 0.475684, 0.642352, 
+0.268930, 0.211049, 0.174181, 
+0.039345, -0.147746, -0.177481, 
+-0.365164, -0.230328, -0.551503, 
+-0.603005, -0.497587, -0.389864, 
+-0.757514, -0.706011, -0.675487, 
+-0.528946, -0.943851, -1.078690, 
+-1.087600, -0.955467, -1.483190, 
+-1.348360, -1.483190, -1.348360, 
+-1.085930, -0.938010, -0.943851, 
+-1.078690, -0.681476, -0.540944, 
+-0.757514, -0.706011, -0.540614, 
+-0.367058, -0.551503, -0.603005, 
+-0.365164, -0.230328, -0.138552, 
+-0.206896, 0.174181, 0.039345, 
+0.268468, 0.222269, 0.475684, 
+0.642352, 0.759791, 0.719381, 
+1.142350, 0.975686, 1.257520, 
+1.206010, 0.972837, 0.836119, 
+1.051500, 1.103010, 0.214572, 
+0.407449, 0.288323, 0.449827, 
+-0.086700, -0.027358, -0.200560, 
+-0.166595, -0.271802, -0.418426, 
+-0.426131, -0.551441, -0.096117, 
+-0.206969, -0.027946, -0.184969, 
+0.211136, 0.260131, 0.407172, 
+0.430781, 0.718277, 0.885068, 
+0.753103, 0.957692, 0.866273, 
+1.024530, 0.934099, 1.093820, 
+0.348422, 0.524463, 0.404903, 
+0.587376, 0.054525, 0.146431, 
+-0.078026, -0.007795, -0.329488, 
+-0.302967, -0.488115, -0.439332, 
+-0.498515, -0.453527, -0.633059, 
+-0.570251, -0.655787, -0.791132, 
+-0.802013, -0.956872, -1.094720, 
+-1.208560, -1.207720, -1.339910, 
+-0.693371, -0.795404, -0.803594, 
+-0.945068, -0.464668, -0.631721, 
+-0.458968, -0.547876, -0.332878, 
+-0.485696, -0.286385, -0.408481, 
+0.053021, -0.058200, 0.110846, 
+-0.002086, 0.381384, 0.416784, 
+0.551133, 0.613261, 0.872474, 
+0.901208, 1.038300, 1.084660, 
+0.737459, 0.751250, 0.890002, 
+0.915210, 0.010915, 0.000000, 
+0.951057, 0.951057, 0.587785, 
+1.538840, -0.587785, 0.000000, 
+-0.951057, -1.538840, -0.951057, 
+-0.015847, 0.475529, 0.951057, 
+0.492248, 1.244950, 1.063310, 
+0.274399, 0.293893, -0.293892, 
+-0.296404, -1.063310, -1.244950, 
+-0.453865, -0.951057, -0.475529, 
+0.466620, 0.792932, -0.013913, 
+-0.748783, -0.497528, 0.021382, 
+-0.017158, 0.172591, 0.330125, 
+0.458568, 0.457971, 0.137740, 
+0.299049, 0.588394, 0.667324, 
+0.432341, 0.634346, 0.117322, 
+0.193603, 0.211702, 0.098278, 
+-0.199438, -0.077304, -0.082243, 
+-0.216296, -0.458634, -0.592374, 
+-0.563926, -0.680404, -0.135751, 
+-0.302942, -0.469005, -0.453640, 
+-0.182727, -0.314240, 0.024270, 
+0.021546, 0.158510, 0.317019, 
+0.485799, 0.492951, 0.792548, 
+0.634038, 0.951057, 0.951057, 
+0.777915, 0.613430, 0.951057, 
+0.951057, 0.793994, 0.635800, 
+1.049020, 1.146990, 1.084480, 
+0.924310, 1.440880, 1.342910, 
+1.380330, 1.221820, 0.948209, 
+0.856297, 0.746293, 0.904802, 
+0.476242, 0.393234, 0.489821, 
+0.391857, 0.194471, 0.075751, 
+0.097964, 0.195929, -0.097964, 
+-0.195928, -0.173234, -0.078922, 
+-0.489821, -0.391856, -0.467007, 
+-0.397859, -0.746293, -0.904802, 
+-0.945206, -0.849559, -1.380330, 
+-1.221820, -1.440880, -1.342910, 
+-1.100830, -0.923191, -1.049020, 
+-1.146990, -0.774515, -0.621542, 
+-0.951057, -0.951057, -0.803055, 
+-0.635255, -0.951057, -0.951057, 
+-0.792548, -0.634038, -0.454301, 
+-0.479369, -0.158510, -0.317019, 
+0.149331, 0.178643, 0.295860, 
+0.306275, 0.246225, 0.406610, 
+0.367954, 0.498458, -0.014929, 
+0.084917, -0.094272, -0.000726, 
+-0.245374, -0.340755, -0.435351, 
+-0.490564, -0.176355, -0.294974, 
+-0.148010, -0.328246, 0.136248, 
+0.182883, 0.328957, 0.309903, 
+0.646446, 0.622546, 0.800859, 
+0.801803, 0.643087, 0.654473, 
+0.796963, 0.816799, 0.748189, 
+0.898148, 0.819092, 0.970033, 
+1.049540, 1.162660, 1.115310, 
+1.266700, 0.539960, 0.728396, 
+0.656318, 0.783152, 0.302716, 
+0.403858, 0.219006, 0.309252, 
+0.009667, 0.107161, -0.118211, 
+0.011786, -0.284704, -0.179858, 
+-0.411567, -0.296676, -0.570168, 
+-0.635348, -0.726784, -0.793211, 
+-1.016860, -1.112970, -1.170910, 
+-1.281530, -0.737860, -0.851723, 
+-0.883732, -0.995070, -0.653153, 
+-0.769939, -0.631918, -0.797688, 
+-0.616289, -0.806819, -0.638485, 
+-0.790356, -0.136399, -0.322769, 
+-0.165339, -0.309622]
+DataCollection/Mesh/PartitionInfo/Size: 2
+DataCollection/Mesh/PartitionInfo/Type: FMS_UINT64
+DataCollection/Mesh/PartitionInfo/Values: [0, 1]
+DataCollection/Mesh/NumDomainNames: 1
+DataCollection/Mesh/NumComponents: 1
+DataCollection/Mesh/NumTags: 0
+DataCollection/Mesh/DomainNames/0/Name: Domain
+DataCollection/Mesh/DomainNames/0/NumDomains: 1
+DataCollection/Mesh/DomainNames/0/Domains/0/Dimension: 2
+DataCollection/Mesh/DomainNames/0/Domains/0/NumVertices: 31
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/EntityType: FMS_EDGE
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/NumEntities: 50
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Size: 100
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Type: FMS_INT32
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Values: [11, 0, 26, 
+11, 26, 14, 
+14, 0, 27, 
+14, 27, 17, 
+17, 0, 28, 
+17, 28, 20, 
+20, 0, 29, 
+20, 29, 23, 
+23, 0, 30, 
+23, 30, 11, 
+11, 1, 12, 
+1, 26, 12, 
+12, 3, 13, 
+3, 26, 13, 
+13, 2, 14, 
+2, 15, 2, 
+27, 15, 15, 
+5, 16, 5, 
+27, 16, 16, 
+4, 17, 4, 
+18, 4, 28, 
+18, 18, 7, 
+19, 7, 28, 
+19, 19, 6, 
+20, 6, 21, 
+6, 29, 21, 
+21, 9, 22, 
+9, 29, 22, 
+22, 8, 23, 
+8, 24, 8, 
+30, 24, 24, 
+10, 25, 10, 
+30, 25, 25, 1]
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/EntityType: FMS_QUADRILATERAL
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/NumEntities: 20
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Size: 80
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Type: FMS_INT32
+DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Values: [0, 1, 2, 
+3, 3, 4, 
+5, 6, 6, 
+7, 8, 9, 
+9, 10, 11, 
+12, 12, 13, 
+14, 0, 15, 
+16, 17, 1, 
+17, 18, 19, 
+20, 2, 20, 
+21, 22, 22, 
+23, 24, 4, 
+24, 25, 26, 
+27, 5, 27, 
+28, 29, 29, 
+30, 31, 7, 
+31, 32, 33, 
+34, 8, 34, 
+35, 36, 36, 
+37, 38, 10, 
+38, 39, 40, 
+41, 11, 41, 
+42, 43, 43, 
+44, 45, 13, 
+45, 46, 47, 
+48, 14, 48, 
+49, 15]
+DataCollection/Mesh/Components/0/Name: volume
+DataCollection/Mesh/Components/0/Dimension: 2
+DataCollection/Mesh/Components/0/NumEntities: 20
+DataCollection/Mesh/Components/0/Coordinates: Coords
+DataCollection/Mesh/Components/0/NumParts: 1
+DataCollection/Mesh/Components/0/Parts/0/DomainName: Domain
+DataCollection/Mesh/Components/0/Parts/0/DomainID: 0
+DataCollection/Mesh/Components/0/Parts/0/FullDomain: Yes
+DataCollection/Mesh/Components/0/Relations/Size: 0
+DataCollection/Mesh/Components/0/Relations/Type: FMS_UINT64

doc/CodeDocumentation.conf.in

@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = v4.2.1
+PROJECT_NUMBER         = v4.3.1
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

examples/CMakeLists.txt

@@ -84,8 +84,9 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
   get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
   string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
 
-  if (NOT (${TEST_NAME} MATCHES "ex0p?"))
-    set(THIS_TEST_OPTIONS "-no-vis")
+  set(THIS_TEST_OPTIONS "-no-vis")
+  if (${TEST_NAME} MATCHES "ex0p?")
+    set(THIS_TEST_OPTIONS)
   endif()
   if (${TEST_NAME} MATCHES "ex10p*")
     list(APPEND THIS_TEST_OPTIONS "-tf" "5")
@@ -107,6 +108,34 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
   endif()
 endforeach()
 
+# Add CUDA/HIP tests.
+set(DEVICE_EXAMPLES
+    # serial examples with device support:
+    ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
+    # parallel examples with device support:
+    ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p ex24p ex25p ex26p)
+set(MFEM_TEST_DEVICE)
+if (MFEM_USE_CUDA)
+  set(MFEM_TEST_DEVICE "cuda")
+elseif (MFEM_USE_HIP)
+  set(MFEM_TEST_DEVICE "hip")
+endif()
+if (MFEM_TEST_DEVICE)
+  foreach(TEST_NAME ${DEVICE_EXAMPLES})
+    set(THIS_TEST_OPTIONS "-no-vis" "-d" "${MFEM_TEST_DEVICE}")
+    if (NOT (${TEST_NAME} MATCHES ".*p$"))
+      add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_ser
+        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+    elseif (MFEM_USE_MPI)
+      add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_np=${MFEM_MPI_NP}
+        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
+        ${MPIEXEC_PREFLAGS}
+        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+        ${MPIEXEC_POSTFLAGS})
+    endif()
+  endforeach()
+endif()
+
 # If STRUMPACK is enabled, add a test run that uses it.
 if (MFEM_USE_STRUMPACK)
   add_test(NAME ex11p_strumpack_np=${MFEM_MPI_NP}

examples/arpack/ex11.cpp

@@ -0,0 +1,286 @@
+//                       MFEM Example 11 - Serial Version
+//
+// Compile with: make ex11
+//
+// Sample runs:  ex11 -m ../data/square-disc.mesh
+//               ex11 -m ../data/star.mesh
+//               ex11 -m ../data/star-mixed.mesh
+//               ex11 -m ../data/periodic-annulus-sector.msh
+//               ex11 -m ../data/square-disc-p2.vtk -o 2
+//               ex11 -m ../data/square-disc-p3.mesh -o 3
+//               ex11 -m ../data/square-disc-nurbs.mesh -o -1
+//               ex11 -m ../data/disc-nurbs.mesh -o -1 -n 20
+//               ex11 -m ../data/star-surf.mesh
+//               ex11 -m ../data/square-disc-surf.mesh
+//               ex11 -m ../data/inline-segment.mesh
+//               ex11 -m ../data/inline-quad.mesh
+//               ex11 -m ../data/inline-tri.mesh
+//               ex11 -m ../data/amr-quad.mesh
+//               ex11 -m ../data/amr-hex.mesh
+//               ex11 -m ../data/mobius-strip.mesh -n 8
+//
+// Description:  This example code demonstrates the use of MFEM to solve the
+//               eigenvalue problem -Delta u = lambda u with homogeneous
+//               Dirichlet boundary conditions.
+//
+//               We compute a number of the lowest eigenmodes by discretizing
+//               the Laplacian and Mass operators using a FE space of the
+//               specified order, or an isoparametric/isogeometric space if
+//               order < 1 (quadratic for quadratic curvilinear mesh, NURBS for
+//               NURBS mesh, etc.)
+//
+//               The example highlights the use of the ARPACK eigenvalue solver
+//               (regular inverse mode). Reusing a single GLVis visualization
+//               window for multiple eigenfunctions is also illustrated.
+//
+//               We recommend viewing Example 1 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../data/star.mesh";
+   int ser_ref_levels = 3;
+   int order = 1;
+   int nev = 5;
+   double dbc_eig = 1e3;
+   bool visualization = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&nev, "-n", "--num-eigs",
+                  "Number of desired eigenmodes.");
+   args.AddOption(&dbc_eig, "-d", "--dbc-eig",
+                  "Eigenvalues associated with Dirichlet BC "
+                  "(should be larger than the maximum desired eigenvalue).");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 2. Read the (serial) mesh from the given mesh file on all processors. We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh;
+   ifstream imesh(mesh_file);
+   if (!imesh)
+   {
+      cerr << "\nCan not open mesh file: " << mesh_file << '\n' << endl;
+      return 2;
+   }
+   mesh = new Mesh(imesh, 1, 1);
+   imesh.close();
+   int dim = mesh->Dimension();
+
+   // 3. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement (2 by default, or
+   //    specified on the command line with -rs).
+   for (int lev = 0; lev < ser_ref_levels; lev++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 4. Define a finite element space on the mesh. Here we
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+   }
+   else if (mesh->GetNodes())
+   {
+      fec = mesh->GetNodes()->OwnFEC();
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+   int size = fespace->GetVSize();
+
+   cout << "Number of unknowns: " << size << endl;
+
+   // 5. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
+   //    element space. The first corresponds to the Laplacian operator -Delta,
+   //    while the second is a simple mass matrix needed on the right hand side
+   //    of the generalized eigenvalue problem below. The boundary conditions
+   //    are implemented by elimination with special values on the diagonal to
+   //    shift the Dirichlet eigenvalues out of the computational range. After
+   //    serial and parallel assembly we extract the corresponding parallel
+   //    matrices A and M.
+   ConstantCoefficient one(1.0);
+   Array<int> ess_bdr;
+   if (mesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+   }
+
+   BilinearForm *a = new BilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+   if (mesh->bdr_attributes.Size() == 0)
+   {
+      // Add a mass term if the mesh has no boundary, e.g. periodic mesh or
+      // closed surface.
+      a->AddDomainIntegrator(new MassIntegrator(one));
+   }
+   a->Assemble();
+   if (mesh->bdr_attributes.Size() != 0)
+   {
+      a->EliminateEssentialBCDiag(ess_bdr, dbc_eig);
+   }
+   a->Finalize();
+
+   BilinearForm *m = new BilinearForm(fespace);
+   m->AddDomainIntegrator(new MassIntegrator(one));
+   m->Assemble();
+   if (mesh->bdr_attributes.Size() != 0)
+   {
+      // shift the eigenvalue corresponding to eliminated dofs to a large value
+      m->EliminateEssentialBCDiag(ess_bdr, 1.0);
+   }
+   m->Finalize();
+
+   // 6. Define and configure the ARPACK eigensolver
+   ArPackSym * arpack = new ArPackSym();
+   Solver    * solver = NULL;
+
+#ifndef MFEM_USE_SUITESPARSE
+   // 7. Define a simple symmetric Gauss-Seidel preconditioner and use it to
+   //    solve the system A X = B with PCG.
+   cout << "Building CGSolver" << endl;
+   GSSmoother M(m->SpMat());
+   CGSolver * cg_solver = new CGSolver;
+   cg_solver->SetPreconditioner(M);
+   cg_solver->SetRelTol(1.0e-12);
+   solver = cg_solver;
+#else
+   // 7. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+   cout << "Building UMFPackSolver" << endl;
+   UMFPackSolver * umf_solver = new UMFPackSolver;
+   umf_solver->Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+   solver = umf_solver;
+#endif
+   solver->SetOperator(m->SpMat());
+
+   arpack->SetNumModes(nev);
+   arpack->SetMaxIter(400);
+   arpack->SetTol(1e-8);
+   arpack->SetMode(2);
+   arpack->SetPrintLevel(2);
+
+   arpack->SetOperator(*a);
+   arpack->SetMassMatrix(*m);
+   arpack->SetSolver(*solver);
+
+   // 8. Compute the eigenmodes and extract the array of eigenvalues. Define a
+   //    parallel grid function to represent each of the eigenmodes returned by
+   //    the solver.
+   Array<double> eigenvalues;
+   arpack->Solve();
+   arpack->GetEigenvalues(eigenvalues);
+
+   cout << endl;
+   std::ios::fmtflags old_fmt = cout.flags();
+   cout.setf(std::ios::scientific);
+   std::streamsize old_prec = cout.precision(14);
+   for (int i=0; i<nev; i++)
+   {
+      cout << "Eigenvalue lambda   " << eigenvalues[i] << endl;
+   }
+   cout.precision(old_prec);
+   cout.flags(old_fmt);
+   cout << endl;
+
+   GridFunction x(fespace);
+
+   // 9. Save the refined mesh and the modes in parallel. This output can be
+   //    viewed later using GLVis: "glvis -np <np> -m mesh -g mode".
+   {
+      ostringstream mesh_name, mode_name;
+      mesh_name << "ex11.mesh";
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      mesh->Print(mesh_ofs);
+
+      for (int i=0; i<nev; i++)
+      {
+         // convert eigenvector from HypreParVector to ParGridFunction
+         x = arpack->GetEigenvector(i);
+
+         mode_name << "mode_" << setfill('0') << setw(2) << i;
+
+         ofstream mode_ofs(mode_name.str().c_str());
+         mode_ofs.precision(8);
+         x.Save(mode_ofs);
+         mode_name.str("");
+      }
+   }
+
+   // 10. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream mode_sock(vishost, visport);
+      mode_sock.precision(8);
+
+      for (int i=0; i<nev; i++)
+      {
+         cout << "Eigenmode " << i+1 << '/' << nev
+              << ", Lambda = " << eigenvalues[i] << endl;
+
+         // convert eigenvector from HypreParVector to ParGridFunction
+         x = arpack->GetEigenvector(i);
+
+         mode_sock << "solution\n" << *mesh << x << flush
+                   << "window_title 'Eigenmode " << i+1 << '/' << nev
+                   << ", Lambda = " << eigenvalues[i] << "'" << endl;
+
+         char c;
+         cout << "press (q)uit or (c)ontinue --> " << flush;
+         cin >> c;
+
+         if (c != 'c')
+         {
+            break;
+         }
+      }
+      mode_sock.close();
+   }
+
+   // 11. Free the used memory.
+   delete arpack;
+   delete solver;
+   delete m;
+   delete a;
+
+   delete fespace;
+   if (order > 0)
+   {
+      delete fec;
+   }
+   delete mesh;
+
+   return 0;
+}

examples/arpack/makefile

@@ -0,0 +1,69 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../..
+MFEM_BUILD_DIR ?= ../..
+SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/arpack/,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = ex11
+PAR_EXAMPLES = 
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES)
+endif
+RC_FILES = $(patsubst $(SRC)%,%,$(wildcard $(SRC)rc_*))
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES)
+
+# Examples depend on their corresponding rc_* files:
+make-rc-rule = $(1): | $(filter rc_$(1)%,$(RC_FILES))
+$(foreach ex,$(EXAMPLES),$(eval $(call make-rc-rule,$(ex))))
+
+# Rules to copy the rc_* files when building out-of-source:
+ifneq ($(SRC),)
+$(RC_FILES): %: $(SRC)%
+	cp -pf $(<) .
+endif
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean: clean-build clean-exec
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:
+	@rm -rf mesh.* sol.* sol_p.* sol_u.* Example5*
+	@rm -f ex9-mesh.* ex9-init.* ex9-final.* Example9*
+	@rm -f deformed.* velocity.* elastic_energy.*

examples/caliper/README

@@ -0,0 +1,18 @@
+                    Finite Element Discretization Library
+                                   __
+                       _ __ ___   / _|  ___  _ __ ___
+                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
+                      | | | | | ||  _||  __/| | | | | |
+                      |_| |_| |_||_|   \___||_| |_| |_|
+
+                               https://mfem.org
+
+This directory contains modifications of the example codes that illustrate the
+use of MFEM features based on the Caliper performance profiling library.
+
+To build these examples, make sure that MFEM is configured with the option
+"MFEM_USE_CALIPER = YES", see the top-level INSTALL file for details (version
+2.5.0 of Caliper is recommended, though older versions may work too).
+
+We recommend comparing the original example codes with the corresponding files
+in the current directory.

examples/caliper/ex1.cpp

@@ -206,9 +206,9 @@ int main(int argc, char *argv[])
    cout << "Size of linear system: " << A->Height() << endl;
 
    // 11. Solve the linear system A X = B.
-   MFEM_PERF_BEGIN("Solve A X=B");
    if (!pa)
    {
+      MFEM_PERF_SCOPE("Solve A X=B (FA)");
 #ifndef MFEM_USE_SUITESPARSE
       // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
       GSSmoother M((SparseMatrix&)(*A));
@@ -223,6 +223,7 @@ int main(int argc, char *argv[])
    }
    else // Jacobi preconditioning in partial assembly mode
    {
+      MFEM_PERF_SCOPE("Solve A X=B (PA)");
       if (UsesTensorBasis(fespace))
       {
          OperatorJacobiSmoother M(a, ess_tdof_list);
@@ -233,7 +234,6 @@ int main(int argc, char *argv[])
          CG(*A, B, X, 1, 400, 1e-12, 0.0);
       }
    }
-   MFEM_PERF_END("Solve A X=B");
    // 12. Recover the solution as a finite element grid function.
    a.RecoverFEMSolution(X, b, x);
 

examples/caliper/ex1p.cpp

@@ -231,28 +231,29 @@ int main(int argc, char *argv[])
    // 13. Solve the linear system A X = B.
    //     * With full assembly, use the BoomerAMG preconditioner from hypre.
    //     * With partial assembly, use Jacobi smoothing, for now.
-   MFEM_PERF_BEGIN("Solve A X = B");
-   Solver *prec = NULL;
-   if (pa)
    {
-      if (UsesTensorBasis(fespace))
+      MFEM_PERF_SCOPE("Solve A X=B");
+      Solver *prec = NULL;
+      if (pa)
       {
-         prec = new OperatorJacobiSmoother(a, ess_tdof_list);
+         if (UsesTensorBasis(fespace))
+         {
+            prec = new OperatorJacobiSmoother(a, ess_tdof_list);
+         }
       }
+      else
+      {
+         prec = new HypreBoomerAMG;
+      }
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(2000);
+      cg.SetPrintLevel(1);
+      if (prec) { cg.SetPreconditioner(*prec); }
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      delete prec;
    }
-   else
-   {
-      prec = new HypreBoomerAMG;
-   }
-   CGSolver cg(MPI_COMM_WORLD);
-   cg.SetRelTol(1e-12);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(1);
-   if (prec) { cg.SetPreconditioner(*prec); }
-   cg.SetOperator(*A);
-   cg.Mult(B, X);
-   delete prec;
-   MFEM_PERF_END("Solve A X = B");
    // 14. Recover the parallel grid function corresponding to X. This is the
    //     local finite element solution on each processor.
    a.RecoverFEMSolution(X, b, x);

examples/ex1.cpp

@@ -9,6 +9,7 @@
 //               ex1 -m ../data/fichera.mesh
 //               ex1 -m ../data/fichera-mixed.mesh
 //               ex1 -m ../data/toroid-wedge.mesh
+//               ex1 -m ../data/octahedron.mesh -o 1
 //               ex1 -m ../data/periodic-annulus-sector.msh
 //               ex1 -m ../data/periodic-torus-sector.msh
 //               ex1 -m ../data/square-disc-p2.vtk -o 2

examples/ex13p.cpp

@@ -55,6 +55,7 @@ int main(int argc, char *argv[])
    int order = 1;
    int nev = 5;
    bool visualization = 1;
+   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -71,6 +72,8 @@ int main(int argc, char *argv[])
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (!args.Good())
    {
@@ -86,13 +89,18 @@ int main(int argc, char *argv[])
       args.PrintOptions(cout);
    }
 
-   // 3. Read the (serial) mesh from the given mesh file on all processors. We
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   if (myid == 0) { device.Print(); }
+
+   // 4. Read the (serial) mesh from the given mesh file on all processors. We
    //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
    //    and volume meshes with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
-   // 4. Refine the serial mesh on all processors to increase the resolution. In
+   // 5. Refine the serial mesh on all processors to increase the resolution. In
    //    this example we do 'ref_levels' of uniform refinement (2 by default, or
    //    specified on the command line with -rs).
    for (int lev = 0; lev < ser_ref_levels; lev++)
@@ -100,7 +108,7 @@ int main(int argc, char *argv[])
       mesh->UniformRefinement();
    }
 
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution (1 time by
    //    default, or specified on the command line with -rp). Once the parallel
    //    mesh is defined, the serial mesh can be deleted.
@@ -110,9 +118,8 @@ int main(int argc, char *argv[])
    {
       pmesh->UniformRefinement();
    }
-   pmesh->ReorientTetMesh();
 
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
+   // 7. Define a parallel finite element space on the parallel mesh. Here we
    //    use the Nedelec finite elements of the specified order.
    FiniteElementCollection *fec = new ND_FECollection(order, dim);
    ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
@@ -122,7 +129,7 @@ int main(int argc, char *argv[])
       cout << "Number of unknowns: " << size << endl;
    }
 
-   // 7. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
+   // 8. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
    //    element space. The first corresponds to the curl curl, while the second
    //    is a simple mass matrix needed on the right hand side of the
    //    generalized eigenvalue problem below. The boundary conditions are
@@ -164,7 +171,7 @@ int main(int argc, char *argv[])
    delete a;
    delete m;
 
-   // 8. Define and configure the AME eigensolver and the AMS preconditioner for
+   // 9. Define and configure the AME eigensolver and the AMS preconditioner for
    //    A to be used within the solver. Set the matrices which define the
    //    generalized eigenproblem A x = lambda M x.
    HypreAMS *ams = new HypreAMS(*A,fespace);
@@ -180,15 +187,15 @@ int main(int argc, char *argv[])
    ame->SetMassMatrix(*M);
    ame->SetOperator(*A);
 
-   // 9. Compute the eigenmodes and extract the array of eigenvalues. Define a
-   //    parallel grid function to represent each of the eigenmodes returned by
-   //    the solver.
+   // 10. Compute the eigenmodes and extract the array of eigenvalues. Define a
+   //     parallel grid function to represent each of the eigenmodes returned by
+   //     the solver.
    Array<double> eigenvalues;
    ame->Solve();
    ame->GetEigenvalues(eigenvalues);
    ParGridFunction x(fespace);
 
-   // 10. Save the refined mesh and the modes in parallel. This output can be
+   // 11. Save the refined mesh and the modes in parallel. This output can be
    //     viewed later using GLVis: "glvis -np <np> -m mesh -g mode".
    {
       ostringstream mesh_name, mode_name;
@@ -213,7 +220,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 11. Send the solution by socket to a GLVis server.
+   // 12. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       char vishost[] = "localhost";
@@ -253,7 +260,7 @@ int main(int argc, char *argv[])
       mode_sock.close();
    }
 
-   // 12. Free the used memory.
+   // 13. Free the used memory.
    delete ame;
    delete ams;
    delete M;

examples/ex16.cpp

@@ -24,7 +24,10 @@
 //               class ConductionOperator defining C(u)), as well as their
 //               implicit time integration. Note that implementing the method
 //               ConductionOperator::ImplicitSolve is the only requirement for
-//               high-order implicit (SDIRK) time integration.
+//               high-order implicit (SDIRK) time integration. In this example,
+//               the diffusion operator is linearized by evaluating with the
+//               lagged solution from the previous timestep, so there is only
+//               a linear solve.
 //
 //               We recommend viewing examples 2, 9 and 10 before viewing this
 //               example.
@@ -326,8 +329,8 @@ ConductionOperator::ConductionOperator(FiniteElementSpace &f, double al,
 void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
 {
    // Compute:
-   //    du_dt = M^{-1}*-K(u)
-   // for du_dt
+   //    du_dt = M^{-1}*-Ku
+   // for du_dt, where K is linearized by using u from the previous timestep
    Kmat.Mult(u, z);
    z.Neg(); // z = -z
    M_solver.Mult(z, du_dt);
@@ -338,7 +341,7 @@ void ConductionOperator::ImplicitSolve(const double dt,
 {
    // Solve the equation:
    //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
-   // for du_dt
+   // for du_dt, where K is linearized by using u from the previous timestep
    if (!T)
    {
       T = Add(1.0, Mmat, dt, Kmat);

examples/ex16p.cpp

@@ -24,8 +24,11 @@
 //               class ConductionOperator defining C(u)), as well as their
 //               implicit time integration. Note that implementing the method
 //               ConductionOperator::ImplicitSolve is the only requirement for
-//               high-order implicit (SDIRK) time integration. Optional saving
-//               with ADIOS2 (adios2.readthedocs.io) is also illustrated.
+//               high-order implicit (SDIRK) time integration. In this example,
+//               the diffusion operator is linearized by evaluating with the
+//               lagged solution from the previous timestep, so there is only
+//               a linear solve. Optional saving with ADIOS2
+//               (adios2.readthedocs.io) is also illustrated.
 //
 //               We recommend viewing examples 2, 9 and 10 before viewing this
 //               example.
@@ -420,8 +423,8 @@ ConductionOperator::ConductionOperator(ParFiniteElementSpace &f, double al,
 void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
 {
    // Compute:
-   //    du_dt = M^{-1}*-K(u)
-   // for du_dt
+   //    du_dt = M^{-1}*-Ku
+   // for du_dt, where K is linearized by using u from the previous timestep
    Kmat.Mult(u, z);
    z.Neg(); // z = -z
    M_solver.Mult(z, du_dt);
@@ -432,7 +435,7 @@ void ConductionOperator::ImplicitSolve(const double dt,
 {
    // Solve the equation:
    //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
-   // for du_dt
+   // for du_dt, where K is linearized by using u from the previous timestep
    if (!T)
    {
       T = Add(1.0, Mmat, dt, Kmat);

examples/ex19p.cpp

@@ -196,6 +196,12 @@ void InitialDeformation(const Vector &x, Vector &y);
 
 int main(int argc, char *argv[])
 {
+#ifdef HYPRE_USING_CUDA
+   cout << "\nAs of mfem-4.3 and hypre-2.22.0 (July 2021) this example\n"
+        << "is NOT supported with the CUDA version of hypre.\n\n";
+   return 255;
+#endif
+
    // 1. Initialize MPI
    MPI_Session mpi;
    const int myid = mpi.WorldRank();
@@ -438,16 +444,20 @@ JacobianPreconditioner::JacobianPreconditioner(Array<ParFiniteElementSpace *>
 void JacobianPreconditioner::Mult(const Vector &k, Vector &y) const
 {
    // Extract the blocks from the input and output vectors
-   Vector disp_in(k.GetData() + block_trueOffsets[0],
-                  block_trueOffsets[1]-block_trueOffsets[0]);
-   Vector pres_in(k.GetData() + block_trueOffsets[1],
-                  block_trueOffsets[2]-block_trueOffsets[1]);
-
-   Vector disp_out(y.GetData() + block_trueOffsets[0],
+   Vector disp_in;
+   disp_in.MakeRef(const_cast<Vector&>(k), block_trueOffsets[0],
                    block_trueOffsets[1]-block_trueOffsets[0]);
-   Vector pres_out(y.GetData() + block_trueOffsets[1],
+   Vector pres_in;
+   pres_in.MakeRef(const_cast<Vector&>(k), block_trueOffsets[1],
                    block_trueOffsets[2]-block_trueOffsets[1]);
 
+   Vector disp_out;
+   disp_out.MakeRef(y, block_trueOffsets[0],
+                    block_trueOffsets[1]-block_trueOffsets[0]);
+   Vector pres_out;
+   pres_out.MakeRef(y, block_trueOffsets[1],
+                    block_trueOffsets[2]-block_trueOffsets[1]);
+
    Vector temp(block_trueOffsets[1]-block_trueOffsets[0]);
    Vector temp2(block_trueOffsets[1]-block_trueOffsets[0]);
 
@@ -459,6 +469,9 @@ void JacobianPreconditioner::Mult(const Vector &k, Vector &y) const
    subtract(disp_in, temp, temp2);
 
    stiff_pcg->Mult(temp2, disp_out);
+
+   disp_out.SyncAliasMemory(y);
+   pres_out.SyncAliasMemory(y);
 }
 
 void JacobianPreconditioner::SetOperator(const Operator &op)
@@ -473,7 +486,10 @@ void JacobianPreconditioner::SetOperator(const Operator &op)
 
       if (!spaces[0]->GetParMesh()->Nonconforming())
       {
+#ifndef HYPRE_USING_CUDA
+         // Not available yet when hypre is built with CUDA
          stiff_prec_amg->SetElasticityOptions(spaces[0]);
+#endif
       }
 
       stiff_prec = stiff_prec_amg;

examples/ex1p.cpp

@@ -9,6 +9,7 @@
 //               mpirun -np 4 ex1p -m ../data/fichera.mesh
 //               mpirun -np 4 ex1p -m ../data/fichera-mixed.mesh
 //               mpirun -np 4 ex1p -m ../data/toroid-wedge.mesh
+//               mpirun -np 4 ex1p -m ../data/octahedron.mesh -o 1
 //               mpirun -np 4 ex1p -m ../data/periodic-annulus-sector.msh
 //               mpirun -np 4 ex1p -m ../data/periodic-torus-sector.msh
 //               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
@@ -89,7 +90,8 @@ int main(int argc, char *argv[])
    args.AddOption(&device_config, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
 #ifdef MFEM_USE_CEED
-   args.AddOption(&algebraic_ceed, "-a", "--algebraic", "-no-a", "--no-algebraic",
+   args.AddOption(&algebraic_ceed, "-a", "--algebraic",
+                  "-no-a", "--no-algebraic",
                   "Use algebraic Ceed solver");
 #endif
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -197,15 +199,15 @@ int main(int argc, char *argv[])
    b.AddDomainIntegrator(new DomainLFIntegrator(one));
    b.Assemble();
 
-   // 10. Define the solution vector x as a parallel finite element grid function
-   //     corresponding to fespace. Initialize x with initial guess of zero,
-   //     which satisfies the boundary conditions.
+   // 10. Define the solution vector x as a parallel finite element grid
+   //     function corresponding to fespace. Initialize x with initial guess of
+   //     zero, which satisfies the boundary conditions.
    ParGridFunction x(&fespace);
    x = 0.0;
 
    // 11. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //     domain integrator.
+   //     corresponding to the Laplacian operator -Delta, by adding the
+   //     Diffusion domain integrator.
    ParBilinearForm a(&fespace);
    if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
    a.AddDomainIntegrator(new DiffusionIntegrator(one));

examples/ex22.cpp

@@ -13,6 +13,8 @@
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 1
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 2
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 2 -pa
+//               ex22 -m ../data/inline-wedge.mesh -o 1
+//               ex22 -m ../data/inline-pyramid.mesh -o 1
 //               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0
 //
 // Device sample runs:

examples/ex22p.cpp

@@ -13,6 +13,8 @@
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 1
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 2
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 1 -p 2 -pa
+//               mpirun -np 4 ex22p -m ../data/inline-wedge.mesh -o 1
+//               mpirun -np 4 ex22p -m ../data/inline-pyramid.mesh -o 1
 //               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0
 //
 // Device sample runs:

examples/ex24.cpp

@@ -113,7 +113,6 @@ int main(int argc, char *argv[])
          mesh->UniformRefinement();
       }
    }
-   mesh->ReorientTetMesh();
 
    // 5. Define a finite element space on the mesh. Here we use Nedelec or
    //    Raviart-Thomas finite elements of the specified order.

examples/ex24p.cpp

@@ -141,7 +141,6 @@ int main(int argc, char *argv[])
          pmesh->UniformRefinement();
       }
    }
-   pmesh->ReorientTetMesh();
 
    // 7. Define a parallel finite element space on the parallel mesh. Here we
    //    use Nedelec or Raviart-Thomas finite elements of the specified order.

examples/ex25.cpp

@@ -277,10 +277,8 @@ int main(int argc, char *argv[])
       mesh->UniformRefinement();
    }
 
-   // 6. Reorient mesh in case of a tet mesh
-   mesh->ReorientTetMesh();
-
-   // Set element attributes in order to distinguish elements in the PML region
+   // 6. Set element attributes in order to distinguish elements in the
+   //    PML region
    pml->SetAttributes(mesh);
 
    // 7. Define a finite element space on the mesh. Here we use the Nedelec

examples/ex25p.cpp

@@ -316,9 +316,6 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 7a. Reorient mesh in case of a tet mesh
-   pmesh->ReorientTetMesh();
-
    // 8. Set element attributes in order to distinguish elements in the PML
    pml->SetAttributes(pmesh);
 

examples/ex28p.cpp

@@ -81,6 +81,12 @@ Mesh * build_trapezoid_mesh(double offset)
 
 int main(int argc, char *argv[])
 {
+#ifdef HYPRE_USING_CUDA
+   cout << "\nAs of mfem-4.3 and hypre-2.22.0 (July 2021) this example\n"
+        << "is NOT supported with the CUDA version of hypre.\n\n";
+   return 255;
+#endif
+
    // 1. Initialize MPI.
    int num_procs, myid;
    MPI_Init(&argc, &argv);
@@ -360,6 +366,7 @@ int main(int argc, char *argv[])
    }
    delete pmesh;
 
+   // HYPRE_Finalize();
    MPI_Finalize();
 
    return 0;

examples/ex2p.cpp

@@ -61,6 +61,7 @@ int main(int argc, char *argv[])
    bool visualization = 1;
    bool amg_elast = 0;
    bool reorder_space = false;
+   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -78,6 +79,8 @@ int main(int argc, char *argv[])
                   "Enable or disable GLVis visualization.");
    args.AddOption(&reorder_space, "-nodes", "--by-nodes", "-vdim", "--by-vdim",
                   "Use byNODES ordering of vector space instead of byVDIM");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (!args.Good())
    {
@@ -93,7 +96,12 @@ int main(int argc, char *argv[])
       args.PrintOptions(cout);
    }
 
-   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   if (myid == 0) { device.Print(); }
+
+   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
    //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
    //    and volume meshes with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
@@ -109,14 +117,14 @@ int main(int argc, char *argv[])
       return 3;
    }
 
-   // 4. Select the order of the finite element discretization space. For NURBS
+   // 5. Select the order of the finite element discretization space. For NURBS
    //    meshes, we increase the order by degree elevation.
    if (mesh->NURBSext)
    {
       mesh->DegreeElevate(order, order);
    }
 
-   // 5. Refine the serial mesh on all processors to increase the resolution. In
+   // 6. Refine the serial mesh on all processors to increase the resolution. In
    //    this example we do 'ref_levels' of uniform refinement. We choose
    //    'ref_levels' to be the largest number that gives a final mesh with no
    //    more than 1,000 elements.
@@ -129,7 +137,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 7. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution. Once the
    //    parallel mesh is defined, the serial mesh can be deleted.
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -142,7 +150,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   // 8. Define a parallel finite element space on the parallel mesh. Here we
    //    use vector finite elements, i.e. dim copies of a scalar finite element
    //    space. We use the ordering by vector dimension (the last argument of
    //    the FiniteElementSpace constructor) which is expected in the systems
@@ -175,7 +183,7 @@ int main(int argc, char *argv[])
            << "Assembling: " << flush;
    }
 
-   // 8. Determine the list of true (i.e. parallel conforming) essential
+   // 9. Determine the list of true (i.e. parallel conforming) essential
    //    boundary dofs. In this example, the boundary conditions are defined by
    //    marking only boundary attribute 1 from the mesh as essential and
    //    converting it to a list of true dofs.
@@ -184,14 +192,14 @@ int main(int argc, char *argv[])
    ess_bdr[0] = 1;
    fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
 
-   // 9. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system. In this case, b_i equals the
-   //    boundary integral of f*phi_i where f represents a "pull down" force on
-   //    the Neumann part of the boundary and phi_i are the basis functions in
-   //    the finite element fespace. The force is defined by the object f, which
-   //    is a vector of Coefficient objects. The fact that f is non-zero on
-   //    boundary attribute 2 is indicated by the use of piece-wise constants
-   //    coefficient for its last component.
+   // 10. Set up the parallel linear form b(.) which corresponds to the
+   //     right-hand side of the FEM linear system. In this case, b_i equals the
+   //     boundary integral of f*phi_i where f represents a "pull down" force on
+   //     the Neumann part of the boundary and phi_i are the basis functions in
+   //     the finite element fespace. The force is defined by the object f, which
+   //     is a vector of Coefficient objects. The fact that f is non-zero on
+   //     boundary attribute 2 is indicated by the use of piece-wise constants
+   //     coefficient for its last component.
    VectorArrayCoefficient f(dim);
    for (int i = 0; i < dim-1; i++)
    {
@@ -212,13 +220,13 @@ int main(int argc, char *argv[])
    }
    b->Assemble();
 
-   // 10. Define the solution vector x as a parallel finite element grid
+   // 11. Define the solution vector x as a parallel finite element grid
    //     function corresponding to fespace. Initialize x with initial guess of
    //     zero, which satisfies the boundary conditions.
    ParGridFunction x(fespace);
    x = 0.0;
 
-   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
+   // 12. Set up the parallel bilinear form a(.,.) on the finite element space
    //     corresponding to the linear elasticity integrator with piece-wise
    //     constants coefficient lambda and mu.
    Vector lambda(pmesh->attributes.Max());
@@ -233,7 +241,7 @@ int main(int argc, char *argv[])
    ParBilinearForm *a = new ParBilinearForm(fespace);
    a->AddDomainIntegrator(new ElasticityIntegrator(lambda_func, mu_func));
 
-   // 12. Assemble the parallel bilinear form and the corresponding linear
+   // 13. Assemble the parallel bilinear form and the corresponding linear
    //     system, applying any necessary transformations such as: parallel
    //     assembly, eliminating boundary conditions, applying conforming
    //     constraints for non-conforming AMR, static condensation, etc.
@@ -250,7 +258,7 @@ int main(int argc, char *argv[])
       cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
    }
 
-   // 13. Define and apply a parallel PCG solver for A X = B with the BoomerAMG
+   // 14. Define and apply a parallel PCG solver for A X = B with the BoomerAMG
    //     preconditioner from hypre.
    HypreBoomerAMG *amg = new HypreBoomerAMG(A);
    if (amg_elast && !a->StaticCondensationIsEnabled())
@@ -268,11 +276,11 @@ int main(int argc, char *argv[])
    pcg->SetPreconditioner(*amg);
    pcg->Mult(B, X);
 
-   // 14. Recover the parallel grid function corresponding to X. This is the
+   // 15. Recover the parallel grid function corresponding to X. This is the
    //     local finite element solution on each processor.
    a->RecoverFEMSolution(X, *b, x);
 
-   // 15. For non-NURBS meshes, make the mesh curved based on the finite element
+   // 16. For non-NURBS meshes, make the mesh curved based on the finite element
    //     space. This means that we define the mesh elements through a fespace
    //     based transformation of the reference element.  This allows us to save
    //     the displaced mesh as a curved mesh when using high-order finite
@@ -284,7 +292,7 @@ int main(int argc, char *argv[])
       pmesh->SetNodalFESpace(fespace);
    }
 
-   // 16. Save in parallel the displaced mesh and the inverted solution (which
+   // 17. Save in parallel the displaced mesh and the inverted solution (which
    //     gives the backward displacements to the original grid). This output
    //     can be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
    {
@@ -305,7 +313,7 @@ int main(int argc, char *argv[])
       x.Save(sol_ofs);
    }
 
-   // 17. Send the above data by socket to a GLVis server.  Use the "n" and "b"
+   // 18. Send the above data by socket to a GLVis server.  Use the "n" and "b"
    //     keys in GLVis to visualize the displacements.
    if (visualization)
    {
@@ -317,7 +325,7 @@ int main(int argc, char *argv[])
       sol_sock << "solution\n" << *pmesh << x << flush;
    }
 
-   // 18. Free the used memory.
+   // 19. Free the used memory.
    delete pcg;
    delete amg;
    delete a;

examples/ex3.cpp

@@ -16,6 +16,8 @@
 //               ex3 -m ../data/beam-hex-nurbs.mesh
 //               ex3 -m ../data/amr-hex.mesh
 //               ex3 -m ../data/fichera-amr.mesh
+//               ex3 -m ../data/ref-prism.mesh -o 1
+//               ex3 -m ../data/octahedron.mesh -o 1
 //               ex3 -m ../data/star-surf.mesh -o 1
 //               ex3 -m ../data/mobius-strip.mesh -f 0.1
 //               ex3 -m ../data/klein-bottle.mesh -f 0.1
@@ -113,7 +115,6 @@ int main(int argc, char *argv[])
          mesh->UniformRefinement();
       }
    }
-   mesh->ReorientTetMesh();
 
    // 5. Define a finite element space on the mesh. Here we use the Nedelec
    //    finite elements of the specified order.

examples/ex3p.cpp

@@ -16,6 +16,8 @@
 //               mpirun -np 4 ex3p -m ../data/beam-hex-nurbs.mesh
 //               mpirun -np 4 ex3p -m ../data/amr-quad.mesh -o 2
 //               mpirun -np 4 ex3p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex3p -m ../data/ref-prism.mesh -o 1
+//               mpirun -np 4 ex3p -m ../data/octahedron.mesh -o 1
 //               mpirun -np 4 ex3p -m ../data/star-surf.mesh -o 2
 //               mpirun -np 4 ex3p -m ../data/mobius-strip.mesh -o 2 -f 0.1
 //               mpirun -np 4 ex3p -m ../data/klein-bottle.mesh -o 2 -f 0.1
@@ -103,6 +105,7 @@ int main(int argc, char *argv[])
       {
          args.PrintUsage(cout);
       }
+      // HYPRE_Finalize();
       MPI_Finalize();
       return 1;
    }
@@ -138,9 +141,7 @@ int main(int argc, char *argv[])
 
    // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
-   //    meshes need to be reoriented before we can define high-order Nedelec
-   //    spaces on them.
+   //    parallel mesh is defined, the serial mesh can be deleted.
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
    delete mesh;
    {
@@ -150,7 +151,6 @@ int main(int argc, char *argv[])
          pmesh->UniformRefinement();
       }
    }
-   pmesh->ReorientTetMesh();
 
    // 7. Define a parallel finite element space on the parallel mesh. Here we
    //    use the Nedelec finite elements of the specified order.

examples/ex4.cpp

@@ -19,6 +19,8 @@
 //               ex4 -m ../data/amr-hex.mesh
 //               ex4 -m ../data/amr-hex.mesh -o 2 -hb
 //               ex4 -m ../data/fichera-amr.mesh -o 2 -sc
+//               ex4 -m ../data/ref-prism.mesh -o 1
+//               ex4 -m ../data/octahedron.mesh -o 1
 //               ex4 -m ../data/star-surf.mesh -o 1
 //
 // Device sample runs:

examples/ex4p.cpp

@@ -19,6 +19,8 @@
 //               mpirun -np 3 ex4p -m ../data/amr-quad.mesh -o 2 -hb
 //               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -sc
 //               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -hb
+//               mpirun -np 4 ex4p -m ../data/ref-prism.mesh -o 1
+//               mpirun -np 4 ex4p -m ../data/octahedron.mesh -o 1
 //               mpirun -np 4 ex4p -m ../data/star-surf.mesh -o 3 -hb
 //
 // Device sample runs:
@@ -135,9 +137,7 @@ int main(int argc, char *argv[])
 
    // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
-   //    meshes need to be reoriented before we can define high-order Nedelec
-   //    spaces on them (this is needed in the ADS solver below).
+   //    parallel mesh is defined, the serial mesh can be deleted.
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
    delete mesh;
    {
@@ -147,7 +147,6 @@ int main(int argc, char *argv[])
          pmesh->UniformRefinement();
       }
    }
-   pmesh->ReorientTetMesh();
 
    // 7. Define a parallel finite element space on the parallel mesh. Here we
    //    use the Raviart-Thomas finite elements of the specified order.

examples/ex5.cpp

@@ -197,6 +197,7 @@ int main(int argc, char *argv[])
       SparseMatrix &M(mVarf->SpMat());
       SparseMatrix &B(bVarf->SpMat());
       B *= -1.;
+      if (Device::IsEnabled()) { B.BuildTranspose(); }
       Bt = new TransposeOperator(&B);
 
       darcyOp.SetBlock(0,0, &M);
@@ -240,6 +241,7 @@ int main(int argc, char *argv[])
    {
       SparseMatrix &M(mVarf->SpMat());
       M.GetDiag(Md);
+      Md.HostReadWrite();
 
       SparseMatrix &B(bVarf->SpMat());
       MinvBt = Transpose(B);
@@ -287,12 +289,18 @@ int main(int argc, char *argv[])
    chrono.Stop();
 
    if (solver.GetConverged())
+   {
       std::cout << "MINRES converged in " << solver.GetNumIterations()
-                << " iterations with a residual norm of " << solver.GetFinalNorm() << ".\n";
+                << " iterations with a residual norm of "
+                << solver.GetFinalNorm() << ".\n";
+   }
    else
+   {
       std::cout << "MINRES did not converge in " << solver.GetNumIterations()
-                << " iterations. Residual norm is " << solver.GetFinalNorm() << ".\n";
-   std::cout << "MINRES solver took " << chrono.RealTime() << "s. \n";
+                << " iterations. Residual norm is " << solver.GetFinalNorm()
+                << ".\n";
+   }
+   std::cout << "MINRES solver took " << chrono.RealTime() << "s.\n";
 
    // 12. Create the grid functions u and p. Compute the L2 error norms.
    GridFunction u, p;

examples/ex7p.cpp

@@ -47,6 +47,7 @@ int main(int argc, char *argv[])
    int order = 2;
    bool always_snap = false;
    bool visualization = 1;
+   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&elem_type, "-e", "--elem",
@@ -65,6 +66,8 @@ int main(int argc, char *argv[])
                   "--snap-at-the-end",
                   "If true, snap nodes to the sphere initially and after each refinement "
                   "otherwise, snap only after the last refinement");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (!args.Good())
    {
@@ -80,7 +83,12 @@ int main(int argc, char *argv[])
       args.PrintOptions(cout);
    }
 
-   // 3. Generate an initial high-order (surface) mesh on the unit sphere. The
+   // 3. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   if (myid == 0) { device.Print(); }
+
+   // 4. Generate an initial high-order (surface) mesh on the unit sphere. The
    //    Mesh object represents a 2D mesh in 3 spatial dimensions. We first add
    //    the elements and the vertices of the mesh, and then make it high-order
    //    by specifying a finite element space for its nodes.
@@ -146,7 +154,7 @@ int main(int argc, char *argv[])
    FiniteElementSpace nodal_fes(mesh, &fec, mesh->SpaceDimension());
    mesh->SetNodalFESpace(&nodal_fes);
 
-   // 4. Refine the mesh while snapping nodes to the sphere. Number of parallel
+   // 5. Refine the mesh while snapping nodes to the sphere. Number of parallel
    //    refinements is fixed to 2.
    for (int l = 0; l <= ref_levels; l++)
    {
@@ -218,7 +226,7 @@ int main(int argc, char *argv[])
       SnapNodes(*pmesh);
    }
 
-   // 5. Define a finite element space on the mesh. Here we use isoparametric
+   // 6. Define a finite element space on the mesh. Here we use isoparametric
    //    finite elements -- the same as the mesh nodes.
    ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, &fec);
    HYPRE_BigInt size = fespace->GlobalTrueVSize();
@@ -227,7 +235,7 @@ int main(int argc, char *argv[])
       cout << "Number of unknowns: " << size << endl;
    }
 
-   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
+   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
    //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
    //    the basis functions in the finite element fespace.
    ParLinearForm *b = new ParLinearForm(fespace);
@@ -237,27 +245,27 @@ int main(int argc, char *argv[])
    b->AddDomainIntegrator(new DomainLFIntegrator(rhs_coef));
    b->Assemble();
 
-   // 7. Define the solution vector x as a finite element grid function
+   // 8. Define the solution vector x as a finite element grid function
    //    corresponding to fespace. Initialize x with initial guess of zero.
    ParGridFunction x(fespace);
    x = 0.0;
 
-   // 8. Set up the bilinear form a(.,.) on the finite element space
+   // 9. Set up the bilinear form a(.,.) on the finite element space
    //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
    //    and Mass domain integrators.
    ParBilinearForm *a = new ParBilinearForm(fespace);
    a->AddDomainIntegrator(new DiffusionIntegrator(one));
    a->AddDomainIntegrator(new MassIntegrator(one));
 
-   // 9. Assemble the parallel linear system, applying any transformations
-   //    such as: parallel assembly, applying conforming constraints, etc.
+   // 10. Assemble the parallel linear system, applying any transformations
+   //     such as: parallel assembly, applying conforming constraints, etc.
    a->Assemble();
    HypreParMatrix A;
    Vector B, X;
    Array<int> empty_tdof_list;
    a->FormLinearSystem(empty_tdof_list, x, *b, A, X, B);
 
-   // 10. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
+   // 11. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
    //     preconditioner from hypre. Extract the parallel grid function x
    //     corresponding to the finite element approximation X. This is the local
    //     solution on each processor.
@@ -273,14 +281,14 @@ int main(int argc, char *argv[])
    delete a;
    delete b;
 
-   // 11. Compute and print the L^2 norm of the error.
+   // 12. Compute and print the L^2 norm of the error.
    double err = x.ComputeL2Error(sol_coef);
    if (myid == 0)
    {
       cout << "\nL2 norm of error: " << err << endl;
    }
 
-   // 12. Save the refined mesh and the solution. This output can be viewed
+   // 13. Save the refined mesh and the solution. This output can be viewed
    //     later using GLVis: "glvis -np <np> -m sphere_refined -g sol".
    {
       ostringstream mesh_name, sol_name;
@@ -296,7 +304,7 @@ int main(int argc, char *argv[])
       x.Save(sol_ofs);
    }
 
-   // 13. Send the solution by socket to a GLVis server.
+   // 14. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       char vishost[] = "localhost";
@@ -307,7 +315,7 @@ int main(int argc, char *argv[])
       sol_sock << "solution\n" << *pmesh << x << flush;
    }
 
-   // 14. Free the used memory.
+   // 15. Free the used memory.
    delete pcg;
    delete amg;
    delete fespace;

examples/ex8p.cpp

@@ -106,7 +106,6 @@ int main(int argc, char *argv[])
          pmesh->UniformRefinement();
       }
    }
-   pmesh->ReorientTetMesh();
 
    // 6. Define the trial, interfacial (trace) and test DPG spaces:
    //    - The trial space, x0_space, contains the non-interfacial unknowns and

examples/makefile

@@ -26,6 +26,9 @@ SEQ_EXAMPLES = ex0 ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 \
 PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
   ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
   ex25p ex26p ex27p ex28p ex29p
+SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
+PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p \
+  ex24p ex25p ex26p
 
 ifeq ($(MFEM_USE_MPI),NO)
    EXAMPLES = $(SEQ_EXAMPLES)
@@ -99,6 +102,14 @@ RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
 	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
 %-test-seq: %
 	@$(call mfem-test,$<,, Serial example)
+%-test-par-cuda: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel CUDA example,-d cuda)
+%-test-seq-cuda: %
+	@$(call mfem-test,$<,, Serial CUDA example,-d cuda)
+%-test-par-hip: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel HIP example,-d hip)
+%-test-seq-hip: %
+	@$(call mfem-test,$<,, Serial HIP example,-d hip)
 
 # Testing: Specific execution options
 ex0-test-seq: ex0

examples/petsc/ex3p.cpp

@@ -121,9 +121,7 @@ int main(int argc, char *argv[])
 
    // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
-   //    meshes need to be reoriented before we can define high-order Nedelec
-   //    spaces on them.
+   //    parallel mesh is defined, the serial mesh can be deleted.
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
    delete mesh;
    {
@@ -133,7 +131,6 @@ int main(int argc, char *argv[])
          pmesh->UniformRefinement();
       }
    }
-   pmesh->ReorientTetMesh();
 
    // 6. Define a parallel finite element space on the parallel mesh. Here we
    //    use the Nedelec finite elements of the specified order.

examples/petsc/ex4p.cpp

@@ -122,9 +122,7 @@ int main(int argc, char *argv[])
 
    // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
-   //    meshes need to be reoriented before we can define high-order Nedelec
-   //    spaces on them (this is needed in the ADS solver below).
+   //    parallel mesh is defined, the serial mesh can be deleted.
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
    delete mesh;
    {
@@ -134,7 +132,6 @@ int main(int argc, char *argv[])
          pmesh->UniformRefinement();
       }
    }
-   pmesh->ReorientTetMesh();
 
    // 6. Define a parallel finite element space on the parallel mesh. Here we
    //    use the Raviart-Thomas finite elements of the specified order.

examples/spectra/ex11.cpp

@@ -0,0 +1,250 @@
+//                       MFEM Example 11 - Serial Version
+//
+// Compile with: make ex11
+//
+// Sample runs:  ex11 -m ../data/square-disc.mesh
+//               ex11 -m ../data/star.mesh
+//               ex11 -m ../data/star-mixed.mesh
+//               ex11 -m ../data/periodic-annulus-sector.msh
+//               ex11 -m ../data/square-disc-p2.vtk -o 2
+//               ex11 -m ../data/square-disc-p3.mesh -o 3
+//               ex11 -m ../data/square-disc-nurbs.mesh -o -1
+//               ex11 -m ../data/disc-nurbs.mesh -o -1 -n 20
+//               ex11 -m ../data/star-surf.mesh
+//               ex11 -m ../data/square-disc-surf.mesh
+//               ex11 -m ../data/inline-segment.mesh
+//               ex11 -m ../data/inline-quad.mesh
+//               ex11 -m ../data/inline-tri.mesh
+//               ex11 -m ../data/amr-quad.mesh
+//               ex11 -m ../data/amr-hex.mesh
+//               ex11 -m ../data/mobius-strip.mesh -n 8
+//
+// Description:  This example code demonstrates the use of MFEM to solve the
+//               eigenvalue problem -Delta u = lambda u with homogeneous
+//               Dirichlet boundary conditions.
+//
+//               We compute a number of the lowest eigenmodes by discretizing
+//               the Laplacian and Mass operators using a FE space of the
+//               specified order, or an isoparametric/isogeometric space if
+//               order < 1 (quadratic for quadratic curvilinear mesh, NURBS for
+//               NURBS mesh, etc.)
+//
+//               The example highlights the use of the ARPACK eigenvalue solver
+//               (regular inverse mode). Reusing a single GLVis visualization
+//               window for multiple eigenfunctions is also illustrated.
+//
+//               We recommend viewing Example 1 before viewing this example.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../data/star.mesh";
+   int ser_ref_levels = 1;
+   int order = 1;
+   int nev = 5;
+   double dbc_eig = 1e3;
+   bool visualization = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&nev, "-n", "--num-eigs",
+                  "Number of desired eigenmodes.");
+   args.AddOption(&dbc_eig, "-d", "--dbc-eig",
+                  "Eigenvalues associated with Dirichlet BC "
+                  "(should be larger than the maximum desired eigenvalue).");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 2. Read the (serial) mesh from the given mesh file on all processors. We
+   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
+   //    and volume meshes with the same code.
+   Mesh *mesh;
+   ifstream imesh(mesh_file);
+   if (!imesh)
+   {
+      cerr << "\nCan not open mesh file: " << mesh_file << '\n' << endl;
+      return 2;
+   }
+   mesh = new Mesh(imesh, 1, 1);
+   imesh.close();
+   int dim = mesh->Dimension();
+
+   // 3. Refine the serial mesh on all processors to increase the resolution. In
+   //    this example we do 'ref_levels' of uniform refinement (2 by default, or
+   //    specified on the command line with -rs). 
+   for (int lev = 0; lev < ser_ref_levels; lev++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 4. Define a finite element space on the mesh. Here we
+   //    use continuous Lagrange finite elements of the specified order. If
+   //    order < 1, we instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec;
+   if (order > 0)
+   {
+      fec = new H1_FECollection(order, dim);
+   }
+   else if (mesh->GetNodes())
+   {
+      fec = mesh->GetNodes()->OwnFEC();
+   }
+   else
+   {
+      fec = new H1_FECollection(order = 1, dim);
+   }
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+   int size = fespace->GetVSize();
+
+   cout << "Number of unknowns: " << size << endl;
+
+   // 5. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
+   //    element space. The first corresponds to the Laplacian operator -Delta,
+   //    while the second is a simple mass matrix needed on the right hand side
+   //    of the generalized eigenvalue problem below. The boundary conditions
+   //    are implemented by elimination with special values on the diagonal to
+   //    shift the Dirichlet eigenvalues out of the computational range. After
+   //    serial and parallel assembly we extract the corresponding parallel
+   //    matrices A and M.
+   ConstantCoefficient one(1.0);
+   Array<int> ess_bdr;
+   if (mesh->bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh->bdr_attributes.Max());
+      ess_bdr = 1;
+   }
+
+   BilinearForm *a = new BilinearForm(fespace);
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));
+   if (mesh->bdr_attributes.Size() == 0)
+   {
+      // Add a mass term if the mesh has no boundary, e.g. periodic mesh or
+      // closed surface.
+      a->AddDomainIntegrator(new MassIntegrator(one));
+   }
+   a->Assemble();
+   if (mesh->bdr_attributes.Size() != 0)
+   {
+      a->EliminateEssentialBCDiag(ess_bdr, dbc_eig);
+   }
+   a->Finalize();
+
+   BilinearForm *m = new BilinearForm(fespace);
+   m->AddDomainIntegrator(new MassIntegrator(one));
+   m->Assemble();
+   if (mesh->bdr_attributes.Size() != 0)
+   {
+      // shift the eigenvalue corresponding to eliminated dofs to a large value
+      m->EliminateEssentialBCDiag(ess_bdr, 1.0);
+   }
+   m->Finalize();
+
+   // 6. Define and configure the SPECTRA eigensolver and solve problem
+   SpectraEigenSolver spectra;
+
+   spectra.SetNumModes(nev)
+      .SetKrylov(10)
+      .SetMaxIter(5000)
+      .SetTol(1e-5)
+      .SetOperators(*a, *m)
+      .Solve();
+
+   Eigen::VectorXd eigenvalues = spectra.GetEigenvalues(nev);
+
+   // 7. Define a grid function to represent each of the eigenmodes returned by the solver.
+   
+   GridFunction x(fespace);
+
+   // 8. Save the refined mesh and the modes in parallel. 
+   //    This output can be viewed later using GLVis: "glvis -np <np> -m mesh -g mode"
+   {
+      ostringstream mesh_name, mode_name;
+      mesh_name << "ex11.mesh";
+
+      ofstream mesh_ofs(mesh_name.str().c_str());
+      mesh_ofs.precision(8);
+      mesh->Print(mesh_ofs);
+
+      for (int i = 0; i < nev; i++) {
+         // conver Eigen Vector to MFEM Vector
+         Vector eigenvector = VectorConverter<double>::from(spectra.GetEigenvector(i));
+         
+         // convert eigenvector from Vector to GridFunction
+         x = eigenvector;
+
+         mode_name << "mode_" << setfill('0') << setw(2) << i;
+
+         ofstream mode_ofs(mode_name.str().c_str());
+         mode_ofs.precision(8);
+         x.Save(mode_ofs);
+         mode_name.str("");
+      }
+   }
+
+   // 10. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream mode_sock(vishost, visport);
+      mode_sock.precision(8);
+
+      for (int i=0; i<nev; i++)
+      {
+         cout << "Eigenmode " << i+1 << '/' << nev
+              << ", Lambda = " << eigenvalues[i] << endl;
+
+         // convert eigenvector from HypreParVector to ParGridFunction
+         Vector eigenvector = VectorConverter<double>::from(spectra.GetEigenvector(i));
+         x = eigenvector;
+
+         mode_sock << "solution\n" << *mesh << x << flush
+                   << "window_title 'Eigenmode " << i+1 << '/' << nev
+                   << ", Lambda = " << eigenvalues[i] << "'" << endl;
+
+         char c;
+         cout << "press (q)uit or (c)ontinue --> " << flush;
+         cin >> c;
+
+         if (c != 'c')
+         {
+            break;
+         }
+      }
+      mode_sock.close();
+   }
+
+   // 10. Free the used memory.
+   delete m;
+   delete a;
+
+   delete fespace;
+   if (order > 0)
+   {
+      delete fec;
+   }
+   delete mesh;
+
+   return 0;
+}

examples/spectra/makefile

@@ -0,0 +1,67 @@
+# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../..
+MFEM_BUILD_DIR ?= ../..
+SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/spectra/,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = ex11
+PAR_EXAMPLES = 
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES)
+endif
+RC_FILES = $(patsubst $(SRC)%,%,$(wildcard $(SRC)rc_*))
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES)
+
+# Examples depend on their corresponding rc_* files:
+make-rc-rule = $(1): | $(filter rc_$(1)%,$(RC_FILES))
+$(foreach ex,$(EXAMPLES),$(eval $(call make-rc-rule,$(ex))))
+
+# Rules to copy the rc_* files when building out-of-source:
+ifneq ($(SRC),)
+$(RC_FILES): %: $(SRC)%
+	cp -pf $(<) .
+endif
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean: clean-build clean-exec
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:
+	@rm -rf *.mesh mode_*

examples/superlu/ex1p.cpp

@@ -282,6 +282,10 @@ int main(int argc, char *argv[])
    superlu->SetOperator(*SLU_A);
    superlu->SetPrintStatistics(true);
    superlu->Mult(B, X);
+   superlu->DismantleGrid();
+
+   delete SLU_A;
+   delete superlu;
 
    // 14. Recover the parallel grid function corresponding to X. This is the
    //     local finite element solution on each processor.

fem/CMakeLists.txt

@@ -39,6 +39,7 @@ set(SRCS
   complex_fem.cpp
   convergence.cpp
   datacollection.cpp
+  doftrans.cpp
   eltrans.cpp
   estimators.cpp
   fe.cpp
@@ -105,6 +106,7 @@ set(SRCS
   tmop/tmop_pa_w3.cpp
   tmop/tmop_pa_w3_c0.cpp
   tmop_tools.cpp
+  tmop_amr.cpp
   gslib.cpp
   transfer.cpp
   lor.cpp
@@ -118,6 +120,7 @@ set(HDRS
   complex_fem.hpp
   convergence.hpp
   datacollection.hpp
+  doftrans.hpp
   eltrans.hpp
   estimators.hpp
   fe.hpp
@@ -164,6 +167,7 @@ set(HDRS
   tmop.hpp
   tmop/tmop_pa.hpp
   tmop_tools.hpp
+  tmop_amr.hpp
   gslib.hpp
   transfer.hpp
   lor.hpp
@@ -184,6 +188,11 @@ if (MFEM_USE_ADIOS2)
   list(APPEND HDRS adios2datacollection.hpp)
 endif()
 
+if (MFEM_USE_FMS)
+  list(APPEND SRCS fmsdatacollection.cpp fmsconvert.cpp)
+  list(APPEND HDRS fmsdatacollection.hpp fmsconvert.hpp)
+endif()
+
 if (MFEM_USE_MPI)
   list(APPEND SRCS
     pbilinearform.cpp

fem/bilinearform.cpp

@@ -391,6 +391,7 @@ void BilinearForm::Assemble(int skip_zeros)
    }
 
    ElementTransformation *eltrans;
+   DofTransformation * doftrans;
    Mesh *mesh = fes -> GetMesh();
    DenseMatrix elmat, *elmat_p;
 
@@ -424,7 +425,7 @@ void BilinearForm::Assemble(int skip_zeros)
       for (int i = 0; i < fes -> GetNE(); i++)
       {
          int elem_attr = fes->GetMesh()->GetAttribute(i);
-         fes->GetElementVDofs(i, vdofs);
+         doftrans = fes->GetElementVDofs(i, vdofs);
          if (element_matrices)
          {
             elmat_p = &(*element_matrices)(i);
@@ -458,6 +459,11 @@ void BilinearForm::Assemble(int skip_zeros)
             {
                elmat_p = &elmat;
             }
+            if (doftrans)
+            {
+               doftrans->TransformDual(elmat);
+            }
+            elmat_p = &elmat;
          }
          if (static_cond)
          {
@@ -503,7 +509,7 @@ void BilinearForm::Assemble(int skip_zeros)
          if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
 
          const FiniteElement &be = *fes->GetBE(i);
-         fes -> GetBdrElementVDofs (i, vdofs);
+         doftrans = fes -> GetBdrElementVDofs (i, vdofs);
          eltrans = fes -> GetBdrElementTransformation (i);
          int k = 0;
          for (; k < boundary_integs.Size(); k++)
@@ -523,17 +529,22 @@ void BilinearForm::Assemble(int skip_zeros)
             boundary_integs[k]->AssembleElementMatrix(be, *eltrans, elemmat);
             elmat += elemmat;
          }
+         if (doftrans)
+         {
+            doftrans->TransformDual(elmat);
+         }
+         elmat_p = &elmat;
          if (!static_cond)
          {
-            mat->AddSubMatrix(vdofs, vdofs, elmat, skip_zeros);
+            mat->AddSubMatrix(vdofs, vdofs, *elmat_p, skip_zeros);
             if (hybridization)
             {
-               hybridization->AssembleBdrMatrix(i, elmat);
+               hybridization->AssembleBdrMatrix(i, *elmat_p);
             }
          }
          else
          {
-            static_cond->AssembleBdrMatrix(i, elmat);
+            static_cond->AssembleBdrMatrix(i, *elmat_p);
          }
       }
    }
@@ -725,8 +736,8 @@ void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
       {
          // A, X and B point to the same data as mat, x and b
          EliminateVDofsInRHS(ess_tdof_list, x, b);
-         X.NewMemoryAndSize(x.GetMemory(), x.Size(), false);
-         B.NewMemoryAndSize(b.GetMemory(), b.Size(), false);
+         X.MakeRef(x, 0, x.Size());
+         B.MakeRef(b, 0, b.Size());
          if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
       }
    }
@@ -1318,9 +1329,10 @@ void MixedBilinearForm::Assemble (int skip_zeros)
       return;
    }
 
-   Array<int> tr_vdofs, te_vdofs;
    ElementTransformation *eltrans;
-   DenseMatrix elemmat;
+   DofTransformation * dom_dof_trans;
+   DofTransformation * ran_dof_trans;
+   DenseMatrix elmat;
 
    Mesh *mesh = test_fes -> GetMesh();
 
@@ -1333,16 +1345,24 @@ void MixedBilinearForm::Assemble (int skip_zeros)
    {
       for (int i = 0; i < test_fes -> GetNE(); i++)
       {
-         trial_fes -> GetElementVDofs (i, tr_vdofs);
-         test_fes  -> GetElementVDofs (i, te_vdofs);
+         dom_dof_trans = trial_fes -> GetElementVDofs (i, trial_vdofs);
+         ran_dof_trans = test_fes  -> GetElementVDofs (i, test_vdofs);
          eltrans = test_fes -> GetElementTransformation (i);
+
+         elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
+         elmat = 0.0;
          for (int k = 0; k < domain_integs.Size(); k++)
          {
             domain_integs[k] -> AssembleElementMatrix2 (*trial_fes -> GetFE(i),
                                                         *test_fes  -> GetFE(i),
                                                         *eltrans, elemmat);
-            mat -> AddSubMatrix (te_vdofs, tr_vdofs, elemmat, skip_zeros);
+            elmat += elemmat;
          }
+         if (ran_dof_trans || dom_dof_trans)
+         {
+            TransformDual(ran_dof_trans, dom_dof_trans, elmat);
+         }
+         mat -> AddSubMatrix (test_vdofs, trial_vdofs, elmat, skip_zeros);
       }
    }
 
@@ -1374,9 +1394,12 @@ void MixedBilinearForm::Assemble (int skip_zeros)
          const int bdr_attr = mesh->GetBdrAttribute(i);
          if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
 
-         trial_fes -> GetBdrElementVDofs (i, tr_vdofs);
-         test_fes  -> GetBdrElementVDofs (i, te_vdofs);
+         dom_dof_trans = trial_fes -> GetBdrElementVDofs (i, trial_vdofs);
+         ran_dof_trans = test_fes  -> GetBdrElementVDofs (i, test_vdofs);
          eltrans = test_fes -> GetBdrElementTransformation (i);
+
+         elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
+         elmat = 0.0;
          for (int k = 0; k < boundary_integs.Size(); k++)
          {
             if (boundary_integs_marker[k] &&
@@ -1385,29 +1408,34 @@ void MixedBilinearForm::Assemble (int skip_zeros)
             boundary_integs[k]->AssembleElementMatrix2 (*trial_fes -> GetBE(i),
                                                         *test_fes  -> GetBE(i),
                                                         *eltrans, elemmat);
-            mat -> AddSubMatrix (te_vdofs, tr_vdofs, elemmat, skip_zeros);
+            elmat += elemmat;
          }
+         if (ran_dof_trans || dom_dof_trans)
+         {
+            TransformDual(ran_dof_trans, dom_dof_trans, elmat);
+         }
+         mat -> AddSubMatrix (test_vdofs, trial_vdofs, elmat, skip_zeros);
       }
    }
 
    if (trace_face_integs.Size())
    {
       FaceElementTransformations *ftr;
-      Array<int> te_vdofs2;
+      Array<int> test_vdofs2;
       const FiniteElement *trial_face_fe, *test_fe1, *test_fe2;
 
       int nfaces = mesh->GetNumFaces();
       for (int i = 0; i < nfaces; i++)
       {
          ftr = mesh->GetFaceElementTransformations(i);
-         trial_fes->GetFaceVDofs(i, tr_vdofs);
-         test_fes->GetElementVDofs(ftr->Elem1No, te_vdofs);
+         trial_fes->GetFaceVDofs(i, trial_vdofs);
+         test_fes->GetElementVDofs(ftr->Elem1No, test_vdofs);
          trial_face_fe = trial_fes->GetFaceElement(i);
          test_fe1 = test_fes->GetFE(ftr->Elem1No);
          if (ftr->Elem2No >= 0)
          {
-            test_fes->GetElementVDofs(ftr->Elem2No, te_vdofs2);
-            te_vdofs.Append(te_vdofs2);
+            test_fes->GetElementVDofs(ftr->Elem2No, test_vdofs2);
+            test_vdofs.Append(test_vdofs2);
             test_fe2 = test_fes->GetFE(ftr->Elem2No);
          }
          else
@@ -1421,7 +1449,7 @@ void MixedBilinearForm::Assemble (int skip_zeros)
          {
             trace_face_integs[k]->AssembleFaceMatrix(*trial_face_fe, *test_fe1,
                                                      *test_fe2, *ftr, elemmat);
-            mat->AddSubMatrix(te_vdofs, tr_vdofs, elemmat, skip_zeros);
+            mat->AddSubMatrix(test_vdofs, trial_vdofs, elemmat, skip_zeros);
          }
       }
    }
@@ -1461,8 +1489,8 @@ void MixedBilinearForm::Assemble (int skip_zeros)
          ftr = mesh->GetBdrFaceTransformations(i);
          if (ftr)
          {
-            trial_fes->GetFaceVDofs(ftr->ElementNo, tr_vdofs);
-            test_fes->GetElementVDofs(ftr->Elem1No, te_vdofs);
+            trial_fes->GetFaceVDofs(ftr->ElementNo, trial_vdofs);
+            test_fes->GetElementVDofs(ftr->Elem1No, test_vdofs);
             trial_face_fe = trial_fes->GetFaceElement(ftr->ElementNo);
             test_fe1 = test_fes->GetFE(ftr->Elem1No);
             // The test_fe2 object is really a dummy and not used on the
@@ -1479,7 +1507,7 @@ void MixedBilinearForm::Assemble (int skip_zeros)
                                                                  *test_fe1,
                                                                  *test_fe2,
                                                                  *ftr, elemmat);
-               mat->AddSubMatrix(te_vdofs, tr_vdofs, elemmat, skip_zeros);
+               mat->AddSubMatrix(test_vdofs, trial_vdofs, elemmat, skip_zeros);
             }
          }
       }
@@ -1841,6 +1869,8 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
 
    Array<int> dom_vdofs, ran_vdofs;
    ElementTransformation *T;
+   DofTransformation * dom_dof_trans;
+   DofTransformation * ran_dof_trans;
    const FiniteElement *dom_fe, *ran_fe;
    DenseMatrix totelmat, elmat;
 
@@ -1853,8 +1883,8 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
    {
       for (int i = 0; i < test_fes->GetNE(); i++)
       {
-         trial_fes->GetElementVDofs(i, dom_vdofs);
-         test_fes->GetElementVDofs(i, ran_vdofs);
+         dom_dof_trans = trial_fes->GetElementVDofs(i, dom_vdofs);
+         ran_dof_trans = test_fes->GetElementVDofs(i, ran_vdofs);
          T = test_fes->GetElementTransformation(i);
          dom_fe = trial_fes->GetFE(i);
          ran_fe = test_fes->GetFE(i);
@@ -1867,6 +1897,10 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
                                                      elmat);
             totelmat += elmat;
          }
+         if (ran_dof_trans || dom_dof_trans)
+         {
+            TransformPrimal(ran_dof_trans, dom_dof_trans, totelmat);
+         }
          mat->SetSubMatrix(ran_vdofs, dom_vdofs, totelmat, skip_zeros);
       }
    }

fem/bilininteg.hpp

@@ -711,7 +711,7 @@ protected:
    {
       return "MixedScalarDerivativeIntegrator:  "
              "Trial and test spaces must both be scalar fields in 1D "
-             "and the trial space must implement CaldDShape.";
+             "and the trial space must implement CalcDShape.";
    }
 
    inline virtual void CalcTrialShape(const FiniteElement & trial_fe,
@@ -2936,11 +2936,11 @@ public:
 
     - F. Bassi and S. Rebay. A high order discontinuous Galerkin method for
       compressible turbulent flows. In B. Cockburn, G. E. Karniadakis, and
-      C.-W. Shu, editors, Discontinuous Galerkin Methods, pages 77–88. Springer
+      C.-W. Shu, editors, Discontinuous Galerkin Methods, pages 77-88. Springer
       Berlin Heidelberg, 2000.
     - D. N. Arnold, F. Brezzi, B. Cockburn, and L. D. Marini. Unified analysis
       of discontinuous Galerkin methods for elliptic problems. SIAM Journal on
-      Numerical Analysis, 39(5):1749–1779, 2002.
+      Numerical Analysis, 39(5):1749-1779, 2002.
 */
 class DGDiffusionBR2Integrator : public BilinearFormIntegrator
 {

fem/complex_fem.cpp

@@ -1204,17 +1204,30 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
       });
       // Modify offdiagonal blocks (imaginary parts of the matrix) to conform
       // with standard essential BC treatment
-      ess_tdof_list.HostRead();
       if (A_i.Type() == Operator::Hypre_ParCSR)
       {
          HypreParMatrix * Ah;
          A_i.Get(Ah);
          hypre_ParCSRMatrix *Aih = *Ah;
+#ifndef HYPRE_USING_CUDA
+         ess_tdof_list.HostRead();
          for (int k = 0; k < n; k++)
          {
             const int j = ess_tdof_list[k];
             Aih->diag->data[Aih->diag->i[j]] = 0.0;
          }
+#else
+         Ah->HypreReadWrite();
+         const int *d_ess_tdof_list =
+            ess_tdof_list.GetMemory().Read(MemoryClass::DEVICE, n);
+         const int *d_diag_i = Aih->diag->i;
+         double *d_diag_data = Aih->diag->data;
+         CuWrap1D(n, [=] MFEM_DEVICE (int k)
+         {
+            const int j = d_ess_tdof_list[k];
+            d_diag_data[d_diag_i[j]] = 0.0;
+         });
+#endif
       }
       else
       {

fem/doftrans.cpp

@@ -0,0 +1,358 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "fem.hpp"
+
+namespace mfem
+{
+
+void DofTransformation::TransformPrimal(Vector &v) const
+{
+   TransformPrimal(v.GetData());
+}
+
+void DofTransformation::TransformPrimalCols(DenseMatrix &V) const
+{
+   for (int c=0; c<V.Width(); c++)
+   {
+      TransformPrimal(V.GetColumn(c));
+   }
+}
+
+void DofTransformation::TransformDual(Vector &v) const
+{
+   TransformDual(v.GetData());
+}
+
+void DofTransformation::TransformDual(DenseMatrix &V) const
+{
+   TransformDualCols(V);
+   TransformDualRows(V);
+}
+
+void DofTransformation::TransformDualRows(DenseMatrix &V) const
+{
+   Vector row;
+   for (int r=0; r<V.Height(); r++)
+   {
+      V.GetRow(r, row);
+      TransformDual(row);
+      V.SetRow(r, row);
+   }
+}
+
+void DofTransformation::TransformDualCols(DenseMatrix &V) const
+{
+   for (int c=0; c<V.Width(); c++)
+   {
+      TransformDual(V.GetColumn(c));
+   }
+}
+
+void DofTransformation::InvTransformPrimal(Vector &v) const
+{
+   InvTransformPrimal(v.GetData());
+}
+
+void TransformPrimal(const DofTransformation *ran_dof_trans,
+                     const DofTransformation *dom_dof_trans,
+                     DenseMatrix &elmat)
+{
+   if (ran_dof_trans && dom_dof_trans)
+   {
+      ran_dof_trans->TransformPrimalCols(elmat);
+      dom_dof_trans->TransformDualRows(elmat);
+   }
+   else if (ran_dof_trans)
+   {
+      ran_dof_trans->TransformPrimalCols(elmat);
+   }
+   else if (dom_dof_trans)
+   {
+      dom_dof_trans->TransformDualRows(elmat);
+   }
+   else
+   {
+      // If both transformations are NULL this function should not be called
+   }
+}
+
+void TransformDual(const DofTransformation *ran_dof_trans,
+                   const DofTransformation *dom_dof_trans,
+                   DenseMatrix &elmat)
+{
+   if (ran_dof_trans && dom_dof_trans)
+   {
+      ran_dof_trans->TransformDualCols(elmat);
+      dom_dof_trans->TransformDualRows(elmat);
+   }
+   else if (ran_dof_trans)
+   {
+      ran_dof_trans->TransformDualCols(elmat);
+   }
+   else if (dom_dof_trans)
+   {
+      dom_dof_trans->TransformDualRows(elmat);
+   }
+   else
+   {
+      // If both transformations are NULL this function should not be called
+   }
+}
+
+void VDofTransformation::TransformPrimal(double *v) const
+{
+   int size = doftrans_->Size();
+
+   if ((Ordering::Type)ordering_ == Ordering::byNODES || vdim_ == 1)
+   {
+      for (int i=0; i<vdim_; i++)
+      {
+         doftrans_->TransformPrimal(&v[i*size]);
+      }
+   }
+   else
+   {
+      Vector vec(size);
+      for (int i=0; i<vdim_; i++)
+      {
+         for (int j=0; j<size; j++)
+         {
+            vec(j) = v[j*vdim_+i];
+         }
+         doftrans_->TransformPrimal(vec);
+         for (int j=0; j<size; j++)
+         {
+            v[j*vdim_+i] = vec(j);
+         }
+      }
+   }
+}
+
+void VDofTransformation::InvTransformPrimal(double *v) const
+{
+   int size = doftrans_->Height();
+
+   if ((Ordering::Type)ordering_ == Ordering::byNODES)
+   {
+      for (int i=0; i<vdim_; i++)
+      {
+         doftrans_->InvTransformPrimal(&v[i*size]);
+      }
+   }
+   else
+   {
+      Vector vec(size);
+      for (int i=0; i<vdim_; i++)
+      {
+         for (int j=0; j<size; j++)
+         {
+            vec(j) = v[j*vdim_+i];
+         }
+         doftrans_->InvTransformPrimal(vec);
+         for (int j=0; j<size; j++)
+         {
+            v[j*vdim_+i] = vec(j);
+         }
+      }
+   }
+}
+
+void VDofTransformation::TransformDual(double *v) const
+{
+   int size = doftrans_->Size();
+
+   if ((Ordering::Type)ordering_ == Ordering::byNODES)
+   {
+      for (int i=0; i<vdim_; i++)
+      {
+         doftrans_->TransformDual(&v[i*size]);
+      }
+   }
+   else
+   {
+      Vector vec(size);
+      for (int i=0; i<vdim_; i++)
+      {
+         for (int j=0; j<size; j++)
+         {
+            vec(j) = v[j*vdim_+i];
+         }
+         doftrans_->TransformDual(vec);
+         for (int j=0; j<size; j++)
+         {
+            v[j*vdim_+i] = vec(j);
+         }
+      }
+   }
+}
+
+const double ND_DofTransformation::T_data[24] =
+{
+   1.0,  0.0,  0.0,  1.0,
+   -1.0, -1.0,  0.0,  1.0,
+   0.0,  1.0, -1.0, -1.0,
+   1.0,  0.0, -1.0, -1.0,
+   -1.0, -1.0,  1.0,  0.0,
+   0.0,  1.0,  1.0,  0.0
+};
+
+const DenseTensor ND_DofTransformation
+::T(const_cast<double*>(ND_DofTransformation::T_data), 2, 2, 6);
+
+const double ND_DofTransformation::TInv_data[24] =
+{
+   1.0,  0.0,  0.0,  1.0,
+   -1.0, -1.0,  0.0,  1.0,
+   -1.0, -1.0,  1.0,  0.0,
+   1.0,  0.0, -1.0, -1.0,
+   0.0,  1.0, -1.0, -1.0,
+   0.0,  1.0,  1.0,  0.0
+};
+
+const DenseTensor ND_DofTransformation
+::TInv(const_cast<double*>(TInv_data), 2, 2, 6);
+
+ND_DofTransformation::ND_DofTransformation(int size, int p)
+   : DofTransformation(size),
+     order(p)
+{
+}
+
+ND_TriDofTransformation::ND_TriDofTransformation(int p)
+   : ND_DofTransformation(p*(p + 2), p)
+{
+}
+
+void ND_TriDofTransformation::TransformPrimal(double *v) const
+{
+   int nedofs = order; // number of DoFs per edge
+   int nfdofs = order*(order-1); // number of DoFs per face
+
+   double data[2];
+   Vector v2(data, 2);
+
+   // Transform face DoFs
+   for (int f=0; f<1; f++)
+   {
+      for (int i=0; i<nfdofs/2; i++)
+      {
+         v2 = &v[3*nedofs + f*nfdofs + 2*i];
+         T(Fo[f]).Mult(v2, &v[3*nedofs + f*nfdofs + 2*i]);
+      }
+   }
+}
+
+void
+ND_TriDofTransformation::InvTransformPrimal(double *v) const
+{
+   int nedofs = order; // number of DoFs per edge
+   int nfdofs = order*(order-1); // number of DoFs per face
+
+   double data[2];
+   Vector v2(data, 2);
+
+   // Transform face DoFs
+   for (int f=0; f<1; f++)
+   {
+      for (int i=0; i<nfdofs/2; i++)
+      {
+         v2 = &v[3*nedofs + f*nfdofs + 2*i];
+         TInv(Fo[f]).Mult(v2, &v[3*nedofs + f*nfdofs + 2*i]);
+      }
+   }
+}
+
+void
+ND_TriDofTransformation::TransformDual(double *v) const
+{
+   int nedofs = order; // number of DoFs per edge
+   int nfdofs = order*(order-1); // number of DoFs per face
+
+   double data[2];
+   Vector v2(data, 2);
+
+   // Transform face DoFs
+   for (int f=0; f<1; f++)
+   {
+      for (int i=0; i<nfdofs/2; i++)
+      {
+         v2 = &v[3*nedofs + f*nfdofs + 2*i];
+         TInv(Fo[f]).MultTranspose(v2, &v[3*nedofs + f*nfdofs + 2*i]);
+      }
+   }
+}
+
+ND_TetDofTransformation::ND_TetDofTransformation(int p)
+   : ND_DofTransformation(p*(p + 2)*(p + 3)/2, p)
+{
+}
+
+void ND_TetDofTransformation::TransformPrimal(double *v) const
+{
+   int nedofs = order; // number of DoFs per edge
+   int nfdofs = order*(order-1); // number of DoFs per face
+
+   double data[2];
+   Vector v2(data, 2);
+
+   // Transform face DoFs
+   for (int f=0; f<4; f++)
+   {
+      for (int i=0; i<nfdofs/2; i++)
+      {
+         v2 = &v[6*nedofs + f*nfdofs + 2*i];
+         T(Fo[f]).Mult(v2, &v[6*nedofs + f*nfdofs + 2*i]);
+      }
+   }
+}
+
+void
+ND_TetDofTransformation::InvTransformPrimal(double *v) const
+{
+   int nedofs = order; // number of DoFs per edge
+   int nfdofs = order*(order-1); // number of DoFs per face
+
+   double data[2];
+   Vector v2(data, 2);
+
+   // Transform face DoFs
+   for (int f=0; f<4; f++)
+   {
+      for (int i=0; i<nfdofs/2; i++)
+      {
+         v2 = &v[6*nedofs + f*nfdofs + 2*i];
+         TInv(Fo[f]).Mult(v2, &v[6*nedofs + f*nfdofs + 2*i]);
+      }
+   }
+}
+
+void
+ND_TetDofTransformation::TransformDual(double *v) const
+{
+   int nedofs = order; // number of DoFs per edge
+   int nfdofs = order*(order-1); // number of DoFs per face
+
+   double data[2];
+   Vector v2(data, 2);
+
+   // Transform face DoFs
+   for (int f=0; f<4; f++)
+   {
+      for (int i=0; i<nfdofs/2; i++)
+      {
+         v2 = &v[6*nedofs + f*nfdofs + 2*i];
+         TInv(Fo[f]).MultTranspose(v2, &v[6*nedofs + f*nfdofs + 2*i]);
+      }
+   }
+}
+
+} // namespace mfem

fem/doftrans.hpp

@@ -0,0 +1,277 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_DOFTRANSFORM
+#define MFEM_DOFTRANSFORM
+
+#include "../config/config.hpp"
+#include "../linalg/linalg.hpp"
+#include "intrules.hpp"
+#include "fe.hpp"
+
+namespace mfem
+{
+
+/** The DofTransformation class is an abstract base class for a family of
+    transformations that map local degrees of freedom (DoFs), contained within
+    individual elements, to global degrees of freedom, stored within
+    GridFunction objects. These transformations are necessary to ensure that
+    basis functions in neighboring elements align correctly. Closely related but
+    complementary transformations are required for the entries stored in
+    LinearForm and BilinearForm objects. The DofTransformation class is designed
+    to apply the action of both of these types of DoF transformations.
+
+    Let the "primal transformation" be given by the operator T. This means that
+    given a local element vector v the data that must be placed into a
+    GridFunction object is v_t = T * v.
+
+    We also need the inverse of the primal transformation T^{-1} so that we can
+    recover the local element vector from data read out of a GridFunction
+    e.g. v = T^{-1} * v_t.
+
+    We need to preserve the action of our linear forms applied to primal
+    vectors. In other words, if f is the local vector computed by a linear
+    form then f * v = f_t * v_t (where "*" represents an inner product of
+    vectors). This requires that f_t = T^{-T} * f i.e. the "dual transform" is
+    given by the transpose of the inverse of the primal transformation.
+
+    For bilinear forms we require that v^T * A * v = v_t^T * A_t * v_t. This
+    implies that A_t = T^{-T} * A * T^{-1}. This can be accomplished by
+    performing dual transformations of the rows and columns of the matrix A.
+
+    For discrete linear operators the range must be modified with the primal
+    transformation rather than the dual transformation because the result is a
+    primal vector rather than a dual vector. This leads to the transformation
+    D_t = T * D * T^{-1}. This can be accomplished by using a primal
+    transformation on the columns of D and a dual transformation on its rows.
+*/
+class DofTransformation
+{
+protected:
+   int size_;
+
+   Array<int> Fo;
+
+   DofTransformation(int size)
+      : size_(size) {}
+
+public:
+
+   inline int Size() const { return size_; }
+   inline int Height() const { return size_; }
+   inline int NumRows() const { return size_; }
+   inline int Width() const { return size_; }
+   inline int NumCols() const { return size_; }
+
+   /** @brief Configure the transformation using face orientations for the
+       current element. */
+   /// The face_orientation array can be obtained from Mesh::GetElementFaces.
+   inline void SetFaceOrientations(const Array<int> & face_orientation)
+   { Fo = face_orientation; }
+
+   inline const Array<int> & GetFaceOrientations() const { return Fo; }
+
+   /** Transform local DoFs to align with the global DoFs. For example, this
+       transformation can be used to map the local vector computed by
+       FiniteElement::Project() to the transformed vector stored within a
+       GridFunction object. */
+   virtual void TransformPrimal(double *v) const = 0;
+   virtual void TransformPrimal(Vector &v) const;
+
+   /// Transform groups of DoFs stored as dense matrices
+   virtual void TransformPrimalCols(DenseMatrix &V) const;
+
+   /** Inverse transform local DoFs. Used to transform DoFs from a global vector
+       back to their element-local form. For example, this must be used to
+       transform the vector obtained using GridFunction::GetSubVector before it
+       can be used to compute a local interpolation.
+   */
+   virtual void InvTransformPrimal(double *v) const = 0;
+   virtual void InvTransformPrimal(Vector &v) const;
+
+   /** Transform dual DoFs as computed by a LinearFormIntegrator before summing
+       into a LinearForm object. */
+   virtual void TransformDual(double *v) const = 0;
+   virtual void TransformDual(Vector &v) const;
+
+   /** Transform a matrix of dual DoFs entries as computed by a
+       BilinearFormIntegrator before summing into a BilinearForm object. */
+   virtual void TransformDual(DenseMatrix &V) const;
+
+   /// Transform groups of dual DoFs stored as dense matrices
+   virtual void TransformDualRows(DenseMatrix &V) const;
+   virtual void TransformDualCols(DenseMatrix &V) const;
+
+   virtual ~DofTransformation() {}
+};
+
+/** Transform a matrix of DoFs entries from different finite element spaces as
+    computed by a DiscreteInterpolator before copying into a
+    DiscreteLinearOperator.
+*/
+void TransformPrimal(const DofTransformation *ran_dof_trans,
+                     const DofTransformation *dom_dof_trans,
+                     DenseMatrix &elmat);
+
+/** Transform a matrix of dual DoFs entries from different finite element spaces
+    as computed by a BilinearFormIntegrator before summing into a
+    MixedBilinearForm object.
+*/
+void TransformDual(const DofTransformation *ran_dof_trans,
+                   const DofTransformation *dom_dof_trans,
+                   DenseMatrix &elmat);
+
+/** The VDofTransformation class implements a nested transformation where an
+    arbitrary DofTransformation is replicated with a vdim >= 1.
+*/
+class VDofTransformation : public DofTransformation
+{
+private:
+   int vdim_;
+   int ordering_;
+   DofTransformation * doftrans_;
+
+public:
+   /** @brief Default constructor which requires that SetDofTransformation be
+       called before use. */
+   VDofTransformation(int vdim = 1, int ordering = 0)
+      : DofTransformation(0),
+        vdim_(vdim), ordering_(ordering),
+        doftrans_(NULL) {}
+
+   /// Constructor with a known DofTransformation
+   VDofTransformation(DofTransformation & doftrans, int vdim = 1,
+                      int ordering = 0)
+      : DofTransformation(vdim * doftrans.Size()),
+        vdim_(vdim), ordering_(ordering),
+        doftrans_(&doftrans) {}
+
+   /// Set or change the vdim parameter
+   inline void SetVDim(int vdim)
+   {
+      vdim_ = vdim;
+      if (doftrans_)
+      {
+         size_ = vdim_ * doftrans_->Size();
+      }
+   }
+
+   /// Return the current vdim value
+   inline int GetVDim() const { return vdim_; }
+
+   /// Set or change the nested DofTransformation object
+   inline void SetDofTransformation(DofTransformation & doftrans)
+   {
+      size_ = vdim_ * doftrans.Size();
+      doftrans_ = &doftrans;
+   }
+
+   /// Return the nested DofTransformation object
+   inline DofTransformation * GetDofTransformation() const { return doftrans_; }
+
+   inline void SetFaceOrientation(const Array<int> & face_orientation)
+   { Fo = face_orientation; doftrans_->SetFaceOrientations(face_orientation); }
+
+   using DofTransformation::TransformPrimal;
+   using DofTransformation::InvTransformPrimal;
+   using DofTransformation::TransformDual;
+
+   void TransformPrimal(double *v) const;
+   void InvTransformPrimal(double *v) const;
+   void TransformDual(double *v) const;
+};
+
+/** Abstract base class for high-order Nedelec spaces on elements with
+    triangular faces.
+
+    The Nedelec DoFs on the interior of triangular faces come in pairs which
+    share an interpolation point but have different vector directions. These
+    directions depend on the orientation of the face and can therefore differ in
+    neighboring elements. The mapping required to transform these DoFs can be
+    implemented as series of 2x2 linear transformations. The raw data for these
+    linear transformations is stored in the T_data and TInv_data arrays and can
+    be accessed as DenseMatrices using the GetFaceTransform() and
+    GetFaceInverseTransform() methods.
+*/
+class ND_DofTransformation : public DofTransformation
+{
+protected:
+   static const double T_data[24];
+   static const double TInv_data[24];
+   static const DenseTensor T, TInv;
+   int order;
+
+   ND_DofTransformation(int size, int order);
+
+public:
+   // Return the 2x2 transformation operator for the given face orientation
+   static const DenseMatrix & GetFaceTransform(int ori) { return T(ori); }
+
+   // Return the 2x2 inverse transformation operator
+   static const DenseMatrix & GetFaceInverseTransform(int ori)
+   { return TInv(ori); }
+};
+
+/// DoF transformation implementation for the Nedelec basis on triangles
+class ND_TriDofTransformation : public ND_DofTransformation
+{
+public:
+   ND_TriDofTransformation(int order);
+
+   using DofTransformation::TransformPrimal;
+   using DofTransformation::InvTransformPrimal;
+   using DofTransformation::TransformDual;
+
+   void TransformPrimal(double *v) const;
+
+   void InvTransformPrimal(double *v) const;
+
+   void TransformDual(double *v) const;
+};
+
+/// DoF transformation implementation for the Nedelec basis on tetrahedra
+class ND_TetDofTransformation : public ND_DofTransformation
+{
+public:
+   ND_TetDofTransformation(int order);
+
+   using DofTransformation::TransformPrimal;
+   using DofTransformation::InvTransformPrimal;
+   using DofTransformation::TransformDual;
+
+   void TransformPrimal(double *v) const;
+
+   void InvTransformPrimal(double *v) const;
+
+   void TransformDual(double *v) const;
+};
+
+/// DoF transformation implementation for the Nedelec basis on wedge elements
+/** TODO: (Under development) */
+class ND_WedgeDofTransformation : public ND_DofTransformation
+{
+public:
+   ND_WedgeDofTransformation(int order);
+
+   using DofTransformation::TransformPrimal;
+   using DofTransformation::InvTransformPrimal;
+   using DofTransformation::TransformDual;
+
+   void TransformPrimal(double *v) const;
+
+   void InvTransformPrimal(double *v) const;
+
+   void TransformDual(double *v) const;
+};
+
+} // namespace mfem
+
+#endif // MFEM_DOFTRANSFORM

fem/eltrans.cpp

@@ -380,6 +380,7 @@ void IsoparametricTransformation::SetIdentityTransformation(
       case Geometry::TETRAHEDRON : FElem = &TetrahedronFE; break;
       case Geometry::CUBE :        FElem = &HexahedronFE; break;
       case Geometry::PRISM :       FElem = &WedgeFE; break;
+      case Geometry::PYRAMID :     FElem = &PyramidFE; break;
       default:
          MFEM_ABORT("unknown Geometry::Type!");
    }

fem/eltrans.hpp

@@ -316,7 +316,7 @@ public:
 
    /// Set the desired print level, useful for debugging.
    /** The valid options are: -1 - never print (default); 0 - print only errors;
-       1 - print the first and last last iterations; 2 - print every iteration;
+       1 - print the first and last iterations; 2 - print every iteration;
        and 3 - print every iteration including point coordinates. */
    void SetPrintLevel(int pr_level) { print_level = pr_level; }
 

fem/estimators.cpp

@@ -329,7 +329,7 @@ void KellyErrorEstimator::ComputeEstimates()
          error_estimates(e) = sqrt(factor * error_estimates(e));
       }
 
-      total_error = error_estimates.Sum();
+      total_error = error_estimates.Norml2();
       delete flux;
       return;
    }
@@ -452,9 +452,10 @@ void KellyErrorEstimator::ComputeEstimates()
    auto pfes = dynamic_cast<ParFiniteElementSpace*>(xfes);
    MFEM_VERIFY(pfes, "xfes is not a ParFiniteElementSpace pointer");
 
-   double process_local_error = error_estimates.Sum();
+   double process_local_error = pow(error_estimates.Norml2(),2.0);
    MPI_Allreduce(&process_local_error, &total_error, 1, MPI_DOUBLE,
                  MPI_SUM, pfes->GetComm());
+   total_error = sqrt(total_error);
 #endif // MFEM_USE_MPI
 }
 

fem/fe.hpp

@@ -97,7 +97,7 @@ public:
       {
          "Gauss-Legendre", "Gauss-Lobatto", "Positive (Bernstein)",
          "Open uniform", "Closed uniform", "Open half uniform",
-         "Seredipity", "Closed Gauss-Legendre",
+         "Serendipity", "Closed Gauss-Legendre",
          "Integrated Gauss-Lobatto indicator"
       };
       return name[Check(b_type)];
@@ -1126,7 +1126,7 @@ public:
    { dofs = 1.0; }
 };
 
-/// A 1D quadractic finite element with uniformly spaced nodes
+/// A 1D quadratic finite element with uniformly spaced nodes
 class Quad1DFiniteElement : public NodalFiniteElement
 {
 public:
@@ -1313,6 +1313,64 @@ public:
                            DenseMatrix &dshape) const;
 };
 
+/// A linear element defined on a triangular prism
+class LinearWedgeFiniteElement : public NodalFiniteElement
+{
+public:
+   /// Construct the LinearWedgeFiniteElement
+   LinearWedgeFiniteElement();
+
+   /** @brief virtual function which evaluates the values of all
+       shape functions at a given point ip and stores
+       them in the vector shape of dimension Dof (4) */
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+
+   /** @brief virtual function which evaluates the values of all
+       partial derivatives of all shape functions at a given
+       point ip and stores them in the matrix dshape (Dof x Dim) (4 x 3)
+       so that each row contains the derivatives of one shape function */
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+
+   virtual void ProjectDelta(int vertex, Vector &dofs) const
+   { dofs = 0.0; dofs(vertex) = 1.0; }
+
+   /** @brief Get the dofs associated with the given @a face.
+       @a *dofs is set to an internal array of the local dofc on the
+       face, while *ndofs is set to the number of dofs on that face.
+   */
+   virtual void GetFaceDofs(int face, int **dofs, int *ndofs) const;
+};
+
+/// A linear element defined on a square pyramid
+class LinearPyramidFiniteElement : public NodalFiniteElement
+{
+public:
+   /// Construct the LinearPyramidFiniteElement
+   LinearPyramidFiniteElement();
+
+   /** @brief virtual function which evaluates the values of all
+       shape functions at a given point ip and stores
+       them in the vector shape of dimension Dof (4) */
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+
+   /** @brief virtual function which evaluates the values of all
+       partial derivatives of all shape functions at a given
+       point ip and stores them in the matrix dshape (Dof x Dim) (4 x 3)
+       so that each row contains the derivatives of one shape function */
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+
+   virtual void ProjectDelta(int vertex, Vector &dofs) const
+   { dofs = 0.0; dofs(vertex) = 1.0; }
+
+   /** @brief Get the dofs associated with the given @a face.
+       @a *dofs is set to an internal array of the local dofc on the
+       face, while *ndofs is set to the number of dofs on that face.
+   */
+   virtual void GetFaceDofs(int face, int **dofs, int *ndofs) const;
+};
+
 /// A 2D constant element on a triangle
 class P0TriangleFiniteElement : public NodalFiniteElement
 {
@@ -1690,6 +1748,32 @@ public:
    { dofs(0) = 1.0; }
 };
 
+/// A 3D constant element on a wedge
+class P0WdgFiniteElement : public NodalFiniteElement
+{
+public:
+   /// Construct the P0WdgFiniteElement
+   P0WdgFiniteElement ();
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void ProjectDelta(int vertex, Vector &dofs) const
+   { dofs(0) = 1.0; }
+};
+
+/// A 3D constant element on a pyramid
+class P0PyrFiniteElement : public NodalFiniteElement
+{
+public:
+   /// Construct the P0PyrFiniteElement
+   P0PyrFiniteElement ();
+   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
+   virtual void CalcDShape(const IntegrationPoint &ip,
+                           DenseMatrix &dshape) const;
+   virtual void ProjectDelta(int vertex, Vector &dofs) const
+   { dofs(0) = 1.0; }
+};
+
 /** @brief Tensor products of 1D Lagrange1DFiniteElement
     (only degree 2 is functional) */
 class LagrangeHexFiniteElement : public NodalFiniteElement
@@ -1828,6 +1912,10 @@ public:
    using FiniteElement::Project;
    virtual void Project (VectorCoefficient &vc,
                          ElementTransformation &Trans, Vector &dofs) const;
+
+   virtual void ProjectGrad(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &grad) const;
 };
 
 
@@ -1852,6 +1940,66 @@ public:
    using FiniteElement::Project;
    virtual void Project (VectorCoefficient &vc,
                          ElementTransformation &Trans, Vector &dofs) const;
+
+   virtual void ProjectGrad(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &grad) const;
+};
+
+
+/// A 3D 1st order Nedelec element on a wedge
+class Nedelec1WdgFiniteElement : public VectorFiniteElement
+{
+private:
+   static const double tk[9][3];
+
+public:
+   /// Construct the Nedelec1WdgFiniteElement
+   Nedelec1WdgFiniteElement();
+   virtual void CalcVShape(const IntegrationPoint &ip,
+                           DenseMatrix &shape) const;
+   virtual void CalcVShape(ElementTransformation &Trans,
+                           DenseMatrix &shape) const
+   { CalcVShape_ND(Trans, shape); }
+   virtual void CalcCurlShape(const IntegrationPoint &ip,
+                              DenseMatrix &curl_shape) const;
+   virtual void GetLocalInterpolation (ElementTransformation &Trans,
+                                       DenseMatrix &I) const;
+   using FiniteElement::Project;
+   virtual void Project (VectorCoefficient &vc,
+                         ElementTransformation &Trans, Vector &dofs) const;
+
+   virtual void ProjectGrad(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &grad) const;
+};
+
+
+/// A 3D 1st order Nedelec element on a pyramid
+class Nedelec1PyrFiniteElement : public VectorFiniteElement
+{
+private:
+   static const double tk[8][3];
+
+public:
+   /// Construct the Nedelec1PyrFiniteElement
+   Nedelec1PyrFiniteElement();
+   virtual void CalcVShape(const IntegrationPoint &ip,
+                           DenseMatrix &shape) const;
+   virtual void CalcVShape(ElementTransformation &Trans,
+                           DenseMatrix &shape) const
+   { CalcVShape_ND(Trans, shape); }
+   virtual void CalcCurlShape(const IntegrationPoint &ip,
+                              DenseMatrix &curl_shape) const;
+   virtual void GetLocalInterpolation (ElementTransformation &Trans,
+                                       DenseMatrix &I) const;
+   using FiniteElement::Project;
+   virtual void Project (VectorCoefficient &vc,
+                         ElementTransformation &Trans, Vector &dofs) const;
+
+   virtual void ProjectGrad(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &grad) const;
 };
 
 
@@ -1945,6 +2093,77 @@ public:
 };
 
 
+/// A 3D 0th order Raviert-Thomas element on a wedge
+class RT0WdgFiniteElement : public VectorFiniteElement
+{
+private:
+   static const double nk[5][3];
+
+public:
+   /// Construct the RT0WdgFiniteElement
+   RT0WdgFiniteElement();
+
+   virtual void CalcVShape(const IntegrationPoint &ip,
+                           DenseMatrix &shape) const;
+
+   virtual void CalcVShape(ElementTransformation &Trans,
+                           DenseMatrix &shape) const
+   { CalcVShape_RT(Trans, shape); }
+
+   virtual void CalcDivShape(const IntegrationPoint &ip,
+                             Vector &divshape) const;
+
+   virtual void GetLocalInterpolation (ElementTransformation &Trans,
+                                       DenseMatrix &I) const;
+
+   using FiniteElement::Project;
+
+   virtual void Project (VectorCoefficient &vc,
+                         ElementTransformation &Trans, Vector &dofs) const;
+
+   virtual void ProjectCurl(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &curl) const;
+};
+
+
+/// A 3D 0th order Raviert-Thomas element on a pyramid
+class RT0PyrFiniteElement : public VectorFiniteElement
+{
+private:
+   static const double nk[5][3];
+
+   // If true match RT0TetFiniteElement rather than RT_TetrahedronElement(0)
+   bool rt0;
+
+public:
+   /// Construct the RT0PyrFiniteElement
+   RT0PyrFiniteElement(bool rt0tets = true);
+
+   virtual void CalcVShape(const IntegrationPoint &ip,
+                           DenseMatrix &shape) const;
+
+   virtual void CalcVShape(ElementTransformation &Trans,
+                           DenseMatrix &shape) const
+   { CalcVShape_RT(Trans, shape); }
+
+   virtual void CalcDivShape(const IntegrationPoint &ip,
+                             Vector &divshape) const;
+
+   virtual void GetLocalInterpolation (ElementTransformation &Trans,
+                                       DenseMatrix &I) const;
+
+   using FiniteElement::Project;
+
+   virtual void Project (VectorCoefficient &vc,
+                         ElementTransformation &Trans, Vector &dofs) const;
+
+   virtual void ProjectCurl(const FiniteElement &fe,
+                            ElementTransformation &Trans,
+                            DenseMatrix &curl) const;
+};
+
+
 class RotTriLinearHexFiniteElement : public NodalFiniteElement
 {
 public:

fem/fe_coll.cpp

@@ -33,6 +33,9 @@ int FiniteElementCollection::HasFaceDofs(Geometry::Type geom, int p) const
       case Geometry::PRISM:
          return max(GetNumDof(Geometry::TRIANGLE, p),
                     GetNumDof(Geometry::SQUARE, p));
+      case Geometry::PYRAMID:
+         return max(GetNumDof(Geometry::TRIANGLE, p),
+                    GetNumDof(Geometry::SQUARE, p));
       default:
          MFEM_ABORT("unknown geometry type");
    }
@@ -574,6 +577,7 @@ LinearFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
       case Geometry::TETRAHEDRON: return &TetrahedronFE;
       case Geometry::CUBE:        return &ParallelepipedFE;
       case Geometry::PRISM:       return &WedgeFE;
+      case Geometry::PYRAMID:     return &PyramidFE;
       default:
          mfem_error ("LinearFECollection: unknown geometry type.");
    }
@@ -591,6 +595,7 @@ int LinearFECollection::DofForGeometry(Geometry::Type GeomType) const
       case Geometry::TETRAHEDRON: return 0;
       case Geometry::CUBE:        return 0;
       case Geometry::PRISM:       return 0;
+      case Geometry::PYRAMID:     return 0;
       default:
          mfem_error ("LinearFECollection: unknown geometry type.");
    }
@@ -1240,6 +1245,7 @@ Const3DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
       case Geometry::TETRAHEDRON: return &TetrahedronFE;
       case Geometry::CUBE:        return &ParallelepipedFE;
       case Geometry::PRISM:       return &WedgeFE;
+      case Geometry::PYRAMID:     return &PyramidFE;
       default:
          mfem_error ("Const3DFECollection: unknown geometry type.");
    }
@@ -1257,6 +1263,7 @@ int Const3DFECollection::DofForGeometry(Geometry::Type GeomType) const
       case Geometry::TETRAHEDRON: return 1;
       case Geometry::CUBE:        return 1;
       case Geometry::PRISM:       return 1;
+      case Geometry::PYRAMID:     return 1;
       default:
          mfem_error ("Const3DFECollection: unknown geometry type.");
    }
@@ -1277,6 +1284,8 @@ LinearDiscont3DFECollection::FiniteElementForGeometry(
    switch (GeomType)
    {
       case Geometry::TETRAHEDRON: return &TetrahedronFE;
+      case Geometry::PYRAMID:     return &PyramidFE;
+      case Geometry::PRISM:       return &WedgeFE;
       case Geometry::CUBE:        return &ParallelepipedFE;
       default:
          mfem_error ("LinearDiscont3DFECollection: unknown geometry type.");
@@ -1293,6 +1302,8 @@ int LinearDiscont3DFECollection::DofForGeometry(Geometry::Type GeomType) const
       case Geometry::TRIANGLE:    return 0;
       case Geometry::SQUARE:      return 0;
       case Geometry::TETRAHEDRON: return 4;
+      case Geometry::PYRAMID:     return 5;
+      case Geometry::PRISM:       return 6;
       case Geometry::CUBE:        return 8;
       default:
          mfem_error ("LinearDiscont3DFECollection: unknown geometry type.");
@@ -1394,6 +1405,8 @@ ND1_3DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
    {
       case Geometry::CUBE:        return &HexahedronFE;
       case Geometry::TETRAHEDRON: return &TetrahedronFE;
+      case Geometry::PRISM:       return &WedgeFE;
+      case Geometry::PYRAMID:     return &PyramidFE;
       default:
          mfem_error ("ND1_3DFECollection: unknown geometry type.");
    }
@@ -1410,6 +1423,8 @@ int ND1_3DFECollection::DofForGeometry(Geometry::Type GeomType) const
       case Geometry::SQUARE:      return 0;
       case Geometry::TETRAHEDRON: return 0;
       case Geometry::CUBE:        return 0;
+      case Geometry::PRISM:       return 0;
+      case Geometry::PYRAMID:     return 0;
       default:
          mfem_error ("ND1_3DFECollection: unknown geometry type.");
    }
@@ -1439,6 +1454,8 @@ RT0_3DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
       case Geometry::SQUARE:      return &QuadrilateralFE;
       case Geometry::CUBE:        return &HexahedronFE;
       case Geometry::TETRAHEDRON: return &TetrahedronFE;
+      case Geometry::PRISM:       return &WedgeFE;
+      case Geometry::PYRAMID:     return &PyramidFE;
       default:
          mfem_error ("RT0_3DFECollection: unknown geometry type.");
    }
@@ -1455,6 +1472,8 @@ int RT0_3DFECollection::DofForGeometry(Geometry::Type GeomType) const
       case Geometry::SQUARE:      return 1;
       case Geometry::TETRAHEDRON: return 0;
       case Geometry::CUBE:        return 0;
+      case Geometry::PRISM:       return 0;
+      case Geometry::PYRAMID:     return 0;
       default:
          mfem_error ("RT0_3DFECollection: unknown geometry type.");
    }
@@ -1730,6 +1749,7 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype)
          H1_dof[Geometry::TETRAHEDRON] = (TriDof*pm3)/3;
          H1_dof[Geometry::CUBE] = QuadDof*pm1;
          H1_dof[Geometry::PRISM] = TriDof*pm1;
+         H1_dof[Geometry::PYRAMID] = 0;
          if (b_type == BasisType::Positive)
          {
             H1_Elements[Geometry::TETRAHEDRON] = new H1Pos_TetrahedronElement(p);
@@ -1743,6 +1763,7 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype)
             H1_Elements[Geometry::CUBE] = new H1_HexahedronElement(p, btype);
             H1_Elements[Geometry::PRISM] = new H1_WedgeElement(p, btype);
          }
+         H1_Elements[Geometry::PYRAMID] = new LinearPyramidFiniteElement;
 
          const int &TetDof = H1_dof[Geometry::TETRAHEDRON];
          TetDofOrd[0] = new int[24*TetDof];
@@ -1837,6 +1858,21 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype)
    }
 }
 
+const FiniteElement *
+H1_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
+{
+   if (GeomType != Geometry::PYRAMID || this->GetOrder() == 1)
+   {
+      return H1_Elements[GeomType];
+   }
+   else
+   {
+      MFEM_ABORT("H1 Pyramid basis functions are not yet supported "
+                 "for order > 1.");
+      return NULL;
+   }
+}
+
 const int *H1_FECollection::DofOrderForOrientation(Geometry::Type GeomType,
                                                    int Or) const
 {
@@ -2076,9 +2112,12 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
          L2_Elements[Geometry::CUBE] = new L2_HexahedronElement(p, btype);
          L2_Elements[Geometry::PRISM] = new L2_WedgeElement(p, btype);
       }
+      L2_Elements[Geometry::PYRAMID] = new P0PyrFiniteElement;
+
       L2_Elements[Geometry::TETRAHEDRON]->SetMapType(map_type);
       L2_Elements[Geometry::CUBE]->SetMapType(map_type);
       L2_Elements[Geometry::PRISM]->SetMapType(map_type);
+      L2_Elements[Geometry::PYRAMID]->SetMapType(map_type);
       // Trace element use the default Gauss-Legendre nodal points for positive basis
       if (b_type == BasisType::Positive)
       {
@@ -2199,6 +2238,21 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
    }
 }
 
+const FiniteElement *
+L2_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
+{
+   if (GeomType != Geometry::PYRAMID || this->GetOrder() == 0)
+   {
+      return L2_Elements[GeomType];
+   }
+   else
+   {
+      MFEM_ABORT("L2 Pyramid basis functions are not yet supported "
+                 "for order > 0.");
+      return NULL;
+   }
+}
+
 const int *L2_FECollection::DofOrderForOrientation(Geometry::Type GeomType,
                                                    int Or) const
 {
@@ -2290,6 +2344,12 @@ RT_FECollection::RT_FECollection(const int order, const int dim,
 
       RT_Elements[Geometry::CUBE] = new RT_HexahedronElement(p, cb_type, ob_type);
       RT_dof[Geometry::CUBE] = 3*p*pp1*pp1;
+
+      RT_Elements[Geometry::PRISM] = new RT0WdgFiniteElement;
+      RT_dof[Geometry::PRISM] = 0;
+
+      RT_Elements[Geometry::PYRAMID] = new RT0PyrFiniteElement(false);
+      RT_dof[Geometry::PYRAMID] = 0;
    }
    else
    {
@@ -2433,6 +2493,22 @@ void RT_FECollection::InitFaces(const int p, const int dim,
    }
 }
 
+const FiniteElement *
+RT_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
+{
+   if ((GeomType != Geometry::PRISM && GeomType != Geometry::PYRAMID) ||
+       this->GetOrder() == 1)
+   {
+      return RT_Elements[GeomType];
+   }
+   else
+   {
+      MFEM_ABORT("RT Wedge and Pyramid basis functions are not yet supported "
+                 "for order > 0.");
+      return NULL;
+   }
+}
+
 const int *RT_FECollection::DofOrderForOrientation(Geometry::Type GeomType,
                                                    int Or) const
 {
@@ -2656,18 +2732,31 @@ ND_FECollection::ND_FECollection(const int p, const int dim,
       {
          for (int i = 0; i + j <= pm2; i++)
          {
-            int k1 = p*pm1 - (p - j)*(pm1 - j) + 2*i;
-            int k2 = p*pm1 - (p - i)*(pm1 - i) + 2*j;
-            // (0,1,2)
-            TriDofOrd[0][k1  ] = k1;
-            TriDofOrd[0][k1+1] = k1 + 1;
-            // (0,2,1)
-            TriDofOrd[5][k1  ] = k2 + 1;
-            TriDofOrd[5][k1+1] = k2;
+            int k0 = p*pm1 - (p - j)*(pm1 - j) + 2*i;
+            int k1 = 2*pm2 - 2*i + ((2*p-3)-j)*j;
+            int k2 = 2*pm2 - 2*j + ((2*p-3)-i)*i;
+            int k3 = p*pm1 - 2 - 3*j - i - (i+j)*(i+j);
+            int k4 = p*pm1 - 2 - 3*i - j - (i+j)*(i+j);
+            int k5 = p*pm1 - (p - i)*(pm1 - i) + 2*j;
 
-            // The other orientations can not be supported with the current
-            // interface. The method Mesh::ReorientTetMesh will ensure that
-            // only orientations 0 and 5 are generated.
+            // (0,1,2)
+            TriDofOrd[0][k0  ] = k0;
+            TriDofOrd[0][k0+1] = k0 + 1;
+            // (1,0,2)
+            TriDofOrd[1][k0  ] = k1;
+            TriDofOrd[1][k0+1] = k1 + 1;
+            // (2,0,1)
+            TriDofOrd[2][k0  ] = k2;
+            TriDofOrd[2][k0+1] = k2 + 1;
+            // (2,1,0)
+            TriDofOrd[3][k0  ] = k3;
+            TriDofOrd[3][k0+1] = k3 + 1;
+            // (1,2,0)
+            TriDofOrd[4][k0  ] = k4;
+            TriDofOrd[4][k0+1] = k4 + 1;
+            // (0,2,1)
+            TriDofOrd[5][k0  ] = k5;
+            TriDofOrd[5][k0+1] = k5 + 1;
          }
       }
    }
@@ -2680,6 +2769,28 @@ ND_FECollection::ND_FECollection(const int p, const int dim,
       // TODO: cb_type and ob_type for tets
       ND_Elements[Geometry::TETRAHEDRON] = new ND_TetrahedronElement(p);
       ND_dof[Geometry::TETRAHEDRON] = p*pm1*pm2/2;
+
+      ND_Elements[Geometry::PRISM] = new Nedelec1WdgFiniteElement;
+      ND_dof[Geometry::PRISM] = 0;
+
+      ND_Elements[Geometry::PYRAMID] = new Nedelec1PyrFiniteElement;
+      ND_dof[Geometry::PYRAMID] = 0;
+   }
+}
+
+const FiniteElement *
+ND_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
+{
+   if ((GeomType != Geometry::PRISM && GeomType != Geometry::PYRAMID) ||
+       this->GetOrder() == 1)
+   {
+      return ND_Elements[GeomType];
+   }
+   else
+   {
+      MFEM_ABORT("ND Wedge and Pyramid basis functions are not yet supported "
+                 "for order > 1.");
+      return NULL;
    }
 }
 
@@ -2692,11 +2803,6 @@ const int *ND_FECollection::DofOrderForOrientation(Geometry::Type GeomType,
    }
    else if (GeomType == Geometry::TRIANGLE)
    {
-      if (Or != 0 && Or != 5)
-      {
-         MFEM_ABORT("triangle face orientation " << Or << " is not supported! "
-                    "Use Mesh::ReorientTetMesh to fix it.");
-      }
       return TriDofOrd[Or%6];
    }
    else if (GeomType == Geometry::SQUARE)

fem/fe_coll.hpp

@@ -228,8 +228,7 @@ public:
                             const int btype = BasisType::GaussLobatto);
 
    virtual const FiniteElement *FiniteElementForGeometry(
-      Geometry::Type GeomType) const
-   { return H1_Elements[GeomType]; }
+      Geometry::Type GeomType) const;
    virtual int DofForGeometry(Geometry::Type GeomType) const
    { return H1_dof[GeomType]; }
    virtual const int *DofOrderForOrientation(Geometry::Type GeomType,
@@ -302,10 +301,7 @@ public:
                    const int map_type = FiniteElement::VALUE);
 
    virtual const FiniteElement *FiniteElementForGeometry(
-      Geometry::Type GeomType) const
-   {
-      return L2_Elements[GeomType];
-   }
+      Geometry::Type GeomType) const;
    virtual int DofForGeometry(Geometry::Type GeomType) const
    {
       if (L2_Elements[GeomType])
@@ -371,8 +367,7 @@ public:
                    const int ob_type = BasisType::GaussLegendre);
 
    virtual const FiniteElement *FiniteElementForGeometry(
-      Geometry::Type GeomType) const
-   { return RT_Elements[GeomType]; }
+      Geometry::Type GeomType) const;
    virtual int DofForGeometry(Geometry::Type GeomType) const
    { return RT_dof[GeomType]; }
    virtual const int *DofOrderForOrientation(Geometry::Type GeomType,
@@ -430,8 +425,7 @@ public:
                    const int ob_type = BasisType::GaussLegendre);
 
    virtual const FiniteElement *
-   FiniteElementForGeometry(Geometry::Type GeomType) const
-   { return ND_Elements[GeomType]; }
+   FiniteElementForGeometry(Geometry::Type GeomType) const;
 
    virtual int DofForGeometry(Geometry::Type GeomType) const
    { return ND_dof[GeomType]; }
@@ -529,9 +523,10 @@ private:
    const BiLinear2DFiniteElement QuadrilateralFE;
    const Linear3DFiniteElement TetrahedronFE;
    const TriLinear3DFiniteElement ParallelepipedFE;
-   const H1_WedgeElement WedgeFE;
+   const LinearWedgeFiniteElement WedgeFE;
+   const LinearPyramidFiniteElement PyramidFE;
 public:
-   LinearFECollection() : FiniteElementCollection(1), WedgeFE(1) { }
+   LinearFECollection() : FiniteElementCollection(1) { }
 
    virtual const FiniteElement *
    FiniteElementForGeometry(Geometry::Type GeomType) const;
@@ -936,10 +931,11 @@ class Const3DFECollection : public FiniteElementCollection
 private:
    const P0TetFiniteElement TetrahedronFE;
    const P0HexFiniteElement ParallelepipedFE;
-   const L2_WedgeElement WedgeFE;
+   const P0WdgFiniteElement WedgeFE;
+   const P0PyrFiniteElement PyramidFE;
 
 public:
-   Const3DFECollection() : FiniteElementCollection(0), WedgeFE(0) { }
+   Const3DFECollection() : FiniteElementCollection(0) { }
 
    virtual const FiniteElement *
    FiniteElementForGeometry(Geometry::Type GeomType) const;
@@ -960,6 +956,8 @@ class LinearDiscont3DFECollection : public FiniteElementCollection
 {
 private:
    const Linear3DFiniteElement TetrahedronFE;
+   const LinearPyramidFiniteElement PyramidFE;
+   const LinearWedgeFiniteElement WedgeFE;
    const TriLinear3DFiniteElement ParallelepipedFE;
 
 public:
@@ -1036,6 +1034,8 @@ class ND1_3DFECollection : public FiniteElementCollection
 private:
    const Nedelec1HexFiniteElement HexahedronFE;
    const Nedelec1TetFiniteElement TetrahedronFE;
+   const Nedelec1WdgFiniteElement WedgeFE;
+   const Nedelec1PyrFiniteElement PyramidFE;
 
 public:
    ND1_3DFECollection() : FiniteElementCollection(1) { }
@@ -1061,6 +1061,8 @@ private:
    const P0QuadFiniteElement QuadrilateralFE;
    const RT0HexFiniteElement HexahedronFE;
    const RT0TetFiniteElement TetrahedronFE;
+   const RT0WdgFiniteElement WedgeFE;
+   const RT0PyrFiniteElement PyramidFE;
 public:
    RT0_3DFECollection() : FiniteElementCollection(1) { }
 

fem/fem.hpp

@@ -16,6 +16,7 @@
 #include "geom.hpp"
 #include "fe.hpp"
 #include "fe_coll.hpp"
+#include "doftrans.hpp"
 #include "eltrans.hpp"
 #include "coefficient.hpp"
 #include "complex_fem.hpp"
@@ -34,6 +35,7 @@
 #include "staticcond.hpp"
 #include "tmop.hpp"
 #include "tmop_tools.hpp"
+#include "tmop_amr.hpp"
 #include "gslib.hpp"
 #include "restriction.hpp"
 #include "quadinterpolator.hpp"
@@ -64,4 +66,9 @@
 #include "adios2datacollection.hpp"
 #endif
 
+#ifdef MFEM_USE_FMS
+#include "fmsconvert.hpp"
+#include "fmsdatacollection.hpp"
+#endif
+
 #endif

fem/fespace.cpp

@@ -58,9 +58,12 @@ DofsToVDofs<Ordering::byVDIM>(int ndofs, int vdim, Array<int> &dofs)
 
 FiniteElementSpace::FiniteElementSpace()
    : mesh(NULL), fec(NULL), vdim(0), ordering(Ordering::byNODES),
-     ndofs(0), nvdofs(0), nedofs(0), nfdofs(0), nbdofs(0), bdofs(NULL),
-     elem_dof(NULL), bdr_elem_dof(NULL), face_dof(NULL),
+     ndofs(0), nvdofs(0), nedofs(0), nfdofs(0), nbdofs(0),
+     bdofs(NULL),
+     elem_dof(NULL), elem_fos(NULL), bdr_elem_dof(NULL), bdr_elem_fos(NULL),
+     face_dof(NULL),
      NURBSext(NULL), own_ext(false),
+     DoFTrans(0), VDoFTrans(vdim, ordering),
      cP(NULL), cR(NULL), cR_hp(NULL), cP_is_set(false),
      Th(Operator::ANY_TYPE),
      sequence(0), mesh_sequence(0), orders_changed(false), relaxed_hp(false)
@@ -69,6 +72,7 @@ FiniteElementSpace::FiniteElementSpace()
 FiniteElementSpace::FiniteElementSpace(const FiniteElementSpace &orig,
                                        Mesh *mesh,
                                        const FiniteElementCollection *fec)
+   : VDoFTrans(orig.vdim, orig.ordering)
 {
    mesh = mesh ? mesh : orig.mesh;
    fec = fec ? fec : orig.fec;
@@ -259,16 +263,36 @@ void FiniteElementSpace::AdjustVDofs (Array<int> &vdofs)
    }
 }
 
-void FiniteElementSpace::GetElementVDofs(int i, Array<int> &vdofs) const
+DofTransformation *
+FiniteElementSpace::GetElementVDofs(int i, Array<int> &vdofs) const
 {
-   GetElementDofs(i, vdofs);
+   DofTransformation * doftrans = GetElementDofs(i, vdofs);
    DofsToVDofs(vdofs);
+   if (vdim == 1 || doftrans == NULL)
+   {
+      return doftrans;
+   }
+   else
+   {
+      VDoFTrans.SetDofTransformation(*doftrans);
+      return &VDoFTrans;
+   }
 }
 
-void FiniteElementSpace::GetBdrElementVDofs(int i, Array<int> &vdofs) const
+DofTransformation *
+FiniteElementSpace::GetBdrElementVDofs(int i, Array<int> &vdofs) const
 {
-   GetBdrElementDofs(i, vdofs);
+   DofTransformation * doftrans = GetBdrElementDofs(i, vdofs);
    DofsToVDofs(vdofs);
+   if (vdim == 1 || doftrans == NULL)
+   {
+      return doftrans;
+   }
+   else
+   {
+      VDoFTrans.SetDofTransformation(*doftrans);
+      return &VDoFTrans;
+   }
 }
 
 void FiniteElementSpace::GetFaceVDofs(int i, Array<int> &vdofs) const
@@ -307,21 +331,39 @@ void FiniteElementSpace::BuildElementToDofTable() const
 
    // TODO: can we call GetElementDofs only once per element?
    Table *el_dof = new Table;
+   Table *el_fos = (mesh->Dimension() > 2) ? (new Table) : NULL;
    Array<int> dofs;
+   Array<int> F, Fo;
    el_dof -> MakeI (mesh -> GetNE());
+   if (el_fos) { el_fos -> MakeI (mesh -> GetNE()); }
    for (int i = 0; i < mesh -> GetNE(); i++)
    {
       GetElementDofs (i, dofs);
       el_dof -> AddColumnsInRow (i, dofs.Size());
+
+      if (el_fos)
+      {
+         mesh->GetElementFaces(i, F, Fo);
+         el_fos -> AddColumnsInRow (i, Fo.Size());
+      }
    }
    el_dof -> MakeJ();
+   if (el_fos) { el_fos -> MakeJ(); }
    for (int i = 0; i < mesh -> GetNE(); i++)
    {
       GetElementDofs (i, dofs);
       el_dof -> AddConnections (i, (int *)dofs, dofs.Size());
+
+      if (el_fos)
+      {
+         mesh->GetElementFaces(i, F, Fo);
+         el_fos -> AddConnections (i, (int *)Fo, Fo.Size());
+      }
    }
    el_dof -> ShiftUpI();
+   if (el_fos) { el_fos -> ShiftUpI(); }
    elem_dof = el_dof;
+   elem_fos = el_fos;
 }
 
 void FiniteElementSpace::BuildBdrElementToDofTable() const
@@ -375,7 +417,9 @@ void FiniteElementSpace::BuildFaceToDofTable() const
 void FiniteElementSpace::RebuildElementToDofTable()
 {
    delete elem_dof;
+   delete elem_fos;
    elem_dof = NULL;
+   elem_fos = NULL;
    BuildElementToDofTable();
 }
 
@@ -1315,8 +1359,10 @@ const FaceQuadratureInterpolator
 
 SparseMatrix *FiniteElementSpace::RefinementMatrix_main(
    const int coarse_ndofs, const Table &coarse_elem_dof,
-   const DenseTensor localP[]) const
+   const Table *coarse_elem_fos, const DenseTensor localP[]) const
 {
+   /// TODO: Implement DofTransformation support
+
    MFEM_VERIFY(mesh->GetLastOperation() == Mesh::REFINE, "");
 
    Array<int> dofs, coarse_dofs, coarse_vdofs;
@@ -1399,7 +1445,8 @@ void FiniteElementSpace::GetLocalRefinementMatrices(
 }
 
 SparseMatrix* FiniteElementSpace::RefinementMatrix(int old_ndofs,
-                                                   const Table* old_elem_dof)
+                                                   const Table* old_elem_dof,
+                                                   const Table* old_elem_fos)
 {
    MFEM_VERIFY(GetNE() >= old_elem_dof->Size(),
                "Previous mesh is not coarser.");
@@ -1412,13 +1459,16 @@ SparseMatrix* FiniteElementSpace::RefinementMatrix(int old_ndofs,
       GetLocalRefinementMatrices(elem_geoms[i], localP[elem_geoms[i]]);
    }
 
-   return RefinementMatrix_main(old_ndofs, *old_elem_dof, localP);
+   return RefinementMatrix_main(old_ndofs, *old_elem_dof, old_elem_fos,
+                                localP);
 }
 
 FiniteElementSpace::RefinementOperator::RefinementOperator
-(const FiniteElementSpace* fespace, Table* old_elem_dof, int old_ndofs)
+(const FiniteElementSpace* fespace, Table* old_elem_dof, Table* old_elem_fos,
+ int old_ndofs)
    : fespace(fespace)
    , old_elem_dof(old_elem_dof)
+   , old_elem_fos(old_elem_fos)
 {
    MFEM_VERIFY(fespace->GetNE() >= old_elem_dof->Size(),
                "Previous mesh is not coarser.");
@@ -1432,12 +1482,14 @@ FiniteElementSpace::RefinementOperator::RefinementOperator
    {
       fespace->GetLocalRefinementMatrices(elem_geoms[i], localP[elem_geoms[i]]);
    }
+
+   ConstructDoFTrans();
 }
 
 FiniteElementSpace::RefinementOperator::RefinementOperator(
    const FiniteElementSpace *fespace, const FiniteElementSpace *coarse_fes)
    : Operator(fespace->GetVSize(), coarse_fes->GetVSize()),
-     fespace(fespace), old_elem_dof(NULL)
+     fespace(fespace), old_elem_dof(NULL), old_elem_fos(NULL)
 {
    Mesh::GeometryList elem_geoms(*fespace->GetMesh());
 
@@ -1449,11 +1501,50 @@ FiniteElementSpace::RefinementOperator::RefinementOperator(
 
    // Make a copy of the coarse elem_dof Table.
    old_elem_dof = new Table(coarse_fes->GetElementToDofTable());
+
+   // Make a copy of the coarse elem_fos Table if it exists.
+   if (coarse_fes->GetElementToFaceOrientationTable())
+   {
+      old_elem_fos = new Table(*coarse_fes->GetElementToFaceOrientationTable());
+   }
+
+   ConstructDoFTrans();
 }
 
 FiniteElementSpace::RefinementOperator::~RefinementOperator()
 {
    delete old_elem_dof;
+   delete old_elem_fos;
+}
+
+void FiniteElementSpace::RefinementOperator
+::ConstructDoFTrans()
+{
+   old_DoFTrans.SetSize(Geometry::NUM_GEOMETRIES);
+   for (int i=0; i<old_DoFTrans.Size(); i++)
+   {
+      old_DoFTrans[i] = NULL;
+   }
+
+   const FiniteElementCollection *fec = fespace->FEColl();
+   if (dynamic_cast<const ND_FECollection*>(fec))
+   {
+      const FiniteElement * nd_tri =
+         fec->FiniteElementForGeometry(Geometry::TRIANGLE);
+      if (nd_tri)
+      {
+         old_DoFTrans[Geometry::TRIANGLE] =
+            new ND_TriDofTransformation(nd_tri->GetOrder());
+      }
+
+      const FiniteElement * nd_tet =
+         fec->FiniteElementForGeometry(Geometry::TETRAHEDRON);
+      if (nd_tet)
+      {
+         old_DoFTrans[Geometry::TETRAHEDRON] =
+            new ND_TetDofTransformation(nd_tet->GetOrder());
+      }
+   }
 }
 
 void FiniteElementSpace::RefinementOperator
@@ -1462,7 +1553,7 @@ void FiniteElementSpace::RefinementOperator
    Mesh* mesh = fespace->GetMesh();
    const CoarseFineTransformations &rtrans = mesh->GetRefinementTransforms();
 
-   Array<int> dofs, vdofs, old_dofs, old_vdofs;
+   Array<int> dofs, vdofs, old_dofs, old_vdofs, old_Fo;
 
    int vdim = fespace->GetVDim();
    int old_ndofs = width / vdim;
@@ -1477,18 +1568,53 @@ void FiniteElementSpace::RefinementOperator
 
       subY.SetSize(lP.Height());
 
-      fespace->GetElementDofs(k, dofs);
+      DofTransformation *doftrans = fespace->GetElementDofs(k, dofs);
       old_elem_dof->GetRow(emb.parent, old_dofs);
 
-      for (int vd = 0; vd < vdim; vd++)
+      if (!doftrans)
       {
-         dofs.Copy(vdofs);
-         fespace->DofsToVDofs(vd, vdofs);
-         old_dofs.Copy(old_vdofs);
-         fespace->DofsToVDofs(vd, old_vdofs, old_ndofs);
-         x.GetSubVector(old_vdofs, subX);
-         lP.Mult(subX, subY);
-         y.SetSubVector(vdofs, subY);
+         for (int vd = 0; vd < vdim; vd++)
+         {
+            dofs.Copy(vdofs);
+            fespace->DofsToVDofs(vd, vdofs);
+            old_dofs.Copy(old_vdofs);
+            fespace->DofsToVDofs(vd, old_vdofs, old_ndofs);
+            x.GetSubVector(old_vdofs, subX);
+            lP.Mult(subX, subY);
+            y.SetSubVector(vdofs, subY);
+         }
+      }
+      else
+      {
+         old_elem_fos->GetRow(emb.parent, old_Fo);
+         old_DoFTrans[geom]->SetFaceOrientations(old_Fo);
+
+         DofTransformation *new_doftrans = NULL;
+         VDofTransformation *vdoftrans =
+            dynamic_cast<VDofTransformation*>(doftrans);
+         if (vdoftrans)
+         {
+            new_doftrans = doftrans;
+            doftrans = vdoftrans->GetDofTransformation();
+         }
+
+         for (int vd = 0; vd < vdim; vd++)
+         {
+            dofs.Copy(vdofs);
+            fespace->DofsToVDofs(vd, vdofs);
+            old_dofs.Copy(old_vdofs);
+            fespace->DofsToVDofs(vd, old_vdofs, old_ndofs);
+            x.GetSubVector(old_vdofs, subX);
+            old_DoFTrans[geom]->InvTransformPrimal(subX);
+            lP.Mult(subX, subY);
+            doftrans->TransformPrimal(subY);
+            y.SetSubVector(vdofs, subY);
+         }
+
+         if (vdoftrans)
+         {
+            doftrans = new_doftrans;
+         }
       }
    }
 }
@@ -1504,12 +1630,12 @@ void FiniteElementSpace::RefinementOperator
    Array<char> processed(fespace->GetVSize());
    processed = 0;
 
-   Array<int> f_dofs, c_dofs, f_vdofs, c_vdofs;
+   Array<int> f_dofs, c_dofs, f_vdofs, c_vdofs, old_Fo;
 
    int vdim = fespace->GetVDim();
    int old_ndofs = width / vdim;
 
-   Vector subY, subX;
+   Vector subY, subX, subYt, subXt;
 
    for (int k = 0; k < mesh->GetNE(); k++)
    {
@@ -1517,30 +1643,77 @@ void FiniteElementSpace::RefinementOperator
       const Geometry::Type geom = mesh->GetElementBaseGeometry(k);
       const DenseMatrix &lP = localP[geom](emb.matrix);
 
-      fespace->GetElementDofs(k, f_dofs);
+      DofTransformation * doftrans = fespace->GetElementDofs(k, f_dofs);
       old_elem_dof->GetRow(emb.parent, c_dofs);
 
-      subY.SetSize(lP.Width());
-
-      for (int vd = 0; vd < vdim; vd++)
+      if (!doftrans)
       {
-         f_dofs.Copy(f_vdofs);
-         fespace->DofsToVDofs(vd, f_vdofs);
-         c_dofs.Copy(c_vdofs);
-         fespace->DofsToVDofs(vd, c_vdofs, old_ndofs);
+         subY.SetSize(lP.Width());
 
-         x.GetSubVector(f_vdofs, subX);
-
-         for (int p = 0; p < f_dofs.Size(); ++p)
+         for (int vd = 0; vd < vdim; vd++)
          {
-            if (processed[DecodeDof(f_dofs[p])])
+            f_dofs.Copy(f_vdofs);
+            fespace->DofsToVDofs(vd, f_vdofs);
+            c_dofs.Copy(c_vdofs);
+            fespace->DofsToVDofs(vd, c_vdofs, old_ndofs);
+
+            x.GetSubVector(f_vdofs, subX);
+
+            for (int p = 0; p < f_dofs.Size(); ++p)
             {
-               subX[p] = 0.0;
+               if (processed[DecodeDof(f_dofs[p])])
+               {
+                  subX[p] = 0.0;
+               }
             }
+
+            lP.MultTranspose(subX, subY);
+            y.AddElementVector(c_vdofs, subY);
+         }
+      }
+      else
+      {
+         subYt.SetSize(lP.Width());
+
+         old_elem_fos->GetRow(emb.parent, old_Fo);
+         old_DoFTrans[geom]->SetFaceOrientations(old_Fo);
+
+         DofTransformation *new_doftrans = NULL;
+         VDofTransformation *vdoftrans =
+            dynamic_cast<VDofTransformation*>(doftrans);
+         if (vdoftrans)
+         {
+            new_doftrans = doftrans;
+            doftrans = vdoftrans->GetDofTransformation();
          }
 
-         lP.MultTranspose(subX, subY);
-         y.AddElementVector(c_vdofs, subY);
+         for (int vd = 0; vd < vdim; vd++)
+         {
+            f_dofs.Copy(f_vdofs);
+            fespace->DofsToVDofs(vd, f_vdofs);
+            c_dofs.Copy(c_vdofs);
+            fespace->DofsToVDofs(vd, c_vdofs, old_ndofs);
+
+            x.GetSubVector(f_vdofs, subX);
+            old_DoFTrans[geom]->InvTransformPrimal(subX);
+
+            for (int p = 0; p < f_dofs.Size(); ++p)
+            {
+               if (processed[DecodeDof(f_dofs[p])])
+               {
+                  subX[p] = 0.0;
+               }
+            }
+
+            lP.MultTranspose(subX, subY);
+            doftrans->TransformPrimal(subY);
+            y.AddElementVector(c_vdofs, subY);
+         }
+
+         if (vdoftrans)
+         {
+            doftrans = new_doftrans;
+         }
       }
 
       for (int p = 0; p < f_dofs.Size(); ++p)
@@ -1550,6 +1723,7 @@ void FiniteElementSpace::RefinementOperator
    }
 }
 
+/// TODO: Implement DofTransformation support
 FiniteElementSpace::DerefinementOperator::DerefinementOperator(
    const FiniteElementSpace *f_fes, const FiniteElementSpace *c_fes,
    BilinearFormIntegrator *mass_integ)
@@ -1707,8 +1881,11 @@ void FiniteElementSpace::GetLocalDerefinementMatrices(Geometry::Type geom,
 }
 
 SparseMatrix* FiniteElementSpace::DerefinementMatrix(int old_ndofs,
-                                                     const Table* old_elem_dof)
+                                                     const Table* old_elem_dof,
+                                                     const Table* old_elem_fos)
 {
+   /// TODO: Implement DofTransformation support
+
    MFEM_VERIFY(Nonconforming(), "Not implemented for conforming meshes.");
    MFEM_VERIFY(old_ndofs, "Missing previous (finer) space.");
    MFEM_VERIFY(ndofs <= old_ndofs, "Previous space is not finer.");
@@ -1814,6 +1991,7 @@ void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
    this->ordering = (Ordering::Type) ordering;
 
    elem_dof = NULL;
+   elem_fos = NULL;
    face_dof = NULL;
 
    sequence = 0;
@@ -1841,6 +2019,8 @@ void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
       UpdateNURBS();
       cP = cR = cR_hp = NULL;
       cP_is_set = false;
+
+      ConstructDoFTrans();
    }
    else
    {
@@ -1848,9 +2028,41 @@ void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
       own_ext = 0;
       Construct();
    }
+
    BuildElementToDofTable();
 }
 
+void FiniteElementSpace::ConstructDoFTrans()
+{
+   DestroyDoFTrans();
+
+   VDoFTrans.SetVDim(vdim);
+   DoFTrans.SetSize(Geometry::NUM_GEOMETRIES);
+   for (int i=0; i<DoFTrans.Size(); i++)
+   {
+      DoFTrans[i] = NULL;
+   }
+   if (mesh->Dimension() < 3) { return; }
+   if (dynamic_cast<const ND_FECollection*>(fec))
+   {
+      const FiniteElement * nd_tri =
+         fec->FiniteElementForGeometry(Geometry::TRIANGLE);
+      if (nd_tri)
+      {
+         DoFTrans[Geometry::TRIANGLE] =
+            new ND_TriDofTransformation(nd_tri->GetOrder());
+      }
+
+      const FiniteElement * nd_tet =
+         fec->FiniteElementForGeometry(Geometry::TETRAHEDRON);
+      if (nd_tet)
+      {
+         DoFTrans[Geometry::TETRAHEDRON] =
+            new ND_TetDofTransformation(nd_tet->GetOrder());
+      }
+   }
+}
+
 NURBSExtension *FiniteElementSpace::StealNURBSext()
 {
    if (NURBSext && !own_ext)
@@ -1946,7 +2158,9 @@ void FiniteElementSpace::Construct()
                "Variable order space requires a nonconforming mesh.");
 
    elem_dof = NULL;
+   elem_fos = NULL;
    bdr_elem_dof = NULL;
+   bdr_elem_fos = NULL;
    face_dof = NULL;
 
    ndofs = 0;
@@ -2044,6 +2258,8 @@ void FiniteElementSpace::Construct()
 
    ndofs = nvdofs + nedofs + nfdofs + nbdofs;
 
+   ConstructDoFTrans();
+
    // record the current mesh sequence number to detect refinement etc.
    mesh_sequence = mesh->GetSequence();
 
@@ -2295,14 +2511,22 @@ int FiniteElementSpace::GetNVariants(int entity, int index) const
 static const char* msg_orders_changed =
    "Element orders changed, you need to Update() the space first.";
 
-void FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
+DofTransformation *
+FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
 {
    MFEM_VERIFY(!orders_changed, msg_orders_changed);
 
    if (elem_dof)
    {
       elem_dof->GetRow(elem, dofs);
-      return;
+
+      if (DoFTrans[mesh->GetElementBaseGeometry(elem)])
+      {
+         Array<int> Fo;
+         elem_fos -> GetRow (elem, Fo);
+         DoFTrans[mesh->GetElementBaseGeometry(elem)]->SetFaceOrientations(Fo);
+      }
+      return DoFTrans[mesh->GetElementBaseGeometry(elem)];
    }
 
    Array<int> V, E, Eo, F, Fo; // TODO: LocalArray
@@ -2326,6 +2550,11 @@ void FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
       {
          nfd += fec->GetNumDof(mesh->GetFaceGeometry(F[i]), order);
       }
+      if (DoFTrans[mesh->GetElementBaseGeometry(elem)])
+      {
+         DoFTrans[mesh->GetElementBaseGeometry(elem)]
+         -> SetFaceOrientations(Fo);
+      }
    }
 
    dofs.SetSize(0);
@@ -2383,6 +2612,7 @@ void FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
          dofs.Append(bbase + j);
       }
    }
+   return DoFTrans[mesh->GetElementBaseGeometry(elem)];
 }
 
 const FiniteElement *FiniteElementSpace::GetFE(int i) const
@@ -2415,18 +2645,27 @@ const FiniteElement *FiniteElementSpace::GetFE(int i) const
    return FE;
 }
 
-void FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const
+DofTransformation *
+FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const
 {
    MFEM_VERIFY(!orders_changed, msg_orders_changed);
 
    if (bdr_elem_dof)
    {
       bdr_elem_dof->GetRow(bel, dofs);
-      return;
+
+      if (DoFTrans[mesh->GetBdrElementBaseGeometry(bel)])
+      {
+         Array<int> Fo;
+         bdr_elem_fos -> GetRow (bel, Fo);
+         DoFTrans[mesh->GetBdrElementBaseGeometry(bel)]->
+         SetFaceOrientations(Fo);
+      }
+      return DoFTrans[mesh->GetBdrElementBaseGeometry(bel)];
    }
 
-   Array<int> V, E, Eo; // TODO: LocalArray
-   int F, Fo;
+   Array<int> V, E, Eo, Fo; // TODO: LocalArray
+   int F, oF;
 
    int dim = mesh->Dimension();
    auto geom = mesh->GetBdrElementGeometry(bel);
@@ -2445,7 +2684,17 @@ void FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const
 
    if (nv) { mesh->GetBdrElementVertices(bel, V); }
    if (ne) { mesh->GetBdrElementEdges(bel, E, Eo); }
-   if (nf) { mesh->GetBdrElementFace(bel, &F, &Fo); }
+   if (nf)
+   {
+      mesh->GetBdrElementFace(bel, &F, &oF);
+
+      if (DoFTrans[mesh->GetBdrElementBaseGeometry(bel)])
+      {
+         Fo.Append(oF);
+         DoFTrans[mesh->GetBdrElementBaseGeometry(bel)]->
+         SetFaceOrientations(Fo);
+      }
+   }
 
    dofs.SetSize(0);
    dofs.Reserve(nv*V.Size() + ne*E.Size() + nf);
@@ -2478,13 +2727,15 @@ void FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const
    if (nf) // face DOFs
    {
       int fbase = (var_face_dofs.Size() > 0) ? FindFaceDof(F, nf) : F*nf;
-      const int *ind = fec->GetDofOrdering(geom, order, Fo);
+      const int *ind = fec->GetDofOrdering(geom, order, oF);
 
       for (int j = 0; j < nf; j++)
       {
          dofs.Append(EncodeDof(nvdofs + nedofs + fbase, ind[j]));
       }
    }
+
+   return DoFTrans[mesh->GetBdrElementBaseGeometry(bel)];
 }
 
 int FiniteElementSpace::GetFaceDofs(int face, Array<int> &dofs,
@@ -2794,6 +3045,8 @@ void FiniteElementSpace::Destroy()
    }
    E2BFQ_array.SetSize(0);
 
+   DestroyDoFTrans();
+
    dof_elem_array.DeleteAll();
    dof_ldof_array.DeleteAll();
 
@@ -2806,7 +3059,9 @@ void FiniteElementSpace::Destroy()
    else
    {
       delete elem_dof;
+      delete elem_fos;
       delete bdr_elem_dof;
+      delete bdr_elem_fos;
       delete face_dof;
 
       delete [] bdofs;
@@ -2814,6 +3069,15 @@ void FiniteElementSpace::Destroy()
    ceed::RemoveBasisAndRestriction(this);
 }
 
+void FiniteElementSpace::DestroyDoFTrans()
+{
+   for (int i = 0; i < DoFTrans.Size(); i++)
+   {
+      delete DoFTrans[i];
+   }
+   DoFTrans.SetSize(0);
+}
+
 void FiniteElementSpace::GetTransferOperator(
    const FiniteElementSpace &coarse_fes, OperatorHandle &T) const
 {
@@ -2831,6 +3095,8 @@ void FiniteElementSpace::GetTransferOperator(
       }
       T.Reset(RefinementMatrix_main(coarse_fes.GetNDofs(),
                                     coarse_fes.GetElementToDofTable(),
+                                    coarse_fes.
+                                    GetElementToFaceOrientationTable(),
                                     localP));
    }
    else
@@ -2933,6 +3199,7 @@ void FiniteElementSpace::Update(bool want_transform)
    }
 
    Table* old_elem_dof = NULL;
+   Table* old_elem_fos = NULL;
    int old_ndofs;
    bool old_orders_changed = orders_changed;
 
@@ -2940,7 +3207,9 @@ void FiniteElementSpace::Update(bool want_transform)
    if (want_transform)
    {
       old_elem_dof = elem_dof;
+      old_elem_fos = elem_fos;
       elem_dof = NULL;
+      elem_fos = NULL;
       old_ndofs = ndofs;
    }
 
@@ -2966,15 +3235,18 @@ void FiniteElementSpace::Update(bool want_transform)
          {
             if (Th.Type() != Operator::MFEM_SPARSEMAT)
             {
-               Th.Reset(new RefinementOperator(this, old_elem_dof, old_ndofs));
+               Th.Reset(new RefinementOperator(this, old_elem_dof,
+                                               old_elem_fos, old_ndofs));
                // The RefinementOperator takes ownership of 'old_elem_dof', so
                // we no longer own it:
                old_elem_dof = NULL;
+               old_elem_fos = NULL;
             }
             else
             {
                // calculate fully assembled matrix
-               Th.Reset(RefinementMatrix(old_ndofs, old_elem_dof));
+               Th.Reset(RefinementMatrix(old_ndofs, old_elem_dof,
+                                         old_elem_fos));
             }
             break;
          }
@@ -2982,7 +3254,7 @@ void FiniteElementSpace::Update(bool want_transform)
          case Mesh::DEREFINE:
          {
             BuildConformingInterpolation();
-            Th.Reset(DerefinementMatrix(old_ndofs, old_elem_dof));
+            Th.Reset(DerefinementMatrix(old_ndofs, old_elem_dof, old_elem_fos));
             if (cP && cR)
             {
                Th.SetOperatorOwner(false);
@@ -2997,6 +3269,7 @@ void FiniteElementSpace::Update(bool want_transform)
       }
 
       delete old_elem_dof;
+      delete old_elem_fos;
    }
 }
 

fem/fespace.hpp

@@ -16,6 +16,7 @@
 #include "../linalg/sparsemat.hpp"
 #include "../mesh/mesh.hpp"
 #include "fe_coll.hpp"
+#include "doftrans.hpp"
 #include "restriction.hpp"
 #include <iostream>
 #include <unordered_map>
@@ -128,7 +129,9 @@ protected:
 
    // precalculated DOFs for each element, boundary element, and face
    mutable Table *elem_dof; // owned (except in NURBS FE space)
+   mutable Table *elem_fos; // face orientations by element index
    mutable Table *bdr_elem_dof; // owned (except in NURBS FE space)
+   mutable Table *bdr_elem_fos; // bdr face orientations by bdr element index
    mutable Table *face_dof; // owned; in var-order space contains variant 0 DOFs
 
    Array<int> dof_elem_array, dof_ldof_array;
@@ -137,6 +140,9 @@ protected:
    int own_ext;
    mutable Array<int> face_to_be; // NURBS FE space only
 
+   Array<DofTransformation*> DoFTrans;
+   mutable VDofTransformation VDoFTrans;
+
    /** Matrix representing the prolongation from the global conforming dofs to
        a set of intermediate partially conforming dofs, e.g. the dofs associated
        with a "cut" space on a non-conforming mesh. */
@@ -189,6 +195,9 @@ protected:
    void Construct();
    void Destroy();
 
+   void ConstructDoFTrans();
+   void DestroyDoFTrans();
+
    void BuildElementToDofTable() const;
    void BuildBdrElementToDofTable() const;
    void BuildFaceToDofTable() const;
@@ -283,12 +292,19 @@ protected:
       const FiniteElementSpace* fespace;
       DenseTensor localP[Geometry::NumGeom];
       Table* old_elem_dof; // Owned.
+      Table* old_elem_fos; // Owned.
+
+      Array<DofTransformation*> old_DoFTrans;
+      mutable VDofTransformation old_VDoFTrans;
+
+      void ConstructDoFTrans();
 
    public:
       /** Construct the operator based on the elem_dof table of the original
           (coarse) space. The class takes ownership of the table. */
       RefinementOperator(const FiniteElementSpace* fespace,
-                         Table *old_elem_dof/*takes ownership*/, int old_ndofs);
+                         Table *old_elem_dof/*takes ownership*/,
+                         Table *old_elem_fos/*takes ownership*/, int old_ndofs);
       RefinementOperator(const FiniteElementSpace *fespace,
                          const FiniteElementSpace *coarse_fes);
       virtual void Mult(const Vector &x, Vector &y) const;
@@ -302,6 +318,7 @@ protected:
       const FiniteElementSpace *fine_fes; // Not owned.
       DenseTensor localR[Geometry::NumGeom];
       Table *coarse_elem_dof; // Owned.
+      // Table *coarse_elem_fos; // Owned.
       Table coarse_to_fine;
       Array<int> coarse_to_ref_type;
       Array<Geometry::Type> ref_type_to_geom;
@@ -323,6 +340,7 @@ protected:
        the same vector dimension, vdim. */
    SparseMatrix *RefinementMatrix_main(const int coarse_ndofs,
                                        const Table &coarse_elem_dof,
+                                       const Table *coarse_elem_fos,
                                        const DenseTensor localP[]) const;
 
    void GetLocalRefinementMatrices(Geometry::Type geom,
@@ -333,10 +351,12 @@ protected:
    /** Calculate explicit GridFunction interpolation matrix (after mesh
        refinement). NOTE: consider using the RefinementOperator class instead
        of the fully assembled matrix, which can take a lot of memory. */
-   SparseMatrix* RefinementMatrix(int old_ndofs, const Table* old_elem_dof);
+   SparseMatrix* RefinementMatrix(int old_ndofs, const Table* old_elem_dof,
+                                  const Table* old_elem_fos);
 
    /// Calculate GridFunction restriction matrix after mesh derefinement.
-   SparseMatrix* DerefinementMatrix(int old_ndofs, const Table* old_elem_dof);
+   SparseMatrix* DerefinementMatrix(int old_ndofs, const Table* old_elem_dof,
+                                    const Table* old_elem_fos);
 
    /** @brief Return in @a localP the local refinement matrices that map
        between fespaces after mesh refinement. */
@@ -614,10 +634,11 @@ public:
    int GetBdrAttribute(int i) const { return mesh->GetBdrAttribute(i); }
 
    /// Returns indices of degrees of freedom of element 'elem'.
-   virtual void GetElementDofs(int elem, Array<int> &dofs) const;
+   virtual DofTransformation *GetElementDofs(int elem, Array<int> &dofs) const;
 
    /// Returns indices of degrees of freedom for boundary element 'bel'.
-   virtual void GetBdrElementDofs(int bel, Array<int> &dofs) const;
+   virtual DofTransformation *GetBdrElementDofs(int bel,
+                                                Array<int> &dofs) const;
 
    /** @brief Returns the indices of the degrees of freedom for the specified
        face, including the DOFs for the edges and the vertices of the face. */
@@ -666,10 +687,10 @@ public:
    static void AdjustVDofs(Array<int> &vdofs);
 
    /// Returns indexes of degrees of freedom in array dofs for i'th element.
-   void GetElementVDofs(int i, Array<int> &vdofs) const;
+   DofTransformation *GetElementVDofs(int i, Array<int> &vdofs) const;
 
    /// Returns indexes of degrees of freedom for i'th boundary element.
-   void GetBdrElementVDofs(int i, Array<int> &vdofs) const;
+   DofTransformation *GetBdrElementVDofs(int i, Array<int> &vdofs) const;
 
    /// Returns indexes of degrees of freedom for i'th face element (2D and 3D).
    void GetFaceVDofs(int i, Array<int> &vdofs) const;
@@ -695,6 +716,8 @@ public:
        is preserved. */
    void ReorderElementToDofTable();
 
+   const Table *GetElementToFaceOrientationTable() const { return elem_fos; }
+
    /** @brief Return a reference to the internal Table that stores the lists of
        scalar dofs, for each mesh element, as returned by GetElementDofs(). */
    const Table &GetElementToDofTable() const { return *elem_dof; }

fem/fmsconvert.hpp

@@ -0,0 +1,46 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef FMS_CONVERT
+#define FMS_CONVERT
+
+#include "../config/config.hpp"
+#include "datacollection.hpp"
+
+#ifdef MFEM_USE_FMS
+#include <fms.h>
+
+namespace mfem
+{
+
+/** In-memory conversion of FMS data collection to an MFEM data collection.
+    @param dc The FMS data collection to convert.
+    @param[out] mfem_dc A pointer to a new MFEM DataCollection containing the
+    FMS data.
+    @return 0 on success; non-zero on failure.
+*/
+int FmsDataCollectionToDataCollection(FmsDataCollection dc,
+                                      DataCollection **mfem_dc);
+
+/** In-memory conversion of MFEM data collection to an FMS data collection.
+    @param mfem_dc The MFEM data collection to convert.
+    @param[out] dc A pointer to a new FmsDataCollection containing the MFEM
+    data.
+    @return 0 on success; non-zero on failure.
+*/
+int DataCollectionToFmsDataCollection(DataCollection *mfem_dc,
+                                      FmsDataCollection *dc);
+
+} // namespace mfem
+
+#endif
+
+#endif

fem/fmsdatacollection.cpp

@@ -0,0 +1,167 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../config/config.hpp"
+
+#ifdef MFEM_USE_FMS
+
+#include "fem.hpp"
+#include "../general/text.hpp"
+
+#include <fmsio.h>
+
+#include <string>
+#include <sstream>
+
+namespace mfem
+{
+
+// class FMSDataCollection implementation
+
+FMSDataCollection::FMSDataCollection(const std::string& coll_name,
+                                     Mesh *mesh)
+   : DataCollection(coll_name, mesh),
+     fms_protocol("ascii")
+{
+   appendRankToFileName = false; // always include rank in file names
+   cycle = 0;                   // always include cycle in directory names
+}
+
+#ifdef MFEM_USE_MPI
+
+FMSDataCollection::FMSDataCollection(MPI_Comm comm,
+                                     const std::string& coll_name,
+                                     Mesh *mesh)
+   : DataCollection(coll_name, mesh),
+     fms_protocol("ascii")
+{
+   m_comm = comm;
+   MPI_Comm_rank(comm, &myid);
+   MPI_Comm_size(comm, &num_procs);
+   appendRankToFileName = true; // always include rank in file names
+   cycle = 0;                   // always include cycle in directory names
+}
+
+#endif
+
+FMSDataCollection::~FMSDataCollection()
+{
+   // empty
+}
+
+void FMSDataCollection::Save()
+{
+   // Convert this to FmsDataCollection.
+
+   FmsDataCollection dc;
+   if (DataCollectionToFmsDataCollection(this, &dc) == 0)
+   {
+      std::string root(RootFileName());
+      int err = FmsIOWrite(root.c_str(), fms_protocol.c_str(), dc);
+      FmsDataCollectionDestroy(&dc);
+      if (err)
+      {
+         MFEM_ABORT("Error creating FMS file: " << root);
+      }
+   }
+   else
+   {
+      MFEM_ABORT("Error converting data collection");
+   }
+}
+
+void FMSDataCollection::Load(int cycle)
+{
+   DeleteAll();
+   this->cycle = cycle;
+
+   FmsDataCollection dc;
+   std::string root(RootFileName());
+   int err = FmsIORead(root.c_str(), fms_protocol.c_str(), &dc);
+
+   if (err == 0)
+   {
+      DataCollection *mdc = nullptr;
+      if (FmsDataCollectionToDataCollection(dc,&mdc) == 0)
+      {
+         // Tell the data collection we read that it does not own data.
+         // We will steal its data.
+         mdc->SetOwnData(false);
+
+         SetCycle(mdc->GetCycle());
+         SetTime(mdc->GetTime());
+         SetTimeStep(mdc->GetTimeStep());
+         name = mdc->GetCollectionName();
+
+         // Set mdc's mesh as our mesh.
+         SetMesh(mdc->GetMesh());
+
+         // Set mdc's fields/qfields as ours.
+         std::vector<std::string> names;
+         for (const auto &pair : mdc->GetFieldMap())
+         {
+            names.push_back(pair.first);
+            RegisterField(pair.first, pair.second);
+         }
+         for (const auto &name : names)
+         {
+            mdc->DeregisterField(name);
+         }
+
+         names.clear();
+         for (const auto &pair : mdc->GetQFieldMap())
+         {
+            names.push_back(pair.first);
+            RegisterQField(pair.first, pair.second);
+         }
+         for (const auto &name : names)
+         {
+            mdc->DeregisterField(name);
+         }
+
+         // Indicate that we own the data.
+         SetOwnData(true);
+
+         // Delete mdc. We stole its contents.
+         delete mdc;
+      }
+      FmsDataCollectionDestroy(&dc);
+   }
+   else
+   {
+      MFEM_ABORT("Error reading data collection: " << root);
+   }
+}
+
+void FMSDataCollection::SetProtocol(const std::string &protocol)
+{
+   fms_protocol = protocol;
+}
+
+std::string FMSDataCollection::RootFileName()
+{
+   std::string res;
+   if (pad_digits_cycle)
+   {
+      res = prefix_path + name + "_" +
+            to_padded_string(cycle, pad_digits_cycle) +
+            ".fms";
+   }
+   else
+   {
+      res = prefix_path + name + ".fms";
+   }
+   return res;
+}
+
+} // namespace mfem
+
+#endif

fem/fmsdatacollection.hpp

@@ -0,0 +1,74 @@
+// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_FMSDATACOLLECTION
+#define MFEM_FMSDATACOLLECTION
+
+#include "../config/config.hpp"
+
+#ifdef MFEM_USE_FMS
+
+#include "datacollection.hpp"
+#include <fms.h>
+
+namespace mfem
+{
+
+/** @brief Data collection that uses FMS. */
+/** FMSDataCollection lets MFEM read/write data using FMS.
+
+    For more information, see:
+    - FMS project, https://ceed.exascaleproject.org/fms/
+*/
+
+/// Data collection with FMS I/O routines
+class FMSDataCollection : public DataCollection
+{
+protected:
+   // file name helpers
+
+   /// Returns file name for the current cycle
+   std::string RootFileName();
+
+   // holds currently active i/o protocol
+   std::string fms_protocol;
+
+public:
+   /// Constructor. The collection name is used when saving the data.
+   /** If @a mesh is NULL, then the mesh can be set later by calling either
+       SetMesh() or Load(). The latter works only in serial. */
+   FMSDataCollection(const std::string& collection_name,
+                     Mesh *mesh = NULL);
+#ifdef MFEM_USE_MPI
+   /// Construct a parallel FMSDataCollection.
+   FMSDataCollection(MPI_Comm comm, const std::string& collection_name,
+                     Mesh *mesh = NULL);
+#endif
+
+   /// We will delete the mesh and fields if we own them
+   virtual ~FMSDataCollection();
+
+   /// Set the FMS relay i/o protocol to use
+   /** Supported options: ascii (default), json, yaml, hdf5 */
+   void SetProtocol(const std::string &protocol);
+
+   /// Save the collection and a FMS blueprint root file
+   virtual void Save();
+
+   /// Load the collection based blueprint data
+   virtual void Load(int cycle = 0);
+};
+
+} // namespace mfem
+
+#endif
+
+#endif

fem/geom.cpp

@@ -11,15 +11,19 @@
 
 #include "fem.hpp"
 #include "../mesh/wedge.hpp"
+#include "../mesh/pyramid.hpp"
 
 namespace mfem
 {
 
 const char *Geometry::Name[NumGeom] =
-{ "Point", "Segment", "Triangle", "Square", "Tetrahedron", "Cube", "Prism" };
+{
+   "Point", "Segment", "Triangle", "Square", "Tetrahedron", "Cube", "Prism",
+   "Pyramid"
+};
 
 const double Geometry::Volume[NumGeom] =
-{ 1.0, 1.0, 0.5, 1.0, 1./6, 1.0, 0.5 };
+{ 1.0, 1.0, 0.5, 1.0, 1./6, 1.0, 0.5, 1./3 };
 
 Geometry::Geometry()
 {
@@ -139,6 +143,28 @@ Geometry::Geometry()
    GeomVert[6]->IntPoint(5).y = 1.0;
    GeomVert[6]->IntPoint(5).z = 1.0;
 
+   // Vertices for Geometry::PYRAMID
+   GeomVert[7] = new IntegrationRule(5);
+   GeomVert[7]->IntPoint(0).x = 0.0;
+   GeomVert[7]->IntPoint(0).y = 0.0;
+   GeomVert[7]->IntPoint(0).z = 0.0;
+
+   GeomVert[7]->IntPoint(1).x = 1.0;
+   GeomVert[7]->IntPoint(1).y = 0.0;
+   GeomVert[7]->IntPoint(1).z = 0.0;
+
+   GeomVert[7]->IntPoint(2).x = 1.0;
+   GeomVert[7]->IntPoint(2).y = 1.0;
+   GeomVert[7]->IntPoint(2).z = 0.0;
+
+   GeomVert[7]->IntPoint(3).x = 0.0;
+   GeomVert[7]->IntPoint(3).y = 1.0;
+   GeomVert[7]->IntPoint(3).z = 0.0;
+
+   GeomVert[7]->IntPoint(4).x = 0.0;
+   GeomVert[7]->IntPoint(4).y = 0.0;
+   GeomVert[7]->IntPoint(4).z = 1.0;
+
    GeomCenter[POINT].x = 0.0;
    GeomCenter[POINT].y = 0.0;
    GeomCenter[POINT].z = 0.0;
@@ -167,6 +193,10 @@ Geometry::Geometry()
    GeomCenter[PRISM].y = 1.0 / 3.0;
    GeomCenter[PRISM].z = 0.5;
 
+   GeomCenter[PYRAMID].x = 0.375;
+   GeomCenter[PYRAMID].y = 0.375;
+   GeomCenter[PYRAMID].z = 0.25;
+
    GeomToPerfGeomJac[POINT]       = NULL;
    GeomToPerfGeomJac[SEGMENT]     = new DenseMatrix(1);
    GeomToPerfGeomJac[TRIANGLE]    = new DenseMatrix(2);
@@ -174,6 +204,7 @@ Geometry::Geometry()
    GeomToPerfGeomJac[TETRAHEDRON] = new DenseMatrix(3);
    GeomToPerfGeomJac[CUBE]        = new DenseMatrix(3);
    GeomToPerfGeomJac[PRISM]       = new DenseMatrix(3);
+   GeomToPerfGeomJac[PYRAMID]     = new DenseMatrix(3);
 
    PerfGeomToGeomJac[POINT]       = NULL;
    PerfGeomToGeomJac[SEGMENT]     = NULL;
@@ -182,6 +213,7 @@ Geometry::Geometry()
    PerfGeomToGeomJac[TETRAHEDRON] = new DenseMatrix(3);
    PerfGeomToGeomJac[CUBE]        = NULL;
    PerfGeomToGeomJac[PRISM]       = new DenseMatrix(3);
+   PerfGeomToGeomJac[PYRAMID]     = new DenseMatrix(3);
 
    GeomToPerfGeomJac[SEGMENT]->Diag(1.0, 1);
    {
@@ -210,6 +242,14 @@ Geometry::Geometry()
       *GeomToPerfGeomJac[PRISM] = pri_T.Jacobian();
       CalcInverse(pri_T.Jacobian(), *PerfGeomToGeomJac[PRISM]);
    }
+   {
+      IsoparametricTransformation pyr_T;
+      pyr_T.SetFE(&PyramidFE);
+      GetPerfPointMat (PYRAMID, pyr_T.GetPointMat());
+      pyr_T.SetIntPoint(&GeomCenter[PYRAMID]);
+      *GeomToPerfGeomJac[PYRAMID] = pyr_T.Jacobian();
+      CalcInverse(pyr_T.Jacobian(), *PerfGeomToGeomJac[PYRAMID]);
+   }
 }
 
 Geometry::~Geometry()
@@ -233,6 +273,7 @@ const IntegrationRule * Geometry::GetVertices(int GeomType)
       case Geometry::TETRAHEDRON: return GeomVert[4];
       case Geometry::CUBE:        return GeomVert[5];
       case Geometry::PRISM:       return GeomVert[6];
+      case Geometry::PYRAMID:     return GeomVert[7];
       default:
          mfem_error ("Geometry::GetVertices(...)");
    }
@@ -310,6 +351,25 @@ void Geometry::GetRandomPoint(int GeomType, IntegrationPoint &ip)
             ip.y = 1.0 - ip.y;
          }
          break;
+      case Geometry::PYRAMID:
+         ip.x = double(rand()) / RAND_MAX;
+         ip.y = double(rand()) / RAND_MAX;
+         ip.z = double(rand()) / RAND_MAX;
+         if (ip.x + ip.z > 1.0 && ip.y < ip.x)
+         {
+            double x = ip.x;
+            ip.x = ip.y;
+            ip.y = 1.0 - ip.z;
+            ip.z = 1.0 - x;
+         }
+         else if (ip.y + ip.z > 1.0)
+         {
+            double z = ip.z;
+            ip.z = 1.0 - ip.y;
+            ip.y = ip.x;
+            ip.x = 1.0 - z;
+         }
+         break;
       default:
          MFEM_ABORT("Unknown type of reference element!");
    }
@@ -371,6 +431,10 @@ bool Geometry::CheckPoint(int GeomType, const IntegrationPoint &ip)
          if (ip.x < 0.0 || ip.y < 0.0 || ip.x+ip.y > 1.0 ||
              ip.z < 0.0 || ip.z > 1.0) { return false; }
          break;
+      case Geometry::PYRAMID:
+         if (ip.x < 0.0 || ip.y < 0.0 || ip.x+ip.z > 1.0 || ip.y+ip.z > 1.0 ||
+             ip.z < 0.0 || ip.z > 1.0) { return false; }
+         break;
       default:
          MFEM_ABORT("Unknown type of reference element!");
    }
@@ -441,6 +505,17 @@ bool Geometry::CheckPoint(int GeomType, const IntegrationPoint &ip, double eps)
             return false;
          }
          break;
+      case Geometry::PYRAMID:
+         if (internal::FuzzyLT(ip.x, 0.0, eps)
+             || internal::FuzzyLT(ip.y, 0.0, eps)
+             || internal::FuzzyGT(ip.x+ip.z, 1.0, eps)
+             || internal::FuzzyGT(ip.y+ip.z, 1.0, eps)
+             || internal::FuzzyLT(ip.z, 0.0, eps)
+             || internal::FuzzyGT(ip.z, 1.0, eps) )
+         {
+            return false;
+         }
+         break;
       default:
          MFEM_ABORT("Unknown type of reference element!");
    }
@@ -555,6 +630,16 @@ bool Geometry::ProjectPoint(int GeomType, const IntegrationPoint &beg,
          double lbeg[5] = { beg.x, beg.y, beg.z, 1.0-beg.x-beg.y, 1.0-beg.z };
          return internal::IntersectSegment<5,3>(lbeg, lend, end);
       }
+      case Geometry::PYRAMID:
+      {
+         double lend[6] = { end.x, end.y, end.z,
+                            1.0-end.x-end.z, 1.0-end.y-end.z, 1.0-end.z
+                          };
+         double lbeg[6] = { beg.x, beg.y, beg.z,
+                            1.0-beg.x-beg.z, 1.0-beg.y-beg.z, 1.0-beg.z
+                          };
+         return internal::IntersectSegment<6,3>(lbeg, lend, end);
+      }
       default:
          MFEM_ABORT("Unknown type of reference element!");
    }
@@ -652,6 +737,43 @@ bool Geometry::ProjectPoint(int GeomType, IntegrationPoint &ip)
          return in_tri && in_z;
       }
 
+      case PYRAMID:
+      {
+         if (ip.x < 0.0)
+         {
+            ip.x = 0.0;
+            internal::ProjectTriangle(ip.y, ip.z);
+            return false;
+         }
+         if (ip.y < 0.0)
+         {
+            ip.y = 0.0;
+            internal::ProjectTriangle(ip.x, ip.z);
+            return false;
+         }
+         if (ip.z < 0.0)
+         {
+            ip.z = 0.0;
+            if (ip.x > 1.0) { ip.x = 1.0; }
+            if (ip.y > 1.0) { ip.y = 1.0; }
+            return false;
+         }
+         if (ip.x >= ip.y)
+         {
+            bool in_y = true;
+            bool in_tri = internal::ProjectTriangle(ip.x, ip.z);
+            if (ip.y > ip.z) { in_y = false; ip.y = ip.z; }
+            return in_tri && in_y;
+         }
+         else
+         {
+            bool in_x = true;
+            bool in_tri = internal::ProjectTriangle(ip.y, ip.z);
+            if (ip.x > ip.z) { in_x = false; ip.x = ip.z; }
+            return in_tri && in_x;
+         }
+      }
+
       default:
          MFEM_ABORT("Reference element type is not supported!");
    }
@@ -726,6 +848,17 @@ void Geometry::GetPerfPointMat(int GeomType, DenseMatrix &pm)
       }
       break;
 
+      case Geometry::PYRAMID:
+      {
+         pm.SetSize (3, 5);
+         pm(0,0) = 0.0;  pm(1,0) = 0.0;  pm(2,0) = 0.0;
+         pm(0,1) = 1.0;  pm(1,1) = 0.0;  pm(2,1) = 0.0;
+         pm(0,2) = 1.0;  pm(1,2) = 1.0;  pm(2,2) = 0.0;
+         pm(0,3) = 0.0;  pm(1,3) = 1.0;  pm(2,3) = 0.0;
+         pm(0,4) = 0.5;  pm(1,4) = 0.5;  pm(2,4) = 0.7071067811865475;
+      }
+      break;
+
       default:
          mfem_error ("Geometry::GetPerfPointMat (...)");
    }
@@ -744,13 +877,13 @@ void Geometry::JacToPerfJac(int GeomType, const DenseMatrix &J,
    }
 }
 
-const int Geometry::NumBdrArray[NumGeom] = { 0, 2, 3, 4, 4, 6, 5 };
-const int Geometry::Dimension[NumGeom] = { 0, 1, 2, 2, 3, 3, 3 };
+const int Geometry::NumBdrArray[NumGeom] = { 0, 2, 3, 4, 4, 6, 5, 5 };
+const int Geometry::Dimension[NumGeom] = { 0, 1, 2, 2, 3, 3, 3, 3 };
 const int Geometry::DimStart[MaxDim+2] =
 { POINT, SEGMENT, TRIANGLE, TETRAHEDRON, NUM_GEOMETRIES };
-const int Geometry::NumVerts[NumGeom] = { 1, 2, 3, 4, 4, 8, 6 };
-const int Geometry::NumEdges[NumGeom] = { 0, 1, 3, 4, 6, 12, 9 };
-const int Geometry::NumFaces[NumGeom] = { 0, 0, 1, 1, 4, 6, 5 };
+const int Geometry::NumVerts[NumGeom] = { 1, 2, 3, 4, 4, 8, 6, 5 };
+const int Geometry::NumEdges[NumGeom] = { 0, 1, 3, 4, 6, 12, 9, 8 };
+const int Geometry::NumFaces[NumGeom] = { 0, 0, 1, 1, 4, 6, 5, 5 };
 
 const int Geometry::
 Constants<Geometry::POINT>::Orient[1][1] = {{0}};
@@ -897,6 +1030,30 @@ Constants<Geometry::PRISM>::VertToVert::J[9][2] =
    {5, 4}                   // 4,5:4
 };
 
+const int Geometry::
+Constants<Geometry::PYRAMID>::Edges[8][2] =
+{{0, 1}, {1, 2}, {3, 2}, {0, 3}, {0, 4}, {1, 4}, {2, 4}, {3, 4}};
+const int Geometry::
+Constants<Geometry::PYRAMID>::FaceTypes[5] =
+{
+   Geometry::SQUARE,
+   Geometry::TRIANGLE, Geometry::TRIANGLE,
+   Geometry::TRIANGLE, Geometry::TRIANGLE
+};
+const int Geometry::
+Constants<Geometry::PYRAMID>::FaceVert[5][4] =
+{{3, 2, 1, 0}, {0, 1, 4, -1}, {1, 2, 4, -1}, {2, 3, 4, -1}, {3, 0, 4, -1}};
+const int Geometry::
+Constants<Geometry::PYRAMID>::VertToVert::I[5] = {0, 3, 5, 7, 8};
+const int Geometry::
+Constants<Geometry::PYRAMID>::VertToVert::J[8][2] =
+{
+   {1, 0}, {3, 3}, {4, 4}, // 0,1:0   0,3:3  0,4:4
+   {2, 1}, {4, 5},         // 1,2:1   1,4:5
+   {3,-3}, {4, 6},         // 2,3:-3  2,4:6
+   {4, 7}                  // 3,4:7
+};
+
 
 GeometryRefiner::GeometryRefiner()
 {
@@ -1262,6 +1419,104 @@ RefinedGeometry * GeometryRefiner::Refine(Geometry::Type Geom,
          return RG;
       }
 
+      case Geometry::PYRAMID:
+      {
+         const int n = Times;
+         RG = new RefinedGeometry ((n+1)*(n+2)*(2*n+3)/6,
+                                   5*n*(2*n-1)*(2*n+1)/3, 0);
+         RG->Times = Times;
+         RG->ETimes = ETimes;
+         RG->Type = type;
+         // enumerate and define the vertices
+         m = 0;
+         for (k = 0; k <= n; k++)
+         {
+            const double *cpij =
+               poly1d.GetPoints(Times - k, BasisType::GetNodalBasis(type));
+            for (j = 0; j <= n - k; j++)
+               for (i = 0; i <= n - k; i++)
+               {
+                  IntegrationPoint &ip = RG->RefPts.IntPoint(m);
+                  if (type == 0)
+                  {
+                     ip.x = (n > k) ? (double(i) / (n - k)) : 0.0;
+                     ip.y = (n > k) ? (double(j) / (n - k)) : 0.0;
+                     ip.z = double(k) / n;
+                  }
+                  else
+                  {
+                     ip.x = cpij[i] * (1.0 - cp[k]);
+                     ip.y = cpij[j] * (1.0 - cp[k]);
+                     ip.z = cp[k];
+                  }
+                  m++;
+               }
+         }
+         if (m != (n+1)*(n+2)*(2*n+3)/6)
+         {
+            mfem_error("GeometryRefiner::Refine() for PYRAMID #1");
+         }
+         // elements
+         Array<int> &G = RG->RefGeoms;
+         m = 0;
+         for (k = 0; k < n; k++)
+         {
+            int lk = k * (k * (2 * k - 6 * n - 9) + 6 * n * (n + 3) + 13) / 6;
+            int lkp1 = (k + 1) *
+                       (k * (2 * k - 6 * n -5) + 6 * n * (n + 2) + 6) / 6;
+            for (j = 0; j < n - k; j++)
+            {
+               for (i = 0; i < n - k; i++)
+               {
+                  G[m++] = lk + j * (n - k + 1) + i;
+                  G[m++] = lk + j * (n - k + 1) + i + 1;
+                  G[m++] = lk + (j + 1) * (n - k + 1) + i + 1;
+                  G[m++] = lk + (j + 1) * (n - k + 1) + i;
+                  G[m++] = lkp1 + j * (n - k) + i;
+               }
+            }
+            for (j = 0; j < n - k - 1; j++)
+            {
+               for (i = 0; i < n - k - 1; i++)
+               {
+                  G[m++] = lkp1 + j * (n - k) + i;
+                  G[m++] = lkp1 + (j + 1) * (n - k) + i;
+                  G[m++] = lkp1 + (j + 1) * (n - k) + i + 1;
+                  G[m++] = lkp1 + j * (n - k) + i + 1;
+                  G[m++] = lk + (j + 1) * (n - k + 1) + i + 1;
+               }
+            }
+            for (j = 0; j < n - k; j++)
+            {
+               for (i = 0; i < n - k - 1; i++)
+               {
+                  G[m++] = lk + j * (n - k + 1) + i + 1;
+                  G[m++] = lk + (j + 1) * (n - k + 1) + i + 1;
+                  G[m++] = lkp1 + j * (n - k) + i;
+                  G[m++] = lkp1 + j * (n - k) + i + 1;
+                  G[m++] = -1;
+               }
+            }
+            for (j = 0; j < n - k - 1; j++)
+            {
+               for (i = 0; i < n - k; i++)
+               {
+                  G[m++] = lk + (j + 1) * (n - k + 1) + i;
+                  G[m++] = lk + (j + 1) * (n - k + 1) + i + 1;
+                  G[m++] = lkp1 + (j + 1) * (n - k) + i;
+                  G[m++] = lkp1 + j * (n - k) + i;
+                  G[m++] = -1;
+               }
+            }
+         }
+         if (m != 5*n*(2*n-1)*(2*n+1)/3)
+         {
+            mfem_error("GeometryRefiner::Refine() for PYRAMID #2");
+         }
+         RGeom[Geometry::PYRAMID].Append(RG);
+         return RG;
+      }
+
       case Geometry::PRISM:
       {
          const int n = Times;

fem/geom.hpp

@@ -27,6 +27,7 @@ namespace mfem
     Geometry::TETRAHEDRON - w/ vert. (0,0,0),(1,0,0),(0,1,0),(0,0,1)
     Geometry::CUBE - the unit cube
     Geometry::PRISM - w/ vert. (0,0,0),(1,0,0),(0,1,0),(0,0,1),(1,0,1),(0,1,1)
+    Geometry::PYRAMID - w/ vert. (0,0,0),(1,0,0),(1,1,0),(0,1,0),(0,0,1)
 */
 class Geometry
 {
@@ -34,7 +35,7 @@ public:
    enum Type
    {
       INVALID = -1,
-      POINT = 0, SEGMENT, TRIANGLE, SQUARE, TETRAHEDRON, CUBE, PRISM,
+      POINT = 0, SEGMENT, TRIANGLE, SQUARE, TETRAHEDRON, CUBE, PRISM, PYRAMID,
       NUM_GEOMETRIES
    };
 
@@ -251,7 +252,26 @@ template <> struct Geometry::Constants<Geometry::PRISM>
    };
 };
 
-// Defined in fe.cpp to ensure construction after 'mfem::WedgeFE'.
+template <> struct Geometry::Constants<Geometry::PYRAMID>
+{
+   static const int Dimension = 3;
+   static const int NumVert = 5;
+   static const int NumEdges = 8;
+   static const int Edges[NumEdges][2];
+   static const int NumFaces = 5;
+   static const int FaceTypes[NumFaces];
+   static const int MaxFaceVert = 4;
+   static const int FaceVert[NumFaces][MaxFaceVert];
+   // Upper-triangular part of the local vertex-to-vertex graph.
+   struct VertToVert
+   {
+      static const int I[NumVert];
+      static const int J[NumEdges][2]; // {end,edge_idx}
+   };
+};
+
+// Defined in fe.cpp to ensure construction after 'mfem::TriangleFE' and
+// `mfem::TetrahedronFE`.
 extern Geometry Geometries;
 
 

fem/gridfunc.cpp

@@ -218,7 +218,7 @@ void GridFunction::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
 
 void GridFunction::MakeTRef(FiniteElementSpace *f, double *tv)
 {
-   if (!f->GetProlongationMatrix())
+   if (IsIdentityProlongation(f->GetProlongationMatrix()))
    {
       MakeRef(f, tv);
       t_vec.NewDataAndSize(tv, size);
@@ -232,7 +232,8 @@ void GridFunction::MakeTRef(FiniteElementSpace *f, double *tv)
 
 void GridFunction::MakeTRef(FiniteElementSpace *f, Vector &tv, int tv_offset)
 {
-   if (!f->GetProlongationMatrix())
+   tv.UseDevice(true);
+   if (IsIdentityProlongation(f->GetProlongationMatrix()))
    {
       MakeRef(f, tv, tv_offset);
       t_vec.NewMemoryAndSize(data, size, false);
@@ -241,10 +242,7 @@ void GridFunction::MakeTRef(FiniteElementSpace *f, Vector &tv, int tv_offset)
    {
       MFEM_ASSERT(tv.Size() >= tv_offset + f->GetTrueVSize(), "");
       SetSpace(f); // works in parallel
-      tv.UseDevice(true);
-      const int tv_size = f->GetTrueVSize();
-      t_vec.NewMemoryAndSize(Memory<double>(tv.GetMemory(), tv_offset, tv_size),
-                             tv_size, true);
+      t_vec.MakeRef(tv, tv_offset, f->GetTrueVSize());
    }
 }
 
@@ -257,6 +255,8 @@ void GridFunction::SumFluxAndCount(BilinearFormIntegrator &blfi,
    GridFunction &u = *this;
 
    ElementTransformation *Transf;
+   DofTransformation *udoftrans;
+   DofTransformation *fdoftrans;
 
    FiniteElementSpace *ufes = u.FESpace();
    FiniteElementSpace *ffes = flux.FESpace();
@@ -276,15 +276,23 @@ void GridFunction::SumFluxAndCount(BilinearFormIntegrator &blfi,
          continue;
       }
 
-      ufes->GetElementVDofs(i, udofs);
-      ffes->GetElementVDofs(i, fdofs);
+      udoftrans = ufes->GetElementVDofs(i, udofs);
+      fdoftrans = ffes->GetElementVDofs(i, fdofs);
 
       u.GetSubVector(udofs, ul);
+      if (udoftrans)
+      {
+         udoftrans->InvTransformPrimal(ul);
+      }
 
       Transf = ufes->GetElementTransformation(i);
       blfi.ComputeElementFlux(*ufes->GetFE(i), *Transf, ul,
                               *ffes->GetFE(i), fl, wcoef);
 
+      if (fdoftrans)
+      {
+         fdoftrans->TransformPrimal(fl);
+      }
       flux.AddElementVector(fdofs, fl);
 
       FiniteElementSpace::AdjustVDofs(fdofs);
@@ -334,10 +342,10 @@ int GridFunction::VectorDim() const
 void GridFunction::GetTrueDofs(Vector &tv) const
 {
    const SparseMatrix *R = fes->GetRestrictionMatrix();
-   if (!R)
+   if (!R || IsIdentityProlongation(fes->GetProlongationMatrix()))
    {
-      // R is identity -> make tv a reference to *this
-      tv.MakeRef(const_cast<GridFunction &>(*this), 0, size);
+      // R is identity
+      tv = *this; // no real copy if 'tv' and '*this' use the same data
    }
    else
    {
@@ -366,7 +374,7 @@ void GridFunction::GetNodalValues(int i, Array<double> &nval, int vdim) const
 
    int k;
 
-   fes->GetElementVDofs(i, vdofs);
+   DofTransformation * doftrans = fes->GetElementVDofs(i, vdofs);
    const FiniteElement *FElem = fes->GetFE(i);
    const IntegrationRule *ElemVert =
       Geometries.GetVertices(FElem->GetGeomType());
@@ -376,6 +384,10 @@ void GridFunction::GetNodalValues(int i, Array<double> &nval, int vdim) const
    vdim--;
    Vector loc_data;
    GetSubVector(vdofs, loc_data);
+   if (doftrans)
+   {
+      doftrans->InvTransformPrimal(loc_data);
+   }
 
    if (FElem->GetRangeType() == FiniteElement::SCALAR)
    {
@@ -405,7 +417,7 @@ double GridFunction::GetValue(int i, const IntegrationPoint &ip, int vdim)
 const
 {
    Array<int> dofs;
-   fes->GetElementDofs(i, dofs);
+   DofTransformation * doftrans = fes->GetElementDofs(i, dofs);
    fes->DofsToVDofs(vdim-1, dofs);
    Vector DofVal(dofs.Size()), LocVec;
    const FiniteElement *fe = fes->GetFE(i);
@@ -420,6 +432,10 @@ const
       fe->CalcPhysShape(*Tr, DofVal);
    }
    GetSubVector(dofs, LocVec);
+   if (doftrans)
+   {
+      doftrans->InvTransformPrimal(LocVec);
+   }
 
    return (DofVal * LocVec);
 }
@@ -430,9 +446,13 @@ void GridFunction::GetVectorValue(int i, const IntegrationPoint &ip,
    const FiniteElement *FElem = fes->GetFE(i);
    int dof = FElem->GetDof();
    Array<int> vdofs;
-   fes->GetElementVDofs(i, vdofs);
+   DofTransformation * doftrans = fes->GetElementVDofs(i, vdofs);
    Vector loc_data;
    GetSubVector(vdofs, loc_data);
+   if (doftrans)
+   {
+      doftrans->InvTransformPrimal(loc_data);
+   }
    if (FElem->GetRangeType() == FiniteElement::SCALAR)
    {
       Vector shape(dof);
@@ -472,30 +492,35 @@ const
    Array<int> dofs;
    int n = ir.GetNPoints();
    vals.SetSize(n);
-   fes->GetElementDofs(i, dofs);
+   DofTransformation * doftrans = fes->GetElementDofs(i, dofs);
    fes->DofsToVDofs(vdim-1, dofs);
    const FiniteElement *FElem = fes->GetFE(i);
    int dof = FElem->GetDof();
    Vector DofVal(dof), loc_data(dof);
    GetSubVector(dofs, loc_data);
-   if (FElem->GetMapType() == FiniteElement::VALUE)
+   if (doftrans)
    {
-      for (int k = 0; k < n; k++)
-      {
-         FElem->CalcShape(ir.IntPoint(k), DofVal);
-         vals(k) = DofVal * loc_data;
-      }
+      doftrans->InvTransformPrimal(loc_data);
    }
-   else
-   {
-      ElementTransformation *Tr = fes->GetElementTransformation(i);
-      for (int k = 0; k < n; k++)
+   for (int k = 0; k < n; k++)
+      if (FElem->GetMapType() == FiniteElement::VALUE)
       {
-         Tr->SetIntPoint(&ir.IntPoint(k));
-         FElem->CalcPhysShape(*Tr, DofVal);
-         vals(k) = DofVal * loc_data;
+         for (int k = 0; k < n; k++)
+         {
+            FElem->CalcShape(ir.IntPoint(k), DofVal);
+            vals(k) = DofVal * loc_data;
+         }
+      }
+      else
+      {
+         ElementTransformation *Tr = fes->GetElementTransformation(i);
+         for (int k = 0; k < n; k++)
+         {
+            Tr->SetIntPoint(&ir.IntPoint(k));
+            FElem->CalcPhysShape(*Tr, DofVal);
+            vals(k) = DofVal * loc_data;
+         }
       }
-   }
 }
 
 void GridFunction::GetValues(int i, const IntegrationRule &ir, Vector &vals,
@@ -863,11 +888,12 @@ void GridFunction::GetVectorValue(ElementTransformation &T,
 
    Array<int> vdofs;
    const FiniteElement *fe = NULL;
+   DofTransformation * doftrans = NULL;
 
    switch (T.ElementType)
    {
       case ElementTransformation::ELEMENT:
-         fes->GetElementVDofs(T.ElementNo, vdofs);
+         doftrans = fes->GetElementVDofs(T.ElementNo, vdofs);
          fe = fes->GetFE(T.ElementNo);
          break;
       case ElementTransformation::EDGE:
@@ -958,6 +984,10 @@ void GridFunction::GetVectorValue(ElementTransformation &T,
    int dof = fe->GetDof();
    Vector loc_data;
    GetSubVector(vdofs, loc_data);
+   if (doftrans)
+   {
+      doftrans->InvTransformPrimal(loc_data);
+   }
    if (fe->GetRangeType() == FiniteElement::SCALAR)
    {
       Vector shape(dof);
@@ -1000,10 +1030,14 @@ void GridFunction::GetVectorValues(ElementTransformation &T,
    int dof = FElem->GetDof();
 
    Array<int> vdofs;
-   fes->GetElementVDofs(T.ElementNo, vdofs);
-
+   DofTransformation * doftrans = fes->GetElementVDofs(T.ElementNo, vdofs);
    Vector loc_data;
    GetSubVector(vdofs, loc_data);
+   if (doftrans)
+   {
+      doftrans->InvTransformPrimal(loc_data);
+   }
+
    int nip = ir.GetNPoints();
 
    if (FElem->GetRangeType() == FiniteElement::SCALAR)
@@ -1091,6 +1125,8 @@ void GridFunction::GetValuesFrom(const GridFunction &orig_func)
    // Without averaging ...
 
    const FiniteElementSpace *orig_fes = orig_func.FESpace();
+   DofTransformation * doftrans;
+   DofTransformation * orig_doftrans;
    Array<int> vdofs, orig_vdofs;
    Vector shape, loc_values, orig_loc_values;
    int i, j, d, ne, dof, odof, vdim;
@@ -1099,9 +1135,13 @@ void GridFunction::GetValuesFrom(const GridFunction &orig_func)
    vdim = fes->GetVDim();
    for (i = 0; i < ne; i++)
    {
-      fes->GetElementVDofs(i, vdofs);
-      orig_fes->GetElementVDofs(i, orig_vdofs);
+      doftrans = fes->GetElementVDofs(i, vdofs);
+      orig_doftrans = orig_fes->GetElementVDofs(i, orig_vdofs);
       orig_func.GetSubVector(orig_vdofs, orig_loc_values);
+      if (orig_doftrans)
+      {
+         orig_doftrans->InvTransformPrimal(orig_loc_values);
+      }
       const FiniteElement *fe = fes->GetFE(i);
       const FiniteElement *orig_fe = orig_fes->GetFE(i);
       dof = fe->GetDof();
@@ -1119,6 +1159,10 @@ void GridFunction::GetValuesFrom(const GridFunction &orig_func)
                shape * ((const double *)orig_loc_values + d * odof) ;
          }
       }
+      if (doftrans)
+      {
+         doftrans->TransformPrimal(loc_values);
+      }
       SetSubVector(vdofs, loc_values);
    }
 }
@@ -1128,8 +1172,10 @@ void GridFunction::GetBdrValuesFrom(const GridFunction &orig_func)
    // Without averaging ...
 
    const FiniteElementSpace *orig_fes = orig_func.FESpace();
+   // DofTransformation * doftrans;
+   // DofTransformation * orig_doftrans;
    Array<int> vdofs, orig_vdofs;
-   Vector shape, loc_values, orig_loc_values;
+   Vector shape, loc_values, loc_values_t, orig_loc_values, orig_loc_values_t;
    int i, j, d, nbe, dof, odof, vdim;
 
    nbe = fes->GetNBE();
@@ -1167,37 +1213,33 @@ void GridFunction::GetVectorFieldValues(
    Array<int> vdofs;
    ElementTransformation *transf;
 
-   int d, j, k, n, sdim, dof, ind;
+   int d, k, n, sdim, dof;
 
    n = ir.GetNPoints();
-   fes->GetElementVDofs(i, vdofs);
+   DofTransformation * doftrans = fes->GetElementVDofs(i, vdofs);
    const FiniteElement *fe = fes->GetFE(i);
    dof = fe->GetDof();
    sdim = fes->GetMesh()->SpaceDimension();
-   int *dofs = &vdofs[comp*dof];
+   // int *dofs = &vdofs[comp*dof];
    transf = fes->GetElementTransformation(i);
    transf->Transform(ir, tr);
    vals.SetSize(n, sdim);
    DenseMatrix vshape(dof, sdim);
-   double a;
+   Vector loc_data, val(sdim);
+   GetSubVector(vdofs, loc_data);
+   if (doftrans)
+   {
+      doftrans->InvTransformPrimal(loc_data);
+   }
    for (k = 0; k < n; k++)
    {
       const IntegrationPoint &ip = ir.IntPoint(k);
       transf->SetIntPoint(&ip);
       fe->CalcVShape(*transf, vshape);
+      vshape.MultTranspose(loc_data, val);
       for (d = 0; d < sdim; d++)
       {
-         a = 0.0;
-         for (j = 0; j < dof; j++)
-            if ( (ind=dofs[j]) >= 0 )
-            {
-               a += vshape(j, d) * data[ind];
-            }
-            else
-            {
-               a -= vshape(j, d) * data[-1-ind];
-            }
-         vals(k, d) = a;
+         vals(k,d) = val(d);
       }
    }
 }
@@ -1367,9 +1409,13 @@ void GridFunction::GetVectorGradientHat(
    const FiniteElement *FElem = fes->GetFE(elNo);
    int dim = FElem->GetDim(), dof = FElem->GetDof();
    Array<int> vdofs;
-   fes->GetElementVDofs(elNo, vdofs);
+   DofTransformation * doftrans = fes->GetElementVDofs(elNo, vdofs);
    Vector loc_data;
    GetSubVector(vdofs, loc_data);
+   if (doftrans)
+   {
+      doftrans->InvTransformPrimal(loc_data);
+   }
    // assuming scalar FE
    int vdim = fes->GetVDim();
    DenseMatrix dshape(dof, dim);
@@ -1408,9 +1454,13 @@ double GridFunction::GetDivergence(ElementTransformation &T) const
          {
             // Assuming RT-type space
             Array<int> dofs;
-            fes->GetElementDofs(elNo, dofs);
+            DofTransformation * doftrans = fes->GetElementDofs(elNo, dofs);
             Vector loc_data, divshape(fe->GetDof());
             GetSubVector(dofs, loc_data);
+            if (doftrans)
+            {
+               doftrans->InvTransformPrimal(loc_data);
+            }
             fe->CalcDivShape(T.GetIntPoint(), divshape);
             return (loc_data * divshape) / T.Weight();
          }
@@ -1501,9 +1551,13 @@ void GridFunction::GetCurl(ElementTransformation &T, Vector &curl) const
          {
             // Assuming ND-type space
             Array<int> dofs;
-            fes->GetElementDofs(elNo, dofs);
+            DofTransformation * doftrans = fes->GetElementDofs(elNo, dofs);
             Vector loc_data;
             GetSubVector(dofs, loc_data);
+            if (doftrans)
+            {
+               doftrans->InvTransformPrimal(loc_data);
+            }
             DenseMatrix curl_shape(fe->GetDof(), fe->GetDim() == 3 ? 3 : 1);
             fe->CalcCurlShape(T.GetIntPoint(), curl_shape);
             curl.SetSize(curl_shape.Width());
@@ -1645,8 +1699,12 @@ void GridFunction::GetGradients(ElementTransformation &tr,
    DenseMatrix dshape(fe->GetDof(), fe->GetDim());
    Vector lval, gh(fe->GetDim()), gcol;
    Array<int> dofs;
-   fes->GetElementDofs(elNo, dofs);
+   DofTransformation * doftrans = fes->GetElementDofs(elNo, dofs);
    GetSubVector(dofs, lval);
+   if (doftrans)
+   {
+      doftrans->InvTransformPrimal(lval);
+   }
    grad.SetSize(fe->GetDim(), ir.GetNPoints());
    for (int i = 0; i < ir.GetNPoints(); i++)
    {
@@ -1726,6 +1784,8 @@ void GridFunction::GetElementAverages(GridFunction &avgs) const
 {
    MassIntegrator Mi;
    DenseMatrix loc_mass;
+   DofTransformation * te_doftrans;
+   DofTransformation * tr_doftrans;
    Array<int> te_dofs, tr_dofs;
    Vector loc_avgs, loc_this;
    Vector int_psi(avgs.Size());
@@ -1736,11 +1796,19 @@ void GridFunction::GetElementAverages(GridFunction &avgs) const
    {
       Mi.AssembleElementMatrix2(*fes->GetFE(i), *avgs.FESpace()->GetFE(i),
                                 *fes->GetElementTransformation(i), loc_mass);
-      fes->GetElementDofs(i, tr_dofs);
-      avgs.FESpace()->GetElementDofs(i, te_dofs);
+      tr_doftrans = fes->GetElementDofs(i, tr_dofs);
+      te_doftrans = avgs.FESpace()->GetElementDofs(i, te_dofs);
       GetSubVector(tr_dofs, loc_this);
+      if (tr_doftrans)
+      {
+         tr_doftrans->InvTransformPrimal(loc_this);
+      }
       loc_avgs.SetSize(te_dofs.Size());
       loc_mass.Mult(loc_this, loc_avgs);
+      if (te_doftrans)
+      {
+         te_doftrans->TransformPrimal(loc_avgs);
+      }
       avgs.AddElementVector(te_dofs, loc_avgs);
       loc_this = 1.0; // assume the local basis for 'this' sums to 1
       loc_mass.Mult(loc_this, loc_avgs);
@@ -1755,8 +1823,12 @@ void GridFunction::GetElementAverages(GridFunction &avgs) const
 void GridFunction::GetElementDofValues(int el, Vector &dof_vals) const
 {
    Array<int> dof_idx;
-   fes->GetElementVDofs(el, dof_idx);
+   DofTransformation * doftrans = fes->GetElementVDofs(el, dof_idx);
    GetSubVector(dof_idx, dof_vals);
+   if (doftrans)
+   {
+      doftrans->InvTransformPrimal(dof_vals);
+   }
 }
 
 void GridFunction::ProjectGridFunction(const GridFunction &src)
@@ -1792,13 +1864,21 @@ void GridFunction::ProjectGridFunction(const GridFunction &src)
          cached_geom = geom;
       }
 
-      src.fes->GetElementVDofs(i, src_vdofs);
+      DofTransformation * src_doftrans = src.fes->GetElementVDofs(i, src_vdofs);
       src.GetSubVector(src_vdofs, src_lvec);
+      if (src_doftrans)
+      {
+         src_doftrans->InvTransformPrimal(src_lvec);
+      }
       for (int vd = 0; vd < vdim; vd++)
       {
          P.Mult(&src_lvec[vd*P.Width()], &dest_lvec[vd*P.Height()]);
       }
-      fes->GetElementVDofs(i, dest_vdofs);
+      DofTransformation * doftrans = fes->GetElementVDofs(i, dest_vdofs);
+      if (doftrans)
+      {
+         doftrans->TransformPrimal(dest_lvec);
+      }
       SetSubVector(dest_vdofs, dest_lvec);
    }
 }
@@ -1807,10 +1887,15 @@ void GridFunction::ImposeBounds(int i, const Vector &weights,
                                 const Vector &lo_, const Vector &hi_)
 {
    Array<int> vdofs;
-   fes->GetElementVDofs(i, vdofs);
+   DofTransformation * doftrans = fes->GetElementVDofs(i, vdofs);
    int size = vdofs.Size();
    Vector vals, new_vals(size);
+
    GetSubVector(vdofs, vals);
+   if (doftrans)
+   {
+      doftrans->InvTransformPrimal(vals);
+   }
 
    MFEM_ASSERT(weights.Size() == size, "Different # of weights and dofs.");
    MFEM_ASSERT(lo_.Size() == size, "Different # of lower bounds and dofs.");
@@ -1827,6 +1912,10 @@ void GridFunction::ImposeBounds(int i, const Vector &weights,
    slbqp.SetPrintLevel(0); // print messages only if not converged
    slbqp.Mult(vals, new_vals);
 
+   if (doftrans)
+   {
+      doftrans->TransformPrimal(new_vals);
+   }
    SetSubVector(vdofs, new_vals);
 }
 
@@ -1834,10 +1923,14 @@ void GridFunction::ImposeBounds(int i, const Vector &weights,
                                 double min_, double max_)
 {
    Array<int> vdofs;
-   fes->GetElementVDofs(i, vdofs);
+   DofTransformation * doftrans = fes->GetElementVDofs(i, vdofs);
    int size = vdofs.Size();
    Vector vals, new_vals(size);
    GetSubVector(vdofs, vals);
+   if (doftrans)
+   {
+      doftrans->InvTransformPrimal(vals);
+   }
 
    double max_val = vals.Max();
    double min_val = vals.Min();
@@ -1845,6 +1938,10 @@ void GridFunction::ImposeBounds(int i, const Vector &weights,
    if (max_val <= min_)
    {
       new_vals = min_;
+      if (doftrans)
+      {
+         doftrans->TransformPrimal(new_vals);
+      }
       SetSubVector(vdofs, new_vals);
       return;
    }
@@ -2280,6 +2377,7 @@ void GridFunction::ProjectDeltaCoefficient(DeltaCoefficient &delta_coeff,
 void GridFunction::ProjectCoefficient(Coefficient &coeff)
 {
    DeltaCoefficient *delta_c = dynamic_cast<DeltaCoefficient *>(&coeff);
+   DofTransformation * doftrans = NULL;
 
    if (delta_c == NULL)
    {
@@ -2288,9 +2386,13 @@ void GridFunction::ProjectCoefficient(Coefficient &coeff)
 
       for (int i = 0; i < fes->GetNE(); i++)
       {
-         fes->GetElementVDofs(i, vdofs);
+         doftrans = fes->GetElementVDofs(i, vdofs);
          vals.SetSize(vdofs.Size());
          fes->GetFE(i)->Project(coeff, *fes->GetElementTransformation(i), vals);
+         if (doftrans)
+         {
+            doftrans->TransformPrimal(vals);
+         }
          SetSubVector(vdofs, vals);
       }
    }
@@ -2336,11 +2438,17 @@ void GridFunction::ProjectCoefficient(VectorCoefficient &vcoeff)
    Array<int> vdofs;
    Vector vals;
 
+   DofTransformation * doftrans = NULL;
+
    for (i = 0; i < fes->GetNE(); i++)
    {
-      fes->GetElementVDofs(i, vdofs);
+      doftrans = fes->GetElementVDofs(i, vdofs);
       vals.SetSize(vdofs.Size());
       fes->GetFE(i)->Project(vcoeff, *fes->GetElementTransformation(i), vals);
+      if (doftrans)
+      {
+         doftrans->TransformPrimal(vals);
+      }
       SetSubVector(vdofs, vals);
    }
 }
@@ -2403,6 +2511,7 @@ void GridFunction::ProjectCoefficient(Coefficient *coeff[])
    double val;
    const FiniteElement *fe;
    ElementTransformation *transf;
+   // DofTransformation * doftrans;
    Array<int> vdofs;
 
    vdim = fes->GetVDim();
@@ -2412,6 +2521,7 @@ void GridFunction::ProjectCoefficient(Coefficient *coeff[])
       fdof = fe->GetDof();
       transf = fes->GetElementTransformation(i);
       const IntegrationRule &ir = fe->GetNodes();
+      // doftrans = fes->GetElementVDofs(i, vdofs);
       fes->GetElementVDofs(i, vdofs);
       for (j = 0; j < fdof; j++)
       {

fem/gridfunc.hpp

@@ -95,6 +95,12 @@ public:
       : Vector(data, f->GetVSize())
    { fes = f; fec = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }
 
+   /** @brief Construct a GridFunction using previously allocated Vector @a base
+       starting at the given offset, @a base_offset. */
+   GridFunction(FiniteElementSpace *f, Vector &base, int base_offset = 0)
+      : Vector(base, base_offset, f->GetVSize())
+   { fes = f; fec = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }
+
    /// Construct a GridFunction on the given Mesh, using the data from @a input.
    /** The content of @a input should be in the format created by the method
        Save(). The reconstructed FiniteElementSpace and FiniteElementCollection
@@ -130,9 +136,7 @@ public:
        or set. */
    Vector &GetTrueVector() { return t_vec; }
 
-   /// @brief Extract the true-dofs from the GridFunction. If all dofs are true,
-   /// then `tv` will be set to point to the data of `*this`.
-   /** @warning This method breaks const-ness when all dofs are true. */
+   /// Extract the true-dofs from the GridFunction.
    void GetTrueDofs(Vector &tv) const;
 
    /// Shortcut for calling GetTrueDofs() with GetTrueVector() as argument.

fem/gslib.cpp

@@ -19,7 +19,11 @@
 #pragma GCC diagnostic ignored "-Wunused-function"
 #endif
 
+// External GSLIB header (the MFEM header is gslib.hpp)
+namespace gslib
+{
 #include "gslib.h"
+}
 
 #ifdef MFEM_HAVE_GCC_PRAGMA_DIAGNOSTIC
 #pragma GCC diagnostic pop
@@ -34,13 +38,13 @@ FindPointsGSLIB::FindPointsGSLIB()
      dim(-1), points_cnt(0), setupflag(false), default_interp_value(0),
      avgtype(AvgType::ARITHMETIC)
 {
-   gsl_comm = new comm;
-   cr       = new crystal;
+   gsl_comm = new gslib::comm;
+   cr       = new gslib::crystal;
 #ifdef MFEM_USE_MPI
    int initialized;
    MPI_Initialized(&initialized);
    if (!initialized) { MPI_Init(NULL, NULL); }
-   MPI_Comm comm = MPI_COMM_WORLD;;
+   MPI_Comm comm = MPI_COMM_WORLD;
    comm_init(gsl_comm, comm);
 #else
    comm_init(gsl_comm, 0);
@@ -62,8 +66,8 @@ FindPointsGSLIB::FindPointsGSLIB(MPI_Comm comm_)
      dim(-1), points_cnt(0), setupflag(false), default_interp_value(0),
      avgtype(AvgType::ARITHMETIC)
 {
-   gsl_comm = new comm;
-   cr      = new crystal;
+   gsl_comm = new gslib::comm;
+   cr      = new gslib::crystal;
    comm_init(gsl_comm, comm_);
 }
 #endif
@@ -606,7 +610,12 @@ void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
       {
          if (gsl_code[i] == 1) { indl2.Append(i); }
       }
-      if (indl2.Size() == 0) { return; } // no points on element borders
+      int borderPts = indl2.Size();
+#ifdef MFEM_USE_MPI
+      MPI_Allreduce(MPI_IN_PLACE, &borderPts, 1, MPI_INT, MPI_SUM, gsl_comm->c);
+#endif
+      if (borderPts == 0) { return; } // no points on element borders
+
 
       Vector field_out_l2(field_out.Size());
       VectorGridFunctionCoefficient field_in_dg(&field_in);
@@ -728,7 +737,7 @@ void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
       }
 
       // Pack data to send via crystal router
-      struct array *outpt = new array;
+      struct gslib::array *outpt = new gslib::array;
       struct out_pt { double r[3], ival; uint index, el, proc; };
       struct out_pt *pt;
       array_init(struct out_pt, outpt, nptsend);
@@ -788,7 +797,7 @@ void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
          }
 
          // Save index and proc data in a struct
-         struct array *savpt = new array;
+         struct gslib::array *savpt = new gslib::array;
          struct sav_pt { uint index, proc; };
          struct sav_pt *spt;
          array_init(struct sav_pt, savpt, npt);
@@ -806,7 +815,7 @@ void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
          delete outpt;
 
          // Copy data from save struct to send struct and send component wise
-         struct array *sendpt = new array;
+         struct gslib::array *sendpt = new gslib::array;
          struct send_pt { double ival; uint index, proc; };
          struct send_pt *sdpt;
          for (int j = 0; j < ncomp; j++)

fem/gslib.hpp

@@ -17,11 +17,13 @@
 
 #ifdef MFEM_USE_GSLIB
 
+namespace gslib
+{
 struct comm;
 struct findpts_data_2;
 struct findpts_data_3;
-struct array;
 struct crystal;
+}
 
 namespace mfem
 {
@@ -50,10 +52,10 @@ public:
 protected:
    Mesh *mesh, *meshsplit;
    IntegrationRule *ir_simplex;    // IntegrationRule to split quads/hex -> simplex
-   struct findpts_data_2 *fdata2D; // gslib's internal data
-   struct findpts_data_3 *fdata3D; // gslib's internal data
-   struct crystal *cr;             // gslib's internal data
-   struct comm *gsl_comm;          // gslib's internal data
+   struct gslib::findpts_data_2 *fdata2D; // gslib's internal data
+   struct gslib::findpts_data_3 *fdata3D; // gslib's internal data
+   struct gslib::crystal *cr;             // gslib's internal data
+   struct gslib::comm *gsl_comm;          // gslib's internal data
    int dim, points_cnt;
    Array<unsigned int> gsl_code, gsl_proc, gsl_elem, gsl_mfem_elem;
    Vector gsl_mesh, gsl_ref, gsl_dist, gsl_mfem_ref;

fem/intrules.cpp

@@ -910,6 +910,9 @@ IntegrationRules::IntegrationRules(int Ref, int type_):
    TetrahedronIntRules.SetSize(32, h_mt);
    TetrahedronIntRules = NULL;
 
+   PyramidIntRules.SetSize(32, h_mt);
+   PyramidIntRules = NULL;
+
    PrismIntRules.SetSize(32, h_mt);
    PrismIntRules = NULL;
 
@@ -930,6 +933,7 @@ const IntegrationRule &IntegrationRules::Get(int GeomType, int Order)
       case Geometry::TETRAHEDRON: ir_array = &TetrahedronIntRules; break;
       case Geometry::CUBE:        ir_array = &CubeIntRules; break;
       case Geometry::PRISM:       ir_array = &PrismIntRules; break;
+      case Geometry::PYRAMID:     ir_array = &PyramidIntRules; break;
       default:
          mfem_error("IntegrationRules::Get(...) : Unknown geometry type!");
          ir_array = NULL;
@@ -976,6 +980,7 @@ void IntegrationRules::Set(int GeomType, int Order, IntegrationRule &IntRule)
       case Geometry::TETRAHEDRON: ir_array = &TetrahedronIntRules; break;
       case Geometry::CUBE:        ir_array = &CubeIntRules; break;
       case Geometry::PRISM:       ir_array = &PrismIntRules; break;
+      case Geometry::PYRAMID:     ir_array = &PyramidIntRules; break;
       default:
          mfem_error("IntegrationRules::Set(...) : Unknown geometry type!");
          ir_array = NULL;
@@ -1019,6 +1024,7 @@ IntegrationRules::~IntegrationRules()
    DeleteIntRuleArray(TetrahedronIntRules);
    DeleteIntRuleArray(CubeIntRules);
    DeleteIntRuleArray(PrismIntRules);
+   DeleteIntRuleArray(PyramidIntRules);
 }
 
 
@@ -1041,6 +1047,8 @@ IntegrationRule *IntegrationRules::GenerateIntegrationRule(int GeomType,
          return CubeIntegrationRule(Order);
       case Geometry::PRISM:
          return PrismIntegrationRule(Order);
+      case Geometry::PYRAMID:
+         return PyramidIntegrationRule(Order);
       default:
          mfem_error("IntegrationRules::Set(...) : Unknown geometry type!");
          return NULL;
@@ -1648,6 +1656,30 @@ IntegrationRule *IntegrationRules::TetrahedronIntegrationRule(int Order)
    }
 }
 
+// Integration rules for reference pyramid
+IntegrationRule *IntegrationRules::PyramidIntegrationRule(int Order)
+{
+   // This is a simple integration rule adapted from an integration
+   // rule for a cube which seems to be adequate for now. When we
+   // implement high order finite elements for pyramids we should
+   // revisit this and see if we can improve upon it.
+   const IntegrationRule &irc = Get(Geometry::CUBE, Order);
+   int npts = irc.GetNPoints();
+   AllocIntRule(PyramidIntRules, Order);
+   PyramidIntRules[Order] = new IntegrationRule(npts);
+
+   for (int k=0; k<npts; k++)
+   {
+      const IntegrationPoint & ipc = irc.IntPoint(k);
+      IntegrationPoint & ipp = PyramidIntRules[Order]->IntPoint(k);
+      ipp.x = ipc.x * (1.0 - ipc.z);
+      ipp.y = ipc.y * (1.0 - ipc.z);
+      ipp.z = ipc.z;
+      ipp.weight = ipc.weight / 3.0;
+   }
+   return PyramidIntRules[Order];
+}
+
 // Integration rules for reference prism
 IntegrationRule *IntegrationRules::PrismIntegrationRule(int Order)
 {

fem/intrules.hpp

@@ -323,6 +323,7 @@ private:
    Array<IntegrationRule *> TriangleIntRules;
    Array<IntegrationRule *> SquareIntRules;
    Array<IntegrationRule *> TetrahedronIntRules;
+   Array<IntegrationRule *> PyramidIntRules;
    Array<IntegrationRule *> PrismIntRules;
    Array<IntegrationRule *> CubeIntRules;
 
@@ -351,6 +352,7 @@ private:
    IntegrationRule *TriangleIntegrationRule(int Order);
    IntegrationRule *SquareIntegrationRule(int Order);
    IntegrationRule *TetrahedronIntegrationRule(int Order);
+   IntegrationRule *PyramidIntegrationRule(int Order);
    IntegrationRule *PrismIntegrationRule(int Order);
    IntegrationRule *CubeIntegrationRule(int Order);
 

fem/linearform.cpp

@@ -103,6 +103,7 @@ void LinearForm::Assemble()
 {
    Array<int> vdofs;
    ElementTransformation *eltrans;
+   DofTransformation *doftrans;
    Vector elemvect;
 
    int i;
@@ -134,10 +135,14 @@ void LinearForm::Assemble()
             if ( domain_integs_marker[k] == NULL ||
                  (*(domain_integs_marker[k]))[elem_attr-1] == 1 )
             {
-               fes -> GetElementVDofs (i, vdofs);
+               doftrans = fes -> GetElementVDofs (i, vdofs);
                eltrans = fes -> GetElementTransformation (i);
                domain_integs[k]->AssembleRHSElementVect(*fes->GetFE(i),
                                                         *eltrans, elemvect);
+               if (doftrans)
+               {
+                  doftrans->TransformDual(elemvect);
+               }
                AddElementVector (vdofs, elemvect);
             }
          }
@@ -174,7 +179,7 @@ void LinearForm::Assemble()
       {
          const int bdr_attr = mesh->GetBdrAttribute(i);
          if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
-         fes -> GetBdrElementVDofs (i, vdofs);
+         doftrans = fes -> GetBdrElementVDofs (i, vdofs);
          eltrans = fes -> GetBdrElementTransformation (i);
          for (int k=0; k < boundary_integs.Size(); k++)
          {
@@ -184,6 +189,10 @@ void LinearForm::Assemble()
             boundary_integs[k]->AssembleRHSElementVect(*fes->GetBE(i),
                                                        *eltrans, elemvect);
 
+            if (doftrans)
+            {
+               doftrans->TransformDual(elemvect);
+            }
             AddElementVector (vdofs, elemvect);
          }
       }
@@ -267,18 +276,16 @@ void LinearForm::Assemble()
 
 void LinearForm::Update(FiniteElementSpace *f, Vector &v, int v_offset)
 {
+   MFEM_ASSERT(v.Size() >= v_offset + f->GetVSize(), "");
    fes = f;
-   NewMemoryAndSize(Memory<double>(v.GetMemory(), v_offset, f->GetVSize()),
-                    f->GetVSize(), false);
+   v.UseDevice(true);
+   this->Vector::MakeRef(v, v_offset, fes->GetVSize());
    ResetDeltaLocations();
 }
 
 void LinearForm::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
 {
-   MFEM_ASSERT(v.Size() >= v_offset + f->GetVSize(), "");
-   fes = f;
-   v.UseDevice(true);
-   this->Vector::MakeRef(v, v_offset, fes->GetVSize());
+   Update(f, v, v_offset);
 }
 
 void LinearForm::AssembleDelta()

fem/nonlinearform.cpp

@@ -630,7 +630,7 @@ double BlockNonlinearForm::GetEnergyBlocked(const BlockVector &bx) const
 
 double BlockNonlinearForm::GetEnergy(const Vector &x) const
 {
-   xs.Update(x.GetData(), block_offsets);
+   xs.Update(const_cast<Vector&>(x), block_offsets);
    return GetEnergyBlocked(xs);
 }
 
@@ -646,7 +646,9 @@ void BlockNonlinearForm::MultBlocked(const BlockVector &bx,
    Array<const FiniteElement *> fe2(fes.Size());
    ElementTransformation *T;
 
+   by.UseDevice(true);
    by = 0.0;
+   by.SyncToBlocks();
    for (int s=0; s<fes.Size(); ++s)
    {
       el_x_const[s] = el_x[s] = new Vector();
@@ -785,6 +787,8 @@ void BlockNonlinearForm::MultBlocked(const BlockVector &bx,
       delete el_y[s];
       delete el_x[s];
    }
+
+   by.SyncFromBlocks();
 }
 
 const BlockVector &BlockNonlinearForm::Prolongate(const BlockVector &bx) const
@@ -805,8 +809,8 @@ const BlockVector &BlockNonlinearForm::Prolongate(const BlockVector &bx) const
 
 void BlockNonlinearForm::Mult(const Vector &x, Vector &y) const
 {
-   BlockVector bx(x.GetData(), block_trueOffsets);
-   BlockVector by(y.GetData(), block_trueOffsets);
+   BlockVector bx(const_cast<Vector&>(x), block_trueOffsets);
+   BlockVector by(y, block_trueOffsets);
 
    const BlockVector &pbx = Prolongate(bx);
    if (needs_prolongation)
@@ -815,8 +819,8 @@ void BlockNonlinearForm::Mult(const Vector &x, Vector &y) const
    }
    BlockVector &pby = needs_prolongation ? aux2 : by;
 
-   xs.Update(pbx.GetData(), block_offsets);
-   ys.Update(pby.GetData(), block_offsets);
+   xs.Update(const_cast<BlockVector&>(pbx), block_offsets);
+   ys.Update(pby, block_offsets);
    MultBlocked(xs, ys);
 
    for (int s = 0; s < fes.Size(); s++)
@@ -1021,7 +1025,7 @@ void BlockNonlinearForm::ComputeGradientBlocked(const BlockVector &bx) const
 
 Operator &BlockNonlinearForm::GetGradient(const Vector &x) const
 {
-   BlockVector bx(x.GetData(), block_trueOffsets);
+   BlockVector bx(const_cast<Vector&>(x), block_trueOffsets);
    const BlockVector &pbx = Prolongate(bx);
 
    ComputeGradientBlocked(pbx);

fem/nonlininteg.hpp

@@ -121,7 +121,7 @@ public:
        @param[in,out] y  The result Vector: @f$ y += G x @f$. */
    virtual void AddMultGradPA(const Vector &x, Vector &y) const;
 
-   /// Method for computing the diagonal of the gradient with partial assmebly.
+   /// Method for computing the diagonal of the gradient with partial assembly.
    /** The result Vector @a diag is an E-Vector. This method can be called only
        after the method AssembleGradPA() has been called.
 

fem/pbilinearform.cpp

@@ -317,14 +317,11 @@ ParallelEliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
 void ParBilinearForm::TrueAddMult(const Vector &x, Vector &y, const double a)
 const
 {
-   MFEM_VERIFY(interior_face_integs.Size() == 0,
-               "the case of interior face integrators is not"
-               " implemented");
-
    if (X.ParFESpace() != pfes)
    {
       X.SetSpace(pfes);
       Y.SetSpace(pfes);
+      Ytmp.SetSize(pfes->GetTrueVSize());
    }
 
    X.Distribute(&x);
@@ -334,9 +331,13 @@ const
    }
    else
    {
+      MFEM_VERIFY(interior_face_integs.Size() == 0,
+                  "the case of interior face integrators is not"
+                  " implemented");
       mat->Mult(X, Y);
    }
-   pfes->Dof_TrueDof_Matrix()->MultTranspose(a, Y, 1.0, y);
+   pfes->GetProlongationMatrix()->MultTranspose(Y, Ytmp);
+   y.Add(a,Ytmp);
 }
 
 void ParBilinearForm::FormLinearSystem(
@@ -473,7 +474,7 @@ void ParBilinearForm::RecoverFEMSolution(
    else
    {
       // Apply conforming prolongation
-      x.SetSize(P.Height());
+      x.SetSize(P.Height(), GetHypreMemoryType());
       P.Mult(X, x);
    }
 }

fem/pbilinearform.hpp

@@ -33,6 +33,7 @@ protected:
 
    /// Auxiliary objects used in TrueAddMult().
    mutable ParGridFunction X, Y;
+   mutable Vector Ytmp;
 
    OperatorHandle p_mat, p_mat_e;
 

fem/pfespace.cpp

@@ -128,6 +128,9 @@ void ParFiniteElementSpace::ParInit(ParMesh *pm)
    {
       ApplyLDofSigns(*elem_dof);
    }
+
+   // Check for shared trianglular faces with interior Nedelec DoFs
+   CheckNDSTriaDofs();
 }
 
 void ParFiniteElementSpace::Construct()
@@ -464,32 +467,53 @@ void ParFiniteElementSpace::ApplyLDofSigns(Table &el_dof) const
    ApplyLDofSigns(all_dofs);
 }
 
-void ParFiniteElementSpace::GetElementDofs(int i, Array<int> &dofs) const
+DofTransformation *
+ParFiniteElementSpace::GetElementDofs(int i, Array<int> &dofs) const
 {
    if (elem_dof)
    {
       elem_dof->GetRow(i, dofs);
-      return;
+
+      if (DoFTrans[mesh->GetElementBaseGeometry(i)])
+      {
+         Array<int> Fo;
+         elem_fos->GetRow(i, Fo);
+         DoFTrans[mesh->GetElementBaseGeometry(i)]->SetFaceOrientations(Fo);
+         return DoFTrans[mesh->GetElementBaseGeometry(i)];
+      }
+      return NULL;
    }
-   FiniteElementSpace::GetElementDofs(i, dofs);
+   DofTransformation * doftrans = FiniteElementSpace::GetElementDofs(i, dofs);
    if (Conforming())
    {
       ApplyLDofSigns(dofs);
    }
+   return doftrans;
 }
 
-void ParFiniteElementSpace::GetBdrElementDofs(int i, Array<int> &dofs) const
+DofTransformation *
+ParFiniteElementSpace::GetBdrElementDofs(int i, Array<int> &dofs) const
 {
    if (bdr_elem_dof)
    {
       bdr_elem_dof->GetRow(i, dofs);
-      return;
+
+      if (DoFTrans[mesh->GetBdrElementBaseGeometry(i)])
+      {
+         Array<int> Fo;
+         bdr_elem_fos -> GetRow (i, Fo);
+         DoFTrans[mesh->GetBdrElementBaseGeometry(i)]->SetFaceOrientations(Fo);
+         return DoFTrans[mesh->GetBdrElementBaseGeometry(i)];
+      }
+      return NULL;
    }
-   FiniteElementSpace::GetBdrElementDofs(i, dofs);
+   DofTransformation * doftrans =
+      FiniteElementSpace::GetBdrElementDofs(i, dofs);
    if (Conforming())
    {
       ApplyLDofSigns(dofs);
    }
+   return doftrans;
 }
 
 int ParFiniteElementSpace::GetFaceDofs(int i, Array<int> &dofs,
@@ -657,59 +681,266 @@ void ParFiniteElementSpace::GenerateGlobalOffsets() const
    }
 }
 
+void ParFiniteElementSpace::CheckNDSTriaDofs()
+{
+   // Check for Nedelec basis
+   bool nd_basis = dynamic_cast<const ND_FECollection*>(fec);
+   if (!nd_basis)
+   {
+      nd_strias = false;
+      return;
+   }
+
+   // Check for interior face dofs on triangles (the use of TETRAHEDRON
+   // is not an error)
+   bool nd_fdof  = fec->HasFaceDofs(Geometry::TETRAHEDRON,
+                                    GetMaxElementOrder());
+   if (!nd_fdof)
+   {
+      nd_strias = false;
+      return;
+   }
+
+   // Check for shared triangle faces
+   bool strias   = false;
+   {
+      int ngrps = pmesh->GetNGroups();
+      for (int g = 1; g < ngrps; g++)
+      {
+         strias |= pmesh->GroupNTriangles(g);
+      }
+   }
+
+   // Combine results
+   int loc_nd_strias = strias ? 1 : 0;
+   int glb_nd_strias = 0;
+   MPI_Allreduce(&loc_nd_strias, &glb_nd_strias, 1,
+                 MPI_INTEGER, MPI_SUM, MyComm);
+   nd_strias = glb_nd_strias > 0;
+}
+
 void ParFiniteElementSpace::Build_Dof_TrueDof_Matrix() const // matrix P
 {
    MFEM_ASSERT(Conforming(), "wrong code path");
 
    if (P) { return; }
 
-   int ldof  = GetVSize();
-   int ltdof = TrueVSize();
-
-   HYPRE_Int *i_diag = Memory<HYPRE_Int>(ldof+1);
-   HYPRE_Int *j_diag = Memory<HYPRE_Int>(ltdof);
-   int diag_counter;
-
-   HYPRE_Int *i_offd = Memory<HYPRE_Int>(ldof+1);
-   HYPRE_Int *j_offd = Memory<HYPRE_Int>(ldof-ltdof);
-   int offd_counter;
-
-   HYPRE_BigInt *cmap = Memory<HYPRE_BigInt>(ldof-ltdof);
-
-   HYPRE_BigInt *col_starts = GetTrueDofOffsets();
-   HYPRE_BigInt *row_starts = GetDofOffsets();
-
-   Array<Pair<HYPRE_BigInt, int> > cmap_j_offd(ldof-ltdof);
-
-   i_diag[0] = i_offd[0] = 0;
-   diag_counter = offd_counter = 0;
-   for (int i = 0; i < ldof; i++)
+   if (!nd_strias)
    {
-      int ltdof = GetLocalTDofNumber(i);
-      if (ltdof >= 0)
+      // Safe to assume 1-1 correspondence between shared dofs
+      int ldof  = GetVSize();
+      int ltdof = TrueVSize();
+
+      HYPRE_Int *i_diag = Memory<HYPRE_Int>(ldof+1);
+      HYPRE_Int *j_diag = Memory<HYPRE_Int>(ltdof);
+      int diag_counter;
+
+      HYPRE_Int *i_offd = Memory<HYPRE_Int>(ldof+1);
+      HYPRE_Int *j_offd = Memory<HYPRE_Int>(ldof-ltdof);
+      int offd_counter;
+
+      HYPRE_BigInt *cmap   = Memory<HYPRE_BigInt>(ldof-ltdof);
+
+      HYPRE_BigInt *col_starts = GetTrueDofOffsets();
+      HYPRE_BigInt *row_starts = GetDofOffsets();
+
+      Array<Pair<HYPRE_BigInt, int> > cmap_j_offd(ldof-ltdof);
+
+      i_diag[0] = i_offd[0] = 0;
+      diag_counter = offd_counter = 0;
+      for (int i = 0; i < ldof; i++)
       {
-         j_diag[diag_counter++] = ltdof;
+         int ltdof = GetLocalTDofNumber(i);
+         if (ltdof >= 0)
+         {
+            j_diag[diag_counter++] = ltdof;
+         }
+         else
+         {
+            cmap_j_offd[offd_counter].one = GetGlobalTDofNumber(i);
+            cmap_j_offd[offd_counter].two = offd_counter;
+            offd_counter++;
+         }
+         i_diag[i+1] = diag_counter;
+         i_offd[i+1] = offd_counter;
       }
-      else
+
+      SortPairs<HYPRE_BigInt, int>(cmap_j_offd, offd_counter);
+
+      for (int i = 0; i < offd_counter; i++)
       {
-         cmap_j_offd[offd_counter].one = GetGlobalTDofNumber(i);
-         cmap_j_offd[offd_counter].two = offd_counter;
-         offd_counter++;
+         cmap[i] = cmap_j_offd[i].one;
+         j_offd[cmap_j_offd[i].two] = i;
       }
-      i_diag[i+1] = diag_counter;
-      i_offd[i+1] = offd_counter;
+
+      P = new HypreParMatrix(MyComm, MyRank, NRanks, row_starts, col_starts,
+                             i_diag, j_diag, i_offd, j_offd,
+                             cmap, offd_counter);
    }
-
-   SortPairs<HYPRE_BigInt, int>(cmap_j_offd, offd_counter);
-
-   for (int i = 0; i < offd_counter; i++)
+   else
    {
-      cmap[i] = cmap_j_offd[i].one;
-      j_offd[cmap_j_offd[i].two] = i;
-   }
+      // Some shared dofs will be linear combinations of others
+      int ldof  = GetVSize();
+      int ltdof = TrueVSize();
 
-   P = new HypreParMatrix(MyComm, MyRank, NRanks, row_starts, col_starts,
-                          i_diag, j_diag, i_offd, j_offd, cmap, offd_counter);
+      HYPRE_Int gdof  = -1;
+      HYPRE_Int gtdof = -1;
+
+      MPI_Allreduce(&ldof, &gdof, 1, HYPRE_MPI_INT, MPI_SUM, MyComm);
+      MPI_Allreduce(&ltdof, &gtdof, 1, HYPRE_MPI_INT, MPI_SUM, MyComm);
+
+      // Ensure face orientations have been communicated
+      pmesh->ExchangeFaceNbrData();
+
+      // Locate and count non-zeros in off-diagonal portion of P
+      int nnz_offd = 0;
+      Array<int> ldsize(ldof); ldsize = 0;
+      Array<int> ltori(ldof);  ltori = 0; // Local triangle orientations
+      {
+         int ngrps = pmesh->GetNGroups();
+         int nedofs = fec->DofForGeometry(Geometry::SEGMENT);
+         Array<int> sdofs;
+         for (int g = 1; g < ngrps; g++)
+         {
+            if (pmesh->gtopo.IAmMaster(g))
+            {
+               continue;
+            }
+            for (int ei=0; ei<pmesh->GroupNEdges(g); ei++)
+            {
+               this->GetSharedEdgeDofs(g, ei, sdofs);
+               for (int i=0; i<sdofs.Size(); i++)
+               {
+                  int ind = (sdofs[i]>=0) ? sdofs[i] : (-sdofs[i]-1);
+                  if (ldsize[ind] == 0) { nnz_offd++; }
+                  ldsize[ind] = 1;
+               }
+            }
+            for (int fi=0; fi<pmesh->GroupNTriangles(g); fi++)
+            {
+               int face, ori, info1, info2;
+               pmesh->GroupTriangle(g, fi, face, ori);
+               pmesh->GetFaceInfos(face, &info1, &info2);
+               this->GetSharedTriangleDofs(g, fi, sdofs);
+               for (int i=0; i<3*nedofs; i++)
+               {
+                  int ind = (sdofs[i]>=0) ? sdofs[i] : (-sdofs[i]-1);
+                  if (ldsize[ind] == 0) { nnz_offd++; }
+                  ldsize[ind] = 1;
+               }
+               for (int i=3*nedofs; i<sdofs.Size(); i++)
+               {
+                  if (ldsize[sdofs[i]] == 0) { nnz_offd += 2; }
+                  ldsize[sdofs[i]] = 2;
+                  ltori[sdofs[i]]  = info2 % 64;
+               }
+            }
+            for (int fi=0; fi<pmesh->GroupNQuadrilaterals(g); fi++)
+            {
+               this->GetSharedQuadrilateralDofs(g, fi, sdofs);
+               for (int i=0; i<sdofs.Size(); i++)
+               {
+                  int ind = (sdofs[i]>=0) ? sdofs[i] : (-sdofs[i]-1);
+                  if (ldsize[ind] == 0) { nnz_offd++; }
+                  ldsize[ind] = 1;
+               }
+            }
+         }
+      }
+
+      HYPRE_Int *i_diag = new HYPRE_Int[ldof+1];
+      HYPRE_Int *j_diag = new HYPRE_Int[ltdof];
+      double    *d_diag = new double[ltdof];
+      int diag_counter;
+
+      HYPRE_Int *i_offd = new HYPRE_Int[ldof+1];
+      HYPRE_Int *j_offd = new HYPRE_Int[nnz_offd];
+      double    *d_offd = new double[nnz_offd];
+      int offd_counter;
+
+      HYPRE_BigInt *cmap   = new HYPRE_BigInt[ldof-ltdof];
+
+      HYPRE_BigInt *col_starts = GetTrueDofOffsets();
+      HYPRE_BigInt *row_starts = GetDofOffsets();
+
+      Array<Pair<HYPRE_BigInt, int> > cmap_j_offd(ldof-ltdof);
+
+      i_diag[0] = i_offd[0] = 0;
+      diag_counter = offd_counter = 0;
+      int offd_col_counter = 0;
+      for (int i = 0; i < ldof; i++)
+      {
+         int ltdof = GetLocalTDofNumber(i);
+         if (ltdof >= 0)
+         {
+            j_diag[diag_counter]   = ltdof;
+            d_diag[diag_counter++] = 1.0;
+         }
+         else
+         {
+            if (ldsize[i] == 1)
+            {
+               cmap_j_offd[offd_col_counter].one = GetGlobalTDofNumber(i);
+               cmap_j_offd[offd_col_counter].two = offd_counter;
+               offd_counter++;
+               offd_col_counter++;
+            }
+            else
+            {
+               cmap_j_offd[offd_col_counter].one = GetGlobalTDofNumber(i);
+               cmap_j_offd[offd_col_counter].two = offd_counter;
+               offd_counter += 2;
+               offd_col_counter++;
+               i_diag[i+1] = diag_counter;
+               i_offd[i+1] = offd_counter;
+               i++;
+               cmap_j_offd[offd_col_counter].one = GetGlobalTDofNumber(i);
+               cmap_j_offd[offd_col_counter].two = offd_counter;
+               offd_counter += 2;
+               offd_col_counter++;
+            }
+         }
+         i_diag[i+1] = diag_counter;
+         i_offd[i+1] = offd_counter;
+      }
+
+      SortPairs<HYPRE_BigInt, int>(cmap_j_offd, offd_col_counter);
+
+      for (int i = 0; i < nnz_offd; i++)
+      {
+         j_offd[i] = -1;
+         d_offd[i] = 0.0;
+      }
+
+      for (int i = 0; i < offd_col_counter; i++)
+      {
+         cmap[i] = cmap_j_offd[i].one;
+         j_offd[cmap_j_offd[i].two] = i;
+      }
+
+      for (int i = 0; i < ldof; i++)
+      {
+         if (i_offd[i+1] == i_offd[i] + 1)
+         {
+            d_offd[i_offd[i]] = 1.0;
+         }
+         else if (i_offd[i+1] == i_offd[i] + 2)
+         {
+            const double * T = ND_DofTransformation
+                               ::GetFaceTransform(ltori[i]).GetData();
+            j_offd[i_offd[i] + 1] = j_offd[i_offd[i]] + 1;
+            d_offd[i_offd[i]] = T[0]; d_offd[i_offd[i] + 1] = T[2];
+            i++;
+            j_offd[i_offd[i] + 1] = j_offd[i_offd[i]];
+            j_offd[i_offd[i]] = j_offd[i_offd[i] + 1] - 1;
+            d_offd[i_offd[i]] = T[1]; d_offd[i_offd[i] + 1] = T[3];
+         }
+      }
+
+      P = new HypreParMatrix(MyComm, gdof, gtdof, row_starts, col_starts,
+                             i_diag, j_diag, d_diag, i_offd, j_offd, d_offd,
+                             offd_col_counter, cmap);
+   }
 
    SparseMatrix Pdiag;
    P->GetDiag(Pdiag);
@@ -913,6 +1144,8 @@ const Operator *ParFiniteElementSpace::GetProlongationMatrix() const
    {
       if (Pconf) { return Pconf; }
 
+      if (nd_strias) { return Dof_TrueDof_Matrix(); }
+
       if (NRanks == 1)
       {
          Pconf = new IdentityOperator(GetTrueVSize());
@@ -1216,10 +1449,29 @@ void ParFiniteElementSpace::ExchangeFaceNbrData()
    delete [] requests;
 }
 
-void ParFiniteElementSpace::GetFaceNbrElementVDofs(
+DofTransformation *ParFiniteElementSpace::GetFaceNbrElementVDofs(
    int i, Array<int> &vdofs) const
 {
    face_nbr_element_dof.GetRow(i, vdofs);
+
+   DofTransformation *doftrans = NULL;
+   Geometry::Type geom = GetFaceNbrFE(i)->GetGeomType();
+   if (DoFTrans[geom])
+   {
+      Array<int> F, Fo;
+      pmesh->GetFaceNbrElementFaces(pmesh->GetNE() + i, F, Fo);
+      doftrans = DoFTrans[geom];
+      doftrans->SetFaceOrientations(Fo);
+   }
+   if (vdim == 1 || doftrans == NULL)
+   {
+      return doftrans;
+   }
+   else
+   {
+      VDoFTrans.SetDofTransformation(*doftrans);
+      return &VDoFTrans;
+   }
 }
 
 void ParFiniteElementSpace::GetFaceNbrFaceVDofs(int i, Array<int> &vdofs) const
@@ -1278,12 +1530,17 @@ const FiniteElement *ParFiniteElementSpace::GetFaceNbrFaceFE(int i) const
 
 void ParFiniteElementSpace::Lose_Dof_TrueDof_Matrix()
 {
+   P -> StealData();
+#if MFEM_HYPRE_VERSION <= 22200
    hypre_ParCSRMatrix *csrP = (hypre_ParCSRMatrix*)(*P);
    hypre_ParCSRMatrixOwnsRowStarts(csrP) = 1;
    hypre_ParCSRMatrixOwnsColStarts(csrP) = 1;
-   P -> StealData();
    dof_offsets.LoseData();
    tdof_offsets.LoseData();
+#else
+   dof_offsets.DeleteAll();
+   tdof_offsets.DeleteAll();
+#endif
 }
 
 void ParFiniteElementSpace::ConstructTrueDofs()
@@ -2526,7 +2783,8 @@ static int_type* make_j_array(int_type* I, int nrows)
 
 HypreParMatrix*
 ParFiniteElementSpace::RebalanceMatrix(int old_ndofs,
-                                       const Table* old_elem_dof)
+                                       const Table* old_elem_dof,
+                                       const Table* old_elem_fos)
 {
    MFEM_VERIFY(Nonconforming(), "Only supported for nonconforming meshes.");
    MFEM_VERIFY(old_dof_offsets.Size(), "ParFiniteElementSpace::Update needs to "
@@ -2651,7 +2909,8 @@ struct DerefDofMessage
 
 HypreParMatrix*
 ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
-                                                  const Table* old_elem_dof)
+                                                  const Table* old_elem_dof,
+                                                  const Table *old_elem_fos)
 {
    int nrk = HYPRE_AssumedPartitionCheck() ? 2 : NRanks;
 
@@ -2885,19 +3144,10 @@ ParFiniteElementSpace::ParallelDerefinementMatrix(int old_ndofs,
 
    HypreParMatrix* R;
    R = new HypreParMatrix(MyComm, dof_offsets[nrk], old_dof_offsets[nrk],
-                          dof_offsets, old_dof_offsets, diag, offd, cmap);
+                          dof_offsets, old_dof_offsets, diag, offd, cmap,
+                          true);
 
-#ifndef HYPRE_BIGINT
-   diag->LoseData();
-   offd->LoseData();
-#else
-   diag->SetDataOwner(false);
-   offd->SetDataOwner(false);
-#endif
-   delete diag;
-   delete offd;
-
-   R->SetOwnerFlags(3, 3, 1);
+   R->SetOwnerFlags(R->OwnsDiag(), R->OwnsOffd(), 1);
 
    return R;
 }
@@ -3016,13 +3266,16 @@ void ParFiniteElementSpace::Update(bool want_transform)
    }
 
    Table* old_elem_dof = NULL;
+   Table* old_elem_fos = NULL;
    int old_ndofs;
 
    // save old DOF table
    if (want_transform)
    {
       old_elem_dof = elem_dof;
+      old_elem_fos = elem_fos;
       elem_dof = NULL;
+      elem_fos = NULL;
       old_ndofs = ndofs;
       Swap(dof_offsets, old_dof_offsets);
    }
@@ -3044,22 +3297,25 @@ void ParFiniteElementSpace::Update(bool want_transform)
          {
             if (Th.Type() != Operator::MFEM_SPARSEMAT)
             {
-               Th.Reset(new RefinementOperator(this, old_elem_dof, old_ndofs));
+               Th.Reset(new RefinementOperator(this, old_elem_dof,
+                                               old_elem_fos, old_ndofs));
                // The RefinementOperator takes ownership of 'old_elem_dofs', so
                // we no longer own it:
                old_elem_dof = NULL;
+               old_elem_fos = NULL;
             }
             else
             {
                // calculate fully assembled matrix
-               Th.Reset(RefinementMatrix(old_ndofs, old_elem_dof));
+               Th.Reset(RefinementMatrix(old_ndofs, old_elem_dof, old_elem_fos));
             }
             break;
          }
 
          case Mesh::DEREFINE:
          {
-            Th.Reset(ParallelDerefinementMatrix(old_ndofs, old_elem_dof));
+            Th.Reset(ParallelDerefinementMatrix(old_ndofs, old_elem_dof,
+                                                old_elem_fos));
             if (Nonconforming())
             {
                Th.SetOperatorOwner(false);
@@ -3071,7 +3327,7 @@ void ParFiniteElementSpace::Update(bool want_transform)
 
          case Mesh::REBALANCE:
          {
-            Th.Reset(RebalanceMatrix(old_ndofs, old_elem_dof));
+            Th.Reset(RebalanceMatrix(old_ndofs, old_elem_dof, old_elem_fos));
             break;
          }
 
@@ -3080,6 +3336,7 @@ void ParFiniteElementSpace::Update(bool want_transform)
       }
 
       delete old_elem_dof;
+      delete old_elem_fos;
    }
 }
 

fem/pfespace.hpp

@@ -87,6 +87,12 @@ private:
        this is a TransposeOperator wrapping R. */
    mutable Operator *R_transpose;
 
+   /// Flag indicating the existence of shared triangles with interior ND dofs
+   bool nd_strias;
+
+   /// Resets nd_strias flag at constuction or after rebalancing
+   void CheckNDSTriaDofs();
+
    ParNURBSExtension *pNURBSext() const
    { return dynamic_cast<ParNURBSExtension *>(NURBSext); }
 
@@ -174,14 +180,16 @@ private:
        The result is a parallel permutation matrix that can be used to update
        all grid functions defined on this space. */
    HypreParMatrix* RebalanceMatrix(int old_ndofs,
-                                   const Table* old_elem_dof);
+                                   const Table* old_elem_dof,
+                                   const Table* old_elem_fos);
 
    /** Calculate a GridFunction restriction matrix after mesh derefinement.
        The matrix is constructed so that the new grid function interpolates
        the original function, i.e., the original function is evaluated at the
        nodes of the coarse function. */
    HypreParMatrix* ParallelDerefinementMatrix(int old_ndofs,
-                                              const Table *old_elem_dof);
+                                              const Table *old_elem_dof,
+                                              const Table *old_elem_fos);
 
    /// Updates the internal mesh pointer. @warning @a new_mesh must be
    /// <b>topologically identical</b> to the existing mesh. Used if the address
@@ -202,6 +210,8 @@ public:
    int num_face_nbr_dofs;
    // Face-neighbor-element to face-neighbor dof
    Table face_nbr_element_dof;
+   // Face-neighbor-element face orientations
+   Table face_nbr_element_fos;
    // Face-neighbor to ldof in the face-neighbor numbering
    Table face_nbr_ldof;
    // The global ldof indices of the face-neighbor dofs
@@ -279,10 +289,10 @@ public:
    virtual int GetTrueVSize() const { return ltdof_size; }
 
    /// Returns indexes of degrees of freedom in array dofs for i'th element.
-   virtual void GetElementDofs(int i, Array<int> &dofs) const;
+   virtual DofTransformation *GetElementDofs(int i, Array<int> &dofs) const;
 
    /// Returns indexes of degrees of freedom for i'th boundary element.
-   virtual void GetBdrElementDofs(int i, Array<int> &dofs) const;
+   virtual DofTransformation *GetBdrElementDofs(int i, Array<int> &dofs) const;
 
    /** Returns the indexes of the degrees of freedom for i'th face
        including the dofs for the edges and the vertices of the face. */
@@ -291,7 +301,7 @@ public:
    /** Returns pointer to the FiniteElement in the FiniteElementCollection
        associated with i'th element in the mesh object. If @a i is greater than
        or equal to the number of local mesh elements, @a i will be interpreted
-       as a shifted index of a face neigbor element. */
+       as a shifted index of a face neighbor element. */
    virtual const FiniteElement *GetFE(int i) const;
 
    /** Returns an Operator that converts L-vectors to E-vectors on each face.
@@ -382,7 +392,7 @@ public:
    // Face-neighbor functions
    void ExchangeFaceNbrData();
    int GetFaceNbrVSize() const { return num_face_nbr_dofs; }
-   void GetFaceNbrElementVDofs(int i, Array<int> &vdofs) const;
+   DofTransformation *GetFaceNbrElementVDofs(int i, Array<int> &vdofs) const;
    void GetFaceNbrFaceVDofs(int i, Array<int> &vdofs) const;
    const FiniteElement *GetFaceNbrFE(int i) const;
    const FiniteElement *GetFaceNbrFaceFE(int i) const;
@@ -397,6 +407,8 @@ public:
    bool Conforming() const { return pmesh->pncmesh == NULL && !nonconf_P; }
    bool Nonconforming() const { return pmesh->pncmesh != NULL || nonconf_P; }
 
+   bool SharedNDTriangleDofs() const { return nd_strias; }
+
    // Transfer parallel true-dof data from coarse_fes, defined on a coarse mesh,
    // to this FE space, defined on a refined mesh. See full documentation in the
    // base class, FiniteElementSpace::GetTrueTransferOperator.

fem/pgridfunc.cpp

@@ -325,9 +325,14 @@ void ParGridFunction::GetVectorValue(int i, const IntegrationPoint &ip,
    if (nbr_el_no >= 0)
    {
       Array<int> dofs;
-      pfes->GetFaceNbrElementVDofs(nbr_el_no, dofs);
+      DofTransformation * doftrans = pfes->GetFaceNbrElementVDofs(nbr_el_no,
+                                                                  dofs);
       Vector loc_data;
       face_nbr_data.GetSubVector(dofs, loc_data);
+      if (doftrans)
+      {
+         doftrans->InvTransformPrimal(loc_data);
+      }
       const FiniteElement *FElem = pfes->GetFaceNbrFE(nbr_el_no);
       int dof = FElem->GetDof();
       if (FElem->GetRangeType() == FiniteElement::SCALAR)
@@ -437,12 +442,17 @@ void ParGridFunction::GetVectorValue(ElementTransformation &T,
    }
 
    Array<int> vdofs;
-   pfes->GetFaceNbrElementVDofs(nbr_el_no, vdofs);
+   DofTransformation * doftrans = pfes->GetFaceNbrElementVDofs(nbr_el_no,
+                                                               vdofs);
    const FiniteElement *fe = pfes->GetFaceNbrFE(nbr_el_no);
 
    int dof = fe->GetDof();
    Vector loc_data;
    face_nbr_data.GetSubVector(vdofs, loc_data);
+   if (doftrans)
+   {
+      doftrans->InvTransformPrimal(loc_data);
+   }
    if (fe->GetRangeType() == FiniteElement::SCALAR)
    {
       Vector shape(dof);

fem/pgridfunc.hpp

@@ -65,6 +65,11 @@ public:
    ParGridFunction(ParFiniteElementSpace *pf, double *data) :
       GridFunction(pf, data), pfes(pf) { }
 
+   /** @brief Construct a ParGridFunction using previously allocated Vector
+       @a base starting at the given offset, @a base_offset. */
+   ParGridFunction(ParFiniteElementSpace *pf, Vector &base, int base_offset = 0)
+      : GridFunction(pf, base, base_offset), pfes(pf) { }
+
    /// Construct a ParGridFunction using a GridFunction as external data.
    /** The parallel space @a *pf and the space used by @a *gf should match. The
        data from @a *gf is used as the local data of the ParGridFunction on each

fem/pnonlinearform.cpp

@@ -218,7 +218,8 @@ void ParBlockNonlinearForm::SetEssentialBC(const
 
 double ParBlockNonlinearForm::GetEnergy(const Vector &x) const
 {
-   xs_true.Update(x.GetData(), block_trueOffsets);
+   // xs_true is not modified, so const_cast is okay
+   xs_true.Update(const_cast<Vector &>(x), block_trueOffsets);
    xs.Update(block_offsets);
 
    for (int s = 0; s < fes.Size(); ++s)
@@ -237,8 +238,9 @@ double ParBlockNonlinearForm::GetEnergy(const Vector &x) const
 
 void ParBlockNonlinearForm::Mult(const Vector &x, Vector &y) const
 {
-   xs_true.Update(x.GetData(), block_trueOffsets);
-   ys_true.Update(y.GetData(), block_trueOffsets);
+   // xs_true is not modified, so const_cast is okay
+   xs_true.Update(const_cast<Vector &>(x), block_trueOffsets);
+   ys_true.Update(y, block_trueOffsets);
    xs.Update(block_offsets);
    ys.Update(block_offsets);
 
@@ -262,13 +264,17 @@ void ParBlockNonlinearForm::Mult(const Vector &x, Vector &y) const
 
       ys_true.GetBlock(s).SetSubVector(*ess_tdofs[s], 0.0);
    }
+
+   ys_true.SyncFromBlocks();
+   y.SyncMemory(ys_true);
 }
 
 /// Return the local gradient matrix for the given true-dof vector x
 const BlockOperator & ParBlockNonlinearForm::GetLocalGradient(
    const Vector &x) const
 {
-   xs_true.Update(x.GetData(), block_trueOffsets);
+   // xs_true is not modified, so const_cast is okay
+   xs_true.Update(const_cast<Vector &>(x), block_trueOffsets);
    xs.Update(block_offsets);
 
    for (int s=0; s<fes.Size(); ++s)
@@ -277,7 +283,8 @@ const BlockOperator & ParBlockNonlinearForm::GetLocalGradient(
          xs_true.GetBlock(s), xs.GetBlock(s));
    }
 
-   BlockNonlinearForm::ComputeGradientBlocked(xs); // (re)assemble Grad with b.c.
+   // (re)assemble Grad without b.c. into 'Grads'
+   BlockNonlinearForm::ComputeGradientBlocked(xs);
 
    delete BlockGrad;
    BlockGrad = new BlockOperator(block_offsets);

fem/prestriction.cpp

@@ -33,6 +33,10 @@ ParL2FaceRestriction::ParL2FaceRestriction(const ParFiniteElementSpace &fes,
    // If fespace == L2
    const ParFiniteElementSpace &pfes =
       static_cast<const ParFiniteElementSpace&>(this->fes);
+
+   // Ensure the face neighbor data is constructed
+   pfes.GetParMesh()->ExchangeFaceNbrData();
+
    const FiniteElement *fe = pfes.GetFE(0);
    const TensorBasisElement *tfe = dynamic_cast<const TensorBasisElement*>(fe);
    MFEM_VERIFY(tfe != NULL &&

fem/restriction.cpp

@@ -13,6 +13,13 @@
 #include "gridfunc.hpp"
 #include "fespace.hpp"
 #include "../general/forall.hpp"
+#include <climits>
+
+#ifdef MFEM_USE_MPI
+
+#include "pfespace.hpp"
+
+#endif
 
 namespace mfem
 {
@@ -267,35 +274,25 @@ void ElementRestriction::FillSparseMatrix(const Vector &mat_ea,
    FillJAndData(mat_ea, mat);
 }
 
-template <int MaxNbNbr>
 static MFEM_HOST_DEVICE int GetMinElt(const int *my_elts, const int nbElts,
                                       const int *nbr_elts, const int nbrNbElts)
 {
-   // Building the intersection
-   int inter[MaxNbNbr];
-   int cpt = 0;
+   // Find the minimal element index found in both my_elts[] and nbr_elts[]
+   int min_el = INT_MAX;
    for (int i = 0; i < nbElts; i++)
    {
       const int e_i = my_elts[i];
+      if (e_i >= min_el) { continue; }
       for (int j = 0; j < nbrNbElts; j++)
       {
          if (e_i==nbr_elts[j])
          {
-            inter[cpt] = e_i;
-            cpt++;
+            min_el = e_i; // we already know e_i < min_el
+            break;
          }
       }
    }
-   // Finding the minimum
-   int min = inter[0];
-   for (int i = 1; i < cpt; i++)
-   {
-      if (inter[i] < min)
-      {
-         min = inter[i];
-      }
-   }
-   return min;
+   return min_el;
 }
 
 /** Returns the index where a non-zero entry should be added and increment the
@@ -355,7 +352,7 @@ int ElementRestriction::FillI(SparseMatrix &mat) const
                   const int elt = j_E/elt_dofs;
                   j_elts[e_j] = elt;
                }
-               int min_e = GetMinElt<Max>(i_elts, i_nbElts, j_elts, j_nbElts);
+               int min_e = GetMinElt(i_elts, i_nbElts, j_elts, j_nbElts);
                if (e == min_e) // add the nnz only once
                {
                   GetAndIncrementNnzIndex(i_L, I);
@@ -434,7 +431,7 @@ void ElementRestriction::FillJAndData(const Vector &ea_data,
                   j_elts[e_j] = elt;
                   j_B[e_j]    = j_E%elt_dofs;
                }
-               int min_e = GetMinElt<Max>(i_elts, i_nbElts, j_elts, j_nbElts);
+               int min_e = GetMinElt(i_elts, i_nbElts, j_elts, j_nbElts);
                if (e == min_e) // add the nnz only once
                {
                   double val = 0.0;
@@ -684,6 +681,19 @@ H1FaceRestriction::H1FaceRestriction(const FiniteElementSpace &fes,
      gather_indices(nf*dof)
 {
    if (nf==0) { return; }
+
+#ifdef MFEM_USE_MPI
+
+   // If the underlying finite element space is parallel, ensure the face
+   // neighbor information is generated.
+   if (const ParFiniteElementSpace *pfes
+       = dynamic_cast<const ParFiniteElementSpace*>(&fes))
+   {
+      pfes->GetParMesh()->ExchangeFaceNbrData();
+   }
+
+#endif
+
    // If fespace == H1
    const FiniteElement *fe = fes.GetFE(0);
    const TensorBasisElement *tfe = dynamic_cast<const TensorBasisElement*>(fe);

fem/tmop.cpp

@@ -1314,33 +1314,61 @@ static inline void device_copy(double *d_dest, const double *d_src, int size)
 } // namespace internal
 
 #ifdef MFEM_USE_MPI
-void DiscreteAdaptTC::FinalizeParDiscreteTargetSpec(const ParGridFunction
-                                                    &tspec_)
+void DiscreteAdaptTC::FinalizeParDiscreteTargetSpec(const ParGridFunction &t)
 {
    MFEM_VERIFY(adapt_eval, "SetAdaptivityEvaluator() has not been called!")
    MFEM_VERIFY(ncomp > 0, "No target specifications have been set!");
 
-   ParFiniteElementSpace *ptspec_fes = tspec_.ParFESpace();
+   ParFiniteElementSpace *ptspec_fes = t.ParFESpace();
 
    adapt_eval->SetParMetaInfo(*ptspec_fes->GetParMesh(),
                               *ptspec_fes->FEColl(), ncomp);
-   adapt_eval->SetInitialField(*tspec_fes->GetMesh()->GetNodes(), tspec);
+   adapt_eval->SetInitialField(*ptspec_fes->GetMesh()->GetNodes(), tspec);
 
    tspec_sav = tspec;
 
    delete tspec_fesv;
-   tspec_fesv = new FiniteElementSpace(tspec_fes->GetMesh(),
-                                       tspec_fes->FEColl(), ncomp);
+   tspec_fesv = new FiniteElementSpace(ptspec_fes->GetMesh(),
+                                       ptspec_fes->FEColl(), ncomp);
+
+   delete ptspec_fesv;
+   ptspec_fesv = new ParFiniteElementSpace(ptspec_fes->GetParMesh(),
+                                           ptspec_fes->FEColl(), ncomp);
+
+   delete tspec_pgf;
+   tspec_pgf = new ParGridFunction(ptspec_fesv, tspec);
+   tspec_gf = tspec_pgf;
+}
+
+void DiscreteAdaptTC::ParUpdateAfterMeshTopologyChange()
+{
+   ptspec_fesv->Update();
+   if (tspec_fesv)
+   {
+      delete tspec_fesv;
+      tspec_fesv = new FiniteElementSpace(ptspec_fesv->GetMesh(),
+                                          ptspec_fesv->FEColl(), ncomp);
+   }
+   tspec_pgf->Update();
+   tspec_gf = tspec_pgf;
+   tspec.SetDataAndSize(tspec_pgf->GetData(), tspec_pgf->Size());
+   tspec_sav = tspec;
+
+   adapt_eval->SetParMetaInfo(*ptspec_fesv->GetParMesh(),
+                              *ptspec_fesv->FEColl(), ncomp);
+   adapt_eval->SetInitialField(*ptspec_fesv->GetMesh()->GetNodes(), tspec);
 }
 
 void DiscreteAdaptTC::SetTspecAtIndex(int idx, const ParGridFunction &tspec_)
 {
-   const int vdim     = tspec_.FESpace()->GetVDim(),
-             dof_cnt  = tspec_.Size()/vdim;
+   const int vdim = tspec_.FESpace()->GetVDim(),
+             ndof = tspec_.FESpace()->GetNDofs();
+   MFEM_VERIFY(ndof == tspec.Size()/ncomp, "Inconsistency in SetTspecAtIndex.");
+
    const auto tspec__d = tspec_.Read();
    auto tspec_d = tspec.ReadWrite();
-   const int offset = idx*dof_cnt;
-   internal::device_copy(tspec_d + offset, tspec__d, dof_cnt*vdim);
+   const int offset = idx*ndof;
+   internal::device_copy(tspec_d + offset, tspec__d, ndof*vdim);
    FinalizeParDiscreteTargetSpec(tspec_);
 }
 
@@ -1360,78 +1388,71 @@ void DiscreteAdaptTC::SetParDiscreteTargetSkew(const ParGridFunction &tspec_)
    FinalizeParDiscreteTargetSpec(tspec_);
 }
 
-void DiscreteAdaptTC::SetParDiscreteTargetAspectRatio(const ParGridFunction
-                                                      &tspec_)
+void DiscreteAdaptTC::SetParDiscreteTargetAspectRatio(const ParGridFunction &ar)
 {
-   if (aspectratioidx > -1) { SetTspecAtIndex(aspectratioidx, tspec_); return; }
+   if (aspectratioidx > -1) { SetTspecAtIndex(aspectratioidx, ar); return; }
    aspectratioidx = ncomp;
-   SetDiscreteTargetBase(tspec_);
-   FinalizeParDiscreteTargetSpec(tspec_);
+   SetDiscreteTargetBase(ar);
+   FinalizeParDiscreteTargetSpec(ar);
 }
 
-void DiscreteAdaptTC::SetParDiscreteTargetOrientation(const ParGridFunction
-                                                      &tspec_)
+void DiscreteAdaptTC::SetParDiscreteTargetOrientation(const ParGridFunction &o)
 {
-   if (orientationidx > -1) { SetTspecAtIndex(orientationidx, tspec_); return; }
+   if (orientationidx > -1) { SetTspecAtIndex(orientationidx, o); return; }
    orientationidx = ncomp;
-   SetDiscreteTargetBase(tspec_);
-   FinalizeParDiscreteTargetSpec(tspec_);
+   SetDiscreteTargetBase(o);
+   FinalizeParDiscreteTargetSpec(o);
 }
 
 void DiscreteAdaptTC::SetParDiscreteTargetSpec(const ParGridFunction &tspec_)
 {
    SetParDiscreteTargetSize(tspec_);
-   FinalizeParDiscreteTargetSpec(tspec_);
 }
 #endif // MFEM_USE_MPI
 
 void DiscreteAdaptTC::SetDiscreteTargetBase(const GridFunction &tspec_)
 {
-   const int vdim     = tspec_.FESpace()->GetVDim(),
-             dof_cnt  = tspec_.Size()/vdim;
+   const int vdim = tspec_.FESpace()->GetVDim(),
+             ndof = tspec_.FESpace()->GetNDofs();
 
    ncomp += vdim;
 
-   delete tspec_fes;
-   tspec_fes = new FiniteElementSpace(tspec_.FESpace()->GetMesh(),
-                                      tspec_.FESpace()->FEColl(), 1);
-
    // need to append data to tspec
    // make a copy of tspec->tspec_temp, increase its size, and
    // copy data from tspec_temp -> tspec, then add new entries
    Vector tspec_temp = tspec;
    tspec.UseDevice(true);
    tspec_sav.UseDevice(true);
-   tspec.SetSize(ncomp*dof_cnt);
+   tspec.SetSize(ncomp*ndof);
 
    const auto tspec_temp_d = tspec_temp.Read();
    auto tspec_d = tspec.ReadWrite();
    internal::device_copy(tspec_d, tspec_temp_d, tspec_temp.Size());
 
    const auto tspec__d = tspec_.Read();
-   const int offset = (ncomp-vdim)*dof_cnt;
-   internal::device_copy(tspec_d + offset, tspec__d, dof_cnt*vdim);
+   const int offset = (ncomp-vdim)*ndof;
+   internal::device_copy(tspec_d + offset, tspec__d, ndof*vdim);
 }
 
 void DiscreteAdaptTC::SetTspecAtIndex(int idx, const GridFunction &tspec_)
 {
-   const int vdim     = tspec_.FESpace()->GetVDim(),
-             dof_cnt  = tspec_.Size()/vdim;
+   const int vdim = tspec_.FESpace()->GetVDim(),
+             ndof = tspec_.FESpace()->GetNDofs();
+   MFEM_VERIFY(ndof == tspec.Size()/ncomp, "Inconsistency in SetTargetSpec.");
 
    const auto tspec__d = tspec_.Read();
    auto tspec_d = tspec.ReadWrite();
-   const int offset = idx*dof_cnt;
-   internal::device_copy(tspec_d + offset, tspec__d, dof_cnt*vdim);
-   FinalizeSerialDiscreteTargetSpec();
+   const int offset = idx*ndof;
+   internal::device_copy(tspec_d + offset, tspec__d, ndof*vdim);
+   FinalizeSerialDiscreteTargetSpec(tspec_);
 }
 
 void DiscreteAdaptTC::SetSerialDiscreteTargetSize(const GridFunction &tspec_)
 {
-
    if (sizeidx > -1) { SetTspecAtIndex(sizeidx, tspec_); return; }
    sizeidx = ncomp;
    SetDiscreteTargetBase(tspec_);
-   FinalizeSerialDiscreteTargetSpec();
+   FinalizeSerialDiscreteTargetSpec(tspec_);
 }
 
 void DiscreteAdaptTC::SetSerialDiscreteTargetSkew(const GridFunction &tspec_)
@@ -1439,32 +1460,31 @@ void DiscreteAdaptTC::SetSerialDiscreteTargetSkew(const GridFunction &tspec_)
    if (skewidx > -1) { SetTspecAtIndex(skewidx, tspec_); return; }
    skewidx = ncomp;
    SetDiscreteTargetBase(tspec_);
-   FinalizeSerialDiscreteTargetSpec();
+   FinalizeSerialDiscreteTargetSpec(tspec_);
 }
 
-void DiscreteAdaptTC::SetSerialDiscreteTargetAspectRatio(
-   const GridFunction &tspec_)
+void DiscreteAdaptTC::SetSerialDiscreteTargetAspectRatio(const GridFunction &ar)
 {
-   if (aspectratioidx > -1) { SetTspecAtIndex(aspectratioidx, tspec_); return; }
+   if (aspectratioidx > -1) { SetTspecAtIndex(aspectratioidx, ar); return; }
    aspectratioidx = ncomp;
-   SetDiscreteTargetBase(tspec_);
-   FinalizeSerialDiscreteTargetSpec();
+   SetDiscreteTargetBase(ar);
+   FinalizeSerialDiscreteTargetSpec(ar);
 }
 
-void DiscreteAdaptTC::SetSerialDiscreteTargetOrientation(
-   const GridFunction &tspec_)
+void DiscreteAdaptTC::SetSerialDiscreteTargetOrientation(const GridFunction &o)
 {
-   if (orientationidx > -1) { SetTspecAtIndex(orientationidx, tspec_); return; }
+   if (orientationidx > -1) { SetTspecAtIndex(orientationidx, o); return; }
    orientationidx = ncomp;
-   SetDiscreteTargetBase(tspec_);
-   FinalizeSerialDiscreteTargetSpec();
+   SetDiscreteTargetBase(o);
+   FinalizeSerialDiscreteTargetSpec(o);
 }
 
-void DiscreteAdaptTC::FinalizeSerialDiscreteTargetSpec()
+void DiscreteAdaptTC::FinalizeSerialDiscreteTargetSpec(const GridFunction &t)
 {
    MFEM_VERIFY(adapt_eval, "SetAdaptivityEvaluator() has not been called!")
    MFEM_VERIFY(ncomp > 0, "No target specifications have been set!");
 
+   const FiniteElementSpace *tspec_fes = t.FESpace();
    adapt_eval->SetSerialMetaInfo(*tspec_fes->GetMesh(),
                                  *tspec_fes->FEColl(), ncomp);
    adapt_eval->SetInitialField(*tspec_fes->GetMesh()->GetNodes(), tspec);
@@ -1474,12 +1494,40 @@ void DiscreteAdaptTC::FinalizeSerialDiscreteTargetSpec()
    delete tspec_fesv;
    tspec_fesv = new FiniteElementSpace(tspec_fes->GetMesh(),
                                        tspec_fes->FEColl(), ncomp);
+
+   delete tspec_gf;
+   tspec_gf = new GridFunction(tspec_fesv, tspec);
+}
+
+void DiscreteAdaptTC::GetDiscreteTargetSpec(GridFunction &tspec_, int idx)
+{
+   if (idx < 0) { return; }
+   const int ndof = tspec_.FESpace()->GetNDofs(),
+             vdim = tspec_.FESpace()->GetVDim();
+   MFEM_VERIFY(ndof == tspec.Size()/ncomp,
+               "Inconsistency in GetSerialDiscreteTargetSpec.");
+
+   for (int i = 0; i < ndof*vdim; i++)
+   {
+      tspec_(i) = tspec(i + idx*ndof);
+   }
+}
+
+void DiscreteAdaptTC::UpdateAfterMeshTopologyChange()
+{
+   tspec_fesv->Update();
+   tspec_gf->Update();
+   tspec.SetDataAndSize(tspec_gf->GetData(), tspec_gf->Size());
+   tspec_sav = tspec;
+
+   adapt_eval->SetSerialMetaInfo(*tspec_fesv->GetMesh(),
+                                 *tspec_fesv->FEColl(), ncomp);
+   adapt_eval->SetInitialField(*tspec_fesv->GetMesh()->GetNodes(), tspec);
 }
 
 void DiscreteAdaptTC::SetSerialDiscreteTargetSpec(const GridFunction &tspec_)
 {
    SetSerialDiscreteTargetSize(tspec_);
-   FinalizeSerialDiscreteTargetSpec();
 }
 
 
@@ -1509,7 +1557,7 @@ void DiscreteAdaptTC::UpdateTargetSpecificationAtNode(const FiniteElement &el,
    MFEM_VERIFY(tspec.Size() > 0, "Target specification is not set!");
 
    Array<int> dofs;
-   tspec_fes->GetElementDofs(T.ElementNo, dofs);
+   tspec_fesv->GetElementDofs(T.ElementNo, dofs);
    const int cnt = tspec.Size()/ncomp; // dofs per scalar-field
 
    for (int i = 0; i < ncomp; i++)
@@ -1524,7 +1572,7 @@ void DiscreteAdaptTC::RestoreTargetSpecificationAtNode(ElementTransformation &T,
    MFEM_VERIFY(tspec.Size() > 0, "Target specification is not set!");
 
    Array<int> dofs;
-   tspec_fes->GetElementDofs(T.ElementNo, dofs);
+   tspec_fesv->GetElementDofs(T.ElementNo, dofs);
    const int cnt = tspec.Size()/ncomp;
    for (int i = 0; i < ncomp; i++)
    {
@@ -1532,6 +1580,40 @@ void DiscreteAdaptTC::RestoreTargetSpecificationAtNode(ElementTransformation &T,
    }
 }
 
+void DiscreteAdaptTC::SetTspecFromIntRule(int e_id,
+                                          const IntegrationRule &intrule)
+{
+   switch (target_type)
+   {
+      case IDEAL_SHAPE_GIVEN_SIZE:
+      case GIVEN_SHAPE_AND_SIZE:
+      {
+         const int ndofs = tspec_fesv->GetFE(e_id)->GetDof(),
+                   ntspec_dofs = ndofs*ncomp;
+
+         Vector tspec_vals(ntspec_dofs);
+
+         Array<int> dofs;
+         tspec_fesv->GetElementVDofs(e_id, dofs);
+         tspec.GetSubVector(dofs, tspec_vals);
+         DenseMatrix tr;
+         tspec_gf->GetVectorValues(e_id, intrule, tspec_refine, tr);
+         tspec_refine.Transpose();
+         break;
+      }
+      default:
+         MFEM_ABORT("Incompatible target type for discrete adaptation!");
+   }
+}
+
+void DiscreteAdaptTC::SetTspecDataForDerefinement(FiniteElementSpace *fes)
+{
+   coarse_tspec_fesv = fes;
+   const Operator *c_op = fes->GetUpdateOperator();
+   tspec_derefine.SetSize(c_op->Height());
+   c_op->Mult(tspec, tspec_derefine);
+}
+
 void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
                                             const IntegrationRule &ir,
                                             const Vector &elfun,
@@ -1542,6 +1624,8 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
              nqp = ir.GetNPoints();
    Jtrcomp.SetSize(dim, dim, 4*nqp);
 
+   FiniteElementSpace *src_fes = tspec_fesv;
+
    switch (target_type)
    {
       case IDEAL_SHAPE_GIVEN_SIZE:
@@ -1550,7 +1634,7 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
          const DenseMatrix &Wideal =
             Geometries.GetGeomToPerfGeomJac(fe.GetGeomType());
          const int dim = Wideal.Height(),
-                   ndofs = tspec_fes->GetFE(e_id)->GetDof(),
+                   ndofs = tspec_fesv->GetFE(e_id)->GetDof(),
                    ntspec_dofs = ndofs*ncomp;
 
          Vector shape(ndofs), tspec_vals(ntspec_dofs), par_vals,
@@ -1561,11 +1645,29 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
          tspec_fesv->GetElementVDofs(e_id, dofs);
          tspec.UseDevice(true);
          tspec.GetSubVector(dofs, tspec_vals);
+         if (tspec_refine.NumCols() > 0) // Refinement
+         {
+            MFEM_VERIFY(amr_el >= 0, " Target being constructed for an AMR element.");
+            for (int i = 0; i < ncomp; i++)
+            {
+               for (int j = 0; j < ndofs; j++)
+               {
+                  tspec_vals(j + i*ndofs) = tspec_refine(j + amr_el*ndofs, i);
+               }
+            }
+         }
+         else if (tspec_derefine.Size() > 0) // Derefinement
+         {
+            dofs.SetSize(0);
+            coarse_tspec_fesv->GetElementVDofs(e_id, dofs);
+            tspec_derefine.GetSubVector(dofs, tspec_vals);
+            src_fes = coarse_tspec_fesv;
+         }
 
          for (int q = 0; q < nqp; q++)
          {
             const IntegrationPoint &ip = ir.IntPoint(q);
-            tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
+            src_fes->GetFE(e_id)->CalcShape(ip, shape);
             Jtr(q) = Wideal; // Initialize to identity
             for (int d = 0; d < 4; d++)
             {
@@ -1576,9 +1678,16 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
             if (sizeidx != -1) // Set size
             {
                par_vals.SetDataAndSize(tspec_vals.GetData()+sizeidx*ndofs, ndofs);
-               const double min_size = par_vals.Min();
-               MFEM_VERIFY(min_size > 0.0,
-                           "Non-positive size propagated in the target definition.");
+               double min_size = par_vals.Min();//0.001; //
+               if (lim_min_size > 0.)
+               {
+                  min_size = lim_min_size;
+               }
+               else
+               {
+                  MFEM_VERIFY(min_size > 0.0,
+                              "Non-positive size propagated in the target definition.");
+               }
                const double size = std::max(shape * par_vals, min_size);
                Jtr(q).Set(std::pow(size, 1.0/dim), Jtr(q));
                DenseMatrix Jtrcomp_q(Jtrcomp.GetData(0 + 4*q), dim, dim);
@@ -1593,6 +1702,9 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
                {
                   par_vals.SetDataAndSize(tspec_vals.GetData()+
                                           aspectratioidx*ndofs, ndofs);
+                  const double min_size = par_vals.Min();
+                  MFEM_VERIFY(min_size > 0.0,
+                              "Non-positive aspect-ratio propagated in the target definition.");
 
                   const double aspectratio = shape * par_vals;
                   D_rho = 0.;
@@ -1777,7 +1889,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,
 
                grad_phys.Mult(par_vals, grad_ptr_c1);
                Vector grad_q(dim);
-               tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
+               tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
                grad_e_c1.MultTranspose(shape, grad_q);
 
                const double min_size = par_vals.Min();
@@ -1810,7 +1922,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,
 
                   grad_phys.Mult(par_vals, grad_ptr_c1);
                   Vector grad_q(dim);
-                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
+                  tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
                   grad_e_c1.MultTranspose(shape, grad_q);
 
                   const double aspectratio = shape * par_vals;
@@ -1841,7 +1953,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,
                   grad_phys.Mult(par_vals_c2, grad_ptr_c2);
                   grad_phys.Mult(par_vals_c3, grad_ptr_c3);
                   Vector grad_q1(dim), grad_q2(dim), grad_q3(dim);
-                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
+                  tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
                   grad_e_c1.MultTranspose(shape, grad_q1);
                   grad_e_c2.MultTranspose(shape, grad_q2);
                   grad_e_c3.MultTranspose(shape, grad_q3);
@@ -1880,7 +1992,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,
 
                   grad_phys.Mult(par_vals, grad_ptr_c1);
                   Vector grad_q(dim);
-                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
+                  tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
                   grad_e_c1.MultTranspose(shape, grad_q);
 
                   const double skew = shape * par_vals;
@@ -1913,7 +2025,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,
                   grad_phys.Mult(par_vals_c2, grad_ptr_c2);
                   grad_phys.Mult(par_vals_c3, grad_ptr_c3);
                   Vector grad_q1(dim), grad_q2(dim), grad_q3(dim);
-                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
+                  tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
                   grad_e_c1.MultTranspose(shape, grad_q1);
                   grad_e_c2.MultTranspose(shape, grad_q2);
                   grad_e_c3.MultTranspose(shape, grad_q3);
@@ -1960,7 +2072,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,
 
                   grad_phys.Mult(par_vals, grad_ptr_c1);
                   Vector grad_q(dim);
-                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
+                  tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
                   grad_e_c1.MultTranspose(shape, grad_q);
 
                   const double theta = shape * par_vals;
@@ -1991,7 +2103,7 @@ void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,
                   grad_phys.Mult(par_vals_c2, grad_ptr_c2);
                   grad_phys.Mult(par_vals_c3, grad_ptr_c3);
                   Vector grad_q1(dim), grad_q2(dim), grad_q3(dim);
-                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
+                  tspec_fesv->GetFE(e_id)->CalcShape(ip, shape);
                   grad_e_c1.MultTranspose(shape, grad_q1);
                   grad_e_c2.MultTranspose(shape, grad_q2);
                   grad_e_c3.MultTranspose(shape, grad_q3);
@@ -2071,7 +2183,7 @@ void DiscreteAdaptTC::UpdateGradientTargetSpecification(const Vector &x,
 {
    if (use_flag && good_tspec_grad) { return; }
 
-   const int dim = tspec_fes->GetFE(0)->GetDim(),
+   const int dim = tspec_fesv->GetFE(0)->GetDim(),
              cnt = x.Size()/dim;
 
    tspec_pert1h.SetSize(x.Size()*ncomp);
@@ -2097,7 +2209,7 @@ void DiscreteAdaptTC::UpdateHessianTargetSpecification(const Vector &x,
 
    if (use_flag && good_tspec_hess) { return; }
 
-   const int dim    = tspec_fes->GetFE(0)->GetDim(),
+   const int dim    = tspec_fesv->GetFE(0)->GetDim(),
              cnt    = x.Size()/dim,
              totmix = 1+2*(dim-2);
 
@@ -2145,6 +2257,16 @@ void DiscreteAdaptTC::UpdateHessianTargetSpecification(const Vector &x,
    good_tspec_hess = use_flag;
 }
 
+DiscreteAdaptTC::~DiscreteAdaptTC()
+{
+   delete tspec_gf;
+   delete adapt_eval;
+   delete tspec_fesv;
+#ifdef MFEM_USE_MPI
+   delete ptspec_fesv;
+#endif
+}
+
 void AdaptivityEvaluator::SetSerialMetaInfo(const Mesh &m,
                                             const FiniteElementCollection &fec,
                                             int num_comp)
@@ -2258,6 +2380,7 @@ void TMOP_Integrator::EnableAdaptiveLimiting(const ParGridFunction &z0,
                                              AdaptivityEvaluator &ae)
 {
    zeta_0 = &z0;
+   pzeta_0 = &z0;
    delete zeta;
    zeta   = new GridFunction(z0);
    coeff_zeta = &coeff;
@@ -2270,6 +2393,33 @@ void TMOP_Integrator::EnableAdaptiveLimiting(const ParGridFunction &z0,
 }
 #endif
 
+void TMOP_Integrator::UpdateAfterMeshTopologyChange()
+{
+   if (zeta)
+   {
+      zeta->Update();
+      adapt_eval->SetSerialMetaInfo(*zeta->FESpace()->GetMesh(),
+                                    *zeta->FESpace()->FEColl(), 1);
+      adapt_eval->SetInitialField
+      (*zeta->FESpace()->GetMesh()->GetNodes(), *zeta);
+   }
+}
+
+#ifdef MFEM_USE_MPI
+void TMOP_Integrator::ParUpdateAfterMeshTopologyChange()
+{
+   if (zeta)
+   {
+      zeta->Update();
+      adapt_eval->SetParMetaInfo(*pzeta_0->ParFESpace()->GetParMesh(),
+                                 *pzeta_0->ParFESpace()->FEColl(), 1);
+      adapt_eval->SetInitialField
+      (*zeta->FESpace()->GetMesh()->GetNodes(), *zeta);
+   }
+}
+#endif
+
+
 double TMOP_Integrator::GetElementEnergy(const FiniteElement &el,
                                          ElementTransformation &T,
                                          const Vector &elfun)
@@ -2378,6 +2528,145 @@ double TMOP_Integrator::GetElementEnergy(const FiniteElement &el,
 
    return energy;
 }
+
+double TMOP_Integrator::GetRefinementElementEnergy(const FiniteElement &el,
+                                                   ElementTransformation &T,
+                                                   const Vector &elfun,
+                                                   const IntegrationRule &irule)
+{
+   int dof = el.GetDof(), dim = el.GetDim(),
+       NEsplit = elfun.Size() / (dof*dim), el_id = T.ElementNo;
+   double energy = 0.;
+
+   TargetConstructor *tc = const_cast<TargetConstructor *>(targetC);
+   DiscreteAdaptTC *dtc = dynamic_cast<DiscreteAdaptTC *>(tc);
+   // For DiscreteAdaptTC the GridFunctions used to set the targets must be
+   // mapped onto the fine elements.
+   if (dtc) { dtc->SetTspecFromIntRule(el_id, irule); }
+
+   for (int e = 0; e < NEsplit; e++)
+   {
+      DSh.SetSize(dof, dim);
+      Jrt.SetSize(dim);
+      Jpr.SetSize(dim);
+      Jpt.SetSize(dim);
+      Vector elfun_child(dof*dim);
+      for (int i = 0; i < dof; i++)
+      {
+         for (int d = 0; d < dim; d++)
+         {
+            // elfun is (xe1,xe2,...xen,ye1,ye2...yen) and has nodal coordinates
+            // for all the children element of the parent element being considered.
+            // So we must index and get (xek, yek) i.e. nodal coordinates for
+            // the fine element being considered.
+            elfun_child(i + d*dof) = elfun(i + e*dof + d*dof*NEsplit);
+         }
+      }
+      PMatI.UseExternalData(elfun_child.GetData(), dof, dim);
+
+      const IntegrationRule &ir = EnergyIntegrationRule(el);
+
+      double el_energy = 0;
+      DenseTensor Jtr(dim, dim, ir.GetNPoints());
+      if (dtc)
+      {
+         // This is used to index into the tspec vector inside DiscreteAdaptTC.
+         dtc->SetRefinementSubElement(e);
+      }
+      targetC->ComputeElementTargets(el_id, el, ir, elfun_child, Jtr);
+
+      // Define ref->physical transformation, wn a Coefficient is specified.
+      IsoparametricTransformation *Tpr = NULL;
+      if (coeff1 || coeff0)
+      {
+         Tpr = new IsoparametricTransformation;
+         Tpr->SetFE(&el);
+         Tpr->ElementNo = T.ElementNo;
+         Tpr->ElementType = ElementTransformation::ELEMENT;
+         Tpr->Attribute = T.Attribute;
+         Tpr->GetPointMat().Transpose(PMatI); // PointMat = PMatI^T
+      }
+
+      for (int i = 0; i < ir.GetNPoints(); i++)
+      {
+         const IntegrationPoint &ip = ir.IntPoint(i);
+         const DenseMatrix &Jtr_i = Jtr(i);
+         h_metric->SetTargetJacobian(Jtr_i);
+         CalcInverse(Jtr_i, Jrt);
+         const double weight = ip.weight * Jtr_i.Det();
+
+         el.CalcDShape(ip, DSh);
+         MultAtB(PMatI, DSh, Jpr);
+         Mult(Jpr, Jrt, Jpt);
+
+         double val = metric_normal * h_metric->EvalW(Jpt);
+         if (coeff1) { val *= coeff1->Eval(*Tpr, ip); }
+
+         el_energy += weight * val;
+         delete Tpr;
+      }
+      energy += el_energy;
+   }
+   energy /= NEsplit;
+
+   if (dtc) { dtc->ResetRefinementTspecData(); }
+
+   return energy;
+}
+
+double TMOP_Integrator::GetDerefinementElementEnergy(const FiniteElement &el,
+                                                     ElementTransformation &T,
+                                                     const Vector &elfun)
+{
+   int dof = el.GetDof(), dim = el.GetDim();
+   double energy = 0.;
+
+   DSh.SetSize(dof, dim);
+   Jrt.SetSize(dim);
+   Jpr.SetSize(dim);
+   Jpt.SetSize(dim);
+   PMatI.UseExternalData(elfun.GetData(), dof, dim);
+
+   const IntegrationRule &ir = EnergyIntegrationRule(el);
+
+   energy = 0.0;
+   DenseTensor Jtr(dim, dim, ir.GetNPoints());
+   targetC->ComputeElementTargets(T.ElementNo, el, ir, elfun, Jtr);
+
+   // Define ref->physical transformation, wn a Coefficient is specified.
+   IsoparametricTransformation *Tpr = NULL;
+   if (coeff1)
+   {
+      Tpr = new IsoparametricTransformation;
+      Tpr->SetFE(&el);
+      Tpr->ElementNo = T.ElementNo;
+      Tpr->ElementType = ElementTransformation::ELEMENT;
+      Tpr->Attribute = T.Attribute;
+      Tpr->GetPointMat().Transpose(PMatI); // PointMat = PMatI^T
+   }
+
+   for (int i = 0; i < ir.GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir.IntPoint(i);
+      const DenseMatrix &Jtr_i = Jtr(i);
+      h_metric->SetTargetJacobian(Jtr_i);
+      CalcInverse(Jtr_i, Jrt);
+      const double weight = ip.weight * Jtr_i.Det();
+
+      el.CalcDShape(ip, DSh);
+      MultAtB(PMatI, DSh, Jpr);
+      Mult(Jpr, Jrt, Jpt);
+
+      double val = metric_normal * h_metric->EvalW(Jpt);
+      if (coeff1) { val *= coeff1->Eval(*Tpr, ip); }
+
+      energy += weight * val;
+   }
+
+   delete Tpr;
+   return energy;
+}
+
 void TMOP_Integrator::AssembleElementVector(const FiniteElement &el,
                                             ElementTransformation &T,
                                             const Vector &elfun, Vector &elvect)
@@ -3039,7 +3328,7 @@ void TMOP_Integrator::ComputeMinJac(const Vector &x,
    dx = detv_avg_min / dxscale;
 }
 
-void TMOP_Integrator::UpdateAfterMeshChange(const Vector &new_x)
+void TMOP_Integrator::UpdateAfterMeshPositionChange(const Vector &new_x)
 {
    if (discr_tc)
    {
@@ -3168,6 +3457,32 @@ void TMOPComboIntegrator::AssembleElementGrad(const FiniteElement &el,
    }
 }
 
+double TMOPComboIntegrator::GetRefinementElementEnergy(const FiniteElement &el,
+                                                       ElementTransformation &T,
+                                                       const Vector &elfun,
+                                                       const IntegrationRule &irule)
+{
+   double energy= 0.0;
+   for (int i = 0; i < tmopi.Size(); i++)
+   {
+      energy += tmopi[i]->GetRefinementElementEnergy(el, T, elfun, irule);
+   }
+   return energy;
+}
+
+double TMOPComboIntegrator::GetDerefinementElementEnergy(
+   const FiniteElement &el,
+   ElementTransformation &T,
+   const Vector &elfun)
+{
+   double energy= 0.0;
+   for (int i = 0; i < tmopi.Size(); i++)
+   {
+      energy += tmopi[i]->GetDerefinementElementEnergy(el, T, elfun);
+   }
+   return energy;
+}
+
 void TMOPComboIntegrator::EnableNormalization(const GridFunction &x)
 {
    const int cnt = tmopi.Size();

fem/tmop.hpp

@@ -1057,14 +1057,31 @@ protected:
    // eta1(x+h,y), eta2(x+h,y) ... etan(x+h,y), eta1(x,y+h), eta2(x,y+h) ...
    // same for tspec_pert2h and tspec_pertmix.
 
+   // DenseMatrix to hold target_spec values for the (children of the)
+   // element being refined to consider for h-refinement.
+   DenseMatrix tspec_refine;
+   // Vector to hold the target_spec values for the coarse version of the
+   // current mesh. Used for derefinement decision with hr-adaptivity.
+   Vector tspec_derefine;
+
    // Components of Target Jacobian at each quadrature point of an element. This
    // is required for computation of the derivative using chain rule.
    mutable DenseTensor Jtrcomp;
 
    // Note: do not use the Nodes of this space as they may not be on the
    // positions corresponding to the values of tspec.
-   const FiniteElementSpace *tspec_fes;
-   const FiniteElementSpace *tspec_fesv;
+   FiniteElementSpace *tspec_fesv;         //owned
+   FiniteElementSpace *coarse_tspec_fesv;  //not owned, derefinement FESpace
+   GridFunction *tspec_gf;                 //owned, uses tspec and tspec_fes
+   // discrete adaptivity
+#ifdef MFEM_USE_MPI
+   ParFiniteElementSpace *ptspec_fesv;     //owned, needed for derefinement to
+   // get update operator.
+   ParGridFunction *tspec_pgf;             // similar to tspec_gf
+#endif
+
+   int amr_el;
+   double lim_min_size;
 
    // These flags can be used by outside functions to avoid recomputing the
    // tspec and tspec_perth fields again on the same mesh.
@@ -1076,7 +1093,7 @@ protected:
 
    void SetDiscreteTargetBase(const GridFunction &tspec_);
    void SetTspecAtIndex(int idx, const GridFunction &tspec_);
-   void FinalizeSerialDiscreteTargetSpec();
+   void FinalizeSerialDiscreteTargetSpec(const GridFunction &tspec_);
 #ifdef MFEM_USE_MPI
    void SetTspecAtIndex(int idx, const ParGridFunction &tspec_);
    void FinalizeParDiscreteTargetSpec(const ParGridFunction &tspec_);
@@ -1088,16 +1105,16 @@ public:
         ncomp(0),
         sizeidx(-1), skewidx(-1), aspectratioidx(-1), orientationidx(-1),
         tspec(), tspec_sav(), tspec_pert1h(), tspec_pert2h(), tspec_pertmix(),
-        tspec_fes(NULL), tspec_fesv(NULL),
+        tspec_refine(), tspec_derefine(),
+        tspec_fesv(NULL), coarse_tspec_fesv(NULL), tspec_gf(NULL),
+#ifdef MFEM_USE_MPI
+        ptspec_fesv(NULL), tspec_pgf(NULL),
+#endif
+        amr_el(-1), lim_min_size(-0.1),
         good_tspec(false), good_tspec_grad(false), good_tspec_hess(false),
         adapt_eval(NULL) { }
 
-   virtual ~DiscreteAdaptTC()
-   {
-      delete adapt_eval;
-      delete tspec_fes;
-      delete tspec_fesv;
-   }
+   virtual ~DiscreteAdaptTC();
 
    /** @name Target specification methods.
        The following methods are used to specify geometric parameters of the
@@ -1128,6 +1145,20 @@ public:
    void ResetUpdateFlags()
    { good_tspec = good_tspec_grad = good_tspec_hess = false; }
 
+   /// Get one of the discrete fields from tspec.
+   void GetDiscreteTargetSpec(GridFunction &tspec_, int idx);
+   /// Get the FESpace associated with tspec.
+   FiniteElementSpace *GetTSpecFESpace() { return tspec_fesv; }
+   /// Get the entire tspec.
+   GridFunction *GetTSpecData() { return tspec_gf; }
+   /// Update all discrete fields based on tspec and update for AMR
+   void UpdateAfterMeshTopologyChange();
+
+#ifdef MFEM_USE_MPI
+   ParFiniteElementSpace *GetTSpecParFESpace() { return ptspec_fesv; }
+   void ParUpdateAfterMeshTopologyChange();
+#endif
+
    /** Used to update the target specification after the mesh has changed. The
        new mesh positions are given by new_x. If @a use_flags is true, repeated
        calls won't do anything until ResetUpdateFlags() is called. */
@@ -1184,6 +1215,36 @@ public:
                                               const Vector &elfun,
                                               IsoparametricTransformation &Tpr,
                                               DenseTensor &dJtr) const;
+
+   // Generates tspec_vals for target construction using intrule
+   // Used for the refinement component in hr-adaptivity.
+   void SetTspecFromIntRule(int e_id, const IntegrationRule &intrule);
+
+   // Targets based on discrete functions can result in invalid (negative)
+   // size at the quadrature points. This method can be used to set a
+   // minimum target size.
+   void SetMinSizeForTargets(double min_size_) { lim_min_size = min_size_; }
+
+   /// Computes target specification data with respect to the coarse FE space.
+   void SetTspecDataForDerefinement(FiniteElementSpace *fes);
+
+   // Reset refinement data associated with h-adaptivity component.
+   void ResetRefinementTspecData()
+   {
+      tspec_refine.Clear();
+      amr_el = -1;
+   }
+
+   // Reset derefinement data associated with h-adaptivity component.
+   void ResetDerefinementTspecData()
+   {
+      tspec_derefine.Destroy();
+      coarse_tspec_fesv = NULL;
+   }
+
+   // Used to specify the fine element for determining energy of children of a
+   // parent element.
+   void SetRefinementSubElement(int amr_el_) { amr_el = amr_el_; }
 };
 
 class TMOPNewtonSolver;
@@ -1201,6 +1262,7 @@ protected:
    friend class TMOPNewtonSolver;
    friend class TMOPComboIntegrator;
 
+   TMOP_QualityMetric *h_metric;
    TMOP_QualityMetric *metric;        // not owned
    const TargetConstructor *targetC;  // not owned
 
@@ -1227,6 +1289,9 @@ protected:
 
    // Adaptive limiting.
    const GridFunction *zeta_0;       // Not owned.
+#ifdef MFEM_USE_MPI
+   const ParGridFunction *pzeta_0;
+#endif
    GridFunction *zeta;               // Owned. Updated by adapt_eval.
    Coefficient *coeff_zeta;          // Not owned.
    AdaptivityEvaluator *adapt_eval;  // Not owned.
@@ -1337,7 +1402,7 @@ protected:
 #endif
    void ComputeMinJac(const Vector &x, const FiniteElementSpace &fes);
 
-   void UpdateAfterMeshChange(const Vector &new_x);
+   void UpdateAfterMeshPositionChange(const Vector &new_x);
 
    void DisableLimiting()
    {
@@ -1395,11 +1460,13 @@ protected:
    void ComputeAllElementTargets(const Vector &xe = Vector()) const;
 
 public:
-   /** @param[in] m  TMOP_QualityMetric that will be integrated (not owned).
-       @param[in] tc Target-matrix construction algorithm to use (not owned). */
-   TMOP_Integrator(TMOP_QualityMetric *m, TargetConstructor *tc)
-      : metric(m), targetC(tc), IntegRules(NULL), integ_order(-1),
-        coeff1(NULL), metric_normal(1.0),
+   /** @param[in] m    TMOP_QualityMetric for r-adaptivity (not owned).
+       @param[in] tc   Target-matrix construction algorithm to use (not owned).
+       @param[in] hm   TMOP_QualityMetric for h-adaptivity (not owned). */
+   TMOP_Integrator(TMOP_QualityMetric *m, TargetConstructor *tc,
+                   TMOP_QualityMetric *hm)
+      : h_metric(hm), metric(m), targetC(tc), IntegRules(NULL),
+        integ_order(-1), coeff1(NULL), metric_normal(1.0),
         nodes0(NULL), coeff0(NULL),
         lim_dist(NULL), lim_func(NULL), lim_normal(1.0),
         zeta_0(NULL), zeta(NULL), coeff_zeta(NULL), adapt_eval(NULL),
@@ -1407,6 +1474,9 @@ public:
         fdflag(false), dxscale(1.0e3), fd_call_flag(false), exact_action(false)
    { PA.enabled = false; }
 
+   TMOP_Integrator(TMOP_QualityMetric *m, TargetConstructor *tc)
+      : TMOP_Integrator(m, tc, m) { }
+
    ~TMOP_Integrator();
 
    /// Release the device memory of large PA allocations. This will copy device
@@ -1478,6 +1548,22 @@ public:
                                    ElementTransformation &T,
                                    const Vector &elfun);
 
+   /** @brief Computes the mean of the energies of the given element's children.
+
+       In addition to the inputs for GetElementEnergy, this function requires an
+       IntegrationRule to be specified that will give the decomposition of the
+       given element based on the refinement type being considered. */
+   virtual double GetRefinementElementEnergy(const FiniteElement &el,
+                                             ElementTransformation &T,
+                                             const Vector &elfun,
+                                             const IntegrationRule &irule);
+
+   /// This function is similar to GetElementEnergy, but ignores components
+   /// such as limiting etc. to compute the element energy.
+   virtual double GetDerefinementElementEnergy(const FiniteElement &el,
+                                               ElementTransformation &T,
+                                               const Vector &elfun);
+
    virtual void AssembleElementVector(const FiniteElement &el,
                                       ElementTransformation &T,
                                       const Vector &elfun, Vector &elvect);
@@ -1486,6 +1572,13 @@ public:
                                     ElementTransformation &T,
                                     const Vector &elfun, DenseMatrix &elmat);
 
+   TMOP_QualityMetric &GetAMRQualityMetric() { return *h_metric; }
+
+   void UpdateAfterMeshTopologyChange();
+#ifdef MFEM_USE_MPI
+   void ParUpdateAfterMeshTopologyChange();
+#endif
+
    // PA extension
    using NonlinearFormIntegrator::AssemblePA;
    virtual void AssemblePA(const FiniteElementSpace&);
@@ -1564,6 +1657,15 @@ public:
                                     ElementTransformation &T,
                                     const Vector &elfun, DenseMatrix &elmat);
 
+   virtual double GetRefinementElementEnergy(const FiniteElement &el,
+                                             ElementTransformation &T,
+                                             const Vector &elfun,
+                                             const IntegrationRule &irule);
+
+   virtual double GetDerefinementElementEnergy(const FiniteElement &el,
+                                               ElementTransformation &T,
+                                               const Vector &elfun);
+
    /// Normalization factor that considers all integrators in the combination.
    void EnableNormalization(const GridFunction &x);
 #ifdef MFEM_USE_MPI