testing amr results for l-shape domain problem

.gitignore

@@ -252,7 +252,6 @@ miniapps/shifted/ParaViewDistance
 miniapps/shifted/diffusion
 miniapps/shifted/diffusion.mesh
 miniapps/shifted/diffusion.gf
-miniapps/shifted/ParaViewDiffusion
 
 miniapps/tools/display-basis
 miniapps/tools/load-dc
@@ -297,7 +296,6 @@ tests/unit/psedov_tests_*
 tests/unit/tmop_pa_tests_*
 tests/unit/ptmop_pa_tests_*
 tests/unit/ceed_tests
-tests/unit/debug_device_tests
 
 # Test script output
 tests/scripts/*.err

.gitlab-ci.yml

@@ -50,11 +50,10 @@ variables:
   AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
   MFEM_DATA_REPO: https://github.com/mfem/data.git
   ARTIFACTS_DIR: artifacts
-  SLURM_OVERLAP: 1
 
-# The pipeline is divided into stages. Usually, jobs in a given stage wait for
-# the preceding stages to complete before to start. However, we sometimes use
-# the "needs" keyword and express the DAG of jobs for more efficiency.
+# The pipeline is divided into stages. Usually, these are also synchronization
+# points, however, we use "needs" keyword to express the DAG of jobs for more
+# efficiency.
 # - We use setup and setup_baseline phases to download content outside of mfem
 #   directory.
 # - Allocate/Release is where quartz resources are allocated/released once for all.
@@ -88,6 +87,7 @@ setup:
   script:
     - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
     - if [ ! -d data ]; then git clone ${MFEM_DATA_REPO}; fi
+  needs: []
 
 # The setup_baseline job in setup stage_baseline doesn't rely on MFEM git repo.
 # It prepares a pipeline-wide working directory downloading/updating external
@@ -95,7 +95,6 @@ setup:
 # are now using unique directories so repo are never shared with another
 # pipeline. This is not memory efficient (we keep a lot of data), hence this
 # reminder.
-# Note: This job can start immediately.
 setup_baseline:
   tags:
     - shell
@@ -126,10 +125,10 @@ setup_baseline:
   script:
     - srun -p mi60 -t 15 -N 1 tests/gitlab/build_and_test
 
-# Lassen uses a different job scheduler (spectrum lsf) that does not allow
-# pre-allocation the same way slurm does.  We use pdebug queue on lassen to
-# speed-up the allocation.  However this would not be scalable to multiple
-# builds.
+# Lassen uses a different job scheduler (spectrum lsf) that does not
+# allow pre-allocation the same way slurm does.
+# We use pdebug queue on lassen to speed-up the allocation.
+# However this would not be scalable to multiple builds.
 .build_blueos_3_ppc64le_ib_script:
   script:
     - lalloc 1 -W 30 -q pdebug tests/gitlab/build_and_test

.gitlab/lassen.yml

@@ -22,7 +22,6 @@
 
 # Spack helped builds
 # Generic lassen build job, extending build script
-# Note: Lassen jobs can start as soon as the setup job is complete.
 .build_and_test_on_lassen:
   extends: [.build_blueos_3_ppc64le_ib_script, .on_lassen]
   stage: l_build_and_test
@@ -30,5 +29,5 @@
 
 opt_mpi_cuda_xl_16_1_1_8:
   variables:
-    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=70"
+    SPEC: "%xl@16.1.1.8 +mpi +cuda cuda_arch=sm_70"
   extends: .build_and_test_on_lassen

.gitlab/quartz.yml

@@ -16,13 +16,13 @@
     - shell
     - quartz
   rules:
-    # Don't run quartz jobs if...
+    # Don’t run quartz jobs if...
     - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"'
       when: never
-    # Don't run autotest update if...
+    # Don’t run autotest update if...
     - if: '$CI_JOB_NAME =~ /update_autotest/ && $AUTOTEST != "YES"'
       when: never
-    # Don't run autotest update if...
+    # Don’t run autotest update if...
     - if: '$CI_JOB_NAME =~ /q_report/ && $AUTOTEST != "YES"'
       when: never
     # Report success on success status
@@ -37,18 +37,6 @@
     # Default is to run if previous stage succeeded
     - when: on_success
 
-# This is a yaml anchor, it can be used to avoid duplication like here.
-# The code below will simply be pasted wherever the anchor is placed.
-.safe_create_rundir: &safe_create_rundir |
-  if ! mkdir ${rundir}; then
-    n=1
-    while ! mkdir ${rundir}_${n}
-    do
-      n=$((n+1))
-    done
-    rundir=${rundir}_${n}
-  fi
-
 # Allocate
 q_allocate_resources:
   variables:
@@ -77,11 +65,10 @@ q_report_success:
   stage: q_release_resources
   script:
     - echo "Can only run if all the quartz jobs passed"
-    - cd ${AUTOTEST_ROOT}/autotest && git pull
     - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
-    - *safe_create_rundir
+    - cd ${AUTOTEST_ROOT}/autotest && git pull
+    - mkdir -p ${rundir}
     - echo "The Quartz jobs were successful" > ${rundir}/gitlab.out
-    - echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
     - git add ${rundir}
     - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
     - git push origin master
@@ -93,11 +80,10 @@ q_report_failure:
   stage: q_release_resources
   script:
     - echo "Runs if there was at least one failure on quartz"
-    - cd ${AUTOTEST_ROOT}/autotest && git pull
     - rundir="gitlab/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
-    - *safe_create_rundir
+    - cd ${AUTOTEST_ROOT}/autotest && git pull
+    - mkdir -p ${rundir}
     - echo "There was an error while running CI on Quartz" > ${rundir}/gitlab.err
-    - echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/gitlab.err
     - cp ${rundir}/gitlab.err ${rundir}/autotest-email.html
     - git add ${rundir}
     - git commit -am "Gitlab CI log for baseline on quartz with intel ($(date +%Y-%m-%d))"
@@ -108,6 +94,7 @@ q_report_failure:
 .build_and_test_on_quartz:
   extends: [.build_toss_3_x86_64_ib_script, .on_quartz]
   stage: q_build_and_test
+  needs: [setup]
 
 # Build MFEM
 debug_ser_gcc_4_9_3:
@@ -150,11 +137,7 @@ opt_par_gcc_6_1_0_pumi:
     SPEC: "%gcc@6.1.0 +pumi"
   extends: .build_and_test_on_quartz
 
-# Baseline jobs form an independent set of jobs. We use `needs:[]` to specify
-# that "setup-baseline" can start immediately. Then, we have to use needs for
-# each one of the baseline jobs, otherwise they will wait for the rest of the
-# pipeline.
-
+# Baseline
 baselinecheck_mfem_intel_quartz:
   extends: [.baselinecheck_mfem, .on_quartz]
   needs: [setup_baseline]
@@ -164,15 +147,14 @@ update_autotest:
   needs: [baselinecheck_mfem_intel_quartz]
   stage: baseline_to_autotest
   script:
-    - cd ${AUTOTEST_ROOT}/autotest && git pull
     - rundir="quartz/$(date +%Y-%m-%d)-github-${CI_COMMIT_REF_SLUG}"
-    - *safe_create_rundir
+    - cd ${AUTOTEST_ROOT}/autotest && git pull
+    - mkdir -p ${rundir}
     - cp ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/* ${rundir}
     # We create an autotest-email.html file, because that's how we signal that there was a diff (temporary).
     - |
       if [[ -f ${rundir}/*.err ]]
       then
-        echo "See the pipeline here -> $CI_PIPELINE_URL" >> ${rundir}/*.err
         cp ${rundir}/*.err ${rundir}/autotest-email.html
       fi
     - git add ${rundir}

CHANGELOG

@@ -8,87 +8,40 @@
                                https://mfem.org
 
 
-Version 4.3.1 (development)
+Version 4.2.1 (development)
 ===========================
-- Added support for hr-adaptivity using TMOP-based error estimator.
+- Added initial support for GPU-accelerated versions of PETSc that works with
+  MFEM_USE_CUDA if PETSc has been configured with CUDA support. Examples 1 and 9
+  in the examples/petsc directory have been modified to work with --device cuda.
+  Examples with GAMG (ex1p) and SLEPc (ex11p) are also provided.
 
-- Adding lowest order Nedelec and Raviart-Thomas basis functions on wedge
-  shaped elements.
+- Memory management:
+  * Added method Device::SetMemoryTypes that can be used to change the default
+    host and device MemoryTypes before Device setup.
+  * In class MemoryManager, added methods GetDualMemoryType and
+    SetDualMemoryType; dual MemoryTypes are used to determine the second
+    MemoryType (host or device) when only one MemoryType is specified in methods
+    of class Memory.
+  * Added Memory constructor for setting both the host and device MemoryTypes.
+  * Switched the default behavior of device memory allocations so that they
+    are deferred until the device pointer is needed.
+  * Added a second Umpire device MemoryType, DEVICE_UMPIRE_2, with
+    corresponding allocator that can be set with the method
+    MemoryManager::SetUmpireDevice2AllocatorName.
+  * Added HOST_PINNED MemoryType and a pinned host allocator for CUDA and HIP.
 
-- Added initial support for meshes with pyramidal elements, including several
-  pyramidal meshes in the data/ directory and support for the lowest order H1,
-  Nedelec, Raviart-Thomas, and L2 basis functions on pyramids.
+- Added support for Caliper: a library to integrate performance profiling
+  capabilities into applications. See examples/caliper for more details.
 
-- Updated the hypre interface according to changes in hypre-2.22.1. The ADS
-  solver is now fully working on GPUs.
+- Added support for explicit vectorization in the high-performance templated
+  code for Fujitsu's A64FX ARM microprocessor architecture.
 
-- Tetrahedral meshes no longer need to be reordered to support high order
-  Nedelec basis functions. This will allow future support for Nedelec basis
-  functions on wedges and pyramids which are not amenable to reordering. The
-  ReorientTetMesh method of the Mesh and ParMesh classes has been deprecated.
+- Added AlgebraicCeedSolver that does matrix-free algebraic p-multigrid for
+  diffusion problems with the Ceed backend.
 
-
-Version 4.3, released on July 29, 2021
-======================================
-
-Discretization improvements
----------------------------
-- Variable order spaces, p- and hp-refinement. This is the initial (serial)
-  support for variable-order FiniteElementCollection and FiniteElementSpace.
-  The new method FiniteElementSpace::SetElementOrder can be called to set an
-  arbitrary order for each mesh element. The conforming interpolation matrix
-  will now automatically constrain p- and hp- interfaces, enabling general
-  hp-refinement in both 2D and 3D, on uniform or mixed NC meshes. Support for
-  parallel variable-order spaces will follow shortly.
-
-- Extended the support for field transfer between high-order and low-order
-  refined finite element spaces to include: dual fields and H1 fields (both
-  primary and dual). These are illustrated in the lor-transfer miniapp.
-
-- Improved libCEED integration, including support for VectorCoefficient,
-  ConvectionIntegrator, and VectorConvectionNLFIntegrator with libCEED backends.
-
-- Extending support for L2 basis functions using MapTypes VALUE and INTEGRAL in
-  linear interpolators and GridFunction "GetValue" methods.
-
-- Changed the interface for the error estimator and implemented the Kelly error
-  indicator for scalar-valued problems, supported in serial and parallel builds.
-
-- Added support for the "BR2" discontinuous Galerkin discretization for
-  diffusion via DGDiffusionBR2Integrator (see Example 14/14p).
-
-- Added convective and skew-symmetric integrators for the nonlinear term in the
-  Navier-Stokes equations.
-
-- Added new classes DenseSymmetricMatrix and SymmetricMatrixCoefficient for
-  efficient evaluation of symmetric matrix coefficients. This replaces the now
-  deprecated EvalSymmetric in MatrixCoefficient. Added DiagonalMatrixCoefficient
-  for clarity, which is a typedef of VectorCoefficient.
-
-- Added support for nonscalar coefficient with VectorDiffusionIntegrator.
-
-Linear and nonlinear solvers
-----------------------------
-- Added support for AMG preconditioners on GPUs based on the hypre library
-  (version 2.22.0 or later). These include BoomerAMG, AMS and ADS and most
-  MFEM examples that use hypre have been ported to support this functionality.
-  The GPU preconditioners require that both hypre and MFEM are built with CUDA
-  support. Hypre builds with CUDA and unified memory are also supported and
-  can be used with `-d cuda:uvm` as a command-line option.
-
-- Added support for AMG preconditioners for non-symmetric systems (e.g.
-  advection-dominated problems) using hypre's approximate ideal restriction
-  (AIR) AMG. Requires hypre version 2.14.0 or newer. Usage is illustrated in
-  example 9/9p.
-
-- Added new functionality for constructing low-order refined discretizations and
-  solvers, see the LORDiscretization and LORSolver classes. A new basis type for
-  H(curl) and H(div) spaces is introduced to give spectral equivalence. This
-  functionality is illustrated in the LOR solvers miniapp in miniapps/solvers.
-
-- Generalized the Multigrid class to support non-geometric multigrid. Previous
-  functionality, based on FiniteElementSpaceHierarchy, is now available in the
-  derived class GeometricMultigrid.
+- Introduced new options for the mesh-explorer miniapp to visualize the actual
+  element attributes in parallel meshes while retaining the visualization of
+  the domain decomposition.
 
 - Introduced solver interface for linear problems with constraints, a few
   concrete solvers that implement the interface, and a demonstration of their
@@ -99,18 +52,19 @@ Linear and nonlinear solvers
   as described in Barker and Kolev 2020 (https://doi.org/10.1002/nla.2348). See
   Example 3p and linalg/auxiliary.?pp.
 
-- Improved interface for using the Ginkgo library, including: support for matrix-
-  free operators in Ginkgo solvers, new wrappers for Ginkgo preconditioners, HIP
-  support, and reduction of unnecessary data copies.
+- Added a new miniapp block-solvers that compares the performance of various
+  solvers for mixed finite element discretization of the second order scalar
+  elliptic equations. Currently available solvers in the miniapp include a
+  block-diagonal preconditioner that is based on approximate Schur complement
+  (implemented in ex5p), and a newly implemented solver DivFreeSolver, which
+  exploits a multilevel decomposition of the Raviart-Thomas space and its
+  divergence-free subspace. See the miniapps/solvers directory for more details.
 
-- Added initial support for hypre's mixed integer (mixedint) capability, which
-  uses different data types for local and global indices in order to save memory
-  in large problems. This capability requires that hypre was configured with the
-  --enable-mixedint option. Note that this option is currently tested only in
-  ex1p, ex3p, and ex4p, and may not work in more general settings.
+- Added a new miniapp for computing (signed) distance functions to a point
+  source or zero level set. See miniapps/shifted/distance.cpp.
 
-- Added AlgebraicCeedSolver that does matrix-free algebraic p-multigrid for
-  diffusion problems with the Ceed backend.
+- Added matrix-free GPU-enabled implementations of GradientInterpolator and
+  IdentityInterpolator.
 
 - Added interface to MUMPS direct solver. Its usage is demonstrated in ex25p.
   See http://mumps.enseeiht.fr/ for more details. Supported versions >= 5.1.1.
@@ -118,17 +72,6 @@ Linear and nonlinear solvers
 - Added three ESDIRK time integrators: implicit trapezoid rule, L-stable
   ESDIRK-32, and A-stable ESDIRK-33.
 
-- Implemented a variable step-size IMEX (VSSIMEX) method for the Navier miniapp.
-
-- Implemented an adaptive linear solver tolerance option for NewtonSolver based
-  on the algorithm of Eisenstat and Walker.
-
-Meshing improvements
---------------------
-- Added support for reading high-order Lagrange meshes in VTK format. Arbitrary-
-  orders and all element types are supported. See the VTK blog for more info:
-  https://blog.kitware.com/wp-content/uploads/2018/09/Source_Issue_43.pdf.
-
 - Introduced a new non-conforming mesh format that fixes known inconsistencies
   of legacy "MFEM mesh v1.1" NC format and works consistently in both serial and
   parallel. ParMesh::ParPrint can now print non-conforming AMR meshes that can
@@ -137,26 +80,113 @@ Meshing improvements
   NC data files are compatible with serial code, e.g., can be viewed with serial
   GLVis. Loading of legacy NC mesh files is still supported.
 
-- Added FMS support (https://github.com/CEED/FMS) to mfem. FMS can represent
-  unstructured high-order meshes with general high-order finite element fields
-  on them. When enabled, mfem can convert data collections to/from FMS data
-  collections in memory. In addition, an FMS data collection class was added so
-  the convert-dc miniapp can read and generate data files in FMS format.
+- Added support for 1D non-conforming meshes (which can be useful for parallel
+  load balancing and derefinement).
+
+- Added a "scaled Jacobian" visualization option in the Mesh Explorer miniapp to
+  help identify elements with poor mesh quality.
+
+- Added support for the "BR2" discontinuous Galerkin discretization for
+  diffusion via DGDiffusionBR2Integrator (see Example 14/14p).
+
+- Generalized the Multigrid class to support non-geometric multigrid. The
+  previous functionality, based on FiniteElementSpaceHierarchy, is now available
+  in the derived class GeometricMultigrid.
+
+- Upgraded the Catch unit test framework from version 2.13.0 to version 2.13.2.
+
+- The TMOP mesh optimization algorithms were extended to GPU:
+   - QualityMetric #1, #2, #7 and #77 are available in 2D, #302, #303, #315
+     and #321 in 3D
+   - Both AnalyticAdaptTC and DiscreteAdaptTC TargetConstructor are available
+   - Kernels for normalization and limiting have been added
+   - The AdvectorCG now also supports AssemblyLevel::PARTIAL
+
+- Added a new command line boolean option (`--all`) to the unit tests to launch
+  *all* non-regression tests.
+
+- Added support for different modes of QuadratureInterpolator on GPU.
+  The layout (QVectorLayout::byNODES|byVDIM) and the tensor products modes can
+  be enabled before calling the Mult, Values, Derivatives, PhysDerivatives and
+  Determinants methods.
+
+- Implemented a filter method for the Navier miniapp to stabilize highly
+  turbulent flows in direct numerical simulation.
+
+- Added HIP support to the CMake build system.
+
+- Added support for reading high-order Lagrange meshes in VTK format. Arbitrary-
+  orders and all element types are supported. See the VTK blog for more info:
+  https://blog.kitware.com/wp-content/uploads/2018/09/Source_Issue_43.pdf.
+
+- Added support for reading VTK meshes in XML format.
+
+- Added partial assembly and device support to Example 25/25p, with diagonal
+  preconditioning.
+
+- Implemented a variable step-size IMEX (VSSIMEX) method for the Navier miniapp.
 
 - Added new mesh quality metrics and improved the untangling capabilities of the
   TMOP-based mesh optimization algorithms.
 
-- The TMOP mesh optimization algorithms were extended to GPU:
-  * QualityMetric 1, 2, 7, 77 are available in 2D, 302, 303, 315, 321 in 3D
-  * Both AnalyticAdaptTC and DiscreteAdaptTC TargetConstructor are available
-  * Kernels for normalization and limiting have been added
-  * The AdvectorCG now also supports AssemblyLevel::PARTIAL
+- Added convective and skew-symmetric integrators for the nonlinear term in the
+  Navier-Stokes equations.
+
+- Added new miniapp directory mtop/ with optimization-oriented block parametric
+  non-linear form and abstract integrators. Two new miniapps, ParHeat and
+  SeqHeat, demonstrate parallel and sequential implementation of gradients
+  evaluation for linear diffusion with discrete density.
+
+- Changed the interface for the error estimator.
+
+- Implemented the Kelly error indicator for scalar-valued problems, supported
+  in serial and parallel builds.
+
+- Added new classes DenseSymmetricMatrix and SymmetricMatrixCoefficient for
+  efficient evaluation of symmetric matrix coefficients. This replaces the now
+  deprecated EvalSymmetric in MatrixCoefficient. Added DiagonalMatrixCoefficient
+  for clarity, which is a typedef of VectorCoefficient.
+
+- Added support for AMG preconditioners for non-symmetric systems (e.g.
+  advection-dominated problems) using hypre's approximate ideal restriction
+  (AIR) AMG. Requires hypre version 2.14.0 or newer. Usage is illustrated in
+  example 9/9p.
+
+- Implemented an adaptive linear solver tolerance option for NewtonSolver based
+  on the algorithm of Eisenstat and Walker.
+
+- Added support for nonscalar coefficient with VectorDiffusionIntegrator.
+
+- Extending support for L2 basis functions using MapTypes VALUE and INTEGRAL in
+  linear interpolators and GridFunction "GetValue" methods.
+
+- Variable order spaces, p- and hp-refinement. This is the initial (serial)
+  support for variable-order FiniteElementCollection and FiniteElementSpace.
+  The new method FiniteElementSpace::SetElementOrder can be called to set an
+  arbitrary order for each mesh element. The conforming interpolation matrix
+  will now automatically constrain p- and hp- interfaces, enabling general
+  hp-refinement in both 2D and 3D, on uniform or mixed NC meshes. Support for
+  parallel variable-order spaces will follow shortly.
 
 - Added support for creating refined meshes for all element types (e.g. by
   splitting high-order elements into low-order refined elements), including
   mixed meshes. The LOR Transfer miniapp (miniapps/tools/lor-transfer.cpp) now
   supports meshes with any element geometry.
 
+- Testing improvements:
+  * Transitioned from Travis to GitHub Action for testing/CI on GitHub.
+  * Effectively remove Travis from CI.
+  * Use Spack (and Uberenv) to automate TPL building in LLNL GitLab tests.
+  * Added a set of suggested git hooks for developers in config/githooks.
+
+- Added new miniapps demonstrating: 1) the use of GSLIB for overlapping grids,
+  see gslib/schwarz_ex1, and 2) coupling different physics in different domains,
+  see navier/cht. Note that gslib v1.0.7 is require (see INSTALL for details).
+
+- Added a new, very simple example (ex0 and parallel version ex0p). This
+  example solves a simple Poisson problem using H1 elements (the same problem as
+  ex1), but is intended to be extremely simple and approachable for new users.
+
 - Meshes consisting of any type of elements (including mixed meshes) can be
   converted to all-simplex meshes using Mesh::MakeSimplicial.
 
@@ -169,133 +199,42 @@ Meshing improvements
   requisite periodic vertex mappings can be created with
   Mesh::CreatePeriodicVertexMapping.
 
-- Added support for 1D non-conforming meshes (which can be useful for parallel
-  load balancing and derefinement).
+- Added support for transferring dual fields between high-order and low-order
+  refined finite element spaces using the transposed versions of the
+  L2ProjectionGridTransfer operators. This functionality is illustrated in the
+  lor-transfer miniapp.
+
+- Improved interface for using the Ginkgo library, including: support for matrix-
+  free operators in Ginkgo solvers, new wrappers for Ginkgo preconditioners, HIP
+  support, and reduction of unnecessary data copies.
+
+- Added initial support for hypre's mixed integer (mixedint) capability, which
+  uses different data types for local and global indices in order to save memory
+  in large problems. This capability requires that hypre was configured with the
+  --enable-mixedint option. Note that this option is currently tested only in
+  ex1p and may not work in more general settings.
+
+- Added support for transferring fields (primary and dual) between high-order
+  and low-order refined H1 finite element spaces using the
+  L2ProjectionH1GridTransfer operators. This functionality is demonstrated
+  through the lor-transfer miniapp when run with the -h1 option.
+
+- Added new functionality for constructing low-order refined discretizations and
+  solvers, see the LORDiscretization and LORSolver classes. A new basis type for
+  H(curl) and H(div) spaces is introduced to give spectral equivalence. This
+  functionality is illustrated in the LOR solvers miniapp in miniapps/solvers.
 
 - Added sample meshes in the `data` subdirectory showing the reference elements
   of the six currently supported element types; ref-segment.mesh,
   ref-triangle.mesh, ref-square.mesh, ref-tetrahedron.mesh, ref-cube.mesh, and
   ref-prism.mesh.
 
-High-performance computing
---------------------------
-- Added initial support for GPU-accelerated versions of PETSc that works with
-  MFEM_USE_CUDA if PETSc has been configured with CUDA support. Examples 1 and 9
-  in the examples/petsc directory have been modified to work with --device cuda.
-  Examples with GAMG (ex1p) and SLEPc (ex11p) are also provided.
-
-- Added support for explicit vectorization in the high-performance templated
-  code for Fujitsu's A64FX ARM microprocessor architecture.
-
-- Added support for different modes of QuadratureInterpolator on GPU.
-  The layout (QVectorLayout::byNODES|byVDIM) and the tensor products modes can
-  be enabled before calling the Mult, Values, Derivatives, PhysDerivatives and
-  Determinants methods.
-
-- Added method Device::SetMemoryTypes that can be used to change the default
-  host and device MemoryTypes before Device setup.
-
-- In class MemoryManager, added methods GetDualMemoryType and SetDualMemoryType;
-  dual MemoryTypes are used to determine the second MemoryType (host or device)
-  when only one MemoryType is specified in methods of class Memory.
-
-- Added Memory constructor for setting both the host and device MemoryTypes.
-
-- Switched the default behavior of device memory allocations so that they are
-  deferred until the device pointer is needed.
-
-- Added a second Umpire device MemoryType, DEVICE_UMPIRE_2, with corresponding
-  allocator that can be set with the method SetUmpireDevice2AllocatorName.
-
-- Added HOST_PINNED MemoryType and a pinned host allocator for CUDA and HIP.
-
-- Added matrix-free GPU-enabled implementations of GradientInterpolator and
-  IdentityInterpolator.
-
-New and updated examples and miniapps
--------------------------------------
-- Added a new, very simple example (ex0 and parallel version ex0p). This example
-  solves a simple Poisson problem using H1 elements (the same problem as ex1),
-  but is intended to be extremely simple and approachable for new users.
-
-- Added new miniapps demonstrating: 1) the use of GSLIB for overlapping grids,
-  see gslib/schwarz_ex1, and 2) coupling different physics in different domains,
-  see navier/cht. Note that gslib v1.0.7 is require (see INSTALL for details).
-
-- Added a new miniapp for computing (signed) distance functions to a point
-  source or zero level set. See miniapps/shifted/distance.cpp.
-
 - Added a high-order extension of the shifted boundary method to solve PDEs on
   non body-fitted meshes. This is illustrated in the new Shifted Diffusion
   miniapp, see miniapps/shifted/diffusion.cpp.
 
-- Added new miniapp directory mtop/ with optimization-oriented block parametric
-  non-linear form and abstract integrators. Two new miniapps, ParHeat and
-  SeqHeat, demonstrate parallel and sequential implementation of gradients
-  evaluation for linear diffusion with discrete density.
-
-- Added a new miniapp block-solvers that compares the performance of various
-  solvers for mixed finite element discretization of the second order scalar
-  elliptic equations. Currently available solvers in the miniapp include a
-  block-diagonal preconditioner that is based on approximate Schur complement
-  (implemented in ex5p), and a newly implemented solver DivFreeSolver, which
-  exploits a multilevel decomposition of the Raviart-Thomas space and its
-  divergence-free subspace. See the miniapps/solvers directory for more details.
-
-- Introduced new options for the mesh-explorer miniapp to visualize the actual
-  element attributes in parallel meshes while retaining the visualization of the
-  domain decomposition.
-
-- Added partial assembly and device support to Example 25/25p, with diagonal
-  preconditioning.
-
-- Implemented a filter method for the Navier miniapp to stabilize highly
-  turbulent flows in direct numerical simulation.
-
-Improved testing
-----------------
-- Transitioned from Travis to GitHub Action for testing/CI on GitHub.
-
-- Use Spack (and Uberenv) to automate TPL building in LLNL GitLab tests.
-
-- Extended `make test` to include GPU tests when MFEM is built with CUDA or HIP
-  support.
-
-- Added a set of suggested git hooks for developers in config/githooks.
-
-- Added support for Caliper: a library to integrate performance profiling
-  capabilities into applications. See examples/caliper for more details.
-
-- Added a new command line boolean option (`--all`) to the unit tests to launch
-  *all* non-regression tests.
-
-- Upgraded the Catch unit test framework from version 2.13.0 to version 2.13.2.
-
-Miscellaneous
--------------
-- The following integrations have updated minimum version requirements:
-  * CUDA >= 10.1.168
-  * Ginkgo >= 1.4.0
-  * GSLIB >= 1.0.7
-  * HIOP >= 0.4
-  * HYPRE >= 2.20.0 for mixedint support
-  * HYPRE >= 2.22.0 for CUDA support
-  * libCEED >= 0.8
-  * PETSc >= 3.15.0 for CUDA support
-  * RAJA >= 0.13.0
-  see INSTALL for more details.
-
-- Added a "scaled Jacobian" visualization option in the Mesh Explorer miniapp to
-  help identify elements with poor mesh quality.
-
-- Added support for reading VTK meshes in XML format.
-
 - Added makefile rule to generate TAGS table for vi or Emacs users.
 
-- Added HIP support to the CMake build system.
-
-- Various other simplifications, extensions, and bugfixes in the code.
-
 API changes
 -----------
 - Added an abstract interface `mfem::FaceRestriction` for `H1FaceRestriction`
@@ -305,11 +244,20 @@ API changes
   `mfem::FaceRestriction::AddMultTranspose` should replace previous calls to
   `mfem::FaceRestriction::MultTranspose`.
 
+libCEED integration improvements
+--------------------------------
+- Refactor the libCEED integration
+
+- Add support for VectorCoefficient with libCEED backends.
+
+- Add support for ConvectionIntegrator, and VectorConvectionNLFIntegrator with
+  libCEED backends.
+
 
 Version 4.2, released on October 30, 2020
 =========================================
 
-High-performance computing
+High-Performance Computing
 --------------------------
 - Added support for explicit vectorization in the high-performance templated
   code, which can now take advantage of specific classes on the following
@@ -391,6 +339,9 @@ Linear and nonlinear solvers
   matrix with the function HypreParMatrixFromBlocks. This could be useful for
   solving block systems with parallel direct solvers such as STRUMPACK.
 
+- Added CUDA support for SUNDIALS ODE integrators. See the updated SUNDIALS
+  modification of Example 9/9p.
+
 - Added wrappers for hypre's flexible GMRES solver and the new parallel ILU
   preconditioner. The latter requires hypre version 2.19.0 or later.
 
@@ -501,7 +452,7 @@ New and updated examples and miniapps
     L2, with partial assembly support in Example 24/24p.
   * Weak Dirichlet boundary conditions (Nitsche) to the NURBS miniapp.
 
-Data management and visualization
+Data management and Visualization
 ---------------------------------
 - Added support for ADIOS2 for parallel I/O with ParaView visualization. See
   Examples 5, 9, 12, 16. The classes adios2stream and ADIOS2DataCollection

CITATION.cff

@@ -1,66 +0,0 @@
-cff-version: 1.2.0
-message: "If you use MFEM, please cite it as follows."
-authors:
-- family-names: "MFEM Team"
-title: "MFEM: Modular Finite Element Methods [Software]"
-doi: 10.11578/dc.20171025.1248
-url: "https://mfem.org"
-preferred-citation:
-  type: article
-  authors:
-  - family-names: "Anderson"
-    given-names: "Robert"
-    orcid: "https://orcid.org/0000-0002-3508-9944"
-  - family-names: "Andrej"
-    given-names: "Julian"
-    orcid: "https://orcid.org/0000-0001-7661-4840"
-  - family-names: "Barker"
-    given-names: "Andrew"
-    orcid: "https://orcid.org/0000-0003-3572-911X"
-  - family-names: "Bramwell"
-    given-names: "Jamie"
-  - family-names: "Camier"
-    given-names: "Jean-Sylvain"
-    orcid: "https://orcid.org/0000-0003-2421-1999"
-  - family-names: "Cerveny"
-    given-names: "Jakub"
-    orcid: "https://orcid.org/0000-0003-4231-2531"
-  - family-names: "Dobrev"
-    given-names: "Veselin"
-    orcid: "https://orcid.org/0000-0003-1793-5622"
-  - family-names: "Dudouit"
-    given-names: "Yohann"
-    orcid: "https://orcid.org/0000-0001-5831-561X"
-  - family-names: "Fisher"
-    given-names: "Aaron"
-  - family-names: "Kolev"
-    given-names: "Tzanio"
-    orcid: "https://orcid.org/0000-0002-2810-3090"
-  - family-names: "Pazner"
-    given-names: "Will"
-    orcid: "https://orcid.org/0000-0003-4885-2934"
-  - family-names: "Stowell"
-    given-names: "Mark"
-    orcid: "https://orcid.org/0000-0002-5389-7435"
-  - family-names: "Tomov"
-    given-names: "Vladimir"
-    orcid: "https://orcid.org/0000-0002-1846-6816"
-  - family-names: "Akkerman"
-    given-names: "Ido"
-    orcid: "https://orcid.org/0000-0002-5937-0300"
-  - family-names: "Dahm"
-    given-names: "Johann"
-    orcid: "https://orcid.org/0000-0001-9657-3564"
-  - family-names: "Medina"
-    given-names: "David"
-  - family-names: "Zampini"
-    given-names: "Stefano"
-    orcid: "https://orcid.org/0000-0002-0435-0433"
-  doi: "10.1016/j.camwa.2020.06.009"
-  journal: "Computers \\& Mathematics with Applications"
-  month: 1
-  start: 42 # First page number
-  end: 74 # Last page number
-  title: "MFEM: A Modular Finite Element Methods Library"
-  volume: 81
-  year: 2021

CMakeLists.txt

@@ -16,7 +16,7 @@ set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH
 
 # Require C++11 and disable compiler-specific extensions
 set(CMAKE_CXX_STANDARD 11)
-if (MFEM_USE_GINKGO)
+if (MFEM_USE_GINKGO) 
   set(CMAKE_CXX_STANDARD 14)
 endif()
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -54,7 +54,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.3.1)
+set(${PROJECT_NAME}_VERSION 4.2.1)
 
 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -102,7 +102,7 @@ if (MFEM_USE_CUDA)
    endif()
    enable_language(CUDA)
    set(CMAKE_CUDA_STANDARD 11)
-   if (MFEM_USE_GINKGO)
+   if (MFEM_USE_GINKGO) 
      set(CMAKE_CUDA_STANDARD 14)
    endif()
    set(CMAKE_CUDA_STANDARD_REQUIRED ON)
@@ -175,10 +175,6 @@ else()
   set(MFEM_DEBUG OFF)
 endif()
 
-if (WIN32)
-	add_definitions(-D_USE_MATH_DEFINES)
-endif()
-
 # MPI -> hypre; PETSc (optional)
 if (MFEM_USE_MPI)
   find_package(MPI REQUIRED)
@@ -250,7 +246,6 @@ if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
     message(FATAL_ERROR " *** MFEM_USE_LEGACY_OPENMP requires MFEM_THREAD_SAFE=ON.")
   endif()
   find_package(OpenMP REQUIRED)
-  set(OPENMP_LIBRARIES ${OpenMP_CXX_LIBRARIES})
 endif()
 
 # SuiteSparse (before SUNDIALS which may depend on KLU)
@@ -335,10 +330,6 @@ if (MFEM_USE_CONDUIT)
     find_package(Conduit REQUIRED conduit relay blueprint )
 endif()
 
-if (MFEM_USE_FMS)
-    find_package(FMS REQUIRED fms )
-endif()
-
 # Axom/Sidre
 if (MFEM_USE_SIDRE)
   find_package(Axom REQUIRED Axom)
@@ -433,10 +424,9 @@ endif()
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
 set(MFEM_TPLS MPI_CXX OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS PETSC
-    SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB NETCDF
+    SLEPC MESQUITE MUMPS STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
     MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
     CUSPARSE MKL_CPARDISO AMGX CALIPER)
-
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")
@@ -455,9 +445,6 @@ include_directories(${TPL_INCLUDE_DIRS})
 if (OPENMP_FOUND)
   message(STATUS "MFEM: using package OpenMP")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
-  if (MFEM_USE_CUDA)
-    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=${OpenMP_CXX_FLAGS}")
-  endif()
 endif()
 
 message(STATUS "MFEM build type: CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}")

CONTRIBUTING.md

@@ -97,6 +97,7 @@ The MFEM source code has the following structure:
   .
   ├── config
   │   ├── cmake
+  │   │    └── ...
   │   └── githooks
   ├── data
   ├── doc
@@ -134,10 +135,10 @@ The MFEM source code has the following structure:
   └── tests
       ├── convergence
       ├── gitlab
-      ├── mem_manager
       ├── par-mesh-format
       ├── scripts
       └── unit
+          └── ...
 ```
 
 #### Main directories and classes

INSTALL

@@ -474,7 +474,7 @@ MFEM_USE_HIP = YES/NO
    Enables support for AMD devices in MFEM. HIP is a heterogeneous-compute
    interface for portability developed by AMD that can target both AMD and
    NVIDIA GPUs. The variable HIP_ARCH is used to specify the AMD GPU processor
-   used during compilation (by default, HIP_ARCH=gfx900). When enabled, this
+   used during compilation (by default, HIP_ARCH=gfx900).  When enabled, this
    option uses the HIP_* build options, see below.
 
 MFEM_USE_RAJA = YES/NO
@@ -516,13 +516,6 @@ MFEM_USE_CALIPER = YES/NO
    profiling at runtime with Caliper's configuration API. Alternatively, one
    can configure Caliper through environment variables or config files.
 
-MFEM_USE_FMS = YES/NO
-   Enables support for the FMS library which consists of the DataCollection
-   sub-class mfem::FMSDataCollection for I/O in FMS formats, see the header file
-   fem/fmsdatacollection.hpp. In addition, this option enables in-memory
-   convetion routines between FMS's FmsDataCollection structure and MFEM's
-   DataCollection class, see the header file fem/fmsconvert.hpp.
-
 MFEM_BUILD_TAG = (any value)
    An optional tag to characterize the build. Exported to config/config.mk.
    Can be used to identify the MFEM build from other makefiles.
@@ -547,9 +540,8 @@ The specific libraries and their options are:
   See also the "Specific options for hypre" section at the end of this file.
   URL: https://github.com/hypre-space/hypre and https://www.llnl.gov/casc/hypre
   Options: HYPRE_OPT, HYPRE_LIB.
-  Versions: HYPRE >= 2.10.0b  (HYPRE built without CUDA)
-            HYPRE >= 2.20.0   (HYPRE built with '--enable-mixedint')
-            HYPRE >= 2.22.1   (HYPRE built with CUDA)
+  Versions: HYPRE >= 2.10.0b,
+            HYPRE >= 2.20.0 for '--enable-mixedint' support.
 
 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
   MFEM_USE_METIS_5 = YES (default is to use METIS 4).
@@ -623,7 +615,7 @@ The specific libraries and their options are:
   and dependencies of specific modules, see the Ginkgo webpage below.
   URL: https://ginkgo-project.github.io
   Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or Debug).
-  Versions: Ginkgo >= 1.4.0.
+  Versions: Ginkgo >= 1.4.0. 
 
 - AmgX (optional), used when MFEM_USE_AMGX = YES.
   URL: https://github.com/NVIDIA/AMGX
@@ -761,11 +753,6 @@ The specific libraries and their options are:
   URL: https://zlib.net
   Options: ZLIB_OPT, ZLIB_LIB.
 
-- FMS (optional), used when MFEM_USE_FMS = YES.
-  URL: https://github.com/CEED/FMS
-  Options: FMS_OPT, FMS_LIB.
-  Versions: FMS >= 0.2.
-
 Building with CMake
 ===================
 The MFEM build system consists of two steps: configuration and compilation.
@@ -897,7 +884,6 @@ MFEM_USE_RAJA
 MFEM_USE_UMPIRE
 MFEM_USE_SIDRE
 MFEM_USE_CALIPER
-MFEM_USE_FMS
 
 The following options are CMake specific:
 
@@ -952,7 +938,6 @@ The CMake build system adds auto-detection for the following packages/libraries:
  - UMPIRE
  - AXOM - Used when MFEM_USE_SIDRE is enabled
  - CALIPER
- - FMS
 
 The following built-in CMake packages are also used:
 
@@ -970,7 +955,7 @@ config/config.hpp.in:
 
    cp config/config.hpp.in config/_config.hpp
 
-The file config/_config.hpp can then be edited to enable desired options. The
+The file config/_config.hpp can then be edited to enable desired options.  The
 MFEM library is simply a combination of all object files obtained by compiling
 the .cpp source files in the source directories: general, linalg, mesh, and fem.
 
@@ -978,7 +963,7 @@ the .cpp source files in the source directories: general, linalg, mesh, and fem.
 Specifying an MPI job launcher
 ==============================
 By default, MFEM will use 'mpirun -np #' to launch any of its parallel tests or
-miniapps, where # is the number of MPI tasks. An alternate MPI launcher can be
+miniapps, where # is the number of MPI tasks.  An alternate MPI launcher can be
 provided by setting the MFEM_MPIEXEC and MFEM_MPIEXEC_NP config variables.
 
 MFEM will expect the launcher command, plus the command line option to allow it

config/XSDKDefaults.cmake

@@ -256,10 +256,6 @@ IF (DEFINED TPL_ENABLE_SIDRE)
   SET(MFEM_USE_SIDRE ${TPL_ENABLE_SIDRE} CACHE BOOL "Enable Axom/Sidre usage" FORCE)
 ENDIF()
 
-IF (DEFINED TPL_ENABLE_FMS)
-  SET(MFEM_USE_FMS ${TPL_ENABLE_FMS} CACHE BOOL "Enable FMS usage" FORCE)
-ENDIF()
-
 IF (DEFINED TPL_ENABLE_CONDUIT)
   SET(MFEM_USE_CONDUIT ${TPL_ENABLE_CONDUIT} CACHE BOOL "Enable Conduit usage" FORCE)
 ENDIF()

config/cmake/MFEMConfig.cmake.in

@@ -44,7 +44,6 @@ set(MFEM_USE_PETSC @MFEM_USE_PETSC@)
 set(MFEM_USE_SLEPC @MFEM_USE_SLEPC@)
 set(MFEM_USE_MPFR @MFEM_USE_MPFR@)
 set(MFEM_USE_SIDRE @MFEM_USE_SIDRE@)
-set(MFEM_USE_FMS @MFEM_USE_FMS@)
 set(MFEM_USE_CONDUIT @MFEM_USE_CONDUIT@)
 set(MFEM_USE_PUMI @MFEM_USE_PUMI@)
 set(MFEM_USE_CUDA @MFEM_USE_CUDA@)

config/cmake/config.hpp.in

@@ -119,9 +119,6 @@
 // Enable the use of SIMD in the high performance templated classes
 #cmakedefine MFEM_USE_SIMD
 
-// Enable MFEM functionality based on the FMS library
-#cmakedefine MFEM_USE_FMS
-
 // Enable MFEM functionality based on Conduit
 #cmakedefine MFEM_USE_CONDUIT
 

config/cmake/modules/FindFMS.cmake

@@ -1,20 +0,0 @@
-# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - FMS_FOUND
-#   - FMS_LIBRARIES
-#   - FMS_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(FMS FMS FMS_DIR
-  "include" fms.h "lib" fms
-  "Paths to headers required by FMS." "Libraries required by FMS.")

config/config.hpp.in

@@ -117,9 +117,6 @@
 // Enable the use of SIMD in the high performance templated classes
 // #define MFEM_USE_SIMD
 
-// Enable FMS support
-// #define MFEM_USE_FMS
-
 // Enable Conduit support
 // #define MFEM_USE_CONDUIT
 

config/config.mk.in

@@ -43,7 +43,6 @@ MFEM_USE_PETSC         = @MFEM_USE_PETSC@
 MFEM_USE_SLEPC         = @MFEM_USE_SLEPC@
 MFEM_USE_MPFR          = @MFEM_USE_MPFR@
 MFEM_USE_SIDRE         = @MFEM_USE_SIDRE@
-MFEM_USE_FMS           = @MFEM_USE_FMS@
 MFEM_USE_CONDUIT       = @MFEM_USE_CONDUIT@
 MFEM_USE_PUMI          = @MFEM_USE_PUMI@
 MFEM_USE_HIOP          = @MFEM_USE_HIOP@

config/defaults.cmake

@@ -45,7 +45,6 @@ option(MFEM_USE_PETSC "Enable PETSc support." OFF)
 option(MFEM_USE_SLEPC "Enable SLEPc support." OFF)
 option(MFEM_USE_MPFR "Enable MPFR usage." OFF)
 option(MFEM_USE_SIDRE "Enable Axom/Sidre usage" OFF)
-option(MFEM_USE_FMS "Enable FMS usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
 option(MFEM_USE_PUMI "Enable PUMI" OFF)
 option(MFEM_USE_HIOP "Enable HiOp" OFF)
@@ -97,11 +96,6 @@ set(HYPRE_DIR "${MFEM_DIR}/../hypre/src/hypre" CACHE PATH
 # If hypre was compiled to depend on BLAS and LAPACK:
 # set(HYPRE_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
 #     "Packages that HYPRE depends on.")
-if (MFEM_USE_CUDA)
-   # This is only necessary when hypre is built with cuda:
-   set(HYPRE_REQUIRED_LIBRARIES "-lcusparse" "-lcurand" CACHE STRING
-       "Libraries that HYPRE depends on.")
-endif()
 
 set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library.")
 
@@ -138,10 +132,10 @@ set(MUMPS_DIR "${MFEM_DIR}/../MUMPS_5.2.0" CACHE PATH
     "Path to the MUMPS library.")
 # Packages required by MUMPS, depending on how it was compiled.
 set(MUMPS_REQUIRED_PACKAGES "MPI" "BLAS" "METIS" "ScaLAPACK" CACHE STRING
-    "Additional packages required by MUMPS.")
+    "Additional packages required by MUMPS.")    
 # If the MPI package does not find all required Fortran libraries:
 # set(MUMPS_REQUIRED_LIBRARIES "gfortran" "mpi_mpifh" CACHE STRING
-#     "Additional libraries required by MUMPS.")
+#     "Additional libraries required by MUMPS.")    
 
 set(STRUMPACK_DIR "${MFEM_DIR}/../STRUMPACK-build" CACHE PATH
     "Path to the STRUMPACK library.")
@@ -193,12 +187,6 @@ set(SLEPC_ARCH "arch-linux2-c-debug" CACHE STRING "SLEPC build architecture.")
 
 set(MPFR_DIR "" CACHE PATH "Path to the MPFR library.")
 
-set(FMS_DIR "${MFEM_DIR}/../fms" CACHE PATH
-    "Path to the FMS library.")
-# If FMS is built with Conduit:
-# set(FMS_REQUIRED_PACKAGES "Conduit/relay" CACHE STRING
-#     "Additional packages required by FMS.")
-
 set(CONDUIT_DIR "${MFEM_DIR}/../conduit" CACHE PATH
     "Path to the Conduit library.")
 

config/defaults.mk

@@ -136,7 +136,6 @@ MFEM_USE_PETSC         = NO
 MFEM_USE_SLEPC         = NO
 MFEM_USE_MPFR          = NO
 MFEM_USE_SIDRE         = NO
-MFEM_USE_FMS           = NO
 MFEM_USE_CONDUIT       = NO
 MFEM_USE_PUMI          = NO
 MFEM_USE_HIOP          = NO
@@ -175,10 +174,6 @@ LIBUNWIND_LIB = $(if $(NOTMAC),-lunwind -ldl,)
 HYPRE_DIR = @MFEM_DIR@/../hypre/src/hypre
 HYPRE_OPT = -I$(HYPRE_DIR)/include
 HYPRE_LIB = -L$(HYPRE_DIR)/lib -lHYPRE
-ifeq (YES,$(MFEM_USE_CUDA))
-   # This is only necessary when hypre is built with cuda:
-   HYPRE_LIB += -lcusparse -lcurand
-endif
 
 # METIS library configuration
 ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK)$(MFEM_USE_MUMPS),NONONO)
@@ -362,11 +357,6 @@ endif
 MPFR_OPT =
 MPFR_LIB = -lmpfr
 
-# FMS and required libraries configuration
-FMS_DIR = $(MFEM_DIR)/../fms
-FMS_OPT = -I$(FMS_DIR)/include
-FMS_LIB = -Wl,-rpath,$(FMS_DIR)/lib -L$(FMS_DIR)/lib -lfms
-
 # Conduit and required libraries configuration
 CONDUIT_DIR = @MFEM_DIR@/../conduit
 CONDUIT_OPT = -I$(CONDUIT_DIR)/include/conduit

config/test.mk

@@ -57,27 +57,22 @@ TIMECMD := $(word 1,$(TIMECMD))
 ifneq (,$(filter test%,$(MAKECMDGOALS)))
    MAKEFLAGS += -k
 endif
-# Test runs of the examples/miniapps with parameters - check exit code:
-# 0 means success, 255 means the test was skipped, anything else means error
+# Test runs of the examples/miniapps with parameters - check exit code
 mfem-test = \
    printf "   $(3) [$(2) $(1) ... ]: "; \
    $(call $(TIMEFUN),$(TIMECMD),$(2) ./$(1) $(if $(5),,-no-vis )$(4) \
      > $(1).stderr 2>&1); \
-   err="$$3"; \
-   if [ "$$3" = 0 ]; then $(PRINT_OK); \
-   else if [ "$$3" = 255 ]; then $(PRINT_SKIP); err=0; \
-   else $(PRINT_FAILED); cat $(1).stderr; fi; fi; \
-   rm -f $(1).stderr; exit $$err
+   if [ "$$3" = 0 ]; \
+   then $(PRINT_OK); else $(PRINT_FAILED); cat $(1).stderr; fi; \
+   rm -f $(1).stderr; exit $$3
 
 # Test runs of the examples/miniapps - check exit code and if a file exists
-# See mfem-test for the interpretation of the error code
 mfem-test-file = \
    printf "   $(3) [$(2) $(1) ... ]: "; \
    $(call $(TIMEFUN),$(TIMECMD),$(2) ./$(1) -no-vis > $(1).stderr 2>&1); \
    err="$$3"; \
-   if [ "$$3" = 0 ] && [ -e $(4) ]; then $(PRINT_OK); \
-   else if [ "$$3" = 255 ] && [ -e $(4) ]; then $(PRINT_SKIP); err=0; \
-   else $(PRINT_FAILED); cat $(1).stderr; err=64; fi; fi; \
+   if [ "$$3" = 0 ] && [ -e $(4) ]; \
+   then $(PRINT_OK); else $(PRINT_FAILED); cat $(1).stderr; err=64; fi; \
    rm -f $(1).stderr; exit $$err
 
 .PHONY: test test-par-YES test-par-NO test-ser test-par test-clean test-print
@@ -85,26 +80,6 @@ mfem-test-file = \
 # What sets of tests to run in serial and parallel
 test-par-YES: $(PAR_$(MFEM_TESTS):=-test-par) $(SEQ_$(MFEM_TESTS):=-test-seq)
 test-par-NO:  $(SEQ_$(MFEM_TESTS):=-test-seq)
-ifeq ($(MFEM_USE_CUDA),YES)
-.PHONY: test-par-YES-cuda test-par-NO-cuda test-ser-cuda test-par-cuda test-cuda
-test-par-YES: test-par-YES-cuda
-test-par-NO:  test-par-NO-cuda
-test-par-YES-cuda: test-par-cuda test-ser-cuda
-test-par-NO-cuda:  test-ser-cuda
-test-ser-cuda: $(SEQ_DEVICE_$(MFEM_TESTS):=-test-seq-cuda)
-test-par-cuda: $(PAR_DEVICE_$(MFEM_TESTS):=-test-par-cuda)
-test-cuda: test-par-$(MFEM_USE_MPI)-cuda clean-exec
-endif
-ifeq ($(MFEM_USE_HIP),YES)
-.PHONY: test-par-YES-hip test-par-NO-hip test-ser-hip test-par-hip test-hip
-test-par-YES: test-par-YES-hip
-test-par-NO:  test-par-NO-hip
-test-par-YES-hip: test-par-hip test-ser-hip
-test-par-NO-hip:  test-ser-hip
-test-ser-hip: $(SEQ_DEVICE_$(MFEM_TESTS):=-test-seq-hip)
-test-par-hip: $(PAR_DEVICE_$(MFEM_TESTS):=-test-par-hip)
-test-hip: test-par-$(MFEM_USE_MPI)-hip clean-exec
-endif
 test-ser:     test-par-NO
 test-par:     test-par-YES
 test:         all test-par-$(MFEM_USE_MPI) clean-exec

data/inline-pyramid.mesh

@@ -1,9 +0,0 @@
-MFEM INLINE mesh v1.0
-
-type = pyramid
-nx = 4
-ny = 4
-nz = 4
-sx = 1.0
-sy = 1.0
-sz = 1.0

data/octahedron.mesh

@@ -1,43 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-# PYRAMID     = 7
-#
-
-dimension
-3
-
-elements
-2
-1 7 4 3 2 1 0
-1 7 1 2 3 4 5
-
-boundary
-8
-1 2 0 2 1
-2 2 0 3 2
-3 2 0 4 3
-4 2 0 1 4
-5 2 1 2 5
-6 2 2 3 5
-7 2 3 4 5
-8 2 4 1 5
-
-vertices
-6
-3
-0 0 -1
-1 0 0
-0 1 0
--1 0 0
-0 -1 0
-0 0 1

data/star-q3.fms

@@ -1,246 +0,0 @@
-FMS: 100
-DataCollection/Name: star
-DataCollection/NumberOfFieldDescriptors: 1
-DataCollection/FieldDescriptors/0/Name: CoordsDescriptor
-DataCollection/FieldDescriptors/0/ComponentName: volume
-DataCollection/FieldDescriptors/0/Type: 0
-DataCollection/FieldDescriptors/0/FixedOrder/Size: 3
-DataCollection/FieldDescriptors/0/FixedOrder/Type: FMS_UINT64
-DataCollection/FieldDescriptors/0/FixedOrder/Values: [0, 1, 3]
-DataCollection/FieldDescriptors/0/NumDofs: 211
-DataCollection/NumberOfFields: 1
-DataCollection/Fields/0/Name: Coords
-DataCollection/Fields/0/LayoutType: 0
-DataCollection/Fields/0/NumberOfVectorComponents: 2
-DataCollection/Fields/0/FieldDescriptorName: CoordsDescriptor
-DataCollection/Fields/0/Data/Size: 422
-DataCollection/Fields/0/Data/Type: FMS_DOUBLE
-DataCollection/Fields/0/Data/Values: [-0.016886, 1.000000, 0.309017, 
-1.309020, -0.809017, -0.500000, 
--0.809017, -1.618030, 0.309017, 
--0.500000, 1.309020, 0.519420, 
-1.154510, 0.809019, 0.147680, 
--0.095492, -0.654508, -0.415586, 
--1.213520, -1.213520, -0.392210, 
--0.654508, -0.095492, 0.139949, 
-0.809019, 1.154510, 0.660184, 
--0.264063, -0.800064, -0.231060, 
-0.663691, 0.183114, 0.317639, 
-0.543082, 0.598483, 0.345112, 
-0.478298, 0.027703, 0.095229, 
-0.012368, -0.092534, -0.334412, 
--0.313767, -0.140526, -0.293881, 
--0.534056, -0.660290, -0.537646, 
--0.655590, -0.121396, -0.274504, 
--0.346497, -0.296570, 0.004737, 
--0.098835, 0.069287, 0.082675, 
-0.318799, 0.467183, 0.564505, 
-0.595190, 0.846237, 0.671735, 
-1.051500, 1.103010, 0.964008, 
-0.821603, 1.257520, 1.206010, 
-1.142350, 0.975686, 0.781273, 
-0.717257, 0.475684, 0.642352, 
-0.268930, 0.211049, 0.174181, 
-0.039345, -0.147746, -0.177481, 
--0.365164, -0.230328, -0.551503, 
--0.603005, -0.497587, -0.389864, 
--0.757514, -0.706011, -0.675487, 
--0.528946, -0.943851, -1.078690, 
--1.087600, -0.955467, -1.483190, 
--1.348360, -1.483190, -1.348360, 
--1.085930, -0.938010, -0.943851, 
--1.078690, -0.681476, -0.540944, 
--0.757514, -0.706011, -0.540614, 
--0.367058, -0.551503, -0.603005, 
--0.365164, -0.230328, -0.138552, 
--0.206896, 0.174181, 0.039345, 
-0.268468, 0.222269, 0.475684, 
-0.642352, 0.759791, 0.719381, 
-1.142350, 0.975686, 1.257520, 
-1.206010, 0.972837, 0.836119, 
-1.051500, 1.103010, 0.214572, 
-0.407449, 0.288323, 0.449827, 
--0.086700, -0.027358, -0.200560, 
--0.166595, -0.271802, -0.418426, 
--0.426131, -0.551441, -0.096117, 
--0.206969, -0.027946, -0.184969, 
-0.211136, 0.260131, 0.407172, 
-0.430781, 0.718277, 0.885068, 
-0.753103, 0.957692, 0.866273, 
-1.024530, 0.934099, 1.093820, 
-0.348422, 0.524463, 0.404903, 
-0.587376, 0.054525, 0.146431, 
--0.078026, -0.007795, -0.329488, 
--0.302967, -0.488115, -0.439332, 
--0.498515, -0.453527, -0.633059, 
--0.570251, -0.655787, -0.791132, 
--0.802013, -0.956872, -1.094720, 
--1.208560, -1.207720, -1.339910, 
--0.693371, -0.795404, -0.803594, 
--0.945068, -0.464668, -0.631721, 
--0.458968, -0.547876, -0.332878, 
--0.485696, -0.286385, -0.408481, 
-0.053021, -0.058200, 0.110846, 
--0.002086, 0.381384, 0.416784, 
-0.551133, 0.613261, 0.872474, 
-0.901208, 1.038300, 1.084660, 
-0.737459, 0.751250, 0.890002, 
-0.915210, 0.010915, 0.000000, 
-0.951057, 0.951057, 0.587785, 
-1.538840, -0.587785, 0.000000, 
--0.951057, -1.538840, -0.951057, 
--0.015847, 0.475529, 0.951057, 
-0.492248, 1.244950, 1.063310, 
-0.274399, 0.293893, -0.293892, 
--0.296404, -1.063310, -1.244950, 
--0.453865, -0.951057, -0.475529, 
-0.466620, 0.792932, -0.013913, 
--0.748783, -0.497528, 0.021382, 
--0.017158, 0.172591, 0.330125, 
-0.458568, 0.457971, 0.137740, 
-0.299049, 0.588394, 0.667324, 
-0.432341, 0.634346, 0.117322, 
-0.193603, 0.211702, 0.098278, 
--0.199438, -0.077304, -0.082243, 
--0.216296, -0.458634, -0.592374, 
--0.563926, -0.680404, -0.135751, 
--0.302942, -0.469005, -0.453640, 
--0.182727, -0.314240, 0.024270, 
-0.021546, 0.158510, 0.317019, 
-0.485799, 0.492951, 0.792548, 
-0.634038, 0.951057, 0.951057, 
-0.777915, 0.613430, 0.951057, 
-0.951057, 0.793994, 0.635800, 
-1.049020, 1.146990, 1.084480, 
-0.924310, 1.440880, 1.342910, 
-1.380330, 1.221820, 0.948209, 
-0.856297, 0.746293, 0.904802, 
-0.476242, 0.393234, 0.489821, 
-0.391857, 0.194471, 0.075751, 
-0.097964, 0.195929, -0.097964, 
--0.195928, -0.173234, -0.078922, 
--0.489821, -0.391856, -0.467007, 
--0.397859, -0.746293, -0.904802, 
--0.945206, -0.849559, -1.380330, 
--1.221820, -1.440880, -1.342910, 
--1.100830, -0.923191, -1.049020, 
--1.146990, -0.774515, -0.621542, 
--0.951057, -0.951057, -0.803055, 
--0.635255, -0.951057, -0.951057, 
--0.792548, -0.634038, -0.454301, 
--0.479369, -0.158510, -0.317019, 
-0.149331, 0.178643, 0.295860, 
-0.306275, 0.246225, 0.406610, 
-0.367954, 0.498458, -0.014929, 
-0.084917, -0.094272, -0.000726, 
--0.245374, -0.340755, -0.435351, 
--0.490564, -0.176355, -0.294974, 
--0.148010, -0.328246, 0.136248, 
-0.182883, 0.328957, 0.309903, 
-0.646446, 0.622546, 0.800859, 
-0.801803, 0.643087, 0.654473, 
-0.796963, 0.816799, 0.748189, 
-0.898148, 0.819092, 0.970033, 
-1.049540, 1.162660, 1.115310, 
-1.266700, 0.539960, 0.728396, 
-0.656318, 0.783152, 0.302716, 
-0.403858, 0.219006, 0.309252, 
-0.009667, 0.107161, -0.118211, 
-0.011786, -0.284704, -0.179858, 
--0.411567, -0.296676, -0.570168, 
--0.635348, -0.726784, -0.793211, 
--1.016860, -1.112970, -1.170910, 
--1.281530, -0.737860, -0.851723, 
--0.883732, -0.995070, -0.653153, 
--0.769939, -0.631918, -0.797688, 
--0.616289, -0.806819, -0.638485, 
--0.790356, -0.136399, -0.322769, 
--0.165339, -0.309622]
-DataCollection/Mesh/PartitionInfo/Size: 2
-DataCollection/Mesh/PartitionInfo/Type: FMS_UINT64
-DataCollection/Mesh/PartitionInfo/Values: [0, 1]
-DataCollection/Mesh/NumDomainNames: 1
-DataCollection/Mesh/NumComponents: 1
-DataCollection/Mesh/NumTags: 0
-DataCollection/Mesh/DomainNames/0/Name: Domain
-DataCollection/Mesh/DomainNames/0/NumDomains: 1
-DataCollection/Mesh/DomainNames/0/Domains/0/Dimension: 2
-DataCollection/Mesh/DomainNames/0/Domains/0/NumVertices: 31
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/EntityType: FMS_EDGE
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/NumEntities: 50
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Size: 100
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Type: FMS_INT32
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/0/Values: [11, 0, 26, 
-11, 26, 14, 
-14, 0, 27, 
-14, 27, 17, 
-17, 0, 28, 
-17, 28, 20, 
-20, 0, 29, 
-20, 29, 23, 
-23, 0, 30, 
-23, 30, 11, 
-11, 1, 12, 
-1, 26, 12, 
-12, 3, 13, 
-3, 26, 13, 
-13, 2, 14, 
-2, 15, 2, 
-27, 15, 15, 
-5, 16, 5, 
-27, 16, 16, 
-4, 17, 4, 
-18, 4, 28, 
-18, 18, 7, 
-19, 7, 28, 
-19, 19, 6, 
-20, 6, 21, 
-6, 29, 21, 
-21, 9, 22, 
-9, 29, 22, 
-22, 8, 23, 
-8, 24, 8, 
-30, 24, 24, 
-10, 25, 10, 
-30, 25, 25, 1]
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/EntityType: FMS_QUADRILATERAL
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/NumEntities: 20
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Size: 80
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Type: FMS_INT32
-DataCollection/Mesh/DomainNames/0/Domains/0/Entities/1/Values: [0, 1, 2, 
-3, 3, 4, 
-5, 6, 6, 
-7, 8, 9, 
-9, 10, 11, 
-12, 12, 13, 
-14, 0, 15, 
-16, 17, 1, 
-17, 18, 19, 
-20, 2, 20, 
-21, 22, 22, 
-23, 24, 4, 
-24, 25, 26, 
-27, 5, 27, 
-28, 29, 29, 
-30, 31, 7, 
-31, 32, 33, 
-34, 8, 34, 
-35, 36, 36, 
-37, 38, 10, 
-38, 39, 40, 
-41, 11, 41, 
-42, 43, 43, 
-44, 45, 13, 
-45, 46, 47, 
-48, 14, 48, 
-49, 15]
-DataCollection/Mesh/Components/0/Name: volume
-DataCollection/Mesh/Components/0/Dimension: 2
-DataCollection/Mesh/Components/0/NumEntities: 20
-DataCollection/Mesh/Components/0/Coordinates: Coords
-DataCollection/Mesh/Components/0/NumParts: 1
-DataCollection/Mesh/Components/0/Parts/0/DomainName: Domain
-DataCollection/Mesh/Components/0/Parts/0/DomainID: 0
-DataCollection/Mesh/Components/0/Parts/0/FullDomain: Yes
-DataCollection/Mesh/Components/0/Relations/Size: 0
-DataCollection/Mesh/Components/0/Relations/Type: FMS_UINT64

doc/CodeDocumentation.conf.in

@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = v4.3.1
+PROJECT_NUMBER         = v4.2.1
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

examples/CMakeLists.txt

@@ -84,9 +84,8 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
   get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
   string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
 
-  set(THIS_TEST_OPTIONS "-no-vis")
-  if (${TEST_NAME} MATCHES "ex0p?")
-    set(THIS_TEST_OPTIONS)
+  if (NOT (${TEST_NAME} MATCHES "ex0p?"))
+    set(THIS_TEST_OPTIONS "-no-vis")
   endif()
   if (${TEST_NAME} MATCHES "ex10p*")
     list(APPEND THIS_TEST_OPTIONS "-tf" "5")
@@ -108,34 +107,6 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
   endif()
 endforeach()
 
-# Add CUDA/HIP tests.
-set(DEVICE_EXAMPLES
-    # serial examples with device support:
-    ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
-    # parallel examples with device support:
-    ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p ex24p ex25p ex26p)
-set(MFEM_TEST_DEVICE)
-if (MFEM_USE_CUDA)
-  set(MFEM_TEST_DEVICE "cuda")
-elseif (MFEM_USE_HIP)
-  set(MFEM_TEST_DEVICE "hip")
-endif()
-if (MFEM_TEST_DEVICE)
-  foreach(TEST_NAME ${DEVICE_EXAMPLES})
-    set(THIS_TEST_OPTIONS "-no-vis" "-d" "${MFEM_TEST_DEVICE}")
-    if (NOT (${TEST_NAME} MATCHES ".*p$"))
-      add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_ser
-        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-    elseif (MFEM_USE_MPI)
-      add_test(NAME ${TEST_NAME}_${MFEM_TEST_DEVICE}_np=${MFEM_MPI_NP}
-        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-        ${MPIEXEC_PREFLAGS}
-        $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-        ${MPIEXEC_POSTFLAGS})
-    endif()
-  endforeach()
-endif()
-
 # If STRUMPACK is enabled, add a test run that uses it.
 if (MFEM_USE_STRUMPACK)
   add_test(NAME ex11p_strumpack_np=${MFEM_MPI_NP}

examples/caliper/README

@@ -1,18 +0,0 @@
-                    Finite Element Discretization Library
-                                   __
-                       _ __ ___   / _|  ___  _ __ ___
-                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
-                      | | | | | ||  _||  __/| | | | | |
-                      |_| |_| |_||_|   \___||_| |_| |_|
-
-                               https://mfem.org
-
-This directory contains modifications of the example codes that illustrate the
-use of MFEM features based on the Caliper performance profiling library.
-
-To build these examples, make sure that MFEM is configured with the option
-"MFEM_USE_CALIPER = YES", see the top-level INSTALL file for details (version
-2.5.0 of Caliper is recommended, though older versions may work too).
-
-We recommend comparing the original example codes with the corresponding files
-in the current directory.

examples/caliper/ex1.cpp

@@ -206,9 +206,9 @@ int main(int argc, char *argv[])
    cout << "Size of linear system: " << A->Height() << endl;
 
    // 11. Solve the linear system A X = B.
+   MFEM_PERF_BEGIN("Solve A X=B");
    if (!pa)
    {
-      MFEM_PERF_SCOPE("Solve A X=B (FA)");
 #ifndef MFEM_USE_SUITESPARSE
       // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
       GSSmoother M((SparseMatrix&)(*A));
@@ -223,7 +223,6 @@ int main(int argc, char *argv[])
    }
    else // Jacobi preconditioning in partial assembly mode
    {
-      MFEM_PERF_SCOPE("Solve A X=B (PA)");
       if (UsesTensorBasis(fespace))
       {
          OperatorJacobiSmoother M(a, ess_tdof_list);
@@ -234,6 +233,7 @@ int main(int argc, char *argv[])
          CG(*A, B, X, 1, 400, 1e-12, 0.0);
       }
    }
+   MFEM_PERF_END("Solve A X=B");
    // 12. Recover the solution as a finite element grid function.
    a.RecoverFEMSolution(X, b, x);
 

examples/caliper/ex1p.cpp

@@ -231,29 +231,28 @@ int main(int argc, char *argv[])
    // 13. Solve the linear system A X = B.
    //     * With full assembly, use the BoomerAMG preconditioner from hypre.
    //     * With partial assembly, use Jacobi smoothing, for now.
+   MFEM_PERF_BEGIN("Solve A X = B");
+   Solver *prec = NULL;
+   if (pa)
    {
-      MFEM_PERF_SCOPE("Solve A X=B");
-      Solver *prec = NULL;
-      if (pa)
+      if (UsesTensorBasis(fespace))
       {
-         if (UsesTensorBasis(fespace))
-         {
-            prec = new OperatorJacobiSmoother(a, ess_tdof_list);
-         }
+         prec = new OperatorJacobiSmoother(a, ess_tdof_list);
       }
-      else
-      {
-         prec = new HypreBoomerAMG;
-      }
-      CGSolver cg(MPI_COMM_WORLD);
-      cg.SetRelTol(1e-12);
-      cg.SetMaxIter(2000);
-      cg.SetPrintLevel(1);
-      if (prec) { cg.SetPreconditioner(*prec); }
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete prec;
    }
+   else
+   {
+      prec = new HypreBoomerAMG;
+   }
+   CGSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(1);
+   if (prec) { cg.SetPreconditioner(*prec); }
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+   delete prec;
+   MFEM_PERF_END("Solve A X = B");
    // 14. Recover the parallel grid function corresponding to X. This is the
    //     local finite element solution on each processor.
    a.RecoverFEMSolution(X, b, x);

examples/ex1.cpp

@@ -9,7 +9,6 @@
 //               ex1 -m ../data/fichera.mesh
 //               ex1 -m ../data/fichera-mixed.mesh
 //               ex1 -m ../data/toroid-wedge.mesh
-//               ex1 -m ../data/octahedron.mesh -o 1
 //               ex1 -m ../data/periodic-annulus-sector.msh
 //               ex1 -m ../data/periodic-torus-sector.msh
 //               ex1 -m ../data/square-disc-p2.vtk -o 2

examples/ex13p.cpp

@@ -55,7 +55,6 @@ int main(int argc, char *argv[])
    int order = 1;
    int nev = 5;
    bool visualization = 1;
-   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -72,8 +71,6 @@ int main(int argc, char *argv[])
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (!args.Good())
    {
@@ -89,18 +86,13 @@ int main(int argc, char *argv[])
       args.PrintOptions(cout);
    }
 
-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors. We
+   // 3. Read the (serial) mesh from the given mesh file on all processors. We
    //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
    //    and volume meshes with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
    int dim = mesh->Dimension();
 
-   // 5. Refine the serial mesh on all processors to increase the resolution. In
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
    //    this example we do 'ref_levels' of uniform refinement (2 by default, or
    //    specified on the command line with -rs).
    for (int lev = 0; lev < ser_ref_levels; lev++)
@@ -108,7 +100,7 @@ int main(int argc, char *argv[])
       mesh->UniformRefinement();
    }
 
-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution (1 time by
    //    default, or specified on the command line with -rp). Once the parallel
    //    mesh is defined, the serial mesh can be deleted.
@@ -118,8 +110,9 @@ int main(int argc, char *argv[])
    {
       pmesh->UniformRefinement();
    }
+   pmesh->ReorientTetMesh();
 
-   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
    //    use the Nedelec finite elements of the specified order.
    FiniteElementCollection *fec = new ND_FECollection(order, dim);
    ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
@@ -129,7 +122,7 @@ int main(int argc, char *argv[])
       cout << "Number of unknowns: " << size << endl;
    }
 
-   // 8. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
+   // 7. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
    //    element space. The first corresponds to the curl curl, while the second
    //    is a simple mass matrix needed on the right hand side of the
    //    generalized eigenvalue problem below. The boundary conditions are
@@ -171,7 +164,7 @@ int main(int argc, char *argv[])
    delete a;
    delete m;
 
-   // 9. Define and configure the AME eigensolver and the AMS preconditioner for
+   // 8. Define and configure the AME eigensolver and the AMS preconditioner for
    //    A to be used within the solver. Set the matrices which define the
    //    generalized eigenproblem A x = lambda M x.
    HypreAMS *ams = new HypreAMS(*A,fespace);
@@ -187,15 +180,15 @@ int main(int argc, char *argv[])
    ame->SetMassMatrix(*M);
    ame->SetOperator(*A);
 
-   // 10. Compute the eigenmodes and extract the array of eigenvalues. Define a
-   //     parallel grid function to represent each of the eigenmodes returned by
-   //     the solver.
+   // 9. Compute the eigenmodes and extract the array of eigenvalues. Define a
+   //    parallel grid function to represent each of the eigenmodes returned by
+   //    the solver.
    Array<double> eigenvalues;
    ame->Solve();
    ame->GetEigenvalues(eigenvalues);
    ParGridFunction x(fespace);
 
-   // 11. Save the refined mesh and the modes in parallel. This output can be
+   // 10. Save the refined mesh and the modes in parallel. This output can be
    //     viewed later using GLVis: "glvis -np <np> -m mesh -g mode".
    {
       ostringstream mesh_name, mode_name;
@@ -220,7 +213,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 12. Send the solution by socket to a GLVis server.
+   // 11. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       char vishost[] = "localhost";
@@ -260,7 +253,7 @@ int main(int argc, char *argv[])
       mode_sock.close();
    }
 
-   // 13. Free the used memory.
+   // 12. Free the used memory.
    delete ame;
    delete ams;
    delete M;

examples/ex16.cpp

@@ -24,10 +24,7 @@
 //               class ConductionOperator defining C(u)), as well as their
 //               implicit time integration. Note that implementing the method
 //               ConductionOperator::ImplicitSolve is the only requirement for
-//               high-order implicit (SDIRK) time integration. In this example,
-//               the diffusion operator is linearized by evaluating with the
-//               lagged solution from the previous timestep, so there is only
-//               a linear solve.
+//               high-order implicit (SDIRK) time integration.
 //
 //               We recommend viewing examples 2, 9 and 10 before viewing this
 //               example.
@@ -329,8 +326,8 @@ ConductionOperator::ConductionOperator(FiniteElementSpace &f, double al,
 void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
 {
    // Compute:
-   //    du_dt = M^{-1}*-Ku
-   // for du_dt, where K is linearized by using u from the previous timestep
+   //    du_dt = M^{-1}*-K(u)
+   // for du_dt
    Kmat.Mult(u, z);
    z.Neg(); // z = -z
    M_solver.Mult(z, du_dt);
@@ -341,7 +338,7 @@ void ConductionOperator::ImplicitSolve(const double dt,
 {
    // Solve the equation:
    //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
-   // for du_dt, where K is linearized by using u from the previous timestep
+   // for du_dt
    if (!T)
    {
       T = Add(1.0, Mmat, dt, Kmat);

examples/ex16p.cpp

@@ -24,11 +24,8 @@
 //               class ConductionOperator defining C(u)), as well as their
 //               implicit time integration. Note that implementing the method
 //               ConductionOperator::ImplicitSolve is the only requirement for
-//               high-order implicit (SDIRK) time integration. In this example,
-//               the diffusion operator is linearized by evaluating with the
-//               lagged solution from the previous timestep, so there is only
-//               a linear solve. Optional saving with ADIOS2
-//               (adios2.readthedocs.io) is also illustrated.
+//               high-order implicit (SDIRK) time integration. Optional saving
+//               with ADIOS2 (adios2.readthedocs.io) is also illustrated.
 //
 //               We recommend viewing examples 2, 9 and 10 before viewing this
 //               example.
@@ -423,8 +420,8 @@ ConductionOperator::ConductionOperator(ParFiniteElementSpace &f, double al,
 void ConductionOperator::Mult(const Vector &u, Vector &du_dt) const
 {
    // Compute:
-   //    du_dt = M^{-1}*-Ku
-   // for du_dt, where K is linearized by using u from the previous timestep
+   //    du_dt = M^{-1}*-K(u)
+   // for du_dt
    Kmat.Mult(u, z);
    z.Neg(); // z = -z
    M_solver.Mult(z, du_dt);
@@ -435,7 +432,7 @@ void ConductionOperator::ImplicitSolve(const double dt,
 {
    // Solve the equation:
    //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
-   // for du_dt, where K is linearized by using u from the previous timestep
+   // for du_dt
    if (!T)
    {
       T = Add(1.0, Mmat, dt, Kmat);

examples/ex19p.cpp

@@ -196,12 +196,6 @@ void InitialDeformation(const Vector &x, Vector &y);
 
 int main(int argc, char *argv[])
 {
-#ifdef HYPRE_USING_CUDA
-   cout << "\nAs of mfem-4.3 and hypre-2.22.0 (July 2021) this example\n"
-        << "is NOT supported with the CUDA version of hypre.\n\n";
-   return 255;
-#endif
-
    // 1. Initialize MPI
    MPI_Session mpi;
    const int myid = mpi.WorldRank();
@@ -444,19 +438,15 @@ JacobianPreconditioner::JacobianPreconditioner(Array<ParFiniteElementSpace *>
 void JacobianPreconditioner::Mult(const Vector &k, Vector &y) const
 {
    // Extract the blocks from the input and output vectors
-   Vector disp_in;
-   disp_in.MakeRef(const_cast<Vector&>(k), block_trueOffsets[0],
-                   block_trueOffsets[1]-block_trueOffsets[0]);
-   Vector pres_in;
-   pres_in.MakeRef(const_cast<Vector&>(k), block_trueOffsets[1],
-                   block_trueOffsets[2]-block_trueOffsets[1]);
+   Vector disp_in(k.GetData() + block_trueOffsets[0],
+                  block_trueOffsets[1]-block_trueOffsets[0]);
+   Vector pres_in(k.GetData() + block_trueOffsets[1],
+                  block_trueOffsets[2]-block_trueOffsets[1]);
 
-   Vector disp_out;
-   disp_out.MakeRef(y, block_trueOffsets[0],
-                    block_trueOffsets[1]-block_trueOffsets[0]);
-   Vector pres_out;
-   pres_out.MakeRef(y, block_trueOffsets[1],
-                    block_trueOffsets[2]-block_trueOffsets[1]);
+   Vector disp_out(y.GetData() + block_trueOffsets[0],
+                   block_trueOffsets[1]-block_trueOffsets[0]);
+   Vector pres_out(y.GetData() + block_trueOffsets[1],
+                   block_trueOffsets[2]-block_trueOffsets[1]);
 
    Vector temp(block_trueOffsets[1]-block_trueOffsets[0]);
    Vector temp2(block_trueOffsets[1]-block_trueOffsets[0]);
@@ -469,9 +459,6 @@ void JacobianPreconditioner::Mult(const Vector &k, Vector &y) const
    subtract(disp_in, temp, temp2);
 
    stiff_pcg->Mult(temp2, disp_out);
-
-   disp_out.SyncAliasMemory(y);
-   pres_out.SyncAliasMemory(y);
 }
 
 void JacobianPreconditioner::SetOperator(const Operator &op)
@@ -486,10 +473,7 @@ void JacobianPreconditioner::SetOperator(const Operator &op)
 
       if (!spaces[0]->GetParMesh()->Nonconforming())
       {
-#ifndef HYPRE_USING_CUDA
-         // Not available yet when hypre is built with CUDA
          stiff_prec_amg->SetElasticityOptions(spaces[0]);
-#endif
       }
 
       stiff_prec = stiff_prec_amg;

examples/ex1p.cpp

@@ -9,7 +9,6 @@
 //               mpirun -np 4 ex1p -m ../data/fichera.mesh
 //               mpirun -np 4 ex1p -m ../data/fichera-mixed.mesh
 //               mpirun -np 4 ex1p -m ../data/toroid-wedge.mesh
-//               mpirun -np 4 ex1p -m ../data/octahedron.mesh -o 1
 //               mpirun -np 4 ex1p -m ../data/periodic-annulus-sector.msh
 //               mpirun -np 4 ex1p -m ../data/periodic-torus-sector.msh
 //               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
@@ -90,8 +89,7 @@ int main(int argc, char *argv[])
    args.AddOption(&device_config, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
 #ifdef MFEM_USE_CEED
-   args.AddOption(&algebraic_ceed, "-a", "--algebraic",
-                  "-no-a", "--no-algebraic",
+   args.AddOption(&algebraic_ceed, "-a", "--algebraic", "-no-a", "--no-algebraic",
                   "Use algebraic Ceed solver");
 #endif
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -199,15 +197,15 @@ int main(int argc, char *argv[])
    b.AddDomainIntegrator(new DomainLFIntegrator(one));
    b.Assemble();
 
-   // 10. Define the solution vector x as a parallel finite element grid
-   //     function corresponding to fespace. Initialize x with initial guess of
-   //     zero, which satisfies the boundary conditions.
+   // 10. Define the solution vector x as a parallel finite element grid function
+   //     corresponding to fespace. Initialize x with initial guess of zero,
+   //     which satisfies the boundary conditions.
    ParGridFunction x(&fespace);
    x = 0.0;
 
    // 11. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the
-   //     Diffusion domain integrator.
+   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
+   //     domain integrator.
    ParBilinearForm a(&fespace);
    if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
    a.AddDomainIntegrator(new DiffusionIntegrator(one));

examples/ex22.cpp

@@ -13,8 +13,6 @@
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 1
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 2
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 2 -pa
-//               ex22 -m ../data/inline-wedge.mesh -o 1
-//               ex22 -m ../data/inline-pyramid.mesh -o 1
 //               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0
 //
 // Device sample runs:

examples/ex22p.cpp

@@ -13,8 +13,6 @@
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 1
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 2
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 1 -p 2 -pa
-//               mpirun -np 4 ex22p -m ../data/inline-wedge.mesh -o 1
-//               mpirun -np 4 ex22p -m ../data/inline-pyramid.mesh -o 1
 //               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0
 //
 // Device sample runs:

examples/ex24.cpp

@@ -113,6 +113,7 @@ int main(int argc, char *argv[])
          mesh->UniformRefinement();
       }
    }
+   mesh->ReorientTetMesh();
 
    // 5. Define a finite element space on the mesh. Here we use Nedelec or
    //    Raviart-Thomas finite elements of the specified order.

examples/ex24p.cpp

@@ -141,6 +141,7 @@ int main(int argc, char *argv[])
          pmesh->UniformRefinement();
       }
    }
+   pmesh->ReorientTetMesh();
 
    // 7. Define a parallel finite element space on the parallel mesh. Here we
    //    use Nedelec or Raviart-Thomas finite elements of the specified order.

examples/ex25.cpp

@@ -92,7 +92,7 @@ class PMLDiagMatrixCoefficient : public VectorCoefficient
 {
 private:
    CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML *, Vector &);
+   void (*Function)(const Vector &, CartesianPML * , Vector &);
 public:
    PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
                                               Vector &),
@@ -277,8 +277,10 @@ int main(int argc, char *argv[])
       mesh->UniformRefinement();
    }
 
-   // 6. Set element attributes in order to distinguish elements in the
-   //    PML region
+   // 6. Reorient mesh in case of a tet mesh
+   mesh->ReorientTetMesh();
+
+   // Set element attributes in order to distinguish elements in the PML region
    pml->SetAttributes(mesh);
 
    // 7. Define a finite element space on the mesh. Here we use the Nedelec

examples/ex25p.cpp

@@ -92,7 +92,7 @@ class PMLDiagMatrixCoefficient : public VectorCoefficient
 {
 private:
    CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML *, Vector &);
+   void (*Function)(const Vector &, CartesianPML * , Vector &);
 public:
    PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
                                               Vector &),
@@ -316,6 +316,9 @@ int main(int argc, char *argv[])
       }
    }
 
+   // 7a. Reorient mesh in case of a tet mesh
+   pmesh->ReorientTetMesh();
+
    // 8. Set element attributes in order to distinguish elements in the PML
    pml->SetAttributes(pmesh);
 

examples/ex28p.cpp

@@ -81,12 +81,6 @@ Mesh * build_trapezoid_mesh(double offset)
 
 int main(int argc, char *argv[])
 {
-#ifdef HYPRE_USING_CUDA
-   cout << "\nAs of mfem-4.3 and hypre-2.22.0 (July 2021) this example\n"
-        << "is NOT supported with the CUDA version of hypre.\n\n";
-   return 255;
-#endif
-
    // 1. Initialize MPI.
    int num_procs, myid;
    MPI_Init(&argc, &argv);
@@ -366,7 +360,6 @@ int main(int argc, char *argv[])
    }
    delete pmesh;
 
-   // HYPRE_Finalize();
    MPI_Finalize();
 
    return 0;

examples/ex2p.cpp

@@ -61,7 +61,6 @@ int main(int argc, char *argv[])
    bool visualization = 1;
    bool amg_elast = 0;
    bool reorder_space = false;
-   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -79,8 +78,6 @@ int main(int argc, char *argv[])
                   "Enable or disable GLVis visualization.");
    args.AddOption(&reorder_space, "-nodes", "--by-nodes", "-vdim", "--by-vdim",
                   "Use byNODES ordering of vector space instead of byVDIM");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (!args.Good())
    {
@@ -96,12 +93,7 @@ int main(int argc, char *argv[])
       args.PrintOptions(cout);
    }
 
-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
    //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
    //    and volume meshes with the same code.
    Mesh *mesh = new Mesh(mesh_file, 1, 1);
@@ -117,14 +109,14 @@ int main(int argc, char *argv[])
       return 3;
    }
 
-   // 5. Select the order of the finite element discretization space. For NURBS
+   // 4. Select the order of the finite element discretization space. For NURBS
    //    meshes, we increase the order by degree elevation.
    if (mesh->NURBSext)
    {
       mesh->DegreeElevate(order, order);
    }
 
-   // 6. Refine the serial mesh on all processors to increase the resolution. In
+   // 5. Refine the serial mesh on all processors to increase the resolution. In
    //    this example we do 'ref_levels' of uniform refinement. We choose
    //    'ref_levels' to be the largest number that gives a final mesh with no
    //    more than 1,000 elements.
@@ -137,7 +129,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 7. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution. Once the
    //    parallel mesh is defined, the serial mesh can be deleted.
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -150,7 +142,7 @@ int main(int argc, char *argv[])
       }
    }
 
-   // 8. Define a parallel finite element space on the parallel mesh. Here we
+   // 7. Define a parallel finite element space on the parallel mesh. Here we
    //    use vector finite elements, i.e. dim copies of a scalar finite element
    //    space. We use the ordering by vector dimension (the last argument of
    //    the FiniteElementSpace constructor) which is expected in the systems
@@ -183,7 +175,7 @@ int main(int argc, char *argv[])
            << "Assembling: " << flush;
    }
 
-   // 9. Determine the list of true (i.e. parallel conforming) essential
+   // 8. Determine the list of true (i.e. parallel conforming) essential
    //    boundary dofs. In this example, the boundary conditions are defined by
    //    marking only boundary attribute 1 from the mesh as essential and
    //    converting it to a list of true dofs.
@@ -192,14 +184,14 @@ int main(int argc, char *argv[])
    ess_bdr[0] = 1;
    fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
 
-   // 10. Set up the parallel linear form b(.) which corresponds to the
-   //     right-hand side of the FEM linear system. In this case, b_i equals the
-   //     boundary integral of f*phi_i where f represents a "pull down" force on
-   //     the Neumann part of the boundary and phi_i are the basis functions in
-   //     the finite element fespace. The force is defined by the object f, which
-   //     is a vector of Coefficient objects. The fact that f is non-zero on
-   //     boundary attribute 2 is indicated by the use of piece-wise constants
-   //     coefficient for its last component.
+   // 9. Set up the parallel linear form b(.) which corresponds to the
+   //    right-hand side of the FEM linear system. In this case, b_i equals the
+   //    boundary integral of f*phi_i where f represents a "pull down" force on
+   //    the Neumann part of the boundary and phi_i are the basis functions in
+   //    the finite element fespace. The force is defined by the object f, which
+   //    is a vector of Coefficient objects. The fact that f is non-zero on
+   //    boundary attribute 2 is indicated by the use of piece-wise constants
+   //    coefficient for its last component.
    VectorArrayCoefficient f(dim);
    for (int i = 0; i < dim-1; i++)
    {
@@ -220,13 +212,13 @@ int main(int argc, char *argv[])
    }
    b->Assemble();
 
-   // 11. Define the solution vector x as a parallel finite element grid
+   // 10. Define the solution vector x as a parallel finite element grid
    //     function corresponding to fespace. Initialize x with initial guess of
    //     zero, which satisfies the boundary conditions.
    ParGridFunction x(fespace);
    x = 0.0;
 
-   // 12. Set up the parallel bilinear form a(.,.) on the finite element space
+   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
    //     corresponding to the linear elasticity integrator with piece-wise
    //     constants coefficient lambda and mu.
    Vector lambda(pmesh->attributes.Max());
@@ -241,7 +233,7 @@ int main(int argc, char *argv[])
    ParBilinearForm *a = new ParBilinearForm(fespace);
    a->AddDomainIntegrator(new ElasticityIntegrator(lambda_func, mu_func));
 
-   // 13. Assemble the parallel bilinear form and the corresponding linear
+   // 12. Assemble the parallel bilinear form and the corresponding linear
    //     system, applying any necessary transformations such as: parallel
    //     assembly, eliminating boundary conditions, applying conforming
    //     constraints for non-conforming AMR, static condensation, etc.
@@ -258,7 +250,7 @@ int main(int argc, char *argv[])
       cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
    }
 
-   // 14. Define and apply a parallel PCG solver for A X = B with the BoomerAMG
+   // 13. Define and apply a parallel PCG solver for A X = B with the BoomerAMG
    //     preconditioner from hypre.
    HypreBoomerAMG *amg = new HypreBoomerAMG(A);
    if (amg_elast && !a->StaticCondensationIsEnabled())
@@ -276,11 +268,11 @@ int main(int argc, char *argv[])
    pcg->SetPreconditioner(*amg);
    pcg->Mult(B, X);
 
-   // 15. Recover the parallel grid function corresponding to X. This is the
+   // 14. Recover the parallel grid function corresponding to X. This is the
    //     local finite element solution on each processor.
    a->RecoverFEMSolution(X, *b, x);
 
-   // 16. For non-NURBS meshes, make the mesh curved based on the finite element
+   // 15. For non-NURBS meshes, make the mesh curved based on the finite element
    //     space. This means that we define the mesh elements through a fespace
    //     based transformation of the reference element.  This allows us to save
    //     the displaced mesh as a curved mesh when using high-order finite
@@ -292,7 +284,7 @@ int main(int argc, char *argv[])
       pmesh->SetNodalFESpace(fespace);
    }
 
-   // 17. Save in parallel the displaced mesh and the inverted solution (which
+   // 16. Save in parallel the displaced mesh and the inverted solution (which
    //     gives the backward displacements to the original grid). This output
    //     can be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
    {
@@ -313,7 +305,7 @@ int main(int argc, char *argv[])
       x.Save(sol_ofs);
    }
 
-   // 18. Send the above data by socket to a GLVis server.  Use the "n" and "b"
+   // 17. Send the above data by socket to a GLVis server.  Use the "n" and "b"
    //     keys in GLVis to visualize the displacements.
    if (visualization)
    {
@@ -325,7 +317,7 @@ int main(int argc, char *argv[])
       sol_sock << "solution\n" << *pmesh << x << flush;
    }
 
-   // 19. Free the used memory.
+   // 18. Free the used memory.
    delete pcg;
    delete amg;
    delete a;

examples/ex3.cpp

@@ -16,8 +16,6 @@
 //               ex3 -m ../data/beam-hex-nurbs.mesh
 //               ex3 -m ../data/amr-hex.mesh
 //               ex3 -m ../data/fichera-amr.mesh
-//               ex3 -m ../data/ref-prism.mesh -o 1
-//               ex3 -m ../data/octahedron.mesh -o 1
 //               ex3 -m ../data/star-surf.mesh -o 1
 //               ex3 -m ../data/mobius-strip.mesh -f 0.1
 //               ex3 -m ../data/klein-bottle.mesh -f 0.1
@@ -115,6 +113,7 @@ int main(int argc, char *argv[])
          mesh->UniformRefinement();
       }
    }
+   mesh->ReorientTetMesh();
 
    // 5. Define a finite element space on the mesh. Here we use the Nedelec
    //    finite elements of the specified order.

examples/ex3p.cpp

@@ -16,8 +16,6 @@
 //               mpirun -np 4 ex3p -m ../data/beam-hex-nurbs.mesh
 //               mpirun -np 4 ex3p -m ../data/amr-quad.mesh -o 2
 //               mpirun -np 4 ex3p -m ../data/amr-hex.mesh
-//               mpirun -np 4 ex3p -m ../data/ref-prism.mesh -o 1
-//               mpirun -np 4 ex3p -m ../data/octahedron.mesh -o 1
 //               mpirun -np 4 ex3p -m ../data/star-surf.mesh -o 2
 //               mpirun -np 4 ex3p -m ../data/mobius-strip.mesh -o 2 -f 0.1
 //               mpirun -np 4 ex3p -m ../data/klein-bottle.mesh -o 2 -f 0.1
@@ -105,7 +103,6 @@ int main(int argc, char *argv[])
       {
          args.PrintUsage(cout);
       }
-      // HYPRE_Finalize();
       MPI_Finalize();
       return 1;
    }
@@ -141,7 +138,9 @@ int main(int argc, char *argv[])
 
    // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
+   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
+   //    meshes need to be reoriented before we can define high-order Nedelec
+   //    spaces on them.
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
    delete mesh;
    {
@@ -151,6 +150,7 @@ int main(int argc, char *argv[])
          pmesh->UniformRefinement();
       }
    }
+   pmesh->ReorientTetMesh();
 
    // 7. Define a parallel finite element space on the parallel mesh. Here we
    //    use the Nedelec finite elements of the specified order.

examples/ex4.cpp

@@ -19,8 +19,6 @@
 //               ex4 -m ../data/amr-hex.mesh
 //               ex4 -m ../data/amr-hex.mesh -o 2 -hb
 //               ex4 -m ../data/fichera-amr.mesh -o 2 -sc
-//               ex4 -m ../data/ref-prism.mesh -o 1
-//               ex4 -m ../data/octahedron.mesh -o 1
 //               ex4 -m ../data/star-surf.mesh -o 1
 //
 // Device sample runs:

examples/ex4p.cpp

@@ -19,8 +19,6 @@
 //               mpirun -np 3 ex4p -m ../data/amr-quad.mesh -o 2 -hb
 //               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -sc
 //               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -hb
-//               mpirun -np 4 ex4p -m ../data/ref-prism.mesh -o 1
-//               mpirun -np 4 ex4p -m ../data/octahedron.mesh -o 1
 //               mpirun -np 4 ex4p -m ../data/star-surf.mesh -o 3 -hb
 //
 // Device sample runs:
@@ -137,7 +135,9 @@ int main(int argc, char *argv[])
 
    // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
+   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
+   //    meshes need to be reoriented before we can define high-order Nedelec
+   //    spaces on them (this is needed in the ADS solver below).
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
    delete mesh;
    {
@@ -147,6 +147,7 @@ int main(int argc, char *argv[])
          pmesh->UniformRefinement();
       }
    }
+   pmesh->ReorientTetMesh();
 
    // 7. Define a parallel finite element space on the parallel mesh. Here we
    //    use the Raviart-Thomas finite elements of the specified order.

examples/ex5.cpp

@@ -197,7 +197,6 @@ int main(int argc, char *argv[])
       SparseMatrix &M(mVarf->SpMat());
       SparseMatrix &B(bVarf->SpMat());
       B *= -1.;
-      if (Device::IsEnabled()) { B.BuildTranspose(); }
       Bt = new TransposeOperator(&B);
 
       darcyOp.SetBlock(0,0, &M);
@@ -241,7 +240,6 @@ int main(int argc, char *argv[])
    {
       SparseMatrix &M(mVarf->SpMat());
       M.GetDiag(Md);
-      Md.HostReadWrite();
 
       SparseMatrix &B(bVarf->SpMat());
       MinvBt = Transpose(B);
@@ -289,18 +287,12 @@ int main(int argc, char *argv[])
    chrono.Stop();
 
    if (solver.GetConverged())
-   {
       std::cout << "MINRES converged in " << solver.GetNumIterations()
-                << " iterations with a residual norm of "
-                << solver.GetFinalNorm() << ".\n";
-   }
+                << " iterations with a residual norm of " << solver.GetFinalNorm() << ".\n";
    else
-   {
       std::cout << "MINRES did not converge in " << solver.GetNumIterations()
-                << " iterations. Residual norm is " << solver.GetFinalNorm()
-                << ".\n";
-   }
-   std::cout << "MINRES solver took " << chrono.RealTime() << "s.\n";
+                << " iterations. Residual norm is " << solver.GetFinalNorm() << ".\n";
+   std::cout << "MINRES solver took " << chrono.RealTime() << "s. \n";
 
    // 12. Create the grid functions u and p. Compute the L2 error norms.
    GridFunction u, p;

examples/ex7p.cpp

@@ -47,7 +47,6 @@ int main(int argc, char *argv[])
    int order = 2;
    bool always_snap = false;
    bool visualization = 1;
-   const char *device_config = "cpu";
 
    OptionsParser args(argc, argv);
    args.AddOption(&elem_type, "-e", "--elem",
@@ -66,8 +65,6 @@ int main(int argc, char *argv[])
                   "--snap-at-the-end",
                   "If true, snap nodes to the sphere initially and after each refinement "
                   "otherwise, snap only after the last refinement");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
    args.Parse();
    if (!args.Good())
    {
@@ -83,12 +80,7 @@ int main(int argc, char *argv[])
       args.PrintOptions(cout);
    }
 
-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Generate an initial high-order (surface) mesh on the unit sphere. The
+   // 3. Generate an initial high-order (surface) mesh on the unit sphere. The
    //    Mesh object represents a 2D mesh in 3 spatial dimensions. We first add
    //    the elements and the vertices of the mesh, and then make it high-order
    //    by specifying a finite element space for its nodes.
@@ -154,7 +146,7 @@ int main(int argc, char *argv[])
    FiniteElementSpace nodal_fes(mesh, &fec, mesh->SpaceDimension());
    mesh->SetNodalFESpace(&nodal_fes);
 
-   // 5. Refine the mesh while snapping nodes to the sphere. Number of parallel
+   // 4. Refine the mesh while snapping nodes to the sphere. Number of parallel
    //    refinements is fixed to 2.
    for (int l = 0; l <= ref_levels; l++)
    {
@@ -226,7 +218,7 @@ int main(int argc, char *argv[])
       SnapNodes(*pmesh);
    }
 
-   // 6. Define a finite element space on the mesh. Here we use isoparametric
+   // 5. Define a finite element space on the mesh. Here we use isoparametric
    //    finite elements -- the same as the mesh nodes.
    ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, &fec);
    HYPRE_BigInt size = fespace->GlobalTrueVSize();
@@ -235,7 +227,7 @@ int main(int argc, char *argv[])
       cout << "Number of unknowns: " << size << endl;
    }
 
-   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
+   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
    //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
    //    the basis functions in the finite element fespace.
    ParLinearForm *b = new ParLinearForm(fespace);
@@ -245,27 +237,27 @@ int main(int argc, char *argv[])
    b->AddDomainIntegrator(new DomainLFIntegrator(rhs_coef));
    b->Assemble();
 
-   // 8. Define the solution vector x as a finite element grid function
+   // 7. Define the solution vector x as a finite element grid function
    //    corresponding to fespace. Initialize x with initial guess of zero.
    ParGridFunction x(fespace);
    x = 0.0;
 
-   // 9. Set up the bilinear form a(.,.) on the finite element space
+   // 8. Set up the bilinear form a(.,.) on the finite element space
    //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
    //    and Mass domain integrators.
    ParBilinearForm *a = new ParBilinearForm(fespace);
    a->AddDomainIntegrator(new DiffusionIntegrator(one));
    a->AddDomainIntegrator(new MassIntegrator(one));
 
-   // 10. Assemble the parallel linear system, applying any transformations
-   //     such as: parallel assembly, applying conforming constraints, etc.
+   // 9. Assemble the parallel linear system, applying any transformations
+   //    such as: parallel assembly, applying conforming constraints, etc.
    a->Assemble();
    HypreParMatrix A;
    Vector B, X;
    Array<int> empty_tdof_list;
    a->FormLinearSystem(empty_tdof_list, x, *b, A, X, B);
 
-   // 11. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
+   // 10. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
    //     preconditioner from hypre. Extract the parallel grid function x
    //     corresponding to the finite element approximation X. This is the local
    //     solution on each processor.
@@ -281,14 +273,14 @@ int main(int argc, char *argv[])
    delete a;
    delete b;
 
-   // 12. Compute and print the L^2 norm of the error.
+   // 11. Compute and print the L^2 norm of the error.
    double err = x.ComputeL2Error(sol_coef);
    if (myid == 0)
    {
       cout << "\nL2 norm of error: " << err << endl;
    }
 
-   // 13. Save the refined mesh and the solution. This output can be viewed
+   // 12. Save the refined mesh and the solution. This output can be viewed
    //     later using GLVis: "glvis -np <np> -m sphere_refined -g sol".
    {
       ostringstream mesh_name, sol_name;
@@ -304,7 +296,7 @@ int main(int argc, char *argv[])
       x.Save(sol_ofs);
    }
 
-   // 14. Send the solution by socket to a GLVis server.
+   // 13. Send the solution by socket to a GLVis server.
    if (visualization)
    {
       char vishost[] = "localhost";
@@ -315,7 +307,7 @@ int main(int argc, char *argv[])
       sol_sock << "solution\n" << *pmesh << x << flush;
    }
 
-   // 15. Free the used memory.
+   // 14. Free the used memory.
    delete pcg;
    delete amg;
    delete fespace;

examples/ex8p.cpp

@@ -106,6 +106,7 @@ int main(int argc, char *argv[])
          pmesh->UniformRefinement();
       }
    }
+   pmesh->ReorientTetMesh();
 
    // 6. Define the trial, interfacial (trace) and test DPG spaces:
    //    - The trial space, x0_space, contains the non-interfacial unknowns and

examples/fd/element-smoother.cpp

@@ -0,0 +1,353 @@
+#include "element-smoother.hpp"
+
+using namespace std;
+using namespace mfem;
+
+
+ElementSmoother::ElementSmoother(ParFiniteElementSpace * fes_,  
+Array<int> ess_bdr, Coefficient * cf_ )
+: Solver(fes_->GetTrueVSize()), fes(fes_), cf(cf_)
+{  
+   comm = fes->GetComm();
+   MPI_Comm_size(comm, &num_procs);
+   MPI_Comm_rank(comm, &myid);
+   Pr = fes->GetProlongationMatrix();
+   ParMesh * pmesh = fes->GetParMesh();
+   dim = pmesh->Dimension();
+   eidx.SetSize(dim);
+   eidx[0] = 0;
+   eidx[1] = 1;
+   if (dim == 3) eidx[2] = 8;
+   nrelems = pmesh->GetNE();
+   tpcf.SetSize(nrelems);
+   int vsize = fes->GetVSize();
+   ovlp_count.SetSize(vsize);
+   ovlp_count = 0.0;
+   // Construct overlap count for each dof &
+   // count x y z edges sharing a vertex
+   for (int i=0; i<nrelems; i++)
+   {
+      Array<int> elem_dofs;
+      fes->GetElementDofs(i,elem_dofs);
+      for (int j = 0; j<elem_dofs.Size(); j++)
+      {
+         ovlp_count(elem_dofs[j]) +=1.0;
+      }
+   }
+
+
+   Vector tovlp_count(fes->TrueVSize());
+   if (Pr)
+   {
+      Pr->MultTranspose(ovlp_count,tovlp_count);
+      Pr->Mult(tovlp_count, ovlp_count);
+   }
+
+   DenseMatrix edge_counts;
+   // GetVertexToEdgeCount(pmesh,edge_counts);
+   double * data = edge_counts.GetData();
+
+   // helper H1 fespace for communication of vertex info
+   H1_FECollection fec(1, dim); 
+   ParFiniteElementSpace aux_fes(const_cast<ParMesh *>(pmesh), &fec);
+   Vector tedge_counts(aux_fes.TrueVSize());
+   Vector temp(aux_fes.GetVSize());
+
+
+   // const Operator * Ph = aux_fes.GetProlongationMatrix();
+
+   // if (Ph)
+   // {
+   //    for (int d=0; d<dim; d++)
+   //    {
+   //       temp.SetData(&data[d*aux_fes.GetVSize()]);
+   //       Ph->MultTranspose(temp,tedge_counts);
+   //       Ph->Mult(tedge_counts, temp);
+   //    }
+   // }
+
+   for (int i = 0; i < vsize; i++)
+   {
+      ovlp_count(i) = 1.0/sqrt(ovlp_count(i)); 
+   }
+
+   fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   TPElementTransformation TPTrans(*fes);
+   
+   int nredges = pmesh->GetNEdges();
+   edge_orient.SetSize(nredges); // orientation of edges wrt the element
+   Array<DenseMatrix *> EdgeGrad(nredges);
+   Array<DenseMatrix *> EdgeMass(nredges);
+   Array<DenseMatrix *> AssembledEdgeGrad(nredges);
+   Array<DenseMatrix *> AssembledEdgeMass(nredges);
+   // Initialize
+   for (int i = 0; i<nredges; i++)
+   {
+      AssembledEdgeGrad[i] = nullptr;
+      AssembledEdgeMass[i] = nullptr;
+      EdgeGrad[i] = nullptr;
+      EdgeMass[i] = nullptr;
+   }
+
+   // loop through element to calculate Edge matrices
+   Array<int> emarker(nredges); emarker = 0;
+   for (int iel = 0; iel<nrelems; iel++)
+   {
+      tpcf[iel] = new ElementTPFunctionCoefficient(*fes,iel,*cf);
+      Array<int> edges, cor;
+      pmesh->GetElementEdges(iel,edges,cor);
+      for (int ii = 0; ii<dim; ii++) 
+      {   
+         int i = eidx[ii];
+         int edge = edges[i];
+         if (emarker[edge]) continue;
+         edge_orient[edge] = cor[i];
+         const FiniteElement * fe = fes->GetEdgeElement(edge);
+
+         tpcf[iel]->SetCoord(ii);
+         tpcf[iel]->SetOrient(edge_orient[edge]);
+         tpcf[iel]->ResetCounter(ii);
+         IntegrationRule *irs = TensorIntegrationRule(1,fe->GetOrder());
+         EdgeGrad[edge] = new DenseMatrix(fe->GetDof());
+         EdgeMass[edge] = new DenseMatrix(fe->GetDof());
+
+         int j;
+         Vector * Q;
+         switch (ii)
+         {
+         case 0: 
+            j = 1; 
+            Q = tpcf[iel]->GetVecX();
+            break;
+         case 1: 
+            j = 0; 
+            Q = tpcf[iel]->GetVecY();
+            break;  
+         default:
+            j=2; 
+            Q = tpcf[iel]->GetVecZ();
+            break;
+         }
+
+         Vector * vecG = TPTrans.GetTPTransformation(iel,ii,ii);
+         Vector * vecM = nullptr;
+         Vector *vecM1 = nullptr;
+         Vector *vecM2 = nullptr;
+         if (dim == 2)
+         {
+            vecM = TPTrans.GetTPTransformation(iel,ii,j);
+         }
+         else
+         {
+            switch (ii)
+            {
+            case 0:
+            {
+               vecM1 = TPTrans.GetTPTransformation(iel,ii,1);
+               vecM2 = TPTrans.GetTPTransformation(iel,ii,2);
+            }
+            break;
+            case 1:
+            {
+               vecM1 = TPTrans.GetTPTransformation(iel,ii,0);
+               vecM2 = TPTrans.GetTPTransformation(iel,ii,2);
+            }            
+            default:
+            {
+               vecM1 = TPTrans.GetTPTransformation(iel,ii,0);
+               vecM2 = TPTrans.GetTPTransformation(iel,ii,1);
+            }
+               break;
+            }
+            vecM = new Vector(vecM1->Size());
+            for (int i=0; i<vecM->Size(); i++)
+            {
+               // (*vecM)(i) = ((*vecM1)(i)+(*vecM2)(i))/2.0;
+               // (*vecM)(i) = (*vecM1)(i);
+               (*vecM)(i) = 1.0/((1./(*vecM1)(i)+1./(*vecM2)(i))/2.0);
+            }   
+         }
+         // GetDiffusionEdgeMatrix(edge,fes,*vecG,*Q,irs,*EdgeGrad[edge],edge_orient[edge]);
+         // GetMassEdgeMatrix(edge,fes,*vecM,*Q,irs,*EdgeMass[edge],edge_orient[edge]);
+         Get1DMatrices(fes,edge, edge_orient[edge],
+                       *vecG, *vecM,*Q,irs,*EdgeGrad[edge],*EdgeMass[edge]);
+         emarker[edge] = 1;
+      }
+   }
+
+   
+   for (int iel=0; iel<nrelems; iel++)
+   {
+      Array<int> edges, cor;
+      pmesh->GetElementEdges(iel,edges,cor);
+
+      for (int ii = 0; ii<dim; ii++) 
+      {   
+         int i = eidx[ii];
+         int k = edges[i];
+         const FiniteElement *fe = fes->GetEdgeElement(k);
+         int ndof = fe->GetDof();
+         DenseMatrix Grad(ndof); Grad = *EdgeGrad[k];
+         DenseMatrix Mass(ndof); Mass = *EdgeMass[k];
+
+         Array<int> vert;
+         pmesh->GetEdgeVertices(k,vert);
+         for (int i = 0; i<2; i++)
+         {
+            int vertex = vert[i];
+            Array<int> vertdofs;
+            fes->GetVertexDofs(vertex,vertdofs);
+            // double count = edge_counts(vertdofs[0],ii); 
+
+            // Grad(i,i) *= count;
+            // Mass(i,i) *= count;
+            Grad(i,i) *= 2.;
+            Mass(i,i) *= 2.;
+         }
+         const Array<int> &dmap =
+         dynamic_cast<const TensorBasisElement&>(*fe).GetDofMap();
+         SparseMatrix * P = new SparseMatrix(dmap.Size());
+         for (int j = 0; j<dmap.Size(); j++)
+         {
+            P->Set(dmap[j],j, 1.0);
+         }
+         P->Finalize();
+         // Map from MFEM ordering to TensorProduct Ordering
+         AssembledEdgeGrad[k] = RAP(Grad,*P);
+         AssembledEdgeMass[k] = RAP(Mass,*P);
+         delete P;
+      }
+   }
+
+   Array<Array<int> * > tmap;
+   TensorProductEssentialDofsMaps(ess_tdof_list, fes, tmap, dofmap);
+
+
+   Array<DenseMatrix * > G(nredges);
+   Array<DenseMatrix * > M(nredges);
+
+   for (int ie = 0; ie<nredges; ie++)
+   {
+      if (!emarker[ie]) continue;
+      if (!AssembledEdgeMass[ie])
+      {
+         cout << "ie = " << ie << endl;
+         MFEM_ABORT("Memory allocation incosistency 2");
+      }
+      const FiniteElement * fe = fes->GetEdgeElement(ie);
+      int n = fe->GetDof() - tmap[ie]->Size();
+      G[ie] = new DenseMatrix(n);
+      M[ie] = new DenseMatrix(n);
+      const Array<int> &dmap =
+      dynamic_cast<const TensorBasisElement&>(*fe).GetDofMap();
+      // modify tmap to use tensor product index;
+      Array<int> dmapt(dmap.Size());
+      for (int i = 0; i<dmap.Size(); i++)
+      {
+         dmapt[dmap[i]] = i;
+      }
+
+      // Eliminate indices corresponding to tmap from matrices Grad1D_A and Mass1D_A
+      // construct Map;
+      Array<int> tmap_marker(fe->GetDof());
+      tmap_marker = 0;
+      for (int i = 0; i<tmap[ie]->Size(); i++)
+      {
+         int j = (*tmap[ie])[i];
+         tmap_marker[j] = 1;
+      }
+      Array<int> dof_list;
+      for (int i =0; i<fe->GetDof(); i++)
+      {
+         if (tmap_marker[i]) continue;
+         dof_list.Append(dmapt[i]);
+      }
+      dof_list.Sort();
+      for (int i=0; i<dof_list.Size(); i++)
+      {
+         int iii = (edge_orient[ie] == 1) ? i : n - i - 1; 
+         int ii = dof_list[i];
+         for (int j=0; j<dof_list.Size(); j++)
+         {
+            int jjj = (edge_orient[ie] == 1) ? j : n - j - 1; 
+            int jj = dof_list[j];
+            (*G[ie])(iii,jjj) = (*AssembledEdgeGrad[ie])(ii,jj);
+            (*M[ie])(iii,jjj) = (*AssembledEdgeMass[ie])(ii,jj);
+         }
+      }
+   }
+
+   elem_inv.SetSize(nrelems);
+   for (int iel = 0; iel<nrelems; iel++)
+   {
+      elem_inv[iel] = nullptr;
+      Array<int> edges, cor;
+      pmesh->GetElementEdges(iel,edges,cor);
+      Array<DenseMatrix *> Gv(dim);
+      Array<DenseMatrix *> Mv(dim);
+      bool msize = true;
+      for (int d = 0; d<dim; d++)
+      {
+         Gv[dim - d - 1] = G[edges[eidx[d]]];
+         Mv[dim - d - 1] = M[edges[eidx[d]]];
+         if (Mv[dim-d-1]->Size() == 0) msize = false;
+      }
+      if (msize) elem_inv[iel] = new FDSolver(Gv,Mv);
+   }
+}
+
+
+void ElementSmoother::Mult(const Vector &r, Vector &z) const
+{
+   z = r;
+   z.SetSubVectorComplement(ess_tdof_list,0.0);
+   Vector rnew(fes->GetVSize());
+   Vector znew(fes->GetVSize()); 
+   Vector ztemp(fes->GetTrueVSize()); 
+   ztemp = 0.0;
+   znew = 0.0;
+
+   // const SparseMatrix * R = fes->GetRestrictionMatrix();
+   if (Pr)
+   {
+      Pr->Mult(r,rnew);
+   }
+   else
+   {
+      rnew = r;
+   }
+   for (int iel=0; iel<nrelems; iel++)
+   {
+      if (!elem_inv[iel]) continue;
+      int n = dofmap[iel]->Size();
+
+      Vector rloc(n);
+      rnew.GetSubVector(*dofmap[iel],rloc);
+      // pre-scale
+      for (int i = 0; i<n; i++)
+      {
+         int j = (*dofmap[iel])[i];
+         rloc[i] *= ovlp_count[j]; 
+      }
+      Vector zloc(n);
+      elem_inv[iel]->Mult(rloc,zloc);
+      // post-scale
+      for (int i = 0; i<n; i++)
+      {
+         int j = (*dofmap[iel])[i];
+         zloc[i] *= ovlp_count[j]; 
+      }
+
+      znew.AddElementVector(*dofmap[iel],zloc);
+   }
+   if (Pr)
+   {
+      Pr->MultTranspose(znew,ztemp);
+   }
+   else
+   {
+      ztemp = znew;
+   }
+   // R->Mult(znew,ztemp);
+   z += ztemp;
+}

examples/fd/element-smoother.hpp

@@ -0,0 +1,30 @@
+#pragma once
+#include "smoother-util.hpp"
+
+using namespace std;
+using namespace mfem;
+
+class ElementSmoother: public Solver
+{
+private:
+   int num_procs, myid;
+   MPI_Comm comm;
+   int nrelems;
+   int dim;
+   ParFiniteElementSpace * fes = nullptr;
+   const Operator * Pr = nullptr;
+   Coefficient * cf = nullptr;
+   Array<int> eidx; // edge local index
+   Array<int> edge_orient; // orientation of edges wrt the element
+   Array<FDSolver *> elem_inv;
+   Array<int> ess_tdof_list;
+   Vector ovlp_count;
+   Array<Array<int> * > dofmap;
+   Array<ElementTPFunctionCoefficient *> tpcf;
+public:
+   ElementSmoother(ParFiniteElementSpace * fes_, Array<int> ess_bdr, Coefficient * cf_=nullptr);
+   virtual void SetOperator(const Operator &op) { }
+   virtual void Mult(const Vector &r, Vector &z) const;
+   virtual void MultTranspose(const Vector &r, Vector &z) const { Mult(r,z); }
+   virtual ~ElementSmoother(){};
+};

examples/fd/ex_amr_diffusionp.cpp

@@ -0,0 +1,289 @@
+//                                MFEM Example 26
+//
+
+#include "mfem.hpp"
+#include "exact_sol.hpp"
+#include <fstream>
+#include <iostream>
+#include <random>
+#include "element-smoother.hpp"
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 0. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 1. Parse command-line options.
+   const char *mesh_file = "l-shape-benchmark.mesh";
+   int init_geometric_refinements = 0;
+   int pinit_geometric_refinements = 0;
+   int geometric_refinements = 0;
+   int order_refinements = 2;
+   const char *device_config = "cpu";
+   bool visualization = true;
+   int order = 1;
+   int solver = 0;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order", "Finite element order.");
+   args.AddOption(&solver, "-solver", "--solver", "Solver: 0:MG-Cheb-Jac, 1: MG-Cheb-ElemSmoother");
+   args.AddOption(&init_geometric_refinements, "-ref", "--initial-geometric-refinements",
+                  "Number of serial geometric refinements defining the coarse mesh.");
+   args.AddOption(&pinit_geometric_refinements, "-pref", "--initial-geometric-refinements",
+                  "Number of parallel geometric refinements defining the coarse mesh.");                  
+   args.AddOption(&geometric_refinements, "-gr", "--geometric-refinements",
+                  "Number of geometric refinements done prior to order refinements.");
+   args.AddOption(&order_refinements, "-or", "--order-refinements",
+                  "Number of order refinements. Finest level in the hierarchy has order 2^{or}.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+
+   for (int l = 0; l < init_geometric_refinements; l++)
+   {
+      mesh->UniformRefinement();
+   }
+   mesh->EnsureNCMesh();
+   ParMesh * pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   mesh->Clear();
+   {
+      for (int l = 0; l < pinit_geometric_refinements; l++)
+      {
+         pmesh->UniformRefinement();
+      }
+   }
+
+   FiniteElementCollection *fec = new H1_FECollection(order, dim);
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+
+   Array<int> ess_bdr;
+   if(pmesh->bdr_attributes.Size())
+   {   
+      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+   }
+
+
+   ParGridFunction x(fespace);
+   FunctionCoefficient ex_coeff(lshape_exact);
+   x.ProjectCoefficient(ex_coeff);
+
+   // ------------------------------------------------- 
+   // Bilinear and linear forms
+   // ------------------------------------------------- 
+   ConstantCoefficient cf(1.0);
+   ParBilinearForm a(fespace);
+   a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+   DiffusionIntegrator * aa = new DiffusionIntegrator(cf);
+   // int order1 = fespace->GetElementOrder(0);
+   IntegrationRule *irs = TensorIntegrationRule(*fespace,order); 
+   aa->SetIntegrationRule(*irs);
+   a.AddDomainIntegrator(aa);
+
+   ParLinearForm b(fespace);
+   FunctionCoefficient rhscf(lshape_rhs);
+   b.AddDomainIntegrator(new DomainLFIntegrator(rhscf));
+   // ------------------------------------------------- 
+
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+
+   socketstream sout;
+   if (visualization)
+   {
+      sout.open(vishost, visport);
+      if (!sout)
+      {
+         if (myid == 0)
+         {
+            cout << "Unable to connect to GLVis server at "
+                 << vishost << ':' << visport << endl;
+            cout << "GLVis visualization disabled.\n";
+         }
+         visualization = false;
+      }
+
+      sout.precision(8);
+
+      sout << "parallel " << num_procs << " " << myid << "\n";
+      sout << "solution\n" << *pmesh << x << flush;
+   }
+
+   L2_FECollection flux_fec(order, dim);
+   ParFiniteElementSpace flux_fes(pmesh, &flux_fec, dim);
+   FiniteElementCollection *smooth_flux_fec = NULL;
+   ParFiniteElementSpace *smooth_flux_fes = NULL;
+   smooth_flux_fec = new RT_FECollection(order-1, dim);
+   smooth_flux_fes = new ParFiniteElementSpace(pmesh, smooth_flux_fec, 1);
+   L2ZienkiewiczZhuEstimator estimator(*aa, x, flux_fes, *smooth_flux_fes);
+   ThresholdRefiner refiner(estimator);
+   refiner.SetTotalErrorFraction(0.7);
+   refiner.SetNCLimit(1);
+   StopWatch chrono;
+   Array<double> ts0, ts1, tsol;
+   int ref_amr = 20;   
+   Array<int> iter;
+   Array<int> dofs;
+
+   ostringstream file_name;
+   file_name << "lshape-amr_" << order << ".csv";
+   ofstream conv(file_name.str().c_str());
+   conv << "DOFs " << ", " << "it-Cheb-Jac" << ", " << "it-ChebElemSmoother" << endl;
+
+   for (int it = 0; it < ref_amr ; it++)
+   {
+      HYPRE_BigInt global_dofs = fespace->GlobalTrueVSize();
+      if (myid == 0)
+      {
+         cout << "\nAMR iteration " << it << endl;
+         cout << "Number of unknowns: " << global_dofs << endl;
+      }
+
+      Array<int> ess_tdof_list;
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+      a.Assemble();
+      b.Assemble();
+
+      OperatorPtr A;
+      Vector B, X;
+      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+
+      chrono.Clear();
+      chrono.Start();
+      chrono.Stop();
+      ts0.Append(chrono.RealTime());
+      chrono.Clear();
+      chrono.Start();
+      Solver * prec1 = nullptr;
+      Solver * prec2 = nullptr;
+      Solver * S = nullptr;
+      // if (solver)
+      // {
+         S = new ElementSmoother(fespace,ess_bdr, &cf); 
+         prec1 = new OperatorChebyshevSmoother(*A, *S, 4, MPI_COMM_WORLD,7);
+      // }
+      // else
+      // {
+         Vector diag(fespace->GetTrueVSize());
+         a.AssembleDiagonal(diag);
+         prec2 = new OperatorChebyshevSmoother(*A, diag,ess_tdof_list, 4, MPI_COMM_WORLD,10);
+      // }
+
+      chrono.Stop();
+      ts1.Append(chrono.RealTime());
+
+
+
+      int print_level = 3;
+      int max_iter = 2000;
+      double rtol = 1e-8;
+
+      CGSolver pcg(MPI_COMM_WORLD);
+      pcg.SetPrintLevel(print_level);
+      pcg.SetMaxIter(max_iter);
+      pcg.SetRelTol(rtol);
+      pcg.SetOperator(*A);
+      pcg.SetPreconditioner(*prec1);
+
+
+      // chrono.Clear();
+      // chrono.Start();
+      Vector Y = X;
+      pcg.Mult(B,Y);
+      int iter1 = pcg.GetNumIterations();
+
+
+      pcg.SetPreconditioner(*prec2);
+      pcg.Mult(B,X);
+      int iter2 = pcg.GetNumIterations();
+
+      // chrono.Stop();
+      // tsol.Append(chrono.RealTime());
+      // iter.Append(pcg.GetNumIterations());
+      // dofs.Append(global_dofs);
+
+      delete S; 
+      delete prec1;
+      delete prec2;
+
+      conv << global_dofs << ", " << iter1 << ", " << iter2 << endl;
+
+
+      a.RecoverFEMSolution(X,b,x);
+
+      if (visualization)
+      {
+         sout << "parallel " << num_procs << " " << myid << "\n";
+         sout << "solution\n" << *pmesh << x << flush;
+      }
+      refiner.Apply(*pmesh);
+      if (refiner.Stop())
+      {
+         if (myid == 0)
+         {
+            cout << "Stopping criterion satisfied. Stop." << endl;
+         }
+         break;
+      }
+      fespace->Update();
+      x.Update();
+      x.ProjectCoefficient(ex_coeff);
+
+      a.Update();
+      b.Update();
+   }
+
+
+
+   if (myid==0)
+   {
+      cout << "ts0 total  = " << ts0.Sum() << endl;
+      cout << "ts1 total  = " << ts1.Sum() << endl;
+      cout << "tsol total = " << tsol.Sum() << endl;
+   }
+
+   if (myid == 0)
+   {
+      cout << "num iterations = "; iter.Print(cout, iter.Size());
+      cout << "dofs = "; dofs.Print(cout, dofs.Size());
+   }
+
+   delete smooth_flux_fes;
+   delete smooth_flux_fec;
+   delete pmesh;
+
+   // 13. Free the used memory.
+   MPI_Finalize();
+   return 0;
+}
+
+
+

examples/fd/ex_diffusionp.cpp

@@ -0,0 +1,670 @@
+//                                MFEM Example 26
+//
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include <random>
+#include "element-smoother.hpp"
+using namespace std;
+using namespace mfem;
+
+class DiffusionMultigrid : public GeometricMultigrid
+{
+private:
+   Coefficient * cf = nullptr;
+   int smoother_kind = 0;
+   // 0: Jacobi, 1:Chebychev, 2: element-smoother(matrix-free)
+   HypreBoomerAMG* amg;
+
+public:
+   // Constructs a diffusion multigrid for the ParFiniteElementSpaceHierarchy
+   // and the array of essential boundaries
+   DiffusionMultigrid(ParFiniteElementSpaceHierarchy& fespaces,
+                      Array<int>& ess_bdr, Coefficient * cf_,int smoother_ = 0)
+      : GeometricMultigrid(fespaces), cf(cf_), smoother_kind(smoother_)
+   {
+      ConstructCoarseOperatorAndSolver(fespaces.GetFESpaceAtLevel(0), ess_bdr,cf);
+
+      for (int level = 1; level < fespaces.GetNumLevels(); ++level)
+      {
+         ConstructOperatorAndSmoother(fespaces.GetFESpaceAtLevel(level), ess_bdr,cf);
+      }
+   }
+
+   virtual ~DiffusionMultigrid()
+   {
+      delete amg;
+   }
+
+private:
+   void ConstructBilinearForm(ParFiniteElementSpace& fespace, Array<int>& ess_bdr,
+                              bool partial_assembly, Coefficient * cf)
+   {
+      ParBilinearForm* form = new ParBilinearForm(&fespace);
+      if (partial_assembly)
+      {
+         form->SetAssemblyLevel(AssemblyLevel::PARTIAL);
+      }
+      form->AddDomainIntegrator(new DiffusionIntegrator(*cf));
+      form->Assemble();
+      bfs.Append(form);
+
+      essentialTrueDofs.Append(new Array<int>());
+      fespace.GetEssentialTrueDofs(ess_bdr, *essentialTrueDofs.Last());
+   }
+
+   void ConstructCoarseOperatorAndSolver(ParFiniteElementSpace& coarse_fespace,
+                                         Array<int>& ess_bdr, 
+                                         Coefficient * cf)
+   {
+      ConstructBilinearForm(coarse_fespace, ess_bdr, false, cf);
+
+      HypreParMatrix* hypreCoarseMat = new HypreParMatrix();
+      bfs.Last()->FormSystemMatrix(*essentialTrueDofs.Last(), *hypreCoarseMat);
+
+      amg = new HypreBoomerAMG(*hypreCoarseMat);
+      amg->SetPrintLevel(-1);
+
+      CGSolver* pcg = new CGSolver(MPI_COMM_WORLD);
+      pcg->SetPrintLevel(-1);
+      pcg->SetMaxIter(10);
+      pcg->SetRelTol(sqrt(1e-8));
+      pcg->SetAbsTol(0.0);
+      pcg->SetOperator(*hypreCoarseMat);
+      pcg->SetPreconditioner(*amg);
+
+      AddLevel(hypreCoarseMat, pcg, true, true);
+   }
+
+   void ConstructOperatorAndSmoother(ParFiniteElementSpace& fespace,
+                                     Array<int>& ess_bdr, Coefficient *cf)
+   {
+      ConstructBilinearForm(fespace, ess_bdr, true, cf);
+
+      OperatorPtr opr;
+      opr.SetType(Operator::ANY_TYPE);
+      bfs.Last()->FormSystemMatrix(*essentialTrueDofs.Last(), opr);
+      opr.SetOperatorOwner(false);
+      Solver * smoother = nullptr;
+      Vector diag;
+      if (smoother_kind < 2 )
+      {
+         diag.SetSize(fespace.GetTrueVSize());
+         bfs.Last()->AssembleDiagonal(diag);
+      }
+
+      switch (smoother_kind)
+      {
+      case 0:
+         smoother = new OperatorJacobiSmoother(diag,*essentialTrueDofs.Last(),0.6667);
+         break;
+      case 1:
+         smoother = new OperatorChebyshevSmoother(opr.Ptr(), diag,
+                                       *essentialTrueDofs.Last(), 5);
+         break;   
+      case 2:
+         smoother = new ElementSmoother(&fespace,ess_bdr,cf);
+         break;      
+      case 3:
+         {
+            ElementSmoother * sm = new ElementSmoother(&fespace,ess_bdr,cf);
+            smoother = new OperatorChebyshevSmoother(*opr,*sm,5,fespace.GetComm());
+            break;      
+         }
+      default:
+         MFEM_ABORT("Wrong smoother choice");
+         break;
+      }
+      AddLevel(opr.Ptr(), smoother, true, true);
+   }
+};
+
+int dim;
+int  exact   = 0;
+bool tpcoeff = true;
+
+double f_exact(const Vector & x);
+double u_exact(const Vector & x);
+void usol(const Vector & x, double &u, Vector & Grad, double & d2u);
+
+double DiffusionCoeff(const Vector & x);
+double TPDiffusionCoeff(const Vector & x, int coord);
+void DiffusionCoeffGrad(const Vector & x, Vector & Grad);
+
+
+int main(int argc, char *argv[])
+{
+   // 0. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../data/inline-quad.mesh";
+   int init_geometric_refinements = 0;
+   int pinit_geometric_refinements = 0;
+   int geometric_refinements = 0;
+   int order_refinements = 2;
+   const char *device_config = "cpu";
+   bool visualization = true;
+   int order = 1;
+   double skew_factor = 0.0;
+   double scale_factor = 1.0;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order", "Finite element order.");
+   args.AddOption(&init_geometric_refinements, "-ref", "--initial-geometric-refinements",
+                  "Number of serial geometric refinements defining the coarse mesh.");
+   args.AddOption(&pinit_geometric_refinements, "-pref", "--initial-geometric-refinements",
+                  "Number of parallel geometric refinements defining the coarse mesh.");                  
+   args.AddOption(&geometric_refinements, "-gr", "--geometric-refinements",
+                  "Number of geometric refinements done prior to order refinements.");
+   args.AddOption(&order_refinements, "-or", "--order-refinements",
+                  "Number of order refinements. Finest level in the hierarchy has order 2^{or}.");
+   args.AddOption(&tpcoeff, "-tpcoeff", "--tp-coefficient", "-no-tpcoeff",
+               "--no-tp-coefficient", "Tensor product diffusion coefficient or not");
+   args.AddOption(&exact, "-exact", "--exact", "Exact Solution flag: 0: unknown");   
+   args.AddOption(&skew_factor, "-c", "--skew_factor", "Skew_factor");   
+   args.AddOption(&scale_factor, "-s", "--scale_factor", "Scale_factor");   
+   
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // 2. Enable hardware devices such as GPUs, and programming models such as
+   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
+   Device device(device_config);
+   if (myid == 0) { device.Print(); }
+
+
+   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
+   //    the same code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   // int nx = pow(2,init_geometric_refinements);
+   // int ny = pow(2,init_geometric_refinements);
+   // Mesh *mesh = new Mesh(1,1,1,mfem::Element::HEXAHEDRON,true,1.0,2.0,3.0,false);
+   // Mesh *mesh = new Mesh(1,1,mfem::Element::QUADRILATERAL,true,1.0,1.0,false);
+   // Mesh *mesh = new Mesh(1,4.0);
+   // move nodes
+   dim = mesh->Dimension();
+   mesh->EnsureNodes();
+   
+   mesh->SetCurvature(3);
+   GridFunction * nodes = mesh->GetNodes();
+   // *nodes +=1.0;
+   // *nodes *=0.5;
+   double c = skew_factor;
+   double s = scale_factor;
+   if (dim == 2)
+   {
+      for (int i=0; i<nodes->Size()/2; i++)
+      {
+      // double temp = (*nodes)(2*i);
+      // (*nodes)(2*i) += (*nodes)(2*i)*(*nodes)(2*i) + c*pow((*nodes)(2*i+1),2);
+      // (*nodes)(2*i) +=  c*pow((*nodes)(2*i+1),2);
+         (*nodes)(2*i) +=  c*pow((*nodes)(2*i+1),2);
+      // (*nodes)(2*i+1) = (*nodes)(2*i+1)*(*nodes)(2*i+1) + c*pow(temp,2);
+      }
+   }
+   else
+   {
+      for (int i=0; i<nodes->Size()/3; i++)
+      {
+         // (*nodes)(3*i) +=  c*pow((*nodes)(3*i+1),2);
+         // (*nodes)(3*i+1) +=  c*pow((*nodes)(3*i+2),3);
+         (*nodes)(3*i+2) +=  c*pow((*nodes)(3*i),2);
+      }
+   }
+
+   for (int i=0; i<nodes->Size(); i++)
+   {
+      (*nodes)(i) *=  s;
+   }
+
+   dim = mesh->Dimension();
+   // mesh->EnsureNCMesh();
+
+   // 4. Refine the mesh to increase the resolution and order
+   {
+      for (int l = 0; l < init_geometric_refinements; l++)
+      {
+         mesh->UniformRefinement();
+      }
+   }
+
+   ParMesh * pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   mesh->Clear();
+   {
+      for (int l = 0; l < pinit_geometric_refinements; l++)
+      {
+         pmesh->UniformRefinement();
+      }
+   }
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream mesh_sock(vishost, visport);
+      mesh_sock << "parallel " << num_procs << " " << myid << "\n";
+      mesh_sock.precision(8);
+      mesh_sock << "mesh\n" << *pmesh << flush;
+   }
+
+   FiniteElementCollection *fec = new H1_FECollection(order, dim);
+   ParFiniteElementSpace *coarse_fespace = new ParFiniteElementSpace(pmesh, fec);
+   ParFiniteElementSpaceHierarchy fespaces(pmesh, coarse_fespace, true, true);
+
+
+   Coefficient * cf = nullptr;
+   if (exact)
+   {
+      cf = new FunctionCoefficient(DiffusionCoeff);
+   }
+   else
+   {
+      cf = new ConstantCoefficient(1.0);
+   }
+
+   Array<FiniteElementCollection*> collections;
+   collections.Append(fec);
+   for (int level = 0; level < geometric_refinements; ++level)
+   {
+      fespaces.AddUniformlyRefinedLevel();
+   }
+   for (int level = 0; level < order_refinements; ++level)
+   {
+      // order++;
+      order *=2;
+      // collections.Append(new H1_FECollection(std::pow(2, level+1), dim));
+      collections.Append(new H1_FECollection(order, dim));
+      fespaces.AddOrderRefinedLevel(collections.Last());
+   }
+
+   HYPRE_Int size = fespaces.GetFinestFESpace().GlobalTrueVSize();
+
+   if (myid == 0)
+   {
+      cout << "Number of finite element unknowns: " << size << endl;
+      cout << "Order = " << order << endl;
+   }
+
+   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
+   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
+   //    the basis functions in the finite element fespace.
+   FunctionCoefficient f(f_exact);
+   ConstantCoefficient one(1.0);
+     
+   
+   Array<int> ess_bdr;
+   if(pmesh->bdr_attributes.Size())
+   {   
+      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
+      ess_bdr = 1;
+   }
+   ParGridFunction x(&fespaces.GetFinestFESpace());
+   // GridFunction gf_coeff(&fespaces.GetFinestFESpace());
+   ParMesh * ref_mesh = fespaces.GetFinestFESpace().GetParMesh();
+   L2_FECollection * l2fec = new L2_FECollection(order,dim);
+   ParFiniteElementSpace * l2fes = new ParFiniteElementSpace(ref_mesh,l2fec);
+   ParGridFunction gf_coeff(l2fes);
+   // gf_coeff.ProjectCoefficient(*cf);
+   gf_coeff.ProjectDiscCoefficient(*cf,mfem::GridFunction::AvgType::ARITHMETIC);
+   
+   int print_level = 3;
+   int max_iter = 2000;
+   double rtol = 1e-8;
+   StopWatch chrono;
+   // for (int i = 0; i<=6; i++)
+   for (int i = 0; i<=6; i++)
+   {
+      OperatorPtr A;
+      Vector B, X;
+      Solver * prec = nullptr;
+      CGSolver pcg(MPI_COMM_WORLD);
+      pcg.SetPrintLevel(print_level);
+      pcg.SetMaxIter(max_iter);
+      pcg.SetRelTol(rtol);
+      // i=1; Chebychev-Jacobi-MG
+      // i=2; Chebychev-Element-MG
+      // i=3; Chebychev-Jacobi-Smoother
+      // i=4; Element-Smoother 
+      // i=5; Chebychev-Element-Smoother 
+      ParLinearForm *b = new ParLinearForm(&fespaces.GetFinestFESpace());
+      if (exact)
+      {
+         b->AddDomainIntegrator(new DomainLFIntegrator(f));
+      }
+      else
+      {
+         b->AddDomainIntegrator(new DomainLFIntegrator(one));
+      }
+      b->Assemble();
+      FunctionCoefficient u_ex(u_exact);
+      x = 0.0;
+      if (exact) x.ProjectCoefficient(u_ex);
+
+      if (i<4)
+      {
+         prec = new DiffusionMultigrid(fespaces, ess_bdr, cf,i);
+         dynamic_cast<DiffusionMultigrid *>(prec)->
+                  SetCycleType(Multigrid::CycleType::VCYCLE, 1, 1);
+         dynamic_cast<DiffusionMultigrid *>(prec)->
+                  FormFineLinearSystem(x, *b, A, X, B);
+         if (i == 0)
+         {
+            if (myid == 0)
+               cout << "\nJacobi-MG " << endl;
+         }         
+         else if (i == 1)
+         {
+            if (myid == 0)
+               cout << "\nJacobi-Chebychev-MG " << endl;
+         }
+         else if (i == 2)
+         {
+            if (myid == 0)
+               cout << "\nElement-Smoother-MG " << endl;
+         }
+         else 
+         {
+            if (myid == 0)
+               cout << "\nElement-Chebychev-MG " << endl;
+         }
+         pcg.SetOperator(*A);
+         if (prec) { pcg.SetPreconditioner(*prec); }
+         chrono.Clear();
+         chrono.Start();         
+         pcg.Mult(B,X);
+         chrono.Stop();      
+         if (myid == 0)
+            cout<< "PCG::mult time = " << chrono.RealTime() << endl;       
+        // Recover the solution as a finite element grid function.
+        dynamic_cast<DiffusionMultigrid *>(prec)->RecoverFineFEMSolution(X, *b, x);
+        delete prec;
+      }
+      else
+      {
+         ParBilinearForm a(&fespaces.GetFinestFESpace());
+         a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+         DiffusionIntegrator * aa = new DiffusionIntegrator(*cf);
+         int order1 = fespaces.GetFinestFESpace().GetOrder(0);
+         IntegrationRule *irs = TensorIntegrationRule(fespaces.GetFinestFESpace(),order1); 
+         aa->SetIntegrationRule(*irs);
+         a.AddDomainIntegrator(aa);
+         a.Assemble();
+         
+         Array<int> ess_tdof_list;
+         fespaces.GetFinestFESpace().GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+         a.FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
+         if (i==4)
+         {
+            Vector diag(fespaces.GetFinestFESpace().GetTrueVSize());
+            a.AssembleDiagonal(diag);
+            prec = new OperatorChebyshevSmoother(A.Ptr(), diag,ess_tdof_list, 1, MPI_COMM_WORLD);
+            if (myid == 0)
+               cout << "\nJacobi-Chebychev " << endl;
+         }
+         else if (i==5)
+         {
+            prec = new ElementSmoother(&fespaces.GetFinestFESpace(),ess_bdr, cf); 
+            if (myid == 0)
+               cout << "\nElementSmoother " << endl;
+         }
+         else
+         {
+            ElementSmoother *S = new ElementSmoother(&fespaces.GetFinestFESpace(),ess_bdr, cf); 
+            prec = new OperatorChebyshevSmoother(*A, *S, 1, MPI_COMM_WORLD);
+            if (myid == 0)
+               cout << "\nElement-Chebychev " << endl;
+         }
+         pcg.SetOperator(*A);
+         if (prec) { pcg.SetPreconditioner(*prec); }
+         chrono.Clear();
+         chrono.Start();         
+         pcg.Mult(B,X);
+         chrono.Stop();
+         if (myid == 0)
+            cout<< "PCG::mult time = " << chrono.RealTime() << endl;        
+         delete prec;
+
+         a.RecoverFEMSolution(X,*b,x);
+      }
+      delete b;
+   }
+
+   // 12. Send the solution by socket to a GLVis server.
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *fespaces.GetFinestFESpace().GetMesh() << x <<
+               flush;
+      socketstream coeff_sock(vishost, visport);
+      coeff_sock << "parallel " << num_procs << " " << myid << "\n";
+      coeff_sock.precision(8);
+      coeff_sock << "solution\n" << *fespaces.GetFinestFESpace().GetMesh() << gf_coeff <<
+               flush;         
+   }
+
+   // 13. Free the used memory.
+   for (int level = 0; level < collections.Size(); ++level)
+   {
+      delete collections[level];
+   }
+
+   MPI_Finalize();
+   return 0;
+}
+
+
+double f_exact(const Vector & x)
+{
+   // -div (f * grad(u)) = (f * gradu(0))_x + (f*gradu(1))_y + + (f*gradu(2))_z
+   // = f_x*gradu(0) + f * gradu(0)_x + f_y * gradu(1) + f* gradu(1)_y + f_z * gradu(2) + f* gradu(2)_z
+   // = f_x*gradu(0) + f_y * gradu(1) + f_z * gradu(2) + f*d2u
+   double u;
+   double d2u;
+   Vector gradu;
+   usol(x,u,gradu,d2u);
+   Vector gradf;
+   DiffusionCoeffGrad(x,gradf);
+   double f = DiffusionCoeff(x);
+   double val = gradf * gradu + f*d2u;
+   return -val;
+}
+
+double u_exact(const Vector & x)
+{
+   double u;
+   Vector gradu;
+   double d2u;
+   usol(x,u,gradu,d2u);
+   return u;
+}
+
+void usol(const Vector & x, double &u, Vector & Grad, double & d2u)
+{
+   Grad.SetSize(dim);
+   if (exact == 1)
+   {
+      Vector alpha(dim); alpha = 5.0;
+      // Vector alpha(dim); alpha = 0.5;
+      double s = alpha * x; // dot product
+      u = sin(M_PI*s);
+      d2u = 0.0;
+      for (int i = 0; i<dim; i++)
+      {
+         Grad[i] = alpha(i) * M_PI * cos(M_PI*s);
+         d2u += alpha(i)*alpha(i); 
+      }
+      d2u = - M_PI*M_PI * d2u * u;
+   }
+   else if (exact == 2)
+   {
+      double c_0 = 1.2;
+      double k_0 = 3.0;
+      double c_1 = 2.3;
+      double k_1 = 5.0;
+      double c_2 = 1.3;
+      double k_2 = 1.0;
+
+      double alpha = c_0 + k_0 * x(0);
+      double beta  = c_1 + k_1 * x(1);
+      double gamma = 1.0;
+      
+      if (dim == 2)
+      {
+         u = sin(M_PI * alpha) *  sin(M_PI * beta);
+         Grad[0] = M_PI*k_0 * cos(alpha) * sin(M_PI*beta);
+         Grad[1] = M_PI*k_1 * cos(beta) * sin(M_PI*alpha);
+      }
+      else if (dim == 3)
+      {
+         gamma  = c_2 + k_2 * x(2);
+         u = sin(M_PI * alpha) *  sin(M_PI * beta) * sin(M_PI*gamma);
+         Grad[0] = M_PI*k_0 * cos(alpha) * sin(M_PI*beta) * sin(M_PI*gamma);
+         Grad[1] = M_PI*k_1 * cos(beta) * sin(M_PI*alpha) * sin(M_PI*gamma);
+         Grad[2] = M_PI*k_2 * cos(gamma) * sin(M_PI*alpha) * sin(M_PI*beta);
+      }
+
+      double u_xx = - M_PI * M_PI * k_0 * k_0 * u;
+      double u_yy = - M_PI * M_PI * k_1 * k_1 * u;
+      double u_zz = - M_PI * M_PI * k_2 * k_2 * u;
+      d2u = u_xx + u_yy; 
+      if (dim == 3 ) d2u += u_zz;
+   }
+}
+
+double TPDiffusionCoeff(const Vector & x, int coord)
+{
+   double val;
+   switch (coord)
+   {
+      case  0: val = 4.+3.*x(0); break;
+      case  1: val = 0.5+7.*x(1)*x(1); break;
+      case  2: val = (0.1+2.*x(2)); break;
+      default: 
+         val = (4.+3.*x(0))*(0.5+7.*x(1)*x(1)); 
+         if (dim == 3 ) val *= (0.1+2.*x(2));
+      break;
+      // case  0: val = x(0); break;
+      // case  1: val = 1.0; break;
+      // case  2: val = 1.0; break;
+      // default: val = x(0); break;
+      // case  0: val = 3.0; break;
+      // case  1: val = 2.0; break;
+      // case  2: val = 1.0; break;
+      // default: val = 6.0; break;
+   }
+   return val;
+   // return 2.0;
+
+}
+
+double DiffusionCoeff(const Vector & x)
+{
+   double val;
+   if (tpcoeff)
+   {
+      val = (4.+3.*x(0))*(0.5+7.*x(1)*x(1));
+      if (dim == 3) val *= (0.1+2.*x(2));
+   }
+   else
+   {
+      // val = 2.0+cos(x.Sum());
+      Vector cf(dim);
+      // cf(0) = 0.1; cf(1) = 3.; 
+      cf(0) = 1.0; cf(1) = 2.0; 
+      // if (dim == 3) cf(2) = -7.8;
+      if (dim == 3) cf(2) = +1.8;
+      // double dd = x * cf + 1.5* x(1)*x(1);
+      double dd = x * cf;
+      // // double dd = x * cf;
+      // // val = exp(cos(dd));
+      val = exp(dd);
+
+      // Vector alpha(dim); alpha = 5.0;
+      // double s = alpha * x; // dot product
+      // val = 2.0+sin(M_PI*s);
+
+   }   
+   return val;
+}
+
+
+void DiffusionCoeffGrad(const Vector & x, Vector & Grad)
+{
+   Grad.SetSize(dim);
+   if (tpcoeff)
+   {
+      if (dim == 2)
+      {
+         Grad[0] = 3.* (0.5+7.*x(1)*x(1));
+         Grad[1] = 14.* x(1) * (4.+3.*x(0));
+      }
+      else
+      {
+         Grad[0] = 3.* (0.5+7.*x(1)*x(1))*(0.1+2.*x(2));
+         Grad[1] = 14.* x(1) * (4.+3.*x(0))*(0.1+2.*x(2));
+         Grad[2] = 2.*(4.+3.*x(0))*(0.5+7.*x(1)*x(1));
+      }
+   }
+   else
+   {
+      Vector cf(dim);
+      // cf(0) = 0.1; cf(1) = 3.; 
+      cf(0) = 1.0; cf(1) = 2.0; 
+      // if (dim == 3) cf(2) = -7.8;
+      if (dim == 3) cf(2) = 1.8;
+      // double dd = x * cf  + 1.5* x(1)*x(1);
+      double dd = x * cf;
+      Vector alpha(dim); alpha = 5.0;
+
+      // for (int d = 0; d<dim; d++)
+      // {
+      // //    // Grad[d] = -sin(x.Sum());
+      // //    // Grad[d] = -cf(d) * exp(cos(dd))*sin(dd);
+      // //    Grad[d] = cf(d) * exp(dd);
+      //    Grad[d] = alpha(d) * M_PI * cos(M_PI*s);
+      // }
+      if (dim == 2)
+      {
+         Grad[0] = (cf(0) )*exp(dd);
+         Grad[1] = (cf(1) + 3.0*x(1))*exp(dd);
+      }
+      else
+      {
+         // Grad[0] = (cf(0) + 1.5 * x(1))*exp(dd);
+         Grad[0] = cf(0)*exp(dd);
+         Grad[1] = cf(1)*exp(dd);
+         Grad[2] = cf(2)*exp(dd);
+      }
+   }
+}
+
+

examples/fd/exact_sol.cpp

@@ -0,0 +1,39 @@
+
+#include "exact_sol.hpp"
+
+double lshape_exact(const Vector & pt)
+{
+   double x = pt[0];
+   double y = pt[1];
+   double r = sqrt(x*x + y*y);
+   double alpha = 2. / 3.;
+
+   double theta = atan2(y, x);
+   if (y < 0) { theta += 2 * M_PI; }
+
+   return pow(r,alpha) * sin(alpha * theta);
+}
+
+void lshape_grad(const Vector & x, Vector & grad)
+{
+
+}
+
+double lshape_rhs(const Vector & x)
+{
+   return 0.0;
+}
+
+double wavefront_exact(const Vector & x)
+{
+   return 0.;
+}
+
+void wavefront_grad(const Vector & x, Vector & grad)
+{
+
+}
+double wavefront_rhs(const Vector & x)
+{
+   return 0.;
+}

examples/fd/exact_sol.hpp

@@ -0,0 +1,15 @@
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+double lshape_exact(const Vector & x);
+void lshape_grad(const Vector & x, Vector & grad);
+double lshape_rhs(const Vector & x);
+
+double wavefront_exact(const Vector & x);
+void wavefront_grad(const Vector & x, Vector & grad);
+double wavefront_rhs(const Vector & x);

examples/fd/l-shape-benchmark.mesh

@@ -1,5 +1,4 @@
 MFEM mesh v1.0
-
 #
 # MFEM Geometry Types (see mesh/geom.hpp):
 #
@@ -10,29 +9,36 @@ MFEM mesh v1.0
 # TETRAHEDRON = 4
 # CUBE        = 5
 # PRISM       = 6
-# PYRAMID     = 7
 #
 
 dimension
-3
+2
 
 elements
-1
-1 7 0 1 2 3 4
+3
+1 3 0 3 4 1
+1 3 3 6 7 4
+1 3 4 5 2 1
 
 boundary
-5
-1 3 3 2 1 0
-2 2 0 1 4
-3 2 1 2 4
-4 2 2 3 4
-5 2 3 0 4
+8
+1 1 0 1
+1 1 1 2
+1 1 2 5
+2 1 5 4
+2 1 4 7
+1 1 7 6
+1 1 6 3
+1 1 3 0
 
 vertices
-5
-3
-0 0 0
-1 0 0
-1 1 0
-0 1 0
-0 0 1
+8
+2
+-1 1
+0 1
+1 1
+-1 0
+0 0
+1 0
+-1 -1
+0 -1

examples/fd/makefile

@@ -9,33 +9,51 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
+
 # Use the MFEM build directory
 MFEM_DIR ?= ../..
 MFEM_BUILD_DIR ?= ../..
+SRC = $(.,)
 CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
 
-TESTS = dangling-aliases
-
 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)
 
+SEQ_EXAMPLES = ex_diffusion 
+PAR_EXAMPLES = ex_diffusionp  ex_amr_diffusionp
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
 .SUFFIXES:
-.SUFFIXES: .o .cpp
+.SUFFIXES: .o .cpp .mk
 .PHONY: all clean
+.PRECIOUS: %.o
 
-# Remove built-in rule
+COMMON_O= element-smoother.o smoother-util.o exact_sol.o
+
+# Remove built-in rules
 %: %.cpp
+%.o: %.cpp
 
-# Replace the default implicit rule for *.cpp files
-%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+all: $(EXAMPLES)
 
-all: $(TESTS)
+# Rules for building the EXAMPLES
+
+%: $(SRC)%.cpp $(COMMON_O) $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(COMMON_O) $(MFEM_LIBS)
+
+# Rules for compiling miniapp dependencies
+$(COMMON_O) $($(EXAMPLES)): \
+%.o: $(SRC)%.cpp $(SRC)%.hpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $(<) -o $(@)
 
 # Generate an error message if the MFEM library is not built and exit
 $(MFEM_LIB_FILE):
 	$(error The MFEM library is not built)
 
 clean:
-	rm -f *.o *~ $(TESTS)
-	rm -rf *.dSYM *.TVD.*breakpoints
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints

examples/fd/smoother-util.cpp

@@ -0,0 +1,746 @@
+
+
+#include "smoother-util.hpp"
+
+IntegrationRule * TensorIntegrationRule(const FiniteElementSpace & fes, int order) 
+{
+   Mesh *mesh = fes.GetMesh();
+   IntegrationRule * ir;
+   int ir_order = 2*order+2;
+   IntegrationRules IntRule(0, Quadrature1D::GaussLobatto);
+   // IntegrationRules IntRule(0, Quadrature1D::GaussLegendre);
+
+   int dim = mesh->Dimension();
+   switch (dim)
+   {
+   case 1: 
+      ir = new IntegrationRule(IntRules.Get(mfem::Geometry::SEGMENT, ir_order));
+      break;
+   case 2: 
+      ir = new IntegrationRule(IntRules.Get(mfem::Geometry::SQUARE, ir_order));
+      break;   
+   default:
+      ir = new IntegrationRule(IntRules.Get(mfem::Geometry::CUBE, ir_order));
+      break;
+   }
+   return ir;
+}
+IntegrationRule * TensorIntegrationRule(int dim, int order) 
+{
+   IntegrationRule * ir;
+   int ir_order = 2*order+2;
+   // IntegrationRules IntRule(0, Quadrature1D::GaussLegendre);
+   IntegrationRules IntRule(0, Quadrature1D::GaussLobatto);
+   switch (dim)
+   {
+   case 1: 
+      ir = new IntegrationRule(IntRules.Get(mfem::Geometry::SEGMENT, ir_order));
+      break;
+   case 2: 
+      ir = new IntegrationRule(IntRules.Get(mfem::Geometry::SQUARE, ir_order));
+      break;   
+   default:
+      ir = new IntegrationRule(IntRules.Get(mfem::Geometry::CUBE, ir_order));
+      break;
+   }
+   return ir;
+}
+
+void KronMult(const Vector & x, const Vector & y, Vector & z)
+{
+   int n = x.Size();
+   int m = y.Size();
+   z.SetSize(n*m);
+   for (int i=0; i<n; i++)
+   {
+      for (int j = 0; j<m; j++)
+      {
+         z(i*m+j) = x(i)*y(j);
+      }
+   }
+}
+
+void KronMult(const Vector & x, const Vector & y, const Vector & z, Vector & w)
+{
+   Vector xy;
+   KronMult(x,y,xy);
+   KronMult(xy,z,w);
+}
+
+void AlterLS(DenseMatrix & T, Vector & vecA, Vector & vecB)
+{
+   int n = T.Height();
+   int m = T.Width();
+   Vector x(n);
+   Vector temp(m);
+   double s;
+   Vector y(m); y.Randomize(1); y /= y.Norml2();
+   int maxit = 3;
+   for (int i=0; i<maxit; i++)
+   {
+      T.Mult(y,x); x /= x.Norml2();
+      T.MultTranspose(x,temp); y=temp; y/= y.Norml2();
+      s = InnerProduct(y,temp);
+   }
+   vecA = x; vecA *= sqrt(s);
+   vecB = y; vecB *= sqrt(s);
+}
+
+void AlterLS(DenseTensor & T, Vector & vecA, Vector & vecB, Vector & vecC)
+{
+   int n = T.SizeI();
+   int m = T.SizeJ();
+   int l = T.SizeK();
+
+   DenseMatrix A0(n,l*m);
+   DenseMatrix A1(m,l*n);
+   DenseMatrix A2(l,n*m);
+
+   for (int i=0; i<n; ++i)
+   {
+      for (int j=0; j<m; ++j)
+      {
+         for (int k=0; k<l; ++k)
+         {
+            A0(i,j+k*m) = T(i,j,k);
+            A1(j,i+k*n) = T(i,j,k);
+            A2(k,i+j*n) = T(i,j,k);
+         }
+      }
+   }
+
+   double s;
+   Vector x(n);
+   Vector temp;
+   // Vector temp;
+   Vector y(m); y.Randomize(1); y/=y.Norml2();
+   Vector z(l); z.Randomize(2); z/=z.Norml2();
+
+   int maxit = 3;
+   for (int i = 0; i<maxit; i++)
+   {
+      KronMult(z,y,temp); A0.Mult(temp,x); s=x.Norml2(); x/=s;
+      KronMult(z,x,temp); A1.Mult(temp,y); s=y.Norml2(); y/=s;
+      KronMult(y,x,temp); A2.Mult(temp,z); s=z.Norml2(); z/=s;
+   }
+   vecA = x; vecA *= cbrt(s);
+   vecB = y; vecB *= cbrt(s);
+   vecC = z; vecC *= cbrt(s);
+}
+
+ElementTPFunctionCoefficient::ElementTPFunctionCoefficient(FiniteElementSpace &fes, int iel, Coefficient &cf)
+{
+   coeff_avg = 0.0;
+   dim = fes.GetMesh()->Dimension();
+   ElementTransformation * Tr;
+   const IntegrationRule * ir;
+   const FiniteElement * fe = fes.GetFE(iel);
+   ir = TensorIntegrationRule(fes,fe->GetOrder()); 
+   int nint1D = (dim == 2) ? sqrt(ir->GetNPoints()) : cbrt(ir->GetNPoints());
+   int nrintx = nint1D;
+   int nrinty = nint1D;
+   int nrintz = (dim ==2) ? 0 : nint1D;
+   if (dim == 2)
+   {
+      A.SetSize(nrintx,nrinty);
+      Tr = fes.GetElementTransformation(iel);
+      nint = ir->GetNPoints();
+      for (int i = 0; i < nint; i++)
+      {
+         int nint1D = sqrt(ir->GetNPoints());
+         int iy = i/nint1D;
+         int ix = i%nint1D;
+         const IntegrationPoint &ip = ir->IntPoint(i);
+         Vector x(dim);
+         Tr->Transform(ip,x);
+         double val = cf.Eval(*Tr, ip);
+         A(ix,iy) = val;
+         coeff_avg += val;
+      }
+      AlterLS(A,VecX,VecY);
+      coeff_avg /= (double)nint;
+   }
+   else
+   {
+      T.SetSize(nrintx,nrinty,nrintz);
+      Tr = fes.GetElementTransformation(iel);
+      for (int i = 0; i < ir->GetNPoints(); i++)
+      {
+         int nint1D = cbrt(ir->GetNPoints());
+         int iz = i/(nint1D*nint1D);
+         int iy = (i - iz*nint1D*nint1D)/nint1D;
+         int ix = (i - iz*nint1D*nint1D)%nint1D;
+         const IntegrationPoint &ip = ir->IntPoint(i);
+         Tr->SetIntPoint (&ip);
+         double val = cf.Eval(*Tr, ip);
+         T(ix,iy,iz) = val;
+      }
+      AlterLS(T,VecX,VecY,VecZ);
+   }
+   delete ir;
+}
+
+
+double ElementTPFunctionCoefficient::Eval(ElementTransformation &T, const IntegrationPoint &ip)
+{
+   double val = 0.0;
+   switch (coord)
+   {
+      case 0: 
+      {
+         if (orient == 1)
+         {
+            val = VecX(nintx++);
+         }
+         else
+         {
+            int nend = VecX.Size();   
+            val = VecX(nend-1-nintx++);
+         }
+      }
+      break;
+      case 1: 
+      {
+         if (orient == 1)
+         {
+            val = VecY(ninty++);
+         }
+         else
+         {
+            int nend = VecY.Size();   
+            val = VecY(nend-1-ninty++);
+         }
+      } 
+      break;
+      case 2: 
+      {
+         if (orient == 1)
+         {
+            val = VecZ(nintz++); 
+         }
+         else
+         {
+            int nend = VecZ.Size();   
+            val = VecZ(nend-1-nintz++);
+         }
+      }
+      break;
+      case -1:
+      {
+         int nint1D = sqrt(nint);
+         int iy = mint/nint1D;
+         int ix = mint%nint1D;
+         val = VecX(ix) * VecY(iy);
+         mint++;
+      }
+      break;
+      case -2:
+      {
+         int nint1D = cbrt(nint);
+         int iz = mint/(nint1D*nint1D);
+         int iy = (mint - iz*nint1D*nint1D)/nint1D;
+         int ix = (mint - iz*nint1D*nint1D)%nint1D;
+         val = VecX(ix) * VecY(iy) * VecZ(iz);
+         mint++;
+      }
+      break;
+      default: MFEM_ABORT("ElementTPFunctionCoefficient::Eval: Wrong coord choice"); 
+         break;
+   }
+   return val;
+}
+
+
+void TPElementTransformation::Setup2D()
+{
+   Mesh * mesh = fes->GetMesh();
+   MFEM_VERIFY(dim == 2, "Wrong dimension");
+   int nel = mesh->GetNE();
+   TransA1D.SetSize(nel,dim);
+   TransB1D.SetSize(nel,dim);
+   // Get ElementTransformations for the 2D elements
+   for (int iel = 0; iel <nel; iel++)
+   {
+      // allocate memory for Trans1D
+      for (int d = 0; d<dim; d++)
+      {
+         TransA1D[iel][d] = new Vector;
+         TransB1D[iel][d] = new Vector;
+      }
+      ElementTransformation * T = mesh->GetElementTransformation(iel);
+      // Populate integration points and get the K = adj(J)/sqrt(detJ);
+      // Store K_11^2 + K_12^2 
+      //       K_21^2 + K_22^2 
+
+      const FiniteElement * fe = fes->GetFE(iel);
+      const IntegrationRule * ir = TensorIntegrationRule(*fes,fe->GetOrder()); 
+      int nint = ir->GetNPoints();
+      int nint1D = sqrt(nint);
+      DenseMatrix A, B;
+      A.SetSize(nint1D,nint1D);
+      B.SetSize(nint1D,nint1D);
+
+      for (int i = 0; i < nint; i++)
+      {
+         int iy = i/nint1D;
+         int ix = i%nint1D;
+         const IntegrationPoint &ip = ir->IntPoint(i);
+         T->SetIntPoint(&ip);
+
+         double detJ = T->Weight();
+         const DenseMatrix & adjJ = T->AdjugateJacobian();
+         DenseMatrix JtJ(adjJ.Height());
+         MultAtB(adjJ,adjJ,JtJ);
+         JtJ *= 1.0/abs(detJ);
+         // DenseMatrix adjtt(adjJ.Height());
+         // MultAtB(adjJ, adjJ, adjtt);
+         double valA = pow(abs(adjJ(0,0))+abs(adjJ(0,1)),2)/abs(detJ);
+         // double valA = abs(JtJ(0,0))+abs(JtJ(0,1));
+         double valB = pow(abs(adjJ(1,1))+abs(adjJ(1,0)),2)/abs(detJ);
+         // double valB = abs(JtJ(1,1))+abs(JtJ(1,0));
+         A(ix,iy) = valA;
+         B(ix,iy) = valB;
+      }
+      AlterLS(A,*TransA1D[iel][0],*TransA1D[iel][1]);
+      AlterLS(B,*TransB1D[iel][0],*TransB1D[iel][1]);
+   }
+}
+void TPElementTransformation::Setup3D()
+{
+   Mesh * mesh = fes->GetMesh();
+   MFEM_VERIFY(dim == 3, "Wrong dimension");
+   int nel = mesh->GetNE();
+   TransA1D.SetSize(nel,dim);
+   TransB1D.SetSize(nel,dim);
+   TransC1D.SetSize(nel,dim);
+   // Get ElementTransformations for the 3D elements
+   for (int iel = 0; iel <nel; iel++)
+   {
+      // allocate memory for Trans1D
+      for (int d = 0; d<dim; d++)
+      {
+         TransA1D[iel][d] = new Vector;
+         TransB1D[iel][d] = new Vector;
+         TransC1D[iel][d] = new Vector;
+      }
+      ElementTransformation * T = mesh->GetElementTransformation(iel);
+      // Populate integrations points and get the K = adj(J)/sqrt(detJ);
+      // Store K_11^2 + K_12^2 + K_13^2
+      //       K_21^2 + K_22^2 + K_23^2
+      //       K_31^2 + K_32^2 + K_33^2
+
+      const FiniteElement * fe = fes->GetFE(iel);
+      const IntegrationRule * ir = TensorIntegrationRule(*fes,fe->GetOrder()); 
+      int nint = ir->GetNPoints();
+      int nint1D = cbrt(nint);
+      DenseTensor A, B, C;
+      A.SetSize(nint1D,nint1D,nint1D);
+      B.SetSize(nint1D,nint1D,nint1D);
+      C.SetSize(nint1D,nint1D,nint1D);
+
+      for (int i = 0; i < nint; i++)
+      {
+         int iz = i/(nint1D*nint1D);
+         int iy = (i - iz*nint1D*nint1D)/nint1D;
+         int ix = (i - iz*nint1D*nint1D)%nint1D;
+
+         const IntegrationPoint &ip = ir->IntPoint(i);
+         T->SetIntPoint(&ip);
+
+         double detJ = T->Weight();
+         const DenseMatrix & adjJ = T->AdjugateJacobian();
+         // DenseMatrix JtJ(adjJ.Height());
+         // MultAtB(adjJ,adjJ,JtJ);
+         // JtJ *= 1.0/abs(detJ);
+         // double valA = abs(JtJ(0,0))+abs(JtJ(0,1))+abs(JtJ(0,2));
+         double valA = pow(abs(adjJ(0,0))+abs(adjJ(0,1))+abs(adjJ(0,2)),2)/abs(detJ);
+         // double valB = abs(JtJ(1,0))+abs(JtJ(1,1))+abs(JtJ(1,2));
+         double valB = pow(abs(adjJ(1,0))+abs(adjJ(1,1))+abs(adjJ(1,2)),2)/abs(detJ);
+         // double valC = abs(JtJ(2,0))+abs(JtJ(2,1))+abs(JtJ(2,2));
+         double valC = pow(abs(adjJ(2,0))+abs(adjJ(2,1))+abs(adjJ(2,2)),2)/abs(detJ);
+         A(ix,iy,iz) = valA;
+         B(ix,iy,iz) = valB;
+         C(ix,iy,iz) = valC;
+      }
+      AlterLS(A,*TransA1D[iel][0],*TransA1D[iel][1],*TransA1D[iel][2]);
+      AlterLS(B,*TransB1D[iel][0],*TransB1D[iel][1],*TransB1D[iel][2]);
+      AlterLS(C,*TransC1D[iel][0],*TransC1D[iel][1],*TransC1D[iel][2]);
+      // for (int i = 0; i<dim; i++)
+      // {
+      //    cout << "TransA["<<i<<"] = "; TransA1D[iel][i]->Print(cout, TransA1D[iel][i]->Size()); 
+      //    cout << "TransB["<<i<<"] = "; TransB1D[iel][i]->Print(cout, TransB1D[iel][i]->Size()); 
+      //    cout << "TransC["<<i<<"] = "; TransC1D[iel][i]->Print(cout, TransC1D[iel][i]->Size()); 
+      // }
+      // cin.get();
+   }
+}
+
+TPElementTransformation::TPElementTransformation(FiniteElementSpace &fes_) 
+: fes(&fes_)
+{
+   dim = fes->GetMesh()->Dimension();
+   if (dim == 2) 
+   {
+      Setup2D();
+   }
+   else
+   {
+      Setup3D();
+   }
+} 
+
+
+void GetVertexToEdgeCount(const Mesh * mesh, DenseMatrix & edge_counts)
+{
+   // serial
+   int dim = mesh->Dimension();
+   int nv = mesh->GetNV();
+   int ne = mesh->GetNEdges();
+   int nel = mesh->GetNE();
+
+   Array<int> ibeg(dim), iend(dim), inc(dim);
+   // loop through axis
+   for (int axis = 0; axis<dim; axis++)
+   {
+      switch (axis)
+      {
+         case 0:  ibeg[axis] = 0; inc[axis] = 2; iend[axis] = (dim == 2) ? 4 : 8  ; break; // "x" edges
+         case 1:  ibeg[axis] = 1; inc[axis] = 2; iend[axis] = (dim == 2) ? 4 : 8  ; break; // "y" edges
+         case 2:  ibeg[axis] = 8; inc[axis] = 1; iend[axis] = 12; break;                   // "z edges"
+         default: MFEM_ABORT("This should be unreachable"); break; 
+      }
+   }
+
+   Array<bool> edge_marker(ne); 
+   edge_counts.SetSize(nv,dim);
+   edge_counts = 0.0;
+   Array<int> edge_owned;
+   bool par = false;
+#ifdef MFEM_USE_MPI
+   const ParMesh * pmesh = dynamic_cast<const ParMesh *>(mesh);
+   if (pmesh) par = true;
+#endif   
+   edge_owned.SetSize(ne); edge_owned = 0;
+   ND_FECollection fec(1, dim);
+   if (par) 
+   {
+      ParFiniteElementSpace aux_fes(const_cast<ParMesh *>(pmesh), &fec);
+      int mytoffset = aux_fes.GetMyTDofOffset();
+      int tsize = aux_fes.GetTrueVSize();
+      Array<int> dofs;
+      for (int i=0; i<ne; ++i)
+      {
+         aux_fes.GetEdgeDofs(i, dofs);
+         const int ldof = (dofs[0] >= 0) ? dofs[0] : -1 - dofs[0];
+         int sign = aux_fes.GetLocalTDofNumber(ldof);
+         if (sign == -1) continue; // just a hack for now to work with AMR, need to rethink this.
+         int gdof = aux_fes.GetGlobalTDofNumber(ldof);
+         if (gdof >= mytoffset && gdof < mytoffset+tsize)
+         {
+            edge_owned[i] = 1;
+         }
+      }
+   }
+   else
+   {
+      edge_owned = 1;
+      if (mesh->Nonconforming())
+      {
+         NCMesh * ncmesh = mesh->ncmesh;
+         const mfem::NCMesh::NCList &nclist = ncmesh->GetEdgeList();
+         Array<mfem::NCMesh::Slave> ncslaves = nclist.slaves;
+         int ns = ncslaves.Size();
+         for (int i=0; i<ns; ++i)
+         {
+             int j = ncslaves[i].index;
+            edge_owned[j] = 0;
+         }
+      }
+   }
+
+   for (int d=0; d<dim; d++)
+   {
+      edge_marker = false;
+      for (int iel = 0; iel<nel; iel++)
+      {
+         Array<int> elem_edges, cor;
+         mesh->GetElementEdges(iel,elem_edges,cor);
+         for (int i = ibeg[d]; i<iend[d]; i += inc[d])
+         {
+            int edge = elem_edges[i];
+            if (edge_owned.Size())
+            {
+               // skip contributions from edge not owned by the proc
+               if (!edge_owned[edge]) continue; 
+            }
+            if (edge_marker[edge]) continue;
+            Array<int>edge_vert;
+            mesh->GetEdgeVertices(edge,edge_vert);
+            for (int j = 0; j < 2; j++)
+            {
+               int vertex = edge_vert[j];
+               edge_counts(vertex,d) += 1.0;
+            }
+            edge_marker[edge] = true;
+         }
+      }
+   }   
+}
+
+void GetDiffusionEdgeMatrix(int iedge, FiniteElementSpace * fes, 
+                               Vector & Jac1D, Vector & Coeff1D,
+                               const IntegrationRule *ir,  
+                               DenseMatrix &elmat, int orient)
+{
+   const FiniteElement * el = fes->GetEdgeElement(iedge);
+   int nd = el->GetDof();
+   int dim = el->GetDim();
+
+   DenseMatrix dshape(nd,dim);
+   elmat.SetSize(nd);
+   elmat = 0.0;
+   int nint = ir->GetNPoints();
+   
+   for (int i = 0; i < nint; i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+      double w = ip.weight;
+      el->CalcDShape(ip, dshape);
+      int j = orient == -1 ? nint-i-1 : i;
+      double val = Coeff1D(j) * Jac1D(i);
+      w *= val;
+      AddMult_a_AAt(w, dshape, elmat);
+   }
+}
+
+void GetMassEdgeMatrix(int iedge, FiniteElementSpace * fes, 
+                       Vector & Jac1D, Vector & Coeff1D,
+                       const IntegrationRule *ir,  
+                       DenseMatrix &elmat, int orient)
+{
+   const FiniteElement * el = fes->GetEdgeElement(iedge);
+   int nd = el->GetDof();
+   Vector shape(nd);
+   elmat.SetSize(nd);
+   elmat = 0.0;
+   int nint = ir->GetNPoints();
+   for (int i = 0; i < nint; i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+      double w = ip.weight;
+      el->CalcShape(ip, shape);
+      int j = orient == -1 ? nint-i-1 : i;
+      double val = Coeff1D(j) * Jac1D(i);
+      w *= val;
+      AddMult_a_VVt(w, shape, elmat);
+   }
+}
+
+void Get1DMatrices(FiniteElementSpace * fes, int iedge, int orient, 
+                               Vector & JacL, Vector & JacM, Vector & Coeff1D,
+                               const IntegrationRule *ir,  
+                               DenseMatrix &L, DenseMatrix & M)
+{
+   const FiniteElement * el = fes->GetEdgeElement(iedge);
+   int dim = el->GetDim();
+   int nd = el->GetDof();
+
+   DenseMatrix dshape(nd,dim);
+   Vector shape(nd);
+   L.SetSize(nd); L = 0.0;
+   M.SetSize(nd); M = 0.0;
+   int nint = ir->GetNPoints();
+   for (int i = 0; i < nint; i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+      double w = ip.weight;
+      el->CalcDShape(ip, dshape);
+      el->CalcShape(ip, shape);
+      int j = orient == -1 ? nint-i-1 : i;
+      double wL = w*Coeff1D(j) * JacL(j);
+      double wM = w*Coeff1D(j) * JacM(j);
+      AddMult_a_AAt(wL, dshape, L);
+      AddMult_a_VVt(wM, shape, M);
+   } 
+}                               
+
+
+
+void TensorProductEssentialDofsMaps(const Array<int> & ess_tdof_list, 
+                                    const ParFiniteElementSpace * fes, 
+                                    Array<Array<int> *> & tmap, // local edge map
+                                    Array<Array<int>* > & non_ess_dofs) // element map
+{
+
+   MPI_Comm comm = fes->GetComm();
+   int num_procs,myid;
+   MPI_Comm_size(comm, &num_procs);
+   MPI_Comm_rank(comm, &myid);
+
+   // 1. Find the element local dofs that are essential and then identify 
+   //    edge vertices (in local numbering) that are produced from
+   //    Gather the vertex dofs to be eliminated for each edge in tmap.
+   // 2. Reconstruct the essential dofs for each element (this list might be different 
+   //    for each element e.g. l-shape/fichera mesh, where to keep the kronecker product 
+   //    structure an essential dof is not eliminated, see vertex (v) below)
+   //   .  .  .
+   //   .  .  .
+   //   .  .  v  .  .
+   //   .  .  .  .  .
+   //   .  .  .  .  .
+   // ----------------------------------------------------------------------
+   //
+   ParMesh * pmesh = fes->GetParMesh();
+   int dim = pmesh->Dimension();
+   int nredges = pmesh->GetNEdges();
+   int tsize = fes->GetTrueVSize();
+   int vsize = fes->GetVSize();
+   Vector tess_tdof_marker(tsize); tess_tdof_marker = 0.0;
+   Vector ess_tdof_marker(vsize); 
+   
+   for (int i = 0; i<ess_tdof_list.Size(); i++)
+   {
+      int tdof = ess_tdof_list[i];
+
+      tess_tdof_marker[tdof] = 1.0;
+   }
+
+   fes->GetProlongationMatrix()->Mult(tess_tdof_marker, ess_tdof_marker);
+
+   tmap.SetSize(nredges);
+   for (int i = 0; i<nredges; i++) { tmap[i] = new Array<int>(0); } 
+   int nel = pmesh->GetNE();
+   non_ess_dofs.SetSize(nel);
+   for (int iel = 0; iel<nel; iel++)
+   {
+      non_ess_dofs[iel] = new Array<int>(0);
+      Array<int> local_dofs; 
+      Array<int> local_tdofs; 
+      const FiniteElement &fe = *fes->GetFE(iel);
+      // mfem to Tensor basis map
+      const Array<int> &dmap =
+      dynamic_cast<const TensorBasisElement&>(fe).GetDofMap();
+      Array<int> dmapt(dmap.Size());
+      for (int i = 0; i<dmapt.Size(); i++)
+      {
+         dmapt[dmap[i]] = i;
+      }
+      Array<int> elem_dofs;
+      fes->GetElementDofs(iel,elem_dofs);
+      // get local index of ess_dofs
+      int n = elem_dofs.Size();
+      // loop through the vertices
+      for (int i = 0; i< elem_dofs.Size(); i++)
+      {
+         int ldof = elem_dofs[i];
+         if (!ess_tdof_marker[ldof]) // if not essential dof
+         {
+            local_dofs.Append(dmapt[i]); // append in local element dofs
+            local_tdofs.Append(ldof); // append in local element dofs
+         }
+      }
+      
+      // Find the possible vertex local dofs on the edges to be removed
+      int n1D = (dim == 2) ? sqrt(n) : cbrt(n);
+      Array<int> edges, cor;
+      pmesh->GetElementEdges(iel,edges,cor);
+      Array<int> eidx(dim);
+      eidx[0] = 0;
+      eidx[1] = 1;
+      if (dim == 3) eidx[2] = 8;
+      // mark edge local dofs
+      Array<Array<int> *> markers(dim);
+      for (int d = 0; d<dim; d++)
+      {
+         Array<int> marker(n1D); marker = 0;
+         for (int i = 0; i<local_dofs.Size(); i++)
+         {
+            int j = local_dofs[i];
+            int c = j/(n1D*n1D);
+            int l = c*n1D*n1D;
+            int k = (d == 0) ? (j-l)%n1D : d == 1 ? (j-l)/n1D : c; 
+            marker[k] = 1;
+         }
+         markers[d] = new Array<int>(marker);
+         // pick up the edge and orientation
+         int edge = edges[eidx[d]];
+         int orient = cor[eidx[d]];
+         const FiniteElement &fe = *fes->GetEdgeElement(edge);
+         // edge tensor product map
+         const Array<int> &emap =
+         dynamic_cast<const TensorBasisElement&>(fe).GetDofMap();
+         Array<int> edge_ldofs;
+         for (int i = 0; i<marker.Size(); i++)
+         {
+            if (!marker[i])
+            {
+               if (orient == 1)
+               {
+                  edge_ldofs.Append(emap[i]);
+               }
+               else
+               {
+                  int k = (emap[i] == 1) ? 0 : 1;
+                  edge_ldofs.Append(k);
+               }
+            }
+         }
+         edge_ldofs.Sort(); edge_ldofs.Unique();   
+         tmap[edge]->Append(edge_ldofs);
+         tmap[edge]->Sort();
+         tmap[edge]->Unique();
+      }
+      if (dim == 2)
+      {
+         for (int j = 0; j<n1D; j++)
+         {
+            if ((*markers[1])[j])
+            {
+               for (int i = 0; i<n1D; i++)
+               {
+                  if ((*markers[0])[i])
+                  {
+                     int ldof = n1D*j+i;
+                     non_ess_dofs[iel]->Append(ldof);
+                  }
+               }
+            }
+         }
+      }
+      else
+      {
+         for (int k = 0; k<n1D; k++)
+         {
+            if ((*markers[2])[k])
+            {
+               for (int j = 0; j<n1D; j++)
+               {
+                  if ((*markers[1])[j])
+                  {
+                     for (int i = 0; i<n1D; i++)
+                     {
+                        if ((*markers[0])[i])
+                        {
+                           int ldof = n1D*n1D*k + n1D*j+i;
+                           non_ess_dofs[iel]->Append(ldof);
+                        }
+                     }
+                  }
+               }
+            }
+         }
+      }
+      non_ess_dofs[iel]->Sort();
+      non_ess_dofs[iel]->Unique();
+      for (int i = 0; i<non_ess_dofs[iel]->Size(); i++)
+      {
+         int ldof = (*non_ess_dofs[iel])[i];
+         int tdof = elem_dofs[dmap[ldof]];
+         (*non_ess_dofs[iel])[i] = tdof;
+      }
+      for (int d=0; d<dim; d++)
+      {
+         delete markers[d];
+      }
+   }
+};

examples/fd/smoother-util.hpp

@@ -0,0 +1,112 @@
+
+#pragma once
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+IntegrationRule * TensorIntegrationRule(const FiniteElementSpace & fes, int order);
+IntegrationRule * TensorIntegrationRule(int dim, int order);
+
+void KronMult(const Vector & x, const Vector & y, Vector & z);
+void KronMult(const Vector & x, const Vector & y, const Vector & z, Vector & w);
+
+
+void AlterLS(DenseMatrix & T, Vector & vecA, Vector & vecB);
+
+void AlterLS(DenseTensor & T, Vector & vecA, Vector & vecB, Vector & vecC);
+
+class ElementTPFunctionCoefficient : public Coefficient//
+{
+private:
+   int dim;
+   DenseMatrix A;
+   DenseTensor T;
+   double coeff_avg;
+   Vector VecX;
+   Vector VecY;
+   Vector VecZ;
+   int orient;
+   int nint; // total num of integrations points
+   int mint=0; // counter for all the integrations points
+   int nintx = 0; // counter for the x integrations points
+   int ninty = 0; // counter for the y integration points
+   int nintz = 0; // counter for the z integration points
+   int coord = 0; // (indication flag for x,y or z coordinate)
+
+public:
+   ElementTPFunctionCoefficient(FiniteElementSpace &fes, int iel, Coefficient &cf);
+   double GetCoeffAvg() {return coeff_avg;}
+   void ResetCounters() { mint = nintx = ninty = nintz = 0; }
+   void ResetCounter(int c) 
+   { 
+      switch (c)
+      {
+         case 0: nintx = 0; break;
+         case 1: ninty = 0; break;
+         case 2: nintz = 0; break;
+         default: mint = 0; break;
+      }
+   }
+   void SetCoord(int coord_) { coord = coord_; }
+   void SetOrient(int orient_) { orient = orient_; }
+   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip);
+   Vector * GetVecX(){return &VecX;}
+   Vector * GetVecY(){return &VecY;}
+   Vector * GetVecZ(){return &VecZ;}
+   virtual ~ElementTPFunctionCoefficient() { }
+};
+
+
+class TPElementTransformation
+{
+private:
+   int dim;
+   FiniteElementSpace * fes = nullptr;
+   Array2D<Vector *> TransA1D;
+   Array2D<Vector *> TransB1D;
+   Array2D<Vector *> TransC1D;
+   void Setup2D();
+   void Setup3D();
+public:
+   TPElementTransformation(FiniteElementSpace &fes_);
+   Vector * GetTPTransformation(int iel, int coord, int which_coeff)
+   {
+      switch(which_coeff)
+      {
+         case 0: return TransA1D[iel][coord]; break;
+         case 1: return TransB1D[iel][coord]; break;
+         case 2: 
+         {
+            MFEM_VERIFY(dim == 3, "Wrong coeff for this dimension");
+            return TransC1D[iel][coord]; 
+            break;
+         }
+         default: MFEM_ABORT("Wrong coeff selection"); return 0; break; 
+      }
+   }
+   ~TPElementTransformation() { }
+};
+
+void GetVertexToEdgeCount(const Mesh * mesh, DenseMatrix & edge_counts);
+
+void Get1DMatrices(FiniteElementSpace * fes, int iedge, int orient, 
+                               Vector & JacL, Vector & JacM, Vector & Coeff1D,
+                               const IntegrationRule *ir,  
+                               DenseMatrix &L, DenseMatrix & M);
+
+void GetDiffusionEdgeMatrix(int iedge, FiniteElementSpace * fes, 
+                               Vector & Jac1D, Vector & Coeff1D,
+                               const IntegrationRule *ir,  
+                               DenseMatrix &elmat, int orient);
+void GetMassEdgeMatrix(int iedge, FiniteElementSpace * fes, 
+                       Vector & Jac1D, Vector & Coeff1D,
+                       const IntegrationRule *ir,  
+                       DenseMatrix &elmat, int orient);
+
+void TensorProductEssentialDofsMaps(const Array<int> & ess_tdof_list, 
+                                    const ParFiniteElementSpace * fes, 
+                                    Array<Array<int> *> & tmap, // local edge map
+                                    Array<Array<int>* > & non_ess_dofs); // element map

examples/makefile

@@ -26,9 +26,6 @@ SEQ_EXAMPLES = ex0 ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 \
 PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
   ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
   ex25p ex26p ex27p ex28p ex29p
-SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
-PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p \
-  ex24p ex25p ex26p
 
 ifeq ($(MFEM_USE_MPI),NO)
    EXAMPLES = $(SEQ_EXAMPLES)
@@ -102,14 +99,6 @@ RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
 	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
 %-test-seq: %
 	@$(call mfem-test,$<,, Serial example)
-%-test-par-cuda: %
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel CUDA example,-d cuda)
-%-test-seq-cuda: %
-	@$(call mfem-test,$<,, Serial CUDA example,-d cuda)
-%-test-par-hip: %
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel HIP example,-d hip)
-%-test-seq-hip: %
-	@$(call mfem-test,$<,, Serial HIP example,-d hip)
 
 # Testing: Specific execution options
 ex0-test-seq: ex0

examples/petsc/ex3p.cpp

@@ -121,7 +121,9 @@ int main(int argc, char *argv[])
 
    // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
+   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
+   //    meshes need to be reoriented before we can define high-order Nedelec
+   //    spaces on them.
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
    delete mesh;
    {
@@ -131,6 +133,7 @@ int main(int argc, char *argv[])
          pmesh->UniformRefinement();
       }
    }
+   pmesh->ReorientTetMesh();
 
    // 6. Define a parallel finite element space on the parallel mesh. Here we
    //    use the Nedelec finite elements of the specified order.

examples/petsc/ex4p.cpp

@@ -122,7 +122,9 @@ int main(int argc, char *argv[])
 
    // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
    //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
+   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
+   //    meshes need to be reoriented before we can define high-order Nedelec
+   //    spaces on them (this is needed in the ADS solver below).
    ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
    delete mesh;
    {
@@ -132,6 +134,7 @@ int main(int argc, char *argv[])
          pmesh->UniformRefinement();
       }
    }
+   pmesh->ReorientTetMesh();
 
    // 6. Define a parallel finite element space on the parallel mesh. Here we
    //    use the Raviart-Thomas finite elements of the specified order.

examples/superlu/ex1p.cpp

@@ -282,10 +282,6 @@ int main(int argc, char *argv[])
    superlu->SetOperator(*SLU_A);
    superlu->SetPrintStatistics(true);
    superlu->Mult(B, X);
-   superlu->DismantleGrid();
-
-   delete SLU_A;
-   delete superlu;
 
    // 14. Recover the parallel grid function corresponding to X. This is the
    //     local finite element solution on each processor.

fem/CMakeLists.txt

@@ -39,7 +39,6 @@ set(SRCS
   complex_fem.cpp
   convergence.cpp
   datacollection.cpp
-  doftrans.cpp
   eltrans.cpp
   estimators.cpp
   fe.cpp
@@ -106,7 +105,6 @@ set(SRCS
   tmop/tmop_pa_w3.cpp
   tmop/tmop_pa_w3_c0.cpp
   tmop_tools.cpp
-  tmop_amr.cpp
   gslib.cpp
   transfer.cpp
   lor.cpp
@@ -120,7 +118,6 @@ set(HDRS
   complex_fem.hpp
   convergence.hpp
   datacollection.hpp
-  doftrans.hpp
   eltrans.hpp
   estimators.hpp
   fe.hpp
@@ -167,7 +164,6 @@ set(HDRS
   tmop.hpp
   tmop/tmop_pa.hpp
   tmop_tools.hpp
-  tmop_amr.hpp
   gslib.hpp
   transfer.hpp
   lor.hpp
@@ -188,11 +184,6 @@ if (MFEM_USE_ADIOS2)
   list(APPEND HDRS adios2datacollection.hpp)
 endif()
 
-if (MFEM_USE_FMS)
-  list(APPEND SRCS fmsdatacollection.cpp fmsconvert.cpp)
-  list(APPEND HDRS fmsdatacollection.hpp fmsconvert.hpp)
-endif()
-
 if (MFEM_USE_MPI)
   list(APPEND SRCS
     pbilinearform.cpp

fem/bilinearform.cpp

@@ -391,7 +391,6 @@ void BilinearForm::Assemble(int skip_zeros)
    }
 
    ElementTransformation *eltrans;
-   DofTransformation * doftrans;
    Mesh *mesh = fes -> GetMesh();
    DenseMatrix elmat, *elmat_p;
 
@@ -425,7 +424,7 @@ void BilinearForm::Assemble(int skip_zeros)
       for (int i = 0; i < fes -> GetNE(); i++)
       {
          int elem_attr = fes->GetMesh()->GetAttribute(i);
-         doftrans = fes->GetElementVDofs(i, vdofs);
+         fes->GetElementVDofs(i, vdofs);
          if (element_matrices)
          {
             elmat_p = &(*element_matrices)(i);
@@ -459,11 +458,6 @@ void BilinearForm::Assemble(int skip_zeros)
             {
                elmat_p = &elmat;
             }
-            if (doftrans)
-            {
-               doftrans->TransformDual(elmat);
-            }
-            elmat_p = &elmat;
          }
          if (static_cond)
          {
@@ -509,7 +503,7 @@ void BilinearForm::Assemble(int skip_zeros)
          if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
 
          const FiniteElement &be = *fes->GetBE(i);
-         doftrans = fes -> GetBdrElementVDofs (i, vdofs);
+         fes -> GetBdrElementVDofs (i, vdofs);
          eltrans = fes -> GetBdrElementTransformation (i);
          int k = 0;
          for (; k < boundary_integs.Size(); k++)
@@ -529,22 +523,17 @@ void BilinearForm::Assemble(int skip_zeros)
             boundary_integs[k]->AssembleElementMatrix(be, *eltrans, elemmat);
             elmat += elemmat;
          }
-         if (doftrans)
-         {
-            doftrans->TransformDual(elmat);
-         }
-         elmat_p = &elmat;
          if (!static_cond)
          {
-            mat->AddSubMatrix(vdofs, vdofs, *elmat_p, skip_zeros);
+            mat->AddSubMatrix(vdofs, vdofs, elmat, skip_zeros);
             if (hybridization)
             {
-               hybridization->AssembleBdrMatrix(i, *elmat_p);
+               hybridization->AssembleBdrMatrix(i, elmat);
             }
          }
          else
          {
-            static_cond->AssembleBdrMatrix(i, *elmat_p);
+            static_cond->AssembleBdrMatrix(i, elmat);
          }
       }
    }
@@ -736,8 +725,8 @@ void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
       {
          // A, X and B point to the same data as mat, x and b
          EliminateVDofsInRHS(ess_tdof_list, x, b);
-         X.MakeRef(x, 0, x.Size());
-         B.MakeRef(b, 0, b.Size());
+         X.NewMemoryAndSize(x.GetMemory(), x.Size(), false);
+         B.NewMemoryAndSize(b.GetMemory(), b.Size(), false);
          if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
       }
    }
@@ -1329,10 +1318,9 @@ void MixedBilinearForm::Assemble (int skip_zeros)
       return;
    }
 
+   Array<int> tr_vdofs, te_vdofs;
    ElementTransformation *eltrans;
-   DofTransformation * dom_dof_trans;
-   DofTransformation * ran_dof_trans;
-   DenseMatrix elmat;
+   DenseMatrix elemmat;
 
    Mesh *mesh = test_fes -> GetMesh();
 
@@ -1345,24 +1333,16 @@ void MixedBilinearForm::Assemble (int skip_zeros)
    {
       for (int i = 0; i < test_fes -> GetNE(); i++)
       {
-         dom_dof_trans = trial_fes -> GetElementVDofs (i, trial_vdofs);
-         ran_dof_trans = test_fes  -> GetElementVDofs (i, test_vdofs);
+         trial_fes -> GetElementVDofs (i, tr_vdofs);
+         test_fes  -> GetElementVDofs (i, te_vdofs);
          eltrans = test_fes -> GetElementTransformation (i);
-
-         elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
-         elmat = 0.0;
          for (int k = 0; k < domain_integs.Size(); k++)
          {
             domain_integs[k] -> AssembleElementMatrix2 (*trial_fes -> GetFE(i),
                                                         *test_fes  -> GetFE(i),
                                                         *eltrans, elemmat);
-            elmat += elemmat;
+            mat -> AddSubMatrix (te_vdofs, tr_vdofs, elemmat, skip_zeros);
          }
-         if (ran_dof_trans || dom_dof_trans)
-         {
-            TransformDual(ran_dof_trans, dom_dof_trans, elmat);
-         }
-         mat -> AddSubMatrix (test_vdofs, trial_vdofs, elmat, skip_zeros);
       }
    }
 
@@ -1394,12 +1374,9 @@ void MixedBilinearForm::Assemble (int skip_zeros)
          const int bdr_attr = mesh->GetBdrAttribute(i);
          if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
 
-         dom_dof_trans = trial_fes -> GetBdrElementVDofs (i, trial_vdofs);
-         ran_dof_trans = test_fes  -> GetBdrElementVDofs (i, test_vdofs);
+         trial_fes -> GetBdrElementVDofs (i, tr_vdofs);
+         test_fes  -> GetBdrElementVDofs (i, te_vdofs);
          eltrans = test_fes -> GetBdrElementTransformation (i);
-
-         elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
-         elmat = 0.0;
          for (int k = 0; k < boundary_integs.Size(); k++)
          {
             if (boundary_integs_marker[k] &&
@@ -1408,34 +1385,29 @@ void MixedBilinearForm::Assemble (int skip_zeros)
             boundary_integs[k]->AssembleElementMatrix2 (*trial_fes -> GetBE(i),
                                                         *test_fes  -> GetBE(i),
                                                         *eltrans, elemmat);
-            elmat += elemmat;
+            mat -> AddSubMatrix (te_vdofs, tr_vdofs, elemmat, skip_zeros);
          }
-         if (ran_dof_trans || dom_dof_trans)
-         {
-            TransformDual(ran_dof_trans, dom_dof_trans, elmat);
-         }
-         mat -> AddSubMatrix (test_vdofs, trial_vdofs, elmat, skip_zeros);
       }
    }
 
    if (trace_face_integs.Size())
    {
       FaceElementTransformations *ftr;
-      Array<int> test_vdofs2;
+      Array<int> te_vdofs2;
       const FiniteElement *trial_face_fe, *test_fe1, *test_fe2;
 
       int nfaces = mesh->GetNumFaces();
       for (int i = 0; i < nfaces; i++)
       {
          ftr = mesh->GetFaceElementTransformations(i);
-         trial_fes->GetFaceVDofs(i, trial_vdofs);
-         test_fes->GetElementVDofs(ftr->Elem1No, test_vdofs);
+         trial_fes->GetFaceVDofs(i, tr_vdofs);
+         test_fes->GetElementVDofs(ftr->Elem1No, te_vdofs);
          trial_face_fe = trial_fes->GetFaceElement(i);
          test_fe1 = test_fes->GetFE(ftr->Elem1No);
          if (ftr->Elem2No >= 0)
          {
-            test_fes->GetElementVDofs(ftr->Elem2No, test_vdofs2);
-            test_vdofs.Append(test_vdofs2);
+            test_fes->GetElementVDofs(ftr->Elem2No, te_vdofs2);
+            te_vdofs.Append(te_vdofs2);
             test_fe2 = test_fes->GetFE(ftr->Elem2No);
          }
          else
@@ -1449,7 +1421,7 @@ void MixedBilinearForm::Assemble (int skip_zeros)
          {
             trace_face_integs[k]->AssembleFaceMatrix(*trial_face_fe, *test_fe1,
                                                      *test_fe2, *ftr, elemmat);
-            mat->AddSubMatrix(test_vdofs, trial_vdofs, elemmat, skip_zeros);
+            mat->AddSubMatrix(te_vdofs, tr_vdofs, elemmat, skip_zeros);
          }
       }
    }
@@ -1489,8 +1461,8 @@ void MixedBilinearForm::Assemble (int skip_zeros)
          ftr = mesh->GetBdrFaceTransformations(i);
          if (ftr)
          {
-            trial_fes->GetFaceVDofs(ftr->ElementNo, trial_vdofs);
-            test_fes->GetElementVDofs(ftr->Elem1No, test_vdofs);
+            trial_fes->GetFaceVDofs(ftr->ElementNo, tr_vdofs);
+            test_fes->GetElementVDofs(ftr->Elem1No, te_vdofs);
             trial_face_fe = trial_fes->GetFaceElement(ftr->ElementNo);
             test_fe1 = test_fes->GetFE(ftr->Elem1No);
             // The test_fe2 object is really a dummy and not used on the
@@ -1507,7 +1479,7 @@ void MixedBilinearForm::Assemble (int skip_zeros)
                                                                  *test_fe1,
                                                                  *test_fe2,
                                                                  *ftr, elemmat);
-               mat->AddSubMatrix(test_vdofs, trial_vdofs, elemmat, skip_zeros);
+               mat->AddSubMatrix(te_vdofs, tr_vdofs, elemmat, skip_zeros);
             }
          }
       }
@@ -1869,8 +1841,6 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
 
    Array<int> dom_vdofs, ran_vdofs;
    ElementTransformation *T;
-   DofTransformation * dom_dof_trans;
-   DofTransformation * ran_dof_trans;
    const FiniteElement *dom_fe, *ran_fe;
    DenseMatrix totelmat, elmat;
 
@@ -1883,8 +1853,8 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
    {
       for (int i = 0; i < test_fes->GetNE(); i++)
       {
-         dom_dof_trans = trial_fes->GetElementVDofs(i, dom_vdofs);
-         ran_dof_trans = test_fes->GetElementVDofs(i, ran_vdofs);
+         trial_fes->GetElementVDofs(i, dom_vdofs);
+         test_fes->GetElementVDofs(i, ran_vdofs);
          T = test_fes->GetElementTransformation(i);
          dom_fe = trial_fes->GetFE(i);
          ran_fe = test_fes->GetFE(i);
@@ -1897,10 +1867,6 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
                                                      elmat);
             totelmat += elmat;
          }
-         if (ran_dof_trans || dom_dof_trans)
-         {
-            TransformPrimal(ran_dof_trans, dom_dof_trans, totelmat);
-         }
          mat->SetSubMatrix(ran_vdofs, dom_vdofs, totelmat, skip_zeros);
       }
    }

fem/bilininteg.hpp

@@ -711,7 +711,7 @@ protected:
    {
       return "MixedScalarDerivativeIntegrator:  "
              "Trial and test spaces must both be scalar fields in 1D "
-             "and the trial space must implement CalcDShape.";
+             "and the trial space must implement CaldDShape.";
    }
 
    inline virtual void CalcTrialShape(const FiniteElement & trial_fe,
@@ -2936,11 +2936,11 @@ public:
 
     - F. Bassi and S. Rebay. A high order discontinuous Galerkin method for
       compressible turbulent flows. In B. Cockburn, G. E. Karniadakis, and
-      C.-W. Shu, editors, Discontinuous Galerkin Methods, pages 77-88. Springer
+      C.-W. Shu, editors, Discontinuous Galerkin Methods, pages 77–88. Springer
       Berlin Heidelberg, 2000.
     - D. N. Arnold, F. Brezzi, B. Cockburn, and L. D. Marini. Unified analysis
       of discontinuous Galerkin methods for elliptic problems. SIAM Journal on
-      Numerical Analysis, 39(5):1749-1779, 2002.
+      Numerical Analysis, 39(5):1749–1779, 2002.
 */
 class DGDiffusionBR2Integrator : public BilinearFormIntegrator
 {

fem/bilininteg_convection_ea.cpp

@@ -17,14 +17,14 @@ namespace mfem
 {
 
 template<int T_D1D = 0, int T_Q1D = 0>
-void EAConvectionAssemble1D(const int NE,
-                            const Array<double> &b,
-                            const Array<double> &g,
-                            const Vector &padata,
-                            Vector &eadata,
-                            const bool add,
-                            const int d1d = 0,
-                            const int q1d = 0)
+static void EAConvectionAssemble1D(const int NE,
+                                   const Array<double> &b,
+                                   const Array<double> &g,
+                                   const Vector &padata,
+                                   Vector &eadata,
+                                   const bool add,
+                                   const int d1d = 0,
+                                   const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -69,14 +69,14 @@ void EAConvectionAssemble1D(const int NE,
 }
 
 template<int T_D1D = 0, int T_Q1D = 0>
-void EAConvectionAssemble2D(const int NE,
-                            const Array<double> &b,
-                            const Array<double> &g,
-                            const Vector &padata,
-                            Vector &eadata,
-                            const bool add,
-                            const int d1d = 0,
-                            const int q1d = 0)
+static void EAConvectionAssemble2D(const int NE,
+                                   const Array<double> &b,
+                                   const Array<double> &g,
+                                   const Vector &padata,
+                                   Vector &eadata,
+                                   const bool add,
+                                   const int d1d = 0,
+                                   const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -146,14 +146,14 @@ void EAConvectionAssemble2D(const int NE,
 }
 
 template<int T_D1D = 0, int T_Q1D = 0>
-void EAConvectionAssemble3D(const int NE,
-                            const Array<double> &b,
-                            const Array<double> &g,
-                            const Vector &padata,
-                            Vector &eadata,
-                            const bool add,
-                            const int d1d = 0,
-                            const int q1d = 0)
+static void EAConvectionAssemble3D(const int NE,
+                                   const Array<double> &b,
+                                   const Array<double> &g,
+                                   const Vector &padata,
+                                   Vector &eadata,
+                                   const bool add,
+                                   const int d1d = 0,
+                                   const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;

fem/bilininteg_convection_pa.cpp

@@ -21,13 +21,13 @@ namespace mfem
 // PA Convection Integrator
 
 // PA Convection Assemble 2D kernel
-void PAConvectionSetup2D(const int NQ,
-                         const int NE,
-                         const Array<double> &w,
-                         const Vector &j,
-                         const Vector &vel,
-                         const double alpha,
-                         Vector &op)
+static void PAConvectionSetup2D(const int NQ,
+                                const int NE,
+                                const Array<double> &w,
+                                const Vector &j,
+                                const Vector &vel,
+                                const double alpha,
+                                Vector &op)
 {
    constexpr int DIM = 2;
 
@@ -60,13 +60,13 @@ void PAConvectionSetup2D(const int NQ,
 }
 
 // PA Convection Assemble 3D kernel
-void PAConvectionSetup3D(const int NQ,
-                         const int NE,
-                         const Array<double> &w,
-                         const Vector &j,
-                         const Vector &vel,
-                         const double alpha,
-                         Vector &op)
+static void PAConvectionSetup3D(const int NQ,
+                                const int NE,
+                                const Array<double> &w,
+                                const Vector &j,
+                                const Vector &vel,
+                                const double alpha,
+                                Vector &op)
 {
    constexpr int DIM = 3;
    constexpr int SDIM = DIM;
@@ -135,7 +135,7 @@ static void PAConvectionSetup(const int dim,
 }
 
 // PA Convection Apply 2D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
+template<int T_D1D = 0, int T_Q1D = 0> static
 void PAConvectionApply2D(const int ne,
                          const Array<double> &b,
                          const Array<double> &g,
@@ -254,7 +254,7 @@ void PAConvectionApply2D(const int ne,
 }
 
 // Optimized PA Convection Apply 2D kernel
-template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
+template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0> static
 void SmemPAConvectionApply2D(const int ne,
                              const Array<double> &b,
                              const Array<double> &g,
@@ -382,7 +382,7 @@ void SmemPAConvectionApply2D(const int ne,
 }
 
 // PA Convection Apply 3D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
+template<int T_D1D = 0, int T_Q1D = 0> static
 void PAConvectionApply3D(const int ne,
                          const Array<double> &b,
                          const Array<double> &g,
@@ -563,7 +563,7 @@ void PAConvectionApply3D(const int ne,
 }
 
 // Optimized PA Convection Apply 3D kernel
-template<int T_D1D = 0, int T_Q1D = 0>
+template<int T_D1D = 0, int T_Q1D = 0> static
 void SmemPAConvectionApply3D(const int ne,
                              const Array<double> &b,
                              const Array<double> &g,

fem/bilininteg_dgtrace_ea.cpp

@@ -16,12 +16,12 @@
 namespace mfem
 {
 
-void EADGTraceAssemble1DInt(const int NF,
-                            const Array<double> &basis,
-                            const Vector &padata,
-                            Vector &eadata_int,
-                            Vector &eadata_ext,
-                            const bool add)
+static void EADGTraceAssemble1DInt(const int NF,
+                                   const Array<double> &basis,
+                                   const Vector &padata,
+                                   Vector &eadata_int,
+                                   Vector &eadata_ext,
+                                   const bool add)
 {
    auto D = Reshape(padata.Read(), 2, 2, NF);
    auto A_int = Reshape(eadata_int.ReadWrite(), 2, NF);
@@ -50,11 +50,11 @@ void EADGTraceAssemble1DInt(const int NF,
    });
 }
 
-void EADGTraceAssemble1DBdr(const int NF,
-                            const Array<double> &basis,
-                            const Vector &padata,
-                            Vector &eadata_bdr,
-                            const bool add)
+static void EADGTraceAssemble1DBdr(const int NF,
+                                   const Array<double> &basis,
+                                   const Vector &padata,
+                                   Vector &eadata_bdr,
+                                   const bool add)
 {
    auto D = Reshape(padata.Read(), 2, 2, NF);
    auto A_bdr = Reshape(eadata_bdr.ReadWrite(), NF);
@@ -72,14 +72,14 @@ void EADGTraceAssemble1DBdr(const int NF,
 }
 
 template<int T_D1D = 0, int T_Q1D = 0>
-void EADGTraceAssemble2DInt(const int NF,
-                            const Array<double> &basis,
-                            const Vector &padata,
-                            Vector &eadata_int,
-                            Vector &eadata_ext,
-                            const bool add,
-                            const int d1d = 0,
-                            const int q1d = 0)
+static void EADGTraceAssemble2DInt(const int NF,
+                                   const Array<double> &basis,
+                                   const Vector &padata,
+                                   Vector &eadata_int,
+                                   Vector &eadata_ext,
+                                   const bool add,
+                                   const int d1d = 0,
+                                   const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -128,13 +128,13 @@ void EADGTraceAssemble2DInt(const int NF,
 }
 
 template<int T_D1D = 0, int T_Q1D = 0>
-void EADGTraceAssemble2DBdr(const int NF,
-                            const Array<double> &basis,
-                            const Vector &padata,
-                            Vector &eadata_bdr,
-                            const bool add,
-                            const int d1d = 0,
-                            const int q1d = 0)
+static void EADGTraceAssemble2DBdr(const int NF,
+                                   const Array<double> &basis,
+                                   const Vector &padata,
+                                   Vector &eadata_bdr,
+                                   const bool add,
+                                   const int d1d = 0,
+                                   const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -170,14 +170,14 @@ void EADGTraceAssemble2DBdr(const int NF,
 }
 
 template<int T_D1D = 0, int T_Q1D = 0>
-void EADGTraceAssemble3DInt(const int NF,
-                            const Array<double> &basis,
-                            const Vector &padata,
-                            Vector &eadata_int,
-                            Vector &eadata_ext,
-                            const bool add,
-                            const int d1d = 0,
-                            const int q1d = 0)
+static void EADGTraceAssemble3DInt(const int NF,
+                                   const Array<double> &basis,
+                                   const Vector &padata,
+                                   Vector &eadata_int,
+                                   Vector &eadata_ext,
+                                   const bool add,
+                                   const int d1d = 0,
+                                   const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -268,13 +268,13 @@ void EADGTraceAssemble3DInt(const int NF,
 }
 
 template<int T_D1D = 0, int T_Q1D = 0>
-void EADGTraceAssemble3DBdr(const int NF,
-                            const Array<double> &basis,
-                            const Vector &padata,
-                            Vector &eadata_bdr,
-                            const bool add,
-                            const int d1d = 0,
-                            const int q1d = 0)
+static void EADGTraceAssemble3DBdr(const int NF,
+                                   const Array<double> &basis,
+                                   const Vector &padata,
+                                   Vector &eadata_bdr,
+                                   const bool add,
+                                   const int d1d = 0,
+                                   const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;

fem/bilininteg_dgtrace_pa.cpp

@@ -19,16 +19,16 @@ using namespace std;
 namespace mfem
 {
 // PA DG Trace Integrator
-void PADGTraceSetup2D(const int Q1D,
-                      const int NF,
-                      const Array<double> &w,
-                      const Vector &det,
-                      const Vector &nor,
-                      const Vector &rho,
-                      const Vector &vel,
-                      const double alpha,
-                      const double beta,
-                      Vector &op)
+static void PADGTraceSetup2D(const int Q1D,
+                             const int NF,
+                             const Array<double> &w,
+                             const Vector &det,
+                             const Vector &nor,
+                             const Vector &rho,
+                             const Vector &vel,
+                             const double alpha,
+                             const double beta,
+                             Vector &op)
 {
    const int VDIM = 2;
 
@@ -61,16 +61,16 @@ void PADGTraceSetup2D(const int Q1D,
    });
 }
 
-void PADGTraceSetup3D(const int Q1D,
-                      const int NF,
-                      const Array<double> &w,
-                      const Vector &det,
-                      const Vector &nor,
-                      const Vector &rho,
-                      const Vector &vel,
-                      const double alpha,
-                      const double beta,
-                      Vector &op)
+static void PADGTraceSetup3D(const int Q1D,
+                             const int NF,
+                             const Array<double> &w,
+                             const Vector &det,
+                             const Vector &nor,
+                             const Vector &rho,
+                             const Vector &vel,
+                             const double alpha,
+                             const double beta,
+                             Vector &op)
 {
    const int VDIM = 3;
 
@@ -301,7 +301,7 @@ void DGTraceIntegrator::AssemblePABoundaryFaces(const FiniteElementSpace& fes)
 }
 
 // PA DGTrace Apply 2D kernel for Gauss-Lobatto/Bernstein
-template<int T_D1D = 0, int T_Q1D = 0>
+template<int T_D1D = 0, int T_Q1D = 0> static
 void PADGTraceApply2D(const int NF,
                       const Array<double> &b,
                       const Array<double> &bt,
@@ -392,7 +392,7 @@ void PADGTraceApply2D(const int NF,
 }
 
 // PA DGTrace Apply 3D kernel for Gauss-Lobatto/Bernstein
-template<int T_D1D = 0, int T_Q1D = 0>
+template<int T_D1D = 0, int T_Q1D = 0> static
 void PADGTraceApply3D(const int NF,
                       const Array<double> &b,
                       const Array<double> &bt,
@@ -537,7 +537,7 @@ void PADGTraceApply3D(const int NF,
 }
 
 // Optimized PA DGTrace Apply 3D kernel for Gauss-Lobatto/Bernstein
-template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
+template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0> static
 void SmemPADGTraceApply3D(const int NF,
                           const Array<double> &b,
                           const Array<double> &bt,
@@ -701,7 +701,7 @@ static void PADGTraceApply(const int dim,
 }
 
 // PA DGTrace Apply 2D kernel for Gauss-Lobatto/Bernstein
-template<int T_D1D = 0, int T_Q1D = 0>
+template<int T_D1D = 0, int T_Q1D = 0> static
 void PADGTraceApplyTranspose2D(const int NF,
                                const Array<double> &b,
                                const Array<double> &bt,
@@ -797,7 +797,7 @@ void PADGTraceApplyTranspose2D(const int NF,
 }
 
 // PA DGTrace Apply Transpose 3D kernel for Gauss-Lobatto/Bernstein
-template<int T_D1D = 0, int T_Q1D = 0>
+template<int T_D1D = 0, int T_Q1D = 0> static
 void PADGTraceApplyTranspose3D(const int NF,
                                const Array<double> &b,
                                const Array<double> &bt,
@@ -953,7 +953,7 @@ void PADGTraceApplyTranspose3D(const int NF,
 }
 
 // Optimized PA DGTrace Apply Transpose 3D kernel for Gauss-Lobatto/Bernstein
-template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
+template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0> static
 void SmemPADGTraceApplyTranspose3D(const int NF,
                                    const Array<double> &b,
                                    const Array<double> &bt,

fem/bilininteg_diffusion_ea.cpp

@@ -17,14 +17,14 @@ namespace mfem
 {
 
 template<int T_D1D = 0, int T_Q1D = 0>
-void EADiffusionAssemble1D(const int NE,
-                           const Array<double> &b,
-                           const Array<double> &g,
-                           const Vector &padata,
-                           Vector &eadata,
-                           const bool add,
-                           const int d1d = 0,
-                           const int q1d = 0)
+static void EADiffusionAssemble1D(const int NE,
+                                  const Array<double> &b,
+                                  const Array<double> &g,
+                                  const Vector &padata,
+                                  Vector &eadata,
+                                  const bool add,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -68,14 +68,14 @@ void EADiffusionAssemble1D(const int NE,
 }
 
 template<int T_D1D = 0, int T_Q1D = 0>
-void EADiffusionAssemble2D(const int NE,
-                           const Array<double> &b,
-                           const Array<double> &g,
-                           const Vector &padata,
-                           Vector &eadata,
-                           const bool add,
-                           const int d1d = 0,
-                           const int q1d = 0)
+static void EADiffusionAssemble2D(const int NE,
+                                  const Array<double> &b,
+                                  const Array<double> &g,
+                                  const Vector &padata,
+                                  Vector &eadata,
+                                  const bool add,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -145,14 +145,14 @@ void EADiffusionAssemble2D(const int NE,
 }
 
 template<int T_D1D = 0, int T_Q1D = 0>
-void EADiffusionAssemble3D(const int NE,
-                           const Array<double> &b,
-                           const Array<double> &g,
-                           const Vector &padata,
-                           Vector &eadata,
-                           const bool add,
-                           const int d1d = 0,
-                           const int q1d = 0)
+static void EADiffusionAssemble3D(const int NE,
+                                  const Array<double> &b,
+                                  const Array<double> &g,
+                                  const Vector &padata,
+                                  Vector &eadata,
+                                  const bool add,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;

fem/bilininteg_diffusion_pa.cpp

@@ -496,14 +496,14 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
 }
 
 template<int T_D1D = 0, int T_Q1D = 0>
-void PADiffusionDiagonal2D(const int NE,
-                           const bool symmetric,
-                           const Array<double> &b,
-                           const Array<double> &g,
-                           const Vector &d,
-                           Vector &y,
-                           const int d1d = 0,
-                           const int q1d = 0)
+static void PADiffusionDiagonal2D(const int NE,
+                                  const bool symmetric,
+                                  const Array<double> &b,
+                                  const Array<double> &g,
+                                  const Vector &d,
+                                  Vector &y,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -562,14 +562,14 @@ void PADiffusionDiagonal2D(const int NE,
 
 // Shared memory PA Diffusion Diagonal 2D kernel
 template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
-void SmemPADiffusionDiagonal2D(const int NE,
-                               const bool symmetric,
-                               const Array<double> &b_,
-                               const Array<double> &g_,
-                               const Vector &d_,
-                               Vector &y_,
-                               const int d1d = 0,
-                               const int q1d = 0)
+static void SmemPADiffusionDiagonal2D(const int NE,
+                                      const bool symmetric,
+                                      const Array<double> &b_,
+                                      const Array<double> &g_,
+                                      const Vector &d_,
+                                      Vector &y_,
+                                      const int d1d = 0,
+                                      const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -656,14 +656,14 @@ void SmemPADiffusionDiagonal2D(const int NE,
 }
 
 template<int T_D1D = 0, int T_Q1D = 0>
-void PADiffusionDiagonal3D(const int NE,
-                           const bool symmetric,
-                           const Array<double> &b,
-                           const Array<double> &g,
-                           const Vector &d,
-                           Vector &y,
-                           const int d1d = 0,
-                           const int q1d = 0)
+static void PADiffusionDiagonal3D(const int NE,
+                                  const bool symmetric,
+                                  const Array<double> &b,
+                                  const Array<double> &g,
+                                  const Vector &d,
+                                  Vector &y,
+                                  const int d1d = 0,
+                                  const int q1d = 0)
 {
    constexpr int DIM = 3;
    const int D1D = T_D1D ? T_D1D : d1d;
@@ -757,14 +757,14 @@ void PADiffusionDiagonal3D(const int NE,
 
 // Shared memory PA Diffusion Diagonal 3D kernel
 template<int T_D1D = 0, int T_Q1D = 0>
-void SmemPADiffusionDiagonal3D(const int NE,
-                               const bool symmetric,
-                               const Array<double> &b_,
-                               const Array<double> &g_,
-                               const Vector &d_,
-                               Vector &y_,
-                               const int d1d = 0,
-                               const int q1d = 0)
+static void SmemPADiffusionDiagonal3D(const int NE,
+                                      const bool symmetric,
+                                      const Array<double> &b_,
+                                      const Array<double> &g_,
+                                      const Vector &d_,
+                                      Vector &y_,
+                                      const int d1d = 0,
+                                      const int q1d = 0)
 {
    constexpr int DIM = 3;
    const int D1D = T_D1D ? T_D1D : d1d;
@@ -1034,17 +1034,17 @@ static void OccaPADiffusionApply3D(const int D1D,
 
 // PA Diffusion Apply 2D kernel
 template<int T_D1D = 0, int T_Q1D = 0>
-void PADiffusionApply2D(const int NE,
-                        const bool symmetric,
-                        const Array<double> &b_,
-                        const Array<double> &g_,
-                        const Array<double> &bt_,
-                        const Array<double> &gt_,
-                        const Vector &d_,
-                        const Vector &x_,
-                        Vector &y_,
-                        const int d1d = 0,
-                        const int q1d = 0)
+static void PADiffusionApply2D(const int NE,
+                               const bool symmetric,
+                               const Array<double> &b_,
+                               const Array<double> &g_,
+                               const Array<double> &bt_,
+                               const Array<double> &gt_,
+                               const Vector &d_,
+                               const Vector &x_,
+                               Vector &y_,
+                               const int d1d = 0,
+                               const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -1156,15 +1156,15 @@ void PADiffusionApply2D(const int NE,
 
 // Shared memory PA Diffusion Apply 2D kernel
 template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
-void SmemPADiffusionApply2D(const int NE,
-                            const bool symmetric,
-                            const Array<double> &b_,
-                            const Array<double> &g_,
-                            const Vector &d_,
-                            const Vector &x_,
-                            Vector &y_,
-                            const int d1d = 0,
-                            const int q1d = 0)
+static void SmemPADiffusionApply2D(const int NE,
+                                   const bool symmetric,
+                                   const Array<double> &b_,
+                                   const Array<double> &g_,
+                                   const Vector &d_,
+                                   const Vector &x_,
+                                   Vector &y_,
+                                   const int d1d = 0,
+                                   const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -1314,16 +1314,16 @@ void SmemPADiffusionApply2D(const int NE,
 
 // PA Diffusion Apply 3D kernel
 template<int T_D1D = 0, int T_Q1D = 0>
-void PADiffusionApply3D(const int NE,
-                        const bool symmetric,
-                        const Array<double> &b,
-                        const Array<double> &g,
-                        const Array<double> &bt,
-                        const Array<double> &gt,
-                        const Vector &d_,
-                        const Vector &x_,
-                        Vector &y_,
-                        int d1d = 0, int q1d = 0)
+static void PADiffusionApply3D(const int NE,
+                               const bool symmetric,
+                               const Array<double> &b,
+                               const Array<double> &g,
+                               const Array<double> &bt,
+                               const Array<double> &gt,
+                               const Vector &d_,
+                               const Vector &x_,
+                               Vector &y_,
+                               int d1d = 0, int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -1533,15 +1533,15 @@ static MFEM_HOST_DEVICE inline double sign(const int q, const int d)
 }
 
 template<int T_D1D = 0, int T_Q1D = 0>
-void SmemPADiffusionApply3D(const int NE,
-                            const bool symmetric,
-                            const Array<double> &b_,
-                            const Array<double> &g_,
-                            const Vector &d_,
-                            const Vector &x_,
-                            Vector &y_,
-                            const int d1d = 0,
-                            const int q1d = 0)
+static void SmemPADiffusionApply3D(const int NE,
+                                   const bool symmetric,
+                                   const Array<double> &b_,
+                                   const Array<double> &g_,
+                                   const Vector &d_,
+                                   const Vector &x_,
+                                   Vector &y_,
+                                   const int d1d = 0,
+                                   const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;

fem/bilininteg_divergence.cpp

@@ -21,12 +21,12 @@ namespace mfem
 // PA Divergence Integrator
 
 // PA Divergence Assemble 2D kernel
-void PADivergenceSetup2D(const int Q1D,
-                         const int NE,
-                         const Array<double> &w,
-                         const Vector &j,
-                         const double COEFF,
-                         Vector &op)
+static void PADivergenceSetup2D(const int Q1D,
+                                const int NE,
+                                const Array<double> &w,
+                                const Vector &j,
+                                const double COEFF,
+                                Vector &op)
 {
    const int NQ = Q1D*Q1D;
    auto W = w.Read();
@@ -51,12 +51,12 @@ void PADivergenceSetup2D(const int Q1D,
 }
 
 // PA Divergence Assemble 3D kernel
-void PADivergenceSetup3D(const int Q1D,
-                         const int NE,
-                         const Array<double> &w,
-                         const Vector &j,
-                         const double COEFF,
-                         Vector &op)
+static void PADivergenceSetup3D(const int Q1D,
+                                const int NE,
+                                const Array<double> &w,
+                                const Vector &j,
+                                const double COEFF,
+                                Vector &op)
 {
    const int NQ = Q1D*Q1D*Q1D;
    auto W = w.Read();
@@ -160,16 +160,16 @@ void VectorDivergenceIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
 
 // PA Divergence Apply 2D kernel
 template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0>
-void PADivergenceApply2D(const int NE,
-                         const Array<double> &b,
-                         const Array<double> &g,
-                         const Array<double> &bt,
-                         const Vector &op_,
-                         const Vector &x_,
-                         Vector &y_,
-                         const int tr_d1d = 0,
-                         const int te_d1d = 0,
-                         const int q1d = 0)
+static void PADivergenceApply2D(const int NE,
+                                const Array<double> &b,
+                                const Array<double> &g,
+                                const Array<double> &bt,
+                                const Vector &op_,
+                                const Vector &x_,
+                                Vector &y_,
+                                const int tr_d1d = 0,
+                                const int te_d1d = 0,
+                                const int q1d = 0)
 {
    const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
    const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -281,16 +281,16 @@ void PADivergenceApply2D(const int NE,
 // Shared memory PA Divergence Apply 2D kernel
 template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0,
          const int T_NBZ = 0>
-void SmemPADivergenceApply2D(const int NE,
-                             const Array<double> &b_,
-                             const Array<double> &g_,
-                             const Array<double> &bt_,
-                             const Vector &op_,
-                             const Vector &x_,
-                             Vector &y_,
-                             const int tr_d1d = 0,
-                             const int te_d1d = 0,
-                             const int q1d = 0)
+static void SmemPADivergenceApply2D(const int NE,
+                                    const Array<double> &b_,
+                                    const Array<double> &g_,
+                                    const Array<double> &bt_,
+                                    const Vector &op_,
+                                    const Vector &x_,
+                                    Vector &y_,
+                                    const int tr_d1d = 0,
+                                    const int te_d1d = 0,
+                                    const int q1d = 0)
 {
    // TODO
    MFEM_ASSERT(false, "SHARED MEM NOT PROGRAMMED YET");
@@ -298,16 +298,16 @@ void SmemPADivergenceApply2D(const int NE,
 
 // PA Divergence Apply 2D kernel transpose
 template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0>
-void PADivergenceApplyTranspose2D(const int NE,
-                                  const Array<double> &bt,
-                                  const Array<double> &gt,
-                                  const Array<double> &b,
-                                  const Vector &op_,
-                                  const Vector &x_,
-                                  Vector &y_,
-                                  const int tr_d1d = 0,
-                                  const int te_d1d = 0,
-                                  const int q1d = 0)
+static void PADivergenceApplyTranspose2D(const int NE,
+                                         const Array<double> &bt,
+                                         const Array<double> &gt,
+                                         const Array<double> &b,
+                                         const Vector &op_,
+                                         const Vector &x_,
+                                         Vector &y_,
+                                         const int tr_d1d = 0,
+                                         const int te_d1d = 0,
+                                         const int q1d = 0)
 {
    const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
    const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -414,16 +414,16 @@ void PADivergenceApplyTranspose2D(const int NE,
 
 // PA Vector Divergence Apply 3D kernel
 template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0>
-void PADivergenceApply3D(const int NE,
-                         const Array<double> &b,
-                         const Array<double> &g,
-                         const Array<double> &bt,
-                         const Vector &op_,
-                         const Vector &x_,
-                         Vector &y_,
-                         int tr_d1d = 0,
-                         int te_d1d = 0,
-                         int q1d = 0)
+static void PADivergenceApply3D(const int NE,
+                                const Array<double> &b,
+                                const Array<double> &g,
+                                const Array<double> &bt,
+                                const Vector &op_,
+                                const Vector &x_,
+                                Vector &y_,
+                                int tr_d1d = 0,
+                                int te_d1d = 0,
+                                int q1d = 0)
 {
    const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
    const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -597,16 +597,16 @@ void PADivergenceApply3D(const int NE,
 
 // PA Vector Divergence Apply 3D kernel
 template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0>
-void PADivergenceApplyTranspose3D(const int NE,
-                                  const Array<double> &bt,
-                                  const Array<double> &gt,
-                                  const Array<double> &b,
-                                  const Vector &op_,
-                                  const Vector &x_,
-                                  Vector &y_,
-                                  int tr_d1d = 0,
-                                  int te_d1d = 0,
-                                  int q1d = 0)
+static void PADivergenceApplyTranspose3D(const int NE,
+                                         const Array<double> &bt,
+                                         const Array<double> &gt,
+                                         const Array<double> &b,
+                                         const Vector &op_,
+                                         const Vector &x_,
+                                         Vector &y_,
+                                         int tr_d1d = 0,
+                                         int te_d1d = 0,
+                                         int q1d = 0)
 {
    const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
    const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -775,16 +775,16 @@ void PADivergenceApplyTranspose3D(const int NE,
 
 // Shared memory PA Vector Divergence Apply 3D kernel
 template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0>
-void SmemPADivergenceApply3D(const int NE,
-                             const Array<double> &b_,
-                             const Array<double> &g_,
-                             const Array<double> &bt_,
-                             const Vector &q_,
-                             const Vector &x_,
-                             Vector &y_,
-                             const int tr_d1d = 0,
-                             const int te_d1d = 0,
-                             const int q1d = 0)
+static void SmemPADivergenceApply3D(const int NE,
+                                    const Array<double> &b_,
+                                    const Array<double> &g_,
+                                    const Array<double> &bt_,
+                                    const Vector &q_,
+                                    const Vector &x_,
+                                    Vector &y_,
+                                    const int tr_d1d = 0,
+                                    const int te_d1d = 0,
+                                    const int q1d = 0)
 {
    const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
    const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;

fem/bilininteg_gradient.cpp

@@ -70,12 +70,12 @@ namespace mfem
    the \b MFEM_SHARED keyword for local arrays. */
 
 // PA Gradient Assemble 2D kernel
-void PAGradientSetup2D(const int Q1D,
-                       const int NE,
-                       const Array<double> &w,
-                       const Vector &j,
-                       const Vector &c,
-                       Vector &op)
+static void PAGradientSetup2D(const int Q1D,
+                              const int NE,
+                              const Array<double> &w,
+                              const Vector &j,
+                              const Vector &c,
+                              Vector &op)
 {
    const int NQ = Q1D*Q1D;
    auto W = w.Read();
@@ -105,12 +105,12 @@ void PAGradientSetup2D(const int Q1D,
 }
 
 // PA Gradient Assemble 3D kernel
-void PAGradientSetup3D(const int Q1D,
-                       const int NE,
-                       const Array<double> &w,
-                       const Vector &j,
-                       const Vector &c,
-                       Vector &op)
+static void PAGradientSetup3D(const int Q1D,
+                              const int NE,
+                              const Array<double> &w,
+                              const Vector &j,
+                              const Vector &c,
+                              Vector &op)
 {
    const int NQ = Q1D*Q1D*Q1D;
    auto W = w.Read();
@@ -254,16 +254,16 @@ void GradientIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
 
 // PA Gradient Apply 2D kernel
 template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
-void PAGradientApply2D(const int NE,
-                       const Array<double> &b,
-                       const Array<double> &g,
-                       const Array<double> &bt,
-                       const Vector &op_,
-                       const Vector &x_,
-                       Vector &y_,
-                       const int tr_d1d = 0,
-                       const int te_d1d = 0,
-                       const int q1d = 0)
+static void PAGradientApply2D(const int NE,
+                              const Array<double> &b,
+                              const Array<double> &g,
+                              const Array<double> &bt,
+                              const Vector &op_,
+                              const Vector &x_,
+                              Vector &y_,
+                              const int tr_d1d = 0,
+                              const int te_d1d = 0,
+                              const int q1d = 0)
 {
    const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
    const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -384,16 +384,16 @@ static void PAGradientApplyTranspose2D(const int NE,
 
 // PA Gradient Apply 3D kernel
 template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0>
-void PAGradientApply3D(const int NE,
-                       const Array<double> &b,
-                       const Array<double> &g,
-                       const Array<double> &bt,
-                       const Vector &op_,
-                       const Vector &x_,
-                       Vector &y_,
-                       int tr_d1d = 0,
-                       int te_d1d = 0,
-                       int q1d = 0)
+static void PAGradientApply3D(const int NE,
+                              const Array<double> &b,
+                              const Array<double> &g,
+                              const Array<double> &bt,
+                              const Vector &op_,
+                              const Vector &x_,
+                              Vector &y_,
+                              int tr_d1d = 0,
+                              int te_d1d = 0,
+                              int q1d = 0)
 {
    const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
    const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
@@ -579,16 +579,16 @@ static void PAGradientApplyTranspose3D(const int NE,
 
 // Shared memory PA Gradient Apply 3D kernel
 template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0>
-void SmemPAGradientApply3D(const int NE,
-                           const Array<double> &b_,
-                           const Array<double> &g_,
-                           const Array<double> &bt_,
-                           const Vector &d_,
-                           const Vector &x_,
-                           Vector &y_,
-                           const int tr_d1d = 0,
-                           const int te_d1d = 0,
-                           const int q1d = 0)
+static void SmemPAGradientApply3D(const int NE,
+                                  const Array<double> &b_,
+                                  const Array<double> &g_,
+                                  const Array<double> &bt_,
+                                  const Vector &d_,
+                                  const Vector &x_,
+                                  Vector &y_,
+                                  const int tr_d1d = 0,
+                                  const int te_d1d = 0,
+                                  const int q1d = 0)
 {
    const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
    const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;

fem/bilininteg_hcurl.cpp

@@ -791,12 +791,12 @@ void SmemPAHcurlMassApply3D(const int D1D,
 }
 
 // PA H(curl) curl-curl assemble 2D kernel
-void PACurlCurlSetup2D(const int Q1D,
-                       const int NE,
-                       const Array<double> &w,
-                       const Vector &j,
-                       Vector &coeff,
-                       Vector &op)
+static void PACurlCurlSetup2D(const int Q1D,
+                              const int NE,
+                              const Array<double> &w,
+                              const Vector &j,
+                              Vector &coeff,
+                              Vector &op)
 {
    const int NQ = Q1D*Q1D;
    auto W = w.Read();
@@ -818,13 +818,13 @@ void PACurlCurlSetup2D(const int Q1D,
 }
 
 // PA H(curl) curl-curl assemble 3D kernel
-void PACurlCurlSetup3D(const int Q1D,
-                       const int coeffDim,
-                       const int NE,
-                       const Array<double> &w,
-                       const Vector &j,
-                       Vector &coeff,
-                       Vector &op)
+static void PACurlCurlSetup3D(const int Q1D,
+                              const int coeffDim,
+                              const int NE,
+                              const Array<double> &w,
+                              const Vector &j,
+                              Vector &coeff,
+                              Vector &op)
 {
    const int NQ = Q1D*Q1D*Q1D;
    const bool symmetric = (coeffDim != 9);
@@ -1045,16 +1045,16 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
    }
 }
 
-void PACurlCurlApply2D(const int D1D,
-                       const int Q1D,
-                       const int NE,
-                       const Array<double> &bo,
-                       const Array<double> &bot,
-                       const Array<double> &gc,
-                       const Array<double> &gct,
-                       const Vector &pa_data,
-                       const Vector &x,
-                       Vector &y)
+static void PACurlCurlApply2D(const int D1D,
+                              const int Q1D,
+                              const int NE,
+                              const Array<double> &bo,
+                              const Array<double> &bot,
+                              const Array<double> &gc,
+                              const Array<double> &gct,
+                              const Vector &pa_data,
+                              const Vector &x,
+                              Vector &y)
 {
    constexpr static int VDIM = 2;
    constexpr static int MAX_D1D = HCURL_MAX_D1D;
@@ -1166,19 +1166,19 @@ void PACurlCurlApply2D(const int D1D,
 }
 
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-void PACurlCurlApply3D(const int D1D,
-                       const int Q1D,
-                       const bool symmetric,
-                       const int NE,
-                       const Array<double> &bo,
-                       const Array<double> &bc,
-                       const Array<double> &bot,
-                       const Array<double> &bct,
-                       const Array<double> &gc,
-                       const Array<double> &gct,
-                       const Vector &pa_data,
-                       const Vector &x,
-                       Vector &y)
+static void PACurlCurlApply3D(const int D1D,
+                              const int Q1D,
+                              const bool symmetric,
+                              const int NE,
+                              const Array<double> &bo,
+                              const Array<double> &bc,
+                              const Array<double> &bot,
+                              const Array<double> &bct,
+                              const Array<double> &gc,
+                              const Array<double> &gct,
+                              const Vector &pa_data,
+                              const Vector &x,
+                              Vector &y)
 {
    MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
    MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
@@ -1677,19 +1677,19 @@ void PACurlCurlApply3D(const int D1D,
 }
 
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-void SmemPACurlCurlApply3D(const int D1D,
-                           const int Q1D,
-                           const bool symmetric,
-                           const int NE,
-                           const Array<double> &bo,
-                           const Array<double> &bc,
-                           const Array<double> &bot,
-                           const Array<double> &bct,
-                           const Array<double> &gc,
-                           const Array<double> &gct,
-                           const Vector &pa_data,
-                           const Vector &x,
-                           Vector &y)
+static void SmemPACurlCurlApply3D(const int D1D,
+                                  const int Q1D,
+                                  const bool symmetric,
+                                  const int NE,
+                                  const Array<double> &bo,
+                                  const Array<double> &bc,
+                                  const Array<double> &bot,
+                                  const Array<double> &bct,
+                                  const Array<double> &gc,
+                                  const Array<double> &gct,
+                                  const Vector &pa_data,
+                                  const Vector &x,
+                                  Vector &y)
 {
    MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
    MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
@@ -2032,13 +2032,13 @@ void CurlCurlIntegrator::AddMultPA(const Vector &x, Vector &y) const
    }
 }
 
-void PACurlCurlAssembleDiagonal2D(const int D1D,
-                                  const int Q1D,
-                                  const int NE,
-                                  const Array<double> &bo,
-                                  const Array<double> &gc,
-                                  const Vector &pa_data,
-                                  Vector &diag)
+static void PACurlCurlAssembleDiagonal2D(const int D1D,
+                                         const int Q1D,
+                                         const int NE,
+                                         const Array<double> &bo,
+                                         const Array<double> &gc,
+                                         const Vector &pa_data,
+                                         Vector &diag)
 {
    constexpr static int VDIM = 2;
    constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
@@ -2087,16 +2087,16 @@ void PACurlCurlAssembleDiagonal2D(const int D1D,
 }
 
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-void PACurlCurlAssembleDiagonal3D(const int D1D,
-                                  const int Q1D,
-                                  const bool symmetric,
-                                  const int NE,
-                                  const Array<double> &bo,
-                                  const Array<double> &bc,
-                                  const Array<double> &go,
-                                  const Array<double> &gc,
-                                  const Vector &pa_data,
-                                  Vector &diag)
+static void PACurlCurlAssembleDiagonal3D(const int D1D,
+                                         const int Q1D,
+                                         const bool symmetric,
+                                         const int NE,
+                                         const Array<double> &bo,
+                                         const Array<double> &bc,
+                                         const Array<double> &go,
+                                         const Array<double> &gc,
+                                         const Vector &pa_data,
+                                         Vector &diag)
 {
    constexpr static int VDIM = 3;
    MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
@@ -2273,16 +2273,16 @@ void PACurlCurlAssembleDiagonal3D(const int D1D,
 }
 
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-void SmemPACurlCurlAssembleDiagonal3D(const int D1D,
-                                      const int Q1D,
-                                      const bool symmetric,
-                                      const int NE,
-                                      const Array<double> &bo,
-                                      const Array<double> &bc,
-                                      const Array<double> &go,
-                                      const Array<double> &gc,
-                                      const Vector &pa_data,
-                                      Vector &diag)
+static void SmemPACurlCurlAssembleDiagonal3D(const int D1D,
+                                             const int Q1D,
+                                             const bool symmetric,
+                                             const int NE,
+                                             const Array<double> &bo,
+                                             const Array<double> &bc,
+                                             const Array<double> &go,
+                                             const Array<double> &gc,
+                                             const Vector &pa_data,
+                                             Vector &diag)
 {
    MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
    MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
@@ -2955,18 +2955,18 @@ void MixedVectorCurlIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
 // Apply to x corresponding to DOF's in H(curl) (trial), whose curl is
 // integrated against H(curl) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-void PAHcurlL2Apply3D(const int D1D,
-                      const int Q1D,
-                      const int coeffDim,
-                      const int NE,
-                      const Array<double> &bo,
-                      const Array<double> &bc,
-                      const Array<double> &bot,
-                      const Array<double> &bct,
-                      const Array<double> &gc,
-                      const Vector &pa_data,
-                      const Vector &x,
-                      Vector &y)
+static void PAHcurlL2Apply3D(const int D1D,
+                             const int Q1D,
+                             const int coeffDim,
+                             const int NE,
+                             const Array<double> &bo,
+                             const Array<double> &bc,
+                             const Array<double> &bot,
+                             const Array<double> &bct,
+                             const Array<double> &gc,
+                             const Vector &pa_data,
+                             const Vector &x,
+                             Vector &y)
 {
    MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
    MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
@@ -3297,16 +3297,16 @@ void PAHcurlL2Apply3D(const int D1D,
 // Apply to x corresponding to DOF's in H(curl) (trial), whose curl is
 // integrated against H(curl) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-void SmemPAHcurlL2Apply3D(const int D1D,
-                          const int Q1D,
-                          const int coeffDim,
-                          const int NE,
-                          const Array<double> &bo,
-                          const Array<double> &bc,
-                          const Array<double> &gc,
-                          const Vector &pa_data,
-                          const Vector &x,
-                          Vector &y)
+static void SmemPAHcurlL2Apply3D(const int D1D,
+                                 const int Q1D,
+                                 const int coeffDim,
+                                 const int NE,
+                                 const Array<double> &bo,
+                                 const Array<double> &bc,
+                                 const Array<double> &gc,
+                                 const Vector &pa_data,
+                                 const Vector &x,
+                                 Vector &y)
 {
    MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
    MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
@@ -3585,18 +3585,18 @@ void SmemPAHcurlL2Apply3D(const int D1D,
 // Apply to x corresponding to DOF's in H(curl) (trial), whose curl is
 // integrated against H(div) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-void PAHcurlHdivApply3D(const int D1D,
-                        const int D1Dtest,
-                        const int Q1D,
-                        const int NE,
-                        const Array<double> &bo,
-                        const Array<double> &bc,
-                        const Array<double> &bot,
-                        const Array<double> &bct,
-                        const Array<double> &gc,
-                        const Vector &pa_data,
-                        const Vector &x,
-                        Vector &y)
+static void PAHcurlHdivApply3D(const int D1D,
+                               const int D1Dtest,
+                               const int Q1D,
+                               const int NE,
+                               const Array<double> &bo,
+                               const Array<double> &bc,
+                               const Array<double> &bot,
+                               const Array<double> &bct,
+                               const Array<double> &gc,
+                               const Vector &pa_data,
+                               const Vector &x,
+                               Vector &y)
 {
    MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
    MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
@@ -4071,18 +4071,18 @@ void MixedVectorWeakCurlIntegrator::AssemblePA(const FiniteElementSpace
 // Apply to x corresponding to DOF's in H(curl) (trial), integrated against curl
 // of H(curl) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-void PAHcurlL2Apply3DTranspose(const int D1D,
-                               const int Q1D,
-                               const int coeffDim,
-                               const int NE,
-                               const Array<double> &bo,
-                               const Array<double> &bc,
-                               const Array<double> &bot,
-                               const Array<double> &bct,
-                               const Array<double> &gct,
-                               const Vector &pa_data,
-                               const Vector &x,
-                               Vector &y)
+static void PAHcurlL2Apply3DTranspose(const int D1D,
+                                      const int Q1D,
+                                      const int coeffDim,
+                                      const int NE,
+                                      const Array<double> &bo,
+                                      const Array<double> &bc,
+                                      const Array<double> &bot,
+                                      const Array<double> &bct,
+                                      const Array<double> &gct,
+                                      const Vector &pa_data,
+                                      const Vector &x,
+                                      Vector &y)
 {
    // See PAHcurlL2Apply3D for comments.
 
@@ -4413,16 +4413,16 @@ void PAHcurlL2Apply3DTranspose(const int D1D,
 }
 
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-void SmemPAHcurlL2Apply3DTranspose(const int D1D,
-                                   const int Q1D,
-                                   const int coeffDim,
-                                   const int NE,
-                                   const Array<double> &bo,
-                                   const Array<double> &bc,
-                                   const Array<double> &gc,
-                                   const Vector &pa_data,
-                                   const Vector &x,
-                                   Vector &y)
+static void SmemPAHcurlL2Apply3DTranspose(const int D1D,
+                                          const int Q1D,
+                                          const int coeffDim,
+                                          const int NE,
+                                          const Array<double> &bo,
+                                          const Array<double> &bc,
+                                          const Array<double> &gc,
+                                          const Vector &pa_data,
+                                          const Vector &x,
+                                          Vector &y)
 {
    MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
    MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
@@ -4675,13 +4675,13 @@ void MixedVectorWeakCurlIntegrator::AddMultPA(const Vector &x, Vector &y) const
 // Apply to x corresponding to DOFs in H^1 (domain) the (topological) gradient
 // to get a dof in H(curl) (range). You can think of the range as the "test" space
 // and the domain as the "trial" space, but there's no integration.
-void PAHcurlApplyGradient2D(const int c_dofs1D,
-                            const int o_dofs1D,
-                            const int NE,
-                            const Array<double> &B_,
-                            const Array<double> &G_,
-                            const Vector &x_,
-                            Vector &y_)
+static void PAHcurlApplyGradient2D(const int c_dofs1D,
+                                   const int o_dofs1D,
+                                   const int NE,
+                                   const Array<double> &B_,
+                                   const Array<double> &G_,
+                                   const Vector &x_,
+                                   Vector &y_)
 {
    auto B = Reshape(B_.Read(), c_dofs1D, c_dofs1D);
    auto G = Reshape(G_.Read(), o_dofs1D, c_dofs1D);
@@ -4753,12 +4753,12 @@ void PAHcurlApplyGradient2D(const int c_dofs1D,
 }
 
 // Specialization of PAHcurlApplyGradient2D to the case where B is identity
-void PAHcurlApplyGradient2DBId(const int c_dofs1D,
-                               const int o_dofs1D,
-                               const int NE,
-                               const Array<double> &G_,
-                               const Vector &x_,
-                               Vector &y_)
+static void PAHcurlApplyGradient2DBId(const int c_dofs1D,
+                                      const int o_dofs1D,
+                                      const int NE,
+                                      const Array<double> &G_,
+                                      const Vector &x_,
+                                      Vector &y_)
 {
    auto G = Reshape(G_.Read(), o_dofs1D, c_dofs1D);
 
@@ -4822,7 +4822,7 @@ void PAHcurlApplyGradient2DBId(const int c_dofs1D,
    });
 }
 
-void PAHcurlApplyGradientTranspose2D(
+static void PAHcurlApplyGradientTranspose2D(
    const int c_dofs1D, const int o_dofs1D, const int NE,
    const Array<double> &B_, const Array<double> &G_,
    const Vector &x_, Vector &y_)
@@ -4898,7 +4898,7 @@ void PAHcurlApplyGradientTranspose2D(
 
 // Specialization of PAHcurlApplyGradientTranspose2D to the case where
 // B is identity
-void PAHcurlApplyGradientTranspose2DBId(
+static void PAHcurlApplyGradientTranspose2DBId(
    const int c_dofs1D, const int o_dofs1D, const int NE,
    const Array<double> &G_,
    const Vector &x_, Vector &y_)
@@ -4965,13 +4965,13 @@ void PAHcurlApplyGradientTranspose2DBId(
    });
 }
 
-void PAHcurlApplyGradient3D(const int c_dofs1D,
-                            const int o_dofs1D,
-                            const int NE,
-                            const Array<double> &B_,
-                            const Array<double> &G_,
-                            const Vector &x_,
-                            Vector &y_)
+static void PAHcurlApplyGradient3D(const int c_dofs1D,
+                                   const int o_dofs1D,
+                                   const int NE,
+                                   const Array<double> &B_,
+                                   const Array<double> &G_,
+                                   const Vector &x_,
+                                   Vector &y_)
 {
    auto B = Reshape(B_.Read(), c_dofs1D, c_dofs1D);
    auto G = Reshape(G_.Read(), o_dofs1D, c_dofs1D);
@@ -5154,12 +5154,12 @@ void PAHcurlApplyGradient3D(const int c_dofs1D,
 }
 
 // Specialization of PAHcurlApplyGradient3D to the case where
-void PAHcurlApplyGradient3DBId(const int c_dofs1D,
-                               const int o_dofs1D,
-                               const int NE,
-                               const Array<double> &G_,
-                               const Vector &x_,
-                               Vector &y_)
+static void PAHcurlApplyGradient3DBId(const int c_dofs1D,
+                                      const int o_dofs1D,
+                                      const int NE,
+                                      const Array<double> &G_,
+                                      const Vector &x_,
+                                      Vector &y_)
 {
    auto G = Reshape(G_.Read(), o_dofs1D, c_dofs1D);
 
@@ -5322,7 +5322,7 @@ void PAHcurlApplyGradient3DBId(const int c_dofs1D,
    });
 }
 
-void PAHcurlApplyGradientTranspose3D(
+static void PAHcurlApplyGradientTranspose3D(
    const int c_dofs1D, const int o_dofs1D, const int NE,
    const Array<double> &B_, const Array<double> &G_,
    const Vector &x_, Vector &y_)
@@ -5507,7 +5507,7 @@ void PAHcurlApplyGradientTranspose3D(
 }
 
 // Specialization of PAHcurlApplyGradientTranspose3D to the case where
-void PAHcurlApplyGradientTranspose3DBId(
+static void PAHcurlApplyGradientTranspose3DBId(
    const int c_dofs1D, const int o_dofs1D, const int NE,
    const Array<double> &G_,
    const Vector &x_, Vector &y_)
@@ -5789,14 +5789,14 @@ void GradientInterpolator::AddMultTransposePA(const Vector &x, Vector &y) const
    }
 }
 
-void PAHcurlVecH1IdentityApply3D(const int c_dofs1D,
-                                 const int o_dofs1D,
-                                 const int NE,
-                                 const Array<double> &Bclosed,
-                                 const Array<double> &Bopen,
-                                 const Vector &pa_data,
-                                 const Vector &x_,
-                                 Vector &y_)
+static void PAHcurlVecH1IdentityApply3D(const int c_dofs1D,
+                                        const int o_dofs1D,
+                                        const int NE,
+                                        const Array<double> &Bclosed,
+                                        const Array<double> &Bopen,
+                                        const Vector &pa_data,
+                                        const Vector &x_,
+                                        Vector &y_)
 {
    auto Bc = Reshape(Bclosed.Read(), c_dofs1D, c_dofs1D);
    auto Bo = Reshape(Bopen.Read(), o_dofs1D, c_dofs1D);
@@ -6002,14 +6002,14 @@ void PAHcurlVecH1IdentityApply3D(const int c_dofs1D,
    });
 }
 
-void PAHcurlVecH1IdentityApplyTranspose3D(const int c_dofs1D,
-                                          const int o_dofs1D,
-                                          const int NE,
-                                          const Array<double> &Bclosed,
-                                          const Array<double> &Bopen,
-                                          const Vector &pa_data,
-                                          const Vector &x_,
-                                          Vector &y_)
+static void PAHcurlVecH1IdentityApplyTranspose3D(const int c_dofs1D,
+                                                 const int o_dofs1D,
+                                                 const int NE,
+                                                 const Array<double> &Bclosed,
+                                                 const Array<double> &Bopen,
+                                                 const Vector &pa_data,
+                                                 const Vector &x_,
+                                                 Vector &y_)
 {
    auto Bc = Reshape(Bclosed.Read(), c_dofs1D, c_dofs1D);
    auto Bo = Reshape(Bopen.Read(), o_dofs1D, c_dofs1D);
@@ -6228,14 +6228,14 @@ void PAHcurlVecH1IdentityApplyTranspose3D(const int c_dofs1D,
    });
 }
 
-void PAHcurlVecH1IdentityApply2D(const int c_dofs1D,
-                                 const int o_dofs1D,
-                                 const int NE,
-                                 const Array<double> &Bclosed,
-                                 const Array<double> &Bopen,
-                                 const Vector &pa_data,
-                                 const Vector &x_,
-                                 Vector &y_)
+static void PAHcurlVecH1IdentityApply2D(const int c_dofs1D,
+                                        const int o_dofs1D,
+                                        const int NE,
+                                        const Array<double> &Bclosed,
+                                        const Array<double> &Bopen,
+                                        const Vector &pa_data,
+                                        const Vector &x_,
+                                        Vector &y_)
 {
    auto Bc = Reshape(Bclosed.Read(), c_dofs1D, c_dofs1D);
    auto Bo = Reshape(Bopen.Read(), o_dofs1D, c_dofs1D);
@@ -6327,14 +6327,14 @@ void PAHcurlVecH1IdentityApply2D(const int c_dofs1D,
    });
 }
 
-void PAHcurlVecH1IdentityApplyTranspose2D(const int c_dofs1D,
-                                          const int o_dofs1D,
-                                          const int NE,
-                                          const Array<double> &Bclosed,
-                                          const Array<double> &Bopen,
-                                          const Vector &pa_data,
-                                          const Vector &x_,
-                                          Vector &y_)
+static void PAHcurlVecH1IdentityApplyTranspose2D(const int c_dofs1D,
+                                                 const int o_dofs1D,
+                                                 const int NE,
+                                                 const Array<double> &Bclosed,
+                                                 const Array<double> &Bopen,
+                                                 const Vector &pa_data,
+                                                 const Vector &x_,
+                                                 Vector &y_)
 {
    auto Bc = Reshape(Bclosed.Read(), c_dofs1D, c_dofs1D);
    auto Bo = Reshape(Bopen.Read(), o_dofs1D, c_dofs1D);

fem/bilininteg_hdiv.cpp

@@ -539,12 +539,12 @@ void PAHdivMassApply3D(const int D1D,
 
 // PA H(div) div-div assemble 2D kernel
 // NOTE: this is identical to PACurlCurlSetup3D
-void PADivDivSetup2D(const int Q1D,
-                     const int NE,
-                     const Array<double> &w,
-                     const Vector &j,
-                     Vector &coeff_,
-                     Vector &op)
+static void PADivDivSetup2D(const int Q1D,
+                            const int NE,
+                            const Array<double> &w,
+                            const Vector &j,
+                            Vector &coeff_,
+                            Vector &op)
 {
    const int NQ = Q1D*Q1D;
    auto W = w.Read();
@@ -565,12 +565,12 @@ void PADivDivSetup2D(const int Q1D,
    });
 }
 
-void PADivDivSetup3D(const int Q1D,
-                     const int NE,
-                     const Array<double> &w,
-                     const Vector &j,
-                     Vector &coeff_,
-                     Vector &op)
+static void PADivDivSetup3D(const int Q1D,
+                            const int NE,
+                            const Array<double> &w,
+                            const Vector &j,
+                            Vector &coeff_,
+                            Vector &op)
 {
    const int NQ = Q1D*Q1D*Q1D;
    auto W = w.Read();
@@ -599,16 +599,16 @@ void PADivDivSetup3D(const int Q1D,
    });
 }
 
-void PADivDivApply2D(const int D1D,
-                     const int Q1D,
-                     const int NE,
-                     const Array<double> &Bo_,
-                     const Array<double> &Gc_,
-                     const Array<double> &Bot_,
-                     const Array<double> &Gct_,
-                     const Vector &op_,
-                     const Vector &x_,
-                     Vector &y_)
+static void PADivDivApply2D(const int D1D,
+                            const int Q1D,
+                            const int NE,
+                            const Array<double> &Bo_,
+                            const Array<double> &Gc_,
+                            const Array<double> &Bot_,
+                            const Array<double> &Gct_,
+                            const Vector &op_,
+                            const Vector &x_,
+                            Vector &y_)
 {
    constexpr static int VDIM = 2;
    constexpr static int MAX_D1D = HDIV_MAX_D1D;
@@ -718,16 +718,16 @@ void PADivDivApply2D(const int D1D,
    }); // end of element loop
 }
 
-void PADivDivApply3D(const int D1D,
-                     const int Q1D,
-                     const int NE,
-                     const Array<double> &Bo_,
-                     const Array<double> &Gc_,
-                     const Array<double> &Bot_,
-                     const Array<double> &Gct_,
-                     const Vector &op_,
-                     const Vector &x_,
-                     Vector &y_)
+static void PADivDivApply3D(const int D1D,
+                            const int Q1D,
+                            const int NE,
+                            const Array<double> &Bo_,
+                            const Array<double> &Gc_,
+                            const Array<double> &Bot_,
+                            const Array<double> &Gct_,
+                            const Vector &op_,
+                            const Vector &x_,
+                            Vector &y_)
 {
    MFEM_VERIFY(D1D <= HDIV_MAX_D1D, "Error: D1D > HDIV_MAX_D1D");
    MFEM_VERIFY(Q1D <= HDIV_MAX_Q1D, "Error: Q1D > HDIV_MAX_Q1D");
@@ -967,13 +967,13 @@ void DivDivIntegrator::AddMultPA(const Vector &x, Vector &y) const
    }
 }
 
-void PADivDivAssembleDiagonal2D(const int D1D,
-                                const int Q1D,
-                                const int NE,
-                                const Array<double> &Bo_,
-                                const Array<double> &Gc_,
-                                const Vector &op_,
-                                Vector &diag_)
+static void PADivDivAssembleDiagonal2D(const int D1D,
+                                       const int Q1D,
+                                       const int NE,
+                                       const Array<double> &Bo_,
+                                       const Array<double> &Gc_,
+                                       const Vector &op_,
+                                       Vector &diag_)
 {
    constexpr static int VDIM = 2;
    constexpr static int MAX_Q1D = HDIV_MAX_Q1D;
@@ -1023,13 +1023,13 @@ void PADivDivAssembleDiagonal2D(const int D1D,
    });
 }
 
-void PADivDivAssembleDiagonal3D(const int D1D,
-                                const int Q1D,
-                                const int NE,
-                                const Array<double> &Bo_,
-                                const Array<double> &Gc_,
-                                const Vector &op_,
-                                Vector &diag_)
+static void PADivDivAssembleDiagonal3D(const int D1D,
+                                       const int Q1D,
+                                       const int NE,
+                                       const Array<double> &Bo_,
+                                       const Array<double> &Gc_,
+                                       const Vector &op_,
+                                       Vector &diag_)
 {
    MFEM_VERIFY(D1D <= HDIV_MAX_D1D, "Error: D1D > HDIV_MAX_D1D");
    MFEM_VERIFY(Q1D <= HDIV_MAX_Q1D, "Error: Q1D > HDIV_MAX_Q1D");
@@ -1104,11 +1104,11 @@ void DivDivIntegrator::AssembleDiagonalPA(Vector& diag)
 }
 
 // PA H(div)-L2 (div u, p) assemble 2D kernel
-void PADivL2Setup2D(const int Q1D,
-                    const int NE,
-                    const Array<double> &w,
-                    Vector &coeff_,
-                    Vector &op)
+static void PADivL2Setup2D(const int Q1D,
+                           const int NE,
+                           const Array<double> &w,
+                           Vector &coeff_,
+                           Vector &op)
 {
    const int NQ = Q1D*Q1D;
    auto W = w.Read();
@@ -1123,11 +1123,11 @@ void PADivL2Setup2D(const int Q1D,
    });
 }
 
-void PADivL2Setup3D(const int Q1D,
-                    const int NE,
-                    const Array<double> &w,
-                    Vector &coeff_,
-                    Vector &op)
+static void PADivL2Setup3D(const int Q1D,
+                           const int NE,
+                           const Array<double> &w,
+                           Vector &coeff_,
+                           Vector &op)
 {
    const int NQ = Q1D*Q1D*Q1D;
    auto W = w.Read();
@@ -1225,16 +1225,16 @@ VectorFEDivergenceIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
 
 // Apply to x corresponding to DOF's in H(div) (trial), whose divergence is
 // integrated against L_2 test functions corresponding to y.
-void PAHdivL2Apply3D(const int D1D,
-                     const int Q1D,
-                     const int L2D1D,
-                     const int NE,
-                     const Array<double> &Bo_,
-                     const Array<double> &Gc_,
-                     const Array<double> &L2Bot_,
-                     const Vector &op_,
-                     const Vector &x_,
-                     Vector &y_)
+static void PAHdivL2Apply3D(const int D1D,
+                            const int Q1D,
+                            const int L2D1D,
+                            const int NE,
+                            const Array<double> &Bo_,
+                            const Array<double> &Gc_,
+                            const Array<double> &L2Bot_,
+                            const Vector &op_,
+                            const Vector &x_,
+                            Vector &y_)
 {
    MFEM_VERIFY(D1D <= HDIV_MAX_D1D, "Error: D1D > HDIV_MAX_D1D");
    MFEM_VERIFY(Q1D <= HDIV_MAX_Q1D, "Error: Q1D > HDIV_MAX_Q1D");
@@ -1388,16 +1388,16 @@ void PAHdivL2Apply3D(const int D1D,
 
 // Apply to x corresponding to DOF's in H(div) (trial), whose divergence is
 // integrated against L_2 test functions corresponding to y.
-void PAHdivL2Apply2D(const int D1D,
-                     const int Q1D,
-                     const int L2D1D,
-                     const int NE,
-                     const Array<double> &Bo_,
-                     const Array<double> &Gc_,
-                     const Array<double> &L2Bot_,
-                     const Vector &op_,
-                     const Vector &x_,
-                     Vector &y_)
+static void PAHdivL2Apply2D(const int D1D,
+                            const int Q1D,
+                            const int L2D1D,
+                            const int NE,
+                            const Array<double> &Bo_,
+                            const Array<double> &Gc_,
+                            const Array<double> &L2Bot_,
+                            const Vector &op_,
+                            const Vector &x_,
+                            Vector &y_)
 {
    constexpr static int VDIM = 2;
    constexpr static int MAX_D1D = HDIV_MAX_D1D;
@@ -1494,16 +1494,16 @@ void PAHdivL2Apply2D(const int D1D,
    }); // end of element loop
 }
 
-void PAHdivL2ApplyTranspose3D(const int D1D,
-                              const int Q1D,
-                              const int L2D1D,
-                              const int NE,
-                              const Array<double> &L2Bo_,
-                              const Array<double> &Gct_,
-                              const Array<double> &Bot_,
-                              const Vector &op_,
-                              const Vector &x_,
-                              Vector &y_)
+static void PAHdivL2ApplyTranspose3D(const int D1D,
+                                     const int Q1D,
+                                     const int L2D1D,
+                                     const int NE,
+                                     const Array<double> &L2Bo_,
+                                     const Array<double> &Gct_,
+                                     const Array<double> &Bot_,
+                                     const Vector &op_,
+                                     const Vector &x_,
+                                     Vector &y_)
 {
    MFEM_VERIFY(D1D <= HDIV_MAX_D1D, "Error: D1D > HDIV_MAX_D1D");
    MFEM_VERIFY(Q1D <= HDIV_MAX_Q1D, "Error: Q1D > HDIV_MAX_Q1D");
@@ -1656,16 +1656,16 @@ void PAHdivL2ApplyTranspose3D(const int D1D,
    }); // end of element loop
 }
 
-void PAHdivL2ApplyTranspose2D(const int D1D,
-                              const int Q1D,
-                              const int L2D1D,
-                              const int NE,
-                              const Array<double> &L2Bo_,
-                              const Array<double> &Gct_,
-                              const Array<double> &Bot_,
-                              const Vector &op_,
-                              const Vector &x_,
-                              Vector &y_)
+static void PAHdivL2ApplyTranspose2D(const int D1D,
+                                     const int Q1D,
+                                     const int L2D1D,
+                                     const int NE,
+                                     const Array<double> &L2Bo_,
+                                     const Array<double> &Gct_,
+                                     const Array<double> &Bot_,
+                                     const Vector &op_,
+                                     const Vector &x_,
+                                     Vector &y_)
 {
    constexpr static int VDIM = 2;
    constexpr static int MAX_D1D = HDIV_MAX_D1D;
@@ -1791,16 +1791,16 @@ void VectorFEDivergenceIntegrator::AddMultTransposePA(const Vector &x,
    }
 }
 
-void PAHdivL2AssembleDiagonal_ADAt_3D(const int D1D,
-                                      const int Q1D,
-                                      const int L2D1D,
-                                      const int NE,
-                                      const Array<double> &L2Bo_,
-                                      const Array<double> &Gct_,
-                                      const Array<double> &Bot_,
-                                      const Vector &op_,
-                                      const Vector &D_,
-                                      Vector &diag_)
+static void PAHdivL2AssembleDiagonal_ADAt_3D(const int D1D,
+                                             const int Q1D,
+                                             const int L2D1D,
+                                             const int NE,
+                                             const Array<double> &L2Bo_,
+                                             const Array<double> &Gct_,
+                                             const Array<double> &Bot_,
+                                             const Vector &op_,
+                                             const Vector &D_,
+                                             Vector &diag_)
 {
    MFEM_VERIFY(D1D <= HDIV_MAX_D1D, "Error: D1D > HDIV_MAX_D1D");
    MFEM_VERIFY(Q1D <= HDIV_MAX_Q1D, "Error: Q1D > HDIV_MAX_Q1D");
@@ -1916,16 +1916,16 @@ void PAHdivL2AssembleDiagonal_ADAt_3D(const int D1D,
    }); // end of element loop
 }
 
-void PAHdivL2AssembleDiagonal_ADAt_2D(const int D1D,
-                                      const int Q1D,
-                                      const int L2D1D,
-                                      const int NE,
-                                      const Array<double> &L2Bo_,
-                                      const Array<double> &Gct_,
-                                      const Array<double> &Bot_,
-                                      const Vector &op_,
-                                      const Vector &D_,
-                                      Vector &diag_)
+static void PAHdivL2AssembleDiagonal_ADAt_2D(const int D1D,
+                                             const int Q1D,
+                                             const int L2D1D,
+                                             const int NE,
+                                             const Array<double> &L2Bo_,
+                                             const Array<double> &Gct_,
+                                             const Array<double> &Bot_,
+                                             const Vector &op_,
+                                             const Vector &D_,
+                                             Vector &diag_)
 {
    constexpr static int VDIM = 2;
 

fem/bilininteg_mass_ea.cpp

@@ -17,13 +17,13 @@ namespace mfem
 {
 
 template<int T_D1D = 0, int T_Q1D = 0>
-void EAMassAssemble1D(const int NE,
-                      const Array<double> &basis,
-                      const Vector &padata,
-                      Vector &eadata,
-                      const bool add,
-                      const int d1d = 0,
-                      const int q1d = 0)
+static void EAMassAssemble1D(const int NE,
+                             const Array<double> &basis,
+                             const Vector &padata,
+                             Vector &eadata,
+                             const bool add,
+                             const int d1d = 0,
+                             const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -67,13 +67,13 @@ void EAMassAssemble1D(const int NE,
 }
 
 template<int T_D1D = 0, int T_Q1D = 0>
-void EAMassAssemble2D(const int NE,
-                      const Array<double> &basis,
-                      const Vector &padata,
-                      Vector &eadata,
-                      const bool add,
-                      const int d1d = 0,
-                      const int q1d = 0)
+static void EAMassAssemble2D(const int NE,
+                             const Array<double> &basis,
+                             const Vector &padata,
+                             Vector &eadata,
+                             const bool add,
+                             const int d1d = 0,
+                             const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -139,13 +139,13 @@ void EAMassAssemble2D(const int NE,
 }
 
 template<int T_D1D = 0, int T_Q1D = 0>
-void EAMassAssemble3D(const int NE,
-                      const Array<double> &basis,
-                      const Vector &padata,
-                      Vector &eadata,
-                      const bool add,
-                      const int d1d = 0,
-                      const int q1d = 0)
+static void EAMassAssemble3D(const int NE,
+                             const Array<double> &basis,
+                             const Vector &padata,
+                             Vector &eadata,
+                             const bool add,
+                             const int d1d = 0,
+                             const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;

fem/bilininteg_mass_pa.cpp

@@ -155,12 +155,12 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 }
 
 template<int T_D1D = 0, int T_Q1D = 0>
-void PAMassAssembleDiagonal2D(const int NE,
-                              const Array<double> &b,
-                              const Vector &d,
-                              Vector &y,
-                              const int d1d = 0,
-                              const int q1d = 0)
+static void PAMassAssembleDiagonal2D(const int NE,
+                                     const Array<double> &b,
+                                     const Vector &d,
+                                     Vector &y,
+                                     const int d1d = 0,
+                                     const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -201,12 +201,12 @@ void PAMassAssembleDiagonal2D(const int NE,
 }
 
 template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
-void SmemPAMassAssembleDiagonal2D(const int NE,
-                                  const Array<double> &b_,
-                                  const Vector &d_,
-                                  Vector &y_,
-                                  const int d1d = 0,
-                                  const int q1d = 0)
+static void SmemPAMassAssembleDiagonal2D(const int NE,
+                                         const Array<double> &b_,
+                                         const Vector &d_,
+                                         Vector &y_,
+                                         const int d1d = 0,
+                                         const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -267,12 +267,12 @@ void SmemPAMassAssembleDiagonal2D(const int NE,
 }
 
 template<int T_D1D = 0, int T_Q1D = 0>
-void PAMassAssembleDiagonal3D(const int NE,
-                              const Array<double> &b,
-                              const Vector &d,
-                              Vector &y,
-                              const int d1d = 0,
-                              const int q1d = 0)
+static void PAMassAssembleDiagonal3D(const int NE,
+                                     const Array<double> &b,
+                                     const Vector &d,
+                                     Vector &y,
+                                     const int d1d = 0,
+                                     const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -336,12 +336,12 @@ void PAMassAssembleDiagonal3D(const int NE,
 }
 
 template<int T_D1D = 0, int T_Q1D = 0>
-void SmemPAMassAssembleDiagonal3D(const int NE,
-                                  const Array<double> &b_,
-                                  const Vector &d_,
-                                  Vector &y_,
-                                  const int d1d = 0,
-                                  const int q1d = 0)
+static void SmemPAMassAssembleDiagonal3D(const int NE,
+                                         const Array<double> &b_,
+                                         const Vector &d_,
+                                         Vector &y_,
+                                         const int d1d = 0,
+                                         const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -569,14 +569,14 @@ static void OccaPAMassApply3D(const int D1D,
 #endif // MFEM_USE_OCCA
 
 template<int T_D1D = 0, int T_Q1D = 0>
-void PAMassApply2D(const int NE,
-                   const Array<double> &b_,
-                   const Array<double> &bt_,
-                   const Vector &d_,
-                   const Vector &x_,
-                   Vector &y_,
-                   const int d1d = 0,
-                   const int q1d = 0)
+static void PAMassApply2D(const int NE,
+                          const Array<double> &b_,
+                          const Array<double> &bt_,
+                          const Vector &d_,
+                          const Vector &x_,
+                          Vector &y_,
+                          const int d1d = 0,
+                          const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -661,14 +661,14 @@ void PAMassApply2D(const int NE,
 }
 
 template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
-void SmemPAMassApply2D(const int NE,
-                       const Array<double> &b_,
-                       const Array<double> &bt_,
-                       const Vector &d_,
-                       const Vector &x_,
-                       Vector &y_,
-                       const int d1d = 0,
-                       const int q1d = 0)
+static void SmemPAMassApply2D(const int NE,
+                              const Array<double> &b_,
+                              const Array<double> &bt_,
+                              const Vector &d_,
+                              const Vector &x_,
+                              Vector &y_,
+                              const int d1d = 0,
+                              const int q1d = 0)
 {
    MFEM_CONTRACT_VAR(bt_);
    const int D1D = T_D1D ? T_D1D : d1d;
@@ -784,14 +784,14 @@ void SmemPAMassApply2D(const int NE,
 }
 
 template<int T_D1D = 0, int T_Q1D = 0>
-void PAMassApply3D(const int NE,
-                   const Array<double> &b_,
-                   const Array<double> &bt_,
-                   const Vector &d_,
-                   const Vector &x_,
-                   Vector &y_,
-                   const int d1d = 0,
-                   const int q1d = 0)
+static void PAMassApply3D(const int NE,
+                          const Array<double> &b_,
+                          const Array<double> &bt_,
+                          const Vector &d_,
+                          const Vector &x_,
+                          Vector &y_,
+                          const int d1d = 0,
+                          const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -925,14 +925,14 @@ void PAMassApply3D(const int NE,
 }
 
 template<int T_D1D = 0, int T_Q1D = 0>
-void SmemPAMassApply3D(const int NE,
-                       const Array<double> &b_,
-                       const Array<double> &bt_,
-                       const Vector &d_,
-                       const Vector &x_,
-                       Vector &y_,
-                       const int d1d = 0,
-                       const int q1d = 0)
+static void SmemPAMassApply3D(const int NE,
+                              const Array<double> &b_,
+                              const Array<double> &bt_,
+                              const Vector &d_,
+                              const Vector &x_,
+                              Vector &y_,
+                              const int d1d = 0,
+                              const int q1d = 0)
 {
    MFEM_CONTRACT_VAR(bt_);
    const int D1D = T_D1D ? T_D1D : d1d;

fem/bilininteg_vecdiffusion.cpp

@@ -22,12 +22,12 @@ namespace mfem
 // PA Vector Diffusion Integrator
 
 // PA Diffusion Assemble 2D kernel
-void PAVectorDiffusionSetup2D(const int Q1D,
-                              const int NE,
-                              const Array<double> &w,
-                              const Vector &j,
-                              const Vector &c,
-                              Vector &op)
+static void PAVectorDiffusionSetup2D(const int Q1D,
+                                     const int NE,
+                                     const Array<double> &w,
+                                     const Vector &j,
+                                     const Vector &c,
+                                     Vector &op)
 {
    const int NQ = Q1D*Q1D;
    auto W = w.Read();
@@ -59,12 +59,12 @@ void PAVectorDiffusionSetup2D(const int Q1D,
 }
 
 // PA Diffusion Assemble 3D kernel
-void PAVectorDiffusionSetup3D(const int Q1D,
-                              const int NE,
-                              const Array<double> &w,
-                              const Vector &j,
-                              const Vector &c,
-                              Vector &op)
+static void PAVectorDiffusionSetup3D(const int Q1D,
+                                     const int NE,
+                                     const Array<double> &w,
+                                     const Vector &j,
+                                     const Vector &c,
+                                     Vector &op)
 {
    const int NQ = Q1D*Q1D*Q1D;
    auto W = w.Read();
@@ -251,7 +251,7 @@ void VectorDiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
 }
 
 // PA Diffusion Apply 2D kernel
-template<int T_D1D = 0, int T_Q1D = 0, int T_VDIM = 0>
+template<int T_D1D = 0, int T_Q1D = 0, int T_VDIM = 0> static
 void PAVectorDiffusionApply2D(const int NE,
                               const Array<double> &b,
                               const Array<double> &g,
@@ -374,7 +374,7 @@ void PAVectorDiffusionApply2D(const int NE,
 
 // PA Diffusion Apply 3D kernel
 template<const int T_D1D = 0,
-         const int T_Q1D = 0>
+         const int T_Q1D = 0> static
 void PAVectorDiffusionApply3D(const int NE,
                               const Array<double> &b,
                               const Array<double> &g,
@@ -606,13 +606,13 @@ void VectorDiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
 }
 
 template<int T_D1D = 0, int T_Q1D = 0>
-void PAVectorDiffusionDiagonal2D(const int NE,
-                                 const Array<double> &b,
-                                 const Array<double> &g,
-                                 const Vector &d,
-                                 Vector &y,
-                                 const int d1d = 0,
-                                 const int q1d = 0)
+static void PAVectorDiffusionDiagonal2D(const int NE,
+                                        const Array<double> &b,
+                                        const Array<double> &g,
+                                        const Vector &d,
+                                        Vector &y,
+                                        const int d1d = 0,
+                                        const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -673,13 +673,13 @@ void PAVectorDiffusionDiagonal2D(const int NE,
 }
 
 template<int T_D1D = 0, int T_Q1D = 0>
-void PAVectorDiffusionDiagonal3D(const int NE,
-                                 const Array<double> &b,
-                                 const Array<double> &g,
-                                 const Vector &d,
-                                 Vector &y,
-                                 const int d1d = 0,
-                                 const int q1d = 0)
+static void PAVectorDiffusionDiagonal3D(const int NE,
+                                        const Array<double> &b,
+                                        const Array<double> &g,
+                                        const Vector &d,
+                                        Vector &y,
+                                        const int d1d = 0,
+                                        const int q1d = 0)
 {
    constexpr int DIM = 3;
    const int D1D = T_D1D ? T_D1D : d1d;

fem/bilininteg_vecmass.cpp

@@ -104,14 +104,14 @@ void VectorMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 
 template<const int T_D1D = 0,
          const int T_Q1D = 0>
-void PAVectorMassApply2D(const int NE,
-                         const Array<double> &B_,
-                         const Array<double> &Bt_,
-                         const Vector &op_,
-                         const Vector &x_,
-                         Vector &y_,
-                         const int d1d = 0,
-                         const int q1d = 0)
+static void PAVectorMassApply2D(const int NE,
+                                const Array<double> &B_,
+                                const Array<double> &Bt_,
+                                const Vector &op_,
+                                const Vector &x_,
+                                Vector &y_,
+                                const int d1d = 0,
+                                const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -201,14 +201,14 @@ void PAVectorMassApply2D(const int NE,
 
 template<const int T_D1D = 0,
          const int T_Q1D = 0>
-void PAVectorMassApply3D(const int NE,
-                         const Array<double> &B_,
-                         const Array<double> &Bt_,
-                         const Vector &op_,
-                         const Vector &x_,
-                         Vector &y_,
-                         const int d1d = 0,
-                         const int q1d = 0)
+static void PAVectorMassApply3D(const int NE,
+                                const Array<double> &B_,
+                                const Array<double> &Bt_,
+                                const Vector &op_,
+                                const Vector &x_,
+                                Vector &y_,
+                                const int d1d = 0,
+                                const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -379,13 +379,13 @@ void VectorMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 }
 
 template<const int T_D1D = 0, const int T_Q1D = 0>
-void PAVectorMassAssembleDiagonal2D(const int NE,
-                                    const Array<double> &B_,
-                                    const Array<double> &Bt_,
-                                    const Vector &op_,
-                                    Vector &diag_,
-                                    const int d1d = 0,
-                                    const int q1d = 0)
+static void PAVectorMassAssembleDiagonal2D(const int NE,
+                                           const Array<double> &B_,
+                                           const Array<double> &Bt_,
+                                           const Vector &op_,
+                                           Vector &diag_,
+                                           const int d1d = 0,
+                                           const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -431,13 +431,13 @@ void PAVectorMassAssembleDiagonal2D(const int NE,
 }
 
 template<const int T_D1D = 0, const int T_Q1D = 0>
-void PAVectorMassAssembleDiagonal3D(const int NE,
-                                    const Array<double> &B_,
-                                    const Array<double> &Bt_,
-                                    const Vector &op_,
-                                    Vector &diag_,
-                                    const int d1d = 0,
-                                    const int q1d = 0)
+static void PAVectorMassAssembleDiagonal3D(const int NE,
+                                           const Array<double> &B_,
+                                           const Array<double> &Bt_,
+                                           const Vector &op_,
+                                           Vector &diag_,
+                                           const int d1d = 0,
+                                           const int q1d = 0)
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;

fem/complex_fem.cpp

@@ -1204,30 +1204,17 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
       });
       // Modify offdiagonal blocks (imaginary parts of the matrix) to conform
       // with standard essential BC treatment
+      ess_tdof_list.HostRead();
       if (A_i.Type() == Operator::Hypre_ParCSR)
       {
          HypreParMatrix * Ah;
          A_i.Get(Ah);
          hypre_ParCSRMatrix *Aih = *Ah;
-#ifndef HYPRE_USING_CUDA
-         ess_tdof_list.HostRead();
          for (int k = 0; k < n; k++)
          {
             const int j = ess_tdof_list[k];
             Aih->diag->data[Aih->diag->i[j]] = 0.0;
          }
-#else
-         Ah->HypreReadWrite();
-         const int *d_ess_tdof_list =
-            ess_tdof_list.GetMemory().Read(MemoryClass::DEVICE, n);
-         const int *d_diag_i = Aih->diag->i;
-         double *d_diag_data = Aih->diag->data;
-         CuWrap1D(n, [=] MFEM_DEVICE (int k)
-         {
-            const int j = d_ess_tdof_list[k];
-            d_diag_data[d_diag_i[j]] = 0.0;
-         });
-#endif
       }
       else
       {

fem/doftrans.cpp

@@ -1,358 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "fem.hpp"
-
-namespace mfem
-{
-
-void DofTransformation::TransformPrimal(Vector &v) const
-{
-   TransformPrimal(v.GetData());
-}
-
-void DofTransformation::TransformPrimalCols(DenseMatrix &V) const
-{
-   for (int c=0; c<V.Width(); c++)
-   {
-      TransformPrimal(V.GetColumn(c));
-   }
-}
-
-void DofTransformation::TransformDual(Vector &v) const
-{
-   TransformDual(v.GetData());
-}
-
-void DofTransformation::TransformDual(DenseMatrix &V) const
-{
-   TransformDualCols(V);
-   TransformDualRows(V);
-}
-
-void DofTransformation::TransformDualRows(DenseMatrix &V) const
-{
-   Vector row;
-   for (int r=0; r<V.Height(); r++)
-   {
-      V.GetRow(r, row);
-      TransformDual(row);
-      V.SetRow(r, row);
-   }
-}
-
-void DofTransformation::TransformDualCols(DenseMatrix &V) const
-{
-   for (int c=0; c<V.Width(); c++)
-   {
-      TransformDual(V.GetColumn(c));
-   }
-}
-
-void DofTransformation::InvTransformPrimal(Vector &v) const
-{
-   InvTransformPrimal(v.GetData());
-}
-
-void TransformPrimal(const DofTransformation *ran_dof_trans,
-                     const DofTransformation *dom_dof_trans,
-                     DenseMatrix &elmat)
-{
-   if (ran_dof_trans && dom_dof_trans)
-   {
-      ran_dof_trans->TransformPrimalCols(elmat);
-      dom_dof_trans->TransformDualRows(elmat);
-   }
-   else if (ran_dof_trans)
-   {
-      ran_dof_trans->TransformPrimalCols(elmat);
-   }
-   else if (dom_dof_trans)
-   {
-      dom_dof_trans->TransformDualRows(elmat);
-   }
-   else
-   {
-      // If both transformations are NULL this function should not be called
-   }
-}
-
-void TransformDual(const DofTransformation *ran_dof_trans,
-                   const DofTransformation *dom_dof_trans,
-                   DenseMatrix &elmat)
-{
-   if (ran_dof_trans && dom_dof_trans)
-   {
-      ran_dof_trans->TransformDualCols(elmat);
-      dom_dof_trans->TransformDualRows(elmat);
-   }
-   else if (ran_dof_trans)
-   {
-      ran_dof_trans->TransformDualCols(elmat);
-   }
-   else if (dom_dof_trans)
-   {
-      dom_dof_trans->TransformDualRows(elmat);
-   }
-   else
-   {
-      // If both transformations are NULL this function should not be called
-   }
-}
-
-void VDofTransformation::TransformPrimal(double *v) const
-{
-   int size = doftrans_->Size();
-
-   if ((Ordering::Type)ordering_ == Ordering::byNODES || vdim_ == 1)
-   {
-      for (int i=0; i<vdim_; i++)
-      {
-         doftrans_->TransformPrimal(&v[i*size]);
-      }
-   }
-   else
-   {
-      Vector vec(size);
-      for (int i=0; i<vdim_; i++)
-      {
-         for (int j=0; j<size; j++)
-         {
-            vec(j) = v[j*vdim_+i];
-         }
-         doftrans_->TransformPrimal(vec);
-         for (int j=0; j<size; j++)
-         {
-            v[j*vdim_+i] = vec(j);
-         }
-      }
-   }
-}
-
-void VDofTransformation::InvTransformPrimal(double *v) const
-{
-   int size = doftrans_->Height();
-
-   if ((Ordering::Type)ordering_ == Ordering::byNODES)
-   {
-      for (int i=0; i<vdim_; i++)
-      {
-         doftrans_->InvTransformPrimal(&v[i*size]);
-      }
-   }
-   else
-   {
-      Vector vec(size);
-      for (int i=0; i<vdim_; i++)
-      {
-         for (int j=0; j<size; j++)
-         {
-            vec(j) = v[j*vdim_+i];
-         }
-         doftrans_->InvTransformPrimal(vec);
-         for (int j=0; j<size; j++)
-         {
-            v[j*vdim_+i] = vec(j);
-         }
-      }
-   }
-}
-
-void VDofTransformation::TransformDual(double *v) const
-{
-   int size = doftrans_->Size();
-
-   if ((Ordering::Type)ordering_ == Ordering::byNODES)
-   {
-      for (int i=0; i<vdim_; i++)
-      {
-         doftrans_->TransformDual(&v[i*size]);
-      }
-   }
-   else
-   {
-      Vector vec(size);
-      for (int i=0; i<vdim_; i++)
-      {
-         for (int j=0; j<size; j++)
-         {
-            vec(j) = v[j*vdim_+i];
-         }
-         doftrans_->TransformDual(vec);
-         for (int j=0; j<size; j++)
-         {
-            v[j*vdim_+i] = vec(j);
-         }
-      }
-   }
-}
-
-const double ND_DofTransformation::T_data[24] =
-{
-   1.0,  0.0,  0.0,  1.0,
-   -1.0, -1.0,  0.0,  1.0,
-   0.0,  1.0, -1.0, -1.0,
-   1.0,  0.0, -1.0, -1.0,
-   -1.0, -1.0,  1.0,  0.0,
-   0.0,  1.0,  1.0,  0.0
-};
-
-const DenseTensor ND_DofTransformation
-::T(const_cast<double*>(ND_DofTransformation::T_data), 2, 2, 6);
-
-const double ND_DofTransformation::TInv_data[24] =
-{
-   1.0,  0.0,  0.0,  1.0,
-   -1.0, -1.0,  0.0,  1.0,
-   -1.0, -1.0,  1.0,  0.0,
-   1.0,  0.0, -1.0, -1.0,
-   0.0,  1.0, -1.0, -1.0,
-   0.0,  1.0,  1.0,  0.0
-};
-
-const DenseTensor ND_DofTransformation
-::TInv(const_cast<double*>(TInv_data), 2, 2, 6);
-
-ND_DofTransformation::ND_DofTransformation(int size, int p)
-   : DofTransformation(size),
-     order(p)
-{
-}
-
-ND_TriDofTransformation::ND_TriDofTransformation(int p)
-   : ND_DofTransformation(p*(p + 2), p)
-{
-}
-
-void ND_TriDofTransformation::TransformPrimal(double *v) const
-{
-   int nedofs = order; // number of DoFs per edge
-   int nfdofs = order*(order-1); // number of DoFs per face
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform face DoFs
-   for (int f=0; f<1; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[3*nedofs + f*nfdofs + 2*i];
-         T(Fo[f]).Mult(v2, &v[3*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-void
-ND_TriDofTransformation::InvTransformPrimal(double *v) const
-{
-   int nedofs = order; // number of DoFs per edge
-   int nfdofs = order*(order-1); // number of DoFs per face
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform face DoFs
-   for (int f=0; f<1; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[3*nedofs + f*nfdofs + 2*i];
-         TInv(Fo[f]).Mult(v2, &v[3*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-void
-ND_TriDofTransformation::TransformDual(double *v) const
-{
-   int nedofs = order; // number of DoFs per edge
-   int nfdofs = order*(order-1); // number of DoFs per face
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform face DoFs
-   for (int f=0; f<1; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[3*nedofs + f*nfdofs + 2*i];
-         TInv(Fo[f]).MultTranspose(v2, &v[3*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-ND_TetDofTransformation::ND_TetDofTransformation(int p)
-   : ND_DofTransformation(p*(p + 2)*(p + 3)/2, p)
-{
-}
-
-void ND_TetDofTransformation::TransformPrimal(double *v) const
-{
-   int nedofs = order; // number of DoFs per edge
-   int nfdofs = order*(order-1); // number of DoFs per face
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform face DoFs
-   for (int f=0; f<4; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[6*nedofs + f*nfdofs + 2*i];
-         T(Fo[f]).Mult(v2, &v[6*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-void
-ND_TetDofTransformation::InvTransformPrimal(double *v) const
-{
-   int nedofs = order; // number of DoFs per edge
-   int nfdofs = order*(order-1); // number of DoFs per face
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform face DoFs
-   for (int f=0; f<4; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[6*nedofs + f*nfdofs + 2*i];
-         TInv(Fo[f]).Mult(v2, &v[6*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-void
-ND_TetDofTransformation::TransformDual(double *v) const
-{
-   int nedofs = order; // number of DoFs per edge
-   int nfdofs = order*(order-1); // number of DoFs per face
-
-   double data[2];
-   Vector v2(data, 2);
-
-   // Transform face DoFs
-   for (int f=0; f<4; f++)
-   {
-      for (int i=0; i<nfdofs/2; i++)
-      {
-         v2 = &v[6*nedofs + f*nfdofs + 2*i];
-         TInv(Fo[f]).MultTranspose(v2, &v[6*nedofs + f*nfdofs + 2*i]);
-      }
-   }
-}
-
-} // namespace mfem

fem/doftrans.hpp

@@ -1,277 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_DOFTRANSFORM
-#define MFEM_DOFTRANSFORM
-
-#include "../config/config.hpp"
-#include "../linalg/linalg.hpp"
-#include "intrules.hpp"
-#include "fe.hpp"
-
-namespace mfem
-{
-
-/** The DofTransformation class is an abstract base class for a family of
-    transformations that map local degrees of freedom (DoFs), contained within
-    individual elements, to global degrees of freedom, stored within
-    GridFunction objects. These transformations are necessary to ensure that
-    basis functions in neighboring elements align correctly. Closely related but
-    complementary transformations are required for the entries stored in
-    LinearForm and BilinearForm objects. The DofTransformation class is designed
-    to apply the action of both of these types of DoF transformations.
-
-    Let the "primal transformation" be given by the operator T. This means that
-    given a local element vector v the data that must be placed into a
-    GridFunction object is v_t = T * v.
-
-    We also need the inverse of the primal transformation T^{-1} so that we can
-    recover the local element vector from data read out of a GridFunction
-    e.g. v = T^{-1} * v_t.
-
-    We need to preserve the action of our linear forms applied to primal
-    vectors. In other words, if f is the local vector computed by a linear
-    form then f * v = f_t * v_t (where "*" represents an inner product of
-    vectors). This requires that f_t = T^{-T} * f i.e. the "dual transform" is
-    given by the transpose of the inverse of the primal transformation.
-
-    For bilinear forms we require that v^T * A * v = v_t^T * A_t * v_t. This
-    implies that A_t = T^{-T} * A * T^{-1}. This can be accomplished by
-    performing dual transformations of the rows and columns of the matrix A.
-
-    For discrete linear operators the range must be modified with the primal
-    transformation rather than the dual transformation because the result is a
-    primal vector rather than a dual vector. This leads to the transformation
-    D_t = T * D * T^{-1}. This can be accomplished by using a primal
-    transformation on the columns of D and a dual transformation on its rows.
-*/
-class DofTransformation
-{
-protected:
-   int size_;
-
-   Array<int> Fo;
-
-   DofTransformation(int size)
-      : size_(size) {}
-
-public:
-
-   inline int Size() const { return size_; }
-   inline int Height() const { return size_; }
-   inline int NumRows() const { return size_; }
-   inline int Width() const { return size_; }
-   inline int NumCols() const { return size_; }
-
-   /** @brief Configure the transformation using face orientations for the
-       current element. */
-   /// The face_orientation array can be obtained from Mesh::GetElementFaces.
-   inline void SetFaceOrientations(const Array<int> & face_orientation)
-   { Fo = face_orientation; }
-
-   inline const Array<int> & GetFaceOrientations() const { return Fo; }
-
-   /** Transform local DoFs to align with the global DoFs. For example, this
-       transformation can be used to map the local vector computed by
-       FiniteElement::Project() to the transformed vector stored within a
-       GridFunction object. */
-   virtual void TransformPrimal(double *v) const = 0;
-   virtual void TransformPrimal(Vector &v) const;
-
-   /// Transform groups of DoFs stored as dense matrices
-   virtual void TransformPrimalCols(DenseMatrix &V) const;
-
-   /** Inverse transform local DoFs. Used to transform DoFs from a global vector
-       back to their element-local form. For example, this must be used to
-       transform the vector obtained using GridFunction::GetSubVector before it
-       can be used to compute a local interpolation.
-   */
-   virtual void InvTransformPrimal(double *v) const = 0;
-   virtual void InvTransformPrimal(Vector &v) const;
-
-   /** Transform dual DoFs as computed by a LinearFormIntegrator before summing
-       into a LinearForm object. */
-   virtual void TransformDual(double *v) const = 0;
-   virtual void TransformDual(Vector &v) const;
-
-   /** Transform a matrix of dual DoFs entries as computed by a
-       BilinearFormIntegrator before summing into a BilinearForm object. */
-   virtual void TransformDual(DenseMatrix &V) const;
-
-   /// Transform groups of dual DoFs stored as dense matrices
-   virtual void TransformDualRows(DenseMatrix &V) const;
-   virtual void TransformDualCols(DenseMatrix &V) const;
-
-   virtual ~DofTransformation() {}
-};
-
-/** Transform a matrix of DoFs entries from different finite element spaces as
-    computed by a DiscreteInterpolator before copying into a
-    DiscreteLinearOperator.
-*/
-void TransformPrimal(const DofTransformation *ran_dof_trans,
-                     const DofTransformation *dom_dof_trans,
-                     DenseMatrix &elmat);
-
-/** Transform a matrix of dual DoFs entries from different finite element spaces
-    as computed by a BilinearFormIntegrator before summing into a
-    MixedBilinearForm object.
-*/
-void TransformDual(const DofTransformation *ran_dof_trans,
-                   const DofTransformation *dom_dof_trans,
-                   DenseMatrix &elmat);
-
-/** The VDofTransformation class implements a nested transformation where an
-    arbitrary DofTransformation is replicated with a vdim >= 1.
-*/
-class VDofTransformation : public DofTransformation
-{
-private:
-   int vdim_;
-   int ordering_;
-   DofTransformation * doftrans_;
-
-public:
-   /** @brief Default constructor which requires that SetDofTransformation be
-       called before use. */
-   VDofTransformation(int vdim = 1, int ordering = 0)
-      : DofTransformation(0),
-        vdim_(vdim), ordering_(ordering),
-        doftrans_(NULL) {}
-
-   /// Constructor with a known DofTransformation
-   VDofTransformation(DofTransformation & doftrans, int vdim = 1,
-                      int ordering = 0)
-      : DofTransformation(vdim * doftrans.Size()),
-        vdim_(vdim), ordering_(ordering),
-        doftrans_(&doftrans) {}
-
-   /// Set or change the vdim parameter
-   inline void SetVDim(int vdim)
-   {
-      vdim_ = vdim;
-      if (doftrans_)
-      {
-         size_ = vdim_ * doftrans_->Size();
-      }
-   }
-
-   /// Return the current vdim value
-   inline int GetVDim() const { return vdim_; }
-
-   /// Set or change the nested DofTransformation object
-   inline void SetDofTransformation(DofTransformation & doftrans)
-   {
-      size_ = vdim_ * doftrans.Size();
-      doftrans_ = &doftrans;
-   }
-
-   /// Return the nested DofTransformation object
-   inline DofTransformation * GetDofTransformation() const { return doftrans_; }
-
-   inline void SetFaceOrientation(const Array<int> & face_orientation)
-   { Fo = face_orientation; doftrans_->SetFaceOrientations(face_orientation); }
-
-   using DofTransformation::TransformPrimal;
-   using DofTransformation::InvTransformPrimal;
-   using DofTransformation::TransformDual;
-
-   void TransformPrimal(double *v) const;
-   void InvTransformPrimal(double *v) const;
-   void TransformDual(double *v) const;
-};
-
-/** Abstract base class for high-order Nedelec spaces on elements with
-    triangular faces.
-
-    The Nedelec DoFs on the interior of triangular faces come in pairs which
-    share an interpolation point but have different vector directions. These
-    directions depend on the orientation of the face and can therefore differ in
-    neighboring elements. The mapping required to transform these DoFs can be
-    implemented as series of 2x2 linear transformations. The raw data for these
-    linear transformations is stored in the T_data and TInv_data arrays and can
-    be accessed as DenseMatrices using the GetFaceTransform() and
-    GetFaceInverseTransform() methods.
-*/
-class ND_DofTransformation : public DofTransformation
-{
-protected:
-   static const double T_data[24];
-   static const double TInv_data[24];
-   static const DenseTensor T, TInv;
-   int order;
-
-   ND_DofTransformation(int size, int order);
-
-public:
-   // Return the 2x2 transformation operator for the given face orientation
-   static const DenseMatrix & GetFaceTransform(int ori) { return T(ori); }
-
-   // Return the 2x2 inverse transformation operator
-   static const DenseMatrix & GetFaceInverseTransform(int ori)
-   { return TInv(ori); }
-};
-
-/// DoF transformation implementation for the Nedelec basis on triangles
-class ND_TriDofTransformation : public ND_DofTransformation
-{
-public:
-   ND_TriDofTransformation(int order);
-
-   using DofTransformation::TransformPrimal;
-   using DofTransformation::InvTransformPrimal;
-   using DofTransformation::TransformDual;
-
-   void TransformPrimal(double *v) const;
-
-   void InvTransformPrimal(double *v) const;
-
-   void TransformDual(double *v) const;
-};
-
-/// DoF transformation implementation for the Nedelec basis on tetrahedra
-class ND_TetDofTransformation : public ND_DofTransformation
-{
-public:
-   ND_TetDofTransformation(int order);
-
-   using DofTransformation::TransformPrimal;
-   using DofTransformation::InvTransformPrimal;
-   using DofTransformation::TransformDual;
-
-   void TransformPrimal(double *v) const;
-
-   void InvTransformPrimal(double *v) const;
-
-   void TransformDual(double *v) const;
-};
-
-/// DoF transformation implementation for the Nedelec basis on wedge elements
-/** TODO: (Under development) */
-class ND_WedgeDofTransformation : public ND_DofTransformation
-{
-public:
-   ND_WedgeDofTransformation(int order);
-
-   using DofTransformation::TransformPrimal;
-   using DofTransformation::InvTransformPrimal;
-   using DofTransformation::TransformDual;
-
-   void TransformPrimal(double *v) const;
-
-   void InvTransformPrimal(double *v) const;
-
-   void TransformDual(double *v) const;
-};
-
-} // namespace mfem
-
-#endif // MFEM_DOFTRANSFORM

fem/eltrans.cpp

@@ -380,7 +380,6 @@ void IsoparametricTransformation::SetIdentityTransformation(
       case Geometry::TETRAHEDRON : FElem = &TetrahedronFE; break;
       case Geometry::CUBE :        FElem = &HexahedronFE; break;
       case Geometry::PRISM :       FElem = &WedgeFE; break;
-      case Geometry::PYRAMID :     FElem = &PyramidFE; break;
       default:
          MFEM_ABORT("unknown Geometry::Type!");
    }

fem/eltrans.hpp

@@ -316,7 +316,7 @@ public:
 
    /// Set the desired print level, useful for debugging.
    /** The valid options are: -1 - never print (default); 0 - print only errors;
-       1 - print the first and last iterations; 2 - print every iteration;
+       1 - print the first and last last iterations; 2 - print every iteration;
        and 3 - print every iteration including point coordinates. */
    void SetPrintLevel(int pr_level) { print_level = pr_level; }
 

fem/estimators.cpp

@@ -329,7 +329,7 @@ void KellyErrorEstimator::ComputeEstimates()
          error_estimates(e) = sqrt(factor * error_estimates(e));
       }
 
-      total_error = error_estimates.Norml2();
+      total_error = error_estimates.Sum();
       delete flux;
       return;
    }
@@ -452,10 +452,9 @@ void KellyErrorEstimator::ComputeEstimates()
    auto pfes = dynamic_cast<ParFiniteElementSpace*>(xfes);
    MFEM_VERIFY(pfes, "xfes is not a ParFiniteElementSpace pointer");
 
-   double process_local_error = pow(error_estimates.Norml2(),2.0);
+   double process_local_error = error_estimates.Sum();
    MPI_Allreduce(&process_local_error, &total_error, 1, MPI_DOUBLE,
                  MPI_SUM, pfes->GetComm());
-   total_error = sqrt(total_error);
 #endif // MFEM_USE_MPI
 }
 

fem/fe.hpp

@@ -97,7 +97,7 @@ public:
       {
          "Gauss-Legendre", "Gauss-Lobatto", "Positive (Bernstein)",
          "Open uniform", "Closed uniform", "Open half uniform",
-         "Serendipity", "Closed Gauss-Legendre",
+         "Seredipity", "Closed Gauss-Legendre",
          "Integrated Gauss-Lobatto indicator"
       };
       return name[Check(b_type)];
@@ -1126,7 +1126,7 @@ public:
    { dofs = 1.0; }
 };
 
-/// A 1D quadratic finite element with uniformly spaced nodes
+/// A 1D quadractic finite element with uniformly spaced nodes
 class Quad1DFiniteElement : public NodalFiniteElement
 {
 public:
@@ -1313,64 +1313,6 @@ public:
                            DenseMatrix &dshape) const;
 };
 
-/// A linear element defined on a triangular prism
-class LinearWedgeFiniteElement : public NodalFiniteElement
-{
-public:
-   /// Construct the LinearWedgeFiniteElement
-   LinearWedgeFiniteElement();
-
-   /** @brief virtual function which evaluates the values of all
-       shape functions at a given point ip and stores
-       them in the vector shape of dimension Dof (4) */
-   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
-
-   /** @brief virtual function which evaluates the values of all
-       partial derivatives of all shape functions at a given
-       point ip and stores them in the matrix dshape (Dof x Dim) (4 x 3)
-       so that each row contains the derivatives of one shape function */
-   virtual void CalcDShape(const IntegrationPoint &ip,
-                           DenseMatrix &dshape) const;
-
-   virtual void ProjectDelta(int vertex, Vector &dofs) const
-   { dofs = 0.0; dofs(vertex) = 1.0; }
-
-   /** @brief Get the dofs associated with the given @a face.
-       @a *dofs is set to an internal array of the local dofc on the
-       face, while *ndofs is set to the number of dofs on that face.
-   */
-   virtual void GetFaceDofs(int face, int **dofs, int *ndofs) const;
-};
-
-/// A linear element defined on a square pyramid
-class LinearPyramidFiniteElement : public NodalFiniteElement
-{
-public:
-   /// Construct the LinearPyramidFiniteElement
-   LinearPyramidFiniteElement();
-
-   /** @brief virtual function which evaluates the values of all
-       shape functions at a given point ip and stores
-       them in the vector shape of dimension Dof (4) */
-   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
-
-   /** @brief virtual function which evaluates the values of all
-       partial derivatives of all shape functions at a given
-       point ip and stores them in the matrix dshape (Dof x Dim) (4 x 3)
-       so that each row contains the derivatives of one shape function */
-   virtual void CalcDShape(const IntegrationPoint &ip,
-                           DenseMatrix &dshape) const;
-
-   virtual void ProjectDelta(int vertex, Vector &dofs) const
-   { dofs = 0.0; dofs(vertex) = 1.0; }
-
-   /** @brief Get the dofs associated with the given @a face.
-       @a *dofs is set to an internal array of the local dofc on the
-       face, while *ndofs is set to the number of dofs on that face.
-   */
-   virtual void GetFaceDofs(int face, int **dofs, int *ndofs) const;
-};
-
 /// A 2D constant element on a triangle
 class P0TriangleFiniteElement : public NodalFiniteElement
 {
@@ -1748,32 +1690,6 @@ public:
    { dofs(0) = 1.0; }
 };
 
-/// A 3D constant element on a wedge
-class P0WdgFiniteElement : public NodalFiniteElement
-{
-public:
-   /// Construct the P0WdgFiniteElement
-   P0WdgFiniteElement ();
-   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
-   virtual void CalcDShape(const IntegrationPoint &ip,
-                           DenseMatrix &dshape) const;
-   virtual void ProjectDelta(int vertex, Vector &dofs) const
-   { dofs(0) = 1.0; }
-};
-
-/// A 3D constant element on a pyramid
-class P0PyrFiniteElement : public NodalFiniteElement
-{
-public:
-   /// Construct the P0PyrFiniteElement
-   P0PyrFiniteElement ();
-   virtual void CalcShape(const IntegrationPoint &ip, Vector &shape) const;
-   virtual void CalcDShape(const IntegrationPoint &ip,
-                           DenseMatrix &dshape) const;
-   virtual void ProjectDelta(int vertex, Vector &dofs) const
-   { dofs(0) = 1.0; }
-};
-
 /** @brief Tensor products of 1D Lagrange1DFiniteElement
     (only degree 2 is functional) */
 class LagrangeHexFiniteElement : public NodalFiniteElement
@@ -1912,10 +1828,6 @@ public:
    using FiniteElement::Project;
    virtual void Project (VectorCoefficient &vc,
                          ElementTransformation &Trans, Vector &dofs) const;
-
-   virtual void ProjectGrad(const FiniteElement &fe,
-                            ElementTransformation &Trans,
-                            DenseMatrix &grad) const;
 };
 
 
@@ -1940,66 +1852,6 @@ public:
    using FiniteElement::Project;
    virtual void Project (VectorCoefficient &vc,
                          ElementTransformation &Trans, Vector &dofs) const;
-
-   virtual void ProjectGrad(const FiniteElement &fe,
-                            ElementTransformation &Trans,
-                            DenseMatrix &grad) const;
-};
-
-
-/// A 3D 1st order Nedelec element on a wedge
-class Nedelec1WdgFiniteElement : public VectorFiniteElement
-{
-private:
-   static const double tk[9][3];
-
-public:
-   /// Construct the Nedelec1WdgFiniteElement
-   Nedelec1WdgFiniteElement();
-   virtual void CalcVShape(const IntegrationPoint &ip,
-                           DenseMatrix &shape) const;
-   virtual void CalcVShape(ElementTransformation &Trans,
-                           DenseMatrix &shape) const
-   { CalcVShape_ND(Trans, shape); }
-   virtual void CalcCurlShape(const IntegrationPoint &ip,
-                              DenseMatrix &curl_shape) const;
-   virtual void GetLocalInterpolation (ElementTransformation &Trans,
-                                       DenseMatrix &I) const;
-   using FiniteElement::Project;
-   virtual void Project (VectorCoefficient &vc,
-                         ElementTransformation &Trans, Vector &dofs) const;
-
-   virtual void ProjectGrad(const FiniteElement &fe,
-                            ElementTransformation &Trans,
-                            DenseMatrix &grad) const;
-};
-
-
-/// A 3D 1st order Nedelec element on a pyramid
-class Nedelec1PyrFiniteElement : public VectorFiniteElement
-{
-private:
-   static const double tk[8][3];
-
-public:
-   /// Construct the Nedelec1PyrFiniteElement
-   Nedelec1PyrFiniteElement();
-   virtual void CalcVShape(const IntegrationPoint &ip,
-                           DenseMatrix &shape) const;
-   virtual void CalcVShape(ElementTransformation &Trans,
-                           DenseMatrix &shape) const
-   { CalcVShape_ND(Trans, shape); }
-   virtual void CalcCurlShape(const IntegrationPoint &ip,
-                              DenseMatrix &curl_shape) const;
-   virtual void GetLocalInterpolation (ElementTransformation &Trans,
-                                       DenseMatrix &I) const;
-   using FiniteElement::Project;
-   virtual void Project (VectorCoefficient &vc,
-                         ElementTransformation &Trans, Vector &dofs) const;
-
-   virtual void ProjectGrad(const FiniteElement &fe,
-                            ElementTransformation &Trans,
-                            DenseMatrix &grad) const;
 };
 
 
@@ -2093,77 +1945,6 @@ public:
 };
 
 
-/// A 3D 0th order Raviert-Thomas element on a wedge
-class RT0WdgFiniteElement : public VectorFiniteElement
-{
-private:
-   static const double nk[5][3];
-
-public:
-   /// Construct the RT0WdgFiniteElement
-   RT0WdgFiniteElement();
-
-   virtual void CalcVShape(const IntegrationPoint &ip,
-                           DenseMatrix &shape) const;
-
-   virtual void CalcVShape(ElementTransformation &Trans,
-                           DenseMatrix &shape) const
-   { CalcVShape_RT(Trans, shape); }
-
-   virtual void CalcDivShape(const IntegrationPoint &ip,
-                             Vector &divshape) const;
-
-   virtual void GetLocalInterpolation (ElementTransformation &Trans,
-                                       DenseMatrix &I) const;
-
-   using FiniteElement::Project;
-
-   virtual void Project (VectorCoefficient &vc,
-                         ElementTransformation &Trans, Vector &dofs) const;
-
-   virtual void ProjectCurl(const FiniteElement &fe,
-                            ElementTransformation &Trans,
-                            DenseMatrix &curl) const;
-};
-
-
-/// A 3D 0th order Raviert-Thomas element on a pyramid
-class RT0PyrFiniteElement : public VectorFiniteElement
-{
-private:
-   static const double nk[5][3];
-
-   // If true match RT0TetFiniteElement rather than RT_TetrahedronElement(0)
-   bool rt0;
-
-public:
-   /// Construct the RT0PyrFiniteElement
-   RT0PyrFiniteElement(bool rt0tets = true);
-
-   virtual void CalcVShape(const IntegrationPoint &ip,
-                           DenseMatrix &shape) const;
-
-   virtual void CalcVShape(ElementTransformation &Trans,
-                           DenseMatrix &shape) const
-   { CalcVShape_RT(Trans, shape); }
-
-   virtual void CalcDivShape(const IntegrationPoint &ip,
-                             Vector &divshape) const;
-
-   virtual void GetLocalInterpolation (ElementTransformation &Trans,
-                                       DenseMatrix &I) const;
-
-   using FiniteElement::Project;
-
-   virtual void Project (VectorCoefficient &vc,
-                         ElementTransformation &Trans, Vector &dofs) const;
-
-   virtual void ProjectCurl(const FiniteElement &fe,
-                            ElementTransformation &Trans,
-                            DenseMatrix &curl) const;
-};
-
-
 class RotTriLinearHexFiniteElement : public NodalFiniteElement
 {
 public:

fem/fe_coll.cpp

@@ -33,9 +33,6 @@ int FiniteElementCollection::HasFaceDofs(Geometry::Type geom, int p) const
       case Geometry::PRISM:
          return max(GetNumDof(Geometry::TRIANGLE, p),
                     GetNumDof(Geometry::SQUARE, p));
-      case Geometry::PYRAMID:
-         return max(GetNumDof(Geometry::TRIANGLE, p),
-                    GetNumDof(Geometry::SQUARE, p));
       default:
          MFEM_ABORT("unknown geometry type");
    }
@@ -577,7 +574,6 @@ LinearFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
       case Geometry::TETRAHEDRON: return &TetrahedronFE;
       case Geometry::CUBE:        return &ParallelepipedFE;
       case Geometry::PRISM:       return &WedgeFE;
-      case Geometry::PYRAMID:     return &PyramidFE;
       default:
          mfem_error ("LinearFECollection: unknown geometry type.");
    }
@@ -595,7 +591,6 @@ int LinearFECollection::DofForGeometry(Geometry::Type GeomType) const
       case Geometry::TETRAHEDRON: return 0;
       case Geometry::CUBE:        return 0;
       case Geometry::PRISM:       return 0;
-      case Geometry::PYRAMID:     return 0;
       default:
          mfem_error ("LinearFECollection: unknown geometry type.");
    }
@@ -1245,7 +1240,6 @@ Const3DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
       case Geometry::TETRAHEDRON: return &TetrahedronFE;
       case Geometry::CUBE:        return &ParallelepipedFE;
       case Geometry::PRISM:       return &WedgeFE;
-      case Geometry::PYRAMID:     return &PyramidFE;
       default:
          mfem_error ("Const3DFECollection: unknown geometry type.");
    }
@@ -1263,7 +1257,6 @@ int Const3DFECollection::DofForGeometry(Geometry::Type GeomType) const
       case Geometry::TETRAHEDRON: return 1;
       case Geometry::CUBE:        return 1;
       case Geometry::PRISM:       return 1;
-      case Geometry::PYRAMID:     return 1;
       default:
          mfem_error ("Const3DFECollection: unknown geometry type.");
    }
@@ -1284,8 +1277,6 @@ LinearDiscont3DFECollection::FiniteElementForGeometry(
    switch (GeomType)
    {
       case Geometry::TETRAHEDRON: return &TetrahedronFE;
-      case Geometry::PYRAMID:     return &PyramidFE;
-      case Geometry::PRISM:       return &WedgeFE;
       case Geometry::CUBE:        return &ParallelepipedFE;
       default:
          mfem_error ("LinearDiscont3DFECollection: unknown geometry type.");
@@ -1302,8 +1293,6 @@ int LinearDiscont3DFECollection::DofForGeometry(Geometry::Type GeomType) const
       case Geometry::TRIANGLE:    return 0;
       case Geometry::SQUARE:      return 0;
       case Geometry::TETRAHEDRON: return 4;
-      case Geometry::PYRAMID:     return 5;
-      case Geometry::PRISM:       return 6;
       case Geometry::CUBE:        return 8;
       default:
          mfem_error ("LinearDiscont3DFECollection: unknown geometry type.");
@@ -1405,8 +1394,6 @@ ND1_3DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
    {
       case Geometry::CUBE:        return &HexahedronFE;
       case Geometry::TETRAHEDRON: return &TetrahedronFE;
-      case Geometry::PRISM:       return &WedgeFE;
-      case Geometry::PYRAMID:     return &PyramidFE;
       default:
          mfem_error ("ND1_3DFECollection: unknown geometry type.");
    }
@@ -1423,8 +1410,6 @@ int ND1_3DFECollection::DofForGeometry(Geometry::Type GeomType) const
       case Geometry::SQUARE:      return 0;
       case Geometry::TETRAHEDRON: return 0;
       case Geometry::CUBE:        return 0;
-      case Geometry::PRISM:       return 0;
-      case Geometry::PYRAMID:     return 0;
       default:
          mfem_error ("ND1_3DFECollection: unknown geometry type.");
    }
@@ -1454,8 +1439,6 @@ RT0_3DFECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
       case Geometry::SQUARE:      return &QuadrilateralFE;
       case Geometry::CUBE:        return &HexahedronFE;
       case Geometry::TETRAHEDRON: return &TetrahedronFE;
-      case Geometry::PRISM:       return &WedgeFE;
-      case Geometry::PYRAMID:     return &PyramidFE;
       default:
          mfem_error ("RT0_3DFECollection: unknown geometry type.");
    }
@@ -1472,8 +1455,6 @@ int RT0_3DFECollection::DofForGeometry(Geometry::Type GeomType) const
       case Geometry::SQUARE:      return 1;
       case Geometry::TETRAHEDRON: return 0;
       case Geometry::CUBE:        return 0;
-      case Geometry::PRISM:       return 0;
-      case Geometry::PYRAMID:     return 0;
       default:
          mfem_error ("RT0_3DFECollection: unknown geometry type.");
    }
@@ -1749,7 +1730,6 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype)
          H1_dof[Geometry::TETRAHEDRON] = (TriDof*pm3)/3;
          H1_dof[Geometry::CUBE] = QuadDof*pm1;
          H1_dof[Geometry::PRISM] = TriDof*pm1;
-         H1_dof[Geometry::PYRAMID] = 0;
          if (b_type == BasisType::Positive)
          {
             H1_Elements[Geometry::TETRAHEDRON] = new H1Pos_TetrahedronElement(p);
@@ -1763,7 +1743,6 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype)
             H1_Elements[Geometry::CUBE] = new H1_HexahedronElement(p, btype);
             H1_Elements[Geometry::PRISM] = new H1_WedgeElement(p, btype);
          }
-         H1_Elements[Geometry::PYRAMID] = new LinearPyramidFiniteElement;
 
          const int &TetDof = H1_dof[Geometry::TETRAHEDRON];
          TetDofOrd[0] = new int[24*TetDof];
@@ -1858,21 +1837,6 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype)
    }
 }
 
-const FiniteElement *
-H1_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
-{
-   if (GeomType != Geometry::PYRAMID || this->GetOrder() == 1)
-   {
-      return H1_Elements[GeomType];
-   }
-   else
-   {
-      MFEM_ABORT("H1 Pyramid basis functions are not yet supported "
-                 "for order > 1.");
-      return NULL;
-   }
-}
-
 const int *H1_FECollection::DofOrderForOrientation(Geometry::Type GeomType,
                                                    int Or) const
 {
@@ -2112,12 +2076,9 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
          L2_Elements[Geometry::CUBE] = new L2_HexahedronElement(p, btype);
          L2_Elements[Geometry::PRISM] = new L2_WedgeElement(p, btype);
       }
-      L2_Elements[Geometry::PYRAMID] = new P0PyrFiniteElement;
-
       L2_Elements[Geometry::TETRAHEDRON]->SetMapType(map_type);
       L2_Elements[Geometry::CUBE]->SetMapType(map_type);
       L2_Elements[Geometry::PRISM]->SetMapType(map_type);
-      L2_Elements[Geometry::PYRAMID]->SetMapType(map_type);
       // Trace element use the default Gauss-Legendre nodal points for positive basis
       if (b_type == BasisType::Positive)
       {
@@ -2238,21 +2199,6 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
    }
 }
 
-const FiniteElement *
-L2_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
-{
-   if (GeomType != Geometry::PYRAMID || this->GetOrder() == 0)
-   {
-      return L2_Elements[GeomType];
-   }
-   else
-   {
-      MFEM_ABORT("L2 Pyramid basis functions are not yet supported "
-                 "for order > 0.");
-      return NULL;
-   }
-}
-
 const int *L2_FECollection::DofOrderForOrientation(Geometry::Type GeomType,
                                                    int Or) const
 {
@@ -2344,12 +2290,6 @@ RT_FECollection::RT_FECollection(const int order, const int dim,
 
       RT_Elements[Geometry::CUBE] = new RT_HexahedronElement(p, cb_type, ob_type);
       RT_dof[Geometry::CUBE] = 3*p*pp1*pp1;
-
-      RT_Elements[Geometry::PRISM] = new RT0WdgFiniteElement;
-      RT_dof[Geometry::PRISM] = 0;
-
-      RT_Elements[Geometry::PYRAMID] = new RT0PyrFiniteElement(false);
-      RT_dof[Geometry::PYRAMID] = 0;
    }
    else
    {
@@ -2493,22 +2433,6 @@ void RT_FECollection::InitFaces(const int p, const int dim,
    }
 }
 
-const FiniteElement *
-RT_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
-{
-   if ((GeomType != Geometry::PRISM && GeomType != Geometry::PYRAMID) ||
-       this->GetOrder() == 1)
-   {
-      return RT_Elements[GeomType];
-   }
-   else
-   {
-      MFEM_ABORT("RT Wedge and Pyramid basis functions are not yet supported "
-                 "for order > 0.");
-      return NULL;
-   }
-}
-
 const int *RT_FECollection::DofOrderForOrientation(Geometry::Type GeomType,
                                                    int Or) const
 {
@@ -2732,31 +2656,18 @@ ND_FECollection::ND_FECollection(const int p, const int dim,
       {
          for (int i = 0; i + j <= pm2; i++)
          {
-            int k0 = p*pm1 - (p - j)*(pm1 - j) + 2*i;
-            int k1 = 2*pm2 - 2*i + ((2*p-3)-j)*j;
-            int k2 = 2*pm2 - 2*j + ((2*p-3)-i)*i;
-            int k3 = p*pm1 - 2 - 3*j - i - (i+j)*(i+j);
-            int k4 = p*pm1 - 2 - 3*i - j - (i+j)*(i+j);
-            int k5 = p*pm1 - (p - i)*(pm1 - i) + 2*j;
-
+            int k1 = p*pm1 - (p - j)*(pm1 - j) + 2*i;
+            int k2 = p*pm1 - (p - i)*(pm1 - i) + 2*j;
             // (0,1,2)
-            TriDofOrd[0][k0  ] = k0;
-            TriDofOrd[0][k0+1] = k0 + 1;
-            // (1,0,2)
-            TriDofOrd[1][k0  ] = k1;
-            TriDofOrd[1][k0+1] = k1 + 1;
-            // (2,0,1)
-            TriDofOrd[2][k0  ] = k2;
-            TriDofOrd[2][k0+1] = k2 + 1;
-            // (2,1,0)
-            TriDofOrd[3][k0  ] = k3;
-            TriDofOrd[3][k0+1] = k3 + 1;
-            // (1,2,0)
-            TriDofOrd[4][k0  ] = k4;
-            TriDofOrd[4][k0+1] = k4 + 1;
+            TriDofOrd[0][k1  ] = k1;
+            TriDofOrd[0][k1+1] = k1 + 1;
             // (0,2,1)
-            TriDofOrd[5][k0  ] = k5;
-            TriDofOrd[5][k0+1] = k5 + 1;
+            TriDofOrd[5][k1  ] = k2 + 1;
+            TriDofOrd[5][k1+1] = k2;
+
+            // The other orientations can not be supported with the current
+            // interface. The method Mesh::ReorientTetMesh will ensure that
+            // only orientations 0 and 5 are generated.
          }
       }
    }
@@ -2769,28 +2680,6 @@ ND_FECollection::ND_FECollection(const int p, const int dim,
       // TODO: cb_type and ob_type for tets
       ND_Elements[Geometry::TETRAHEDRON] = new ND_TetrahedronElement(p);
       ND_dof[Geometry::TETRAHEDRON] = p*pm1*pm2/2;
-
-      ND_Elements[Geometry::PRISM] = new Nedelec1WdgFiniteElement;
-      ND_dof[Geometry::PRISM] = 0;
-
-      ND_Elements[Geometry::PYRAMID] = new Nedelec1PyrFiniteElement;
-      ND_dof[Geometry::PYRAMID] = 0;
-   }
-}
-
-const FiniteElement *
-ND_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
-{
-   if ((GeomType != Geometry::PRISM && GeomType != Geometry::PYRAMID) ||
-       this->GetOrder() == 1)
-   {
-      return ND_Elements[GeomType];
-   }
-   else
-   {
-      MFEM_ABORT("ND Wedge and Pyramid basis functions are not yet supported "
-                 "for order > 1.");
-      return NULL;
    }
 }
 
@@ -2803,6 +2692,11 @@ const int *ND_FECollection::DofOrderForOrientation(Geometry::Type GeomType,
    }
    else if (GeomType == Geometry::TRIANGLE)
    {
+      if (Or != 0 && Or != 5)
+      {
+         MFEM_ABORT("triangle face orientation " << Or << " is not supported! "
+                    "Use Mesh::ReorientTetMesh to fix it.");
+      }
       return TriDofOrd[Or%6];
    }
    else if (GeomType == Geometry::SQUARE)

fem/fe_coll.hpp

@@ -228,7 +228,8 @@ public:
                             const int btype = BasisType::GaussLobatto);
 
    virtual const FiniteElement *FiniteElementForGeometry(
-      Geometry::Type GeomType) const;
+      Geometry::Type GeomType) const
+   { return H1_Elements[GeomType]; }
    virtual int DofForGeometry(Geometry::Type GeomType) const
    { return H1_dof[GeomType]; }
    virtual const int *DofOrderForOrientation(Geometry::Type GeomType,
@@ -301,7 +302,10 @@ public:
                    const int map_type = FiniteElement::VALUE);
 
    virtual const FiniteElement *FiniteElementForGeometry(
-      Geometry::Type GeomType) const;
+      Geometry::Type GeomType) const
+   {
+      return L2_Elements[GeomType];
+   }
    virtual int DofForGeometry(Geometry::Type GeomType) const
    {
       if (L2_Elements[GeomType])
@@ -367,7 +371,8 @@ public:
                    const int ob_type = BasisType::GaussLegendre);
 
    virtual const FiniteElement *FiniteElementForGeometry(
-      Geometry::Type GeomType) const;
+      Geometry::Type GeomType) const
+   { return RT_Elements[GeomType]; }
    virtual int DofForGeometry(Geometry::Type GeomType) const
    { return RT_dof[GeomType]; }
    virtual const int *DofOrderForOrientation(Geometry::Type GeomType,
@@ -425,7 +430,8 @@ public:
                    const int ob_type = BasisType::GaussLegendre);
 
    virtual const FiniteElement *
-   FiniteElementForGeometry(Geometry::Type GeomType) const;
+   FiniteElementForGeometry(Geometry::Type GeomType) const
+   { return ND_Elements[GeomType]; }
 
    virtual int DofForGeometry(Geometry::Type GeomType) const
    { return ND_dof[GeomType]; }
@@ -523,10 +529,9 @@ private:
    const BiLinear2DFiniteElement QuadrilateralFE;
    const Linear3DFiniteElement TetrahedronFE;
    const TriLinear3DFiniteElement ParallelepipedFE;
-   const LinearWedgeFiniteElement WedgeFE;
-   const LinearPyramidFiniteElement PyramidFE;
+   const H1_WedgeElement WedgeFE;
 public:
-   LinearFECollection() : FiniteElementCollection(1) { }
+   LinearFECollection() : FiniteElementCollection(1), WedgeFE(1) { }
 
    virtual const FiniteElement *
    FiniteElementForGeometry(Geometry::Type GeomType) const;
@@ -931,11 +936,10 @@ class Const3DFECollection : public FiniteElementCollection
 private:
    const P0TetFiniteElement TetrahedronFE;
    const P0HexFiniteElement ParallelepipedFE;
-   const P0WdgFiniteElement WedgeFE;
-   const P0PyrFiniteElement PyramidFE;
+   const L2_WedgeElement WedgeFE;
 
 public:
-   Const3DFECollection() : FiniteElementCollection(0) { }
+   Const3DFECollection() : FiniteElementCollection(0), WedgeFE(0) { }
 
    virtual const FiniteElement *
    FiniteElementForGeometry(Geometry::Type GeomType) const;
@@ -956,8 +960,6 @@ class LinearDiscont3DFECollection : public FiniteElementCollection
 {
 private:
    const Linear3DFiniteElement TetrahedronFE;
-   const LinearPyramidFiniteElement PyramidFE;
-   const LinearWedgeFiniteElement WedgeFE;
    const TriLinear3DFiniteElement ParallelepipedFE;
 
 public:
@@ -1034,8 +1036,6 @@ class ND1_3DFECollection : public FiniteElementCollection
 private:
    const Nedelec1HexFiniteElement HexahedronFE;
    const Nedelec1TetFiniteElement TetrahedronFE;
-   const Nedelec1WdgFiniteElement WedgeFE;
-   const Nedelec1PyrFiniteElement PyramidFE;
 
 public:
    ND1_3DFECollection() : FiniteElementCollection(1) { }
@@ -1061,8 +1061,6 @@ private:
    const P0QuadFiniteElement QuadrilateralFE;
    const RT0HexFiniteElement HexahedronFE;
    const RT0TetFiniteElement TetrahedronFE;
-   const RT0WdgFiniteElement WedgeFE;
-   const RT0PyrFiniteElement PyramidFE;
 public:
    RT0_3DFECollection() : FiniteElementCollection(1) { }
 

fem/fem.hpp

@@ -16,7 +16,6 @@
 #include "geom.hpp"
 #include "fe.hpp"
 #include "fe_coll.hpp"
-#include "doftrans.hpp"
 #include "eltrans.hpp"
 #include "coefficient.hpp"
 #include "complex_fem.hpp"
@@ -35,7 +34,6 @@
 #include "staticcond.hpp"
 #include "tmop.hpp"
 #include "tmop_tools.hpp"
-#include "tmop_amr.hpp"
 #include "gslib.hpp"
 #include "restriction.hpp"
 #include "quadinterpolator.hpp"
@@ -66,9 +64,4 @@
 #include "adios2datacollection.hpp"
 #endif
 
-#ifdef MFEM_USE_FMS
-#include "fmsconvert.hpp"
-#include "fmsdatacollection.hpp"
-#endif
-
 #endif

fem/fespace.cpp

@@ -58,12 +58,9 @@ DofsToVDofs<Ordering::byVDIM>(int ndofs, int vdim, Array<int> &dofs)
 
 FiniteElementSpace::FiniteElementSpace()
    : mesh(NULL), fec(NULL), vdim(0), ordering(Ordering::byNODES),
-     ndofs(0), nvdofs(0), nedofs(0), nfdofs(0), nbdofs(0),
-     bdofs(NULL),
-     elem_dof(NULL), elem_fos(NULL), bdr_elem_dof(NULL), bdr_elem_fos(NULL),
-     face_dof(NULL),
+     ndofs(0), nvdofs(0), nedofs(0), nfdofs(0), nbdofs(0), bdofs(NULL),
+     elem_dof(NULL), bdr_elem_dof(NULL), face_dof(NULL),
      NURBSext(NULL), own_ext(false),
-     DoFTrans(0), VDoFTrans(vdim, ordering),
      cP(NULL), cR(NULL), cR_hp(NULL), cP_is_set(false),
      Th(Operator::ANY_TYPE),
      sequence(0), mesh_sequence(0), orders_changed(false), relaxed_hp(false)
@@ -72,7 +69,6 @@ FiniteElementSpace::FiniteElementSpace()
 FiniteElementSpace::FiniteElementSpace(const FiniteElementSpace &orig,
                                        Mesh *mesh,
                                        const FiniteElementCollection *fec)
-   : VDoFTrans(orig.vdim, orig.ordering)
 {
    mesh = mesh ? mesh : orig.mesh;
    fec = fec ? fec : orig.fec;
@@ -263,36 +259,16 @@ void FiniteElementSpace::AdjustVDofs (Array<int> &vdofs)
    }
 }
 
-DofTransformation *
-FiniteElementSpace::GetElementVDofs(int i, Array<int> &vdofs) const
+void FiniteElementSpace::GetElementVDofs(int i, Array<int> &vdofs) const
 {
-   DofTransformation * doftrans = GetElementDofs(i, vdofs);
+   GetElementDofs(i, vdofs);
    DofsToVDofs(vdofs);
-   if (vdim == 1 || doftrans == NULL)
-   {
-      return doftrans;
-   }
-   else
-   {
-      VDoFTrans.SetDofTransformation(*doftrans);
-      return &VDoFTrans;
-   }
 }
 
-DofTransformation *
-FiniteElementSpace::GetBdrElementVDofs(int i, Array<int> &vdofs) const
+void FiniteElementSpace::GetBdrElementVDofs(int i, Array<int> &vdofs) const
 {
-   DofTransformation * doftrans = GetBdrElementDofs(i, vdofs);
+   GetBdrElementDofs(i, vdofs);
    DofsToVDofs(vdofs);
-   if (vdim == 1 || doftrans == NULL)
-   {
-      return doftrans;
-   }
-   else
-   {
-      VDoFTrans.SetDofTransformation(*doftrans);
-      return &VDoFTrans;
-   }
 }
 
 void FiniteElementSpace::GetFaceVDofs(int i, Array<int> &vdofs) const
@@ -331,39 +307,21 @@ void FiniteElementSpace::BuildElementToDofTable() const
 
    // TODO: can we call GetElementDofs only once per element?
    Table *el_dof = new Table;
-   Table *el_fos = (mesh->Dimension() > 2) ? (new Table) : NULL;
    Array<int> dofs;
-   Array<int> F, Fo;
    el_dof -> MakeI (mesh -> GetNE());
-   if (el_fos) { el_fos -> MakeI (mesh -> GetNE()); }
    for (int i = 0; i < mesh -> GetNE(); i++)
    {
       GetElementDofs (i, dofs);
       el_dof -> AddColumnsInRow (i, dofs.Size());
-
-      if (el_fos)
-      {
-         mesh->GetElementFaces(i, F, Fo);
-         el_fos -> AddColumnsInRow (i, Fo.Size());
-      }
    }
    el_dof -> MakeJ();
-   if (el_fos) { el_fos -> MakeJ(); }
    for (int i = 0; i < mesh -> GetNE(); i++)
    {
       GetElementDofs (i, dofs);
       el_dof -> AddConnections (i, (int *)dofs, dofs.Size());
-
-      if (el_fos)
-      {
-         mesh->GetElementFaces(i, F, Fo);
-         el_fos -> AddConnections (i, (int *)Fo, Fo.Size());
-      }
    }
    el_dof -> ShiftUpI();
-   if (el_fos) { el_fos -> ShiftUpI(); }
    elem_dof = el_dof;
-   elem_fos = el_fos;
 }
 
 void FiniteElementSpace::BuildBdrElementToDofTable() const
@@ -417,9 +375,7 @@ void FiniteElementSpace::BuildFaceToDofTable() const
 void FiniteElementSpace::RebuildElementToDofTable()
 {
    delete elem_dof;
-   delete elem_fos;
    elem_dof = NULL;
-   elem_fos = NULL;
    BuildElementToDofTable();
 }
 
@@ -1359,10 +1315,8 @@ const FaceQuadratureInterpolator
 
 SparseMatrix *FiniteElementSpace::RefinementMatrix_main(
    const int coarse_ndofs, const Table &coarse_elem_dof,
-   const Table *coarse_elem_fos, const DenseTensor localP[]) const
+   const DenseTensor localP[]) const
 {
-   /// TODO: Implement DofTransformation support
-
    MFEM_VERIFY(mesh->GetLastOperation() == Mesh::REFINE, "");
 
    Array<int> dofs, coarse_dofs, coarse_vdofs;
@@ -1445,8 +1399,7 @@ void FiniteElementSpace::GetLocalRefinementMatrices(
 }
 
 SparseMatrix* FiniteElementSpace::RefinementMatrix(int old_ndofs,
-                                                   const Table* old_elem_dof,
-                                                   const Table* old_elem_fos)
+                                                   const Table* old_elem_dof)
 {
    MFEM_VERIFY(GetNE() >= old_elem_dof->Size(),
                "Previous mesh is not coarser.");
@@ -1459,16 +1412,13 @@ SparseMatrix* FiniteElementSpace::RefinementMatrix(int old_ndofs,
       GetLocalRefinementMatrices(elem_geoms[i], localP[elem_geoms[i]]);
    }
 
-   return RefinementMatrix_main(old_ndofs, *old_elem_dof, old_elem_fos,
-                                localP);
+   return RefinementMatrix_main(old_ndofs, *old_elem_dof, localP);
 }
 
 FiniteElementSpace::RefinementOperator::RefinementOperator
-(const FiniteElementSpace* fespace, Table* old_elem_dof, Table* old_elem_fos,
- int old_ndofs)
+(const FiniteElementSpace* fespace, Table* old_elem_dof, int old_ndofs)
    : fespace(fespace)
    , old_elem_dof(old_elem_dof)
-   , old_elem_fos(old_elem_fos)
 {
    MFEM_VERIFY(fespace->GetNE() >= old_elem_dof->Size(),
                "Previous mesh is not coarser.");
@@ -1482,14 +1432,12 @@ FiniteElementSpace::RefinementOperator::RefinementOperator
    {
       fespace->GetLocalRefinementMatrices(elem_geoms[i], localP[elem_geoms[i]]);
    }
-
-   ConstructDoFTrans();
 }
 
 FiniteElementSpace::RefinementOperator::RefinementOperator(
    const FiniteElementSpace *fespace, const FiniteElementSpace *coarse_fes)
    : Operator(fespace->GetVSize(), coarse_fes->GetVSize()),
-     fespace(fespace), old_elem_dof(NULL), old_elem_fos(NULL)
+     fespace(fespace), old_elem_dof(NULL)
 {
    Mesh::GeometryList elem_geoms(*fespace->GetMesh());
 
@@ -1501,50 +1449,11 @@ FiniteElementSpace::RefinementOperator::RefinementOperator(
 
    // Make a copy of the coarse elem_dof Table.
    old_elem_dof = new Table(coarse_fes->GetElementToDofTable());
-
-   // Make a copy of the coarse elem_fos Table if it exists.
-   if (coarse_fes->GetElementToFaceOrientationTable())
-   {
-      old_elem_fos = new Table(*coarse_fes->GetElementToFaceOrientationTable());
-   }
-
-   ConstructDoFTrans();
 }
 
 FiniteElementSpace::RefinementOperator::~RefinementOperator()
 {
    delete old_elem_dof;
-   delete old_elem_fos;
-}
-
-void FiniteElementSpace::RefinementOperator
-::ConstructDoFTrans()
-{
-   old_DoFTrans.SetSize(Geometry::NUM_GEOMETRIES);
-   for (int i=0; i<old_DoFTrans.Size(); i++)
-   {
-      old_DoFTrans[i] = NULL;
-   }
-
-   const FiniteElementCollection *fec = fespace->FEColl();
-   if (dynamic_cast<const ND_FECollection*>(fec))
-   {
-      const FiniteElement * nd_tri =
-         fec->FiniteElementForGeometry(Geometry::TRIANGLE);
-      if (nd_tri)
-      {
-         old_DoFTrans[Geometry::TRIANGLE] =
-            new ND_TriDofTransformation(nd_tri->GetOrder());
-      }
-
-      const FiniteElement * nd_tet =
-         fec->FiniteElementForGeometry(Geometry::TETRAHEDRON);
-      if (nd_tet)
-      {
-         old_DoFTrans[Geometry::TETRAHEDRON] =
-            new ND_TetDofTransformation(nd_tet->GetOrder());
-      }
-   }
 }
 
 void FiniteElementSpace::RefinementOperator
@@ -1553,7 +1462,7 @@ void FiniteElementSpace::RefinementOperator
    Mesh* mesh = fespace->GetMesh();
    const CoarseFineTransformations &rtrans = mesh->GetRefinementTransforms();
 
-   Array<int> dofs, vdofs, old_dofs, old_vdofs, old_Fo;
+   Array<int> dofs, vdofs, old_dofs, old_vdofs;
 
    int vdim = fespace->GetVDim();
    int old_ndofs = width / vdim;
@@ -1568,53 +1477,18 @@ void FiniteElementSpace::RefinementOperator
 
       subY.SetSize(lP.Height());
 
-      DofTransformation *doftrans = fespace->GetElementDofs(k, dofs);
+      fespace->GetElementDofs(k, dofs);
       old_elem_dof->GetRow(emb.parent, old_dofs);
 
-      if (!doftrans)
+      for (int vd = 0; vd < vdim; vd++)
       {
-         for (int vd = 0; vd < vdim; vd++)
-         {
-            dofs.Copy(vdofs);
-            fespace->DofsToVDofs(vd, vdofs);
-            old_dofs.Copy(old_vdofs);
-            fespace->DofsToVDofs(vd, old_vdofs, old_ndofs);
-            x.GetSubVector(old_vdofs, subX);
-            lP.Mult(subX, subY);
-            y.SetSubVector(vdofs, subY);
-         }
-      }
-      else
-      {
-         old_elem_fos->GetRow(emb.parent, old_Fo);
-         old_DoFTrans[geom]->SetFaceOrientations(old_Fo);
-
-         DofTransformation *new_doftrans = NULL;
-         VDofTransformation *vdoftrans =
-            dynamic_cast<VDofTransformation*>(doftrans);
-         if (vdoftrans)
-         {
-            new_doftrans = doftrans;
-            doftrans = vdoftrans->GetDofTransformation();
-         }
-
-         for (int vd = 0; vd < vdim; vd++)
-         {
-            dofs.Copy(vdofs);
-            fespace->DofsToVDofs(vd, vdofs);
-            old_dofs.Copy(old_vdofs);
-            fespace->DofsToVDofs(vd, old_vdofs, old_ndofs);
-            x.GetSubVector(old_vdofs, subX);
-            old_DoFTrans[geom]->InvTransformPrimal(subX);
-            lP.Mult(subX, subY);
-            doftrans->TransformPrimal(subY);
-            y.SetSubVector(vdofs, subY);
-         }
-
-         if (vdoftrans)
-         {
-            doftrans = new_doftrans;
-         }
+         dofs.Copy(vdofs);
+         fespace->DofsToVDofs(vd, vdofs);
+         old_dofs.Copy(old_vdofs);
+         fespace->DofsToVDofs(vd, old_vdofs, old_ndofs);
+         x.GetSubVector(old_vdofs, subX);
+         lP.Mult(subX, subY);
+         y.SetSubVector(vdofs, subY);
       }
    }
 }
@@ -1630,12 +1504,12 @@ void FiniteElementSpace::RefinementOperator
    Array<char> processed(fespace->GetVSize());
    processed = 0;
 
-   Array<int> f_dofs, c_dofs, f_vdofs, c_vdofs, old_Fo;
+   Array<int> f_dofs, c_dofs, f_vdofs, c_vdofs;
 
    int vdim = fespace->GetVDim();
    int old_ndofs = width / vdim;
 
-   Vector subY, subX, subYt, subXt;
+   Vector subY, subX;
 
    for (int k = 0; k < mesh->GetNE(); k++)
    {
@@ -1643,77 +1517,30 @@ void FiniteElementSpace::RefinementOperator
       const Geometry::Type geom = mesh->GetElementBaseGeometry(k);
       const DenseMatrix &lP = localP[geom](emb.matrix);
 
-      DofTransformation * doftrans = fespace->GetElementDofs(k, f_dofs);
+      fespace->GetElementDofs(k, f_dofs);
       old_elem_dof->GetRow(emb.parent, c_dofs);
 
-      if (!doftrans)
+      subY.SetSize(lP.Width());
+
+      for (int vd = 0; vd < vdim; vd++)
       {
-         subY.SetSize(lP.Width());
+         f_dofs.Copy(f_vdofs);
+         fespace->DofsToVDofs(vd, f_vdofs);
+         c_dofs.Copy(c_vdofs);
+         fespace->DofsToVDofs(vd, c_vdofs, old_ndofs);
 
-         for (int vd = 0; vd < vdim; vd++)
+         x.GetSubVector(f_vdofs, subX);
+
+         for (int p = 0; p < f_dofs.Size(); ++p)
          {
-            f_dofs.Copy(f_vdofs);
-            fespace->DofsToVDofs(vd, f_vdofs);
-            c_dofs.Copy(c_vdofs);
-            fespace->DofsToVDofs(vd, c_vdofs, old_ndofs);
-
-            x.GetSubVector(f_vdofs, subX);
-
-            for (int p = 0; p < f_dofs.Size(); ++p)
+            if (processed[DecodeDof(f_dofs[p])])
             {
-               if (processed[DecodeDof(f_dofs[p])])
-               {
-                  subX[p] = 0.0;
-               }
+               subX[p] = 0.0;
             }
-
-            lP.MultTranspose(subX, subY);
-            y.AddElementVector(c_vdofs, subY);
-         }
-      }
-      else
-      {
-         subYt.SetSize(lP.Width());
-
-         old_elem_fos->GetRow(emb.parent, old_Fo);
-         old_DoFTrans[geom]->SetFaceOrientations(old_Fo);
-
-         DofTransformation *new_doftrans = NULL;
-         VDofTransformation *vdoftrans =
-            dynamic_cast<VDofTransformation*>(doftrans);
-         if (vdoftrans)
-         {
-            new_doftrans = doftrans;
-            doftrans = vdoftrans->GetDofTransformation();
          }
 
-         for (int vd = 0; vd < vdim; vd++)
-         {
-            f_dofs.Copy(f_vdofs);
-            fespace->DofsToVDofs(vd, f_vdofs);
-            c_dofs.Copy(c_vdofs);
-            fespace->DofsToVDofs(vd, c_vdofs, old_ndofs);
-
-            x.GetSubVector(f_vdofs, subX);
-            old_DoFTrans[geom]->InvTransformPrimal(subX);
-
-            for (int p = 0; p < f_dofs.Size(); ++p)
-            {
-               if (processed[DecodeDof(f_dofs[p])])
-               {
-                  subX[p] = 0.0;
-               }
-            }
-
-            lP.MultTranspose(subX, subY);
-            doftrans->TransformPrimal(subY);
-            y.AddElementVector(c_vdofs, subY);
-         }
-
-         if (vdoftrans)
-         {
-            doftrans = new_doftrans;
-         }
+         lP.MultTranspose(subX, subY);
+         y.AddElementVector(c_vdofs, subY);
       }
 
       for (int p = 0; p < f_dofs.Size(); ++p)
@@ -1723,7 +1550,6 @@ void FiniteElementSpace::RefinementOperator
    }
 }
 
-/// TODO: Implement DofTransformation support
 FiniteElementSpace::DerefinementOperator::DerefinementOperator(
    const FiniteElementSpace *f_fes, const FiniteElementSpace *c_fes,
    BilinearFormIntegrator *mass_integ)
@@ -1881,11 +1707,8 @@ void FiniteElementSpace::GetLocalDerefinementMatrices(Geometry::Type geom,
 }
 
 SparseMatrix* FiniteElementSpace::DerefinementMatrix(int old_ndofs,
-                                                     const Table* old_elem_dof,
-                                                     const Table* old_elem_fos)
+                                                     const Table* old_elem_dof)
 {
-   /// TODO: Implement DofTransformation support
-
    MFEM_VERIFY(Nonconforming(), "Not implemented for conforming meshes.");
    MFEM_VERIFY(old_ndofs, "Missing previous (finer) space.");
    MFEM_VERIFY(ndofs <= old_ndofs, "Previous space is not finer.");
@@ -1991,7 +1814,6 @@ void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
    this->ordering = (Ordering::Type) ordering;
 
    elem_dof = NULL;
-   elem_fos = NULL;
    face_dof = NULL;
 
    sequence = 0;
@@ -2019,8 +1841,6 @@ void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
       UpdateNURBS();
       cP = cR = cR_hp = NULL;
       cP_is_set = false;
-
-      ConstructDoFTrans();
    }
    else
    {
@@ -2028,41 +1848,9 @@ void FiniteElementSpace::Constructor(Mesh *mesh, NURBSExtension *NURBSext,
       own_ext = 0;
       Construct();
    }
-
    BuildElementToDofTable();
 }
 
-void FiniteElementSpace::ConstructDoFTrans()
-{
-   DestroyDoFTrans();
-
-   VDoFTrans.SetVDim(vdim);
-   DoFTrans.SetSize(Geometry::NUM_GEOMETRIES);
-   for (int i=0; i<DoFTrans.Size(); i++)
-   {
-      DoFTrans[i] = NULL;
-   }
-   if (mesh->Dimension() < 3) { return; }
-   if (dynamic_cast<const ND_FECollection*>(fec))
-   {
-      const FiniteElement * nd_tri =
-         fec->FiniteElementForGeometry(Geometry::TRIANGLE);
-      if (nd_tri)
-      {
-         DoFTrans[Geometry::TRIANGLE] =
-            new ND_TriDofTransformation(nd_tri->GetOrder());
-      }
-
-      const FiniteElement * nd_tet =
-         fec->FiniteElementForGeometry(Geometry::TETRAHEDRON);
-      if (nd_tet)
-      {
-         DoFTrans[Geometry::TETRAHEDRON] =
-            new ND_TetDofTransformation(nd_tet->GetOrder());
-      }
-   }
-}
-
 NURBSExtension *FiniteElementSpace::StealNURBSext()
 {
    if (NURBSext && !own_ext)
@@ -2158,9 +1946,7 @@ void FiniteElementSpace::Construct()
                "Variable order space requires a nonconforming mesh.");
 
    elem_dof = NULL;
-   elem_fos = NULL;
    bdr_elem_dof = NULL;
-   bdr_elem_fos = NULL;
    face_dof = NULL;
 
    ndofs = 0;
@@ -2258,8 +2044,6 @@ void FiniteElementSpace::Construct()
 
    ndofs = nvdofs + nedofs + nfdofs + nbdofs;
 
-   ConstructDoFTrans();
-
    // record the current mesh sequence number to detect refinement etc.
    mesh_sequence = mesh->GetSequence();
 
@@ -2511,22 +2295,14 @@ int FiniteElementSpace::GetNVariants(int entity, int index) const
 static const char* msg_orders_changed =
    "Element orders changed, you need to Update() the space first.";
 
-DofTransformation *
-FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
+void FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
 {
    MFEM_VERIFY(!orders_changed, msg_orders_changed);
 
    if (elem_dof)
    {
       elem_dof->GetRow(elem, dofs);
-
-      if (DoFTrans[mesh->GetElementBaseGeometry(elem)])
-      {
-         Array<int> Fo;
-         elem_fos -> GetRow (elem, Fo);
-         DoFTrans[mesh->GetElementBaseGeometry(elem)]->SetFaceOrientations(Fo);
-      }
-      return DoFTrans[mesh->GetElementBaseGeometry(elem)];
+      return;
    }
 
    Array<int> V, E, Eo, F, Fo; // TODO: LocalArray
@@ -2550,11 +2326,6 @@ FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
       {
          nfd += fec->GetNumDof(mesh->GetFaceGeometry(F[i]), order);
       }
-      if (DoFTrans[mesh->GetElementBaseGeometry(elem)])
-      {
-         DoFTrans[mesh->GetElementBaseGeometry(elem)]
-         -> SetFaceOrientations(Fo);
-      }
    }
 
    dofs.SetSize(0);
@@ -2612,7 +2383,6 @@ FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
          dofs.Append(bbase + j);
       }
    }
-   return DoFTrans[mesh->GetElementBaseGeometry(elem)];
 }
 
 const FiniteElement *FiniteElementSpace::GetFE(int i) const
@@ -2645,27 +2415,18 @@ const FiniteElement *FiniteElementSpace::GetFE(int i) const
    return FE;
 }
 
-DofTransformation *
-FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const
+void FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const
 {
    MFEM_VERIFY(!orders_changed, msg_orders_changed);
 
    if (bdr_elem_dof)
    {
       bdr_elem_dof->GetRow(bel, dofs);
-
-      if (DoFTrans[mesh->GetBdrElementBaseGeometry(bel)])
-      {
-         Array<int> Fo;
-         bdr_elem_fos -> GetRow (bel, Fo);
-         DoFTrans[mesh->GetBdrElementBaseGeometry(bel)]->
-         SetFaceOrientations(Fo);
-      }
-      return DoFTrans[mesh->GetBdrElementBaseGeometry(bel)];
+      return;
    }
 
-   Array<int> V, E, Eo, Fo; // TODO: LocalArray
-   int F, oF;
+   Array<int> V, E, Eo; // TODO: LocalArray
+   int F, Fo;
 
    int dim = mesh->Dimension();
    auto geom = mesh->GetBdrElementGeometry(bel);
@@ -2684,17 +2445,7 @@ FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const
 
    if (nv) { mesh->GetBdrElementVertices(bel, V); }
    if (ne) { mesh->GetBdrElementEdges(bel, E, Eo); }
-   if (nf)
-   {
-      mesh->GetBdrElementFace(bel, &F, &oF);
-
-      if (DoFTrans[mesh->GetBdrElementBaseGeometry(bel)])
-      {
-         Fo.Append(oF);
-         DoFTrans[mesh->GetBdrElementBaseGeometry(bel)]->
-         SetFaceOrientations(Fo);
-      }
-   }
+   if (nf) { mesh->GetBdrElementFace(bel, &F, &Fo); }
 
    dofs.SetSize(0);
    dofs.Reserve(nv*V.Size() + ne*E.Size() + nf);
@@ -2727,15 +2478,13 @@ FiniteElementSpace::GetBdrElementDofs(int bel, Array<int> &dofs) const
    if (nf) // face DOFs
    {
       int fbase = (var_face_dofs.Size() > 0) ? FindFaceDof(F, nf) : F*nf;
-      const int *ind = fec->GetDofOrdering(geom, order, oF);
+      const int *ind = fec->GetDofOrdering(geom, order, Fo);
 
       for (int j = 0; j < nf; j++)
       {
          dofs.Append(EncodeDof(nvdofs + nedofs + fbase, ind[j]));
       }
    }
-
-   return DoFTrans[mesh->GetBdrElementBaseGeometry(bel)];
 }
 
 int FiniteElementSpace::GetFaceDofs(int face, Array<int> &dofs,
@@ -3045,8 +2794,6 @@ void FiniteElementSpace::Destroy()
    }
    E2BFQ_array.SetSize(0);
 
-   DestroyDoFTrans();
-
    dof_elem_array.DeleteAll();
    dof_ldof_array.DeleteAll();
 
@@ -3059,9 +2806,7 @@ void FiniteElementSpace::Destroy()
    else
    {
       delete elem_dof;
-      delete elem_fos;
       delete bdr_elem_dof;
-      delete bdr_elem_fos;
       delete face_dof;
 
       delete [] bdofs;
@@ -3069,15 +2814,6 @@ void FiniteElementSpace::Destroy()
    ceed::RemoveBasisAndRestriction(this);
 }
 
-void FiniteElementSpace::DestroyDoFTrans()
-{
-   for (int i = 0; i < DoFTrans.Size(); i++)
-   {
-      delete DoFTrans[i];
-   }
-   DoFTrans.SetSize(0);
-}
-
 void FiniteElementSpace::GetTransferOperator(
    const FiniteElementSpace &coarse_fes, OperatorHandle &T) const
 {
@@ -3095,8 +2831,6 @@ void FiniteElementSpace::GetTransferOperator(
       }
       T.Reset(RefinementMatrix_main(coarse_fes.GetNDofs(),
                                     coarse_fes.GetElementToDofTable(),
-                                    coarse_fes.
-                                    GetElementToFaceOrientationTable(),
                                     localP));
    }
    else
@@ -3199,7 +2933,6 @@ void FiniteElementSpace::Update(bool want_transform)
    }
 
    Table* old_elem_dof = NULL;
-   Table* old_elem_fos = NULL;
    int old_ndofs;
    bool old_orders_changed = orders_changed;
 
@@ -3207,9 +2940,7 @@ void FiniteElementSpace::Update(bool want_transform)
    if (want_transform)
    {
       old_elem_dof = elem_dof;
-      old_elem_fos = elem_fos;
       elem_dof = NULL;
-      elem_fos = NULL;
       old_ndofs = ndofs;
    }
 
@@ -3235,18 +2966,15 @@ void FiniteElementSpace::Update(bool want_transform)
          {
             if (Th.Type() != Operator::MFEM_SPARSEMAT)
             {
-               Th.Reset(new RefinementOperator(this, old_elem_dof,
-                                               old_elem_fos, old_ndofs));
+               Th.Reset(new RefinementOperator(this, old_elem_dof, old_ndofs));
                // The RefinementOperator takes ownership of 'old_elem_dof', so
                // we no longer own it:
                old_elem_dof = NULL;
-               old_elem_fos = NULL;
             }
             else
             {
                // calculate fully assembled matrix
-               Th.Reset(RefinementMatrix(old_ndofs, old_elem_dof,
-                                         old_elem_fos));
+               Th.Reset(RefinementMatrix(old_ndofs, old_elem_dof));
             }
             break;
          }
@@ -3254,7 +2982,7 @@ void FiniteElementSpace::Update(bool want_transform)
          case Mesh::DEREFINE:
          {
             BuildConformingInterpolation();
-            Th.Reset(DerefinementMatrix(old_ndofs, old_elem_dof, old_elem_fos));
+            Th.Reset(DerefinementMatrix(old_ndofs, old_elem_dof));
             if (cP && cR)
             {
                Th.SetOperatorOwner(false);
@@ -3269,7 +2997,6 @@ void FiniteElementSpace::Update(bool want_transform)
       }
 
       delete old_elem_dof;
-      delete old_elem_fos;
    }
 }
 

fem/fespace.hpp

@@ -16,7 +16,6 @@
 #include "../linalg/sparsemat.hpp"
 #include "../mesh/mesh.hpp"
 #include "fe_coll.hpp"
-#include "doftrans.hpp"
 #include "restriction.hpp"
 #include <iostream>
 #include <unordered_map>
@@ -129,9 +128,7 @@ protected:
 
    // precalculated DOFs for each element, boundary element, and face
    mutable Table *elem_dof; // owned (except in NURBS FE space)
-   mutable Table *elem_fos; // face orientations by element index
    mutable Table *bdr_elem_dof; // owned (except in NURBS FE space)
-   mutable Table *bdr_elem_fos; // bdr face orientations by bdr element index
    mutable Table *face_dof; // owned; in var-order space contains variant 0 DOFs
 
    Array<int> dof_elem_array, dof_ldof_array;
@@ -140,9 +137,6 @@ protected:
    int own_ext;
    mutable Array<int> face_to_be; // NURBS FE space only
 
-   Array<DofTransformation*> DoFTrans;
-   mutable VDofTransformation VDoFTrans;
-
    /** Matrix representing the prolongation from the global conforming dofs to
        a set of intermediate partially conforming dofs, e.g. the dofs associated
        with a "cut" space on a non-conforming mesh. */
@@ -195,9 +189,6 @@ protected:
    void Construct();
    void Destroy();
 
-   void ConstructDoFTrans();
-   void DestroyDoFTrans();
-
    void BuildElementToDofTable() const;
    void BuildBdrElementToDofTable() const;
    void BuildFaceToDofTable() const;
@@ -292,19 +283,12 @@ protected:
       const FiniteElementSpace* fespace;
       DenseTensor localP[Geometry::NumGeom];
       Table* old_elem_dof; // Owned.
-      Table* old_elem_fos; // Owned.
-
-      Array<DofTransformation*> old_DoFTrans;
-      mutable VDofTransformation old_VDoFTrans;
-
-      void ConstructDoFTrans();
 
    public:
       /** Construct the operator based on the elem_dof table of the original
           (coarse) space. The class takes ownership of the table. */
       RefinementOperator(const FiniteElementSpace* fespace,
-                         Table *old_elem_dof/*takes ownership*/,
-                         Table *old_elem_fos/*takes ownership*/, int old_ndofs);
+                         Table *old_elem_dof/*takes ownership*/, int old_ndofs);
       RefinementOperator(const FiniteElementSpace *fespace,
                          const FiniteElementSpace *coarse_fes);
       virtual void Mult(const Vector &x, Vector &y) const;
@@ -318,7 +302,6 @@ protected:
       const FiniteElementSpace *fine_fes; // Not owned.
       DenseTensor localR[Geometry::NumGeom];
       Table *coarse_elem_dof; // Owned.
-      // Table *coarse_elem_fos; // Owned.
       Table coarse_to_fine;
       Array<int> coarse_to_ref_type;
       Array<Geometry::Type> ref_type_to_geom;
@@ -340,7 +323,6 @@ protected:
        the same vector dimension, vdim. */
    SparseMatrix *RefinementMatrix_main(const int coarse_ndofs,
                                        const Table &coarse_elem_dof,
-                                       const Table *coarse_elem_fos,
                                        const DenseTensor localP[]) const;
 
    void GetLocalRefinementMatrices(Geometry::Type geom,
@@ -351,12 +333,10 @@ protected:
    /** Calculate explicit GridFunction interpolation matrix (after mesh
        refinement). NOTE: consider using the RefinementOperator class instead
        of the fully assembled matrix, which can take a lot of memory. */
-   SparseMatrix* RefinementMatrix(int old_ndofs, const Table* old_elem_dof,
-                                  const Table* old_elem_fos);
+   SparseMatrix* RefinementMatrix(int old_ndofs, const Table* old_elem_dof);
 
    /// Calculate GridFunction restriction matrix after mesh derefinement.
-   SparseMatrix* DerefinementMatrix(int old_ndofs, const Table* old_elem_dof,
-                                    const Table* old_elem_fos);
+   SparseMatrix* DerefinementMatrix(int old_ndofs, const Table* old_elem_dof);
 
    /** @brief Return in @a localP the local refinement matrices that map
        between fespaces after mesh refinement. */
@@ -634,11 +614,10 @@ public:
    int GetBdrAttribute(int i) const { return mesh->GetBdrAttribute(i); }
 
    /// Returns indices of degrees of freedom of element 'elem'.
-   virtual DofTransformation *GetElementDofs(int elem, Array<int> &dofs) const;
+   virtual void GetElementDofs(int elem, Array<int> &dofs) const;
 
    /// Returns indices of degrees of freedom for boundary element 'bel'.
-   virtual DofTransformation *GetBdrElementDofs(int bel,
-                                                Array<int> &dofs) const;
+   virtual void GetBdrElementDofs(int bel, Array<int> &dofs) const;
 
    /** @brief Returns the indices of the degrees of freedom for the specified
        face, including the DOFs for the edges and the vertices of the face. */
@@ -687,10 +666,10 @@ public:
    static void AdjustVDofs(Array<int> &vdofs);
 
    /// Returns indexes of degrees of freedom in array dofs for i'th element.
-   DofTransformation *GetElementVDofs(int i, Array<int> &vdofs) const;
+   void GetElementVDofs(int i, Array<int> &vdofs) const;
 
    /// Returns indexes of degrees of freedom for i'th boundary element.
-   DofTransformation *GetBdrElementVDofs(int i, Array<int> &vdofs) const;
+   void GetBdrElementVDofs(int i, Array<int> &vdofs) const;
 
    /// Returns indexes of degrees of freedom for i'th face element (2D and 3D).
    void GetFaceVDofs(int i, Array<int> &vdofs) const;
@@ -716,8 +695,6 @@ public:
        is preserved. */
    void ReorderElementToDofTable();
 
-   const Table *GetElementToFaceOrientationTable() const { return elem_fos; }
-
    /** @brief Return a reference to the internal Table that stores the lists of
        scalar dofs, for each mesh element, as returned by GetElementDofs(). */
    const Table &GetElementToDofTable() const { return *elem_dof; }

fem/fmsconvert.hpp

@@ -1,46 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef FMS_CONVERT
-#define FMS_CONVERT
-
-#include "../config/config.hpp"
-#include "datacollection.hpp"
-
-#ifdef MFEM_USE_FMS
-#include <fms.h>
-
-namespace mfem
-{
-
-/** In-memory conversion of FMS data collection to an MFEM data collection.
-    @param dc The FMS data collection to convert.
-    @param[out] mfem_dc A pointer to a new MFEM DataCollection containing the
-    FMS data.
-    @return 0 on success; non-zero on failure.
-*/
-int FmsDataCollectionToDataCollection(FmsDataCollection dc,
-                                      DataCollection **mfem_dc);
-
-/** In-memory conversion of MFEM data collection to an FMS data collection.
-    @param mfem_dc The MFEM data collection to convert.
-    @param[out] dc A pointer to a new FmsDataCollection containing the MFEM
-    data.
-    @return 0 on success; non-zero on failure.
-*/
-int DataCollectionToFmsDataCollection(DataCollection *mfem_dc,
-                                      FmsDataCollection *dc);
-
-} // namespace mfem
-
-#endif
-
-#endif

fem/fmsdatacollection.cpp

@@ -1,167 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../config/config.hpp"
-
-#ifdef MFEM_USE_FMS
-
-#include "fem.hpp"
-#include "../general/text.hpp"
-
-#include <fmsio.h>
-
-#include <string>
-#include <sstream>
-
-namespace mfem
-{
-
-// class FMSDataCollection implementation
-
-FMSDataCollection::FMSDataCollection(const std::string& coll_name,
-                                     Mesh *mesh)
-   : DataCollection(coll_name, mesh),
-     fms_protocol("ascii")
-{
-   appendRankToFileName = false; // always include rank in file names
-   cycle = 0;                   // always include cycle in directory names
-}
-
-#ifdef MFEM_USE_MPI
-
-FMSDataCollection::FMSDataCollection(MPI_Comm comm,
-                                     const std::string& coll_name,
-                                     Mesh *mesh)
-   : DataCollection(coll_name, mesh),
-     fms_protocol("ascii")
-{
-   m_comm = comm;
-   MPI_Comm_rank(comm, &myid);
-   MPI_Comm_size(comm, &num_procs);
-   appendRankToFileName = true; // always include rank in file names
-   cycle = 0;                   // always include cycle in directory names
-}
-
-#endif
-
-FMSDataCollection::~FMSDataCollection()
-{
-   // empty
-}
-
-void FMSDataCollection::Save()
-{
-   // Convert this to FmsDataCollection.
-
-   FmsDataCollection dc;
-   if (DataCollectionToFmsDataCollection(this, &dc) == 0)
-   {
-      std::string root(RootFileName());
-      int err = FmsIOWrite(root.c_str(), fms_protocol.c_str(), dc);
-      FmsDataCollectionDestroy(&dc);
-      if (err)
-      {
-         MFEM_ABORT("Error creating FMS file: " << root);
-      }
-   }
-   else
-   {
-      MFEM_ABORT("Error converting data collection");
-   }
-}
-
-void FMSDataCollection::Load(int cycle)
-{
-   DeleteAll();
-   this->cycle = cycle;
-
-   FmsDataCollection dc;
-   std::string root(RootFileName());
-   int err = FmsIORead(root.c_str(), fms_protocol.c_str(), &dc);
-
-   if (err == 0)
-   {
-      DataCollection *mdc = nullptr;
-      if (FmsDataCollectionToDataCollection(dc,&mdc) == 0)
-      {
-         // Tell the data collection we read that it does not own data.
-         // We will steal its data.
-         mdc->SetOwnData(false);
-
-         SetCycle(mdc->GetCycle());
-         SetTime(mdc->GetTime());
-         SetTimeStep(mdc->GetTimeStep());
-         name = mdc->GetCollectionName();
-
-         // Set mdc's mesh as our mesh.
-         SetMesh(mdc->GetMesh());
-
-         // Set mdc's fields/qfields as ours.
-         std::vector<std::string> names;
-         for (const auto &pair : mdc->GetFieldMap())
-         {
-            names.push_back(pair.first);
-            RegisterField(pair.first, pair.second);
-         }
-         for (const auto &name : names)
-         {
-            mdc->DeregisterField(name);
-         }
-
-         names.clear();
-         for (const auto &pair : mdc->GetQFieldMap())
-         {
-            names.push_back(pair.first);
-            RegisterQField(pair.first, pair.second);
-         }
-         for (const auto &name : names)
-         {
-            mdc->DeregisterField(name);
-         }
-
-         // Indicate that we own the data.
-         SetOwnData(true);
-
-         // Delete mdc. We stole its contents.
-         delete mdc;
-      }
-      FmsDataCollectionDestroy(&dc);
-   }
-   else
-   {
-      MFEM_ABORT("Error reading data collection: " << root);
-   }
-}
-
-void FMSDataCollection::SetProtocol(const std::string &protocol)
-{
-   fms_protocol = protocol;
-}
-
-std::string FMSDataCollection::RootFileName()
-{
-   std::string res;
-   if (pad_digits_cycle)
-   {
-      res = prefix_path + name + "_" +
-            to_padded_string(cycle, pad_digits_cycle) +
-            ".fms";
-   }
-   else
-   {
-      res = prefix_path + name + ".fms";
-   }
-   return res;
-}
-
-} // namespace mfem
-
-#endif

fem/fmsdatacollection.hpp

@@ -1,74 +0,0 @@
-// Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_FMSDATACOLLECTION
-#define MFEM_FMSDATACOLLECTION
-
-#include "../config/config.hpp"
-
-#ifdef MFEM_USE_FMS
-
-#include "datacollection.hpp"
-#include <fms.h>
-
-namespace mfem
-{
-
-/** @brief Data collection that uses FMS. */
-/** FMSDataCollection lets MFEM read/write data using FMS.
-
-    For more information, see:
-    - FMS project, https://ceed.exascaleproject.org/fms/
-*/
-
-/// Data collection with FMS I/O routines
-class FMSDataCollection : public DataCollection
-{
-protected:
-   // file name helpers
-
-   /// Returns file name for the current cycle
-   std::string RootFileName();
-
-   // holds currently active i/o protocol
-   std::string fms_protocol;
-
-public:
-   /// Constructor. The collection name is used when saving the data.
-   /** If @a mesh is NULL, then the mesh can be set later by calling either
-       SetMesh() or Load(). The latter works only in serial. */
-   FMSDataCollection(const std::string& collection_name,
-                     Mesh *mesh = NULL);
-#ifdef MFEM_USE_MPI
-   /// Construct a parallel FMSDataCollection.
-   FMSDataCollection(MPI_Comm comm, const std::string& collection_name,
-                     Mesh *mesh = NULL);
-#endif
-
-   /// We will delete the mesh and fields if we own them
-   virtual ~FMSDataCollection();
-
-   /// Set the FMS relay i/o protocol to use
-   /** Supported options: ascii (default), json, yaml, hdf5 */
-   void SetProtocol(const std::string &protocol);
-
-   /// Save the collection and a FMS blueprint root file
-   virtual void Save();
-
-   /// Load the collection based blueprint data
-   virtual void Load(int cycle = 0);
-};
-
-} // namespace mfem
-
-#endif
-
-#endif

fem/geom.cpp

@@ -11,19 +11,15 @@
 
 #include "fem.hpp"
 #include "../mesh/wedge.hpp"
-#include "../mesh/pyramid.hpp"
 
 namespace mfem
 {
 
 const char *Geometry::Name[NumGeom] =
-{
-   "Point", "Segment", "Triangle", "Square", "Tetrahedron", "Cube", "Prism",
-   "Pyramid"
-};
+{ "Point", "Segment", "Triangle", "Square", "Tetrahedron", "Cube", "Prism" };
 
 const double Geometry::Volume[NumGeom] =
-{ 1.0, 1.0, 0.5, 1.0, 1./6, 1.0, 0.5, 1./3 };
+{ 1.0, 1.0, 0.5, 1.0, 1./6, 1.0, 0.5 };
 
 Geometry::Geometry()
 {
@@ -143,28 +139,6 @@ Geometry::Geometry()
    GeomVert[6]->IntPoint(5).y = 1.0;
    GeomVert[6]->IntPoint(5).z = 1.0;
 
-   // Vertices for Geometry::PYRAMID
-   GeomVert[7] = new IntegrationRule(5);
-   GeomVert[7]->IntPoint(0).x = 0.0;
-   GeomVert[7]->IntPoint(0).y = 0.0;
-   GeomVert[7]->IntPoint(0).z = 0.0;
-
-   GeomVert[7]->IntPoint(1).x = 1.0;
-   GeomVert[7]->IntPoint(1).y = 0.0;
-   GeomVert[7]->IntPoint(1).z = 0.0;
-
-   GeomVert[7]->IntPoint(2).x = 1.0;
-   GeomVert[7]->IntPoint(2).y = 1.0;
-   GeomVert[7]->IntPoint(2).z = 0.0;
-
-   GeomVert[7]->IntPoint(3).x = 0.0;
-   GeomVert[7]->IntPoint(3).y = 1.0;
-   GeomVert[7]->IntPoint(3).z = 0.0;
-
-   GeomVert[7]->IntPoint(4).x = 0.0;
-   GeomVert[7]->IntPoint(4).y = 0.0;
-   GeomVert[7]->IntPoint(4).z = 1.0;
-
    GeomCenter[POINT].x = 0.0;
    GeomCenter[POINT].y = 0.0;
    GeomCenter[POINT].z = 0.0;
@@ -193,10 +167,6 @@ Geometry::Geometry()
    GeomCenter[PRISM].y = 1.0 / 3.0;
    GeomCenter[PRISM].z = 0.5;
 
-   GeomCenter[PYRAMID].x = 0.375;
-   GeomCenter[PYRAMID].y = 0.375;
-   GeomCenter[PYRAMID].z = 0.25;
-
    GeomToPerfGeomJac[POINT]       = NULL;
    GeomToPerfGeomJac[SEGMENT]     = new DenseMatrix(1);
    GeomToPerfGeomJac[TRIANGLE]    = new DenseMatrix(2);
@@ -204,7 +174,6 @@ Geometry::Geometry()
    GeomToPerfGeomJac[TETRAHEDRON] = new DenseMatrix(3);
    GeomToPerfGeomJac[CUBE]        = new DenseMatrix(3);
    GeomToPerfGeomJac[PRISM]       = new DenseMatrix(3);
-   GeomToPerfGeomJac[PYRAMID]     = new DenseMatrix(3);
 
    PerfGeomToGeomJac[POINT]       = NULL;
    PerfGeomToGeomJac[SEGMENT]     = NULL;
@@ -213,7 +182,6 @@ Geometry::Geometry()
    PerfGeomToGeomJac[TETRAHEDRON] = new DenseMatrix(3);
    PerfGeomToGeomJac[CUBE]        = NULL;
    PerfGeomToGeomJac[PRISM]       = new DenseMatrix(3);
-   PerfGeomToGeomJac[PYRAMID]     = new DenseMatrix(3);
 
    GeomToPerfGeomJac[SEGMENT]->Diag(1.0, 1);
    {
@@ -242,14 +210,6 @@ Geometry::Geometry()
       *GeomToPerfGeomJac[PRISM] = pri_T.Jacobian();
       CalcInverse(pri_T.Jacobian(), *PerfGeomToGeomJac[PRISM]);
    }
-   {
-      IsoparametricTransformation pyr_T;
-      pyr_T.SetFE(&PyramidFE);
-      GetPerfPointMat (PYRAMID, pyr_T.GetPointMat());
-      pyr_T.SetIntPoint(&GeomCenter[PYRAMID]);
-      *GeomToPerfGeomJac[PYRAMID] = pyr_T.Jacobian();
-      CalcInverse(pyr_T.Jacobian(), *PerfGeomToGeomJac[PYRAMID]);
-   }
 }
 
 Geometry::~Geometry()
@@ -273,7 +233,6 @@ const IntegrationRule * Geometry::GetVertices(int GeomType)
       case Geometry::TETRAHEDRON: return GeomVert[4];
       case Geometry::CUBE:        return GeomVert[5];
       case Geometry::PRISM:       return GeomVert[6];
-      case Geometry::PYRAMID:     return GeomVert[7];
       default:
          mfem_error ("Geometry::GetVertices(...)");
    }
@@ -351,25 +310,6 @@ void Geometry::GetRandomPoint(int GeomType, IntegrationPoint &ip)
             ip.y = 1.0 - ip.y;
          }
          break;
-      case Geometry::PYRAMID:
-         ip.x = double(rand()) / RAND_MAX;
-         ip.y = double(rand()) / RAND_MAX;
-         ip.z = double(rand()) / RAND_MAX;
-         if (ip.x + ip.z > 1.0 && ip.y < ip.x)
-         {
-            double x = ip.x;
-            ip.x = ip.y;
-            ip.y = 1.0 - ip.z;
-            ip.z = 1.0 - x;
-         }
-         else if (ip.y + ip.z > 1.0)
-         {
-            double z = ip.z;
-            ip.z = 1.0 - ip.y;
-            ip.y = ip.x;
-            ip.x = 1.0 - z;
-         }
-         break;
       default:
          MFEM_ABORT("Unknown type of reference element!");
    }
@@ -431,10 +371,6 @@ bool Geometry::CheckPoint(int GeomType, const IntegrationPoint &ip)
          if (ip.x < 0.0 || ip.y < 0.0 || ip.x+ip.y > 1.0 ||
              ip.z < 0.0 || ip.z > 1.0) { return false; }
          break;
-      case Geometry::PYRAMID:
-         if (ip.x < 0.0 || ip.y < 0.0 || ip.x+ip.z > 1.0 || ip.y+ip.z > 1.0 ||
-             ip.z < 0.0 || ip.z > 1.0) { return false; }
-         break;
       default:
          MFEM_ABORT("Unknown type of reference element!");
    }
@@ -505,17 +441,6 @@ bool Geometry::CheckPoint(int GeomType, const IntegrationPoint &ip, double eps)
             return false;
          }
          break;
-      case Geometry::PYRAMID:
-         if (internal::FuzzyLT(ip.x, 0.0, eps)
-             || internal::FuzzyLT(ip.y, 0.0, eps)
-             || internal::FuzzyGT(ip.x+ip.z, 1.0, eps)
-             || internal::FuzzyGT(ip.y+ip.z, 1.0, eps)
-             || internal::FuzzyLT(ip.z, 0.0, eps)
-             || internal::FuzzyGT(ip.z, 1.0, eps) )
-         {
-            return false;
-         }
-         break;
       default:
          MFEM_ABORT("Unknown type of reference element!");
    }
@@ -630,16 +555,6 @@ bool Geometry::ProjectPoint(int GeomType, const IntegrationPoint &beg,
          double lbeg[5] = { beg.x, beg.y, beg.z, 1.0-beg.x-beg.y, 1.0-beg.z };
          return internal::IntersectSegment<5,3>(lbeg, lend, end);
       }
-      case Geometry::PYRAMID:
-      {
-         double lend[6] = { end.x, end.y, end.z,
-                            1.0-end.x-end.z, 1.0-end.y-end.z, 1.0-end.z
-                          };
-         double lbeg[6] = { beg.x, beg.y, beg.z,
-                            1.0-beg.x-beg.z, 1.0-beg.y-beg.z, 1.0-beg.z
-                          };
-         return internal::IntersectSegment<6,3>(lbeg, lend, end);
-      }
       default:
          MFEM_ABORT("Unknown type of reference element!");
    }
@@ -737,43 +652,6 @@ bool Geometry::ProjectPoint(int GeomType, IntegrationPoint &ip)
          return in_tri && in_z;
       }
 
-      case PYRAMID:
-      {
-         if (ip.x < 0.0)
-         {
-            ip.x = 0.0;
-            internal::ProjectTriangle(ip.y, ip.z);
-            return false;
-         }
-         if (ip.y < 0.0)
-         {
-            ip.y = 0.0;
-            internal::ProjectTriangle(ip.x, ip.z);
-            return false;
-         }
-         if (ip.z < 0.0)
-         {
-            ip.z = 0.0;
-            if (ip.x > 1.0) { ip.x = 1.0; }
-            if (ip.y > 1.0) { ip.y = 1.0; }
-            return false;
-         }
-         if (ip.x >= ip.y)
-         {
-            bool in_y = true;
-            bool in_tri = internal::ProjectTriangle(ip.x, ip.z);
-            if (ip.y > ip.z) { in_y = false; ip.y = ip.z; }
-            return in_tri && in_y;
-         }
-         else
-         {
-            bool in_x = true;
-            bool in_tri = internal::ProjectTriangle(ip.y, ip.z);
-            if (ip.x > ip.z) { in_x = false; ip.x = ip.z; }
-            return in_tri && in_x;
-         }
-      }
-
       default:
          MFEM_ABORT("Reference element type is not supported!");
    }
@@ -848,17 +726,6 @@ void Geometry::GetPerfPointMat(int GeomType, DenseMatrix &pm)
       }
       break;
 
-      case Geometry::PYRAMID:
-      {
-         pm.SetSize (3, 5);
-         pm(0,0) = 0.0;  pm(1,0) = 0.0;  pm(2,0) = 0.0;
-         pm(0,1) = 1.0;  pm(1,1) = 0.0;  pm(2,1) = 0.0;
-         pm(0,2) = 1.0;  pm(1,2) = 1.0;  pm(2,2) = 0.0;
-         pm(0,3) = 0.0;  pm(1,3) = 1.0;  pm(2,3) = 0.0;
-         pm(0,4) = 0.5;  pm(1,4) = 0.5;  pm(2,4) = 0.7071067811865475;
-      }
-      break;
-
       default:
          mfem_error ("Geometry::GetPerfPointMat (...)");
    }
@@ -877,13 +744,13 @@ void Geometry::JacToPerfJac(int GeomType, const DenseMatrix &J,
    }
 }
 
-const int Geometry::NumBdrArray[NumGeom] = { 0, 2, 3, 4, 4, 6, 5, 5 };
-const int Geometry::Dimension[NumGeom] = { 0, 1, 2, 2, 3, 3, 3, 3 };
+const int Geometry::NumBdrArray[NumGeom] = { 0, 2, 3, 4, 4, 6, 5 };
+const int Geometry::Dimension[NumGeom] = { 0, 1, 2, 2, 3, 3, 3 };
 const int Geometry::DimStart[MaxDim+2] =
 { POINT, SEGMENT, TRIANGLE, TETRAHEDRON, NUM_GEOMETRIES };
-const int Geometry::NumVerts[NumGeom] = { 1, 2, 3, 4, 4, 8, 6, 5 };
-const int Geometry::NumEdges[NumGeom] = { 0, 1, 3, 4, 6, 12, 9, 8 };
-const int Geometry::NumFaces[NumGeom] = { 0, 0, 1, 1, 4, 6, 5, 5 };
+const int Geometry::NumVerts[NumGeom] = { 1, 2, 3, 4, 4, 8, 6 };
+const int Geometry::NumEdges[NumGeom] = { 0, 1, 3, 4, 6, 12, 9 };
+const int Geometry::NumFaces[NumGeom] = { 0, 0, 1, 1, 4, 6, 5 };
 
 const int Geometry::
 Constants<Geometry::POINT>::Orient[1][1] = {{0}};
@@ -1030,30 +897,6 @@ Constants<Geometry::PRISM>::VertToVert::J[9][2] =
    {5, 4}                   // 4,5:4
 };
 
-const int Geometry::
-Constants<Geometry::PYRAMID>::Edges[8][2] =
-{{0, 1}, {1, 2}, {3, 2}, {0, 3}, {0, 4}, {1, 4}, {2, 4}, {3, 4}};
-const int Geometry::
-Constants<Geometry::PYRAMID>::FaceTypes[5] =
-{
-   Geometry::SQUARE,
-   Geometry::TRIANGLE, Geometry::TRIANGLE,
-   Geometry::TRIANGLE, Geometry::TRIANGLE
-};
-const int Geometry::
-Constants<Geometry::PYRAMID>::FaceVert[5][4] =
-{{3, 2, 1, 0}, {0, 1, 4, -1}, {1, 2, 4, -1}, {2, 3, 4, -1}, {3, 0, 4, -1}};
-const int Geometry::
-Constants<Geometry::PYRAMID>::VertToVert::I[5] = {0, 3, 5, 7, 8};
-const int Geometry::
-Constants<Geometry::PYRAMID>::VertToVert::J[8][2] =
-{
-   {1, 0}, {3, 3}, {4, 4}, // 0,1:0   0,3:3  0,4:4
-   {2, 1}, {4, 5},         // 1,2:1   1,4:5
-   {3,-3}, {4, 6},         // 2,3:-3  2,4:6
-   {4, 7}                  // 3,4:7
-};
-
 
 GeometryRefiner::GeometryRefiner()
 {
@@ -1419,104 +1262,6 @@ RefinedGeometry * GeometryRefiner::Refine(Geometry::Type Geom,
          return RG;
       }
 
-      case Geometry::PYRAMID:
-      {
-         const int n = Times;
-         RG = new RefinedGeometry ((n+1)*(n+2)*(2*n+3)/6,
-                                   5*n*(2*n-1)*(2*n+1)/3, 0);
-         RG->Times = Times;
-         RG->ETimes = ETimes;
-         RG->Type = type;
-         // enumerate and define the vertices
-         m = 0;
-         for (k = 0; k <= n; k++)
-         {
-            const double *cpij =
-               poly1d.GetPoints(Times - k, BasisType::GetNodalBasis(type));
-            for (j = 0; j <= n - k; j++)
-               for (i = 0; i <= n - k; i++)
-               {
-                  IntegrationPoint &ip = RG->RefPts.IntPoint(m);
-                  if (type == 0)
-                  {
-                     ip.x = (n > k) ? (double(i) / (n - k)) : 0.0;
-                     ip.y = (n > k) ? (double(j) / (n - k)) : 0.0;
-                     ip.z = double(k) / n;
-                  }
-                  else
-                  {
-                     ip.x = cpij[i] * (1.0 - cp[k]);
-                     ip.y = cpij[j] * (1.0 - cp[k]);
-                     ip.z = cp[k];
-                  }
-                  m++;
-               }
-         }
-         if (m != (n+1)*(n+2)*(2*n+3)/6)
-         {
-            mfem_error("GeometryRefiner::Refine() for PYRAMID #1");
-         }
-         // elements
-         Array<int> &G = RG->RefGeoms;
-         m = 0;
-         for (k = 0; k < n; k++)
-         {
-            int lk = k * (k * (2 * k - 6 * n - 9) + 6 * n * (n + 3) + 13) / 6;
-            int lkp1 = (k + 1) *
-                       (k * (2 * k - 6 * n -5) + 6 * n * (n + 2) + 6) / 6;
-            for (j = 0; j < n - k; j++)
-            {
-               for (i = 0; i < n - k; i++)
-               {
-                  G[m++] = lk + j * (n - k + 1) + i;
-                  G[m++] = lk + j * (n - k + 1) + i + 1;
-                  G[m++] = lk + (j + 1) * (n - k + 1) + i + 1;
-                  G[m++] = lk + (j + 1) * (n - k + 1) + i;
-                  G[m++] = lkp1 + j * (n - k) + i;
-               }
-            }
-            for (j = 0; j < n - k - 1; j++)
-            {
-               for (i = 0; i < n - k - 1; i++)
-               {
-                  G[m++] = lkp1 + j * (n - k) + i;
-                  G[m++] = lkp1 + (j + 1) * (n - k) + i;
-                  G[m++] = lkp1 + (j + 1) * (n - k) + i + 1;
-                  G[m++] = lkp1 + j * (n - k) + i + 1;
-                  G[m++] = lk + (j + 1) * (n - k + 1) + i + 1;
-               }
-            }
-            for (j = 0; j < n - k; j++)
-            {
-               for (i = 0; i < n - k - 1; i++)
-               {
-                  G[m++] = lk + j * (n - k + 1) + i + 1;
-                  G[m++] = lk + (j + 1) * (n - k + 1) + i + 1;
-                  G[m++] = lkp1 + j * (n - k) + i;
-                  G[m++] = lkp1 + j * (n - k) + i + 1;
-                  G[m++] = -1;
-               }
-            }
-            for (j = 0; j < n - k - 1; j++)
-            {
-               for (i = 0; i < n - k; i++)
-               {
-                  G[m++] = lk + (j + 1) * (n - k + 1) + i;
-                  G[m++] = lk + (j + 1) * (n - k + 1) + i + 1;
-                  G[m++] = lkp1 + (j + 1) * (n - k) + i;
-                  G[m++] = lkp1 + j * (n - k) + i;
-                  G[m++] = -1;
-               }
-            }
-         }
-         if (m != 5*n*(2*n-1)*(2*n+1)/3)
-         {
-            mfem_error("GeometryRefiner::Refine() for PYRAMID #2");
-         }
-         RGeom[Geometry::PYRAMID].Append(RG);
-         return RG;
-      }
-
       case Geometry::PRISM:
       {
          const int n = Times;

fem/geom.hpp

@@ -27,7 +27,6 @@ namespace mfem
     Geometry::TETRAHEDRON - w/ vert. (0,0,0),(1,0,0),(0,1,0),(0,0,1)
     Geometry::CUBE - the unit cube
     Geometry::PRISM - w/ vert. (0,0,0),(1,0,0),(0,1,0),(0,0,1),(1,0,1),(0,1,1)
-    Geometry::PYRAMID - w/ vert. (0,0,0),(1,0,0),(1,1,0),(0,1,0),(0,0,1)
 */
 class Geometry
 {
@@ -35,7 +34,7 @@ public:
    enum Type
    {
       INVALID = -1,
-      POINT = 0, SEGMENT, TRIANGLE, SQUARE, TETRAHEDRON, CUBE, PRISM, PYRAMID,
+      POINT = 0, SEGMENT, TRIANGLE, SQUARE, TETRAHEDRON, CUBE, PRISM,
       NUM_GEOMETRIES
    };
 
@@ -252,26 +251,7 @@ template <> struct Geometry::Constants<Geometry::PRISM>
    };
 };
 
-template <> struct Geometry::Constants<Geometry::PYRAMID>
-{
-   static const int Dimension = 3;
-   static const int NumVert = 5;
-   static const int NumEdges = 8;
-   static const int Edges[NumEdges][2];
-   static const int NumFaces = 5;
-   static const int FaceTypes[NumFaces];
-   static const int MaxFaceVert = 4;
-   static const int FaceVert[NumFaces][MaxFaceVert];
-   // Upper-triangular part of the local vertex-to-vertex graph.
-   struct VertToVert
-   {
-      static const int I[NumVert];
-      static const int J[NumEdges][2]; // {end,edge_idx}
-   };
-};
-
-// Defined in fe.cpp to ensure construction after 'mfem::TriangleFE' and
-// `mfem::TetrahedronFE`.
+// Defined in fe.cpp to ensure construction after 'mfem::WedgeFE'.
 extern Geometry Geometries;
 
 

fem/gridfunc.cpp

@@ -218,7 +218,7 @@ void GridFunction::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
 
 void GridFunction::MakeTRef(FiniteElementSpace *f, double *tv)
 {
-   if (IsIdentityProlongation(f->GetProlongationMatrix()))
+   if (!f->GetProlongationMatrix())
    {
       MakeRef(f, tv);
       t_vec.NewDataAndSize(tv, size);
@@ -232,8 +232,7 @@ void GridFunction::MakeTRef(FiniteElementSpace *f, double *tv)
 
 void GridFunction::MakeTRef(FiniteElementSpace *f, Vector &tv, int tv_offset)
 {
-   tv.UseDevice(true);
-   if (IsIdentityProlongation(f->GetProlongationMatrix()))
+   if (!f->GetProlongationMatrix())
    {
       MakeRef(f, tv, tv_offset);
       t_vec.NewMemoryAndSize(data, size, false);
@@ -242,7 +241,10 @@ void GridFunction::MakeTRef(FiniteElementSpace *f, Vector &tv, int tv_offset)
    {
       MFEM_ASSERT(tv.Size() >= tv_offset + f->GetTrueVSize(), "");
       SetSpace(f); // works in parallel
-      t_vec.MakeRef(tv, tv_offset, f->GetTrueVSize());
+      tv.UseDevice(true);
+      const int tv_size = f->GetTrueVSize();
+      t_vec.NewMemoryAndSize(Memory<double>(tv.GetMemory(), tv_offset, tv_size),
+                             tv_size, true);
    }
 }
 
@@ -255,8 +257,6 @@ void GridFunction::SumFluxAndCount(BilinearFormIntegrator &blfi,
    GridFunction &u = *this;
 
    ElementTransformation *Transf;
-   DofTransformation *udoftrans;
-   DofTransformation *fdoftrans;
 
    FiniteElementSpace *ufes = u.FESpace();
    FiniteElementSpace *ffes = flux.FESpace();
@@ -276,23 +276,15 @@ void GridFunction::SumFluxAndCount(BilinearFormIntegrator &blfi,
          continue;
       }
 
-      udoftrans = ufes->GetElementVDofs(i, udofs);
-      fdoftrans = ffes->GetElementVDofs(i, fdofs);
+      ufes->GetElementVDofs(i, udofs);
+      ffes->GetElementVDofs(i, fdofs);
 
       u.GetSubVector(udofs, ul);
-      if (udoftrans)
-      {
-         udoftrans->InvTransformPrimal(ul);
-      }
 
       Transf = ufes->GetElementTransformation(i);
       blfi.ComputeElementFlux(*ufes->GetFE(i), *Transf, ul,
                               *ffes->GetFE(i), fl, wcoef);
 
-      if (fdoftrans)
-      {
-         fdoftrans->TransformPrimal(fl);
-      }
       flux.AddElementVector(fdofs, fl);
 
       FiniteElementSpace::AdjustVDofs(fdofs);
@@ -342,10 +334,10 @@ int GridFunction::VectorDim() const
 void GridFunction::GetTrueDofs(Vector &tv) const
 {
    const SparseMatrix *R = fes->GetRestrictionMatrix();
-   if (!R || IsIdentityProlongation(fes->GetProlongationMatrix()))
+   if (!R)
    {
-      // R is identity
-      tv = *this; // no real copy if 'tv' and '*this' use the same data
+      // R is identity -> make tv a reference to *this
+      tv.MakeRef(const_cast<GridFunction &>(*this), 0, size);
    }
    else
    {
@@ -374,7 +366,7 @@ void GridFunction::GetNodalValues(int i, Array<double> &nval, int vdim) const
 
    int k;
 
-   DofTransformation * doftrans = fes->GetElementVDofs(i, vdofs);
+   fes->GetElementVDofs(i, vdofs);
    const FiniteElement *FElem = fes->GetFE(i);
    const IntegrationRule *ElemVert =
       Geometries.GetVertices(FElem->GetGeomType());
@@ -384,10 +376,6 @@ void GridFunction::GetNodalValues(int i, Array<double> &nval, int vdim) const
    vdim--;
    Vector loc_data;
    GetSubVector(vdofs, loc_data);
-   if (doftrans)
-   {
-      doftrans->InvTransformPrimal(loc_data);
-   }
 
    if (FElem->GetRangeType() == FiniteElement::SCALAR)
    {
@@ -417,7 +405,7 @@ double GridFunction::GetValue(int i, const IntegrationPoint &ip, int vdim)
 const
 {
    Array<int> dofs;
-   DofTransformation * doftrans = fes->GetElementDofs(i, dofs);
+   fes->GetElementDofs(i, dofs);
    fes->DofsToVDofs(vdim-1, dofs);
    Vector DofVal(dofs.Size()), LocVec;
    const FiniteElement *fe = fes->GetFE(i);
@@ -432,10 +420,6 @@ const
       fe->CalcPhysShape(*Tr, DofVal);
    }
    GetSubVector(dofs, LocVec);
-   if (doftrans)
-   {
-      doftrans->InvTransformPrimal(LocVec);
-   }
 
    return (DofVal * LocVec);
 }
@@ -446,13 +430,9 @@ void GridFunction::GetVectorValue(int i, const IntegrationPoint &ip,
    const FiniteElement *FElem = fes->GetFE(i);
    int dof = FElem->GetDof();
    Array<int> vdofs;
-   DofTransformation * doftrans = fes->GetElementVDofs(i, vdofs);
+   fes->GetElementVDofs(i, vdofs);
    Vector loc_data;
    GetSubVector(vdofs, loc_data);
-   if (doftrans)
-   {
-      doftrans->InvTransformPrimal(loc_data);
-   }
    if (FElem->GetRangeType() == FiniteElement::SCALAR)
    {
       Vector shape(dof);
@@ -492,35 +472,30 @@ const
    Array<int> dofs;
    int n = ir.GetNPoints();
    vals.SetSize(n);
-   DofTransformation * doftrans = fes->GetElementDofs(i, dofs);
+   fes->GetElementDofs(i, dofs);
    fes->DofsToVDofs(vdim-1, dofs);
    const FiniteElement *FElem = fes->GetFE(i);
    int dof = FElem->GetDof();
    Vector DofVal(dof), loc_data(dof);
    GetSubVector(dofs, loc_data);
-   if (doftrans)
+   if (FElem->GetMapType() == FiniteElement::VALUE)
    {
-      doftrans->InvTransformPrimal(loc_data);
+      for (int k = 0; k < n; k++)
+      {
+         FElem->CalcShape(ir.IntPoint(k), DofVal);
+         vals(k) = DofVal * loc_data;
+      }
    }
-   for (int k = 0; k < n; k++)
-      if (FElem->GetMapType() == FiniteElement::VALUE)
+   else
+   {
+      ElementTransformation *Tr = fes->GetElementTransformation(i);
+      for (int k = 0; k < n; k++)
       {
-         for (int k = 0; k < n; k++)
-         {
-            FElem->CalcShape(ir.IntPoint(k), DofVal);
-            vals(k) = DofVal * loc_data;
-         }
-      }
-      else
-      {
-         ElementTransformation *Tr = fes->GetElementTransformation(i);
-         for (int k = 0; k < n; k++)
-         {
-            Tr->SetIntPoint(&ir.IntPoint(k));
-            FElem->CalcPhysShape(*Tr, DofVal);
-            vals(k) = DofVal * loc_data;
-         }
+         Tr->SetIntPoint(&ir.IntPoint(k));
+         FElem->CalcPhysShape(*Tr, DofVal);
+         vals(k) = DofVal * loc_data;
       }
+   }
 }
 
 void GridFunction::GetValues(int i, const IntegrationRule &ir, Vector &vals,
@@ -888,12 +863,11 @@ void GridFunction::GetVectorValue(ElementTransformation &T,
 
    Array<int> vdofs;
    const FiniteElement *fe = NULL;
-   DofTransformation * doftrans = NULL;
 
    switch (T.ElementType)
    {
       case ElementTransformation::ELEMENT:
-         doftrans = fes->GetElementVDofs(T.ElementNo, vdofs);
+         fes->GetElementVDofs(T.ElementNo, vdofs);
          fe = fes->GetFE(T.ElementNo);
          break;
       case ElementTransformation::EDGE:
@@ -984,10 +958,6 @@ void GridFunction::GetVectorValue(ElementTransformation &T,
    int dof = fe->GetDof();
    Vector loc_data;
    GetSubVector(vdofs, loc_data);
-   if (doftrans)
-   {
-      doftrans->InvTransformPrimal(loc_data);
-   }
    if (fe->GetRangeType() == FiniteElement::SCALAR)
    {
       Vector shape(dof);
@@ -1030,14 +1000,10 @@ void GridFunction::GetVectorValues(ElementTransformation &T,
    int dof = FElem->GetDof();
 
    Array<int> vdofs;
-   DofTransformation * doftrans = fes->GetElementVDofs(T.ElementNo, vdofs);
+   fes->GetElementVDofs(T.ElementNo, vdofs);
+
    Vector loc_data;
    GetSubVector(vdofs, loc_data);
-   if (doftrans)
-   {
-      doftrans->InvTransformPrimal(loc_data);
-   }
-
    int nip = ir.GetNPoints();
 
    if (FElem->GetRangeType() == FiniteElement::SCALAR)
@@ -1125,8 +1091,6 @@ void GridFunction::GetValuesFrom(const GridFunction &orig_func)
    // Without averaging ...
 
    const FiniteElementSpace *orig_fes = orig_func.FESpace();
-   DofTransformation * doftrans;
-   DofTransformation * orig_doftrans;
    Array<int> vdofs, orig_vdofs;
    Vector shape, loc_values, orig_loc_values;
    int i, j, d, ne, dof, odof, vdim;
@@ -1135,13 +1099,9 @@ void GridFunction::GetValuesFrom(const GridFunction &orig_func)
    vdim = fes->GetVDim();
    for (i = 0; i < ne; i++)
    {
-      doftrans = fes->GetElementVDofs(i, vdofs);
-      orig_doftrans = orig_fes->GetElementVDofs(i, orig_vdofs);
+      fes->GetElementVDofs(i, vdofs);
+      orig_fes->GetElementVDofs(i, orig_vdofs);
       orig_func.GetSubVector(orig_vdofs, orig_loc_values);
-      if (orig_doftrans)
-      {
-         orig_doftrans->InvTransformPrimal(orig_loc_values);
-      }
       const FiniteElement *fe = fes->GetFE(i);
       const FiniteElement *orig_fe = orig_fes->GetFE(i);
       dof = fe->GetDof();
@@ -1159,10 +1119,6 @@ void GridFunction::GetValuesFrom(const GridFunction &orig_func)
                shape * ((const double *)orig_loc_values + d * odof) ;
          }
       }
-      if (doftrans)
-      {
-         doftrans->TransformPrimal(loc_values);
-      }
       SetSubVector(vdofs, loc_values);
    }
 }
@@ -1172,10 +1128,8 @@ void GridFunction::GetBdrValuesFrom(const GridFunction &orig_func)
    // Without averaging ...
 
    const FiniteElementSpace *orig_fes = orig_func.FESpace();
-   // DofTransformation * doftrans;
-   // DofTransformation * orig_doftrans;
    Array<int> vdofs, orig_vdofs;
-   Vector shape, loc_values, loc_values_t, orig_loc_values, orig_loc_values_t;
+   Vector shape, loc_values, orig_loc_values;
    int i, j, d, nbe, dof, odof, vdim;
 
    nbe = fes->GetNBE();
@@ -1213,33 +1167,37 @@ void GridFunction::GetVectorFieldValues(
    Array<int> vdofs;
    ElementTransformation *transf;
 
-   int d, k, n, sdim, dof;
+   int d, j, k, n, sdim, dof, ind;
 
    n = ir.GetNPoints();
-   DofTransformation * doftrans = fes->GetElementVDofs(i, vdofs);
+   fes->GetElementVDofs(i, vdofs);
    const FiniteElement *fe = fes->GetFE(i);
    dof = fe->GetDof();
    sdim = fes->GetMesh()->SpaceDimension();
-   // int *dofs = &vdofs[comp*dof];
+   int *dofs = &vdofs[comp*dof];
    transf = fes->GetElementTransformation(i);
    transf->Transform(ir, tr);
    vals.SetSize(n, sdim);
    DenseMatrix vshape(dof, sdim);
-   Vector loc_data, val(sdim);
-   GetSubVector(vdofs, loc_data);
-   if (doftrans)
-   {
-      doftrans->InvTransformPrimal(loc_data);
-   }
+   double a;
    for (k = 0; k < n; k++)
    {
       const IntegrationPoint &ip = ir.IntPoint(k);
       transf->SetIntPoint(&ip);
       fe->CalcVShape(*transf, vshape);
-      vshape.MultTranspose(loc_data, val);
       for (d = 0; d < sdim; d++)
       {
-         vals(k,d) = val(d);
+         a = 0.0;
+         for (j = 0; j < dof; j++)
+            if ( (ind=dofs[j]) >= 0 )
+            {
+               a += vshape(j, d) * data[ind];
+            }
+            else
+            {
+               a -= vshape(j, d) * data[-1-ind];
+            }
+         vals(k, d) = a;
       }
    }
 }
@@ -1409,13 +1367,9 @@ void GridFunction::GetVectorGradientHat(
    const FiniteElement *FElem = fes->GetFE(elNo);
    int dim = FElem->GetDim(), dof = FElem->GetDof();
    Array<int> vdofs;
-   DofTransformation * doftrans = fes->GetElementVDofs(elNo, vdofs);
+   fes->GetElementVDofs(elNo, vdofs);
    Vector loc_data;
    GetSubVector(vdofs, loc_data);
-   if (doftrans)
-   {
-      doftrans->InvTransformPrimal(loc_data);
-   }
    // assuming scalar FE
    int vdim = fes->GetVDim();
    DenseMatrix dshape(dof, dim);
@@ -1454,13 +1408,9 @@ double GridFunction::GetDivergence(ElementTransformation &T) const
          {
             // Assuming RT-type space
             Array<int> dofs;
-            DofTransformation * doftrans = fes->GetElementDofs(elNo, dofs);
+            fes->GetElementDofs(elNo, dofs);
             Vector loc_data, divshape(fe->GetDof());
             GetSubVector(dofs, loc_data);
-            if (doftrans)
-            {
-               doftrans->InvTransformPrimal(loc_data);
-            }
             fe->CalcDivShape(T.GetIntPoint(), divshape);
             return (loc_data * divshape) / T.Weight();
          }
@@ -1551,13 +1501,9 @@ void GridFunction::GetCurl(ElementTransformation &T, Vector &curl) const
          {
             // Assuming ND-type space
             Array<int> dofs;
-            DofTransformation * doftrans = fes->GetElementDofs(elNo, dofs);
+            fes->GetElementDofs(elNo, dofs);
             Vector loc_data;
             GetSubVector(dofs, loc_data);
-            if (doftrans)
-            {
-               doftrans->InvTransformPrimal(loc_data);
-            }
             DenseMatrix curl_shape(fe->GetDof(), fe->GetDim() == 3 ? 3 : 1);
             fe->CalcCurlShape(T.GetIntPoint(), curl_shape);
             curl.SetSize(curl_shape.Width());
@@ -1699,12 +1645,8 @@ void GridFunction::GetGradients(ElementTransformation &tr,
    DenseMatrix dshape(fe->GetDof(), fe->GetDim());
    Vector lval, gh(fe->GetDim()), gcol;
    Array<int> dofs;
-   DofTransformation * doftrans = fes->GetElementDofs(elNo, dofs);
+   fes->GetElementDofs(elNo, dofs);
    GetSubVector(dofs, lval);
-   if (doftrans)
-   {
-      doftrans->InvTransformPrimal(lval);
-   }
    grad.SetSize(fe->GetDim(), ir.GetNPoints());
    for (int i = 0; i < ir.GetNPoints(); i++)
    {
@@ -1784,8 +1726,6 @@ void GridFunction::GetElementAverages(GridFunction &avgs) const
 {
    MassIntegrator Mi;
    DenseMatrix loc_mass;
-   DofTransformation * te_doftrans;
-   DofTransformation * tr_doftrans;
    Array<int> te_dofs, tr_dofs;
    Vector loc_avgs, loc_this;
    Vector int_psi(avgs.Size());
@@ -1796,19 +1736,11 @@ void GridFunction::GetElementAverages(GridFunction &avgs) const
    {
       Mi.AssembleElementMatrix2(*fes->GetFE(i), *avgs.FESpace()->GetFE(i),
                                 *fes->GetElementTransformation(i), loc_mass);
-      tr_doftrans = fes->GetElementDofs(i, tr_dofs);
-      te_doftrans = avgs.FESpace()->GetElementDofs(i, te_dofs);
+      fes->GetElementDofs(i, tr_dofs);
+      avgs.FESpace()->GetElementDofs(i, te_dofs);
       GetSubVector(tr_dofs, loc_this);
-      if (tr_doftrans)
-      {
-         tr_doftrans->InvTransformPrimal(loc_this);
-      }
       loc_avgs.SetSize(te_dofs.Size());
       loc_mass.Mult(loc_this, loc_avgs);
-      if (te_doftrans)
-      {
-         te_doftrans->TransformPrimal(loc_avgs);
-      }
       avgs.AddElementVector(te_dofs, loc_avgs);
       loc_this = 1.0; // assume the local basis for 'this' sums to 1
       loc_mass.Mult(loc_this, loc_avgs);
@@ -1823,12 +1755,8 @@ void GridFunction::GetElementAverages(GridFunction &avgs) const
 void GridFunction::GetElementDofValues(int el, Vector &dof_vals) const
 {
    Array<int> dof_idx;
-   DofTransformation * doftrans = fes->GetElementVDofs(el, dof_idx);
+   fes->GetElementVDofs(el, dof_idx);
    GetSubVector(dof_idx, dof_vals);
-   if (doftrans)
-   {
-      doftrans->InvTransformPrimal(dof_vals);
-   }
 }
 
 void GridFunction::ProjectGridFunction(const GridFunction &src)
@@ -1864,21 +1792,13 @@ void GridFunction::ProjectGridFunction(const GridFunction &src)
          cached_geom = geom;
       }
 
-      DofTransformation * src_doftrans = src.fes->GetElementVDofs(i, src_vdofs);
+      src.fes->GetElementVDofs(i, src_vdofs);
       src.GetSubVector(src_vdofs, src_lvec);
-      if (src_doftrans)
-      {
-         src_doftrans->InvTransformPrimal(src_lvec);
-      }
       for (int vd = 0; vd < vdim; vd++)
       {
          P.Mult(&src_lvec[vd*P.Width()], &dest_lvec[vd*P.Height()]);
       }
-      DofTransformation * doftrans = fes->GetElementVDofs(i, dest_vdofs);
-      if (doftrans)
-      {
-         doftrans->TransformPrimal(dest_lvec);
-      }
+      fes->GetElementVDofs(i, dest_vdofs);
       SetSubVector(dest_vdofs, dest_lvec);
    }
 }
@@ -1887,15 +1807,10 @@ void GridFunction::ImposeBounds(int i, const Vector &weights,
                                 const Vector &lo_, const Vector &hi_)
 {
    Array<int> vdofs;
-   DofTransformation * doftrans = fes->GetElementVDofs(i, vdofs);
+   fes->GetElementVDofs(i, vdofs);
    int size = vdofs.Size();
    Vector vals, new_vals(size);
-
    GetSubVector(vdofs, vals);
-   if (doftrans)
-   {
-      doftrans->InvTransformPrimal(vals);
-   }
 
    MFEM_ASSERT(weights.Size() == size, "Different # of weights and dofs.");
    MFEM_ASSERT(lo_.Size() == size, "Different # of lower bounds and dofs.");
@@ -1912,10 +1827,6 @@ void GridFunction::ImposeBounds(int i, const Vector &weights,
    slbqp.SetPrintLevel(0); // print messages only if not converged
    slbqp.Mult(vals, new_vals);
 
-   if (doftrans)
-   {
-      doftrans->TransformPrimal(new_vals);
-   }
    SetSubVector(vdofs, new_vals);
 }
 
@@ -1923,14 +1834,10 @@ void GridFunction::ImposeBounds(int i, const Vector &weights,
                                 double min_, double max_)
 {
    Array<int> vdofs;
-   DofTransformation * doftrans = fes->GetElementVDofs(i, vdofs);
+   fes->GetElementVDofs(i, vdofs);
    int size = vdofs.Size();
    Vector vals, new_vals(size);
    GetSubVector(vdofs, vals);
-   if (doftrans)
-   {
-      doftrans->InvTransformPrimal(vals);
-   }
 
    double max_val = vals.Max();
    double min_val = vals.Min();
@@ -1938,10 +1845,6 @@ void GridFunction::ImposeBounds(int i, const Vector &weights,
    if (max_val <= min_)
    {
       new_vals = min_;
-      if (doftrans)
-      {
-         doftrans->TransformPrimal(new_vals);
-      }
       SetSubVector(vdofs, new_vals);
       return;
    }
@@ -2377,7 +2280,6 @@ void GridFunction::ProjectDeltaCoefficient(DeltaCoefficient &delta_coeff,
 void GridFunction::ProjectCoefficient(Coefficient &coeff)
 {
    DeltaCoefficient *delta_c = dynamic_cast<DeltaCoefficient *>(&coeff);
-   DofTransformation * doftrans = NULL;
 
    if (delta_c == NULL)
    {
@@ -2386,13 +2288,9 @@ void GridFunction::ProjectCoefficient(Coefficient &coeff)
 
       for (int i = 0; i < fes->GetNE(); i++)
       {
-         doftrans = fes->GetElementVDofs(i, vdofs);
+         fes->GetElementVDofs(i, vdofs);
          vals.SetSize(vdofs.Size());
          fes->GetFE(i)->Project(coeff, *fes->GetElementTransformation(i), vals);
-         if (doftrans)
-         {
-            doftrans->TransformPrimal(vals);
-         }
          SetSubVector(vdofs, vals);
       }
    }
@@ -2438,17 +2336,11 @@ void GridFunction::ProjectCoefficient(VectorCoefficient &vcoeff)
    Array<int> vdofs;
    Vector vals;
 
-   DofTransformation * doftrans = NULL;
-
    for (i = 0; i < fes->GetNE(); i++)
    {
-      doftrans = fes->GetElementVDofs(i, vdofs);
+      fes->GetElementVDofs(i, vdofs);
       vals.SetSize(vdofs.Size());
       fes->GetFE(i)->Project(vcoeff, *fes->GetElementTransformation(i), vals);
-      if (doftrans)
-      {
-         doftrans->TransformPrimal(vals);
-      }
       SetSubVector(vdofs, vals);
    }
 }
@@ -2511,7 +2403,6 @@ void GridFunction::ProjectCoefficient(Coefficient *coeff[])
    double val;
    const FiniteElement *fe;
    ElementTransformation *transf;
-   // DofTransformation * doftrans;
    Array<int> vdofs;
 
    vdim = fes->GetVDim();
@@ -2521,7 +2412,6 @@ void GridFunction::ProjectCoefficient(Coefficient *coeff[])
       fdof = fe->GetDof();
       transf = fes->GetElementTransformation(i);
       const IntegrationRule &ir = fe->GetNodes();
-      // doftrans = fes->GetElementVDofs(i, vdofs);
       fes->GetElementVDofs(i, vdofs);
       for (j = 0; j < fdof; j++)
       {

fem/gridfunc.hpp

@@ -95,12 +95,6 @@ public:
       : Vector(data, f->GetVSize())
    { fes = f; fec = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }
 
-   /** @brief Construct a GridFunction using previously allocated Vector @a base
-       starting at the given offset, @a base_offset. */
-   GridFunction(FiniteElementSpace *f, Vector &base, int base_offset = 0)
-      : Vector(base, base_offset, f->GetVSize())
-   { fes = f; fec = NULL; fes_sequence = f->GetSequence(); UseDevice(true); }
-
    /// Construct a GridFunction on the given Mesh, using the data from @a input.
    /** The content of @a input should be in the format created by the method
        Save(). The reconstructed FiniteElementSpace and FiniteElementCollection
@@ -136,7 +130,9 @@ public:
        or set. */
    Vector &GetTrueVector() { return t_vec; }
 
-   /// Extract the true-dofs from the GridFunction.
+   /// @brief Extract the true-dofs from the GridFunction. If all dofs are true,
+   /// then `tv` will be set to point to the data of `*this`.
+   /** @warning This method breaks const-ness when all dofs are true. */
    void GetTrueDofs(Vector &tv) const;
 
    /// Shortcut for calling GetTrueDofs() with GetTrueVector() as argument.