Extended PA for some vector mass and curl integrators to handle diagonal matrix coefficients. Added unit tests.

Implemented PA for MixedVectorCurlIntegrator and MixedVectorWeakCurlIntegrator. Added unit tests for these integrators.
2020-03-31 11:56:02 -07:00 · 2020-03-30 18:02:46 -07:00
325 changed files with 6867 additions and 50205 deletions
@@ -16,7 +16,7 @@ install:
 - set PATH=C:\Program Files\Microsoft MPI\Bin;%PATH%

 # Install METIS
- ps: Start-FileDownload 'https://mfem.github.io/tpls/metis-5.1.0.tar.gz'
+- ps: Start-FileDownload 'http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz'
 - 7z x metis-5.1.0.tar.gz -so | 7z x -si -ttar > nul
 - cd metis-5.1.0
 - ps: ( get-content "GKlib\gk_arch.h") | % { If ($_.ReadCount -ge 52) {$_ -replace "#ifdef __MSC__","#ifdef DISABLE_THIS_ANCIENT_MSC_CHECK"} Else {$_} } | set-content "GKlib\gk_arch.h"
@@ -26,17 +26,17 @@ install:
 - cd ..

 # Install hypre
- ps: Start-FileDownload 'https://github.com/hypre-space/hypre/archive/v2.19.0.tar.gz'
- 7z x v2.19.0.tar.gz -so | 7z x -si -ttar > nul
- cd hypre-2.19.0/src
- cmake -H. -Bbuild -DMPI_C_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DMPI_C_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include"
+- ps: Start-FileDownload 'https://github.com/hypre-space/hypre/archive/V2-10-0b.tar.gz'
+- 7z x V2-10-0b.tar.gz -so | 7z x -si -ttar > nul
+- cd hypre-2-10-0b
+- cmake -H. -Bbuild -DHYPRE_USING_FEI=OFF -DMPI_C_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DMPI_C_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include"
 - cmake --build build
 - cmake --build build --target install
- cd ../..
+- cd ..

 # MFEM
 before_build:
- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DHYPRE_DIR=%cd%\hypre-2.19.0\src\hypre -DMETIS_LIBRARIES=%cd%\metis-5.1.0\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%cd%\metis-5.1.0\include
+- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DHYPRE_LIBRARIES=%cd%\hypre-2-10-0b\hypre\lib\HYPRE.lib -DHYPRE_INCLUDE_DIRS=%cd%\hypre-2-10-0b\hypre\include -DHYPRE_VERSION=21000 -DMETIS_LIBRARIES=%cd%\metis-5.1.0\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%cd%\metis-5.1.0\include
 - cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_serial -DMFEM_USE_MPI=FALSE

 build_script:
@@ -29,8 +29,6 @@ config/sample-runs-build.log
 doc/CodeDocumentation.conf
 doc/CodeDocumentation.html
 doc/CodeDocumentation
-doc/undoc.log
-doc/warnings.log

 # Temporary files created by the tests.
 *.stderr
@@ -50,7 +48,6 @@ examples/ex1[04-9]
 examples/ex1[0-9]p
 examples/ex2[0-9]
 examples/ex2[0-9]p
-examples/ex25-gpu

 examples/refined.mesh
 examples/displaced.mesh
@@ -75,10 +72,6 @@ examples/deformed.*
 examples/velocity.*
 examples/elastic_energy.*
 examples/mode_*
-examples/ex5-p-*.bp
-examples/ex9-p-*.bp
-examples/ex12-p-*.bp
-examples/ex16-p-*.bp
 examples/ex16.mesh
 examples/ex16-mesh.*
 examples/ex16-init.*
@@ -123,7 +116,7 @@ examples/sundials/ex16-final.*
 examples/sundials/Example16*

 examples/petsc/ex[1-69]p
-examples/petsc/ex1[0-1]p
+examples/petsc/ex10p

 examples/petsc/mesh.*
 examples/petsc/sol.*
@@ -138,7 +131,6 @@ examples/petsc/Example9*
 examples/petsc/deformed.*
 examples/petsc/velocity.*
 examples/petsc/elastic_energy.*
-examples/petsc/mode_*

 examples/pumi/ex1
 examples/pumi/ex[126]p
@@ -171,11 +163,8 @@ miniapps/meshing/twist
 miniapps/meshing/mesh-explorer
 miniapps/meshing/shaper
 miniapps/meshing/extruder
-miniapps/meshing/trimmer
 miniapps/meshing/mesh-optimizer
 miniapps/meshing/pmesh-optimizer
-miniapps/meshing/minimal-surface
-miniapps/meshing/pminimal-surface

 miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
@@ -185,7 +174,6 @@ miniapps/meshing/mesh-explorer.mesh
 miniapps/meshing/partitioning.txt
 miniapps/meshing/shaper.mesh
 miniapps/meshing/extruder.mesh
-miniapps/meshing/trimmer.mesh
 miniapps/meshing/optimized*
 miniapps/meshing/perturbed*

@@ -237,17 +225,6 @@ miniapps/gslib/field-diff
 miniapps/gslib/findpts
 miniapps/gslib/pfindpts

-miniapps/navier/navier_mms
-miniapps/navier/navier_kovasznay
-miniapps/navier/navier_tgv
-miniapps/navier/navier_shear
-miniapps/navier/navier_3dfoc
-miniapps/navier/tgv_out*.txt
-miniapps/navier/*_output
-
-miniapps/adjoint/cvsRoberts_ASAi_dns
-miniapps/adjoint/adjoint_advection_diffusion
-
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
@@ -11,17 +11,13 @@

 language: cpp

+sudo: false
+
 stages:
  - checks
  - tests
  - optional

-env:
-  global:
-    - HYPRE_ARCHIVE=v2.19.0.tar.gz
-      HYPRE_URL=https://github.com/hypre-space/hypre/archive/$HYPRE_ARCHIVE
-      HYPRE_TOP_DIR=hypre-2.19.0
-
 jobs:
  include:

@@ -144,7 +140,8 @@ jobs:
           NPROCS=2
      cache:
        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
          - $TRAVIS_BUILD_DIR/../metis-4.0
      before_cache:
        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
@@ -174,7 +171,8 @@ jobs:
           NPROCS=2
      cache:
        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
          - $TRAVIS_BUILD_DIR/../metis-4.0
      before_cache:
        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
@@ -197,7 +195,7 @@ jobs:
        - cd ${TRAVIS_BUILD_DIR}/build
        - cmake ..
             -DMFEM_USE_MPI=ON
-             -DHYPRE_DIR=${TRAVIS_BUILD_DIR}/../$HYPRE_TOP_DIR/src/hypre
+             -DHYPRE_DIR=${TRAVIS_BUILD_DIR}/../hypre-2.10.0b/src/hypre
             -DMFEM_MPI_NP=$NPROCS
        - make -j3 mfem examples
        - cd ${TRAVIS_BUILD_DIR}/build/tests/unit
@@ -205,7 +203,8 @@ jobs:
        - ctest --output-on-failure
      cache:
        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
          - $TRAVIS_BUILD_DIR/../metis-4.0
      before_cache:
        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
@@ -250,7 +249,8 @@ jobs:
           TMPDIR=/tmp
      cache:
        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
          - $TRAVIS_BUILD_DIR/../metis-4.0
          - $HOME/local-cached
      before_cache:
@@ -270,7 +270,8 @@ jobs:
           TMPDIR=/tmp
      cache:
        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
          - $TRAVIS_BUILD_DIR/../metis-4.0
          - $HOME/local-cached
      before_cache:
@@ -336,18 +337,18 @@ install:

   # hypre
   - if [ $MPI == "YES" ]; then
-        if [ ! -e $HYPRE_TOP_DIR/src/hypre/lib/libHYPRE.a ]; then
-           wget $HYPRE_URL;
-           rm -rf $HYPRE_TOP_DIR;
-           tar xvzf $HYPRE_ARCHIVE;
-           cd $HYPRE_TOP_DIR/src;
-           ./configure --disable-fortran CC=mpicc CXX=mpic++;
+        if [ ! -e hypre-2.10.0b/src/hypre/lib/libHYPRE.a ]; then
+           wget https://computation.llnl.gov/project/linear_solvers/download/hypre-2.10.0b.tar.gz --no-check-certificate;
+           rm -rf hypre-2.10.0b;
+           tar xvzf hypre-2.10.0b.tar.gz;
+           cd hypre-2.10.0b/src;
+           ./configure --disable-fortran --without-fei CC=mpicc CXX=mpic++;
           make -j3;
           cd ../..;
        else
-            echo "Reusing cached $HYPRE_TOP_DIR/";
+            echo "Reusing cached hypre-2.10.0b/";
        fi;
-        ln -s $HYPRE_TOP_DIR hypre;
+        ln -s hypre-2.10.0b hypre;
     else
        echo "Serial build, not using hypre";
     fi
@@ -355,7 +356,7 @@ install:
   # METIS
   - if [ $MPI == "YES" ]; then
        if [ ! -e metis-4.0/libmetis.a ]; then
-           wget https://mfem.github.io/tpls/metis-4.0.3.tar.gz;
+           wget http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/OLD/metis-4.0.3.tar.gz;
           tar xvzf metis-4.0.3.tar.gz;
           make -j3 -C metis-4.0.3/Lib CC="$CC" OPTFLAGS="-O2";
           rm -rf metis-4.0;
@@ -369,10 +370,8 @@ script:
   # Compiler
   - if [ $MPI == "YES" ]; then
        export MYCXX=mpic++;
-        export MAKE_CXX_FLAG=MPICXX=$MYCXX;
     else
        export MYCXX="$CXX";
-        export MAKE_CXX_FLAG=CXX=$MYCXX;
     fi

   # Print the compiler version
@@ -385,9 +384,12 @@ script:
     if [ "$CODECOV" == "YES" ]; then
        CPPFLAGS="--coverage -g";
     fi;
+     if [ "$CXX" == "clang++" ]; then
+        export MFEM_PERF_SW=clang;
+     fi

   # Configure the library
-   - make config MFEM_USE_MPI=$MPI MFEM_DEBUG=$DEBUG $MAKE_CXX_FLAG
+   - make config MFEM_USE_MPI=$MPI MFEM_DEBUG=$DEBUG MFEM_CXX="$MYCXX"
        MFEM_MPI_NP=$NPROCS CPPFLAGS="$CPPFLAGS"
   # Show the configuration
   - make info
@@ -23,149 +23,19 @@ Meshing improvements
  Hessian for r-adaptivity using discrete fields, and allows use of skewness
  and orientation based metrics.

- Added support for r-adaptivity with more than one discrete field. This allows
-  the user to specify different discrete functions for controlling the
-  size, aspect-ratio, orientation, and skew of elements in the mesh.
-
- Added TMOP capability for approximate tangential mesh relaxation.
-
- Added support for reading periodic meshes in Gmsh format (version 2.2). See
-  for example the periodic-annulus-sector and periodic-torus-sector files in
-  the data directory.
-
- Added complete action of the TMOP Integrator to account for the spatial
-  derivatives of discrete and analytic targets.
-
-Performance improvements
------------------------
- Added support for explicit vectorization in the high-performance templated
-  code, which can now take advantage of specific intrinsics classes on the
-  following architectures:
-    - x86 (SSE/AVX/AVX2/AVX512),
-    - Power8 & Power9 (VSX),
-    - BG/Q (QPX).
-  These are now enabled by default, and can be disabled with MFEM_USE_SIMD=NO.
-  See the new file linalg/simd.hpp and the new directory linalg/simd.
-
-Improved GPU capabilities
-------------------------
- Added support for Chebyshev accelerated polynomial smoother on GPU.
-
- Optimized AMD/HIP kernel support.
-
- Added a Full Assembly mode compatible with Device kernel execution. This
-  assembly level builds on top of the current Element Assembly kernels to
-  compute a global sparse matrix. All integrators supported by element assembly
-  are also supported by full assembly. See the '-fa' option in Example 9.
-
- Added support for BlockOperator on GPU. See the updated Example 5.
-
- Added partial assembly and GPU support for ComplexOperator,
-  [Par]ComplexGridFunction, [Par]ComplexLinearForm, and [Par]SesquilinearForm.
-  See the updated Example 22.
-
-Discretization improvements
---------------------------
- Added support for matrix-free interpolation and restriction operators between
-  continuous H1 finite element spaces of different order on the same mesh or
-  with the same order on uniformly refined meshes.
-
- Added support for simplices in GSLIB-FindPoints.
-
- Added support for H1 and L2 element matrix assembly in the mass, convection,
-  diffusion, transpose, and the face DG trace integrators. This is compatible
-  with GPU device execution and is illustrated in Example 9/9p, see the option
-  '-ea'. When enabled, this level of assembly stores independent dense matrices
-  for the elements, and independent dense matrices for the faces in the DG case.
-
- Added new partial assembly kernels for H(div) bilinear forms, as well as
-  VectorFEDivergenceIntegrator.
-
- Improved the documentation of the GridFunction GetValue and GetVectorValue
-  methods. Expanded the GetValue and GetVectorValue methods which accept an
-  ElementTransformation argument to support evaluation on boundary elements
-  and, in the continuous field case, arbitrary mesh edges and faces.
-
- Added new coefficient and vector coefficient classes for QuadratureFunctions.
-  Additionally, new LinearForm integrators were also added which make use of
-  these new QuadratureFunction coefficient classes.
-
- Added support face integrals on the boundaries of NURBS meshes.
-
-Linear and nonlinear solvers
----------------------------
- Added power method to iteratively estimate the largest eigenvalue and the
-  corresponding eigenvector of an operator.
-
- Added initial support for h- and p-multigrid solvers and preconditioners for
-  matrix-based and matrix-free discretizations with basic GPU capability.
-
- Added a new IterativeSolverMonitor class that allows to monitor the residual
-  and solution during the solving process of an IterativeSolver after every
-  iteration.
-
- Added support for the CVODES package in SUNDIALS which provides ODE
-  solvers with sensitivity analysis capabilities. See the CVODESSolver
-  class and the new adjoint miniapps below.
-
- Block arrays of parallel matrices can now be merged into a single parallel
-  matrix with the function HypreParMatrixFromBlocks. This could be useful for
-  solving block systems with parallel direct solvers such as STRUMPACK.
-
- In SLISolver, changed the residual inner product from (Br,r) to (Br,Br) so the
-  solver can work with non-SPD preconditioner B.
-
- Added support for the SLEPc eigensolver package.
-
 New and updated examples and miniapps
 -------------------------------------
+- Adding a simple meshing miniapp, Twist, which demonstrates MFEM's strategy of
+  stitching together opposite surfaces of a mesh to create a topologically
+  periodic mesh.
+
 - Added a new example, Example 25/25p, to demonstrate the use of a Perfectly
  Matched Layer (PML) for the simulation of electromagnetic wave propagation.
  The example defines and solves several indefinite Maxwell problems.

- Added a new Example 26/26p to demonstrate the construction of a matrix-free
-  geometric and p-multigrid preconditioner for the Laplace problem.
-
- Added a new example, Example 27/27p, to demonstrate the enforcement of various
-  boundary conditions with the Laplace operator. The example shows the procedure
-  for applying Dirichlet, Neumann (both homogeneous and inhomogeneous), Robin,
-  and periodic boundary conditions with either H1 or DG discretizations.
-
- Added a new miniapp, Navier, that solves the time-dependent Navier-Stokes
-  equations of incompressible fluid dynamics. See the miniapps/navier directory
-  for more details.
-
- Added a new miniapps/adjoint directory with two miniapps demonstrating how to
-  solve adjoint problems in MFEM using the CVODES package in SUNDIALS. Both of
-  these miniapps require the MFEM_USE_SUNDIALS configuration option.
-  * The cvsRoberts_ASAi_dns miniapp solves a backward adjoint problem for a
-    system of ODEs, evaluating both forward and adjoint quadratures in serial.
-  * The adjoint_advection_diffusion miniapp solves a backward adjoint problem
-    for an advection diffusion PDE, evaluating adjoint quadratures in parallel.
-
- Ported Example 11p to SLEPc, to demonstrate solving the Laplace eigenvalue
-  equation with the shift-and-invert spectral transformation method.
-
- Added a simple meshing miniapp, Twist, which demonstrates MFEM's strategy of
-  stitching together opposite surfaces of a mesh to create a topologically
-  periodic mesh.
-
- Added a new meshing miniapp, Minimal Surface, which solves Plateau's problem:
-  the Dirichlet problem for the minimal surface equation.
-
- Added full assembly support in Example 9/9p.
-
- Added a new test problem in Example 24/24p, demonstrating a mixed bilinear
-  form for H1, H(curl), H(div) and L_2, with partial assembly support.
-
- Added weak Dirichlet boundary conditions (Nitsche) to the NURBS miniapp.
-
- Added a simple mesh editing miniapp, Trimmer, which trims away portions of a
-  mesh based on element attributes. Any newly exposed boundary elements are
-  assigned attribute numbers related to the trimmed element attributes.
-
- Added partial assembly and device support to Example 4/4p, Example 5/5p,
-  Example 22/22p, and Example 25/25p, with diagonal preconditioning.
+Discretization improvements
+---------------------------
+- Added support for simplices in GSLIB-FindPoints.

 Improved testing
 ----------------
@@ -176,16 +46,8 @@ Improved testing

 Miscellaneous
 -------------
- Added support for ADIOS2 for parallel I/O with ParaView visualization. The
-  classes adios2stream and ADIOS2DataCollection are introduced in mfem as the
-  interfaces to generate ADIOS2 Binary Pack (BP4) directory datasets for the
-  entire spatial and temporal data. In addition, ADIOS2 allows for setting a
-  user-defined number of data substreams/subfiles. See examples 5, 9, 12, 16.
-
- The integration order used in the ComputeLpError and ComputeElementLpError
-  methods of class GridFunction has been increased.
-
- Various other simplifications, extensions, and bugfixes in the code.
+- In SLISolver, changed the residual inner product from (Br,r) to (Br,Br) so the
+  solver can work with non-SPD preconditioner B.


 Version 4.1, released on March 10, 2020
@@ -149,13 +149,9 @@ if (MFEM_USE_MPI)
      message(FATAL_ERROR "PETSc version >= 3.8.0 is required")
    endif()
    set(PETSC_INCLUDE_DIRS ${PETSC_INCLUDES})
-    if (MFEM_USE_SLEPC)
-      find_package(SLEPc REQUIRED config)
-      message(STATUS "Found SLEPc version ${SLEPC_VERSION}")
-    endif()
  endif()
 else()
-  set(PKGS_NEED_MPI SUPERLU PETSC SLEPC STRUMPACK PUMI)
+  set(PKGS_NEED_MPI SUPERLU PETSC STRUMPACK PUMI)
  foreach(PKG IN LISTS PKGS_NEED_MPI)
    if (MFEM_USE_${PKG})
      message(STATUS "Disabling package ${PKG} - requires MPI")
@@ -211,10 +207,10 @@ endif()
 # SUNDIALS
 if (MFEM_USE_SUNDIALS)
  if (NOT MFEM_USE_MPI)
-    find_package(SUNDIALS REQUIRED NVector_Serial CVODES ARKODE KINSOL)
+    find_package(SUNDIALS REQUIRED NVector_Serial CVODE ARKODE KINSOL)
  else()
    find_package(SUNDIALS REQUIRED
-      NVector_Serial NVector_Parallel NVector_ParHyp CVODES ARKODE KINSOL)
+      NVector_Serial NVector_Parallel NVector_ParHyp CVODE ARKODE KINSOL)
  endif()
 endif()

@@ -327,11 +323,6 @@ if (MFEM_USE_UMPIRE)
   find_package(UMPIRE REQUIRED)
 endif()

-# ADIOS2 for parallel I/O
-if (MFEM_USE_ADIOS2)
-   find_package(ADIOS2 REQUIRED)
-endif()
-
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
  if (APPLE)
@@ -356,8 +347,8 @@ endif()
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
 set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
-    SLEPC MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
-    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2)
+    MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
+    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")
@@ -109,7 +109,6 @@ The MFEM source code has the following structure:
  ├── linalg
  ├── mesh
  ├── miniapps
-  │   ├── adjoint
  │   ├── common
  │   ├── electromagnetics
  │   ├── gslib
@@ -384,13 +383,9 @@ Before a PR can be merged, it should satisfy the following:
    - [ ] Is this a new feature users need to be aware of? New or updated example or miniapp?
    - [ ] Does it make sense to create a new section in the `CHANGELOG` to group with other related features?
 - [ ] Update `INSTALL`:
-    - [ ] Had a new optional library been added? If so, what range of versions of this library are required? (*Make sure the external library is compatible with our BSD license, e.g. it is not licensed under GPL!*)
-    - [ ] Have the version ranges for any required or optional libraries changed?
+    - [ ] Had a new optional library been added? (*Make sure the external library is compatible with our BSD license, e.g. it is not licensed under GPL!*)
    - [ ] Does `make` or `cmake` have a new target?
    - [ ] Did the requirements or the installation process change? *(rare)*
- [ ] Update continuous integration server configurations if necessary (e.g. with new version requirements for each of MFEM's dependencies)
-    - [ ] `.travis.yml`
-    - [ ] `.appveyor.yml`
 - [ ] Update `.gitignore`:
    - [ ] Check if `make distclean; git status` shows any files that were generated from the source by the project (not an IDE) but we don't want to track in the repository.
    - [ ] Add new patterns (just for the new files above) and re-run the above test.
@@ -504,10 +499,6 @@ MFEM uses a `master`/`next`-branch workflow as described below:
    - [ ] `makefile`
    - [ ] `CMakeLists.txt`
    - [ ] `doc/CodeDocumentation.conf.in`
- [ ] Check that version requirements for each of MFEM's dependencies are documented in `INSTALL` and up-to-date
- [ ] Check that continuous integration server configurations reflect the dependency version requirements of the new release
-    - [ ] `.travis.yml`
-    - [ ] `.appveyor.yml`
 - [ ] (LLNL only) Make sure all `README.html` files in the source repo are up to date.
 - [ ] Tag the repository:

@@ -383,10 +383,6 @@ MFEM_USE_PETSC = YES/NO
   and other features based on the PETSc package. When enabled, this option uses
   the PETSC_* library options, see below.

-MFEM_USE_SLEPC = YES/NO
-   Enable MFEM eigensolvers based on the SLEPc package. When enabled, this
-   option uses the SLEPC_* library options, see below.
-
 MFEM_USE_MPFR = YES/NO
   MPFR is a library for multiple-precision floating-point computations. This
   option enables the use of MPFR in MFEM, e.g. for precise computation of 1D
@@ -400,12 +396,6 @@ MFEM_USE_SIDRE = YES/NO
   blueprint specification. When enabled, this option requires installation of
   HDF5 (see also MFEM_USE_NETCDF), Conduit and LLNL's axom project.

-MFEM_USE_SIMD = YES/NO
-   Enables the high performance templated classes to use architecture dependent
-   SIMD intrinsics instead of the generic implementation of class AutoSIMD in
-   linalg/simd/auto.hpp. This option should be combined with suitable
-   compiler options, such as -march=native, to enable optimal vectorization.
-
 MFEM_USE_CONDUIT = YES/NO
   Enables support for converting MFEM Mesh and Grid Function objects to and
   from Conduit Mesh Blueprint Descriptions (https://github.com/LLNL/conduit/)
@@ -413,11 +403,6 @@ MFEM_USE_CONDUIT = YES/NO
   an installation of Conduit. If Conduit was built with HDF5 support, it also
   requires an installation of HDF5 (see also MFEM_USE_NETCDF).

-MFEM_USE_ADIOS2 = YES/NO
-   Enables support for ADIOS2, version 2 of the adaptable input output system
-   for scientific data management. In MFEM, ADIOS2 provides parallel I/O with
-   ParaView visualization.
-
 MFEM_USE_ZLIB = YES/NO
   Enables use of on-the-fly gzip compressed streams. With this feature enabled
   (YES), MFEM can compress its output files on-the-fly. In addition, it can
@@ -436,8 +421,6 @@ MFEM_USE_PUMI = YES/NO
   data management system that is capable of handling general non-manifold
   models and effectively supports automated adaptive analysis. PUMI enables
   support for parallel unstructured mesh modifications in MFEM.
-   The develop branch of PUMI repository (https://github.com/SCOREC/core)
-   should be used for most updated features.

 MFEM_USE_UMPIRE = YES/NO
   Enables support for Umpire, a resource management library that allows the
@@ -509,13 +492,11 @@ The specific libraries and their options are:
 - HYPRE, required for the parallel build, i.e. when MFEM_USE_MPI = YES.
  URL: https://github.com/hypre-space/hypre and https://www.llnl.gov/casc/hypre
  Options: HYPRE_OPT, HYPRE_LIB.
-  Versions: HYPRE >= 2.10.0b.

 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
  MFEM_USE_METIS_5 = YES (default is to use METIS 4).
  URL: http://glaros.dtc.umn.edu/gkhome/metis/metis/overview
  Options: METIS_OPT, METIS_LIB.
-  Versions: METIS 4.0.3 or 5.1.0.

 - LAPACK (optional), used when MFEM_USE_LAPACK = YES. Alternative, optimized
  implementations can also be used, e.g. the ATLAS project.
@@ -539,7 +520,6 @@ The specific libraries and their options are:
  both MPI and hypre.
  URL: http://computation.llnl.gov/projects/sundials/sundials-software
  Options: SUNDIALS_OPT, SUNDIALS_LIB.
-  Versions: SUNDIALS >= 5.0.0.

 - Mesquite (optional), used when MFEM_USE_MESQUITE = YES.
  URL: http://trilinos.org/oldsite/packages/mesquite
@@ -548,7 +528,6 @@ The specific libraries and their options are:
 - SuiteSparse (optional), used when MFEM_USE_SUITESPARSE = YES.
  URL: http://faculty.cse.tamu.edu/davis/suitesparse.html
  Options: SUITESPARSE_OPT, SUITESPARSE_LIB.
-  Versions: SuiteSparse >= 4.5.4, older versions may work too.

 - SuperLU_DIST (optional), used when MFEM_USE_SUPERLU = YES. Note that
  SuperLU_DIST requires ParMETIS, which includes METIS 5 in its distribution.
@@ -556,7 +535,6 @@ The specific libraries and their options are:
  same location.
  URL: http://crd-legacy.lbl.gov/~xiaoye/SuperLU
  Options: SUPERLU_OPT, SUPERLU_LIB.
-  Versions: SuperLU_DIST >= 5.1.0.

 - STRUMPACK (optional), used when MFEM_USE_STRUMPACK = YES. Note that STRUMPACK
  requires the PT-Scotch and Scalapack libraries as well as ParMETIS, which
@@ -566,7 +544,6 @@ The specific libraries and their options are:
  2.0.0 or later.
  URL: http://portal.nersc.gov/project/sparse/strumpack
  Options: STRUMPACK_OPT, STRUMPACK_LIB.
-  Versions: STRUMPACK >= 3.0.0, requires HYPRE < 2.16.0.

 - Ginkgo (optional), used when MFEM_USE_GINKGO = YES. Note that Ginkgo needs a
  C++ compiler that supports the C++-11 standard. For additional requirements
@@ -579,7 +556,6 @@ The specific libraries and their options are:
  one can get the library through the Homebrew package manager (http://brew.sh).
  URL: http://gnutls.org
  Options: GNUTLS_OPT, GNUTLS_LIB.
-  Versions: GnuTLS >= 2.12.0, older versions may work too.

 - NetCDF (optional), used when MFEM_USE_NETCDF = YES, required for reading Cubit
  mesh files. Also requires installation of HDF5 and ZLIB, as explained at the
@@ -587,7 +563,6 @@ The specific libraries and their options are:
  don't need the C++ or parallel versions.
  URL: www.unidata.ucar.edu/software/netcdf
  Options: NETCDF_OPT, NETCDF_LIB.
-  Versions: NetCDF >= 4.4.0.

 - PETSc (optional), used when MFEM_USE_PETSC = YES. Version 3.8 or higher of
  the PETSC dev branch is required. The MFEM and PETSc builds can share common
@@ -599,13 +574,6 @@ The specific libraries and their options are:
                --with-shared-libraries=0
  URL: https://www.mcs.anl.gov/petsc
  Options: PETSC_OPT, PETSC_LIB.
-  Versions: PETSc >= 3.8.0.
-
- SLEPc (optional), used when MFEM_USE_SLEPC = YES. SLEPc depends on PETSc and
-  uses some of the PETSc options when compiled.
-  URL: https://slepc.upv.es/
-  Options: SLEPC_OPT, SLEPC_LIB.
-  Versions: SLEPc >= 3.8.0.

 - Sidre (optional), part of LLNL's axom project, used when MFEM_USE_SIDRE = YES.
  Starting with MFEM v4.1, Axom version 0.3.1 or later is required.
@@ -613,23 +581,16 @@ The specific libraries and their options are:
       https://github.com/LLNL/conduit (Conduit)
       https://support.hdfgroup.org/HDF5 (HDF5)
  Options: SIDRE_OPT, SIDRE_LIB.
-  Versions: Axom >= 0.3.1.

 - Conduit (optional), used when MFEM_USE_CONDUIT = YES. Conduit Mesh Blueprint
  support requires Conduit >= v0.3.1 and VisIt >= v2.13.1 to read the output.
  URL: https://github.com/LLNL/conduit (Conduit)
       https://support.hdfgroup.org/HDF5 (HDF5)
  Options: CONDUIT_OPT, CONDUIT_LIB.
-  Versions: Conduit >= 0.3.1.
-
- ADIOS2 (optional) used when MFEM_USE_ADIOS2 = YES.
-  URL: https://adios2.readthedocs.io/

 - PUMI (optional), used when MFEM_USE_PUMI = YES.
  URL: https://scorec.rpi.edu/pumi
-       https://github.com/SCOREC/core
  Options: PUMI_OPT, PUMI_LIB.
-  Versions: PUMI >= 2.2.3.

 - HiOp (optional), used when MFEM_USE_HIOP = YES.
  URL: https://github.com/LLNL/hiop
@@ -643,12 +604,10 @@ The specific libraries and their options are:
  MFEM_USE_GSLIB=YES.
  URL: https://github.com/gslib/gslib/archive/v1.0.5.tar.gz
  Options: GSLIB_OPT, GSLIB_LIB.
-  Versions: GSLIB >= 1.0.5.

 - CUDA (optional), used when MFEM_USE_CUDA = YES.
  URL: https://developer.nvidia.com/cuda-toolkit
  Options: CUDA_CXX, CUDA_ARCH, CUDA_OPT, CUDA_LIB.
-  Versions: CUDA >= 9.1, older versions may work too.

 - HIP (optional), used when MFEM_USE_HIP = YES.
  URL: https://rocm.github.io/ROCmInstall.html
@@ -657,24 +616,21 @@ The specific libraries and their options are:
 - OCCA (optional), used when MFEM_USE_OCCA = YES.
  URL: https://libocca.org
  Options: OCCA_DIR, OCCA_OPT, OCCA_LIB.
-  Versions: OCCA >= 1.0.9.

- libCEED (optional), used when MFEM_USE_CEED = YES.
+- libCEED (optional), used when MFEM_USE_CEED = YES. Requires libCEED v0.6
+  or later version, specifically, git-hash 3d05795 or later.
  URL: https://github.com/CEED/libCEED
       https://ceed.exascaleproject.org/libceed
  Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED >= 0.6, git-hash a970f63.

 - RAJA (optional), used when MFEM_USE_RAJA = YES.
  Beginning with MFEM v4.1, only RAJA v0.10.0+ is supported.
  URL: https://github.com/LLNL/RAJA
  Options: RAJA_DIR, RAJA_OPT, RAJA_LIB.
-  Versions: RAJA >= 0.10.0.

 - Umpire, used when MFEM_USE_UMPIRE = YES.
  URL: https://github.com/LLNL/Umpire
  Options: UMPIRE_DIR, UMPIRE_OPT, UMPIRE_LIB.
-  Versions: Umpire >= 2.0.0.

 - MPFR (optional), used when MFEM_USE_MPFR = YES.
  URL: http://mpfr.org, it depends on the GMP library: https://gmplib.org
@@ -244,10 +244,6 @@ IF (DEFINED TPL_ENABLE_PETSC)
  SET(MFEM_USE_PETSC ${TPL_ENABLE_PETSC} CACHE BOOL "Enable PETSc support." FORCE)
 ENDIF()

-IF (DEFINED TPL_ENABLE_SLEPC)
-  SET(MFEM_USE_SLEPC ${TPL_ENABLE_SLEPC} CACHE BOOL "Enable SLEPc support." FORCE)
-ENDIF()
-
 IF (DEFINED TPL_ENABLE_MPFR)
  SET(MFEM_USE_MPFR ${TPL_ENABLE_MPFR} CACHE BOOL "Enable MPFR usage." FORCE)
 ENDIF()
@@ -38,7 +38,6 @@ set(MFEM_USE_GNUTLS @MFEM_USE_GNUTLS@)
 set(MFEM_USE_GSLIB @MFEM_USE_GSLIB@)
 set(MFEM_USE_NETCDF @MFEM_USE_NETCDF@)
 set(MFEM_USE_PETSC @MFEM_USE_PETSC@)
-set(MFEM_USE_SLEPC @MFEM_USE_SLEPC@)
 set(MFEM_USE_MPFR @MFEM_USE_MPFR@)
 set(MFEM_USE_SIDRE @MFEM_USE_SIDRE@)
 set(MFEM_USE_CONDUIT @MFEM_USE_CONDUIT@)
@@ -48,8 +47,6 @@ set(MFEM_USE_OCCA @MFEM_USE_OCCA@)
 set(MFEM_USE_RAJA @MFEM_USE_RAJA@)
 set(MFEM_USE_CEED @MFEM_USE_CEED@)
 set(MFEM_USE_UMPIRE @MFEM_USE_UMPIRE@)
-set(MFEM_USE_SIMD @MFEM_USE_SIMD@)
-set(MFEM_USE_ADIOS2 @MFEM_USE_ADIOS2@)

 set(MFEM_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
 set(MFEM_CXX_FLAGS "@CMAKE_CXX_FLAGS@")
@@ -104,15 +104,9 @@
 // Enable MFEM functionality based on the PETSc library
 #cmakedefine MFEM_USE_PETSC

-// Enable MFEM functionality based on the SLEPc library
-#cmakedefine MFEM_USE_SLEPC
-
 // Enable MFEM functionality based on the Sidre library
 #cmakedefine MFEM_USE_SIDRE

-// Enable the use of SIMD in the high performance templated classes
-#cmakedefine MFEM_USE_SIMD
-
 // Enable MFEM functionality based on Conduit
 #cmakedefine MFEM_USE_CONDUIT

@@ -138,9 +132,6 @@
 // Enable MFEM functionality based on the Umpire library
 #cmakedefine MFEM_USE_UMPIRE

-// Enable MFEM functionality based on the ADIOS2 library
-#cmakedefine MFEM_USE_ADIOS2
-
 // Which library functions to use in class StopWatch for measuring time.
 // For a list of the available options, see INSTALL.
 // If not defined, an option is selected automatically.
@@ -1,52 +0,0 @@
-#------------------------------------------------------------------------------#
-# Distributed under the OSI-approved Apache License, Version 2.0.  See
-# accompanying file Copyright.txt for details.
-#------------------------------------------------------------------------------#
-#
-# FindADIOS2
-# -----------
-#
-# Try to find the ADIOS2 library
-#
-# This module defines the following variables:
-#
-#   ADIOS2_FOUND        - System has ADIOS2
-#   ADIOS2_INCLUDE_DIRS - The ADIOS2 include directory
-#   ADIOS2_LIBRARIES    - Link these to use ADIOS2
-#
-# and the following imported targets:
-#   ADIOS2::ADIOS2 - The ADIOS2 compression library target
-#
-# You can also set the following variable to help guide the search:
-#   ADIOS2_DIR - The install prefix for ADIOS2 containing the
-#              include and lib folders
-#              Note: this can be set as a CMake variable or an
-#                    environment variable.  If specified as a CMake
-#                    variable, it will override any setting specified
-#                    as an environment variable.
-
-if(NOT ADIOS2_FOUND)
-  if((NOT ADIOS2_DIR) AND (NOT (ENV{ADIOS2_DIR} STREQUAL "")))
-    set(ADIOS2_DIR "$ENV{ADIOS2_DIR}")
-  endif()
-  if(ADIOS2_DIR)
-    set(ADIOS2_INCLUDE_OPTS HINTS ${ADIOS2_DIR}/include NO_DEFAULT_PATHS)
-    set(ADIOS2_LIBRARY_OPTS
-      HINTS ${ADIOS2_DIR}/lib ${ADIOS2_DIR}/lib64
-      NO_DEFAULT_PATHS
-    )
-  endif()
-
-  find_path(ADIOS2_INCLUDE_DIR adios2.h ${ADIOS2_INCLUDE_OPTS})
-  find_library(ADIOS2_LIBRARY NAMES adios2 ${ADIOS2_LIBRARY_OPTS}) 
-
-  include(FindPackageHandleStandardArgs)
-  find_package_handle_standard_args(ADIOS2
-    FOUND_VAR ADIOS2_FOUND
-    REQUIRED_VARS ADIOS2_LIBRARY ADIOS2_INCLUDE_DIR
-  )
-  if(ADIOS2_FOUND)
-    set(ADIOS2_INCLUDE_DIRS ${ADIOS2_INCLUDE_DIR})
-    set(ADIOS2_LIBRARIES ${ADIOS2_LIBRARY})
-  endif()
-endif()
@@ -1,44 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Sets the following variables:
-#   - SLEPC_FOUND
-#   - SLEPC_INCLUDE_DIRS
-#   - SLEPC_LIBRARIES
-
-set(SLEPc_REQUIRED_PACKAGES "PETSC" CACHE STRING
-  "Additional packages required by SLEPc")
-
-include(MfemCmakeUtilities)
-mfem_find_package(SLEPc SLEPC SLEPC_DIR
-  "include" "slepceps.h"
-  "${PETSC_ARCH}/lib" "slepc" # add NAMES_PER_DIR?
-  "Paths to headers required by SLEPc."
-  "Libraries required by SLEPc."
-  ADD_COMPONENT "config" "${PETSC_ARCH}/include" "slepcconf.h" "" ""
-  CHECK_BUILD SLEPC_VERSION_OK TRUE
-"
-#include \"petsc.h\"
-#include \"slepceps.h\"
-int main()
-{
-  PetscErrorCode ierr;
-  int argc = 0;
-  char** argv = NULL;
-  ierr = SlepcInitialize(&argc, &argv, PETSC_NULL, PETSC_NULL);
-  EPS eps;
-  ierr = EPSCreate(PETSC_COMM_SELF, &eps); CHKERRQ(ierr);
-  ierr = EPSDestroy(&eps); CHKERRQ(ierr);
-  ierr = SlepcFinalize(); CHKERRQ(ierr);
-  return 0;
-}
-"
-  )
@@ -25,6 +25,5 @@ mfem_find_package(SUNDIALS SUNDIALS SUNDIALS_DIR
  ADD_COMPONENT NVector_ParHyp
    "include" nvector/nvector_parhyp.h "lib" sundials_nvecparhyp
  ADD_COMPONENT CVODE "include" cvode/cvode.h "lib" sundials_cvode
-  ADD_COMPONENT CVODES "include" cvodes/cvodes.h "lib" sundials_cvodes
  ADD_COMPONENT ARKODE "include" arkode/arkode.h "lib" sundials_arkode
  ADD_COMPONENT KINSOL "include" kinsol/kinsol.h "lib" sundials_kinsol)
@@ -731,9 +731,9 @@ function(mfem_export_mk_files)
      MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP MFEM_USE_LEGACY_OPENMP
      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GNUTLS
-      MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE
+      MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_MPFR MFEM_USE_SIDRE
      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA
-      MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2)
+      MFEM_USE_UMPIRE)
  foreach(var ${CONFIG_MK_BOOL_VARS})
    if (${var})
      set(${var} YES)
@@ -743,7 +743,6 @@ function(mfem_export_mk_files)
  endforeach()
  # TODO: Add support for MFEM_USE_CUDA=YES
  set(MFEM_CXX ${CMAKE_CXX_COMPILER})
-  set(MFEM_HOST_CXX ${MFEM_CXX})
  set(MFEM_CPPFLAGS "")
  string(STRIP "${CMAKE_CXX_FLAGS_${BUILD_TYPE}} ${CMAKE_CXX_FLAGS}"
         MFEM_CXXFLAGS)
@@ -48,9 +48,6 @@
 #ifdef MFEM_USE_PETSC
 #error Building with PETSc (MFEM_USE_PETSC=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
-#ifdef MFEM_USE_SLEPC
-#error Building with SLEPc (MFEM_USE_SLEPC=YES) requires MPI (MFEM_USE_MPI=YES)
-#endif
 #ifdef MFEM_USE_PUMI
 #error Building with PUMI (MFEM_USE_PUMI=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
@@ -106,9 +106,6 @@
 // Enable Sidre support
 // #define MFEM_USE_SIDRE

-// Enable the use of SIMD in the high performance templated classes
-// #define MFEM_USE_SIMD
-
 // Enable Conduit support
 // #define MFEM_USE_CONDUIT

@@ -118,9 +115,6 @@
 // Enable functionality based on the PETSc library
 // #define MFEM_USE_PETSC

-// Enable functionality based on the SLEPc library
-// #define MFEM_USE_SLEPC
-
 // Enable functionality based on the MPFR library.
 // #define MFEM_USE_MPFR

@@ -153,9 +147,6 @@
 // Enable functionality based on the Umpire library.
 // #define MFEM_USE_UMPIRE

-// Enable IO functionality based on the ADIOS2 library.
-// #define MFEM_USE_ADIOS2
-
 // Version of HYPRE used for building MFEM.
 // #define MFEM_HYPRE_VERSION @MFEM_HYPRE_VERSION@

@@ -37,7 +37,6 @@ MFEM_USE_GINKGO        = @MFEM_USE_GINKGO@
 MFEM_USE_GNUTLS        = @MFEM_USE_GNUTLS@
 MFEM_USE_NETCDF        = @MFEM_USE_NETCDF@
 MFEM_USE_PETSC         = @MFEM_USE_PETSC@
-MFEM_USE_SLEPC         = @MFEM_USE_SLEPC@
 MFEM_USE_MPFR          = @MFEM_USE_MPFR@
 MFEM_USE_SIDRE         = @MFEM_USE_SIDRE@
 MFEM_USE_CONDUIT       = @MFEM_USE_CONDUIT@
@@ -50,12 +49,9 @@ MFEM_USE_RAJA          = @MFEM_USE_RAJA@
 MFEM_USE_OCCA          = @MFEM_USE_OCCA@
 MFEM_USE_CEED          = @MFEM_USE_CEED@
 MFEM_USE_UMPIRE        = @MFEM_USE_UMPIRE@
-MFEM_USE_SIMD          = @MFEM_USE_SIMD@
-MFEM_USE_ADIOS2        = @MFEM_USE_ADIOS2@

 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@
-MFEM_HOST_CXX  = @MFEM_HOST_CXX@
 MFEM_CPPFLAGS  = @MFEM_CPPFLAGS@
 MFEM_CXXFLAGS  = @MFEM_CXXFLAGS@
 MFEM_TPLFLAGS  = @MFEM_TPLFLAGS@
@@ -39,7 +39,6 @@ option(MFEM_USE_GNUTLS "Enable GNUTLS usage" OFF)
 option(MFEM_USE_GSLIB "Enable GSLIB usage" OFF)
 option(MFEM_USE_NETCDF "Enable NETCDF usage" OFF)
 option(MFEM_USE_PETSC "Enable PETSc support." OFF)
-option(MFEM_USE_SLEPC "Enable SLEPc support." OFF)
 option(MFEM_USE_MPFR "Enable MPFR usage." OFF)
 option(MFEM_USE_SIDRE "Enable Axom/Sidre usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
@@ -50,8 +49,6 @@ option(MFEM_USE_OCCA "Enable OCCA" OFF)
 option(MFEM_USE_RAJA "Enable RAJA" OFF)
 option(MFEM_USE_CEED "Enable CEED" OFF)
 option(MFEM_USE_UMPIRE "Enable Umpire" OFF)
-option(MFEM_USE_SIMD "Enable use of SIMD intrinsics" ON)
-option(MFEM_USE_ADIOS2 "Enable ADIOS2" OFF)

 set(MFEM_MPI_NP 4 CACHE STRING "Number of processes used for MPI tests")

@@ -88,8 +85,6 @@ set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library."

 set(LIBUNWIND_DIR "" CACHE PATH "Path to Libunwind.")

-# For sundials_nvecparhyp and nvecparallel remember to build with MPI_ENABLED=ON
-# and modify cmake variables for hypre for sundials
 set(SUNDIALS_DIR "${MFEM_DIR}/../sundials-5.0.0/instdir" CACHE PATH
    "Path to the SUNDIALS library.")
 # The following may be necessary, if SUNDIALS was built with KLU:
@@ -158,10 +153,6 @@ set(PETSC_DIR "${MFEM_DIR}/../petsc" CACHE PATH
    "Path to the PETSc main directory.")
 set(PETSC_ARCH "arch-linux2-c-debug" CACHE STRING "PETSc build architecture.")

-set(SLEPC_DIR "${MFEM_DIR}/../slepc" CACHE PATH
-    "Path to the SLEPc main directory.")
-set(SLEPC_ARCH "arch-linux2-c-debug" CACHE STRING "SLEPC build architecture.")
-
 set(MPFR_DIR "" CACHE PATH "Path to the MPFR library.")

 set(CONDUIT_DIR "${MFEM_DIR}/../conduit" CACHE PATH
@@ -125,7 +125,6 @@ MFEM_USE_GINKGO        = NO
 MFEM_USE_GNUTLS        = NO
 MFEM_USE_NETCDF        = NO
 MFEM_USE_PETSC         = NO
-MFEM_USE_SLEPC         = NO
 MFEM_USE_MPFR          = NO
 MFEM_USE_SIDRE         = NO
 MFEM_USE_CONDUIT       = NO
@@ -138,8 +137,6 @@ MFEM_USE_RAJA          = NO
 MFEM_USE_OCCA          = NO
 MFEM_USE_CEED          = NO
 MFEM_USE_UMPIRE        = NO
-MFEM_USE_SIMD          = YES
-MFEM_USE_ADIOS2        = NO

 # Compile and link options for zlib.
 ZLIB_DIR =
@@ -189,12 +186,10 @@ OPENMP_LIB =
 POSIX_CLOCKS_LIB = -lrt

 # SUNDIALS library configuration
-# For sundials_nvecparhyp and nvecparallel remember to build with MPI_ENABLED=ON
-# and modify cmake variables for hypre for sundials
 SUNDIALS_DIR = @MFEM_DIR@/../sundials-5.0.0/instdir
 SUNDIALS_OPT = -I$(SUNDIALS_DIR)/include
 SUNDIALS_LIB = -Wl,-rpath,$(SUNDIALS_DIR)/lib64 -L$(SUNDIALS_DIR)/lib64\
- -lsundials_arkode -lsundials_cvodes -lsundials_nvecserial -lsundials_kinsol
+ -lsundials_arkode -lsundials_cvode -lsundials_nvecserial -lsundials_kinsol

 ifeq ($(MFEM_USE_MPI),YES)
   SUNDIALS_LIB += -lsundials_nvecparhyp -lsundials_nvecparallel
@@ -279,20 +274,6 @@ ifeq ($(PETSC_FOUND),YES)
      -L$(abspath $(PETSC_DIR))/lib -lpetsc $(PETSC_LIB)
 endif

-SLEPC_DIR := $(MFEM_DIR)/../slepc
-SLEPC_VARS := $(SLEPC_DIR)/lib/slepc/conf/slepc_variables
-SLEPC_FOUND := $(if $(wildcard $(SLEPC_VARS)),YES,)
-SLEPC_INC_VAR = SLEPC_INCLUDE
-SLEPC_LIB_VAR = SLEPC_EXTERNAL_LIB
-ifeq ($(SLEPC_FOUND),YES)
-   SLEPC_OPT := $(shell sed -n "s/$(SLEPC_INC_VAR) *= *//p" $(SLEPC_VARS))
-   # Some additional external libraries might be defined in this file
-   -include ${SLEPC_DIR}/${PETSC_ARCH}/lib/slepc/conf/slepcvariables
-   SLEPC_LIB := $(shell sed -n "s/$(SLEPC_LIB_VAR) *= *//p" $(SLEPC_VARS))
-   SLEPC_LIB := -Wl,-rpath,$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib\
-      -L$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib -lslepc $(SLEPC_LIB)
-endif
-
 # MPFR library configuration
 MPFR_OPT =
 MPFR_LIB = -lmpfr
@@ -47,7 +47,7 @@ groups_serial=(
   "Meshing miniapps:"
   "miniapps/meshing"
   "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp
-    mesh-optimizer.cpp minimal-surface.cpp"'
+    mesh-optimizer.cpp"'
 )
 # Parallel groups
 groups_parallel=(
@@ -72,7 +72,7 @@ groups_parallel=(
 '"meshing"
   "Meshing miniapps:"
   "miniapps/meshing"
-   "pmesh-optimizer.cpp pminimal-surface.cpp"'
+   "pmesh-optimizer.cpp"'
 '"electromagnetics"
   "Electromagnetics miniapps:"
   "miniapps/electromagnetics"
@@ -101,7 +101,7 @@ groups_all=(
   "Meshing miniapps:"
   "miniapps/meshing"
   "mobius-strip.cpp klein-bottle.cpp extruder.cpp toroid.cpp
-    {,p}mesh-optimizer.cpp {,p}minimal-surface.cpp"'
+    {,p}mesh-optimizer.cpp"'
 '"electromagnetics"
   "Electromagnetics miniapps:"
   "miniapps/electromagnetics"
@@ -29,20 +29,8 @@
 #define MFEM_ALWAYS_INLINE
 #endif

-// --- MFEM_VECTORIZE_LOOP (disabled)
-#if (__cplusplus >= 201103L) && !defined(MFEM_DEBUG) && defined(__GNUC__)
-//#define MFEM_VECTORIZE_LOOP _Pragma("GCC ivdep")
-#define MFEM_VECTORIZE_LOOP
-#else
-#define MFEM_VECTORIZE_LOOP
-#endif
-
-// MFEM_TEMPLATE_BLOCK_SIZE is the block size used by the template matrix-matrix
-// multiply, Mult_AB, defined in tmatrix.hpp. This parameter will generally
-// require tuning to determine good value. It is probably highly influenced by
-// the SIMD width when Mult_AB is used with a SIMD type like AutoSIMD.
 #define MFEM_TEMPLATE_BLOCK_SIZE 4
-
+#define MFEM_SIMD_SIZE 32
 #define MFEM_TEMPLATE_ENABLE_SERIALIZE

 // #define MFEM_TEMPLATE_ELTRANS_HAS_NODE_DOFS
@@ -50,6 +38,11 @@
 // #define MFEM_TEMPLATE_FIELD_EVAL_DATA_HAS_DOFS
 #define MFEM_TEMPLATE_INTRULE_COEFF_PRECOMP

+// derived macros
+#define MFEM_ROUNDUP(val,base) ((((val)+(base)-1)/(base))*(base))
+#define MFEM_ALIGN_SIZE(size,type) \
+   MFEM_ROUNDUP(size,(MFEM_SIMD_SIZE)/sizeof(type))
+
 #ifdef MFEM_COUNT_FLOPS
 namespace mfem
 {
@@ -1,37 +0,0 @@
-SetFactory("OpenCASCADE");
-
-R1 = 1.0;
-R2 = 2.0;
-
-Point(1) = {0.0, 0, 0, 1.0};
-Point(2) = {R1, 0, 0, 1.0};
-Point(3) = {R2, 0, 0, 1.0};
-Point(4) = {R1*Cos(Pi/3), R1*Sin(Pi/3), 0, 1.0};
-Point(5) = {R2*Cos(Pi/3), R2*Sin(Pi/3), 0, 1.0};
-Line(1) = {2, 3};
-Line(2) = {4, 5};
-Circle(3) = {2, 1, 4};
-Circle(4) = {3, 1, 5};
-Curve Loop(5) = {1, 4, -2, -3};
-Plane Surface(1) = {5};
-
-Transfinite Curve{1} = 7;
-Transfinite Curve{2} = 7;
-Transfinite Curve{3} = 4;
-Transfinite Curve{4} = 10;
-
-// Set a rotation periodicity constraint:
-Periodic Line{1} = {2} Rotate{{0,0,1}, {0,0,0}, -Pi/3};
-
-// Tag surfaces and volumes with positive integers
-Physical Curve(1) = {3};
-Physical Curve(2) = {4};
-Physical Curve(3) = {1};
-Physical Curve(4) = {2};
-Physical Surface(1) = {1};
-
-// Generate 2D mesh
-Mesh 2;
-Mesh.MshFileVersion = 2.2;
-
-Save "periodic-annulus-sector.msh";
@@ -1,185 +0,0 @@
-$MeshFormat
-2.2 0 8
-$EndMeshFormat
-$Nodes
-55
-1 1 0 0
-2 2 0 0
-3 0.5000000000000001 0.8660254037844386 0
-4 1 1.732050807568877 0
-5 1.166666666666667 0 0
-6 1.333333333333333 0 0
-7 1.5 0 0
-8 1.666666666666667 0 0
-9 1.833333333333333 0 0
-10 0.5833333333333335 1.010362971081845 0
-11 0.6666666666666667 1.154700538379251 0
-12 0.7500000000000002 1.299038105676658 0
-13 0.8333333333333335 1.443375672974064 0
-14 0.9166666666666669 1.587713240271471 0
-15 0.9396926207859085 0.3420201433256683 0
-16 0.7660444431189786 0.6427876096865386 0
-17 1.986476715483886 0.2321858282504602 0
-18 1.946089741159648 0.4612317414848793 0
-19 1.879385241571817 0.6840402866513365 0
-20 1.787265280646825 0.8975983604009234 0
-21 1.670975622825874 1.09901795614161 0
-22 1.532088886237958 1.285575219373077 0
-23 1.372483275737469 1.454747283146095 0
-24 1.194317183405575 1.604246385510085 0
-25 1.425989114816062 0.1915326920916892 0
-26 0.8788667344146573 1.13917645290495 0
-27 1.630372059110754 0.7154531062316609 0
-28 1.436395769298814 1.053728612482506 0
-29 1.081023776188756 0.6241293681829633 0
-30 1.168737372335971 1.428012728596308 0
-31 1.821063986059922 0.298149890497067 0
-32 1.234707097211386 0.3469796339295647 0
-33 1.377747393186519 0.6200150626754309 0
-34 1.457047681210906 0.3890895843559762 0
-35 0.917846726184522 0.8957978954532204 0
-36 1.218335619030348 0.9017812086952638 0
-37 1.066623110765233 1.061857005744772 0
-38 1.587029716281926 0.1355955181472859 0
-39 1.744445799211916 0.1441515753740107 0
-40 1.25 0.1443375672974065 0
-41 1.453660070628011 0.8435769396609902 0
-42 1.741367044061892 0.499612708014486 0
-43 1.30550638526547 1.257610469847477 0
-44 1.118213276932792 0.1666674689105279 0
-45 0.9109440214958271 1.306610291787315 0
-46 0.9970618258753989 1.438658589955562 0
-47 0.7499999999999998 1.010362971081845 0
-48 0.7034449005273667 0.8850673702175776 0
-49 1.605449512513618 0.9269067082200894 0
-50 1.561654019115059 0.5298592532912715 0
-51 1.229782222487711 1.096820457143683 0
-52 1.617066998712459 0.3090202662210922 0
-53 1.079645953234324 1.246963713711438 0
-54 1.877063966817811 0.1348974588243076 0
-55 1.055356609656722 1.558136350380461 0
-$EndNodes
-$Elements
-108
-1 1 2 3 1 1 5
-2 1 2 3 1 5 6
-3 1 2 3 1 6 7
-4 1 2 3 1 7 8
-5 1 2 3 1 8 9
-6 1 2 3 1 9 2
-7 1 2 4 2 3 10
-8 1 2 4 2 10 11
-9 1 2 4 2 11 12
-10 1 2 4 2 12 13
-11 1 2 4 2 13 14
-12 1 2 4 2 14 4
-13 1 2 1 3 1 15
-14 1 2 1 3 15 16
-15 1 2 1 3 16 3
-16 1 2 2 4 2 17
-17 1 2 2 4 17 18
-18 1 2 2 4 18 19
-19 1 2 2 4 19 20
-20 1 2 2 4 20 21
-21 1 2 2 4 21 22
-22 1 2 2 4 22 23
-23 1 2 2 4 23 24
-24 1 2 2 4 24 4
-25 2 2 1 1 32 40 25
-26 2 2 1 1 25 34 32
-27 2 2 1 1 33 41 36
-28 2 2 1 1 38 52 25
-29 2 2 1 1 33 36 29
-30 2 2 1 1 26 47 35
-31 2 2 1 1 35 37 26
-32 2 2 1 1 25 52 34
-33 2 2 1 1 32 44 40
-34 2 2 1 1 15 32 29
-35 2 2 1 1 15 29 16
-36 2 2 1 1 36 41 28
-37 2 2 1 1 32 33 29
-38 2 2 1 1 50 52 42
-39 2 2 1 1 32 34 33
-40 2 2 1 1 42 52 31
-41 2 2 1 1 43 53 51
-42 2 2 1 1 27 41 33
-43 2 2 1 1 26 53 45
-44 2 2 1 1 18 31 17
-45 2 2 1 1 29 35 16
-46 2 2 1 1 29 36 35
-47 2 2 1 1 24 30 23
-48 2 2 1 1 30 53 43
-49 2 2 1 1 17 54 2
-50 2 2 1 1 4 55 24
-51 2 2 1 1 28 51 36
-52 2 2 1 1 47 48 35
-53 2 2 1 1 36 37 35
-54 2 2 1 1 37 53 26
-55 2 2 1 1 22 28 21
-56 2 2 1 1 20 27 19
-57 2 2 1 1 33 50 27
-58 2 2 1 1 15 44 32
-59 2 2 1 1 18 42 31
-60 2 2 1 1 30 43 23
-61 2 2 1 1 35 48 16
-62 2 2 1 1 31 54 17
-63 2 2 1 1 9 39 8
-64 2 2 1 1 8 38 7
-65 2 2 1 1 7 25 6
-66 2 2 1 1 22 43 28
-67 2 2 1 1 23 43 22
-68 2 2 1 1 39 54 31
-69 2 2 1 1 19 42 18
-70 2 2 1 1 24 55 30
-71 2 2 1 1 27 42 19
-72 2 2 1 1 13 46 14
-73 2 2 1 1 51 53 37
-74 2 2 1 1 39 52 38
-75 2 2 1 1 6 40 5
-76 2 2 1 1 34 52 50
-77 2 2 1 1 12 45 13
-78 2 2 1 1 30 55 46
-79 2 2 1 1 10 47 11
-80 2 2 1 1 8 39 38
-81 2 2 1 1 28 49 21
-82 2 2 1 1 7 38 25
-83 2 2 1 1 41 49 28
-84 2 2 1 1 20 49 27
-85 2 2 1 1 11 26 12
-86 2 2 1 1 27 49 41
-87 2 2 1 1 31 52 39
-88 2 2 1 1 25 40 6
-89 2 2 1 1 2 54 9
-90 2 2 1 1 14 55 4
-91 2 2 1 1 45 53 46
-92 2 2 1 1 45 46 13
-93 2 2 1 1 5 44 1
-94 2 2 1 1 21 49 20
-95 2 2 1 1 46 53 30
-96 2 2 1 1 3 48 10
-97 2 2 1 1 34 50 33
-98 2 2 1 1 36 51 37
-99 2 2 1 1 26 45 12
-100 2 2 1 1 11 47 26
-101 2 2 1 1 27 50 42
-102 2 2 1 1 40 44 5
-103 2 2 1 1 43 51 28
-104 2 2 1 1 10 48 47
-105 2 2 1 1 9 54 39
-106 2 2 1 1 46 55 14
-107 2 2 1 1 1 44 15
-108 2 2 1 1 16 48 3
-$EndElements
-$Periodic
-1
-1 1 2
-Affine 0.5000000000000001 0.8660254037844386 0 0 -0.8660254037844386 0.5000000000000001 0 0 0 0 1 0 0 0 0 1
-7
-9 14
-6 11
-8 13
-5 10
-7 12
-2 4
-1 3
-$EndPeriodic
@@ -1,25 +0,0 @@
-SetFactory("OpenCASCADE");
-
-R = 1.5;
-r = 0.5;
-
-Torus(1) = {0,0,0, R, r, Pi/3};
-
-pts() = PointsOf{ Volume{1}; };
-
-Characteristic Length{ pts() } = 0.25;
-
-// Set a rotation periodicity constraint:
-Periodic Surface{3} = {2} Rotate{{0,0,1}, {0,0,0}, Pi/3};
-
-// Tag surfaces and volumes with positive integers
-Physical Surface(1) = {1};
-Physical Surface(2) = {2};
-Physical Surface(3) = {3};
-Physical Volume(1) = {1};
-
-// Generate 3D mesh
-Mesh 3;
-
-Mesh.MshFileVersion = 2.2;
-Save "periodic-torus-sector.msh";
@@ -1,155 +0,0 @@
-MFEM NURBS mesh v1.0
-
-dimension
-2
-
-elements
-5
-1 3 0 3 7 4
-1 3 3 2 6 7
-1 3 2 1 5 6
-1 3 1 0 4 5
-1 3 2 8 9 1
-
-boundary
-10
-1 1 0 3
-2 1 3 2
-2 1 1 0
-2 1 2 8
-2 1 9 1
-3 1 7 4
-3 1 6 7
-3 1 5 6
-3 1 4 5
-4 1 8 9
-
-edges
-15
-0 0 4
-0 3 7
-0 1 5
-0 2 6
-1 0 3
-1 4 7
-2 3 2
-2 7 6
-2 1 0
-2 5 4
-1 2 1
-1 6 5
-1 8 9
-3 2 8
-3 1 9
-
-vertices
-10
-
-patches
-
-knotvectors
-2
-2 3 0 0 0 1 1 1
-2 4 0 0 0 0.5 1 1 1
-
-dimension
-2
-
-controlpoints_cartesian
-5 5 1
-5 3.92523e-16 1
-5 -5 1
-2.47593 2.47593 1
-4.95187 6.06429e-16 0.707107
-2.47593 -2.47593 1
-0.424264 0.424264 1
-0.848528 1.03915e-16 0.707107
-0.424264 -0.424264 1
-0.353553 0.353553 1
-0.707107 8.65956e-17 0.707107
-0.353553 -0.353553 1
-
-knotvectors
-2
-2 3 0 0 0 1 1 1
-2 4 0 0 0 0.5 1 1 1
-
-dimension
-2
-
-controlpoints_cartesian
-5 -5 1
-1.17757e-15 -5 1
-5 -5 1
-2.47593 -2.47593 1
-9.09644e-16 -4.95187 0.707107
-2.47593 -2.47593 1
-0.424264 -0.424264 1
-1.55872e-16 -0.848528 0.707107
-0.424264 -0.424264 1
-0.353553 -0.353553 1
-1.29893e-16 -0.707107 0.707107
-0.353553 -0.353553 1
-
-knotvectors
-2
-2 3 0 0 0 1 1 1
-2 4 0 0 0 0.5 1 1 1
-
-dimension
-2
-
-controlpoints_cartesian
-5 -5 1
-5 -1.17757e-15 1
-5 5 1
-2.47593 -2.47593 1
-4.95187 -1.21286e-15 0.707107
-2.47593 2.47593 1
-0.424264 -0.424264 1
-0.848528 -2.07829e-16 0.707107
-0.424264 0.424264 1
-0.353553 -0.353553 1
-0.707107 -1.73191e-16 0.707107
-0.353553 0.353553 1
-
-knotvectors
-2
-2 3 0 0 0 1 1 1
-2 4 0 0 0 0.5 1 1 1
-
-dimension
-2
-
-controlpoints_cartesian
-5 5 1
-3.92523e-16 5 1
-5 5 1
-2.47593 2.47593 1
-3.03215e-16 4.95187 0.707107
-2.47593 2.47593 1
-0.424264 0.424264 1
-5.19574e-17 0.848528 0.707107
-0.424264 0.424264 1
-0.353553 0.353553 1
-4.32978e-17 0.707107 0.707107
-0.353553 0.353553 1
-
-knotvectors
-2
-2 3 0 0 0 1 1 1
-2 3 0 0 0 1 1 1
-
-dimension
-2
-
-controlpoints_cartesian
-5 -5 1
-10 -5 1
-15 -5 1
-5 0 1
-10 0 1
-15 0 1
-5 5 1
-10 5 1
-15 5 1
@@ -16,21 +16,36 @@ if (DOXYGEN_FOUND)
  configure_file(${CMAKE_CURRENT_SOURCE_DIR}/CodeDocumentation.conf.in
    ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.conf @ONLY)

+  if (UNIX)
+    # Only create symlinks if UNIX operating system
+    add_custom_target(doc
+      COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.conf
+      COMMAND ${CMAKE_COMMAND} -E remove -f ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.html
+      COMMAND ${CMAKE_COMMAND} -E create_symlink
+      ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation/html/index.html
+      ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.html
+      BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation/html/index.html
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+      COMMENT "Generating API documentation with Doxygen to CodeDocumentation.html"
+      VERBATIM)

-  add_custom_target(doc
-    COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.conf
-    COMMAND echo "<meta http-equiv=\"REFRESH\" content=\"0;URL=CodeDocumentation/html/index.html\">" > ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.html
-    BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation/html/index.html
-    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-    COMMENT "Generating API documentation with Doxygen to CodeDocumentation.html"
-    VERBATIM)
-
-  add_custom_target(clean-doc
-    COMMAND ${CMAKE_COMMAND} -E remove -f ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.html
-    COMMAND ${CMAKE_COMMAND} -E remove -f ${CMAKE_CURRENT_BINARY_DIR}/warnings.log
-    COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation
-    COMMENT "Removing API documentation"
-    VERBATIM)
+    add_custom_target(clean-doc
+      COMMAND ${CMAKE_COMMAND} -E remove -f ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.html
+      COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation
+      COMMENT "Removing API documentation"
+      VERBATIM)

+  else (UNIX)
+    add_custom_target(doc
+      COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.conf
+      BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation/html/index.html
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+      COMMENT "Generating API documentation with Doxygen to CodeDocumentation/html/index.html"
+      VERBATIM)

+    add_custom_target(clean-doc
+      COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation
+      COMMENT "Removing API documentation"
+      VERBATIM)
+  endif (UNIX)
 endif (DOXYGEN_FOUND)
@@ -51,7 +51,7 @@ PROJECT_BRIEF          = "Finite element discretization library"
 # pixels and the maximum width should not exceed 200 pixels. Doxygen will copy
 # the logo to the output directory.

-PROJECT_LOGO           = web/logo-small.png
+PROJECT_LOGO           =

 # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path
 # into which the generated documentation will be written. If a relative path is
@@ -746,7 +746,7 @@ WARN_FORMAT            = "$file:$line: $text"
 # messages should be written. If left blank the output is written to standard
 # error (stderr).

-WARN_LOGFILE           = warnings.log
+WARN_LOGFILE           =

 #---------------------------------------------------------------------------
 # Configuration options related to the input files
@@ -770,12 +770,10 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/examples/pumi \
                         @MFEM_SOURCE_DIR@/examples/hiop \
                         @MFEM_SOURCE_DIR@/examples/sundials \
-                         @MFEM_SOURCE_DIR@/miniapps/adjoint \			 
                         @MFEM_SOURCE_DIR@/miniapps/common \
                         @MFEM_SOURCE_DIR@/miniapps/electromagnetics \
                         @MFEM_SOURCE_DIR@/miniapps/gslib \
                         @MFEM_SOURCE_DIR@/miniapps/meshing \
-                         @MFEM_SOURCE_DIR@/miniapps/navier \
                         @MFEM_SOURCE_DIR@/miniapps/nurbs \
                         @MFEM_SOURCE_DIR@/miniapps/performance \
                         @MFEM_SOURCE_DIR@/miniapps/tools \
@@ -1471,7 +1469,7 @@ MATHJAX_FORMAT         = HTML-CSS
 # The default value is: http://cdn.mathjax.org/mathjax/latest.
 # This tag requires that the tag USE_MATHJAX is set to YES.

-MATHJAX_RELPATH        = http://cdn.mathjax.org/mathjax/latest
+MATHJAX_RELPATH        = https://cdn.llnl.gov/mathjax/2.7.2

 # The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax
 # extension names that should be enabled during MathJax rendering. For example
@@ -88,8 +88,6 @@ namespace mfem {
 * - <a class="el" href="ex24p_8cpp_source.html">Example 24p</a>: parallel mixed finite element spaces and interpolators
 * - <a class="el" href="ex25_8cpp_source.html">Example 25</a>: simulation of electromagnetic wave propagation using a Perfectly Matched Layer (PML)
 * - <a class="el" href="ex25p_8cpp_source.html">Example 25p</a>: parallel simulation of electromagnetic wave propagation using a Perfectly Matched Layer (PML)
- * - <a class="el" href="ex26_8cpp_source.html">Example 26</a>: multigrid preconditioner for the Laplace problem using nodal H1 FEM
- * - <a class="el" href="ex26p_8cpp_source.html">Example 26p</a>: parallel multigrid preconditioner for the Laplace problem using nodal H1 FEM
 *
 * <H4>SUNDIALS Examples</H4>
 * - Variants of Examples
@@ -101,9 +99,6 @@ namespace mfem {
 *   and
 *   <a class="el" href="sundials_2ex16p_8cpp_source.html">16p</a>
 *   demonstrating the use of MFEM's \link sundials.hpp SUNDIALS classes\endlink
- * - CVODES adjoint examples:
- *   <a class="el" href="cvsRoberts__ASAi__dns_8cpp_source.html">serial ODE system</a>,
- *   <a class="el" href="adjoint__advection__diffusion_8cpp_source.html">parallel advection-diffusion</a>
 *
 * <H4>PETSc Examples</H4>
 * - Variants of Examples
@@ -143,18 +138,14 @@ namespace mfem {
 * - <a class="el" href="tesla_8cpp_source.html">Tesla</a>: simple magnetostatics simulation code
 * - <a class="el" href="maxwell_8cpp_source.html">Maxwell</a>: simple transient full-wave electromagnetics simulation code
 * - <a class="el" href="joule_8cpp_source.html">Joule</a>: transient magnetics and Joule heating miniapp
- * - <a class="el" href="classmfem_1_1navier_1_1NavierSolver.html">Navier</a>: solve the transient incompressible Navier-Stokes equations
-
-* - <a class="el" href="mobius-strip_8cpp_source.html">Mobius Strip</a>: generate various Mobius strip-like meshes
+ * - <a class="el" href="mobius-strip_8cpp_source.html">Mobius Strip</a>: generate various Mobius strip-like meshes
 * - <a class="el" href="klein-bottle_8cpp_source.html">Klein Bottle</a>: generate three types of Klein bottle surfaces
 * - <a class="el" href="toroid_8cpp_source.html">Toroid</a>: generate simple toroidal meshes
 * - <a class="el" href="twist_8cpp_source.html">Twist</a>: generate simple periodic meshes
- * - <a class="el" href="minimal-surface_8cpp_source.html">Minimal Surface</a>: compute minimal surfaces, <a class="el" href="minimal-surface_8cpp_source.html">serial</a> and <a class="el" href="pminimal-surface_8cpp_source.html">parallel</a> versions
 * - <a class="el" href="shaper_8cpp_source.html">Shaper</a>: resolve material interfaces by mesh refinement
 * - <a class="el" href="extruder_8cpp_source.html">Extruder</a>: extrude a low-dimensional mesh into a higher dimension
 * - <a class="el" href="mesh-explorer_8cpp_source.html">Mesh Explorer</a>: visualize and manipulate meshes
 * - <a class="el" href="mesh-optimizer_8cpp_source.html">Mesh Optimizer</a>: optimize high-order meshes, <a class="el" href="mesh-optimizer_8cpp_source.html">serial</a> and <a class="el" href="pmesh-optimizer_8cpp_source.html">parallel</a> versions
- * - <a class="el" href="trimmer_8cpp_source.html">Trimmer</a>: trim elements from existing meshes
 * - <a class="el" href="display-basis_8cpp_source.html">Display Basis</a>: visualize finite element basis functions
 * - <a class="el" href="get-values_8cpp_source.html">Get Values</a>: extract field values via DataCollection classes
 * - <a class="el" href="load-dc_8cpp_source.html">Load DC</a>: visualize fields saved via DataCollection classes
@@ -9,25 +9,18 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

-SHELL = /bin/bash
 MFEM_DIR ?= ..
 DOXYGEN_CONF = CodeDocumentation.conf

-
 # doxygen uses: graphviz, latex
 html: $(DOXYGEN_CONF)
-	@# Generate the html documentation
-	@doxygen $(DOXYGEN_CONF)
-	@echo "<meta http-equiv=\"REFRESH\" content=\"0;URL=CodeDocumentation/html/index.html\">" > CodeDocumentation.html
-	@cat warnings.log
-	@# Generate the log of undocumented methods
-	@( cat $(DOXYGEN_CONF) ; echo "GENERATE_HTML=NO" ; echo "EXTRACT_ALL=NO" ; echo "WARN_LOGFILE=undoc.log" ; echo "QUIET=YES" ) | doxygen - &> /dev/null
+	doxygen $(DOXYGEN_CONF)
+	rm -f CodeDocumentation.html
+	ln -s CodeDocumentation/html/index.html CodeDocumentation.html

 clean:
 	rm -rf $(DOXYGEN_CONF) CodeDocumentation CodeDocumentation.html *~
-	rm -rf  undoc.log warnings.log

 $(DOXYGEN_CONF): $(MFEM_DIR)/doc/$(DOXYGEN_CONF).in
-	@sed -e 's%@MFEM_SOURCE_DIR@%$(MFEM_DIR)%g' $(<) \
+	sed -e 's%@MFEM_SOURCE_DIR@%$(MFEM_DIR)%g' $(<) \
 	  > $(DOXYGEN_CONF)
-
@@ -32,8 +32,6 @@ list(APPEND ALL_EXE_SRCS
  ex23.cpp
  ex24.cpp
  ex25.cpp
-  ex26.cpp
-  ex27.cpp
  )

 if (MFEM_USE_MPI)
@@ -62,8 +60,6 @@ if (MFEM_USE_MPI)
    ex22p.cpp
    ex24p.cpp
    ex25p.cpp
-    ex26p.cpp
-    ex27p.cpp
    )
 endif()

@@ -83,15 +79,13 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
    list(APPEND THIS_TEST_OPTIONS "-tf" "5")
  elseif(${TEST_NAME} MATCHES "ex15p*")
    list(APPEND THIS_TEST_OPTIONS "-e" "1")
-  elseif(${TEST_NAME} MATCHES "ex27p*")
-    list(APPEND THIS_TEST_OPTIONS "-dg")
  endif()

  if (NOT (${TEST_NAME} MATCHES ".*p$"))
    add_test(NAME ${TEST_NAME}_ser
      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
  else()
-    add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
+    add_test(NAME ${TEST_NAME}_np=4
      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
      ${MPIEXEC_PREFLAGS}
      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
@@ -101,22 +95,13 @@ endforeach()

 # If STRUMPACK is enabled, add a test run that uses it.
 if (MFEM_USE_STRUMPACK)
-  add_test(NAME ex11p_strumpack_np=${MFEM_MPI_NP}
+  add_test(NAME ex11p_strumpack_np=4
    COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
    ${MPIEXEC_PREFLAGS}
    $<TARGET_FILE:ex11p> "-no-vis" "--strumpack"
    ${MPIEXEC_POSTFLAGS})
 endif()

-# If SuperLU_DIST is enabled, add a test run that uses it.
-if (MFEM_USE_SUPERLU)
-  add_test(NAME ex11p_superlu_np=${MFEM_MPI_NP}
-    COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-    ${MPIEXEC_PREFLAGS}
-    $<TARGET_FILE:ex11p> "-no-vis" "--superlu"
-    ${MPIEXEC_POSTFLAGS})
-endif()
-
 # Include the examples/sundials directory if SUNDIALS is enabled.
 if (MFEM_USE_SUNDIALS)
  add_subdirectory(sundials)
@@ -9,8 +9,6 @@
 //               ex1 -m ../data/fichera.mesh
 //               ex1 -m ../data/fichera-mixed.mesh
 //               ex1 -m ../data/toroid-wedge.mesh
-//               ex1 -m ../data/periodic-annulus-sector.msh
-//               ex1 -m ../data/periodic-torus-sector.msh
 //               ex1 -m ../data/square-disc-p2.vtk -o 2
 //               ex1 -m ../data/square-disc-p3.mesh -o 3
 //               ex1 -m ../data/square-disc-nurbs.mesh -o -1
@@ -34,8 +32,7 @@
 //               ex1 -pa -d raja-omp
 //               ex1 -pa -d occa-omp
 //               ex1 -pa -d ceed-cpu
-//             * ex1 -pa -d ceed-cuda
-//               ex1 -pa -d ceed-cuda:/gpu/cuda/shared
+//               ex1 -pa -d ceed-cuda
 //               ex1 -m ../data/beam-hex.mesh -pa -d cuda
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cuda:/gpu/cuda/ref
@@ -103,8 +100,8 @@ int main(int argc, char *argv[])
   // 3. Read the mesh from the given mesh file. We can handle triangular,
   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
   //    the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();

   // 4. Refine the mesh to increase the resolution. In this example we do
   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
@@ -112,10 +109,10 @@ int main(int argc, char *argv[])
   //    elements.
   {
      int ref_levels =
-         (int)floor(log(50000./mesh.GetNE())/log(2.)/dim);
+         (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
      for (int l = 0; l < ref_levels; l++)
      {
-         mesh.UniformRefinement();
+         mesh->UniformRefinement();
      }
   }

@@ -123,70 +120,66 @@ int main(int argc, char *argv[])
   //    Lagrange finite elements of the specified order. If order < 1, we
   //    instead use an isoparametric/isogeometric space.
   FiniteElementCollection *fec;
-   bool delete_fec;
   if (order > 0)
   {
      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
   }
-   else if (mesh.GetNodes())
+   else if (mesh->GetNodes())
   {
-      fec = mesh.GetNodes()->OwnFEC();
-      delete_fec = false;
+      fec = mesh->GetNodes()->OwnFEC();
      cout << "Using isoparametric FEs: " << fec->Name() << endl;
   }
   else
   {
      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
   }
-   FiniteElementSpace fespace(&mesh, fec);
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
   cout << "Number of finite element unknowns: "
-        << fespace.GetTrueVSize() << endl;
+        << fespace->GetTrueVSize() << endl;

   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
   //    In this example, the boundary conditions are defined by marking all
   //    the boundary attributes from the mesh as essential (Dirichlet) and
   //    converting them to a list of true dofs.
   Array<int> ess_tdof_list;
-   if (mesh.bdr_attributes.Size())
+   if (mesh->bdr_attributes.Size())
   {
-      Array<int> ess_bdr(mesh.bdr_attributes.Max());
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
   //    the basis functions in the finite element fespace.
-   LinearForm b(&fespace);
+   LinearForm *b = new LinearForm(fespace);
   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->Assemble();

   // 8. Define the solution vector x as a finite element grid function
   //    corresponding to fespace. Initialize x with initial guess of zero,
   //    which satisfies the boundary conditions.
-   GridFunction x(&fespace);
+   GridFunction x(fespace);
   x = 0.0;

   // 9. Set up the bilinear form a(.,.) on the finite element space
   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
   //    domain integrator.
-   BilinearForm a(&fespace);
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   BilinearForm *a = new BilinearForm(fespace);
+   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));

   // 10. Assemble the bilinear form and the corresponding linear system,
   //     applying any necessary transformations such as: eliminating boundary
   //     conditions, applying conforming constraints for non-conforming AMR,
   //     static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
-   a.Assemble();
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();

   OperatorPtr A;
   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);

   cout << "Size of linear system: " << A->Height() << endl;

@@ -207,9 +200,9 @@ int main(int argc, char *argv[])
   }
   else // Jacobi preconditioning in partial assembly mode
   {
-      if (UsesTensorBasis(fespace))
+      if (UsesTensorBasis(*fespace))
      {
-         OperatorJacobiSmoother M(a, ess_tdof_list);
+         OperatorJacobiSmoother M(*a, ess_tdof_list);
         PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
      }
      else
@@ -219,13 +212,13 @@ int main(int argc, char *argv[])
   }

   // 12. Recover the solution as a finite element grid function.
-   a.RecoverFEMSolution(X, b, x);
+   a->RecoverFEMSolution(X, *b, x);

   // 13. Save the refined mesh and the solution. This output can be viewed later
   //     using GLVis: "glvis -m refined.mesh -g sol.gf".
   ofstream mesh_ofs("refined.mesh");
   mesh_ofs.precision(8);
-   mesh.Print(mesh_ofs);
+   mesh->Print(mesh_ofs);
   ofstream sol_ofs("sol.gf");
   sol_ofs.precision(8);
   x.Save(sol_ofs);
@@ -237,14 +230,15 @@ int main(int argc, char *argv[])
      int  visport   = 19916;
      socketstream sol_sock(vishost, visport);
      sol_sock.precision(8);
-      sol_sock << "solution\n" << mesh << x << flush;
+      sol_sock << "solution\n" << *mesh << x << flush;
   }

   // 15. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete mesh;

   return 0;
 }
@@ -8,8 +8,6 @@
 //               mpirun -np 4 ex11p -m ../data/escher.mesh
 //               mpirun -np 4 ex11p -m ../data/fichera.mesh
 //               mpirun -np 4 ex11p -m ../data/fichera-mixed.mesh
-//               mpirun -np 4 ex11p -m ../data/periodic-annulus-sector.msh
-//               mpirun -np 4 ex11p -m ../data/periodic-torus-sector.msh -rs 1
 //               mpirun -np 4 ex11p -m ../data/toroid-wedge.mesh -o 2
 //               mpirun -np 4 ex11p -m ../data/square-disc-p2.vtk -o 2
 //               mpirun -np 4 ex11p -m ../data/square-disc-p3.mesh -o 3
@@ -33,8 +33,7 @@
 //               The example highlights the use of the LOBPCG eigenvalue solver
 //               together with the BoomerAMG preconditioner in HYPRE. Reusing a
 //               single GLVis visualization window for multiple eigenfunctions
-//               and optional saving with ADIOS2 (adios2.readthedocs.io) streams
-//               are also illustrated.
+//               is also illustrated.
 //
 //               We recommend viewing examples 2 and 11 before viewing this
 //               example.
@@ -61,7 +60,6 @@ int main(int argc, char *argv[])
   int seed = 66;
   bool visualization = 1;
   bool amg_elast = 0;
-   bool adios2 = false;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -79,9 +77,6 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
-   args.AddOption(&adios2, "-adios2", "--adios2-streams", "-no-adios2",
-                  "--no-adios2-streams",
-                  "Save data using adios2 streams.");
   args.Parse();
   if (!args.Good())
   {
@@ -291,28 +286,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 13. Optionally output a BP (binary pack) file using ADIOS2. This can be
-   //     visualized with the ParaView VTX reader.
-#ifdef MFEM_USE_ADIOS2
-   if (adios2)
-   {
-      std::string postfix(mesh_file);
-      postfix.erase(0, std::string("../data/").size() );
-      postfix += "_o" + std::to_string(order);
-
-      adios2stream adios2output("ex12-p-" + postfix + ".bp",
-                                adios2stream::openmode::out, MPI_COMM_WORLD);
-      pmesh->Print(adios2output);
-      for (int i=0; i<nev; i++)
-      {
-         x = lobpcg->GetEigenvector(i);
-         // x is a temporary that must be saved immediately
-         x.Save(adios2output, "mode_" + std::to_string(i));
-      }
-   }
-#endif
-
-   // 14. Send the above data by socket to a GLVis server. Use the "n" and "b"
+   // 13. Send the above data by socket to a GLVis server. Use the "n" and "b"
   //     keys in GLVis to visualize the displacements.
   if (visualization)
   {
@@ -352,7 +326,7 @@ int main(int argc, char *argv[])
      mode_sock.close();
   }

-   // 15. Free the used memory.
+   // 14. Free the used memory.
   delete lobpcg;
   delete amg;
   delete M;
@@ -35,38 +35,6 @@
 using namespace std;
 using namespace mfem;

-class CustomSolverMonitor : public IterativeSolverMonitor
-{
-public:
-   CustomSolverMonitor(const ParMesh *m,
-                       ParGridFunction *f) :
-      pmesh(m),
-      pgf(f) {}
-
-   void MonitorSolution(int i, double norm, const Vector &x, bool final)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      int  num_procs, myid;
-
-      MPI_Comm_size(pmesh->GetComm(),&num_procs);
-      MPI_Comm_rank(pmesh->GetComm(),&myid);
-
-      pgf->SetFromTrueDofs(x);
-
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << *pgf
-               << "window_title 'Iteration no " << i << "'"
-               << "keys rRjlc\n" << flush;
-   }
-
-private:
-   const ParMesh *pmesh;
-   ParGridFunction *pgf;
-};
-
 int main(int argc, char *argv[])
 {
   // 1. Initialize MPI.
@@ -220,7 +188,6 @@ int main(int argc, char *argv[])
   }
   else
   {
-      CustomSolverMonitor monitor(pmesh, &x);
      GMRESSolver gmres(MPI_COMM_WORLD);
      gmres.SetAbsTol(0.0);
      gmres.SetRelTol(1e-12);
@@ -229,7 +196,6 @@ int main(int argc, char *argv[])
      gmres.SetPrintLevel(1);
      gmres.SetOperator(*A);
      gmres.SetPreconditioner(*amg);
-      gmres.SetMonitor(monitor);
      gmres.Mult(*B, *X);
   }
   delete amg;
@@ -24,8 +24,7 @@
 //               class ConductionOperator defining C(u)), as well as their
 //               implicit time integration. Note that implementing the method
 //               ConductionOperator::ImplicitSolve is the only requirement for
-//               high-order implicit (SDIRK) time integration. Optional saving
-//               with ADIOS2 (adios2.readthedocs.io) is also illustrated.
+//               high-order implicit (SDIRK) time integration.
 //
 //               We recommend viewing examples 2, 9 and 10 before viewing this
 //               example.
@@ -109,7 +108,6 @@ int main(int argc, char *argv[])
   bool visualization = true;
   bool visit = false;
   int vis_steps = 5;
-   bool adios2 = false;

   int precision = 8;
   cout.precision(precision);
@@ -142,9 +140,6 @@ int main(int argc, char *argv[])
                  "Save data files for VisIt (visit.llnl.gov) visualization.");
   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
                  "Visualize every n-th timestep.");
-   args.AddOption(&adios2, "-adios2", "--adios2-streams", "-no-adios2",
-                  "--no-adios2-streams",
-                  "Save data using adios2 streams.");
   args.Parse();
   if (!args.Good())
   {
@@ -253,27 +248,6 @@ int main(int argc, char *argv[])
      visit_dc.Save();
   }

-   // Optionally output a BP (binary pack) file using ADIOS2. This can be
-   // visualized with the ParaView VTX reader.
-#ifdef MFEM_USE_ADIOS2
-   ADIOS2DataCollection* adios2_dc = NULL;
-   if (adios2)
-   {
-      std::string postfix(mesh_file);
-      postfix.erase(0, std::string("../data/").size() );
-      postfix += "_o" + std::to_string(order);
-      postfix += "_solver" + std::to_string(ode_solver_type);
-      const std::string collection_name = "ex16-p-" + postfix + ".bp";
-
-      adios2_dc = new ADIOS2DataCollection(MPI_COMM_WORLD, collection_name, pmesh);
-      adios2_dc->SetParameter("SubStreams", std::to_string(num_procs/2) );
-      adios2_dc->RegisterField("temperature", &u_gf);
-      adios2_dc->SetCycle(0);
-      adios2_dc->SetTime(0.0);
-      adios2_dc->Save();
-   }
-#endif
-
   socketstream sout;
   if (visualization)
   {
@@ -343,26 +317,10 @@ int main(int argc, char *argv[])
            visit_dc.SetTime(t);
            visit_dc.Save();
         }
-
-#ifdef MFEM_USE_ADIOS2
-         if (adios2)
-         {
-            adios2_dc->SetCycle(ti);
-            adios2_dc->SetTime(t);
-            adios2_dc->Save();
-         }
-#endif
      }
      oper.SetParameters(u);
   }

-#ifdef MFEM_USE_ADIOS2
-   if (adios2)
-   {
-      delete adios2_dc;
-   }
-#endif
-
   // 11. Save the final solution in parallel. This output can be viewed later
   //     using GLVis: "glvis -np <np> -m ex16-mesh -g ex16-final".
   {
@@ -88,6 +88,8 @@ private:
   Vector funval2;
   Vector nor;
   Vector fluxN;
+   IntegrationPoint eip1;
+   IntegrationPoint eip2;

 public:
   FaceIntegrator(RiemannSolver &rsolver_, const int dim);
@@ -416,24 +418,27 @@ void FaceIntegrator::AssembleFaceVector(const FiniteElement &el1,
   {
      intorder++;
   }
-   const IntegrationRule *ir = &IntRules.Get(Tr.GetGeometryType(), intorder);
+   const IntegrationRule *ir = &IntRules.Get(Tr.FaceGeom, intorder);

   for (int i = 0; i < ir->GetNPoints(); i++)
   {
      const IntegrationPoint &ip = ir->IntPoint(i);

-      Tr.SetAllIntPoints(&ip); // set face and element int. points
+      Tr.Loc1.Transform(ip, eip1);
+      Tr.Loc2.Transform(ip, eip2);

      // Calculate basis functions on both elements at the face
-      el1.CalcShape(Tr.GetElement1IntPoint(), shape1);
-      el2.CalcShape(Tr.GetElement2IntPoint(), shape2);
+      el1.CalcShape(eip1, shape1);
+      el2.CalcShape(eip2, shape2);

      // Interpolate elfun at the point
      elfun1_mat.MultTranspose(shape1, funval1);
      elfun2_mat.MultTranspose(shape2, funval2);

+      Tr.Face->SetIntPoint(&ip);
+
      // Get the normal vector and the flux on the face
-      CalcOrtho(Tr.Jacobian(), nor);
+      CalcOrtho(Tr.Face->Jacobian(), nor);
      const double mcs = rsolver.Eval(funval1, funval2, nor, fluxN);

      // Update max char speed
@@ -38,42 +38,6 @@
 using namespace std;
 using namespace mfem;

-class GeneralResidualMonitor : public IterativeSolverMonitor
-{
-public:
-   GeneralResidualMonitor(const std::string& prefix_, int print_lvl)
-      : prefix(prefix_)
-   {
-      print_level = print_lvl;
-   }
-
-   virtual void MonitorResidual(int it, double norm, const Vector &r, bool final);
-
-private:
-   const std::string prefix;
-   int print_level;
-   mutable double norm0;
-};
-
-void GeneralResidualMonitor::MonitorResidual(int it, double norm,
-                                             const Vector &r, bool final)
-{
-   if (print_level == 1 || (print_level == 3 && (final || it == 0)))
-   {
-      mfem::out << prefix << " iteration " << setw(2) << it
-                << " : ||r|| = " << norm;
-      if (it > 0)
-      {
-         mfem::out << ",  ||r||/||r_0|| = " << norm/norm0;
-      }
-      else
-      {
-         norm0 = norm;
-      }
-      mfem::out << '\n';
-   }
-}
-
 // Custom block preconditioner for the Jacobian of the incompressible nonlinear
 // elasticity operator. It has the form
 //
@@ -139,11 +103,9 @@ protected:

   // Newton solver for the hyperelastic operator
   NewtonSolver newton_solver;
-   GeneralResidualMonitor newton_monitor;

   // Solver for the Jacobian solve in the Newton method
   Solver *j_solver;
-   GeneralResidualMonitor j_monitor;

   // Preconditioner for the Jacobian
   Solver *j_prec;
@@ -448,8 +410,7 @@ RubberOperator::RubberOperator(Array<FiniteElementSpace *> &fes,
                               int iter,
                               Coefficient &c_mu)
   : Operator(fes[0]->GetVSize() + fes[1]->GetVSize()),
-     newton_solver(), newton_monitor("Newton", 1),
-     j_monitor("  GMRES", 3), mu(c_mu), block_offsets(offsets)
+     newton_solver(), mu(c_mu), block_offsets(offsets)
 {
   Array<Vector *> rhs(2);
   rhs = NULL; // Set all entries in the array
@@ -485,8 +446,7 @@ RubberOperator::RubberOperator(Array<FiniteElementSpace *> &fes,
   j_gmres->SetRelTol(1e-12);
   j_gmres->SetAbsTol(1e-12);
   j_gmres->SetMaxIter(300);
-   j_gmres->SetPrintLevel(-1);
-   j_gmres->SetMonitor(j_monitor);
+   j_gmres->SetPrintLevel(0);
   j_gmres->SetPreconditioner(*j_prec);
   j_solver = j_gmres;

@@ -494,8 +454,7 @@ RubberOperator::RubberOperator(Array<FiniteElementSpace *> &fes,
   newton_solver.iterative_mode = true;
   newton_solver.SetSolver(*j_solver);
   newton_solver.SetOperator(*this);
-   newton_solver.SetPrintLevel(-1);
-   newton_solver.SetMonitor(newton_monitor);
+   newton_solver.SetPrintLevel(1);
   newton_solver.SetRelTol(rel_tol);
   newton_solver.SetAbsTol(abs_tol);
   newton_solver.SetMaxIter(iter);
@@ -38,56 +38,6 @@
 using namespace std;
 using namespace mfem;

-class GeneralResidualMonitor : public IterativeSolverMonitor
-{
-public:
-   GeneralResidualMonitor(MPI_Comm comm, const std::string& prefix_,
-                          int print_lvl)
-      : prefix(prefix_)
-   {
-#ifndef MFEM_USE_MPI
-      print_level = print_lvl;
-#else
-      int rank;
-      MPI_Comm_rank(comm, &rank);
-      if (rank == 0)
-      {
-         print_level = print_lvl;
-      }
-      else
-      {
-         print_level = -1;
-      }
-#endif
-   }
-
-   virtual void MonitorResidual(int it, double norm, const Vector &r, bool final);
-
-private:
-   const std::string prefix;
-   int print_level;
-   mutable double norm0;
-};
-
-void GeneralResidualMonitor::MonitorResidual(int it, double norm,
-                                             const Vector &r, bool final)
-{
-   if (print_level == 1 || (print_level == 3 && (final || it == 0)))
-   {
-      mfem::out << prefix << " iteration " << setw(2) << it
-                << " : ||r|| = " << norm;
-      if (it > 0)
-      {
-         mfem::out << ",  ||r||/||r_0|| = " << norm/norm0;
-      }
-      else
-      {
-         norm0 = norm;
-      }
-      mfem::out << '\n';
-   }
-}
-
 // Custom block preconditioner for the Jacobian of the incompressible nonlinear
 // elasticity operator. It has the form
 //
@@ -153,11 +103,9 @@ protected:

   // Newton solver for the hyperelastic operator
   NewtonSolver newton_solver;
-   GeneralResidualMonitor newton_monitor;

   // Solver for the Jacobian solve in the Newton method
   Solver *j_solver;
-   GeneralResidualMonitor j_monitor;

   // Preconditioner for the Jacobian
   Solver *j_prec;
@@ -511,10 +459,7 @@ RubberOperator::RubberOperator(Array<ParFiniteElementSpace *> &fes,
                               int iter,
                               Coefficient &c_mu)
   : Operator(fes[0]->TrueVSize() + fes[1]->TrueVSize()),
-     newton_solver(fes[0]->GetComm()),
-     newton_monitor(fes[0]->GetComm(), "Newton", 1),
-     j_monitor(fes[0]->GetComm(), "  GMRES", 3),
-     mu(c_mu), block_trueOffsets(trueOffsets)
+     newton_solver(fes[0]->GetComm()), mu(c_mu), block_trueOffsets(trueOffsets)
 {
   Array<Vector *> rhs(2);
   rhs = NULL; // Set all entries in the array
@@ -554,8 +499,7 @@ RubberOperator::RubberOperator(Array<ParFiniteElementSpace *> &fes,
   j_gmres->SetRelTol(1e-12);
   j_gmres->SetAbsTol(1e-12);
   j_gmres->SetMaxIter(300);
-   j_gmres->SetPrintLevel(-1);
-   j_gmres->SetMonitor(j_monitor);
+   j_gmres->SetPrintLevel(0);
   j_gmres->SetPreconditioner(*j_prec);
   j_solver = j_gmres;

@@ -563,8 +507,7 @@ RubberOperator::RubberOperator(Array<ParFiniteElementSpace *> &fes,
   newton_solver.iterative_mode = true;
   newton_solver.SetSolver(*j_solver);
   newton_solver.SetOperator(*this);
-   newton_solver.SetPrintLevel(-1);
-   newton_solver.SetMonitor(newton_monitor);
+   newton_solver.SetPrintLevel(1);
   newton_solver.SetRelTol(rel_tol);
   newton_solver.SetAbsTol(abs_tol);
   newton_solver.SetMaxIter(iter);
@@ -9,8 +9,6 @@
 //               mpirun -np 4 ex1p -m ../data/fichera.mesh
 //               mpirun -np 4 ex1p -m ../data/fichera-mixed.mesh
 //               mpirun -np 4 ex1p -m ../data/toroid-wedge.mesh
-//               mpirun -np 4 ex1p -m ../data/periodic-annulus-sector.msh
-//               mpirun -np 4 ex1p -m ../data/periodic-torus-sector.msh
 //               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
 //               mpirun -np 4 ex1p -m ../data/square-disc-p3.mesh -o 3
 //               mpirun -np 4 ex1p -m ../data/square-disc-nurbs.mesh -o -1
@@ -32,8 +30,7 @@
 //               mpirun -np 4 ex1p -pa -d occa-cuda
 //               mpirun -np 4 ex1p -pa -d raja-omp
 //               mpirun -np 4 ex1p -pa -d ceed-cpu
-//             * mpirun -np 4 ex1p -pa -d ceed-cuda
-//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
+//               mpirun -np 4 ex1p -pa -d ceed-cuda
 //               mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //
 // Description:  This example code demonstrates the use of MFEM to define a
@@ -112,8 +109,8 @@ int main(int argc, char *argv[])
   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();

   // 5. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ref_levels' of uniform refinement. We choose
@@ -121,23 +118,23 @@ int main(int argc, char *argv[])
   //    more than 10,000 elements.
   {
      int ref_levels =
-         (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
+         (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
      for (int l = 0; l < ref_levels; l++)
      {
-         mesh.UniformRefinement();
+         mesh->UniformRefinement();
      }
   }

   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
   {
      int par_ref_levels = 2;
      for (int l = 0; l < par_ref_levels; l++)
      {
-         pmesh.UniformRefinement();
+         pmesh->UniformRefinement();
      }
   }

@@ -145,16 +142,13 @@ int main(int argc, char *argv[])
   //    use continuous Lagrange finite elements of the specified order. If
   //    order < 1, we instead use an isoparametric/isogeometric space.
   FiniteElementCollection *fec;
-   bool delete_fec;
   if (order > 0)
   {
      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
   }
-   else if (pmesh.GetNodes())
+   else if (pmesh->GetNodes())
   {
-      fec = pmesh.GetNodes()->OwnFEC();
-      delete_fec = false;
+      fec = pmesh->GetNodes()->OwnFEC();
      if (myid == 0)
      {
         cout << "Using isoparametric FEs: " << fec->Name() << endl;
@@ -163,10 +157,9 @@ int main(int argc, char *argv[])
   else
   {
      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
   }
-   ParFiniteElementSpace fespace(&pmesh, fec);
-   HYPRE_Int size = fespace.GlobalTrueVSize();
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
   if (myid == 0)
   {
      cout << "Number of finite element unknowns: " << size << endl;
@@ -177,44 +170,44 @@ int main(int argc, char *argv[])
   //    by marking all the boundary attributes from the mesh as essential
   //    (Dirichlet) and converting them to a list of true dofs.
   Array<int> ess_tdof_list;
-   if (pmesh.bdr_attributes.Size())
+   if (pmesh->bdr_attributes.Size())
   {
-      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

   // 9. Set up the parallel linear form b(.) which corresponds to the
   //    right-hand side of the FEM linear system, which in this case is
   //    (1,phi_i) where phi_i are the basis functions in fespace.
-   ParLinearForm b(&fespace);
+   ParLinearForm *b = new ParLinearForm(fespace);
   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->Assemble();

   // 10. Define the solution vector x as a parallel finite element grid function
   //     corresponding to fespace. Initialize x with initial guess of zero,
   //     which satisfies the boundary conditions.
-   ParGridFunction x(&fespace);
+   ParGridFunction x(fespace);
   x = 0.0;

   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
   //     domain integrator.
-   ParBilinearForm a(&fespace);
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));

   // 12. Assemble the parallel bilinear form and the corresponding linear
   //     system, applying any necessary transformations such as: parallel
   //     assembly, eliminating boundary conditions, applying conforming
   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
-   a.Assemble();
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();

   OperatorPtr A;
   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);

   // 13. Solve the linear system A X = B.
   //     * With full assembly, use the BoomerAMG preconditioner from hypre.
@@ -222,9 +215,9 @@ int main(int argc, char *argv[])
   Solver *prec = NULL;
   if (pa)
   {
-      if (UsesTensorBasis(fespace))
+      if (UsesTensorBasis(*fespace))
      {
-         prec = new OperatorJacobiSmoother(a, ess_tdof_list);
+         prec = new OperatorJacobiSmoother(*a, ess_tdof_list);
      }
   }
   else
@@ -242,7 +235,7 @@ int main(int argc, char *argv[])

   // 14. Recover the parallel grid function corresponding to X. This is the
   //     local finite element solution on each processor.
-   a.RecoverFEMSolution(X, b, x);
+   a->RecoverFEMSolution(X, *b, x);

   // 15. Save the refined mesh and the solution in parallel. This output can
   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
@@ -253,7 +246,7 @@ int main(int argc, char *argv[])

      ofstream mesh_ofs(mesh_name.str().c_str());
      mesh_ofs.precision(8);
-      pmesh.Print(mesh_ofs);
+      pmesh->Print(mesh_ofs);

      ofstream sol_ofs(sol_name.str().c_str());
      sol_ofs.precision(8);
@@ -268,14 +261,16 @@ int main(int argc, char *argv[])
      socketstream sol_sock(vishost, visport);
      sol_sock << "parallel " << num_procs << " " << myid << "\n";
      sol_sock.precision(8);
-      sol_sock << "solution\n" << pmesh << x << flush;
+      sol_sock << "solution\n" << *pmesh << x << flush;
   }

   // 17. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete pmesh;
+
   MPI_Finalize();

   return 0;
@@ -13,11 +13,6 @@
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 2
 //               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0
 //
-// Device sample runs:
-//               ex22 -m ../data/inline-quad.mesh -o 3 -p 1 -pa -d cuda
-//               ex22 -m ../data/inline-hex.mesh -o 2 -p 2 -pa -d cuda
-//               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0 -pa -d cuda
-//
 // Description:  This example code demonstrates the use of MFEM to define and
 //               solve simple complex-valued linear systems. It implements three
 //               variants of a damped harmonic oscillator:
@@ -81,8 +76,6 @@ int main(int argc, char *argv[])
   bool visualization = 1;
   bool herm_conv = true;
   bool exact_sol = true;
-   bool pa = false;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -113,10 +106,6 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -146,18 +135,13 @@ int main(int argc, char *argv[])
   ComplexOperator::Convention conv =
      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;

-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes
   //    with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 4. Refine the mesh to increase resolution. In this example we do
+   // 3. Refine the mesh to increase resolution. In this example we do
   //    'ref_levels' of uniform refinement where the user specifies
   //    the number of levels with the '-r' option.
   for (int l = 0; l < ref_levels; l++)
@@ -165,7 +149,7 @@ int main(int argc, char *argv[])
      mesh->UniformRefinement();
   }

-   // 5. Define a finite element space on the mesh. Here we use continuous
+   // 4. Define a finite element space on the mesh. Here we use continuous
   //    Lagrange, Nedelec, or Raviart-Thomas finite elements of the specified
   //    order.
   if (dim == 1 && prob != 0 )
@@ -187,7 +171,7 @@ int main(int argc, char *argv[])
   cout << "Number of finite element unknowns: " << fespace->GetTrueVSize()
        << endl;

-   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
+   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
   //    In this example, the boundary conditions are defined based on the type
   //    of mesh and the problem type.
   Array<int> ess_tdof_list;
@@ -199,12 +183,12 @@ int main(int argc, char *argv[])
      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

-   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
+   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
   //    the FEM linear system.
   ComplexLinearForm b(fespace, conv);
   b.Vector::operator=(0.0);

-   // 8. Define the solution vector u as a complex finite element grid function
+   // 7. Define the solution vector u as a complex finite element grid function
   //    corresponding to fespace. Initialize u with initial guess of 1+0i or
   //    the exact solution if it is known.
   ComplexGridFunction u(fespace);
@@ -226,6 +210,7 @@ int main(int argc, char *argv[])
   VectorConstantCoefficient zeroVecCoef(zeroVec);
   VectorConstantCoefficient oneVecCoef(oneVec);

+   u = 0.0;
   switch (prob)
   {
      case 0:
@@ -278,7 +263,7 @@ int main(int argc, char *argv[])
                 << "window_title 'Exact: Imaginary Part'" << flush;
   }

-   // 9. Set up the sesquilinear form a(.,.) on the finite element space
+   // 8. Set up the sesquilinear form a(.,.) on the finite element space
   //    corresponding to the damped harmonic oscillator operator of the
   //    appropriate type:
   //
@@ -297,7 +282,6 @@ int main(int argc, char *argv[])
   ConstantCoefficient negMassCoef(omega_ * omega_ * epsilon_);

   SesquilinearForm *a = new SesquilinearForm(fespace, conv);
-   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   switch (prob)
   {
      case 0:
@@ -321,7 +305,7 @@ int main(int argc, char *argv[])
      default: break; // This should be unreachable
   }

-   // 9a. Set up the bilinear form for the preconditioner corresponding to the
+   // 8a. Set up the bilinear form for the preconditioner corresponding to the
   //     appropriate operator
   //
   //      0) A scalar H1 field
@@ -334,8 +318,6 @@ int main(int argc, char *argv[])
   //         -Grad(a Div) - omega^2 b + omega c
   //
   BilinearForm *pcOp = new BilinearForm(fespace);
-   if (pa) { pcOp->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-
   switch (prob)
   {
      case 0:
@@ -356,9 +338,9 @@ int main(int argc, char *argv[])
      default: break; // This should be unreachable
   }

-   // 10. Assemble the form and the corresponding linear system, applying any
-   //     necessary transformations such as: assembly, eliminating boundary
-   //     conditions, conforming constraints for non-conforming AMR, etc.
+   // 9. Assemble the form and the corresponding linear system, applying any
+   //    necessary transformations such as: assembly, eliminating boundary
+   //    conditions, conforming constraints for non-conforming AMR, etc.
   a->Assemble();
   pcOp->Assemble();

@@ -366,17 +348,28 @@ int main(int argc, char *argv[])
   Vector B, U;

   a->FormLinearSystem(ess_tdof_list, u, b, A, U, B);
+   u = 0.0;
+   U = 0.0;

-   cout << "Size of linear system: " << A->Width() << endl << endl;
+   OperatorHandle PCOp;
+   pcOp->FormSystemMatrix(ess_tdof_list, PCOp);

-   // 11. Define and apply a GMRES solver for AU=B with a block diagonal
+   {
+      ComplexSparseMatrix * Asp =
+         dynamic_cast<ComplexSparseMatrix*>(A.Ptr());
+
+      cout << "Size of linear system: "
+           << 2 * Asp->real().Width() << endl << endl;
+   }
+
+   // 10. Define and apply a GMRES solver for AU=B with a block diagonal
   //     preconditioner based on the appropriate sparse smoother.
   {
      Array<int> blockOffsets;
      blockOffsets.SetSize(3);
      blockOffsets[0] = 0;
-      blockOffsets[1] = A->Height() / 2;
-      blockOffsets[2] = A->Height() / 2;
+      blockOffsets[1] = PCOp.Ptr()->Height();
+      blockOffsets[2] = PCOp.Ptr()->Height();
      blockOffsets.PartialSum();

      BlockDiagonalPreconditioner BDP(blockOffsets);
@@ -384,31 +377,22 @@ int main(int argc, char *argv[])
      Operator * pc_r = NULL;
      Operator * pc_i = NULL;

-      if (pa)
+      double s = 1.0;
+      switch (prob)
      {
-         pc_r = new OperatorJacobiSmoother(*pcOp, ess_tdof_list);
+         case 0:
+            pc_r = new DSmoother(*PCOp.As<SparseMatrix>());
+            break;
+         case 1:
+            pc_r = new GSSmoother(*PCOp.As<SparseMatrix>());
+            s = -1.0;
+            break;
+         case 2:
+            pc_r = new DSmoother(*PCOp.As<SparseMatrix>());
+            break;
+
+         default: break; // This should be unreachable
      }
-      else
-      {
-         OperatorHandle PCOp;
-         pcOp->SetDiagonalPolicy(mfem::Operator::DIAG_ONE);
-         pcOp->FormSystemMatrix(ess_tdof_list, PCOp);
-         switch (prob)
-         {
-            case 0:
-               pc_r = new DSmoother(*PCOp.As<SparseMatrix>());
-               break;
-            case 1:
-               pc_r = new GSSmoother(*PCOp.As<SparseMatrix>());
-               break;
-            case 2:
-               pc_r = new DSmoother(*PCOp.As<SparseMatrix>());
-               break;
-            default:
-               break; // This should be unreachable
-         }
-      }
-      double s = (prob != 1) ? 1.0 : -1.0;
      pc_i = new ScaledOperator(pc_r,
                                (conv == ComplexOperator::HERMITIAN) ?
                                s:-s);
@@ -426,11 +410,9 @@ int main(int argc, char *argv[])
      gmres.Mult(B, U);
   }

-   // 12. Recover the solution as a finite element grid function and compute the
+   // 11. Recover the solution as a finite element grid function and compute the
   //     errors if the exact solution is known.
   a->RecoverFEMSolution(U, b, u);
-   u.real().SyncMemory(u);
-   u.imag().SyncMemory(u);

   if (exact_sol)
   {
@@ -460,7 +442,7 @@ int main(int argc, char *argv[])
      cout << endl;
   }

-   // 13. Save the refined mesh and the solution. This output can be viewed
+   // 12. Save the refined mesh and the solution. This output can be viewed
   //     later using GLVis: "glvis -m mesh -g sol".
   {
      ofstream mesh_ofs("refined.mesh");
@@ -475,7 +457,7 @@ int main(int argc, char *argv[])
      u.imag().Save(sol_i_ofs);
   }

-   // 14. Send the solution by socket to a GLVis server.
+   // 13. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -534,7 +516,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 15. Free the used memory.
+   // 14. Free the used memory.
   delete a;
   delete u_exact;
   delete pcOp;
@@ -13,11 +13,6 @@
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 2
 //               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0
 //
-// Device sample runs:
-//               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 1 -p 1 -pa -d cuda
-//               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 1 -p 2 -pa -d cuda
-//               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0 -pa -d cuda
-//
 // Description:  This example code demonstrates the use of MFEM to define and
 //               solve simple complex-valued linear systems. It implements three
 //               variants of a damped harmonic oscillator:
@@ -46,6 +41,7 @@
 //               We recommend viewing examples 1, 3 and 4 before viewing this
 //               example.

+
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
@@ -88,8 +84,6 @@ int main(int argc, char *argv[])
   bool visualization = 1;
   bool herm_conv = true;
   bool exact_sol = true;
-   bool pa = false;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -122,10 +116,6 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -162,24 +152,19 @@ int main(int argc, char *argv[])
   ComplexOperator::Convention conv =
      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;

-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors. We
+   // 3. Read the (serial) mesh from the given mesh file on all processors. We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 5. Refine the serial mesh on all processors to increase the resolution.
+   // 4. Refine the serial mesh on all processors to increase the resolution.
   for (int l = 0; l < ser_ref_levels; l++)
   {
      mesh->UniformRefinement();
   }

-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -189,7 +174,7 @@ int main(int argc, char *argv[])
      pmesh->UniformRefinement();
   }

-   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use continuous Lagrange, Nedelec, or Raviart-Thomas finite elements of
   //    the specified order.
   if (dim == 1 && prob != 0 )
@@ -217,7 +202,7 @@ int main(int argc, char *argv[])
      cout << "Number of finite element unknowns: " << size << endl;
   }

-   // 8. Determine the list of true (i.e. parallel conforming) essential
+   // 7. Determine the list of true (i.e. parallel conforming) essential
   //    boundary dofs. In this example, the boundary conditions are defined
   //    based on the type of mesh and the problem type.
   Array<int> ess_tdof_list;
@@ -229,14 +214,14 @@ int main(int argc, char *argv[])
      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

-   // 9. Set up the parallel linear form b(.) which corresponds to the
+   // 8. Set up the parallel linear form b(.) which corresponds to the
   //    right-hand side of the FEM linear system.
   ParComplexLinearForm b(fespace, conv);
   b.Vector::operator=(0.0);

-   // 10. Define the solution vector u as a parallel complex finite element grid
-   //     function corresponding to fespace. Initialize u with initial guess of
-   //     1+0i or the exact solution if it is known.
+   // 9. Define the solution vector u as a parallel complex finite element grid
+   //    function corresponding to fespace. Initialize u with initial guess of
+   //    1+0i or the exact solution if it is known.
   ParComplexGridFunction u(fespace);
   ParComplexGridFunction * u_exact = NULL;
   if (exact_sol) { u_exact = new ParComplexGridFunction(fespace); }
@@ -256,6 +241,7 @@ int main(int argc, char *argv[])
   VectorConstantCoefficient zeroVecCoef(zeroVec);
   VectorConstantCoefficient oneVecCoef(oneVec);

+   u = 0.0;
   switch (prob)
   {
      case 0:
@@ -310,7 +296,7 @@ int main(int argc, char *argv[])
                 << "window_title 'Exact: Imaginary Part'" << flush;
   }

-   // 11. Set up the parallel sesquilinear form a(.,.) on the finite element
+   // 10. Set up the parallel sesquilinear form a(.,.) on the finite element
   //     space corresponding to the damped harmonic oscillator operator of the
   //     appropriate type:
   //
@@ -329,7 +315,6 @@ int main(int argc, char *argv[])
   ConstantCoefficient negMassCoef(omega_ * omega_ * epsilon_);

   ParSesquilinearForm *a = new ParSesquilinearForm(fespace, conv);
-   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   switch (prob)
   {
      case 0:
@@ -353,7 +338,7 @@ int main(int argc, char *argv[])
      default: break; // This should be unreachable
   }

-   // 11a. Set up the parallel bilinear form for the preconditioner
+   // 10a. Set up the parallel bilinear form for the preconditioner
   //      corresponding to the appropriate operator
   //
   //      0) A scalar H1 field
@@ -366,7 +351,6 @@ int main(int argc, char *argv[])
   //         -Grad(a Div) - omega^2 b + omega c
   //
   ParBilinearForm *pcOp = new ParBilinearForm(fespace);
-   if (pa) { pcOp->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   switch (prob)
   {
      case 0:
@@ -387,7 +371,7 @@ int main(int argc, char *argv[])
      default: break; // This should be unreachable
   }

-   // 12. Assemble the parallel bilinear form and the corresponding linear
+   // 11. Assemble the parallel bilinear form and the corresponding linear
   //     system, applying any necessary transformations such as: parallel
   //     assembly, eliminating boundary conditions, applying conforming
   //     constraints for non-conforming AMR, etc.
@@ -398,22 +382,30 @@ int main(int argc, char *argv[])
   Vector B, U;

   a->FormLinearSystem(ess_tdof_list, u, b, A, U, B);
+   u = 0.0;
+   U = 0.0;
+
+   OperatorHandle PCOp;
+   pcOp->FormSystemMatrix(ess_tdof_list, PCOp);

   if (myid == 0)
   {
+      ComplexHypreParMatrix * Ahyp =
+         dynamic_cast<ComplexHypreParMatrix*>(A.Ptr());
+
      cout << "Size of linear system: "
-           << 2 * fespace->GlobalTrueVSize() << endl << endl;
+           << 2 * Ahyp->real().GetGlobalNumRows() << endl << endl;
   }

-   // 13. Define and apply a parallel FGMRES solver for AU=B with a block
+   // 12. Define and apply a parallel FGMRES solver for AU=B with a block
   //     diagonal preconditioner based on the appropriate multigrid
   //     preconditioner from hypre.
   {
      Array<int> blockTrueOffsets;
      blockTrueOffsets.SetSize(3);
      blockTrueOffsets[0] = 0;
-      blockTrueOffsets[1] = A->Height() / 2;
-      blockTrueOffsets[2] = A->Height() / 2;
+      blockTrueOffsets[1] = PCOp.Ptr()->Height();
+      blockTrueOffsets[2] = PCOp.Ptr()->Height();
      blockTrueOffsets.PartialSum();

      BlockDiagonalPreconditioner BDP(blockTrueOffsets);
@@ -421,34 +413,25 @@ int main(int argc, char *argv[])
      Operator * pc_r = NULL;
      Operator * pc_i = NULL;

-      if (pa)
+      switch (prob)
      {
-         pc_r = new OperatorJacobiSmoother(*pcOp, ess_tdof_list);
-      }
-      else
-      {
-         OperatorHandle PCOp;
-         pcOp->FormSystemMatrix(ess_tdof_list, PCOp);
-         switch (prob)
-         {
-            case 0:
-               pc_r = new HypreBoomerAMG(*PCOp.As<HypreParMatrix>());
-               break;
-            case 1:
+         case 0:
+            pc_r = new HypreBoomerAMG(*PCOp.As<HypreParMatrix>());
+            break;
+         case 1:
+            pc_r = new HypreAMS(*PCOp.As<HypreParMatrix>(), fespace);
+            break;
+         case 2:
+            if (dim == 2 )
+            {
               pc_r = new HypreAMS(*PCOp.As<HypreParMatrix>(), fespace);
-               break;
-            case 2:
-               if (dim == 2 )
-               {
-                  pc_r = new HypreAMS(*PCOp.As<HypreParMatrix>(), fespace);
-               }
-               else
-               {
-                  pc_r = new HypreADS(*PCOp.As<HypreParMatrix>(), fespace);
-               }
-               break;
-            default: break; // This should be unreachable
-         }
+            }
+            else
+            {
+               pc_r = new HypreADS(*PCOp.As<HypreParMatrix>(), fespace);
+            }
+            break;
+         default: break; // This should be unreachable
      }
      pc_i = new ScaledOperator(pc_r,
                                (conv == ComplexOperator::HERMITIAN) ?
@@ -466,11 +449,9 @@ int main(int argc, char *argv[])
      fgmres.SetPrintLevel(1);
      fgmres.Mult(B, U);
   }
-   // 14. Recover the parallel grid function corresponding to U. This is the
+   // 13. Recover the parallel grid function corresponding to U. This is the
   //     local finite element solution on each processor.
   a->RecoverFEMSolution(U, b, u);
-   u.real().SyncMemory(u);
-   u.imag().SyncMemory(u);

   if (exact_sol)
   {
@@ -503,7 +484,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 15. Save the refined mesh and the solution in parallel. This output can be
+   // 14. Save the refined mesh and the solution in parallel. This output can be
   //     viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
   {
      ostringstream mesh_name, sol_r_name, sol_i_name;
@@ -523,7 +504,7 @@ int main(int argc, char *argv[])
      u.imag().Save(sol_i_ofs);
   }

-   // 16. Send the solution by socket to a GLVis server.
+   // 15. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -588,7 +569,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 17. Free the used memory.
+   // 16. Free the used memory.
   delete a;
   delete u_exact;
   delete pcOp;
@@ -6,8 +6,6 @@
 //               ex24 -m ../data/square-disc.mesh -o 2
 //               ex24 -m ../data/beam-tet.mesh
 //               ex24 -m ../data/beam-hex.mesh -o 2 -pa
-//               ex24 -m ../data/beam-hex.mesh -o 2 -pa -p 1
-//               ex24 -m ../data/beam-hex.mesh -o 2 -pa -p 2
 //               ex24 -m ../data/escher.mesh
 //               ex24 -m ../data/escher.mesh -o 2
 //               ex24 -m ../data/fichera.mesh
@@ -25,16 +23,11 @@
 //               ex24 -m ../data/beam-hex.mesh -pa -d cuda
 //
 // Description:  This example code illustrates usage of mixed finite element
-//               spaces, with three variants:
+//               spaces. Using two different approaches, we project a gradient
+//               of a function in H^1 to H(curl). Other spaces and example
+//               computations are to be added in the future.
 //
-//               1) (grad p, u) for p in H^1 tested against u in H(curl)
-//               2) (curl v, u) for v in H(curl) tested against u in H(div), 3D
-//               3) (div v, q) for v in H(div) tested against q in L_2
-//
-//               Using different approaches, we project the gradient, curl, or
-//               divergence to the appropriate space.
-//
-//               We recommend viewing examples 1, 3, and 5 before viewing this
+//               We recommend viewing examples 1 and 3 before viewing this
 //               example.

 #include "mfem.hpp"
@@ -46,19 +39,14 @@ using namespace mfem;

 double p_exact(const Vector &x);
 void gradp_exact(const Vector &, Vector &);
-double div_gradp_exact(const Vector &x);
-void v_exact(const Vector &x, Vector &v);
-void curlv_exact(const Vector &x, Vector &cv);

 int dim;
-double freq = 1.0, kappa;

 int main(int argc, char *argv[])
 {
   // 1. Parse command-line options.
   const char *mesh_file = "../data/beam-hex.mesh";
   int order = 1;
-   int prob = 0;
   bool static_cond = false;
   bool pa = false;
   const char *device_config = "cpu";
@@ -69,8 +57,6 @@ int main(int argc, char *argv[])
                  "Mesh file to use.");
   args.AddOption(&order, "-o", "--order",
                  "Finite element order (polynomial degree).");
-   args.AddOption(&prob, "-p", "--problem-type",
-                  "Choose between 0: grad, 1: curl, 2: div");
   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
@@ -88,7 +74,6 @@ int main(int argc, char *argv[])
      return 1;
   }
   args.PrintOptions(cout);
-   kappa = freq * M_PI;

   // 2. Enable hardware devices such as GPUs, and programming models such as
   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
@@ -115,129 +100,72 @@ int main(int argc, char *argv[])
   }
   mesh->ReorientTetMesh();

-   // 5. Define a finite element space on the mesh. Here we use Nedelec or
-   //    Raviart-Thomas finite elements of the specified order.
-   FiniteElementCollection *trial_fec = NULL;
-   FiniteElementCollection *test_fec = NULL;
+   // 5. Define a parallel finite element space on the parallel mesh. Here we
+   //    use the Nedelec finite elements of the specified order.
+   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   FiniteElementCollection *H1fec = new H1_FECollection(order, dim);
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+   FiniteElementSpace *H1fespace = new FiniteElementSpace(mesh, H1fec);

-   if (prob == 0)
-   {
-      trial_fec = new H1_FECollection(order, dim);
-      test_fec = new ND_FECollection(order, dim);
-   }
-   else if (prob == 1)
-   {
-      trial_fec = new ND_FECollection(order, dim);
-      test_fec = new RT_FECollection(order-1, dim);
-   }
-   else
-   {
-      trial_fec = new RT_FECollection(order-1, dim);
-      test_fec = new L2_FECollection(order-1, dim);
-   }
+   int size = fespace->GetTrueVSize();
+   int H1size = H1fespace->GetTrueVSize();
+   cout << "Number of Nedelec finite element unknowns: " << size << endl;
+   cout << "Number of H1 finite element unknowns: " << H1size << endl;

-   FiniteElementSpace trial_fes(mesh, trial_fec);
-   FiniteElementSpace test_fes(mesh, test_fec);
-
-   int trial_size = trial_fes.GetTrueVSize();
-   int test_size = test_fes.GetTrueVSize();
-
-   if (prob == 0)
-   {
-      cout << "Number of Nedelec finite element unknowns: " << test_size << endl;
-      cout << "Number of H1 finite element unknowns: " << trial_size << endl;
-   }
-   else if (prob == 1)
-   {
-      cout << "Number of Nedelec finite element unknowns: " << trial_size << endl;
-      cout << "Number of Raviart-Thomas finite element unknowns: " << test_size <<
-           endl;
-   }
-   else
-   {
-      cout << "Number of Raviart-Thomas finite element unknowns: "
-           << trial_size << endl;
-      cout << "Number of L2 finite element unknowns: " << test_size << endl;
-   }
-
-   // 6. Define the solution vector as a finite element grid function
-   //    corresponding to the trial fespace.
-   GridFunction gftest(&test_fes);
-   GridFunction gftrial(&trial_fes);
-   GridFunction x(&test_fes);
+   // 6. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x by projecting the exact
+   //    solution. Note that only values from the boundary edges will be used
+   //    when eliminating the non-homogeneous boundary condition to modify the
+   //    r.h.s. vector b.
+   GridFunction x(fespace);
   FunctionCoefficient p_coef(p_exact);
+   GridFunction p(H1fespace);
+   p.ProjectCoefficient(p_coef);
+   p.SetTrueVector();
+   p.SetFromTrueVector();
+
   VectorFunctionCoefficient gradp_coef(sdim, gradp_exact);
-   VectorFunctionCoefficient v_coef(sdim, v_exact);
-   VectorFunctionCoefficient curlv_coef(sdim, curlv_exact);
-   FunctionCoefficient divgradp_coef(div_gradp_exact);

-   if (prob == 0)
-   {
-      gftrial.ProjectCoefficient(p_coef);
-   }
-   else if (prob == 1)
-   {
-      gftrial.ProjectCoefficient(v_coef);
-   }
-   else
-   {
-      gftrial.ProjectCoefficient(gradp_coef);
-   }
-
-   gftrial.SetTrueVector();
-   gftrial.SetFromTrueVector();
-
-   // 7. Set up the bilinear forms for L2 projection.
-   ConstantCoefficient one(1.0);
-   BilinearForm a(&test_fes);
-   MixedBilinearForm a_mixed(&trial_fes, &test_fes);
+   // 7. Set up the bilinear forms.
+   Coefficient *muinv = new ConstantCoefficient(1.0);
+   Coefficient *sigma = new ConstantCoefficient(1.0);
+   BilinearForm *a = new BilinearForm(fespace);
+   MixedBilinearForm *a_NDH1 = new MixedBilinearForm(H1fespace, fespace);
   if (pa)
   {
-      a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      a_mixed.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+      a->SetAssemblyLevel(AssemblyLevel::PARTIAL);
+      a_NDH1->SetAssemblyLevel(AssemblyLevel::PARTIAL);
   }

-   if (prob == 0)
-   {
-      a.AddDomainIntegrator(new VectorFEMassIntegrator(one));
-      a_mixed.AddDomainIntegrator(new MixedVectorGradientIntegrator(one));
-   }
-   else if (prob == 1)
-   {
-      a.AddDomainIntegrator(new VectorFEMassIntegrator(one));
-      a_mixed.AddDomainIntegrator(new MixedVectorCurlIntegrator(one));
-   }
-   else
-   {
-      a.AddDomainIntegrator(new MassIntegrator(one));
-      a_mixed.AddDomainIntegrator(new VectorFEDivergenceIntegrator(one));
-   }
+   // First approach: L2 projection
+   a->AddDomainIntegrator(new VectorFEMassIntegrator(*sigma));
+   a_NDH1->AddDomainIntegrator(new MixedVectorGradientIntegrator(*muinv));

-   // 8. Assemble the bilinear form and the corresponding linear system,
-   //    applying any necessary transformations such as: eliminating boundary
-   //    conditions, applying conforming constraints for non-conforming AMR,
-   //    static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
+   // 8. Assemble the parallel bilinear form and the corresponding linear
+   //    system, applying any necessary transformations such as: parallel
+   //    assembly, eliminating boundary conditions, applying conforming
+   //    constraints for non-conforming AMR, static condensation, etc.
+   if (static_cond) { a->EnableStaticCondensation(); }

-   a.Assemble();
-   if (!pa) { a.Finalize(); }
+   a->Assemble();
+   if (!pa) { a->Finalize(); }

-   a_mixed.Assemble();
-   if (!pa) { a_mixed.Finalize(); }
+   a_NDH1->Assemble();
+   if (!pa) { a_NDH1->Finalize(); }

   if (pa)
   {
-      a_mixed.Mult(gftrial, x);
+      a_NDH1->Mult(p, x);
   }
   else
   {
-      SparseMatrix& mixed = a_mixed.SpMat();
-      mixed.Mult(gftrial, x);
+      SparseMatrix& NDH1 = a_NDH1->SpMat();
+      NDH1.Mult(p, x);
   }

   // 9. Define and apply a PCG solver for Ax = b with Jacobi preconditioner.
   {
-      GridFunction rhs(&test_fes);
+      GridFunction rhs(fespace);
      rhs = x;
      x = 0.0;

@@ -248,15 +176,15 @@ int main(int argc, char *argv[])
      if (pa)
      {
         Array<int> ess_tdof_list; // empty
-         OperatorJacobiSmoother Jacobi(a, ess_tdof_list);
+         OperatorJacobiSmoother Jacobi(*a, ess_tdof_list);

-         cg.SetOperator(a);
+         cg.SetOperator(*a);
         cg.SetPreconditioner(Jacobi);
         cg.Mult(rhs, x);
      }
      else
      {
-         SparseMatrix& Amat = a.SpMat();
+         SparseMatrix& Amat = a->SpMat();
         DSmoother Jacobi(Amat);

         cg.SetOperator(Amat);
@@ -265,89 +193,33 @@ int main(int argc, char *argv[])
      }
   }

-   // 10. Compute the same field by applying a DiscreteInterpolator.
-   GridFunction discreteInterpolant(&test_fes);
-   DiscreteLinearOperator dlo(&trial_fes, &test_fes);
-   if (prob == 0)
-   {
-      dlo.AddDomainInterpolator(new GradientInterpolator());
-   }
-   else if (prob == 1)
-   {
-      dlo.AddDomainInterpolator(new CurlInterpolator());
-   }
-   else
-   {
-      dlo.AddDomainInterpolator(new DivergenceInterpolator());
-   }
+   // 10. Second approach: compute the same solution by applying
+   //     GradientInterpolator in H(curl).
+   DiscreteLinearOperator grad(H1fespace, fespace);
+   grad.AddDomainInterpolator(new GradientInterpolator());
+   grad.Assemble();

-   dlo.Assemble();
-   dlo.Mult(gftrial, discreteInterpolant);
+   GridFunction gradp(fespace);
+   grad.Mult(p, gradp);

-   // 11. Compute the projection of the exact field.
-   GridFunction exact_proj(&test_fes);
-   if (prob == 0)
-   {
-      exact_proj.ProjectCoefficient(gradp_coef);
-   }
-   else if (prob == 1)
-   {
-      exact_proj.ProjectCoefficient(curlv_coef);
-   }
-   else
-   {
-      exact_proj.ProjectCoefficient(divgradp_coef);
-   }
+   // 11. Compute the projection of the exact grad p.
+   GridFunction exact_gradp(fespace);
+   exact_gradp.ProjectCoefficient(gradp_coef);
+   exact_gradp.SetTrueVector();
+   exact_gradp.SetFromTrueVector();

-   exact_proj.SetTrueVector();
-   exact_proj.SetFromTrueVector();
-
-   // 12. Compute and print the L_2 norm of the error.
-   if (prob == 0)
+   // 12. Compute and print the L^2 norm of the error.
   {
      double errSol = x.ComputeL2Error(gradp_coef);
-      double errInterp = discreteInterpolant.ComputeL2Error(gradp_coef);
-      double errProj = exact_proj.ComputeL2Error(gradp_coef);
+      double errInterp = gradp.ComputeL2Error(gradp_coef);
+      double errProj = exact_gradp.ComputeL2Error(gradp_coef);

      cout << "\n Solution of (E_h,v) = (grad p_h,v) for E_h and v in H(curl): "
-           "|| E_h - grad p ||_{L_2} = " << errSol << '\n' << endl;
+           "|| E_h - grad p ||_{L^2} = " << errSol << '\n' << endl;
      cout << " Gradient interpolant E_h = grad p_h in H(curl): || E_h - grad p"
-           " ||_{L_2} = " << errInterp << '\n' << endl;
+           "||_{L^2} = " << errInterp << '\n' << endl;
      cout << " Projection E_h of exact grad p in H(curl): || E_h - grad p "
-           "||_{L_2} = " << errProj << '\n' << endl;
-   }
-   else if (prob == 1)
-   {
-      double errSol = x.ComputeL2Error(curlv_coef);
-      double errInterp = discreteInterpolant.ComputeL2Error(curlv_coef);
-      double errProj = exact_proj.ComputeL2Error(curlv_coef);
-
-      cout << "\n Solution of (E_h,w) = (curl v_h,w) for E_h and w in H(div): "
-           "|| E_h - curl v ||_{L_2} = " << errSol << '\n' << endl;
-      cout << " Curl interpolant E_h = curl v_h in H(div): || E_h - curl v "
-           "||_{L_2} = " << errInterp << '\n' << endl;
-      cout << " Projection E_h of exact curl v in H(div): || E_h - curl v "
-           "||_{L_2} = " << errProj << '\n' << endl;
-   }
-   else
-   {
-      int order_quad = max(2, 2*order+1);
-      const IntegrationRule *irs[Geometry::NumGeom];
-      for (int i=0; i < Geometry::NumGeom; ++i)
-      {
-         irs[i] = &(IntRules.Get(i, order_quad));
-      }
-
-      double errSol = x.ComputeL2Error(divgradp_coef, irs);
-      double errInterp = discreteInterpolant.ComputeL2Error(divgradp_coef, irs);
-      double errProj = exact_proj.ComputeL2Error(divgradp_coef, irs);
-
-      cout << "\n Solution of (f_h,q) = (div v_h,q) for f_h and q in L_2: "
-           "|| f_h - div v ||_{L_2} = " << errSol << '\n' << endl;
-      cout << " Divergence interpolant f_h = div v_h in L_2: || f_h - div v "
-           "||_{L_2} = " << errInterp << '\n' << endl;
-      cout << " Projection f_h of exact div v in L_2: || f_h - div v "
-           "||_{L_2} = " << errProj << '\n' << endl;
+           "||_{L^2} = " << errProj << '\n' << endl;
   }

   // 13. Save the refined mesh and the solution. This output can be viewed
@@ -370,8 +242,14 @@ int main(int argc, char *argv[])
   }

   // 15. Free the used memory.
-   delete trial_fec;
-   delete test_fec;
+   delete a;
+   delete a_NDH1;
+   delete sigma;
+   delete muinv;
+   delete fespace;
+   delete H1fespace;
+   delete fec;
+   delete H1fec;
   delete mesh;

   return 0;
@@ -406,47 +284,3 @@ void gradp_exact(const Vector &x, Vector &f)
      if (x.Size() == 3) { f(2) = 0.0; }
   }
 }
-
-double div_gradp_exact(const Vector &x)
-{
-   if (dim == 3)
-   {
-      return -3.0 * sin(x(0)) * sin(x(1)) * sin(x(2));
-   }
-   else if (dim == 2)
-   {
-      return -2.0 * sin(x(0)) * sin(x(1));
-   }
-
-   return 0.0;
-}
-
-void v_exact(const Vector &x, Vector &v)
-{
-   if (dim == 3)
-   {
-      v(0) = sin(kappa * x(1));
-      v(1) = sin(kappa * x(2));
-      v(2) = sin(kappa * x(0));
-   }
-   else
-   {
-      v(0) = sin(kappa * x(1));
-      v(1) = sin(kappa * x(0));
-      if (x.Size() == 3) { v(2) = 0.0; }
-   }
-}
-
-void curlv_exact(const Vector &x, Vector &cv)
-{
-   if (dim == 3)
-   {
-      cv(0) = -kappa * cos(kappa * x(2));
-      cv(1) = -kappa * cos(kappa * x(0));
-      cv(2) = -kappa * cos(kappa * x(1));
-   }
-   else
-   {
-      cv = 0.0;
-   }
-}
@@ -6,8 +6,6 @@
 //               mpirun -np 4 ex24p -m ../data/square-disc.mesh -o 2
 //               mpirun -np 4 ex24p -m ../data/beam-tet.mesh
 //               mpirun -np 4 ex24p -m ../data/beam-hex.mesh -o 2 -pa
-//               mpirun -np 4 ex24p -m ../data/beam-hex.mesh -o 2 -pa -p 1
-//               mpirun -np 4 ex24p -m ../data/beam-hex.mesh -o 2 -pa -p 2
 //               mpirun -np 4 ex24p -m ../data/escher.mesh
 //               mpirun -np 4 ex24p -m ../data/escher.mesh -o 2
 //               mpirun -np 4 ex24p -m ../data/fichera.mesh
@@ -25,16 +23,11 @@
 //               mpirun -np 4 ex24p -m ../data/beam-hex.mesh -pa -d cuda
 //
 // Description:  This example code illustrates usage of mixed finite element
-//               spaces, with three variants:
+//               spaces. Using two different approaches, we project a gradient
+//               of a function in H^1 to H(curl). Other spaces and example
+//               computations are to be added in the future.
 //
-//               1) (grad p, u) for p in H^1 tested against u in H(curl)
-//               2) (curl v, u) for v in H(curl) tested against u in H(div), 3D
-//               3) (div v, q) for v in H(div) tested against q in L_2
-//
-//               Using different approaches, we project the gradient, curl, or
-//               divergence to the appropriate space.
-//
-//               We recommend viewing examples 1, 3, and 5 before viewing this
+//               We recommend viewing examples 1 and 3 before viewing this
 //               example.

 #include "mfem.hpp"
@@ -46,12 +39,8 @@ using namespace mfem;

 double p_exact(const Vector &x);
 void gradp_exact(const Vector &, Vector &);
-double div_gradp_exact(const Vector &x);
-void v_exact(const Vector &x, Vector &v);
-void curlv_exact(const Vector &x, Vector &cv);

 int dim;
-double freq = 1.0, kappa;

 int main(int argc, char *argv[])
 {
@@ -64,7 +53,6 @@ int main(int argc, char *argv[])
   // 2. Parse command-line options.
   const char *mesh_file = "../data/beam-hex.mesh";
   int order = 1;
-   int prob = 0;
   bool static_cond = false;
   bool pa = false;
   const char *device_config = "cpu";
@@ -75,8 +63,6 @@ int main(int argc, char *argv[])
                  "Mesh file to use.");
   args.AddOption(&order, "-o", "--order",
                  "Finite element order (polynomial degree).");
-   args.AddOption(&prob, "-p", "--problem-type",
-                  "Choose between 0: grad, 1: curl, 2: div");
   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
@@ -101,7 +87,6 @@ int main(int argc, char *argv[])
   {
      args.PrintOptions(cout);
   }
-   kappa = freq * M_PI;

   // 3. Enable hardware devices such as GPUs, and programming models such as
   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
@@ -144,137 +129,80 @@ int main(int argc, char *argv[])
   pmesh->ReorientTetMesh();

   // 7. Define a parallel finite element space on the parallel mesh. Here we
-   //    use Nedelec or Raviart-Thomas finite elements of the specified order.
-   FiniteElementCollection *trial_fec = NULL;
-   FiniteElementCollection *test_fec = NULL;
-
-   if (prob == 0)
-   {
-      trial_fec = new H1_FECollection(order, dim);
-      test_fec = new ND_FECollection(order, dim);
-   }
-   else if (prob == 1)
-   {
-      trial_fec = new ND_FECollection(order, dim);
-      test_fec = new RT_FECollection(order-1, dim);
-   }
-   else
-   {
-      trial_fec = new RT_FECollection(order-1, dim);
-      test_fec = new L2_FECollection(order-1, dim);
-   }
-
-   ParFiniteElementSpace trial_fes(pmesh, trial_fec);
-   ParFiniteElementSpace test_fes(pmesh, test_fec);
-
-   HYPRE_Int trial_size = trial_fes.GlobalTrueVSize();
-   HYPRE_Int test_size = test_fes.GlobalTrueVSize();
-
+   //    use the Nedelec finite elements of the specified order.
+   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   FiniteElementCollection *H1fec = new H1_FECollection(order, dim);
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   ParFiniteElementSpace *H1fespace = new ParFiniteElementSpace(pmesh, H1fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   HYPRE_Int H1size = H1fespace->GlobalTrueVSize();
   if (myid == 0)
   {
-      if (prob == 0)
-      {
-         cout << "Number of Nedelec finite element unknowns: " << test_size << endl;
-         cout << "Number of H1 finite element unknowns: " << trial_size << endl;
-      }
-      else if (prob == 1)
-      {
-         cout << "Number of Nedelec finite element unknowns: " << trial_size << endl;
-         cout << "Number of Raviart-Thomas finite element unknowns: " << test_size <<
-              endl;
-      }
-      else
-      {
-         cout << "Number of Raviart-Thomas finite element unknowns: "
-              << trial_size << endl;
-         cout << "Number of L2 finite element unknowns: " << test_size << endl;
-      }
+      cout << "Number of Nedelec finite element unknowns: " << size << endl;
+      cout << "Number of H1 finite element unknowns: " << H1size << endl;
   }

-   // 8. Define the solution vector as a parallel finite element grid function
-   //    corresponding to the trial fespace.
-   ParGridFunction gftest(&test_fes);
-   ParGridFunction gftrial(&trial_fes);
-   ParGridFunction x(&test_fes);
+   // 8. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x by projecting the exact
+   //    solution. Note that only values from the boundary edges will be used
+   //    when eliminating the non-homogeneous boundary condition to modify the
+   //    r.h.s. vector b.
+   ParGridFunction x(fespace);
   FunctionCoefficient p_coef(p_exact);
+   ParGridFunction p(H1fespace);
+   p.ProjectCoefficient(p_coef);
+   p.SetTrueVector();
+   p.SetFromTrueVector();
+
   VectorFunctionCoefficient gradp_coef(sdim, gradp_exact);
-   VectorFunctionCoefficient v_coef(sdim, v_exact);
-   VectorFunctionCoefficient curlv_coef(sdim, curlv_exact);
-   FunctionCoefficient divgradp_coef(div_gradp_exact);

-   if (prob == 0)
-   {
-      gftrial.ProjectCoefficient(p_coef);
-   }
-   else if (prob == 1)
-   {
-      gftrial.ProjectCoefficient(v_coef);
-   }
-   else
-   {
-      gftrial.ProjectCoefficient(gradp_coef);
-   }
-
-   gftrial.SetTrueVector();
-   gftrial.SetFromTrueVector();
-
-   // 9. Set up the parallel bilinear forms for L2 projection.
-   ConstantCoefficient one(1.0);
-   ParBilinearForm a(&test_fes);
-   ParMixedBilinearForm a_mixed(&trial_fes, &test_fes);
+   // 9. Set up the parallel bilinear forms.
+   Coefficient *muinv = new ConstantCoefficient(1.0);
+   Coefficient *sigma = new ConstantCoefficient(1.0);
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   ParMixedBilinearForm *a_NDH1 = new ParMixedBilinearForm(H1fespace, fespace);
   if (pa)
   {
-      a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      a_mixed.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+      a->SetAssemblyLevel(AssemblyLevel::PARTIAL);
+      a_NDH1->SetAssemblyLevel(AssemblyLevel::PARTIAL);
   }

-   if (prob == 0)
-   {
-      a.AddDomainIntegrator(new VectorFEMassIntegrator(one));
-      a_mixed.AddDomainIntegrator(new MixedVectorGradientIntegrator(one));
-   }
-   else if (prob == 1)
-   {
-      a.AddDomainIntegrator(new VectorFEMassIntegrator(one));
-      a_mixed.AddDomainIntegrator(new MixedVectorCurlIntegrator(one));
-   }
-   else
-   {
-      a.AddDomainIntegrator(new MassIntegrator(one));
-      a_mixed.AddDomainIntegrator(new VectorFEDivergenceIntegrator(one));
-   }
+   // First approach: L2 projection
+   a->AddDomainIntegrator(new VectorFEMassIntegrator(*sigma));
+   a_NDH1->AddDomainIntegrator(new MixedVectorGradientIntegrator(*muinv));

   // 10. Assemble the parallel bilinear form and the corresponding linear
   //     system, applying any necessary transformations such as: parallel
   //     assembly, eliminating boundary conditions, applying conforming
   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
+   if (static_cond) { a->EnableStaticCondensation(); }

-   a.Assemble();
-   if (!pa) { a.Finalize(); }
+   a->Assemble();
+   if (!pa) { a->Finalize(); }

-   a_mixed.Assemble();
-   if (!pa) { a_mixed.Finalize(); }
+   a_NDH1->Assemble();
+   if (!pa) { a_NDH1->Finalize(); }

-   Vector B(test_fes.GetTrueVSize());
-   Vector X(test_fes.GetTrueVSize());
+   Vector B(fespace->GetTrueVSize());
+   Vector X(fespace->GetTrueVSize());

   if (pa)
   {
-      ParLinearForm b(&test_fes); // used as a vector
-      a_mixed.Mult(gftrial, b); // process-local multiplication
-      b.ParallelAssemble(B);
+      ParLinearForm *b = new ParLinearForm(fespace); // used as a vector
+      a_NDH1->Mult(p, *b); // process-local multiplication
+      b->ParallelAssemble(B);
+      delete b;
   }
   else
   {
-      HypreParMatrix *mixed = a_mixed.ParallelAssemble();
+      HypreParMatrix *NDH1 = a_NDH1->ParallelAssemble();

-      Vector P(trial_fes.GetTrueVSize());
-      gftrial.GetTrueDofs(P);
+      Vector P(H1fespace->GetTrueVSize());
+      p.GetTrueDofs(P);

-      mixed->Mult(P,B);
+      NDH1->Mult(P,B);

-      delete mixed;
+      delete NDH1;
   }

   // 11. Define and apply a parallel PCG solver for AX=B with Jacobi
@@ -284,9 +212,9 @@ int main(int argc, char *argv[])
      Array<int> ess_tdof_list; // empty

      OperatorPtr A;
-      a.FormSystemMatrix(ess_tdof_list, A);
+      a->FormSystemMatrix(ess_tdof_list, A);

-      OperatorJacobiSmoother Jacobi(a, ess_tdof_list);
+      OperatorJacobiSmoother Jacobi(*a, ess_tdof_list);

      CGSolver cg(MPI_COMM_WORLD);
      cg.SetRelTol(1e-12);
@@ -299,7 +227,7 @@ int main(int argc, char *argv[])
   }
   else
   {
-      HypreParMatrix *Amat = a.ParallelAssemble();
+      HypreParMatrix *Amat = a->ParallelAssemble();
      HypreDiagScale Jacobi(*Amat);
      HyprePCG pcg(*Amat);
      pcg.SetTol(1e-12);
@@ -314,97 +242,35 @@ int main(int argc, char *argv[])

   x.SetFromTrueDofs(X);

-   // 12. Compute the same field by applying a DiscreteInterpolator.
-   ParGridFunction discreteInterpolant(&test_fes);
-   ParDiscreteLinearOperator dlo(&trial_fes, &test_fes);
-   if (prob == 0)
-   {
-      dlo.AddDomainInterpolator(new GradientInterpolator());
-   }
-   else if (prob == 1)
-   {
-      dlo.AddDomainInterpolator(new CurlInterpolator());
-   }
-   else
-   {
-      dlo.AddDomainInterpolator(new DivergenceInterpolator());
-   }
+   // 12. Second approach: compute the same solution by applying
+   //     GradientInterpolator in H(curl).
+   ParDiscreteLinearOperator grad(H1fespace, fespace);
+   grad.AddDomainInterpolator(new GradientInterpolator());
+   grad.Assemble();

-   dlo.Assemble();
-   dlo.Mult(gftrial, discreteInterpolant);
+   ParGridFunction gradp(fespace);
+   grad.Mult(p, gradp);

-   // 13. Compute the projection of the exact field.
-   ParGridFunction exact_proj(&test_fes);
-   if (prob == 0)
-   {
-      exact_proj.ProjectCoefficient(gradp_coef);
-   }
-   else if (prob == 1)
-   {
-      exact_proj.ProjectCoefficient(curlv_coef);
-   }
-   else
-   {
-      exact_proj.ProjectCoefficient(divgradp_coef);
-   }
+   // 13. Compute the projection of the exact grad p.
+   ParGridFunction exact_gradp(fespace);
+   exact_gradp.ProjectCoefficient(gradp_coef);
+   exact_gradp.SetTrueVector();
+   exact_gradp.SetFromTrueVector();

-   exact_proj.SetTrueVector();
-   exact_proj.SetFromTrueVector();
-
-   // 14. Compute and print the L_2 norm of the error.
-   if (prob == 0)
+   // 14. Compute and print the L^2 norm of the error.
   {
      double errSol = x.ComputeL2Error(gradp_coef);
-      double errInterp = discreteInterpolant.ComputeL2Error(gradp_coef);
-      double errProj = exact_proj.ComputeL2Error(gradp_coef);
+      double errInterp = gradp.ComputeL2Error(gradp_coef);
+      double errProj = exact_gradp.ComputeL2Error(gradp_coef);

      if (myid == 0)
      {
-         cout << "\n Solution of (E_h,v) = (grad p_h,v) for E_h and v in H(curl)"
-              ": || E_h - grad p ||_{L_2} = " << errSol << '\n' << endl;
-         cout << " Gradient interpolant E_h = grad p_h in H(curl): || E_h - grad"
-              " p ||_{L_2} = " << errInterp << '\n' << endl;
+         cout << "\n Solution of (E_h,v) = (grad p_h,v) for E_h and v in "
+              "H(curl): || E_h - grad p ||_{L^2} = " << errSol << '\n' << endl;
+         cout << " Gradient interpolant E_h = grad p_h in H(curl): || E_h - "
+              "grad p ||_{L^2} = " << errInterp << '\n' << endl;
         cout << " Projection E_h of exact grad p in H(curl): || E_h - grad p "
-              "||_{L_2} = " << errProj << '\n' << endl;
-      }
-   }
-   else if (prob == 1)
-   {
-      double errSol = x.ComputeL2Error(curlv_coef);
-      double errInterp = discreteInterpolant.ComputeL2Error(curlv_coef);
-      double errProj = exact_proj.ComputeL2Error(curlv_coef);
-
-      if (myid == 0)
-      {
-         cout << "\n Solution of (E_h,w) = (curl v_h,w) for E_h and w in "
-              "H(div): || E_h - curl v ||_{L_2} = " << errSol << '\n' << endl;
-         cout << " Curl interpolant E_h = curl v_h in H(div): || E_h - curl v "
-              "||_{L_2} = " << errInterp << '\n' << endl;
-         cout << " Projection E_h of exact curl v in H(div): || E_h - curl v "
-              "||_{L_2} = " << errProj << '\n' << endl;
-      }
-   }
-   else
-   {
-      int order_quad = max(2, 2*order+1);
-      const IntegrationRule *irs[Geometry::NumGeom];
-      for (int i=0; i < Geometry::NumGeom; ++i)
-      {
-         irs[i] = &(IntRules.Get(i, order_quad));
-      }
-
-      double errSol = x.ComputeL2Error(divgradp_coef, irs);
-      double errInterp = discreteInterpolant.ComputeL2Error(divgradp_coef, irs);
-      double errProj = exact_proj.ComputeL2Error(divgradp_coef, irs);
-
-      if (myid == 0)
-      {
-         cout << "\n Solution of (f_h,q) = (div v_h,q) for f_h and q in L_2: "
-              "|| f_h - div v ||_{L_2} = " << errSol << '\n' << endl;
-         cout << " Divergence interpolant f_h = div v_h in L_2: || f_h - div v"
-              " ||_{L_2} = " << errInterp << '\n' << endl;
-         cout << " Projection f_h of exact div v in L_2: || f_h - div v "
-              "||_{L_2} = " << errProj << '\n' << endl;
+              "||_{L^2} = " << errProj << '\n' << endl;
      }
   }

@@ -436,8 +302,14 @@ int main(int argc, char *argv[])
   }

   // 17. Free the used memory.
-   delete trial_fec;
-   delete test_fec;
+   delete a;
+   delete a_NDH1;
+   delete sigma;
+   delete muinv;
+   delete fespace;
+   delete H1fespace;
+   delete fec;
+   delete H1fec;
   delete pmesh;

   MPI_Finalize();
@@ -474,47 +346,3 @@ void gradp_exact(const Vector &x, Vector &f)
      if (x.Size() == 3) { f(2) = 0.0; }
   }
 }
-
-double div_gradp_exact(const Vector &x)
-{
-   if (dim == 3)
-   {
-      return -3.0 * sin(x(0)) * sin(x(1)) * sin(x(2));
-   }
-   else if (dim == 2)
-   {
-      return -2.0 * sin(x(0)) * sin(x(1));
-   }
-
-   return 0.0;
-}
-
-void v_exact(const Vector &x, Vector &v)
-{
-   if (dim == 3)
-   {
-      v(0) = sin(kappa * x(1));
-      v(1) = sin(kappa * x(2));
-      v(2) = sin(kappa * x(0));
-   }
-   else
-   {
-      v(0) = sin(kappa * x(1));
-      v(1) = sin(kappa * x(0));
-      if (x.Size() == 3) { v(2) = 0.0; }
-   }
-}
-
-void curlv_exact(const Vector &x, Vector &cv)
-{
-   if (dim == 3)
-   {
-      cv(0) = -kappa * cos(kappa * x(2));
-      cv(1) = -kappa * cos(kappa * x(0));
-      cv(2) = -kappa * cos(kappa * x(1));
-   }
-   else
-   {
-      cv = 0.0;
-   }
-}
@@ -10,10 +10,6 @@
 //               ex25 -o 2 -f 8.0 -ref 3 -prob 4 -m ../data/inline-quad.mesh
 //               ex25 -o 2 -f 2.0 -ref 1 -prob 4 -m ../data/inline-hex.mesh
 //
-// Device sample runs:
-//               ex25 -o 2 -f 8.0 -ref 3 -prob 4 -m ../data/inline-quad.mesh -pa -d cuda
-//               ex25 -o 2 -f 2.0 -ref 1 -prob 4 -m ../data/inline-hex.mesh -pa -d cuda
-//
 // Description:  This example code solves a simple electromagnetic wave
 //               propagation problem corresponding to the second order
 //               indefinite Maxwell equation
@@ -86,24 +82,24 @@ public:
 };

 // Class for returning the PML coefficients of the bilinear form
-class PMLDiagMatrixCoefficient : public VectorCoefficient
+class PMLMatrixCoefficient : public MatrixCoefficient
 {
 private:
   CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML * , Vector &);
+   void (*Function)(const Vector &, CartesianPML * , DenseMatrix &);
 public:
-   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
-                                              Vector &),
-                            CartesianPML * pml_)
-      : VectorCoefficient(dim), pml(pml_), Function(F)
+   PMLMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+                                          DenseMatrix &),
+                        CartesianPML * pml_)
+      : MatrixCoefficient(dim), pml(pml_), Function(F)
   {}
-   virtual void Eval(Vector &K, ElementTransformation &T,
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                     const IntegrationPoint &ip)
   {
      double x[3];
      Vector transip(x, 3);
      T.Transform(ip, transip);
-      K.SetSize(vdim);
+      K.SetSize(height, width);
      (*Function)(transip, pml, K);
   }
 };
@@ -120,13 +116,13 @@ void source(const Vector &x, Vector & f);

 // Functions for computing the necessary coefficients after PML stretching.
 // J is the Jacobian matrix of the stretching function
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);

-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);

 Array2D<double> comp_domain_bdr;
 Array2D<double> domain_bdr;
@@ -157,8 +153,6 @@ int main(int argc, char *argv[])
   double freq = 5.0;
   bool herm_conv = true;
   bool visualization = 1;
-   bool pa = false;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -180,21 +174,12 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();

   if (iprob > 4) { iprob = 4; }
   prob = (prob_type)iprob;

-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Setup the mesh
+   // 2. Setup the mesh
   if (!mesh_file)
   {
      exact_known = true;
@@ -235,7 +220,7 @@ int main(int argc, char *argv[])
   // Setup PML length
   Array2D<double> length(dim, 2); length = 0.0;

-   // 4. Setup the Cartesian PML region.
+   // 3. Setup the Cartesian PML region.
   switch (prob)
   {
      case disc:
@@ -261,19 +246,19 @@ int main(int argc, char *argv[])
   comp_domain_bdr = pml->GetCompDomainBdr();
   domain_bdr = pml->GetDomainBdr();

-   // 5. Refine the mesh to increase the resolution.
+   // 4. Refine the mesh to increase the resolution.
   for (int l = 0; l < ref_levels; l++)
   {
      mesh->UniformRefinement();
   }

-   // 6. Reorient mesh in case of a tet mesh
+   // 5. Reorient mesh in case of a tet mesh
   mesh->ReorientTetMesh();

   // Set element attributes in order to distinguish elements in the PML region
   pml->SetAttributes(mesh);

-   // 7. Define a finite element space on the mesh. Here we use the Nedelec
+   // 6. Define a finite element space on the mesh. Here we use the Nedelec
   //    finite elements of the specified order.
   FiniteElementCollection *fec = new ND_FECollection(order, dim);
   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
@@ -281,7 +266,7 @@ int main(int argc, char *argv[])

   cout << "Number of finite element unknowns: " << size << endl;

-   // 8. Determine the list of true essential boundary dofs. In this example,
+   // 7. Determine the list of true essential boundary dofs. In this example,
   //    the boundary conditions are defined based on the specific mesh and the
   //    problem type.
   Array<int> ess_tdof_list;
@@ -323,12 +308,12 @@ int main(int argc, char *argv[])
   }
   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);

-   // 9. Setup Complex Operator convention
+   // 8. Setup Complex Operator convention
   ComplexOperator::Convention conv =
      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;

-   // 10. Set up the linear form b(.) which corresponds to the right-hand side of
-   //     the FEM linear system.
+   // 9. Set up the linear form b(.) which corresponds to the right-hand side of
+   //    the FEM linear system.
   VectorFunctionCoefficient f(dim, source);
   ComplexLinearForm b(fespace, conv);
   if (prob == load_src)
@@ -338,7 +323,7 @@ int main(int argc, char *argv[])
   b.Vector::operator=(0.0);
   b.Assemble();

-   // 11. Define the solution vector x as a complex finite element grid function
+   // 10. Define the solution vector x as a complex finite element grid function
   //     corresponding to fespace.
   ComplexGridFunction x(fespace);
   x = 0.0;
@@ -346,7 +331,7 @@ int main(int argc, char *argv[])
   VectorFunctionCoefficient E_Im(dim, E_bdr_data_Im);
   x.ProjectBdrCoefficientTangent(E_Re, E_Im, ess_bdr);

-   // 12. Set up the sesquilinear form a(.,.)
+   // 11. Set up the sesquilinear form a(.,.)
   //
   //     In Comp
   //     Domain:   1/mu (Curl E, Curl F) - omega^2 * epsilon (E,F)
@@ -380,19 +365,19 @@ int main(int argc, char *argv[])
   a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_omeg),NULL);

   int cdim = (dim == 2) ? 1 : dim;
-   PMLDiagMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
-   PMLDiagMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
-   ScalarVectorProductCoefficient c1_Re(muinv,pml_c1_Re);
-   ScalarVectorProductCoefficient c1_Im(muinv,pml_c1_Im);
-   VectorRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
-   VectorRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
+   PMLMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
+   PMLMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
+   ScalarMatrixProductCoefficient c1_Re(muinv,pml_c1_Re);
+   ScalarMatrixProductCoefficient c1_Im(muinv,pml_c1_Im);
+   MatrixRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);

-   PMLDiagMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
-   PMLDiagMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
-   ScalarVectorProductCoefficient c2_Re(omeg,pml_c2_Re);
-   ScalarVectorProductCoefficient c2_Im(omeg,pml_c2_Im);
-   VectorRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
-   VectorRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
+   PMLMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
+   PMLMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
+   ScalarMatrixProductCoefficient c2_Re(omeg,pml_c2_Re);
+   ScalarMatrixProductCoefficient c2_Im(omeg,pml_c2_Im);
+   MatrixRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);

   // Integrators inside the PML region
   a.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_Re),
@@ -400,29 +385,30 @@ int main(int argc, char *argv[])
   a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_Re),
                         new VectorFEMassIntegrator(restr_c2_Im));

-   // 13. Assemble the bilinear form and the corresponding linear system,
+   // 12. Assemble the bilinear form and the corresponding linear system,
   //     applying any necessary transformations such as: assembly, eliminating
   //     boundary conditions, applying conforming constraints for
   //     non-conforming AMR, etc.
-#ifndef MFEM_USE_SUITESPARSE
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-#endif
-   a.Assemble(0);
+   a.Assemble();

-   OperatorPtr A;
+   OperatorHandle Ah;
   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+   a.FormLinearSystem(ess_tdof_list, x, b, Ah, X, B);
+
+   // 13. Transform to monolithic SparseMatrix
+   SparseMatrix *A = Ah.As<ComplexSparseMatrix>()->GetSystemMatrix();
+
+   cout << "Size of linear system: " << A->Height() << endl;

   // 14. Solve using a direct or an iterative solver
 #ifdef MFEM_USE_SUITESPARSE
   {
-      if (pa) { cout << "PA not available with MFEM_USE_SUITESPARSE" << endl; }
-      ComplexUMFPackSolver csolver(*A.As<ComplexSparseMatrix>());
-      csolver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      csolver.SetPrintLevel(1);
-      csolver.Mult(B, X);
+      UMFPackSolver  solver(*A);
+      solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      solver.Mult(B, X);
   }
 #else
+
   // 14a. Set up the Bilinear form a(.,.) for the preconditioner
   //
   //    In Comp
@@ -438,64 +424,45 @@ int main(int argc, char *argv[])
      prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_muinv));
      prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_absomeg));

-      PMLDiagMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
-      ScalarVectorProductCoefficient c1_abs(muinv,pml_c1_abs);
-      VectorRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);
+      PMLMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
+      ScalarMatrixProductCoefficient c1_abs(muinv,pml_c1_abs);
+      MatrixRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);

-      PMLDiagMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
-      ScalarVectorProductCoefficient c2_abs(absomeg,pml_c2_abs);
-      VectorRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);
+      PMLMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
+      ScalarMatrixProductCoefficient c2_abs(absomeg,pml_c2_abs);
+      MatrixRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);

      prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_abs));
      prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_abs));

-      if (pa) { prec.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
      prec.Assemble();

+      OperatorHandle PCOpAh;
+      prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
+
      // 14b. Define and apply a GMRES solver for AU=B with a block diagonal
-      //      preconditioner based on the Gauss-Seidel or Jacobi sparse smoother.
+      //      preconditioner based on the Gauss-Seidel sparse smoother.
      Array<int> offsets(3);
      offsets[0] = 0;
      offsets[1] = fespace->GetTrueVSize();
      offsets[2] = fespace->GetTrueVSize();
      offsets.PartialSum();

-      Operator *pc_r = nullptr;
-      Operator *pc_i = nullptr;
-      int s = (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0;
-      if (pa)
-      {
-         // Jacobi Smoother
-         OperatorJacobiSmoother *d00 = new OperatorJacobiSmoother(prec, ess_tdof_list);
-         ScaledOperator *d11 = new ScaledOperator(d00, s);
-         pc_r = d00;
-         pc_i = d11;
-      }
-      else
-      {
-         OperatorPtr PCOpAh;
-         prec.SetDiagonalPolicy(mfem::Operator::DIAG_ONE);
-         prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
-
-         // Gauss-Seidel Smoother
-         GSSmoother *gs00 = new GSSmoother(*PCOpAh.As<SparseMatrix>());
-         ScaledOperator *gs11 = new ScaledOperator(gs00, s);
-         pc_r = gs00;
-         pc_i = gs11;
-      }
-
-      BlockDiagonalPreconditioner BlockDP(offsets);
-      BlockDP.SetDiagonalBlock(0, pc_r);
-      BlockDP.SetDiagonalBlock(1, pc_i);
+      GSSmoother gs00(*PCOpAh.As<SparseMatrix>());
+      BlockDiagonalPreconditioner BlockGS(offsets);
+      ScaledOperator gs11(&gs00,
+                          (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0);
+      BlockGS.SetDiagonalBlock(0,&gs00);
+      BlockGS.SetDiagonalBlock(1,&gs11);

      GMRESSolver gmres;
      gmres.SetPrintLevel(1);
      gmres.SetKDim(200);
-      gmres.SetMaxIter(pa ? 5000 : 2000);
+      gmres.SetMaxIter(2000);
      gmres.SetRelTol(1e-5);
      gmres.SetAbsTol(0.0);
      gmres.SetOperator(*A);
-      gmres.SetPreconditioner(BlockDP);
+      gmres.SetPreconditioner(BlockGS);
      gmres.Mult(B, X);
   }
 #endif
@@ -507,8 +474,10 @@ int main(int argc, char *argv[])
   // If exact is known compute the error
   if (exact_known)
   {
+      ComplexGridFunction x_gf(fespace);
      VectorFunctionCoefficient E_ex_Re(dim, E_exact_Re);
      VectorFunctionCoefficient E_ex_Im(dim, E_exact_Im);
+      x_gf.ProjectCoefficient(E_ex_Re, E_ex_Im);
      int order_quad = max(2, 2 * order + 1);
      const IntegrationRule *irs[Geometry::NumGeom];
      for (int i = 0; i < Geometry::NumGeom; ++i)
@@ -604,6 +573,7 @@ int main(int argc, char *argv[])
   }

   // 18. Free the used memory.
+   delete A;
   delete pml;
   delete fespace;
   delete fec;
@@ -801,7 +771,7 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
   }
 }

-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det(1.0, 0.0);
@@ -812,13 +782,14 @@ void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = (det / pow(dxs[i], 2)).real();
+      M(i, i) = (det / pow(dxs[i], 2)).real();
   }
 }

-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -829,13 +800,14 @@ void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = (det / pow(dxs[i], 2)).imag();
+      M(i, i) = (det / pow(dxs[i], 2)).imag();
   }
 }

-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -846,13 +818,14 @@ void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = abs(det / pow(dxs[i], 2));
+      M(i, i) = abs(det / pow(dxs[i], 2));
   }
 }

-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det(1.0, 0.0);
@@ -866,18 +839,19 @@ void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D)
   // in the 2D case the coefficient is scalar 1/det(J)
   if (dim == 2)
   {
-      D = (1.0 / det).real();
+      M = (1.0 / det).real();
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = (pow(dxs[i], 2) / det).real();
+         M(i, i) = (pow(dxs[i], 2) / det).real();
      }
   }
 }

-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -890,18 +864,19 @@ void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D)

   if (dim == 2)
   {
-      D = (1.0 / det).imag();
+      M = (1.0 / det).imag();
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = (pow(dxs[i], 2) / det).imag();
+         M(i, i) = (pow(dxs[i], 2) / det).imag();
      }
   }
 }

-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -914,13 +889,14 @@ void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D)

   if (dim == 2)
   {
-      D = abs(1.0 / det);
+      M = abs(1.0 / det);
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = abs(pow(dxs[i], 2) / det);
+         M(i, i) = abs(pow(dxs[i], 2) / det);
      }
   }
 }
@@ -10,10 +10,6 @@
 //               mpirun -np 4 ex25p -o 2 -f 8.0 -rs 2 -rp 2 -prob 4 -m ../data/inline-quad.mesh
 //               mpirun -np 4 ex25p -o 2 -f 2.0 -rs 1 -rp 1 -prob 4 -m ../data/inline-hex.mesh
 //
-// Device sample runs:
-//               mpirun -np 4 ex25p -o 1 -f 3.0 -rs 3 -rp 1 -prob 2 -pa -d cuda
-//               mpirun -np 4 ex25p -o 2 -f 1.0 -rs 1 -rp 1 -prob 3 -pa -d cuda
-//
 // Description:  This example code solves a simple electromagnetic wave
 //               propagation problem corresponding to the second order
 //               indefinite Maxwell equation
@@ -86,24 +82,24 @@ public:
 };

 // Class for returning the PML coefficients of the bilinear form
-class PMLDiagMatrixCoefficient : public VectorCoefficient
+class PMLMatrixCoefficient : public MatrixCoefficient
 {
 private:
   CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML * , Vector &);
+   void (*Function)(const Vector &, CartesianPML * , DenseMatrix &);
 public:
-   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
-                                              Vector &),
-                            CartesianPML * pml_)
-      : VectorCoefficient(dim), pml(pml_), Function(F)
+   PMLMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+                                          DenseMatrix &),
+                        CartesianPML * pml_)
+      : MatrixCoefficient(dim), pml(pml_), Function(F)
   {}
-   virtual void Eval(Vector &K, ElementTransformation &T,
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                     const IntegrationPoint &ip)
   {
      double x[3];
      Vector transip(x, 3);
      T.Transform(ip, transip);
-      K.SetSize(vdim);
+      K.SetSize(height, width);
      (*Function)(transip, pml, K);
   }
 };
@@ -120,13 +116,13 @@ void source(const Vector &x, Vector & f);

 // Functions for computing the necessary coefficients after PML stretching.
 // J is the Jacobian matrix of the stretching function
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);

-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);

 Array2D<double> comp_domain_bdr;
 Array2D<double> domain_bdr;
@@ -164,8 +160,6 @@ int main(int argc, char *argv[])
   double freq = 5.0;
   bool herm_conv = true;
   bool visualization = 1;
-   bool pa = false;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -189,21 +183,12 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();

   if (iprob > 4) { iprob = 4; }
   prob = (prob_type)iprob;

-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 4. Setup the (serial) mesh on all processors.
+   // 3. Setup the (serial) mesh on all processors.
   if (!mesh_file)
   {
      exact_known = true;
@@ -251,7 +236,7 @@ int main(int argc, char *argv[])
   // Setup PML length
   Array2D<double> length(dim, 2); length = 0.0;

-   // 5. Setup the Cartesian PML region.
+   // 4. Setup the Cartesian PML region.
   switch (prob)
   {
      case disc:
@@ -277,13 +262,13 @@ int main(int argc, char *argv[])
   comp_domain_bdr = pml->GetCompDomainBdr();
   domain_bdr = pml->GetDomainBdr();

-   // 6. Refine the serial mesh on all processors to increase the resolution.
+   // 5. Refine the serial mesh on all processors to increase the resolution.
   for (int l = 0; l < ref_levels; l++)
   {
      mesh->UniformRefinement();
   }

-   // 7. Define a parallel mesh by a partitioning of the serial mesh.
+   // 6. Define a parallel mesh by a partitioning of the serial mesh.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
   delete mesh;
   {
@@ -293,13 +278,13 @@ int main(int argc, char *argv[])
      }
   }

-   // 7a. Reorient mesh in case of a tet mesh
+   // 6a. Reorient mesh in case of a tet mesh
   pmesh->ReorientTetMesh();

-   // 8. Set element attributes in order to distinguish elements in the PML
+   // 7. Set element attributes in order to distinguish elements in the PML
   pml->SetAttributes(pmesh);

-   // 9. Define a parallel finite element space on the parallel mesh. Here we
+   // 8. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Nedelec finite elements of the specified order.
   FiniteElementCollection *fec = new ND_FECollection(order, dim);
   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
@@ -309,9 +294,9 @@ int main(int argc, char *argv[])
      cout << "Number of finite element unknowns: " << size << endl;
   }

-   // 10. Determine the list of true (i.e. parallel conforming) essential
-   //     boundary dofs. In this example, the boundary conditions are defined
-   //     based on the specific mesh and the problem type.
+   // 9. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    based on the specific mesh and the problem type.
   Array<int> ess_tdof_list;
   Array<int> ess_bdr;
   if (pmesh->bdr_attributes.Size())
@@ -351,11 +336,11 @@ int main(int argc, char *argv[])
   }
   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);

-   // 11. Setup Complex Operator convention
+   // 10. Setup Complex Operator convention
   ComplexOperator::Convention conv =
      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;

-   // 12. Set up the parallel linear form b(.) which corresponds to the
+   // 11. Set up the parallel linear form b(.) which corresponds to the
   //     right-hand side of the FEM linear system.
   VectorFunctionCoefficient f(dim, source);
   ParComplexLinearForm b(fespace, conv);
@@ -366,7 +351,7 @@ int main(int argc, char *argv[])
   b.Vector::operator=(0.0);
   b.Assemble();

-   // 13. Define the solution vector x as a parallel complex finite element grid
+   // 12. Define the solution vector x as a parallel complex finite element grid
   //     function corresponding to fespace.
   ParComplexGridFunction x(fespace);
   x = 0.0;
@@ -374,7 +359,7 @@ int main(int argc, char *argv[])
   VectorFunctionCoefficient E_Im(dim, E_bdr_data_Im);
   x.ProjectBdrCoefficientTangent(E_Re, E_Im, ess_bdr);

-   // 14. Set up the parallel sesquilinear form a(.,.)
+   // 13. Set up the parallel sesquilinear form a(.,.)
   //
   //     In Comp
   //     Domain:   1/mu (Curl E, Curl F) - omega^2 * epsilon (E,F)
@@ -408,19 +393,19 @@ int main(int argc, char *argv[])
   a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_omeg),NULL);

   int cdim = (dim == 2) ? 1 : dim;
-   PMLDiagMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
-   PMLDiagMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
-   ScalarVectorProductCoefficient c1_Re(muinv,pml_c1_Re);
-   ScalarVectorProductCoefficient c1_Im(muinv,pml_c1_Im);
-   VectorRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
-   VectorRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
+   PMLMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
+   PMLMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
+   ScalarMatrixProductCoefficient c1_Re(muinv,pml_c1_Re);
+   ScalarMatrixProductCoefficient c1_Im(muinv,pml_c1_Im);
+   MatrixRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);

-   PMLDiagMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
-   PMLDiagMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
-   ScalarVectorProductCoefficient c2_Re(omeg,pml_c2_Re);
-   ScalarVectorProductCoefficient c2_Im(omeg,pml_c2_Im);
-   VectorRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
-   VectorRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
+   PMLMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
+   PMLMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
+   ScalarMatrixProductCoefficient c2_Re(omeg,pml_c2_Re);
+   ScalarMatrixProductCoefficient c2_Im(omeg,pml_c2_Im);
+   MatrixRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);

   // Integrators inside the PML region
   a.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_Re),
@@ -428,25 +413,27 @@ int main(int argc, char *argv[])
   a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_Re),
                         new VectorFEMassIntegrator(restr_c2_Im));

-   // 15. Assemble the parallel bilinear form and the corresponding linear
+   // 14. Assemble the parallel bilinear form and the corresponding linear
   //     system, applying any necessary transformations such as: parallel
   //     assembly, eliminating boundary conditions, applying conforming
   //     constraints for non-conforming AMR, etc.
-#ifndef MFEM_USE_SUPERLU
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-#endif
   a.Assemble();

-   OperatorPtr Ah;
+   OperatorHandle Ah;
   Vector B, X;
   a.FormLinearSystem(ess_tdof_list, x, b, Ah, X, B);

+   // 15. Transform to monolithic HypreParMatrix
+   HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
+
+   if (myid == 0)
+   {
+      cout << "Size of linear system: " << A->GetGlobalNumRows() << endl;
+   }
+
   // 16. Solve using a direct or an iterative solver
 #ifdef MFEM_USE_SUPERLU
   {
-      if (pa) { cout << "PA not available with MFEM_USE_SUPERLU" << endl; }
-      // Transform to monolithic HypreParMatrix
-      HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
      SuperLURowLocMatrix SA(*A);
      SuperLUSolver superlu(MPI_COMM_WORLD);
      superlu.SetPrintStatistics(false);
@@ -454,9 +441,9 @@ int main(int argc, char *argv[])
      superlu.SetColumnPermutation(superlu::PARMETIS);
      superlu.SetOperator(SA);
      superlu.Mult(B, X);
-      delete A;
   }
 #else
+
   // 16a. Set up the parallel Bilinear form a(.,.) for the preconditioner
   //
   //    In Comp
@@ -472,20 +459,22 @@ int main(int argc, char *argv[])
      prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_muinv));
      prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_absomeg));

-      PMLDiagMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
-      ScalarVectorProductCoefficient c1_abs(muinv,pml_c1_abs);
-      VectorRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);
+      PMLMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
+      ScalarMatrixProductCoefficient c1_abs(muinv,pml_c1_abs);
+      MatrixRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);

-      PMLDiagMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
-      ScalarVectorProductCoefficient c2_abs(absomeg,pml_c2_abs);
-      VectorRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);
+      PMLMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
+      ScalarMatrixProductCoefficient c2_abs(absomeg,pml_c2_abs);
+      MatrixRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);

      prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_abs));
      prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_abs));

-      if (pa) { prec.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
      prec.Assemble();

+      OperatorHandle PCOpAh;
+      prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
+
      // 16b. Define and apply a parallel GMRES solver for AU=B with a block
      //      diagonal preconditioner based on hypre's AMS preconditioner.
      Array<int> offsets(3);
@@ -494,41 +483,21 @@ int main(int argc, char *argv[])
      offsets[2] = fespace->GetTrueVSize();
      offsets.PartialSum();

-      Operator *pc_r = nullptr;
-      Operator *pc_i = nullptr;
-      int s = (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0;
-      if (pa)
-      {
-         // Jacobi Smoother
-         OperatorJacobiSmoother *d00 = new OperatorJacobiSmoother(prec, ess_tdof_list);
-         ScaledOperator *d11 = new ScaledOperator(d00, s);
-         pc_r = d00;
-         pc_i = d11;
-      }
-      else
-      {
-         OperatorPtr PCOpAh;
-         prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
-
-         // Hypre AMS
-         HypreAMS *ams00 = new HypreAMS(*PCOpAh.As<HypreParMatrix>(), fespace);
-         ScaledOperator *ams11 = new ScaledOperator(ams00, s);
-         pc_r = ams00;
-         pc_i = ams11;
-      }
-
-      BlockDiagonalPreconditioner BlockDP(offsets);
-      BlockDP.SetDiagonalBlock(0, pc_r);
-      BlockDP.SetDiagonalBlock(1, pc_i);
+      HypreAMS ams00(*PCOpAh.As<HypreParMatrix>(),fespace);
+      BlockDiagonalPreconditioner BlockAMS(offsets);
+      ScaledOperator ams11(&ams00,
+                           (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0);
+      BlockAMS.SetDiagonalBlock(0,&ams00);
+      BlockAMS.SetDiagonalBlock(1,&ams11);

      GMRESSolver gmres(MPI_COMM_WORLD);
      gmres.SetPrintLevel(1);
      gmres.SetKDim(200);
-      gmres.SetMaxIter(pa ? 5000 : 2000);
+      gmres.SetMaxIter(2000);
      gmres.SetRelTol(1e-5);
      gmres.SetAbsTol(0.0);
-      gmres.SetOperator(*Ah);
-      gmres.SetPreconditioner(BlockDP);
+      gmres.SetOperator(*A);
+      gmres.SetPreconditioner(BlockAMS);
      gmres.Mult(B, X);
   }
 #endif
@@ -540,8 +509,10 @@ int main(int argc, char *argv[])
   // If exact is known compute the error
   if (exact_known)
   {
+      ParComplexGridFunction x_gf(fespace);
      VectorFunctionCoefficient E_ex_Re(dim, E_exact_Re);
      VectorFunctionCoefficient E_ex_Im(dim, E_exact_Im);
+      x_gf.ProjectCoefficient(E_ex_Re, E_ex_Im);
      int order_quad = max(2, 2 * order + 1);
      const IntegrationRule *irs[Geometry::NumGeom];
      for (int i = 0; i < Geometry::NumGeom; ++i)
@@ -658,6 +629,7 @@ int main(int argc, char *argv[])
   }

   // 20. Free the used memory.
+   delete A;
   delete pml;
   delete fespace;
   delete fec;
@@ -856,7 +828,7 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
   }
 }

-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det(1.0, 0.0);
@@ -867,13 +839,14 @@ void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = (det / pow(dxs[i], 2)).real();
+      M(i, i) = (det / pow(dxs[i], 2)).real();
   }
 }

-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -884,13 +857,14 @@ void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = (det / pow(dxs[i], 2)).imag();
+      M(i, i) = (det / pow(dxs[i], 2)).imag();
   }
 }

-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -901,13 +875,14 @@ void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = abs(det / pow(dxs[i], 2));
+      M(i, i) = abs(det / pow(dxs[i], 2));
   }
 }

-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det(1.0, 0.0);
@@ -921,18 +896,19 @@ void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D)
   // in the 2D case the coefficient is scalar 1/det(J)
   if (dim == 2)
   {
-      D = (1.0 / det).real();
+      M = (1.0 / det).real();
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = (pow(dxs[i], 2) / det).real();
+         M(i, i) = (pow(dxs[i], 2) / det).real();
      }
   }
 }

-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -945,18 +921,19 @@ void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D)

   if (dim == 2)
   {
-      D = (1.0 / det).imag();
+      M = (1.0 / det).imag();
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = (pow(dxs[i], 2) / det).imag();
+         M(i, i) = (pow(dxs[i], 2) / det).imag();
      }
   }
 }

-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -969,13 +946,14 @@ void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D)

   if (dim == 2)
   {
-      D = abs(1.0 / det);
+      M = abs(1.0 / det);
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = abs(pow(dxs[i], 2) / det);
+         M(i, i) = abs(pow(dxs[i], 2) / det);
      }
   }
 }
@@ -1,255 +0,0 @@
-//                                MFEM Example 26
-//
-// Compile with: make ex26
-//
-// Sample runs:  ex26 -m ../data/star.mesh
-//               ex26 -m ../data/fichera.mesh
-//               ex26 -m ../data/beam-hex.mesh
-//
-// Device sample runs:
-//               ex26 -d cuda
-//               ex26 -d raja-cuda
-//               ex26 -d occa-cuda
-//               ex26 -d raja-omp
-//               ex26 -d occa-omp
-//               ex26 -d ceed-cpu
-//               ex26 -d ceed-cuda
-//               ex26 -m ../data/beam-hex.mesh -d cuda
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions
-//               as in Example 1.
-//
-//               It highlights on the creation of a hierarchy of discretization
-//               spaces with partial assembly and the construction of an
-//               efficient multigrid preconditioner for the iterative solver.
-//
-//               We recommend viewing Example 1 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Class for constructing a multigrid preconditioner for the diffusion operator.
-// This example multigrid preconditioner class demonstrates the creation of the
-// diffusion bilinear forms and operators using partial assembly for all spaces
-// in the FiniteElementSpaceHierarchy. The preconditioner uses a CG solver on
-// the coarsest level and second order Chebyshev accelerated smoothers on the
-// other levels.
-class DiffusionMultigrid : public Multigrid
-{
-private:
-   ConstantCoefficient one;
-
-public:
-   // Constructs a diffusion multigrid for the given FiniteElementSpaceHierarchy
-   // and the array of essential boundaries
-   DiffusionMultigrid(FiniteElementSpaceHierarchy& fespaces, Array<int>& ess_bdr)
-      : Multigrid(fespaces), one(1.0)
-   {
-      ConstructCoarseOperatorAndSolver(fespaces.GetFESpaceAtLevel(0), ess_bdr);
-
-      for (int level = 1; level < fespaces.GetNumLevels(); ++level)
-      {
-         ConstructOperatorAndSmoother(fespaces.GetFESpaceAtLevel(level), ess_bdr);
-      }
-   }
-
-private:
-   void ConstructBilinearForm(FiniteElementSpace& fespace, Array<int>& ess_bdr)
-   {
-      BilinearForm* form = new BilinearForm(&fespace);
-      form->SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      form->AddDomainIntegrator(new DiffusionIntegrator(one));
-      form->Assemble();
-      bfs.Append(form);
-
-      essentialTrueDofs.Append(new Array<int>());
-      fespace.GetEssentialTrueDofs(ess_bdr, *essentialTrueDofs.Last());
-   }
-
-   void ConstructCoarseOperatorAndSolver(FiniteElementSpace& coarse_fespace,
-                                         Array<int>& ess_bdr)
-   {
-      ConstructBilinearForm(coarse_fespace, ess_bdr);
-
-      OperatorPtr opr;
-      opr.SetType(Operator::ANY_TYPE);
-      bfs.Last()->FormSystemMatrix(*essentialTrueDofs.Last(), opr);
-      opr.SetOperatorOwner(false);
-
-      CGSolver* pcg = new CGSolver();
-      pcg->SetPrintLevel(-1);
-      pcg->SetMaxIter(200);
-      pcg->SetRelTol(sqrt(1e-4));
-      pcg->SetAbsTol(0.0);
-      pcg->SetOperator(*opr.Ptr());
-
-      AddLevel(opr.Ptr(), pcg, true, true);
-   }
-
-   void ConstructOperatorAndSmoother(FiniteElementSpace& fespace,
-                                     Array<int>& ess_bdr)
-   {
-      ConstructBilinearForm(fespace, ess_bdr);
-
-      OperatorPtr opr;
-      opr.SetType(Operator::ANY_TYPE);
-      bfs.Last()->FormSystemMatrix(*essentialTrueDofs.Last(), opr);
-      opr.SetOperatorOwner(false);
-
-      Vector diag(fespace.GetTrueVSize());
-      bfs.Last()->AssembleDiagonal(diag);
-
-      Solver* smoother = new OperatorChebyshevSmoother(opr.Ptr(), diag,
-                                                       *essentialTrueDofs.Last(), 2);
-      AddLevel(opr.Ptr(), smoother, true, true);
-   }
-};
-
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../data/star.mesh";
-   int geometric_refinements = 0;
-   int order_refinements = 2;
-   const char *device_config = "cpu";
-   bool visualization = true;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&geometric_refinements, "-gr", "--geometric-refinements",
-                  "Number of geometric refinements done prior to order refinements.");
-   args.AddOption(&order_refinements, "-or", "--order-refinements",
-                  "Number of order refinements. Finest level in the hierarchy has order 2^{or}.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
-   //    the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 4. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
-   //    largest number that gives a final mesh with no more than 50,000
-   //    elements.
-   {
-      int ref_levels =
-         (int)floor(log(5000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh->UniformRefinement();
-      }
-   }
-
-   // 5. Define a finite element space hierarchy on the mesh. Here we use
-   //    continuous Lagrange finite elements. We start with order 1 on the
-   //    coarse level and geometrically refine the spaces by the specified
-   //    amount. Afterwards, we increase the order of the finite elements
-   //    by a factor of 2 for each additional level.
-   FiniteElementCollection *fec = new H1_FECollection(1, dim);
-   FiniteElementSpace *coarse_fespace = new FiniteElementSpace(mesh, fec);
-   FiniteElementSpaceHierarchy fespaces(mesh, coarse_fespace, true, true);
-
-   Array<FiniteElementCollection*> collections;
-   collections.Append(fec);
-   for (int level = 0; level < geometric_refinements; ++level)
-   {
-      fespaces.AddUniformlyRefinedLevel();
-   }
-   for (int level = 0; level < order_refinements; ++level)
-   {
-      collections.Append(new H1_FECollection(std::pow(2, level+1), dim));
-      fespaces.AddOrderRefinedLevel(collections.Last());
-   }
-
-   cout << "Number of finite element unknowns: "
-        << fespaces.GetFinestFESpace().GetTrueVSize() << endl;
-
-   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
-   //    the basis functions in the finite element fespace.
-   LinearForm *b = new LinearForm(&fespaces.GetFinestFESpace());
-   ConstantCoefficient one(1.0);
-   b->AddDomainIntegrator(new DomainLFIntegrator(one));
-   b->Assemble();
-
-   // 7. Define the solution vector x as a finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero,
-   //    which satisfies the boundary conditions.
-   GridFunction x(&fespaces.GetFinestFESpace());
-   x = 0.0;
-
-   // 8. Create the multigrid operator using the previously created
-   //    FiniteElementSpaceHierarchy and additional boundary information. This operator
-   //    is then used to create the MultigridSolver as a preconditioner in the
-   //    iterative solver.
-   Array<int> ess_bdr(mesh->bdr_attributes.Max());
-   ess_bdr = 1;
-
-   DiffusionMultigrid M(fespaces, ess_bdr);
-   M.SetCycleType(Multigrid::CycleType::VCYCLE, 1, 1);
-
-   OperatorPtr A;
-   Vector B, X;
-   M.FormFineLinearSystem(x, *b, A, X, B);
-   cout << "Size of linear system: " << A->Height() << endl;
-
-   // 9. Solve the linear system A X = B.
-   PCG(*A, M, B, X, 1, 2000, 1e-12, 0.0);
-
-   // 10. Recover the solution as a finite element grid function.
-   M.RecoverFineFEMSolution(X, *b, x);
-
-   // 11. Save the refined mesh and the solution. This output can be viewed later
-   //     using GLVis: "glvis -m refined.mesh -g sol.gf".
-   ofstream mesh_ofs("refined.mesh");
-   mesh_ofs.precision(8);
-   fespaces.GetFinestFESpace().GetMesh()->Print(mesh_ofs);
-   ofstream sol_ofs("sol.gf");
-   sol_ofs.precision(8);
-   x.Save(sol_ofs);
-
-   // 12. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *fespaces.GetFinestFESpace().GetMesh() << x <<
-               flush;
-   }
-
-   // 13. Free the used memory.
-   delete b;
-   for (int level = 0; level < collections.Size(); ++level)
-   {
-      delete collections[level];
-   }
-
-   return 0;
-}
@@ -1,317 +0,0 @@
-//                       MFEM Example 26 - Parallel Version
-//
-// Compile with: make ex26p
-//
-// Sample runs:  mpirun -np 4 ex26p -m ../data/star.mesh
-//               mpirun -np 4 ex26p -m ../data/fichera.mesh
-//               mpirun -np 4 ex26p -m ../data/beam-hex.mesh
-//
-// Device sample runs:
-//               mpirun -np 4 ex26p -d cuda
-//               mpirun -np 4 ex26p -d occa-cuda
-//               mpirun -np 4 ex26p -d raja-omp
-//               mpirun -np 4 ex26p -d ceed-cpu
-//               mpirun -np 4 ex26p -d ceed-cuda
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions
-//               as in Example 1.
-//
-//               It highlights on the creation of a hierarchy of discretization
-//               spaces with partial assembly and the construction of an
-//               efficient multigrid preconditioner for the iterative solver.
-//
-//               We recommend viewing Example 1 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Class for constructing a multigrid preconditioner for the diffusion operator.
-// This example multigrid preconditioner class demonstrates the creation of the
-// parallel diffusion bilinear forms and operators using partial assembly for
-// all spaces except the coarsest one in the ParFiniteElementSpaceHierarchy.
-// The multigrid uses a PCG solver preconditioned with AMG on the coarsest level
-// and second order Chebyshev accelerated smoothers on the other levels.
-class DiffusionMultigrid : public Multigrid
-{
-private:
-   ConstantCoefficient one;
-   HypreBoomerAMG* amg;
-
-public:
-   // Constructs a diffusion multigrid for the ParFiniteElementSpaceHierarchy
-   // and the array of essential boundaries
-   DiffusionMultigrid(ParFiniteElementSpaceHierarchy& fespaces,
-                      Array<int>& ess_bdr)
-      : Multigrid(fespaces), one(1.0)
-   {
-      ConstructCoarseOperatorAndSolver(fespaces.GetFESpaceAtLevel(0), ess_bdr);
-
-      for (int level = 1; level < fespaces.GetNumLevels(); ++level)
-      {
-         ConstructOperatorAndSmoother(fespaces.GetFESpaceAtLevel(level), ess_bdr);
-      }
-   }
-
-   virtual ~DiffusionMultigrid()
-   {
-      delete amg;
-   }
-
-private:
-   void ConstructBilinearForm(ParFiniteElementSpace& fespace, Array<int>& ess_bdr,
-                              bool partial_assembly)
-   {
-      ParBilinearForm* form = new ParBilinearForm(&fespace);
-      if (partial_assembly)
-      {
-         form->SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      }
-      form->AddDomainIntegrator(new DiffusionIntegrator(one));
-      form->Assemble();
-      bfs.Append(form);
-
-      essentialTrueDofs.Append(new Array<int>());
-      fespace.GetEssentialTrueDofs(ess_bdr, *essentialTrueDofs.Last());
-   }
-
-   void ConstructCoarseOperatorAndSolver(ParFiniteElementSpace& coarse_fespace,
-                                         Array<int>& ess_bdr)
-   {
-      ConstructBilinearForm(coarse_fespace, ess_bdr, false);
-
-      HypreParMatrix* hypreCoarseMat = new HypreParMatrix();
-      bfs.Last()->FormSystemMatrix(*essentialTrueDofs.Last(), *hypreCoarseMat);
-
-      amg = new HypreBoomerAMG(*hypreCoarseMat);
-      amg->SetPrintLevel(-1);
-
-      CGSolver* pcg = new CGSolver(MPI_COMM_WORLD);
-      pcg->SetPrintLevel(-1);
-      pcg->SetMaxIter(10);
-      pcg->SetRelTol(sqrt(1e-4));
-      pcg->SetAbsTol(0.0);
-      pcg->SetOperator(*hypreCoarseMat);
-      pcg->SetPreconditioner(*amg);
-
-      AddLevel(hypreCoarseMat, pcg, true, true);
-   }
-
-   void ConstructOperatorAndSmoother(ParFiniteElementSpace& fespace,
-                                     Array<int>& ess_bdr)
-   {
-      ConstructBilinearForm(fespace, ess_bdr, true);
-
-      OperatorPtr opr;
-      opr.SetType(Operator::ANY_TYPE);
-      bfs.Last()->FormSystemMatrix(*essentialTrueDofs.Last(), opr);
-      opr.SetOperatorOwner(false);
-
-      Vector diag(fespace.GetTrueVSize());
-      bfs.Last()->AssembleDiagonal(diag);
-
-      Solver* smoother = new OperatorChebyshevSmoother(opr.Ptr(), diag,
-                                                       *essentialTrueDofs.Last(), 2, fespace.GetParMesh()->GetComm());
-
-      AddLevel(opr.Ptr(), smoother, true, true);
-   }
-};
-
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../data/star.mesh";
-   int geometric_refinements = 0;
-   int order_refinements = 2;
-   const char *device_config = "cpu";
-   bool visualization = true;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&geometric_refinements, "-gr", "--geometric-refinements",
-                  "Number of geometric refinements done prior to order refinements.");
-   args.AddOption(&order_refinements, "-or", "--order-refinements",
-                  "Number of order refinements. Finest level in the hierarchy has order 2^{or}.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 5. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 1,000 elements.
-   {
-      int ref_levels =
-         (int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh->UniformRefinement();
-      }
-   }
-
-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   {
-      int par_ref_levels = 2;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         pmesh->UniformRefinement();
-      }
-   }
-
-   // 7. Define a parallel finite element space hierarchy on the parallel mesh.
-   //    Here we use continuous Lagrange finite elements. We start with order 1
-   //    on the coarse level and geometrically refine the spaces by the specified
-   //    amount. Afterwards, we increase the order of the finite elements by a
-   //    factor of 2 for each additional level.
-   FiniteElementCollection *fec = new H1_FECollection(1, dim);
-   ParFiniteElementSpace *coarse_fespace = new ParFiniteElementSpace(pmesh, fec);
-
-   Array<FiniteElementCollection*> collections;
-   collections.Append(fec);
-   ParFiniteElementSpaceHierarchy* fespaces = new ParFiniteElementSpaceHierarchy(
-      pmesh, coarse_fespace, true, true);
-   for (int level = 0; level < geometric_refinements; ++level)
-   {
-      fespaces->AddUniformlyRefinedLevel();
-   }
-   for (int level = 0; level < order_refinements; ++level)
-   {
-      collections.Append(new H1_FECollection(std::pow(2, level+1), dim));
-      fespaces->AddOrderRefinedLevel(collections.Last());
-   }
-
-   HYPRE_Int size = fespaces->GetFinestFESpace().GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-   }
-
-   // 8. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (1,phi_i) where phi_i are the basis functions in fespace.
-   ParLinearForm *b = new ParLinearForm(&fespaces->GetFinestFESpace());
-   ConstantCoefficient one(1.0);
-   b->AddDomainIntegrator(new DomainLFIntegrator(one));
-   b->Assemble();
-
-   // 9. Define the solution vector x as a parallel finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero,
-   //    which satisfies the boundary conditions.
-   ParGridFunction x(&fespaces->GetFinestFESpace());
-   x = 0.0;
-
-   // 10. Create the multigrid operator using the previously created parallel
-   //     FiniteElementSpaceHierarchy and additional boundary information. This operator
-   //     is then used to create the MultigridSolver as a preconditioner in the
-   //     iterative solver.
-   Array<int> ess_bdr(pmesh->bdr_attributes.Max());
-   if (pmesh->bdr_attributes.Size())
-   {
-      ess_bdr = 1;
-   }
-
-   DiffusionMultigrid* M = new DiffusionMultigrid(*fespaces, ess_bdr);
-   M->SetCycleType(Multigrid::CycleType::VCYCLE, 1, 1);
-
-   OperatorPtr A;
-   Vector X, B;
-   M->FormFineLinearSystem(x, *b, A, X, B);
-
-   // 11. Solve the linear system A X = B.
-   CGSolver cg(MPI_COMM_WORLD);
-   cg.SetRelTol(1e-12);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(1);
-   cg.SetOperator(*A);
-   cg.SetPreconditioner(*M);
-   cg.Mult(B, X);
-
-   // 12. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   M->RecoverFineFEMSolution(X, *b, x);
-
-   // 13. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      fespaces->GetFinestFESpace().GetParMesh()->Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 14. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *fespaces->GetFinestFESpace().GetParMesh()
-               << x << flush;
-   }
-
-   // 15. Free the used memory.
-   delete M;
-   delete b;
-   delete fespaces;
-   for (int level = 0; level < collections.Size(); ++level)
-   {
-      delete collections[level];
-   }
-
-   MPI_Finalize();
-
-   return 0;
-}
@@ -1,736 +0,0 @@
-//                       MFEM Example 27 - Serial Version
-//
-// Compile with: make ex27
-//
-// Sample runs:  ex27
-//               ex27 -dg
-//               ex27 -dg -dbc 8 -nbc -2
-//               ex27 -rbc-a 1 -rbc-b 8
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 0 with a variety of boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order using a continuous or discontinuous space.  We then
-//               apply Dirichlet, Neumann (both homogeneous and inhomogeneous),
-//               Robin, and Periodic boundary conditions on different portions
-//               of a predefined mesh.
-//
-//               The predefined mesh consists of a rectangle with two
-//               holes removed (see below).  The narrow ends of the
-//               mesh are connected to form a Periodic boundary
-//               condition.  The lower edge (tagged with attribute 1)
-//               receives an inhomogeneous Neumann boundary condition.
-//               A Robin boundary condition is applied to upper edge
-//               (attribute 2).  The circular hole on the left
-//               (attribute 3) enforces a Dirichlet boundary
-//               condition.  Finally, a natural boundary condition, or
-//               homogeneous Neumann BC, is applied to the circular
-//               hole on the right (attribute 4).
-//
-//                  Attribute 3    ^ y  Attribute 2
-//                        \        |      /
-//                     +-----------+-----------+
-//                     |    \_     |     _     |
-//                     |    / \    |    / \    |
-//                  <--+---+---+---+---+---+---+--> x
-//                     |    \_/    |    \_/    |
-//                     |           |      \    |
-//                     +-----------+-----------+       (hole radii are
-//                          /      |        \            adjustable)
-//                  Attribute 1    v    Attribute 4
-//
-//               The boundary conditions are defined as (where u is
-//               the solution field):
-//                  Dirichlet: u = d
-//                  Neumann:   n.Grad(u) = g
-//                  Robin:     n.Grad(u) + a u = b
-//
-//               The user can adjust the values of 'd', 'g', 'a', and
-//               'b' with command line options.
-//
-//               This example highlights the differing implementations of
-//               boundary conditions with continuous and discontinuous Galerkin
-//               formulations of the Laplace problem.
-//
-//               We recommend viewing examples 1 and 14 before viewing this
-//               example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-static double a_ = 0.2;
-
-// Normal to hole with boundary attribute 4
-void n4Vec(const Vector &x, Vector &n) { n = x; n[0] -= 0.5; n /= -n.Norml2(); }
-
-Mesh * GenerateSerialMesh(int ref);
-
-// Compute the average value of alpha*n.Grad(sol) + beta*sol over the boundary
-// attributes marked in bdr_marker.  Also computes the L2 norm of
-// alpha*n.Grad(sol) + beta*sol - gamma  over the same boundary.
-double IntegrateBC(const GridFunction &sol, const Array<int> &bdr_marker,
-                   double alpha, double beta, double gamma,
-                   double &err);
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   int ser_ref_levels = 2;
-   int order = 1;
-   double sigma = -1.0;
-   double kappa = -1.0;
-   bool h1 = true;
-   bool visualization = true;
-
-   double mat_val = 1.0;
-   double dbc_val = 0.0;
-   double nbc_val = 1.0;
-   double rbc_a_val = 1.0; // du/dn + a * u = b
-   double rbc_b_val = 1.0;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&h1, "-h1", "--continuous", "-dg", "--discontinuous",
-                  "Select continuous \"H1\" or discontinuous \"DG\" basis.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&sigma, "-s", "--sigma",
-                  "One of the two DG penalty parameters, typically +1/-1."
-                  " See the documentation of class DGDiffusionIntegrator.");
-   args.AddOption(&kappa, "-k", "--kappa",
-                  "One of the two DG penalty parameters, should be positive."
-                  " Negative values are replaced with (order+1)^2.");
-   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&mat_val, "-mat", "--material-value",
-                  "Constant value for material coefficient "
-                  "in the Laplace operator.");
-   args.AddOption(&dbc_val, "-dbc", "--dirichlet-value",
-                  "Constant value for Dirichlet Boundary Condition.");
-   args.AddOption(&nbc_val, "-nbc", "--neumann-value",
-                  "Constant value for Neumann Boundary Condition.");
-   args.AddOption(&rbc_a_val, "-rbc-a", "--robin-a-value",
-                  "Constant 'a' value for Robin Boundary Condition: "
-                  "du/dn + a * u = b.");
-   args.AddOption(&rbc_b_val, "-rbc-b", "--robin-b-value",
-                  "Constant 'b' value for Robin Boundary Condition: "
-                  "du/dn + a * u = b.");
-   args.AddOption(&a_, "-a", "--radius",
-                  "Radius of holes in the mesh.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(mfem::out);
-      return 1;
-   }
-   if (kappa < 0 && !h1)
-   {
-      kappa = (order+1)*(order+1);
-   }
-   args.PrintOptions(mfem::out);
-
-   if (a_ < 0.01)
-   {
-      mfem::out << "Hole radius too small, resetting to 0.01.\n";
-      a_ = 0.01;
-   }
-   if (a_ > 0.49)
-   {
-      mfem::out << "Hole radius too large, resetting to 0.49.\n";
-      a_ = 0.49;
-   }
-
-   // 2. Construct the (serial) mesh and refine it if requested.
-   Mesh *mesh = GenerateSerialMesh(ser_ref_levels);
-   int dim = mesh->Dimension();
-
-   // 3. Define a finite element space on the serial mesh.  Here we
-   //    use either continuous Lagrange finite elements or discontinuous
-   //    Galerkin finite elements of the specified order.
-   FiniteElementCollection *fec =
-      h1 ? (FiniteElementCollection*)new H1_FECollection(order, dim) :
-      (FiniteElementCollection*)new DG_FECollection(order, dim);
-   FiniteElementSpace fespace(mesh, fec);
-   int size = fespace.GetTrueVSize();
-   mfem::out << "Number of finite element unknowns: " << size << endl;
-
-   // 4. Create "marker arrays" to define the portions of the boundary
-   //    associated with each type of boundary condition.  These arrays
-   //    have an entry corresponding to each boundary attribute.
-   //    Placing a '1' in entry i marks attribute i+1 as being
-   //    active, '0' is inactive.
-   Array<int> nbc_bdr(mesh->bdr_attributes.Max());
-   Array<int> rbc_bdr(mesh->bdr_attributes.Max());
-   Array<int> dbc_bdr(mesh->bdr_attributes.Max());
-
-   nbc_bdr = 0; nbc_bdr[0] = 1;
-   rbc_bdr = 0; rbc_bdr[1] = 1;
-   dbc_bdr = 0; dbc_bdr[2] = 1;
-
-   Array<int> ess_tdof_list(0);
-   if (h1 && mesh->bdr_attributes.Size())
-   {
-      // For a continuous basis the linear system must be modifed to enforce
-      // an essential (Dirichlet) boundary condition.  In the DG case this is
-      // not necessary as the boundary condition will only be enforced weakly.
-      fespace.GetEssentialTrueDofs(dbc_bdr, ess_tdof_list);
-   }
-
-   // 5. Setup the various coefficients needed for the Laplace operator and
-   //    the various boundary conditions.  In general these coefficients could
-   //    be functions of position but here we use only constants.
-   ConstantCoefficient matCoef(mat_val);
-   ConstantCoefficient dbcCoef(dbc_val);
-   ConstantCoefficient nbcCoef(nbc_val);
-   ConstantCoefficient rbcACoef(rbc_a_val);
-   ConstantCoefficient rbcBCoef(rbc_b_val);
-
-   // Since the n.Grad(u) terms arise by integrating -Div(m Grad(u)) by parts
-   // we must introduce the coefficient 'm' into the boundary conditions.
-   // Therefore, in the case of the Neumann BC, we actually enforce
-   // m n.Grad(u) = m g rather than simply n.Grad(u) = g.
-   ProductCoefficient m_nbcCoef(matCoef, nbcCoef);
-   ProductCoefficient m_rbcACoef(matCoef, rbcACoef);
-   ProductCoefficient m_rbcBCoef(matCoef, rbcBCoef);
-
-   // 6. Define the solution vector u as a finite element grid function
-   //    corresponding to fespace. Initialize u with initial guess of zero.
-   GridFunction u(&fespace);
-   u = 0.0;
-
-   // 7. Set up the bilinear form a(.,.) on the finite element space
-   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //    domain integrator.
-   BilinearForm a(&fespace);
-   a.AddDomainIntegrator(new DiffusionIntegrator(matCoef));
-   if (h1)
-   {
-      // Add a Mass integrator on the Robin boundary
-      a.AddBoundaryIntegrator(new MassIntegrator(m_rbcACoef), rbc_bdr);
-   }
-   else
-   {
-      // Add the interfacial portion of the Lapalce operator
-      a.AddInteriorFaceIntegrator(new DGDiffusionIntegrator(matCoef,
-                                                            sigma, kappa));
-
-      // Counteract the n.Grad(u) term on the Dirichlet portion of the boundary
-      a.AddBdrFaceIntegrator(new DGDiffusionIntegrator(matCoef, sigma, kappa),
-                             dbc_bdr);
-
-      // Augment the n.Grad(u) term with a*u on the Robin portion of boundary
-      a.AddBdrFaceIntegrator(new BoundaryMassIntegrator(m_rbcACoef),
-                             rbc_bdr);
-   }
-   a.Assemble();
-
-   // 8. Assemble the linear form for the right hand side vector.
-   LinearForm b(&fespace);
-
-   if (h1)
-   {
-      // Set the Dirchlet values in the solution vector
-      u.ProjectBdrCoefficient(dbcCoef, dbc_bdr);
-
-      // Add the desired value for n.Grad(u) on the Neumann boundary
-      b.AddBoundaryIntegrator(new BoundaryLFIntegrator(m_nbcCoef), nbc_bdr);
-
-      // Add the desired value for n.Grad(u) + a*u on the Robin boundary
-      b.AddBoundaryIntegrator(new BoundaryLFIntegrator(m_rbcBCoef), rbc_bdr);
-   }
-   else
-   {
-      // Add the desired value for the Dirchlet boundary
-      b.AddBdrFaceIntegrator(new DGDirichletLFIntegrator(dbcCoef, matCoef,
-                                                         sigma, kappa),
-                             dbc_bdr);
-
-      // Add the desired value for n.Grad(u) on the Neumann boundary
-      b.AddBdrFaceIntegrator(new BoundaryLFIntegrator(m_nbcCoef),
-                             nbc_bdr);
-
-      // Add the desired value for n.Grad(u) + a*u on the Robin boundary
-      b.AddBdrFaceIntegrator(new BoundaryLFIntegrator(m_rbcBCoef),
-                             rbc_bdr);
-   }
-   b.Assemble();
-
-   // 9. Construct the linear system.
-   OperatorPtr A;
-   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, u, b, A, X, B);
-
-#ifndef MFEM_USE_SUITESPARSE
-   // 10. Define a simple symmetric Gauss-Seidel preconditioner and use it to
-   //     solve the system AX=B with PCG in the symmetric case, and GMRES in the
-   //     non-symmetric one.
-   {
-      GSSmoother M((SparseMatrix&)(*A));
-      if (sigma == -1.0)
-      {
-         PCG(*A, M, B, X, 1, 500, 1e-12, 0.0);
-      }
-      else
-      {
-         GMRES(*A, M, B, X, 1, 500, 10, 1e-12, 0.0);
-      }
-   }
-#else
-   // 11. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the
-   //     system.
-   UMFPackSolver umf_solver;
-   umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-   umf_solver.SetOperator(*A);
-   umf_solver.Mult(B, X);
-#endif
-
-   // 12. Recover the grid function corresponding to U. This is the
-   //     local finite element solution.
-   a.RecoverFEMSolution(X, b, u);
-
-   // 13. Build a mass matrix to help solve for n.Grad(u) where 'n' is
-   //     a surface normal.
-   BilinearForm m(&fespace);
-   m.AddDomainIntegrator(new MassIntegrator);
-   m.Assemble();
-
-   ess_tdof_list.SetSize(0);
-   OperatorPtr M;
-   m.FormSystemMatrix(ess_tdof_list, M);
-
-   // 14. Compute the various boundary integrals.
-   mfem::out << endl
-             << "Verifying boundary conditions" << endl
-             << "=============================" << endl;
-   {
-      // Integrate the solution on the Dirichlet boundary and compare
-      // to the expected value.
-      double err, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, err);
-
-      bool hom_dbc = (dbc_val == 0.0);
-      err /=  hom_dbc ? 1.0 : fabs(dbc_val);
-      mfem::out << "Average of solution on Gamma_dbc:\t"
-                << avg << ", \t"
-                << (hom_dbc ? "absolute" : "relative")
-                << " error " << err << endl;
-   }
-   {
-      // Integrate n.Grad(u) on the inhomogeneous Neumann boundary and
-      // compare to the expected value.
-      double err, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, err);
-
-      bool hom_nbc = (nbc_val == 0.0);
-      err /=  hom_nbc ? 1.0 : fabs(nbc_val);
-      mfem::out << "Average of n.Grad(u) on Gamma_nbc:\t"
-                << avg << ", \t"
-                << (hom_nbc ? "absolute" : "relative")
-                << " error " << err << endl;
-   }
-   {
-      // Integrate n.Grad(u) on the homogeneous Neumann boundary and compare
-      // to the expected value of zero.
-      Array<int> nbc0_bdr(mesh->bdr_attributes.Max());
-      nbc0_bdr = 0;
-      nbc0_bdr[3] = 1;
-
-      double err, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, err);
-
-      bool hom_nbc = true;
-      mfem::out << "Average of n.Grad(u) on Gamma_nbc0:\t"
-                << avg << ", \t"
-                << (hom_nbc ? "absolute" : "relative")
-                << " error " << err << endl;
-   }
-   {
-      // Integrate n.Grad(u) + a * u on the Robin boundary and compare to
-      // the expected value.
-      double err, avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val, err);
-
-      bool hom_rbc = (rbc_b_val == 0.0);
-      err /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
-      mfem::out << "Average of n.Grad(u)+a*u on Gamma_rbc:\t"
-                << avg << ", \t"
-                << (hom_rbc ? "absolute" : "relative")
-                << " error " << err << endl;
-   }
-
-   // 15. Save the refined mesh and the solution. This output can be viewed
-   //     later using GLVis: "glvis -m refined.mesh -g sol.gf".
-   {
-      ofstream mesh_ofs("refined.mesh");
-      mesh_ofs.precision(8);
-      mesh->Print(mesh_ofs);
-      ofstream sol_ofs("sol.gf");
-      sol_ofs.precision(8);
-      u.Save(sol_ofs);
-   }
-
-   // 16. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      string title_str = h1 ? "H1" : "DG";
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *mesh << u
-               << "window_title '" << title_str << " Solution'"
-               << " keys 'mmc'" << flush;
-   }
-
-   // 17. Free the used memory.
-   delete fec;
-   delete mesh;
-
-   return 0;
-}
-
-void quad_trans(double u, double v, double &x, double &y, bool log = false)
-{
-   double a = a_; // Radius of disc
-
-   double d = 4.0 * a * (M_SQRT2 - 2.0 * a) * (1.0 - 2.0 * v);
-
-   double v0 = (1.0 + M_SQRT2) * (M_SQRT2 * a - 2.0 * v) *
-               ((4.0 - 3 * M_SQRT2) * a +
-                (8.0 * (M_SQRT2 - 1.0) * a - 2.0) * v) / d;
-
-   double r = 2.0 * ((M_SQRT2 - 1.0) * a * a * (1.0 - 4.0 *v) +
-                     2.0 * (1.0 + M_SQRT2 *
-                            (1.0 + 2.0 * (2.0 * a - M_SQRT2 - 1.0) * a)) * v * v
-                    ) / d;
-
-   double t = asin(v / r) * u / v;
-   if (log)
-   {
-      mfem::out << "u, v, r, v0, t "
-                << u << " " << v << " " << r << " " << v0 << " " << t
-                << endl;
-   }
-   x = r * sin(t);
-   y = r * cos(t) - v0;
-}
-
-void trans(const Vector &u, Vector &x)
-{
-   double tol = 1e-4;
-
-   if (u[1] > 0.5 - tol || u[1] < -0.5 + tol)
-   {
-      x = u;
-      return;
-   }
-   if (u[0] > 1.0 - tol || u[0] < -1.0 + tol || fabs(u[0]) < tol)
-   {
-      x = u;
-      return;
-   }
-
-   if (u[0] > 0.0)
-   {
-      if (u[1] > fabs(u[0] - 0.5))
-      {
-         quad_trans(u[0] - 0.5, u[1], x[0], x[1]);
-         x[0] += 0.5;
-         return;
-      }
-      if (u[1] < -fabs(u[0] - 0.5))
-      {
-         quad_trans(u[0] - 0.5, -u[1], x[0], x[1]);
-         x[0] += 0.5;
-         x[1] *= -1.0;
-         return;
-      }
-      if (u[0] - 0.5 > fabs(u[1]))
-      {
-         quad_trans(u[1], u[0] - 0.5, x[1], x[0]);
-         x[0] += 0.5;
-         return;
-      }
-      if (u[0] - 0.5 < -fabs(u[1]))
-      {
-         quad_trans(u[1], 0.5 - u[0], x[1], x[0]);
-         x[0] *= -1.0;
-         x[0] += 0.5;
-         return;
-      }
-   }
-   else
-   {
-      if (u[1] > fabs(u[0] + 0.5))
-      {
-         quad_trans(u[0] + 0.5, u[1], x[0], x[1]);
-         x[0] -= 0.5;
-         return;
-      }
-      if (u[1] < -fabs(u[0] + 0.5))
-      {
-         quad_trans(u[0] + 0.5, -u[1], x[0], x[1]);
-         x[0] -= 0.5;
-         x[1] *= -1.0;
-         return;
-      }
-      if (u[0] + 0.5 > fabs(u[1]))
-      {
-         quad_trans(u[1], u[0] + 0.5, x[1], x[0]);
-         x[0] -= 0.5;
-         return;
-      }
-      if (u[0] + 0.5 < -fabs(u[1]))
-      {
-         quad_trans(u[1], -0.5 - u[0], x[1], x[0]);
-         x[0] *= -1.0;
-         x[0] -= 0.5;
-         return;
-      }
-   }
-   x = u;
-}
-
-Mesh * GenerateSerialMesh(int ref)
-{
-   Mesh * mesh = new Mesh(2, 29, 16, 24, 2);
-
-   int vi[4];
-
-   for (int i=0; i<2; i++)
-   {
-      int o = 13 * i;
-      vi[0] = o + 0; vi[1] = o + 3; vi[2] = o + 4; vi[3] = o + 1;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 1; vi[1] = o + 4; vi[2] = o + 5; vi[3] = o + 2;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 5; vi[1] = o + 8; vi[2] = o + 9; vi[3] = o + 2;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 8; vi[1] = o + 12; vi[2] = o + 15; vi[3] = o + 9;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 11; vi[1] = o + 14; vi[2] = o + 15; vi[3] = o + 12;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 10; vi[1] = o + 13; vi[2] = o + 14; vi[3] = o + 11;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 6; vi[1] = o + 13; vi[2] = o + 10; vi[3] = o + 7;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 0; vi[1] = o + 6; vi[2] = o + 7; vi[3] = o + 3;
-      mesh->AddQuad(vi);
-   }
-
-   vi[0] =  0; vi[1] =  6; mesh->AddBdrSegment(vi, 1);
-   vi[0] =  6; vi[1] = 13; mesh->AddBdrSegment(vi, 1);
-   vi[0] = 13; vi[1] = 19; mesh->AddBdrSegment(vi, 1);
-   vi[0] = 19; vi[1] = 26; mesh->AddBdrSegment(vi, 1);
-
-   vi[0] = 28; vi[1] = 22; mesh->AddBdrSegment(vi, 2);
-   vi[0] = 22; vi[1] = 15; mesh->AddBdrSegment(vi, 2);
-   vi[0] = 15; vi[1] =  9; mesh->AddBdrSegment(vi, 2);
-   vi[0] =  9; vi[1] =  2; mesh->AddBdrSegment(vi, 2);
-
-   for (int i=0; i<2; i++)
-   {
-      int o = 13 * i;
-      vi[0] = o +  7; vi[1] = o +  3; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o + 10; vi[1] = o +  7; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o + 11; vi[1] = o + 10; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o + 12; vi[1] = o + 11; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o +  8; vi[1] = o + 12; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o +  5; vi[1] = o +  8; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o +  4; vi[1] = o +  5; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o +  3; vi[1] = o +  4; mesh->AddBdrSegment(vi, 3 + i);
-   }
-
-   double d[2];
-   double a = a_ / M_SQRT2;
-
-   d[0] = -1.0; d[1] = -0.5; mesh->AddVertex(d);
-   d[0] = -1.0; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] = -1.0; d[1] =  0.5; mesh->AddVertex(d);
-
-   d[0] = -0.5 - a; d[1] =   -a; mesh->AddVertex(d);
-   d[0] = -0.5 - a; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] = -0.5 - a; d[1] =    a; mesh->AddVertex(d);
-
-   d[0] = -0.5; d[1] = -0.5; mesh->AddVertex(d);
-   d[0] = -0.5; d[1] =   -a; mesh->AddVertex(d);
-   d[0] = -0.5; d[1] =    a; mesh->AddVertex(d);
-   d[0] = -0.5; d[1] =  0.5; mesh->AddVertex(d);
-
-   d[0] = -0.5 + a; d[1] =   -a; mesh->AddVertex(d);
-   d[0] = -0.5 + a; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] = -0.5 + a; d[1] =    a; mesh->AddVertex(d);
-
-   d[0] =  0.0; d[1] = -0.5; mesh->AddVertex(d);
-   d[0] =  0.0; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] =  0.0; d[1] =  0.5; mesh->AddVertex(d);
-
-   d[0] =  0.5 - a; d[1] =   -a; mesh->AddVertex(d);
-   d[0] =  0.5 - a; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] =  0.5 - a; d[1] =    a; mesh->AddVertex(d);
-
-   d[0] =  0.5; d[1] = -0.5; mesh->AddVertex(d);
-   d[0] =  0.5; d[1] =   -a; mesh->AddVertex(d);
-   d[0] =  0.5; d[1] =    a; mesh->AddVertex(d);
-   d[0] =  0.5; d[1] =  0.5; mesh->AddVertex(d);
-
-   d[0] =  0.5 + a; d[1] =   -a; mesh->AddVertex(d);
-   d[0] =  0.5 + a; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] =  0.5 + a; d[1] =    a; mesh->AddVertex(d);
-
-   d[0] =  1.0; d[1] = -0.5; mesh->AddVertex(d);
-   d[0] =  1.0; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] =  1.0; d[1] =  0.5; mesh->AddVertex(d);
-
-   mesh->FinalizeTopology();
-
-   mesh->SetCurvature(1, true);
-
-   // Stitch the ends of the stack together
-   {
-      Array<int> v2v(mesh->GetNV());
-      for (int i = 0; i < v2v.Size() - 3; i++)
-      {
-         v2v[i] = i;
-      }
-      // identify vertices on the narrow ends of the rectangle
-      v2v[v2v.Size() - 3] = 0;
-      v2v[v2v.Size() - 2] = 1;
-      v2v[v2v.Size() - 1] = 2;
-
-      // renumber elements
-      for (int i = 0; i < mesh->GetNE(); i++)
-      {
-         Element *el = mesh->GetElement(i);
-         int *v = el->GetVertices();
-         int nv = el->GetNVertices();
-         for (int j = 0; j < nv; j++)
-         {
-            v[j] = v2v[v[j]];
-         }
-      }
-      // renumber boundary elements
-      for (int i = 0; i < mesh->GetNBE(); i++)
-      {
-         Element *el = mesh->GetBdrElement(i);
-         int *v = el->GetVertices();
-         int nv = el->GetNVertices();
-         for (int j = 0; j < nv; j++)
-         {
-            v[j] = v2v[v[j]];
-         }
-      }
-      mesh->RemoveUnusedVertices();
-      mesh->RemoveInternalBoundaries();
-   }
-   mesh->SetCurvature(3, true);
-
-   for (int l = 0; l < ref; l++)
-   {
-      mesh->UniformRefinement();
-   }
-
-   mesh->Transform(trans);
-
-   return mesh;
-}
-
-double IntegrateBC(const GridFunction &x, const Array<int> &bdr,
-                   double alpha, double beta, double gamma,
-                   double &err)
-{
-   double nrm = 0.0;
-   double avg = 0.0;
-   err = 0.0;
-
-   const bool a_is_zero = alpha == 0.0;
-   const bool b_is_zero = beta == 0.0;
-
-   const FiniteElementSpace &fes = *x.FESpace();
-   MFEM_ASSERT(fes.GetVDim() == 1, "");
-   Mesh &mesh = *fes.GetMesh();
-   Vector shape, loc_dofs, w_nor;
-   DenseMatrix dshape;
-   Array<int> dof_ids;
-   for (int i = 0; i < mesh.GetNBE(); i++)
-   {
-      if (bdr[mesh.GetBdrAttribute(i)-1] == 0) { continue; }
-
-      FaceElementTransformations *FTr = mesh.GetBdrFaceTransformations(i);
-      if (FTr == nullptr) { continue; }
-
-      const FiniteElement &fe = *fes.GetFE(FTr->Elem1No);
-      MFEM_ASSERT(fe.GetMapType() == FiniteElement::VALUE, "");
-      const int int_order = 2*fe.GetOrder() + 3;
-      const IntegrationRule &ir = IntRules.Get(FTr->FaceGeom, int_order);
-
-      fes.GetElementDofs(FTr->Elem1No, dof_ids);
-      x.GetSubVector(dof_ids, loc_dofs);
-      if (!a_is_zero)
-      {
-         const int sdim = FTr->Face->GetSpaceDim();
-         w_nor.SetSize(sdim);
-         dshape.SetSize(fe.GetDof(), sdim);
-      }
-      if (!b_is_zero)
-      {
-         shape.SetSize(fe.GetDof());
-      }
-      for (int j = 0; j < ir.GetNPoints(); j++)
-      {
-         const IntegrationPoint &ip = ir.IntPoint(j);
-         IntegrationPoint eip;
-         FTr->Loc1.Transform(ip, eip);
-         FTr->Face->SetIntPoint(&ip);
-         double face_weight = FTr->Face->Weight();
-         double val = 0.0;
-         if (!a_is_zero)
-         {
-            FTr->Elem1->SetIntPoint(&eip);
-            fe.CalcPhysDShape(*FTr->Elem1, dshape);
-            CalcOrtho(FTr->Face->Jacobian(), w_nor);
-            val += alpha * dshape.InnerProduct(w_nor, loc_dofs) / face_weight;
-         }
-         if (!b_is_zero)
-         {
-            fe.CalcShape(eip, shape);
-            val += beta * (shape * loc_dofs);
-         }
-
-         // Measure the length of the boundary
-         nrm += ip.weight * face_weight;
-
-         // Integrate alpha * n.Grad(x) + beta * x
-         avg += val * ip.weight * face_weight;
-
-         // Integrate |alpha * n.Grad(x) + beta * x - gamma|^2
-         val -= gamma;
-         err += (val*val) * ip.weight * face_weight;
-      }
-   }
-
-   // Normalize by the length of the boundary
-   if (std::abs(nrm) > 0.0)
-   {
-      err /= nrm;
-      avg /= nrm;
-   }
-
-   // Compute l2 norm of the error in the boundary condition
-   // (negative quadrature weights may produce negative 'err')
-   err = (err >= 0.0) ? sqrt(err) : -sqrt(-err);
-
-   // Return the average value of alpha * n.Grad(x) + beta * x
-   return avg;
-}
@@ -1,773 +0,0 @@
-//                       MFEM Example 27 - Parallel Version
-//
-// Compile with: make ex27p
-//
-// Sample runs:  mpirun -np 4 ex27p
-//               mpirun -np 4 ex27p -dg
-//               mpirun -np 4 ex27p -dg -dbc 8 -nbc -2
-//               mpirun -np 4 ex27p -rbc-a 1 -rbc-b 8
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 0 with a variety of boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order using a continuous or discontinuous space.  We then
-//               apply Dirichlet, Neumann (both homogeneous and inhomogeneous),
-//               Robin, and Periodic boundary conditions on different portions
-//               of a predefined mesh.
-//
-//               The predefined mesh consists of a rectangle with two
-//               holes removed (see below).  The narrow ends of the
-//               mesh are connected to form a Periodic boundary
-//               condition.  The lower edge (tagged with attribute 1)
-//               receives an inhomogeneous Neumann boundary condition.
-//               A Robin boundary condition is applied to upper edge
-//               (attribute 2).  The circular hole on the left
-//               (attribute 3) enforces a Dirichlet boundary
-//               condition.  Finally, a natural boundary condition, or
-//               homogeneous Neumann BC, is applied to the circular
-//               hole on the right (attribute 4).
-//
-//                  Attribute 3    ^ y  Attribute 2
-//                        \        |      /
-//                     +-----------+-----------+
-//                     |    \_     |     _     |
-//                     |    / \    |    / \    |
-//                  <--+---+---+---+---+---+---+--> x
-//                     |    \_/    |    \_/    |
-//                     |           |      \    |
-//                     +-----------+-----------+       (hole radii are
-//                          /      |        \            adjustable)
-//                  Attribute 1    v    Attribute 4
-//
-//               The boundary conditions are defined as (where u is
-//               the solution field):
-//                  Dirichlet: u = d
-//                  Neumann:   n.Grad(u) = g
-//                  Robin:     n.Grad(u) + a u = b
-//
-//               The user can adjust the values of 'd', 'g', 'a', and
-//               'b' with command line options.
-//
-//               This example highlights the differing implementations of
-//               boundary conditions with continuous and discontinuous Galerkin
-//               formulations of the Laplace problem.
-//
-//               We recommend viewing examples 1 and 14 before viewing this
-//               example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-static double a_ = 0.2;
-
-// Normal to hole with boundary attribute 4
-void n4Vec(const Vector &x, Vector &n) { n = x; n[0] -= 0.5; n /= -n.Norml2(); }
-
-Mesh * GenerateSerialMesh(int ref);
-
-// Compute the average value of alpha*n.Grad(sol) + beta*sol over the boundary
-// attributes marked in bdr_marker.  Also computes the L2 norm of
-// alpha*n.Grad(sol) + beta*sol - gamma  over the same boundary.
-double IntegrateBC(const ParGridFunction &sol, const Array<int> &bdr_marker,
-                   double alpha, double beta, double gamma,
-                   double &err);
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   MPI_Session mpi;
-   if (!mpi.Root()) { mfem::out.Disable(); mfem::err.Disable(); }
-
-   // 2. Parse command-line options.
-   int ser_ref_levels = 2;
-   int par_ref_levels = 1;
-   int order = 1;
-   double sigma = -1.0;
-   double kappa = -1.0;
-   bool h1 = true;
-   bool visualization = true;
-
-   double mat_val = 1.0;
-   double dbc_val = 0.0;
-   double nbc_val = 1.0;
-   double rbc_a_val = 1.0; // du/dn + a * u = b
-   double rbc_b_val = 1.0;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&h1, "-h1", "--continuous", "-dg", "--discontinuous",
-                  "Select continuous \"H1\" or discontinuous \"DG\" basis.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&sigma, "-s", "--sigma",
-                  "One of the two DG penalty parameters, typically +1/-1."
-                  " See the documentation of class DGDiffusionIntegrator.");
-   args.AddOption(&kappa, "-k", "--kappa",
-                  "One of the two DG penalty parameters, should be positive."
-                  " Negative values are replaced with (order+1)^2.");
-   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
-                  "Number of times to refine the mesh uniformly in parallel.");
-   args.AddOption(&mat_val, "-mat", "--material-value",
-                  "Constant value for material coefficient "
-                  "in the Laplace operator.");
-   args.AddOption(&dbc_val, "-dbc", "--dirichlet-value",
-                  "Constant value for Dirichlet Boundary Condition.");
-   args.AddOption(&nbc_val, "-nbc", "--neumann-value",
-                  "Constant value for Neumann Boundary Condition.");
-   args.AddOption(&rbc_a_val, "-rbc-a", "--robin-a-value",
-                  "Constant 'a' value for Robin Boundary Condition: "
-                  "du/dn + a * u = b.");
-   args.AddOption(&rbc_b_val, "-rbc-b", "--robin-b-value",
-                  "Constant 'b' value for Robin Boundary Condition: "
-                  "du/dn + a * u = b.");
-   args.AddOption(&a_, "-a", "--radius",
-                  "Radius of holes in the mesh.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(mfem::out);
-      return 1;
-   }
-   if (kappa < 0 && !h1)
-   {
-      kappa = (order+1)*(order+1);
-   }
-   args.PrintOptions(mfem::out);
-
-   if (a_ < 0.01)
-   {
-      mfem::out << "Hole radius too small, resetting to 0.01.\n";
-      a_ = 0.01;
-   }
-   if (a_ > 0.49)
-   {
-      mfem::out << "Hole radius too large, resetting to 0.49.\n";
-      a_ = 0.49;
-   }
-
-   // 3. Construct the (serial) mesh and refine it if requested.
-   Mesh *mesh = GenerateSerialMesh(ser_ref_levels);
-   int dim = mesh->Dimension();
-
-   // 4. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   for (int l = 0; l < par_ref_levels; l++)
-   {
-      pmesh.UniformRefinement();
-   }
-
-   // 5. Define a parallel finite element space on the parallel mesh. Here we
-   //    use either continuous Lagrange finite elements or discontinuous
-   //    Galerkin finite elements of the specified order.
-   FiniteElementCollection *fec =
-      h1 ? (FiniteElementCollection*)new H1_FECollection(order, dim) :
-      (FiniteElementCollection*)new DG_FECollection(order, dim);
-   ParFiniteElementSpace fespace(&pmesh, fec);
-   HYPRE_Int size = fespace.GlobalTrueVSize();
-   mfem::out << "Number of finite element unknowns: " << size << endl;
-
-   // 6. Create "marker arrays" to define the portions of the boundary
-   //    associated with each type of boundary condition.  These arrays
-   //    have an entry corresponding to each boundary attribute.
-   //    Placing a '1' in entry i marks attribute i+1 as being
-   //    active, '0' is inactive.
-   Array<int> nbc_bdr(pmesh.bdr_attributes.Max());
-   Array<int> rbc_bdr(pmesh.bdr_attributes.Max());
-   Array<int> dbc_bdr(pmesh.bdr_attributes.Max());
-
-   nbc_bdr = 0; nbc_bdr[0] = 1;
-   rbc_bdr = 0; rbc_bdr[1] = 1;
-   dbc_bdr = 0; dbc_bdr[2] = 1;
-
-   Array<int> ess_tdof_list(0);
-   if (h1 && pmesh.bdr_attributes.Size())
-   {
-      // For a continuous basis the linear system must be modifed to enforce
-      // an essential (Dirichlet) boundary condition.  In the DG case this is
-      // not necessary as the boundary condition will only be enforced weakly.
-      fespace.GetEssentialTrueDofs(dbc_bdr, ess_tdof_list);
-   }
-
-   // 7. Setup the various coefficients needed for the Laplace operator and
-   //    the various boundary conditions.  In general these coefficients could
-   //    be functions of position but here we use only constants.
-   ConstantCoefficient matCoef(mat_val);
-   ConstantCoefficient dbcCoef(dbc_val);
-   ConstantCoefficient nbcCoef(nbc_val);
-   ConstantCoefficient rbcACoef(rbc_a_val);
-   ConstantCoefficient rbcBCoef(rbc_b_val);
-
-   // Since the n.Grad(u) terms arise by integrating -Div(m Grad(u)) by parts
-   // we must introduce the coefficient 'm' into the boundary conditions.
-   // Therefore, in the case of the Neumann BC, we actually enforce
-   // m n.Grad(u) = m g rather than simply n.Grad(u) = g.
-   ProductCoefficient m_nbcCoef(matCoef, nbcCoef);
-   ProductCoefficient m_rbcACoef(matCoef, rbcACoef);
-   ProductCoefficient m_rbcBCoef(matCoef, rbcBCoef);
-
-   // 8. Define the solution vector u as a parallel finite element grid function
-   //    corresponding to fespace. Initialize u with initial guess of zero.
-   ParGridFunction u(&fespace);
-   u = 0.0;
-
-   // 9. Set up the parallel bilinear form a(.,.) on the finite element space
-   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //    domain integrator.
-   ParBilinearForm a(&fespace);
-   a.AddDomainIntegrator(new DiffusionIntegrator(matCoef));
-   if (h1)
-   {
-      // Add a Mass integrator on the Robin boundary
-      a.AddBoundaryIntegrator(new MassIntegrator(m_rbcACoef), rbc_bdr);
-   }
-   else
-   {
-      // Add the interfacial portion of the Lapalce operator
-      a.AddInteriorFaceIntegrator(new DGDiffusionIntegrator(matCoef,
-                                                            sigma, kappa));
-
-      // Counteract the n.Grad(u) term on the Dirichlet portion of the boundary
-      a.AddBdrFaceIntegrator(new DGDiffusionIntegrator(matCoef, sigma, kappa),
-                             dbc_bdr);
-
-      // Augment the n.Grad(u) term with a*u on the Robin portion of boundary
-      a.AddBdrFaceIntegrator(new BoundaryMassIntegrator(m_rbcACoef),
-                             rbc_bdr);
-   }
-   a.Assemble();
-
-   // 10. Assemble the parallel linear form for the right hand side vector.
-   ParLinearForm b(&fespace);
-
-   if (h1)
-   {
-      // Set the Dirchlet values in the solution vector
-      u.ProjectBdrCoefficient(dbcCoef, dbc_bdr);
-
-      // Add the desired value for n.Grad(u) on the Neumann boundary
-      b.AddBoundaryIntegrator(new BoundaryLFIntegrator(m_nbcCoef), nbc_bdr);
-
-      // Add the desired value for n.Grad(u) + a*u on the Robin boundary
-      b.AddBoundaryIntegrator(new BoundaryLFIntegrator(m_rbcBCoef), rbc_bdr);
-   }
-   else
-   {
-      // Add the desired value for the Dirchlet boundary
-      b.AddBdrFaceIntegrator(new DGDirichletLFIntegrator(dbcCoef, matCoef,
-                                                         sigma, kappa),
-                             dbc_bdr);
-
-      // Add the desired value for n.Grad(u) on the Neumann boundary
-      b.AddBdrFaceIntegrator(new BoundaryLFIntegrator(m_nbcCoef),
-                             nbc_bdr);
-
-      // Add the desired value for n.Grad(u) + a*u on the Robin boundary
-      b.AddBdrFaceIntegrator(new BoundaryLFIntegrator(m_rbcBCoef),
-                             rbc_bdr);
-   }
-   b.Assemble();
-
-   // 11. Construct the linear system.
-   OperatorPtr A;
-   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, u, b, A, X, B);
-
-   // 12. Solve the linear system A X = B.
-   HypreSolver *amg = new HypreBoomerAMG;
-   if (h1 || sigma == -1.0)
-   {
-      HyprePCG pcg(MPI_COMM_WORLD);
-      pcg.SetTol(1e-12);
-      pcg.SetMaxIter(200);
-      pcg.SetPrintLevel(2);
-      pcg.SetPreconditioner(*amg);
-      pcg.SetOperator(*A);
-      pcg.Mult(B, X);
-   }
-   else
-   {
-      GMRESSolver gmres(MPI_COMM_WORLD);
-      gmres.SetAbsTol(0.0);
-      gmres.SetRelTol(1e-12);
-      gmres.SetMaxIter(200);
-      gmres.SetKDim(10);
-      gmres.SetPrintLevel(1);
-      gmres.SetPreconditioner(*amg);
-      gmres.SetOperator(*A);
-      gmres.Mult(B, X);
-   }
-   delete amg;
-
-   // 13. Recover the parallel grid function corresponding to U. This is the
-   //     local finite element solution on each processor.
-   a.RecoverFEMSolution(X, b, u);
-
-   // 14. Build a mass matrix to help solve for n.Grad(u) where 'n' is
-   //     a surface normal.
-   ParBilinearForm m(&fespace);
-   m.AddDomainIntegrator(new MassIntegrator);
-   m.Assemble();
-
-   ess_tdof_list.SetSize(0);
-   OperatorPtr M;
-   m.FormSystemMatrix(ess_tdof_list, M);
-
-   // 15. Compute the various boundary integrals.
-   mfem::out << endl
-             << "Verifying boundary conditions" << endl
-             << "=============================" << endl;
-   {
-      // Integrate the solution on the Dirichlet boundary and compare
-      // to the expected value.
-      double err, avg = IntegrateBC(u, dbc_bdr, 0.0, 1.0, dbc_val, err);
-
-      bool hom_dbc = (dbc_val == 0.0);
-      err /=  hom_dbc ? 1.0 : fabs(dbc_val);
-      mfem::out << "Average of solution on Gamma_dbc:\t"
-                << avg << ", \t"
-                << (hom_dbc ? "absolute" : "relative")
-                << " error " << err << endl;
-   }
-   {
-      // Integrate n.Grad(u) on the inhomogeneous Neumann boundary and
-      // compare to the expected value.
-      double err, avg = IntegrateBC(u, nbc_bdr, 1.0, 0.0, nbc_val, err);
-
-      bool hom_nbc = (nbc_val == 0.0);
-      err /=  hom_nbc ? 1.0 : fabs(nbc_val);
-      mfem::out << "Average of n.Grad(u) on Gamma_nbc:\t"
-                << avg << ", \t"
-                << (hom_nbc ? "absolute" : "relative")
-                << " error " << err << endl;
-   }
-   {
-      // Integrate n.Grad(u) on the homogeneous Neumann boundary and compare
-      // to the expected value of zero.
-      Array<int> nbc0_bdr(pmesh.bdr_attributes.Max());
-      nbc0_bdr = 0;
-      nbc0_bdr[3] = 1;
-
-      double err, avg = IntegrateBC(u, nbc0_bdr, 1.0, 0.0, 0.0, err);
-
-      bool hom_nbc = true;
-      mfem::out << "Average of n.Grad(u) on Gamma_nbc0:\t"
-                << avg << ", \t"
-                << (hom_nbc ? "absolute" : "relative")
-                << " error " << err << endl;
-   }
-   {
-      // Integrate n.Grad(u) + a * u on the Robin boundary and compare to
-      // the expected value.
-      double err, avg = IntegrateBC(u, rbc_bdr, 1.0, rbc_a_val, rbc_b_val, err);
-
-      bool hom_rbc = (rbc_b_val == 0.0);
-      err /=  hom_rbc ? 1.0 : fabs(rbc_b_val);
-      mfem::out << "Average of n.Grad(u)+a*u on Gamma_rbc:\t"
-                << avg << ", \t"
-                << (hom_rbc ? "absolute" : "relative")
-                << " error " << err << endl;
-   }
-
-   // 16. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << mpi.WorldRank();
-      sol_name << "sol." << setfill('0') << setw(6) << mpi.WorldRank();
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh.Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      u.Save(sol_ofs);
-   }
-
-   // 17. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      string title_str = h1 ? "H1" : "DG";
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << mpi.WorldSize()
-               << " " << mpi.WorldRank() << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << pmesh << u
-               << "window_title '" << title_str << " Solution'"
-               << " keys 'mmc'" << flush;
-   }
-
-   // 18. Free the used memory.
-   delete fec;
-
-   return 0;
-}
-
-void quad_trans(double u, double v, double &x, double &y, bool log = false)
-{
-   double a = a_; // Radius of disc
-
-   double d = 4.0 * a * (M_SQRT2 - 2.0 * a) * (1.0 - 2.0 * v);
-
-   double v0 = (1.0 + M_SQRT2) * (M_SQRT2 * a - 2.0 * v) *
-               ((4.0 - 3 * M_SQRT2) * a +
-                (8.0 * (M_SQRT2 - 1.0) * a - 2.0) * v) / d;
-
-   double r = 2.0 * ((M_SQRT2 - 1.0) * a * a * (1.0 - 4.0 *v) +
-                     2.0 * (1.0 + M_SQRT2 *
-                            (1.0 + 2.0 * (2.0 * a - M_SQRT2 - 1.0) * a)) * v * v
-                    ) / d;
-
-   double t = asin(v / r) * u / v;
-   if (log)
-   {
-      mfem::out << "u, v, r, v0, t "
-                << u << " " << v << " " << r << " " << v0 << " " << t
-                << endl;
-   }
-   x = r * sin(t);
-   y = r * cos(t) - v0;
-}
-
-void trans(const Vector &u, Vector &x)
-{
-   double tol = 1e-4;
-
-   if (u[1] > 0.5 - tol || u[1] < -0.5 + tol)
-   {
-      x = u;
-      return;
-   }
-   if (u[0] > 1.0 - tol || u[0] < -1.0 + tol || fabs(u[0]) < tol)
-   {
-      x = u;
-      return;
-   }
-
-   if (u[0] > 0.0)
-   {
-      if (u[1] > fabs(u[0] - 0.5))
-      {
-         quad_trans(u[0] - 0.5, u[1], x[0], x[1]);
-         x[0] += 0.5;
-         return;
-      }
-      if (u[1] < -fabs(u[0] - 0.5))
-      {
-         quad_trans(u[0] - 0.5, -u[1], x[0], x[1]);
-         x[0] += 0.5;
-         x[1] *= -1.0;
-         return;
-      }
-      if (u[0] - 0.5 > fabs(u[1]))
-      {
-         quad_trans(u[1], u[0] - 0.5, x[1], x[0]);
-         x[0] += 0.5;
-         return;
-      }
-      if (u[0] - 0.5 < -fabs(u[1]))
-      {
-         quad_trans(u[1], 0.5 - u[0], x[1], x[0]);
-         x[0] *= -1.0;
-         x[0] += 0.5;
-         return;
-      }
-   }
-   else
-   {
-      if (u[1] > fabs(u[0] + 0.5))
-      {
-         quad_trans(u[0] + 0.5, u[1], x[0], x[1]);
-         x[0] -= 0.5;
-         return;
-      }
-      if (u[1] < -fabs(u[0] + 0.5))
-      {
-         quad_trans(u[0] + 0.5, -u[1], x[0], x[1]);
-         x[0] -= 0.5;
-         x[1] *= -1.0;
-         return;
-      }
-      if (u[0] + 0.5 > fabs(u[1]))
-      {
-         quad_trans(u[1], u[0] + 0.5, x[1], x[0]);
-         x[0] -= 0.5;
-         return;
-      }
-      if (u[0] + 0.5 < -fabs(u[1]))
-      {
-         quad_trans(u[1], -0.5 - u[0], x[1], x[0]);
-         x[0] *= -1.0;
-         x[0] -= 0.5;
-         return;
-      }
-   }
-   x = u;
-}
-
-Mesh * GenerateSerialMesh(int ref)
-{
-   Mesh * mesh = new Mesh(2, 29, 16, 24, 2);
-
-   int vi[4];
-
-   for (int i=0; i<2; i++)
-   {
-      int o = 13 * i;
-      vi[0] = o + 0; vi[1] = o + 3; vi[2] = o + 4; vi[3] = o + 1;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 1; vi[1] = o + 4; vi[2] = o + 5; vi[3] = o + 2;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 5; vi[1] = o + 8; vi[2] = o + 9; vi[3] = o + 2;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 8; vi[1] = o + 12; vi[2] = o + 15; vi[3] = o + 9;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 11; vi[1] = o + 14; vi[2] = o + 15; vi[3] = o + 12;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 10; vi[1] = o + 13; vi[2] = o + 14; vi[3] = o + 11;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 6; vi[1] = o + 13; vi[2] = o + 10; vi[3] = o + 7;
-      mesh->AddQuad(vi);
-
-      vi[0] = o + 0; vi[1] = o + 6; vi[2] = o + 7; vi[3] = o + 3;
-      mesh->AddQuad(vi);
-   }
-
-   vi[0] =  0; vi[1] =  6; mesh->AddBdrSegment(vi, 1);
-   vi[0] =  6; vi[1] = 13; mesh->AddBdrSegment(vi, 1);
-   vi[0] = 13; vi[1] = 19; mesh->AddBdrSegment(vi, 1);
-   vi[0] = 19; vi[1] = 26; mesh->AddBdrSegment(vi, 1);
-
-   vi[0] = 28; vi[1] = 22; mesh->AddBdrSegment(vi, 2);
-   vi[0] = 22; vi[1] = 15; mesh->AddBdrSegment(vi, 2);
-   vi[0] = 15; vi[1] =  9; mesh->AddBdrSegment(vi, 2);
-   vi[0] =  9; vi[1] =  2; mesh->AddBdrSegment(vi, 2);
-
-   for (int i=0; i<2; i++)
-   {
-      int o = 13 * i;
-      vi[0] = o +  7; vi[1] = o +  3; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o + 10; vi[1] = o +  7; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o + 11; vi[1] = o + 10; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o + 12; vi[1] = o + 11; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o +  8; vi[1] = o + 12; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o +  5; vi[1] = o +  8; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o +  4; vi[1] = o +  5; mesh->AddBdrSegment(vi, 3 + i);
-      vi[0] = o +  3; vi[1] = o +  4; mesh->AddBdrSegment(vi, 3 + i);
-   }
-
-   double d[2];
-   double a = a_ / M_SQRT2;
-
-   d[0] = -1.0; d[1] = -0.5; mesh->AddVertex(d);
-   d[0] = -1.0; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] = -1.0; d[1] =  0.5; mesh->AddVertex(d);
-
-   d[0] = -0.5 - a; d[1] =   -a; mesh->AddVertex(d);
-   d[0] = -0.5 - a; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] = -0.5 - a; d[1] =    a; mesh->AddVertex(d);
-
-   d[0] = -0.5; d[1] = -0.5; mesh->AddVertex(d);
-   d[0] = -0.5; d[1] =   -a; mesh->AddVertex(d);
-   d[0] = -0.5; d[1] =    a; mesh->AddVertex(d);
-   d[0] = -0.5; d[1] =  0.5; mesh->AddVertex(d);
-
-   d[0] = -0.5 + a; d[1] =   -a; mesh->AddVertex(d);
-   d[0] = -0.5 + a; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] = -0.5 + a; d[1] =    a; mesh->AddVertex(d);
-
-   d[0] =  0.0; d[1] = -0.5; mesh->AddVertex(d);
-   d[0] =  0.0; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] =  0.0; d[1] =  0.5; mesh->AddVertex(d);
-
-   d[0] =  0.5 - a; d[1] =   -a; mesh->AddVertex(d);
-   d[0] =  0.5 - a; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] =  0.5 - a; d[1] =    a; mesh->AddVertex(d);
-
-   d[0] =  0.5; d[1] = -0.5; mesh->AddVertex(d);
-   d[0] =  0.5; d[1] =   -a; mesh->AddVertex(d);
-   d[0] =  0.5; d[1] =    a; mesh->AddVertex(d);
-   d[0] =  0.5; d[1] =  0.5; mesh->AddVertex(d);
-
-   d[0] =  0.5 + a; d[1] =   -a; mesh->AddVertex(d);
-   d[0] =  0.5 + a; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] =  0.5 + a; d[1] =    a; mesh->AddVertex(d);
-
-   d[0] =  1.0; d[1] = -0.5; mesh->AddVertex(d);
-   d[0] =  1.0; d[1] =  0.0; mesh->AddVertex(d);
-   d[0] =  1.0; d[1] =  0.5; mesh->AddVertex(d);
-
-   mesh->FinalizeTopology();
-
-   mesh->SetCurvature(1, true);
-
-   // Stitch the ends of the stack together
-   {
-      Array<int> v2v(mesh->GetNV());
-      for (int i = 0; i < v2v.Size() - 3; i++)
-      {
-         v2v[i] = i;
-      }
-      // identify vertices on the narrow ends of the rectangle
-      v2v[v2v.Size() - 3] = 0;
-      v2v[v2v.Size() - 2] = 1;
-      v2v[v2v.Size() - 1] = 2;
-
-      // renumber elements
-      for (int i = 0; i < mesh->GetNE(); i++)
-      {
-         Element *el = mesh->GetElement(i);
-         int *v = el->GetVertices();
-         int nv = el->GetNVertices();
-         for (int j = 0; j < nv; j++)
-         {
-            v[j] = v2v[v[j]];
-         }
-      }
-      // renumber boundary elements
-      for (int i = 0; i < mesh->GetNBE(); i++)
-      {
-         Element *el = mesh->GetBdrElement(i);
-         int *v = el->GetVertices();
-         int nv = el->GetNVertices();
-         for (int j = 0; j < nv; j++)
-         {
-            v[j] = v2v[v[j]];
-         }
-      }
-      mesh->RemoveUnusedVertices();
-      mesh->RemoveInternalBoundaries();
-   }
-   mesh->SetCurvature(3, true);
-
-   for (int l = 0; l < ref; l++)
-   {
-      mesh->UniformRefinement();
-   }
-
-   mesh->Transform(trans);
-
-   return mesh;
-}
-
-double IntegrateBC(const ParGridFunction &x, const Array<int> &bdr,
-                   double alpha, double beta, double gamma,
-                   double &glb_err)
-{
-   double loc_vals[3];
-   double &nrm = loc_vals[0];
-   double &avg = loc_vals[1];
-   double &err = loc_vals[2];
-
-   nrm = 0.0;
-   avg = 0.0;
-   err = 0.0;
-
-   const bool a_is_zero = alpha == 0.0;
-   const bool b_is_zero = beta == 0.0;
-
-   const ParFiniteElementSpace &fes = *x.ParFESpace();
-   MFEM_ASSERT(fes.GetVDim() == 1, "");
-   ParMesh &mesh = *fes.GetParMesh();
-   Vector shape, loc_dofs, w_nor;
-   DenseMatrix dshape;
-   Array<int> dof_ids;
-   for (int i = 0; i < mesh.GetNBE(); i++)
-   {
-      if (bdr[mesh.GetBdrAttribute(i)-1] == 0) { continue; }
-
-      FaceElementTransformations *FTr = mesh.GetBdrFaceTransformations(i);
-      if (FTr == nullptr) { continue; }
-
-      const FiniteElement &fe = *fes.GetFE(FTr->Elem1No);
-      MFEM_ASSERT(fe.GetMapType() == FiniteElement::VALUE, "");
-      const int int_order = 2*fe.GetOrder() + 3;
-      const IntegrationRule &ir = IntRules.Get(FTr->FaceGeom, int_order);
-
-      fes.GetElementDofs(FTr->Elem1No, dof_ids);
-      x.GetSubVector(dof_ids, loc_dofs);
-      if (!a_is_zero)
-      {
-         const int sdim = FTr->Face->GetSpaceDim();
-         w_nor.SetSize(sdim);
-         dshape.SetSize(fe.GetDof(), sdim);
-      }
-      if (!b_is_zero)
-      {
-         shape.SetSize(fe.GetDof());
-      }
-      for (int j = 0; j < ir.GetNPoints(); j++)
-      {
-         const IntegrationPoint &ip = ir.IntPoint(j);
-         IntegrationPoint eip;
-         FTr->Loc1.Transform(ip, eip);
-         FTr->Face->SetIntPoint(&ip);
-         double face_weight = FTr->Face->Weight();
-         double val = 0.0;
-         if (!a_is_zero)
-         {
-            FTr->Elem1->SetIntPoint(&eip);
-            fe.CalcPhysDShape(*FTr->Elem1, dshape);
-            CalcOrtho(FTr->Face->Jacobian(), w_nor);
-            val += alpha * dshape.InnerProduct(w_nor, loc_dofs) / face_weight;
-         }
-         if (!b_is_zero)
-         {
-            fe.CalcShape(eip, shape);
-            val += beta * (shape * loc_dofs);
-         }
-
-         // Measure the length of the boundary
-         nrm += ip.weight * face_weight;
-
-         // Integrate alpha * n.Grad(x) + beta * x
-         avg += val * ip.weight * face_weight;
-
-         // Integrate |alpha * n.Grad(x) + beta * x - gamma|^2
-         val -= gamma;
-         err += (val*val) * ip.weight * face_weight;
-      }
-   }
-
-   double glb_vals[3];
-   MPI_Allreduce(loc_vals, glb_vals, 3, MPI_DOUBLE, MPI_SUM, fes.GetComm());
-
-   double glb_nrm = glb_vals[0];
-   double glb_avg = glb_vals[1];
-   glb_err = glb_vals[2];
-
-   // Normalize by the length of the boundary
-   if (std::abs(glb_nrm) > 0.0)
-   {
-      glb_err /= glb_nrm;
-      glb_avg /= glb_nrm;
-   }
-
-   // Compute l2 norm of the error in the boundary condition
-   // (negative quadrature weights may produce negative 'err')
-   glb_err = (glb_err >= 0.0) ? sqrt(glb_err) : -sqrt(-glb_err);
-
-   // Return the average value of alpha * n.Grad(x) + beta * x
-   return glb_avg;
-}
@@ -6,7 +6,6 @@
 //               ex4 -m ../data/star.mesh
 //               ex4 -m ../data/beam-tet.mesh
 //               ex4 -m ../data/beam-hex.mesh
-//               ex4 -m ../data/beam-hex.mesh -o 2 -pa
 //               ex4 -m ../data/escher.mesh
 //               ex4 -m ../data/fichera.mesh -o 2 -hb
 //               ex4 -m ../data/fichera-q2.vtk
@@ -21,12 +20,6 @@
 //               ex4 -m ../data/fichera-amr.mesh -o 2 -sc
 //               ex4 -m ../data/star-surf.mesh -o 1
 //
-// Device sample runs:
-//               ex4 -m ../data/star.mesh -pa -d cuda
-//               ex4 -m ../data/star.mesh -pa -d raja-cuda
-//               ex4 -m ../data/star.mesh -pa -d raja-omp
-//               ex4 -m ../data/beam-hex.mesh -pa -d cuda
-//
 // Description:  This example code solves a simple 2D/3D H(div) diffusion
 //               problem corresponding to the second order definite equation
 //               -grad(alpha div F) + beta F = f with boundary condition F dot n
@@ -62,8 +55,6 @@ int main(int argc, char *argv[])
   bool set_bc = true;
   bool static_cond = false;
   bool hybridization = false;
-   bool pa = false;
-   const char *device_config = "cpu";
   bool visualization = 1;

   OptionsParser args(argc, argv);
@@ -79,10 +70,6 @@ int main(int argc, char *argv[])
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&hybridization, "-hb", "--hybridization", "-no-hb",
                  "--no-hybridization", "Enable hybridization.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -95,19 +82,14 @@ int main(int argc, char *argv[])
   args.PrintOptions(cout);
   kappa = freq * M_PI;

-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
   //    quadrilateral, tetrahedral, hexahedral, surface and volume, as well as
   //    periodic meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();
   int sdim = mesh->SpaceDimension();

-   // 4. Refine the mesh to increase the resolution. In this example we do
+   // 3. Refine the mesh to increase the resolution. In this example we do
   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
   //    largest number that gives a final mesh with no more than 25,000
   //    elements.
@@ -120,14 +102,14 @@ int main(int argc, char *argv[])
      }
   }

-   // 5. Define a finite element space on the mesh. Here we use the
+   // 4. Define a finite element space on the mesh. Here we use the
   //    Raviart-Thomas finite elements of the specified order.
   FiniteElementCollection *fec = new RT_FECollection(order-1, dim);
   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
   cout << "Number of finite element unknowns: "
        << fespace->GetTrueVSize() << endl;

-   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
+   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
   //    In this example, the boundary conditions are defined by marking all
   //    the boundary attributes from the mesh as essential (Dirichlet) and
   //    converting them to a list of true dofs.
@@ -139,7 +121,7 @@ int main(int argc, char *argv[])
      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

-   // 7. Set up the linear form b(.) which corresponds to the right-hand side
+   // 6. Set up the linear form b(.) which corresponds to the right-hand side
   //    of the FEM linear system, which in this case is (f,phi_i) where f is
   //    given by the function f_exact and phi_i are the basis functions in the
   //    finite element fespace.
@@ -148,7 +130,7 @@ int main(int argc, char *argv[])
   b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
   b->Assemble();

-   // 8. Define the solution vector x as a finite element grid function
+   // 7. Define the solution vector x as a finite element grid function
   //    corresponding to fespace. Initialize x by projecting the exact
   //    solution. Note that only values from the boundary faces will be used
   //    when eliminating the non-homogeneous boundary condition to modify the
@@ -157,17 +139,16 @@ int main(int argc, char *argv[])
   VectorFunctionCoefficient F(sdim, F_exact);
   x.ProjectCoefficient(F);

-   // 9. Set up the bilinear form corresponding to the H(div) diffusion operator
+   // 8. Set up the bilinear form corresponding to the H(div) diffusion operator
   //    grad alpha div + beta I, by adding the div-div and the mass domain
   //    integrators.
   Coefficient *alpha = new ConstantCoefficient(1.0);
   Coefficient *beta  = new ConstantCoefficient(1.0);
   BilinearForm *a = new BilinearForm(fespace);
-   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   a->AddDomainIntegrator(new DivDivIntegrator(*alpha));
   a->AddDomainIntegrator(new VectorFEMassIntegrator(*beta));

-   // 10. Assemble the bilinear form and the corresponding linear system,
+   // 9. Assemble the bilinear form and the corresponding linear system,
   //    applying any necessary transformations such as: eliminating boundary
   //    conditions, applying conforming constraints for non-conforming AMR,
   //    static condensation, hybridization, etc.
@@ -186,47 +167,32 @@ int main(int argc, char *argv[])
   }
   a->Assemble();

-   OperatorPtr A;
+   SparseMatrix A;
   Vector B, X;
   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);

-   cout << "Size of linear system: " << A->Height() << endl;
+   cout << "Size of linear system: " << A.Height() << endl;

-   // 11. Solve the linear system A X = B.
-   if (!pa)
-   {
 #ifndef MFEM_USE_SUITESPARSE
-      // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
-      GSSmoother M((SparseMatrix&)(*A));
-      PCG(*A, M, B, X, 1, 10000, 1e-20, 0.0);
+   // 10. Define a simple symmetric Gauss-Seidel preconditioner and use it to
+   //     solve the system A X = B with PCG.
+   GSSmoother M(A);
+   PCG(A, M, B, X, 1, 10000, 1e-20, 0.0);
 #else
-      // If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-      UMFPackSolver umf_solver;
-      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      umf_solver.SetOperator(*A);
-      umf_solver.Mult(B, X);
+   // 10. If compiled with SuiteSparse support, use UMFPACK to solve the system.
+   UMFPackSolver umf_solver;
+   umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+   umf_solver.SetOperator(A);
+   umf_solver.Mult(B, X);
 #endif
-   }
-   else // Jacobi preconditioning in partial assembly mode
-   {
-      if (UsesTensorBasis(*fespace))
-      {
-         OperatorJacobiSmoother M(*a, ess_tdof_list);
-         PCG(*A, M, B, X, 1, 10000, 1e-20, 0.0);
-      }
-      else
-      {
-         CG(*A, B, X, 1, 10000, 1e-20, 0.0);
-      }
-   }

-   // 12. Recover the solution as a finite element grid function.
+   // 11. Recover the solution as a finite element grid function.
   a->RecoverFEMSolution(X, *b, x);

-   // 13. Compute and print the L^2 norm of the error.
+   // 12. Compute and print the L^2 norm of the error.
   cout << "\n|| F_h - F ||_{L^2} = " << x.ComputeL2Error(F) << '\n' << endl;

-   // 14. Save the refined mesh and the solution. This output can be viewed
+   // 13. Save the refined mesh and the solution. This output can be viewed
   //     later using GLVis: "glvis -m refined.mesh -g sol.gf".
   {
      ofstream mesh_ofs("refined.mesh");
@@ -237,7 +203,7 @@ int main(int argc, char *argv[])
      x.Save(sol_ofs);
   }

-   // 15. Send the solution by socket to a GLVis server.
+   // 14. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -247,7 +213,7 @@ int main(int argc, char *argv[])
      sol_sock << "solution\n" << *mesh << x << flush;
   }

-   // 16. Free the used memory.
+   // 15. Free the used memory.
   delete hfes;
   delete hfec;
   delete a;
@@ -269,7 +235,7 @@ void F_exact(const Vector &p, Vector &F)

   double x = p(0);
   double y = p(1);
-   // double z = (dim == 3) ? p(2) : 0.0; // Uncomment if F is changed to depend on z
+   // double z = (dim == 3) ? p(2) : 0.0;

   F(0) = cos(kappa*x)*sin(kappa*y);
   F(1) = cos(kappa*y)*sin(kappa*x);
@@ -286,7 +252,7 @@ void f_exact(const Vector &p, Vector &f)

   double x = p(0);
   double y = p(1);
-   // double z = (dim == 3) ? p(2) : 0.0; // Uncomment if f is changed to depend on z
+   // double z = (dim == 3) ? p(2) : 0.0;

   double temp = 1 + 2*kappa*kappa;

@@ -6,7 +6,6 @@
 //               mpirun -np 4 ex4p -m ../data/star.mesh
 //               mpirun -np 4 ex4p -m ../data/beam-tet.mesh
 //               mpirun -np 4 ex4p -m ../data/beam-hex.mesh
-//               mpirun -np 4 ex4p -m ../data/beam-hex.mesh -o 2 -pa
 //               mpirun -np 4 ex4p -m ../data/escher.mesh -o 2 -sc
 //               mpirun -np 4 ex4p -m ../data/fichera.mesh -o 2 -hb
 //               mpirun -np 4 ex4p -m ../data/fichera-q2.vtk
@@ -16,17 +15,10 @@
 //               mpirun -np 4 ex4p -m ../data/periodic-square.mesh -no-bc
 //               mpirun -np 4 ex4p -m ../data/periodic-cube.mesh -no-bc
 //               mpirun -np 4 ex4p -m ../data/amr-quad.mesh
-//               mpirun -np 3 ex4p -m ../data/amr-quad.mesh -o 2 -hb
 //               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -sc
 //               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -hb
 //               mpirun -np 4 ex4p -m ../data/star-surf.mesh -o 3 -hb
 //
-// Device sample runs:
-//               mpirun -np 4 ex4p -m ../data/star.mesh -pa -d cuda
-//               mpirun -np 4 ex4p -m ../data/star.mesh -pa -d raja-cuda
-//               mpirun -np 4 ex4p -m ../data/star.mesh -pa -d raja-omp
-//               mpirun -np 4 ex4p -m ../data/beam-hex.mesh -pa -d cuda
-//
 // Description:  This example code solves a simple 2D/3D H(div) diffusion
 //               problem corresponding to the second order definite equation
 //               -grad(alpha div F) + beta F = f with boundary condition F dot n
@@ -68,8 +60,6 @@ int main(int argc, char *argv[])
   bool set_bc = true;
   bool static_cond = false;
   bool hybridization = false;
-   bool pa = false;
-   const char *device_config = "cpu";
   bool visualization = 1;

   OptionsParser args(argc, argv);
@@ -85,10 +75,6 @@ int main(int argc, char *argv[])
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&hybridization, "-hb", "--hybridization", "-no-hb",
                  "--no-hybridization", "Enable hybridization.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -108,19 +94,14 @@ int main(int argc, char *argv[])
   }
   kappa = freq * M_PI;

-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume, as well as periodic meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();
   int sdim = mesh->SpaceDimension();

-   // 5. Refine the serial mesh on all processors to increase the resolution. In
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ref_levels' of uniform refinement. We choose
   //    'ref_levels' to be the largest number that gives a final mesh with no
   //    more than 1,000 elements.
@@ -133,7 +114,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
   //    meshes need to be reoriented before we can define high-order Nedelec
@@ -149,7 +130,7 @@ int main(int argc, char *argv[])
   }
   pmesh->ReorientTetMesh();

-   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Raviart-Thomas finite elements of the specified order.
   FiniteElementCollection *fec = new RT_FECollection(order-1, dim);
   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
@@ -159,7 +140,7 @@ int main(int argc, char *argv[])
      cout << "Number of finite element unknowns: " << size << endl;
   }

-   // 8. Determine the list of true (i.e. parallel conforming) essential
+   // 7. Determine the list of true (i.e. parallel conforming) essential
   //    boundary dofs. In this example, the boundary conditions are defined
   //    by marking all the boundary attributes from the mesh as essential
   //    (Dirichlet) and converting them to a list of true dofs.
@@ -171,7 +152,7 @@ int main(int argc, char *argv[])
      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

-   // 9. Set up the parallel linear form b(.) which corresponds to the
+   // 8. Set up the parallel linear form b(.) which corresponds to the
   //    right-hand side of the FEM linear system, which in this case is
   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
   //    basis functions in the finite element fespace.
@@ -180,7 +161,7 @@ int main(int argc, char *argv[])
   b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
   b->Assemble();

-   // 10. Define the solution vector x as a parallel finite element grid function
+   // 9. Define the solution vector x as a parallel finite element grid function
   //    corresponding to fespace. Initialize x by projecting the exact
   //    solution. Note that only values from the boundary faces will be used
   //    when eliminating the non-homogeneous boundary condition to modify the
@@ -189,17 +170,16 @@ int main(int argc, char *argv[])
   VectorFunctionCoefficient F(sdim, F_exact);
   x.ProjectCoefficient(F);

-   // 11. Set up the parallel bilinear form corresponding to the H(div)
+   // 10. Set up the parallel bilinear form corresponding to the H(div)
   //     diffusion operator grad alpha div + beta I, by adding the div-div and
   //     the mass domain integrators.
   Coefficient *alpha = new ConstantCoefficient(1.0);
   Coefficient *beta  = new ConstantCoefficient(1.0);
   ParBilinearForm *a = new ParBilinearForm(fespace);
-   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   a->AddDomainIntegrator(new DivDivIntegrator(*alpha));
   a->AddDomainIntegrator(new VectorFEMassIntegrator(*beta));

-   // 12. Assemble the parallel bilinear form and the corresponding linear
+   // 11. Assemble the parallel bilinear form and the corresponding linear
   //     system, applying any necessary transformations such as: parallel
   //     assembly, eliminating boundary conditions, applying conforming
   //     constraints for non-conforming AMR, static condensation,
@@ -219,43 +199,41 @@ int main(int argc, char *argv[])
   }
   a->Assemble();

-   OperatorPtr A;
+   HypreParMatrix A;
   Vector B, X;
   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);

-   if (myid == 0 && !pa)
+   HYPRE_Int glob_size = A.GetGlobalNumRows();
+   if (myid == 0)
   {
-      cout << "Size of linear system: "
-           << A.As<HypreParMatrix>()->GetGlobalNumRows() << endl;
+      cout << "Size of linear system: " << glob_size << endl;
   }

-   // 13. Define and apply a parallel PCG solver for A X = B with the 2D AMS or
+   // 12. Define and apply a parallel PCG solver for A X = B with the 2D AMS or
   //     the 3D ADS preconditioners from hypre. If using hybridization, the
-   //     system is preconditioned with hypre's BoomerAMG. In the partial
-   //     assembly case, use Jacobi preconditioning.
-   Solver *prec = NULL;
-   CGSolver *pcg = new CGSolver(MPI_COMM_WORLD);
-   pcg->SetOperator(*A);
+   //     system is preconditioned with hypre's BoomerAMG.
+   HypreSolver *prec = NULL;
+   CGSolver *pcg = new CGSolver(A.GetComm());
+   pcg->SetOperator(A);
   pcg->SetRelTol(1e-12);
-   pcg->SetMaxIter(2000);
+   pcg->SetMaxIter(500);
   pcg->SetPrintLevel(1);
-   if (hybridization) { prec = new HypreBoomerAMG(*A.As<HypreParMatrix>()); }
-   else if (pa) { prec = new OperatorJacobiSmoother(*a, ess_tdof_list); }
+   if (hybridization) { prec = new HypreBoomerAMG(A); }
   else
   {
      ParFiniteElementSpace *prec_fespace =
         (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
-      if (dim == 2)   { prec = new HypreAMS(*A.As<HypreParMatrix>(), prec_fespace); }
-      else            { prec = new HypreADS(*A.As<HypreParMatrix>(), prec_fespace); }
+      if (dim == 2)   { prec = new HypreAMS(A, prec_fespace); }
+      else            { prec = new HypreADS(A, prec_fespace); }
   }
   pcg->SetPreconditioner(*prec);
   pcg->Mult(B, X);

-   // 14. Recover the parallel grid function corresponding to X. This is the
+   // 13. Recover the parallel grid function corresponding to X. This is the
   //     local finite element solution on each processor.
   a->RecoverFEMSolution(X, *b, x);

-   // 15. Compute and print the L^2 norm of the error.
+   // 14. Compute and print the L^2 norm of the error.
   {
      double err = x.ComputeL2Error(F);
      if (myid == 0)
@@ -264,7 +242,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 16. Save the refined mesh and the solution in parallel. This output can
+   // 15. Save the refined mesh and the solution in parallel. This output can
   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
   {
      ostringstream mesh_name, sol_name;
@@ -280,7 +258,7 @@ int main(int argc, char *argv[])
      x.Save(sol_ofs);
   }

-   // 17. Send the solution by socket to a GLVis server.
+   // 16. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -291,7 +269,7 @@ int main(int argc, char *argv[])
      sol_sock << "solution\n" << *pmesh << x << flush;
   }

-   // 18. Free the used memory.
+   // 17. Free the used memory.
   delete pcg;
   delete prec;
   delete hfes;
@@ -317,7 +295,7 @@ void F_exact(const Vector &p, Vector &F)

   double x = p(0);
   double y = p(1);
-   // double z = (dim == 3) ? p(2) : 0.0; // Uncomment if F is changed to depend on z
+   // double z = (dim == 3) ? p(2) : 0.0;

   F(0) = cos(kappa*x)*sin(kappa*y);
   F(1) = cos(kappa*y)*sin(kappa*x);
@@ -334,7 +312,7 @@ void f_exact(const Vector &p, Vector &f)

   double x = p(0);
   double y = p(1);
-   // double z = (dim == 3) ? p(2) : 0.0; // Uncomment if f is changed to depend on z
+   // double z = (dim == 3) ? p(2) : 0.0;

   double temp = 1 + 2*kappa*kappa;

@@ -4,19 +4,11 @@
 //
 // Sample runs:  ex5 -m ../data/square-disc.mesh
 //               ex5 -m ../data/star.mesh
-//               ex5 -m ../data/star.mesh -pa
 //               ex5 -m ../data/beam-tet.mesh
 //               ex5 -m ../data/beam-hex.mesh
-//               ex5 -m ../data/beam-hex.mesh -pa
 //               ex5 -m ../data/escher.mesh
 //               ex5 -m ../data/fichera.mesh
 //
-// Device sample runs:
-//               ex5 -m ../data/star.mesh -pa -d cuda
-//               ex5 -m ../data/star.mesh -pa -d raja-cuda
-//               ex5 -m ../data/star.mesh -pa -d raja-omp
-//               ex5 -m ../data/beam-hex.mesh -pa -d cuda
-//
 // Description:  This example code solves a simple 2D/3D mixed Darcy problem
 //               corresponding to the saddle point system
 //                                 k*u + grad p = f
@@ -55,8 +47,6 @@ int main(int argc, char *argv[])
   // 1. Parse command-line options.
   const char *mesh_file = "../data/star.mesh";
   int order = 1;
-   bool pa = false;
-   const char *device_config = "cpu";
   bool visualization = 1;

   OptionsParser args(argc, argv);
@@ -64,10 +54,6 @@ int main(int argc, char *argv[])
                  "Mesh file to use.");
   args.AddOption(&order, "-o", "--order",
                  "Finite element order (polynomial degree).");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -79,18 +65,13 @@ int main(int argc, char *argv[])
   }
   args.PrintOptions(cout);

-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
   //    the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 4. Refine the mesh to increase the resolution. In this example we do
+   // 3. Refine the mesh to increase the resolution. In this example we do
   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
   //    largest number that gives a final mesh with no more than 10,000
   //    elements.
@@ -103,7 +84,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 5. Define a finite element space on the mesh. Here we use the
+   // 4. Define a finite element space on the mesh. Here we use the
   //    Raviart-Thomas finite elements of the specified order.
   FiniteElementCollection *hdiv_coll(new RT_FECollection(order, dim));
   FiniteElementCollection *l2_coll(new L2_FECollection(order, dim));
@@ -111,7 +92,7 @@ int main(int argc, char *argv[])
   FiniteElementSpace *R_space = new FiniteElementSpace(mesh, hdiv_coll);
   FiniteElementSpace *W_space = new FiniteElementSpace(mesh, l2_coll);

-   // 6. Define the BlockStructure of the problem, i.e. define the array of
+   // 5. Define the BlockStructure of the problem, i.e. define the array of
   //    offsets for each variable. The last component of the Array is the sum
   //    of the dimensions of each block.
   Array<int> block_offsets(3); // number of variables + 1
@@ -126,7 +107,7 @@ int main(int argc, char *argv[])
   std::cout << "dim(R+W) = " << block_offsets.Last() << "\n";
   std::cout << "***********************************************************\n";

-   // 7. Define the coefficients, analytical solution, and rhs of the PDE.
+   // 6. Define the coefficients, analytical solution, and rhs of the PDE.
   ConstantCoefficient k(1.0);

   VectorFunctionCoefficient fcoeff(dim, fFun);
@@ -136,28 +117,25 @@ int main(int argc, char *argv[])
   VectorFunctionCoefficient ucoeff(dim, uFun_ex);
   FunctionCoefficient pcoeff(pFun_ex);

-   // 8. Allocate memory (x, rhs) for the analytical solution and the right hand
+   // 7. Allocate memory (x, rhs) for the analytical solution and the right hand
   //    side.  Define the GridFunction u,p for the finite element solution and
   //    linear forms fform and gform for the right hand side.  The data
   //    allocated by x and rhs are passed as a reference to the grid functions
   //    (u,p) and the linear forms (fform, gform).
-   MemoryType mt = device.GetMemoryType();
-   BlockVector x(block_offsets, mt), rhs(block_offsets, mt);
+   BlockVector x(block_offsets), rhs(block_offsets);

   LinearForm *fform(new LinearForm);
   fform->Update(R_space, rhs.GetBlock(0), 0);
   fform->AddDomainIntegrator(new VectorFEDomainLFIntegrator(fcoeff));
   fform->AddBoundaryIntegrator(new VectorFEBoundaryFluxLFIntegrator(fnatcoeff));
   fform->Assemble();
-   fform->SyncAliasMemory(rhs);

   LinearForm *gform(new LinearForm);
   gform->Update(W_space, rhs.GetBlock(1), 0);
   gform->AddDomainIntegrator(new DomainLFIntegrator(gcoeff));
   gform->Assemble();
-   gform->SyncAliasMemory(rhs);

-   // 9. Assemble the finite element matrices for the Darcy operator
+   // 8. Assemble the finite element matrices for the Darcy operator
   //
   //                            D = [ M  B^T ]
   //                                [ B   0  ]
@@ -168,103 +146,55 @@ int main(int argc, char *argv[])
   BilinearForm *mVarf(new BilinearForm(R_space));
   MixedBilinearForm *bVarf(new MixedBilinearForm(R_space, W_space));

-   if (pa) { mVarf->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   mVarf->AddDomainIntegrator(new VectorFEMassIntegrator(k));
   mVarf->Assemble();
-   if (!pa) { mVarf->Finalize(); }
+   mVarf->Finalize();
+   SparseMatrix &M(mVarf->SpMat());

-   if (pa) { bVarf->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   bVarf->AddDomainIntegrator(new VectorFEDivergenceIntegrator);
   bVarf->Assemble();
-   if (!pa) { bVarf->Finalize(); }
+   bVarf->Finalize();
+   SparseMatrix & B(bVarf->SpMat());
+   B *= -1.;
+   SparseMatrix *BT = Transpose(B);

-   BlockOperator darcyOp(block_offsets);
+   BlockMatrix darcyMatrix(block_offsets);
+   darcyMatrix.SetBlock(0,0, &M);
+   darcyMatrix.SetBlock(0,1, BT);
+   darcyMatrix.SetBlock(1,0, &B);

-   TransposeOperator *Bt = NULL;
-
-   if (pa)
-   {
-      Bt = new TransposeOperator(bVarf);
-
-      darcyOp.SetBlock(0,0, mVarf);
-      darcyOp.SetBlock(0,1, Bt, -1.0);
-      darcyOp.SetBlock(1,0, bVarf, -1.0);
-   }
-   else
-   {
-      SparseMatrix &M(mVarf->SpMat());
-      SparseMatrix &B(bVarf->SpMat());
-      B *= -1.;
-      Bt = new TransposeOperator(&B);
-
-      darcyOp.SetBlock(0,0, &M);
-      darcyOp.SetBlock(0,1, Bt);
-      darcyOp.SetBlock(1,0, &B);
-   }
-
-   // 10. Construct the operators for preconditioner
+   // 9. Construct the operators for preconditioner
   //
   //                 P = [ diag(M)         0         ]
   //                     [  0       B diag(M)^-1 B^T ]
   //
   //     Here we use Symmetric Gauss-Seidel to approximate the inverse of the
   //     pressure Schur Complement
-   SparseMatrix *MinvBt = NULL;
-   Vector Md(mVarf->Height());
+   SparseMatrix *MinvBt = Transpose(B);
+   Vector Md(M.Height());
+   M.GetDiag(Md);
+   for (int i = 0; i < Md.Size(); i++)
+   {
+      MinvBt->ScaleRow(i, 1./Md(i));
+   }
+   SparseMatrix *S = Mult(B, *MinvBt);

-   BlockDiagonalPreconditioner darcyPrec(block_offsets);
   Solver *invM, *invS;
-   SparseMatrix *S = NULL;
-
-   if (pa)
-   {
-      mVarf->AssembleDiagonal(Md);
-      auto Md_host = Md.HostRead();
-      Vector invMd(mVarf->Height());
-      for (int i=0; i<mVarf->Height(); ++i)
-      {
-         invMd(i) = 1.0 / Md_host[i];
-      }
-
-      Vector BMBt_diag(bVarf->Height());
-      bVarf->AssembleDiagonal_ADAt(invMd, BMBt_diag);
-
-      Array<int> ess_tdof_list;  // empty
-
-      invM = new OperatorJacobiSmoother(Md, ess_tdof_list);
-      invS = new OperatorJacobiSmoother(BMBt_diag, ess_tdof_list);
-   }
-   else
-   {
-      SparseMatrix &M(mVarf->SpMat());
-      M.GetDiag(Md);
-
-      SparseMatrix &B(bVarf->SpMat());
-      MinvBt = Transpose(B);
-
-      for (int i = 0; i < Md.Size(); i++)
-      {
-         MinvBt->ScaleRow(i, 1./Md(i));
-      }
-
-      S = Mult(B, *MinvBt);
-
-      invM = new DSmoother(M);
-
+   invM = new DSmoother(M);
 #ifndef MFEM_USE_SUITESPARSE
-      invS = new GSSmoother(*S);
+   invS = new GSSmoother(*S);
 #else
-      invS = new UMFPackSolver(*S);
+   invS = new UMFPackSolver(*S);
 #endif
-   }

   invM->iterative_mode = false;
   invS->iterative_mode = false;

+   BlockDiagonalPreconditioner darcyPrec(block_offsets);
   darcyPrec.SetDiagonalBlock(0, invM);
   darcyPrec.SetDiagonalBlock(1, invS);

-   // 11. Solve the linear system with MINRES.
+   // 10. Solve the linear system with MINRES.
   //     Check the norm of the unpreconditioned residual.
   int maxIter(1000);
   double rtol(1.e-6);
@@ -276,12 +206,11 @@ int main(int argc, char *argv[])
   solver.SetAbsTol(atol);
   solver.SetRelTol(rtol);
   solver.SetMaxIter(maxIter);
-   solver.SetOperator(darcyOp);
+   solver.SetOperator(darcyMatrix);
   solver.SetPreconditioner(darcyPrec);
   solver.SetPrintLevel(1);
   x = 0.0;
   solver.Mult(rhs, x);
-   if (device.IsEnabled()) { x.HostRead(); }
   chrono.Stop();

   if (solver.GetConverged())
@@ -292,7 +221,7 @@ int main(int argc, char *argv[])
                << " iterations. Residual norm is " << solver.GetFinalNorm() << ".\n";
   std::cout << "MINRES solver took " << chrono.RealTime() << "s. \n";

-   // 12. Create the grid functions u and p. Compute the L2 error norms.
+   // 11. Create the grid functions u and p. Compute the L2 error norms.
   GridFunction u, p;
   u.MakeRef(R_space, x.GetBlock(0), 0);
   p.MakeRef(W_space, x.GetBlock(1), 0);
@@ -312,7 +241,7 @@ int main(int argc, char *argv[])
   std::cout << "|| u_h - u_ex || / || u_ex || = " << err_u / norm_u << "\n";
   std::cout << "|| p_h - p_ex || / || p_ex || = " << err_p / norm_p << "\n";

-   // 13. Save the mesh and the solution. This output can be viewed later using
+   // 12. Save the mesh and the solution. This output can be viewed later using
   //     GLVis: "glvis -m ex5.mesh -g sol_u.gf" or "glvis -m ex5.mesh -g
   //     sol_p.gf".
   {
@@ -329,13 +258,13 @@ int main(int argc, char *argv[])
      p.Save(p_ofs);
   }

-   // 14. Save data in the VisIt format
+   // 13. Save data in the VisIt format
   VisItDataCollection visit_dc("Example5", mesh);
   visit_dc.RegisterField("velocity", &u);
   visit_dc.RegisterField("pressure", &p);
   visit_dc.Save();

-   // 15. Save data in the ParaView format
+   // 14. Save data in the ParaView format
   ParaViewDataCollection paraview_dc("Example5", mesh);
   paraview_dc.SetPrefixPath("ParaView");
   paraview_dc.SetLevelsOfDetail(order);
@@ -347,7 +276,7 @@ int main(int argc, char *argv[])
   paraview_dc.RegisterField("pressure",&p);
   paraview_dc.Save();

-   // 16. Send the solution by socket to a GLVis server.
+   // 15. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -360,14 +289,14 @@ int main(int argc, char *argv[])
      p_sock << "solution\n" << *mesh << p << "window_title 'Pressure'" << endl;
   }

-   // 17. Free the used memory.
+   // 16. Free the used memory.
   delete fform;
   delete gform;
   delete invM;
   delete invS;
   delete S;
-   delete Bt;
   delete MinvBt;
+   delete BT;
   delete mVarf;
   delete bVarf;
   delete W_space;
@@ -4,19 +4,11 @@
 //
 // Sample runs:  mpirun -np 4 ex5p -m ../data/square-disc.mesh
 //               mpirun -np 4 ex5p -m ../data/star.mesh
-//               mpirun -np 4 ex5p -m ../data/star.mesh -r 2 -pa
 //               mpirun -np 4 ex5p -m ../data/beam-tet.mesh
 //               mpirun -np 4 ex5p -m ../data/beam-hex.mesh
-//               mpirun -np 4 ex5p -m ../data/beam-hex.mesh -pa
 //               mpirun -np 4 ex5p -m ../data/escher.mesh
 //               mpirun -np 4 ex5p -m ../data/fichera.mesh
 //
-// Device sample runs:
-//               mpirun -np 4 ex5p -m ../data/star.mesh -r 2 -pa -d cuda
-//               mpirun -np 4 ex5p -m ../data/star.mesh -r 2 -pa -d raja-cuda
-//               mpirun -np 4 ex5p -m ../data/star.mesh -r 2 -pa -d raja-omp
-//               mpirun -np 4 ex5p -m ../data/beam-hex.mesh -pa -d cuda
-//
 // Description:  This example code solves a simple 2D/3D mixed Darcy problem
 //               corresponding to the saddle point system
 //                                 k*u + grad p = f
@@ -30,8 +22,6 @@
 //               The example demonstrates the use of the BlockMatrix class, as
 //               well as the collective saving of several grid functions in
 //               VisIt (visit.llnl.gov) and ParaView (paraview.org) formats.
-//               Optional saving with ADIOS2 (adios2.readthedocs.io) streams is
-//               also illustrated.
 //
 //               We recommend viewing examples 1-4 before viewing this example.

@@ -62,34 +52,21 @@ int main(int argc, char *argv[])

   // 2. Parse command-line options.
   const char *mesh_file = "../data/star.mesh";
-   int ref_levels = -1;
   int order = 1;
   bool par_format = false;
-   bool pa = false;
-   const char *device_config = "cpu";
   bool visualization = 1;
-   bool adios2 = false;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
                  "Mesh file to use.");
-   args.AddOption(&ref_levels, "-r", "--refine",
-                  "Number of times to refine the mesh uniformly.");
   args.AddOption(&order, "-o", "--order",
                  "Finite element order (polynomial degree).");
   args.AddOption(&par_format, "-pf", "--parallel-format", "-sf",
                  "--serial-format",
                  "Format to use when saving the results for VisIt.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
-   args.AddOption(&adios2, "-adios2", "--adios2-streams", "-no-adios2",
-                  "--no-adios2-streams",
-                  "Save data using adios2 streams.");
   args.Parse();
   if (!args.Good())
   {
@@ -105,34 +82,26 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 5. Refine the serial mesh on all processors to increase the resolution. In
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ref_levels' of uniform refinement. We choose
   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 10,000 elements, unless the user specifies it as input.
+   //    more than 10,000 elements.
   {
-      if (ref_levels == -1)
-      {
-         ref_levels = (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
-      }
-
+      int ref_levels =
+         (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
      for (int l = 0; l < ref_levels; l++)
      {
         mesh->UniformRefinement();
      }
   }

-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -145,7 +114,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Raviart-Thomas finite elements of the specified order.
   FiniteElementCollection *hdiv_coll(new RT_FECollection(order, dim));
   FiniteElementCollection *l2_coll(new L2_FECollection(order, dim));
@@ -165,7 +134,7 @@ int main(int argc, char *argv[])
      std::cout << "***********************************************************\n";
   }

-   // 8. Define the two BlockStructure of the problem.  block_offsets is used
+   // 7. Define the two BlockStructure of the problem.  block_offsets is used
   //    for Vector based on dof (like ParGridFunction or ParLinearForm),
   //    block_trueOffstes is used for Vector based on trueDof (HypreParVector
   //    for the rhs and solution of the linear system).  The offsets computed
@@ -182,7 +151,7 @@ int main(int argc, char *argv[])
   block_trueOffsets[2] = W_space->TrueVSize();
   block_trueOffsets.PartialSum();

-   // 9. Define the coefficients, analytical solution, and rhs of the PDE.
+   // 8. Define the coefficients, analytical solution, and rhs of the PDE.
   ConstantCoefficient k(1.0);

   VectorFunctionCoefficient fcoeff(dim, fFun);
@@ -192,30 +161,25 @@ int main(int argc, char *argv[])
   VectorFunctionCoefficient ucoeff(dim, uFun_ex);
   FunctionCoefficient pcoeff(pFun_ex);

-   // 10. Define the parallel grid function and parallel linear forms, solution
-   //     vector and rhs.
-   MemoryType mt = device.GetMemoryType();
-   BlockVector x(block_offsets, mt), rhs(block_offsets, mt);
-   BlockVector trueX(block_trueOffsets, mt), trueRhs(block_trueOffsets, mt);
+   // 9. Define the parallel grid function and parallel linear forms, solution
+   //    vector and rhs.
+   BlockVector x(block_offsets), rhs(block_offsets);
+   BlockVector trueX(block_trueOffsets), trueRhs(block_trueOffsets);

   ParLinearForm *fform(new ParLinearForm);
   fform->Update(R_space, rhs.GetBlock(0), 0);
   fform->AddDomainIntegrator(new VectorFEDomainLFIntegrator(fcoeff));
   fform->AddBoundaryIntegrator(new VectorFEBoundaryFluxLFIntegrator(fnatcoeff));
   fform->Assemble();
-   fform->SyncAliasMemory(rhs);
   fform->ParallelAssemble(trueRhs.GetBlock(0));
-   trueRhs.GetBlock(0).SyncAliasMemory(trueRhs);

   ParLinearForm *gform(new ParLinearForm);
   gform->Update(W_space, rhs.GetBlock(1), 0);
   gform->AddDomainIntegrator(new DomainLFIntegrator(gcoeff));
   gform->Assemble();
-   gform->SyncAliasMemory(rhs);
   gform->ParallelAssemble(trueRhs.GetBlock(1));
-   trueRhs.GetBlock(1).SyncAliasMemory(trueRhs);

-   // 11. Assemble the finite element matrices for the Darcy operator
+   // 10. Assemble the finite element matrices for the Darcy operator
   //
   //                            D = [ M  B^T ]
   //                                [ B   0  ]
@@ -226,93 +190,44 @@ int main(int argc, char *argv[])
   ParBilinearForm *mVarf(new ParBilinearForm(R_space));
   ParMixedBilinearForm *bVarf(new ParMixedBilinearForm(R_space, W_space));

-   HypreParMatrix *M = NULL;
-   HypreParMatrix *B = NULL;
+   HypreParMatrix *M, *B;

-   if (pa) { mVarf->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   mVarf->AddDomainIntegrator(new VectorFEMassIntegrator(k));
   mVarf->Assemble();
-   if (!pa) { mVarf->Finalize(); }
+   mVarf->Finalize();
+   M = mVarf->ParallelAssemble();

-   if (pa) { bVarf->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   bVarf->AddDomainIntegrator(new VectorFEDivergenceIntegrator);
   bVarf->Assemble();
-   if (!pa) { bVarf->Finalize(); }
+   bVarf->Finalize();
+   B = bVarf->ParallelAssemble();
+   (*B) *= -1;
+
+   HypreParMatrix *BT = B->Transpose();

   BlockOperator *darcyOp = new BlockOperator(block_trueOffsets);
+   darcyOp->SetBlock(0,0, M);
+   darcyOp->SetBlock(0,1, BT);
+   darcyOp->SetBlock(1,0, B);

-   Array<int> empty_tdof_list;  // empty
-   OperatorPtr opM, opB;
-
-   TransposeOperator *Bt = NULL;
-
-   if (pa)
-   {
-      mVarf->FormSystemMatrix(empty_tdof_list, opM);
-      bVarf->FormRectangularSystemMatrix(empty_tdof_list, empty_tdof_list, opB);
-      Bt = new TransposeOperator(opB.Ptr());
-
-      darcyOp->SetBlock(0,0, opM.Ptr());
-      darcyOp->SetBlock(0,1, Bt, -1.0);
-      darcyOp->SetBlock(1,0, opB.Ptr(), -1.0);
-   }
-   else
-   {
-      M = mVarf->ParallelAssemble();
-      B = bVarf->ParallelAssemble();
-      (*B) *= -1;
-      Bt = new TransposeOperator(B);
-
-      darcyOp->SetBlock(0,0, M);
-      darcyOp->SetBlock(0,1, Bt);
-      darcyOp->SetBlock(1,0, B);
-   }
-
-   // 12. Construct the operators for preconditioner
+   // 11. Construct the operators for preconditioner
   //
   //                 P = [ diag(M)         0         ]
   //                     [  0       B diag(M)^-1 B^T ]
   //
   //     Here we use Symmetric Gauss-Seidel to approximate the inverse of the
   //     pressure Schur Complement.
-   HypreParMatrix *MinvBt = NULL;
-   HypreParVector *Md = NULL;
-   HypreParMatrix *S = NULL;
-   Vector Md_PA;
-   Solver *invM, *invS;
+   HypreParMatrix *MinvBt = B->Transpose();
+   HypreParVector *Md = new HypreParVector(MPI_COMM_WORLD, M->GetGlobalNumRows(),
+                                           M->GetRowStarts());
+   M->GetDiag(*Md);

-   if (pa)
-   {
-      Md_PA.SetSize(R_space->GetTrueVSize());
-      mVarf->AssembleDiagonal(Md_PA);
-      auto Md_host = Md_PA.HostRead();
-      Vector invMd(Md_PA.Size());
-      for (int i=0; i<Md_PA.Size(); ++i)
-      {
-         invMd(i) = 1.0 / Md_host[i];
-      }
+   MinvBt->InvScaleRows(*Md);
+   HypreParMatrix *S = ParMult(B, MinvBt);

-      Vector BMBt_diag(W_space->GetTrueVSize());
-      bVarf->AssembleDiagonal_ADAt(invMd, BMBt_diag);
-
-      Array<int> ess_tdof_list;  // empty
-
-      invM = new OperatorJacobiSmoother(Md_PA, ess_tdof_list);
-      invS = new OperatorJacobiSmoother(BMBt_diag, ess_tdof_list);
-   }
-   else
-   {
-      Md = new HypreParVector(MPI_COMM_WORLD, M->GetGlobalNumRows(),
-                              M->GetRowStarts());
-      M->GetDiag(*Md);
-
-      MinvBt = B->Transpose();
-      MinvBt->InvScaleRows(*Md);
-      S = ParMult(B, MinvBt);
-
-      invM = new HypreDiagScale(*M);
-      invS = new HypreBoomerAMG(*S);
-   }
+   HypreSolver *invM, *invS;
+   invM = new HypreDiagScale(*M);
+   invS = new HypreBoomerAMG(*S);

   invM->iterative_mode = false;
   invS->iterative_mode = false;
@@ -322,9 +237,9 @@ int main(int argc, char *argv[])
   darcyPr->SetDiagonalBlock(0, invM);
   darcyPr->SetDiagonalBlock(1, invS);

-   // 13. Solve the linear system with MINRES.
+   // 12. Solve the linear system with MINRES.
   //     Check the norm of the unpreconditioned residual.
-   int maxIter(pa ? 1000 : 500);
+   int maxIter(500);
   double rtol(1.e-6);
   double atol(1.e-10);

@@ -339,7 +254,6 @@ int main(int argc, char *argv[])
   solver.SetPrintLevel(verbose);
   trueX = 0.0;
   solver.Mult(trueRhs, trueX);
-   if (device.IsEnabled()) { trueX.HostRead(); }
   chrono.Stop();

   if (verbose)
@@ -353,7 +267,7 @@ int main(int argc, char *argv[])
      std::cout << "MINRES solver took " << chrono.RealTime() << "s. \n";
   }

-   // 14. Extract the parallel grid function corresponding to the finite element
+   // 13. Extract the parallel grid function corresponding to the finite element
   //     approximation X. This is the local solution on each processor. Compute
   //     L2 error norms.
   ParGridFunction *u(new ParGridFunction);
@@ -381,7 +295,7 @@ int main(int argc, char *argv[])
      std::cout << "|| p_h - p_ex || / || p_ex || = " << err_p / norm_p << "\n";
   }

-   // 15. Save the refined mesh and the solution in parallel. This output can be
+   // 14. Save the refined mesh and the solution in parallel. This output can be
   //     viewed later using GLVis: "glvis -np <np> -m mesh -g sol_*".
   {
      ostringstream mesh_name, u_name, p_name;
@@ -402,7 +316,7 @@ int main(int argc, char *argv[])
      p->Save(p_ofs);
   }

-   // 16. Save data in the VisIt format
+   // 15. Save data in the VisIt format
   VisItDataCollection visit_dc("Example5-Parallel", pmesh);
   visit_dc.RegisterField("velocity", u);
   visit_dc.RegisterField("pressure", p);
@@ -411,7 +325,7 @@ int main(int argc, char *argv[])
                      DataCollection::PARALLEL_FORMAT);
   visit_dc.Save();

-   // 17. Save data in the ParaView format
+   // 16. Save data in the ParaView format
   ParaViewDataCollection paraview_dc("Example5P", pmesh);
   paraview_dc.SetPrefixPath("ParaView");
   paraview_dc.SetLevelsOfDetail(order);
@@ -423,27 +337,7 @@ int main(int argc, char *argv[])
   paraview_dc.RegisterField("pressure",p);
   paraview_dc.Save();

-   // 18. Optionally output a BP (binary pack) file using ADIOS2. This can be
-   //     visualized with the ParaView VTX reader.
-#ifdef MFEM_USE_ADIOS2
-   if (adios2)
-   {
-      std::string postfix(mesh_file);
-      postfix.erase(0, std::string("../data/").size() );
-      postfix += "_o" + std::to_string(order);
-      const std::string collection_name = "ex5-p_" + postfix + ".bp";
-
-      ADIOS2DataCollection adios2_dc(MPI_COMM_WORLD, collection_name, pmesh);
-      adios2_dc.SetLevelsOfDetail(1);
-      adios2_dc.SetCycle(1);
-      adios2_dc.SetTime(0.0);
-      adios2_dc.RegisterField("velocity",u);
-      adios2_dc.RegisterField("pressure",p);
-      adios2_dc.Save();
-   }
-#endif
-
-   // 19. Send the solution by socket to a GLVis server.
+   // 17. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -463,7 +357,7 @@ int main(int argc, char *argv[])
             << endl;
   }

-   // 20. Free the used memory.
+   // 18. Free the used memory.
   delete fform;
   delete gform;
   delete u;
@@ -475,7 +369,7 @@ int main(int argc, char *argv[])
   delete S;
   delete Md;
   delete MinvBt;
-   delete Bt;
+   delete BT;
   delete B;
   delete M;
   delete mVarf;
@@ -20,7 +20,7 @@
 //               ex6 -pa -d occa-cuda
 //               ex6 -pa -d raja-omp
 //               ex6 -pa -d ceed-cpu
-//             * ex6 -pa -d ceed-cuda
+//               * ex6 -pa -d ceed-cuda
 //               ex6 -pa -d ceed-cuda:/gpu/cuda/shared
 //
 // Description:  This is a version of Example 1 with a simple adaptive mesh
@@ -20,7 +20,7 @@
 //               mpirun -np 4 ex6p -pa -d occa-cuda
 //               mpirun -np 4 ex6p -pa -d raja-omp
 //               mpirun -np 4 ex6p -pa -d ceed-cpu
-//             * mpirun -np 4 ex6p -pa -d ceed-cuda
+//               * mpirun -np 4 ex6p -pa -d ceed-cuda
 //               mpirun -np 4 ex6p -pa -d ceed-cuda:/gpu/cuda/shared
 //
 // Description:  This is a version of Example 1 with a simple adaptive mesh
@@ -279,11 +279,4 @@ void SnapNodes(Mesh &mesh)
         nodes(nodes.FESpace()->DofToVDof(i, d)) = node(d);
      }
   }
-   if (mesh.Nonconforming())
-   {
-      // Snap hanging nodes to the master side.
-      Vector tnodes;
-      nodes.GetTrueDofs(tnodes);
-      nodes.SetFromTrueDofs(tnodes);
-   }
 }
@@ -348,11 +348,4 @@ void SnapNodes(Mesh &mesh)
         nodes(nodes.FESpace()->DofToVDof(i, d)) = node(d);
      }
   }
-   if (mesh.Nonconforming())
-   {
-      // Snap hanging nodes to the master side.
-      Vector tnodes;
-      nodes.GetTrueDofs(tnodes);
-      nodes.SetFromTrueDofs(tnodes);
-   }
 }
@@ -19,12 +19,8 @@
 //
 // Device sample runs:
 //    ex9 -pa
-//    ex9 -ea
-//    ex9 -fa
 //    ex9 -pa -m ../data/periodic-cube.mesh
 //    ex9 -pa -m ../data/periodic-cube.mesh -d cuda
-//    ex9 -ea -m ../data/periodic-cube.mesh -d cuda
-//    ex9 -fa -m ../data/periodic-cube.mesh -d cuda
 //
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
@@ -146,8 +142,6 @@ int main(int argc, char *argv[])
   int ref_levels = 2;
   int order = 3;
   bool pa = false;
-   bool ea = false;
-   bool fa = false;
   const char *device_config = "cpu";
   int ode_solver_type = 4;
   double t_final = 10.0;
@@ -172,10 +166,6 @@ int main(int argc, char *argv[])
                  "Order (degree) of the finite elements.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&ea, "-ea", "--element-assembly", "-no-ea",
-                  "--no-element-assembly", "Enable Element Assembly.");
-   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
-                  "--no-full-assembly", "Enable Full Assembly.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
@@ -279,16 +269,6 @@ int main(int argc, char *argv[])
      m.SetAssemblyLevel(AssemblyLevel::PARTIAL);
      k.SetAssemblyLevel(AssemblyLevel::PARTIAL);
   }
-   else if (ea)
-   {
-      m.SetAssemblyLevel(AssemblyLevel::ELEMENT);
-      k.SetAssemblyLevel(AssemblyLevel::ELEMENT);
-   }
-   else if (fa)
-   {
-      m.SetAssemblyLevel(AssemblyLevel::FULL);
-      k.SetAssemblyLevel(AssemblyLevel::FULL);
-   }
   m.AddDomainIntegrator(new MassIntegrator);
   k.AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
   k.AddInteriorFaceIntegrator(
@@ -448,19 +428,20 @@ int main(int argc, char *argv[])
 FE_Evolution::FE_Evolution(BilinearForm &_M, BilinearForm &_K, const Vector &_b)
   : TimeDependentOperator(_M.Height()), M(_M), K(_K), b(_b), z(_M.Height())
 {
+   bool pa = M.GetAssemblyLevel() == AssemblyLevel::PARTIAL;
   Array<int> ess_tdof_list;
-   if (M.GetAssemblyLevel() == AssemblyLevel::LEGACYFULL)
-   {
-      M_prec = new DSmoother(M.SpMat());
-      M_solver.SetOperator(M.SpMat());
-      dg_solver = new DG_Solver(M.SpMat(), K.SpMat(), *M.FESpace());
-   }
-   else
+   if (pa)
   {
      M_prec = new OperatorJacobiSmoother(M, ess_tdof_list);
      M_solver.SetOperator(M);
      dg_solver = NULL;
   }
+   else
+   {
+      M_prec = new DSmoother(M.SpMat());
+      dg_solver = new DG_Solver(M.SpMat(), K.SpMat(), *M.FESpace());
+      M_solver.SetOperator(M.SpMat());
+   }
   M_solver.SetPreconditioner(*M_prec);
   M_solver.iterative_mode = false;
   M_solver.SetRelTol(1e-9);
@@ -16,16 +16,11 @@
 //    mpirun -np 4 ex9p -m ../data/disc-nurbs.mesh -p 2 -rp 1 -dt 0.005 -tf 9
 //    mpirun -np 4 ex9p -m ../data/periodic-square.mesh -p 3 -rp 2 -dt 0.0025 -tf 9 -vs 20
 //    mpirun -np 4 ex9p -m ../data/periodic-cube.mesh -p 0 -o 2 -rp 1 -dt 0.01 -tf 8
-//    mpirun -np 3 ex9p -m ../data/amr-hex.mesh -p 1 -rs 1 -rp 0 -dt 0.005 -tf 0.5
 //
 // Device sample runs:
 //    mpirun -np 4 ex9p -pa
-//    mpirun -np 4 ex9p -ea
-//    mpirun -np 4 ex9p -fa
 //    mpirun -np 4 ex9p -pa -m ../data/periodic-cube.mesh
 //    mpirun -np 4 ex9p -pa -m ../data/periodic-cube.mesh -d cuda
-//    mpirun -np 4 ex9p -ea -m ../data/periodic-cube.mesh -d cuda
-//    mpirun -np 4 ex9p -fa -m ../data/periodic-cube.mesh -d cuda
 //
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
@@ -36,10 +31,9 @@
 //               and explicit ODE time integrators, the definition of periodic
 //               boundary conditions through periodic meshes, as well as the use
 //               of GLVis for persistent visualization of a time-evolving
-//               solution. Saving of time-dependent data files for visualization
-//               with VisIt (visit.llnl.gov) and ParaView (paraview.org), as
-//               well as the optional saving with ADIOS2 (adios2.readthedocs.io)
-//               are also illustrated.
+//               solution. The saving of time-dependent data files for external
+//               visualization with VisIt (visit.llnl.gov) and ParaView
+//               (paraview.org) is also illustrated.

 #include "mfem.hpp"
 #include <fstream>
@@ -166,8 +160,6 @@ int main(int argc, char *argv[])
   int par_ref_levels = 0;
   int order = 3;
   bool pa = false;
-   bool ea = false;
-   bool fa = false;
   const char *device_config = "cpu";
   int ode_solver_type = 4;
   double t_final = 10.0;
@@ -175,7 +167,6 @@ int main(int argc, char *argv[])
   bool visualization = true;
   bool visit = false;
   bool paraview = false;
-   bool adios2 = false;
   bool binary = false;
   int vis_steps = 5;

@@ -195,10 +186,6 @@ int main(int argc, char *argv[])
                  "Order (degree) of the finite elements.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&ea, "-ea", "--element-assembly", "-no-ea",
-                  "--no-element-assembly", "Enable Element Assembly.");
-   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
-                  "--no-full-assembly", "Enable Full Assembly.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
@@ -221,9 +208,6 @@ int main(int argc, char *argv[])
   args.AddOption(&paraview, "-paraview", "--paraview-datafiles", "-no-paraview",
                  "--no-paraview-datafiles",
                  "Save data files for ParaView (paraview.org) visualization.");
-   args.AddOption(&adios2, "-adios2", "--adios2-streams", "-no-adios2",
-                  "--no-adios2-streams",
-                  "Save data using adios2 streams.");
   args.AddOption(&binary, "-binary", "--binary-datafiles", "-ascii",
                  "--ascii-datafiles",
                  "Use binary (Sidre) or ascii format for VisIt data files.");
@@ -330,17 +314,6 @@ int main(int argc, char *argv[])
      m->SetAssemblyLevel(AssemblyLevel::PARTIAL);
      k->SetAssemblyLevel(AssemblyLevel::PARTIAL);
   }
-   else if (ea)
-   {
-      m->SetAssemblyLevel(AssemblyLevel::ELEMENT);
-      k->SetAssemblyLevel(AssemblyLevel::ELEMENT);
-   }
-   else if (fa)
-   {
-      m->SetAssemblyLevel(AssemblyLevel::FULL);
-      k->SetAssemblyLevel(AssemblyLevel::FULL);
-   }
-
   m->AddDomainIntegrator(new MassIntegrator);
   k->AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
   k->AddInteriorFaceIntegrator(
@@ -421,28 +394,6 @@ int main(int argc, char *argv[])
      pd->Save();
   }

-   // Optionally output a BP (binary pack) file using ADIOS2. This can be
-   // visualized with the ParaView VTX reader.
-#ifdef MFEM_USE_ADIOS2
-   ADIOS2DataCollection *adios2_dc = NULL;
-   if (adios2)
-   {
-      std::string postfix(mesh_file);
-      postfix.erase(0, std::string("../data/").size() );
-      postfix += "_o" + std::to_string(order);
-      const std::string collection_name = "ex9-p-" + postfix + ".bp";
-
-      adios2_dc = new ADIOS2DataCollection(MPI_COMM_WORLD, collection_name, pmesh);
-      // output data substreams are half the number of mpi processes
-      adios2_dc->SetParameter("SubStreams", std::to_string(num_procs/2) );
-      // adios2_dc->SetLevelsOfDetail(2);
-      adios2_dc->RegisterField("solution", u);
-      adios2_dc->SetCycle(0);
-      adios2_dc->SetTime(0.0);
-      adios2_dc->Save();
-   }
-#endif
-
   socketstream sout;
   if (visualization)
   {
@@ -521,16 +472,6 @@ int main(int argc, char *argv[])
            pd->SetTime(t);
            pd->Save();
         }
-
-#ifdef MFEM_USE_ADIOS2
-         // transient solutions can be visualized with ParaView
-         if (adios2)
-         {
-            adios2_dc->SetCycle(ti);
-            adios2_dc->SetTime(t);
-            adios2_dc->Save();
-         }
-#endif
      }
   }

@@ -556,12 +497,6 @@ int main(int argc, char *argv[])
   delete pmesh;
   delete ode_solver;
   delete pd;
-#ifdef MFEM_USE_ADIOS2
-   if (adios2)
-   {
-      delete adios2_dc;
-   }
-#endif
   delete dc;

   MPI_Finalize();
@@ -577,21 +512,28 @@ FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
     M_solver(_M.ParFESpace()->GetComm()),
     z(_M.Height())
 {
-   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACYFULL)
-   {
-      M.Reset(_M.ParallelAssemble(), true);
-      K.Reset(_K.ParallelAssemble(), true);
-   }
-   else
+   bool pa = _M.GetAssemblyLevel()==AssemblyLevel::PARTIAL;
+
+   if (pa)
   {
      M.Reset(&_M, false);
      K.Reset(&_K, false);
   }
+   else
+   {
+      M.Reset(_M.ParallelAssemble(), true);
+      K.Reset(_K.ParallelAssemble(), true);
+   }

   M_solver.SetOperator(*M);

   Array<int> ess_tdof_list;
-   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACYFULL)
+   if (pa)
+   {
+      M_prec = new OperatorJacobiSmoother(_M, ess_tdof_list);
+      dg_solver = NULL;
+   }
+   else
   {
      HypreParMatrix &M_mat = *M.As<HypreParMatrix>();
      HypreParMatrix &K_mat = *K.As<HypreParMatrix>();
@@ -600,11 +542,6 @@ FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,

      dg_solver = new DG_Solver(M_mat, K_mat, *_M.FESpace());
   }
-   else
-   {
-      M_prec = new OperatorJacobiSmoother(_M, ess_tdof_list);
-      dg_solver = NULL;
-   }

   M_solver.SetPreconditioner(*M_prec);
   M_solver.iterative_mode = false;
@@ -22,10 +22,9 @@ MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)

 SEQ_EXAMPLES = ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 ex17\
-  ex18 ex19 ex20 ex21 ex22 ex23 ex24 ex25 ex26 ex27
+  ex18 ex19 ex20 ex21 ex22 ex23 ex24 ex25
 PAR_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p ex12p\
-  ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p ex25p\
-  ex26p ex27p
+ ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p ex25p

 ifeq ($(MFEM_USE_MPI),NO)
   EXAMPLES = $(SEQ_EXAMPLES)
@@ -104,21 +103,12 @@ ex15-test-seq: ex15
 	@$(call mfem-test,$<,, Serial example,-e 1)
 ex15p-test-par: ex15p
 	@$(call mfem-test,$<, $(RUN_MPI), Parallel example,-e 1)
-ex27-test-seq: ex27
-	@$(call mfem-test,$<,, Serial example,-dg)
-ex27p-test-par: ex27p
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel example,-dg)
 # Testing: optional tests
 ifeq ($(MFEM_USE_STRUMPACK),YES)
 ex11p-test-strumpack: ex11p
 	@$(call mfem-test,$<, $(RUN_MPI), STRUMPACK example,--strumpack)
 test-par-YES: ex11p-test-strumpack
 endif
-ifeq ($(MFEM_USE_SUPERLU),YES)
-ex11p-test-superlu: ex11p
-	@$(call mfem-test,$<, $(RUN_MPI), SuperLU_DIST example,--superlu)
-test-par-YES: ex11p-test-superlu
-endif

 # Testing: "test" target and mfem-test* variables are defined in config/test.mk

@@ -138,7 +128,6 @@ clean-exec:
 	@rm -f sphere_refined.* sol.* sol_u.* sol_p.* sol_r.* sol_i.*
 	@rm -f ex9.mesh ex9-mesh.* ex9-init.* ex9-final.*
 	@rm -f deformed.* velocity.* elastic_energy.* mode_*
-	@rm -f ex5-p-*.bp ex9-p-*.bp ex12-p-*.bp ex16-p-*.bp
 	@rm -f ex16.mesh ex16-mesh.* ex16-init.* ex16-final.*
 	@rm -f vortex-mesh.* vortex.mesh vortex-?-init.* vortex-?-final.*
 	@rm -f deformation.* pressure.*
@@ -34,15 +34,6 @@ if (MFEM_USE_MPI)
    )
 endif()

-if (MFEM_USE_SLEPC)
-  list(APPEND PETSC_EXAMPLES_SRCS
-    ex11p.cpp
-    )
-  list(APPEND PETSC_RC_FILES
-    rc_ex11p_lobpcg rc_ex11p_gd
-    )
-endif()
-
 # Include the source directory where mfem.hpp and mfem-performance.hpp are.
 include_directories(BEFORE ${PROJECT_BINARY_DIR})

@@ -87,22 +78,12 @@ set(EX9_E_ARGS       -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts
 set(EX9_ES_ARGS      -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl --no-step)
 set(EX9_IS_ARGS      -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_impl --implicit -tf 0.5)
 set(EX10_ARGS        -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p -tf 30 -s 3 -rs 2 -dt 3)
-if (MFEM_USE_SLEPC)
-  set(EX11_ARGS_SINV   -m ../../data/star.mesh --useslepc)
-  set(EX11_ARGS_LOBPCG -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg)
-  set(EX11_ARGS_GD -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_gd)
-endif()

 # Add the tests: one test per command-line-variable.
-set(TEST_OPTIONS_VARS
-    EX1_ARGS_W EX1_ARGS_P EX2_ARGS EX3_ARGS EX4_ARGS EX4_HYB_ARGS
-    EX5_BDDC_LB_ARGS EX5_BDDC_GB_ARGS EX5_FSPL_ARGS EX6_ARGS EX6_NONOVL_ARGS
-    EX9_E_ARGS EX9_ES_ARGS EX9_IS_ARGS EX10_ARGS)
-if (MFEM_USE_SLEPC)
-    list(APPEND TEST_OPTIONS_VARS EX11_ARGS_SINV EX11_ARGS_LOBPCG EX11_ARGS_GD)
-endif()
-
-foreach(TEST_OPTIONS_VAR ${TEST_OPTIONS_VARS})
+foreach(TEST_OPTIONS_VAR
+        EX1_ARGS_W EX1_ARGS_P EX2_ARGS EX3_ARGS EX4_ARGS EX4_HYB_ARGS
+        EX5_BDDC_LB_ARGS EX5_BDDC_GB_ARGS EX5_FSPL_ARGS EX6_ARGS EX6_NONOVL_ARGS
+        EX9_E_ARGS EX9_ES_ARGS EX9_IS_ARGS EX10_ARGS)
  string(REGEX REPLACE "^(.+)_ARGS" "\\1" TEST_NAME_UC ${TEST_OPTIONS_VAR})
  string(REGEX REPLACE "^([^_]+)" "\\1P" TEST_NAME_UC ${TEST_NAME_UC})
  string(TOLOWER ${TEST_NAME_UC} TEST_NAME_FULL)
@@ -1,440 +0,0 @@
-//                       MFEM Example 11 - Parallel Version
-//                              PETSc Modification
-//
-// Compile with: make ex11p
-//
-// Sample runs:  mpirun -np 4 ex11p -m ../../data/star.mesh
-//               mpirun -np 4 ex11p -m ../../data/star.mesh --slepcopts rc_ex11p_lobpcg
-//               mpirun -np 4 ex11p -m ../../data/star.mesh --slepcopts rc_ex11p_gd
-//
-// Description:  This example code demonstrates the use of MFEM to solve the
-//               eigenvalue problem -Delta u = lambda u with homogeneous
-//               Dirichlet boundary conditions.
-//
-//               We compute a number of the lowest eigenmodes by discretizing
-//               the Laplacian and Mass operators using a FE space of the
-//               specified order, or an isoparametric/isogeometric space if
-//               order < 1 (quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example demonstrates the use of the SLEPc eigensolver as an
-//               alternative to the LOBPCG eigenvalue solver. The shift and
-//               invert spectral transformation is used to help the convergence
-//               to the smaller eigenvalues. Alternative solver parameters can
-//               be passed in a file with "-slepcopts".
-//
-//               Reusing a single GLVis visualization window for multiple
-//               eigenfunctions is also illustrated.
-//
-//               We recommend viewing Example 1 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-#ifndef MFEM_USE_SLEPC
-#error This examples requires that MFEM is build with MFEM_USE_SLEPC=YES
-#endif
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../../data/star.mesh";
-   int ser_ref_levels = 2;
-   int par_ref_levels = 1;
-   int order = 1;
-   int nev = 5;
-   int seed = 75;
-   bool slu_solver  = false;
-   bool sp_solver = false;
-   bool visualization = 1;
-   bool use_slepc = true;
-   const char *slepcrc_file = "";
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
-                  "Number of times to refine the mesh uniformly in parallel.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&nev, "-n", "--num-eigs",
-                  "Number of desired eigenmodes.");
-   args.AddOption(&seed, "-s", "--seed",
-                  "Random seed used to initialize LOBPCG.");
-#ifdef MFEM_USE_SUPERLU
-   args.AddOption(&slu_solver, "-slu", "--superlu", "-no-slu",
-                  "--no-superlu", "Use the SuperLU Solver.");
-#endif
-#ifdef MFEM_USE_STRUMPACK
-   args.AddOption(&sp_solver, "-sp", "--strumpack", "-no-sp",
-                  "--no-strumpack", "Use the STRUMPACK Solver.");
-#endif
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&use_slepc, "-useslepc","--useslepc","-no-slepc",
-                  "--no-slepc","Use or not SLEPc to solve the eigenvalue problem");
-   args.AddOption(&slepcrc_file, "-slepcopts", "--slepcopts",
-                  "SlepcOptions file to use.");
-   args.Parse();
-   if (slu_solver && sp_solver)
-   {
-      if (myid == 0)
-         cout << "WARNING: Both SuperLU and STRUMPACK have been selected,"
-              << " please choose either one." << endl
-              << "         Defaulting to SuperLU." << endl;
-      sp_solver = false;
-   }
-   // The command line options are also passed to the STRUMPACK
-   // solver. So do not exit if some options are not recognized.
-   if (!sp_solver)
-   {
-      if (!args.Good())
-      {
-         if (myid == 0)
-         {
-            args.PrintUsage(cout);
-         }
-         MPI_Finalize();
-         return 1;
-      }
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 2b. We initialize SLEPc. This internally initializes PETSc as well.
-   MFEMInitializeSlepc(NULL,NULL,slepcrc_file,NULL);
-
-   // 3. Read the (serial) mesh from the given mesh file on all processors. We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 4. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement (2 by default, or
-   //    specified on the command line with -rs).
-   for (int lev = 0; lev < ser_ref_levels; lev++)
-   {
-      mesh->UniformRefinement();
-   }
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution (1 time by
-   //    default, or specified on the command line with -rp). Once the parallel
-   //    mesh is defined, the serial mesh can be deleted.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   for (int lev = 0; lev < par_ref_levels; lev++)
-   {
-      pmesh->UniformRefinement();
-   }
-
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
-   //    use continuous Lagrange finite elements of the specified order. If
-   //    order < 1, we instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-   }
-   else if (pmesh->GetNodes())
-   {
-      fec = pmesh->GetNodes()->OwnFEC();
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-   }
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_Int size = fespace->GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of unknowns: " << size << endl;
-   }
-
-   // 7. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
-   //    element space. The first corresponds to the Laplacian operator -Delta,
-   //    while the second is a simple mass matrix needed on the right hand side
-   //    of the generalized eigenvalue problem below. The boundary conditions
-   //    are implemented by elimination with special values on the diagonal to
-   //    shift the Dirichlet eigenvalues out of the computational range. After
-   //    serial and parallel assembly we extract the corresponding parallel
-   //    matrices A and M.
-   ConstantCoefficient one(1.0);
-   Array<int> ess_bdr;
-   if (pmesh->bdr_attributes.Size())
-   {
-      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
-      ess_bdr = 1;
-   }
-
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-   if (pmesh->bdr_attributes.Size() == 0)
-   {
-      // Add a mass term if the mesh has no boundary, e.g. periodic mesh or
-      // closed surface.
-      a->AddDomainIntegrator(new MassIntegrator(one));
-   }
-   a->Assemble();
-   a->EliminateEssentialBCDiag(ess_bdr, 1.0);
-   a->Finalize();
-
-   ParBilinearForm *m = new ParBilinearForm(fespace);
-   m->AddDomainIntegrator(new MassIntegrator(one));
-   m->Assemble();
-   // shift the eigenvalue corresponding to eliminated dofs to a large value
-   m->EliminateEssentialBCDiag(ess_bdr, numeric_limits<double>::min());
-   m->Finalize();
-
-   PetscParMatrix *pA = NULL, *pM = NULL;
-   HypreParMatrix *A = NULL, *M = NULL;
-   Operator::Type tid =
-      !use_slepc ? Operator::Hypre_ParCSR : Operator::PETSC_MATAIJ;
-   OperatorHandle Ah(tid), Mh(tid);
-
-   a->ParallelAssemble(Ah);
-   if (!use_slepc) { Ah.Get(A); }
-   else { Ah.Get(pA); }
-   Ah.SetOperatorOwner(false);
-
-   m->ParallelAssemble(Mh);
-   if (!use_slepc) {Mh.Get(M); }
-   else {Mh.Get(pM); }
-   Mh.SetOperatorOwner(false);
-
-#if defined(MFEM_USE_SUPERLU) || defined(MFEM_USE_STRUMPACK)
-   Operator * Arow = NULL;
-#ifdef MFEM_USE_SUPERLU
-   if (slu_solver)
-   {
-      Arow = new SuperLURowLocMatrix(*A);
-   }
-#endif
-#ifdef MFEM_USE_STRUMPACK
-   if (sp_solver)
-   {
-      Arow = new STRUMPACKRowLocMatrix(*A);
-   }
-#endif
-#endif
-
-   delete a;
-   delete m;
-
-   // 8. Define and configure the LOBPCG eigensolver and the BoomerAMG
-   //    preconditioner for A to be used within the solver. Set the matrices
-   //    which define the generalized eigenproblem A x = lambda M x.
-   Solver * precond = NULL;
-   if (!use_slepc)
-   {
-      if (!slu_solver && !sp_solver)
-      {
-         HypreBoomerAMG * amg = new HypreBoomerAMG(*A);
-         amg->SetPrintLevel(0);
-         precond = amg;
-      }
-      else
-      {
-#ifdef MFEM_USE_SUPERLU
-         if (slu_solver)
-         {
-            SuperLUSolver * superlu = new SuperLUSolver(MPI_COMM_WORLD);
-            superlu->SetPrintStatistics(false);
-            superlu->SetSymmetricPattern(true);
-            superlu->SetColumnPermutation(superlu::PARMETIS);
-            superlu->SetOperator(*Arow);
-            precond = superlu;
-         }
-#endif
-#ifdef MFEM_USE_STRUMPACK
-         if (sp_solver)
-         {
-            STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
-            strumpack->SetPrintFactorStatistics(true);
-            strumpack->SetPrintSolveStatistics(false);
-            strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-            strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-            strumpack->DisableMatching();
-            strumpack->SetOperator(*Arow);
-            strumpack->SetFromCommandLine();
-            precond = strumpack;
-         }
-#endif
-      }
-   }
-
-   HypreLOBPCG * lobpcg = NULL;
-   SlepcEigenSolver * slepc = NULL;
-   if (!use_slepc)
-   {
-
-      lobpcg = new HypreLOBPCG(MPI_COMM_WORLD);
-      lobpcg->SetNumModes(nev);
-      lobpcg->SetRandomSeed(seed);
-      lobpcg->SetPreconditioner(*precond);
-      lobpcg->SetMaxIter(200);
-      lobpcg->SetTol(1e-8);
-      lobpcg->SetPrecondUsageMode(1);
-      lobpcg->SetPrintLevel(1);
-      lobpcg->SetMassMatrix(*M);
-      lobpcg->SetOperator(*A);
-   }
-   else
-   {
-      slepc = new SlepcEigenSolver(MPI_COMM_WORLD);
-      slepc->SetNumModes(nev);
-      slepc->SetWhichEigenpairs(SlepcEigenSolver::TARGET_REAL);
-      slepc->SetTarget(0.0);
-      slepc->SetSpectralTransformation(SlepcEigenSolver::SHIFT_INVERT);
-      slepc->SetOperators(*pA,*pM);
-   }
-
-   // 9. Compute the eigenmodes and extract the array of eigenvalues. Define a
-   //    parallel grid function to represent each of the eigenmodes returned by
-   //    the solver.
-   Array<double> eigenvalues;
-   if (!use_slepc)
-   {
-      lobpcg->Solve();
-      lobpcg->GetEigenvalues(eigenvalues);
-   }
-   else
-   {
-      slepc->Solve();
-      eigenvalues.SetSize(nev);
-      for (int i=0; i<nev; i++)
-      {
-         slepc->GetEigenvalue(i,eigenvalues[i]);
-      }
-   }
-   Vector temp(fespace->GetTrueVSize());
-   ParGridFunction x(fespace);
-
-   // 10. Save the refined mesh and the modes in parallel. This output can be
-   //     viewed later using GLVis: "glvis -np <np> -m mesh -g mode".
-   {
-      ostringstream mesh_name, mode_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      for (int i=0; i<nev; i++)
-      {
-         // convert eigenvector from HypreParVector to ParGridFunction
-         if (!use_slepc)
-         {
-            x = lobpcg->GetEigenvector(i);
-         }
-         else
-         {
-            slepc->GetEigenvector(i,temp);
-            x.Distribute(temp);
-
-         }
-
-         mode_name << "mode_" << setfill('0') << setw(2) << i << "."
-                   << setfill('0') << setw(6) << myid;
-
-         ofstream mode_ofs(mode_name.str().c_str());
-         mode_ofs.precision(8);
-         x.Save(mode_ofs);
-         mode_name.str("");
-      }
-   }
-
-   // 11. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream mode_sock(vishost, visport);
-      mode_sock.precision(8);
-
-      for (int i=0; i<nev; i++)
-      {
-         if ( myid == 0 )
-         {
-            cout << "Eigenmode " << i+1 << '/' << nev
-                 << ", Lambda = " << eigenvalues[i] << endl;
-         }
-
-         // convert eigenvector from HypreParVector to ParGridFunction
-         if (!use_slepc)
-         {
-            x = lobpcg->GetEigenvector(i);
-         }
-         else
-         {
-            slepc->GetEigenvector(i,temp);
-            x.Distribute(temp);
-         }
-
-         mode_sock << "parallel " << num_procs << " " << myid << "\n"
-                   << "solution\n" << *pmesh << x << flush
-                   << "window_title 'Eigenmode " << i+1 << '/' << nev
-                   << ", Lambda = " << eigenvalues[i] << "'" << endl;
-
-         char c;
-         if (myid == 0)
-         {
-            cout << "press (q)uit or (c)ontinue --> " << flush;
-            cin >> c;
-         }
-         MPI_Bcast(&c, 1, MPI_CHAR, 0, MPI_COMM_WORLD);
-
-         if (c != 'c')
-         {
-            break;
-         }
-      }
-      mode_sock.close();
-   }
-
-   // 12. Free the used memory.
-   if (!use_slepc)
-   {
-      delete lobpcg;
-   }
-   else
-   {
-      delete slepc;
-   }
-   delete precond;
-   delete M;
-   delete A;
-#if defined(MFEM_USE_SUPERLU) || defined(MFEM_USE_STRUMPACK)
-   delete Arow;
-#endif
-
-   delete fespace;
-   if (order > 0)
-   {
-      delete fec;
-   }
-   delete pmesh;
-
-   // We finalize SLEPc
-   MFEMFinalizeSlepc();
-   MPI_Finalize();
-
-   return 0;
-}
@@ -23,9 +23,6 @@ MFEM_LIB_FILE = mfem_is_not_built

 SEQ_EXAMPLES =
 PAR_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex9p ex10p
-ifeq ($(MFEM_USE_SLEPC),YES)
-   PAR_EXAMPLES += ex11p
-endif
 ifeq ($(MFEM_USE_MPI),NO)
   EXAMPLES = $(SEQ_EXAMPLES)
 else
@@ -90,9 +87,6 @@ EX10_ARGS        := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p
 EX10_MF_ARGS     := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_mf -tf 6 -s 3 -rs 0 -dt 3
 EX10_MFOP_ARGS   := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_mfop -tf 6 -s 3 -rs 0 -dt 3
 EX10_JFNK_ARGS   := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_jfnk --jfnk -tf 6 -s 3 -rs 0 -dt 3
-EX11_ARGS_SINV   := -m ../../data/star.mesh --useslepc
-EX11_ARGS_LOBPCG := -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg
-EX11_ARGS_GD     := -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_gd

 ex1p-test-par: ex1p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX1_ARGS_W))
@@ -120,12 +114,6 @@ ex10p-test-par: ex10p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_MF_ARGS))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_MFOP_ARGS))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_JFNK_ARGS))
-ifeq ($(MFEM_USE_SLEPC),YES)
-ex11p-test-par: ex11p
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_SINV))
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_LOBPCG))
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_GD))
-endif

 # Testing: "test" target and mfem-test* variables are defined in config/test.mk

@@ -1,6 +0,0 @@
-# Options for the eigenvalue solver
-eps_view
-eps_converged_reason
-eps_type gd
-# Options for the spectral transform
-st_type precond
@@ -1,11 +0,0 @@
-# Options for the eigenvalue solver
-eps_monitor
-eps_converged_reason
-eps_view_values
-eps_type lobpcg
-eps_gen_hermitian
-eps_smallest_real
-eps_lobpcg_blocksize 5
-# Options for the spectral transform
-st_type precond
-st_pc_type gamg
@@ -32,13 +32,6 @@
 //               is used for the Finite Element order and "-go" is used for the
 //               geometry order. Note that they can be used independently, i.e.
 //               "-o 8 -go 3" solves for 8th order FE on a third order geometry.
-//
-// NOTE:         Model/Mesh files for this example are in the (large) data file
-//               repository of MFEM here https://github.com/mfem/data under the
-//               folder named "pumi", which consists of the following sub-folders:
-//               a) geom -->  model files
-//               b) parallel --> parallel pumi mesh files
-//               c) serial --> serial pumi mesh files

 #include "mfem.hpp"
 #include <fstream>
@@ -36,14 +36,6 @@
 //               option "-o" is used for the Finite Element order and "-go" for
 //               the geometry order. Note that they can be used independently:
 //               "-o 8 -go 3" solves for 8th order FE on third order geometry.
-//
-// NOTE:         Model/Mesh files for this example are in the (large) data file
-//               repository of MFEM here https://github.com/mfem/data under the
-//               folder named "pumi", which consists of the following sub-folders:
-//               a) geom -->  model files
-//               b) parallel --> parallel pumi mesh files
-//               c) serial --> serial pumi mesh files
-

 #include "mfem.hpp"
 #include <fstream>
@@ -43,14 +43,6 @@
 //               also illustrated.
 //
 //               We recommend viewing Example 1 before viewing this example.
-//
-// NOTE:         Model/Mesh files for this example are in the (large) data file
-//               repository of MFEM here https://github.com/mfem/data under the
-//               folder named "pumi", which consists of the following sub-folders:
-//               a) geom -->  model files
-//               b) parallel --> parallel pumi mesh files
-//               c) serial --> serial pumi mesh files
-

 #include "mfem.hpp"
 #include <fstream>
@@ -1,7 +1,7 @@
 //                       MFEM Example 6 - Parallel Version
 //                              PUMI Modification
 //
-// Compile with: make ex6p
+// Compile with: make ex1p
 //
 // Sample runs:  mpirun -np 8 ex6p
 //
@@ -18,13 +18,6 @@
 //               is added to modify the "adapt_ratio" which is the fraction of
 //               allowable error that scales the output size field of the error
 //               estimator.
-//
-// NOTE:         Model/Mesh files for this example are in the (large) data file
-//               repository of MFEM here https://github.com/mfem/data under the
-//               folder named "pumi", which consists of the following sub-folders:
-//               a) geom -->  model files
-//               b) parallel --> parallel pumi mesh files
-//               c) serial --> serial pumi mesh files

 #include "mfem.hpp"
 #include <fstream>
@@ -339,6 +332,7 @@ int main(int argc, char *argv[])

      apf::destroyField(Tmag_field);
      apf::destroyField(ipfield);
+      apf::destroyNumbering(pumi_mesh->findNumbering("LocalVertexNumbering"));

      // 18. Perform MesAdapt.
      ma::Input* erinput = ma::configure(pumi_mesh, sizefield);
@@ -13,20 +13,13 @@ set(SRCS
  bilinearform.cpp
  bilinearform_ext.cpp
  bilininteg.cpp
-  bilininteg_convection_pa.cpp
-  bilininteg_convection_ea.cpp
-  bilininteg_dgtrace_pa.cpp
-  bilininteg_dgtrace_ea.cpp
-  bilininteg_diffusion_pa.cpp
-  bilininteg_diffusion_ea.cpp
+  bilininteg_convection.cpp
+  bilininteg_dgtrace.cpp
+  bilininteg_diffusion.cpp
  bilininteg_divergence.cpp
  bilininteg_hcurl.cpp
-  bilininteg_hdiv.cpp
-  bilininteg_vectorfe.cpp
  bilininteg_gradient.cpp
-  bilininteg_mass_pa.cpp
-  bilininteg_mass_ea.cpp
-  bilininteg_transpose_ea.cpp
+  bilininteg_mass.cpp
  bilininteg_vecdiffusion.cpp
  bilininteg_vecmass.cpp
  coefficient.cpp
@@ -43,11 +36,9 @@ set(SRCS
  intrules.cpp
  linearform.cpp
  lininteg.cpp
-  multigrid.cpp
  nonlinearform.cpp
  nonlinearform_ext.cpp
  nonlininteg.cpp
-  fespacehierarchy.cpp
  nonlininteg_vectorconvection.cpp
  quadinterpolator.cpp
  quadinterpolator_face.cpp
@@ -56,7 +47,6 @@ set(SRCS
  tmop.cpp
  tmop_tools.cpp
  gslib.cpp
-  transfer.cpp
  )

 set(HDRS
@@ -78,14 +68,12 @@ set(HDRS
  intrules.hpp
  linearform.hpp
  lininteg.hpp
-  multigrid.hpp
  nonlinearform.hpp
  nonlinearform_ext.hpp
  nonlininteg.hpp
  quadinterpolator.hpp
  quadinterpolator_face.hpp
  restriction.hpp
-  fespacehierarchy.hpp
  staticcond.hpp
  tbilinearform.hpp
  tbilininteg.hpp
@@ -98,7 +86,6 @@ set(HDRS
  tmop.hpp
  tmop_tools.hpp
  gslib.hpp
-  transfer.hpp
  )

 if (MFEM_USE_SIDRE)
@@ -111,11 +98,6 @@ if (MFEM_USE_CONDUIT)
  list(APPEND HDRS conduitdatacollection.hpp)
 endif()

-if (MFEM_USE_ADIOS2)
-  list(APPEND SRCS adios2datacollection.cpp)
-  list(APPEND HDRS adios2datacollection.hpp)
-endif()
-
 if (MFEM_USE_MPI)
  list(APPEND SRCS
    pbilinearform.cpp
@@ -1,92 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-//
-// Created on: Jan 7, 2020
-// Author: William F Godoy godoywf@ornl.gov
-// adios2: Adaptable Input/Output System https://github.com/ornladios/ADIOS2
-
-#include "adios2datacollection.hpp"
-
-#ifdef MFEM_USE_ADIOS2
-
-namespace mfem
-{
-
-#ifdef MFEM_USE_MPI
-ADIOS2DataCollection::ADIOS2DataCollection(MPI_Comm comm,
-                                           const std::string& collection_name, Mesh* mesh,
-                                           const std::string engine_type) : DataCollection(collection_name, mesh),
-   stream( new adios2stream(name, adios2stream::openmode::out, comm, engine_type) )
-{
-   SetMesh(mesh);
-}
-#else
-ADIOS2DataCollection::ADIOS2DataCollection(
-   const std::string& collection_name, Mesh* mesh,
-   const std::string engine_type): DataCollection(collection_name, mesh),
-   stream( new adios2stream(name, adios2stream::openmode::out, engine_type) )
-{
-   SetMesh(mesh);
-}
-#endif
-
-ADIOS2DataCollection::~ADIOS2DataCollection()
-{
-   stream->Close();
-}
-
-void ADIOS2DataCollection::Save()
-{
-   stream->BeginStep();
-
-   // only save mesh once (moving mesh, not yet supported)
-   if (stream->CurrentStep() == 0)
-   {
-      if (mesh == nullptr)
-      {
-         const std::string error_message =
-            "MFEM ADIOS2DataCollection Save error: Mesh is null. Please call SetMesh before Save\n";
-         mfem_error(error_message.c_str());
-      }
-      stream->Print(*mesh);
-   }
-
-   // reduce footprint
-   if (myid == 0)
-   {
-      stream->SetTime(time);
-      stream->SetCycle(cycle);
-   }
-
-   for (const auto& field : field_map)
-   {
-      const std::string& variable_name = field.first;
-      field.second->Save(*stream.get(), variable_name);
-   }
-
-   stream->EndStep();
-}
-
-void ADIOS2DataCollection::SetParameter(const std::string key,
-                                        const std::string value) noexcept
-{
-   stream->SetParameter(key, value);
-}
-
-void ADIOS2DataCollection::SetLevelsOfDetail(const int levels_of_detail)
-noexcept
-{
-   stream->SetRefinementLevel(levels_of_detail);
-}
-
-} //end namespace mfem
-
-#endif // MFEM_USE_ADIOS2
@@ -1,93 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-//
-// Created on: Jan 7, 2020
-// Author: William F Godoy godoywf@ornl.gov
-// adios2: Adaptable Input/Output System https://github.com/ornladios/ADIOS2
-
-#ifndef MFEM_ADIOS2DATACOLLECTION
-#define MFEM_ADIOS2DATACOLLECTION
-
-#include "../config/config.hpp"
-
-#ifdef MFEM_USE_ADIOS2
-
-#include "../general/adios2stream.hpp"
-#include "datacollection.hpp"
-
-#include <memory>  // std::unique_ptr
-#include <string>
-
-namespace mfem
-{
-
-class ADIOS2DataCollection : public DataCollection
-{
-public:
-#ifdef MFEM_USE_MPI
-   /**
-    * Parallel constructor. Important: scope of this object must be within
-    * MPI_Init and MPI_Finalize otherwise. The destructor will call the Close
-    * function. Either object must live in a try/catch block (inside try) or use
-    * raw pointers calling delete before MPI_Finalize.
-    * @param comm MPI communicator setting the datacollection domain
-    * @param collection_name unique name for saving data
-    * @param mesh can be set at the constructor level or later by calling
-    * SetMesh()
-    * @param engine_type adios2 engine type
-    */
-   ADIOS2DataCollection(MPI_Comm comm, const std::string& collection_name,
-                        Mesh* mesh = nullptr,
-                        const std::string engine_type = "BPFile");
-#else
-   /**
-    * Serial constructor
-    * @param collection_name unique name for saving data
-    * @param mesh can be set at the constructor level or later by calling
-    * SetMesh()
-    * @param engine_type adios2 engine type
-    * @throws std::invalid_argument (user input error) or std::runtime_error
-    *         (system error)
-    */
-   ADIOS2DataCollection(const std::string& collection_name, Mesh* mesh = nullptr,
-                        const std::string engine_type = "BPFile");
-#endif
-
-   virtual ~ADIOS2DataCollection();
-
-   /** Save the collection */
-   virtual void Save();
-
-   /**
-    * Pass a parameter unique to adios2datacollection
-    * For available parameters:
-    * See https://adios2.readthedocs.io/en/latest/engines/engines.html
-    * The most common is: key=SubStreams value=1 to nprocs (MPI processes)
-    * @param key parameter key
-    * @param value parameter value
-    */
-   void SetParameter(const std::string key, const std::string value) noexcept;
-
-   /**
-    * Sets the levels of detail for the global grid refinement
-    * @param levels_of_detail (default = 1)
-    */
-   void SetLevelsOfDetail(const int levels_of_detail) noexcept;
-
-private:
-   std::unique_ptr<adios2stream> stream;
-};
-
-}  // namespace mfem
-
-#endif // MFEM_USE_ADIOS2
-
-#endif /* MFEM_ADIOS2DATACOLLECTION */
@@ -76,7 +76,7 @@ BilinearForm::BilinearForm(FiniteElementSpace * f)
   precompute_sparsity = 0;
   diag_policy = DIAG_KEEP;

-   assembly = AssemblyLevel::LEGACYFULL;
+   assembly = AssemblyLevel::FULL;
   batch = 1;
   ext = NULL;
 }
@@ -94,7 +94,7 @@ BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)
   precompute_sparsity = ps;
   diag_policy = DIAG_KEEP;

-   assembly = AssemblyLevel::LEGACYFULL;
+   assembly = AssemblyLevel::FULL;
   batch = 1;
   ext = NULL;

@@ -121,13 +121,13 @@ void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
   assembly = assembly_level;
   switch (assembly)
   {
-      case AssemblyLevel::LEGACYFULL:
-         break;
      case AssemblyLevel::FULL:
-         ext = new FABilinearFormExtension(this);
+         // ext = new FABilinearFormExtension(this);
+         // Use the original BilinearForm implementation for now
         break;
      case AssemblyLevel::ELEMENT:
-         ext = new EABilinearFormExtension(this);
+         mfem_error("Element assembly not supported yet... stay tuned!");
+         // ext = new EABilinearFormExtension(this);
         break;
      case AssemblyLevel::PARTIAL:
         ext = new PABilinearFormExtension(this);
@@ -144,7 +144,7 @@ void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
 void BilinearForm::EnableStaticCondensation()
 {
   delete static_cond;
-   if (assembly != AssemblyLevel::LEGACYFULL)
+   if (assembly != AssemblyLevel::FULL)
   {
      static_cond = NULL;
      MFEM_WARNING("Static condensation not supported for this assembly level");
@@ -169,7 +169,7 @@ void BilinearForm::EnableHybridization(FiniteElementSpace *constr_space,
                                       const Array<int> &ess_tdof_list)
 {
   delete hybridization;
-   if (assembly != AssemblyLevel::LEGACYFULL)
+   if (assembly != AssemblyLevel::FULL)
   {
      delete constr_integ;
      hybridization = NULL;
@@ -224,7 +224,7 @@ MatrixInverse * BilinearForm::Inverse() const

 void BilinearForm::Finalize (int skip_zeros)
 {
-   if (assembly == AssemblyLevel::LEGACYFULL)
+   if (assembly == AssemblyLevel::FULL)
   {
      if (!static_cond) { mat->Finalize(skip_zeros); }
      if (mat_e) { mat_e->Finalize(skip_zeros); }
@@ -467,17 +467,8 @@ void BilinearForm::Assemble(int skip_zeros)
         const FiniteElement &be = *fes->GetBE(i);
         fes -> GetBdrElementVDofs (i, vdofs);
         eltrans = fes -> GetBdrElementTransformation (i);
-         int k = 0;
-         for (; k < bbfi.Size(); k++)
-         {
-            if (bbfi_marker[k] &&
-                (*bbfi_marker[k])[bdr_attr-1] == 0) { continue; }
-
-            bbfi[k]->AssembleElementMatrix(be, *eltrans, elmat);
-            k++;
-            break;
-         }
-         for (; k < bbfi.Size(); k++)
+         bbfi[0]->AssembleElementMatrix(be, *eltrans, elmat);
+         for (int k = 1; k < bbfi.Size(); k++)
         {
            if (bbfi_marker[k] &&
                (*bbfi_marker[k])[bdr_attr-1] == 0) { continue; }
@@ -640,7 +631,8 @@ void BilinearForm::AssembleDiagonal(Vector &diag) const
   }
   else
   {
-      mat->GetDiag(diag);
+      MFEM_ABORT("Not implemented. Maybe assemble your bilinear form into a "
+                 "matrix and use SparseMatrix::GetDiag?");
   }
 }

@@ -1083,7 +1075,7 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
   mat = NULL;
   mat_e = NULL;
   extern_bfs = 0;
-   assembly = AssemblyLevel::LEGACYFULL;
+   assembly = AssemblyLevel::FULL;
   ext = NULL;
 }

@@ -1108,7 +1100,7 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
   bbfi_marker = mbf->bbfi_marker;
   btfbfi_marker = mbf->btfbfi_marker;

-   assembly = AssemblyLevel::LEGACYFULL;
+   assembly = AssemblyLevel::FULL;
   ext = NULL;
 }

@@ -1121,8 +1113,6 @@ void MixedBilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
   assembly = assembly_level;
   switch (assembly)
   {
-      case AssemblyLevel::LEGACYFULL:
-         break;
      case AssemblyLevel::FULL:
         // ext = new FAMixedBilinearFormExtension(this);
         // Use the original BilinearForm implementation for now
@@ -1193,7 +1183,7 @@ void MixedBilinearForm::AddMultTranspose(const Vector & x, Vector & y,

 MatrixInverse * MixedBilinearForm::Inverse() const
 {
-   if (assembly != AssemblyLevel::LEGACYFULL)
+   if (assembly != AssemblyLevel::FULL)
   {
      MFEM_WARNING("MixedBilinearForm::Inverse not possible with this assembly level!");
      return NULL;
@@ -1206,7 +1196,7 @@ MatrixInverse * MixedBilinearForm::Inverse() const

 void MixedBilinearForm::Finalize (int skip_zeros)
 {
-   if (assembly == AssemblyLevel::LEGACYFULL)
+   if (assembly == AssemblyLevel::FULL)
   {
      mat -> Finalize (skip_zeros);
   }
@@ -1433,57 +1423,9 @@ void MixedBilinearForm::Assemble (int skip_zeros)
   }
 }

-void MixedBilinearForm::AssembleDiagonal_ADAt(const Vector &D,
-                                              Vector &diag) const
-{
-   if (ext)
-   {
-      MFEM_ASSERT(diag.Size() == test_fes->GetTrueVSize(),
-                  "Vector for holding diagonal has wrong size!");
-      MFEM_ASSERT(D.Size() == trial_fes->GetTrueVSize(),
-                  "Vector for holding diagonal has wrong size!");
-      const Operator *P_trial = trial_fes->GetProlongationMatrix();
-      const Operator *P_test = test_fes->GetProlongationMatrix();
-      if (!IsIdentityProlongation(P_trial))
-      {
-         Vector local_D(P_trial->Height());
-         P_trial->Mult(D, local_D);
-
-         if (!IsIdentityProlongation(P_test))
-         {
-            Vector local_diag(P_test->Height());
-            ext->AssembleDiagonal_ADAt(local_D, local_diag);
-            P_test->MultTranspose(local_diag, diag);
-         }
-         else
-         {
-            ext->AssembleDiagonal_ADAt(local_D, diag);
-         }
-      }
-      else
-      {
-         if (!IsIdentityProlongation(P_test))
-         {
-            Vector local_diag(P_test->Height());
-            ext->AssembleDiagonal_ADAt(D, local_diag);
-            P_test->MultTranspose(local_diag, diag);
-         }
-         else
-         {
-            ext->AssembleDiagonal_ADAt(D, diag);
-         }
-      }
-   }
-   else
-   {
-      MFEM_ABORT("Not implemented. Maybe assemble your bilinear form into a "
-                 "matrix and use SparseMatrix functions?");
-   }
-}
-
 void MixedBilinearForm::ConformingAssemble()
 {
-   if (assembly != AssemblyLevel::LEGACYFULL)
+   if (assembly != AssemblyLevel::FULL)
   {
      MFEM_WARNING("Conforming assemble not supported for this assembly level!");
      return;
@@ -25,15 +25,12 @@
 namespace mfem
 {

-/** @brief Enumeration defining the assembly level for bilinear and nonlinear
-    form classes derived from Operator. */
+/// Enumeration defining the assembly level for bilinear and nonlinear form
+/// classes derived from Operator.
 enum class AssemblyLevel
 {
-   /// Legacy fully assembled form, i.e. a global sparse matrix in MFEM, Hypre
-   /// or PETSC format. This assembly is ALWAYS performed on the host.
-   LEGACYFULL = 0,
-   /// Fully assembled form, i.e. a global sparse matrix in MFEM format. This
-   /// assembly is compatible with device execution.
+   /// Fully assembled form, i.e. a global sparse matrix in MFEM, Hypre or PETSC
+   /// format.
   FULL,
   /// Form assembled at element level, which computes and stores dense element
   /// matrices.
@@ -47,19 +44,15 @@ enum class AssemblyLevel
 };


-/** @brief A "square matrix" operator for the associated FE space and
-    BLFIntegrators The sum of all the BLFIntegrators can be used form the matrix
-    M. This class also supports other assembly levels specified via the
-    SetAssemblyLevel() function. */
+/** Class for bilinear form - "Matrix" with associated FE space and
+    BLFIntegrators. */
 class BilinearForm : public Matrix
 {
 protected:
-   /// Sparse matrix \f$ M \f$ to be associated with the form. Owned.
+   /// Sparse matrix to be associated with the form. Owned.
   SparseMatrix *mat;

-   /** @brief Sparse Matrix \f$ M_e \f$ used to store the eliminations
-        from the b.c.  Owned.
-       \f$ M + M_e = M_{original} \f$ */
+   /// Matrix used to eliminate b.c. Owned.
   SparseMatrix *mat_e;

   /// FE space on which the form lives. Not owned.
@@ -69,12 +62,12 @@ protected:
   AssemblyLevel assembly;
   /// Element batch size used in the form action (1, 8, num_elems, etc.)
   int batch;
-   /** @brief Extension for supporting Full Assembly (FA), Element Assembly (EA),
+   /** Extension for supporting Full Assembly (FA), Element Assembly (EA),
       Partial Assembly (PA), or Matrix Free assembly (MF). */
   BilinearFormExtension *ext;

-   /** @brief Indicates the Mesh::sequence corresponding to the current state of
-       the BilinearForm. */
+   /// Indicates the Mesh::sequence corresponding to the current state of the
+   /// BilinearForm.
   long sequence;

   /** @brief Indicates the BilinearFormIntegrator%s stored in #dbfi, #bbfi,
@@ -122,7 +115,7 @@ protected:
      static_cond = NULL; hybridization = NULL;
      precompute_sparsity = 0;
      diag_policy = DIAG_KEEP;
-      assembly = AssemblyLevel::LEGACYFULL;
+      assembly = AssemblyLevel::FULL;
      batch = 1;
      ext = NULL;
   }
@@ -154,43 +147,35 @@ public:
   /// Get the size of the BilinearForm as a square matrix.
   int Size() const { return height; }

-   /// Set the desired assembly level.
-   /** Valid choices are:
-
-       - AssemblyLevel::FULL  (default)
-       - AssemblyLevel::PARTIAL
-       - AssemblyLevel::ELEMENT
-       - AssemblyLevel::NONE
-
-       This method must be called before assembly. */
+   /// Set the desired assembly level. The default is AssemblyLevel::FULL.
+   /** This method must be called before assembly. */
   void SetAssemblyLevel(AssemblyLevel assembly_level);

-   /// Returns the assembly level
-   AssemblyLevel GetAssemblyLevel() const { return assembly; }
+   /// Get the assembly level
+   AssemblyLevel GetAssemblyLevel() {return assembly;}

-   /** @brief Enable the use of static condensation. For details see the
-       description for class StaticCondensation in fem/staticcond.hpp This method
-       should be called before assembly. If the number of unknowns after static
+   /** Enable the use of static condensation. For details see the description
+       for class StaticCondensation in fem/staticcond.hpp This method should be
+       called before assembly. If the number of unknowns after static
       condensation is not reduced, it is not enabled. */
   void EnableStaticCondensation();

-   /** @brief Check if static condensation was actually enabled by a previous
-       call to EnableStaticCondensation(). */
+   /** Check if static condensation was actually enabled by a previous call to
+       EnableStaticCondensation(). */
   bool StaticCondensationIsEnabled() const { return static_cond; }

   /// Return the trace FE space associated with static condensation.
   FiniteElementSpace *SCFESpace() const
   { return static_cond ? static_cond->GetTraceFESpace() : NULL; }

-   /// Enable hybridization.
-   /** For details see the description for class
+   /** Enable hybridization; for details see the description for class
       Hybridization in fem/hybridization.hpp. This method should be called
       before assembly. */
   void EnableHybridization(FiniteElementSpace *constr_space,
                            BilinearFormIntegrator *constr_integ,
                            const Array<int> &ess_tdof_list);

-   /** @brief For scalar FE spaces, precompute the sparsity pattern of the matrix
+   /** For scalar FE spaces, precompute the sparsity pattern of the matrix
       (assuming dense element matrices) based on the types of integrators
       present in the bilinear form. */
   void UsePrecomputedSparsity(int ps = 1) { precompute_sparsity = ps; }
@@ -209,16 +194,15 @@ public:
   /// Use the sparsity of @a A to allocate the internal SparseMatrix.
   void UseSparsity(SparseMatrix &A);

-   /// Pre-allocate the internal SparseMatrix before assembly.
-   /**  If the flag 'precompute sparsity'
-       is set, the matrix is allocated in CSR format (i.e.
+   /** Pre-allocate the internal SparseMatrix before assembly. If the flag
+       'precompute sparsity' is set, the matrix is allocated in CSR format (i.e.
       finalized) and the entries are initialized with zeros. */
   void AllocateMatrix() { if (mat == NULL) { AllocMat(); } }

-   /// Access all the integrators added with AddDomainIntegrator().
+   /// Access all integrators added with AddDomainIntegrator().
   Array<BilinearFormIntegrator*> *GetDBFI() { return &dbfi; }

-   /// Access all the integrators added with AddBoundaryIntegrator().
+   /// Access all integrators added with AddBoundaryIntegrator().
   Array<BilinearFormIntegrator*> *GetBBFI() { return &bbfi; }
   /** @brief Access all boundary markers added with AddBoundaryIntegrator().
       If no marker was specified when the integrator was added, the
@@ -235,85 +219,64 @@ public:
       corresponding pointer (to Array<int>) will be NULL. */
   Array<Array<int>*> *GetBFBFI_Marker() { return &bfbfi_marker; }

-   /// Returns a reference to: \f$ M_{ij} \f$
   const double &operator()(int i, int j) { return (*mat)(i,j); }

-   /// Returns a reference to: \f$ M_{ij} \f$
+   /// Returns reference to a_{ij}.
   virtual double &Elem(int i, int j);

-   /// Returns constant reference to: \f$ M_{ij} \f$
+   /// Returns constant reference to a_{ij}.
   virtual const double &Elem(int i, int j) const;

-   /// Matrix vector multiplication:  \f$ y = M x \f$
+   /// Matrix vector multiplication.
   virtual void Mult(const Vector &x, Vector &y) const;

-   /** @brief Matrix vector multiplication with the original uneliminated
-       matrix.  The original matrix is \f$ M + M_e \f$ so we have:
-       \f$ y = M x + M_e x \f$ */
   void FullMult(const Vector &x, Vector &y) const
   { mat->Mult(x, y); mat_e->AddMult(x, y); }

-   /// Add the matrix vector multiple to a vector:  \f$ y += a M x \f$
   virtual void AddMult(const Vector &x, Vector &y, const double a = 1.0) const
   { mat -> AddMult (x, y, a); }

-   /** @brief Add the original uneliminated matrix vector multiple to a vector.
-       The original matrix is \f$ M + Me \f$ so we have:
-       \f$ y += M x + M_e x \f$ */
   void FullAddMult(const Vector &x, Vector &y) const
   { mat->AddMult(x, y); mat_e->AddMult(x, y); }

-   /// Add the matrix transpose vector multiplication:  \f$ y += a M^T x \f$
   virtual void AddMultTranspose(const Vector & x, Vector & y,
                                 const double a = 1.0) const
   { mat->AddMultTranspose(x, y, a); }

-   /** @brief Add the original uneliminated matrix transpose vector
-       multiple to a vector. The original matrix is \f$ M + M_e \f$
-       so we have: \f$ y += M^T x + {M_e}^T x \f$ */
   void FullAddMultTranspose(const Vector & x, Vector & y) const
   { mat->AddMultTranspose(x, y); mat_e->AddMultTranspose(x, y); }

-   /// Matrix transpose vector multiplication:  \f$ y = M^T x \f$
   virtual void MultTranspose(const Vector & x, Vector & y) const
   { y = 0.0; AddMultTranspose (x, y); }

-   /// Compute \f$ y^T M x \f$
   double InnerProduct(const Vector &x, const Vector &y) const
   { return mat->InnerProduct (x, y); }

-   /// Returns a pointer to (approximation) of the matrix inverse:  \f$ M^{-1} \f$
+   /// Returns a pointer to (approximation) of the matrix inverse.
   virtual MatrixInverse *Inverse() const;

   /// Finalizes the matrix initialization.
   virtual void Finalize(int skip_zeros = 1);

-   /// Returns a const reference to the sparse matrix.
+   /// Returns a reference to the sparse matrix
   const SparseMatrix &SpMat() const
   {
      MFEM_VERIFY(mat, "mat is NULL and can't be dereferenced");
      return *mat;
   }
-
-   /// Returns a reference to the sparse matrix:  \f$ M \f$
   SparseMatrix &SpMat()
   {
      MFEM_VERIFY(mat, "mat is NULL and can't be dereferenced");
      return *mat;
   }
-
-   /**  @brief Nullifies the internal matrix \f$ M \f$ and returns a pointer
-        to it.  Used for transfering ownership. */
   SparseMatrix *LoseMat() { SparseMatrix *tmp = mat; mat = NULL; return tmp; }

-   /// Returns a const reference to the sparse matrix of eliminated b.c.: \f$ M_e \f$
+   /// Returns a reference to the sparse matrix of eliminated b.c.
   const SparseMatrix &SpMatElim() const
   {
      MFEM_VERIFY(mat_e, "mat_e is NULL and can't be dereferenced");
      return *mat_e;
   }
-
-   /// Returns a reference to the sparse matrix of eliminated b.c.: \f$ M_e \f$
   SparseMatrix &SpMatElim()
   {
      MFEM_VERIFY(mat_e, "mat_e is NULL and can't be dereferenced");
@@ -348,7 +311,6 @@ public:
   void AddBdrFaceIntegrator(BilinearFormIntegrator *bfi,
                             Array<int> &bdr_marker);

-   /// Sets all sparse values of \f$ M \f$ and \f$ M_e \f$ to 'a'.
   void operator=(const double a)
   {
      if (mat != NULL) { *mat = a; }
@@ -366,10 +328,10 @@ public:
       for an AMR mesh. */
   void AssembleDiagonal(Vector &diag) const;

-   /// Get the finite element space prolongation operator.
+   /// Get the finite element space prolongation matrix
   virtual const Operator *GetProlongation() const
   { return fes->GetConformingProlongation(); }
-   /// Get the finite element space restriction operator
+   /// Get the finite element space restriction matrix
   virtual const Operator *GetRestriction() const
   { return fes->GetConformingRestriction(); }
   /// Get the output finite element space prolongation matrix
@@ -529,12 +491,10 @@ public:
                                 double value);

   /// Eliminate the given @a vdofs. NOTE: here, @a vdofs is a list of DOFs.
-   /** In this case the eliminations are applied to the internal \f$ M \f$
-       and @a rhs without storing the elimination matrix \f$ M_e \f$. */
   void EliminateVDofs(const Array<int> &vdofs, const Vector &sol, Vector &rhs,
                       DiagonalPolicy dpolicy = DIAG_ONE);

-   /// Eliminate the given @a vdofs, storing the eliminated part internally in \f$ M_e \f$.
+   /// Eliminate the given @a vdofs, storing the eliminated part internally.
   /** This method works in conjunction with EliminateVDofsInRHS() and allows
       elimination of boundary conditions in multiple right-hand sides. In this
       method, @a vdofs is a list of DOFs. */
@@ -563,29 +523,21 @@ public:
   void EliminateVDofsInRHS(const Array<int> &vdofs, const Vector &x,
                            Vector &b);

-   /// Compute inner product for full uneliminated matrix \f$ y^T M x + y^T M_e x \f$
   double FullInnerProduct(const Vector &x, const Vector &y) const
   { return mat->InnerProduct(x, y) + mat_e->InnerProduct(x, y); }

-   /// Update the @a FiniteElementSpace and delete all data associated with the old one.
   virtual void Update(FiniteElementSpace *nfes = NULL);

   /// (DEPRECATED) Return the FE space associated with the BilinearForm.
   /** @deprecated Use FESpace() instead. */
-   MFEM_DEPRECATED FiniteElementSpace *GetFES() { return fes; }
+   FiniteElementSpace *GetFES() { return fes; }

   /// Return the FE space associated with the BilinearForm.
   FiniteElementSpace *FESpace() { return fes; }
   /// Read-only access to the associated FiniteElementSpace.
   const FiniteElementSpace *FESpace() const { return fes; }

-   /// Sets diagonal policy used upon construction of the linear system.
-   /** Policies include:
-
-       - DIAG_ZERO (Set the diagonal values to zero)
-       - DIAG_ONE  (Set the diagonal values to one)
-       - DIAG_KEEP (Keep the diagonal values)
-   */
+   /// Sets diagonal policy used upon construction of the linear system
   void SetDiagonalPolicy(DiagonalPolicy policy);

   /// Indicate that integrators are not owned by the BilinearForm
@@ -598,16 +550,16 @@ public:

 /**
   Class for assembling of bilinear forms `a(u,v)` defined on different
-   trial and test spaces. The assembled matrix `M` is such that
+   trial and test spaces. The assembled matrix `A` is such that

-       a(u,v) = V^t M U
+       a(u,v) = V^t A U

   where `U` and `V` are the vectors representing the functions `u` and `v`,
   respectively.  The first argument, `u`, of `a(,)` is in the trial space
   and the second argument, `v`, is in the test space. Thus,

-       # of rows of M = dimension of the test space and
-       # of cols of M = dimension of the trial space.
+       # of rows of A = dimension of the test space and
+       # of cols of A = dimension of the trial space.

   Both trial and test spaces should be defined on the same mesh.
 */
@@ -676,15 +628,11 @@ public:
                     FiniteElementSpace *te_fes,
                     MixedBilinearForm *mbf);

-   /// Returns a reference to: \f$ M_{ij} \f$
   virtual double &Elem(int i, int j);

-   /// Returns a reference to: \f$ M_{ij} \f$
   virtual const double &Elem(int i, int j) const;

-   /// Matrix multiplication: \f$ y = M x \f$
   virtual void Mult(const Vector & x, Vector & y) const;
-
   virtual void AddMult(const Vector & x, Vector & y,
                        const double a = 1.0) const;

@@ -694,7 +642,6 @@ public:

   virtual MatrixInverse *Inverse() const;

-   /// Finalizes the matrix initialization.
   virtual void Finalize(int skip_zeros = 1);

   /** Extract the associated matrix as SparseMatrix blocks. The number of
@@ -702,14 +649,8 @@ public:
       test and trial spaces, respectively. */
   void GetBlocks(Array2D<SparseMatrix *> &blocks) const;

-   /// Returns a const reference to the sparse matrix:  \f$ M \f$
   const SparseMatrix &SpMat() const { return *mat; }
-
-   /// Returns a reference to the sparse matrix:  \f$ M \f$
   SparseMatrix &SpMat() { return *mat; }
-
-   /**  @brief Nullifies the internal matrix \f$ M \f$ and returns a pointer
-        to it.  Used for transfering ownership. */
   SparseMatrix *LoseMat() { SparseMatrix *tmp = mat; mat = NULL; return tmp; }

   /// Adds a domain integrator. Assumes ownership of @a bfi.
@@ -756,7 +697,6 @@ public:
       corresponding pointer (to Array<int>) will be NULL. */
   Array<Array<int>*> *GetBTFBFI_Marker() { return &btfbfi_marker; }

-   /// Sets all sparse values of \f$ M \f$ to @a a.
   void operator=(const double a) { *mat = a; }

   /// Set the desired assembly level. The default is AssemblyLevel::FULL.
@@ -765,10 +705,6 @@ public:

   void Assemble(int skip_zeros = 1);

-   /** @brief Assemble the diagonal of ADA^T into diag, where A is this mixed
-       bilinear form and D is a diagonal. */
-   void AssembleDiagonal_ADAt(const Vector &D, Vector &diag) const;
-
   /// Get the input finite element space prolongation matrix
   virtual const Operator *GetProlongation() const
   { return trial_fes->GetProlongationMatrix(); }
@@ -15,7 +15,6 @@
 #include "../general/forall.hpp"
 #include "bilinearform.hpp"
 #include "libceed/ceed.hpp"
-#include "pgridfunc.hpp"

 namespace mfem
 {
@@ -48,7 +47,7 @@ PABilinearFormExtension::PABilinearFormExtension(BilinearForm *form)
   bdr_face_restrict_lex = NULL;
 }

-void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
+void PABilinearFormExtension::SetupRestrictionOperators()
 {
   ElementDofOrdering ordering = UsesTensorBasis(*a->FESpace())?
                                 ElementDofOrdering::LEXICOGRAPHIC:
@@ -66,8 +65,7 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
   if (int_face_restrict_lex == NULL && a->GetFBFI()->Size() > 0)
   {
      int_face_restrict_lex = trialFes->GetFaceRestriction(
-                                 ElementDofOrdering::LEXICOGRAPHIC,
-                                 FaceType::Interior);
+                                 ElementDofOrdering::LEXICOGRAPHIC, FaceType::Interior);
      faceIntX.SetSize(int_face_restrict_lex->Height(), Device::GetMemoryType());
      faceIntY.SetSize(int_face_restrict_lex->Height(), Device::GetMemoryType());
      faceIntY.UseDevice(true); // ensure 'faceIntY = 0.0' is done on device
@@ -76,9 +74,7 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
   if (bdr_face_restrict_lex == NULL && a->GetBFBFI()->Size() > 0)
   {
      bdr_face_restrict_lex = trialFes->GetFaceRestriction(
-                                 ElementDofOrdering::LEXICOGRAPHIC,
-                                 FaceType::Boundary,
-                                 m);
+                                 ElementDofOrdering::LEXICOGRAPHIC, FaceType::Boundary);
      faceBdrX.SetSize(bdr_face_restrict_lex->Height(), Device::GetMemoryType());
      faceBdrY.SetSize(bdr_face_restrict_lex->Height(), Device::GetMemoryType());
      faceBdrY.UseDevice(true); // ensure 'faceBoundY = 0.0' is done on device
@@ -87,7 +83,7 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)

 void PABilinearFormExtension::Assemble()
 {
-   SetupRestrictionOperators(L2FaceValues::DoubleValued);
+   SetupRestrictionOperators();

   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
   const int integratorCount = integrators.Size();
@@ -291,462 +287,6 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
   }
 }

-// Data and methods for element-assembled bilinear forms
-EABilinearFormExtension::EABilinearFormExtension(BilinearForm *form)
-   : PABilinearFormExtension(form),
-     factorize_face_terms(form->FESpace()->IsDGSpace())
-{
-}
-
-void EABilinearFormExtension::Assemble()
-{
-   SetupRestrictionOperators(L2FaceValues::SingleValued);
-
-   ne = trialFes->GetMesh()->GetNE();
-   elemDofs = trialFes->GetFE(0)->GetDof();
-
-   ea_data.SetSize(ne*elemDofs*elemDofs, Device::GetMemoryType());
-   ea_data.UseDevice(true);
-   ea_data = 0.0;
-
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int integratorCount = integrators.Size();
-   for (int i = 0; i < integratorCount; ++i)
-   {
-      integrators[i]->AssembleEA(*a->FESpace(), ea_data);
-   }
-
-   faceDofs = trialFes ->
-              GetTraceElement(0, trialFes->GetMesh()->GetFaceBaseGeometry(0)) ->
-              GetDof();
-
-   Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
-   const int intFaceIntegratorCount = intFaceIntegrators.Size();
-   if (intFaceIntegratorCount>0)
-   {
-      nf_int = trialFes->GetNFbyType(FaceType::Interior);
-      ea_data_int.SetSize(2*nf_int*faceDofs*faceDofs, Device::GetMemoryType());
-      ea_data_ext.SetSize(2*nf_int*faceDofs*faceDofs, Device::GetMemoryType());
-      ea_data_int = 0.0;
-      ea_data_ext = 0.0;
-   }
-   for (int i = 0; i < intFaceIntegratorCount; ++i)
-   {
-      intFaceIntegrators[i]->AssembleEAInteriorFaces(*a->FESpace(),
-                                                     ea_data_int,
-                                                     ea_data_ext);
-   }
-
-   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
-   const int boundFaceIntegratorCount = bdrFaceIntegrators.Size();
-   if (boundFaceIntegratorCount>0)
-   {
-      nf_bdr = trialFes->GetNFbyType(FaceType::Boundary);
-      ea_data_bdr.SetSize(nf_bdr*faceDofs*faceDofs, Device::GetMemoryType());
-      ea_data_bdr = 0.0;
-   }
-   for (int i = 0; i < boundFaceIntegratorCount; ++i)
-   {
-      bdrFaceIntegrators[i]->AssembleEABoundaryFaces(*a->FESpace(),ea_data_bdr);
-   }
-
-   if (factorize_face_terms && int_face_restrict_lex)
-   {
-      auto restFint = dynamic_cast<const L2FaceRestriction&>(*int_face_restrict_lex);
-      restFint.AddFaceMatricesToElementMatrices(ea_data_int, ea_data);
-   }
-   if (factorize_face_terms && bdr_face_restrict_lex)
-   {
-      auto restFbdr = dynamic_cast<const L2FaceRestriction&>(*bdr_face_restrict_lex);
-      restFbdr.AddFaceMatricesToElementMatrices(ea_data_bdr, ea_data);
-   }
-}
-
-void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
-{
-   // Apply the Element Restriction
-   const bool useRestrict = !DeviceCanUseCeed() && elem_restrict;
-   if (!useRestrict)
-   {
-      y.UseDevice(true); // typically this is a large vector, so store on device
-      y = 0.0;
-   }
-   else
-   {
-      elem_restrict->Mult(x, localX);
-      localY = 0.0;
-   }
-   // Apply the Element Matrices
-   const int NDOFS = elemDofs;
-   auto X = Reshape(useRestrict?localX.Read():x.Read(), NDOFS, ne);
-   auto Y = Reshape(useRestrict?localY.ReadWrite():y.ReadWrite(), NDOFS, ne);
-   auto A = Reshape(ea_data.Read(), NDOFS, NDOFS, ne);
-   MFEM_FORALL(glob_j, ne*NDOFS,
-   {
-      const int e = glob_j/NDOFS;
-      const int j = glob_j%NDOFS;
-      double res = 0.0;
-      for (int i = 0; i < NDOFS; i++)
-      {
-         res += A(i, j, e)*X(i, e);
-      }
-      Y(j, e) += res;
-   });
-   // Apply the Element Restriction transposed
-   if (useRestrict)
-   {
-      elem_restrict->MultTranspose(localY, y);
-   }
-
-   // Treatment of interior faces
-   Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
-   const int iFISz = intFaceIntegrators.Size();
-   if (int_face_restrict_lex && iFISz>0)
-   {
-      // Apply the Interior Face Restriction
-      int_face_restrict_lex->Mult(x, faceIntX);
-      if (faceIntX.Size()>0)
-      {
-         faceIntY = 0.0;
-         // Apply the interior face matrices
-         const int NDOFS = faceDofs;
-         auto X = Reshape(faceIntX.Read(), NDOFS, 2, nf_int);
-         auto Y = Reshape(faceIntY.ReadWrite(), NDOFS, 2, nf_int);
-         if (!factorize_face_terms)
-         {
-            auto A_int = Reshape(ea_data_int.Read(), NDOFS, NDOFS, 2, nf_int);
-            MFEM_FORALL(glob_j, nf_int*NDOFS,
-            {
-               const int f = glob_j/NDOFS;
-               const int j = glob_j%NDOFS;
-               double res = 0.0;
-               for (int i = 0; i < NDOFS; i++)
-               {
-                  res += A_int(i, j, 0, f)*X(i, 0, f);
-               }
-               Y(j, 0, f) += res;
-               res = 0.0;
-               for (int i = 0; i < NDOFS; i++)
-               {
-                  res += A_int(i, j, 1, f)*X(i, 1, f);
-               }
-               Y(j, 1, f) += res;
-            });
-         }
-         auto A_ext = Reshape(ea_data_ext.Read(), NDOFS, NDOFS, 2, nf_int);
-         MFEM_FORALL(glob_j, nf_int*NDOFS,
-         {
-            const int f = glob_j/NDOFS;
-            const int j = glob_j%NDOFS;
-            double res = 0.0;
-            for (int i = 0; i < NDOFS; i++)
-            {
-               res += A_ext(i, j, 0, f)*X(i, 0, f);
-            }
-            Y(j, 1, f) += res;
-            res = 0.0;
-            for (int i = 0; i < NDOFS; i++)
-            {
-               res += A_ext(i, j, 1, f)*X(i, 1, f);
-            }
-            Y(j, 0, f) += res;
-         });
-         // Apply the Interior Face Restriction transposed
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
-      }
-   }
-
-   // Treatment of boundary faces
-   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
-   const int bFISz = bdrFaceIntegrators.Size();
-   if (!factorize_face_terms && bdr_face_restrict_lex && bFISz>0)
-   {
-      // Apply the Boundary Face Restriction
-      bdr_face_restrict_lex->Mult(x, faceBdrX);
-      if (faceBdrX.Size()>0)
-      {
-         faceBdrY = 0.0;
-         // Apply the boundary face matrices
-         const int NDOFS = faceDofs;
-         auto X = Reshape(faceBdrX.Read(), NDOFS, nf_bdr);
-         auto Y = Reshape(faceBdrY.ReadWrite(), NDOFS, nf_bdr);
-         auto A = Reshape(ea_data_bdr.Read(), NDOFS, NDOFS, nf_bdr);
-         MFEM_FORALL(glob_j, nf_bdr*NDOFS,
-         {
-            const int f = glob_j/NDOFS;
-            const int j = glob_j%NDOFS;
-            double res = 0.0;
-            for (int i = 0; i < NDOFS; i++)
-            {
-               res += A(i, j, f)*X(i, f);
-            }
-            Y(j, f) += res;
-         });
-         // Apply the Boundary Face Restriction transposed
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
-      }
-   }
-}
-
-void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
-{
-   // Apply the Element Restriction
-   const bool useRestrict = DeviceCanUseCeed() || !elem_restrict;
-   if (!useRestrict)
-   {
-      y.UseDevice(true); // typically this is a large vector, so store on device
-      y = 0.0;
-   }
-   else
-   {
-      elem_restrict->Mult(x, localX);
-      localY = 0.0;
-   }
-   // Apply the Element Matrices transposed
-   const int NDOFS = elemDofs;
-   auto X = Reshape(useRestrict?localX.Read():x.Read(), NDOFS, ne);
-   auto Y = Reshape(useRestrict?localY.ReadWrite():y.ReadWrite(), NDOFS, ne);
-   auto A = Reshape(ea_data.Read(), NDOFS, NDOFS, ne);
-   MFEM_FORALL(glob_j, ne*NDOFS,
-   {
-      const int e = glob_j/NDOFS;
-      const int j = glob_j%NDOFS;
-      double res = 0.0;
-      for (int i = 0; i < NDOFS; i++)
-      {
-         res += A(j, i, e)*X(i, e);
-      }
-      Y(j, e) += res;
-   });
-   // Apply the Element Restriction transposed
-   if (useRestrict)
-   {
-      elem_restrict->MultTranspose(localY, y);
-   }
-
-   // Treatment of interior faces
-   Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
-   const int iFISz = intFaceIntegrators.Size();
-   if (int_face_restrict_lex && iFISz>0)
-   {
-      // Apply the Interior Face Restriction
-      int_face_restrict_lex->Mult(x, faceIntX);
-      if (faceIntX.Size()>0)
-      {
-         faceIntY = 0.0;
-         // Apply the interior face matrices transposed
-         const int NDOFS = faceDofs;
-         auto X = Reshape(faceIntX.Read(), NDOFS, 2, nf_int);
-         auto Y = Reshape(faceIntY.ReadWrite(), NDOFS, 2, nf_int);
-         if (!factorize_face_terms)
-         {
-            auto A_int = Reshape(ea_data_int.Read(), NDOFS, NDOFS, 2, nf_int);
-            MFEM_FORALL(glob_j, nf_int*NDOFS,
-            {
-               const int f = glob_j/NDOFS;
-               const int j = glob_j%NDOFS;
-               double res = 0.0;
-               for (int i = 0; i < NDOFS; i++)
-               {
-                  res += A_int(j, i, 0, f)*X(i, 0, f);
-               }
-               Y(j, 0, f) += res;
-               res = 0.0;
-               for (int i = 0; i < NDOFS; i++)
-               {
-                  res += A_int(j, i, 1, f)*X(i, 1, f);
-               }
-               Y(j, 1, f) += res;
-            });
-         }
-         auto A_ext = Reshape(ea_data_ext.Read(), NDOFS, NDOFS, 2, nf_int);
-         MFEM_FORALL(glob_j, nf_int*NDOFS,
-         {
-            const int f = glob_j/NDOFS;
-            const int j = glob_j%NDOFS;
-            double res = 0.0;
-            for (int i = 0; i < NDOFS; i++)
-            {
-               res += A_ext(j, i, 0, f)*X(i, 0, f);
-            }
-            Y(j, 1, f) += res;
-            res = 0.0;
-            for (int i = 0; i < NDOFS; i++)
-            {
-               res += A_ext(j, i, 1, f)*X(i, 1, f);
-            }
-            Y(j, 0, f) += res;
-         });
-         // Apply the Interior Face Restriction transposed
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
-      }
-   }
-
-   // Treatment of boundary faces
-   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
-   const int bFISz = bdrFaceIntegrators.Size();
-   if (!factorize_face_terms && bdr_face_restrict_lex && bFISz>0)
-   {
-      // Apply the Boundary Face Restriction
-      bdr_face_restrict_lex->Mult(x, faceBdrX);
-      if (faceBdrX.Size()>0)
-      {
-         faceBdrY = 0.0;
-         // Apply the boundary face matrices transposed
-         const int NDOFS = faceDofs;
-         auto X = Reshape(faceBdrX.Read(), NDOFS, nf_bdr);
-         auto Y = Reshape(faceBdrY.ReadWrite(), NDOFS, nf_bdr);
-         auto A = Reshape(ea_data_bdr.Read(), NDOFS, NDOFS, nf_bdr);
-         MFEM_FORALL(glob_j, nf_bdr*NDOFS,
-         {
-            const int f = glob_j/NDOFS;
-            const int j = glob_j%NDOFS;
-            double res = 0.0;
-            for (int i = 0; i < NDOFS; i++)
-            {
-               res += A(j, i, f)*X(i, f);
-            }
-            Y(j, f) += res;
-         });
-         // Apply the Boundary Face Restriction transposed
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
-      }
-   }
-}
-
-// Data and methods for fully-assembled bilinear forms
-FABilinearFormExtension::FABilinearFormExtension(BilinearForm *form)
-   : EABilinearFormExtension(form),
-     mat(form->FESpace()->GetVSize(),form->FESpace()->GetVSize(),0),
-     face_mat(form->FESpace()->GetVSize(),0,0),
-     use_face_mat(false)
-{
-#ifdef MFEM_USE_MPI
-   if ( ParFiniteElementSpace* pfes =
-           dynamic_cast<ParFiniteElementSpace*>(form->FESpace()) )
-   {
-      if (pfes->IsDGSpace())
-      {
-         use_face_mat = true;
-         pfes->ExchangeFaceNbrData();
-         face_mat.SetWidth(pfes->GetFaceNbrVSize());
-      }
-   }
-#endif
-}
-
-void FABilinearFormExtension::Assemble()
-{
-   EABilinearFormExtension::Assemble();
-   FiniteElementSpace &fes = *a->FESpace();
-   if (fes.IsDGSpace())
-   {
-      const L2ElementRestriction *restE =
-         static_cast<const L2ElementRestriction*>(elem_restrict);
-      const L2FaceRestriction *restF =
-         static_cast<const L2FaceRestriction*>(int_face_restrict_lex);
-      // 1. Fill I
-      //  1.1 Increment with restE
-      restE->FillI(mat);
-      //  1.2 Increment with restF
-      if (restF) { restF->FillI(mat, face_mat); }
-      //  1.3 Sum the non-zeros in I
-      auto h_I = mat.HostReadWriteI();
-      int cpt = 0;
-      const int vd = fes.GetVDim();
-      const int ndofs = ne*elemDofs*vd;
-      for (int i = 0; i < ndofs; i++)
-      {
-         const int nnz = h_I[i];
-         h_I[i] = cpt;
-         cpt += nnz;
-      }
-      const int nnz = cpt;
-      h_I[ndofs] = nnz;
-      mat.GetMemoryJ().New(nnz, mat.GetMemoryJ().GetMemoryType());
-      mat.GetMemoryData().New(nnz, mat.GetMemoryData().GetMemoryType());
-      if (use_face_mat && restF)
-      {
-         auto h_I_face = face_mat.HostReadWriteI();
-         int cpt = 0;
-         for (int i = 0; i < ndofs; i++)
-         {
-            const int nnz = h_I_face[i];
-            h_I_face[i] = cpt;
-            cpt += nnz;
-         }
-         const int nnz_face = cpt;
-         h_I_face[ndofs] = nnz_face;
-         face_mat.GetMemoryJ().New(nnz_face,
-                                   face_mat.GetMemoryJ().GetMemoryType());
-         face_mat.GetMemoryData().New(nnz_face,
-                                      face_mat.GetMemoryData().GetMemoryType());
-      }
-      // 2. Fill J and Data
-      // 2.1 Fill J and Data with Elem ea_data
-      restE->FillJAndData(ea_data, mat);
-      // 2.2 Fill J and Data with Face ea_data_ext
-      if (restF) { restF->FillJAndData(ea_data_ext, mat, face_mat); }
-      // 2.3 Shift indirections in I back to original
-      auto I = mat.HostReadWriteI();
-      for (int i = ndofs; i > 0; i--)
-      {
-         I[i] = I[i-1];
-      }
-      I[0] = 0;
-      if (use_face_mat && restF)
-      {
-         auto I_face = face_mat.HostReadWriteI();
-         for (int i = ndofs; i > 0; i--)
-         {
-            I_face[i] = I_face[i-1];
-         }
-         I_face[0] = 0;
-      }
-   }
-   else // continuous Galerkin case
-   {
-      const ElementRestriction &rest =
-         static_cast<const ElementRestriction&>(*elem_restrict);
-      rest.FillSparseMatrix(ea_data, mat);
-   }
-}
-
-void FABilinearFormExtension::Mult(const Vector &x, Vector &y) const
-{
-   mat.Mult(x, y);
-#ifdef MFEM_USE_MPI
-   if (const ParFiniteElementSpace *pfes =
-          dynamic_cast<const ParFiniteElementSpace*>(testFes))
-   {
-      ParGridFunction x_gf;
-      x_gf.MakeRef(const_cast<ParFiniteElementSpace*>(pfes),
-                   const_cast<Vector&>(x),0);
-      x_gf.ExchangeFaceNbrData();
-      Vector &shared_x = x_gf.FaceNbrData();
-      if (shared_x.Size()) { face_mat.AddMult(shared_x, y); }
-   }
-#endif
-}
-
-void FABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
-{
-   mat.MultTranspose(x, y);
-#ifdef MFEM_USE_MPI
-   if (const ParFiniteElementSpace *pfes =
-          dynamic_cast<const ParFiniteElementSpace*>(testFes))
-   {
-      ParGridFunction x_gf;
-      x_gf.MakeRef(const_cast<ParFiniteElementSpace*>(pfes),
-                   const_cast<Vector&>(x),0);
-      x_gf.ExchangeFaceNbrData();
-      Vector &shared_x = x_gf.FaceNbrData();
-      if (shared_x.Size()) { face_mat.AddMultTranspose(shared_x, y); }
-   }
-#endif
-}
-
-
 MixedBilinearFormExtension::MixedBilinearFormExtension(MixedBilinearForm *form)
   : Operator(form->Height(), form->Width()), a(form)
 {
@@ -947,68 +487,4 @@ void PAMixedBilinearFormExtension::AddMultTranspose(const Vector &x, Vector &y,
   }
 }

-void PAMixedBilinearFormExtension::AssembleDiagonal_ADAt(const Vector &D,
-                                                         Vector &diag) const
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-
-   const int iSz = integrators.Size();
-
-   if (elem_restrict_trial)
-   {
-      const ElementRestriction* H1elem_restrict_trial =
-         dynamic_cast<const ElementRestriction*>(elem_restrict_trial);
-      if (H1elem_restrict_trial)
-      {
-         H1elem_restrict_trial->MultUnsigned(D, localTrial);
-      }
-      else
-      {
-         elem_restrict_trial->Mult(D, localTrial);
-      }
-   }
-
-   if (elem_restrict_test)
-   {
-      localTest = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         if (elem_restrict_trial)
-         {
-            integrators[i]->AssembleDiagonalPA_ADAt(localTrial, localTest);
-         }
-         else
-         {
-            integrators[i]->AssembleDiagonalPA_ADAt(D, localTest);
-         }
-      }
-      const ElementRestriction* H1elem_restrict_test =
-         dynamic_cast<const ElementRestriction*>(elem_restrict_test);
-      if (H1elem_restrict_test)
-      {
-         H1elem_restrict_test->MultTransposeUnsigned(localTest, diag);
-      }
-      else
-      {
-         elem_restrict_test->MultTranspose(localTest, diag);
-      }
-   }
-   else
-   {
-      diag.UseDevice(true); // typically this is a large vector, so store on device
-      diag = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         if (elem_restrict_trial)
-         {
-            integrators[i]->AssembleDiagonalPA_ADAt(localTrial, diag);
-         }
-         else
-         {
-            integrators[i]->AssembleDiagonalPA_ADAt(D, diag);
-         }
-      }
-   }
-}
-
 } // namespace mfem
@@ -22,12 +22,9 @@ namespace mfem
 class BilinearForm;
 class MixedBilinearForm;

-/// Class extending the BilinearForm class to support different AssemblyLevels.
-/**  FA - Full Assembly
-     PA - Partial Assembly
-     EA - Element Assembly
-     MF - Matrix Free
-*/
+
+/** @brief Class extending the BilinearForm class to support the different
+    AssemblyLevel%s. */
 class BilinearFormExtension : public Operator
 {
 protected:
@@ -45,7 +42,6 @@ public:
   /// Get the finite element space restriction matrix
   virtual const Operator *GetRestriction() const;

-   /// Assemble at the level given for the BilinearFormExtension subclass
   virtual void Assemble() = 0;

   virtual void AssembleDiagonal(Vector &diag) const
@@ -62,6 +58,46 @@ public:
   virtual void Update() = 0;
 };

+/// Data and methods for fully-assembled bilinear forms
+class FABilinearFormExtension : public BilinearFormExtension
+{
+public:
+   FABilinearFormExtension(BilinearForm *form)
+      : BilinearFormExtension(form) { }
+
+   /// TODO
+   void Assemble() {}
+   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A) {}
+   void FormLinearSystem(const Array<int> &ess_tdof_list,
+                         Vector &x, Vector &b,
+                         OperatorHandle &A, Vector &X, Vector &B,
+                         int copy_interior = 0) {}
+   void Mult(const Vector &x, Vector &y) const {}
+   void MultTranspose(const Vector &x, Vector &y) const {}
+   void Update() {}
+   ~FABilinearFormExtension() {}
+};
+
+/// Data and methods for element-assembled bilinear forms
+class EABilinearFormExtension : public BilinearFormExtension
+{
+public:
+   EABilinearFormExtension(BilinearForm *form)
+      : BilinearFormExtension(form) { }
+
+   /// TODO
+   void Assemble() {}
+   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A) {}
+   void FormLinearSystem(const Array<int> &ess_tdof_list,
+                         Vector &x, Vector &b,
+                         OperatorHandle &A, Vector &X, Vector &B,
+                         int copy_interior = 0) {}
+   void Mult(const Vector &x, Vector &y) const {}
+   void MultTranspose(const Vector &x, Vector &y) const {}
+   void Update() {}
+   ~EABilinearFormExtension() {}
+};
+
 /// Data and methods for partially-assembled bilinear forms
 class PABilinearFormExtension : public BilinearFormExtension
 {
@@ -77,6 +113,7 @@ protected:
 public:
   PABilinearFormExtension(BilinearForm*);

+   void SetupRestrictionOperators();
   void Assemble();
   void AssembleDiagonal(Vector &diag) const;
   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A);
@@ -84,53 +121,14 @@ public:
                         Vector &x, Vector &b,
                         OperatorHandle &A, Vector &X, Vector &B,
                         int copy_interior = 0);
+
   void Mult(const Vector &x, Vector &y) const;
   void MultTranspose(const Vector &x, Vector &y) const;
   void Update();
-
-protected:
-   void SetupRestrictionOperators(const L2FaceValues m);
 };

-/// Data and methods for element-assembled bilinear forms
-class EABilinearFormExtension : public PABilinearFormExtension
-{
-protected:
-   int ne;
-   int elemDofs;
-   // The element matrices are stored row major
-   Vector ea_data;
-   int nf_int, nf_bdr;
-   int faceDofs;
-   Vector ea_data_int, ea_data_ext, ea_data_bdr;
-   bool factorize_face_terms;

-public:
-   EABilinearFormExtension(BilinearForm *form);
-
-   void Assemble();
-   void Mult(const Vector &x, Vector &y) const;
-   void MultTranspose(const Vector &x, Vector &y) const;
-};
-
-/// Data and methods for fully-assembled bilinear forms
-class FABilinearFormExtension : public EABilinearFormExtension
-{
-private:
-   SparseMatrix mat;
-   /// face_mat handles parallelism for DG face terms.
-   SparseMatrix face_mat;
-   bool use_face_mat;
-
-public:
-   FABilinearFormExtension(BilinearForm *form);
-
-   void Assemble();
-   void Mult(const Vector &x, Vector &y) const;
-   void MultTranspose(const Vector &x, Vector &y) const;
-};
-
-/// Data and methods for matrix-free bilinear forms NOT YET IMPLEMENTED.
+/// Data and methods for matrix-free bilinear forms
 class MFBilinearFormExtension : public BilinearFormExtension
 {
 public:
@@ -150,12 +148,8 @@ public:
   ~MFBilinearFormExtension() {}
 };

-/// Class extending the MixedBilinearForm class to support different AssemblyLevels.
-/**  FA - Full Assembly
-     PA - Partial Assembly
-     EA - Element Assembly
-     MF - Matrix Free
-*/
+/** @brief Class extending the MixedBilinearForm class to support the different
+    AssemblyLevel%s. */
 class MixedBilinearFormExtension : public Operator
 {
 protected:
@@ -192,8 +186,6 @@ public:
   virtual void AddMultTranspose(const Vector &x, Vector &y,
                                 const double c=1.0) const = 0;

-   virtual void AssembleDiagonal_ADAt(const Vector &D, Vector &diag) const = 0;
-
   virtual void Update() = 0;
 };

@@ -244,9 +236,6 @@ public:
   void MultTranspose(const Vector &x, Vector &y) const;
   /// y += c*A^T*x
   void AddMultTranspose(const Vector &x, Vector &y, const double c=1.0) const;
-   /// Assemble the diagonal of ADA^T for a diagonal vector D.
-   void AssembleDiagonal_ADAt(const Vector &D, Vector &diag) const;
-
   /// Update internals for when a new MixedBilinearForm is given to this class
   void Update();
 };
@@ -47,37 +47,7 @@ void BilinearFormIntegrator::AssemblePABoundaryFaces(const FiniteElementSpace&)

 void BilinearFormIntegrator::AssembleDiagonalPA(Vector &)
 {
-   mfem_error ("BilinearFormIntegrator::AssembleDiagonalPA(...)\n"
-               "   is not implemented for this class.");
-}
-
-void BilinearFormIntegrator::AssembleEA(const FiniteElementSpace &fes,
-                                        Vector &emat)
-{
-   mfem_error ("BilinearFormIntegrator::AssembleEA(...)\n"
-               "   is not implemented for this class.");
-}
-
-void BilinearFormIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace
-                                                     &fes,
-                                                     Vector &ea_data_int,
-                                                     Vector &ea_data_ext)
-{
-   mfem_error ("BilinearFormIntegrator::AssembleEAInteriorFaces(...)\n"
-               "   is not implemented for this class.");
-}
-
-void BilinearFormIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace
-                                                     &fes,
-                                                     Vector &ea_data_bdr)
-{
-   mfem_error ("BilinearFormIntegrator::AssembleEABoundaryFaces(...)\n"
-               "   is not implemented for this class.");
-}
-
-void BilinearFormIntegrator::AssembleDiagonalPA_ADAt(const Vector &, Vector &)
-{
-   MFEM_ABORT("BilinearFormIntegrator::AssembleDiagonalPA_ADAt(...)\n"
+   MFEM_ABORT("BilinearFormIntegrator::AssembleDiagonalPA(...)\n"
              "   is not implemented for this class.");
 }

@@ -919,25 +889,22 @@ void BoundaryMassIntegrator::AssembleFaceMatrix(
   {
      int order = 2 * el1.GetOrder();

-      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+      ir = &IntRules.Get(Trans.FaceGeom, order);
   }

   elmat = 0.0;
   for (int i = 0; i < ir->GetNPoints(); i++)
   {
      const IntegrationPoint &ip = ir->IntPoint(i);
-
-      // Set the integration point in the face and the neighboring element
-      Trans.SetAllIntPoints(&ip);
-
-      // Access the neighboring element's integration point
-      const IntegrationPoint &eip = Trans.GetElement1IntPoint();
+      IntegrationPoint eip;
+      Trans.Loc1.Transform(ip, eip);
      el1.CalcShape(eip, shape);

-      w = Trans.Weight() * ip.weight;
+      Trans.Face->SetIntPoint(&ip);
+      w = Trans.Face->Weight() * ip.weight;
      if (Q)
      {
-         w *= Q -> Eval(Trans, ip);
+         w *= Q -> Eval(*Trans.Face, ip);
      }

      AddMult_a_VVt(w, shape, elmat);
@@ -1522,7 +1489,6 @@ void CurlCurlIntegrator::AssembleElementMatrix
   double w;

 #ifdef MFEM_THREAD_SAFE
-   Vector D;
   DenseMatrix curlshape(nd,dimc), curlshape_dFt(nd,dimc), M;
 #else
   curlshape.SetSize(nd,dimc);
@@ -1530,7 +1496,6 @@ void CurlCurlIntegrator::AssembleElementMatrix
 #endif
   elmat.SetSize(nd);
   if (MQ) { M.SetSize(dimc); }
-   if (DQ) { D.SetSize(dimc); }

   const IntegrationRule *ir = IntRule;
   if (ir == NULL)
@@ -1574,12 +1539,6 @@ void CurlCurlIntegrator::AssembleElementMatrix
         Mult(curlshape_dFt, M, curlshape);
         AddMultABt(curlshape, curlshape_dFt, elmat);
      }
-      else if (DQ)
-      {
-         DQ->Eval(D, Trans, ip);
-         D *= w;
-         AddMultADAt(curlshape_dFt, D, elmat);
-      }
      else if (Q)
      {
         w *= Q->Eval(Trans, ip);
@@ -2015,7 +1974,7 @@ void VectorFEMassIntegrator::AssembleElementMatrix2(
      D.SetSize(VQ ? VQ->GetVDim() : 0);
      K.SetSize(MQ ? MQ->GetVDim() : 0, MQ ? MQ->GetVDim() : 0);
 #endif
-      DenseMatrix tmp(test_vshape.Height(), K.Width());
+      DenseMatrix tmp(trial_vshape.Height(), K.Width());

      elmat.SetSize (test_dof, trial_dof);

@@ -2176,17 +2135,17 @@ void VectorDiffusionIntegrator::AssembleElementMatrix(
   ElementTransformation &Trans,
   DenseMatrix &elmat)
 {
-   const int dim = el.GetDim();
-   const int dof = el.GetDof();
-   const int sdim = Trans.GetSpaceDim();
-   const bool square = (dim == sdim);
-   double w;
+   int dim = el.GetDim();
+   int dof = el.GetDof();

-   elmat.SetSize(sdim * dof);
+   double norm;

-   dshape.SetSize(dof, dim);
-   dshapedxt.SetSize(dof, sdim);
-   pelmat.SetSize(dof);
+   elmat.SetSize (dim * dof);
+
+   Jinv.  SetSize (dim);
+   dshape.SetSize (dof, dim);
+   gshape.SetSize (dof, dim);
+   pelmat.SetSize (dof);

   const IntegrationRule *ir = IntRule;
   if (ir == NULL)
@@ -2204,29 +2163,35 @@ void VectorDiffusionIntegrator::AssembleElementMatrix(
   }

   elmat = 0.0;
-   pelmat = 0.0;

   for (int i = 0; i < ir -> GetNPoints(); i++)
   {
      const IntegrationPoint &ip = ir->IntPoint(i);
+
      el.CalcDShape (ip, dshape);
+
      Trans.SetIntPoint (&ip);
-      w = Trans.Weight();
-      w = ip.weight / (square ? w : w*w*w);
-      // AdjugateJacobian = / adj(J),         if J is square
-      //                    \ adj(J^t.J).J^t, otherwise
-      Mult(dshape, Trans.AdjugateJacobian(), dshapedxt);
-      if (Q) {  w *= Q -> Eval (Trans, ip); }
-      AddMult_a_AAt(w, dshapedxt, pelmat);
-   }
-   for (int d = 0; d < sdim; d++)
-   {
-      for (int k = 0; k < dof; k++)
+      norm = ip.weight * Trans.Weight();
+      CalcInverse (Trans.Jacobian(), Jinv);
+
+      Mult (dshape, Jinv, gshape);
+
+      MultAAt (gshape, pelmat);
+
+      if (Q)
      {
-         for (int l = 0; l < dof; l++)
-         {
-            elmat(dof*d+k, dof*d+l) = pelmat(k, l);
-         }
+         norm *= Q -> Eval (Trans, ip);
+      }
+
+      pelmat *= norm;
+
+      for (int d = 0; d < dim; d++)
+      {
+         for (int k = 0; k < dof; k++)
+            for (int l = 0; l < dof; l++)
+            {
+               elmat (dof*d+k, dof*d+l) += pelmat (k, l);
+            }
      }
   }
 }
@@ -2576,23 +2541,23 @@ void DGTraceIntegrator::AssembleFaceMatrix(const FiniteElement &el1,
      {
         order++;
      }
-      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+      ir = &IntRules.Get(Trans.FaceGeom, order);
   }

   for (int p = 0; p < ir->GetNPoints(); p++)
   {
      const IntegrationPoint &ip = ir->IntPoint(p);
-
-      // Set the integration point in the face and the neighboring elements
-      Trans.SetAllIntPoints(&ip);
-
-      // Access the neighboring elements' integration points
-      // Note: eip2 will only contain valid data if Elem2 exists
-      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
-      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
-
+      IntegrationPoint eip1, eip2;
+      Trans.Loc1.Transform(ip, eip1);
+      if (ndof2)
+      {
+         Trans.Loc2.Transform(ip, eip2);
+      }
      el1.CalcShape(eip1, shape1);

+      Trans.Face->SetIntPoint(&ip);
+      Trans.Elem1->SetIntPoint(&eip1);
+
      u->Eval(vu, *Trans.Elem1, eip1);

      if (dim == 1)
@@ -2601,7 +2566,7 @@ void DGTraceIntegrator::AssembleFaceMatrix(const FiniteElement &el1,
      }
      else
      {
-         CalcOrtho(Trans.Jacobian(), nor);
+         CalcOrtho(Trans.Face->Jacobian(), nor);
      }

      un = vu * nor;
@@ -2616,6 +2581,7 @@ void DGTraceIntegrator::AssembleFaceMatrix(const FiniteElement &el1,
         double rho_p;
         if (un >= 0.0 && ndof2)
         {
+            Trans.Elem2->SetIntPoint(&eip2);
            rho_p = rho->Eval(*Trans.Elem2, eip2);
         }
         else
@@ -2731,7 +2697,7 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(
      {
         order = 2*el1.GetOrder();
      }
-      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+      ir = &IntRules.Get(Trans.FaceGeom, order);
   }

   // assemble: < {(Q \nabla u).n},[v] >      --> elmat
@@ -2739,26 +2705,22 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(
   for (int p = 0; p < ir->GetNPoints(); p++)
   {
      const IntegrationPoint &ip = ir->IntPoint(p);
+      IntegrationPoint eip1, eip2;

-      // Set the integration point in the face and the neighboring elements
-      Trans.SetAllIntPoints(&ip);
-
-      // Access the neighboring elements' integration points
-      // Note: eip2 will only contain valid data if Elem2 exists
-      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
-      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
-
+      Trans.Loc1.Transform(ip, eip1);
+      Trans.Face->SetIntPoint(&ip);
      if (dim == 1)
      {
         nor(0) = 2*eip1.x - 1.0;
      }
      else
      {
-         CalcOrtho(Trans.Jacobian(), nor);
+         CalcOrtho(Trans.Face->Jacobian(), nor);
      }

      el1.CalcShape(eip1, shape1);
      el1.CalcDShape(eip1, dshape1);
+      Trans.Elem1->SetIntPoint(&eip1);
      w = ip.weight/Trans.Elem1->Weight();
      if (ndof2)
      {
@@ -2804,8 +2766,10 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(

      if (ndof2)
      {
+         Trans.Loc2.Transform(ip, eip2);
         el2.CalcShape(eip2, shape2);
         el2.CalcDShape(eip2, dshape2);
+         Trans.Elem2->SetIntPoint(&eip2);
         w = ip.weight/2/Trans.Elem2->Weight();
         if (!MQ)
         {
@@ -3015,20 +2979,16 @@ void DGElasticityIntegrator::AssembleFaceMatrix(
   {
      // a simple choice for the integration order; is this OK?
      const int order = 2 * max(el1.GetOrder(), ndofs2 ? el2.GetOrder() : 0);
-      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+      ir = &IntRules.Get(Trans.FaceGeom, order);
   }

   for (int pind = 0; pind < ir->GetNPoints(); ++pind)
   {
      const IntegrationPoint &ip = ir->IntPoint(pind);
-
-      // Set the integration point in the face and the neighboring elements
-      Trans.SetAllIntPoints(&ip);
-
-      // Access the neighboring elements' integration points
-      // Note: eip2 will only contain valid data if Elem2 exists
-      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
-      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
+      IntegrationPoint eip1, eip2; // integration point in the reference space
+      Trans.Loc1.Transform(ip, eip1);
+      Trans.Face->SetIntPoint(&ip);
+      Trans.Elem1->SetIntPoint(&eip1);

      el1.CalcShape(eip1, shape1);
      el1.CalcDShape(eip1, dshape1);
@@ -3042,12 +3002,14 @@ void DGElasticityIntegrator::AssembleFaceMatrix(
      }
      else
      {
-         CalcOrtho(Trans.Jacobian(), nor);
+         CalcOrtho(Trans.Face->Jacobian(), nor);
      }

      double w, wLM;
      if (ndofs2)
      {
+         Trans.Loc2.Transform(ip, eip2);
+         Trans.Elem2->SetIntPoint(&eip2);
         el2.CalcShape(eip2, shape2);
         el2.CalcDShape(eip2, dshape2);
         CalcAdjugate(Trans.Elem2->Jacobian(), adjJ);
@@ -3177,36 +3139,33 @@ void TraceJumpIntegrator::AssembleFaceMatrix(
      order += trial_face_fe.GetOrder();
      if (trial_face_fe.GetMapType() == FiniteElement::VALUE)
      {
-         order += Trans.OrderW();
+         order += Trans.Face->OrderW();
      }
-      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+      ir = &IntRules.Get(Trans.FaceGeom, order);
   }

   for (int p = 0; p < ir->GetNPoints(); p++)
   {
      const IntegrationPoint &ip = ir->IntPoint(p);
-
-      // Set the integration point in the face and the neighboring elements
-      Trans.SetAllIntPoints(&ip);
-
-      // Access the neighboring elements' integration points
-      // Note: eip2 will only contain valid data if Elem2 exists
-      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
-      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
-
+      IntegrationPoint eip1, eip2;
      // Trace finite element shape function
+      Trans.Face->SetIntPoint(&ip);
      trial_face_fe.CalcShape(ip, face_shape);
      // Side 1 finite element shape function
+      Trans.Loc1.Transform(ip, eip1);
      test_fe1.CalcShape(eip1, shape1);
+      Trans.Elem1->SetIntPoint(&eip1);
      if (ndof2)
      {
         // Side 2 finite element shape function
+         Trans.Loc2.Transform(ip, eip2);
         test_fe2.CalcShape(eip2, shape2);
+         Trans.Elem2->SetIntPoint(&eip2);
      }
      w = ip.weight;
      if (trial_face_fe.GetMapType() == FiniteElement::VALUE)
      {
-         w *= Trans.Weight();
+         w *= Trans.Face->Weight();
      }
      face_shape *= w;
      for (i = 0; i < ndof1; i++)
@@ -3271,7 +3230,7 @@ void NormalTraceJumpIntegrator::AssembleFaceMatrix(
         order = test_fe1.GetOrder() - 1;
      }
      order += trial_face_fe.GetOrder();
-      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+      ir = &IntRules.Get(Trans.FaceGeom, order);
   }

   for (int p = 0; p < ir->GetNPoints(); p++)
@@ -20,13 +20,6 @@
 namespace mfem
 {

-// Local maximum size of dofs and quads in 1D
-constexpr int HCURL_MAX_D1D = 5;
-constexpr int HCURL_MAX_Q1D = 6;
-
-constexpr int HDIV_MAX_D1D = 5;
-constexpr int HDIV_MAX_Q1D = 6;
-
 /// Abstract base class BilinearFormIntegrator
 class BilinearFormIntegrator : public NonlinearFormIntegrator
 {
@@ -64,9 +57,6 @@ public:
   /// Assemble diagonal and add it to Vector @a diag.
   virtual void AssembleDiagonalPA(Vector &diag);

-   /// Assemble diagonal of ADA^T (A is this integrator) and add it to @a diag.
-   virtual void AssembleDiagonalPA_ADAt(const Vector &D, Vector &diag);
-
   /// Method for partially assembled action.
   /** Perform the action of integrator on the input @a x and add the result to
       the output @a y. Both @a x and @a y are E-vectors, i.e. they represent
@@ -85,22 +75,6 @@ public:
       called. */
   virtual void AddMultTransposePA(const Vector &x, Vector &y) const;

-   /// Method defining element assembly.
-   /** The result of the element assembly is added and stored in the @a emat
-       Vector. */
-   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat);
-   /** Used with BilinearFormIntegrators that have different spaces. */
-   // virtual void AssembleEA(const FiniteElementSpace &trial_fes,
-   //                         const FiniteElementSpace &test_fes,
-   //                         Vector &emat);
-
-   virtual void AssembleEAInteriorFaces(const FiniteElementSpace &fes,
-                                        Vector &ea_data_int,
-                                        Vector &ea_data_ext);
-
-   virtual void AssembleEABoundaryFaces(const FiniteElementSpace &fes,
-                                        Vector &ea_data_bdr);
-
   /// Given a particular Finite Element computes the element matrix elmat.
   virtual void AssembleElementMatrix(const FiniteElement &el,
                                      ElementTransformation &Trans,
@@ -206,8 +180,6 @@ public:
   virtual ~BilinearFormIntegrator() { }
 };

-/** Wraps a given @a BilinearFormIntegrator and transposes the resulting element
-    matrices. See for example ex9, ex9p. */
 class TransposeIntegrator : public BilinearFormIntegrator
 {
 private:
@@ -262,15 +234,6 @@ public:
      bfi->AddMultTransposePA(x, y);
   }

-   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat);
-
-   virtual void AssembleEAInteriorFaces(const FiniteElementSpace &fes,
-                                        Vector &ea_data_int,
-                                        Vector &ea_data_ext);
-
-   virtual void AssembleEABoundaryFaces(const FiniteElementSpace &fes,
-                                        Vector &ea_data_bdr);
-
   virtual ~TransposeIntegrator() { if (own_bfi) { delete bfi; } }
 };

@@ -490,16 +453,6 @@ public:
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat);

-   /// Support for use in BilinearForm. Can be used only when appropriate.
-   /** Appropriate use cases are classes derived from
-       MixedScalarVectorIntegrator where the trial and test spaces can be the
-       same. Examples of such classes are: MixedVectorDivergenceIntegrator,
-       MixedScalarWeakDivergenceIntegrator, etc. */
-   virtual void AssembleElementMatrix(const FiniteElement &fe,
-                                      ElementTransformation &Trans,
-                                      DenseMatrix &elmat)
-   { AssembleElementMatrix2(fe, fe, Trans, elmat); }
-
 protected:

   MixedScalarVectorIntegrator(VectorCoefficient &vq, bool _transpose = false,
@@ -1572,7 +1525,7 @@ public:
 };

 /** Class for integrating the bilinear form a(u,v) := (-V u, Grad v) in 2D or 3D
-    and where V is a vector coefficient, u is in H1 or L2 and v is in H1. */
+    and where V is a vector coefficient, u is in H1 and v is in H1. */
 class MixedScalarWeakDivergenceIntegrator : public MixedScalarVectorIntegrator
 {
 public:
@@ -1693,7 +1646,6 @@ protected:
      trial_fe.CalcPhysCurlShape(Trans, shape);
   }

-   using BilinearFormIntegrator::AssemblePA;
   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
                           const FiniteElementSpace &test_fes);

@@ -1704,10 +1656,8 @@ private:
   Vector pa_data;
   const DofToQuad *mapsO;         ///< Not owned. DOF-to-quad map, open.
   const DofToQuad *mapsC;         ///< Not owned. DOF-to-quad map, closed.
-   const DofToQuad *mapsOtest;     ///< Not owned. DOF-to-quad map, open.
-   const DofToQuad *mapsCtest;     ///< Not owned. DOF-to-quad map, closed.
   const GeometricFactors *geom;   ///< Not owned
-   int dim, ne, dofs1D, dofs1Dtest,quad1D, testType, trialType, coeffDim;
+   int dim, ne, dofs1D, quad1D, testType, trialType, coeffDim;
 };

 /** Class for integrating the bilinear form a(u,v) := (Q u, curl v) in 3D and
@@ -1748,7 +1698,6 @@ protected:
      test_fe.CalcPhysCurlShape(Trans, shape);
   }

-   using BilinearFormIntegrator::AssemblePA;
   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
                           const FiniteElementSpace &test_fes);

@@ -1952,8 +1901,6 @@ public:

   virtual void AssemblePA(const FiniteElementSpace &fes);

-   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat);
-
   virtual void AssembleDiagonalPA(Vector &diag);

   virtual void AddMultPA(const Vector&, Vector&) const;
@@ -2027,8 +1974,6 @@ public:

   virtual void AssemblePA(const FiniteElementSpace &fes);

-   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat);
-
   virtual void AssembleDiagonalPA(Vector &diag);

   virtual void AddMultPA(const Vector&, Vector&) const;
@@ -2040,7 +1985,6 @@ public:
   void SetupPA(const FiniteElementSpace &fes, const bool force = false);
 };

-/** Mass integrator (u, v) restricted to the boundary of a domain */
 class BoundaryMassIntegrator : public MassIntegrator
 {
 public:
@@ -2083,8 +2027,6 @@ public:

   virtual void AssemblePA(const FiniteElementSpace&);

-   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat);
-
   virtual void AddMultPA(const Vector&, Vector&) const;

   static const IntegrationRule &GetRule(const FiniteElement &el,
@@ -2184,25 +2126,11 @@ class VectorFEDivergenceIntegrator : public BilinearFormIntegrator
 protected:
   Coefficient *Q;

-   using BilinearFormIntegrator::AssemblePA;
-   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
-                           const FiniteElementSpace &test_fes);
-
-   virtual void AddMultPA(const Vector&, Vector&) const;
-   virtual void AddMultTransposePA(const Vector&, Vector&) const;
-
 private:
 #ifndef MFEM_THREAD_SAFE
   Vector divshape, shape;
 #endif

-   // PA extension
-   Vector pa_data;
-   const DofToQuad *mapsO;         ///< Not owned. DOF-to-quad map, open.
-   const DofToQuad *L2mapsO;       ///< Not owned. DOF-to-quad map, open.
-   const DofToQuad *mapsC;         ///< Not owned. DOF-to-quad map, closed.
-   int dim, ne, dofs1D, L2dofs1D, quad1D;
-
 public:
   VectorFEDivergenceIntegrator() { Q = NULL; }
   VectorFEDivergenceIntegrator(Coefficient &q) { Q = &q; }
@@ -2213,8 +2141,6 @@ public:
                                       const FiniteElement &test_fe,
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat);
-
-   virtual void AssembleDiagonalPA_ADAt(const Vector &D, Vector &diag);
 };


@@ -2300,14 +2226,12 @@ class CurlCurlIntegrator: public BilinearFormIntegrator
 private:
   Vector vec, pointflux;
 #ifndef MFEM_THREAD_SAFE
-   Vector D;
   DenseMatrix curlshape, curlshape_dFt, M;
   DenseMatrix vshape, projcurl;
 #endif

 protected:
   Coefficient *Q;
-   VectorCoefficient *DQ;
   MatrixCoefficient *MQ;

   // PA extension
@@ -2316,17 +2240,12 @@ protected:
   const DofToQuad *mapsC;         ///< Not owned. DOF-to-quad map, closed.
   const GeometricFactors *geom;   ///< Not owned
   int dim, ne, nq, dofs1D, quad1D;
-   bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient

 public:
-   CurlCurlIntegrator() { Q = NULL; DQ = NULL; MQ = NULL; }
+   CurlCurlIntegrator() { Q = NULL; MQ = NULL; }
   /// Construct a bilinear form integrator for Nedelec elements
-   CurlCurlIntegrator(Coefficient &q, const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), Q(&q) { DQ = NULL; MQ = NULL; }
-   CurlCurlIntegrator(VectorCoefficient &dq, const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), DQ(&dq) { Q = NULL; MQ = NULL; }
-   CurlCurlIntegrator(MatrixCoefficient &mq, const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), MQ(&mq) { Q = NULL; DQ = NULL; }
+   CurlCurlIntegrator(Coefficient &q) : Q(&q) { MQ = NULL; }
+   CurlCurlIntegrator(MatrixCoefficient &m) : MQ(&m) { Q = NULL; }

   /* Given a particular Finite Element, compute the
      element curl-curl matrix elmat */
@@ -2404,11 +2323,8 @@ protected:
   Vector pa_data;
   const DofToQuad *mapsO;         ///< Not owned. DOF-to-quad map, open.
   const DofToQuad *mapsC;         ///< Not owned. DOF-to-quad map, closed.
-   const DofToQuad *mapsOtest;     ///< Not owned. DOF-to-quad map, open.
-   const DofToQuad *mapsCtest;     ///< Not owned. DOF-to-quad map, closed.
   const GeometricFactors *geom;   ///< Not owned
-   int dim, ne, nq, dofs1D, dofs1Dtest, quad1D, trial_fetype, test_fetype;
-   bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient
+   int dim, ne, nq, dofs1D, quad1D;

 public:
   VectorFEMassIntegrator() { Init(NULL, NULL, NULL); }
@@ -2429,8 +2345,6 @@ public:

   using BilinearFormIntegrator::AssemblePA;
   virtual void AssemblePA(const FiniteElementSpace &fes);
-   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
-                           const FiniteElementSpace &test_fes);
   virtual void AddMultPA(const Vector &x, Vector &y) const;
   virtual void AssembleDiagonalPA(Vector& diag);
 };
@@ -2489,23 +2403,11 @@ class DivDivIntegrator: public BilinearFormIntegrator
 protected:
   Coefficient *Q;

-   using BilinearFormIntegrator::AssemblePA;
-   virtual void AssemblePA(const FiniteElementSpace &fes);
-   virtual void AddMultPA(const Vector &x, Vector &y) const;
-   virtual void AssembleDiagonalPA(Vector& diag);
-
 private:
 #ifndef MFEM_THREAD_SAFE
   Vector divshape;
 #endif

-   // PA extension
-   Vector pa_data;
-   const DofToQuad *mapsO;         ///< Not owned. DOF-to-quad map, open.
-   const DofToQuad *mapsC;         ///< Not owned. DOF-to-quad map, closed.
-   const GeometricFactors *geom;   ///< Not owned
-   int dim, ne, dofs1D, quad1D;
-
 public:
   DivDivIntegrator() { Q = NULL; }
   DivDivIntegrator(Coefficient &q) : Q(&q) { }
@@ -2529,12 +2431,14 @@ protected:
   // PA extension
   const DofToQuad *maps;         ///< Not owned
   const GeometricFactors *geom;  ///< Not owned
-   int dim, sdim, ne, dofs1D, quad1D;
+   int dim, ne, dofs1D, quad1D;
   Vector pa_data;

 private:
-   DenseMatrix dshape, dshapedxt, pelmat;
-   DenseMatrix Jinv, gshape;
+   DenseMatrix Jinv;
+   DenseMatrix dshape;
+   DenseMatrix gshape;
+   DenseMatrix pelmat;

 public:
   VectorDiffusionIntegrator() { Q = NULL; }
@@ -2658,13 +2562,6 @@ public:

   virtual void AddMultPA(const Vector&, Vector&) const;

-   virtual void AssembleEAInteriorFaces(const FiniteElementSpace& fes,
-                                        Vector &ea_data_int,
-                                        Vector &ea_data_ext);
-
-   virtual void AssembleEABoundaryFaces(const FiniteElementSpace& fes,
-                                        Vector &ea_data_bdr);
-
   static const IntegrationRule &GetRule(Geometry::Type geom, int order,
                                         FaceElementTransformations &T);

@@ -788,20 +788,6 @@ void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   {
      vel = cQ->GetVec();
   }
-   else if (VectorQuadratureFunctionCoefficient* cQ =
-               dynamic_cast<VectorQuadratureFunctionCoefficient*>(Q))
-   {
-      const QuadratureFunction &qFun = cQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == dim * nq * ne,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-
-      qFun.Read();
-      vel.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
   else
   {
      vel.SetSize(dim * nq * ne);
@@ -1,258 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-
-namespace mfem
-{
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void EAConvectionAssemble1D(const int NE,
-                                   const Array<double> &b,
-                                   const Array<double> &g,
-                                   const Vector &padata,
-                                   Vector &eadata,
-                                   const int d1d = 0,
-                                   const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto G = Reshape(g.Read(), Q1D, D1D);
-   auto D = Reshape(padata.Read(), Q1D, NE);
-   auto A = Reshape(eadata.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      double r_Gi[MQ1];
-      double r_Bj[MQ1];
-      for (int q = 0; q < Q1D; q++)
-      {
-         r_Gi[q] = G(q,MFEM_THREAD_ID(x));
-         r_Bj[q] = B(q,MFEM_THREAD_ID(y));
-      }
-      MFEM_FOREACH_THREAD(i1,x,D1D)
-      {
-         MFEM_FOREACH_THREAD(j1,y,D1D)
-         {
-            double val = 0.0;
-            for (int k1 = 0; k1 < Q1D; ++k1)
-            {
-               val += r_Bj[k1] * D(k1, e) * r_Gi[k1];
-            }
-            A(i1, j1, e) += val;
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void EAConvectionAssemble2D(const int NE,
-                                   const Array<double> &b,
-                                   const Array<double> &g,
-                                   const Vector &padata,
-                                   Vector &eadata,
-                                   const int d1d = 0,
-                                   const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto G = Reshape(g.Read(), Q1D, D1D);
-   auto D = Reshape(padata.Read(), Q1D, Q1D, 2, NE);
-   auto A = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      double r_B[MQ1][MD1];
-      double r_G[MQ1][MD1];
-      for (int d = 0; d < D1D; d++)
-      {
-         for (int q = 0; q < Q1D; q++)
-         {
-            r_B[q][d] = B(q,d);
-            r_G[q][d] = G(q,d);
-         }
-      }
-      MFEM_SHARED double s_D[MQ1][MQ1][2];
-      MFEM_FOREACH_THREAD(k1,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(k2,y,Q1D)
-         {
-            s_D[k1][k2][0] = D(k1,k2,0,e);
-            s_D[k1][k2][1] = D(k1,k2,1,e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(i1,x,D1D)
-      {
-         MFEM_FOREACH_THREAD(i2,y,D1D)
-         {
-            for (int j1 = 0; j1 < D1D; ++j1)
-            {
-               for (int j2 = 0; j2 < D1D; ++j2)
-               {
-                  double val = 0.0;
-                  for (int k1 = 0; k1 < Q1D; ++k1)
-                  {
-                     for (int k2 = 0; k2 < Q1D; ++k2)
-                     {
-                        val += (r_G[k1][i1] * r_B[k2][i2] * s_D[k1][k2][0]
-                                + r_B[k1][i1] * r_G[k2][i2] * s_D[k1][k2][1])
-                               * r_B[k1][j1]* r_B[k2][j2];
-                     }
-                  }
-                  A(i1, i2, j1, j2, e) += val;
-               }
-            }
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void EAConvectionAssemble3D(const int NE,
-                                   const Array<double> &b,
-                                   const Array<double> &g,
-                                   const Vector &padata,
-                                   Vector &eadata,
-                                   const int d1d = 0,
-                                   const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto G = Reshape(g.Read(), Q1D, D1D);
-   auto D = Reshape(padata.Read(), Q1D, Q1D, Q1D, 3, NE);
-   auto A = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, D1D, D1D, D1D,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      double r_B[MQ1][MD1];
-      double r_G[MQ1][MD1];
-      for (int d = 0; d < D1D; d++)
-      {
-         for (int q = 0; q < Q1D; q++)
-         {
-            r_B[q][d] = B(q,d);
-            r_G[q][d] = G(q,d);
-         }
-      }
-      MFEM_FOREACH_THREAD(i1,x,D1D)
-      {
-         MFEM_FOREACH_THREAD(i2,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(i3,z,D1D)
-            {
-               for (int j1 = 0; j1 < D1D; ++j1)
-               {
-                  for (int j2 = 0; j2 < D1D; ++j2)
-                  {
-                     for (int j3 = 0; j3 < D1D; ++j3)
-                     {
-                        double val = 0.0;
-                        for (int k1 = 0; k1 < Q1D; ++k1)
-                        {
-                           for (int k2 = 0; k2 < Q1D; ++k2)
-                           {
-                              for (int k3 = 0; k3 < Q1D; ++k3)
-                              {
-                                 double D0 = D(k1,k2,k3,0,e);
-                                 double D1 = D(k1,k2,k3,1,e);
-                                 double D2 = D(k1,k2,k3,2,e);
-                                 val += (r_G[k1][i1] * r_B[k2][i2] * r_B[k3][i3] * D0
-                                         + r_B[k1][i1] * r_G[k2][i2] * r_B[k3][i3] * D1
-                                         + r_B[k1][i1] * r_B[k2][i2] * r_G[k3][i3] * D2)
-                                        * r_B[k1][j1] * r_B[k2][j2] * r_B[k3][j3];
-                              }
-                           }
-                        }
-                        A(i1, i2, i3, j1, j2, j3, e) += val;
-                     }
-                  }
-               }
-            }
-         }
-      }
-   });
-}
-
-void ConvectionIntegrator::AssembleEA(const FiniteElementSpace &fes,
-                                      Vector &ea_data)
-{
-   AssemblePA(fes);
-   const int ne = fes.GetMesh()->GetNE();
-   const Array<double> &B = maps->B;
-   const Array<double> &G = maps->G;
-   if (dim == 1)
-   {
-      switch ((dofs1D << 4 ) | quad1D)
-      {
-         case 0x22: return EAConvectionAssemble1D<2,2>(ne,B,G,pa_data,ea_data);
-         case 0x33: return EAConvectionAssemble1D<3,3>(ne,B,G,pa_data,ea_data);
-         case 0x44: return EAConvectionAssemble1D<4,4>(ne,B,G,pa_data,ea_data);
-         case 0x55: return EAConvectionAssemble1D<5,5>(ne,B,G,pa_data,ea_data);
-         case 0x66: return EAConvectionAssemble1D<6,6>(ne,B,G,pa_data,ea_data);
-         case 0x77: return EAConvectionAssemble1D<7,7>(ne,B,G,pa_data,ea_data);
-         case 0x88: return EAConvectionAssemble1D<8,8>(ne,B,G,pa_data,ea_data);
-         case 0x99: return EAConvectionAssemble1D<9,9>(ne,B,G,pa_data,ea_data);
-         default:   return EAConvectionAssemble1D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
-      }
-   }
-   else if (dim == 2)
-   {
-      switch ((dofs1D << 4 ) | quad1D)
-      {
-         case 0x22: return EAConvectionAssemble2D<2,2>(ne,B,G,pa_data,ea_data);
-         case 0x33: return EAConvectionAssemble2D<3,3>(ne,B,G,pa_data,ea_data);
-         case 0x44: return EAConvectionAssemble2D<4,4>(ne,B,G,pa_data,ea_data);
-         case 0x55: return EAConvectionAssemble2D<5,5>(ne,B,G,pa_data,ea_data);
-         case 0x66: return EAConvectionAssemble2D<6,6>(ne,B,G,pa_data,ea_data);
-         case 0x77: return EAConvectionAssemble2D<7,7>(ne,B,G,pa_data,ea_data);
-         case 0x88: return EAConvectionAssemble2D<8,8>(ne,B,G,pa_data,ea_data);
-         case 0x99: return EAConvectionAssemble2D<9,9>(ne,B,G,pa_data,ea_data);
-         default:   return EAConvectionAssemble2D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
-      }
-   }
-   else if (dim == 3)
-   {
-      switch ((dofs1D << 4 ) | quad1D)
-      {
-         case 0x23: return EAConvectionAssemble3D<2,3>(ne,B,G,pa_data,ea_data);
-         case 0x34: return EAConvectionAssemble3D<3,4>(ne,B,G,pa_data,ea_data);
-         case 0x45: return EAConvectionAssemble3D<4,5>(ne,B,G,pa_data,ea_data);
-         case 0x56: return EAConvectionAssemble3D<5,6>(ne,B,G,pa_data,ea_data);
-         case 0x67: return EAConvectionAssemble3D<6,7>(ne,B,G,pa_data,ea_data);
-         case 0x78: return EAConvectionAssemble3D<7,8>(ne,B,G,pa_data,ea_data);
-         case 0x89: return EAConvectionAssemble3D<8,9>(ne,B,G,pa_data,ea_data);
-         default:   return EAConvectionAssemble3D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
-      }
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-
-}
@@ -167,19 +167,6 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
      r.SetSize(1);
      r(0) = c_rho->constant;
   }
-   else if (QuadratureFunctionCoefficient* c_rho =
-               dynamic_cast<QuadratureFunctionCoefficient*>(rho))
-   {
-      const QuadratureFunction &qFun = c_rho->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == nq * nf,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      r.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
   else
   {
      r.SetSize(nq * nf);
@@ -213,20 +200,6 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
   {
      vel = c_u->GetVec();
   }
-   else if (VectorQuadratureFunctionCoefficient* c_u =
-               dynamic_cast<VectorQuadratureFunctionCoefficient*>(u))
-   {
-      // Assumed to be in lexicographical ordering
-      const QuadratureFunction &qFun = c_u->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == dim * nq * nf,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      vel.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
   else
   {
      vel.SetSize(dim * nq * nf);
@@ -1,414 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-
-namespace mfem
-{
-
-static void EADGTraceAssemble1DInt(const int NF,
-                                   const Array<double> &basis,
-                                   const Vector &padata,
-                                   Vector &eadata_int,
-                                   Vector &eadata_ext)
-{
-   auto D = Reshape(padata.Read(), 2, 2, NF);
-   auto A_int = Reshape(eadata_int.ReadWrite(), 2, NF);
-   auto A_ext = Reshape(eadata_ext.ReadWrite(), 2, NF);
-   MFEM_FORALL(f, NF,
-   {
-      double val_int0, val_int1, val_ext01, val_ext10;
-      val_int0  = D(0, 0, f);
-      val_ext10 = D(1, 0, f);
-      val_ext01 = D(0, 1, f);
-      val_int1  = D(1, 1, f);
-      A_int(0, f) += val_int0;
-      A_int(1, f) += val_int1;
-      A_ext(0, f) += val_ext01;
-      A_ext(1, f) += val_ext10;
-   });
-}
-
-static void EADGTraceAssemble1DBdr(const int NF,
-                                   const Array<double> &basis,
-                                   const Vector &padata,
-                                   Vector &eadata_bdr)
-{
-   auto D = Reshape(padata.Read(), 2, 2, NF);
-   auto A_bdr = Reshape(eadata_bdr.ReadWrite(), NF);
-   MFEM_FORALL(f, NF,
-   {
-      A_bdr(f) += D(0, 0, f);
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void EADGTraceAssemble2DInt(const int NF,
-                                   const Array<double> &basis,
-                                   const Vector &padata,
-                                   Vector &eadata_int,
-                                   Vector &eadata_ext,
-                                   const int d1d = 0,
-                                   const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(basis.Read(), Q1D, D1D);
-   auto D = Reshape(padata.Read(), Q1D, 2, 2, NF);
-   auto A_int = Reshape(eadata_int.ReadWrite(), D1D, D1D, 2, NF);
-   auto A_ext = Reshape(eadata_ext.ReadWrite(), D1D, D1D, 2, NF);
-   MFEM_FORALL_3D(f, NF, D1D, D1D, 1,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      MFEM_FOREACH_THREAD(i1,x,D1D)
-      {
-         MFEM_FOREACH_THREAD(j1,y,D1D)
-         {
-            double val_int0 = 0.0;
-            double val_int1 = 0.0;
-            double val_ext01 = 0.0;
-            double val_ext10 = 0.0;
-            for (int k1 = 0; k1 < Q1D; ++k1)
-            {
-               val_int0  += B(k1,i1) * B(k1,j1) * D(k1, 0, 0, f);
-               val_ext01 += B(k1,i1) * B(k1,j1) * D(k1, 0, 1, f);
-               val_ext10 += B(k1,i1) * B(k1,j1) * D(k1, 1, 0, f);
-               val_int1  += B(k1,i1) * B(k1,j1) * D(k1, 1, 1, f);
-            }
-            A_int(i1, j1, 0, f) += val_int0;
-            A_int(i1, j1, 1, f) += val_int1;
-            A_ext(i1, j1, 0, f) += val_ext01;
-            A_ext(i1, j1, 1, f) += val_ext10;
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void EADGTraceAssemble2DBdr(const int NF,
-                                   const Array<double> &basis,
-                                   const Vector &padata,
-                                   Vector &eadata_bdr,
-                                   const int d1d = 0,
-                                   const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(basis.Read(), Q1D, D1D);
-   auto D = Reshape(padata.Read(), Q1D, 2, 2, NF);
-   auto A_bdr = Reshape(eadata_bdr.ReadWrite(), D1D, D1D, NF);
-   MFEM_FORALL_3D(f, NF, D1D, D1D, 1,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      MFEM_FOREACH_THREAD(i1,x,D1D)
-      {
-         MFEM_FOREACH_THREAD(j1,y,D1D)
-         {
-            double val_bdr = 0.0;
-            for (int k1 = 0; k1 < Q1D; ++k1)
-            {
-               val_bdr  += B(k1,i1) * B(k1,j1) * D(k1, 0, 0, f);
-            }
-            A_bdr(i1, j1, f) += val_bdr;
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void EADGTraceAssemble3DInt(const int NF,
-                                   const Array<double> &basis,
-                                   const Vector &padata,
-                                   Vector &eadata_int,
-                                   Vector &eadata_ext,
-                                   const int d1d = 0,
-                                   const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(basis.Read(), Q1D, D1D);
-   auto D = Reshape(padata.Read(), Q1D, Q1D, 2, 2, NF);
-   auto A_int = Reshape(eadata_int.ReadWrite(), D1D, D1D, D1D, D1D, 2, NF);
-   auto A_ext = Reshape(eadata_ext.ReadWrite(), D1D, D1D, D1D, D1D, 2, NF);
-   MFEM_FORALL_3D(f, NF, D1D, D1D, 1,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      double r_B[MQ1][MD1];
-      for (int d = 0; d < D1D; d++)
-      {
-         for (int q = 0; q < Q1D; q++)
-         {
-            r_B[q][d] = B(q,d);
-         }
-      }
-      MFEM_SHARED double s_D[MQ1][MQ1][2][2];
-      for (int i=0; i < 2; i++)
-      {
-         for (int j=0; j < 2; j++)
-         {
-            MFEM_FOREACH_THREAD(k1,x,Q1D)
-            {
-               MFEM_FOREACH_THREAD(k2,y,Q1D)
-               {
-                  s_D[k1][k2][i][j] = D(k1,k2,i,j,f);
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(i1,x,D1D)
-      {
-         MFEM_FOREACH_THREAD(i2,y,D1D)
-         {
-            for (int j1 = 0; j1 < D1D; ++j1)
-            {
-               for (int j2 = 0; j2 < D1D; ++j2)
-               {
-                  double val_int0 = 0.0;
-                  double val_int1 = 0.0;
-                  double val_ext01 = 0.0;
-                  double val_ext10 = 0.0;
-                  for (int k1 = 0; k1 < Q1D; ++k1)
-                  {
-                     for (int k2 = 0; k2 < Q1D; ++k2)
-                     {
-                        val_int0 += r_B[k1][i1] * r_B[k1][j1]
-                                    * r_B[k2][i2] * r_B[k2][j2]
-                                    * s_D[k1][k2][0][0];
-                        val_int1 += r_B[k1][i1] * r_B[k1][j1]
-                                    * r_B[k2][i2] * r_B[k2][j2]
-                                    * s_D[k1][k2][1][1];
-                        val_ext01+= r_B[k1][i1] * r_B[k1][j1]
-                                    * r_B[k2][i2] * r_B[k2][j2]
-                                    * s_D[k1][k2][0][1];
-                        val_ext10+= r_B[k1][i1] * r_B[k1][j1]
-                                    * r_B[k2][i2] * r_B[k2][j2]
-                                    * s_D[k1][k2][1][0];
-                     }
-                  }
-                  A_int(i1, i2, j1, j2, 0, f) += val_int0;
-                  A_int(i1, i2, j1, j2, 1, f) += val_int1;
-                  A_ext(i1, i2, j1, j2, 0, f) += val_ext01;
-                  A_ext(i1, i2, j1, j2, 1, f) += val_ext10;
-               }
-            }
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void EADGTraceAssemble3DBdr(const int NF,
-                                   const Array<double> &basis,
-                                   const Vector &padata,
-                                   Vector &eadata_bdr,
-                                   const int d1d = 0,
-                                   const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(basis.Read(), Q1D, D1D);
-   auto D = Reshape(padata.Read(), Q1D, Q1D, 2, 2, NF);
-   auto A_bdr = Reshape(eadata_bdr.ReadWrite(), D1D, D1D, D1D, D1D, NF);
-   MFEM_FORALL_3D(f, NF, D1D, D1D, 1,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      double r_B[MQ1][MD1];
-      for (int d = 0; d < D1D; d++)
-      {
-         for (int q = 0; q < Q1D; q++)
-         {
-            r_B[q][d] = B(q,d);
-         }
-      }
-      MFEM_SHARED double s_D[MQ1][MQ1][2][2];
-      for (int i=0; i < 2; i++)
-      {
-         for (int j=0; j < 2; j++)
-         {
-            MFEM_FOREACH_THREAD(k1,x,Q1D)
-            {
-               MFEM_FOREACH_THREAD(k2,y,Q1D)
-               {
-                  s_D[k1][k2][i][j] = D(k1,k2,i,j,f);
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(i1,x,D1D)
-      {
-         MFEM_FOREACH_THREAD(i2,y,D1D)
-         {
-            for (int j1 = 0; j1 < D1D; ++j1)
-            {
-               for (int j2 = 0; j2 < D1D; ++j2)
-               {
-                  double val_bdr = 0.0;
-                  for (int k1 = 0; k1 < Q1D; ++k1)
-                  {
-                     for (int k2 = 0; k2 < Q1D; ++k2)
-                     {
-                        val_bdr += r_B[k1][i1] * r_B[k1][j1]
-                                   * r_B[k2][i2] * r_B[k2][j2]
-                                   * s_D[k1][k2][0][0];
-                     }
-                  }
-                  A_bdr(i1, i2, j1, j2, f) += val_bdr;
-               }
-            }
-         }
-      }
-   });
-}
-
-void DGTraceIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
-                                                Vector &ea_data_int,
-                                                Vector &ea_data_ext)
-{
-   SetupPA(fes, FaceType::Interior);
-   nf = fes.GetNFbyType(FaceType::Interior);
-   if (nf==0) { return; }
-   const Array<double> &B = maps->B;
-   if (dim == 1)
-   {
-      return EADGTraceAssemble1DInt(nf,B,pa_data,ea_data_int,ea_data_ext);
-   }
-   else if (dim == 2)
-   {
-      switch ((dofs1D << 4 ) | quad1D)
-      {
-         case 0x22:
-            return EADGTraceAssemble2DInt<2,2>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
-         case 0x33:
-            return EADGTraceAssemble2DInt<3,3>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
-         case 0x44:
-            return EADGTraceAssemble2DInt<4,4>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
-         case 0x55:
-            return EADGTraceAssemble2DInt<5,5>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
-         case 0x66:
-            return EADGTraceAssemble2DInt<6,6>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
-         case 0x77:
-            return EADGTraceAssemble2DInt<7,7>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
-         case 0x88:
-            return EADGTraceAssemble2DInt<8,8>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
-         case 0x99:
-            return EADGTraceAssemble2DInt<9,9>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
-         default:
-            return EADGTraceAssemble2DInt(nf,B,pa_data,ea_data_int,
-                                          ea_data_ext,dofs1D,quad1D);
-      }
-   }
-   else if (dim == 3)
-   {
-      switch ((dofs1D << 4 ) | quad1D)
-      {
-         case 0x23:
-            return EADGTraceAssemble3DInt<2,3>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
-         case 0x34:
-            return EADGTraceAssemble3DInt<3,4>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
-         case 0x45:
-            return EADGTraceAssemble3DInt<4,5>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
-         case 0x56:
-            return EADGTraceAssemble3DInt<5,6>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
-         case 0x67:
-            return EADGTraceAssemble3DInt<6,7>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
-         case 0x78:
-            return EADGTraceAssemble3DInt<7,8>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
-         case 0x89:
-            return EADGTraceAssemble3DInt<8,9>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext);
-         default:
-            return EADGTraceAssemble3DInt(nf,B,pa_data,ea_data_int,
-                                          ea_data_ext,dofs1D,quad1D);
-      }
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-
-void DGTraceIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace& fes,
-                                                Vector &ea_data_bdr)
-{
-   SetupPA(fes, FaceType::Boundary);
-   nf = fes.GetNFbyType(FaceType::Boundary);
-   if (nf==0) { return; }
-   const Array<double> &B = maps->B;
-   if (dim == 1)
-   {
-      return EADGTraceAssemble1DBdr(nf,B,pa_data,ea_data_bdr);
-   }
-   else if (dim == 2)
-   {
-      switch ((dofs1D << 4 ) | quad1D)
-      {
-         case 0x22: return EADGTraceAssemble2DBdr<2,2>(nf,B,pa_data,ea_data_bdr);
-         case 0x33: return EADGTraceAssemble2DBdr<3,3>(nf,B,pa_data,ea_data_bdr);
-         case 0x44: return EADGTraceAssemble2DBdr<4,4>(nf,B,pa_data,ea_data_bdr);
-         case 0x55: return EADGTraceAssemble2DBdr<5,5>(nf,B,pa_data,ea_data_bdr);
-         case 0x66: return EADGTraceAssemble2DBdr<6,6>(nf,B,pa_data,ea_data_bdr);
-         case 0x77: return EADGTraceAssemble2DBdr<7,7>(nf,B,pa_data,ea_data_bdr);
-         case 0x88: return EADGTraceAssemble2DBdr<8,8>(nf,B,pa_data,ea_data_bdr);
-         case 0x99: return EADGTraceAssemble2DBdr<9,9>(nf,B,pa_data,ea_data_bdr);
-         default:
-            return EADGTraceAssemble2DBdr(nf,B,pa_data,ea_data_bdr,dofs1D,quad1D);
-      }
-   }
-   else if (dim == 3)
-   {
-      switch ((dofs1D << 4 ) | quad1D)
-      {
-         case 0x23: return EADGTraceAssemble3DBdr<2,3>(nf,B,pa_data,ea_data_bdr);
-         case 0x34: return EADGTraceAssemble3DBdr<3,4>(nf,B,pa_data,ea_data_bdr);
-         case 0x45: return EADGTraceAssemble3DBdr<4,5>(nf,B,pa_data,ea_data_bdr);
-         case 0x56: return EADGTraceAssemble3DBdr<5,6>(nf,B,pa_data,ea_data_bdr);
-         case 0x67: return EADGTraceAssemble3DBdr<6,7>(nf,B,pa_data,ea_data_bdr);
-         case 0x78: return EADGTraceAssemble3DBdr<7,8>(nf,B,pa_data,ea_data_bdr);
-         case 0x89: return EADGTraceAssemble3DBdr<8,9>(nf,B,pa_data,ea_data_bdr);
-         default:
-            return EADGTraceAssemble3DBdr(nf,B,pa_data,ea_data_bdr,dofs1D,quad1D);
-      }
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-
-}
@@ -81,20 +81,12 @@ static void OccaPADiffusionSetup3D(const int D1D,
 #endif // MFEM_USE_OCCA

 // PA Diffusion Assemble 2D kernel
-template<const int T_SDIM>
 static void PADiffusionSetup2D(const int Q1D,
                               const int NE,
                               const Array<double> &w,
                               const Vector &j,
                               const Vector &c,
-                               Vector &d);
-template<>
-void PADiffusionSetup2D<2>(const int Q1D,
-                           const int NE,
-                           const Array<double> &w,
-                           const Vector &j,
-                           const Vector &c,
-                           Vector &d)
+                               Vector &d)
 {
   const int NQ = Q1D*Q1D;
   const bool const_c = c.Size() == 1;
@@ -120,48 +112,6 @@ void PADiffusionSetup2D<2>(const int Q1D,
   });
 }

-// PA Diffusion Assemble 2D kernel with 3D node coords
-template<>
-void PADiffusionSetup2D<3>(const int Q1D,
-                           const int NE,
-                           const Array<double> &w,
-                           const Vector &j,
-                           const Vector &c,
-                           Vector &d)
-{
-   constexpr int DIM = 2;
-   constexpr int SDIM = 3;
-   const int NQ = Q1D*Q1D;
-   const bool const_c = c.Size() == 1;
-
-   auto W = w.Read();
-   auto J = Reshape(j.Read(), NQ, SDIM, DIM, NE);
-   auto C = const_c ? Reshape(c.Read(), 1, 1) : Reshape(c.Read(), NQ, NE);
-   auto D = Reshape(d.Write(), NQ, 3, NE);
-   MFEM_FORALL(e, NE,
-   {
-      for (int q = 0; q < NQ; ++q)
-      {
-         const double wq = W[q];
-         const double J11 = J(q,0,0,e);
-         const double J21 = J(q,1,0,e);
-         const double J31 = J(q,2,0,e);
-         const double J12 = J(q,0,1,e);
-         const double J22 = J(q,1,1,e);
-         const double J32 = J(q,2,1,e);
-         const double E = J11*J11 + J21*J21 + J31*J31;
-         const double G = J12*J12 + J22*J22 + J32*J32;
-         const double F = J11*J12 + J21*J22 + J31*J32;
-         const double iw = 1.0 / sqrt(E*G - F*F);
-         const double coeff = const_c ? C(0,0) : C(q,e);
-         const double alpha = wq * coeff * iw;
-         D(q,0,e) =  alpha * G; // 1,1
-         D(q,1,e) = -alpha * F; // 1,2
-         D(q,2,e) =  alpha * E; // 2,2
-      }
-   });
-}
-
 // PA Diffusion Assemble 3D kernel
 static void PADiffusionSetup3D(const int Q1D,
                               const int NE,
@@ -216,7 +166,6 @@ static void PADiffusionSetup3D(const int Q1D,
 }

 static void PADiffusionSetup(const int dim,
-                             const int sdim,
                             const int D1D,
                             const int Q1D,
                             const int NE,
@@ -234,11 +183,8 @@ static void PADiffusionSetup(const int dim,
         OccaPADiffusionSetup2D(D1D, Q1D, NE, W, J, C, D);
         return;
      }
-#else
-      MFEM_CONTRACT_VAR(D1D);
 #endif // MFEM_USE_OCCA
-      if (sdim == 2) { PADiffusionSetup2D<2>(Q1D, NE, W, J, C, D); }
-      if (sdim == 3) { PADiffusionSetup2D<3>(Q1D, NE, W, J, C, D); }
+      PADiffusionSetup2D(Q1D, NE, W, J, C, D);
   }
   if (dim == 3)
   {
@@ -271,8 +217,6 @@ void DiffusionIntegrator::SetupPA(const FiniteElementSpace &fes,
      InitCeedCoeff(Q, ptr);
      return CeedPADiffusionAssemble(fes, *ir, *ptr);
   }
-#else
-   MFEM_CONTRACT_VAR(force);
 #endif
   const int dims = el.GetDim();
   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
@@ -280,7 +224,6 @@ void DiffusionIntegrator::SetupPA(const FiniteElementSpace &fes,
   dim = mesh->Dimension();
   ne = fes.GetNE();
   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
-   const int sdim = mesh->SpaceDimension();
   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;
@@ -296,19 +239,6 @@ void DiffusionIntegrator::SetupPA(const FiniteElementSpace &fes,
      coeff.SetSize(1);
      coeff(0) = cQ->constant;
   }
-   else if (QuadratureFunctionCoefficient* cQ =
-               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
-   {
-      const QuadratureFunction &qFun = cQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == ne*nq,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
   else
   {
      coeff.SetSize(nq * ne);
@@ -322,8 +252,8 @@ void DiffusionIntegrator::SetupPA(const FiniteElementSpace &fes,
         }
      }
   }
-   PADiffusionSetup(dim, sdim, dofs1D, quad1D, ne, ir->GetWeights(), geom->J,
-                    coeff, pa_data);
+   PADiffusionSetup(dim, dofs1D, quad1D, ne, ir->GetWeights(), geom->J, coeff,
+                    pa_data);
 }

 void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
@@ -980,6 +910,8 @@ template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
 static void SmemPADiffusionApply2D(const int NE,
                                   const Array<double> &b_,
                                   const Array<double> &g_,
+                                   const Array<double> &bt_,
+                                   const Array<double> &gt_,
                                   const Vector &d_,
                                   const Vector &x_,
                                   Vector &y_,
@@ -1320,37 +1252,13 @@ static void PADiffusionApply3D(const int NE,
   });
 }

-// Half of B and G are stored in shared to get B, Bt, G and Gt.
-// Indices computation for SmemPADiffusionApply3D.
-static MFEM_HOST_DEVICE inline int qi(const int q, const int d, const int Q)
-{
-   return (q<=d) ? q : Q-1-q;
-}
-
-static MFEM_HOST_DEVICE inline int dj(const int q, const int d, const int D)
-{
-   return (q<=d) ? d : D-1-d;
-}
-
-static MFEM_HOST_DEVICE inline int qk(const int q, const int d, const int Q)
-{
-   return (q<=d) ? Q-1-q : q;
-}
-
-static MFEM_HOST_DEVICE inline int dl(const int q, const int d, const int D)
-{
-   return (q<=d) ? D-1-d : d;
-}
-
-static MFEM_HOST_DEVICE inline double sign(const int q, const int d)
-{
-   return (q<=d) ? -1.0 : 1.0;
-}
-
+// Shared memory PA Diffusion Apply 3D kernel
 template<int T_D1D = 0, int T_Q1D = 0>
 static void SmemPADiffusionApply3D(const int NE,
                                   const Array<double> &b_,
                                   const Array<double> &g_,
+                                   const Array<double> &bt_,
+                                   const Array<double> &gt_,
                                   const Vector &d_,
                                   const Vector &x_,
                                   Vector &y_,
@@ -1359,27 +1267,28 @@ static void SmemPADiffusionApply3D(const int NE,
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int M1Q = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int M1D = T_D1D ? T_D1D : MAX_D1D;
-   MFEM_VERIFY(D1D <= M1D, "");
-   MFEM_VERIFY(Q1D <= M1Q, "");
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
   auto b = Reshape(b_.Read(), Q1D, D1D);
   auto g = Reshape(g_.Read(), Q1D, D1D);
-   auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, 6, NE);
+   auto d = Reshape(d_.Read(), Q1D*Q1D*Q1D, 6, NE);
   auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
   {
+      const int tidz = MFEM_THREAD_ID(z);
      const int D1D = T_D1D ? T_D1D : d1d;
      const int Q1D = T_Q1D ? T_Q1D : q1d;
      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
-      MFEM_SHARED double sBG[MQ1*MD1];
-      double (*B)[MD1] = (double (*)[MD1]) sBG;
-      double (*G)[MD1] = (double (*)[MD1]) sBG;
-      double (*Bt)[MQ1] = (double (*)[MQ1]) sBG;
-      double (*Gt)[MQ1] = (double (*)[MQ1]) sBG;
+      constexpr int MDQ = MQ1 > MD1 ? MQ1 : MD1;
+      MFEM_SHARED double sBG[2][MQ1*MD1];
+      double (*B)[MD1] = (double (*)[MD1]) (sBG+0);
+      double (*G)[MD1] = (double (*)[MD1]) (sBG+1);
+      double (*Bt)[MQ1] = (double (*)[MQ1]) (sBG+0);
+      double (*Gt)[MQ1] = (double (*)[MQ1]) (sBG+1);
      MFEM_SHARED double sm0[3][MDQ*MDQ*MDQ];
      MFEM_SHARED double sm1[3][MDQ*MDQ*MDQ];
      double (*X)[MD1][MD1]    = (double (*)[MD1][MD1]) (sm0+2);
@@ -1397,127 +1306,108 @@ static void SmemPADiffusionApply3D(const int NE,
      double (*QDD0)[MD1][MD1] = (double (*)[MD1][MD1]) (sm0+0);
      double (*QDD1)[MD1][MD1] = (double (*)[MD1][MD1]) (sm0+1);
      double (*QDD2)[MD1][MD1] = (double (*)[MD1][MD1]) (sm0+2);
-      MFEM_FOREACH_THREAD(dy,y,D1D)
+      MFEM_FOREACH_THREAD(dz,z,D1D)
      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
         {
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
+            MFEM_FOREACH_THREAD(dx,x,D1D)
            {
               X[dz][dy][dx] = x(dx,dy,dz,e);
            }
         }
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+      }
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D)
         {
-            const int i = qi(qx,dy,Q1D);
-            const int j = dj(qx,dy,D1D);
-            const int k = qk(qx,dy,Q1D);
-            const int l = dl(qx,dy,D1D);
-            B[i][j] = b(qx,dy);
-            G[k][l] = g(qx,dy) * sign(qx,dy);
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B[q][d] = b(q,d);
+               G[q][d] = g(q,d);
+            }
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
+      MFEM_FOREACH_THREAD(dz,z,D1D)
      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
         {
-            double u[D1D], v[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++) { u[dz] = v[dz] = 0.0; }
-            MFEM_UNROLL(MD1)
-            for (int dx = 0; dx < D1D; ++dx)
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
            {
-               const int i = qi(qx,dx,Q1D);
-               const int j = dj(qx,dx,D1D);
-               const int k = qk(qx,dx,Q1D);
-               const int l = dl(qx,dx,D1D);
-               const double s = sign(qx,dx);
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; ++dz)
+               double u = 0.0;
+               double v = 0.0;
+               for (int dx = 0; dx < D1D; ++dx)
               {
                  const double coords = X[dz][dy][dx];
-                  u[dz] += coords * B[i][j];
-                  v[dz] += coords * G[k][l] * s;
+                  u += coords * B[qx][dx];
+                  v += coords * G[qx][dx];
               }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               DDQ0[dz][dy][qx] = u[dz];
-               DDQ1[dz][dy][qx] = v[dz];
+               DDQ0[dz][dy][qx] = u;
+               DDQ1[dz][dy][qx] = v;
            }
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      MFEM_FOREACH_THREAD(dz,z,D1D)
      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
         {
-            double u[D1D], v[D1D], w[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++) { u[dz] = v[dz] = w[dz] = 0.0; }
-            MFEM_UNROLL(MD1)
-            for (int dy = 0; dy < D1D; ++dy)
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
            {
-               const int i = qi(qy,dy,Q1D);
-               const int j = dj(qy,dy,D1D);
-               const int k = qk(qy,dy,Q1D);
-               const int l = dl(qy,dy,D1D);
-               const double s = sign(qy,dy);
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; dz++)
+               double u = 0.0;
+               double v = 0.0;
+               double w = 0.0;
+               for (int dy = 0; dy < D1D; ++dy)
               {
-                  u[dz] += DDQ1[dz][dy][qx] * B[i][j];
-                  v[dz] += DDQ0[dz][dy][qx] * G[k][l] * s;
-                  w[dz] += DDQ0[dz][dy][qx] * B[i][j];
+                  u += DDQ1[dz][dy][qx] * B[qy][dy];
+                  v += DDQ0[dz][dy][qx] * G[qy][dy];
+                  w += DDQ0[dz][dy][qx] * B[qy][dy];
               }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++)
-            {
-               DQQ0[dz][qy][qx] = u[dz];
-               DQQ1[dz][qy][qx] = v[dz];
-               DQQ2[dz][qy][qx] = w[dz];
+               DQQ0[dz][qy][qx] = u;
+               DQQ1[dz][qy][qx] = v;
+               DQQ2[dz][qy][qx] = w;
            }
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
         {
-            double u[Q1D], v[Q1D], w[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; qz++) { u[qz] = v[qz] = w[qz] = 0.0; }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
            {
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; qz++)
+               double u = 0.0;
+               double v = 0.0;
+               double w = 0.0;
+               for (int dz = 0; dz < D1D; ++dz)
               {
-                  const int i = qi(qz,dz,Q1D);
-                  const int j = dj(qz,dz,D1D);
-                  const int k = qk(qz,dz,Q1D);
-                  const int l = dl(qz,dz,D1D);
-                  const double s = sign(qz,dz);
-                  u[qz] += DQQ0[dz][qy][qx] * B[i][j];
-                  v[qz] += DQQ1[dz][qy][qx] * B[i][j];
-                  w[qz] += DQQ2[dz][qy][qx] * G[k][l] * s;
+                  u += DQQ0[dz][qy][qx] * B[qz][dz];
+                  v += DQQ1[dz][qy][qx] * B[qz][dz];
+                  w += DQQ2[dz][qy][qx] * G[qz][dz];
               }
+               QQQ0[qz][qy][qx] = u;
+               QQQ1[qz][qy][qx] = v;
+               QQQ2[qz][qy][qx] = w;
            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; qz++)
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
            {
-               const double O11 = d(qx,qy,qz,0,e);
-               const double O12 = d(qx,qy,qz,1,e);
-               const double O13 = d(qx,qy,qz,2,e);
-               const double O22 = d(qx,qy,qz,3,e);
-               const double O23 = d(qx,qy,qz,4,e);
-               const double O33 = d(qx,qy,qz,5,e);
-               const double gX = u[qz];
-               const double gY = v[qz];
-               const double gZ = w[qz];
+               const int q = qx + ((qy*Q1D) + (qz*Q1D*Q1D));
+               const double O11 = d(q,0,e);
+               const double O12 = d(q,1,e);
+               const double O13 = d(q,2,e);
+               const double O22 = d(q,3,e);
+               const double O23 = d(q,4,e);
+               const double O33 = d(q,5,e);
+               const double gX = QQQ0[qz][qy][qx];
+               const double gY = QQQ1[qz][qy][qx];
+               const double gZ = QQQ2[qz][qy][qx];
               QQQ0[qz][qy][qx] = (O11*gX) + (O12*gY) + (O13*gZ);
               QQQ1[qz][qy][qx] = (O12*gX) + (O22*gY) + (O23*gZ);
               QQQ2[qz][qy][qx] = (O13*gX) + (O23*gY) + (O33*gZ);
@@ -1525,112 +1415,78 @@ static void SmemPADiffusionApply3D(const int NE,
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(d,y,D1D)
+      if (tidz == 0)
      {
-         MFEM_FOREACH_THREAD(q,x,Q1D)
+         MFEM_FOREACH_THREAD(d,y,D1D)
         {
-            const int i = qi(q,d,Q1D);
-            const int j = dj(q,d,D1D);
-            const int k = qk(q,d,Q1D);
-            const int l = dl(q,d,D1D);
-            Bt[j][i] = b(q,d);
-            Gt[l][k] = g(q,d) * sign(q,d);
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               Bt[d][q] = b(q,d);
+               Gt[d][q] = g(q,d);
+            }
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
         {
-            double u[Q1D], v[Q1D], w[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz) { u[qz] = v[qz] = w[qz] = 0.0; }
-            MFEM_UNROLL(MQ1)
-            for (int qx = 0; qx < Q1D; ++qx)
+            MFEM_FOREACH_THREAD(dx,x,D1D)
            {
-               const int i = qi(qx,dx,Q1D);
-               const int j = dj(qx,dx,D1D);
-               const int k = qk(qx,dx,Q1D);
-               const int l = dl(qx,dx,D1D);
-               const double s = sign(qx,dx);
-               MFEM_UNROLL(MQ1)
+               double u = 0.0;
+               double v = 0.0;
+               double w = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  u += QQQ0[qz][qy][qx] * Gt[dx][qx];
+                  v += QQQ1[qz][qy][qx] * Bt[dx][qx];
+                  w += QQQ2[qz][qy][qx] * Bt[dx][qx];
+               }
+               QQD0[qz][qy][dx] = u;
+               QQD1[qz][qy][dx] = v;
+               QQD2[qz][qy][dx] = w;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               double u = 0.0;
+               double v = 0.0;
+               double w = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  u += QQD0[qz][qy][dx] * Bt[dy][qy];
+                  v += QQD1[qz][qy][dx] * Gt[dy][qy];
+                  w += QQD2[qz][qy][dx] * Bt[dy][qy];
+               }
+               QDD0[qz][dy][dx] = u;
+               QDD1[qz][dy][dx] = v;
+               QDD2[qz][dy][dx] = w;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dz,z,D1D)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               double u = 0.0;
+               double v = 0.0;
+               double w = 0.0;
               for (int qz = 0; qz < Q1D; ++qz)
               {
-                  u[qz] += QQQ0[qz][qy][qx] * Gt[l][k] * s;
-                  v[qz] += QQQ1[qz][qy][qx] * Bt[j][i];
-                  w[qz] += QQQ2[qz][qy][qx] * Bt[j][i];
+                  u += QDD0[qz][dy][dx] * Bt[dz][qz];
+                  v += QDD1[qz][dy][dx] * Bt[dz][qz];
+                  w += QDD2[qz][dy][dx] * Gt[dz][qz];
               }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               QQD0[qz][qy][dx] = u[qz];
-               QQD1[qz][qy][dx] = v[qz];
-               QQD2[qz][qy][dx] = w[qz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[Q1D], v[Q1D], w[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz) { u[qz] = v[qz] = w[qz] = 0.0; }
-            MFEM_UNROLL(MQ1)
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const int i = qi(qy,dy,Q1D);
-               const int j = dj(qy,dy,D1D);
-               const int k = qk(qy,dy,Q1D);
-               const int l = dl(qy,dy,D1D);
-               const double s = sign(qy,dy);
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u[qz] += QQD0[qz][qy][dx] * Bt[j][i];
-                  v[qz] += QQD1[qz][qy][dx] * Gt[l][k] * s;
-                  w[qz] += QQD2[qz][qy][dx] * Bt[j][i];
-               }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               QDD0[qz][dy][dx] = u[qz];
-               QDD1[qz][dy][dx] = v[qz];
-               QDD2[qz][dy][dx] = w[qz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[D1D], v[D1D], w[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz) { u[dz] = v[dz] = w[dz] = 0.0; }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  const int i = qi(qz,dz,Q1D);
-                  const int j = dj(qz,dz,D1D);
-                  const int k = qk(qz,dz,Q1D);
-                  const int l = dl(qz,dz,D1D);
-                  const double s = sign(qz,dz);
-                  u[dz] += QDD0[qz][dy][dx] * Bt[j][i];
-                  v[dz] += QDD1[qz][dy][dx] * Bt[j][i];
-                  w[dz] += QDD2[qz][dy][dx] * Gt[l][k] * s;
-               }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               y(dx,dy,dz,e) += (u[dz] + v[dz] + w[dz]);
+               y(dx,dy,dz,e) += (u + v + w);
            }
         }
      }
@@ -1665,37 +1521,34 @@ static void PADiffusionApply(const int dim,
      MFEM_ABORT("OCCA PADiffusionApply unknown kernel!");
   }
 #endif // MFEM_USE_OCCA
-   const int ID = (D1D << 4 ) | Q1D;
-
   if (dim == 2)
   {
-      switch (ID)
+      switch ((D1D << 4 ) | Q1D)
      {
-         case 0x22: return SmemPADiffusionApply2D<2,2,16>(NE,B,G,D,X,Y);
-         case 0x33: return SmemPADiffusionApply2D<3,3,16>(NE,B,G,D,X,Y);
-         case 0x44: return SmemPADiffusionApply2D<4,4,8>(NE,B,G,D,X,Y);
-         case 0x55: return SmemPADiffusionApply2D<5,5,8>(NE,B,G,D,X,Y);
-         case 0x66: return SmemPADiffusionApply2D<6,6,4>(NE,B,G,D,X,Y);
-         case 0x77: return SmemPADiffusionApply2D<7,7,4>(NE,B,G,D,X,Y);
-         case 0x88: return SmemPADiffusionApply2D<8,8,2>(NE,B,G,D,X,Y);
-         case 0x99: return SmemPADiffusionApply2D<9,9,2>(NE,B,G,D,X,Y);
+         case 0x22: return SmemPADiffusionApply2D<2,2,16>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x33: return SmemPADiffusionApply2D<3,3,16>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x44: return SmemPADiffusionApply2D<4,4,8>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x55: return SmemPADiffusionApply2D<5,5,8>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x66: return SmemPADiffusionApply2D<6,6,4>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x77: return SmemPADiffusionApply2D<7,7,4>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x88: return SmemPADiffusionApply2D<8,8,2>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x99: return SmemPADiffusionApply2D<9,9,2>(NE,B,G,Bt,Gt,D,X,Y);
         default:   return PADiffusionApply2D(NE,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
      }
   }
-
-   if (dim == 3)
+   else if (dim == 3)
   {
-      switch (ID)
+      switch ((D1D << 4 ) | Q1D)
      {
-         case 0x23: return SmemPADiffusionApply3D<2,3>(NE,B,G,D,X,Y);
-         case 0x34: return SmemPADiffusionApply3D<3,4>(NE,B,G,D,X,Y);
-         case 0x45: return SmemPADiffusionApply3D<4,5>(NE,B,G,D,X,Y);
-         case 0x46: return SmemPADiffusionApply3D<4,6>(NE,B,G,D,X,Y);
-         case 0x56: return SmemPADiffusionApply3D<5,6>(NE,B,G,D,X,Y);
-         case 0x58: return SmemPADiffusionApply3D<5,8>(NE,B,G,D,X,Y);
-         case 0x67: return SmemPADiffusionApply3D<6,7>(NE,B,G,D,X,Y);
-         case 0x78: return SmemPADiffusionApply3D<7,8>(NE,B,G,D,X,Y);
-         case 0x89: return SmemPADiffusionApply3D<8,9>(NE,B,G,D,X,Y);
+         case 0x23: return SmemPADiffusionApply3D<2,3>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x34: return SmemPADiffusionApply3D<3,4>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x45: return SmemPADiffusionApply3D<4,5>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x46: return SmemPADiffusionApply3D<4,6>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x56: return SmemPADiffusionApply3D<5,6>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x58: return SmemPADiffusionApply3D<5,8>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x67: return SmemPADiffusionApply3D<6,7>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x78: return SmemPADiffusionApply3D<7,8>(NE,B,G,Bt,Gt,D,X,Y);
+         case 0x89: return SmemPADiffusionApply3D<8,9>(NE,B,G,Bt,Gt,D,X,Y);
         default:   return PADiffusionApply3D(NE,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
      }
   }
@@ -1,275 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-
-namespace mfem
-{
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void EADiffusionAssemble1D(const int NE,
-                                  const Array<double> &b,
-                                  const Array<double> &g,
-                                  const Vector &padata,
-                                  Vector &eadata,
-                                  const int d1d = 0,
-                                  const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto G = Reshape(g.Read(), Q1D, D1D);
-   auto D = Reshape(padata.Read(), Q1D, NE);
-   auto A = Reshape(eadata.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      double r_Gi[MQ1];
-      double r_Gj[MQ1];
-      for (int q = 0; q < Q1D; q++)
-      {
-         r_Gi[q] = G(q,MFEM_THREAD_ID(x));
-         r_Gj[q] = G(q,MFEM_THREAD_ID(y));
-      }
-      MFEM_FOREACH_THREAD(i1,x,D1D)
-      {
-         MFEM_FOREACH_THREAD(j1,y,D1D)
-         {
-            double val = 0.0;
-            for (int k1 = 0; k1 < Q1D; ++k1)
-            {
-               val += r_Gj[k1] * D(k1, e) * r_Gi[k1];
-            }
-            A(i1, j1, e) += val;
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void EADiffusionAssemble2D(const int NE,
-                                  const Array<double> &b,
-                                  const Array<double> &g,
-                                  const Vector &padata,
-                                  Vector &eadata,
-                                  const int d1d = 0,
-                                  const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto G = Reshape(g.Read(), Q1D, D1D);
-   auto D = Reshape(padata.Read(), Q1D, Q1D, 3, NE);
-   auto A = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      double r_B[MQ1][MD1];
-      double r_G[MQ1][MD1];
-      for (int d = 0; d < D1D; d++)
-      {
-         for (int q = 0; q < Q1D; q++)
-         {
-            r_B[q][d] = B(q,d);
-            r_G[q][d] = G(q,d);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(i1,x,D1D)
-      {
-         MFEM_FOREACH_THREAD(i2,y,D1D)
-         {
-            for (int j1 = 0; j1 < D1D; ++j1)
-            {
-               for (int j2 = 0; j2 < D1D; ++j2)
-               {
-                  double val = 0.0;
-                  for (int k1 = 0; k1 < Q1D; ++k1)
-                  {
-                     for (int k2 = 0; k2 < Q1D; ++k2)
-                     {
-                        double bgi = r_G[k1][i1] * r_B[k2][i2];
-                        double gbi = r_B[k1][i1] * r_G[k2][i2];
-                        double bgj = r_G[k1][j1] * r_B[k2][j2];
-                        double gbj = r_B[k1][j1] * r_G[k2][j2];
-                        double D00 = D(k1,k2,0,e);
-                        double D10 = D(k1,k2,1,e);
-                        double D01 = D10;
-                        double D11 = D(k1,k2,2,e);
-                        val += bgi * D00 * bgj
-                               + gbi * D01 * bgj
-                               + bgi * D10 * gbj
-                               + gbi * D11 * gbj;
-                     }
-                  }
-                  A(i1, i2, j1, j2, e) += val;
-               }
-            }
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void EADiffusionAssemble3D(const int NE,
-                                  const Array<double> &g,
-                                  const Array<double> &b,
-                                  const Vector &padata,
-                                  Vector &eadata,
-                                  const int d1d = 0,
-                                  const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto G = Reshape(g.Read(), Q1D, D1D);
-   auto D = Reshape(padata.Read(), Q1D, Q1D, Q1D, 6, NE);
-   auto A = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, D1D, D1D, D1D,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      double r_B[MQ1][MD1];
-      double r_G[MQ1][MD1];
-      for (int d = 0; d < D1D; d++)
-      {
-         for (int q = 0; q < Q1D; q++)
-         {
-            r_B[q][d] = B(q,d);
-            r_G[q][d] = G(q,d);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(i1,x,D1D)
-      {
-         MFEM_FOREACH_THREAD(i2,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(i3,z,D1D)
-            {
-               for (int j1 = 0; j1 < D1D; ++j1)
-               {
-                  for (int j2 = 0; j2 < D1D; ++j2)
-                  {
-                     for (int j3 = 0; j3 < D1D; ++j3)
-                     {
-                        double val = 0.0;
-                        for (int k1 = 0; k1 < Q1D; ++k1)
-                        {
-                           for (int k2 = 0; k2 < Q1D; ++k2)
-                           {
-                              for (int k3 = 0; k3 < Q1D; ++k3)
-                              {
-                                 double bbgi = r_G[k1][i1] * r_B[k2][i2] * r_B[k3][i3];
-                                 double bgbi = r_B[k1][i1] * r_G[k2][i2] * r_B[k3][i3];
-                                 double gbbi = r_B[k1][i1] * r_B[k2][i2] * r_G[k3][i3];
-                                 double bbgj = r_G[k1][j1] * r_B[k2][j2] * r_B[k3][j3];
-                                 double bgbj = r_B[k1][j1] * r_G[k2][j2] * r_B[k3][j3];
-                                 double gbbj = r_B[k1][j1] * r_B[k2][j2] * r_G[k3][j3];
-                                 double D00 = D(k1,k2,k3,0,e);
-                                 double D10 = D(k1,k2,k3,1,e);
-                                 double D20 = D(k1,k2,k3,2,e);
-                                 double D01 = D10;
-                                 double D11 = D(k1,k2,k3,3,e);
-                                 double D21 = D(k1,k2,k3,4,e);
-                                 double D02 = D20;
-                                 double D12 = D21;
-                                 double D22 = D(k1,k2,k3,5,e);
-                                 val += bbgi * D00 * bbgj
-                                        + bgbi * D10 * bbgj
-                                        + gbbi * D20 * bbgj
-                                        + bbgi * D01 * bgbj
-                                        + bgbi * D11 * bgbj
-                                        + gbbi * D21 * bgbj
-                                        + bbgi * D02 * gbbj
-                                        + bgbi * D12 * gbbj
-                                        + gbbi * D22 * gbbj;
-                              }
-                           }
-                        }
-                        A(i1, i2, i3, j1, j2, j3, e) += val;
-                     }
-                  }
-               }
-            }
-         }
-      }
-   });
-}
-
-void DiffusionIntegrator::AssembleEA(const FiniteElementSpace &fes,
-                                     Vector &ea_data)
-{
-   AssemblePA(fes);
-   const int ne = fes.GetMesh()->GetNE();
-   const Array<double> &B = maps->B;
-   const Array<double> &G = maps->G;
-   if (dim == 1)
-   {
-      switch ((dofs1D << 4 ) | quad1D)
-      {
-         case 0x22: return EADiffusionAssemble1D<2,2>(ne,B,G,pa_data,ea_data);
-         case 0x33: return EADiffusionAssemble1D<3,3>(ne,B,G,pa_data,ea_data);
-         case 0x44: return EADiffusionAssemble1D<4,4>(ne,B,G,pa_data,ea_data);
-         case 0x55: return EADiffusionAssemble1D<5,5>(ne,B,G,pa_data,ea_data);
-         case 0x66: return EADiffusionAssemble1D<6,6>(ne,B,G,pa_data,ea_data);
-         case 0x77: return EADiffusionAssemble1D<7,7>(ne,B,G,pa_data,ea_data);
-         case 0x88: return EADiffusionAssemble1D<8,8>(ne,B,G,pa_data,ea_data);
-         case 0x99: return EADiffusionAssemble1D<9,9>(ne,B,G,pa_data,ea_data);
-         default:   return EADiffusionAssemble1D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
-      }
-   }
-   else if (dim == 2)
-   {
-      switch ((dofs1D << 4 ) | quad1D)
-      {
-         case 0x22: return EADiffusionAssemble2D<2,2>(ne,B,G,pa_data,ea_data);
-         case 0x33: return EADiffusionAssemble2D<3,3>(ne,B,G,pa_data,ea_data);
-         case 0x44: return EADiffusionAssemble2D<4,4>(ne,B,G,pa_data,ea_data);
-         case 0x55: return EADiffusionAssemble2D<5,5>(ne,B,G,pa_data,ea_data);
-         case 0x66: return EADiffusionAssemble2D<6,6>(ne,B,G,pa_data,ea_data);
-         case 0x77: return EADiffusionAssemble2D<7,7>(ne,B,G,pa_data,ea_data);
-         case 0x88: return EADiffusionAssemble2D<8,8>(ne,B,G,pa_data,ea_data);
-         case 0x99: return EADiffusionAssemble2D<9,9>(ne,B,G,pa_data,ea_data);
-         default:   return EADiffusionAssemble2D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
-      }
-   }
-   else if (dim == 3)
-   {
-      switch ((dofs1D << 4 ) | quad1D)
-      {
-         case 0x23: return EADiffusionAssemble3D<2,3>(ne,B,G,pa_data,ea_data);
-         case 0x34: return EADiffusionAssemble3D<3,4>(ne,B,G,pa_data,ea_data);
-         case 0x45: return EADiffusionAssemble3D<4,5>(ne,B,G,pa_data,ea_data);
-         case 0x56: return EADiffusionAssemble3D<5,6>(ne,B,G,pa_data,ea_data);
-         case 0x67: return EADiffusionAssemble3D<6,7>(ne,B,G,pa_data,ea_data);
-         case 0x78: return EADiffusionAssemble3D<7,8>(ne,B,G,pa_data,ea_data);
-         case 0x89: return EADiffusionAssemble3D<8,9>(ne,B,G,pa_data,ea_data);
-         default:   return EADiffusionAssemble3D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
-      }
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-
-}
@@ -41,8 +41,6 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes, const bool force)
      InitCeedCoeff(Q, ptr);
      return CeedPAMassAssemble(fes, *ir, *ptr);
   }
-#else
-   MFEM_CONTRACT_VAR(force);
 #endif
   dim = mesh->Dimension();
   ne = fes.GetMesh()->GetNE();
@@ -64,19 +62,6 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes, const bool force)
      coeff.SetSize(1);
      coeff(0) = cQ->constant;
   }
-   else if (QuadratureFunctionCoefficient* cQ =
-               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
-   {
-      const QuadratureFunction &qFun = cQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == nq * ne,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
   else
   {
      coeff.SetSize(nq * ne);
@@ -654,7 +639,6 @@ static void SmemPAMassApply2D(const int NE,
                              const int d1d = 0,
                              const int q1d = 0)
 {
-   MFEM_CONTRACT_VAR(bt_);
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
@@ -918,7 +902,6 @@ static void SmemPAMassApply3D(const int NE,
                              const int d1d = 0,
                              const int q1d = 0)
 {
-   MFEM_CONTRACT_VAR(bt_);
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
   constexpr int M1Q = T_Q1D ? T_Q1D : MAX_Q1D;
@@ -1,255 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-
-namespace mfem
-{
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void EAMassAssemble1D(const int NE,
-                             const Array<double> &basis,
-                             const Vector &padata,
-                             Vector &eadata,
-                             const int d1d = 0,
-                             const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(basis.Read(), Q1D, D1D);
-   auto D = Reshape(padata.Read(), Q1D, NE);
-   auto M = Reshape(eadata.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      double r_Bi[MQ1];
-      double r_Bj[MQ1];
-      for (int q = 0; q < Q1D; q++)
-      {
-         r_Bi[q] = B(q,MFEM_THREAD_ID(x));
-         r_Bj[q] = B(q,MFEM_THREAD_ID(y));
-      }
-      MFEM_FOREACH_THREAD(i1,x,D1D)
-      {
-         MFEM_FOREACH_THREAD(j1,y,D1D)
-         {
-            double val = 0.0;
-            for (int k1 = 0; k1 < Q1D; ++k1)
-            {
-               val += r_Bi[k1] * r_Bj[k1] * D(k1, e);
-            }
-            M(i1, j1, e) += val;
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void EAMassAssemble2D(const int NE,
-                             const Array<double> &basis,
-                             const Vector &padata,
-                             Vector &eadata,
-                             const int d1d = 0,
-                             const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(basis.Read(), Q1D, D1D);
-   auto D = Reshape(padata.Read(), Q1D, Q1D, NE);
-   auto M = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      double r_B[MQ1][MD1];
-      for (int d = 0; d < D1D; d++)
-      {
-         for (int q = 0; q < Q1D; q++)
-         {
-            r_B[q][d] = B(q,d);
-         }
-      }
-      MFEM_SHARED double s_D[MQ1][MQ1];
-      MFEM_FOREACH_THREAD(k1,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(k2,y,Q1D)
-         {
-            s_D[k1][k2] = D(k1,k2,e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(i1,x,D1D)
-      {
-         MFEM_FOREACH_THREAD(i2,y,D1D)
-         {
-            for (int j1 = 0; j1 < D1D; ++j1)
-            {
-               for (int j2 = 0; j2 < D1D; ++j2)
-               {
-                  double val = 0.0;
-                  for (int k1 = 0; k1 < Q1D; ++k1)
-                  {
-                     for (int k2 = 0; k2 < Q1D; ++k2)
-                     {
-                        val += r_B[k1][i1] * r_B[k1][j1]
-                               * r_B[k2][i2] * r_B[k2][j2]
-                               * s_D[k1][k2];
-                     }
-                  }
-                  M(i1, i2, j1, j2, e) += val;
-               }
-            }
-         }
-      }
-   });
-}
-
-template<int T_D1D = 0, int T_Q1D = 0>
-static void EAMassAssemble3D(const int NE,
-                             const Array<double> &basis,
-                             const Vector &padata,
-                             Vector &eadata,
-                             const int d1d = 0,
-                             const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(basis.Read(), Q1D, D1D);
-   auto D = Reshape(padata.Read(), Q1D, Q1D, Q1D, NE);
-   auto M = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, D1D, D1D, D1D,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      double r_B[MQ1][MD1];
-      for (int d = 0; d < D1D; d++)
-      {
-         for (int q = 0; q < Q1D; q++)
-         {
-            r_B[q][d] = B(q,d);
-         }
-      }
-      MFEM_SHARED double s_D[MQ1][MQ1][MQ1];
-      MFEM_FOREACH_THREAD(k1,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(k2,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(k3,z,Q1D)
-            {
-               s_D[k1][k2][k3] = D(k1,k2,k3,e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(i1,x,D1D)
-      {
-         MFEM_FOREACH_THREAD(i2,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(i3,z,D1D)
-            {
-               for (int j1 = 0; j1 < D1D; ++j1)
-               {
-                  for (int j2 = 0; j2 < D1D; ++j2)
-                  {
-                     for (int j3 = 0; j3 < D1D; ++j3)
-                     {
-                        double val = 0.0;
-                        for (int k1 = 0; k1 < Q1D; ++k1)
-                        {
-                           for (int k2 = 0; k2 < Q1D; ++k2)
-                           {
-                              for (int k3 = 0; k3 < Q1D; ++k3)
-                              {
-                                 val += r_B[k1][i1] * r_B[k1][j1]
-                                        * r_B[k2][i2] * r_B[k2][j2]
-                                        * r_B[k3][i3] * r_B[k3][j3]
-                                        * s_D[k1][k2][k3];
-                              }
-                           }
-                        }
-                        M(i1, i2, i3, j1, j2, j3, e) += val;
-                     }
-                  }
-               }
-            }
-         }
-      }
-   });
-}
-
-void MassIntegrator::AssembleEA(const FiniteElementSpace &fes,
-                                Vector &ea_data)
-{
-   AssemblePA(fes);
-   const int ne = fes.GetMesh()->GetNE();
-   const Array<double> &B = maps->B;
-   if (dim == 1)
-   {
-      switch ((dofs1D << 4 ) | quad1D)
-      {
-         case 0x22: return EAMassAssemble1D<2,2>(ne,B,pa_data,ea_data);
-         case 0x33: return EAMassAssemble1D<3,3>(ne,B,pa_data,ea_data);
-         case 0x44: return EAMassAssemble1D<4,4>(ne,B,pa_data,ea_data);
-         case 0x55: return EAMassAssemble1D<5,5>(ne,B,pa_data,ea_data);
-         case 0x66: return EAMassAssemble1D<6,6>(ne,B,pa_data,ea_data);
-         case 0x77: return EAMassAssemble1D<7,7>(ne,B,pa_data,ea_data);
-         case 0x88: return EAMassAssemble1D<8,8>(ne,B,pa_data,ea_data);
-         case 0x99: return EAMassAssemble1D<9,9>(ne,B,pa_data,ea_data);
-         default:   return EAMassAssemble1D(ne,B,pa_data,ea_data,dofs1D,quad1D);
-      }
-   }
-   else if (dim == 2)
-   {
-      switch ((dofs1D << 4 ) | quad1D)
-      {
-         case 0x22: return EAMassAssemble2D<2,2>(ne,B,pa_data,ea_data);
-         case 0x33: return EAMassAssemble2D<3,3>(ne,B,pa_data,ea_data);
-         case 0x44: return EAMassAssemble2D<4,4>(ne,B,pa_data,ea_data);
-         case 0x55: return EAMassAssemble2D<5,5>(ne,B,pa_data,ea_data);
-         case 0x66: return EAMassAssemble2D<6,6>(ne,B,pa_data,ea_data);
-         case 0x77: return EAMassAssemble2D<7,7>(ne,B,pa_data,ea_data);
-         case 0x88: return EAMassAssemble2D<8,8>(ne,B,pa_data,ea_data);
-         case 0x99: return EAMassAssemble2D<9,9>(ne,B,pa_data,ea_data);
-         default:   return EAMassAssemble2D(ne,B,pa_data,ea_data,dofs1D,quad1D);
-      }
-   }
-   else if (dim == 3)
-   {
-      switch ((dofs1D << 4 ) | quad1D)
-      {
-         case 0x23: return EAMassAssemble3D<2,3>(ne,B,pa_data,ea_data);
-         case 0x34: return EAMassAssemble3D<3,4>(ne,B,pa_data,ea_data);
-         case 0x45: return EAMassAssemble3D<4,5>(ne,B,pa_data,ea_data);
-         case 0x56: return EAMassAssemble3D<5,6>(ne,B,pa_data,ea_data);
-         case 0x67: return EAMassAssemble3D<6,7>(ne,B,pa_data,ea_data);
-         case 0x78: return EAMassAssemble3D<7,8>(ne,B,pa_data,ea_data);
-         case 0x89: return EAMassAssemble3D<8,9>(ne,B,pa_data,ea_data);
-         default:   return EAMassAssemble3D(ne,B,pa_data,ea_data,dofs1D,quad1D);
-      }
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-
-}
@@ -1,103 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-
-namespace mfem
-{
-
-void TransposeIntegrator::AssembleEA(const FiniteElementSpace &fes,
-                                     Vector &ea_data)
-{
-   Vector ea_data_tmp(ea_data.Size());
-   ea_data_tmp = 0.0;
-   bfi->AssembleEA(fes, ea_data_tmp);
-   const int ne = fes.GetNE();
-   if (ne == 0) { return; }
-   const int dofs = fes.GetFE(0)->GetDof();
-   auto A = Reshape(ea_data_tmp.Write(), dofs, dofs, ne);
-   auto AT = Reshape(ea_data.ReadWrite(), dofs, dofs, ne);
-   MFEM_FORALL(e, ne,
-   {
-      for (int i = 0; i < dofs; i++)
-      {
-         for (int j = 0; j < dofs; j++)
-         {
-            const double a = A(i, j, e);
-            AT(j, i, e) += a;
-         }
-      }
-   });
-}
-
-void TransposeIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
-                                                  Vector &ea_data_int,
-                                                  Vector &ea_data_ext)
-{
-   const int nf = fes.GetNFbyType(FaceType::Interior);
-   if (nf == 0) { return; }
-   Vector ea_data_int_tmp(ea_data_int.Size());
-   Vector ea_data_ext_tmp(ea_data_ext.Size());
-   ea_data_int_tmp = 0.0;
-   ea_data_ext_tmp = 0.0;
-   bfi->AssembleEAInteriorFaces(fes, ea_data_int_tmp, ea_data_ext_tmp);
-   const int faceDofs = fes.GetTraceElement(0,
-                                            fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
-   auto A_int = Reshape(ea_data_int_tmp.Read(), faceDofs, faceDofs, 2, nf);
-   auto A_ext = Reshape(ea_data_ext_tmp.Read(), faceDofs, faceDofs, 2, nf);
-   auto AT_int = Reshape(ea_data_int.ReadWrite(), faceDofs, faceDofs, 2, nf);
-   auto AT_ext = Reshape(ea_data_ext.ReadWrite(), faceDofs, faceDofs, 2, nf);
-   MFEM_FORALL(f, nf,
-   {
-      for (int i = 0; i < faceDofs; i++)
-      {
-         for (int j = 0; j < faceDofs; j++)
-         {
-            const double a_int0 = A_int(i, j, 0, f);
-            const double a_int1 = A_int(i, j, 1, f);
-            const double a_ext0 = A_ext(i, j, 0, f);
-            const double a_ext1 = A_ext(i, j, 1, f);
-            AT_int(j, i, 0, f) += a_int0;
-            AT_int(j, i, 1, f) += a_int1;
-            AT_ext(j, i, 0, f) += a_ext1;
-            AT_ext(j, i, 1, f) += a_ext0;
-         }
-      }
-   });
-}
-
-void TransposeIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace& fes,
-                                                  Vector &ea_data_bdr)
-{
-   const int nf = fes.GetNFbyType(FaceType::Boundary);
-   if (nf == 0) { return; }
-   Vector ea_data_bdr_tmp(ea_data_bdr.Size());
-   ea_data_bdr_tmp = 0.0;
-   bfi->AssembleEABoundaryFaces(fes, ea_data_bdr_tmp);
-   const int faceDofs = fes.GetTraceElement(0,
-                                            fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
-   auto A_bdr = Reshape(ea_data_bdr_tmp.Read(), faceDofs, faceDofs, nf);
-   auto AT_bdr = Reshape(ea_data_bdr.ReadWrite(), faceDofs, faceDofs, nf);
-   MFEM_FORALL(f, nf,
-   {
-      for (int i = 0; i < faceDofs; i++)
-      {
-         for (int j = 0; j < faceDofs; j++)
-         {
-            const double a_bdr = A_bdr(i, j, f);
-            AT_bdr(j, i, f) += a_bdr;
-         }
-      }
-   });
-}
-
-}
@@ -101,6 +101,7 @@ static void PAVectorDiffusionSetup3D(const int Q1D,
 }

 static void PAVectorDiffusionSetup(const int dim,
+                                   const int D1D,
                                   const int Q1D,
                                   const int NE,
                                   const Array<double> &W,
@@ -133,7 +134,6 @@ void VectorDiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
   const int nq = ir->GetNPoints();
   dim = mesh->Dimension();
-   sdim = mesh->SpaceDimension();
   ne = fes.GetNE();
   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
@@ -147,83 +147,46 @@ void VectorDiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
      MFEM_VERIFY(cQ != NULL, "only ConstantCoefficient is supported!");
      coeff = cQ->constant;
   }
-   const Array<double> &w = ir->GetWeights();
-   const Vector &j = geom->J;
-   Vector &d = pa_data;
-   if (dim == 1) { MFEM_ABORT("dim==1 not supported in PAVectorDiffusionSetup"); }
-   if (dim == 2 && sdim == 3)
-   {
-      constexpr int DIM = 2;
-      constexpr int SDIM = 3;
-      const int NQ = quad1D*quad1D;
-      auto W = w.Read();
-      auto J = Reshape(j.Read(), NQ, SDIM, DIM, ne);
-      auto D = Reshape(d.Write(), NQ, SDIM, ne);
-      MFEM_FORALL(e, ne,
-      {
-         for (int q = 0; q < NQ; ++q)
-         {
-            const double wq = W[q];
-            const double J11 = J(q,0,0,e);
-            const double J21 = J(q,1,0,e);
-            const double J31 = J(q,2,0,e);
-            const double J12 = J(q,0,1,e);
-            const double J22 = J(q,1,1,e);
-            const double J32 = J(q,2,1,e);
-            const double E = J11*J11 + J21*J21 + J31*J31;
-            const double G = J12*J12 + J22*J22 + J32*J32;
-            const double F = J11*J12 + J21*J22 + J31*J32;
-            const double iw = 1.0 / sqrt(E*G - F*F);
-            const double alpha = wq * coeff * iw;
-            D(q,0,e) =  alpha * G; // 1,1
-            D(q,1,e) = -alpha * F; // 1,2
-            D(q,2,e) =  alpha * E; // 2,2
-         }
-      });
-   }
-   else
-   {
-      PAVectorDiffusionSetup(dim, quad1D, ne, w, j, coeff, d);
-   }
+   PAVectorDiffusionSetup(dim, dofs1D, quad1D, ne, ir->GetWeights(), geom->J,
+                          coeff, pa_data);
 }

 // PA Diffusion Apply 2D kernel
-template<int T_D1D = 0, int T_Q1D = 0, int T_VDIM = 0> static
+template<int T_D1D = 0, int T_Q1D = 0> static
 void PAVectorDiffusionApply2D(const int NE,
                              const Array<double> &b,
                              const Array<double> &g,
                              const Array<double> &bt,
                              const Array<double> &gt,
-                              const Vector &d_,
-                              const Vector &x_,
-                              Vector &y_,
+                              const Vector &_op,
+                              const Vector &_x,
+                              Vector &_y,
                              const int d1d = 0,
-                              const int q1d = 0,
-                              const int vdim = 0)
+                              const int q1d = 0)
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
+   constexpr int VDIM = 2;
   MFEM_VERIFY(D1D <= MAX_D1D, "");
   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
   auto B = Reshape(b.Read(), Q1D, D1D);
   auto G = Reshape(g.Read(), Q1D, D1D);
   auto Bt = Reshape(bt.Read(), D1D, Q1D);
   auto Gt = Reshape(gt.Read(), D1D, Q1D);
-   auto D = Reshape(d_.Read(), Q1D*Q1D, 3, NE);
-   auto x = Reshape(x_.Read(), D1D, D1D, VDIM, NE);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, VDIM, NE);
+   auto op = Reshape(_op.Read(), Q1D*Q1D, 3, NE);
+   auto x = Reshape(_x.Read(), D1D, D1D, VDIM, NE);
+   auto y = Reshape(_y.ReadWrite(), D1D, D1D, VDIM, NE);
   MFEM_FORALL(e, NE,
   {
      const int D1D = T_D1D ? T_D1D : d1d;
      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      // the following variables are evaluated at compile time
      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;

-      double grad[max_Q1D][max_Q1D][2];
-      for (int c = 0; c < VDIM; c++)
+      for (int c = 0; c < VDIM; ++ c)
      {
+         double grad[max_Q1D][max_Q1D][2];
         for (int qy = 0; qy < Q1D; ++qy)
         {
            for (int qx = 0; qx < Q1D; ++qx)
@@ -266,11 +229,14 @@ void PAVectorDiffusionApply2D(const int NE,
            for (int qx = 0; qx < Q1D; ++qx)
            {
               const int q = qx + qy * Q1D;
-               const double O11 = D(q,0,e);
-               const double O12 = D(q,1,e);
-               const double O22 = D(q,2,e);
+
+               const double O11 = op(q,0,e);
+               const double O12 = op(q,1,e);
+               const double O22 = op(q,2,e);
+
               const double gradX = grad[qy][qx][0];
               const double gradY = grad[qy][qx][1];
+
               grad[qy][qx][0] = (O11 * gradX) + (O12 * gradY);
               grad[qy][qx][1] = (O12 * gradX) + (O22 * gradY);
            }
@@ -280,8 +246,8 @@ void PAVectorDiffusionApply2D(const int NE,
            double gradX[max_D1D][2];
            for (int dx = 0; dx < D1D; ++dx)
            {
-               gradX[dx][0] = 0.0;
-               gradX[dx][1] = 0.0;
+               gradX[dx][0] = 0;
+               gradX[dx][1] = 0;
            }
            for (int qx = 0; qx < Q1D; ++qx)
            {
@@ -503,36 +469,35 @@ void PAVectorDiffusionApply3D(const int NE,
   });
 }

+static void PAVectorDiffusionApply(const int dim,
+                                   const int D1D,
+                                   const int Q1D,
+                                   const int NE,
+                                   const Array<double> &B,
+                                   const Array<double> &G,
+                                   const Array<double> &Bt,
+                                   const Array<double> &Gt,
+                                   const Vector &op,
+                                   const Vector &x,
+                                   Vector &y)
+{
+   if (dim == 2)
+   {
+      return PAVectorDiffusionApply2D(NE,B,G,Bt,Gt,op,x,y,D1D,Q1D);
+   }
+   if (dim == 3)
+   {
+      return PAVectorDiffusionApply3D(NE,B,G,Bt,Gt,op,x,y,D1D,Q1D);
+   }
+   MFEM_ABORT("Unknown kernel.");
+}
+
 // PA Diffusion Apply kernel
 void VectorDiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
-   const int D1D = dofs1D;
-   const int Q1D = quad1D;
-   const Array<double> &B = maps->B;
-   const Array<double> &G = maps->G;
-   const Array<double> &Bt = maps->Bt;
-   const Array<double> &Gt = maps->Gt;
-   const Vector &D = pa_data;
-
-   if (dim == 2 && sdim == 3)
-   {
-      switch ((dofs1D << 4 ) | quad1D)
-      {
-         case 0x22: return PAVectorDiffusionApply2D<2,2,3>(ne,B,G,Bt,Gt,D,x,y);
-         case 0x33: return PAVectorDiffusionApply2D<3,3,3>(ne,B,G,Bt,Gt,D,x,y);
-         case 0x44: return PAVectorDiffusionApply2D<4,4,3>(ne,B,G,Bt,Gt,D,x,y);
-         case 0x55: return PAVectorDiffusionApply2D<5,5,3>(ne,B,G,Bt,Gt,D,x,y);
-         default:
-            return PAVectorDiffusionApply2D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D,sdim);
-      }
-   }
-   if (dim == 2 && sdim == 2)
-   { return PAVectorDiffusionApply2D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D,sdim); }
-
-   if (dim == 3 && sdim == 3)
-   { return PAVectorDiffusionApply3D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D); }
-
-   MFEM_ABORT("Unknown kernel.");
+   PAVectorDiffusionApply(dim, dofs1D, quad1D, ne,
+                          maps->B, maps->G, maps->Bt, maps->Gt,
+                          pa_data, x, y);
 }

 template<int T_D1D = 0, int T_Q1D = 0>
@@ -49,7 +49,7 @@ double FunctionCoefficient::Eval(ElementTransformation & T,
 double GridFunctionCoefficient::Eval (ElementTransformation &T,
                                      const IntegrationPoint &ip)
 {
-   return GridF -> GetValue (T, ip, Component);
+   return GridF -> GetValue (T.ElementNo, ip, Component);
 }

 double TransformedCoefficient::Eval(ElementTransformation &T,
@@ -160,13 +160,13 @@ void VectorArrayCoefficient::Eval(Vector &V, ElementTransformation &T,
 }

 VectorGridFunctionCoefficient::VectorGridFunctionCoefficient (
-   const GridFunction *gf)
+   GridFunction *gf)
   : VectorCoefficient ((gf) ? gf -> VectorDim() : 0)
 {
   GridFunc = gf;
 }

-void VectorGridFunctionCoefficient::SetGridFunction(const GridFunction *gf)
+void VectorGridFunctionCoefficient::SetGridFunction(GridFunction *gf)
 {
   GridFunc = gf; vdim = (gf) ? gf -> VectorDim() : 0;
 }
@@ -174,7 +174,7 @@ void VectorGridFunctionCoefficient::SetGridFunction(const GridFunction *gf)
 void VectorGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
                                         const IntegrationPoint &ip)
 {
-   GridFunc->GetVectorValue(T, ip, V);
+   GridFunc->GetVectorValue(T.ElementNo, ip, V);
 }

 void VectorGridFunctionCoefficient::Eval(
@@ -184,14 +184,14 @@ void VectorGridFunctionCoefficient::Eval(
 }

 GradientGridFunctionCoefficient::GradientGridFunctionCoefficient (
-   const GridFunction *gf)
+   GridFunction *gf)
   : VectorCoefficient((gf) ?
                       gf -> FESpace() -> GetMesh() -> SpaceDimension() : 0)
 {
   GridFunc = gf;
 }

-void GradientGridFunctionCoefficient::SetGridFunction(const GridFunction *gf)
+void GradientGridFunctionCoefficient::SetGridFunction(GridFunction *gf)
 {
   GridFunc = gf; vdim = (gf) ?
                         gf -> FESpace() -> GetMesh() -> SpaceDimension() : 0;
@@ -209,24 +209,18 @@ void GradientGridFunctionCoefficient::Eval(
   GridFunc->GetGradients(T, ir, M);
 }

-CurlGridFunctionCoefficient::CurlGridFunctionCoefficient(
-   const GridFunction *gf)
-   : VectorCoefficient(0)
+CurlGridFunctionCoefficient::CurlGridFunctionCoefficient (
+   GridFunction *gf)
+   : VectorCoefficient ((gf) ?
+                        gf -> FESpace() -> GetMesh() -> SpaceDimension() : 0)
 {
-   SetGridFunction(gf);
+   GridFunc = gf;
 }

-void CurlGridFunctionCoefficient::SetGridFunction(const GridFunction *gf)
+void CurlGridFunctionCoefficient::SetGridFunction(GridFunction *gf)
 {
-   if (gf)
-   {
-      int sdim = gf -> FESpace() -> GetMesh() -> SpaceDimension();
-      MFEM_VERIFY(sdim == 2 || sdim == 3,
-                  "CurlGridFunctionCoefficient "
-                  "only defind for spaces of dimension 2 or 3.");
-   }
-   GridFunc = gf;
-   vdim = (gf) ? (2 * gf -> FESpace() -> GetMesh() -> SpaceDimension() - 3) : 0;
+   GridFunc = gf; vdim = (gf) ?
+                         gf -> FESpace() -> GetMesh() -> SpaceDimension() : 0;
 }

 void CurlGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
@@ -236,7 +230,7 @@ void CurlGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
 }

 DivergenceGridFunctionCoefficient::DivergenceGridFunctionCoefficient (
-   const GridFunction *gf) : Coefficient()
+   GridFunction *gf) : Coefficient()
 {
   GridFunc = gf;
 }
@@ -319,31 +313,6 @@ void MatrixFunctionCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
   }
 }

-void MatrixFunctionCoefficient::EvalSymmetric(Vector &K,
-                                              ElementTransformation &T,
-                                              const IntegrationPoint &ip)
-{
-   MFEM_VERIFY(symmetric && height == width && height < 4 && SymmFunction,
-               "MatrixFunctionCoefficient is not symmetric");
-
-   double x[3];
-   Vector transip(x, 3);
-
-   T.Transform(ip, transip);
-
-   K.SetSize((width * (width + 1)) / 2); // 1x1: 1, 2x2: 3, 3x3: 6
-
-   if (SymmFunction)
-   {
-      (*SymmFunction)(transip, K);
-   }
-
-   if (Q)
-   {
-      K *= Q->Eval(T, ip, GetTime());
-   }
-}
-
 MatrixArrayCoefficient::MatrixArrayCoefficient (int dim)
   : MatrixCoefficient (dim)
 {
@@ -447,43 +416,13 @@ double DeterminantCoefficient::Eval(ElementTransformation &T,
   return ma.Det();
 }

-VectorSumCoefficient::VectorSumCoefficient(int dim)
-   : VectorCoefficient(dim),
-     ACoef(NULL), BCoef(NULL),
-     A(dim), B(dim),
-     alphaCoef(NULL), betaCoef(NULL),
-     alpha(1.0), beta(1.0)
-{
-   A = 0.0; B = 0.0;
-}
-
-VectorSumCoefficient::VectorSumCoefficient(VectorCoefficient &_A,
-                                           VectorCoefficient &_B,
+VectorSumCoefficient::VectorSumCoefficient(VectorCoefficient &A,
+                                           VectorCoefficient &B,
                                           double _alpha, double _beta)
-   : VectorCoefficient(_A.GetVDim()),
-     ACoef(&_A), BCoef(&_B),
-     A(_A.GetVDim()), B(_A.GetVDim()),
-     alphaCoef(NULL), betaCoef(NULL),
-     alpha(_alpha), beta(_beta)
+   : VectorCoefficient(A.GetVDim()), a(&A), b(&B), alpha(_alpha), beta(_beta),
+     va(A.GetVDim())
 {
-   MFEM_ASSERT(_A.GetVDim() == _B.GetVDim(),
-               "VectorSumCoefficient:  "
-               "Arguments must have the same dimension.");
-}
-
-VectorSumCoefficient::VectorSumCoefficient(VectorCoefficient &_A,
-                                           VectorCoefficient &_B,
-                                           Coefficient &_alpha,
-                                           Coefficient &_beta)
-   : VectorCoefficient(_A.GetVDim()),
-     ACoef(&_A), BCoef(&_B),
-     A(_A.GetVDim()),
-     B(_A.GetVDim()),
-     alphaCoef(&_alpha),
-     betaCoef(&_beta),
-     alpha(0.0), beta(0.0)
-{
-   MFEM_ASSERT(_A.GetVDim() == _B.GetVDim(),
+   MFEM_ASSERT(A.GetVDim() == B.GetVDim(),
               "VectorSumCoefficient:  "
               "Arguments must have the same dimension.");
 }
@@ -491,47 +430,26 @@ VectorSumCoefficient::VectorSumCoefficient(VectorCoefficient &_A,
 void VectorSumCoefficient::Eval(Vector &V, ElementTransformation &T,
                                const IntegrationPoint &ip)
 {
-   V.SetSize(A.Size());
-   if (    ACoef) { ACoef->Eval(A, T, ip); }
-   if (    BCoef) { BCoef->Eval(B, T, ip); }
-   if (alphaCoef) { alpha = alphaCoef->Eval(T, ip); }
-   if ( betaCoef) { beta  = betaCoef->Eval(T, ip); }
-   add(alpha, A, beta, B, V);
+   b->Eval(V, T, ip);
+   if ( beta != 1.0 ) { V *= beta; }
+   a->Eval(va, T, ip);
+   V.Add(alpha, va);
 }

-ScalarVectorProductCoefficient::ScalarVectorProductCoefficient(
-   double A,
-   VectorCoefficient &B)
-   : VectorCoefficient(B.GetVDim()), aConst(A), a(NULL), b(&B)
-{}
-
 ScalarVectorProductCoefficient::ScalarVectorProductCoefficient(
   Coefficient &A,
   VectorCoefficient &B)
-   : VectorCoefficient(B.GetVDim()), aConst(0.0), a(&A), b(&B)
+   : VectorCoefficient(B.GetVDim()), a(&A), b(&B)
 {}

 void ScalarVectorProductCoefficient::Eval(Vector &V, ElementTransformation &T,
                                          const IntegrationPoint &ip)
 {
-   double sa = (a == NULL) ? aConst : a->Eval(T, ip);
+   double sa = a->Eval(T, ip);
   b->Eval(V, T, ip);
   V *= sa;
 }

-NormalizedVectorCoefficient::NormalizedVectorCoefficient(VectorCoefficient &A,
-                                                         double _tol)
-   : VectorCoefficient(A.GetVDim()), a(&A), tol(_tol)
-{}
-
-void NormalizedVectorCoefficient::Eval(Vector &V, ElementTransformation &T,
-                                       const IntegrationPoint &ip)
-{
-   a->Eval(V, T, ip);
-   double nv = V.Norml2();
-   V *= (nv > tol) ? (1.0/nv) : 0.0;
-}
-
 VectorCrossProductCoefficient::VectorCrossProductCoefficient(
   VectorCoefficient &A,
   VectorCoefficient &B)
@@ -553,18 +471,17 @@ void VectorCrossProductCoefficient::Eval(Vector &V, ElementTransformation &T,
   V[2] = va[0] * vb[1] - va[1] * vb[0];
 }

-MatrixVectorProductCoefficient::MatrixVectorProductCoefficient(
-   MatrixCoefficient &A, VectorCoefficient &B)
+MatVecCoefficient::MatVecCoefficient(MatrixCoefficient &A,
+                                     VectorCoefficient &B)
   : VectorCoefficient(A.GetHeight()), a(&A), b(&B),
     ma(A.GetHeight(), A.GetWidth()), vb(B.GetVDim())
 {
   MFEM_ASSERT(A.GetWidth() == B.GetVDim(),
-               "MatrixVectorProductCoefficient:  "
-               "Arguments have incompatible dimensions.");
+               "MatVecCoefficient:  Arguments have incompatible dimensions.");
 }

-void MatrixVectorProductCoefficient::Eval(Vector &V, ElementTransformation &T,
-                                          const IntegrationPoint &ip)
+void MatVecCoefficient::Eval(Vector &V, ElementTransformation &T,
+                             const IntegrationPoint &ip)
 {
   a->Eval(ma, T, ip);
   b->Eval(vb, T, ip);
@@ -600,23 +517,17 @@ void MatrixSumCoefficient::Eval(DenseMatrix &M, ElementTransformation &T,
   M.Add(alpha, ma);
 }

-ScalarMatrixProductCoefficient::ScalarMatrixProductCoefficient(
-   double A,
-   MatrixCoefficient &B)
-   : MatrixCoefficient(B.GetHeight(), B.GetWidth()), aConst(A), a(NULL), b(&B)
-{}
-
 ScalarMatrixProductCoefficient::ScalarMatrixProductCoefficient(
   Coefficient &A,
   MatrixCoefficient &B)
-   : MatrixCoefficient(B.GetHeight(), B.GetWidth()), aConst(0.0), a(&A), b(&B)
+   : MatrixCoefficient(B.GetHeight(), B.GetWidth()), a(&A), b(&B)
 {}

 void ScalarMatrixProductCoefficient::Eval(DenseMatrix &M,
                                          ElementTransformation &T,
                                          const IntegrationPoint &ip)
 {
-   double sa = (a == NULL) ? aConst : a->Eval(T, ip);
+   double sa = a->Eval(T, ip);
   b->Eval(M, T, ip);
   M *= sa;
 }
@@ -670,30 +581,6 @@ void OuterProductCoefficient::Eval(DenseMatrix &M, ElementTransformation &T,
   }
 }

-CrossCrossCoefficient::CrossCrossCoefficient(Coefficient &A,
-                                             VectorCoefficient &K)
-   : MatrixCoefficient(K.GetVDim(), K.GetVDim()), aConst(0.0), a(&A), k(&K),
-     vk(K.GetVDim())
-{}
-
-void CrossCrossCoefficient::Eval(DenseMatrix &M, ElementTransformation &T,
-                                 const IntegrationPoint &ip)
-{
-   k->Eval(vk, T, ip);
-   M.SetSize(vk.Size(), vk.Size());
-   M = 0.0;
-   double k2 = vk*vk;
-   for (int i=0; i<vk.Size(); i++)
-   {
-      M(i, i) = k2;
-      for (int j=0; j<vk.Size(); j++)
-      {
-         M(i, j) -= vk[i] * vk[j];
-      }
-   }
-   M *= ((a == NULL ) ? aConst : a->Eval(T, ip) );
-}
-
 double LpNormLoop(double p, Coefficient &coeff, Mesh &mesh,
                  const IntegrationRule *irs[])
 {
@@ -871,61 +758,4 @@ double ComputeGlobalLpNorm(double p, VectorCoefficient &coeff, ParMesh &pmesh,
 }
 #endif

-VectorQuadratureFunctionCoefficient::VectorQuadratureFunctionCoefficient(
-   QuadratureFunction &qf)
-   : VectorCoefficient(qf.GetVDim()), QuadF(qf), index(0) { }
-
-void VectorQuadratureFunctionCoefficient::SetComponent(int _index, int _length)
-{
-   MFEM_VERIFY(_index >= 0, "Index must be >= 0");
-   MFEM_VERIFY(_index < QuadF.GetVDim(),
-               "Index must be < QuadratureFunction length");
-   index = _index;
-
-   MFEM_VERIFY(_length > 0, "Length must be > 0");
-   MFEM_VERIFY(_length <= QuadF.GetVDim() - index,
-               "Length must be <= (QuadratureFunction length - index)");
-
-   vdim = _length;
-}
-
-void VectorQuadratureFunctionCoefficient::Eval(Vector &V,
-                                               ElementTransformation &T,
-                                               const IntegrationPoint &ip)
-{
-   QuadF.HostRead();
-
-   if (index == 0 && vdim == QuadF.GetVDim())
-   {
-      QuadF.GetElementValues(T.ElementNo, ip.index, V);
-   }
-   else
-   {
-      Vector temp;
-      QuadF.GetElementValues(T.ElementNo, ip.index, temp);
-      V.SetSize(vdim);
-      for (int i = 0; i < vdim; i++)
-      {
-         V(i) = temp(index + i);
-      }
-   }
-
-   return;
-}
-
-QuadratureFunctionCoefficient::QuadratureFunctionCoefficient(
-   QuadratureFunction &qf) : QuadF(qf)
-{
-   MFEM_VERIFY(qf.GetVDim() == 1, "QuadratureFunction's vdim must be 1");
-}
-
-double QuadratureFunctionCoefficient::Eval(ElementTransformation &T,
-                                           const IntegrationPoint &ip)
-{
-   QuadF.HostRead();
-   Vector temp(1);
-   QuadF.GetElementValues(T.ElementNo, ip.index, temp);
-   return temp[0];
-}
-
 }
@@ -99,8 +99,8 @@ public:
                     ComplexOperator::Convention
                     convention = ComplexOperator::HERMITIAN);

-   /** @brief Create a ComplexLinearForm on the FiniteElementSpace @a fes, using
-       the same integrators as the LinearForms @a lf_r (real) and @a lf_i (imag).
+   /** @brief Create a ComplexLinearForm on the FiniteElementSpace @a f, using
+       the same integrators as the LinearForms @a lfr (real) and @a lfi (imag) .

       The pointer @a fes is not owned by the newly constructed object.

@@ -195,8 +195,8 @@ private:
   BilinearForm *blfr;
   BilinearForm *blfi;

-   /* These methods check if the real/imag parts of the sesquilinear form are
-      not empty */
+   /* These methods check if the real/imag parts of the sesqulinear form are not
+      empty */
   bool RealInteg();
   bool ImagInteg();

@@ -204,7 +204,7 @@ public:
   SesquilinearForm(FiniteElementSpace *fes,
                    ComplexOperator::Convention
                    convention = ComplexOperator::HERMITIAN);
-   /** @brief Create a SesquilinearForm on the FiniteElementSpace @a fes, using
+   /** @brief Create a SesquilinearForm on the FiniteElementSpace @a f, using
       the same integrators as the BilinearForms @a bfr and @a bfi .

       The pointer @a fes is not owned by the newly constructed object.
@@ -219,21 +219,6 @@ public:
   void SetConvention(const ComplexOperator::Convention &
                      convention) { conv = convention; }

-   /// Set the desired assembly level.
-   /** Valid choices are:
-
-       - AssemblyLevel::FULL  (default)
-       - AssemblyLevel::PARTIAL
-       - AssemblyLevel::ELEMENT
-       - AssemblyLevel::NONE
-
-       This method must be called before assembly. */
-   void SetAssemblyLevel(AssemblyLevel assembly_level)
-   {
-      blfr->SetAssemblyLevel(assembly_level);
-      blfi->SetAssemblyLevel(assembly_level);
-   }
-
   BilinearForm & real() { return *blfr; }
   BilinearForm & imag() { return *blfi; }
   const BilinearForm & real() const { return *blfr; }
@@ -324,7 +309,7 @@ protected:
 public:

   /* @brief Construct a ParComplexGridFunction associated with the
-      ParFiniteElementSpace @a *pf. */
+      ParFiniteElementSpace @a *f. */
   ParComplexGridFunction(ParFiniteElementSpace *pf);

   void Update();
@@ -416,8 +401,8 @@ public:
                        convention = ComplexOperator::HERMITIAN);

   /** @brief Create a ParComplexLinearForm on the ParFiniteElementSpace @a pf,
-       using the same integrators as the LinearForms @a plf_r (real) and
-       @a plf_i (imag).
+       using the same integrators as the LinearForms @a plfr (real) and @a plfi
+       (imag) .

      The pointer @a fes is not owned by the newly constructed object.

@@ -493,7 +478,7 @@ public:
 /** Class for a parallel sesquilinear form

    A sesquilinear form is a generalization of a bilinear form to complex-valued
-    fields. Sesquilinear forms are linear in the second argument but the
+    fields. Sesquilinear forms are linear in the second argument but but the
    first argument involves a complex conjugate in the sense that:

                a(alpha u, beta v) = conj(alpha) beta a(u, v)
@@ -539,21 +524,6 @@ public:
   void SetConvention(const ComplexOperator::Convention &
                      convention) { conv = convention; }

-   /// Set the desired assembly level.
-   /** Valid choices are:
-
-       - AssemblyLevel::FULL  (default)
-       - AssemblyLevel::PARTIAL
-       - AssemblyLevel::ELEMENT
-       - AssemblyLevel::NONE
-
-       This method must be called before assembly. */
-   void SetAssemblyLevel(AssemblyLevel assembly_level)
-   {
-      pblfr->SetAssemblyLevel(assembly_level);
-      pblfi->SetAssemblyLevel(assembly_level);
-   }
-
   ParBilinearForm & real() { return *pblfr; }
   ParBilinearForm & imag() { return *pblfi; }
   const ParBilinearForm & real() const { return *pblfr; }
@@ -415,6 +415,9 @@ void VisItDataCollection::SetMesh(MPI_Comm comm, Mesh *new_mesh)
 void VisItDataCollection::RegisterField(const std::string& name,
                                        GridFunction *gf)
 {
+   DataCollection::RegisterField(name, gf);
+   field_info_map[name] = VisItFieldInfo("nodes", gf->VectorDim());
+
   int LOD = 1;
   if (gf->FESpace()->GetNURBSext())
   {
@@ -428,27 +431,6 @@ void VisItDataCollection::RegisterField(const std::string& name,
      }
   }

-   DataCollection::RegisterField(name, gf);
-   field_info_map[name] = VisItFieldInfo("nodes", gf->VectorDim(), LOD);
-   visit_levels_of_detail = std::max(visit_levels_of_detail, LOD);
-}
-
-void VisItDataCollection::RegisterQField(const std::string& name,
-                                         QuadratureFunction *qf)
-{
-   int LOD = -1;
-   Mesh *mesh = qf->GetSpace()->GetMesh();
-   for (int e=0; e<qf->GetSpace()->GetNE(); e++)
-   {
-      int locLOD = GlobGeometryRefiner.GetRefinementLevelFromElems(
-                      mesh->GetElementBaseGeometry(e),
-                      qf->GetElementIntRule(e).GetNPoints());
-
-      LOD = std::max(LOD,locLOD);
-   }
-
-   DataCollection::RegisterQField(name, qf);
-   field_info_map[name] = VisItFieldInfo("elements", 1, LOD);
   visit_levels_of_detail = std::max(visit_levels_of_detail, LOD);
 }

@@ -616,28 +598,14 @@ void VisItDataCollection::LoadFields()
      // TODO: 1) load parallel GridFunction on one processor
      if (serial)
      {
-         if ((it->second).association == "nodes")
-         {
-            field_map.Register(it->first, new GridFunction(mesh, file), own_data);
-         }
-         else if ((it->second).association == "elements")
-         {
-            q_field_map.Register(it->first, new QuadratureFunction(mesh, file), own_data);
-         }
+         field_map.Register(it->first, new GridFunction(mesh, file), own_data);
      }
      else
      {
 #ifdef MFEM_USE_MPI
-         if ((it->second).association == "nodes")
-         {
-            field_map.Register(
-               it->first,
-               new ParGridFunction(dynamic_cast<ParMesh*>(mesh), file), own_data);
-         }
-         else if ((it->second).association == "elements")
-         {
-            q_field_map.Register(it->first, new QuadratureFunction(mesh, file), own_data);
-         }
+         field_map.Register(
+            it->first,
+            new ParGridFunction(dynamic_cast<ParMesh*>(mesh), file), own_data);
 #else
         error = READ_ERROR;
         MFEM_WARNING("Reading parallel format in serial is not supported");
@@ -672,7 +640,7 @@ std::string VisItDataCollection::GetVisItRootString()
   {
      ftags["assoc"] = picojson::value((it->second).association);
      ftags["comps"] = picojson::value(to_string((it->second).num_components));
-      ftags["lod"] = picojson::value(to_string((it->second).lod));
+      ftags["lod"] = picojson::value(to_string(visit_levels_of_detail));
      field["path"] = picojson::value(path_str + it->first + file_ext_format);
      field["tags"] = picojson::value(ftags);
      fields[it->first] = picojson::value(field);
@@ -771,6 +739,12 @@ ParaViewDataCollection::ParaViewDataCollection(const std::string&
 #endif
 }

+void ParaViewDataCollection::RegisterField(const std::string& field_name,
+                                           mfem::GridFunction *gf)
+{
+   DataCollection::RegisterField(field_name,gf);
+}
+
 void ParaViewDataCollection::SetLevelsOfDetail(int levels_of_detail_)
 {
   levels_of_detail = levels_of_detail_;
@@ -841,7 +815,7 @@ void ParaViewDataCollection::Save()
   // the directory is created

   // create pvd file if needed
-   if (myid == 0 && !pvd_stream.is_open())
+   if (!pvd_stream.is_open())
   {
      std::string dpath=GenerateCollectionPath();
      std::string pvdname=dpath+"/"+GeneratePVDFileName();
@@ -391,10 +391,9 @@ class VisItFieldInfo
 public:
   std::string association;
   int num_components;
-   int lod;
-   VisItFieldInfo() { association = ""; num_components = 0; lod = 1;}
-   VisItFieldInfo(std::string _association, int _num_components, int _lod = 1)
-   { association = _association; num_components = _num_components; lod =_lod;}
+   VisItFieldInfo() { association = ""; num_components = 0; }
+   VisItFieldInfo(std::string _association, int _num_components)
+   { association = _association; num_components = _num_components; }
 };

 /// Data collection with VisIt I/O routines
@@ -446,12 +445,6 @@ public:
   /// Add a grid function to the collection and update the root file
   virtual void RegisterField(const std::string& field_name, GridFunction *gf);

-   /// Add a quadrature function to the collection and update the root file.
-   /** Visualization of quadrature function is not supported in VisIt(3.12).
-       A patch has been sent to VisIt developers in June 2020. */
-   virtual void RegisterQField(const std::string& q_field_name,
-                               QuadratureFunction *qf);
-
   /// Set VisIt parameter: default levels of detail for the MultiresControl
   void SetLevelsOfDetail(int levels_of_detail);

@@ -508,6 +501,10 @@ public:
   ParaViewDataCollection(const std::string& collection_name,
                          mfem::Mesh *mesh_ = NULL);

+   /// Add a grid function to the collection
+   virtual void RegisterField(const std::string& field_name,
+                              mfem::GridFunction *gf) override;
+
   /// Set refinement levels - every element is uniformly split based on
   /// levels_of_detail_
   void SetLevelsOfDetail(int levels_of_detail_);
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Dylan Copeland	b51732c827	Extended PA for some vector mass and curl integrators to handle diagonal matrix coefficients. Added unit tests.	2020-03-31 11:56:02 -07:00
Dylan Copeland	d838c6b33b	Implemented PA for MixedVectorCurlIntegrator and MixedVectorWeakCurlIntegrator. Added unit tests for these integrators.	2020-03-30 18:02:46 -07:00